2023-08-28T01:50:39.147 INFO:root:teuthology version: 0.0.1.dev185+g449a1bc 2023-08-28T01:50:39.147 DEBUG:teuthology.run:Teuthology command: teuthology --archive /home/teuthworker/archive/teuthology-2023-08-26_14:23:13-upgrade:pacific-x-reef-distro-default-smithi/7381875 --owner scheduled_teuthology@teuthology --verbose --description upgrade:pacific-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} --name teuthology-2023-08-26_14:23:13-upgrade:pacific-x-reef-distro-default-smithi -- /home/teuthworker/archive/teuthology-2023-08-26_14:23:13-upgrade:pacific-x-reef-distro-default-smithi/7381875/orig.config.yaml 2023-08-28T01:50:39.206 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-08-28T01:50:39.300 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2023-08-26_14:23:13-upgrade:pacific-x-reef-distro-default-smithi/7381875 branch: reef description: upgrade:pacific-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} email: ceph-qa@ceph.io first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' first_in_suite: false job_id: '7381875' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run basic import/export cli tests for rbd ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run basic import/export cli tests for rbd ' name: teuthology-2023-08-26_14:23:13-upgrade:pacific-x-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: rhel os_version: '8.6' overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: 9cf208207dfe23ffccb33308b22e51570cb45e51 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 9cf208207dfe23ffccb33308b22e51570cb45e51 selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: reef sha1: 81fef490342f7b9544e89fe32a9e51929064494d owner: scheduled_teuthology@teuthology priority: 100 repo: https://github.com/ceph/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' seed: 2117 sha1: 9cf208207dfe23ffccb33308b22e51570cb45e51 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: pacific clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/test_librbd.sh - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' suite: upgrade:pacific-x suite_branch: reef suite_path: /home/teuthworker/src/github.com_ceph_ceph_81fef490342f7b9544e89fe32a9e51929064494d/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph.git suite_sha1: 81fef490342f7b9544e89fe32a9e51929064494d targets: smithi143.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCpUxo5ay/ucdkX9rleNMm8Ofp24XRXUtJPOK3Ji3XBpTiUgmmwneaY6316+3CG+k4K0p3IebanuyTvZApjm02/dNCvu8yD3rjEpWVgIuqYH8O3lEdN9DXcCX+7xHJE0m18+ZPsJQVsNrNJxY+5zqyNnBc8jpZq2LStsWwPOKPWhauANRkOBFSGDoLppWZbLx/yJs+R0qoNLLK9LpKQTUnqsapXvbD27OpPahzj/qbNxFPPyY6XrrldZ6qEO+EjJOujFLH3gzv0kJu10DCuzxrpz6ston4RPi1O3GgcHnF4Mqr/W9VwZ94UTMh+k4dIFRVqJEKrcwRHa5kFsvH/WhVx0Ygr/MiV/Q86mmV7JT+/MpzIRTtWyei5L9IBeZIE9cwzj+mOk6SD/t1roIVEGVV/uF2AyFDVBPr6F8oZ822OGpUyyaDsWTitPNdTX4TC3zdSrLc9tyDa1P74eH/ZLeTSf+C6pM/cepOhQaa6XzpDiPSuJUm9HOQxUlEAvcqa7Ac= smithi169.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCpUxo5ay/ucdkX9rleNMm8Ofp24XRXUtJPOK3Ji3XBpTiUgmmwneaY6316+3CG+k4K0p3IebanuyTvZApjm02/dNCvu8yD3rjEpWVgIuqYH8O3lEdN9DXcCX+7xHJE0m18+ZPsJQVsNrNJxY+5zqyNnBc8jpZq2LStsWwPOKPWhauANRkOBFSGDoLppWZbLx/yJs+R0qoNLLK9LpKQTUnqsapXvbD27OpPahzj/qbNxFPPyY6XrrldZ6qEO+EjJOujFLH3gzv0kJu10DCuzxrpz6ston4RPi1O3GgcHnF4Mqr/W9VwZ94UTMh+k4dIFRVqJEKrcwRHa5kFsvH/WhVx0Ygr/MiV/Q86mmV7JT+/MpzIRTtWyei5L9IBeZIE9cwzj+mOk6SD/t1roIVEGVV/uF2AyFDVBPr6F8oZ822OGpUyyaDsWTitPNdTX4TC3zdSrLc9tyDa1P74eH/ZLeTSf+C6pM/cepOhQaa6XzpDiPSuJUm9HOQxUlEAvcqa7Ac= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools:3.0 --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - install: branch: pacific exclude_packages: - ceph-volume - cephadm: cephadm_branch: pacific cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:pacific - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 449a1bc2027504e7b3c3d7b30fa4178906581da7 timestamp: 2023-08-26_14:23:13 tube: smithi user: teuthology verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.29856 2023-08-28T01:50:39.300 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph_81fef490342f7b9544e89fe32a9e51929064494d/qa; will attempt to use it 2023-08-28T01:50:39.301 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph_81fef490342f7b9544e89fe32a9e51929064494d/qa/tasks 2023-08-28T01:50:39.301 INFO:teuthology.run_tasks:Running task internal.check_packages... 2023-08-28T01:50:39.303 INFO:teuthology.task.internal:Checking packages... 2023-08-28T01:50:39.324 INFO:teuthology.task.internal:Checking packages for os_type 'rhel', flavor 'default' and ceph hash '9cf208207dfe23ffccb33308b22e51570cb45e51' 2023-08-28T01:50:39.325 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-08-28T01:50:39.325 INFO:teuthology.packaging:ref: None 2023-08-28T01:50:39.325 INFO:teuthology.packaging:tag: None 2023-08-28T01:50:39.325 INFO:teuthology.packaging:branch: reef 2023-08-28T01:50:39.325 INFO:teuthology.packaging:sha1: 9cf208207dfe23ffccb33308b22e51570cb45e51 2023-08-28T01:50:39.325 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=reef 2023-08-28T01:50:39.582 INFO:teuthology.task.internal:Found packages for ceph version 18.2.0-191.g02042ef2 2023-08-28T01:50:39.583 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2023-08-28T01:50:39.591 INFO:teuthology.task.internal:no buildpackages task found 2023-08-28T01:50:39.591 INFO:teuthology.run_tasks:Running task internal.save_config... 2023-08-28T01:50:39.598 INFO:teuthology.task.internal:Saving configuration 2023-08-28T01:50:39.620 INFO:teuthology.run_tasks:Running task internal.check_lock... 2023-08-28T01:50:39.626 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-08-28T01:50:39.650 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi143.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-08-26_14:23:13-upgrade:pacific-x-reef-distro-default-smithi/7381875', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-08-28 01:45:26.004969', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-08-28T01:50:39.673 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi169.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-08-26_14:23:13-upgrade:pacific-x-reef-distro-default-smithi/7381875', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-08-28 01:45:26.006865', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-08-28T01:50:39.673 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2023-08-28T01:50:39.679 INFO:teuthology.task.internal:roles: ubuntu@smithi143.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2023-08-28T01:50:39.680 INFO:teuthology.task.internal:roles: ubuntu@smithi169.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2023-08-28T01:50:39.680 INFO:teuthology.run_tasks:Running task console_log... 2023-08-28T01:50:39.743 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7fec7b379670>, signals=[15]) 2023-08-28T01:50:39.744 INFO:teuthology.run_tasks:Running task internal.connect... 2023-08-28T01:50:39.751 INFO:teuthology.task.internal:Opening connections... 2023-08-28T01:50:39.751 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi143.front.sepia.ceph.com 2023-08-28T01:50:39.752 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi143.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-08-28T01:50:39.820 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi169.front.sepia.ceph.com 2023-08-28T01:50:39.821 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi169.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-08-28T01:50:39.888 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2023-08-28T01:50:39.894 DEBUG:teuthology.orchestra.run.smithi143:> uname -m 2023-08-28T01:50:39.914 INFO:teuthology.orchestra.run.smithi143.stdout:x86_64 2023-08-28T01:50:39.914 DEBUG:teuthology.orchestra.run.smithi143:> cat /etc/os-release 2023-08-28T01:50:39.973 INFO:teuthology.orchestra.run.smithi143.stdout:NAME="Red Hat Enterprise Linux" 2023-08-28T01:50:39.973 INFO:teuthology.orchestra.run.smithi143.stdout:VERSION="8.6 (Ootpa)" 2023-08-28T01:50:39.973 INFO:teuthology.orchestra.run.smithi143.stdout:ID="rhel" 2023-08-28T01:50:39.974 INFO:teuthology.orchestra.run.smithi143.stdout:ID_LIKE="fedora" 2023-08-28T01:50:39.974 INFO:teuthology.orchestra.run.smithi143.stdout:VERSION_ID="8.6" 2023-08-28T01:50:39.974 INFO:teuthology.orchestra.run.smithi143.stdout:PLATFORM_ID="platform:el8" 2023-08-28T01:50:39.974 INFO:teuthology.orchestra.run.smithi143.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2023-08-28T01:50:39.974 INFO:teuthology.orchestra.run.smithi143.stdout:ANSI_COLOR="0;31" 2023-08-28T01:50:39.974 INFO:teuthology.orchestra.run.smithi143.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2023-08-28T01:50:39.974 INFO:teuthology.orchestra.run.smithi143.stdout:HOME_URL="https://www.redhat.com/" 2023-08-28T01:50:39.974 INFO:teuthology.orchestra.run.smithi143.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2023-08-28T01:50:39.974 INFO:teuthology.orchestra.run.smithi143.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-08-28T01:50:39.974 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T01:50:39.974 INFO:teuthology.orchestra.run.smithi143.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2023-08-28T01:50:39.974 INFO:teuthology.orchestra.run.smithi143.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2023-08-28T01:50:39.975 INFO:teuthology.orchestra.run.smithi143.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2023-08-28T01:50:39.975 INFO:teuthology.orchestra.run.smithi143.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2023-08-28T01:50:39.975 INFO:teuthology.lock.ops:Updating smithi143.front.sepia.ceph.com on lock server 2023-08-28T01:50:40.001 DEBUG:teuthology.orchestra.run.smithi169:> uname -m 2023-08-28T01:50:40.020 INFO:teuthology.orchestra.run.smithi169.stdout:x86_64 2023-08-28T01:50:40.021 DEBUG:teuthology.orchestra.run.smithi169:> cat /etc/os-release 2023-08-28T01:50:40.080 INFO:teuthology.orchestra.run.smithi169.stdout:NAME="Red Hat Enterprise Linux" 2023-08-28T01:50:40.080 INFO:teuthology.orchestra.run.smithi169.stdout:VERSION="8.6 (Ootpa)" 2023-08-28T01:50:40.081 INFO:teuthology.orchestra.run.smithi169.stdout:ID="rhel" 2023-08-28T01:50:40.081 INFO:teuthology.orchestra.run.smithi169.stdout:ID_LIKE="fedora" 2023-08-28T01:50:40.081 INFO:teuthology.orchestra.run.smithi169.stdout:VERSION_ID="8.6" 2023-08-28T01:50:40.081 INFO:teuthology.orchestra.run.smithi169.stdout:PLATFORM_ID="platform:el8" 2023-08-28T01:50:40.081 INFO:teuthology.orchestra.run.smithi169.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2023-08-28T01:50:40.081 INFO:teuthology.orchestra.run.smithi169.stdout:ANSI_COLOR="0;31" 2023-08-28T01:50:40.081 INFO:teuthology.orchestra.run.smithi169.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2023-08-28T01:50:40.081 INFO:teuthology.orchestra.run.smithi169.stdout:HOME_URL="https://www.redhat.com/" 2023-08-28T01:50:40.081 INFO:teuthology.orchestra.run.smithi169.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2023-08-28T01:50:40.081 INFO:teuthology.orchestra.run.smithi169.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-08-28T01:50:40.081 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T01:50:40.081 INFO:teuthology.orchestra.run.smithi169.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2023-08-28T01:50:40.082 INFO:teuthology.orchestra.run.smithi169.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2023-08-28T01:50:40.082 INFO:teuthology.orchestra.run.smithi169.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2023-08-28T01:50:40.082 INFO:teuthology.orchestra.run.smithi169.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2023-08-28T01:50:40.082 INFO:teuthology.lock.ops:Updating smithi169.front.sepia.ceph.com on lock server 2023-08-28T01:50:40.101 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2023-08-28T01:50:40.110 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2023-08-28T01:50:40.123 INFO:teuthology.task.internal:Checking for old test directory... 2023-08-28T01:50:40.124 DEBUG:teuthology.orchestra.run.smithi143:> test '!' -e /home/ubuntu/cephtest 2023-08-28T01:50:40.126 DEBUG:teuthology.orchestra.run.smithi169:> test '!' -e /home/ubuntu/cephtest 2023-08-28T01:50:40.145 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2023-08-28T01:50:40.152 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2023-08-28T01:50:40.152 DEBUG:teuthology.orchestra.run.smithi143:> test -z $(ls -A /var/lib/ceph) 2023-08-28T01:50:40.186 DEBUG:teuthology.orchestra.run.smithi169:> test -z $(ls -A /var/lib/ceph) 2023-08-28T01:50:40.218 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2023-08-28T01:50:40.271 INFO:teuthology.run_tasks:Running task kernel... 2023-08-28T01:50:40.286 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2023-08-28T01:50:40.286 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro'}, 'mon.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2023-08-28T01:50:40.286 DEBUG:teuthology.orchestra.run.smithi143:> test -f /run/.containerenv -o -f /.dockerenv 2023-08-28T01:50:40.287 DEBUG:teuthology.orchestra.run.smithi169:> test -f /run/.containerenv -o -f /.dockerenv 2023-08-28T01:50:40.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-08-28T01:50:40.305 DEBUG:teuthology.orchestra.run.smithi169:> uname -r 2023-08-28T01:50:40.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-08-28T01:50:40.306 DEBUG:teuthology.orchestra.run.smithi143:> uname -r 2023-08-28T01:50:40.363 INFO:teuthology.orchestra.run.smithi169.stdout:4.18.0-372.9.1.el8.x86_64 2023-08-28T01:50:40.364 INFO:teuthology.task.kernel:Running kernel on smithi169: 4.18.0-372.9.1.el8.x86_64 2023-08-28T01:50:40.365 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum install -y kernel 2023-08-28T01:50:40.366 INFO:teuthology.orchestra.run.smithi143.stdout:4.18.0-372.9.1.el8.x86_64 2023-08-28T01:50:40.366 INFO:teuthology.task.kernel:Running kernel on smithi143: 4.18.0-372.9.1.el8.x86_64 2023-08-28T01:50:40.366 DEBUG:teuthology.orchestra.run.smithi143:> sudo yum install -y kernel 2023-08-28T01:50:41.506 INFO:teuthology.orchestra.run.smithi169.stdout:Updating Subscription Management repositories. 2023-08-28T01:50:41.506 INFO:teuthology.orchestra.run.smithi169.stdout:Unable to read consumer identity 2023-08-28T01:50:41.583 INFO:teuthology.orchestra.run.smithi143.stdout:Updating Subscription Management repositories. 2023-08-28T01:50:41.584 INFO:teuthology.orchestra.run.smithi143.stdout:Unable to read consumer identity 2023-08-28T01:50:42.974 INFO:teuthology.orchestra.run.smithi169.stdout:Copr repo for python3-asyncssh owned by ceph 16 kB/s | 3.5 kB 00:00 2023-08-28T01:50:42.983 INFO:teuthology.orchestra.run.smithi143.stdout:Copr repo for python3-asyncssh owned by ceph 24 kB/s | 3.5 kB 00:00 2023-08-28T01:50:44.691 INFO:teuthology.orchestra.run.smithi143.stdout:Extra Packages for Enterprise Linux 9.3 MB/s | 16 MB 00:01 2023-08-28T01:50:44.999 INFO:teuthology.orchestra.run.smithi169.stdout:Extra Packages for Enterprise Linux 7.2 MB/s | 16 MB 00:02 2023-08-28T01:50:48.125 INFO:teuthology.orchestra.run.smithi143.stdout:lab-extras 431 kB/s | 24 kB 00:00 2023-08-28T01:50:48.408 INFO:teuthology.orchestra.run.smithi169.stdout:lab-extras 633 kB/s | 24 kB 00:00 2023-08-28T01:50:48.769 INFO:teuthology.orchestra.run.smithi143.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2023-08-28T01:50:48.812 INFO:teuthology.orchestra.run.smithi143.stdout:Dependencies resolved. 2023-08-28T01:50:48.813 INFO:teuthology.orchestra.run.smithi143.stdout:Nothing to do. 2023-08-28T01:50:48.814 INFO:teuthology.orchestra.run.smithi143.stdout:Complete! 2023-08-28T01:50:48.975 DEBUG:teuthology.orchestra.run.smithi143:> echo no | sudo yum reinstall kernel || true 2023-08-28T01:50:49.086 INFO:teuthology.orchestra.run.smithi169.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2023-08-28T01:50:49.149 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-08-28T01:50:49.150 INFO:teuthology.orchestra.run.smithi169.stdout:Nothing to do. 2023-08-28T01:50:49.150 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-08-28T01:50:49.283 DEBUG:teuthology.orchestra.run.smithi169:> echo no | sudo yum reinstall kernel || true 2023-08-28T01:50:49.306 INFO:teuthology.orchestra.run.smithi143.stdout:Updating Subscription Management repositories. 2023-08-28T01:50:49.306 INFO:teuthology.orchestra.run.smithi143.stdout:Unable to read consumer identity 2023-08-28T01:50:49.567 INFO:teuthology.orchestra.run.smithi143.stdout:Last metadata expiration check: 0:00:01 ago on Mon 28 Aug 2023 01:50:48 AM UTC. 2023-08-28T01:50:49.610 INFO:teuthology.orchestra.run.smithi143.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-08-28T01:50:49.615 INFO:teuthology.orchestra.run.smithi169.stdout:Updating Subscription Management repositories. 2023-08-28T01:50:49.616 INFO:teuthology.orchestra.run.smithi169.stdout:Unable to read consumer identity 2023-08-28T01:50:49.653 INFO:teuthology.orchestra.run.smithi143.stderr:Error: No packages marked for reinstall. 2023-08-28T01:50:49.699 DEBUG:teuthology.orchestra.run.smithi143:> sudo yum reinstall -y kernel || true 2023-08-28T01:50:49.878 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:00:01 ago on Mon 28 Aug 2023 01:50:48 AM UTC. 2023-08-28T01:50:49.919 INFO:teuthology.orchestra.run.smithi169.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-08-28T01:50:49.963 INFO:teuthology.orchestra.run.smithi169.stderr:Error: No packages marked for reinstall. 2023-08-28T01:50:50.010 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum reinstall -y kernel || true 2023-08-28T01:50:50.027 INFO:teuthology.orchestra.run.smithi143.stdout:Updating Subscription Management repositories. 2023-08-28T01:50:50.027 INFO:teuthology.orchestra.run.smithi143.stdout:Unable to read consumer identity 2023-08-28T01:50:50.283 INFO:teuthology.orchestra.run.smithi143.stdout:Last metadata expiration check: 0:00:02 ago on Mon 28 Aug 2023 01:50:48 AM UTC. 2023-08-28T01:50:50.325 INFO:teuthology.orchestra.run.smithi143.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-08-28T01:50:50.338 INFO:teuthology.orchestra.run.smithi169.stdout:Updating Subscription Management repositories. 2023-08-28T01:50:50.338 INFO:teuthology.orchestra.run.smithi169.stdout:Unable to read consumer identity 2023-08-28T01:50:50.363 INFO:teuthology.orchestra.run.smithi143.stderr:Error: No packages marked for reinstall. 2023-08-28T01:50:50.410 DEBUG:teuthology.orchestra.run.smithi143:> rpm -q kernel | sort -rV | head -n 1 2023-08-28T01:50:50.490 INFO:teuthology.orchestra.run.smithi143.stdout:kernel-4.18.0-372.9.1.el8.x86_64 2023-08-28T01:50:50.490 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-372.9.1.el8.x86_64 2023-08-28T01:50:50.490 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2023-08-28T01:50:50.490 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2023-08-28T01:50:50.490 DEBUG:teuthology.orchestra.run.smithi143:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-08-28T01:50:50.523 INFO:teuthology.orchestra.run.smithi143.stdout:ttyS1 2023-08-28T01:50:50.535 DEBUG:teuthology.parallel:result is None 2023-08-28T01:50:50.592 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:00:02 ago on Mon 28 Aug 2023 01:50:48 AM UTC. 2023-08-28T01:50:50.631 INFO:teuthology.orchestra.run.smithi169.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-08-28T01:50:50.657 INFO:teuthology.orchestra.run.smithi169.stderr:Error: No packages marked for reinstall. 2023-08-28T01:50:50.703 DEBUG:teuthology.orchestra.run.smithi169:> rpm -q kernel | sort -rV | head -n 1 2023-08-28T01:50:50.783 INFO:teuthology.orchestra.run.smithi169.stdout:kernel-4.18.0-372.9.1.el8.x86_64 2023-08-28T01:50:50.783 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-372.9.1.el8.x86_64 2023-08-28T01:50:50.783 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2023-08-28T01:50:50.784 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2023-08-28T01:50:50.784 DEBUG:teuthology.orchestra.run.smithi169:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-08-28T01:50:50.809 INFO:teuthology.orchestra.run.smithi169.stdout:ttyS1 2023-08-28T01:50:50.832 DEBUG:teuthology.parallel:result is None 2023-08-28T01:50:50.832 INFO:teuthology.run_tasks:Running task internal.base... 2023-08-28T01:50:50.840 INFO:teuthology.task.internal:Creating test directory... 2023-08-28T01:50:50.840 DEBUG:teuthology.orchestra.run.smithi143:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-08-28T01:50:50.842 DEBUG:teuthology.orchestra.run.smithi169:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-08-28T01:50:50.890 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2023-08-28T01:50:50.898 INFO:teuthology.run_tasks:Running task internal.archive... 2023-08-28T01:50:50.904 INFO:teuthology.task.internal:Creating archive directory... 2023-08-28T01:50:50.904 DEBUG:teuthology.orchestra.run.smithi143:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-08-28T01:50:50.907 DEBUG:teuthology.orchestra.run.smithi169:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-08-28T01:50:50.952 INFO:teuthology.run_tasks:Running task internal.coredump... 2023-08-28T01:50:50.959 INFO:teuthology.task.internal:Enabling coredump saving... 2023-08-28T01:50:50.959 DEBUG:teuthology.orchestra.run.smithi143:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-08-28T01:50:50.965 DEBUG:teuthology.orchestra.run.smithi169:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-08-28T01:50:50.990 INFO:teuthology.orchestra.run.smithi143.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-08-28T01:50:51.000 INFO:teuthology.orchestra.run.smithi143.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-08-28T01:50:51.038 INFO:teuthology.orchestra.run.smithi169.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-08-28T01:50:51.048 INFO:teuthology.orchestra.run.smithi169.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-08-28T01:50:51.049 INFO:teuthology.run_tasks:Running task internal.sudo... 2023-08-28T01:50:51.059 INFO:teuthology.task.internal:Configuring sudo... 2023-08-28T01:50:51.060 DEBUG:teuthology.orchestra.run.smithi143:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-08-28T01:50:51.062 DEBUG:teuthology.orchestra.run.smithi169:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-08-28T01:50:51.088 INFO:teuthology.run_tasks:Running task internal.syslog... 2023-08-28T01:50:51.097 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2023-08-28T01:50:51.098 DEBUG:teuthology.orchestra.run.smithi143:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-08-28T01:50:51.129 DEBUG:teuthology.orchestra.run.smithi169:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-08-28T01:50:51.145 DEBUG:teuthology.orchestra.run.smithi143:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-08-28T01:50:51.227 DEBUG:teuthology.orchestra.run.smithi143:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-08-28T01:50:51.276 DEBUG:teuthology.orchestra.run.smithi143:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-08-28T01:50:51.358 DEBUG:teuthology.orchestra.run.smithi143:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-08-28T01:50:51.400 DEBUG:teuthology.orchestra.run.smithi143:> set -ex 2023-08-28T01:50:51.400 DEBUG:teuthology.orchestra.run.smithi143:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-08-28T01:50:51.467 DEBUG:teuthology.orchestra.run.smithi169:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-08-28T01:50:51.511 DEBUG:teuthology.orchestra.run.smithi169:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-08-28T01:50:51.561 DEBUG:teuthology.orchestra.run.smithi169:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-08-28T01:50:51.642 DEBUG:teuthology.orchestra.run.smithi169:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-08-28T01:50:51.684 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2023-08-28T01:50:51.684 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-08-28T01:50:51.751 DEBUG:teuthology.orchestra.run.smithi143:> sudo service rsyslog restart 2023-08-28T01:50:51.753 DEBUG:teuthology.orchestra.run.smithi169:> sudo service rsyslog restart 2023-08-28T01:50:51.809 INFO:teuthology.orchestra.run.smithi143.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-08-28T01:50:51.843 INFO:teuthology.orchestra.run.smithi169.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-08-28T01:50:52.248 INFO:teuthology.run_tasks:Running task internal.timer... 2023-08-28T01:50:52.261 INFO:teuthology.task.internal:Starting timer... 2023-08-28T01:50:52.261 INFO:teuthology.run_tasks:Running task pcp... 2023-08-28T01:50:52.271 INFO:teuthology.run_tasks:Running task selinux... 2023-08-28T01:50:52.279 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2023-08-28T01:50:52.280 DEBUG:teuthology.orchestra.run.smithi143:> sudo service auditd rotate 2023-08-28T01:50:52.368 INFO:teuthology.orchestra.run.smithi143.stdout:Rotating logs: 2023-08-28T01:50:52.369 DEBUG:teuthology.orchestra.run.smithi169:> sudo service auditd rotate 2023-08-28T01:50:52.436 INFO:teuthology.orchestra.run.smithi169.stdout:Rotating logs: 2023-08-28T01:50:52.438 DEBUG:teuthology.task.selinux:Getting current SELinux state 2023-08-28T01:50:52.438 DEBUG:teuthology.orchestra.run.smithi143:> /usr/sbin/getenforce 2023-08-28T01:50:52.461 INFO:teuthology.orchestra.run.smithi143.stdout:Permissive 2023-08-28T01:50:52.461 DEBUG:teuthology.orchestra.run.smithi169:> /usr/sbin/getenforce 2023-08-28T01:50:52.486 INFO:teuthology.orchestra.run.smithi169.stdout:Permissive 2023-08-28T01:50:52.486 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi143.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi169.front.sepia.ceph.com': 'permissive'} 2023-08-28T01:50:52.486 DEBUG:teuthology.orchestra.run.smithi143:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-08-28T01:50:52.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-08-28T01:50:52.529 DEBUG:teuthology.orchestra.run.smithi169:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-08-28T01:50:52.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-08-28T01:50:52.555 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2023-08-28T01:50:52.555 DEBUG:teuthology.orchestra.run.smithi143:> sudo /usr/sbin/setenforce permissive 2023-08-28T01:50:52.594 DEBUG:teuthology.orchestra.run.smithi169:> sudo /usr/sbin/setenforce permissive 2023-08-28T01:50:52.622 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2023-08-28T01:50:52.632 INFO:teuthology.repo_utils:/home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main was just updated or references a specific commit; assuming it is current 2023-08-28T01:50:52.632 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2023-08-28T01:50:52.652 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2023-08-28T01:50:52.653 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi143.front.sepia.ceph.com,smithi169.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2023-08-28T01:59:25.775 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi143.front.sepia.ceph.com'), Remote(name='ubuntu@smithi169.front.sepia.ceph.com')] 2023-08-28T01:59:25.801 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi143.front.sepia.ceph.com' 2023-08-28T01:59:25.802 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi143.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-08-28T01:59:25.871 DEBUG:teuthology.orchestra.run.smithi143:> true 2023-08-28T01:59:25.939 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi143.front.sepia.ceph.com' 2023-08-28T01:59:25.940 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi169.front.sepia.ceph.com' 2023-08-28T01:59:25.940 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi169.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-08-28T01:59:26.008 DEBUG:teuthology.orchestra.run.smithi169:> true 2023-08-28T01:59:26.080 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi169.front.sepia.ceph.com' 2023-08-28T01:59:26.080 INFO:teuthology.run_tasks:Running task clock... 2023-08-28T01:59:26.092 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2023-08-28T01:59:26.092 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-08-28T01:59:26.092 DEBUG:teuthology.orchestra.run.smithi143:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-08-28T01:59:26.095 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-08-28T01:59:26.095 DEBUG:teuthology.orchestra.run.smithi169:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-08-28T01:59:26.125 INFO:teuthology.orchestra.run.smithi143.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-08-28T01:59:26.139 INFO:teuthology.orchestra.run.smithi143.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-08-28T01:59:26.155 INFO:teuthology.orchestra.run.smithi169.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-08-28T01:59:26.170 INFO:teuthology.orchestra.run.smithi169.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-08-28T01:59:26.217 INFO:teuthology.orchestra.run.smithi143.stderr:sudo: ntpd: command not found 2023-08-28T01:59:26.229 INFO:teuthology.orchestra.run.smithi143.stdout:506 Cannot talk to daemon 2023-08-28T01:59:26.243 INFO:teuthology.orchestra.run.smithi143.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-08-28T01:59:26.245 INFO:teuthology.orchestra.run.smithi169.stderr:sudo: ntpd: command not found 2023-08-28T01:59:26.257 INFO:teuthology.orchestra.run.smithi169.stdout:506 Cannot talk to daemon 2023-08-28T01:59:26.257 INFO:teuthology.orchestra.run.smithi143.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-08-28T01:59:26.271 INFO:teuthology.orchestra.run.smithi169.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-08-28T01:59:26.285 INFO:teuthology.orchestra.run.smithi169.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-08-28T01:59:26.333 INFO:teuthology.orchestra.run.smithi143.stderr:bash: ntpq: command not found 2023-08-28T01:59:26.336 INFO:teuthology.orchestra.run.smithi143.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-08-28T01:59:26.337 INFO:teuthology.orchestra.run.smithi143.stdout:=============================================================================== 2023-08-28T01:59:26.337 INFO:teuthology.orchestra.run.smithi143.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-08-28T01:59:26.337 INFO:teuthology.orchestra.run.smithi143.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-08-28T01:59:26.337 INFO:teuthology.orchestra.run.smithi143.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-08-28T01:59:26.337 INFO:teuthology.orchestra.run.smithi143.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-08-28T01:59:26.365 INFO:teuthology.orchestra.run.smithi169.stderr:bash: ntpq: command not found 2023-08-28T01:59:26.369 INFO:teuthology.orchestra.run.smithi169.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-08-28T01:59:26.369 INFO:teuthology.orchestra.run.smithi169.stdout:=============================================================================== 2023-08-28T01:59:26.369 INFO:teuthology.orchestra.run.smithi169.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-08-28T01:59:26.369 INFO:teuthology.orchestra.run.smithi169.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-08-28T01:59:26.369 INFO:teuthology.orchestra.run.smithi169.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-08-28T01:59:26.369 INFO:teuthology.orchestra.run.smithi169.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-08-28T01:59:26.370 INFO:teuthology.run_tasks:Running task pexec... 2023-08-28T01:59:26.383 INFO:teuthology.task.pexec:Executing custom commands... 2023-08-28T01:59:26.383 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi143.front.sepia.ceph.com 2023-08-28T01:59:26.383 DEBUG:teuthology.orchestra.run.smithi143:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-08-28T01:59:26.384 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi169.front.sepia.ceph.com 2023-08-28T01:59:26.384 DEBUG:teuthology.orchestra.run.smithi169:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-08-28T01:59:26.755 INFO:teuthology.orchestra.run.smithi143.stdout:Updating Subscription Management repositories. 2023-08-28T01:59:26.784 INFO:teuthology.orchestra.run.smithi169.stdout:Updating Subscription Management repositories. 2023-08-28T01:59:27.553 INFO:teuthology.orchestra.run.smithi143.stdout:Last metadata expiration check: 0:01:23 ago on Mon 28 Aug 2023 01:58:04 AM UTC. 2023-08-28T01:59:27.582 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:01:23 ago on Mon 28 Aug 2023 01:58:04 AM UTC. 2023-08-28T01:59:29.321 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-08-28T01:59:29.322 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-08-28T01:59:29.322 INFO:teuthology.orchestra.run.smithi169.stdout: Package Architecture Version Repository Size 2023-08-28T01:59:29.323 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-08-28T01:59:29.323 INFO:teuthology.orchestra.run.smithi169.stdout:Resetting modules: 2023-08-28T01:59:29.323 INFO:teuthology.orchestra.run.smithi169.stdout: container-tools 2023-08-28T01:59:29.323 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T01:59:29.323 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction Summary 2023-08-28T01:59:29.323 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-08-28T01:59:29.323 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T01:59:29.420 INFO:teuthology.orchestra.run.smithi143.stdout:Dependencies resolved. 2023-08-28T01:59:29.421 INFO:teuthology.orchestra.run.smithi143.stdout:================================================================================ 2023-08-28T01:59:29.422 INFO:teuthology.orchestra.run.smithi143.stdout: Package Architecture Version Repository Size 2023-08-28T01:59:29.422 INFO:teuthology.orchestra.run.smithi143.stdout:================================================================================ 2023-08-28T01:59:29.422 INFO:teuthology.orchestra.run.smithi143.stdout:Resetting modules: 2023-08-28T01:59:29.422 INFO:teuthology.orchestra.run.smithi143.stdout: container-tools 2023-08-28T01:59:29.422 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T01:59:29.422 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction Summary 2023-08-28T01:59:29.422 INFO:teuthology.orchestra.run.smithi143.stdout:================================================================================ 2023-08-28T01:59:29.422 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T01:59:30.285 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:01:25 ago on Mon 28 Aug 2023 01:58:04 AM UTC. 2023-08-28T01:59:30.382 INFO:teuthology.orchestra.run.smithi143.stdout:Last metadata expiration check: 0:01:25 ago on Mon 28 Aug 2023 01:58:04 AM UTC. 2023-08-28T01:59:32.838 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-08-28T01:59:32.988 INFO:teuthology.orchestra.run.smithi143.stdout:Complete! 2023-08-28T01:59:33.294 INFO:teuthology.orchestra.run.smithi169.stdout:Updating Subscription Management repositories. 2023-08-28T01:59:33.453 INFO:teuthology.orchestra.run.smithi143.stdout:Updating Subscription Management repositories. 2023-08-28T01:59:34.073 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:01:29 ago on Mon 28 Aug 2023 01:58:04 AM UTC. 2023-08-28T01:59:34.220 INFO:teuthology.orchestra.run.smithi143.stdout:Last metadata expiration check: 0:01:29 ago on Mon 28 Aug 2023 01:58:04 AM UTC. 2023-08-28T01:59:35.912 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-08-28T01:59:35.922 INFO:teuthology.orchestra.run.smithi143.stdout:Dependencies resolved. 2023-08-28T01:59:36.006 INFO:teuthology.orchestra.run.smithi169.stdout:========================================================================================================================== 2023-08-28T01:59:36.006 INFO:teuthology.orchestra.run.smithi169.stdout: Package Arch Version Repository Size 2023-08-28T01:59:36.006 INFO:teuthology.orchestra.run.smithi169.stdout:========================================================================================================================== 2023-08-28T01:59:36.006 INFO:teuthology.orchestra.run.smithi169.stdout:Installing group/module packages: 2023-08-28T01:59:36.006 INFO:teuthology.orchestra.run.smithi169.stdout: buildah x86_64 1.19.9-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.8 M 2023-08-28T01:59:36.006 INFO:teuthology.orchestra.run.smithi169.stdout: cockpit-podman noarch 29-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 1.1 M 2023-08-28T01:59:36.006 INFO:teuthology.orchestra.run.smithi169.stdout: crun x86_64 0.18-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 185 k 2023-08-28T01:59:36.006 INFO:teuthology.orchestra.run.smithi169.stdout: skopeo x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.4 M 2023-08-28T01:59:36.006 INFO:teuthology.orchestra.run.smithi169.stdout: toolbox x86_64 0.0.99.3-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.2 M 2023-08-28T01:59:36.007 INFO:teuthology.orchestra.run.smithi169.stdout: udica noarch 0.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-08-28T01:59:36.007 INFO:teuthology.orchestra.run.smithi169.stdout:Installing dependencies: 2023-08-28T01:59:36.007 INFO:teuthology.orchestra.run.smithi169.stdout: yajl x86_64 2.1.0-10.el8 rhel-8-for-x86_64-appstream-rpms 41 k 2023-08-28T01:59:36.007 INFO:teuthology.orchestra.run.smithi169.stdout:Downgrading: 2023-08-28T01:59:36.007 INFO:teuthology.orchestra.run.smithi169.stdout: conmon x86_64 2:2.0.26-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-08-28T01:59:36.007 INFO:teuthology.orchestra.run.smithi169.stdout: container-selinux noarch 2:2.178.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 53 k 2023-08-28T01:59:36.007 INFO:teuthology.orchestra.run.smithi169.stdout: containernetworking-plugins x86_64 0.9.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 19 M 2023-08-28T01:59:36.007 INFO:teuthology.orchestra.run.smithi169.stdout: containers-common x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 91 k 2023-08-28T01:59:36.007 INFO:teuthology.orchestra.run.smithi169.stdout: criu x86_64 3.15-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 511 k 2023-08-28T01:59:36.007 INFO:teuthology.orchestra.run.smithi169.stdout: fuse-overlayfs x86_64 1.4.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 72 k 2023-08-28T01:59:36.007 INFO:teuthology.orchestra.run.smithi169.stdout: libslirp x86_64 4.3.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 69 k 2023-08-28T01:59:36.008 INFO:teuthology.orchestra.run.smithi169.stdout: podman x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 11 M 2023-08-28T01:59:36.008 INFO:teuthology.orchestra.run.smithi169.stdout: podman-catatonit x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 322 k 2023-08-28T01:59:36.008 INFO:teuthology.orchestra.run.smithi169.stdout: podman-docker noarch 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 56 k 2023-08-28T01:59:36.008 INFO:teuthology.orchestra.run.smithi169.stdout: runc x86_64 1.0.0-73.rc95.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.9 M 2023-08-28T01:59:36.008 INFO:teuthology.orchestra.run.smithi169.stdout: slirp4netns x86_64 1.1.8-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-08-28T01:59:36.008 INFO:teuthology.orchestra.run.smithi169.stdout:Installing module profiles: 2023-08-28T01:59:36.008 INFO:teuthology.orchestra.run.smithi169.stdout: container-tools/common 2023-08-28T01:59:36.008 INFO:teuthology.orchestra.run.smithi169.stdout:Enabling module streams: 2023-08-28T01:59:36.008 INFO:teuthology.orchestra.run.smithi169.stdout: container-tools 3.0 2023-08-28T01:59:36.008 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T01:59:36.008 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction Summary 2023-08-28T01:59:36.008 INFO:teuthology.orchestra.run.smithi169.stdout:========================================================================================================================== 2023-08-28T01:59:36.009 INFO:teuthology.orchestra.run.smithi169.stdout:Install 7 Packages 2023-08-28T01:59:36.009 INFO:teuthology.orchestra.run.smithi169.stdout:Downgrade 12 Packages 2023-08-28T01:59:36.009 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T01:59:36.009 INFO:teuthology.orchestra.run.smithi169.stdout:Total download size: 51 M 2023-08-28T01:59:36.009 INFO:teuthology.orchestra.run.smithi169.stdout:Downloading Packages: 2023-08-28T01:59:36.013 INFO:teuthology.orchestra.run.smithi143.stdout:========================================================================================================================== 2023-08-28T01:59:36.014 INFO:teuthology.orchestra.run.smithi143.stdout: Package Arch Version Repository Size 2023-08-28T01:59:36.014 INFO:teuthology.orchestra.run.smithi143.stdout:========================================================================================================================== 2023-08-28T01:59:36.014 INFO:teuthology.orchestra.run.smithi143.stdout:Installing group/module packages: 2023-08-28T01:59:36.014 INFO:teuthology.orchestra.run.smithi143.stdout: buildah x86_64 1.19.9-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.8 M 2023-08-28T01:59:36.014 INFO:teuthology.orchestra.run.smithi143.stdout: cockpit-podman noarch 29-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 1.1 M 2023-08-28T01:59:36.014 INFO:teuthology.orchestra.run.smithi143.stdout: crun x86_64 0.18-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 185 k 2023-08-28T01:59:36.014 INFO:teuthology.orchestra.run.smithi143.stdout: skopeo x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.4 M 2023-08-28T01:59:36.014 INFO:teuthology.orchestra.run.smithi143.stdout: toolbox x86_64 0.0.99.3-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.2 M 2023-08-28T01:59:36.014 INFO:teuthology.orchestra.run.smithi143.stdout: udica noarch 0.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-08-28T01:59:36.014 INFO:teuthology.orchestra.run.smithi143.stdout:Installing dependencies: 2023-08-28T01:59:36.014 INFO:teuthology.orchestra.run.smithi143.stdout: yajl x86_64 2.1.0-10.el8 rhel-8-for-x86_64-appstream-rpms 41 k 2023-08-28T01:59:36.015 INFO:teuthology.orchestra.run.smithi143.stdout:Downgrading: 2023-08-28T01:59:36.015 INFO:teuthology.orchestra.run.smithi143.stdout: conmon x86_64 2:2.0.26-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-08-28T01:59:36.015 INFO:teuthology.orchestra.run.smithi143.stdout: container-selinux noarch 2:2.178.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 53 k 2023-08-28T01:59:36.015 INFO:teuthology.orchestra.run.smithi143.stdout: containernetworking-plugins x86_64 0.9.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 19 M 2023-08-28T01:59:36.015 INFO:teuthology.orchestra.run.smithi143.stdout: containers-common x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 91 k 2023-08-28T01:59:36.015 INFO:teuthology.orchestra.run.smithi143.stdout: criu x86_64 3.15-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 511 k 2023-08-28T01:59:36.015 INFO:teuthology.orchestra.run.smithi143.stdout: fuse-overlayfs x86_64 1.4.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 72 k 2023-08-28T01:59:36.015 INFO:teuthology.orchestra.run.smithi143.stdout: libslirp x86_64 4.3.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 69 k 2023-08-28T01:59:36.015 INFO:teuthology.orchestra.run.smithi143.stdout: podman x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 11 M 2023-08-28T01:59:36.015 INFO:teuthology.orchestra.run.smithi143.stdout: podman-catatonit x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 322 k 2023-08-28T01:59:36.015 INFO:teuthology.orchestra.run.smithi143.stdout: podman-docker noarch 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 56 k 2023-08-28T01:59:36.015 INFO:teuthology.orchestra.run.smithi143.stdout: runc x86_64 1.0.0-73.rc95.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.9 M 2023-08-28T01:59:36.015 INFO:teuthology.orchestra.run.smithi143.stdout: slirp4netns x86_64 1.1.8-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-08-28T01:59:36.016 INFO:teuthology.orchestra.run.smithi143.stdout:Installing module profiles: 2023-08-28T01:59:36.016 INFO:teuthology.orchestra.run.smithi143.stdout: container-tools/common 2023-08-28T01:59:36.016 INFO:teuthology.orchestra.run.smithi143.stdout:Enabling module streams: 2023-08-28T01:59:36.016 INFO:teuthology.orchestra.run.smithi143.stdout: container-tools 3.0 2023-08-28T01:59:36.016 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T01:59:36.016 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction Summary 2023-08-28T01:59:36.016 INFO:teuthology.orchestra.run.smithi143.stdout:========================================================================================================================== 2023-08-28T01:59:36.016 INFO:teuthology.orchestra.run.smithi143.stdout:Install 7 Packages 2023-08-28T01:59:36.016 INFO:teuthology.orchestra.run.smithi143.stdout:Downgrade 12 Packages 2023-08-28T01:59:36.016 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T01:59:36.017 INFO:teuthology.orchestra.run.smithi143.stdout:Total download size: 51 M 2023-08-28T01:59:36.017 INFO:teuthology.orchestra.run.smithi143.stdout:Downloading Packages: 2023-08-28T01:59:36.101 INFO:teuthology.orchestra.run.smithi169.stdout:(1/19): slirp4netns-1.1.8-1.module+el8.6.0+1487 565 kB/s | 51 kB 00:00 2023-08-28T01:59:36.109 INFO:teuthology.orchestra.run.smithi143.stdout:(1/19): containers-common-1.2.4-1.module+el8.6. 992 kB/s | 91 kB 00:00 2023-08-28T01:59:36.134 INFO:teuthology.orchestra.run.smithi169.stdout:(2/19): containers-common-1.2.4-1.module+el8.6. 731 kB/s | 91 kB 00:00 2023-08-28T01:59:36.135 INFO:teuthology.orchestra.run.smithi143.stdout:(2/19): slirp4netns-1.1.8-1.module+el8.6.0+1487 439 kB/s | 51 kB 00:00 2023-08-28T01:59:36.160 INFO:teuthology.orchestra.run.smithi143.stdout:(3/19): container-selinux-2.178.0-2.module+el8. 372 kB/s | 53 kB 00:00 2023-08-28T01:59:36.160 INFO:teuthology.orchestra.run.smithi169.stdout:(3/19): container-selinux-2.178.0-2.module+el8. 353 kB/s | 53 kB 00:00 2023-08-28T01:59:36.193 INFO:teuthology.orchestra.run.smithi169.stdout:(4/19): criu-3.15-1.module+el8.6.0+14874+644362 5.4 MB/s | 511 kB 00:00 2023-08-28T01:59:36.193 INFO:teuthology.orchestra.run.smithi143.stdout:(4/19): criu-3.15-1.module+el8.6.0+14874+644362 6.0 MB/s | 511 kB 00:00 2023-08-28T01:59:36.218 INFO:teuthology.orchestra.run.smithi169.stdout:(5/19): libslirp-4.3.1-1.module+el8.6.0+14874+6 824 kB/s | 69 kB 00:00 2023-08-28T01:59:36.219 INFO:teuthology.orchestra.run.smithi143.stdout:(5/19): libslirp-4.3.1-1.module+el8.6.0+14874+6 824 kB/s | 69 kB 00:00 2023-08-28T01:59:36.252 INFO:teuthology.orchestra.run.smithi169.stdout:(6/19): fuse-overlayfs-1.4.0-2.module+el8.6.0+1 1.2 MB/s | 72 kB 00:00 2023-08-28T01:59:36.252 INFO:teuthology.orchestra.run.smithi143.stdout:(6/19): fuse-overlayfs-1.4.0-2.module+el8.6.0+1 1.2 MB/s | 72 kB 00:00 2023-08-28T01:59:36.311 INFO:teuthology.orchestra.run.smithi143.stdout:(7/19): runc-1.0.0-73.rc95.module+el8.6.0+14874 19 MB/s | 2.9 MB 00:00 2023-08-28T01:59:36.327 INFO:teuthology.orchestra.run.smithi169.stdout:(7/19): runc-1.0.0-73.rc95.module+el8.6.0+14874 17 MB/s | 2.9 MB 00:00 2023-08-28T01:59:36.336 INFO:teuthology.orchestra.run.smithi143.stdout:(8/19): podman-docker-3.0.1-9.module+el8.6.0+14 668 kB/s | 56 kB 00:00 2023-08-28T01:59:36.361 INFO:teuthology.orchestra.run.smithi143.stdout:(9/19): conmon-2.0.26-1.module+el8.6.0+14874+64 1.0 MB/s | 51 kB 00:00 2023-08-28T01:59:36.378 INFO:teuthology.orchestra.run.smithi169.stdout:(8/19): podman-docker-3.0.1-9.module+el8.6.0+14 446 kB/s | 56 kB 00:00 2023-08-28T01:59:36.403 INFO:teuthology.orchestra.run.smithi169.stdout:(9/19): conmon-2.0.26-1.module+el8.6.0+14874+64 677 kB/s | 51 kB 00:00 2023-08-28T01:59:36.429 INFO:teuthology.orchestra.run.smithi143.stdout:(10/19): podman-catatonit-3.0.1-9.module+el8.6. 4.7 MB/s | 322 kB 00:00 2023-08-28T01:59:36.453 INFO:teuthology.orchestra.run.smithi169.stdout:(10/19): podman-catatonit-3.0.1-9.module+el8.6. 6.3 MB/s | 322 kB 00:00 2023-08-28T01:59:36.487 INFO:teuthology.orchestra.run.smithi143.stdout:(11/19): yajl-2.1.0-10.el8.x86_64.rpm 700 kB/s | 41 kB 00:00 2023-08-28T01:59:36.495 INFO:teuthology.orchestra.run.smithi169.stdout:(11/19): yajl-2.1.0-10.el8.x86_64.rpm 982 kB/s | 41 kB 00:00 2023-08-28T01:59:36.612 INFO:teuthology.orchestra.run.smithi169.stdout:(12/19): toolbox-0.0.99.3-1.module+el8.6.0+1487 19 MB/s | 2.2 MB 00:00 2023-08-28T01:59:36.663 INFO:teuthology.orchestra.run.smithi143.stdout:(12/19): podman-3.0.1-9.module+el8.6.0+14874+64 25 MB/s | 11 MB 00:00 2023-08-28T01:59:36.713 INFO:teuthology.orchestra.run.smithi143.stdout:(13/19): toolbox-0.0.99.3-1.module+el8.6.0+1487 9.7 MB/s | 2.2 MB 00:00 2023-08-28T01:59:36.754 INFO:teuthology.orchestra.run.smithi169.stdout:(13/19): podman-3.0.1-9.module+el8.6.0+14874+64 20 MB/s | 11 MB 00:00 2023-08-28T01:59:36.905 INFO:teuthology.orchestra.run.smithi143.stdout:(14/19): skopeo-1.2.4-1.module+el8.6.0+14874+64 26 MB/s | 6.4 MB 00:00 2023-08-28T01:59:36.921 INFO:teuthology.orchestra.run.smithi169.stdout:(14/19): skopeo-1.2.4-1.module+el8.6.0+14874+64 21 MB/s | 6.4 MB 00:00 2023-08-28T01:59:37.030 INFO:teuthology.orchestra.run.smithi169.stdout:(15/19): buildah-1.19.9-3.module+el8.6.0+14874+ 25 MB/s | 6.8 MB 00:00 2023-08-28T01:59:37.105 INFO:teuthology.orchestra.run.smithi143.stdout:(15/19): containernetworking-plugins-0.9.1-1.mo 25 MB/s | 19 MB 00:00 2023-08-28T01:59:37.156 INFO:teuthology.orchestra.run.smithi143.stdout:(16/19): cockpit-podman-29-2.module+el8.6.0+148 4.3 MB/s | 1.1 MB 00:00 2023-08-28T01:59:37.181 INFO:teuthology.orchestra.run.smithi143.stdout:(17/19): crun-0.18-3.module+el8.6.0+14874+64436 2.4 MB/s | 185 kB 00:00 2023-08-28T01:59:37.230 INFO:teuthology.orchestra.run.smithi169.stdout:(16/19): containernetworking-plugins-0.9.1-1.mo 22 MB/s | 19 MB 00:00 2023-08-28T01:59:37.255 INFO:teuthology.orchestra.run.smithi169.stdout:(17/19): crun-0.18-3.module+el8.6.0+14874+64436 820 kB/s | 185 kB 00:00 2023-08-28T01:59:37.256 INFO:teuthology.orchestra.run.smithi143.stdout:(18/19): buildah-1.19.9-3.module+el8.6.0+14874+ 12 MB/s | 6.8 MB 00:00 2023-08-28T01:59:37.272 INFO:teuthology.orchestra.run.smithi169.stdout:(18/19): udica-0.2.4-1.module+el8.6.0+14874+644 1.2 MB/s | 51 kB 00:00 2023-08-28T01:59:37.281 INFO:teuthology.orchestra.run.smithi143.stdout:(19/19): udica-0.2.4-1.module+el8.6.0+14874+644 403 kB/s | 51 kB 00:00 2023-08-28T01:59:37.282 INFO:teuthology.orchestra.run.smithi143.stdout:-------------------------------------------------------------------------------- 2023-08-28T01:59:37.282 INFO:teuthology.orchestra.run.smithi143.stdout:Total 40 MB/s | 51 MB 00:01 2023-08-28T01:59:37.306 INFO:teuthology.orchestra.run.smithi169.stdout:(19/19): cockpit-podman-29-2.module+el8.6.0+148 2.8 MB/s | 1.1 MB 00:00 2023-08-28T01:59:37.306 INFO:teuthology.orchestra.run.smithi169.stdout:-------------------------------------------------------------------------------- 2023-08-28T01:59:37.306 INFO:teuthology.orchestra.run.smithi169.stdout:Total 39 MB/s | 51 MB 00:01 2023-08-28T01:59:37.769 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction check 2023-08-28T01:59:37.787 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction check 2023-08-28T01:59:37.812 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction check succeeded. 2023-08-28T01:59:37.812 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction test 2023-08-28T01:59:37.829 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction check succeeded. 2023-08-28T01:59:37.829 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction test 2023-08-28T01:59:38.241 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction test succeeded. 2023-08-28T01:59:38.245 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction 2023-08-28T01:59:38.257 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction test succeeded. 2023-08-28T01:59:38.261 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction 2023-08-28T01:59:39.391 INFO:teuthology.orchestra.run.smithi143.stdout: Preparing : 1/1 2023-08-28T01:59:39.450 INFO:teuthology.orchestra.run.smithi169.stdout: Preparing : 1/1 2023-08-28T01:59:39.454 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299 1/1 2023-08-28T01:59:39.506 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299 1/1 2023-08-28T01:59:39.638 INFO:teuthology.orchestra.run.smithi143.stdout: Downgrading : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-08-28T01:59:39.647 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-08-28T01:59:39.662 INFO:teuthology.orchestra.run.smithi169.stdout: Downgrading : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-08-28T01:59:39.664 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-08-28T01:59:39.671 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-08-28T01:59:39.689 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-08-28T01:59:39.846 INFO:teuthology.orchestra.run.smithi143.stdout: Downgrading : container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-08-28T01:59:39.887 INFO:teuthology.orchestra.run.smithi169.stdout: Downgrading : container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-08-28T01:59:56.872 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-08-28T01:59:57.106 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-08-28T01:59:58.656 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : yajl-2.1.0-10.el8.x86_64 3/31 2023-08-28T01:59:58.873 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : yajl-2.1.0-10.el8.x86_64 3/31 2023-08-28T01:59:58.901 INFO:teuthology.orchestra.run.smithi143.stdout: Downgrading : containernetworking-plugins-0.9.1-1.module+el8.6.0 4/31 2023-08-28T01:59:59.093 INFO:teuthology.orchestra.run.smithi143.stdout: Downgrading : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 5/31 2023-08-28T01:59:59.132 INFO:teuthology.orchestra.run.smithi169.stdout: Downgrading : containernetworking-plugins-0.9.1-1.module+el8.6.0 4/31 2023-08-28T01:59:59.266 INFO:teuthology.orchestra.run.smithi143.stdout: Downgrading : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 6/31 2023-08-28T01:59:59.330 INFO:teuthology.orchestra.run.smithi169.stdout: Downgrading : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 5/31 2023-08-28T01:59:59.494 INFO:teuthology.orchestra.run.smithi143.stdout: Downgrading : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 7/31 2023-08-28T01:59:59.495 INFO:teuthology.orchestra.run.smithi143.stdout: Downgrading : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 8/31 2023-08-28T01:59:59.495 INFO:teuthology.orchestra.run.smithi143.stdout:warning: /etc/containers/registries.conf created as /etc/containers/registries.conf.rpmnew 2023-08-28T01:59:59.495 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T01:59:59.521 INFO:teuthology.orchestra.run.smithi169.stdout: Downgrading : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 6/31 2023-08-28T01:59:59.726 INFO:teuthology.orchestra.run.smithi169.stdout: Downgrading : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 7/31 2023-08-28T01:59:59.726 INFO:teuthology.orchestra.run.smithi169.stdout: Downgrading : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 8/31 2023-08-28T01:59:59.726 INFO:teuthology.orchestra.run.smithi169.stdout:warning: /etc/containers/registries.conf created as /etc/containers/registries.conf.rpmnew 2023-08-28T01:59:59.727 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:00:00.365 INFO:teuthology.orchestra.run.smithi143.stdout: Downgrading : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 9/31 2023-08-28T02:00:00.578 INFO:teuthology.orchestra.run.smithi169.stdout: Downgrading : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 9/31 2023-08-28T02:00:00.635 INFO:teuthology.orchestra.run.smithi143.stdout: Downgrading : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 10/31 2023-08-28T02:00:00.775 INFO:teuthology.orchestra.run.smithi169.stdout: Downgrading : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 10/31 2023-08-28T02:00:02.107 INFO:teuthology.orchestra.run.smithi143.stdout: Downgrading : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 11/31 2023-08-28T02:00:02.207 INFO:teuthology.orchestra.run.smithi169.stdout: Downgrading : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 11/31 2023-08-28T02:00:02.370 INFO:teuthology.orchestra.run.smithi143.stdout: Downgrading : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 12/31 2023-08-28T02:00:02.503 INFO:teuthology.orchestra.run.smithi169.stdout: Downgrading : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 12/31 2023-08-28T02:00:02.734 INFO:teuthology.orchestra.run.smithi143.stdout: Downgrading : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 13/31 2023-08-28T02:00:02.905 INFO:teuthology.orchestra.run.smithi169.stdout: Downgrading : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 13/31 2023-08-28T02:00:03.030 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 14/31 2023-08-28T02:00:03.206 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 14/31 2023-08-28T02:00:03.761 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 15/31 2023-08-28T02:00:03.958 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 15/31 2023-08-28T02:00:04.501 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 16/31 2023-08-28T02:00:04.688 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 16/31 2023-08-28T02:00:04.733 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 17/31 2023-08-28T02:00:04.898 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 17/31 2023-08-28T02:00:04.924 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 18/31 2023-08-28T02:00:05.111 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 18/31 2023-08-28T02:00:05.118 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 19/31 2023-08-28T02:00:05.257 INFO:teuthology.orchestra.run.smithi143.stdout: Cleanup : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 20/31 2023-08-28T02:00:05.257 INFO:teuthology.orchestra.run.smithi143.stdout: Cleanup : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-08-28T02:00:05.291 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 19/31 2023-08-28T02:00:05.422 INFO:teuthology.orchestra.run.smithi169.stdout: Cleanup : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 20/31 2023-08-28T02:00:05.422 INFO:teuthology.orchestra.run.smithi169.stdout: Cleanup : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-08-28T02:00:05.571 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-08-28T02:00:05.735 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-08-28T02:00:05.770 INFO:teuthology.orchestra.run.smithi143.stdout: Cleanup : containers-common-2:1-27.module+el8.6.0+14877+f643 22/31 2023-08-28T02:00:05.770 INFO:teuthology.orchestra.run.smithi143.stdout: Cleanup : container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-08-28T02:00:05.934 INFO:teuthology.orchestra.run.smithi169.stdout: Cleanup : containers-common-2:1-27.module+el8.6.0+14877+f643 22/31 2023-08-28T02:00:05.934 INFO:teuthology.orchestra.run.smithi169.stdout: Cleanup : container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-08-28T02:00:05.935 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-08-28T02:00:06.084 INFO:teuthology.orchestra.run.smithi143.stdout: Cleanup : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-08-28T02:00:06.109 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-08-28T02:00:06.241 INFO:teuthology.orchestra.run.smithi169.stdout: Cleanup : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-08-28T02:00:06.243 INFO:teuthology.orchestra.run.smithi143.stdout: Cleanup : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 25/31 2023-08-28T02:00:06.401 INFO:teuthology.orchestra.run.smithi143.stdout: Cleanup : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 26/31 2023-08-28T02:00:06.442 INFO:teuthology.orchestra.run.smithi169.stdout: Cleanup : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 25/31 2023-08-28T02:00:06.587 INFO:teuthology.orchestra.run.smithi143.stdout: Cleanup : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 27/31 2023-08-28T02:00:06.600 INFO:teuthology.orchestra.run.smithi169.stdout: Cleanup : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 26/31 2023-08-28T02:00:06.743 INFO:teuthology.orchestra.run.smithi143.stdout: Cleanup : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 28/31 2023-08-28T02:00:06.792 INFO:teuthology.orchestra.run.smithi169.stdout: Cleanup : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 27/31 2023-08-28T02:00:06.884 INFO:teuthology.orchestra.run.smithi143.stdout: Cleanup : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 29/31 2023-08-28T02:00:06.933 INFO:teuthology.orchestra.run.smithi169.stdout: Cleanup : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 28/31 2023-08-28T02:00:07.031 INFO:teuthology.orchestra.run.smithi143.stdout: Cleanup : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 30/31 2023-08-28T02:00:07.031 INFO:teuthology.orchestra.run.smithi143.stdout: Cleanup : containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-08-28T02:00:07.074 INFO:teuthology.orchestra.run.smithi169.stdout: Cleanup : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 29/31 2023-08-28T02:00:07.184 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-08-28T02:00:07.240 INFO:teuthology.orchestra.run.smithi169.stdout: Cleanup : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 30/31 2023-08-28T02:00:07.241 INFO:teuthology.orchestra.run.smithi169.stdout: Cleanup : containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-08-28T02:00:07.399 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-08-28T02:00:07.694 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 31/31 2023-08-28T02:00:07.934 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 31/31 2023-08-28T02:00:08.068 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-08-28T02:00:08.068 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 1/31 2023-08-28T02:00:08.068 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : containers-common-2:1-27.module+el8.6.0+14877+f643 2/31 2023-08-28T02:00:08.068 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : container-selinux-2:2.178.0-2.module+el8.6.0+14874 3/31 2023-08-28T02:00:08.068 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : container-selinux-2:2.179.1-1.module+el8.6.0+14877 4/31 2023-08-28T02:00:08.068 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 5/31 2023-08-28T02:00:08.068 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 6/31 2023-08-28T02:00:08.069 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 7/31 2023-08-28T02:00:08.069 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 8/31 2023-08-28T02:00:08.069 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 9/31 2023-08-28T02:00:08.069 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 10/31 2023-08-28T02:00:08.069 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 11/31 2023-08-28T02:00:08.069 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 12/31 2023-08-28T02:00:08.069 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 13/31 2023-08-28T02:00:08.069 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 14/31 2023-08-28T02:00:08.069 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 15/31 2023-08-28T02:00:08.069 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 16/31 2023-08-28T02:00:08.069 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 17/31 2023-08-28T02:00:08.069 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 18/31 2023-08-28T02:00:08.070 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 19/31 2023-08-28T02:00:08.070 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 20/31 2023-08-28T02:00:08.070 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : containernetworking-plugins-0.9.1-1.module+el8.6.0 21/31 2023-08-28T02:00:08.070 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : containernetworking-plugins-1:1.0.1-2.module+el8.6 22/31 2023-08-28T02:00:08.070 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 23/31 2023-08-28T02:00:08.070 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-08-28T02:00:08.070 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : yajl-2.1.0-10.el8.x86_64 25/31 2023-08-28T02:00:08.070 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 26/31 2023-08-28T02:00:08.070 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 27/31 2023-08-28T02:00:08.071 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 28/31 2023-08-28T02:00:08.071 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 29/31 2023-08-28T02:00:08.071 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 30/31 2023-08-28T02:00:08.331 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-08-28T02:00:08.331 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 1/31 2023-08-28T02:00:08.331 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : containers-common-2:1-27.module+el8.6.0+14877+f643 2/31 2023-08-28T02:00:08.331 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : container-selinux-2:2.178.0-2.module+el8.6.0+14874 3/31 2023-08-28T02:00:08.331 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : container-selinux-2:2.179.1-1.module+el8.6.0+14877 4/31 2023-08-28T02:00:08.331 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 5/31 2023-08-28T02:00:08.331 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 6/31 2023-08-28T02:00:08.331 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 7/31 2023-08-28T02:00:08.332 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 8/31 2023-08-28T02:00:08.332 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 9/31 2023-08-28T02:00:08.332 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 10/31 2023-08-28T02:00:08.332 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 11/31 2023-08-28T02:00:08.332 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 12/31 2023-08-28T02:00:08.332 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 13/31 2023-08-28T02:00:08.332 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 14/31 2023-08-28T02:00:08.332 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 15/31 2023-08-28T02:00:08.332 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 16/31 2023-08-28T02:00:08.332 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 17/31 2023-08-28T02:00:08.332 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 18/31 2023-08-28T02:00:08.332 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 19/31 2023-08-28T02:00:08.333 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 20/31 2023-08-28T02:00:08.333 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : containernetworking-plugins-0.9.1-1.module+el8.6.0 21/31 2023-08-28T02:00:08.333 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : containernetworking-plugins-1:1.0.1-2.module+el8.6 22/31 2023-08-28T02:00:08.333 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 23/31 2023-08-28T02:00:08.333 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-08-28T02:00:08.333 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : yajl-2.1.0-10.el8.x86_64 25/31 2023-08-28T02:00:08.333 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 26/31 2023-08-28T02:00:08.333 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 27/31 2023-08-28T02:00:08.333 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 28/31 2023-08-28T02:00:08.333 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 29/31 2023-08-28T02:00:08.333 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 30/31 2023-08-28T02:00:09.453 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 31/31 2023-08-28T02:00:09.453 INFO:teuthology.orchestra.run.smithi143.stdout:Last metadata expiration check: 0:02:04 ago on Mon 28 Aug 2023 01:58:04 AM UTC. 2023-08-28T02:00:09.724 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 31/31 2023-08-28T02:00:09.724 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:02:05 ago on Mon 28 Aug 2023 01:58:04 AM UTC. 2023-08-28T02:00:12.245 INFO:teuthology.orchestra.run.smithi143.stdout:Installed products updated. 2023-08-28T02:00:12.371 INFO:teuthology.orchestra.run.smithi169.stdout:Installed products updated. 2023-08-28T02:00:12.452 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:00:12.452 INFO:teuthology.orchestra.run.smithi143.stdout:Downgraded: 2023-08-28T02:00:12.452 INFO:teuthology.orchestra.run.smithi143.stdout: conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x86_64 2023-08-28T02:00:12.452 INFO:teuthology.orchestra.run.smithi143.stdout: container-selinux-2:2.178.0-2.module+el8.6.0+14874+64436299.noarch 2023-08-28T02:00:12.453 INFO:teuthology.orchestra.run.smithi143.stdout: containernetworking-plugins-0.9.1-1.module+el8.6.0+14874+64436299.x86_64 2023-08-28T02:00:12.453 INFO:teuthology.orchestra.run.smithi143.stdout: containers-common-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-08-28T02:00:12.453 INFO:teuthology.orchestra.run.smithi143.stdout: criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 2023-08-28T02:00:12.453 INFO:teuthology.orchestra.run.smithi143.stdout: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299.x86_64 2023-08-28T02:00:12.453 INFO:teuthology.orchestra.run.smithi143.stdout: libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86_64 2023-08-28T02:00:12.453 INFO:teuthology.orchestra.run.smithi143.stdout: podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-08-28T02:00:12.453 INFO:teuthology.orchestra.run.smithi143.stdout: podman-catatonit-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-08-28T02:00:12.453 INFO:teuthology.orchestra.run.smithi143.stdout: podman-docker-3.0.1-9.module+el8.6.0+14874+64436299.noarch 2023-08-28T02:00:12.453 INFO:teuthology.orchestra.run.smithi143.stdout: runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x86_64 2023-08-28T02:00:12.453 INFO:teuthology.orchestra.run.smithi143.stdout: slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299.x86_64 2023-08-28T02:00:12.453 INFO:teuthology.orchestra.run.smithi143.stdout:Installed: 2023-08-28T02:00:12.453 INFO:teuthology.orchestra.run.smithi143.stdout: buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86_64 2023-08-28T02:00:12.454 INFO:teuthology.orchestra.run.smithi143.stdout: cockpit-podman-29-2.module+el8.6.0+14874+64436299.noarch 2023-08-28T02:00:12.454 INFO:teuthology.orchestra.run.smithi143.stdout: crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 2023-08-28T02:00:12.454 INFO:teuthology.orchestra.run.smithi143.stdout: skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-08-28T02:00:12.454 INFO:teuthology.orchestra.run.smithi143.stdout: toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x86_64 2023-08-28T02:00:12.454 INFO:teuthology.orchestra.run.smithi143.stdout: udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 2023-08-28T02:00:12.454 INFO:teuthology.orchestra.run.smithi143.stdout: yajl-2.1.0-10.el8.x86_64 2023-08-28T02:00:12.454 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:00:12.454 INFO:teuthology.orchestra.run.smithi143.stdout:Complete! 2023-08-28T02:00:12.567 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:00:12.568 INFO:teuthology.orchestra.run.smithi169.stdout:Downgraded: 2023-08-28T02:00:12.568 INFO:teuthology.orchestra.run.smithi169.stdout: conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x86_64 2023-08-28T02:00:12.568 INFO:teuthology.orchestra.run.smithi169.stdout: container-selinux-2:2.178.0-2.module+el8.6.0+14874+64436299.noarch 2023-08-28T02:00:12.568 INFO:teuthology.orchestra.run.smithi169.stdout: containernetworking-plugins-0.9.1-1.module+el8.6.0+14874+64436299.x86_64 2023-08-28T02:00:12.568 INFO:teuthology.orchestra.run.smithi169.stdout: containers-common-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-08-28T02:00:12.568 INFO:teuthology.orchestra.run.smithi169.stdout: criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 2023-08-28T02:00:12.568 INFO:teuthology.orchestra.run.smithi169.stdout: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299.x86_64 2023-08-28T02:00:12.568 INFO:teuthology.orchestra.run.smithi169.stdout: libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86_64 2023-08-28T02:00:12.568 INFO:teuthology.orchestra.run.smithi169.stdout: podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-08-28T02:00:12.568 INFO:teuthology.orchestra.run.smithi169.stdout: podman-catatonit-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-08-28T02:00:12.568 INFO:teuthology.orchestra.run.smithi169.stdout: podman-docker-3.0.1-9.module+el8.6.0+14874+64436299.noarch 2023-08-28T02:00:12.568 INFO:teuthology.orchestra.run.smithi169.stdout: runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x86_64 2023-08-28T02:00:12.569 INFO:teuthology.orchestra.run.smithi169.stdout: slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299.x86_64 2023-08-28T02:00:12.569 INFO:teuthology.orchestra.run.smithi169.stdout:Installed: 2023-08-28T02:00:12.569 INFO:teuthology.orchestra.run.smithi169.stdout: buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86_64 2023-08-28T02:00:12.569 INFO:teuthology.orchestra.run.smithi169.stdout: cockpit-podman-29-2.module+el8.6.0+14874+64436299.noarch 2023-08-28T02:00:12.569 INFO:teuthology.orchestra.run.smithi169.stdout: crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 2023-08-28T02:00:12.569 INFO:teuthology.orchestra.run.smithi169.stdout: skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-08-28T02:00:12.569 INFO:teuthology.orchestra.run.smithi169.stdout: toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x86_64 2023-08-28T02:00:12.569 INFO:teuthology.orchestra.run.smithi169.stdout: udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 2023-08-28T02:00:12.569 INFO:teuthology.orchestra.run.smithi169.stdout: yajl-2.1.0-10.el8.x86_64 2023-08-28T02:00:12.569 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:00:12.569 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-08-28T02:00:12.649 DEBUG:teuthology.parallel:result is None 2023-08-28T02:00:12.788 DEBUG:teuthology.parallel:result is None 2023-08-28T02:00:12.789 INFO:teuthology.run_tasks:Running task install... 2023-08-28T02:00:12.799 DEBUG:teuthology.task.install:project ceph 2023-08-28T02:00:12.799 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': '9cf208207dfe23ffccb33308b22e51570cb45e51'}} 2023-08-28T02:00:12.799 DEBUG:teuthology.task.install:config {'branch': 'pacific', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': '9cf208207dfe23ffccb33308b22e51570cb45e51'} 2023-08-28T02:00:12.799 INFO:teuthology.task.install:Using flavor: default 2023-08-28T02:00:12.805 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2023-08-28T02:00:12.805 INFO:teuthology.task.install:extra packages: [] 2023-08-28T02:00:12.805 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'pacific', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '9cf208207dfe23ffccb33308b22e51570cb45e51', 'tag': None, 'wait_for_package': False} 2023-08-28T02:00:12.805 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-08-28T02:00:12.806 INFO:teuthology.packaging:ref: None 2023-08-28T02:00:12.806 INFO:teuthology.packaging:tag: None 2023-08-28T02:00:12.806 INFO:teuthology.packaging:branch: pacific 2023-08-28T02:00:12.806 INFO:teuthology.packaging:sha1: 9cf208207dfe23ffccb33308b22e51570cb45e51 2023-08-28T02:00:12.806 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=pacific 2023-08-28T02:00:12.808 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'pacific', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '9cf208207dfe23ffccb33308b22e51570cb45e51', 'tag': None, 'wait_for_package': False} 2023-08-28T02:00:12.808 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-08-28T02:00:12.808 INFO:teuthology.packaging:ref: None 2023-08-28T02:00:12.808 INFO:teuthology.packaging:tag: None 2023-08-28T02:00:12.808 INFO:teuthology.packaging:branch: pacific 2023-08-28T02:00:12.808 INFO:teuthology.packaging:sha1: 9cf208207dfe23ffccb33308b22e51570cb45e51 2023-08-28T02:00:12.808 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=pacific 2023-08-28T02:00:13.038 INFO:teuthology.task.install.rpm:Pulling from https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/centos/8/flavors/default/ 2023-08-28T02:00:13.039 INFO:teuthology.task.install.rpm:Package version is 16.2.13-565.g595355f4 2023-08-28T02:00:13.146 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-08-28T02:00:13.146 DEBUG:teuthology.orchestra.run.smithi143:> set -ex 2023-08-28T02:00:13.147 DEBUG:teuthology.orchestra.run.smithi143:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-08-28T02:00:13.149 INFO:teuthology.task.install.rpm:Pulling from https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/centos/8/flavors/default/ 2023-08-28T02:00:13.150 INFO:teuthology.task.install.rpm:Package version is 16.2.13-565.g595355f4 2023-08-28T02:00:13.181 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-08-28T02:00:13.181 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-08-28T02:00:13.181 INFO:teuthology.packaging:ref: None 2023-08-28T02:00:13.181 INFO:teuthology.packaging:tag: None 2023-08-28T02:00:13.182 INFO:teuthology.packaging:branch: pacific 2023-08-28T02:00:13.182 INFO:teuthology.packaging:sha1: 9cf208207dfe23ffccb33308b22e51570cb45e51 2023-08-28T02:00:13.182 DEBUG:teuthology.orchestra.run.smithi143:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/pacific/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-08-28T02:00:13.250 DEBUG:teuthology.orchestra.run.smithi143:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-08-28T02:00:13.314 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://4.chacra.ceph.com/r/ceph/pacific/595355f450898939abedfa92883f9a78b18c37aa/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-08-28T02:00:13.314 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2023-08-28T02:00:13.314 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-08-28T02:00:13.327 DEBUG:teuthology.orchestra.run.smithi143:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-08-28T02:00:13.343 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-08-28T02:00:13.343 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-08-28T02:00:13.343 INFO:teuthology.packaging:ref: None 2023-08-28T02:00:13.343 INFO:teuthology.packaging:tag: None 2023-08-28T02:00:13.343 INFO:teuthology.packaging:branch: pacific 2023-08-28T02:00:13.343 INFO:teuthology.packaging:sha1: 9cf208207dfe23ffccb33308b22e51570cb45e51 2023-08-28T02:00:13.343 DEBUG:teuthology.orchestra.run.smithi169:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/pacific/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-08-28T02:00:13.394 INFO:teuthology.orchestra.run.smithi143.stdout:check_obsoletes = 1 2023-08-28T02:00:13.396 DEBUG:teuthology.orchestra.run.smithi143:> sudo yum clean all 2023-08-28T02:00:13.412 DEBUG:teuthology.orchestra.run.smithi169:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-08-28T02:00:13.489 DEBUG:teuthology.orchestra.run.smithi169:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-08-28T02:00:13.556 INFO:teuthology.orchestra.run.smithi169.stdout:check_obsoletes = 1 2023-08-28T02:00:13.557 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum clean all 2023-08-28T02:00:13.766 INFO:teuthology.orchestra.run.smithi143.stdout:Updating Subscription Management repositories. 2023-08-28T02:00:13.927 INFO:teuthology.orchestra.run.smithi169.stdout:Updating Subscription Management repositories. 2023-08-28T02:00:14.019 INFO:teuthology.orchestra.run.smithi143.stdout:44 files removed 2023-08-28T02:00:14.066 DEBUG:teuthology.orchestra.run.smithi143:> sudo yum -y install ceph-radosgw 2023-08-28T02:00:14.160 INFO:teuthology.orchestra.run.smithi169.stdout:44 files removed 2023-08-28T02:00:14.206 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install ceph-radosgw 2023-08-28T02:00:14.398 INFO:teuthology.orchestra.run.smithi143.stdout:Updating Subscription Management repositories. 2023-08-28T02:00:14.537 INFO:teuthology.orchestra.run.smithi169.stdout:Updating Subscription Management repositories. 2023-08-28T02:00:15.123 INFO:teuthology.orchestra.run.smithi143.stdout:ceph packages for x86_64 234 kB/s | 76 kB 00:00 2023-08-28T02:00:15.308 INFO:teuthology.orchestra.run.smithi169.stdout:ceph packages for x86_64 193 kB/s | 76 kB 00:00 2023-08-28T02:00:15.474 INFO:teuthology.orchestra.run.smithi143.stdout:ceph noarch packages 48 kB/s | 15 kB 00:00 2023-08-28T02:00:15.609 INFO:teuthology.orchestra.run.smithi169.stdout:ceph noarch packages 57 kB/s | 15 kB 00:00 2023-08-28T02:00:15.807 INFO:teuthology.orchestra.run.smithi143.stdout:ceph source packages 4.6 kB/s | 1.4 kB 00:00 2023-08-28T02:00:15.933 INFO:teuthology.orchestra.run.smithi143.stdout:Copr repo for python3-asyncssh owned by ceph 33 kB/s | 3.5 kB 00:00 2023-08-28T02:00:15.950 INFO:teuthology.orchestra.run.smithi169.stdout:ceph source packages 4.5 kB/s | 1.4 kB 00:00 2023-08-28T02:00:16.102 INFO:teuthology.orchestra.run.smithi169.stdout:Copr repo for python3-asyncssh owned by ceph 26 kB/s | 3.5 kB 00:00 2023-08-28T02:00:17.161 INFO:teuthology.orchestra.run.smithi143.stdout:Extra Packages for Enterprise Linux 13 MB/s | 16 MB 00:01 2023-08-28T02:00:17.371 INFO:teuthology.orchestra.run.smithi169.stdout:Extra Packages for Enterprise Linux 13 MB/s | 16 MB 00:01 2023-08-28T02:00:20.651 INFO:teuthology.orchestra.run.smithi143.stdout:lab-extras 288 kB/s | 24 kB 00:00 2023-08-28T02:00:20.827 INFO:teuthology.orchestra.run.smithi169.stdout:lab-extras 590 kB/s | 24 kB 00:00 2023-08-28T02:00:21.552 INFO:teuthology.orchestra.run.smithi143.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 64 MB/s | 53 MB 00:00 2023-08-28T02:00:21.729 INFO:teuthology.orchestra.run.smithi169.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 57 MB/s | 47 MB 00:00 2023-08-28T02:00:29.630 INFO:teuthology.orchestra.run.smithi169.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 59 MB/s | 53 MB 00:00 2023-08-28T02:00:29.788 INFO:teuthology.orchestra.run.smithi143.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 58 MB/s | 47 MB 00:00 2023-08-28T02:00:40.714 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-08-28T02:00:40.717 INFO:teuthology.orchestra.run.smithi169.stdout:==================================================================================================== 2023-08-28T02:00:40.717 INFO:teuthology.orchestra.run.smithi169.stdout: Package Arch Version Repository Size 2023-08-28T02:00:40.717 INFO:teuthology.orchestra.run.smithi169.stdout:==================================================================================================== 2023-08-28T02:00:40.717 INFO:teuthology.orchestra.run.smithi169.stdout:Installing: 2023-08-28T02:00:40.717 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-radosgw x86_64 2:16.2.13-565.g595355f4.el8 ceph 12 M 2023-08-28T02:00:40.717 INFO:teuthology.orchestra.run.smithi169.stdout:Upgrading: 2023-08-28T02:00:40.718 INFO:teuthology.orchestra.run.smithi169.stdout: librados2 x86_64 2:16.2.13-565.g595355f4.el8 ceph 3.7 M 2023-08-28T02:00:40.718 INFO:teuthology.orchestra.run.smithi169.stdout: librbd1 x86_64 2:16.2.13-565.g595355f4.el8 ceph 4.2 M 2023-08-28T02:00:40.718 INFO:teuthology.orchestra.run.smithi169.stdout:Installing dependencies: 2023-08-28T02:00:40.718 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-base x86_64 2:16.2.13-565.g595355f4.el8 ceph 6.1 M 2023-08-28T02:00:40.718 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-common x86_64 2:16.2.13-565.g595355f4.el8 ceph 24 M 2023-08-28T02:00:40.718 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-selinux x86_64 2:16.2.13-565.g595355f4.el8 ceph 26 k 2023-08-28T02:00:40.718 INFO:teuthology.orchestra.run.smithi169.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-08-28T02:00:40.718 INFO:teuthology.orchestra.run.smithi169.stdout: leveldb x86_64 1.22-1.el8 epel 181 k 2023-08-28T02:00:40.718 INFO:teuthology.orchestra.run.smithi169.stdout: libcephfs2 x86_64 2:16.2.13-565.g595355f4.el8 ceph 824 k 2023-08-28T02:00:40.719 INFO:teuthology.orchestra.run.smithi169.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-08-28T02:00:40.719 INFO:teuthology.orchestra.run.smithi169.stdout: librabbitmq x86_64 0.9.0-3.el8 rhel-8-for-x86_64-baseos-rpms 47 k 2023-08-28T02:00:40.719 INFO:teuthology.orchestra.run.smithi169.stdout: libradosstriper1 x86_64 2:16.2.13-565.g595355f4.el8 ceph 494 k 2023-08-28T02:00:40.719 INFO:teuthology.orchestra.run.smithi169.stdout: librdkafka x86_64 0.11.4-3.el8 rhel-8-for-x86_64-appstream-rpms 354 k 2023-08-28T02:00:40.719 INFO:teuthology.orchestra.run.smithi169.stdout: librgw2 x86_64 2:16.2.13-565.g595355f4.el8 ceph 3.8 M 2023-08-28T02:00:40.719 INFO:teuthology.orchestra.run.smithi169.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-08-28T02:00:40.719 INFO:teuthology.orchestra.run.smithi169.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2023-08-28T02:00:40.719 INFO:teuthology.orchestra.run.smithi169.stdout: python3-ceph-argparse x86_64 2:16.2.13-565.g595355f4.el8 ceph 45 k 2023-08-28T02:00:40.719 INFO:teuthology.orchestra.run.smithi169.stdout: python3-ceph-common x86_64 2:16.2.13-565.g595355f4.el8 ceph 99 k 2023-08-28T02:00:40.720 INFO:teuthology.orchestra.run.smithi169.stdout: python3-cephfs x86_64 2:16.2.13-565.g595355f4.el8 ceph 215 k 2023-08-28T02:00:40.720 INFO:teuthology.orchestra.run.smithi169.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2023-08-28T02:00:40.720 INFO:teuthology.orchestra.run.smithi169.stdout: python3-rados x86_64 2:16.2.13-565.g595355f4.el8 ceph 390 k 2023-08-28T02:00:40.720 INFO:teuthology.orchestra.run.smithi169.stdout: python3-rbd x86_64 2:16.2.13-565.g595355f4.el8 ceph 369 k 2023-08-28T02:00:40.720 INFO:teuthology.orchestra.run.smithi169.stdout: python3-rgw x86_64 2:16.2.13-565.g595355f4.el8 ceph 114 k 2023-08-28T02:00:40.720 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:00:40.720 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction Summary 2023-08-28T02:00:40.721 INFO:teuthology.orchestra.run.smithi169.stdout:==================================================================================================== 2023-08-28T02:00:40.721 INFO:teuthology.orchestra.run.smithi169.stdout:Install 21 Packages 2023-08-28T02:00:40.721 INFO:teuthology.orchestra.run.smithi169.stdout:Upgrade 2 Packages 2023-08-28T02:00:40.721 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:00:40.721 INFO:teuthology.orchestra.run.smithi169.stdout:Total download size: 57 M 2023-08-28T02:00:40.721 INFO:teuthology.orchestra.run.smithi169.stdout:Downloading Packages: 2023-08-28T02:00:40.754 INFO:teuthology.orchestra.run.smithi143.stdout:Dependencies resolved. 2023-08-28T02:00:40.757 INFO:teuthology.orchestra.run.smithi143.stdout:==================================================================================================== 2023-08-28T02:00:40.757 INFO:teuthology.orchestra.run.smithi143.stdout: Package Arch Version Repository Size 2023-08-28T02:00:40.757 INFO:teuthology.orchestra.run.smithi143.stdout:==================================================================================================== 2023-08-28T02:00:40.757 INFO:teuthology.orchestra.run.smithi143.stdout:Installing: 2023-08-28T02:00:40.757 INFO:teuthology.orchestra.run.smithi143.stdout: ceph-radosgw x86_64 2:16.2.13-565.g595355f4.el8 ceph 12 M 2023-08-28T02:00:40.758 INFO:teuthology.orchestra.run.smithi143.stdout:Upgrading: 2023-08-28T02:00:40.758 INFO:teuthology.orchestra.run.smithi143.stdout: librados2 x86_64 2:16.2.13-565.g595355f4.el8 ceph 3.7 M 2023-08-28T02:00:40.758 INFO:teuthology.orchestra.run.smithi143.stdout: librbd1 x86_64 2:16.2.13-565.g595355f4.el8 ceph 4.2 M 2023-08-28T02:00:40.758 INFO:teuthology.orchestra.run.smithi143.stdout:Installing dependencies: 2023-08-28T02:00:40.758 INFO:teuthology.orchestra.run.smithi143.stdout: ceph-base x86_64 2:16.2.13-565.g595355f4.el8 ceph 6.1 M 2023-08-28T02:00:40.758 INFO:teuthology.orchestra.run.smithi143.stdout: ceph-common x86_64 2:16.2.13-565.g595355f4.el8 ceph 24 M 2023-08-28T02:00:40.759 INFO:teuthology.orchestra.run.smithi143.stdout: ceph-selinux x86_64 2:16.2.13-565.g595355f4.el8 ceph 26 k 2023-08-28T02:00:40.759 INFO:teuthology.orchestra.run.smithi143.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-08-28T02:00:40.759 INFO:teuthology.orchestra.run.smithi143.stdout: leveldb x86_64 1.22-1.el8 epel 181 k 2023-08-28T02:00:40.759 INFO:teuthology.orchestra.run.smithi143.stdout: libcephfs2 x86_64 2:16.2.13-565.g595355f4.el8 ceph 824 k 2023-08-28T02:00:40.759 INFO:teuthology.orchestra.run.smithi143.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-08-28T02:00:40.759 INFO:teuthology.orchestra.run.smithi143.stdout: librabbitmq x86_64 0.9.0-3.el8 rhel-8-for-x86_64-baseos-rpms 47 k 2023-08-28T02:00:40.760 INFO:teuthology.orchestra.run.smithi143.stdout: libradosstriper1 x86_64 2:16.2.13-565.g595355f4.el8 ceph 494 k 2023-08-28T02:00:40.760 INFO:teuthology.orchestra.run.smithi143.stdout: librdkafka x86_64 0.11.4-3.el8 rhel-8-for-x86_64-appstream-rpms 354 k 2023-08-28T02:00:40.760 INFO:teuthology.orchestra.run.smithi143.stdout: librgw2 x86_64 2:16.2.13-565.g595355f4.el8 ceph 3.8 M 2023-08-28T02:00:40.760 INFO:teuthology.orchestra.run.smithi143.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-08-28T02:00:40.760 INFO:teuthology.orchestra.run.smithi143.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2023-08-28T02:00:40.760 INFO:teuthology.orchestra.run.smithi143.stdout: python3-ceph-argparse x86_64 2:16.2.13-565.g595355f4.el8 ceph 45 k 2023-08-28T02:00:40.761 INFO:teuthology.orchestra.run.smithi143.stdout: python3-ceph-common x86_64 2:16.2.13-565.g595355f4.el8 ceph 99 k 2023-08-28T02:00:40.761 INFO:teuthology.orchestra.run.smithi143.stdout: python3-cephfs x86_64 2:16.2.13-565.g595355f4.el8 ceph 215 k 2023-08-28T02:00:40.761 INFO:teuthology.orchestra.run.smithi143.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2023-08-28T02:00:40.761 INFO:teuthology.orchestra.run.smithi143.stdout: python3-rados x86_64 2:16.2.13-565.g595355f4.el8 ceph 390 k 2023-08-28T02:00:40.761 INFO:teuthology.orchestra.run.smithi143.stdout: python3-rbd x86_64 2:16.2.13-565.g595355f4.el8 ceph 369 k 2023-08-28T02:00:40.761 INFO:teuthology.orchestra.run.smithi143.stdout: python3-rgw x86_64 2:16.2.13-565.g595355f4.el8 ceph 114 k 2023-08-28T02:00:40.762 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:00:40.762 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction Summary 2023-08-28T02:00:40.762 INFO:teuthology.orchestra.run.smithi143.stdout:==================================================================================================== 2023-08-28T02:00:40.762 INFO:teuthology.orchestra.run.smithi143.stdout:Install 21 Packages 2023-08-28T02:00:40.762 INFO:teuthology.orchestra.run.smithi143.stdout:Upgrade 2 Packages 2023-08-28T02:00:40.762 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:00:40.763 INFO:teuthology.orchestra.run.smithi143.stdout:Total download size: 57 M 2023-08-28T02:00:40.763 INFO:teuthology.orchestra.run.smithi143.stdout:Downloading Packages: 2023-08-28T02:00:41.656 INFO:teuthology.orchestra.run.smithi169.stdout:(1/23): ceph-base-16.2.13-565.g595355f4.el8.x86 6.9 MB/s | 6.1 MB 00:00 2023-08-28T02:00:41.723 INFO:teuthology.orchestra.run.smithi169.stdout:(2/23): ceph-selinux-16.2.13-565.g595355f4.el8. 384 kB/s | 26 kB 00:00 2023-08-28T02:00:41.899 INFO:teuthology.orchestra.run.smithi169.stdout:(3/23): libcephfs2-16.2.13-565.g595355f4.el8.x8 4.6 MB/s | 824 kB 00:00 2023-08-28T02:00:41.990 INFO:teuthology.orchestra.run.smithi169.stdout:(4/23): libradosstriper1-16.2.13-565.g595355f4. 5.3 MB/s | 494 kB 00:00 2023-08-28T02:00:42.349 INFO:teuthology.orchestra.run.smithi169.stdout:(5/23): ceph-radosgw-16.2.13-565.g595355f4.el8. 7.4 MB/s | 12 MB 00:01 2023-08-28T02:00:42.424 INFO:teuthology.orchestra.run.smithi169.stdout:(6/23): python3-ceph-argparse-16.2.13-565.g5953 603 kB/s | 45 kB 00:00 2023-08-28T02:00:42.474 INFO:teuthology.orchestra.run.smithi169.stdout:(7/23): librgw2-16.2.13-565.g595355f4.el8.x86_6 7.8 MB/s | 3.8 MB 00:00 2023-08-28T02:00:42.500 INFO:teuthology.orchestra.run.smithi169.stdout:(8/23): python3-ceph-common-16.2.13-565.g595355 1.3 MB/s | 99 kB 00:00 2023-08-28T02:00:42.533 INFO:teuthology.orchestra.run.smithi169.stdout:(9/23): python3-cephfs-16.2.13-565.g595355f4.el 3.6 MB/s | 215 kB 00:00 2023-08-28T02:00:42.592 INFO:teuthology.orchestra.run.smithi169.stdout:(10/23): python3-rados-16.2.13-565.g595355f4.el 4.2 MB/s | 390 kB 00:00 2023-08-28T02:00:42.617 INFO:teuthology.orchestra.run.smithi169.stdout:(11/23): python3-rbd-16.2.13-565.g595355f4.el8. 4.3 MB/s | 369 kB 00:00 2023-08-28T02:00:42.659 INFO:teuthology.orchestra.run.smithi169.stdout:(12/23): python3-rgw-16.2.13-565.g595355f4.el8. 1.7 MB/s | 114 kB 00:00 2023-08-28T02:00:43.301 INFO:teuthology.orchestra.run.smithi169.stdout:(13/23): ceph-common-16.2.13-565.g595355f4.el8. 9.6 MB/s | 24 MB 00:02 2023-08-28T02:00:43.305 INFO:teuthology.orchestra.run.smithi143.stdout:(1/23): ceph-radosgw-16.2.13-565.g595355f4.el8. 4.8 MB/s | 12 MB 00:02 2023-08-28T02:00:43.326 INFO:teuthology.orchestra.run.smithi169.stdout:(14/23): leveldb-1.22-1.el8.x86_64.rpm 271 kB/s | 181 kB 00:00 2023-08-28T02:00:43.338 INFO:teuthology.orchestra.run.smithi143.stdout:(2/23): ceph-selinux-16.2.13-565.g595355f4.el8. 778 kB/s | 26 kB 00:00 2023-08-28T02:00:43.418 INFO:teuthology.orchestra.run.smithi169.stdout:(15/23): gperftools-libs-2.7-9.el8.x86_64.rpm 382 kB/s | 306 kB 00:00 2023-08-28T02:00:43.421 INFO:teuthology.orchestra.run.smithi143.stdout:(3/23): libcephfs2-16.2.13-565.g595355f4.el8.x8 9.7 MB/s | 824 kB 00:00 2023-08-28T02:00:43.460 INFO:teuthology.orchestra.run.smithi169.stdout:(16/23): libunwind-1.3.1-3.el8.x86_64.rpm 562 kB/s | 75 kB 00:00 2023-08-28T02:00:43.488 INFO:teuthology.orchestra.run.smithi143.stdout:(4/23): libradosstriper1-16.2.13-565.g595355f4. 7.2 MB/s | 494 kB 00:00 2023-08-28T02:00:43.494 INFO:teuthology.orchestra.run.smithi169.stdout:(17/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 3.4 MB/s | 259 kB 00:00 2023-08-28T02:00:43.553 INFO:teuthology.orchestra.run.smithi169.stdout:(18/23): python3-prettytable-0.7.2-14.el8.noarc 477 kB/s | 44 kB 00:00 2023-08-28T02:00:43.578 INFO:teuthology.orchestra.run.smithi169.stdout:(19/23): librdkafka-0.11.4-3.el8.x86_64.rpm 4.4 MB/s | 354 kB 00:00 2023-08-28T02:00:43.595 INFO:teuthology.orchestra.run.smithi169.stdout:(20/23): librabbitmq-0.9.0-3.el8.x86_64.rpm 1.1 MB/s | 47 kB 00:00 2023-08-28T02:00:43.722 INFO:teuthology.orchestra.run.smithi143.stdout:(5/23): librgw2-16.2.13-565.g595355f4.el8.x86_6 16 MB/s | 3.8 MB 00:00 2023-08-28T02:00:43.728 INFO:teuthology.orchestra.run.smithi169.stdout:(21/23): liboath-2.6.2-3.el8.x86_64.rpm 138 kB/s | 59 kB 00:00 2023-08-28T02:00:43.781 INFO:teuthology.orchestra.run.smithi143.stdout:(6/23): python3-ceph-argparse-16.2.13-565.g5953 774 kB/s | 45 kB 00:00 2023-08-28T02:00:43.856 INFO:teuthology.orchestra.run.smithi143.stdout:(7/23): python3-ceph-common-16.2.13-565.g595355 1.3 MB/s | 99 kB 00:00 2023-08-28T02:00:43.940 INFO:teuthology.orchestra.run.smithi143.stdout:(8/23): python3-cephfs-16.2.13-565.g595355f4.el 2.5 MB/s | 215 kB 00:00 2023-08-28T02:00:44.007 INFO:teuthology.orchestra.run.smithi143.stdout:(9/23): python3-rados-16.2.13-565.g595355f4.el8 5.7 MB/s | 390 kB 00:00 2023-08-28T02:00:44.020 INFO:teuthology.orchestra.run.smithi169.stdout:(22/23): librbd1-16.2.13-565.g595355f4.el8.x86_ 9.8 MB/s | 4.2 MB 00:00 2023-08-28T02:00:44.065 INFO:teuthology.orchestra.run.smithi143.stdout:(10/23): python3-rbd-16.2.13-565.g595355f4.el8. 6.2 MB/s | 369 kB 00:00 2023-08-28T02:00:44.104 INFO:teuthology.orchestra.run.smithi169.stdout:(23/23): librados2-16.2.13-565.g595355f4.el8.x8 7.1 MB/s | 3.7 MB 00:00 2023-08-28T02:00:44.109 INFO:teuthology.orchestra.run.smithi169.stdout:-------------------------------------------------------------------------------- 2023-08-28T02:00:44.109 INFO:teuthology.orchestra.run.smithi169.stdout:Total 17 MB/s | 57 MB 00:03 2023-08-28T02:00:44.115 INFO:teuthology.orchestra.run.smithi143.stdout:(11/23): python3-rgw-16.2.13-565.g595355f4.el8. 2.2 MB/s | 114 kB 00:00 2023-08-28T02:00:44.155 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction check 2023-08-28T02:00:44.202 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction check succeeded. 2023-08-28T02:00:44.202 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction test 2023-08-28T02:00:44.300 INFO:teuthology.orchestra.run.smithi143.stdout:(12/23): ceph-base-16.2.13-565.g595355f4.el8.x8 1.8 MB/s | 6.1 MB 00:03 2023-08-28T02:00:44.527 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction test succeeded. 2023-08-28T02:00:44.530 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction 2023-08-28T02:00:44.549 INFO:teuthology.orchestra.run.smithi143.stdout:(13/23): gperftools-libs-2.7-9.el8.x86_64.rpm 706 kB/s | 306 kB 00:00 2023-08-28T02:00:44.617 INFO:teuthology.orchestra.run.smithi143.stdout:(14/23): leveldb-1.22-1.el8.x86_64.rpm 572 kB/s | 181 kB 00:00 2023-08-28T02:00:44.650 INFO:teuthology.orchestra.run.smithi143.stdout:(15/23): liboath-2.6.2-3.el8.x86_64.rpm 586 kB/s | 59 kB 00:00 2023-08-28T02:00:44.709 INFO:teuthology.orchestra.run.smithi143.stdout:(16/23): libunwind-1.3.1-3.el8.x86_64.rpm 814 kB/s | 75 kB 00:00 2023-08-28T02:00:44.756 INFO:teuthology.orchestra.run.smithi143.stdout:(17/23): librabbitmq-0.9.0-3.el8.x86_64.rpm 445 kB/s | 47 kB 00:00 2023-08-28T02:00:44.810 INFO:teuthology.orchestra.run.smithi143.stdout:(18/23): python3-prettytable-0.7.2-14.el8.noarc 910 kB/s | 44 kB 00:00 2023-08-28T02:00:44.835 INFO:teuthology.orchestra.run.smithi143.stdout:(19/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 2.0 MB/s | 259 kB 00:00 2023-08-28T02:00:44.869 INFO:teuthology.orchestra.run.smithi143.stdout:(20/23): librdkafka-0.11.4-3.el8.x86_64.rpm 5.9 MB/s | 354 kB 00:00 2023-08-28T02:00:45.152 INFO:teuthology.orchestra.run.smithi143.stdout:(21/23): librbd1-16.2.13-565.g595355f4.el8.x86_ 15 MB/s | 4.2 MB 00:00 2023-08-28T02:00:45.554 INFO:teuthology.orchestra.run.smithi169.stdout: Preparing : 1/1 2023-08-28T02:00:45.712 INFO:teuthology.orchestra.run.smithi143.stdout:(22/23): librados2-16.2.13-565.g595355f4.el8.x8 4.3 MB/s | 3.7 MB 00:00 2023-08-28T02:00:45.717 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-08-28T02:00:46.110 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-08-28T02:00:46.317 INFO:teuthology.orchestra.run.smithi169.stdout: Upgrading : librados2-2:16.2.13-565.g595355f4.el8.x86_64 2/25 2023-08-28T02:00:46.395 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: librados2-2:16.2.13-565.g595355f4.el8.x86_64 2/25 2023-08-28T02:00:46.614 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-rados-2:16.2.13-565.g595355f4.el8.x86_64 3/25 2023-08-28T02:00:46.848 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : liboath-2.6.2-3.el8.x86_64 4/25 2023-08-28T02:00:47.059 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : libcephfs2-2:16.2.13-565.g595355f4.el8.x86_64 5/25 2023-08-28T02:00:47.470 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: libcephfs2-2:16.2.13-565.g595355f4.el8.x86_64 5/25 2023-08-28T02:00:47.667 INFO:teuthology.orchestra.run.smithi169.stdout: Upgrading : librbd1-2:16.2.13-565.g595355f4.el8.x86_64 6/25 2023-08-28T02:00:47.734 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: librbd1-2:16.2.13-565.g595355f4.el8.x86_64 6/25 2023-08-28T02:00:47.991 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 7/25 2023-08-28T02:00:48.142 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 8/25 2023-08-28T02:00:48.620 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 8/25 2023-08-28T02:00:48.825 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : librgw2-2:16.2.13-565.g595355f4.el8.x86_64 9/25 2023-08-28T02:00:48.902 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: librgw2-2:16.2.13-565.g595355f4.el8.x86_64 9/25 2023-08-28T02:00:49.097 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : leveldb-1.22-1.el8.x86_64 10/25 2023-08-28T02:00:49.272 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-ceph-argparse-2:16.2.13-565.g595355f4.el8. 11/25 2023-08-28T02:00:49.444 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-cephfs-2:16.2.13-565.g595355f4.el8.x86_64 12/25 2023-08-28T02:00:49.684 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-rgw-2:16.2.13-565.g595355f4.el8.x86_64 13/25 2023-08-28T02:00:49.875 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-rbd-2:16.2.13-565.g595355f4.el8.x86_64 14/25 2023-08-28T02:00:50.053 INFO:teuthology.orchestra.run.smithi143.stdout:(23/23): ceph-common-16.2.13-565.g595355f4.el8. 2.6 MB/s | 24 MB 00:09 2023-08-28T02:00:50.057 INFO:teuthology.orchestra.run.smithi143.stdout:-------------------------------------------------------------------------------- 2023-08-28T02:00:50.057 INFO:teuthology.orchestra.run.smithi143.stdout:Total 6.2 MB/s | 57 MB 00:09 2023-08-28T02:00:50.075 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : libradosstriper1-2:16.2.13-565.g595355f4.el8.x86_6 15/25 2023-08-28T02:00:50.103 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction check 2023-08-28T02:00:50.146 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: libradosstriper1-2:16.2.13-565.g595355f4.el8.x86_6 15/25 2023-08-28T02:00:50.150 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction check succeeded. 2023-08-28T02:00:50.150 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction test 2023-08-28T02:00:50.260 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-08-28T02:00:50.476 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction test succeeded. 2023-08-28T02:00:50.479 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction 2023-08-28T02:00:50.493 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 17/25 2023-08-28T02:00:50.711 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-08-28T02:00:50.867 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-ceph-common-2:16.2.13-565.g595355f4.el8.x8 19/25 2023-08-28T02:00:51.529 INFO:teuthology.orchestra.run.smithi143.stdout: Preparing : 1/1 2023-08-28T02:00:51.735 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-08-28T02:00:52.149 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-08-28T02:00:52.334 INFO:teuthology.orchestra.run.smithi143.stdout: Upgrading : librados2-2:16.2.13-565.g595355f4.el8.x86_64 2/25 2023-08-28T02:00:52.418 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: librados2-2:16.2.13-565.g595355f4.el8.x86_64 2/25 2023-08-28T02:00:52.590 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-rados-2:16.2.13-565.g595355f4.el8.x86_64 3/25 2023-08-28T02:00:52.823 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : liboath-2.6.2-3.el8.x86_64 4/25 2023-08-28T02:00:53.034 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : libcephfs2-2:16.2.13-565.g595355f4.el8.x86_64 5/25 2023-08-28T02:00:53.442 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: libcephfs2-2:16.2.13-565.g595355f4.el8.x86_64 5/25 2023-08-28T02:00:53.659 INFO:teuthology.orchestra.run.smithi143.stdout: Upgrading : librbd1-2:16.2.13-565.g595355f4.el8.x86_64 6/25 2023-08-28T02:00:53.942 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: librbd1-2:16.2.13-565.g595355f4.el8.x86_64 6/25 2023-08-28T02:00:54.126 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 7/25 2023-08-28T02:00:54.193 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 7/25 2023-08-28T02:00:54.508 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: ceph-common-2:16.2.13-565.g595355f4.el8.x86_64 20/25 2023-08-28T02:00:54.704 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 8/25 2023-08-28T02:00:54.767 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : ceph-common-2:16.2.13-565.g595355f4.el8.x86_64 20/25 2023-08-28T02:00:54.884 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : librgw2-2:16.2.13-565.g595355f4.el8.x86_64 9/25 2023-08-28T02:00:54.966 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: librgw2-2:16.2.13-565.g595355f4.el8.x86_64 9/25 2023-08-28T02:00:55.156 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : leveldb-1.22-1.el8.x86_64 10/25 2023-08-28T02:00:55.280 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-ceph-argparse-2:16.2.13-565.g595355f4.el8. 11/25 2023-08-28T02:00:55.453 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-cephfs-2:16.2.13-565.g595355f4.el8.x86_64 12/25 2023-08-28T02:00:55.501 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: ceph-common-2:16.2.13-565.g595355f4.el8.x86_64 20/25 2023-08-28T02:00:55.660 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-rgw-2:16.2.13-565.g595355f4.el8.x86_64 13/25 2023-08-28T02:00:55.792 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : ceph-base-2:16.2.13-565.g595355f4.el8.x86_64 21/25 2023-08-28T02:00:55.863 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: ceph-base-2:16.2.13-565.g595355f4.el8.x86_64 21/25 2023-08-28T02:00:55.867 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-rbd-2:16.2.13-565.g595355f4.el8.x86_64 14/25 2023-08-28T02:00:56.009 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : libradosstriper1-2:16.2.13-565.g595355f4.el8.x86_6 15/25 2023-08-28T02:00:56.049 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : ceph-selinux-2:16.2.13-565.g595355f4.el8.x86_64 22/25 2023-08-28T02:00:56.077 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: libradosstriper1-2:16.2.13-565.g595355f4.el8.x86_6 15/25 2023-08-28T02:00:56.210 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-08-28T02:00:56.410 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 17/25 2023-08-28T02:00:56.669 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-08-28T02:00:56.818 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-ceph-common-2:16.2.13-565.g595355f4.el8.x8 19/25 2023-08-28T02:01:00.491 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: ceph-common-2:16.2.13-565.g595355f4.el8.x86_64 20/25 2023-08-28T02:01:00.758 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : ceph-common-2:16.2.13-565.g595355f4.el8.x86_64 20/25 2023-08-28T02:01:01.462 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: ceph-common-2:16.2.13-565.g595355f4.el8.x86_64 20/25 2023-08-28T02:01:01.777 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : ceph-base-2:16.2.13-565.g595355f4.el8.x86_64 21/25 2023-08-28T02:01:01.861 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: ceph-base-2:16.2.13-565.g595355f4.el8.x86_64 21/25 2023-08-28T02:01:01.966 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : ceph-selinux-2:16.2.13-565.g595355f4.el8.x86_64 22/25 2023-08-28T02:01:14.724 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: ceph-selinux-2:16.2.13-565.g595355f4.el8.x86_64 22/25 2023-08-28T02:01:14.724 INFO:teuthology.orchestra.run.smithi169.stdout:skipping the directory /sys 2023-08-28T02:01:14.724 INFO:teuthology.orchestra.run.smithi169.stdout:skipping the directory /proc 2023-08-28T02:01:14.724 INFO:teuthology.orchestra.run.smithi169.stdout:skipping the directory /dev 2023-08-28T02:01:14.724 INFO:teuthology.orchestra.run.smithi169.stdout:skipping the directory /run 2023-08-28T02:01:14.724 INFO:teuthology.orchestra.run.smithi169.stdout:skipping the directory /mnt 2023-08-28T02:01:14.724 INFO:teuthology.orchestra.run.smithi169.stdout:skipping the directory /var/tmp 2023-08-28T02:01:14.725 INFO:teuthology.orchestra.run.smithi169.stdout:skipping the directory /home 2023-08-28T02:01:14.725 INFO:teuthology.orchestra.run.smithi169.stdout:skipping the directory /tmp 2023-08-28T02:01:14.725 INFO:teuthology.orchestra.run.smithi169.stdout:skipping the directory /dev 2023-08-28T02:01:14.725 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:01:15.897 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : ceph-radosgw-2:16.2.13-565.g595355f4.el8.x86_64 23/25 2023-08-28T02:01:15.962 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: ceph-radosgw-2:16.2.13-565.g595355f4.el8.x86_64 23/25 2023-08-28T02:01:15.962 INFO:teuthology.orchestra.run.smithi169.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-08-28T02:01:16.207 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-08-28T02:01:16.207 INFO:teuthology.orchestra.run.smithi169.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-08-28T02:01:16.829 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-08-28T02:01:16.829 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : ceph-base-2:16.2.13-565.g595355f4.el8.x86_64 1/25 2023-08-28T02:01:16.829 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : ceph-common-2:16.2.13-565.g595355f4.el8.x86_64 2/25 2023-08-28T02:01:16.829 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : ceph-radosgw-2:16.2.13-565.g595355f4.el8.x86_64 3/25 2023-08-28T02:01:16.829 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : ceph-selinux-2:16.2.13-565.g595355f4.el8.x86_64 4/25 2023-08-28T02:01:16.829 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : libcephfs2-2:16.2.13-565.g595355f4.el8.x86_64 5/25 2023-08-28T02:01:16.830 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : libradosstriper1-2:16.2.13-565.g595355f4.el8.x86_6 6/25 2023-08-28T02:01:16.830 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : librgw2-2:16.2.13-565.g595355f4.el8.x86_64 7/25 2023-08-28T02:01:16.830 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-ceph-argparse-2:16.2.13-565.g595355f4.el8. 8/25 2023-08-28T02:01:16.830 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-ceph-common-2:16.2.13-565.g595355f4.el8.x8 9/25 2023-08-28T02:01:16.830 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-cephfs-2:16.2.13-565.g595355f4.el8.x86_64 10/25 2023-08-28T02:01:16.830 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-rados-2:16.2.13-565.g595355f4.el8.x86_64 11/25 2023-08-28T02:01:16.830 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-rbd-2:16.2.13-565.g595355f4.el8.x86_64 12/25 2023-08-28T02:01:16.830 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-rgw-2:16.2.13-565.g595355f4.el8.x86_64 13/25 2023-08-28T02:01:16.831 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 14/25 2023-08-28T02:01:16.831 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : leveldb-1.22-1.el8.x86_64 15/25 2023-08-28T02:01:16.831 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 16/25 2023-08-28T02:01:16.831 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 17/25 2023-08-28T02:01:16.831 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 18/25 2023-08-28T02:01:16.831 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 19/25 2023-08-28T02:01:16.831 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 20/25 2023-08-28T02:01:16.831 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 21/25 2023-08-28T02:01:16.836 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : librados2-2:16.2.13-565.g595355f4.el8.x86_64 22/25 2023-08-28T02:01:16.836 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-08-28T02:01:16.836 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : librbd1-2:16.2.13-565.g595355f4.el8.x86_64 24/25 2023-08-28T02:01:18.237 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-08-28T02:01:18.238 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:00:49 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:01:20.132 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: ceph-selinux-2:16.2.13-565.g595355f4.el8.x86_64 22/25 2023-08-28T02:01:20.132 INFO:teuthology.orchestra.run.smithi143.stdout:skipping the directory /sys 2023-08-28T02:01:20.132 INFO:teuthology.orchestra.run.smithi143.stdout:skipping the directory /proc 2023-08-28T02:01:20.132 INFO:teuthology.orchestra.run.smithi143.stdout:skipping the directory /dev 2023-08-28T02:01:20.132 INFO:teuthology.orchestra.run.smithi143.stdout:skipping the directory /run 2023-08-28T02:01:20.132 INFO:teuthology.orchestra.run.smithi143.stdout:skipping the directory /mnt 2023-08-28T02:01:20.132 INFO:teuthology.orchestra.run.smithi143.stdout:skipping the directory /var/tmp 2023-08-28T02:01:20.132 INFO:teuthology.orchestra.run.smithi143.stdout:skipping the directory /home 2023-08-28T02:01:20.132 INFO:teuthology.orchestra.run.smithi143.stdout:skipping the directory /tmp 2023-08-28T02:01:20.133 INFO:teuthology.orchestra.run.smithi143.stdout:skipping the directory /dev 2023-08-28T02:01:20.133 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:01:20.524 INFO:teuthology.orchestra.run.smithi169.stdout:Installed products updated. 2023-08-28T02:01:20.710 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:01:20.711 INFO:teuthology.orchestra.run.smithi169.stdout:Upgraded: 2023-08-28T02:01:20.711 INFO:teuthology.orchestra.run.smithi169.stdout: librados2-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:01:20.711 INFO:teuthology.orchestra.run.smithi169.stdout: librbd1-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:01:20.711 INFO:teuthology.orchestra.run.smithi169.stdout:Installed: 2023-08-28T02:01:20.711 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-base-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:01:20.711 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-common-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:01:20.711 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-radosgw-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:01:20.711 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-selinux-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:01:20.711 INFO:teuthology.orchestra.run.smithi169.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-08-28T02:01:20.711 INFO:teuthology.orchestra.run.smithi169.stdout: leveldb-1.22-1.el8.x86_64 2023-08-28T02:01:20.711 INFO:teuthology.orchestra.run.smithi169.stdout: libcephfs2-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:01:20.711 INFO:teuthology.orchestra.run.smithi169.stdout: liboath-2.6.2-3.el8.x86_64 2023-08-28T02:01:20.712 INFO:teuthology.orchestra.run.smithi169.stdout: librabbitmq-0.9.0-3.el8.x86_64 2023-08-28T02:01:20.712 INFO:teuthology.orchestra.run.smithi169.stdout: libradosstriper1-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:01:20.712 INFO:teuthology.orchestra.run.smithi169.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-08-28T02:01:20.712 INFO:teuthology.orchestra.run.smithi169.stdout: librgw2-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:01:20.712 INFO:teuthology.orchestra.run.smithi169.stdout: libunwind-1.3.1-3.el8.x86_64 2023-08-28T02:01:20.712 INFO:teuthology.orchestra.run.smithi169.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-08-28T02:01:20.712 INFO:teuthology.orchestra.run.smithi169.stdout: python3-ceph-argparse-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:01:20.712 INFO:teuthology.orchestra.run.smithi169.stdout: python3-ceph-common-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:01:20.712 INFO:teuthology.orchestra.run.smithi169.stdout: python3-cephfs-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:01:20.712 INFO:teuthology.orchestra.run.smithi169.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-08-28T02:01:20.712 INFO:teuthology.orchestra.run.smithi169.stdout: python3-rados-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:01:20.712 INFO:teuthology.orchestra.run.smithi169.stdout: python3-rbd-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:01:20.712 INFO:teuthology.orchestra.run.smithi169.stdout: python3-rgw-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:01:20.713 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:01:20.713 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-08-28T02:01:20.902 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install ceph-test 2023-08-28T02:01:21.253 INFO:teuthology.orchestra.run.smithi169.stdout:Updating Subscription Management repositories. 2023-08-28T02:01:21.274 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : ceph-radosgw-2:16.2.13-565.g595355f4.el8.x86_64 23/25 2023-08-28T02:01:21.338 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: ceph-radosgw-2:16.2.13-565.g595355f4.el8.x86_64 23/25 2023-08-28T02:01:21.338 INFO:teuthology.orchestra.run.smithi143.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-08-28T02:01:21.574 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-08-28T02:01:21.574 INFO:teuthology.orchestra.run.smithi143.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-08-28T02:01:22.072 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:00:53 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:01:22.177 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-08-28T02:01:22.178 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : ceph-base-2:16.2.13-565.g595355f4.el8.x86_64 1/25 2023-08-28T02:01:22.178 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : ceph-common-2:16.2.13-565.g595355f4.el8.x86_64 2/25 2023-08-28T02:01:22.178 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : ceph-radosgw-2:16.2.13-565.g595355f4.el8.x86_64 3/25 2023-08-28T02:01:22.178 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : ceph-selinux-2:16.2.13-565.g595355f4.el8.x86_64 4/25 2023-08-28T02:01:22.178 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : libcephfs2-2:16.2.13-565.g595355f4.el8.x86_64 5/25 2023-08-28T02:01:22.178 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : libradosstriper1-2:16.2.13-565.g595355f4.el8.x86_6 6/25 2023-08-28T02:01:22.178 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : librgw2-2:16.2.13-565.g595355f4.el8.x86_64 7/25 2023-08-28T02:01:22.178 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-ceph-argparse-2:16.2.13-565.g595355f4.el8. 8/25 2023-08-28T02:01:22.178 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-ceph-common-2:16.2.13-565.g595355f4.el8.x8 9/25 2023-08-28T02:01:22.179 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-cephfs-2:16.2.13-565.g595355f4.el8.x86_64 10/25 2023-08-28T02:01:22.179 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-rados-2:16.2.13-565.g595355f4.el8.x86_64 11/25 2023-08-28T02:01:22.179 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-rbd-2:16.2.13-565.g595355f4.el8.x86_64 12/25 2023-08-28T02:01:22.179 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-rgw-2:16.2.13-565.g595355f4.el8.x86_64 13/25 2023-08-28T02:01:22.179 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 14/25 2023-08-28T02:01:22.179 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : leveldb-1.22-1.el8.x86_64 15/25 2023-08-28T02:01:22.179 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 16/25 2023-08-28T02:01:22.179 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 17/25 2023-08-28T02:01:22.179 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 18/25 2023-08-28T02:01:22.180 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 19/25 2023-08-28T02:01:22.180 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 20/25 2023-08-28T02:01:22.180 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 21/25 2023-08-28T02:01:22.180 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : librados2-2:16.2.13-565.g595355f4.el8.x86_64 22/25 2023-08-28T02:01:22.180 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-08-28T02:01:22.180 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : librbd1-2:16.2.13-565.g595355f4.el8.x86_64 24/25 2023-08-28T02:01:23.468 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-08-28T02:01:23.469 INFO:teuthology.orchestra.run.smithi169.stdout:============================================================================================ 2023-08-28T02:01:23.470 INFO:teuthology.orchestra.run.smithi169.stdout: Package Arch Version Repository Size 2023-08-28T02:01:23.470 INFO:teuthology.orchestra.run.smithi169.stdout:============================================================================================ 2023-08-28T02:01:23.470 INFO:teuthology.orchestra.run.smithi169.stdout:Installing: 2023-08-28T02:01:23.470 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-test x86_64 2:16.2.13-565.g595355f4.el8 ceph 52 M 2023-08-28T02:01:23.470 INFO:teuthology.orchestra.run.smithi169.stdout:Installing dependencies: 2023-08-28T02:01:23.470 INFO:teuthology.orchestra.run.smithi169.stdout: jq x86_64 1.6-3.el8 rhel-8-for-x86_64-appstream-rpms 202 k 2023-08-28T02:01:23.470 INFO:teuthology.orchestra.run.smithi169.stdout: libcephsqlite x86_64 2:16.2.13-565.g595355f4.el8 ceph 174 k 2023-08-28T02:01:23.470 INFO:teuthology.orchestra.run.smithi169.stdout: oniguruma x86_64 6.8.2-2.el8 rhel-8-for-x86_64-appstream-rpms 187 k 2023-08-28T02:01:23.470 INFO:teuthology.orchestra.run.smithi169.stdout: socat x86_64 1.7.4.1-1.el8 rhel-8-for-x86_64-appstream-rpms 323 k 2023-08-28T02:01:23.470 INFO:teuthology.orchestra.run.smithi169.stdout: xmlstarlet x86_64 1.6.1-20.el8 lab-extras 70 k 2023-08-28T02:01:23.470 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:01:23.471 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction Summary 2023-08-28T02:01:23.471 INFO:teuthology.orchestra.run.smithi169.stdout:============================================================================================ 2023-08-28T02:01:23.471 INFO:teuthology.orchestra.run.smithi169.stdout:Install 6 Packages 2023-08-28T02:01:23.471 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:01:23.471 INFO:teuthology.orchestra.run.smithi169.stdout:Total download size: 53 M 2023-08-28T02:01:23.472 INFO:teuthology.orchestra.run.smithi169.stdout:Installed size: 195 M 2023-08-28T02:01:23.472 INFO:teuthology.orchestra.run.smithi169.stdout:Downloading Packages: 2023-08-28T02:01:23.521 INFO:teuthology.orchestra.run.smithi169.stdout:(1/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 1.4 MB/s | 70 kB 00:00 2023-08-28T02:01:23.571 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-08-28T02:01:23.571 INFO:teuthology.orchestra.run.smithi143.stdout:Last metadata expiration check: 0:00:54 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:01:23.613 INFO:teuthology.orchestra.run.smithi169.stdout:(2/6): socat-1.7.4.1-1.el8.x86_64.rpm 3.8 MB/s | 323 kB 00:00 2023-08-28T02:01:23.672 INFO:teuthology.orchestra.run.smithi169.stdout:(3/6): oniguruma-6.8.2-2.el8.x86_64.rpm 3.1 MB/s | 187 kB 00:00 2023-08-28T02:01:23.724 INFO:teuthology.orchestra.run.smithi169.stdout:(4/6): libcephsqlite-16.2.13-565.g595355f4.el8. 690 kB/s | 174 kB 00:00 2023-08-28T02:01:23.758 INFO:teuthology.orchestra.run.smithi169.stdout:(5/6): jq-1.6-3.el8.x86_64.rpm 2.4 MB/s | 202 kB 00:00 2023-08-28T02:01:25.869 INFO:teuthology.orchestra.run.smithi143.stdout:Installed products updated. 2023-08-28T02:01:26.056 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:01:26.056 INFO:teuthology.orchestra.run.smithi143.stdout:Upgraded: 2023-08-28T02:01:26.056 INFO:teuthology.orchestra.run.smithi143.stdout: librados2-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:01:26.056 INFO:teuthology.orchestra.run.smithi143.stdout: librbd1-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:01:26.056 INFO:teuthology.orchestra.run.smithi143.stdout:Installed: 2023-08-28T02:01:26.056 INFO:teuthology.orchestra.run.smithi143.stdout: ceph-base-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:01:26.057 INFO:teuthology.orchestra.run.smithi143.stdout: ceph-common-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:01:26.057 INFO:teuthology.orchestra.run.smithi143.stdout: ceph-radosgw-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:01:26.057 INFO:teuthology.orchestra.run.smithi143.stdout: ceph-selinux-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:01:26.057 INFO:teuthology.orchestra.run.smithi143.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-08-28T02:01:26.057 INFO:teuthology.orchestra.run.smithi143.stdout: leveldb-1.22-1.el8.x86_64 2023-08-28T02:01:26.057 INFO:teuthology.orchestra.run.smithi143.stdout: libcephfs2-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:01:26.057 INFO:teuthology.orchestra.run.smithi143.stdout: liboath-2.6.2-3.el8.x86_64 2023-08-28T02:01:26.057 INFO:teuthology.orchestra.run.smithi143.stdout: librabbitmq-0.9.0-3.el8.x86_64 2023-08-28T02:01:26.057 INFO:teuthology.orchestra.run.smithi143.stdout: libradosstriper1-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:01:26.057 INFO:teuthology.orchestra.run.smithi143.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-08-28T02:01:26.057 INFO:teuthology.orchestra.run.smithi143.stdout: librgw2-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:01:26.057 INFO:teuthology.orchestra.run.smithi143.stdout: libunwind-1.3.1-3.el8.x86_64 2023-08-28T02:01:26.057 INFO:teuthology.orchestra.run.smithi143.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-08-28T02:01:26.058 INFO:teuthology.orchestra.run.smithi143.stdout: python3-ceph-argparse-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:01:26.058 INFO:teuthology.orchestra.run.smithi143.stdout: python3-ceph-common-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:01:26.058 INFO:teuthology.orchestra.run.smithi143.stdout: python3-cephfs-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:01:26.058 INFO:teuthology.orchestra.run.smithi143.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-08-28T02:01:26.058 INFO:teuthology.orchestra.run.smithi143.stdout: python3-rados-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:01:26.058 INFO:teuthology.orchestra.run.smithi143.stdout: python3-rbd-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:01:26.058 INFO:teuthology.orchestra.run.smithi143.stdout: python3-rgw-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:01:26.058 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:01:26.058 INFO:teuthology.orchestra.run.smithi143.stdout:Complete! 2023-08-28T02:01:26.156 INFO:teuthology.orchestra.run.smithi169.stdout:(6/6): ceph-test-16.2.13-565.g595355f4.el8.x86_ 19 MB/s | 52 MB 00:02 2023-08-28T02:01:26.158 INFO:teuthology.orchestra.run.smithi169.stdout:-------------------------------------------------------------------------------- 2023-08-28T02:01:26.158 INFO:teuthology.orchestra.run.smithi169.stdout:Total 20 MB/s | 53 MB 00:02 2023-08-28T02:01:26.196 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction check 2023-08-28T02:01:26.218 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction check succeeded. 2023-08-28T02:01:26.218 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction test 2023-08-28T02:01:26.232 DEBUG:teuthology.orchestra.run.smithi143:> sudo yum -y install ceph-test 2023-08-28T02:01:26.499 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction test succeeded. 2023-08-28T02:01:26.503 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction 2023-08-28T02:01:26.579 INFO:teuthology.orchestra.run.smithi143.stdout:Updating Subscription Management repositories. 2023-08-28T02:01:26.990 INFO:teuthology.orchestra.run.smithi169.stdout: Preparing : 1/1 2023-08-28T02:01:27.144 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 1/6 2023-08-28T02:01:27.234 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 1/6 2023-08-28T02:01:27.403 INFO:teuthology.orchestra.run.smithi143.stdout:Last metadata expiration check: 0:00:58 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:01:27.431 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : jq-1.6-3.el8.x86_64 2/6 2023-08-28T02:01:27.628 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 3/6 2023-08-28T02:01:27.786 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 4/6 2023-08-28T02:01:27.961 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : libcephsqlite-2:16.2.13-565.g595355f4.el8.x86_64 5/6 2023-08-28T02:01:28.780 INFO:teuthology.orchestra.run.smithi143.stdout:Dependencies resolved. 2023-08-28T02:01:28.781 INFO:teuthology.orchestra.run.smithi143.stdout:============================================================================================ 2023-08-28T02:01:28.782 INFO:teuthology.orchestra.run.smithi143.stdout: Package Arch Version Repository Size 2023-08-28T02:01:28.782 INFO:teuthology.orchestra.run.smithi143.stdout:============================================================================================ 2023-08-28T02:01:28.782 INFO:teuthology.orchestra.run.smithi143.stdout:Installing: 2023-08-28T02:01:28.782 INFO:teuthology.orchestra.run.smithi143.stdout: ceph-test x86_64 2:16.2.13-565.g595355f4.el8 ceph 52 M 2023-08-28T02:01:28.782 INFO:teuthology.orchestra.run.smithi143.stdout:Installing dependencies: 2023-08-28T02:01:28.782 INFO:teuthology.orchestra.run.smithi143.stdout: jq x86_64 1.6-3.el8 rhel-8-for-x86_64-appstream-rpms 202 k 2023-08-28T02:01:28.782 INFO:teuthology.orchestra.run.smithi143.stdout: libcephsqlite x86_64 2:16.2.13-565.g595355f4.el8 ceph 174 k 2023-08-28T02:01:28.782 INFO:teuthology.orchestra.run.smithi143.stdout: oniguruma x86_64 6.8.2-2.el8 rhel-8-for-x86_64-appstream-rpms 187 k 2023-08-28T02:01:28.782 INFO:teuthology.orchestra.run.smithi143.stdout: socat x86_64 1.7.4.1-1.el8 rhel-8-for-x86_64-appstream-rpms 323 k 2023-08-28T02:01:28.782 INFO:teuthology.orchestra.run.smithi143.stdout: xmlstarlet x86_64 1.6.1-20.el8 lab-extras 70 k 2023-08-28T02:01:28.782 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:01:28.783 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction Summary 2023-08-28T02:01:28.783 INFO:teuthology.orchestra.run.smithi143.stdout:============================================================================================ 2023-08-28T02:01:28.783 INFO:teuthology.orchestra.run.smithi143.stdout:Install 6 Packages 2023-08-28T02:01:28.783 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:01:28.783 INFO:teuthology.orchestra.run.smithi143.stdout:Total download size: 53 M 2023-08-28T02:01:28.783 INFO:teuthology.orchestra.run.smithi143.stdout:Installed size: 195 M 2023-08-28T02:01:28.784 INFO:teuthology.orchestra.run.smithi143.stdout:Downloading Packages: 2023-08-28T02:01:28.825 INFO:teuthology.orchestra.run.smithi143.stdout:(1/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 1.7 MB/s | 70 kB 00:00 2023-08-28T02:01:28.900 INFO:teuthology.orchestra.run.smithi143.stdout:(2/6): socat-1.7.4.1-1.el8.x86_64.rpm 4.8 MB/s | 323 kB 00:00 2023-08-28T02:01:28.942 INFO:teuthology.orchestra.run.smithi143.stdout:(3/6): oniguruma-6.8.2-2.el8.x86_64.rpm 4.4 MB/s | 187 kB 00:00 2023-08-28T02:01:28.984 INFO:teuthology.orchestra.run.smithi143.stdout:(4/6): jq-1.6-3.el8.x86_64.rpm 4.7 MB/s | 202 kB 00:00 2023-08-28T02:01:29.026 INFO:teuthology.orchestra.run.smithi143.stdout:(5/6): libcephsqlite-16.2.13-565.g595355f4.el8. 718 kB/s | 174 kB 00:00 2023-08-28T02:01:31.318 INFO:teuthology.orchestra.run.smithi143.stdout:(6/6): ceph-test-16.2.13-565.g595355f4.el8.x86_ 20 MB/s | 52 MB 00:02 2023-08-28T02:01:31.319 INFO:teuthology.orchestra.run.smithi143.stdout:-------------------------------------------------------------------------------- 2023-08-28T02:01:31.320 INFO:teuthology.orchestra.run.smithi143.stdout:Total 21 MB/s | 53 MB 00:02 2023-08-28T02:01:31.357 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction check 2023-08-28T02:01:31.378 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction check succeeded. 2023-08-28T02:01:31.378 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction test 2023-08-28T02:01:31.660 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction test succeeded. 2023-08-28T02:01:31.663 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction 2023-08-28T02:01:32.122 INFO:teuthology.orchestra.run.smithi143.stdout: Preparing : 1/1 2023-08-28T02:01:32.253 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 1/6 2023-08-28T02:01:32.331 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 1/6 2023-08-28T02:01:32.523 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : jq-1.6-3.el8.x86_64 2/6 2023-08-28T02:01:32.535 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: libcephsqlite-2:16.2.13-565.g595355f4.el8.x86_64 5/6 2023-08-28T02:01:32.728 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 3/6 2023-08-28T02:01:32.807 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : ceph-test-2:16.2.13-565.g595355f4.el8.x86_64 6/6 2023-08-28T02:01:32.882 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 4/6 2023-08-28T02:01:33.035 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: ceph-test-2:16.2.13-565.g595355f4.el8.x86_64 6/6 2023-08-28T02:01:33.036 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : ceph-test-2:16.2.13-565.g595355f4.el8.x86_64 1/6 2023-08-28T02:01:33.036 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : libcephsqlite-2:16.2.13-565.g595355f4.el8.x86_64 2/6 2023-08-28T02:01:33.036 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 3/6 2023-08-28T02:01:33.036 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 4/6 2023-08-28T02:01:33.036 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 5/6 2023-08-28T02:01:33.036 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : libcephsqlite-2:16.2.13-565.g595355f4.el8.x86_64 5/6 2023-08-28T02:01:34.171 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : jq-1.6-3.el8.x86_64 6/6 2023-08-28T02:01:34.171 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:01:05 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:01:36.490 INFO:teuthology.orchestra.run.smithi169.stdout:Installed products updated. 2023-08-28T02:01:36.674 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:01:36.674 INFO:teuthology.orchestra.run.smithi169.stdout:Installed: 2023-08-28T02:01:36.674 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-test-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:01:36.674 INFO:teuthology.orchestra.run.smithi169.stdout: jq-1.6-3.el8.x86_64 2023-08-28T02:01:36.674 INFO:teuthology.orchestra.run.smithi169.stdout: libcephsqlite-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:01:36.675 INFO:teuthology.orchestra.run.smithi169.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-08-28T02:01:36.675 INFO:teuthology.orchestra.run.smithi169.stdout: socat-1.7.4.1-1.el8.x86_64 2023-08-28T02:01:36.675 INFO:teuthology.orchestra.run.smithi169.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-08-28T02:01:36.675 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:01:36.675 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-08-28T02:01:36.807 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install ceph 2023-08-28T02:01:37.142 INFO:teuthology.orchestra.run.smithi169.stdout:Updating Subscription Management repositories. 2023-08-28T02:01:37.566 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: libcephsqlite-2:16.2.13-565.g595355f4.el8.x86_64 5/6 2023-08-28T02:01:37.799 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : ceph-test-2:16.2.13-565.g595355f4.el8.x86_64 6/6 2023-08-28T02:01:37.946 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:01:08 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:01:38.024 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: ceph-test-2:16.2.13-565.g595355f4.el8.x86_64 6/6 2023-08-28T02:01:38.024 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : ceph-test-2:16.2.13-565.g595355f4.el8.x86_64 1/6 2023-08-28T02:01:38.024 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : libcephsqlite-2:16.2.13-565.g595355f4.el8.x86_64 2/6 2023-08-28T02:01:38.025 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 3/6 2023-08-28T02:01:38.025 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 4/6 2023-08-28T02:01:38.025 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 5/6 2023-08-28T02:01:39.177 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : jq-1.6-3.el8.x86_64 6/6 2023-08-28T02:01:39.177 INFO:teuthology.orchestra.run.smithi143.stdout:Last metadata expiration check: 0:01:10 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:01:39.310 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-08-28T02:01:39.313 INFO:teuthology.orchestra.run.smithi169.stdout:======================================================================================================= 2023-08-28T02:01:39.313 INFO:teuthology.orchestra.run.smithi169.stdout: Package Arch Version Repository Size 2023-08-28T02:01:39.313 INFO:teuthology.orchestra.run.smithi169.stdout:======================================================================================================= 2023-08-28T02:01:39.313 INFO:teuthology.orchestra.run.smithi169.stdout:Installing: 2023-08-28T02:01:39.314 INFO:teuthology.orchestra.run.smithi169.stdout: ceph x86_64 2:16.2.13-565.g595355f4.el8 ceph 6.4 k 2023-08-28T02:01:39.314 INFO:teuthology.orchestra.run.smithi169.stdout:Installing dependencies: 2023-08-28T02:01:39.314 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-mds x86_64 2:16.2.13-565.g595355f4.el8 ceph 2.4 M 2023-08-28T02:01:39.314 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-mgr x86_64 2:16.2.13-565.g595355f4.el8 ceph 1.6 M 2023-08-28T02:01:39.314 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-mgr-modules-core noarch 2:16.2.13-565.g595355f4.el8 ceph-noarch 244 k 2023-08-28T02:01:39.314 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-mon x86_64 2:16.2.13-565.g595355f4.el8 ceph 5.0 M 2023-08-28T02:01:39.314 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-osd x86_64 2:16.2.13-565.g595355f4.el8 ceph 18 M 2023-08-28T02:01:39.314 INFO:teuthology.orchestra.run.smithi169.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-08-28T02:01:39.314 INFO:teuthology.orchestra.run.smithi169.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-08-28T02:01:39.314 INFO:teuthology.orchestra.run.smithi169.stdout: python3-cffi x86_64 1.11.5-5.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2023-08-28T02:01:39.314 INFO:teuthology.orchestra.run.smithi169.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-08-28T02:01:39.315 INFO:teuthology.orchestra.run.smithi169.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-08-28T02:01:39.315 INFO:teuthology.orchestra.run.smithi169.stdout: python3-cryptography x86_64 3.2.1-5.el8 rhel-8-for-x86_64-baseos-rpms 559 k 2023-08-28T02:01:39.315 INFO:teuthology.orchestra.run.smithi169.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-08-28T02:01:39.315 INFO:teuthology.orchestra.run.smithi169.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-08-28T02:01:39.315 INFO:teuthology.orchestra.run.smithi169.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-08-28T02:01:39.315 INFO:teuthology.orchestra.run.smithi169.stdout: python3-mako noarch 1.0.6-13.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2023-08-28T02:01:39.315 INFO:teuthology.orchestra.run.smithi169.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2023-08-28T02:01:39.315 INFO:teuthology.orchestra.run.smithi169.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-08-28T02:01:39.315 INFO:teuthology.orchestra.run.smithi169.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-08-28T02:01:39.315 INFO:teuthology.orchestra.run.smithi169.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-08-28T02:01:39.315 INFO:teuthology.orchestra.run.smithi169.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2023-08-28T02:01:39.316 INFO:teuthology.orchestra.run.smithi169.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2023-08-28T02:01:39.316 INFO:teuthology.orchestra.run.smithi169.stdout: python3-pytz noarch 2017.2-9.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2023-08-28T02:01:39.316 INFO:teuthology.orchestra.run.smithi169.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-08-28T02:01:39.316 INFO:teuthology.orchestra.run.smithi169.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-08-28T02:01:39.316 INFO:teuthology.orchestra.run.smithi169.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-08-28T02:01:39.316 INFO:teuthology.orchestra.run.smithi169.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-08-28T02:01:39.316 INFO:teuthology.orchestra.run.smithi169.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-08-28T02:01:39.316 INFO:teuthology.orchestra.run.smithi169.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-08-28T02:01:39.316 INFO:teuthology.orchestra.run.smithi169.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-08-28T02:01:39.316 INFO:teuthology.orchestra.run.smithi169.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2023-08-28T02:01:39.316 INFO:teuthology.orchestra.run.smithi169.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-08-28T02:01:39.316 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:01:39.317 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction Summary 2023-08-28T02:01:39.317 INFO:teuthology.orchestra.run.smithi169.stdout:======================================================================================================= 2023-08-28T02:01:39.317 INFO:teuthology.orchestra.run.smithi169.stdout:Install 32 Packages 2023-08-28T02:01:39.317 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:01:39.317 INFO:teuthology.orchestra.run.smithi169.stdout:Total download size: 31 M 2023-08-28T02:01:39.317 INFO:teuthology.orchestra.run.smithi169.stdout:Installed size: 101 M 2023-08-28T02:01:39.318 INFO:teuthology.orchestra.run.smithi169.stdout:Downloading Packages: 2023-08-28T02:01:39.498 INFO:teuthology.orchestra.run.smithi169.stdout:(1/32): ceph-16.2.13-565.g595355f4.el8.x86_64.r 45 kB/s | 6.4 kB 00:00 2023-08-28T02:01:40.040 INFO:teuthology.orchestra.run.smithi169.stdout:(2/32): ceph-mgr-16.2.13-565.g595355f4.el8.x86_ 2.3 MB/s | 1.6 MB 00:00 2023-08-28T02:01:40.207 INFO:teuthology.orchestra.run.smithi169.stdout:(3/32): ceph-mon-16.2.13-565.g595355f4.el8.x86_ 7.1 MB/s | 5.0 MB 00:00 2023-08-28T02:01:40.257 INFO:teuthology.orchestra.run.smithi169.stdout:(4/32): ceph-mds-16.2.13-565.g595355f4.el8.x86_ 2.7 MB/s | 2.4 MB 00:00 2023-08-28T02:01:40.282 INFO:teuthology.orchestra.run.smithi169.stdout:(5/32): ceph-mgr-modules-core-16.2.13-565.g5953 3.2 MB/s | 244 kB 00:00 2023-08-28T02:01:40.572 INFO:teuthology.orchestra.run.smithi169.stdout:(6/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rpm 138 kB/s | 44 kB 00:00 2023-08-28T02:01:40.650 INFO:teuthology.orchestra.run.smithi169.stdout:(7/32): python3-beautifulsoup4-4.6.3-2.el8.1.no 505 kB/s | 185 kB 00:00 2023-08-28T02:01:40.775 INFO:teuthology.orchestra.run.smithi169.stdout:(8/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 853 kB/s | 173 kB 00:00 2023-08-28T02:01:40.834 INFO:teuthology.orchestra.run.smithi169.stdout:(9/32): python3-cherrypy-18.4.0-1.el8.noarch.rp 2.0 MB/s | 384 kB 00:00 2023-08-28T02:01:40.859 INFO:teuthology.orchestra.run.smithi169.stdout:(10/32): python3-jaraco-6.2-6.el8.noarch.rpm 133 kB/s | 11 kB 00:00 2023-08-28T02:01:40.918 INFO:teuthology.orchestra.run.smithi169.stdout:(11/32): python3-jaraco-functools-2.0-4.el8.noa 210 kB/s | 18 kB 00:00 2023-08-28T02:01:40.943 INFO:teuthology.orchestra.run.smithi169.stdout:(12/32): python3-logutils-0.3.5-11.el8.noarch.r 580 kB/s | 49 kB 00:00 2023-08-28T02:01:41.018 INFO:teuthology.orchestra.run.smithi169.stdout:(13/32): python3-more-itertools-7.2.0-3.el8.noa 585 kB/s | 59 kB 00:00 2023-08-28T02:01:41.052 INFO:teuthology.orchestra.run.smithi169.stdout:(14/32): python3-pecan-1.3.2-9.el8.noarch.rpm 2.5 MB/s | 283 kB 00:00 2023-08-28T02:01:41.102 INFO:teuthology.orchestra.run.smithi169.stdout:(15/32): python3-portend-2.6-1.el8.noarch.rpm 186 kB/s | 16 kB 00:00 2023-08-28T02:01:41.136 INFO:teuthology.orchestra.run.smithi169.stdout:(16/32): python3-simplegeneric-0.8.1-17.el8.noa 223 kB/s | 19 kB 00:00 2023-08-28T02:01:41.186 INFO:teuthology.orchestra.run.smithi169.stdout:(17/32): python3-singledispatch-3.4.0.3-18.el8. 286 kB/s | 24 kB 00:00 2023-08-28T02:01:41.220 INFO:teuthology.orchestra.run.smithi169.stdout:(18/32): python3-tempora-1.14.1-5.el8.noarch.rp 350 kB/s | 29 kB 00:00 2023-08-28T02:01:41.270 INFO:teuthology.orchestra.run.smithi169.stdout:(19/32): python3-trustme-0.6.0-4.el8.noarch.rpm 330 kB/s | 27 kB 00:00 2023-08-28T02:01:41.329 INFO:teuthology.orchestra.run.smithi169.stdout:(20/32): python3-waitress-1.4.3-1.el8.noarch.rp 2.3 MB/s | 255 kB 00:00 2023-08-28T02:01:41.421 INFO:teuthology.orchestra.run.smithi169.stdout:(21/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 1.6 MB/s | 251 kB 00:00 2023-08-28T02:01:41.446 INFO:teuthology.orchestra.run.smithi169.stdout:(22/32): python3-webtest-2.0.33-1.el8.noarch.rp 732 kB/s | 85 kB 00:00 2023-08-28T02:01:41.496 INFO:teuthology.orchestra.run.smithi169.stdout:(23/32): python3-zc-lockfile-2.0-2.el8.noarch.r 305 kB/s | 23 kB 00:00 2023-08-28T02:01:41.522 INFO:teuthology.orchestra.run.smithi169.stdout:(24/32): python3-pytz-2017.2-9.el8.noarch.rpm 719 kB/s | 54 kB 00:00 2023-08-28T02:01:41.552 INFO:teuthology.orchestra.run.smithi143.stdout:Installed products updated. 2023-08-28T02:01:41.580 INFO:teuthology.orchestra.run.smithi169.stdout:(25/32): python3-mako-1.0.6-13.el8.noarch.rpm 2.9 MB/s | 157 kB 00:00 2023-08-28T02:01:41.597 INFO:teuthology.orchestra.run.smithi169.stdout:(26/32): python3-markupsafe-0.23-19.el8.x86_64. 387 kB/s | 39 kB 00:00 2023-08-28T02:01:41.648 INFO:teuthology.orchestra.run.smithi169.stdout:(27/32): python3-werkzeug-0.12.2-4.el8.noarch.r 6.7 MB/s | 457 kB 00:00 2023-08-28T02:01:41.673 INFO:teuthology.orchestra.run.smithi169.stdout:(28/32): python3-pyOpenSSL-19.0.0-1.el8.noarch. 1.3 MB/s | 103 kB 00:00 2023-08-28T02:01:41.706 INFO:teuthology.orchestra.run.smithi169.stdout:(29/32): python3-pycparser-2.14-14.el8.noarch.r 1.8 MB/s | 109 kB 00:00 2023-08-28T02:01:41.735 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:01:41.735 INFO:teuthology.orchestra.run.smithi143.stdout:Installed: 2023-08-28T02:01:41.735 INFO:teuthology.orchestra.run.smithi143.stdout: ceph-test-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:01:41.735 INFO:teuthology.orchestra.run.smithi143.stdout: jq-1.6-3.el8.x86_64 2023-08-28T02:01:41.736 INFO:teuthology.orchestra.run.smithi143.stdout: libcephsqlite-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:01:41.736 INFO:teuthology.orchestra.run.smithi143.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-08-28T02:01:41.736 INFO:teuthology.orchestra.run.smithi143.stdout: socat-1.7.4.1-1.el8.x86_64 2023-08-28T02:01:41.736 INFO:teuthology.orchestra.run.smithi143.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-08-28T02:01:41.736 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:01:41.736 INFO:teuthology.orchestra.run.smithi143.stdout:Complete! 2023-08-28T02:01:41.748 INFO:teuthology.orchestra.run.smithi169.stdout:(30/32): python3-cryptography-3.2.1-5.el8.x86_6 7.3 MB/s | 559 kB 00:00 2023-08-28T02:01:41.773 INFO:teuthology.orchestra.run.smithi169.stdout:(31/32): python3-cffi-1.11.5-5.el8.x86_64.rpm 3.5 MB/s | 238 kB 00:00 2023-08-28T02:01:41.871 DEBUG:teuthology.orchestra.run.smithi143:> sudo yum -y install ceph 2023-08-28T02:01:41.957 INFO:teuthology.orchestra.run.smithi169.stdout:(32/32): ceph-osd-16.2.13-565.g595355f4.el8.x86 9.6 MB/s | 18 MB 00:01 2023-08-28T02:01:41.960 INFO:teuthology.orchestra.run.smithi169.stdout:-------------------------------------------------------------------------------- 2023-08-28T02:01:41.960 INFO:teuthology.orchestra.run.smithi169.stdout:Total 12 MB/s | 31 MB 00:02 2023-08-28T02:01:42.051 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction check 2023-08-28T02:01:42.090 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction check succeeded. 2023-08-28T02:01:42.090 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction test 2023-08-28T02:01:42.207 INFO:teuthology.orchestra.run.smithi143.stdout:Updating Subscription Management repositories. 2023-08-28T02:01:42.306 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction test succeeded. 2023-08-28T02:01:42.310 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction 2023-08-28T02:01:43.114 INFO:teuthology.orchestra.run.smithi143.stdout:Last metadata expiration check: 0:01:14 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:01:43.435 INFO:teuthology.orchestra.run.smithi169.stdout: Preparing : 1/1 2023-08-28T02:01:43.680 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-08-28T02:01:43.920 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-08-28T02:01:44.134 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 3/32 2023-08-28T02:01:44.480 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 4/32 2023-08-28T02:01:44.480 INFO:teuthology.orchestra.run.smithi143.stdout:Dependencies resolved. 2023-08-28T02:01:44.483 INFO:teuthology.orchestra.run.smithi143.stdout:======================================================================================================= 2023-08-28T02:01:44.483 INFO:teuthology.orchestra.run.smithi143.stdout: Package Arch Version Repository Size 2023-08-28T02:01:44.484 INFO:teuthology.orchestra.run.smithi143.stdout:======================================================================================================= 2023-08-28T02:01:44.484 INFO:teuthology.orchestra.run.smithi143.stdout:Installing: 2023-08-28T02:01:44.484 INFO:teuthology.orchestra.run.smithi143.stdout: ceph x86_64 2:16.2.13-565.g595355f4.el8 ceph 6.4 k 2023-08-28T02:01:44.484 INFO:teuthology.orchestra.run.smithi143.stdout:Installing dependencies: 2023-08-28T02:01:44.484 INFO:teuthology.orchestra.run.smithi143.stdout: ceph-mds x86_64 2:16.2.13-565.g595355f4.el8 ceph 2.4 M 2023-08-28T02:01:44.484 INFO:teuthology.orchestra.run.smithi143.stdout: ceph-mgr x86_64 2:16.2.13-565.g595355f4.el8 ceph 1.6 M 2023-08-28T02:01:44.484 INFO:teuthology.orchestra.run.smithi143.stdout: ceph-mgr-modules-core noarch 2:16.2.13-565.g595355f4.el8 ceph-noarch 244 k 2023-08-28T02:01:44.484 INFO:teuthology.orchestra.run.smithi143.stdout: ceph-mon x86_64 2:16.2.13-565.g595355f4.el8 ceph 5.0 M 2023-08-28T02:01:44.484 INFO:teuthology.orchestra.run.smithi143.stdout: ceph-osd x86_64 2:16.2.13-565.g595355f4.el8 ceph 18 M 2023-08-28T02:01:44.484 INFO:teuthology.orchestra.run.smithi143.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-08-28T02:01:44.484 INFO:teuthology.orchestra.run.smithi143.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-08-28T02:01:44.485 INFO:teuthology.orchestra.run.smithi143.stdout: python3-cffi x86_64 1.11.5-5.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2023-08-28T02:01:44.485 INFO:teuthology.orchestra.run.smithi143.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-08-28T02:01:44.485 INFO:teuthology.orchestra.run.smithi143.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-08-28T02:01:44.485 INFO:teuthology.orchestra.run.smithi143.stdout: python3-cryptography x86_64 3.2.1-5.el8 rhel-8-for-x86_64-baseos-rpms 559 k 2023-08-28T02:01:44.485 INFO:teuthology.orchestra.run.smithi143.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-08-28T02:01:44.485 INFO:teuthology.orchestra.run.smithi143.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-08-28T02:01:44.485 INFO:teuthology.orchestra.run.smithi143.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-08-28T02:01:44.485 INFO:teuthology.orchestra.run.smithi143.stdout: python3-mako noarch 1.0.6-13.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2023-08-28T02:01:44.485 INFO:teuthology.orchestra.run.smithi143.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2023-08-28T02:01:44.485 INFO:teuthology.orchestra.run.smithi143.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-08-28T02:01:44.485 INFO:teuthology.orchestra.run.smithi143.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-08-28T02:01:44.485 INFO:teuthology.orchestra.run.smithi143.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-08-28T02:01:44.486 INFO:teuthology.orchestra.run.smithi143.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2023-08-28T02:01:44.486 INFO:teuthology.orchestra.run.smithi143.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2023-08-28T02:01:44.486 INFO:teuthology.orchestra.run.smithi143.stdout: python3-pytz noarch 2017.2-9.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2023-08-28T02:01:44.486 INFO:teuthology.orchestra.run.smithi143.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-08-28T02:01:44.486 INFO:teuthology.orchestra.run.smithi143.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-08-28T02:01:44.486 INFO:teuthology.orchestra.run.smithi143.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-08-28T02:01:44.486 INFO:teuthology.orchestra.run.smithi143.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-08-28T02:01:44.486 INFO:teuthology.orchestra.run.smithi143.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-08-28T02:01:44.486 INFO:teuthology.orchestra.run.smithi143.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-08-28T02:01:44.486 INFO:teuthology.orchestra.run.smithi143.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-08-28T02:01:44.486 INFO:teuthology.orchestra.run.smithi143.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2023-08-28T02:01:44.486 INFO:teuthology.orchestra.run.smithi143.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-08-28T02:01:44.487 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:01:44.487 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction Summary 2023-08-28T02:01:44.487 INFO:teuthology.orchestra.run.smithi143.stdout:======================================================================================================= 2023-08-28T02:01:44.487 INFO:teuthology.orchestra.run.smithi143.stdout:Install 32 Packages 2023-08-28T02:01:44.487 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:01:44.487 INFO:teuthology.orchestra.run.smithi143.stdout:Total download size: 31 M 2023-08-28T02:01:44.487 INFO:teuthology.orchestra.run.smithi143.stdout:Installed size: 101 M 2023-08-28T02:01:44.488 INFO:teuthology.orchestra.run.smithi143.stdout:Downloading Packages: 2023-08-28T02:01:44.728 INFO:teuthology.orchestra.run.smithi143.stdout:(1/32): ceph-16.2.13-565.g595355f4.el8.x86_64.r 36 kB/s | 6.4 kB 00:00 2023-08-28T02:01:44.819 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 5/32 2023-08-28T02:01:44.828 INFO:teuthology.orchestra.run.smithi143.stdout:(2/32): ceph-mgr-16.2.13-565.g595355f4.el8.x86_ 5.8 MB/s | 1.6 MB 00:00 2023-08-28T02:01:44.957 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 6/32 2023-08-28T02:01:45.133 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 7/32 2023-08-28T02:01:45.146 INFO:teuthology.orchestra.run.smithi143.stdout:(3/32): ceph-mds-16.2.13-565.g595355f4.el8.x86_ 4.1 MB/s | 2.4 MB 00:00 2023-08-28T02:01:45.263 INFO:teuthology.orchestra.run.smithi143.stdout:(4/32): ceph-mgr-modules-core-16.2.13-565.g5953 2.0 MB/s | 244 kB 00:00 2023-08-28T02:01:45.363 INFO:teuthology.orchestra.run.smithi143.stdout:(5/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rpm 436 kB/s | 44 kB 00:00 2023-08-28T02:01:45.441 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 8/32 2023-08-28T02:01:45.455 INFO:teuthology.orchestra.run.smithi143.stdout:(6/32): python3-beautifulsoup4-4.6.3-2.el8.1.no 2.0 MB/s | 185 kB 00:00 2023-08-28T02:01:45.505 INFO:teuthology.orchestra.run.smithi143.stdout:(7/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 3.4 MB/s | 173 kB 00:00 2023-08-28T02:01:45.572 INFO:teuthology.orchestra.run.smithi143.stdout:(8/32): python3-cherrypy-18.4.0-1.el8.noarch.rp 5.6 MB/s | 384 kB 00:00 2023-08-28T02:01:45.614 INFO:teuthology.orchestra.run.smithi143.stdout:(9/32): python3-jaraco-6.2-6.el8.noarch.rpm 267 kB/s | 11 kB 00:00 2023-08-28T02:01:45.656 INFO:teuthology.orchestra.run.smithi143.stdout:(10/32): python3-jaraco-functools-2.0-4.el8.noa 422 kB/s | 18 kB 00:00 2023-08-28T02:01:45.686 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 9/32 2023-08-28T02:01:45.698 INFO:teuthology.orchestra.run.smithi143.stdout:(11/32): python3-logutils-0.3.5-11.el8.noarch.r 1.1 MB/s | 49 kB 00:00 2023-08-28T02:01:45.890 INFO:teuthology.orchestra.run.smithi143.stdout:(12/32): ceph-osd-16.2.13-565.g595355f4.el8.x86 17 MB/s | 18 MB 00:01 2023-08-28T02:01:45.915 INFO:teuthology.orchestra.run.smithi143.stdout:(13/32): python3-more-itertools-7.2.0-3.el8.noa 270 kB/s | 59 kB 00:00 2023-08-28T02:01:45.918 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 10/32 2023-08-28T02:01:45.957 INFO:teuthology.orchestra.run.smithi143.stdout:(14/32): python3-portend-2.6-1.el8.noarch.rpm 373 kB/s | 16 kB 00:00 2023-08-28T02:01:46.049 INFO:teuthology.orchestra.run.smithi143.stdout:(15/32): ceph-mon-16.2.13-565.g595355f4.el8.x86 3.8 MB/s | 5.0 MB 00:01 2023-08-28T02:01:46.075 INFO:teuthology.orchestra.run.smithi143.stdout:(16/32): python3-simplegeneric-0.8.1-17.el8.noa 160 kB/s | 19 kB 00:00 2023-08-28T02:01:46.117 INFO:teuthology.orchestra.run.smithi143.stdout:(17/32): python3-tempora-1.14.1-5.el8.noarch.rp 702 kB/s | 29 kB 00:00 2023-08-28T02:01:46.142 INFO:teuthology.orchestra.run.smithi143.stdout:(18/32): python3-singledispatch-3.4.0.3-18.el8. 259 kB/s | 24 kB 00:00 2023-08-28T02:01:46.163 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 11/32 2023-08-28T02:01:46.167 INFO:teuthology.orchestra.run.smithi143.stdout:(19/32): python3-pecan-1.3.2-9.el8.noarch.rpm 1.0 MB/s | 283 kB 00:00 2023-08-28T02:01:46.193 INFO:teuthology.orchestra.run.smithi143.stdout:(20/32): python3-trustme-0.6.0-4.el8.noarch.rpm 363 kB/s | 27 kB 00:00 2023-08-28T02:01:46.218 INFO:teuthology.orchestra.run.smithi143.stdout:(21/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 4.9 MB/s | 251 kB 00:00 2023-08-28T02:01:46.260 INFO:teuthology.orchestra.run.smithi143.stdout:(22/32): python3-waitress-1.4.3-1.el8.noarch.rp 2.1 MB/s | 255 kB 00:00 2023-08-28T02:01:46.285 INFO:teuthology.orchestra.run.smithi143.stdout:(23/32): python3-zc-lockfile-2.0-2.el8.noarch.r 342 kB/s | 23 kB 00:00 2023-08-28T02:01:46.311 INFO:teuthology.orchestra.run.smithi143.stdout:(24/32): python3-webtest-2.0.33-1.el8.noarch.rp 726 kB/s | 85 kB 00:00 2023-08-28T02:01:46.331 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 12/32 2023-08-28T02:01:46.361 INFO:teuthology.orchestra.run.smithi143.stdout:(25/32): python3-cffi-1.11.5-5.el8.x86_64.rpm 4.7 MB/s | 238 kB 00:00 2023-08-28T02:01:46.386 INFO:teuthology.orchestra.run.smithi143.stdout:(26/32): python3-pycparser-2.14-14.el8.noarch.r 867 kB/s | 109 kB 00:00 2023-08-28T02:01:46.412 INFO:teuthology.orchestra.run.smithi143.stdout:(27/32): python3-pytz-2017.2-9.el8.noarch.rpm 1.1 MB/s | 54 kB 00:00 2023-08-28T02:01:46.437 INFO:teuthology.orchestra.run.smithi143.stdout:(28/32): python3-markupsafe-0.23-19.el8.x86_64. 770 kB/s | 39 kB 00:00 2023-08-28T02:01:46.463 INFO:teuthology.orchestra.run.smithi143.stdout:(29/32): python3-mako-1.0.6-13.el8.noarch.rpm 3.0 MB/s | 157 kB 00:00 2023-08-28T02:01:46.488 INFO:teuthology.orchestra.run.smithi143.stdout:(30/32): python3-cryptography-3.2.1-5.el8.x86_6 2.7 MB/s | 559 kB 00:00 2023-08-28T02:01:46.513 INFO:teuthology.orchestra.run.smithi143.stdout:(31/32): python3-pyOpenSSL-19.0.0-1.el8.noarch. 2.0 MB/s | 103 kB 00:00 2023-08-28T02:01:46.546 INFO:teuthology.orchestra.run.smithi143.stdout:(32/32): python3-werkzeug-0.12.2-4.el8.noarch.r 4.1 MB/s | 457 kB 00:00 2023-08-28T02:01:46.548 INFO:teuthology.orchestra.run.smithi143.stdout:-------------------------------------------------------------------------------- 2023-08-28T02:01:46.548 INFO:teuthology.orchestra.run.smithi143.stdout:Total 15 MB/s | 31 MB 00:02 2023-08-28T02:01:46.590 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 13/32 2023-08-28T02:01:46.634 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction check 2023-08-28T02:01:46.672 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction check succeeded. 2023-08-28T02:01:46.672 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction test 2023-08-28T02:01:46.756 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 14/32 2023-08-28T02:01:46.886 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction test succeeded. 2023-08-28T02:01:46.889 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction 2023-08-28T02:01:46.970 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 15/32 2023-08-28T02:01:47.125 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 16/32 2023-08-28T02:01:47.312 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 17/32 2023-08-28T02:01:47.455 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 18/32 2023-08-28T02:01:47.653 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 19/32 2023-08-28T02:01:47.895 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 20/32 2023-08-28T02:01:47.978 INFO:teuthology.orchestra.run.smithi143.stdout: Preparing : 1/1 2023-08-28T02:01:48.080 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-08-28T02:01:48.180 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-08-28T02:01:48.369 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-08-28T02:01:48.467 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-08-28T02:01:48.663 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-08-28T02:01:48.687 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 3/32 2023-08-28T02:01:48.867 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 24/32 2023-08-28T02:01:48.909 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 4/32 2023-08-28T02:01:49.149 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 5/32 2023-08-28T02:01:49.198 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 25/32 2023-08-28T02:01:49.362 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 6/32 2023-08-28T02:01:49.553 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-08-28T02:01:49.584 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 7/32 2023-08-28T02:01:49.945 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 8/32 2023-08-28T02:01:50.219 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 9/32 2023-08-28T02:01:50.383 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 10/32 2023-08-28T02:01:50.551 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 11/32 2023-08-28T02:01:50.585 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : ceph-mgr-modules-core-2:16.2.13-565.g595355f4.el8. 27/32 2023-08-28T02:01:50.751 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 12/32 2023-08-28T02:01:50.757 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : ceph-mgr-2:16.2.13-565.g595355f4.el8.x86_64 28/32 2023-08-28T02:01:51.004 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 13/32 2023-08-28T02:01:51.182 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 14/32 2023-08-28T02:01:51.363 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 15/32 2023-08-28T02:01:51.528 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 16/32 2023-08-28T02:01:51.744 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 17/32 2023-08-28T02:01:51.889 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 18/32 2023-08-28T02:01:52.086 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 19/32 2023-08-28T02:01:52.248 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: ceph-mgr-2:16.2.13-565.g595355f4.el8.x86_64 28/32 2023-08-28T02:01:52.287 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 20/32 2023-08-28T02:01:52.459 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : ceph-osd-2:16.2.13-565.g595355f4.el8.x86_64 29/32 2023-08-28T02:01:52.480 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-08-28T02:01:52.735 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-08-28T02:01:52.884 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: ceph-osd-2:16.2.13-565.g595355f4.el8.x86_64 29/32 2023-08-28T02:01:52.996 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-08-28T02:01:53.032 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : ceph-mon-2:16.2.13-565.g595355f4.el8.x86_64 30/32 2023-08-28T02:01:53.193 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 24/32 2023-08-28T02:01:53.249 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: ceph-mon-2:16.2.13-565.g595355f4.el8.x86_64 30/32 2023-08-28T02:01:53.415 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : ceph-mds-2:16.2.13-565.g595355f4.el8.x86_64 31/32 2023-08-28T02:01:53.433 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: ceph-mds-2:16.2.13-565.g595355f4.el8.x86_64 31/32 2023-08-28T02:01:53.531 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 25/32 2023-08-28T02:01:53.546 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : ceph-2:16.2.13-565.g595355f4.el8.x86_64 32/32 2023-08-28T02:01:53.836 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-08-28T02:01:53.912 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: ceph-2:16.2.13-565.g595355f4.el8.x86_64 32/32 2023-08-28T02:01:53.913 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : ceph-2:16.2.13-565.g595355f4.el8.x86_64 1/32 2023-08-28T02:01:53.913 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : ceph-mds-2:16.2.13-565.g595355f4.el8.x86_64 2/32 2023-08-28T02:01:53.913 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : ceph-mgr-2:16.2.13-565.g595355f4.el8.x86_64 3/32 2023-08-28T02:01:53.913 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : ceph-mon-2:16.2.13-565.g595355f4.el8.x86_64 4/32 2023-08-28T02:01:53.913 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : ceph-osd-2:16.2.13-565.g595355f4.el8.x86_64 5/32 2023-08-28T02:01:53.913 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : ceph-mgr-modules-core-2:16.2.13-565.g595355f4.el8. 6/32 2023-08-28T02:01:53.913 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 7/32 2023-08-28T02:01:53.913 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 8/32 2023-08-28T02:01:53.913 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 9/32 2023-08-28T02:01:53.913 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 10/32 2023-08-28T02:01:53.913 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 11/32 2023-08-28T02:01:53.913 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 12/32 2023-08-28T02:01:53.914 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 13/32 2023-08-28T02:01:53.914 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 14/32 2023-08-28T02:01:53.914 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 15/32 2023-08-28T02:01:53.914 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-portend-2.6-1.el8.noarch 16/32 2023-08-28T02:01:53.914 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 17/32 2023-08-28T02:01:53.914 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 18/32 2023-08-28T02:01:53.914 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 19/32 2023-08-28T02:01:53.914 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 20/32 2023-08-28T02:01:53.914 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 21/32 2023-08-28T02:01:53.914 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 22/32 2023-08-28T02:01:53.914 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 23/32 2023-08-28T02:01:53.914 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 24/32 2023-08-28T02:01:53.914 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 25/32 2023-08-28T02:01:53.915 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 26/32 2023-08-28T02:01:53.915 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 27/32 2023-08-28T02:01:53.915 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 28/32 2023-08-28T02:01:53.916 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 29/32 2023-08-28T02:01:53.916 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 30/32 2023-08-28T02:01:53.916 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 31/32 2023-08-28T02:01:54.144 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : ceph-mgr-modules-core-2:16.2.13-565.g595355f4.el8. 27/32 2023-08-28T02:01:54.302 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : ceph-mgr-2:16.2.13-565.g595355f4.el8.x86_64 28/32 2023-08-28T02:01:55.452 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 32/32 2023-08-28T02:01:55.452 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:01:26 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:01:55.797 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: ceph-mgr-2:16.2.13-565.g595355f4.el8.x86_64 28/32 2023-08-28T02:01:55.983 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : ceph-osd-2:16.2.13-565.g595355f4.el8.x86_64 29/32 2023-08-28T02:01:56.408 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: ceph-osd-2:16.2.13-565.g595355f4.el8.x86_64 29/32 2023-08-28T02:01:56.583 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : ceph-mon-2:16.2.13-565.g595355f4.el8.x86_64 30/32 2023-08-28T02:01:56.799 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: ceph-mon-2:16.2.13-565.g595355f4.el8.x86_64 30/32 2023-08-28T02:01:56.958 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : ceph-mds-2:16.2.13-565.g595355f4.el8.x86_64 31/32 2023-08-28T02:01:56.975 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: ceph-mds-2:16.2.13-565.g595355f4.el8.x86_64 31/32 2023-08-28T02:01:57.122 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : ceph-2:16.2.13-565.g595355f4.el8.x86_64 32/32 2023-08-28T02:01:57.469 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: ceph-2:16.2.13-565.g595355f4.el8.x86_64 32/32 2023-08-28T02:01:57.470 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : ceph-2:16.2.13-565.g595355f4.el8.x86_64 1/32 2023-08-28T02:01:57.470 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : ceph-mds-2:16.2.13-565.g595355f4.el8.x86_64 2/32 2023-08-28T02:01:57.470 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : ceph-mgr-2:16.2.13-565.g595355f4.el8.x86_64 3/32 2023-08-28T02:01:57.470 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : ceph-mon-2:16.2.13-565.g595355f4.el8.x86_64 4/32 2023-08-28T02:01:57.470 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : ceph-osd-2:16.2.13-565.g595355f4.el8.x86_64 5/32 2023-08-28T02:01:57.470 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : ceph-mgr-modules-core-2:16.2.13-565.g595355f4.el8. 6/32 2023-08-28T02:01:57.470 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 7/32 2023-08-28T02:01:57.470 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 8/32 2023-08-28T02:01:57.470 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 9/32 2023-08-28T02:01:57.471 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 10/32 2023-08-28T02:01:57.471 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 11/32 2023-08-28T02:01:57.471 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 12/32 2023-08-28T02:01:57.471 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 13/32 2023-08-28T02:01:57.471 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 14/32 2023-08-28T02:01:57.471 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 15/32 2023-08-28T02:01:57.471 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-portend-2.6-1.el8.noarch 16/32 2023-08-28T02:01:57.471 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 17/32 2023-08-28T02:01:57.471 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 18/32 2023-08-28T02:01:57.471 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 19/32 2023-08-28T02:01:57.472 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 20/32 2023-08-28T02:01:57.472 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 21/32 2023-08-28T02:01:57.472 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 22/32 2023-08-28T02:01:57.472 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 23/32 2023-08-28T02:01:57.472 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 24/32 2023-08-28T02:01:57.472 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 25/32 2023-08-28T02:01:57.472 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 26/32 2023-08-28T02:01:57.472 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 27/32 2023-08-28T02:01:57.472 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 28/32 2023-08-28T02:01:57.473 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 29/32 2023-08-28T02:01:57.473 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 30/32 2023-08-28T02:01:57.473 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 31/32 2023-08-28T02:01:57.798 INFO:teuthology.orchestra.run.smithi169.stdout:Installed products updated. 2023-08-28T02:01:57.985 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:01:57.985 INFO:teuthology.orchestra.run.smithi169.stdout:Installed: 2023-08-28T02:01:57.985 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:01:57.985 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-mds-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:01:57.985 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-mgr-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:01:57.985 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-mgr-modules-core-2:16.2.13-565.g595355f4.el8.noarch 2023-08-28T02:01:57.985 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-mon-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:01:57.985 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-osd-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:01:57.985 INFO:teuthology.orchestra.run.smithi169.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-08-28T02:01:57.985 INFO:teuthology.orchestra.run.smithi169.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-08-28T02:01:57.985 INFO:teuthology.orchestra.run.smithi169.stdout: python3-cffi-1.11.5-5.el8.x86_64 2023-08-28T02:01:57.986 INFO:teuthology.orchestra.run.smithi169.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-08-28T02:01:57.986 INFO:teuthology.orchestra.run.smithi169.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-08-28T02:01:57.986 INFO:teuthology.orchestra.run.smithi169.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2023-08-28T02:01:57.986 INFO:teuthology.orchestra.run.smithi169.stdout: python3-jaraco-6.2-6.el8.noarch 2023-08-28T02:01:57.986 INFO:teuthology.orchestra.run.smithi169.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-08-28T02:01:57.986 INFO:teuthology.orchestra.run.smithi169.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-08-28T02:01:57.986 INFO:teuthology.orchestra.run.smithi169.stdout: python3-mako-1.0.6-13.el8.noarch 2023-08-28T02:01:57.986 INFO:teuthology.orchestra.run.smithi169.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-08-28T02:01:57.986 INFO:teuthology.orchestra.run.smithi169.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-08-28T02:01:57.986 INFO:teuthology.orchestra.run.smithi169.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-08-28T02:01:57.986 INFO:teuthology.orchestra.run.smithi169.stdout: python3-portend-2.6-1.el8.noarch 2023-08-28T02:01:57.986 INFO:teuthology.orchestra.run.smithi169.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-08-28T02:01:57.987 INFO:teuthology.orchestra.run.smithi169.stdout: python3-pycparser-2.14-14.el8.noarch 2023-08-28T02:01:57.987 INFO:teuthology.orchestra.run.smithi169.stdout: python3-pytz-2017.2-9.el8.noarch 2023-08-28T02:01:57.987 INFO:teuthology.orchestra.run.smithi169.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-08-28T02:01:57.987 INFO:teuthology.orchestra.run.smithi169.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-08-28T02:01:57.987 INFO:teuthology.orchestra.run.smithi169.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-08-28T02:01:57.987 INFO:teuthology.orchestra.run.smithi169.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-08-28T02:01:57.987 INFO:teuthology.orchestra.run.smithi169.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-08-28T02:01:57.987 INFO:teuthology.orchestra.run.smithi169.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-08-28T02:01:57.987 INFO:teuthology.orchestra.run.smithi169.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-08-28T02:01:57.987 INFO:teuthology.orchestra.run.smithi169.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-08-28T02:01:57.987 INFO:teuthology.orchestra.run.smithi169.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-08-28T02:01:57.987 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:01:57.988 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-08-28T02:01:58.138 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install ceph-base 2023-08-28T02:01:58.486 INFO:teuthology.orchestra.run.smithi169.stdout:Updating Subscription Management repositories. 2023-08-28T02:01:59.011 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 32/32 2023-08-28T02:01:59.011 INFO:teuthology.orchestra.run.smithi143.stdout:Last metadata expiration check: 0:01:30 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:01:59.356 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:01:30 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:02:00.606 INFO:teuthology.orchestra.run.smithi169.stdout:Package ceph-base-2:16.2.13-565.g595355f4.el8.x86_64 is already installed. 2023-08-28T02:02:00.717 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-08-28T02:02:00.718 INFO:teuthology.orchestra.run.smithi169.stdout:Nothing to do. 2023-08-28T02:02:00.718 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-08-28T02:02:00.827 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install cephadm 2023-08-28T02:02:01.172 INFO:teuthology.orchestra.run.smithi169.stdout:Updating Subscription Management repositories. 2023-08-28T02:02:01.319 INFO:teuthology.orchestra.run.smithi143.stdout:Installed products updated. 2023-08-28T02:02:01.509 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:02:01.509 INFO:teuthology.orchestra.run.smithi143.stdout:Installed: 2023-08-28T02:02:01.509 INFO:teuthology.orchestra.run.smithi143.stdout: ceph-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:02:01.509 INFO:teuthology.orchestra.run.smithi143.stdout: ceph-mds-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:02:01.509 INFO:teuthology.orchestra.run.smithi143.stdout: ceph-mgr-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:02:01.509 INFO:teuthology.orchestra.run.smithi143.stdout: ceph-mgr-modules-core-2:16.2.13-565.g595355f4.el8.noarch 2023-08-28T02:02:01.509 INFO:teuthology.orchestra.run.smithi143.stdout: ceph-mon-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:02:01.509 INFO:teuthology.orchestra.run.smithi143.stdout: ceph-osd-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:02:01.509 INFO:teuthology.orchestra.run.smithi143.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-08-28T02:02:01.509 INFO:teuthology.orchestra.run.smithi143.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-08-28T02:02:01.510 INFO:teuthology.orchestra.run.smithi143.stdout: python3-cffi-1.11.5-5.el8.x86_64 2023-08-28T02:02:01.510 INFO:teuthology.orchestra.run.smithi143.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-08-28T02:02:01.510 INFO:teuthology.orchestra.run.smithi143.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-08-28T02:02:01.510 INFO:teuthology.orchestra.run.smithi143.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2023-08-28T02:02:01.510 INFO:teuthology.orchestra.run.smithi143.stdout: python3-jaraco-6.2-6.el8.noarch 2023-08-28T02:02:01.510 INFO:teuthology.orchestra.run.smithi143.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-08-28T02:02:01.510 INFO:teuthology.orchestra.run.smithi143.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-08-28T02:02:01.510 INFO:teuthology.orchestra.run.smithi143.stdout: python3-mako-1.0.6-13.el8.noarch 2023-08-28T02:02:01.510 INFO:teuthology.orchestra.run.smithi143.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-08-28T02:02:01.510 INFO:teuthology.orchestra.run.smithi143.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-08-28T02:02:01.510 INFO:teuthology.orchestra.run.smithi143.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-08-28T02:02:01.510 INFO:teuthology.orchestra.run.smithi143.stdout: python3-portend-2.6-1.el8.noarch 2023-08-28T02:02:01.511 INFO:teuthology.orchestra.run.smithi143.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-08-28T02:02:01.511 INFO:teuthology.orchestra.run.smithi143.stdout: python3-pycparser-2.14-14.el8.noarch 2023-08-28T02:02:01.511 INFO:teuthology.orchestra.run.smithi143.stdout: python3-pytz-2017.2-9.el8.noarch 2023-08-28T02:02:01.511 INFO:teuthology.orchestra.run.smithi143.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-08-28T02:02:01.511 INFO:teuthology.orchestra.run.smithi143.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-08-28T02:02:01.511 INFO:teuthology.orchestra.run.smithi143.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-08-28T02:02:01.511 INFO:teuthology.orchestra.run.smithi143.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-08-28T02:02:01.511 INFO:teuthology.orchestra.run.smithi143.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-08-28T02:02:01.511 INFO:teuthology.orchestra.run.smithi143.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-08-28T02:02:01.511 INFO:teuthology.orchestra.run.smithi143.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-08-28T02:02:01.511 INFO:teuthology.orchestra.run.smithi143.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-08-28T02:02:01.511 INFO:teuthology.orchestra.run.smithi143.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-08-28T02:02:01.512 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:02:01.512 INFO:teuthology.orchestra.run.smithi143.stdout:Complete! 2023-08-28T02:02:01.668 DEBUG:teuthology.orchestra.run.smithi143:> sudo yum -y install ceph-base 2023-08-28T02:02:02.017 INFO:teuthology.orchestra.run.smithi143.stdout:Updating Subscription Management repositories. 2023-08-28T02:02:02.059 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:01:33 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:02:02.802 INFO:teuthology.orchestra.run.smithi143.stdout:Last metadata expiration check: 0:01:33 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:02:03.418 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-08-28T02:02:03.418 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-08-28T02:02:03.418 INFO:teuthology.orchestra.run.smithi169.stdout: Package Arch Version Repository Size 2023-08-28T02:02:03.418 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-08-28T02:02:03.418 INFO:teuthology.orchestra.run.smithi169.stdout:Installing: 2023-08-28T02:02:03.418 INFO:teuthology.orchestra.run.smithi169.stdout: cephadm noarch 2:16.2.13-565.g595355f4.el8 ceph-noarch 84 k 2023-08-28T02:02:03.418 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:02:03.418 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction Summary 2023-08-28T02:02:03.418 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-08-28T02:02:03.419 INFO:teuthology.orchestra.run.smithi169.stdout:Install 1 Package 2023-08-28T02:02:03.419 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:02:03.419 INFO:teuthology.orchestra.run.smithi169.stdout:Total download size: 84 k 2023-08-28T02:02:03.419 INFO:teuthology.orchestra.run.smithi169.stdout:Installed size: 348 k 2023-08-28T02:02:03.419 INFO:teuthology.orchestra.run.smithi169.stdout:Downloading Packages: 2023-08-28T02:02:03.681 INFO:teuthology.orchestra.run.smithi169.stdout:cephadm-16.2.13-565.g595355f4.el8.noarch.rpm 319 kB/s | 84 kB 00:00 2023-08-28T02:02:03.681 INFO:teuthology.orchestra.run.smithi169.stdout:-------------------------------------------------------------------------------- 2023-08-28T02:02:03.681 INFO:teuthology.orchestra.run.smithi169.stdout:Total 317 kB/s | 84 kB 00:00 2023-08-28T02:02:03.682 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction check 2023-08-28T02:02:03.689 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction check succeeded. 2023-08-28T02:02:03.689 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction test 2023-08-28T02:02:03.690 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction test succeeded. 2023-08-28T02:02:03.694 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction 2023-08-28T02:02:03.735 INFO:teuthology.orchestra.run.smithi169.stdout: Preparing : 1/1 2023-08-28T02:02:04.045 INFO:teuthology.orchestra.run.smithi143.stdout:Package ceph-base-2:16.2.13-565.g595355f4.el8.x86_64 is already installed. 2023-08-28T02:02:04.153 INFO:teuthology.orchestra.run.smithi143.stdout:Dependencies resolved. 2023-08-28T02:02:04.154 INFO:teuthology.orchestra.run.smithi143.stdout:Nothing to do. 2023-08-28T02:02:04.154 INFO:teuthology.orchestra.run.smithi143.stdout:Complete! 2023-08-28T02:02:04.276 DEBUG:teuthology.orchestra.run.smithi143:> sudo yum -y install cephadm 2023-08-28T02:02:04.620 INFO:teuthology.orchestra.run.smithi143.stdout:Updating Subscription Management repositories. 2023-08-28T02:02:05.431 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: cephadm-2:16.2.13-565.g595355f4.el8.noarch 1/1 2023-08-28T02:02:05.561 INFO:teuthology.orchestra.run.smithi143.stdout:Last metadata expiration check: 0:01:36 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:02:05.618 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : cephadm-2:16.2.13-565.g595355f4.el8.noarch 1/1 2023-08-28T02:02:05.786 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: cephadm-2:16.2.13-565.g595355f4.el8.noarch 1/1 2023-08-28T02:02:06.875 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : cephadm-2:16.2.13-565.g595355f4.el8.noarch 1/1 2023-08-28T02:02:06.875 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:01:37 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:02:06.959 INFO:teuthology.orchestra.run.smithi143.stdout:Dependencies resolved. 2023-08-28T02:02:06.960 INFO:teuthology.orchestra.run.smithi143.stdout:================================================================================ 2023-08-28T02:02:06.960 INFO:teuthology.orchestra.run.smithi143.stdout: Package Arch Version Repository Size 2023-08-28T02:02:06.960 INFO:teuthology.orchestra.run.smithi143.stdout:================================================================================ 2023-08-28T02:02:06.960 INFO:teuthology.orchestra.run.smithi143.stdout:Installing: 2023-08-28T02:02:06.960 INFO:teuthology.orchestra.run.smithi143.stdout: cephadm noarch 2:16.2.13-565.g595355f4.el8 ceph-noarch 84 k 2023-08-28T02:02:06.960 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:02:06.960 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction Summary 2023-08-28T02:02:06.960 INFO:teuthology.orchestra.run.smithi143.stdout:================================================================================ 2023-08-28T02:02:06.960 INFO:teuthology.orchestra.run.smithi143.stdout:Install 1 Package 2023-08-28T02:02:06.960 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:02:06.961 INFO:teuthology.orchestra.run.smithi143.stdout:Total download size: 84 k 2023-08-28T02:02:06.961 INFO:teuthology.orchestra.run.smithi143.stdout:Installed size: 348 k 2023-08-28T02:02:06.961 INFO:teuthology.orchestra.run.smithi143.stdout:Downloading Packages: 2023-08-28T02:02:07.124 INFO:teuthology.orchestra.run.smithi143.stdout:cephadm-16.2.13-565.g595355f4.el8.noarch.rpm 517 kB/s | 84 kB 00:00 2023-08-28T02:02:07.124 INFO:teuthology.orchestra.run.smithi143.stdout:-------------------------------------------------------------------------------- 2023-08-28T02:02:07.124 INFO:teuthology.orchestra.run.smithi143.stdout:Total 513 kB/s | 84 kB 00:00 2023-08-28T02:02:07.125 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction check 2023-08-28T02:02:07.132 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction check succeeded. 2023-08-28T02:02:07.132 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction test 2023-08-28T02:02:07.133 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction test succeeded. 2023-08-28T02:02:07.137 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction 2023-08-28T02:02:07.180 INFO:teuthology.orchestra.run.smithi143.stdout: Preparing : 1/1 2023-08-28T02:02:08.851 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: cephadm-2:16.2.13-565.g595355f4.el8.noarch 1/1 2023-08-28T02:02:08.995 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : cephadm-2:16.2.13-565.g595355f4.el8.noarch 1/1 2023-08-28T02:02:09.163 INFO:teuthology.orchestra.run.smithi169.stdout:Installed products updated. 2023-08-28T02:02:09.163 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: cephadm-2:16.2.13-565.g595355f4.el8.noarch 1/1 2023-08-28T02:02:09.347 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:02:09.347 INFO:teuthology.orchestra.run.smithi169.stdout:Installed: 2023-08-28T02:02:09.348 INFO:teuthology.orchestra.run.smithi169.stdout: cephadm-2:16.2.13-565.g595355f4.el8.noarch 2023-08-28T02:02:09.348 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:02:09.348 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-08-28T02:02:09.502 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install ceph-immutable-object-cache 2023-08-28T02:02:09.862 INFO:teuthology.orchestra.run.smithi169.stdout:Updating Subscription Management repositories. 2023-08-28T02:02:10.268 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : cephadm-2:16.2.13-565.g595355f4.el8.noarch 1/1 2023-08-28T02:02:10.268 INFO:teuthology.orchestra.run.smithi143.stdout:Last metadata expiration check: 0:01:41 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:02:10.669 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:01:41 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:02:12.050 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-08-28T02:02:12.051 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-08-28T02:02:12.051 INFO:teuthology.orchestra.run.smithi169.stdout: Package Arch Version Repo Size 2023-08-28T02:02:12.051 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-08-28T02:02:12.051 INFO:teuthology.orchestra.run.smithi169.stdout:Installing: 2023-08-28T02:02:12.051 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-immutable-object-cache x86_64 2:16.2.13-565.g595355f4.el8 ceph 161 k 2023-08-28T02:02:12.051 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:02:12.052 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction Summary 2023-08-28T02:02:12.052 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-08-28T02:02:12.052 INFO:teuthology.orchestra.run.smithi169.stdout:Install 1 Package 2023-08-28T02:02:12.052 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:02:12.052 INFO:teuthology.orchestra.run.smithi169.stdout:Total download size: 161 k 2023-08-28T02:02:12.052 INFO:teuthology.orchestra.run.smithi169.stdout:Installed size: 470 k 2023-08-28T02:02:12.053 INFO:teuthology.orchestra.run.smithi169.stdout:Downloading Packages: 2023-08-28T02:02:12.245 INFO:teuthology.orchestra.run.smithi169.stdout:ceph-immutable-object-cache-16.2.13-565.g595355 838 kB/s | 161 kB 00:00 2023-08-28T02:02:12.245 INFO:teuthology.orchestra.run.smithi169.stdout:-------------------------------------------------------------------------------- 2023-08-28T02:02:12.245 INFO:teuthology.orchestra.run.smithi169.stdout:Total 833 kB/s | 161 kB 00:00 2023-08-28T02:02:12.246 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction check 2023-08-28T02:02:12.256 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction check succeeded. 2023-08-28T02:02:12.256 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction test 2023-08-28T02:02:12.307 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction test succeeded. 2023-08-28T02:02:12.311 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction 2023-08-28T02:02:12.442 INFO:teuthology.orchestra.run.smithi169.stdout: Preparing : 1/1 2023-08-28T02:02:12.620 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : ceph-immutable-object-cache-2:16.2.13-565.g595355f4. 1/1 2023-08-28T02:02:12.635 INFO:teuthology.orchestra.run.smithi143.stdout:Installed products updated. 2023-08-28T02:02:12.820 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:02:12.820 INFO:teuthology.orchestra.run.smithi143.stdout:Installed: 2023-08-28T02:02:12.820 INFO:teuthology.orchestra.run.smithi143.stdout: cephadm-2:16.2.13-565.g595355f4.el8.noarch 2023-08-28T02:02:12.820 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:02:12.820 INFO:teuthology.orchestra.run.smithi143.stdout:Complete! 2023-08-28T02:02:12.955 DEBUG:teuthology.orchestra.run.smithi143:> sudo yum -y install ceph-immutable-object-cache 2023-08-28T02:02:12.973 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: ceph-immutable-object-cache-2:16.2.13-565.g595355f4. 1/1 2023-08-28T02:02:13.314 INFO:teuthology.orchestra.run.smithi143.stdout:Updating Subscription Management repositories. 2023-08-28T02:02:14.048 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : ceph-immutable-object-cache-2:16.2.13-565.g595355f4. 1/1 2023-08-28T02:02:14.048 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:01:45 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:02:14.133 INFO:teuthology.orchestra.run.smithi143.stdout:Last metadata expiration check: 0:01:45 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:02:15.490 INFO:teuthology.orchestra.run.smithi143.stdout:Dependencies resolved. 2023-08-28T02:02:15.490 INFO:teuthology.orchestra.run.smithi143.stdout:================================================================================ 2023-08-28T02:02:15.491 INFO:teuthology.orchestra.run.smithi143.stdout: Package Arch Version Repo Size 2023-08-28T02:02:15.491 INFO:teuthology.orchestra.run.smithi143.stdout:================================================================================ 2023-08-28T02:02:15.491 INFO:teuthology.orchestra.run.smithi143.stdout:Installing: 2023-08-28T02:02:15.491 INFO:teuthology.orchestra.run.smithi143.stdout: ceph-immutable-object-cache x86_64 2:16.2.13-565.g595355f4.el8 ceph 161 k 2023-08-28T02:02:15.491 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:02:15.491 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction Summary 2023-08-28T02:02:15.491 INFO:teuthology.orchestra.run.smithi143.stdout:================================================================================ 2023-08-28T02:02:15.491 INFO:teuthology.orchestra.run.smithi143.stdout:Install 1 Package 2023-08-28T02:02:15.491 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:02:15.492 INFO:teuthology.orchestra.run.smithi143.stdout:Total download size: 161 k 2023-08-28T02:02:15.492 INFO:teuthology.orchestra.run.smithi143.stdout:Installed size: 470 k 2023-08-28T02:02:15.492 INFO:teuthology.orchestra.run.smithi143.stdout:Downloading Packages: 2023-08-28T02:02:15.772 INFO:teuthology.orchestra.run.smithi143.stdout:ceph-immutable-object-cache-16.2.13-565.g595355 575 kB/s | 161 kB 00:00 2023-08-28T02:02:15.773 INFO:teuthology.orchestra.run.smithi143.stdout:-------------------------------------------------------------------------------- 2023-08-28T02:02:15.773 INFO:teuthology.orchestra.run.smithi143.stdout:Total 573 kB/s | 161 kB 00:00 2023-08-28T02:02:15.773 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction check 2023-08-28T02:02:15.784 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction check succeeded. 2023-08-28T02:02:15.784 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction test 2023-08-28T02:02:15.835 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction test succeeded. 2023-08-28T02:02:15.839 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction 2023-08-28T02:02:15.946 INFO:teuthology.orchestra.run.smithi143.stdout: Preparing : 1/1 2023-08-28T02:02:16.112 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : ceph-immutable-object-cache-2:16.2.13-565.g595355f4. 1/1 2023-08-28T02:02:16.347 INFO:teuthology.orchestra.run.smithi169.stdout:Installed products updated. 2023-08-28T02:02:16.468 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: ceph-immutable-object-cache-2:16.2.13-565.g595355f4. 1/1 2023-08-28T02:02:16.531 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:02:16.532 INFO:teuthology.orchestra.run.smithi169.stdout:Installed: 2023-08-28T02:02:16.532 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-immutable-object-cache-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:02:16.532 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:02:16.532 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-08-28T02:02:16.693 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install ceph-mgr 2023-08-28T02:02:17.040 INFO:teuthology.orchestra.run.smithi169.stdout:Updating Subscription Management repositories. 2023-08-28T02:02:17.507 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : ceph-immutable-object-cache-2:16.2.13-565.g595355f4. 1/1 2023-08-28T02:02:17.507 INFO:teuthology.orchestra.run.smithi143.stdout:Last metadata expiration check: 0:01:48 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:02:17.942 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:01:48 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:02:19.202 INFO:teuthology.orchestra.run.smithi169.stdout:Package ceph-mgr-2:16.2.13-565.g595355f4.el8.x86_64 is already installed. 2023-08-28T02:02:19.311 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-08-28T02:02:19.312 INFO:teuthology.orchestra.run.smithi169.stdout:Nothing to do. 2023-08-28T02:02:19.312 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-08-28T02:02:19.423 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install ceph-mgr-dashboard 2023-08-28T02:02:19.769 INFO:teuthology.orchestra.run.smithi169.stdout:Updating Subscription Management repositories. 2023-08-28T02:02:19.862 INFO:teuthology.orchestra.run.smithi143.stdout:Installed products updated. 2023-08-28T02:02:20.048 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:02:20.048 INFO:teuthology.orchestra.run.smithi143.stdout:Installed: 2023-08-28T02:02:20.048 INFO:teuthology.orchestra.run.smithi143.stdout: ceph-immutable-object-cache-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:02:20.049 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:02:20.049 INFO:teuthology.orchestra.run.smithi143.stdout:Complete! 2023-08-28T02:02:20.187 DEBUG:teuthology.orchestra.run.smithi143:> sudo yum -y install ceph-mgr 2023-08-28T02:02:20.533 INFO:teuthology.orchestra.run.smithi143.stdout:Updating Subscription Management repositories. 2023-08-28T02:02:20.687 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:01:51 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:02:21.334 INFO:teuthology.orchestra.run.smithi143.stdout:Last metadata expiration check: 0:01:52 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:02:22.119 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-08-28T02:02:22.120 INFO:teuthology.orchestra.run.smithi169.stdout:=================================================================================================== 2023-08-28T02:02:22.120 INFO:teuthology.orchestra.run.smithi169.stdout: Package Arch Version Repository Size 2023-08-28T02:02:22.120 INFO:teuthology.orchestra.run.smithi169.stdout:=================================================================================================== 2023-08-28T02:02:22.120 INFO:teuthology.orchestra.run.smithi169.stdout:Installing: 2023-08-28T02:02:22.120 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-mgr-dashboard noarch 2:16.2.13-565.g595355f4.el8 ceph-noarch 3.1 M 2023-08-28T02:02:22.120 INFO:teuthology.orchestra.run.smithi169.stdout:Installing dependencies: 2023-08-28T02:02:22.120 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-grafana-dashboards noarch 2:16.2.13-565.g595355f4.el8 ceph-noarch 24 k 2023-08-28T02:02:22.120 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-prometheus-alerts noarch 2:16.2.13-565.g595355f4.el8 ceph-noarch 15 k 2023-08-28T02:02:22.121 INFO:teuthology.orchestra.run.smithi169.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-08-28T02:02:22.121 INFO:teuthology.orchestra.run.smithi169.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-08-28T02:02:22.121 INFO:teuthology.orchestra.run.smithi169.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-08-28T02:02:22.121 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:02:22.121 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction Summary 2023-08-28T02:02:22.121 INFO:teuthology.orchestra.run.smithi169.stdout:=================================================================================================== 2023-08-28T02:02:22.121 INFO:teuthology.orchestra.run.smithi169.stdout:Install 6 Packages 2023-08-28T02:02:22.121 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:02:22.122 INFO:teuthology.orchestra.run.smithi169.stdout:Total download size: 3.4 M 2023-08-28T02:02:22.122 INFO:teuthology.orchestra.run.smithi169.stdout:Installed size: 22 M 2023-08-28T02:02:22.122 INFO:teuthology.orchestra.run.smithi169.stdout:Downloading Packages: 2023-08-28T02:02:22.654 INFO:teuthology.orchestra.run.smithi143.stdout:Package ceph-mgr-2:16.2.13-565.g595355f4.el8.x86_64 is already installed. 2023-08-28T02:02:22.704 INFO:teuthology.orchestra.run.smithi169.stdout:(1/6): ceph-grafana-dashboards-16.2.13-565.g595 55 kB/s | 24 kB 00:00 2023-08-28T02:02:22.764 INFO:teuthology.orchestra.run.smithi143.stdout:Dependencies resolved. 2023-08-28T02:02:22.765 INFO:teuthology.orchestra.run.smithi143.stdout:Nothing to do. 2023-08-28T02:02:22.765 INFO:teuthology.orchestra.run.smithi143.stdout:Complete! 2023-08-28T02:02:22.875 INFO:teuthology.orchestra.run.smithi169.stdout:(2/6): ceph-prometheus-alerts-16.2.13-565.g5953 24 kB/s | 15 kB 00:00 2023-08-28T02:02:22.917 DEBUG:teuthology.orchestra.run.smithi143:> sudo yum -y install ceph-mgr-dashboard 2023-08-28T02:02:23.235 INFO:teuthology.orchestra.run.smithi169.stdout:(3/6): ceph-mgr-dashboard-16.2.13-565.g595355f4 3.2 MB/s | 3.1 MB 00:00 2023-08-28T02:02:23.261 INFO:teuthology.orchestra.run.smithi143.stdout:Updating Subscription Management repositories. 2023-08-28T02:02:23.569 INFO:teuthology.orchestra.run.smithi169.stdout:(4/6): python3-jwt-1.6.1-2.el8.noarch.rpm 131 kB/s | 43 kB 00:00 2023-08-28T02:02:23.653 INFO:teuthology.orchestra.run.smithi169.stdout:(5/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 35 kB/s | 34 kB 00:00 2023-08-28T02:02:23.678 INFO:teuthology.orchestra.run.smithi169.stdout:(6/6): python3-routes-2.4.1-12.el8.noarch.rpm 245 kB/s | 196 kB 00:00 2023-08-28T02:02:23.681 INFO:teuthology.orchestra.run.smithi169.stdout:-------------------------------------------------------------------------------- 2023-08-28T02:02:23.681 INFO:teuthology.orchestra.run.smithi169.stdout:Total 2.2 MB/s | 3.4 MB 00:01 2023-08-28T02:02:23.694 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction check 2023-08-28T02:02:23.706 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction check succeeded. 2023-08-28T02:02:23.706 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction test 2023-08-28T02:02:23.779 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction test succeeded. 2023-08-28T02:02:23.783 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction 2023-08-28T02:02:24.069 INFO:teuthology.orchestra.run.smithi143.stdout:Last metadata expiration check: 0:01:55 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:02:24.098 INFO:teuthology.orchestra.run.smithi169.stdout: Preparing : 1/1 2023-08-28T02:02:24.303 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 1/6 2023-08-28T02:02:24.499 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 2/6 2023-08-28T02:02:24.737 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 3/6 2023-08-28T02:02:24.867 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : ceph-prometheus-alerts-2:16.2.13-565.g595355f4.el8.n 4/6 2023-08-28T02:02:25.501 INFO:teuthology.orchestra.run.smithi143.stdout:Dependencies resolved. 2023-08-28T02:02:25.502 INFO:teuthology.orchestra.run.smithi143.stdout:=================================================================================================== 2023-08-28T02:02:25.502 INFO:teuthology.orchestra.run.smithi143.stdout: Package Arch Version Repository Size 2023-08-28T02:02:25.502 INFO:teuthology.orchestra.run.smithi143.stdout:=================================================================================================== 2023-08-28T02:02:25.502 INFO:teuthology.orchestra.run.smithi143.stdout:Installing: 2023-08-28T02:02:25.502 INFO:teuthology.orchestra.run.smithi143.stdout: ceph-mgr-dashboard noarch 2:16.2.13-565.g595355f4.el8 ceph-noarch 3.1 M 2023-08-28T02:02:25.502 INFO:teuthology.orchestra.run.smithi143.stdout:Installing dependencies: 2023-08-28T02:02:25.502 INFO:teuthology.orchestra.run.smithi143.stdout: ceph-grafana-dashboards noarch 2:16.2.13-565.g595355f4.el8 ceph-noarch 24 k 2023-08-28T02:02:25.502 INFO:teuthology.orchestra.run.smithi143.stdout: ceph-prometheus-alerts noarch 2:16.2.13-565.g595355f4.el8 ceph-noarch 15 k 2023-08-28T02:02:25.502 INFO:teuthology.orchestra.run.smithi143.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-08-28T02:02:25.502 INFO:teuthology.orchestra.run.smithi143.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-08-28T02:02:25.503 INFO:teuthology.orchestra.run.smithi143.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-08-28T02:02:25.503 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:02:25.503 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction Summary 2023-08-28T02:02:25.503 INFO:teuthology.orchestra.run.smithi143.stdout:=================================================================================================== 2023-08-28T02:02:25.503 INFO:teuthology.orchestra.run.smithi143.stdout:Install 6 Packages 2023-08-28T02:02:25.503 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:02:25.503 INFO:teuthology.orchestra.run.smithi143.stdout:Total download size: 3.4 M 2023-08-28T02:02:25.504 INFO:teuthology.orchestra.run.smithi143.stdout:Installed size: 22 M 2023-08-28T02:02:25.504 INFO:teuthology.orchestra.run.smithi143.stdout:Downloading Packages: 2023-08-28T02:02:25.829 INFO:teuthology.orchestra.run.smithi143.stdout:(1/6): ceph-prometheus-alerts-16.2.13-565.g5953 100 kB/s | 15 kB 00:00 2023-08-28T02:02:25.878 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : ceph-grafana-dashboards-2:16.2.13-565.g595355f4.el8. 5/6 2023-08-28T02:02:25.912 INFO:teuthology.orchestra.run.smithi143.stdout:(2/6): ceph-grafana-dashboards-16.2.13-565.g595 105 kB/s | 24 kB 00:00 2023-08-28T02:02:26.021 INFO:teuthology.orchestra.run.smithi143.stdout:(3/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 175 kB/s | 34 kB 00:00 2023-08-28T02:02:26.080 INFO:teuthology.orchestra.run.smithi143.stdout:(4/6): python3-jwt-1.6.1-2.el8.noarch.rpm 731 kB/s | 43 kB 00:00 2023-08-28T02:02:26.147 INFO:teuthology.orchestra.run.smithi143.stdout:(5/6): python3-routes-2.4.1-12.el8.noarch.rpm 840 kB/s | 196 kB 00:00 2023-08-28T02:02:26.163 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : ceph-mgr-dashboard-2:16.2.13-565.g595355f4.el8.noarc 6/6 2023-08-28T02:02:26.247 INFO:teuthology.orchestra.run.smithi143.stdout:(6/6): ceph-mgr-dashboard-16.2.13-565.g595355f4 5.5 MB/s | 3.1 MB 00:00 2023-08-28T02:02:26.249 INFO:teuthology.orchestra.run.smithi143.stdout:-------------------------------------------------------------------------------- 2023-08-28T02:02:26.249 INFO:teuthology.orchestra.run.smithi143.stdout:Total 4.6 MB/s | 3.4 MB 00:00 2023-08-28T02:02:26.261 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction check 2023-08-28T02:02:26.271 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction check succeeded. 2023-08-28T02:02:26.271 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction test 2023-08-28T02:02:26.339 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction test succeeded. 2023-08-28T02:02:26.343 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction 2023-08-28T02:02:26.403 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: ceph-mgr-dashboard-2:16.2.13-565.g595355f4.el8.noarc 6/6 2023-08-28T02:02:26.403 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : ceph-grafana-dashboards-2:16.2.13-565.g595355f4.el8. 1/6 2023-08-28T02:02:26.403 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : ceph-mgr-dashboard-2:16.2.13-565.g595355f4.el8.noarc 2/6 2023-08-28T02:02:26.403 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : ceph-prometheus-alerts-2:16.2.13-565.g595355f4.el8.n 3/6 2023-08-28T02:02:26.403 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 4/6 2023-08-28T02:02:26.403 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 5/6 2023-08-28T02:02:26.660 INFO:teuthology.orchestra.run.smithi143.stdout: Preparing : 1/1 2023-08-28T02:02:26.828 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 1/6 2023-08-28T02:02:27.008 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 2/6 2023-08-28T02:02:27.171 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 3/6 2023-08-28T02:02:27.325 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : ceph-prometheus-alerts-2:16.2.13-565.g595355f4.el8.n 4/6 2023-08-28T02:02:27.467 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 6/6 2023-08-28T02:02:27.467 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:01:58 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:02:28.358 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : ceph-grafana-dashboards-2:16.2.13-565.g595355f4.el8. 5/6 2023-08-28T02:02:29.010 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : ceph-mgr-dashboard-2:16.2.13-565.g595355f4.el8.noarc 6/6 2023-08-28T02:02:29.254 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: ceph-mgr-dashboard-2:16.2.13-565.g595355f4.el8.noarc 6/6 2023-08-28T02:02:29.255 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : ceph-grafana-dashboards-2:16.2.13-565.g595355f4.el8. 1/6 2023-08-28T02:02:29.255 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : ceph-mgr-dashboard-2:16.2.13-565.g595355f4.el8.noarc 2/6 2023-08-28T02:02:29.255 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : ceph-prometheus-alerts-2:16.2.13-565.g595355f4.el8.n 3/6 2023-08-28T02:02:29.255 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 4/6 2023-08-28T02:02:29.255 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 5/6 2023-08-28T02:02:29.855 INFO:teuthology.orchestra.run.smithi169.stdout:Installed products updated. 2023-08-28T02:02:30.042 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:02:30.042 INFO:teuthology.orchestra.run.smithi169.stdout:Installed: 2023-08-28T02:02:30.043 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-grafana-dashboards-2:16.2.13-565.g595355f4.el8.noarch 2023-08-28T02:02:30.043 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-mgr-dashboard-2:16.2.13-565.g595355f4.el8.noarch 2023-08-28T02:02:30.043 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-prometheus-alerts-2:16.2.13-565.g595355f4.el8.noarch 2023-08-28T02:02:30.043 INFO:teuthology.orchestra.run.smithi169.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-08-28T02:02:30.043 INFO:teuthology.orchestra.run.smithi169.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-08-28T02:02:30.043 INFO:teuthology.orchestra.run.smithi169.stdout: python3-routes-2.4.1-12.el8.noarch 2023-08-28T02:02:30.043 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:02:30.043 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-08-28T02:02:30.225 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install ceph-mgr-diskprediction-local 2023-08-28T02:02:30.288 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 6/6 2023-08-28T02:02:30.288 INFO:teuthology.orchestra.run.smithi143.stdout:Last metadata expiration check: 0:02:01 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:02:30.576 INFO:teuthology.orchestra.run.smithi169.stdout:Updating Subscription Management repositories. 2023-08-28T02:02:31.454 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:02:02 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:02:32.642 INFO:teuthology.orchestra.run.smithi143.stdout:Installed products updated. 2023-08-28T02:02:32.806 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-08-28T02:02:32.807 INFO:teuthology.orchestra.run.smithi169.stdout:======================================================================================================================= 2023-08-28T02:02:32.807 INFO:teuthology.orchestra.run.smithi169.stdout: Package Arch Version Repository Size 2023-08-28T02:02:32.807 INFO:teuthology.orchestra.run.smithi169.stdout:======================================================================================================================= 2023-08-28T02:02:32.807 INFO:teuthology.orchestra.run.smithi169.stdout:Installing: 2023-08-28T02:02:32.807 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-mgr-diskprediction-local noarch 2:16.2.13-565.g595355f4.el8 ceph-noarch 8.6 M 2023-08-28T02:02:32.808 INFO:teuthology.orchestra.run.smithi169.stdout:Installing dependencies: 2023-08-28T02:02:32.808 INFO:teuthology.orchestra.run.smithi169.stdout: libgfortran x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 644 k 2023-08-28T02:02:32.808 INFO:teuthology.orchestra.run.smithi169.stdout: libquadmath x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 171 k 2023-08-28T02:02:32.808 INFO:teuthology.orchestra.run.smithi169.stdout: openblas x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.7 M 2023-08-28T02:02:32.808 INFO:teuthology.orchestra.run.smithi169.stdout: openblas-threads x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.9 M 2023-08-28T02:02:32.808 INFO:teuthology.orchestra.run.smithi169.stdout: python3-numpy x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2023-08-28T02:02:32.808 INFO:teuthology.orchestra.run.smithi169.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2023-08-28T02:02:32.808 INFO:teuthology.orchestra.run.smithi169.stdout: python3-scipy x86_64 1.0.0-21.module+el8.5.0+10916+41bd434d rhel-8-for-x86_64-appstream-rpms 14 M 2023-08-28T02:02:32.808 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:02:32.808 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction Summary 2023-08-28T02:02:32.808 INFO:teuthology.orchestra.run.smithi169.stdout:======================================================================================================================= 2023-08-28T02:02:32.809 INFO:teuthology.orchestra.run.smithi169.stdout:Install 8 Packages 2023-08-28T02:02:32.809 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:02:32.809 INFO:teuthology.orchestra.run.smithi169.stdout:Total download size: 37 M 2023-08-28T02:02:32.809 INFO:teuthology.orchestra.run.smithi169.stdout:Installed size: 210 M 2023-08-28T02:02:32.809 INFO:teuthology.orchestra.run.smithi169.stdout:Downloading Packages: 2023-08-28T02:02:32.823 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:02:32.823 INFO:teuthology.orchestra.run.smithi143.stdout:Installed: 2023-08-28T02:02:32.823 INFO:teuthology.orchestra.run.smithi143.stdout: ceph-grafana-dashboards-2:16.2.13-565.g595355f4.el8.noarch 2023-08-28T02:02:32.823 INFO:teuthology.orchestra.run.smithi143.stdout: ceph-mgr-dashboard-2:16.2.13-565.g595355f4.el8.noarch 2023-08-28T02:02:32.823 INFO:teuthology.orchestra.run.smithi143.stdout: ceph-prometheus-alerts-2:16.2.13-565.g595355f4.el8.noarch 2023-08-28T02:02:32.823 INFO:teuthology.orchestra.run.smithi143.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-08-28T02:02:32.823 INFO:teuthology.orchestra.run.smithi143.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-08-28T02:02:32.823 INFO:teuthology.orchestra.run.smithi143.stdout: python3-routes-2.4.1-12.el8.noarch 2023-08-28T02:02:32.823 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:02:32.824 INFO:teuthology.orchestra.run.smithi143.stdout:Complete! 2023-08-28T02:02:32.948 INFO:teuthology.orchestra.run.smithi169.stdout:(1/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 27 MB/s | 3.7 MB 00:00 2023-08-28T02:02:32.964 DEBUG:teuthology.orchestra.run.smithi143:> sudo yum -y install ceph-mgr-diskprediction-local 2023-08-28T02:02:33.024 INFO:teuthology.orchestra.run.smithi169.stdout:(2/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 2.9 MB/s | 225 kB 00:00 2023-08-28T02:02:33.182 INFO:teuthology.orchestra.run.smithi169.stdout:(3/8): openblas-threads-0.3.15-3.el8.x86_64.rpm 31 MB/s | 4.9 MB 00:00 2023-08-28T02:02:33.310 INFO:teuthology.orchestra.run.smithi143.stdout:Updating Subscription Management repositories. 2023-08-28T02:02:33.332 INFO:teuthology.orchestra.run.smithi169.stdout:(4/8): python3-scipy-1.0.0-21.module+el8.5.0+10 26 MB/s | 14 MB 00:00 2023-08-28T02:02:33.466 INFO:teuthology.orchestra.run.smithi169.stdout:(5/8): ceph-mgr-diskprediction-local-16.2.13-56 13 MB/s | 8.6 MB 00:00 2023-08-28T02:02:33.500 INFO:teuthology.orchestra.run.smithi169.stdout:(6/8): libgfortran-8.5.0-10.1.el8_6.x86_64.rpm 3.8 MB/s | 644 kB 00:00 2023-08-28T02:02:33.600 INFO:teuthology.orchestra.run.smithi169.stdout:(7/8): openblas-0.3.15-3.el8.x86_64.rpm 11 MB/s | 4.7 MB 00:00 2023-08-28T02:02:33.625 INFO:teuthology.orchestra.run.smithi169.stdout:(8/8): libquadmath-8.5.0-10.1.el8_6.x86_64.rpm 1.1 MB/s | 171 kB 00:00 2023-08-28T02:02:33.626 INFO:teuthology.orchestra.run.smithi169.stdout:-------------------------------------------------------------------------------- 2023-08-28T02:02:33.626 INFO:teuthology.orchestra.run.smithi169.stdout:Total 45 MB/s | 37 MB 00:00 2023-08-28T02:02:33.863 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction check 2023-08-28T02:02:33.881 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction check succeeded. 2023-08-28T02:02:33.881 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction test 2023-08-28T02:02:34.103 INFO:teuthology.orchestra.run.smithi143.stdout:Last metadata expiration check: 0:02:05 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:02:34.179 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction test succeeded. 2023-08-28T02:02:34.182 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction 2023-08-28T02:02:34.897 INFO:teuthology.orchestra.run.smithi169.stdout: Preparing : 1/1 2023-08-28T02:02:35.041 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-08-28T02:02:35.175 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-08-28T02:02:35.358 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-08-28T02:02:35.478 INFO:teuthology.orchestra.run.smithi143.stdout:Dependencies resolved. 2023-08-28T02:02:35.479 INFO:teuthology.orchestra.run.smithi143.stdout:======================================================================================================================= 2023-08-28T02:02:35.479 INFO:teuthology.orchestra.run.smithi143.stdout: Package Arch Version Repository Size 2023-08-28T02:02:35.479 INFO:teuthology.orchestra.run.smithi143.stdout:======================================================================================================================= 2023-08-28T02:02:35.479 INFO:teuthology.orchestra.run.smithi143.stdout:Installing: 2023-08-28T02:02:35.479 INFO:teuthology.orchestra.run.smithi143.stdout: ceph-mgr-diskprediction-local noarch 2:16.2.13-565.g595355f4.el8 ceph-noarch 8.6 M 2023-08-28T02:02:35.479 INFO:teuthology.orchestra.run.smithi143.stdout:Installing dependencies: 2023-08-28T02:02:35.479 INFO:teuthology.orchestra.run.smithi143.stdout: libgfortran x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 644 k 2023-08-28T02:02:35.480 INFO:teuthology.orchestra.run.smithi143.stdout: libquadmath x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 171 k 2023-08-28T02:02:35.480 INFO:teuthology.orchestra.run.smithi143.stdout: openblas x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.7 M 2023-08-28T02:02:35.480 INFO:teuthology.orchestra.run.smithi143.stdout: openblas-threads x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.9 M 2023-08-28T02:02:35.480 INFO:teuthology.orchestra.run.smithi143.stdout: python3-numpy x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2023-08-28T02:02:35.480 INFO:teuthology.orchestra.run.smithi143.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2023-08-28T02:02:35.480 INFO:teuthology.orchestra.run.smithi143.stdout: python3-scipy x86_64 1.0.0-21.module+el8.5.0+10916+41bd434d rhel-8-for-x86_64-appstream-rpms 14 M 2023-08-28T02:02:35.480 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:02:35.480 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction Summary 2023-08-28T02:02:35.480 INFO:teuthology.orchestra.run.smithi143.stdout:======================================================================================================================= 2023-08-28T02:02:35.480 INFO:teuthology.orchestra.run.smithi143.stdout:Install 8 Packages 2023-08-28T02:02:35.480 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:02:35.481 INFO:teuthology.orchestra.run.smithi143.stdout:Total download size: 37 M 2023-08-28T02:02:35.481 INFO:teuthology.orchestra.run.smithi143.stdout:Installed size: 210 M 2023-08-28T02:02:35.481 INFO:teuthology.orchestra.run.smithi143.stdout:Downloading Packages: 2023-08-28T02:02:35.545 INFO:teuthology.orchestra.run.smithi143.stdout:(1/8): libgfortran-8.5.0-10.1.el8_6.x86_64.rpm 10 MB/s | 644 kB 00:00 2023-08-28T02:02:35.570 INFO:teuthology.orchestra.run.smithi143.stdout:(2/8): libquadmath-8.5.0-10.1.el8_6.x86_64.rpm 1.9 MB/s | 171 kB 00:00 2023-08-28T02:02:35.687 INFO:teuthology.orchestra.run.smithi143.stdout:(3/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 32 MB/s | 3.7 MB 00:00 2023-08-28T02:02:35.729 INFO:teuthology.orchestra.run.smithi143.stdout:(4/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 5.3 MB/s | 225 kB 00:00 2023-08-28T02:02:35.963 INFO:teuthology.orchestra.run.smithi143.stdout:(5/8): python3-scipy-1.0.0-21.module+el8.5.0+10 33 MB/s | 14 MB 00:00 2023-08-28T02:02:35.974 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-08-28T02:02:36.030 INFO:teuthology.orchestra.run.smithi143.stdout:(6/8): openblas-threads-0.3.15-3.el8.x86_64.rpm 16 MB/s | 4.9 MB 00:00 2023-08-28T02:02:36.138 INFO:teuthology.orchestra.run.smithi143.stdout:(7/8): openblas-0.3.15-3.el8.x86_64.rpm 27 MB/s | 4.7 MB 00:00 2023-08-28T02:02:36.183 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-08-28T02:02:36.238 INFO:teuthology.orchestra.run.smithi143.stdout:(8/8): ceph-mgr-diskprediction-local-16.2.13-56 11 MB/s | 8.6 MB 00:00 2023-08-28T02:02:36.239 INFO:teuthology.orchestra.run.smithi143.stdout:-------------------------------------------------------------------------------- 2023-08-28T02:02:36.240 INFO:teuthology.orchestra.run.smithi143.stdout:Total 48 MB/s | 37 MB 00:00 2023-08-28T02:02:36.478 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction check 2023-08-28T02:02:36.496 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction check succeeded. 2023-08-28T02:02:36.496 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction test 2023-08-28T02:02:36.790 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-08-28T02:02:36.793 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction test succeeded. 2023-08-28T02:02:36.797 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction 2023-08-28T02:02:36.999 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : openblas-0.3.15-3.el8.x86_64 4/8 2023-08-28T02:02:37.329 INFO:teuthology.orchestra.run.smithi143.stdout: Preparing : 1/1 2023-08-28T02:02:37.500 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-08-28T02:02:37.631 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-08-28T02:02:37.671 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: openblas-0.3.15-3.el8.x86_64 4/8 2023-08-28T02:02:37.775 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-08-28T02:02:38.002 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-08-28T02:02:38.241 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-08-28T02:02:38.388 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-08-28T02:02:38.559 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-08-28T02:02:39.154 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-08-28T02:02:39.708 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : openblas-0.3.15-3.el8.x86_64 4/8 2023-08-28T02:02:40.014 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-08-28T02:02:40.426 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: openblas-0.3.15-3.el8.x86_64 4/8 2023-08-28T02:02:40.744 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-08-28T02:02:40.933 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-08-28T02:02:41.279 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 7/8 2023-08-28T02:02:41.448 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : ceph-mgr-diskprediction-local-2:16.2.13-565.g595355f 8/8 2023-08-28T02:02:41.720 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:16.2.13-565.g595355f 8/8 2023-08-28T02:02:41.720 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : ceph-mgr-diskprediction-local-2:16.2.13-565.g595355f 1/8 2023-08-28T02:02:41.720 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 2/8 2023-08-28T02:02:41.721 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 3/8 2023-08-28T02:02:41.721 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 4/8 2023-08-28T02:02:41.721 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : openblas-threads-0.3.15-3.el8.x86_64 5/8 2023-08-28T02:02:41.721 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : openblas-0.3.15-3.el8.x86_64 6/8 2023-08-28T02:02:41.721 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : libgfortran-8.5.0-10.1.el8_6.x86_64 7/8 2023-08-28T02:02:42.705 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-08-28T02:02:42.819 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : libquadmath-8.5.0-10.1.el8_6.x86_64 8/8 2023-08-28T02:02:42.819 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:02:13 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:02:44.030 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 7/8 2023-08-28T02:02:44.199 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : ceph-mgr-diskprediction-local-2:16.2.13-565.g595355f 8/8 2023-08-28T02:02:44.460 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:16.2.13-565.g595355f 8/8 2023-08-28T02:02:44.461 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : ceph-mgr-diskprediction-local-2:16.2.13-565.g595355f 1/8 2023-08-28T02:02:44.461 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-08-28T02:02:44.461 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : libquadmath-8.5.0-10.1.el8_6.x86_64 3/8 2023-08-28T02:02:44.461 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 4/8 2023-08-28T02:02:44.461 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-08-28T02:02:44.461 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-08-28T02:02:44.461 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : openblas-threads-0.3.15-3.el8.x86_64 7/8 2023-08-28T02:02:45.117 INFO:teuthology.orchestra.run.smithi169.stdout:Installed products updated. 2023-08-28T02:02:45.303 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:02:45.303 INFO:teuthology.orchestra.run.smithi169.stdout:Installed: 2023-08-28T02:02:45.303 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-mgr-diskprediction-local-2:16.2.13-565.g595355f4.el8.noarch 2023-08-28T02:02:45.303 INFO:teuthology.orchestra.run.smithi169.stdout: libgfortran-8.5.0-10.1.el8_6.x86_64 2023-08-28T02:02:45.304 INFO:teuthology.orchestra.run.smithi169.stdout: libquadmath-8.5.0-10.1.el8_6.x86_64 2023-08-28T02:02:45.304 INFO:teuthology.orchestra.run.smithi169.stdout: openblas-0.3.15-3.el8.x86_64 2023-08-28T02:02:45.304 INFO:teuthology.orchestra.run.smithi169.stdout: openblas-threads-0.3.15-3.el8.x86_64 2023-08-28T02:02:45.304 INFO:teuthology.orchestra.run.smithi169.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-08-28T02:02:45.304 INFO:teuthology.orchestra.run.smithi169.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-08-28T02:02:45.304 INFO:teuthology.orchestra.run.smithi169.stdout: python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d.x86_64 2023-08-28T02:02:45.304 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:02:45.304 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-08-28T02:02:45.453 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install ceph-mgr-rook 2023-08-28T02:02:45.549 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : openblas-0.3.15-3.el8.x86_64 8/8 2023-08-28T02:02:45.549 INFO:teuthology.orchestra.run.smithi143.stdout:Last metadata expiration check: 0:02:16 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:02:45.799 INFO:teuthology.orchestra.run.smithi169.stdout:Updating Subscription Management repositories. 2023-08-28T02:02:46.616 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:02:17 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:02:47.956 INFO:teuthology.orchestra.run.smithi143.stdout:Installed products updated. 2023-08-28T02:02:47.975 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-08-28T02:02:47.977 INFO:teuthology.orchestra.run.smithi169.stdout:======================================================================================================== 2023-08-28T02:02:47.977 INFO:teuthology.orchestra.run.smithi169.stdout: Package Arch Version Repository Size 2023-08-28T02:02:47.977 INFO:teuthology.orchestra.run.smithi169.stdout:======================================================================================================== 2023-08-28T02:02:47.977 INFO:teuthology.orchestra.run.smithi169.stdout:Installing: 2023-08-28T02:02:47.977 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-mgr-rook noarch 2:16.2.13-565.g595355f4.el8 ceph-noarch 27 k 2023-08-28T02:02:47.977 INFO:teuthology.orchestra.run.smithi169.stdout:Installing dependencies: 2023-08-28T02:02:47.977 INFO:teuthology.orchestra.run.smithi169.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-08-28T02:02:47.977 INFO:teuthology.orchestra.run.smithi169.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-08-28T02:02:47.977 INFO:teuthology.orchestra.run.smithi169.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-08-28T02:02:47.978 INFO:teuthology.orchestra.run.smithi169.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2023-08-28T02:02:47.978 INFO:teuthology.orchestra.run.smithi169.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2023-08-28T02:02:47.978 INFO:teuthology.orchestra.run.smithi169.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-08-28T02:02:47.978 INFO:teuthology.orchestra.run.smithi169.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2023-08-28T02:02:47.978 INFO:teuthology.orchestra.run.smithi169.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2023-08-28T02:02:47.978 INFO:teuthology.orchestra.run.smithi169.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2023-08-28T02:02:47.978 INFO:teuthology.orchestra.run.smithi169.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-08-28T02:02:47.978 INFO:teuthology.orchestra.run.smithi169.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-08-28T02:02:47.978 INFO:teuthology.orchestra.run.smithi169.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-08-28T02:02:47.978 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:02:47.978 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction Summary 2023-08-28T02:02:47.978 INFO:teuthology.orchestra.run.smithi169.stdout:======================================================================================================== 2023-08-28T02:02:47.979 INFO:teuthology.orchestra.run.smithi169.stdout:Install 13 Packages 2023-08-28T02:02:47.979 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:02:47.979 INFO:teuthology.orchestra.run.smithi169.stdout:Total download size: 1.9 M 2023-08-28T02:02:47.979 INFO:teuthology.orchestra.run.smithi169.stdout:Installed size: 24 M 2023-08-28T02:02:47.979 INFO:teuthology.orchestra.run.smithi169.stdout:Downloading Packages: 2023-08-28T02:02:48.145 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:02:48.145 INFO:teuthology.orchestra.run.smithi143.stdout:Installed: 2023-08-28T02:02:48.145 INFO:teuthology.orchestra.run.smithi143.stdout: ceph-mgr-diskprediction-local-2:16.2.13-565.g595355f4.el8.noarch 2023-08-28T02:02:48.145 INFO:teuthology.orchestra.run.smithi143.stdout: libgfortran-8.5.0-10.1.el8_6.x86_64 2023-08-28T02:02:48.145 INFO:teuthology.orchestra.run.smithi143.stdout: libquadmath-8.5.0-10.1.el8_6.x86_64 2023-08-28T02:02:48.145 INFO:teuthology.orchestra.run.smithi143.stdout: openblas-0.3.15-3.el8.x86_64 2023-08-28T02:02:48.145 INFO:teuthology.orchestra.run.smithi143.stdout: openblas-threads-0.3.15-3.el8.x86_64 2023-08-28T02:02:48.145 INFO:teuthology.orchestra.run.smithi143.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-08-28T02:02:48.145 INFO:teuthology.orchestra.run.smithi143.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-08-28T02:02:48.145 INFO:teuthology.orchestra.run.smithi143.stdout: python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d.x86_64 2023-08-28T02:02:48.146 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:02:48.146 INFO:teuthology.orchestra.run.smithi143.stdout:Complete! 2023-08-28T02:02:48.234 INFO:teuthology.orchestra.run.smithi169.stdout:(1/13): ceph-mgr-rook-16.2.13-565.g595355f4.el8 152 kB/s | 27 kB 00:00 2023-08-28T02:02:48.264 INFO:teuthology.orchestra.run.smithi169.stdout:(2/13): python3-certifi-2018.10.15-7.el8.noarch 77 kB/s | 16 kB 00:00 2023-08-28T02:02:48.290 INFO:teuthology.orchestra.run.smithi169.stdout:(3/13): python3-cachetools-3.1.1-4.el8.noarch.r 140 kB/s | 33 kB 00:00 2023-08-28T02:02:48.310 DEBUG:teuthology.orchestra.run.smithi143:> sudo yum -y install ceph-mgr-rook 2023-08-28T02:02:48.415 INFO:teuthology.orchestra.run.smithi169.stdout:(4/13): python3-rsa-4.9-2.el8.noarch.rpm 532 kB/s | 64 kB 00:00 2023-08-28T02:02:48.498 INFO:teuthology.orchestra.run.smithi169.stdout:(5/13): python3-websocket-client-0.56.0-5.el8.n 735 kB/s | 61 kB 00:00 2023-08-28T02:02:48.524 INFO:teuthology.orchestra.run.smithi169.stdout:(6/13): python3-google-auth-1.1.1-10.el8.noarch 281 kB/s | 81 kB 00:00 2023-08-28T02:02:48.582 INFO:teuthology.orchestra.run.smithi169.stdout:(7/13): python3-kubernetes-11.0.0-6.el8.noarch. 3.7 MB/s | 1.2 MB 00:00 2023-08-28T02:02:48.608 INFO:teuthology.orchestra.run.smithi169.stdout:(8/13): python3-jsonpatch-1.21-2.el8.noarch.rpm 317 kB/s | 27 kB 00:00 2023-08-28T02:02:48.659 INFO:teuthology.orchestra.run.smithi143.stdout:Updating Subscription Management repositories. 2023-08-28T02:02:48.666 INFO:teuthology.orchestra.run.smithi169.stdout:(9/13): python3-pyasn1-modules-0.3.7-6.el8.noar 657 kB/s | 110 kB 00:00 2023-08-28T02:02:48.691 INFO:teuthology.orchestra.run.smithi169.stdout:(10/13): python3-jsonpointer-1.10-11.el8.noarch 257 kB/s | 20 kB 00:00 2023-08-28T02:02:48.717 INFO:teuthology.orchestra.run.smithi169.stdout:(11/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 943 kB/s | 126 kB 00:00 2023-08-28T02:02:48.734 INFO:teuthology.orchestra.run.smithi169.stdout:(12/13): python3-requests-oauthlib-1.0.0-1.el8. 635 kB/s | 43 kB 00:00 2023-08-28T02:02:48.759 INFO:teuthology.orchestra.run.smithi169.stdout:(13/13): python3-oauthlib-2.1.0-1.el8.noarch.rp 2.3 MB/s | 155 kB 00:00 2023-08-28T02:02:48.761 INFO:teuthology.orchestra.run.smithi169.stdout:-------------------------------------------------------------------------------- 2023-08-28T02:02:48.761 INFO:teuthology.orchestra.run.smithi169.stdout:Total 2.4 MB/s | 1.9 MB 00:00 2023-08-28T02:02:48.820 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction check 2023-08-28T02:02:48.834 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction check succeeded. 2023-08-28T02:02:48.835 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction test 2023-08-28T02:02:48.905 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction test succeeded. 2023-08-28T02:02:48.909 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction 2023-08-28T02:02:49.392 INFO:teuthology.orchestra.run.smithi169.stdout: Preparing : 1/1 2023-08-28T02:02:49.506 INFO:teuthology.orchestra.run.smithi143.stdout:Last metadata expiration check: 0:02:20 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:02:49.647 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-08-28T02:02:49.875 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-rsa-4.9-2.el8.noarch 2/13 2023-08-28T02:02:50.133 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 3/13 2023-08-28T02:02:50.390 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 4/13 2023-08-28T02:02:50.587 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 5/13 2023-08-28T02:02:50.738 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 6/13 2023-08-28T02:02:50.916 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 7/13 2023-08-28T02:02:50.941 INFO:teuthology.orchestra.run.smithi143.stdout:Dependencies resolved. 2023-08-28T02:02:50.943 INFO:teuthology.orchestra.run.smithi143.stdout:======================================================================================================== 2023-08-28T02:02:50.943 INFO:teuthology.orchestra.run.smithi143.stdout: Package Arch Version Repository Size 2023-08-28T02:02:50.943 INFO:teuthology.orchestra.run.smithi143.stdout:======================================================================================================== 2023-08-28T02:02:50.943 INFO:teuthology.orchestra.run.smithi143.stdout:Installing: 2023-08-28T02:02:50.943 INFO:teuthology.orchestra.run.smithi143.stdout: ceph-mgr-rook noarch 2:16.2.13-565.g595355f4.el8 ceph-noarch 27 k 2023-08-28T02:02:50.943 INFO:teuthology.orchestra.run.smithi143.stdout:Installing dependencies: 2023-08-28T02:02:50.944 INFO:teuthology.orchestra.run.smithi143.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-08-28T02:02:50.944 INFO:teuthology.orchestra.run.smithi143.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-08-28T02:02:50.944 INFO:teuthology.orchestra.run.smithi143.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-08-28T02:02:50.944 INFO:teuthology.orchestra.run.smithi143.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2023-08-28T02:02:50.944 INFO:teuthology.orchestra.run.smithi143.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2023-08-28T02:02:50.944 INFO:teuthology.orchestra.run.smithi143.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-08-28T02:02:50.945 INFO:teuthology.orchestra.run.smithi143.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2023-08-28T02:02:50.945 INFO:teuthology.orchestra.run.smithi143.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2023-08-28T02:02:50.945 INFO:teuthology.orchestra.run.smithi143.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2023-08-28T02:02:50.945 INFO:teuthology.orchestra.run.smithi143.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-08-28T02:02:50.945 INFO:teuthology.orchestra.run.smithi143.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-08-28T02:02:50.946 INFO:teuthology.orchestra.run.smithi143.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-08-28T02:02:50.947 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:02:50.947 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction Summary 2023-08-28T02:02:50.947 INFO:teuthology.orchestra.run.smithi143.stdout:======================================================================================================== 2023-08-28T02:02:50.947 INFO:teuthology.orchestra.run.smithi143.stdout:Install 13 Packages 2023-08-28T02:02:50.948 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:02:50.950 INFO:teuthology.orchestra.run.smithi143.stdout:Total download size: 1.9 M 2023-08-28T02:02:50.950 INFO:teuthology.orchestra.run.smithi143.stdout:Installed size: 24 M 2023-08-28T02:02:50.950 INFO:teuthology.orchestra.run.smithi143.stdout:Downloading Packages: 2023-08-28T02:02:51.112 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 8/13 2023-08-28T02:02:51.263 INFO:teuthology.orchestra.run.smithi143.stdout:(1/13): ceph-mgr-rook-16.2.13-565.g595355f4.el8 198 kB/s | 27 kB 00:00 2023-08-28T02:02:51.329 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-08-28T02:02:51.409 INFO:teuthology.orchestra.run.smithi143.stdout:(2/13): python3-cachetools-3.1.1-4.el8.noarch.r 116 kB/s | 33 kB 00:00 2023-08-28T02:02:51.431 INFO:teuthology.orchestra.run.smithi143.stdout:(3/13): python3-certifi-2018.10.15-7.el8.noarch 53 kB/s | 16 kB 00:00 2023-08-28T02:02:51.546 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 10/13 2023-08-28T02:02:51.598 INFO:teuthology.orchestra.run.smithi143.stdout:(4/13): python3-google-auth-1.1.1-10.el8.noarch 243 kB/s | 81 kB 00:00 2023-08-28T02:02:51.623 INFO:teuthology.orchestra.run.smithi143.stdout:(5/13): python3-rsa-4.9-2.el8.noarch.rpm 335 kB/s | 64 kB 00:00 2023-08-28T02:02:51.673 INFO:teuthology.orchestra.run.smithi143.stdout:(6/13): python3-requests-oauthlib-1.0.0-1.el8.n 851 kB/s | 43 kB 00:00 2023-08-28T02:02:51.699 INFO:teuthology.orchestra.run.smithi143.stdout:(7/13): python3-websocket-client-0.56.0-5.el8.n 608 kB/s | 61 kB 00:00 2023-08-28T02:02:51.724 INFO:teuthology.orchestra.run.smithi143.stdout:(8/13): python3-oauthlib-2.1.0-1.el8.noarch.rpm 3.0 MB/s | 155 kB 00:00 2023-08-28T02:02:51.757 INFO:teuthology.orchestra.run.smithi143.stdout:(9/13): python3-jsonpatch-1.21-2.el8.noarch.rpm 949 kB/s | 27 kB 00:00 2023-08-28T02:02:51.775 INFO:teuthology.orchestra.run.smithi143.stdout:(10/13): python3-pyasn1-modules-0.3.7-6.el8.noa 1.4 MB/s | 110 kB 00:00 2023-08-28T02:02:51.808 INFO:teuthology.orchestra.run.smithi143.stdout:(11/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 2.4 MB/s | 126 kB 00:00 2023-08-28T02:02:51.825 INFO:teuthology.orchestra.run.smithi143.stdout:(12/13): python3-jsonpointer-1.10-11.el8.noarch 400 kB/s | 20 kB 00:00 2023-08-28T02:02:51.858 INFO:teuthology.orchestra.run.smithi143.stdout:(13/13): python3-kubernetes-11.0.0-6.el8.noarch 2.6 MB/s | 1.2 MB 00:00 2023-08-28T02:02:51.859 INFO:teuthology.orchestra.run.smithi143.stdout:-------------------------------------------------------------------------------- 2023-08-28T02:02:51.859 INFO:teuthology.orchestra.run.smithi143.stdout:Total 2.1 MB/s | 1.9 MB 00:00 2023-08-28T02:02:51.917 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction check 2023-08-28T02:02:51.932 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction check succeeded. 2023-08-28T02:02:51.933 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction test 2023-08-28T02:02:52.002 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction test succeeded. 2023-08-28T02:02:52.006 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction 2023-08-28T02:02:52.486 INFO:teuthology.orchestra.run.smithi143.stdout: Preparing : 1/1 2023-08-28T02:02:52.640 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 11/13 2023-08-28T02:02:52.697 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-08-28T02:02:52.896 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 12/13 2023-08-28T02:02:52.900 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-rsa-4.9-2.el8.noarch 2/13 2023-08-28T02:02:53.088 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : ceph-mgr-rook-2:16.2.13-565.g595355f4.el8.noarch 13/13 2023-08-28T02:02:53.112 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 3/13 2023-08-28T02:02:53.255 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 4/13 2023-08-28T02:02:53.313 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: ceph-mgr-rook-2:16.2.13-565.g595355f4.el8.noarch 13/13 2023-08-28T02:02:53.313 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : ceph-mgr-rook-2:16.2.13-565.g595355f4.el8.noarch 1/13 2023-08-28T02:02:53.313 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 2/13 2023-08-28T02:02:53.313 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 3/13 2023-08-28T02:02:53.313 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 4/13 2023-08-28T02:02:53.313 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 5/13 2023-08-28T02:02:53.313 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 6/13 2023-08-28T02:02:53.313 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 7/13 2023-08-28T02:02:53.313 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 8/13 2023-08-28T02:02:53.314 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 9/13 2023-08-28T02:02:53.314 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 10/13 2023-08-28T02:02:53.314 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 11/13 2023-08-28T02:02:53.314 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 12/13 2023-08-28T02:02:53.458 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 5/13 2023-08-28T02:02:53.684 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 6/13 2023-08-28T02:02:53.899 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 7/13 2023-08-28T02:02:54.096 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 8/13 2023-08-28T02:02:54.286 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-08-28T02:02:54.459 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 13/13 2023-08-28T02:02:54.459 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:02:25 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:02:54.484 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 10/13 2023-08-28T02:02:55.557 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 11/13 2023-08-28T02:02:55.771 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 12/13 2023-08-28T02:02:55.914 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : ceph-mgr-rook-2:16.2.13-565.g595355f4.el8.noarch 13/13 2023-08-28T02:02:56.155 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: ceph-mgr-rook-2:16.2.13-565.g595355f4.el8.noarch 13/13 2023-08-28T02:02:56.156 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : ceph-mgr-rook-2:16.2.13-565.g595355f4.el8.noarch 1/13 2023-08-28T02:02:56.156 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 2/13 2023-08-28T02:02:56.156 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 3/13 2023-08-28T02:02:56.156 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 4/13 2023-08-28T02:02:56.156 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 5/13 2023-08-28T02:02:56.156 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 6/13 2023-08-28T02:02:56.156 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 7/13 2023-08-28T02:02:56.156 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 8/13 2023-08-28T02:02:56.156 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 9/13 2023-08-28T02:02:56.156 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 10/13 2023-08-28T02:02:56.156 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 11/13 2023-08-28T02:02:56.157 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 12/13 2023-08-28T02:02:56.747 INFO:teuthology.orchestra.run.smithi169.stdout:Installed products updated. 2023-08-28T02:02:56.934 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:02:56.934 INFO:teuthology.orchestra.run.smithi169.stdout:Installed: 2023-08-28T02:02:56.935 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-mgr-rook-2:16.2.13-565.g595355f4.el8.noarch 2023-08-28T02:02:56.935 INFO:teuthology.orchestra.run.smithi169.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-08-28T02:02:56.935 INFO:teuthology.orchestra.run.smithi169.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-08-28T02:02:56.935 INFO:teuthology.orchestra.run.smithi169.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-08-28T02:02:56.935 INFO:teuthology.orchestra.run.smithi169.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-08-28T02:02:56.935 INFO:teuthology.orchestra.run.smithi169.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-08-28T02:02:56.935 INFO:teuthology.orchestra.run.smithi169.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-08-28T02:02:56.935 INFO:teuthology.orchestra.run.smithi169.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-08-28T02:02:56.935 INFO:teuthology.orchestra.run.smithi169.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-08-28T02:02:56.935 INFO:teuthology.orchestra.run.smithi169.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-08-28T02:02:56.935 INFO:teuthology.orchestra.run.smithi169.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-08-28T02:02:56.936 INFO:teuthology.orchestra.run.smithi169.stdout: python3-rsa-4.9-2.el8.noarch 2023-08-28T02:02:56.936 INFO:teuthology.orchestra.run.smithi169.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-08-28T02:02:56.936 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:02:56.936 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-08-28T02:02:57.056 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install ceph-mgr-cephadm 2023-08-28T02:02:57.261 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 13/13 2023-08-28T02:02:57.261 INFO:teuthology.orchestra.run.smithi143.stdout:Last metadata expiration check: 0:02:28 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:02:57.409 INFO:teuthology.orchestra.run.smithi169.stdout:Updating Subscription Management repositories. 2023-08-28T02:02:58.220 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:02:29 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:02:59.598 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-08-28T02:02:59.599 INFO:teuthology.orchestra.run.smithi169.stdout:=============================================================================================== 2023-08-28T02:02:59.599 INFO:teuthology.orchestra.run.smithi169.stdout: Package Arch Version Repository Size 2023-08-28T02:02:59.599 INFO:teuthology.orchestra.run.smithi169.stdout:=============================================================================================== 2023-08-28T02:02:59.599 INFO:teuthology.orchestra.run.smithi169.stdout:Installing: 2023-08-28T02:02:59.599 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-mgr-cephadm noarch 2:16.2.13-565.g595355f4.el8 ceph-noarch 116 k 2023-08-28T02:02:59.600 INFO:teuthology.orchestra.run.smithi169.stdout:Installing dependencies: 2023-08-28T02:02:59.600 INFO:teuthology.orchestra.run.smithi169.stdout: python3-apipkg noarch 1.5-6.el8 epel 21 k 2023-08-28T02:02:59.600 INFO:teuthology.orchestra.run.smithi169.stdout: python3-babel noarch 2.5.1-7.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2023-08-28T02:02:59.600 INFO:teuthology.orchestra.run.smithi169.stdout: python3-execnet noarch 1.7.1-1.el8 epel 355 k 2023-08-28T02:02:59.600 INFO:teuthology.orchestra.run.smithi169.stdout: python3-jinja2 noarch 2.10.1-3.el8 rhel-8-for-x86_64-appstream-rpms 538 k 2023-08-28T02:02:59.600 INFO:teuthology.orchestra.run.smithi169.stdout: python3-remoto noarch 1.2.1-1.el8 epel 49 k 2023-08-28T02:02:59.600 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:02:59.600 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction Summary 2023-08-28T02:02:59.600 INFO:teuthology.orchestra.run.smithi169.stdout:=============================================================================================== 2023-08-28T02:02:59.600 INFO:teuthology.orchestra.run.smithi169.stdout:Install 6 Packages 2023-08-28T02:02:59.600 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:02:59.601 INFO:teuthology.orchestra.run.smithi169.stdout:Total download size: 5.8 M 2023-08-28T02:02:59.601 INFO:teuthology.orchestra.run.smithi169.stdout:Installed size: 24 M 2023-08-28T02:02:59.601 INFO:teuthology.orchestra.run.smithi169.stdout:Downloading Packages: 2023-08-28T02:02:59.742 INFO:teuthology.orchestra.run.smithi143.stdout:Installed products updated. 2023-08-28T02:02:59.929 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:02:59.929 INFO:teuthology.orchestra.run.smithi143.stdout:Installed: 2023-08-28T02:02:59.929 INFO:teuthology.orchestra.run.smithi143.stdout: ceph-mgr-rook-2:16.2.13-565.g595355f4.el8.noarch 2023-08-28T02:02:59.930 INFO:teuthology.orchestra.run.smithi143.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-08-28T02:02:59.930 INFO:teuthology.orchestra.run.smithi143.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-08-28T02:02:59.930 INFO:teuthology.orchestra.run.smithi143.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-08-28T02:02:59.930 INFO:teuthology.orchestra.run.smithi143.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-08-28T02:02:59.930 INFO:teuthology.orchestra.run.smithi143.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-08-28T02:02:59.930 INFO:teuthology.orchestra.run.smithi143.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-08-28T02:02:59.930 INFO:teuthology.orchestra.run.smithi143.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-08-28T02:02:59.930 INFO:teuthology.orchestra.run.smithi143.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-08-28T02:02:59.931 INFO:teuthology.orchestra.run.smithi143.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-08-28T02:02:59.931 INFO:teuthology.orchestra.run.smithi143.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-08-28T02:02:59.931 INFO:teuthology.orchestra.run.smithi143.stdout: python3-rsa-4.9-2.el8.noarch 2023-08-28T02:02:59.931 INFO:teuthology.orchestra.run.smithi143.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-08-28T02:02:59.931 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:02:59.931 INFO:teuthology.orchestra.run.smithi143.stdout:Complete! 2023-08-28T02:02:59.933 INFO:teuthology.orchestra.run.smithi169.stdout:(1/6): ceph-mgr-cephadm-16.2.13-565.g595355f4.e 398 kB/s | 116 kB 00:00 2023-08-28T02:02:59.984 INFO:teuthology.orchestra.run.smithi169.stdout:(2/6): python3-apipkg-1.5-6.el8.noarch.rpm 62 kB/s | 21 kB 00:00 2023-08-28T02:03:00.063 DEBUG:teuthology.orchestra.run.smithi143:> sudo yum -y install ceph-mgr-cephadm 2023-08-28T02:03:00.076 INFO:teuthology.orchestra.run.smithi169.stdout:(3/6): python3-jinja2-2.10.1-3.el8.noarch.rpm 5.7 MB/s | 538 kB 00:00 2023-08-28T02:03:00.218 INFO:teuthology.orchestra.run.smithi169.stdout:(4/6): python3-babel-2.5.1-7.el8.noarch.rpm 34 MB/s | 4.8 MB 00:00 2023-08-28T02:03:00.293 INFO:teuthology.orchestra.run.smithi169.stdout:(5/6): python3-execnet-1.7.1-1.el8.noarch.rpm 546 kB/s | 355 kB 00:00 2023-08-28T02:03:00.335 INFO:teuthology.orchestra.run.smithi169.stdout:(6/6): python3-remoto-1.2.1-1.el8.noarch.rpm 122 kB/s | 49 kB 00:00 2023-08-28T02:03:00.337 INFO:teuthology.orchestra.run.smithi169.stdout:-------------------------------------------------------------------------------- 2023-08-28T02:03:00.337 INFO:teuthology.orchestra.run.smithi169.stdout:Total 7.9 MB/s | 5.8 MB 00:00 2023-08-28T02:03:00.391 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction check 2023-08-28T02:03:00.402 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction check succeeded. 2023-08-28T02:03:00.402 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction test 2023-08-28T02:03:00.417 INFO:teuthology.orchestra.run.smithi143.stdout:Updating Subscription Management repositories. 2023-08-28T02:03:00.480 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction test succeeded. 2023-08-28T02:03:00.483 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction 2023-08-28T02:03:01.258 INFO:teuthology.orchestra.run.smithi143.stdout:Last metadata expiration check: 0:02:32 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:03:01.461 INFO:teuthology.orchestra.run.smithi169.stdout: Preparing : 1/1 2023-08-28T02:03:01.891 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 1/6 2023-08-28T02:03:02.159 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 2/6 2023-08-28T02:03:02.428 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-apipkg-1.5-6.el8.noarch 3/6 2023-08-28T02:03:02.621 INFO:teuthology.orchestra.run.smithi143.stdout:Dependencies resolved. 2023-08-28T02:03:02.622 INFO:teuthology.orchestra.run.smithi143.stdout:=============================================================================================== 2023-08-28T02:03:02.622 INFO:teuthology.orchestra.run.smithi143.stdout: Package Arch Version Repository Size 2023-08-28T02:03:02.623 INFO:teuthology.orchestra.run.smithi143.stdout:=============================================================================================== 2023-08-28T02:03:02.623 INFO:teuthology.orchestra.run.smithi143.stdout:Installing: 2023-08-28T02:03:02.623 INFO:teuthology.orchestra.run.smithi143.stdout: ceph-mgr-cephadm noarch 2:16.2.13-565.g595355f4.el8 ceph-noarch 116 k 2023-08-28T02:03:02.623 INFO:teuthology.orchestra.run.smithi143.stdout:Installing dependencies: 2023-08-28T02:03:02.623 INFO:teuthology.orchestra.run.smithi143.stdout: python3-apipkg noarch 1.5-6.el8 epel 21 k 2023-08-28T02:03:02.623 INFO:teuthology.orchestra.run.smithi143.stdout: python3-babel noarch 2.5.1-7.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2023-08-28T02:03:02.623 INFO:teuthology.orchestra.run.smithi143.stdout: python3-execnet noarch 1.7.1-1.el8 epel 355 k 2023-08-28T02:03:02.623 INFO:teuthology.orchestra.run.smithi143.stdout: python3-jinja2 noarch 2.10.1-3.el8 rhel-8-for-x86_64-appstream-rpms 538 k 2023-08-28T02:03:02.623 INFO:teuthology.orchestra.run.smithi143.stdout: python3-remoto noarch 1.2.1-1.el8 epel 49 k 2023-08-28T02:03:02.623 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:03:02.623 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction Summary 2023-08-28T02:03:02.623 INFO:teuthology.orchestra.run.smithi143.stdout:=============================================================================================== 2023-08-28T02:03:02.624 INFO:teuthology.orchestra.run.smithi143.stdout:Install 6 Packages 2023-08-28T02:03:02.624 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:03:02.624 INFO:teuthology.orchestra.run.smithi143.stdout:Total download size: 5.8 M 2023-08-28T02:03:02.624 INFO:teuthology.orchestra.run.smithi143.stdout:Installed size: 24 M 2023-08-28T02:03:02.624 INFO:teuthology.orchestra.run.smithi143.stdout:Downloading Packages: 2023-08-28T02:03:02.689 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-execnet-1.7.1-1.el8.noarch 4/6 2023-08-28T02:03:02.928 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : python3-remoto-1.2.1-1.el8.noarch 5/6 2023-08-28T02:03:03.119 INFO:teuthology.orchestra.run.smithi143.stdout:(1/6): python3-apipkg-1.5-6.el8.noarch.rpm 93 kB/s | 21 kB 00:00 2023-08-28T02:03:03.145 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : ceph-mgr-cephadm-2:16.2.13-565.g595355f4.el8.noarch 6/6 2023-08-28T02:03:03.169 INFO:teuthology.orchestra.run.smithi143.stdout:(2/6): ceph-mgr-cephadm-16.2.13-565.g595355f4.e 416 kB/s | 116 kB 00:00 2023-08-28T02:03:03.237 INFO:teuthology.orchestra.run.smithi143.stdout:(3/6): python3-remoto-1.2.1-1.el8.noarch.rpm 417 kB/s | 49 kB 00:00 2023-08-28T02:03:03.278 INFO:teuthology.orchestra.run.smithi143.stdout:(4/6): python3-jinja2-2.10.1-3.el8.noarch.rpm 4.8 MB/s | 538 kB 00:00 2023-08-28T02:03:03.312 INFO:teuthology.orchestra.run.smithi143.stdout:(5/6): python3-execnet-1.7.1-1.el8.noarch.rpm 843 kB/s | 355 kB 00:00 2023-08-28T02:03:03.384 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: ceph-mgr-cephadm-2:16.2.13-565.g595355f4.el8.noarch 6/6 2023-08-28T02:03:03.384 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : ceph-mgr-cephadm-2:16.2.13-565.g595355f4.el8.noarch 1/6 2023-08-28T02:03:03.384 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-apipkg-1.5-6.el8.noarch 2/6 2023-08-28T02:03:03.385 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-execnet-1.7.1-1.el8.noarch 3/6 2023-08-28T02:03:03.385 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-remoto-1.2.1-1.el8.noarch 4/6 2023-08-28T02:03:03.385 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 5/6 2023-08-28T02:03:03.413 INFO:teuthology.orchestra.run.smithi143.stdout:(6/6): python3-babel-2.5.1-7.el8.noarch.rpm 27 MB/s | 4.8 MB 00:00 2023-08-28T02:03:03.414 INFO:teuthology.orchestra.run.smithi143.stdout:-------------------------------------------------------------------------------- 2023-08-28T02:03:03.415 INFO:teuthology.orchestra.run.smithi143.stdout:Total 7.4 MB/s | 5.8 MB 00:00 2023-08-28T02:03:03.470 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction check 2023-08-28T02:03:03.478 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction check succeeded. 2023-08-28T02:03:03.478 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction test 2023-08-28T02:03:03.557 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction test succeeded. 2023-08-28T02:03:03.561 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction 2023-08-28T02:03:04.482 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 6/6 2023-08-28T02:03:04.482 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:02:35 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:03:04.541 INFO:teuthology.orchestra.run.smithi143.stdout: Preparing : 1/1 2023-08-28T02:03:04.906 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 1/6 2023-08-28T02:03:05.127 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 2/6 2023-08-28T02:03:05.395 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-apipkg-1.5-6.el8.noarch 3/6 2023-08-28T02:03:05.622 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-execnet-1.7.1-1.el8.noarch 4/6 2023-08-28T02:03:05.845 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : python3-remoto-1.2.1-1.el8.noarch 5/6 2023-08-28T02:03:06.029 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : ceph-mgr-cephadm-2:16.2.13-565.g595355f4.el8.noarch 6/6 2023-08-28T02:03:06.268 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: ceph-mgr-cephadm-2:16.2.13-565.g595355f4.el8.noarch 6/6 2023-08-28T02:03:06.268 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : ceph-mgr-cephadm-2:16.2.13-565.g595355f4.el8.noarch 1/6 2023-08-28T02:03:06.268 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-apipkg-1.5-6.el8.noarch 2/6 2023-08-28T02:03:06.268 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-execnet-1.7.1-1.el8.noarch 3/6 2023-08-28T02:03:06.268 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-remoto-1.2.1-1.el8.noarch 4/6 2023-08-28T02:03:06.268 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 5/6 2023-08-28T02:03:06.799 INFO:teuthology.orchestra.run.smithi169.stdout:Installed products updated. 2023-08-28T02:03:06.986 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:03:06.986 INFO:teuthology.orchestra.run.smithi169.stdout:Installed: 2023-08-28T02:03:06.986 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-mgr-cephadm-2:16.2.13-565.g595355f4.el8.noarch 2023-08-28T02:03:06.986 INFO:teuthology.orchestra.run.smithi169.stdout: python3-apipkg-1.5-6.el8.noarch 2023-08-28T02:03:06.986 INFO:teuthology.orchestra.run.smithi169.stdout: python3-babel-2.5.1-7.el8.noarch 2023-08-28T02:03:06.986 INFO:teuthology.orchestra.run.smithi169.stdout: python3-execnet-1.7.1-1.el8.noarch 2023-08-28T02:03:06.986 INFO:teuthology.orchestra.run.smithi169.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-08-28T02:03:06.986 INFO:teuthology.orchestra.run.smithi169.stdout: python3-remoto-1.2.1-1.el8.noarch 2023-08-28T02:03:06.987 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:03:06.987 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-08-28T02:03:07.149 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install ceph-fuse 2023-08-28T02:03:07.301 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 6/6 2023-08-28T02:03:07.301 INFO:teuthology.orchestra.run.smithi143.stdout:Last metadata expiration check: 0:02:38 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:03:07.504 INFO:teuthology.orchestra.run.smithi169.stdout:Updating Subscription Management repositories. 2023-08-28T02:03:08.298 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:02:39 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:03:09.658 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-08-28T02:03:09.658 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-08-28T02:03:09.658 INFO:teuthology.orchestra.run.smithi169.stdout: Package Arch Version Repository Size 2023-08-28T02:03:09.659 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-08-28T02:03:09.659 INFO:teuthology.orchestra.run.smithi169.stdout:Installing: 2023-08-28T02:03:09.659 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-fuse 2023-08-28T02:03:09.659 INFO:teuthology.orchestra.run.smithi169.stdout: x86_64 2:16.2.13-565.g595355f4.el8 ceph 910 k 2023-08-28T02:03:09.659 INFO:teuthology.orchestra.run.smithi169.stdout:Installing dependencies: 2023-08-28T02:03:09.659 INFO:teuthology.orchestra.run.smithi169.stdout: fuse x86_64 2.9.7-15.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2023-08-28T02:03:09.659 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:03:09.659 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction Summary 2023-08-28T02:03:09.659 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-08-28T02:03:09.659 INFO:teuthology.orchestra.run.smithi169.stdout:Install 2 Packages 2023-08-28T02:03:09.659 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:03:09.660 INFO:teuthology.orchestra.run.smithi169.stdout:Total download size: 993 k 2023-08-28T02:03:09.660 INFO:teuthology.orchestra.run.smithi169.stdout:Installed size: 2.8 M 2023-08-28T02:03:09.660 INFO:teuthology.orchestra.run.smithi169.stdout:Downloading Packages: 2023-08-28T02:03:09.679 INFO:teuthology.orchestra.run.smithi143.stdout:Installed products updated. 2023-08-28T02:03:09.754 INFO:teuthology.orchestra.run.smithi169.stdout:(1/2): fuse-2.9.7-15.el8.x86_64.rpm 884 kB/s | 83 kB 00:00 2023-08-28T02:03:09.866 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:03:09.866 INFO:teuthology.orchestra.run.smithi143.stdout:Installed: 2023-08-28T02:03:09.866 INFO:teuthology.orchestra.run.smithi143.stdout: ceph-mgr-cephadm-2:16.2.13-565.g595355f4.el8.noarch 2023-08-28T02:03:09.866 INFO:teuthology.orchestra.run.smithi143.stdout: python3-apipkg-1.5-6.el8.noarch 2023-08-28T02:03:09.866 INFO:teuthology.orchestra.run.smithi143.stdout: python3-babel-2.5.1-7.el8.noarch 2023-08-28T02:03:09.866 INFO:teuthology.orchestra.run.smithi143.stdout: python3-execnet-1.7.1-1.el8.noarch 2023-08-28T02:03:09.866 INFO:teuthology.orchestra.run.smithi143.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-08-28T02:03:09.867 INFO:teuthology.orchestra.run.smithi143.stdout: python3-remoto-1.2.1-1.el8.noarch 2023-08-28T02:03:09.867 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:03:09.867 INFO:teuthology.orchestra.run.smithi143.stdout:Complete! 2023-08-28T02:03:10.022 DEBUG:teuthology.orchestra.run.smithi143:> sudo yum -y install ceph-fuse 2023-08-28T02:03:10.079 INFO:teuthology.orchestra.run.smithi169.stdout:(2/2): ceph-fuse-16.2.13-565.g595355f4.el8.x86_ 2.1 MB/s | 910 kB 00:00 2023-08-28T02:03:10.080 INFO:teuthology.orchestra.run.smithi169.stdout:-------------------------------------------------------------------------------- 2023-08-28T02:03:10.080 INFO:teuthology.orchestra.run.smithi169.stdout:Total 2.3 MB/s | 993 kB 00:00 2023-08-28T02:03:10.091 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction check 2023-08-28T02:03:10.112 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction check succeeded. 2023-08-28T02:03:10.112 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction test 2023-08-28T02:03:10.186 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction test succeeded. 2023-08-28T02:03:10.190 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction 2023-08-28T02:03:10.360 INFO:teuthology.orchestra.run.smithi169.stdout: Preparing : 1/1 2023-08-28T02:03:10.374 INFO:teuthology.orchestra.run.smithi143.stdout:Updating Subscription Management repositories. 2023-08-28T02:03:10.651 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : fuse-2.9.7-15.el8.x86_64 1/2 2023-08-28T02:03:10.809 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : ceph-fuse-2:16.2.13-565.g595355f4.el8.x86_64 2/2 2023-08-28T02:03:11.152 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: ceph-fuse-2:16.2.13-565.g595355f4.el8.x86_64 2/2 2023-08-28T02:03:11.152 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : ceph-fuse-2:16.2.13-565.g595355f4.el8.x86_64 1/2 2023-08-28T02:03:11.161 INFO:teuthology.orchestra.run.smithi143.stdout:Last metadata expiration check: 0:02:42 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:03:12.256 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : fuse-2.9.7-15.el8.x86_64 2/2 2023-08-28T02:03:12.256 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:02:43 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:03:12.530 INFO:teuthology.orchestra.run.smithi143.stdout:Dependencies resolved. 2023-08-28T02:03:12.530 INFO:teuthology.orchestra.run.smithi143.stdout:================================================================================ 2023-08-28T02:03:12.530 INFO:teuthology.orchestra.run.smithi143.stdout: Package Arch Version Repository Size 2023-08-28T02:03:12.530 INFO:teuthology.orchestra.run.smithi143.stdout:================================================================================ 2023-08-28T02:03:12.530 INFO:teuthology.orchestra.run.smithi143.stdout:Installing: 2023-08-28T02:03:12.531 INFO:teuthology.orchestra.run.smithi143.stdout: ceph-fuse 2023-08-28T02:03:12.531 INFO:teuthology.orchestra.run.smithi143.stdout: x86_64 2:16.2.13-565.g595355f4.el8 ceph 910 k 2023-08-28T02:03:12.531 INFO:teuthology.orchestra.run.smithi143.stdout:Installing dependencies: 2023-08-28T02:03:12.531 INFO:teuthology.orchestra.run.smithi143.stdout: fuse x86_64 2.9.7-15.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2023-08-28T02:03:12.531 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:03:12.531 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction Summary 2023-08-28T02:03:12.531 INFO:teuthology.orchestra.run.smithi143.stdout:================================================================================ 2023-08-28T02:03:12.531 INFO:teuthology.orchestra.run.smithi143.stdout:Install 2 Packages 2023-08-28T02:03:12.531 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:03:12.532 INFO:teuthology.orchestra.run.smithi143.stdout:Total download size: 993 k 2023-08-28T02:03:12.532 INFO:teuthology.orchestra.run.smithi143.stdout:Installed size: 2.8 M 2023-08-28T02:03:12.532 INFO:teuthology.orchestra.run.smithi143.stdout:Downloading Packages: 2023-08-28T02:03:12.608 INFO:teuthology.orchestra.run.smithi143.stdout:(1/2): fuse-2.9.7-15.el8.x86_64.rpm 1.1 MB/s | 83 kB 00:00 2023-08-28T02:03:12.808 INFO:teuthology.orchestra.run.smithi143.stdout:(2/2): ceph-fuse-16.2.13-565.g595355f4.el8.x86_ 3.2 MB/s | 910 kB 00:00 2023-08-28T02:03:12.809 INFO:teuthology.orchestra.run.smithi143.stdout:-------------------------------------------------------------------------------- 2023-08-28T02:03:12.809 INFO:teuthology.orchestra.run.smithi143.stdout:Total 3.5 MB/s | 993 kB 00:00 2023-08-28T02:03:12.820 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction check 2023-08-28T02:03:12.840 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction check succeeded. 2023-08-28T02:03:12.840 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction test 2023-08-28T02:03:12.914 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction test succeeded. 2023-08-28T02:03:12.918 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction 2023-08-28T02:03:13.084 INFO:teuthology.orchestra.run.smithi143.stdout: Preparing : 1/1 2023-08-28T02:03:13.327 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : fuse-2.9.7-15.el8.x86_64 1/2 2023-08-28T02:03:13.484 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : ceph-fuse-2:16.2.13-565.g595355f4.el8.x86_64 2/2 2023-08-28T02:03:13.808 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: ceph-fuse-2:16.2.13-565.g595355f4.el8.x86_64 2/2 2023-08-28T02:03:13.808 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : ceph-fuse-2:16.2.13-565.g595355f4.el8.x86_64 1/2 2023-08-28T02:03:14.608 INFO:teuthology.orchestra.run.smithi169.stdout:Installed products updated. 2023-08-28T02:03:14.791 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:03:14.791 INFO:teuthology.orchestra.run.smithi169.stdout:Installed: 2023-08-28T02:03:14.791 INFO:teuthology.orchestra.run.smithi169.stdout: ceph-fuse-2:16.2.13-565.g595355f4.el8.x86_64 fuse-2.9.7-15.el8.x86_64 2023-08-28T02:03:14.791 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:03:14.792 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-08-28T02:03:14.880 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : fuse-2.9.7-15.el8.x86_64 2/2 2023-08-28T02:03:14.880 INFO:teuthology.orchestra.run.smithi143.stdout:Last metadata expiration check: 0:02:45 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:03:14.929 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install librados-devel 2023-08-28T02:03:15.285 INFO:teuthology.orchestra.run.smithi169.stdout:Updating Subscription Management repositories. 2023-08-28T02:03:16.140 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:02:47 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:03:17.180 INFO:teuthology.orchestra.run.smithi143.stdout:Installed products updated. 2023-08-28T02:03:17.368 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:03:17.368 INFO:teuthology.orchestra.run.smithi143.stdout:Installed: 2023-08-28T02:03:17.368 INFO:teuthology.orchestra.run.smithi143.stdout: ceph-fuse-2:16.2.13-565.g595355f4.el8.x86_64 fuse-2.9.7-15.el8.x86_64 2023-08-28T02:03:17.368 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:03:17.368 INFO:teuthology.orchestra.run.smithi143.stdout:Complete! 2023-08-28T02:03:17.508 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-08-28T02:03:17.508 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-08-28T02:03:17.508 INFO:teuthology.orchestra.run.smithi169.stdout: Package Arch Version Repo Size 2023-08-28T02:03:17.508 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-08-28T02:03:17.508 INFO:teuthology.orchestra.run.smithi169.stdout:Installing: 2023-08-28T02:03:17.509 INFO:teuthology.orchestra.run.smithi169.stdout: librados-devel x86_64 2:16.2.13-565.g595355f4.el8 ceph 131 k 2023-08-28T02:03:17.509 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:03:17.509 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction Summary 2023-08-28T02:03:17.509 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-08-28T02:03:17.509 INFO:teuthology.orchestra.run.smithi169.stdout:Install 1 Package 2023-08-28T02:03:17.509 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:03:17.510 INFO:teuthology.orchestra.run.smithi169.stdout:Total download size: 131 k 2023-08-28T02:03:17.510 INFO:teuthology.orchestra.run.smithi169.stdout:Installed size: 432 k 2023-08-28T02:03:17.510 INFO:teuthology.orchestra.run.smithi169.stdout:Downloading Packages: 2023-08-28T02:03:17.695 INFO:teuthology.orchestra.run.smithi169.stdout:librados-devel-16.2.13-565.g595355f4.el8.x86_64 706 kB/s | 131 kB 00:00 2023-08-28T02:03:17.696 INFO:teuthology.orchestra.run.smithi169.stdout:-------------------------------------------------------------------------------- 2023-08-28T02:03:17.696 INFO:teuthology.orchestra.run.smithi169.stdout:Total 701 kB/s | 131 kB 00:00 2023-08-28T02:03:17.696 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction check 2023-08-28T02:03:17.701 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction check succeeded. 2023-08-28T02:03:17.702 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction test 2023-08-28T02:03:17.755 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction test succeeded. 2023-08-28T02:03:17.759 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction 2023-08-28T02:03:17.871 INFO:teuthology.orchestra.run.smithi169.stdout: Preparing : 1/1 2023-08-28T02:03:18.012 DEBUG:teuthology.orchestra.run.smithi143:> sudo yum -y install librados-devel 2023-08-28T02:03:18.065 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : librados-devel-2:16.2.13-565.g595355f4.el8.x86_64 1/1 2023-08-28T02:03:18.283 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: librados-devel-2:16.2.13-565.g595355f4.el8.x86_64 1/1 2023-08-28T02:03:18.371 INFO:teuthology.orchestra.run.smithi143.stdout:Updating Subscription Management repositories. 2023-08-28T02:03:19.247 INFO:teuthology.orchestra.run.smithi143.stdout:Last metadata expiration check: 0:02:50 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:03:19.398 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : librados-devel-2:16.2.13-565.g595355f4.el8.x86_64 1/1 2023-08-28T02:03:19.399 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:02:50 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:03:20.612 INFO:teuthology.orchestra.run.smithi143.stdout:Dependencies resolved. 2023-08-28T02:03:20.612 INFO:teuthology.orchestra.run.smithi143.stdout:================================================================================ 2023-08-28T02:03:20.612 INFO:teuthology.orchestra.run.smithi143.stdout: Package Arch Version Repo Size 2023-08-28T02:03:20.612 INFO:teuthology.orchestra.run.smithi143.stdout:================================================================================ 2023-08-28T02:03:20.612 INFO:teuthology.orchestra.run.smithi143.stdout:Installing: 2023-08-28T02:03:20.612 INFO:teuthology.orchestra.run.smithi143.stdout: librados-devel x86_64 2:16.2.13-565.g595355f4.el8 ceph 131 k 2023-08-28T02:03:20.613 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:03:20.613 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction Summary 2023-08-28T02:03:20.613 INFO:teuthology.orchestra.run.smithi143.stdout:================================================================================ 2023-08-28T02:03:20.613 INFO:teuthology.orchestra.run.smithi143.stdout:Install 1 Package 2023-08-28T02:03:20.613 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:03:20.613 INFO:teuthology.orchestra.run.smithi143.stdout:Total download size: 131 k 2023-08-28T02:03:20.613 INFO:teuthology.orchestra.run.smithi143.stdout:Installed size: 432 k 2023-08-28T02:03:20.614 INFO:teuthology.orchestra.run.smithi143.stdout:Downloading Packages: 2023-08-28T02:03:20.806 INFO:teuthology.orchestra.run.smithi143.stdout:librados-devel-16.2.13-565.g595355f4.el8.x86_64 680 kB/s | 131 kB 00:00 2023-08-28T02:03:20.807 INFO:teuthology.orchestra.run.smithi143.stdout:-------------------------------------------------------------------------------- 2023-08-28T02:03:20.807 INFO:teuthology.orchestra.run.smithi143.stdout:Total 676 kB/s | 131 kB 00:00 2023-08-28T02:03:20.807 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction check 2023-08-28T02:03:20.813 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction check succeeded. 2023-08-28T02:03:20.813 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction test 2023-08-28T02:03:20.867 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction test succeeded. 2023-08-28T02:03:20.871 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction 2023-08-28T02:03:21.018 INFO:teuthology.orchestra.run.smithi143.stdout: Preparing : 1/1 2023-08-28T02:03:21.157 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : librados-devel-2:16.2.13-565.g595355f4.el8.x86_64 1/1 2023-08-28T02:03:21.394 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: librados-devel-2:16.2.13-565.g595355f4.el8.x86_64 1/1 2023-08-28T02:03:21.772 INFO:teuthology.orchestra.run.smithi169.stdout:Installed products updated. 2023-08-28T02:03:21.956 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:03:21.956 INFO:teuthology.orchestra.run.smithi169.stdout:Installed: 2023-08-28T02:03:21.956 INFO:teuthology.orchestra.run.smithi169.stdout: librados-devel-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:03:21.956 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:03:21.956 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-08-28T02:03:22.098 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install libcephfs2 2023-08-28T02:03:22.447 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : librados-devel-2:16.2.13-565.g595355f4.el8.x86_64 1/1 2023-08-28T02:03:22.447 INFO:teuthology.orchestra.run.smithi143.stdout:Last metadata expiration check: 0:02:53 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:03:22.455 INFO:teuthology.orchestra.run.smithi169.stdout:Updating Subscription Management repositories. 2023-08-28T02:03:23.269 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:02:54 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:03:24.598 INFO:teuthology.orchestra.run.smithi169.stdout:Package libcephfs2-2:16.2.13-565.g595355f4.el8.x86_64 is already installed. 2023-08-28T02:03:24.708 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-08-28T02:03:24.710 INFO:teuthology.orchestra.run.smithi169.stdout:Nothing to do. 2023-08-28T02:03:24.710 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-08-28T02:03:24.797 INFO:teuthology.orchestra.run.smithi143.stdout:Installed products updated. 2023-08-28T02:03:24.825 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install libcephfs-devel 2023-08-28T02:03:24.980 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:03:24.980 INFO:teuthology.orchestra.run.smithi143.stdout:Installed: 2023-08-28T02:03:24.980 INFO:teuthology.orchestra.run.smithi143.stdout: librados-devel-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:03:24.980 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:03:24.981 INFO:teuthology.orchestra.run.smithi143.stdout:Complete! 2023-08-28T02:03:25.124 DEBUG:teuthology.orchestra.run.smithi143:> sudo yum -y install libcephfs2 2023-08-28T02:03:25.177 INFO:teuthology.orchestra.run.smithi169.stdout:Updating Subscription Management repositories. 2023-08-28T02:03:25.481 INFO:teuthology.orchestra.run.smithi143.stdout:Updating Subscription Management repositories. 2023-08-28T02:03:26.008 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:02:57 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:03:26.305 INFO:teuthology.orchestra.run.smithi143.stdout:Last metadata expiration check: 0:02:57 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:03:27.383 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-08-28T02:03:27.383 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-08-28T02:03:27.383 INFO:teuthology.orchestra.run.smithi169.stdout: Package Arch Version Repo Size 2023-08-28T02:03:27.383 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-08-28T02:03:27.383 INFO:teuthology.orchestra.run.smithi169.stdout:Installing: 2023-08-28T02:03:27.383 INFO:teuthology.orchestra.run.smithi169.stdout: libcephfs-devel x86_64 2:16.2.13-565.g595355f4.el8 ceph 25 k 2023-08-28T02:03:27.383 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:03:27.384 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction Summary 2023-08-28T02:03:27.384 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-08-28T02:03:27.384 INFO:teuthology.orchestra.run.smithi169.stdout:Install 1 Package 2023-08-28T02:03:27.384 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:03:27.384 INFO:teuthology.orchestra.run.smithi169.stdout:Total download size: 25 k 2023-08-28T02:03:27.384 INFO:teuthology.orchestra.run.smithi169.stdout:Installed size: 107 k 2023-08-28T02:03:27.384 INFO:teuthology.orchestra.run.smithi169.stdout:Downloading Packages: 2023-08-28T02:03:27.565 INFO:teuthology.orchestra.run.smithi143.stdout:Package libcephfs2-2:16.2.13-565.g595355f4.el8.x86_64 is already installed. 2023-08-28T02:03:27.603 INFO:teuthology.orchestra.run.smithi169.stdout:libcephfs-devel-16.2.13-565.g595355f4.el8.x86_6 117 kB/s | 25 kB 00:00 2023-08-28T02:03:27.603 INFO:teuthology.orchestra.run.smithi169.stdout:-------------------------------------------------------------------------------- 2023-08-28T02:03:27.604 INFO:teuthology.orchestra.run.smithi169.stdout:Total 116 kB/s | 25 kB 00:00 2023-08-28T02:03:27.604 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction check 2023-08-28T02:03:27.608 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction check succeeded. 2023-08-28T02:03:27.608 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction test 2023-08-28T02:03:27.610 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction test succeeded. 2023-08-28T02:03:27.614 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction 2023-08-28T02:03:27.666 INFO:teuthology.orchestra.run.smithi169.stdout: Preparing : 1/1 2023-08-28T02:03:27.674 INFO:teuthology.orchestra.run.smithi143.stdout:Dependencies resolved. 2023-08-28T02:03:27.675 INFO:teuthology.orchestra.run.smithi143.stdout:Nothing to do. 2023-08-28T02:03:27.675 INFO:teuthology.orchestra.run.smithi143.stdout:Complete! 2023-08-28T02:03:27.800 DEBUG:teuthology.orchestra.run.smithi143:> sudo yum -y install libcephfs-devel 2023-08-28T02:03:27.813 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : libcephfs-devel-2:16.2.13-565.g595355f4.el8.x86_64 1/1 2023-08-28T02:03:28.024 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: libcephfs-devel-2:16.2.13-565.g595355f4.el8.x86_64 1/1 2023-08-28T02:03:28.154 INFO:teuthology.orchestra.run.smithi143.stdout:Updating Subscription Management repositories. 2023-08-28T02:03:28.995 INFO:teuthology.orchestra.run.smithi143.stdout:Last metadata expiration check: 0:02:59 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:03:29.062 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : libcephfs-devel-2:16.2.13-565.g595355f4.el8.x86_64 1/1 2023-08-28T02:03:29.063 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:03:00 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:03:30.363 INFO:teuthology.orchestra.run.smithi143.stdout:Dependencies resolved. 2023-08-28T02:03:30.364 INFO:teuthology.orchestra.run.smithi143.stdout:================================================================================ 2023-08-28T02:03:30.364 INFO:teuthology.orchestra.run.smithi143.stdout: Package Arch Version Repo Size 2023-08-28T02:03:30.364 INFO:teuthology.orchestra.run.smithi143.stdout:================================================================================ 2023-08-28T02:03:30.364 INFO:teuthology.orchestra.run.smithi143.stdout:Installing: 2023-08-28T02:03:30.364 INFO:teuthology.orchestra.run.smithi143.stdout: libcephfs-devel x86_64 2:16.2.13-565.g595355f4.el8 ceph 25 k 2023-08-28T02:03:30.364 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:03:30.364 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction Summary 2023-08-28T02:03:30.364 INFO:teuthology.orchestra.run.smithi143.stdout:================================================================================ 2023-08-28T02:03:30.364 INFO:teuthology.orchestra.run.smithi143.stdout:Install 1 Package 2023-08-28T02:03:30.365 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:03:30.365 INFO:teuthology.orchestra.run.smithi143.stdout:Total download size: 25 k 2023-08-28T02:03:30.365 INFO:teuthology.orchestra.run.smithi143.stdout:Installed size: 107 k 2023-08-28T02:03:30.365 INFO:teuthology.orchestra.run.smithi143.stdout:Downloading Packages: 2023-08-28T02:03:30.588 INFO:teuthology.orchestra.run.smithi143.stdout:libcephfs-devel-16.2.13-565.g595355f4.el8.x86_6 114 kB/s | 25 kB 00:00 2023-08-28T02:03:30.589 INFO:teuthology.orchestra.run.smithi143.stdout:-------------------------------------------------------------------------------- 2023-08-28T02:03:30.589 INFO:teuthology.orchestra.run.smithi143.stdout:Total 114 kB/s | 25 kB 00:00 2023-08-28T02:03:30.589 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction check 2023-08-28T02:03:30.593 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction check succeeded. 2023-08-28T02:03:30.593 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction test 2023-08-28T02:03:30.595 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction test succeeded. 2023-08-28T02:03:30.599 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction 2023-08-28T02:03:30.645 INFO:teuthology.orchestra.run.smithi143.stdout: Preparing : 1/1 2023-08-28T02:03:30.814 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : libcephfs-devel-2:16.2.13-565.g595355f4.el8.x86_64 1/1 2023-08-28T02:03:31.036 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: libcephfs-devel-2:16.2.13-565.g595355f4.el8.x86_64 1/1 2023-08-28T02:03:31.407 INFO:teuthology.orchestra.run.smithi169.stdout:Installed products updated. 2023-08-28T02:03:31.593 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:03:31.634 INFO:teuthology.orchestra.run.smithi169.stdout:Installed: 2023-08-28T02:03:31.634 INFO:teuthology.orchestra.run.smithi169.stdout: libcephfs-devel-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:03:31.635 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:03:31.635 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-08-28T02:03:31.725 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install librados2 2023-08-28T02:03:32.045 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : libcephfs-devel-2:16.2.13-565.g595355f4.el8.x86_64 1/1 2023-08-28T02:03:32.046 INFO:teuthology.orchestra.run.smithi143.stdout:Last metadata expiration check: 0:03:03 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:03:32.085 INFO:teuthology.orchestra.run.smithi169.stdout:Updating Subscription Management repositories. 2023-08-28T02:03:32.881 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:03:03 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:03:34.213 INFO:teuthology.orchestra.run.smithi169.stdout:Package librados2-2:16.2.13-565.g595355f4.el8.x86_64 is already installed. 2023-08-28T02:03:34.316 INFO:teuthology.orchestra.run.smithi143.stdout:Installed products updated. 2023-08-28T02:03:34.322 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-08-28T02:03:34.323 INFO:teuthology.orchestra.run.smithi169.stdout:Nothing to do. 2023-08-28T02:03:34.323 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-08-28T02:03:34.435 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install librbd1 2023-08-28T02:03:34.499 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:03:34.499 INFO:teuthology.orchestra.run.smithi143.stdout:Installed: 2023-08-28T02:03:34.499 INFO:teuthology.orchestra.run.smithi143.stdout: libcephfs-devel-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:03:34.499 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:03:34.500 INFO:teuthology.orchestra.run.smithi143.stdout:Complete! 2023-08-28T02:03:34.642 DEBUG:teuthology.orchestra.run.smithi143:> sudo yum -y install librados2 2023-08-28T02:03:34.791 INFO:teuthology.orchestra.run.smithi169.stdout:Updating Subscription Management repositories. 2023-08-28T02:03:34.994 INFO:teuthology.orchestra.run.smithi143.stdout:Updating Subscription Management repositories. 2023-08-28T02:03:35.606 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:03:06 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:03:35.832 INFO:teuthology.orchestra.run.smithi143.stdout:Last metadata expiration check: 0:03:06 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:03:36.852 INFO:teuthology.orchestra.run.smithi169.stdout:Package librbd1-2:16.2.13-565.g595355f4.el8.x86_64 is already installed. 2023-08-28T02:03:36.962 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-08-28T02:03:36.962 INFO:teuthology.orchestra.run.smithi169.stdout:Nothing to do. 2023-08-28T02:03:36.962 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-08-28T02:03:37.068 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install python3-rados 2023-08-28T02:03:37.082 INFO:teuthology.orchestra.run.smithi143.stdout:Package librados2-2:16.2.13-565.g595355f4.el8.x86_64 is already installed. 2023-08-28T02:03:37.190 INFO:teuthology.orchestra.run.smithi143.stdout:Dependencies resolved. 2023-08-28T02:03:37.191 INFO:teuthology.orchestra.run.smithi143.stdout:Nothing to do. 2023-08-28T02:03:37.191 INFO:teuthology.orchestra.run.smithi143.stdout:Complete! 2023-08-28T02:03:37.316 DEBUG:teuthology.orchestra.run.smithi143:> sudo yum -y install librbd1 2023-08-28T02:03:37.425 INFO:teuthology.orchestra.run.smithi169.stdout:Updating Subscription Management repositories. 2023-08-28T02:03:37.668 INFO:teuthology.orchestra.run.smithi143.stdout:Updating Subscription Management repositories. 2023-08-28T02:03:38.260 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:03:09 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:03:38.500 INFO:teuthology.orchestra.run.smithi143.stdout:Last metadata expiration check: 0:03:09 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:03:39.526 INFO:teuthology.orchestra.run.smithi169.stdout:Package python3-rados-2:16.2.13-565.g595355f4.el8.x86_64 is already installed. 2023-08-28T02:03:39.639 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-08-28T02:03:39.640 INFO:teuthology.orchestra.run.smithi169.stdout:Nothing to do. 2023-08-28T02:03:39.640 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-08-28T02:03:39.772 INFO:teuthology.orchestra.run.smithi143.stdout:Package librbd1-2:16.2.13-565.g595355f4.el8.x86_64 is already installed. 2023-08-28T02:03:39.826 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install python3-rgw 2023-08-28T02:03:39.883 INFO:teuthology.orchestra.run.smithi143.stdout:Dependencies resolved. 2023-08-28T02:03:39.884 INFO:teuthology.orchestra.run.smithi143.stdout:Nothing to do. 2023-08-28T02:03:39.884 INFO:teuthology.orchestra.run.smithi143.stdout:Complete! 2023-08-28T02:03:40.010 DEBUG:teuthology.orchestra.run.smithi143:> sudo yum -y install python3-rados 2023-08-28T02:03:40.180 INFO:teuthology.orchestra.run.smithi169.stdout:Updating Subscription Management repositories. 2023-08-28T02:03:40.363 INFO:teuthology.orchestra.run.smithi143.stdout:Updating Subscription Management repositories. 2023-08-28T02:03:41.028 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:03:12 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:03:41.146 INFO:teuthology.orchestra.run.smithi143.stdout:Last metadata expiration check: 0:03:12 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:03:42.327 INFO:teuthology.orchestra.run.smithi169.stdout:Package python3-rgw-2:16.2.13-565.g595355f4.el8.x86_64 is already installed. 2023-08-28T02:03:42.436 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-08-28T02:03:42.437 INFO:teuthology.orchestra.run.smithi169.stdout:Nothing to do. 2023-08-28T02:03:42.437 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-08-28T02:03:42.468 INFO:teuthology.orchestra.run.smithi143.stdout:Package python3-rados-2:16.2.13-565.g595355f4.el8.x86_64 is already installed. 2023-08-28T02:03:42.552 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install python3-cephfs 2023-08-28T02:03:42.574 INFO:teuthology.orchestra.run.smithi143.stdout:Dependencies resolved. 2023-08-28T02:03:42.575 INFO:teuthology.orchestra.run.smithi143.stdout:Nothing to do. 2023-08-28T02:03:42.575 INFO:teuthology.orchestra.run.smithi143.stdout:Complete! 2023-08-28T02:03:42.697 DEBUG:teuthology.orchestra.run.smithi143:> sudo yum -y install python3-rgw 2023-08-28T02:03:42.906 INFO:teuthology.orchestra.run.smithi169.stdout:Updating Subscription Management repositories. 2023-08-28T02:03:43.051 INFO:teuthology.orchestra.run.smithi143.stdout:Updating Subscription Management repositories. 2023-08-28T02:03:43.700 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:03:14 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:03:43.836 INFO:teuthology.orchestra.run.smithi143.stdout:Last metadata expiration check: 0:03:14 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:03:44.981 INFO:teuthology.orchestra.run.smithi169.stdout:Package python3-cephfs-2:16.2.13-565.g595355f4.el8.x86_64 is already installed. 2023-08-28T02:03:45.091 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-08-28T02:03:45.092 INFO:teuthology.orchestra.run.smithi169.stdout:Nothing to do. 2023-08-28T02:03:45.092 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-08-28T02:03:45.164 INFO:teuthology.orchestra.run.smithi143.stdout:Package python3-rgw-2:16.2.13-565.g595355f4.el8.x86_64 is already installed. 2023-08-28T02:03:45.209 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install python3-rbd 2023-08-28T02:03:45.272 INFO:teuthology.orchestra.run.smithi143.stdout:Dependencies resolved. 2023-08-28T02:03:45.274 INFO:teuthology.orchestra.run.smithi143.stdout:Nothing to do. 2023-08-28T02:03:45.274 INFO:teuthology.orchestra.run.smithi143.stdout:Complete! 2023-08-28T02:03:45.398 DEBUG:teuthology.orchestra.run.smithi143:> sudo yum -y install python3-cephfs 2023-08-28T02:03:45.566 INFO:teuthology.orchestra.run.smithi169.stdout:Updating Subscription Management repositories. 2023-08-28T02:03:45.750 INFO:teuthology.orchestra.run.smithi143.stdout:Updating Subscription Management repositories. 2023-08-28T02:03:46.366 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:03:17 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:03:46.531 INFO:teuthology.orchestra.run.smithi143.stdout:Last metadata expiration check: 0:03:17 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:03:47.614 INFO:teuthology.orchestra.run.smithi169.stdout:Package python3-rbd-2:16.2.13-565.g595355f4.el8.x86_64 is already installed. 2023-08-28T02:03:47.723 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-08-28T02:03:47.724 INFO:teuthology.orchestra.run.smithi169.stdout:Nothing to do. 2023-08-28T02:03:47.724 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-08-28T02:03:47.799 INFO:teuthology.orchestra.run.smithi143.stdout:Package python3-cephfs-2:16.2.13-565.g595355f4.el8.x86_64 is already installed. 2023-08-28T02:03:47.851 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install rbd-fuse 2023-08-28T02:03:47.910 INFO:teuthology.orchestra.run.smithi143.stdout:Dependencies resolved. 2023-08-28T02:03:47.911 INFO:teuthology.orchestra.run.smithi143.stdout:Nothing to do. 2023-08-28T02:03:47.911 INFO:teuthology.orchestra.run.smithi143.stdout:Complete! 2023-08-28T02:03:48.037 DEBUG:teuthology.orchestra.run.smithi143:> sudo yum -y install python3-rbd 2023-08-28T02:03:48.204 INFO:teuthology.orchestra.run.smithi169.stdout:Updating Subscription Management repositories. 2023-08-28T02:03:48.394 INFO:teuthology.orchestra.run.smithi143.stdout:Updating Subscription Management repositories. 2023-08-28T02:03:49.023 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:03:20 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:03:49.180 INFO:teuthology.orchestra.run.smithi143.stdout:Last metadata expiration check: 0:03:20 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:03:50.397 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-08-28T02:03:50.397 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-08-28T02:03:50.398 INFO:teuthology.orchestra.run.smithi169.stdout: Package Architecture Version Repository Size 2023-08-28T02:03:50.398 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-08-28T02:03:50.398 INFO:teuthology.orchestra.run.smithi169.stdout:Installing: 2023-08-28T02:03:50.398 INFO:teuthology.orchestra.run.smithi169.stdout: rbd-fuse x86_64 2:16.2.13-565.g595355f4.el8 ceph 88 k 2023-08-28T02:03:50.398 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:03:50.398 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction Summary 2023-08-28T02:03:50.398 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-08-28T02:03:50.398 INFO:teuthology.orchestra.run.smithi169.stdout:Install 1 Package 2023-08-28T02:03:50.399 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:03:50.399 INFO:teuthology.orchestra.run.smithi169.stdout:Total download size: 88 k 2023-08-28T02:03:50.399 INFO:teuthology.orchestra.run.smithi169.stdout:Installed size: 240 k 2023-08-28T02:03:50.399 INFO:teuthology.orchestra.run.smithi169.stdout:Downloading Packages: 2023-08-28T02:03:50.457 INFO:teuthology.orchestra.run.smithi143.stdout:Package python3-rbd-2:16.2.13-565.g595355f4.el8.x86_64 is already installed. 2023-08-28T02:03:50.566 INFO:teuthology.orchestra.run.smithi169.stdout:rbd-fuse-16.2.13-565.g595355f4.el8.x86_64.rpm 531 kB/s | 88 kB 00:00 2023-08-28T02:03:50.566 INFO:teuthology.orchestra.run.smithi143.stdout:Dependencies resolved. 2023-08-28T02:03:50.567 INFO:teuthology.orchestra.run.smithi169.stdout:-------------------------------------------------------------------------------- 2023-08-28T02:03:50.567 INFO:teuthology.orchestra.run.smithi169.stdout:Total 527 kB/s | 88 kB 00:00 2023-08-28T02:03:50.567 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction check 2023-08-28T02:03:50.567 INFO:teuthology.orchestra.run.smithi143.stdout:Nothing to do. 2023-08-28T02:03:50.567 INFO:teuthology.orchestra.run.smithi143.stdout:Complete! 2023-08-28T02:03:50.576 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction check succeeded. 2023-08-28T02:03:50.576 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction test 2023-08-28T02:03:50.631 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction test succeeded. 2023-08-28T02:03:50.634 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction 2023-08-28T02:03:50.695 DEBUG:teuthology.orchestra.run.smithi143:> sudo yum -y install rbd-fuse 2023-08-28T02:03:50.747 INFO:teuthology.orchestra.run.smithi169.stdout: Preparing : 1/1 2023-08-28T02:03:50.877 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : rbd-fuse-2:16.2.13-565.g595355f4.el8.x86_64 1/1 2023-08-28T02:03:51.052 INFO:teuthology.orchestra.run.smithi143.stdout:Updating Subscription Management repositories. 2023-08-28T02:03:51.145 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: rbd-fuse-2:16.2.13-565.g595355f4.el8.x86_64 1/1 2023-08-28T02:03:51.866 INFO:teuthology.orchestra.run.smithi143.stdout:Last metadata expiration check: 0:03:22 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:03:52.237 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : rbd-fuse-2:16.2.13-565.g595355f4.el8.x86_64 1/1 2023-08-28T02:03:52.237 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:03:23 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:03:53.245 INFO:teuthology.orchestra.run.smithi143.stdout:Dependencies resolved. 2023-08-28T02:03:53.245 INFO:teuthology.orchestra.run.smithi143.stdout:================================================================================ 2023-08-28T02:03:53.246 INFO:teuthology.orchestra.run.smithi143.stdout: Package Architecture Version Repository Size 2023-08-28T02:03:53.246 INFO:teuthology.orchestra.run.smithi143.stdout:================================================================================ 2023-08-28T02:03:53.246 INFO:teuthology.orchestra.run.smithi143.stdout:Installing: 2023-08-28T02:03:53.246 INFO:teuthology.orchestra.run.smithi143.stdout: rbd-fuse x86_64 2:16.2.13-565.g595355f4.el8 ceph 88 k 2023-08-28T02:03:53.246 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:03:53.246 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction Summary 2023-08-28T02:03:53.246 INFO:teuthology.orchestra.run.smithi143.stdout:================================================================================ 2023-08-28T02:03:53.246 INFO:teuthology.orchestra.run.smithi143.stdout:Install 1 Package 2023-08-28T02:03:53.246 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:03:53.247 INFO:teuthology.orchestra.run.smithi143.stdout:Total download size: 88 k 2023-08-28T02:03:53.247 INFO:teuthology.orchestra.run.smithi143.stdout:Installed size: 240 k 2023-08-28T02:03:53.247 INFO:teuthology.orchestra.run.smithi143.stdout:Downloading Packages: 2023-08-28T02:03:53.409 INFO:teuthology.orchestra.run.smithi143.stdout:rbd-fuse-16.2.13-565.g595355f4.el8.x86_64.rpm 545 kB/s | 88 kB 00:00 2023-08-28T02:03:53.410 INFO:teuthology.orchestra.run.smithi143.stdout:-------------------------------------------------------------------------------- 2023-08-28T02:03:53.410 INFO:teuthology.orchestra.run.smithi143.stdout:Total 541 kB/s | 88 kB 00:00 2023-08-28T02:03:53.410 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction check 2023-08-28T02:03:53.419 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction check succeeded. 2023-08-28T02:03:53.420 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction test 2023-08-28T02:03:53.473 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction test succeeded. 2023-08-28T02:03:53.476 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction 2023-08-28T02:03:53.596 INFO:teuthology.orchestra.run.smithi143.stdout: Preparing : 1/1 2023-08-28T02:03:53.736 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : rbd-fuse-2:16.2.13-565.g595355f4.el8.x86_64 1/1 2023-08-28T02:03:53.974 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: rbd-fuse-2:16.2.13-565.g595355f4.el8.x86_64 1/1 2023-08-28T02:03:54.516 INFO:teuthology.orchestra.run.smithi169.stdout:Installed products updated. 2023-08-28T02:03:54.698 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:03:54.698 INFO:teuthology.orchestra.run.smithi169.stdout:Installed: 2023-08-28T02:03:54.699 INFO:teuthology.orchestra.run.smithi169.stdout: rbd-fuse-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:03:54.699 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:03:54.699 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-08-28T02:03:54.872 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install rbd-mirror 2023-08-28T02:03:55.057 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : rbd-fuse-2:16.2.13-565.g595355f4.el8.x86_64 1/1 2023-08-28T02:03:55.057 INFO:teuthology.orchestra.run.smithi143.stdout:Last metadata expiration check: 0:03:26 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:03:55.227 INFO:teuthology.orchestra.run.smithi169.stdout:Updating Subscription Management repositories. 2023-08-28T02:03:56.059 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:03:27 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:03:57.374 INFO:teuthology.orchestra.run.smithi143.stdout:Installed products updated. 2023-08-28T02:03:57.451 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-08-28T02:03:57.452 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-08-28T02:03:57.452 INFO:teuthology.orchestra.run.smithi169.stdout: Package Arch Version Repo Size 2023-08-28T02:03:57.453 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-08-28T02:03:57.453 INFO:teuthology.orchestra.run.smithi169.stdout:Installing: 2023-08-28T02:03:57.453 INFO:teuthology.orchestra.run.smithi169.stdout: rbd-mirror x86_64 2:16.2.13-565.g595355f4.el8 ceph 3.7 M 2023-08-28T02:03:57.453 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:03:57.453 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction Summary 2023-08-28T02:03:57.453 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-08-28T02:03:57.454 INFO:teuthology.orchestra.run.smithi169.stdout:Install 1 Package 2023-08-28T02:03:57.454 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:03:57.454 INFO:teuthology.orchestra.run.smithi169.stdout:Total download size: 3.7 M 2023-08-28T02:03:57.454 INFO:teuthology.orchestra.run.smithi169.stdout:Installed size: 15 M 2023-08-28T02:03:57.454 INFO:teuthology.orchestra.run.smithi169.stdout:Downloading Packages: 2023-08-28T02:03:57.557 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:03:57.558 INFO:teuthology.orchestra.run.smithi143.stdout:Installed: 2023-08-28T02:03:57.558 INFO:teuthology.orchestra.run.smithi143.stdout: rbd-fuse-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:03:57.558 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:03:57.558 INFO:teuthology.orchestra.run.smithi143.stdout:Complete! 2023-08-28T02:03:57.695 DEBUG:teuthology.orchestra.run.smithi143:> sudo yum -y install rbd-mirror 2023-08-28T02:03:57.987 INFO:teuthology.orchestra.run.smithi169.stdout:rbd-mirror-16.2.13-565.g595355f4.el8.x86_64.rpm 6.9 MB/s | 3.7 MB 00:00 2023-08-28T02:03:57.988 INFO:teuthology.orchestra.run.smithi169.stdout:-------------------------------------------------------------------------------- 2023-08-28T02:03:57.988 INFO:teuthology.orchestra.run.smithi169.stdout:Total 6.9 MB/s | 3.7 MB 00:00 2023-08-28T02:03:57.988 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction check 2023-08-28T02:03:57.998 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction check succeeded. 2023-08-28T02:03:57.998 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction test 2023-08-28T02:03:58.048 INFO:teuthology.orchestra.run.smithi143.stdout:Updating Subscription Management repositories. 2023-08-28T02:03:58.066 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction test succeeded. 2023-08-28T02:03:58.070 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction 2023-08-28T02:03:58.515 INFO:teuthology.orchestra.run.smithi169.stdout: Preparing : 1/1 2023-08-28T02:03:58.670 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : rbd-mirror-2:16.2.13-565.g595355f4.el8.x86_64 1/1 2023-08-28T02:03:58.861 INFO:teuthology.orchestra.run.smithi143.stdout:Last metadata expiration check: 0:03:29 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:03:59.025 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: rbd-mirror-2:16.2.13-565.g595355f4.el8.x86_64 1/1 2023-08-28T02:04:00.109 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : rbd-mirror-2:16.2.13-565.g595355f4.el8.x86_64 1/1 2023-08-28T02:04:00.109 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:03:31 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:04:00.239 INFO:teuthology.orchestra.run.smithi143.stdout:Dependencies resolved. 2023-08-28T02:04:00.239 INFO:teuthology.orchestra.run.smithi143.stdout:================================================================================ 2023-08-28T02:04:00.240 INFO:teuthology.orchestra.run.smithi143.stdout: Package Arch Version Repo Size 2023-08-28T02:04:00.240 INFO:teuthology.orchestra.run.smithi143.stdout:================================================================================ 2023-08-28T02:04:00.240 INFO:teuthology.orchestra.run.smithi143.stdout:Installing: 2023-08-28T02:04:00.240 INFO:teuthology.orchestra.run.smithi143.stdout: rbd-mirror x86_64 2:16.2.13-565.g595355f4.el8 ceph 3.7 M 2023-08-28T02:04:00.240 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:04:00.241 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction Summary 2023-08-28T02:04:00.241 INFO:teuthology.orchestra.run.smithi143.stdout:================================================================================ 2023-08-28T02:04:00.241 INFO:teuthology.orchestra.run.smithi143.stdout:Install 1 Package 2023-08-28T02:04:00.242 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:04:00.242 INFO:teuthology.orchestra.run.smithi143.stdout:Total download size: 3.7 M 2023-08-28T02:04:00.242 INFO:teuthology.orchestra.run.smithi143.stdout:Installed size: 15 M 2023-08-28T02:04:00.242 INFO:teuthology.orchestra.run.smithi143.stdout:Downloading Packages: 2023-08-28T02:04:00.772 INFO:teuthology.orchestra.run.smithi143.stdout:rbd-mirror-16.2.13-565.g595355f4.el8.x86_64.rpm 6.9 MB/s | 3.7 MB 00:00 2023-08-28T02:04:00.772 INFO:teuthology.orchestra.run.smithi143.stdout:-------------------------------------------------------------------------------- 2023-08-28T02:04:00.772 INFO:teuthology.orchestra.run.smithi143.stdout:Total 6.9 MB/s | 3.7 MB 00:00 2023-08-28T02:04:00.773 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction check 2023-08-28T02:04:00.785 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction check succeeded. 2023-08-28T02:04:00.785 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction test 2023-08-28T02:04:00.854 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction test succeeded. 2023-08-28T02:04:00.856 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction 2023-08-28T02:04:01.295 INFO:teuthology.orchestra.run.smithi143.stdout: Preparing : 1/1 2023-08-28T02:04:01.437 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : rbd-mirror-2:16.2.13-565.g595355f4.el8.x86_64 1/1 2023-08-28T02:04:01.796 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: rbd-mirror-2:16.2.13-565.g595355f4.el8.x86_64 1/1 2023-08-28T02:04:02.448 INFO:teuthology.orchestra.run.smithi169.stdout:Installed products updated. 2023-08-28T02:04:02.631 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:04:02.632 INFO:teuthology.orchestra.run.smithi169.stdout:Installed: 2023-08-28T02:04:02.632 INFO:teuthology.orchestra.run.smithi169.stdout: rbd-mirror-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:04:02.632 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:04:02.632 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-08-28T02:04:02.806 DEBUG:teuthology.orchestra.run.smithi169:> sudo yum -y install rbd-nbd 2023-08-28T02:04:02.848 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : rbd-mirror-2:16.2.13-565.g595355f4.el8.x86_64 1/1 2023-08-28T02:04:02.943 INFO:teuthology.orchestra.run.smithi143.stdout:Last metadata expiration check: 0:03:33 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:04:03.162 INFO:teuthology.orchestra.run.smithi169.stdout:Updating Subscription Management repositories. 2023-08-28T02:04:03.960 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:03:34 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:04:05.284 INFO:teuthology.orchestra.run.smithi143.stdout:Installed products updated. 2023-08-28T02:04:05.321 INFO:teuthology.orchestra.run.smithi169.stdout:Dependencies resolved. 2023-08-28T02:04:05.321 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-08-28T02:04:05.321 INFO:teuthology.orchestra.run.smithi169.stdout: Package Architecture Version Repository Size 2023-08-28T02:04:05.321 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-08-28T02:04:05.322 INFO:teuthology.orchestra.run.smithi169.stdout:Installing: 2023-08-28T02:04:05.322 INFO:teuthology.orchestra.run.smithi169.stdout: rbd-nbd x86_64 2:16.2.13-565.g595355f4.el8 ceph 173 k 2023-08-28T02:04:05.322 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:04:05.322 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction Summary 2023-08-28T02:04:05.322 INFO:teuthology.orchestra.run.smithi169.stdout:================================================================================ 2023-08-28T02:04:05.322 INFO:teuthology.orchestra.run.smithi169.stdout:Install 1 Package 2023-08-28T02:04:05.322 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:04:05.327 INFO:teuthology.orchestra.run.smithi169.stdout:Total download size: 173 k 2023-08-28T02:04:05.327 INFO:teuthology.orchestra.run.smithi169.stdout:Installed size: 515 k 2023-08-28T02:04:05.327 INFO:teuthology.orchestra.run.smithi169.stdout:Downloading Packages: 2023-08-28T02:04:05.466 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:04:05.466 INFO:teuthology.orchestra.run.smithi143.stdout:Installed: 2023-08-28T02:04:05.467 INFO:teuthology.orchestra.run.smithi143.stdout: rbd-mirror-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:04:05.467 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:04:05.467 INFO:teuthology.orchestra.run.smithi143.stdout:Complete! 2023-08-28T02:04:05.613 DEBUG:teuthology.orchestra.run.smithi143:> sudo yum -y install rbd-nbd 2023-08-28T02:04:05.631 INFO:teuthology.orchestra.run.smithi169.stdout:rbd-nbd-16.2.13-565.g595355f4.el8.x86_64.rpm 575 kB/s | 173 kB 00:00 2023-08-28T02:04:05.631 INFO:teuthology.orchestra.run.smithi169.stdout:-------------------------------------------------------------------------------- 2023-08-28T02:04:05.632 INFO:teuthology.orchestra.run.smithi169.stdout:Total 572 kB/s | 173 kB 00:00 2023-08-28T02:04:05.632 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction check 2023-08-28T02:04:05.643 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction check succeeded. 2023-08-28T02:04:05.643 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction test 2023-08-28T02:04:05.697 INFO:teuthology.orchestra.run.smithi169.stdout:Transaction test succeeded. 2023-08-28T02:04:05.701 INFO:teuthology.orchestra.run.smithi169.stdout:Running transaction 2023-08-28T02:04:05.816 INFO:teuthology.orchestra.run.smithi169.stdout: Preparing : 1/1 2023-08-28T02:04:05.966 INFO:teuthology.orchestra.run.smithi169.stdout: Installing : rbd-nbd-2:16.2.13-565.g595355f4.el8.x86_64 1/1 2023-08-28T02:04:05.970 INFO:teuthology.orchestra.run.smithi143.stdout:Updating Subscription Management repositories. 2023-08-28T02:04:06.188 INFO:teuthology.orchestra.run.smithi169.stdout: Running scriptlet: rbd-nbd-2:16.2.13-565.g595355f4.el8.x86_64 1/1 2023-08-28T02:04:06.784 INFO:teuthology.orchestra.run.smithi143.stdout:Last metadata expiration check: 0:03:37 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:04:07.271 INFO:teuthology.orchestra.run.smithi169.stdout: Verifying : rbd-nbd-2:16.2.13-565.g595355f4.el8.x86_64 1/1 2023-08-28T02:04:07.271 INFO:teuthology.orchestra.run.smithi169.stdout:Last metadata expiration check: 0:03:38 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:04:08.150 INFO:teuthology.orchestra.run.smithi143.stdout:Dependencies resolved. 2023-08-28T02:04:08.151 INFO:teuthology.orchestra.run.smithi143.stdout:================================================================================ 2023-08-28T02:04:08.151 INFO:teuthology.orchestra.run.smithi143.stdout: Package Architecture Version Repository Size 2023-08-28T02:04:08.151 INFO:teuthology.orchestra.run.smithi143.stdout:================================================================================ 2023-08-28T02:04:08.151 INFO:teuthology.orchestra.run.smithi143.stdout:Installing: 2023-08-28T02:04:08.151 INFO:teuthology.orchestra.run.smithi143.stdout: rbd-nbd x86_64 2:16.2.13-565.g595355f4.el8 ceph 173 k 2023-08-28T02:04:08.151 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:04:08.151 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction Summary 2023-08-28T02:04:08.152 INFO:teuthology.orchestra.run.smithi143.stdout:================================================================================ 2023-08-28T02:04:08.152 INFO:teuthology.orchestra.run.smithi143.stdout:Install 1 Package 2023-08-28T02:04:08.152 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:04:08.152 INFO:teuthology.orchestra.run.smithi143.stdout:Total download size: 173 k 2023-08-28T02:04:08.152 INFO:teuthology.orchestra.run.smithi143.stdout:Installed size: 515 k 2023-08-28T02:04:08.152 INFO:teuthology.orchestra.run.smithi143.stdout:Downloading Packages: 2023-08-28T02:04:08.501 INFO:teuthology.orchestra.run.smithi143.stdout:rbd-nbd-16.2.13-565.g595355f4.el8.x86_64.rpm 497 kB/s | 173 kB 00:00 2023-08-28T02:04:08.502 INFO:teuthology.orchestra.run.smithi143.stdout:-------------------------------------------------------------------------------- 2023-08-28T02:04:08.502 INFO:teuthology.orchestra.run.smithi143.stdout:Total 495 kB/s | 173 kB 00:00 2023-08-28T02:04:08.502 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction check 2023-08-28T02:04:08.513 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction check succeeded. 2023-08-28T02:04:08.513 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction test 2023-08-28T02:04:08.567 INFO:teuthology.orchestra.run.smithi143.stdout:Transaction test succeeded. 2023-08-28T02:04:08.570 INFO:teuthology.orchestra.run.smithi143.stdout:Running transaction 2023-08-28T02:04:08.687 INFO:teuthology.orchestra.run.smithi143.stdout: Preparing : 1/1 2023-08-28T02:04:08.841 INFO:teuthology.orchestra.run.smithi143.stdout: Installing : rbd-nbd-2:16.2.13-565.g595355f4.el8.x86_64 1/1 2023-08-28T02:04:09.067 INFO:teuthology.orchestra.run.smithi143.stdout: Running scriptlet: rbd-nbd-2:16.2.13-565.g595355f4.el8.x86_64 1/1 2023-08-28T02:04:09.607 INFO:teuthology.orchestra.run.smithi169.stdout:Installed products updated. 2023-08-28T02:04:09.789 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:04:09.790 INFO:teuthology.orchestra.run.smithi169.stdout:Installed: 2023-08-28T02:04:09.790 INFO:teuthology.orchestra.run.smithi169.stdout: rbd-nbd-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:04:09.790 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:04:09.790 INFO:teuthology.orchestra.run.smithi169.stdout:Complete! 2023-08-28T02:04:09.923 DEBUG:teuthology.parallel:result is None 2023-08-28T02:04:10.118 INFO:teuthology.orchestra.run.smithi143.stdout: Verifying : rbd-nbd-2:16.2.13-565.g595355f4.el8.x86_64 1/1 2023-08-28T02:04:10.118 INFO:teuthology.orchestra.run.smithi143.stdout:Last metadata expiration check: 0:03:41 ago on Mon 28 Aug 2023 02:00:29 AM UTC. 2023-08-28T02:04:12.536 INFO:teuthology.orchestra.run.smithi143.stdout:Installed products updated. 2023-08-28T02:04:12.721 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:04:12.722 INFO:teuthology.orchestra.run.smithi143.stdout:Installed: 2023-08-28T02:04:12.722 INFO:teuthology.orchestra.run.smithi143.stdout: rbd-nbd-2:16.2.13-565.g595355f4.el8.x86_64 2023-08-28T02:04:12.722 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:04:12.722 INFO:teuthology.orchestra.run.smithi143.stdout:Complete! 2023-08-28T02:04:12.855 DEBUG:teuthology.parallel:result is None 2023-08-28T02:04:12.855 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-08-28T02:04:12.855 INFO:teuthology.packaging:ref: None 2023-08-28T02:04:12.855 INFO:teuthology.packaging:tag: None 2023-08-28T02:04:12.855 INFO:teuthology.packaging:branch: pacific 2023-08-28T02:04:12.855 INFO:teuthology.packaging:sha1: 9cf208207dfe23ffccb33308b22e51570cb45e51 2023-08-28T02:04:12.855 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=pacific 2023-08-28T02:04:13.097 DEBUG:teuthology.orchestra.run.smithi143:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-08-28T02:04:13.119 INFO:teuthology.orchestra.run.smithi143.stdout:16.2.13-565.g595355f4.el8 2023-08-28T02:04:13.120 INFO:teuthology.packaging:The installed version of ceph is 16.2.13-565.g595355f4.el8 2023-08-28T02:04:13.120 INFO:teuthology.task.install:The correct ceph version 16.2.13-565.g595355f4 is installed. 2023-08-28T02:04:13.120 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-08-28T02:04:13.120 INFO:teuthology.packaging:ref: None 2023-08-28T02:04:13.120 INFO:teuthology.packaging:tag: None 2023-08-28T02:04:13.120 INFO:teuthology.packaging:branch: pacific 2023-08-28T02:04:13.121 INFO:teuthology.packaging:sha1: 9cf208207dfe23ffccb33308b22e51570cb45e51 2023-08-28T02:04:13.121 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=pacific 2023-08-28T02:04:13.445 DEBUG:teuthology.orchestra.run.smithi169:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-08-28T02:04:13.472 INFO:teuthology.orchestra.run.smithi169.stdout:16.2.13-565.g595355f4.el8 2023-08-28T02:04:13.473 INFO:teuthology.packaging:The installed version of ceph is 16.2.13-565.g595355f4.el8 2023-08-28T02:04:13.473 INFO:teuthology.task.install:The correct ceph version 16.2.13-565.g595355f4 is installed. 2023-08-28T02:04:13.473 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2023-08-28T02:04:13.473 DEBUG:teuthology.orchestra.run.smithi143:> set -ex 2023-08-28T02:04:13.473 DEBUG:teuthology.orchestra.run.smithi143:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-08-28T02:04:13.502 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2023-08-28T02:04:13.502 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-08-28T02:04:13.544 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2023-08-28T02:04:13.545 DEBUG:teuthology.orchestra.run.smithi143:> set -ex 2023-08-28T02:04:13.545 DEBUG:teuthology.orchestra.run.smithi143:> sudo dd of=/usr/bin/daemon-helper 2023-08-28T02:04:13.571 DEBUG:teuthology.orchestra.run.smithi143:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-08-28T02:04:13.638 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2023-08-28T02:04:13.638 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd of=/usr/bin/daemon-helper 2023-08-28T02:04:13.669 DEBUG:teuthology.orchestra.run.smithi169:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-08-28T02:04:13.738 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2023-08-28T02:04:13.738 DEBUG:teuthology.orchestra.run.smithi143:> set -ex 2023-08-28T02:04:13.738 DEBUG:teuthology.orchestra.run.smithi143:> sudo dd of=/usr/bin/adjust-ulimits 2023-08-28T02:04:13.765 DEBUG:teuthology.orchestra.run.smithi143:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-08-28T02:04:13.831 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2023-08-28T02:04:13.831 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd of=/usr/bin/adjust-ulimits 2023-08-28T02:04:13.860 DEBUG:teuthology.orchestra.run.smithi169:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-08-28T02:04:13.928 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2023-08-28T02:04:13.929 DEBUG:teuthology.orchestra.run.smithi143:> set -ex 2023-08-28T02:04:13.929 DEBUG:teuthology.orchestra.run.smithi143:> sudo dd of=/usr/bin/stdin-killer 2023-08-28T02:04:13.955 DEBUG:teuthology.orchestra.run.smithi143:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-08-28T02:04:14.021 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2023-08-28T02:04:14.021 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd of=/usr/bin/stdin-killer 2023-08-28T02:04:14.049 DEBUG:teuthology.orchestra.run.smithi169:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-08-28T02:04:14.117 INFO:teuthology.run_tasks:Running task cephadm... 2023-08-28T02:04:14.188 INFO:tasks.cephadm:Config: {'cephadm_branch': 'pacific', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:pacific', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': '9cf208207dfe23ffccb33308b22e51570cb45e51'} 2023-08-28T02:04:14.189 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:pacific 2023-08-28T02:04:14.189 INFO:tasks.cephadm:Cluster fsid is 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 2023-08-28T02:04:14.189 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2023-08-28T02:04:14.189 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.143', 'mon.c': '[v2:172.21.15.143:3301,v1:172.21.15.143:6790]', 'mon.b': '172.21.15.169'} 2023-08-28T02:04:14.190 INFO:tasks.cephadm:First mon is mon.a on smithi143 2023-08-28T02:04:14.190 INFO:tasks.cephadm:First mgr is y 2023-08-28T02:04:14.190 INFO:tasks.cephadm:Normalizing hostnames... 2023-08-28T02:04:14.190 DEBUG:teuthology.orchestra.run.smithi143:> sudo hostname $(hostname -s) 2023-08-28T02:04:14.216 DEBUG:teuthology.orchestra.run.smithi169:> sudo hostname $(hostname -s) 2023-08-28T02:04:14.244 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref pacific)... 2023-08-28T02:04:14.245 DEBUG:teuthology.orchestra.run.smithi143:> curl --silent https://raw.githubusercontent.com/ceph/ceph/pacific/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-08-28T02:04:14.571 INFO:teuthology.orchestra.run.smithi143.stdout:-rw-rw-r--. 1 ubuntu ubuntu 350873 Aug 28 02:04 /home/ubuntu/cephtest/cephadm 2023-08-28T02:04:14.571 DEBUG:teuthology.orchestra.run.smithi169:> curl --silent https://raw.githubusercontent.com/ceph/ceph/pacific/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-08-28T02:04:14.704 INFO:teuthology.orchestra.run.smithi169.stdout:-rw-rw-r--. 1 ubuntu ubuntu 350873 Aug 28 02:04 /home/ubuntu/cephtest/cephadm 2023-08-28T02:04:14.705 DEBUG:teuthology.orchestra.run.smithi143:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-08-28T02:04:14.723 DEBUG:teuthology.orchestra.run.smithi169:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-08-28T02:04:14.753 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:pacific on all hosts... 2023-08-28T02:04:14.753 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific pull 2023-08-28T02:04:14.765 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific pull 2023-08-28T02:04:15.938 INFO:teuthology.orchestra.run.smithi143.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:pacific... 2023-08-28T02:04:15.994 INFO:teuthology.orchestra.run.smithi169.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:pacific... 2023-08-28T02:05:11.149 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:05:11.149 INFO:teuthology.orchestra.run.smithi143.stdout: "ceph_version": "ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)", 2023-08-28T02:05:11.149 INFO:teuthology.orchestra.run.smithi143.stdout: "image_id": "4099d2e00414f822b7771da6dc5d588eeda3dc16e3b39be700f46017c3047641", 2023-08-28T02:05:11.149 INFO:teuthology.orchestra.run.smithi143.stdout: "repo_digests": [ 2023-08-28T02:05:11.149 INFO:teuthology.orchestra.run.smithi143.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:e2028333a2006b532dd5e4caab422f750a165963636ee26acf78c4fa6593c513" 2023-08-28T02:05:11.149 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:05:11.150 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:05:12.295 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:05:12.296 INFO:teuthology.orchestra.run.smithi169.stdout: "ceph_version": "ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)", 2023-08-28T02:05:12.296 INFO:teuthology.orchestra.run.smithi169.stdout: "image_id": "4099d2e00414f822b7771da6dc5d588eeda3dc16e3b39be700f46017c3047641", 2023-08-28T02:05:12.296 INFO:teuthology.orchestra.run.smithi169.stdout: "repo_digests": [ 2023-08-28T02:05:12.296 INFO:teuthology.orchestra.run.smithi169.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:e2028333a2006b532dd5e4caab422f750a165963636ee26acf78c4fa6593c513" 2023-08-28T02:05:12.296 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:05:12.296 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:05:12.323 DEBUG:teuthology.orchestra.run.smithi143:> sudo mkdir -p /etc/ceph 2023-08-28T02:05:12.360 DEBUG:teuthology.orchestra.run.smithi169:> sudo mkdir -p /etc/ceph 2023-08-28T02:05:12.393 DEBUG:teuthology.orchestra.run.smithi143:> sudo chmod 777 /etc/ceph 2023-08-28T02:05:12.433 DEBUG:teuthology.orchestra.run.smithi169:> sudo chmod 777 /etc/ceph 2023-08-28T02:05:12.462 INFO:tasks.cephadm:Writing seed config... 2023-08-28T02:05:12.463 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2023-08-28T02:05:12.463 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2023-08-28T02:05:12.463 INFO:tasks.cephadm: override: [osd] debug ms = 1 2023-08-28T02:05:12.463 INFO:tasks.cephadm: override: [osd] debug osd = 20 2023-08-28T02:05:12.463 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2023-08-28T02:05:12.463 INFO:tasks.cephadm: override: [global] mon election default strategy = 3 2023-08-28T02:05:12.463 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2023-08-28T02:05:12.464 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2023-08-28T02:05:12.464 INFO:tasks.cephadm: override: [mon] debug mon = 20 2023-08-28T02:05:12.464 INFO:tasks.cephadm: override: [mon] debug ms = 1 2023-08-28T02:05:12.464 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2023-08-28T02:05:12.464 DEBUG:teuthology.orchestra.run.smithi143:> set -ex 2023-08-28T02:05:12.464 DEBUG:teuthology.orchestra.run.smithi143:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2023-08-28T02:05:12.492 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 mon election default strategy = 3 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2023-08-28T02:05:12.492 DEBUG:teuthology.orchestra.run.smithi143:mon.a> sudo journalctl -f -n 0 -u ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@mon.a.service 2023-08-28T02:05:12.534 DEBUG:teuthology.orchestra.run.smithi143:mgr.y> sudo journalctl -f -n 0 -u ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@mgr.y.service 2023-08-28T02:05:12.564 INFO:journalctl@ceph.mon.a.smithi143.stdout:-- Logs begin at Mon 2023-08-28 01:49:41 UTC. -- 2023-08-28T02:05:12.565 INFO:tasks.cephadm:Bootstrapping... 2023-08-28T02:05:12.565 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific -v bootstrap --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.143 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2023-08-28T02:05:12.593 INFO:journalctl@ceph.mgr.y.smithi143.stdout:-- Logs begin at Mon 2023-08-28 01:49:41 UTC. -- 2023-08-28T02:05:12.770 INFO:teuthology.orchestra.run.smithi143.stderr:-------------------------------------------------------------------------------- 2023-08-28T02:05:12.770 INFO:teuthology.orchestra.run.smithi143.stderr:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:pacific', '-v', 'bootstrap', '--fsid', '2fcb0bfa-4547-11ee-817c-1b7a5f1ba315', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.143', '--skip-admin-label'] 2023-08-28T02:05:12.815 INFO:teuthology.orchestra.run.smithi143.stderr:/bin/podman: 3.0.1 2023-08-28T02:05:12.828 INFO:teuthology.orchestra.run.smithi143.stderr:Verifying podman|docker is present... 2023-08-28T02:05:12.871 INFO:teuthology.orchestra.run.smithi143.stderr:/bin/podman: 3.0.1 2023-08-28T02:05:12.885 INFO:teuthology.orchestra.run.smithi143.stderr:Verifying lvm2 is present... 2023-08-28T02:05:12.885 INFO:teuthology.orchestra.run.smithi143.stderr:Verifying time synchronization is in place... 2023-08-28T02:05:12.892 INFO:teuthology.orchestra.run.smithi143.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2023-08-28T02:05:12.898 INFO:teuthology.orchestra.run.smithi143.stderr:systemctl: inactive 2023-08-28T02:05:12.903 INFO:teuthology.orchestra.run.smithi143.stderr:systemctl: enabled 2023-08-28T02:05:12.909 INFO:teuthology.orchestra.run.smithi143.stderr:systemctl: active 2023-08-28T02:05:12.910 INFO:teuthology.orchestra.run.smithi143.stderr:Unit chronyd.service is enabled and running 2023-08-28T02:05:12.910 INFO:teuthology.orchestra.run.smithi143.stderr:Repeating the final host check... 2023-08-28T02:05:12.952 INFO:teuthology.orchestra.run.smithi143.stderr:/bin/podman: 3.0.1 2023-08-28T02:05:12.966 INFO:teuthology.orchestra.run.smithi143.stderr:podman (/bin/podman) version 3.0.1 is present 2023-08-28T02:05:12.966 INFO:teuthology.orchestra.run.smithi143.stderr:systemctl is present 2023-08-28T02:05:12.967 INFO:teuthology.orchestra.run.smithi143.stderr:lvcreate is present 2023-08-28T02:05:12.972 INFO:teuthology.orchestra.run.smithi143.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2023-08-28T02:05:12.977 INFO:teuthology.orchestra.run.smithi143.stderr:systemctl: inactive 2023-08-28T02:05:12.983 INFO:teuthology.orchestra.run.smithi143.stderr:systemctl: enabled 2023-08-28T02:05:12.988 INFO:teuthology.orchestra.run.smithi143.stderr:systemctl: active 2023-08-28T02:05:12.989 INFO:teuthology.orchestra.run.smithi143.stderr:Unit chronyd.service is enabled and running 2023-08-28T02:05:12.989 INFO:teuthology.orchestra.run.smithi143.stderr:Host looks OK 2023-08-28T02:05:12.990 INFO:teuthology.orchestra.run.smithi143.stderr:Cluster fsid: 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 2023-08-28T02:05:12.990 INFO:teuthology.orchestra.run.smithi143.stderr:Acquiring lock 140473025108456 on /run/cephadm/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315.lock 2023-08-28T02:05:12.990 INFO:teuthology.orchestra.run.smithi143.stderr:Lock 140473025108456 acquired on /run/cephadm/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315.lock 2023-08-28T02:05:12.990 INFO:teuthology.orchestra.run.smithi143.stderr:Verifying IP 172.21.15.143 port 3300 ... 2023-08-28T02:05:12.990 INFO:teuthology.orchestra.run.smithi143.stderr:Verifying IP 172.21.15.143 port 6789 ... 2023-08-28T02:05:12.991 INFO:teuthology.orchestra.run.smithi143.stderr:Base mon IP(s) is [172.21.15.143:3300, 172.21.15.143:6789], mon addrv is [v2:172.21.15.143:3300,v1:172.21.15.143:6789] 2023-08-28T02:05:12.993 INFO:teuthology.orchestra.run.smithi143.stderr:/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp metric 100 2023-08-28T02:05:12.994 INFO:teuthology.orchestra.run.smithi143.stderr:/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.143 metric 100 2023-08-28T02:05:12.996 INFO:teuthology.orchestra.run.smithi143.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2023-08-28T02:05:12.996 INFO:teuthology.orchestra.run.smithi143.stderr:/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 1024 pref medium 2023-08-28T02:05:12.996 INFO:teuthology.orchestra.run.smithi143.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 pref medium 2023-08-28T02:05:12.999 INFO:teuthology.orchestra.run.smithi143.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2023-08-28T02:05:12.999 INFO:teuthology.orchestra.run.smithi143.stderr:/sbin/ip: inet6 ::1/128 scope host 2023-08-28T02:05:12.999 INFO:teuthology.orchestra.run.smithi143.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2023-08-28T02:05:12.999 INFO:teuthology.orchestra.run.smithi143.stderr:/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2023-08-28T02:05:12.999 INFO:teuthology.orchestra.run.smithi143.stderr:/sbin/ip: inet6 fe80::1571:f5b1:84c8:a48b/64 scope link noprefixroute 2023-08-28T02:05:12.999 INFO:teuthology.orchestra.run.smithi143.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2023-08-28T02:05:13.000 INFO:teuthology.orchestra.run.smithi143.stderr:Mon IP `172.21.15.143` is in CIDR network `172.21.0.0/20` 2023-08-28T02:05:13.000 INFO:teuthology.orchestra.run.smithi143.stderr:Mon IP `172.21.15.143` is in CIDR network `172.21.0.0/20` 2023-08-28T02:05:13.000 INFO:teuthology.orchestra.run.smithi143.stderr:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2023-08-28T02:05:13.000 INFO:teuthology.orchestra.run.smithi143.stderr:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2023-08-28T02:05:13.001 INFO:teuthology.orchestra.run.smithi143.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:pacific... 2023-08-28T02:05:13.072 INFO:teuthology.orchestra.run.smithi143.stderr:/bin/podman: Trying to pull quay.ceph.io/ceph-ci/ceph:pacific... 2023-08-28T02:05:13.379 INFO:teuthology.orchestra.run.smithi143.stderr:/bin/podman: Getting image source signatures 2023-08-28T02:05:13.380 INFO:teuthology.orchestra.run.smithi143.stderr:/bin/podman: Copying blob sha256:62d5d777e90169e7b1eda90a7a99e4d6a2f4cd41b446d804e196cd4fde26e21c 2023-08-28T02:05:13.380 INFO:teuthology.orchestra.run.smithi143.stderr:/bin/podman: Copying blob sha256:6022c611d474f2b306ddd1edfc8360123ece5266119096f25fde843d7967e636 2023-08-28T02:05:13.380 INFO:teuthology.orchestra.run.smithi143.stderr:/bin/podman: Copying config sha256:4099d2e00414f822b7771da6dc5d588eeda3dc16e3b39be700f46017c3047641 2023-08-28T02:05:13.441 INFO:teuthology.orchestra.run.smithi143.stderr:/bin/podman: Writing manifest to image destination 2023-08-28T02:05:13.441 INFO:teuthology.orchestra.run.smithi143.stderr:/bin/podman: Storing signatures 2023-08-28T02:05:13.603 INFO:teuthology.orchestra.run.smithi143.stderr:/bin/podman: 4099d2e00414f822b7771da6dc5d588eeda3dc16e3b39be700f46017c3047641 2023-08-28T02:05:14.036 INFO:teuthology.orchestra.run.smithi143.stderr:ceph: ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable) 2023-08-28T02:05:14.494 INFO:teuthology.orchestra.run.smithi143.stderr:Ceph version: ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable) 2023-08-28T02:05:14.494 INFO:teuthology.orchestra.run.smithi143.stderr:Extracting ceph user uid/gid from container image... 2023-08-28T02:05:14.825 INFO:teuthology.orchestra.run.smithi143.stderr:stat: 167 167 2023-08-28T02:05:15.285 INFO:teuthology.orchestra.run.smithi143.stderr:Creating initial keys... 2023-08-28T02:05:15.567 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-authtool: AQDbAOxkcBR1IRAA+wEBIyR7LmSqsqNeHS2omQ== 2023-08-28T02:05:16.367 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-authtool: AQDcAOxkuMGBFRAAmuEwYIkkx9upQxV6Tdi8fA== 2023-08-28T02:05:17.075 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-authtool: AQDdAOxkOlUYBBAAkJIcVg21Kv4B4vZqntJg7Q== 2023-08-28T02:05:17.488 INFO:teuthology.orchestra.run.smithi143.stderr:Creating initial monmap... 2023-08-28T02:05:17.825 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2023-08-28T02:05:17.825 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 2023-08-28T02:05:17.825 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-08-28T02:05:18.220 INFO:teuthology.orchestra.run.smithi143.stderr:monmaptool for a [v2:172.21.15.143:3300,v1:172.21.15.143:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2023-08-28T02:05:18.220 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/monmaptool: set fsid to 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 2023-08-28T02:05:18.220 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-08-28T02:05:18.220 INFO:teuthology.orchestra.run.smithi143.stderr: 2023-08-28T02:05:18.220 INFO:teuthology.orchestra.run.smithi143.stderr:Creating mon... 2023-08-28T02:05:18.575 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.547+0000 7f267a1c0880 0 set uid:gid to 167:167 (ceph:ceph) 2023-08-28T02:05:18.575 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.547+0000 7f267a1c0880 1 imported monmap: 2023-08-28T02:05:18.575 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: epoch 0 2023-08-28T02:05:18.575 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 2023-08-28T02:05:18.575 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: last_changed 2023-08-28T02:05:17.823964+0000 2023-08-28T02:05:18.575 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: created 2023-08-28T02:05:17.823964+0000 2023-08-28T02:05:18.575 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: min_mon_release 0 (unknown) 2023-08-28T02:05:18.575 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: election_strategy: 1 2023-08-28T02:05:18.576 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: 0: [v2:172.21.15.143:3300/0,v1:172.21.15.143:6789/0] mon.a 2023-08-28T02:05:18.576 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: 2023-08-28T02:05:18.576 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.547+0000 7f267a1c0880 0 /usr/bin/ceph-mon: set fsid to 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 2023-08-28T02:05:18.576 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: RocksDB version: 6.8.1 2023-08-28T02:05:18.576 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: 2023-08-28T02:05:18.576 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-08-28T02:05:18.576 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Compile date Aug 22 2023 2023-08-28T02:05:18.576 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: DB SUMMARY 2023-08-28T02:05:18.576 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: 2023-08-28T02:05:18.576 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 0, files: 2023-08-28T02:05:18.576 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: 2023-08-28T02:05:18.576 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 2023-08-28T02:05:18.577 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: 2023-08-28T02:05:18.577 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.error_if_exists: 0 2023-08-28T02:05:18.577 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.create_if_missing: 1 2023-08-28T02:05:18.577 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.paranoid_checks: 1 2023-08-28T02:05:18.577 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.env: 0x55f4d5342080 2023-08-28T02:05:18.577 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.fs: Posix File System 2023-08-28T02:05:18.577 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.info_log: 0x55f4d7229aa0 2023-08-28T02:05:18.577 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.max_file_opening_threads: 16 2023-08-28T02:05:18.577 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.statistics: (nil) 2023-08-28T02:05:18.577 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.use_fsync: 0 2023-08-28T02:05:18.577 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.max_log_file_size: 0 2023-08-28T02:05:18.577 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-08-28T02:05:18.577 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.log_file_time_to_roll: 0 2023-08-28T02:05:18.578 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.keep_log_file_num: 1000 2023-08-28T02:05:18.578 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.recycle_log_file_num: 0 2023-08-28T02:05:18.578 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.allow_fallocate: 1 2023-08-28T02:05:18.578 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.allow_mmap_reads: 0 2023-08-28T02:05:18.578 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.allow_mmap_writes: 0 2023-08-28T02:05:18.578 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.use_direct_reads: 0 2023-08-28T02:05:18.578 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-08-28T02:05:18.578 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.create_missing_column_families: 0 2023-08-28T02:05:18.578 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.db_log_dir: 2023-08-28T02:05:18.578 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-08-28T02:05:18.578 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.table_cache_numshardbits: 6 2023-08-28T02:05:18.578 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.max_subcompactions: 1 2023-08-28T02:05:18.579 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.max_background_flushes: -1 2023-08-28T02:05:18.579 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-08-28T02:05:18.579 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-08-28T02:05:18.579 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-08-28T02:05:18.579 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-08-28T02:05:18.579 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-08-28T02:05:18.579 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.advise_random_on_open: 1 2023-08-28T02:05:18.579 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.db_write_buffer_size: 0 2023-08-28T02:05:18.579 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.write_buffer_manager: 0x55f4d7231890 2023-08-28T02:05:18.579 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-08-28T02:05:18.579 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-08-28T02:05:18.579 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-08-28T02:05:18.580 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.use_adaptive_mutex: 0 2023-08-28T02:05:18.580 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.rate_limiter: (nil) 2023-08-28T02:05:18.580 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-08-28T02:05:18.580 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.wal_recovery_mode: 2 2023-08-28T02:05:18.580 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.enable_thread_tracking: 0 2023-08-28T02:05:18.580 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.enable_pipelined_write: 0 2023-08-28T02:05:18.580 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.unordered_write: 0 2023-08-28T02:05:18.580 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-08-28T02:05:18.580 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-08-28T02:05:18.580 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-08-28T02:05:18.580 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-08-28T02:05:18.580 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.row_cache: None 2023-08-28T02:05:18.580 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.wal_filter: None 2023-08-28T02:05:18.581 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-08-28T02:05:18.581 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.allow_ingest_behind: 0 2023-08-28T02:05:18.581 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.preserve_deletes: 0 2023-08-28T02:05:18.581 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.two_write_queues: 0 2023-08-28T02:05:18.581 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.manual_wal_flush: 0 2023-08-28T02:05:18.581 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.atomic_flush: 0 2023-08-28T02:05:18.581 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-08-28T02:05:18.581 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.persist_stats_to_disk: 0 2023-08-28T02:05:18.581 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-08-28T02:05:18.581 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.log_readahead_size: 0 2023-08-28T02:05:18.581 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-08-28T02:05:18.581 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.max_background_jobs: 2 2023-08-28T02:05:18.581 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.max_background_compactions: -1 2023-08-28T02:05:18.582 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-08-28T02:05:18.582 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-08-28T02:05:18.582 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.delayed_write_rate : 16777216 2023-08-28T02:05:18.582 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.max_total_wal_size: 0 2023-08-28T02:05:18.582 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-08-28T02:05:18.582 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.stats_dump_period_sec: 600 2023-08-28T02:05:18.582 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.stats_persist_period_sec: 600 2023-08-28T02:05:18.582 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-08-28T02:05:18.583 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.max_open_files: -1 2023-08-28T02:05:18.583 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.bytes_per_sync: 0 2023-08-28T02:05:18.583 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-08-28T02:05:18.583 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-08-28T02:05:18.583 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.compaction_readahead_size: 0 2023-08-28T02:05:18.583 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Compression algorithms supported: 2023-08-28T02:05:18.583 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-08-28T02:05:18.583 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: kZSTD supported: 0 2023-08-28T02:05:18.583 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: kXpressCompression supported: 0 2023-08-28T02:05:18.583 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: kLZ4HCCompression supported: 1 2023-08-28T02:05:18.583 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: kLZ4Compression supported: 1 2023-08-28T02:05:18.584 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: kBZip2Compression supported: 0 2023-08-28T02:05:18.584 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: kZlibCompression supported: 1 2023-08-28T02:05:18.584 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: kSnappyCompression supported: 1 2023-08-28T02:05:18.584 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-08-28T02:05:18.584 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: [db_impl/db_impl_open.cc:273] Creating manifest 1 2023-08-28T02:05:18.584 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: 2023-08-28T02:05:18.584 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 2023-08-28T02:05:18.584 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: 2023-08-28T02:05:18.584 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-08-28T02:05:18.584 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: 2023-08-28T02:05:18.584 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-08-28T02:05:18.584 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.merge_operator: 2023-08-28T02:05:18.585 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.compaction_filter: None 2023-08-28T02:05:18.585 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.compaction_filter_factory: None 2023-08-28T02:05:18.585 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-08-28T02:05:18.585 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.table_factory: BlockBasedTable 2023-08-28T02:05:18.585 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55f4d713bd20) 2023-08-28T02:05:18.585 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: cache_index_and_filter_blocks: 1 2023-08-28T02:05:18.585 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: cache_index_and_filter_blocks_with_high_priority: 0 2023-08-28T02:05:18.585 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: pin_l0_filter_and_index_blocks_in_cache: 0 2023-08-28T02:05:18.585 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: pin_top_level_index_and_filter: 1 2023-08-28T02:05:18.585 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: index_type: 0 2023-08-28T02:05:18.585 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: data_block_index_type: 0 2023-08-28T02:05:18.586 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: index_shortening: 1 2023-08-28T02:05:18.586 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: data_block_hash_table_util_ratio: 0.750000 2023-08-28T02:05:18.586 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: hash_index_allow_collision: 1 2023-08-28T02:05:18.586 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: checksum: 1 2023-08-28T02:05:18.586 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: no_block_cache: 0 2023-08-28T02:05:18.586 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: block_cache: 0x55f4d7174d10 2023-08-28T02:05:18.586 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: block_cache_name: BinnedLRUCache 2023-08-28T02:05:18.586 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: block_cache_options: 2023-08-28T02:05:18.586 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: capacity : 536870912 2023-08-28T02:05:18.586 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: num_shard_bits : 4 2023-08-28T02:05:18.586 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: strict_capacity_limit : 0 2023-08-28T02:05:18.586 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: high_pri_pool_ratio: 0.000 2023-08-28T02:05:18.587 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: block_cache_compressed: (nil) 2023-08-28T02:05:18.587 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: persistent_cache: (nil) 2023-08-28T02:05:18.587 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: block_size: 4096 2023-08-28T02:05:18.587 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: block_size_deviation: 10 2023-08-28T02:05:18.587 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: block_restart_interval: 16 2023-08-28T02:05:18.587 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: index_block_restart_interval: 1 2023-08-28T02:05:18.587 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: metadata_block_size: 4096 2023-08-28T02:05:18.587 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: partition_filters: 0 2023-08-28T02:05:18.587 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: use_delta_encoding: 1 2023-08-28T02:05:18.587 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: filter_policy: rocksdb.BuiltinBloomFilter 2023-08-28T02:05:18.587 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: whole_key_filtering: 1 2023-08-28T02:05:18.587 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: verify_compression: 0 2023-08-28T02:05:18.588 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: read_amp_bytes_per_bit: 0 2023-08-28T02:05:18.588 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: format_version: 2 2023-08-28T02:05:18.588 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: enable_index_compression: 1 2023-08-28T02:05:18.588 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: block_align: 0 2023-08-28T02:05:18.588 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: 2023-08-28T02:05:18.588 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.write_buffer_size: 33554432 2023-08-28T02:05:18.588 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.max_write_buffer_number: 2 2023-08-28T02:05:18.588 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.compression: NoCompression 2023-08-28T02:05:18.588 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.bottommost_compression: Disabled 2023-08-28T02:05:18.588 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.prefix_extractor: nullptr 2023-08-28T02:05:18.588 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-08-28T02:05:18.588 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.num_levels: 7 2023-08-28T02:05:18.589 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-08-28T02:05:18.589 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-08-28T02:05:18.589 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-08-28T02:05:18.589 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-08-28T02:05:18.589 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-08-28T02:05:18.589 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-08-28T02:05:18.589 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-08-28T02:05:18.589 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-08-28T02:05:18.589 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-08-28T02:05:18.590 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.compression_opts.window_bits: -14 2023-08-28T02:05:18.590 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.compression_opts.level: 32767 2023-08-28T02:05:18.590 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.compression_opts.strategy: 0 2023-08-28T02:05:18.590 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-08-28T02:05:18.590 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-08-28T02:05:18.590 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.compression_opts.enabled: false 2023-08-28T02:05:18.590 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-08-28T02:05:18.590 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-08-28T02:05:18.590 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-08-28T02:05:18.590 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.target_file_size_base: 67108864 2023-08-28T02:05:18.590 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.target_file_size_multiplier: 1 2023-08-28T02:05:18.590 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-08-28T02:05:18.591 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-08-28T02:05:18.591 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-08-28T02:05:18.591 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-08-28T02:05:18.591 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-08-28T02:05:18.591 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-08-28T02:05:18.591 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-08-28T02:05:18.591 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-08-28T02:05:18.591 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-08-28T02:05:18.591 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-08-28T02:05:18.591 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-08-28T02:05:18.591 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-08-28T02:05:18.591 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.arena_block_size: 4194304 2023-08-28T02:05:18.592 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-08-28T02:05:18.592 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-08-28T02:05:18.592 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-08-28T02:05:18.592 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.disable_auto_compactions: 0 2023-08-28T02:05:18.592 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-08-28T02:05:18.592 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-08-28T02:05:18.592 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-08-28T02:05:18.592 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-08-28T02:05:18.592 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-08-28T02:05:18.592 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-08-28T02:05:18.592 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-08-28T02:05:18.593 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-08-28T02:05:18.593 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-08-28T02:05:18.593 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-08-28T02:05:18.593 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.table_properties_collectors: 2023-08-28T02:05:18.593 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.inplace_update_support: 0 2023-08-28T02:05:18.593 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-08-28T02:05:18.593 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-08-28T02:05:18.593 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-08-28T02:05:18.593 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.memtable_huge_page_size: 0 2023-08-28T02:05:18.593 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.bloom_locality: 0 2023-08-28T02:05:18.593 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.max_successive_merges: 0 2023-08-28T02:05:18.593 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-08-28T02:05:18.594 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.paranoid_file_checks: 0 2023-08-28T02:05:18.594 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.force_consistency_checks: 0 2023-08-28T02:05:18.594 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.report_bg_io_stats: 0 2023-08-28T02:05:18.594 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.ttl: 2592000 2023-08-28T02:05:18.594 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.548+0000 7f267a1c0880 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-08-28T02:05:18.594 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.549+0000 7f267a1c0880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-08-28T02:05:18.594 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: 2023-08-28T02:05:18.594 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.549+0000 7f267a1c0880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 0 2023-08-28T02:05:18.595 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: 2023-08-28T02:05:18.595 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.550+0000 7f267a1c0880 4 rocksdb: DB pointer 0x55f4d723f800 2023-08-28T02:05:18.595 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.550+0000 7f2663048700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-08-28T02:05:18.595 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.550+0000 7f2663048700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-08-28T02:05:18.595 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: ** DB Stats ** 2023-08-28T02:05:18.595 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-08-28T02:05:18.595 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-08-28T02:05:18.595 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-08-28T02:05:18.595 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-08-28T02:05:18.595 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-08-28T02:05:18.595 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-08-28T02:05:18.595 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-08-28T02:05:18.596 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: 2023-08-28T02:05:18.596 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-08-28T02:05:18.596 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-08-28T02:05:18.596 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-08-28T02:05:18.596 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-08-28T02:05:18.596 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-08-28T02:05:18.596 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: 2023-08-28T02:05:18.596 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-08-28T02:05:18.596 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-08-28T02:05:18.597 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-08-28T02:05:18.597 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-08-28T02:05:18.597 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: Flush(GB): cumulative 0.000, interval 0.000 2023-08-28T02:05:18.597 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: AddFile(GB): cumulative 0.000, interval 0.000 2023-08-28T02:05:18.597 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: AddFile(Total Files): cumulative 0, interval 0 2023-08-28T02:05:18.597 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: AddFile(L0 Files): cumulative 0, interval 0 2023-08-28T02:05:18.597 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: AddFile(Keys): cumulative 0, interval 0 2023-08-28T02:05:18.597 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-08-28T02:05:18.597 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-08-28T02:05:18.597 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-08-28T02:05:18.597 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: 2023-08-28T02:05:18.598 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: ** File Read Latency Histogram By Level [default] ** 2023-08-28T02:05:18.598 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: 2023-08-28T02:05:18.598 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-08-28T02:05:18.598 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-08-28T02:05:18.598 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-08-28T02:05:18.598 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-08-28T02:05:18.598 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-08-28T02:05:18.598 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: 2023-08-28T02:05:18.598 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-08-28T02:05:18.598 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-08-28T02:05:18.599 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-08-28T02:05:18.599 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-08-28T02:05:18.599 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: Flush(GB): cumulative 0.000, interval 0.000 2023-08-28T02:05:18.599 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: AddFile(GB): cumulative 0.000, interval 0.000 2023-08-28T02:05:18.599 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: AddFile(Total Files): cumulative 0, interval 0 2023-08-28T02:05:18.599 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: AddFile(L0 Files): cumulative 0, interval 0 2023-08-28T02:05:18.599 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: AddFile(Keys): cumulative 0, interval 0 2023-08-28T02:05:18.599 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-08-28T02:05:18.599 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-08-28T02:05:18.599 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-08-28T02:05:18.600 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: 2023-08-28T02:05:18.600 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: ** File Read Latency Histogram By Level [default] ** 2023-08-28T02:05:18.600 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: 2023-08-28T02:05:18.600 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.551+0000 7f267a1c0880 4 rocksdb: [db_impl/db_impl.cc:397] Shutdown: canceling all background work 2023-08-28T02:05:18.600 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.551+0000 7f267a1c0880 4 rocksdb: [db_impl/db_impl.cc:573] Shutdown complete 2023-08-28T02:05:18.600 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph-mon: debug 2023-08-28T02:05:18.551+0000 7f267a1c0880 0 /usr/bin/ceph-mon: created monfs at /var/lib/ceph/mon/ceph-a for mon.a 2023-08-28T02:05:19.012 INFO:teuthology.orchestra.run.smithi143.stderr:create mon.a on 2023-08-28T02:05:19.025 INFO:teuthology.orchestra.run.smithi143.stderr:systemctl: Removed /etc/systemd/system/multi-user.target.wants/ceph.target. 2023-08-28T02:05:19.113 INFO:teuthology.orchestra.run.smithi143.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2023-08-28T02:05:19.208 INFO:teuthology.orchestra.run.smithi143.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315.target → /etc/systemd/system/ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315.target. 2023-08-28T02:05:19.208 INFO:teuthology.orchestra.run.smithi143.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315.target → /etc/systemd/system/ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315.target. 2023-08-28T02:05:19.404 INFO:teuthology.orchestra.run.smithi143.stderr:systemctl: Failed to reset failed state of unit ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@mon.a.service: Unit ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@mon.a.service not loaded. 2023-08-28T02:05:19.411 INFO:teuthology.orchestra.run.smithi143.stderr:systemctl: Created symlink /etc/systemd/system/ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315.target.wants/ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@mon.a.service → /etc/systemd/system/ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@.service. 2023-08-28T02:05:19.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:19 smithi143 systemd[1]: Starting Ceph mon.a for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:05:20.024 INFO:teuthology.orchestra.run.smithi143.stderr:systemctl: disabled 2023-08-28T02:05:20.031 INFO:teuthology.orchestra.run.smithi143.stderr:systemctl: inactive 2023-08-28T02:05:20.031 INFO:teuthology.orchestra.run.smithi143.stderr:firewalld.service is not enabled 2023-08-28T02:05:20.031 INFO:teuthology.orchestra.run.smithi143.stderr:Not possible to enable service . firewalld.service is not available 2023-08-28T02:05:20.032 INFO:teuthology.orchestra.run.smithi143.stderr:Waiting for mon to start... 2023-08-28T02:05:20.032 INFO:teuthology.orchestra.run.smithi143.stderr:Waiting for mon... 2023-08-28T02:05:20.271 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:20 smithi143 bash[99222]: 5045048052e9c323f465ee08270ec281f484ed5b8cb45d030c2a55335d7825ec 2023-08-28T02:05:20.271 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:20 smithi143 systemd[1]: Started Ceph mon.a for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:05:20.271 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:20 smithi143 conmon[99263]: cluster 2023-08-28T02:05:20. 2023-08-28T02:05:20.272 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:20 smithi143 conmon[99263]: 056973+0000 mon.a (mon.0) 0 : [INF] mkfs 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 2023-08-28T02:05:20.272 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:20 smithi143 conmon[99263]: cluster 2023-08-28T02 2023-08-28T02:05:20.272 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:20 smithi143 conmon[99263]: :05:20.038816+0000 mon.a (mon.0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-08-28T02:05:20.544 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: cluster: 2023-08-28T02:05:20.544 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: id: 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 2023-08-28T02:05:20.544 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: health: HEALTH_OK 2023-08-28T02:05:20.544 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: 2023-08-28T02:05:20.544 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: services: 2023-08-28T02:05:20.544 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: mon: 1 daemons, quorum a (age 0.488559s) 2023-08-28T02:05:20.544 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: mgr: no daemons active 2023-08-28T02:05:20.544 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2023-08-28T02:05:20.544 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: 2023-08-28T02:05:20.545 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: data: 2023-08-28T02:05:20.545 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2023-08-28T02:05:20.545 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2023-08-28T02:05:20.545 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2023-08-28T02:05:20.545 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: pgs: 2023-08-28T02:05:20.545 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: 2023-08-28T02:05:21.199 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:21 smithi143 conmon[99263]: cluster 2023-08-28T02:05:20.055672+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-08-28T02:05:21.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:21 smithi143 conmon[99263]: cluster 2023-08-28T02:05:20.055930+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.143:3300/0,v1:172.21.15.143:6789/0]} removed_ranks: {} 2023-08-28T02:05:21.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:21 smithi143 conmon[99263]: cluster 2023-08-28T02:05:20.059185+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap 2023-08-28T02:05:21.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:21 smithi143 conmon[99263]: cluster 2023-08-28T02:05:20.063051+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2023-08-28T02:05:21.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:21 smithi143 conmon[99263]: cluster 2023-08-28T02:05:20.063767+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e1: no daemons active 2023-08-28T02:05:21.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:21 smithi143 conmon[99263]: audit 2023-08-28T02:05:20.544184+0000 mon.a (mon.0) 7 : audit [DBG] from='client.? 172.21.15.143:0/3512303467' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2023-08-28T02:05:21.285 INFO:teuthology.orchestra.run.smithi143.stderr:mon is available 2023-08-28T02:05:21.285 INFO:teuthology.orchestra.run.smithi143.stderr:Assimilating anything we can from ceph.conf... 2023-08-28T02:05:21.805 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: 2023-08-28T02:05:21.805 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: [global] 2023-08-28T02:05:21.806 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: fsid = 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 2023-08-28T02:05:21.806 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: log_to_journald = false 2023-08-28T02:05:21.806 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.143:3300,v1:172.21.15.143:6789] 2023-08-28T02:05:21.806 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2023-08-28T02:05:21.806 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2023-08-28T02:05:21.806 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2023-08-28T02:05:21.806 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2023-08-28T02:05:21.806 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: 2023-08-28T02:05:21.808 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: [mgr] 2023-08-28T02:05:21.809 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2023-08-28T02:05:21.809 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: 2023-08-28T02:05:21.809 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: [osd] 2023-08-28T02:05:21.809 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2023-08-28T02:05:21.809 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2023-08-28T02:05:22.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:22 smithi143 conmon[99263]: audit 2023-08-28T02:05:21.803332+0000 mon.a (mon.0) 2023-08-28T02:05:22.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:22 smithi143 conmon[99263]: 8 : audit [INF] from='client.? 172.21.15.143:0/1421188878' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-08-28T02:05:22.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:22 smithi143 conmon[99263]: audit 2023-08-28T02:05:21.805927+0000 mon.a (mon.0) 9 : audit [INF] from='client.? 172.21.15.143:0/1421188878' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2023-08-28T02:05:22.376 INFO:teuthology.orchestra.run.smithi143.stderr:Generating new minimal ceph.conf... 2023-08-28T02:05:23.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:23 smithi143 conmon[99263]: audit 2023-08-28T02:05:22.928315+0000 mon.a (mon 2023-08-28T02:05:23.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:23 smithi143 conmon[99263]: .0) 10 : audit [DBG] from='client.? 172.21.15.143:0/194299322' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:05:23.534 INFO:teuthology.orchestra.run.smithi143.stderr:Restarting the monitor... 2023-08-28T02:05:23.834 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:23 smithi143 systemd[1]: Stopping Ceph mon.a for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:05:23.835 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:23 smithi143 bash[99767]: Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-mon.a found: no such container 2023-08-28T02:05:23.835 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:23 smithi143 conmon[99263]: debug 2023-08-28T02:05:23.834+0000 7fd87df27700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-08-28T02:05:24.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:23 smithi143 conmon[99263]: debug 2023-08-28T02:05:23.834+0000 7fd87df27700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2023-08-28T02:05:24.699 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:24 smithi143 bash[99883]: 5045048052e9c323f465ee08270ec281f484ed5b8cb45d030c2a55335d7825ec 2023-08-28T02:05:24.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:24 smithi143 systemd[1]: ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@mon.a.service: Succeeded. 2023-08-28T02:05:24.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:24 smithi143 systemd[1]: Stopped Ceph mon.a for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:05:24.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:24 smithi143 systemd[1]: Starting Ceph mon.a for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:05:25.088 INFO:teuthology.orchestra.run.smithi143.stderr:Setting mon public_network to 172.21.0.0/20 2023-08-28T02:05:25.331 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.068+0000 7f0a696b5880 0 set uid:gid to 167:167 (ceph:ceph) 2023-08-28T02:05:25.331 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.068+0000 7f0a696b5880 0 ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable), process ceph-mon, pid 7 2023-08-28T02:05:25.331 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.069+0000 7f0a696b5880 0 pidfile_write: ignore empty --pid-file 2023-08-28T02:05:25.331 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.073+0000 7f0a696b5880 0 load: jerasure load: lrc load: isa 2023-08-28T02:05:25.331 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.073+0000 7f0a696b5880 4 rocksdb: RocksDB version: 6.8.1 2023-08-28T02:05:25.331 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: 2023-08-28T02:05:25.332 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.073+0000 7f0a696b5880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-08-28T02:05:25.332 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.073+0000 7f0a696b5880 4 rocksdb: Compile date Aug 22 2023 2023-08-28T02:05:25.332 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.073+0000 7f0a696b5880 4 rocksdb: DB SUMMARY 2023-08-28T02:05:25.332 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: 2023-08-28T02:05:25.332 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.073+0000 7f0a696b5880 4 rocksdb: CURRENT file: CURRENT 2023-08-28T02:05:25.332 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: 2023-08-28T02:05:25.332 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.073+0000 7f0a696b5880 4 rocksdb: IDENTITY file: IDENTITY 2023-08-28T02:05:25.332 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: 2023-08-28T02:05:25.332 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.073+0000 7f0a696b5880 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 131 Bytes 2023-08-28T02:05:25.332 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: 2023-08-28T02:05:25.332 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.073+0000 7f0a696b5880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000004.sst 2023-08-28T02:05:25.332 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: 2023-08-28T02:05:25.333 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.073+0000 7f0a696b5880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000006.log size: 86713 ; 2023-08-28T02:05:25.333 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: 2023-08-28T02:05:25.333 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.073+0000 7f0a696b5880 4 rocksdb: Options.error_if_exists: 0 2023-08-28T02:05:25.333 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.073+0000 7f0a696b5880 4 rocksdb: Options.create_if_missing: 0 2023-08-28T02:05:25.333 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.073+0000 7f0a696b5880 4 rocksdb: Options.paranoid_checks: 1 2023-08-28T02:05:25.333 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.073+0000 7f0a696b5880 4 rocksdb: Options.env: 0x55cbe8b20080 2023-08-28T02:05:25.333 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.073+0000 7f0a696b5880 4 rocksdb: Options.fs: Posix File System 2023-08-28T02:05:25.333 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.073+0000 7f0a696b5880 4 rocksdb: Options.info_log: 0x55cbeac1fa80 2023-08-28T02:05:25.333 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.073+0000 7f0a696b5880 4 rocksdb: Options.max_file_opening_threads: 16 2023-08-28T02:05:25.333 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.073+0000 7f0a696b5880 4 rocksdb: Options.statistics: (nil) 2023-08-28T02:05:25.333 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.073+0000 7f0a696b5880 4 rocksdb: Options.use_fsync: 0 2023-08-28T02:05:25.333 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.073+0000 7f0a696b5880 4 rocksdb: Options.max_log_file_size: 0 2023-08-28T02:05:25.333 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.073+0000 7f0a696b5880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-08-28T02:05:25.334 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.073+0000 7f0a696b5880 4 rocksdb: Options.log_file_time_to_roll: 0 2023-08-28T02:05:25.334 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.073+0000 7f0a696b5880 4 rocksdb: Options.keep_log_file_num: 1000 2023-08-28T02:05:25.334 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.073+0000 7f0a696b5880 4 rocksdb: Options.recycle_log_file_num: 0 2023-08-28T02:05:25.334 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.073+0000 7f0a696b5880 4 rocksdb: Options.allow_fallocate: 1 2023-08-28T02:05:25.334 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.073+0000 7f0a696b5880 4 rocksdb: Options.allow_mmap_reads: 0 2023-08-28T02:05:25.334 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.073+0000 7f0a696b5880 4 rocksdb: Options.allow_mmap_writes: 0 2023-08-28T02:05:25.334 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.073+0000 7f0a696b5880 4 rocksdb: Options.use_direct_reads: 0 2023-08-28T02:05:25.334 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.073+0000 7f0a696b5880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-08-28T02:05:25.334 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.073+0000 7f0a696b5880 4 rocksdb: Options.create_missing_column_families: 0 2023-08-28T02:05:25.334 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.073+0000 7f0a696b5880 4 rocksdb: Options.db_log_dir: 2023-08-28T02:05:25.334 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.073+0000 7f0a696b5880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-08-28T02:05:25.334 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.073+0000 7f0a696b5880 4 rocksdb: Options.table_cache_numshardbits: 6 2023-08-28T02:05:25.334 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.073+0000 7f0a696b5880 4 rocksdb: Options.max_subcompactions: 1 2023-08-28T02:05:25.335 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.073+0000 7f0a696b5880 4 rocksdb: Options.max_background_flushes: -1 2023-08-28T02:05:25.335 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.073+0000 7f0a696b5880 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-08-28T02:05:25.335 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.073+0000 7f0a696b5880 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-08-28T02:05:25.335 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.073+0000 7f0a696b5880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-08-28T02:05:25.335 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.073+0000 7f0a696b5880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-08-28T02:05:25.335 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.073+0000 7f0a696b5880 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-08-28T02:05:25.335 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.advise_random_on_open: 1 2023-08-28T02:05:25.335 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.db_write_buffer_size: 0 2023-08-28T02:05:25.335 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.write_buffer_manager: 0x55cbeb91a2d0 2023-08-28T02:05:25.335 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-08-28T02:05:25.335 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-08-28T02:05:25.335 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-08-28T02:05:25.335 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.use_adaptive_mutex: 0 2023-08-28T02:05:25.336 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.rate_limiter: (nil) 2023-08-28T02:05:25.336 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-08-28T02:05:25.336 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.wal_recovery_mode: 2 2023-08-28T02:05:25.336 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.enable_thread_tracking: 0 2023-08-28T02:05:25.336 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.enable_pipelined_write: 0 2023-08-28T02:05:25.336 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.unordered_write: 0 2023-08-28T02:05:25.336 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-08-28T02:05:25.336 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-08-28T02:05:25.336 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-08-28T02:05:25.336 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-08-28T02:05:25.336 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.row_cache: None 2023-08-28T02:05:25.336 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.wal_filter: None 2023-08-28T02:05:25.336 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-08-28T02:05:25.337 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.allow_ingest_behind: 0 2023-08-28T02:05:25.337 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.preserve_deletes: 0 2023-08-28T02:05:25.337 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.two_write_queues: 0 2023-08-28T02:05:25.337 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.manual_wal_flush: 0 2023-08-28T02:05:25.337 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.atomic_flush: 0 2023-08-28T02:05:25.337 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-08-28T02:05:25.337 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.persist_stats_to_disk: 0 2023-08-28T02:05:25.337 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-08-28T02:05:25.337 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.log_readahead_size: 0 2023-08-28T02:05:25.337 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-08-28T02:05:25.338 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.max_background_jobs: 2 2023-08-28T02:05:25.338 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.max_background_compactions: -1 2023-08-28T02:05:25.338 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-08-28T02:05:25.338 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-08-28T02:05:25.338 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.delayed_write_rate : 16777216 2023-08-28T02:05:25.339 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.max_total_wal_size: 0 2023-08-28T02:05:25.339 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-08-28T02:05:25.339 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.stats_dump_period_sec: 600 2023-08-28T02:05:25.339 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.stats_persist_period_sec: 600 2023-08-28T02:05:25.339 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-08-28T02:05:25.339 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.max_open_files: -1 2023-08-28T02:05:25.340 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.bytes_per_sync: 0 2023-08-28T02:05:25.340 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-08-28T02:05:25.340 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-08-28T02:05:25.340 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.compaction_readahead_size: 0 2023-08-28T02:05:25.340 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Compression algorithms supported: 2023-08-28T02:05:25.340 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-08-28T02:05:25.340 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: kZSTD supported: 0 2023-08-28T02:05:25.340 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: kXpressCompression supported: 0 2023-08-28T02:05:25.340 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: kLZ4HCCompression supported: 1 2023-08-28T02:05:25.340 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: kLZ4Compression supported: 1 2023-08-28T02:05:25.340 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: kBZip2Compression supported: 0 2023-08-28T02:05:25.340 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: kZlibCompression supported: 1 2023-08-28T02:05:25.340 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: kSnappyCompression supported: 1 2023-08-28T02:05:25.341 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-08-28T02:05:25.341 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000005 2023-08-28T02:05:25.341 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: 2023-08-28T02:05:25.341 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-08-28T02:05:25.341 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: 2023-08-28T02:05:25.341 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-08-28T02:05:25.341 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.merge_operator: 2023-08-28T02:05:25.341 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.compaction_filter: None 2023-08-28T02:05:25.341 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.compaction_filter_factory: None 2023-08-28T02:05:25.341 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-08-28T02:05:25.341 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.table_factory: BlockBasedTable 2023-08-28T02:05:25.341 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55cbeabc9d18) 2023-08-28T02:05:25.341 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: cache_index_and_filter_blocks: 1 2023-08-28T02:05:25.342 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: cache_index_and_filter_blocks_with_high_priority: 0 2023-08-28T02:05:25.342 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-08-28T02:05:25.342 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: pin_top_level_index_and_filter: 1 2023-08-28T02:05:25.342 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: index_type: 0 2023-08-28T02:05:25.342 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: data_block_index_type: 0 2023-08-28T02:05:25.342 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: index_shortening: 1 2023-08-28T02:05:25.342 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: data_block_hash_table_util_ratio: 0.750000 2023-08-28T02:05:25.342 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: hash_index_allow_collision: 1 2023-08-28T02:05:25.342 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: checksum: 1 2023-08-28T02:05:25.342 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: no_block_cache: 0 2023-08-28T02:05:25.342 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: block_cache: 0x55cbeac02f10 2023-08-28T02:05:25.342 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: block_cache_name: BinnedLRUCache 2023-08-28T02:05:25.342 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: block_cache_options: 2023-08-28T02:05:25.343 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: capacity : 536870912 2023-08-28T02:05:25.343 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: num_shard_bits : 4 2023-08-28T02:05:25.343 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: strict_capacity_limit : 0 2023-08-28T02:05:25.343 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: high_pri_pool_ratio: 0.000 2023-08-28T02:05:25.343 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: block_cache_compressed: (nil) 2023-08-28T02:05:25.343 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: persistent_cache: (nil) 2023-08-28T02:05:25.343 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: block_size: 4096 2023-08-28T02:05:25.343 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: block_size_deviation: 10 2023-08-28T02:05:25.343 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: block_restart_interval: 16 2023-08-28T02:05:25.343 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: index_block_restart_interval: 1 2023-08-28T02:05:25.343 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: metadata_block_size: 4096 2023-08-28T02:05:25.343 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: partition_filters: 0 2023-08-28T02:05:25.344 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: use_delta_encoding: 1 2023-08-28T02:05:25.344 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: filter_policy: rocksdb.BuiltinBloomFilter 2023-08-28T02:05:25.344 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: whole_key_filtering: 1 2023-08-28T02:05:25.344 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: verify_compression: 0 2023-08-28T02:05:25.344 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: read_amp_bytes_per_bit: 0 2023-08-28T02:05:25.344 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: format_version: 2 2023-08-28T02:05:25.344 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: enable_index_compression: 1 2023-08-28T02:05:25.344 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: block_align: 0 2023-08-28T02:05:25.344 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: 2023-08-28T02:05:25.344 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.write_buffer_size: 33554432 2023-08-28T02:05:25.344 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.max_write_buffer_number: 2 2023-08-28T02:05:25.344 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.compression: NoCompression 2023-08-28T02:05:25.344 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.bottommost_compression: Disabled 2023-08-28T02:05:25.345 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.prefix_extractor: nullptr 2023-08-28T02:05:25.345 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-08-28T02:05:25.345 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.num_levels: 7 2023-08-28T02:05:25.345 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-08-28T02:05:25.345 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-08-28T02:05:25.345 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-08-28T02:05:25.345 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-08-28T02:05:25.345 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-08-28T02:05:25.345 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-08-28T02:05:25.345 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-08-28T02:05:25.345 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-08-28T02:05:25.345 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-08-28T02:05:25.345 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.compression_opts.window_bits: -14 2023-08-28T02:05:25.346 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.compression_opts.level: 32767 2023-08-28T02:05:25.346 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.compression_opts.strategy: 0 2023-08-28T02:05:25.346 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-08-28T02:05:25.346 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-08-28T02:05:25.346 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.compression_opts.enabled: false 2023-08-28T02:05:25.346 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-08-28T02:05:25.346 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-08-28T02:05:25.346 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-08-28T02:05:25.346 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.target_file_size_base: 67108864 2023-08-28T02:05:25.346 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.target_file_size_multiplier: 1 2023-08-28T02:05:25.346 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-08-28T02:05:25.346 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-08-28T02:05:25.346 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-08-28T02:05:25.347 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-08-28T02:05:25.347 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-08-28T02:05:25.347 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-08-28T02:05:25.347 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-08-28T02:05:25.347 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-08-28T02:05:25.347 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-08-28T02:05:25.347 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-08-28T02:05:25.347 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-08-28T02:05:25.347 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-08-28T02:05:25.347 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.arena_block_size: 4194304 2023-08-28T02:05:25.347 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-08-28T02:05:25.347 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-08-28T02:05:25.348 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-08-28T02:05:25.348 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.disable_auto_compactions: 0 2023-08-28T02:05:25.348 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-08-28T02:05:25.348 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-08-28T02:05:25.348 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-08-28T02:05:25.348 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-08-28T02:05:25.348 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-08-28T02:05:25.348 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-08-28T02:05:25.348 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-08-28T02:05:25.348 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-08-28T02:05:25.348 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-08-28T02:05:25.348 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-08-28T02:05:25.348 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.table_properties_collectors: 2023-08-28T02:05:25.349 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.inplace_update_support: 0 2023-08-28T02:05:25.349 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-08-28T02:05:25.349 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-08-28T02:05:25.349 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-08-28T02:05:25.349 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.memtable_huge_page_size: 0 2023-08-28T02:05:25.349 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.bloom_locality: 0 2023-08-28T02:05:25.349 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.max_successive_merges: 0 2023-08-28T02:05:25.349 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-08-28T02:05:25.349 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.paranoid_file_checks: 0 2023-08-28T02:05:25.349 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.force_consistency_checks: 0 2023-08-28T02:05:25.349 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.report_bg_io_stats: 0 2023-08-28T02:05:25.349 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.ttl: 2592000 2023-08-28T02:05:25.349 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-08-28T02:05:25.350 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 5, log_number is 4,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-08-28T02:05:25.350 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: 2023-08-28T02:05:25.350 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 4 2023-08-28T02:05:25.350 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: 2023-08-28T02:05:25.350 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1693188325075838, "job": 1, "event": "recovery_started", "log_files": [6]} 2023-08-28T02:05:25.350 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.074+0000 7f0a696b5880 4 rocksdb: [db_impl/db_impl_open.cc:760] Recovering log #6 mode 2 2023-08-28T02:05:25.350 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.075+0000 7f0a696b5880 3 rocksdb: [le/block_based/filter_policy.cc:584] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-08-28T02:05:25.350 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.075+0000 7f0a696b5880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1693188325076661, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 7, "file_size": 83313, "table_properties": {"data_size": 81586, "index_size": 332, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 0, "filter_size": 581, "raw_key_size": 9829, "raw_average_key_size": 47, "raw_value_size": 75907, "raw_average_value_size": 368, "num_data_blocks": 11, "num_entries": 206, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1693188325, "oldest_key_time": 3, "file_creation_time": 0}} 2023-08-28T02:05:25.350 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.075+0000 7f0a696b5880 4 rocksdb: [version_set.cc:3826] Creating manifest 8 2023-08-28T02:05:25.350 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: 2023-08-28T02:05:25.350 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.076+0000 7f0a696b5880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1693188325077045, "job": 1, "event": "recovery_finished"} 2023-08-28T02:05:25.350 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.076+0000 7f0a696b5880 4 rocksdb: DB pointer 0x55cbeaccd800 2023-08-28T02:05:25.351 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.076+0000 7f0a504bc700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-08-28T02:05:25.351 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.076+0000 7f0a504bc700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-08-28T02:05:25.351 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: ** DB Stats ** 2023-08-28T02:05:25.351 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: Uptime(secs): 0.0 total, 0.0 interval 2023-08-28T02:05:25.351 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-08-28T02:05:25.351 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-08-28T02:05:25.351 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-08-28T02:05:25.351 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-08-28T02:05:25.351 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-08-28T02:05:25.351 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-08-28T02:05:25.351 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: 2023-08-28T02:05:25.351 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: ** Compaction Stats [default] ** 2023-08-28T02:05:25.351 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-08-28T02:05:25.352 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-08-28T02:05:25.352 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: L0 2/0 83.01 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 141.4 0.00 0.00 1 0.001 0 0 2023-08-28T02:05:25.352 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: Sum 2/0 83.01 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 141.4 0.00 0.00 1 0.001 0 0 2023-08-28T02:05:25.352 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 141.4 0.00 0.00 1 0.001 0 0 2023-08-28T02:05:25.352 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: 2023-08-28T02:05:25.352 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: ** Compaction Stats [default] ** 2023-08-28T02:05:25.352 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-08-28T02:05:25.352 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-08-28T02:05:25.352 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 141.4 0.00 0.00 1 0.001 0 0 2023-08-28T02:05:25.352 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: Uptime(secs): 0.0 total, 0.0 interval 2023-08-28T02:05:25.352 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: Flush(GB): cumulative 0.000, interval 0.000 2023-08-28T02:05:25.352 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: AddFile(GB): cumulative 0.000, interval 0.000 2023-08-28T02:05:25.352 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: AddFile(Total Files): cumulative 0, interval 0 2023-08-28T02:05:25.353 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: AddFile(L0 Files): cumulative 0, interval 0 2023-08-28T02:05:25.353 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: AddFile(Keys): cumulative 0, interval 0 2023-08-28T02:05:25.353 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: Cumulative compaction: 0.00 GB write, 30.77 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-08-28T02:05:25.353 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: Interval compaction: 0.00 GB write, 30.77 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-08-28T02:05:25.353 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-08-28T02:05:25.353 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: 2023-08-28T02:05:25.353 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: ** File Read Latency Histogram By Level [default] ** 2023-08-28T02:05:25.353 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: 2023-08-28T02:05:25.353 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: ** Compaction Stats [default] ** 2023-08-28T02:05:25.353 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-08-28T02:05:25.353 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-08-28T02:05:25.353 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: L0 2/0 83.01 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 141.4 0.00 0.00 1 0.001 0 0 2023-08-28T02:05:25.354 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: Sum 2/0 83.01 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 141.4 0.00 0.00 1 0.001 0 0 2023-08-28T02:05:25.354 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-08-28T02:05:25.354 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: 2023-08-28T02:05:25.354 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: ** Compaction Stats [default] ** 2023-08-28T02:05:25.354 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-08-28T02:05:25.354 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-08-28T02:05:25.354 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 141.4 0.00 0.00 1 0.001 0 0 2023-08-28T02:05:25.354 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 bash[100217]: 3e714a6191278b03ddf36d79b64cfeb2b3f674cf0fdfe867c9e30c7501d13506 2023-08-28T02:05:25.354 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: Uptime(secs): 0.0 total, 0.0 interval 2023-08-28T02:05:25.354 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: Flush(GB): cumulative 0.000, interval 0.000 2023-08-28T02:05:25.354 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: AddFile(GB): cumulative 0.000, interval 0.000 2023-08-28T02:05:25.355 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: AddFile(Total Files): cumulative 0, interval 0 2023-08-28T02:05:25.355 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: AddFile(L0 Files): cumulative 0, interval 0 2023-08-28T02:05:25.355 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: AddFile(Keys): cumulative 0, interval 0 2023-08-28T02:05:25.355 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: Cumulative compaction: 0.00 GB write, 30.38 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-08-28T02:05:25.355 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-08-28T02:05:25.355 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-08-28T02:05:25.355 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: 2023-08-28T02:05:25.355 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: ** File Read Latency Histogram By Level [default] ** 2023-08-28T02:05:25.355 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: 2023-08-28T02:05:25.355 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.077+0000 7f0a696b5880 0 starting mon.a rank 0 at public addrs [v2:172.21.15.143:3300/0,v1:172.21.15.143:6789/0] at bind addrs [v2:172.21.15.143:3300/0,v1:172.21.15.143:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 2023-08-28T02:05:25.355 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.078+0000 7f0a696b5880 1 mon.a@-1(???) e1 preinit fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 2023-08-28T02:05:25.355 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.078+0000 7f0a696b5880 0 mon.a@-1(???).mds e1 new map 2023-08-28T02:05:25.356 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.078+0000 7f0a696b5880 0 mon.a@-1(???).mds e1 print_map 2023-08-28T02:05:25.356 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: e1 2023-08-28T02:05:25.356 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: enable_multiple, ever_enabled_multiple: 1,1 2023-08-28T02:05:25.356 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-08-28T02:05:25.356 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: legacy client fscid: -1 2023-08-28T02:05:25.356 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: 2023-08-28T02:05:25.356 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: No filesystems configured 2023-08-28T02:05:25.356 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: 2023-08-28T02:05:25.356 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.078+0000 7f0a696b5880 0 mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2023-08-28T02:05:25.356 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.078+0000 7f0a696b5880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-08-28T02:05:25.356 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.078+0000 7f0a696b5880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-08-28T02:05:25.356 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.078+0000 7f0a696b5880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-08-28T02:05:25.356 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: audit 2023 2023-08-28T02:05:25.357 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: -08-28T02:05:22.928315+0000 mon.a (mon.0) 10 : 2023-08-28T02:05:25.357 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: audit [DBG] from='client.? 172.21.15.143:0/194299322' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:05:25.357 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: debug 2023-08-28T02:05:25.078+0000 7f0a696b5880 1 mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2023-08-28T02:05:25.357 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 systemd[1]: Started Ceph mon.a for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:05:25.357 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: cluster 2023-08-28T02:05:25.095670+0000 mon.a (mon. 2023-08-28T02:05:25.357 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: 0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-08-28T02:05:25.357 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: cluster 2023-08-28T02:05:25.095749+0000 mon.a (mon.0) 2 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.143:3300/0,v1:172.21.15.143:6789/0]} removed_ranks: {} 2023-08-28T02:05:25.357 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: cluster 2023-08-28T02:05:25.107648+0000 mon.a (mon.0) 3 : cluster [DBG] fsmap 2023-08-28T02:05:25.357 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: cluster 2023-08-28T02:05:25.107704+0000 mon.a (mon.0) 4 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2023-08-28T02:05:25.357 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:25 smithi143 conmon[100258]: cluster 2023-08-28T02:05:25.108242+0000 mon.a (mon.0) 5 : cluster [DBG] mgrmap e1: no daemons active 2023-08-28T02:05:26.259 INFO:teuthology.orchestra.run.smithi143.stderr:Wrote config to /etc/ceph/ceph.conf 2023-08-28T02:05:26.259 INFO:teuthology.orchestra.run.smithi143.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2023-08-28T02:05:26.259 INFO:teuthology.orchestra.run.smithi143.stderr:Creating mgr... 2023-08-28T02:05:26.260 INFO:teuthology.orchestra.run.smithi143.stderr:Verifying port 9283 ... 2023-08-28T02:05:26.387 INFO:teuthology.orchestra.run.smithi143.stderr:systemctl: Failed to reset failed state of unit ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@mgr.y.service: Unit ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@mgr.y.service not loaded. 2023-08-28T02:05:26.393 INFO:teuthology.orchestra.run.smithi143.stderr:systemctl: Created symlink /etc/systemd/system/ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315.target.wants/ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@mgr.y.service → /etc/systemd/system/ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@.service. 2023-08-28T02:05:26.622 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:26 smithi143 systemd[1]: Starting Ceph mgr.y for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:05:26.941 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:26 smithi143 conmon[100258]: audit 2023-08-28T02:05:25.624319+0000 2023-08-28T02:05:26.941 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:26 smithi143 conmon[100258]: mon.a (mon.0) 6 : audit [INF] from='client.? 172.21.15.143:0/1177787772' entity='client.admin' 2023-08-28T02:05:27.081 INFO:teuthology.orchestra.run.smithi143.stderr:systemctl: disabled 2023-08-28T02:05:27.088 INFO:teuthology.orchestra.run.smithi143.stderr:systemctl: inactive 2023-08-28T02:05:27.089 INFO:teuthology.orchestra.run.smithi143.stderr:firewalld.service is not enabled 2023-08-28T02:05:27.089 INFO:teuthology.orchestra.run.smithi143.stderr:Not possible to enable service . firewalld.service is not available 2023-08-28T02:05:27.097 INFO:teuthology.orchestra.run.smithi143.stderr:systemctl: disabled 2023-08-28T02:05:27.105 INFO:teuthology.orchestra.run.smithi143.stderr:systemctl: inactive 2023-08-28T02:05:27.106 INFO:teuthology.orchestra.run.smithi143.stderr:firewalld.service is not enabled 2023-08-28T02:05:27.106 INFO:teuthology.orchestra.run.smithi143.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2023-08-28T02:05:27.106 INFO:teuthology.orchestra.run.smithi143.stderr:Waiting for mgr to start... 2023-08-28T02:05:27.107 INFO:teuthology.orchestra.run.smithi143.stderr:Waiting for mgr... 2023-08-28T02:05:27.339 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:27 smithi143 bash[100667]: 48d3608092c2cdae1485f3415b4bdef79931b86d7d292111cb235bc15fd048c8 2023-08-28T02:05:27.339 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:27 smithi143 systemd[1]: Started Ceph mgr.y for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:05:27.339 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:27 smithi143 conmon[100708]: debug 2023-08-28T02:05:27.194+0000 7fb251645680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-08-28T02:05:27.339 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:27 smithi143 conmon[100708]: debug 2023-08-28T02:05:27.279+0000 7fb251645680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-08-28T02:05:27.651 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: 2023-08-28T02:05:27.654 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: { 2023-08-28T02:05:27.654 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "fsid": "2fcb0bfa-4547-11ee-817c-1b7a5f1ba315", 2023-08-28T02:05:27.654 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "health": { 2023-08-28T02:05:27.654 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-08-28T02:05:27.654 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "checks": {}, 2023-08-28T02:05:27.654 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "mutes": [] 2023-08-28T02:05:27.654 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: }, 2023-08-28T02:05:27.654 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-08-28T02:05:27.654 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "quorum": [ 2023-08-28T02:05:27.654 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: 0 2023-08-28T02:05:27.654 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: ], 2023-08-28T02:05:27.655 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "quorum_names": [ 2023-08-28T02:05:27.655 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "a" 2023-08-28T02:05:27.655 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: ], 2023-08-28T02:05:27.655 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "quorum_age": 2, 2023-08-28T02:05:27.655 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "monmap": { 2023-08-28T02:05:27.655 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "epoch": 1, 2023-08-28T02:05:27.655 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-08-28T02:05:27.655 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "num_mons": 1 2023-08-28T02:05:27.655 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: }, 2023-08-28T02:05:27.655 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "osdmap": { 2023-08-28T02:05:27.655 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "epoch": 1, 2023-08-28T02:05:27.655 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "num_osds": 0, 2023-08-28T02:05:27.656 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-08-28T02:05:27.656 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-08-28T02:05:27.659 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-08-28T02:05:27.659 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-08-28T02:05:27.659 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-08-28T02:05:27.659 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: }, 2023-08-28T02:05:27.659 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "pgmap": { 2023-08-28T02:05:27.659 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-08-28T02:05:27.659 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-08-28T02:05:27.659 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "num_pools": 0, 2023-08-28T02:05:27.659 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "num_objects": 0, 2023-08-28T02:05:27.659 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-08-28T02:05:27.660 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-08-28T02:05:27.660 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-08-28T02:05:27.660 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "bytes_total": 0 2023-08-28T02:05:27.660 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: }, 2023-08-28T02:05:27.660 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "fsmap": { 2023-08-28T02:05:27.660 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "epoch": 1, 2023-08-28T02:05:27.660 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "by_rank": [], 2023-08-28T02:05:27.660 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "up:standby": 0 2023-08-28T02:05:27.660 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: }, 2023-08-28T02:05:27.660 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "mgrmap": { 2023-08-28T02:05:27.660 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "available": false, 2023-08-28T02:05:27.660 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-08-28T02:05:27.660 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "modules": [ 2023-08-28T02:05:27.661 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "iostat", 2023-08-28T02:05:27.661 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "nfs", 2023-08-28T02:05:27.661 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "restful" 2023-08-28T02:05:27.661 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: ], 2023-08-28T02:05:27.661 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "services": {} 2023-08-28T02:05:27.661 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: }, 2023-08-28T02:05:27.661 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "servicemap": { 2023-08-28T02:05:27.661 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "epoch": 1, 2023-08-28T02:05:27.661 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "modified": "2023-08-28T02:05:20.057416+0000", 2023-08-28T02:05:27.661 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "services": {} 2023-08-28T02:05:27.661 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: }, 2023-08-28T02:05:27.661 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "progress_events": {} 2023-08-28T02:05:27.661 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: } 2023-08-28T02:05:27.949 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:27 smithi143 conmon[100258]: audit 2023-08-28T02:05:27.652627+0000 mon.a (mon.0) 7 : audit [DBG] from='client.? 172.21.15.143:0/3415047297' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-08-28T02:05:27.950 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:27 smithi143 conmon[100708]: debug 2023-08-28T02:05:27.664+0000 7fb251645680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-08-28T02:05:28.216 INFO:teuthology.orchestra.run.smithi143.stderr:mgr not available, waiting (1/15)... 2023-08-28T02:05:28.509 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:28 smithi143 conmon[100708]: debug 2023-08-28T02:05:28.427+0000 7fb251645680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-08-28T02:05:28.949 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:28 smithi143 conmon[100708]: debug 2023-08-28T02:05:28.510+0000 7fb251645680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-08-28T02:05:28.950 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:28 smithi143 conmon[100708]: debug 2023-08-28T02:05:28.719+0000 7fb251645680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-08-28T02:05:29.937 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:29 smithi143 conmon[100708]: debug 2023-08-28T02:05:29.647+0000 7fb251645680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-08-28T02:05:29.937 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:29 smithi143 conmon[100708]: debug 2023-08-28T02:05:29.772+0000 7fb251645680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-08-28T02:05:29.937 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:29 smithi143 conmon[100708]: debug 2023-08-28T02:05:29.848+0000 7fb251645680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-08-28T02:05:30.199 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:29 smithi143 conmon[100708]: debug 2023-08-28T02:05:29.937+0000 7fb251645680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-08-28T02:05:30.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:30 smithi143 conmon[100708]: debug 2023-08-28T02:05:30.132+0000 7fb251645680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-08-28T02:05:30.700 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:30 smithi143 conmon[100708]: debug 2023-08-28T02:05:30.433+0000 7fb251645680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-08-28T02:05:30.700 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:30 smithi143 conmon[100708]: debug 2023-08-28T02:05:30.535+0000 7fb251645680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-08-28T02:05:30.750 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: 2023-08-28T02:05:30.750 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: { 2023-08-28T02:05:30.750 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "fsid": "2fcb0bfa-4547-11ee-817c-1b7a5f1ba315", 2023-08-28T02:05:30.750 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "health": { 2023-08-28T02:05:30.750 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-08-28T02:05:30.750 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "checks": {}, 2023-08-28T02:05:30.750 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "mutes": [] 2023-08-28T02:05:30.750 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: }, 2023-08-28T02:05:30.751 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-08-28T02:05:30.751 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "quorum": [ 2023-08-28T02:05:30.751 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: 0 2023-08-28T02:05:30.751 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: ], 2023-08-28T02:05:30.751 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "quorum_names": [ 2023-08-28T02:05:30.751 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "a" 2023-08-28T02:05:30.751 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: ], 2023-08-28T02:05:30.751 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "quorum_age": 5, 2023-08-28T02:05:30.752 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "monmap": { 2023-08-28T02:05:30.752 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "epoch": 1, 2023-08-28T02:05:30.752 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-08-28T02:05:30.752 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "num_mons": 1 2023-08-28T02:05:30.752 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: }, 2023-08-28T02:05:30.752 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "osdmap": { 2023-08-28T02:05:30.752 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "epoch": 1, 2023-08-28T02:05:30.752 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "num_osds": 0, 2023-08-28T02:05:30.752 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-08-28T02:05:30.752 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-08-28T02:05:30.756 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-08-28T02:05:30.756 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-08-28T02:05:30.756 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-08-28T02:05:30.756 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: }, 2023-08-28T02:05:30.757 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "pgmap": { 2023-08-28T02:05:30.757 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-08-28T02:05:30.757 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-08-28T02:05:30.757 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "num_pools": 0, 2023-08-28T02:05:30.757 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "num_objects": 0, 2023-08-28T02:05:30.757 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-08-28T02:05:30.757 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-08-28T02:05:30.757 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-08-28T02:05:30.757 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "bytes_total": 0 2023-08-28T02:05:30.757 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: }, 2023-08-28T02:05:30.758 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "fsmap": { 2023-08-28T02:05:30.758 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "epoch": 1, 2023-08-28T02:05:30.758 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "by_rank": [], 2023-08-28T02:05:30.758 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "up:standby": 0 2023-08-28T02:05:30.758 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: }, 2023-08-28T02:05:30.758 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "mgrmap": { 2023-08-28T02:05:30.758 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "available": false, 2023-08-28T02:05:30.758 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-08-28T02:05:30.758 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "modules": [ 2023-08-28T02:05:30.759 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "iostat", 2023-08-28T02:05:30.759 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "nfs", 2023-08-28T02:05:30.759 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "restful" 2023-08-28T02:05:30.759 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: ], 2023-08-28T02:05:30.759 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "services": {} 2023-08-28T02:05:30.759 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: }, 2023-08-28T02:05:30.759 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "servicemap": { 2023-08-28T02:05:30.759 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "epoch": 1, 2023-08-28T02:05:30.760 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "modified": "2023-08-28T02:05:20.057416+0000", 2023-08-28T02:05:30.760 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "services": {} 2023-08-28T02:05:30.760 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: }, 2023-08-28T02:05:30.760 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "progress_events": {} 2023-08-28T02:05:30.760 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: } 2023-08-28T02:05:31.232 INFO:teuthology.orchestra.run.smithi143.stderr:mgr not available, waiting (2/15)... 2023-08-28T02:05:31.368 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:31 smithi143 conmon[100258]: audit 2023-08-28T02:05:30.748993+0000 mon.a (mon.0 2023-08-28T02:05:31.368 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:31 smithi143 conmon[100258]: ) 8 : audit [DBG] from='client.? 172.21.15.143:0/2373729467' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-08-28T02:05:31.369 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:31 smithi143 conmon[100708]: debug 2023-08-28T02:05:31.288+0000 7fb251645680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-08-28T02:05:31.630 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:31 smithi143 conmon[100708]: debug 2023-08-28T02:05:31.369+0000 7fb251645680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-08-28T02:05:31.631 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:31 smithi143 conmon[100708]: debug 2023-08-28T02:05:31.459+0000 7fb251645680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-08-28T02:05:32.153 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:31 smithi143 conmon[100708]: debug 2023-08-28T02:05:31.631+0000 7fb251645680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-08-28T02:05:32.153 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:31 smithi143 conmon[100708]: debug 2023-08-28T02:05:31.891+0000 7fb251645680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-08-28T02:05:32.153 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:32 smithi143 conmon[100708]: debug 2023-08-28T02:05:32.021+0000 7fb251645680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-08-28T02:05:32.153 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:32 smithi143 conmon[100708]: debug 2023-08-28T02:05:32.153+0000 7fb251645680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-08-28T02:05:32.404 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:32 smithi143 conmon[100708]: debug 2023-08-28T02:05:32.322+0000 7fb251645680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-08-28T02:05:32.699 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:32 smithi143 conmon[100708]: debug 2023-08-28T02:05:32.404+0000 7fb251645680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-08-28T02:05:32.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:32 smithi143 conmon[100258]: cluster 2023-08-28T02:05:32.418957+0000 mon.a (mon.0 2023-08-28T02:05:32.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:32 smithi143 conmon[100258]: ) 9 : cluster [INF] Activating manager daemon y 2023-08-28T02:05:32.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:32 smithi143 conmon[100258]: cluster 2023-08-28T02:05:32.421508+0000 mon.a (mon.0) 10 : cluster [DBG] mgrmap e2: y(active, starting, since 0.00263204s) 2023-08-28T02:05:32.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:32 smithi143 conmon[100258]: audit 2023-08-28T02:05:32. 2023-08-28T02:05:32.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:32 smithi143 conmon[100258]: 422533+0000 mon.a (mon.0) 11 : audit [DBG] 2023-08-28T02:05:32.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:32 smithi143 conmon[100258]: from='mgr.14100 172.21.15.143:0/76848430' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-08-28T02:05:32.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:32 smithi143 conmon[100258]: audit 2023-08- 2023-08-28T02:05:32.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:32 smithi143 conmon[100258]: 28T02:05:32.422758+0000 mon.a (mon 2023-08-28T02:05:32.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:32 smithi143 conmon[100258]: .0) 12 : audit [DBG] from='mgr.14100 172.21.15.143:0/76848430' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-08-28T02:05:32.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:32 smithi143 conmon[100258]: audit 2023-08-28T02:05:32.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:32 smithi143 conmon[100258]: 2023-08-28T02:05 2023-08-28T02:05:32.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:32 smithi143 conmon[100258]: :32.422952+0000 mon.a (mon.0) 13 2023-08-28T02:05:32.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:32 smithi143 conmon[100258]: : audit [DBG] from='mgr.14100 172.21.15.143:0/76848430' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-08-28T02:05:32.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:32 smithi143 conmon[100258]: audit 2023- 2023-08-28T02:05:32.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:32 smithi143 conmon[100258]: 08-28T02:05:32.423123+0000 mon.a 2023-08-28T02:05:32.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:32 smithi143 conmon[100258]: (mon.0) 14 : audit [DBG] from='mgr.14100 172.21.15.143:0/76848430' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T02:05:32.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:32 smithi143 conmon[100258]: audit 2023-08-28T02:05:32.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:32 smithi143 conmon[100258]: 2023-08-28T02:05: 2023-08-28T02:05:32.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:32 smithi143 conmon[100258]: 32.423289+0000 mon.a (mon.0) 15 : 2023-08-28T02:05:32.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:32 smithi143 conmon[100258]: audit [DBG] from='mgr.14100 172.21.15.143:0/76848430' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-08-28T02:05:32.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:32 smithi143 conmon[100258]: cluster 2023 2023-08-28T02:05:32.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:32 smithi143 conmon[100258]: -08-28T02:05:32.436486+0000 2023-08-28T02:05:32.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:32 smithi143 conmon[100258]: mon.a (mon.0) 16 : cluster [INF] Manager daemon y is now available 2023-08-28T02:05:32.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:32 smithi143 conmon[100258]: 2023-08-28T02:05:32.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:32 smithi143 conmon[100258]: audit 2023-08-28T02:05:32.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:32 smithi143 conmon[100258]: 2023-08-28T02:05:32 2023-08-28T02:05:32.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:32 smithi143 conmon[100258]: .446756+0000 mon.a (mon.0) 17 : audit 2023-08-28T02:05:32.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:32 smithi143 conmon[100258]: [INF] from='mgr.14100 172.21.15.143:0/76848430' entity='mgr.y' 2023-08-28T02:05:32.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:32 smithi143 conmon[100258]: audit 2023 2023-08-28T02:05:32.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:32 smithi143 conmon[100258]: -08-28T02:05:32.450076+0000 2023-08-28T02:05:32.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:32 smithi143 conmon[100258]: mon.a (mon.0) 18 : audit [INF] from='mgr.14100 172.21.15.143:0/76848430' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-08-28T02:05:32.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:32 smithi143 conmon[100258]: 2023-08-28T02:05:32.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:32 smithi143 conmon[100258]: audit 2023-08-28T02:05:32.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:32 smithi143 conmon[100258]: 2023-08-28T02:05:32.452529+0000 mon.a (mon 2023-08-28T02:05:32.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:32 smithi143 conmon[100258]: .0) 19 : audit [INF] from='mgr.14100 172.21.15.143:0/76848430' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-08-28T02:05:32.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:32 smithi143 conmon[100258]: audit 2023-08-28T02:05:32.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:32 smithi143 conmon[100258]: 2023-08-28T02:05:32. 2023-08-28T02:05:32.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:32 smithi143 conmon[100258]: 458344+0000 mon.a (mon.0) 20 : audit [INF] 2023-08-28T02:05:32.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:32 smithi143 conmon[100258]: from='mgr.14100 172.21.15.143:0/76848430' entity='mgr.y' 2023-08-28T02:05:32.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:32 smithi143 conmon[100258]: audit 2023-08- 2023-08-28T02:05:32.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:32 smithi143 conmon[100258]: 28T02:05:32.463508+0000 mon.a (mon 2023-08-28T02:05:32.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:32 smithi143 conmon[100258]: .0) 21 : audit [INF] from='mgr.14100 172.21.15.143:0/76848430' entity='mgr.y' 2023-08-28T02:05:33.931 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: 2023-08-28T02:05:33.931 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: { 2023-08-28T02:05:33.932 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "fsid": "2fcb0bfa-4547-11ee-817c-1b7a5f1ba315", 2023-08-28T02:05:33.932 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "health": { 2023-08-28T02:05:33.932 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-08-28T02:05:33.932 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "checks": {}, 2023-08-28T02:05:33.932 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "mutes": [] 2023-08-28T02:05:33.932 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: }, 2023-08-28T02:05:33.932 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-08-28T02:05:33.932 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "quorum": [ 2023-08-28T02:05:33.932 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: 0 2023-08-28T02:05:33.932 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: ], 2023-08-28T02:05:33.932 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "quorum_names": [ 2023-08-28T02:05:33.932 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "a" 2023-08-28T02:05:33.933 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: ], 2023-08-28T02:05:33.933 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "quorum_age": 8, 2023-08-28T02:05:33.933 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "monmap": { 2023-08-28T02:05:33.933 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "epoch": 1, 2023-08-28T02:05:33.933 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-08-28T02:05:33.933 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "num_mons": 1 2023-08-28T02:05:33.933 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: }, 2023-08-28T02:05:33.933 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "osdmap": { 2023-08-28T02:05:33.933 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "epoch": 1, 2023-08-28T02:05:33.934 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "num_osds": 0, 2023-08-28T02:05:33.934 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-08-28T02:05:33.934 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-08-28T02:05:33.937 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-08-28T02:05:33.937 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-08-28T02:05:33.937 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-08-28T02:05:33.937 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: }, 2023-08-28T02:05:33.937 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "pgmap": { 2023-08-28T02:05:33.937 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-08-28T02:05:33.937 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-08-28T02:05:33.937 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "num_pools": 0, 2023-08-28T02:05:33.937 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "num_objects": 0, 2023-08-28T02:05:33.938 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-08-28T02:05:33.938 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-08-28T02:05:33.938 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-08-28T02:05:33.938 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "bytes_total": 0 2023-08-28T02:05:33.938 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: }, 2023-08-28T02:05:33.938 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "fsmap": { 2023-08-28T02:05:33.938 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "epoch": 1, 2023-08-28T02:05:33.938 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "by_rank": [], 2023-08-28T02:05:33.938 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "up:standby": 0 2023-08-28T02:05:33.938 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: }, 2023-08-28T02:05:33.938 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "mgrmap": { 2023-08-28T02:05:33.938 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "available": true, 2023-08-28T02:05:33.939 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-08-28T02:05:33.939 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "modules": [ 2023-08-28T02:05:33.939 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "iostat", 2023-08-28T02:05:33.939 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "nfs", 2023-08-28T02:05:33.939 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "restful" 2023-08-28T02:05:33.939 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: ], 2023-08-28T02:05:33.939 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "services": {} 2023-08-28T02:05:33.939 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: }, 2023-08-28T02:05:33.939 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "servicemap": { 2023-08-28T02:05:33.939 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "epoch": 1, 2023-08-28T02:05:33.939 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "modified": "2023-08-28T02:05:20.057416+0000", 2023-08-28T02:05:33.939 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "services": {} 2023-08-28T02:05:33.939 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: }, 2023-08-28T02:05:33.940 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "progress_events": {} 2023-08-28T02:05:33.940 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: } 2023-08-28T02:05:34.465 INFO:teuthology.orchestra.run.smithi143.stderr:mgr is available 2023-08-28T02:05:34.672 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:34 smithi143 conmon[100258]: cluster 2023-08-28T02:05:33.425784+0000 mon.a (mon.0) 22 : cluster [DBG] mgrmap e3: y(active, since 1.00691s) 2023-08-28T02:05:34.672 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:34 smithi143 conmon[100258]: audit 2023-08-28T02:05:33.930798+0000 mon.a (mon.0) 23 : audit [DBG] from='client.? 172.21.15.143:0/2159488912' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-08-28T02:05:35.064 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: 2023-08-28T02:05:35.064 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: [global] 2023-08-28T02:05:35.064 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: fsid = 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 2023-08-28T02:05:35.065 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: log_to_journald = false 2023-08-28T02:05:35.065 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2023-08-28T02:05:35.065 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2023-08-28T02:05:35.065 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2023-08-28T02:05:35.065 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2023-08-28T02:05:35.065 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: 2023-08-28T02:05:35.066 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: [mgr] 2023-08-28T02:05:35.066 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2023-08-28T02:05:35.066 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: 2023-08-28T02:05:35.066 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: [osd] 2023-08-28T02:05:35.066 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2023-08-28T02:05:35.066 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2023-08-28T02:05:35.573 INFO:teuthology.orchestra.run.smithi143.stderr:Enabling cephadm module... 2023-08-28T02:05:35.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:35 smithi143 conmon[100258]: cluster 2023-08-28T02:05:34.426032+0000 2023-08-28T02:05:35.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:35 smithi143 conmon[100258]: mon.a (mon.0) 24 : cluster [DBG] mgrmap e4: y(active, since 2s) 2023-08-28T02:05:35.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:35 smithi143 conmon[100258]: audit 2023-08-28T02:05:35.063662+0000 mon.a (mon.0) 2023-08-28T02:05:35.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:35 smithi143 conmon[100258]: 25 : audit [INF] from='client.? 172.21.15.143:0/3615201345' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-08-28T02:05:36.699 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:36 smithi143 conmon[100258]: audit 2023-08-28T02:05:36.176443+0000 mon.a (mon.0) 2023-08-28T02:05:36.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:36 smithi143 conmon[100258]: 26 : audit [INF] from='client.? 172.21.15.143:0/1949567847' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2023-08-28T02:05:36.700 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:36 smithi143 conmon[100708]: ignoring --setuser ceph since I am not root 2023-08-28T02:05:36.700 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:36 smithi143 conmon[100708]: ignoring --setgroup ceph since I am not root 2023-08-28T02:05:36.700 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:36 smithi143 conmon[100708]: debug 2023-08-28T02:05:36.596+0000 7f28a7d2b680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-08-28T02:05:36.996 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:36 smithi143 conmon[100708]: debug 2023-08-28T02:05:36.678+0000 7f28a7d2b680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-08-28T02:05:37.426 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:37 smithi143 conmon[100708]: debug 2023-08-28T02:05:37.049+0000 7f28a7d2b680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-08-28T02:05:37.655 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: { 2023-08-28T02:05:37.655 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "epoch": 5, 2023-08-28T02:05:37.655 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "available": true, 2023-08-28T02:05:37.655 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "active_name": "y", 2023-08-28T02:05:37.655 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "num_standby": 0 2023-08-28T02:05:37.655 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: } 2023-08-28T02:05:37.684 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:37 smithi143 conmon[100258]: audit 2023-08-28T02:05:36.429544+0000 mon.a (mon.0) 27 : audit [INF] from='client.? 172.21.15.143:0/1949567847' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2023-08-28T02:05:37.684 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:37 smithi143 conmon[100258]: cluster 2023-08-28T02:05:36.429612+0000 mon.a (mon.0) 28 : cluster [DBG] mgrmap e5: y(active, since 4s) 2023-08-28T02:05:37.949 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:37 smithi143 conmon[100708]: debug 2023-08-28T02:05:37.826+0000 7f28a7d2b680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-08-28T02:05:37.950 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:37 smithi143 conmon[100708]: debug 2023-08-28T02:05:37.916+0000 7f28a7d2b680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-08-28T02:05:38.190 INFO:teuthology.orchestra.run.smithi143.stderr:Waiting for the mgr to restart... 2023-08-28T02:05:38.190 INFO:teuthology.orchestra.run.smithi143.stderr:Waiting for mgr epoch 5... 2023-08-28T02:05:38.388 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:38 smithi143 conmon[100708]: debug 2023-08-28T02:05:38.128+0000 7f28a7d2b680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-08-28T02:05:38.699 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:38 smithi143 conmon[100258]: audit 2023-08-28T02:05:37.655251+0000 mon.a (mon.0) 29 : 2023-08-28T02:05:38.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:38 smithi143 conmon[100258]: audit [DBG] from='client.? 172.21.15.143:0/1496647859' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-08-28T02:05:39.379 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:39 smithi143 conmon[100708]: debug 2023-08-28T02:05:39.089+0000 7f28a7d2b680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-08-28T02:05:39.379 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:39 smithi143 conmon[100708]: debug 2023-08-28T02:05:39.215+0000 7f28a7d2b680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-08-28T02:05:39.380 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:39 smithi143 conmon[100708]: debug 2023-08-28T02:05:39.292+0000 7f28a7d2b680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-08-28T02:05:39.380 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:39 smithi143 conmon[100708]: debug 2023-08-28T02:05:39.380+0000 7f28a7d2b680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-08-28T02:05:39.699 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:39 smithi143 conmon[100708]: debug 2023-08-28T02:05:39.574+0000 7f28a7d2b680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-08-28T02:05:40.199 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:39 smithi143 conmon[100708]: debug 2023-08-28T02:05:39.849+0000 7f28a7d2b680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-08-28T02:05:40.199 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:39 smithi143 conmon[100708]: debug 2023-08-28T02:05:39.945+0000 7f28a7d2b680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-08-28T02:05:40.949 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:40 smithi143 conmon[100708]: debug 2023-08-28T02:05:40.655+0000 7f28a7d2b680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-08-28T02:05:40.950 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:40 smithi143 conmon[100708]: debug 2023-08-28T02:05:40.735+0000 7f28a7d2b680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-08-28T02:05:40.950 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:40 smithi143 conmon[100708]: debug 2023-08-28T02:05:40.825+0000 7f28a7d2b680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-08-28T02:05:41.268 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:41 smithi143 conmon[100708]: debug 2023-08-28T02:05:41.001+0000 7f28a7d2b680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-08-28T02:05:41.530 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:41 smithi143 conmon[100708]: debug 2023-08-28T02:05:41.269+0000 7f28a7d2b680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-08-28T02:05:41.531 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:41 smithi143 conmon[100708]: debug 2023-08-28T02:05:41.399+0000 7f28a7d2b680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-08-28T02:05:41.783 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:41 smithi143 conmon[100708]: debug 2023-08-28T02:05:41.531+0000 7f28a7d2b680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-08-28T02:05:41.783 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:41 smithi143 conmon[100708]: debug 2023-08-28T02:05:41.702+0000 7f28a7d2b680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-08-28T02:05:42.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:41 smithi143 conmon[100708]: debug 2023-08-28T02:05:41.784+0000 7f28a7d2b680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-08-28T02:05:42.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:41 smithi143 conmon[100258]: cluster 2023-08-28T02:05:41.797729+0000 mon.a (mon.0) 30 : cluster [INF] Active manager daemon y restarted 2023-08-28T02:05:42.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:41 smithi143 conmon[100258]: 2023-08-28T02:05:42.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:41 smithi143 conmon[100258]: cluster 2023-08-28T02:05:41.797959+0000 mon.a (mon.0) 31 : cluster [INF] Activating manager daemon y 2023-08-28T02:05:42.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:41 smithi143 conmon[100258]: cluster 2023-08-28T02:05: 2023-08-28T02:05:42.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:41 smithi143 conmon[100258]: 41.800791+0000 mon.a (mon.0) 32 : cluster [DBG] osdmap e2: 0 total, 0 up, 0 in 2023-08-28T02:05:42.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:41 smithi143 conmon[100258]: cluster 2023-08-28T02: 2023-08-28T02:05:42.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:41 smithi143 conmon[100258]: 05:41.800905+0000 mon.a (mon.0) 33 : cluster [DBG] mgrmap e6: y(active, starting, since 0.00302592s) 2023-08-28T02:05:42.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:41 smithi143 conmon[100258]: audit 2023-08-28T02:05:41.801872+0000 2023-08-28T02:05:42.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:41 smithi143 conmon[100258]: mon.a (mon.0) 34 : audit [DBG] from='mgr.14116 172.21.15.143:0/3275888306' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T02:05:42.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:41 smithi143 conmon[100258]: audit 2023-08-28T02:05:41.802451+0000 mon.a (mon.0 2023-08-28T02:05:42.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:41 smithi143 conmon[100258]: ) 35 : audit [DBG] from='mgr.14116 172.21.15.143:0/3275888306' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-08-28T02:05:42.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:41 smithi143 conmon[100258]: audit 2023-08-28T02:05:42.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:41 smithi143 conmon[100258]: 2023-08-28T02:05:41.803109+0000 mon.a (mon.0) 36 : audit [DBG] from='mgr.14116 172.21.15.143:0/3275888306' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-08-28T02:05:42.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:41 smithi143 conmon[100258]: audit 2023-08-28T02:05:41. 2023-08-28T02:05:42.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:41 smithi143 conmon[100258]: 803263+0000 mon.a (mon.0) 37 : audit [DBG] from='mgr.14116 172.21.15.143:0/3275888306' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-08-28T02:05:42.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:41 smithi143 conmon[100258]: audit 2023-08-28T02:05:41 2023-08-28T02:05:42.204 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:41 smithi143 conmon[100258]: .803408+0000 mon.a (mon.0) 38 : audit [DBG] from='mgr.14116 172.21.15.143:0/3275888306' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-08-28T02:05:42.204 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:41 smithi143 conmon[100258]: cluster 2023-08-28T02:05 2023-08-28T02:05:42.204 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:41 smithi143 conmon[100258]: :41.817441+0000 mon.a (mon.0) 39 : cluster [INF] Manager daemon y is now available 2023-08-28T02:05:42.204 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:41 smithi143 conmon[100258]: 2023-08-28T02:05:42.204 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:41 smithi143 conmon[100258]: audit 2023-08-28T02 2023-08-28T02:05:42.204 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:41 smithi143 conmon[100258]: :05:41.829367+0000 mon.a (mon.0) 40 : audit [INF] from='mgr.14116 172.21.15.143:0/3275888306' entity='mgr.y' 2023-08-28T02:05:42.809 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: { 2023-08-28T02:05:42.809 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2023-08-28T02:05:42.809 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "initialized": true 2023-08-28T02:05:42.809 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: } 2023-08-28T02:05:43.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:42 smithi143 conmon[100258]: cephadm 2023-08-28T02:05:41.826839+0000 mgr.y (mgr.14116) 1 : cephadm [INF] Found migration_current of "None". Setting to last migration. 2023-08-28T02:05:43.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:42 smithi143 conmon[100258]: audit 2023-08-28T02:05:41.913719+0000 mon.a (mon.0) 41 : audit [INF] from='mgr.14116 172.21.15.143:0/3275888306' entity='mgr.y' 2023-08-28T02:05:43.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:42 smithi143 conmon[100258]: audit 2023-08-28T02:05:43.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:42 smithi143 conmon[100258]: 2023-08-28T02:05:41.914783+0000 mon.a (mon.0) 42 : audit [DBG] from='mgr.14116 172.21.15.143:0/3275888306' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:05:43.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:42 smithi143 conmon[100258]: audit 2023-08 2023-08-28T02:05:43.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:42 smithi143 conmon[100258]: -28T02:05:41.916011+0000 mon.a (mon 2023-08-28T02:05:43.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:42 smithi143 conmon[100258]: .0) 43 : audit [DBG] from='mgr.14116 172.21.15.143:0/3275888306' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:05:43.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:42 smithi143 conmon[100258]: audit 2023-08-28T 2023-08-28T02:05:43.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:42 smithi143 conmon[100258]: 02:05:41.917551+0000 mon.a (mon.0) 2023-08-28T02:05:43.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:42 smithi143 conmon[100258]: 44 : audit [DBG] from='mgr.14116 172.21.15.143:0/3275888306' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:05:43.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:42 smithi143 conmon[100258]: audit 2023-08-28T02 2023-08-28T02:05:43.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:42 smithi143 conmon[100258]: :05:41.929711+0000 mon.a (mon.0) 2023-08-28T02:05:43.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:42 smithi143 conmon[100258]: 45 : audit [INF] from='mgr.14116 172.21.15.143:0/3275888306' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-08-28T02:05:43.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:42 smithi143 conmon[100258]: audit 2023-08-28T02:05 2023-08-28T02:05:43.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:42 smithi143 conmon[100258]: :41.932118+0000 mon.a (mon.0) 46 : 2023-08-28T02:05:43.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:42 smithi143 conmon[100258]: audit [INF] from='mgr.14116 172.21.15.143:0/3275888306' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-08-28T02:05:43.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:42 smithi143 conmon[100258]: cluster 2023-08-28T02:05:42 2023-08-28T02:05:43.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:42 smithi143 conmon[100258]: .803495+0000 mon.a (mon.0) 47 : cluster 2023-08-28T02:05:43.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:42 smithi143 conmon[100258]: [DBG] mgrmap e7: y(active, since 1.00561s) 2023-08-28T02:05:43.398 INFO:teuthology.orchestra.run.smithi143.stderr:mgr epoch 5 is available 2023-08-28T02:05:43.398 INFO:teuthology.orchestra.run.smithi143.stderr:Setting orchestrator backend to cephadm... 2023-08-28T02:05:44.199 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:43 smithi143 conmon[100258]: audit 2023-08-28T02:05:42.802961+0000 mgr.y (mgr.14116) 2 : audit [DBG] from='client.14120 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-08-28T02:05:44.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:43 smithi143 conmon[100258]: audit 2023-08-28T02: 2023-08-28T02:05:44.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:43 smithi143 conmon[100258]: 05:42.809825+0000 mgr.y (mgr.14116) 3 : audit [DBG] from='client.14120 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-08-28T02:05:45.167 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: value unchanged 2023-08-28T02:05:45.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:44 smithi143 conmon[100258]: cluster 2023-08-28T02:05:43.915860+0000 mon.a (mon.0) 48 : cluster [DBG] mgrmap e8: y(active, since 2s) 2023-08-28T02:05:45.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:44 smithi143 conmon[100258]: audit 2023- 2023-08-28T02:05:45.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:44 smithi143 conmon[100258]: 08-28T02:05:44.033449+0000 mgr.y (mgr.14116) 4 : audit [DBG] from='client.14126 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:05:45.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:44 smithi143 conmon[100258]: audit 2023-08-28T02:05:44.038117+0000 mon.a (mon.0) 49 : audit [INF] from='mgr.14116 172.21.15.143:0/3275888306' entity='mgr.y' 2023-08-28T02:05:45.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:44 smithi143 conmon[100258]: audit 2023-08-28T02:05:45.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:44 smithi143 conmon[100258]: 2023-08-28T02:05:44.088972+0000 mon.a (mon.0) 50 : audit [DBG] from='mgr.14116 172.21.15.143:0/3275888306' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:05:45.697 INFO:teuthology.orchestra.run.smithi143.stderr:Generating ssh key... 2023-08-28T02:05:46.449 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:46 smithi143 conmon[100258]: audit 2023-08-28T02:05:45.168917+0000 mgr.y (mgr.14116) 5 : audit [DBG] from='client.14128 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:05:46.930 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:46 smithi143 conmon[100708]: Generating public/private rsa key pair. 2023-08-28T02:05:46.931 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:46 smithi143 conmon[100708]: Your identification has been saved in /tmp/tmplj0s196p/key. 2023-08-28T02:05:46.931 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:46 smithi143 conmon[100708]: Your public key has been saved in /tmp/tmplj0s196p/key.pub. 2023-08-28T02:05:46.931 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:46 smithi143 conmon[100708]: The key fingerprint is: 2023-08-28T02:05:46.931 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:46 smithi143 conmon[100708]: SHA256:D0fRNMYxphja8kP25dHAJRurqwjXAQXwZUXkzb0iUiI ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 2023-08-28T02:05:46.931 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:46 smithi143 conmon[100708]: The key's randomart image is: 2023-08-28T02:05:46.932 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:46 smithi143 conmon[100708]: +---[RSA 3072]----+ 2023-08-28T02:05:46.932 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:46 smithi143 conmon[100708]: | ....=+==@o. | 2023-08-28T02:05:46.932 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:46 smithi143 conmon[100708]: | . * + B+@ | 2023-08-28T02:05:46.932 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:46 smithi143 conmon[100708]: | E * = B o | 2023-08-28T02:05:46.932 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:46 smithi143 conmon[100708]: | B = + . . | 2023-08-28T02:05:46.932 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:46 smithi143 conmon[100708]: | S = o . | 2023-08-28T02:05:46.932 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:46 smithi143 conmon[100708]: | . B o . | 2023-08-28T02:05:46.932 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:46 smithi143 conmon[100708]: | . . . o | 2023-08-28T02:05:46.932 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:46 smithi143 conmon[100708]: | o . . | 2023-08-28T02:05:46.933 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:46 smithi143 conmon[100708]: | . . | 2023-08-28T02:05:46.933 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:46 smithi143 conmon[100708]: +----[SHA256]-----+ 2023-08-28T02:05:47.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:47 smithi143 conmon[100258]: audit 2023-08-28T02:05:46 2023-08-28T02:05:47.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:47 smithi143 conmon[100258]: .346499+0000 mgr.y (mgr.14116) 6 : audit [DBG] from='client.14130 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:05:47.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:47 smithi143 conmon[100258]: cephadm 2023-08-28T02 2023-08-28T02:05:47.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:47 smithi143 conmon[100258]: :05:46.346689+0000 mgr.y (mgr.14116) 7 : cephadm [INF] Generating ssh key... 2023-08-28T02:05:47.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:47 smithi143 conmon[100258]: audit 2023-08-28T02:05:47.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:47 smithi143 conmon[100258]: 2023-08-28T02:05 2023-08-28T02:05:47.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:47 smithi143 conmon[100258]: :46.490428+0000 mon.a (mon.0) 51 2023-08-28T02:05:47.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:47 smithi143 conmon[100258]: : audit [INF] from='mgr.14116 172.21.15.143:0/3275888306' entity='mgr.y' 2023-08-28T02:05:47.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:47 smithi143 conmon[100258]: audit 2023-08-28T02: 2023-08-28T02:05:47.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:47 smithi143 conmon[100258]: 05:46.492145+0000 mon.a (mon.0) 2023-08-28T02:05:47.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:47 smithi143 conmon[100258]: 52 : audit [INF] from='mgr.14116 172.21.15.143:0/3275888306' entity='mgr.y' 2023-08-28T02:05:47.563 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDakCG94QqO1LnLnI/hQPKm/UtRCVsnGctQhUgSERHsqafP4kb7uS0YkP6u2TEMCMriNUPVU1tpCBzqHTaLssrA0ZH/dt0KaCwUrFLApIaPdF5qyU0G94KCATddgnDV3EhKo9M9M25Q78ryWC0vr7zofdOxhG/S84pDO4d85nPMQHzdI57w/61Oi7Egvt1WQEqf7a7dBmOeyYNQ97YqUMKQnMP59wm+gazxh64/N3EY8irBlqEiutBKYu77xBE5u4Bh463hakVpYPstPZECR9SHFGqC8VwBYvmpJnKJfg0+sdxvqgSJhtinaYW001itWJyoQcr/Zoqwo/He9CGi4BJu8m2YO3XMD8UCOMYlA4gEloNdk9pvkRtpWb1zRKUscoGsN8Xq60nDNtQSnN8RJDhMThdshsPFvrFvCIvxg6SDdsHB8kf2vvg/mxYD4SyfM04AvSjWmLgrVWdRaEdYD8CDPZdE4x2TJbv7iGXHShxWBGTe/S9e7HX9MTxMVN2WQmU= ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 2023-08-28T02:05:48.089 INFO:teuthology.orchestra.run.smithi143.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2023-08-28T02:05:48.089 INFO:teuthology.orchestra.run.smithi143.stderr:Adding key to root@localhost authorized_keys... 2023-08-28T02:05:48.090 INFO:teuthology.orchestra.run.smithi143.stderr:Adding host smithi143... 2023-08-28T02:05:48.807 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:48 smithi143 conmon[100258]: audit 2023-08-28T02:05:47.564644+0000 2023-08-28T02:05:48.807 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:48 smithi143 conmon[100258]: mgr.y (mgr.14116) 8 : audit [DBG] from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:05:48.807 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:48 smithi143 conmon[100708]: [80B blob data] 2023-08-28T02:05:49.354 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: Added host 'smithi143' with addr '172.21.15.143' 2023-08-28T02:05:49.947 INFO:teuthology.orchestra.run.smithi143.stderr:Deploying unmanaged mon service... 2023-08-28T02:05:50.612 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:50 smithi143 conmon[100258]: audit 2023-08-28T02:05:48.710993+0000 mgr.y (mgr.14116) 9 : audit [DBG] from='client.14134 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi143", "addr": "172.21.15.143", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:05:50.612 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:50 smithi143 conmon[100258]: audit 2023-08-28T02:05:49.353770+0000 mon.a (mon.0) 53 : audit [INF] from='mgr.14116 172.21.15.143:0/3275888306' entity='mgr.y' 2023-08-28T02:05:50.612 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:50 smithi143 conmon[100258]: cephadm 2023-08-28T02:05:49.354061+0000 mgr.y (mgr.14116) 10 : cephadm [INF] Added host smithi143 2023-08-28T02:05:50.612 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:50 smithi143 conmon[100258]: audit 2023-08-28T02:05:49.363095+0000 mon.a (mon.0) 54 : audit [DBG] from='mgr.14116 172.21.15.143:0/3275888306' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:05:50.641 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: Scheduled mon update... 2023-08-28T02:05:51.501 INFO:teuthology.orchestra.run.smithi143.stderr:Deploying unmanaged mgr service... 2023-08-28T02:05:51.853 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:51 smithi143 conmon[100258]: audit 2023-08-28T02:05:50.635515+0000 mgr.y (mgr.14116) 11 : audit [DBG] from='client.14136 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:05:51.854 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:51 smithi143 conmon[100258]: cephadm 2023-08-28T02:05:50.637109+0000 mgr.y (mgr.14116) 12 : cephadm [INF] Saving service mon spec with placement count:5 2023-08-28T02:05:51.854 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:51 smithi143 conmon[100258]: audit 2023-08-28T02:05:50.642263+0000 mon.a (mon.0) 55 : audit [INF] from='mgr.14116 172.21.15.143:0/3275888306' entity='mgr.y' 2023-08-28T02:05:52.333 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: Scheduled mgr update... 2023-08-28T02:05:53.008 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:52 smithi143 conmon[100258]: audit 2023-08-28T02:05:51.715704+0000 mon.a 2023-08-28T02:05:53.009 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:52 smithi143 conmon[100258]: (mon.0) 56 : audit [INF] from='mgr.14116 172.21.15.143:0/3275888306' entity='mgr.y' 2023-08-28T02:05:53.009 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:52 smithi143 conmon[100258]: audit 2023-08-28T02:05:51.977297+0000 mon.a (mon.0) 57 : audit [INF] from='mgr.14116 172.21.15.143:0/3275888306' entity='mgr.y' 2023-08-28T02:05:53.009 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:52 smithi143 conmon[100258]: audit 2023-08-28T02:05:52.330574+0000 mgr.y (mgr.14116) 13 : audit [DBG] from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:05:53.009 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:52 smithi143 conmon[100258]: cephadm 2023-08-28T02:05:52.331477+0000 mgr.y (mgr.14116) 14 : cephadm [INF] Saving service mgr spec with placement count:2 2023-08-28T02:05:53.009 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:52 smithi143 conmon[100258]: audit 2023-08-28T02:05:52.334411+0000 mon.a (mon.0) 58 : audit [INF] from='mgr.14116 172.21.15.143:0/3275888306' entity='mgr.y' 2023-08-28T02:05:54.562 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:54 smithi143 conmon[100258]: audit 2023-08-28T02:05:53.369241+0000 mon.a (mon 2023-08-28T02:05:54.562 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:54 smithi143 conmon[100258]: .0) 59 : audit [INF] from='mgr.14116 172.21.15.143:0/3275888306' entity='mgr.y' 2023-08-28T02:05:54.562 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:54 smithi143 conmon[100258]: audit 2023-08-28T02:05:53.777477+0000 mon.a (mon.0) 60 : audit [INF] from='client.? 172.21.15.143:0/3474701581' entity='client.admin' 2023-08-28T02:05:56.087 INFO:teuthology.orchestra.run.smithi143.stderr:Enabling the dashboard module... 2023-08-28T02:05:56.949 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:56 smithi143 conmon[100258]: audit 2023-08-28T02:05:55.511179+0000 mon.a (mon.0) 61 : audit [INF] from='client.? 172.21.15.143:0/998679376' entity='client.admin' 2023-08-28T02:05:57.771 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:57 smithi143 conmon[100258]: audit 2023-08-28T02:05:56.716292+0000 mon.a (mon.0) 62 : audit [INF] from='client.? 172.21.15.143:0/543898148' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2023-08-28T02:05:57.772 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:57 smithi143 conmon[100708]: ignoring --setuser ceph since I am not root 2023-08-28T02:05:57.772 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:57 smithi143 conmon[100708]: ignoring --setgroup ceph since I am not root 2023-08-28T02:05:57.772 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:57 smithi143 conmon[100708]: debug 2023-08-28T02:05:57.687+0000 7f0864c0f680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-08-28T02:05:58.139 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:57 smithi143 conmon[100708]: debug 2023-08-28T02:05:57.772+0000 7f0864c0f680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-08-28T02:05:58.399 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:58 smithi143 conmon[100708]: debug 2023-08-28T02:05:58.140+0000 7f0864c0f680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-08-28T02:05:58.813 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: { 2023-08-28T02:05:58.813 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "epoch": 9, 2023-08-28T02:05:58.813 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "available": true, 2023-08-28T02:05:58.813 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "active_name": "y", 2023-08-28T02:05:58.814 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "num_standby": 0 2023-08-28T02:05:58.814 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: } 2023-08-28T02:05:58.839 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:58 smithi143 conmon[100258]: audit 2023-08-28T02:05:57.518144+0000 mon.a (mon.0) 63 : audit 2023-08-28T02:05:58.840 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:58 smithi143 conmon[100258]: [INF] from='client.? 172.21.15.143:0/543898148' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2023-08-28T02:05:58.840 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:58 smithi143 conmon[100258]: cluster 2023-08-28T02:05:57.518220+0000 mon.a (mon.0) 64 : cluster [DBG] mgrmap e9: y(active, since 15s) 2023-08-28T02:05:59.199 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:58 smithi143 conmon[100708]: debug 2023-08-28T02:05:58.920+0000 7f0864c0f680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-08-28T02:05:59.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:59 smithi143 conmon[100708]: debug 2023-08-28T02:05:59.008+0000 7f0864c0f680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-08-28T02:05:59.395 INFO:teuthology.orchestra.run.smithi143.stderr:Waiting for the mgr to restart... 2023-08-28T02:05:59.395 INFO:teuthology.orchestra.run.smithi143.stderr:Waiting for mgr epoch 9... 2023-08-28T02:05:59.515 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:05:59 smithi143 conmon[100708]: debug 2023-08-28T02:05:59.220+0000 7f0864c0f680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-08-28T02:05:59.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:05:59 smithi143 conmon[100258]: audit 2023-08-28T02:05:58.814324+0000 mon.a (mon.0) 65 : audit [DBG] from='client.? 172.21.15.143:0/2722990115' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-08-28T02:06:00.449 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:06:00 smithi143 conmon[100708]: debug 2023-08-28T02:06:00.191+0000 7f0864c0f680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-08-28T02:06:00.450 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:06:00 smithi143 conmon[100708]: debug 2023-08-28T02:06:00.319+0000 7f0864c0f680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-08-28T02:06:00.450 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:06:00 smithi143 conmon[100708]: debug 2023-08-28T02:06:00.396+0000 7f0864c0f680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-08-28T02:06:00.949 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:06:00 smithi143 conmon[100708]: debug 2023-08-28T02:06:00.486+0000 7f0864c0f680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-08-28T02:06:00.950 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:06:00 smithi143 conmon[100708]: debug 2023-08-28T02:06:00.682+0000 7f0864c0f680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-08-28T02:06:01.449 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:06:00 smithi143 conmon[100708]: debug 2023-08-28T02:06:00.958+0000 7f0864c0f680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-08-28T02:06:01.568 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:06:01 smithi143 conmon[100708]: debug 2023-08-28T02:06:01.055+0000 7f0864c0f680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-08-28T02:06:02.114 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:06:01 smithi143 conmon[100708]: debug 2023-08-28T02:06:01.771+0000 7f0864c0f680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-08-28T02:06:02.114 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:06:01 smithi143 conmon[100708]: debug 2023-08-28T02:06:01.851+0000 7f0864c0f680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-08-28T02:06:02.114 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:06:01 smithi143 conmon[100708]: debug 2023-08-28T02:06:01.942+0000 7f0864c0f680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-08-28T02:06:02.374 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:06:02 smithi143 conmon[100708]: debug 2023-08-28T02:06:02.114+0000 7f0864c0f680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-08-28T02:06:02.642 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:06:02 smithi143 conmon[100708]: debug 2023-08-28T02:06:02.375+0000 7f0864c0f680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-08-28T02:06:02.642 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:06:02 smithi143 conmon[100708]: debug 2023-08-28T02:06:02.504+0000 7f0864c0f680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-08-28T02:06:02.896 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:06:02 smithi143 conmon[100708]: debug 2023-08-28T02:06:02.642+0000 7f0864c0f680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-08-28T02:06:02.896 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:06:02 smithi143 conmon[100708]: debug 2023-08-28T02:06:02.812+0000 7f0864c0f680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-08-28T02:06:03.199 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:06:02 smithi143 conmon[100708]: debug 2023-08-28T02:06:02.897+0000 7f0864c0f680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-08-28T02:06:03.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:06:03 smithi143 conmon[100708]: [80B blob data] 2023-08-28T02:06:03.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:02 smithi143 conmon[100258]: cluster 2023-08-28T02:06:02.911048+0000 mon.a (mon.0) 66 : 2023-08-28T02:06:03.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:02 smithi143 conmon[100258]: cluster [INF] Active manager daemon y restarted 2023-08-28T02:06:03.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:02 smithi143 conmon[100258]: cluster 2023-08-28T02:06:02.911291+0000 mon.a (mon.0) 67 : cluster [INF] Activating manager daemon y 2023-08-28T02:06:03.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:02 smithi143 conmon[100258]: cluster 2023-08-28T02:06:02.914160 2023-08-28T02:06:03.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:02 smithi143 conmon[100258]: +0000 mon.a (mon.0) 68 : cluster [DBG] osdmap e3: 0 total, 0 up, 0 in 2023-08-28T02:06:03.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:02 smithi143 conmon[100258]: cluster 2023-08-28T02:06:03.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:02 smithi143 conmon[100258]: 2023-08-28T02:06:02.914274 2023-08-28T02:06:03.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:02 smithi143 conmon[100258]: +0000 mon.a (mon.0) 69 : cluster 2023-08-28T02:06:03.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:02 smithi143 conmon[100258]: [DBG] mgrmap e10: y(active, starting, since 0.00307748s) 2023-08-28T02:06:03.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:02 smithi143 conmon[100258]: audit 2023-08 2023-08-28T02:06:03.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:02 smithi143 conmon[100258]: -28T02:06:02.915225+0000 mon.a ( 2023-08-28T02:06:03.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:02 smithi143 conmon[100258]: mon.0) 70 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T02:06:03.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:02 smithi143 conmon[100258]: 2023-08-28T02:06:03.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:02 smithi143 conmon[100258]: audit 2023-08-28 2023-08-28T02:06:03.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:02 smithi143 conmon[100258]: T02:06:02.915854+0000 mon.a ( 2023-08-28T02:06:03.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:02 smithi143 conmon[100258]: mon.0) 71 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-08-28T02:06:03.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:02 smithi143 conmon[100258]: 2023-08-28T02:06:03.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:02 smithi143 conmon[100258]: audit 2023-08-28T02:06: 2023-08-28T02:06:03.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:02 smithi143 conmon[100258]: 02.916604+0000 mon.a (mon.0) 72 2023-08-28T02:06:03.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:02 smithi143 conmon[100258]: : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-08-28T02:06:03.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:02 smithi143 conmon[100258]: audit 2023-08- 2023-08-28T02:06:03.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:02 smithi143 conmon[100258]: 28T02:06:02.916785+0000 mon.a (mon 2023-08-28T02:06:03.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:02 smithi143 conmon[100258]: .0) 73 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-08-28T02:06:03.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:02 smithi143 conmon[100258]: 2023-08-28T02:06:03.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:02 smithi143 conmon[100258]: audit 2023-08 2023-08-28T02:06:03.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:02 smithi143 conmon[100258]: -28T02:06:02.916930+0000 mon.a 2023-08-28T02:06:03.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:02 smithi143 conmon[100258]: (mon.0) 74 : audit [DBG] 2023-08-28T02:06:03.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:02 smithi143 conmon[100258]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-08-28T02:06:03.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:02 smithi143 conmon[100258]: cluster 2023-08-28T02:06:03.204 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:02 smithi143 conmon[100258]: 2023-08-28T02:06:02.930362+0000 2023-08-28T02:06:03.204 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:02 smithi143 conmon[100258]: mon.a (mon.0) 75 : cluster 2023-08-28T02:06:03.204 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:02 smithi143 conmon[100258]: [INF] Manager daemon y is now available 2023-08-28T02:06:03.921 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: { 2023-08-28T02:06:03.921 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2023-08-28T02:06:03.921 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: "initialized": true 2023-08-28T02:06:03.921 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: } 2023-08-28T02:06:04.199 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:03 smithi143 conmon[100258]: audit 2023-08-28T02:06:03.023094+0000 mon.a (mon.0) 76 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:06:04.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:03 smithi143 conmon[100258]: audit 2023-08-28T02:06:03.024441+0000 mon.a (mon.0) 77 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:06:04.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:03 smithi143 conmon[100258]: audit 2023-08-28T02:06:03.048151+0000 mon.a (mon.0) 78 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-08-28T02:06:04.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:03 smithi143 conmon[100258]: audit 2023-08-28T02:06:03.050963+0000 mon.a (mon.0) 79 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-08-28T02:06:04.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:03 smithi143 conmon[100258]: cluster 2023-08-28T02:06:03.916051+0000 mon.a (mon.0) 80 : cluster [DBG] mgrmap e11: y(active, since 1.00485s) 2023-08-28T02:06:04.575 INFO:teuthology.orchestra.run.smithi143.stderr:mgr epoch 9 is available 2023-08-28T02:06:04.575 INFO:teuthology.orchestra.run.smithi143.stderr:Generating a dashboard self-signed certificate... 2023-08-28T02:06:05.303 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: Self-signed certificate created 2023-08-28T02:06:05.324 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:04 smithi143 conmon[100258]: audit 2023-08-28T02:06:03.915529+0000 mgr.y (mgr.14146 2023-08-28T02:06:05.325 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:04 smithi143 conmon[100258]: ) 1 : audit [DBG] from='client.14150 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-08-28T02:06:05.325 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:04 smithi143 conmon[100258]: audit 2023-08-28T02:06:03.921891+0000 mgr.y (mgr.14146) 2 : audit [DBG] from='client.14150 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-08-28T02:06:05.325 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:04 smithi143 conmon[100258]: audit 2023-08-28T02:06:04.562368+0000 mon.a (mon.0) 81 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:06.252 INFO:teuthology.orchestra.run.smithi143.stderr:Creating initial admin user... 2023-08-28T02:06:06.449 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:06 smithi143 conmon[100258]: audit 2023-08-28T02:06:05.189478+0000 mgr.y (mgr.14146) 3 : audit [DBG] from='client.14156 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:06:06.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:06 smithi143 conmon[100258]: audit 2023-08-28T02:06:05.301488+0000 mon.a (mon.0) 82 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:06.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:06 smithi143 conmon[100258]: audit 2023-08-28T02:06:05.304060+0000 mon.a (mon.0) 83 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:06.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:06 smithi143 conmon[100258]: cluster 2023-08-28T02:06:05.566543+0000 mon.a (mon.0) 84 : cluster [DBG] mgrmap e12: y(active, since 2s) 2023-08-28T02:06:07.095 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$mcMM0r4PyaZ53GsPGCcpAuiCbI6VPBi8FV7jjMAa6Afi4UmbhZ24q", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1693188367, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2023-08-28T02:06:07.627 INFO:teuthology.orchestra.run.smithi143.stderr:Fetching dashboard port number... 2023-08-28T02:06:08.199 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:08 smithi143 conmon[100258]: audit 2023-08-28T02:06: 2023-08-28T02:06:08.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:08 smithi143 conmon[100258]: 06.880377+0000 mgr.y (mgr.14146) 4 : audit [DBG] from='client.14158 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:06:08.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:08 smithi143 conmon[100258]: audit 2023-08-28T02:06:07.096808+0000 mon.a (mon.0) 85 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:08.270 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: 8443 2023-08-28T02:06:08.854 INFO:teuthology.orchestra.run.smithi143.stderr:systemctl: disabled 2023-08-28T02:06:08.860 INFO:teuthology.orchestra.run.smithi143.stderr:systemctl: inactive 2023-08-28T02:06:08.861 INFO:teuthology.orchestra.run.smithi143.stderr:firewalld.service is not enabled 2023-08-28T02:06:08.861 INFO:teuthology.orchestra.run.smithi143.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2023-08-28T02:06:08.863 INFO:teuthology.orchestra.run.smithi143.stderr:Ceph Dashboard is now available at: 2023-08-28T02:06:08.863 INFO:teuthology.orchestra.run.smithi143.stderr: 2023-08-28T02:06:08.863 INFO:teuthology.orchestra.run.smithi143.stderr: URL: https://smithi143:8443/ 2023-08-28T02:06:08.863 INFO:teuthology.orchestra.run.smithi143.stderr: User: admin 2023-08-28T02:06:08.864 INFO:teuthology.orchestra.run.smithi143.stderr: Password: 9cdv0zegj8 2023-08-28T02:06:08.864 INFO:teuthology.orchestra.run.smithi143.stderr: 2023-08-28T02:06:08.864 INFO:teuthology.orchestra.run.smithi143.stderr:Enabling autotune for osd_memory_target 2023-08-28T02:06:09.109 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:09 smithi143 conmon[100258]: audit 2023-08-28T02:06:08.271925+0000 mon.a (mon.0) 86 : audit 2023-08-28T02:06:09.109 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:09 smithi143 conmon[100258]: [DBG] from='client.? 172.21.15.143:0/3103568531' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2023-08-28T02:06:10.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:10 smithi143 conmon[100258]: cluster 2023-08-28T02:06:09.099670+0000 mon.a (mon.0) 87 : cluster [DBG] mgrmap e13: y(active, since 6s) 2023-08-28T02:06:10.646 INFO:teuthology.orchestra.run.smithi143.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2023-08-28T02:06:11.226 INFO:teuthology.orchestra.run.smithi143.stderr:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2023-08-28T02:06:11.227 INFO:teuthology.orchestra.run.smithi143.stderr: 2023-08-28T02:06:11.227 INFO:teuthology.orchestra.run.smithi143.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2023-08-28T02:06:11.227 INFO:teuthology.orchestra.run.smithi143.stderr: 2023-08-28T02:06:11.227 INFO:teuthology.orchestra.run.smithi143.stderr:Or, if you are only running a single cluster on this host: 2023-08-28T02:06:11.227 INFO:teuthology.orchestra.run.smithi143.stderr: 2023-08-28T02:06:11.227 INFO:teuthology.orchestra.run.smithi143.stderr: sudo /home/ubuntu/cephtest/cephadm shell 2023-08-28T02:06:11.227 INFO:teuthology.orchestra.run.smithi143.stderr: 2023-08-28T02:06:11.227 INFO:teuthology.orchestra.run.smithi143.stderr:Please consider enabling telemetry to help improve Ceph: 2023-08-28T02:06:11.227 INFO:teuthology.orchestra.run.smithi143.stderr: 2023-08-28T02:06:11.227 INFO:teuthology.orchestra.run.smithi143.stderr: ceph telemetry on 2023-08-28T02:06:11.227 INFO:teuthology.orchestra.run.smithi143.stderr: 2023-08-28T02:06:11.228 INFO:teuthology.orchestra.run.smithi143.stderr:For more information see: 2023-08-28T02:06:11.228 INFO:teuthology.orchestra.run.smithi143.stderr: 2023-08-28T02:06:11.228 INFO:teuthology.orchestra.run.smithi143.stderr: https://docs.ceph.com/en/pacific/mgr/telemetry/ 2023-08-28T02:06:11.228 INFO:teuthology.orchestra.run.smithi143.stderr: 2023-08-28T02:06:11.228 INFO:teuthology.orchestra.run.smithi143.stderr:Bootstrap complete. 2023-08-28T02:06:11.265 INFO:tasks.cephadm:Fetching config... 2023-08-28T02:06:11.265 DEBUG:teuthology.orchestra.run.smithi143:> set -ex 2023-08-28T02:06:11.265 DEBUG:teuthology.orchestra.run.smithi143:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2023-08-28T02:06:11.281 INFO:tasks.cephadm:Fetching client.admin keyring... 2023-08-28T02:06:11.281 DEBUG:teuthology.orchestra.run.smithi143:> set -ex 2023-08-28T02:06:11.281 DEBUG:teuthology.orchestra.run.smithi143:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2023-08-28T02:06:11.338 INFO:tasks.cephadm:Fetching mon keyring... 2023-08-28T02:06:11.339 DEBUG:teuthology.orchestra.run.smithi143:> set -ex 2023-08-28T02:06:11.339 DEBUG:teuthology.orchestra.run.smithi143:> sudo dd if=/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/keyring of=/dev/stdout 2023-08-28T02:06:11.405 INFO:tasks.cephadm:Fetching pub ssh key... 2023-08-28T02:06:11.405 DEBUG:teuthology.orchestra.run.smithi143:> set -ex 2023-08-28T02:06:11.405 DEBUG:teuthology.orchestra.run.smithi143:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2023-08-28T02:06:11.463 INFO:tasks.cephadm:Installing pub ssh key for root users... 2023-08-28T02:06:11.463 DEBUG:teuthology.orchestra.run.smithi143:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDakCG94QqO1LnLnI/hQPKm/UtRCVsnGctQhUgSERHsqafP4kb7uS0YkP6u2TEMCMriNUPVU1tpCBzqHTaLssrA0ZH/dt0KaCwUrFLApIaPdF5qyU0G94KCATddgnDV3EhKo9M9M25Q78ryWC0vr7zofdOxhG/S84pDO4d85nPMQHzdI57w/61Oi7Egvt1WQEqf7a7dBmOeyYNQ97YqUMKQnMP59wm+gazxh64/N3EY8irBlqEiutBKYu77xBE5u4Bh463hakVpYPstPZECR9SHFGqC8VwBYvmpJnKJfg0+sdxvqgSJhtinaYW001itWJyoQcr/Zoqwo/He9CGi4BJu8m2YO3XMD8UCOMYlA4gEloNdk9pvkRtpWb1zRKUscoGsN8Xq60nDNtQSnN8RJDhMThdshsPFvrFvCIvxg6SDdsHB8kf2vvg/mxYD4SyfM04AvSjWmLgrVWdRaEdYD8CDPZdE4x2TJbv7iGXHShxWBGTe/S9e7HX9MTxMVN2WQmU= ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-08-28T02:06:11.542 INFO:teuthology.orchestra.run.smithi143.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDakCG94QqO1LnLnI/hQPKm/UtRCVsnGctQhUgSERHsqafP4kb7uS0YkP6u2TEMCMriNUPVU1tpCBzqHTaLssrA0ZH/dt0KaCwUrFLApIaPdF5qyU0G94KCATddgnDV3EhKo9M9M25Q78ryWC0vr7zofdOxhG/S84pDO4d85nPMQHzdI57w/61Oi7Egvt1WQEqf7a7dBmOeyYNQ97YqUMKQnMP59wm+gazxh64/N3EY8irBlqEiutBKYu77xBE5u4Bh463hakVpYPstPZECR9SHFGqC8VwBYvmpJnKJfg0+sdxvqgSJhtinaYW001itWJyoQcr/Zoqwo/He9CGi4BJu8m2YO3XMD8UCOMYlA4gEloNdk9pvkRtpWb1zRKUscoGsN8Xq60nDNtQSnN8RJDhMThdshsPFvrFvCIvxg6SDdsHB8kf2vvg/mxYD4SyfM04AvSjWmLgrVWdRaEdYD8CDPZdE4x2TJbv7iGXHShxWBGTe/S9e7HX9MTxMVN2WQmU= ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 2023-08-28T02:06:11.555 DEBUG:teuthology.orchestra.run.smithi169:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDakCG94QqO1LnLnI/hQPKm/UtRCVsnGctQhUgSERHsqafP4kb7uS0YkP6u2TEMCMriNUPVU1tpCBzqHTaLssrA0ZH/dt0KaCwUrFLApIaPdF5qyU0G94KCATddgnDV3EhKo9M9M25Q78ryWC0vr7zofdOxhG/S84pDO4d85nPMQHzdI57w/61Oi7Egvt1WQEqf7a7dBmOeyYNQ97YqUMKQnMP59wm+gazxh64/N3EY8irBlqEiutBKYu77xBE5u4Bh463hakVpYPstPZECR9SHFGqC8VwBYvmpJnKJfg0+sdxvqgSJhtinaYW001itWJyoQcr/Zoqwo/He9CGi4BJu8m2YO3XMD8UCOMYlA4gEloNdk9pvkRtpWb1zRKUscoGsN8Xq60nDNtQSnN8RJDhMThdshsPFvrFvCIvxg6SDdsHB8kf2vvg/mxYD4SyfM04AvSjWmLgrVWdRaEdYD8CDPZdE4x2TJbv7iGXHShxWBGTe/S9e7HX9MTxMVN2WQmU= ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-08-28T02:06:11.601 INFO:teuthology.orchestra.run.smithi169.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDakCG94QqO1LnLnI/hQPKm/UtRCVsnGctQhUgSERHsqafP4kb7uS0YkP6u2TEMCMriNUPVU1tpCBzqHTaLssrA0ZH/dt0KaCwUrFLApIaPdF5qyU0G94KCATddgnDV3EhKo9M9M25Q78ryWC0vr7zofdOxhG/S84pDO4d85nPMQHzdI57w/61Oi7Egvt1WQEqf7a7dBmOeyYNQ97YqUMKQnMP59wm+gazxh64/N3EY8irBlqEiutBKYu77xBE5u4Bh463hakVpYPstPZECR9SHFGqC8VwBYvmpJnKJfg0+sdxvqgSJhtinaYW001itWJyoQcr/Zoqwo/He9CGi4BJu8m2YO3XMD8UCOMYlA4gEloNdk9pvkRtpWb1zRKUscoGsN8Xq60nDNtQSnN8RJDhMThdshsPFvrFvCIvxg6SDdsHB8kf2vvg/mxYD4SyfM04AvSjWmLgrVWdRaEdYD8CDPZdE4x2TJbv7iGXHShxWBGTe/S9e7HX9MTxMVN2WQmU= ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 2023-08-28T02:06:11.614 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2023-08-28T02:06:11.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:11 smithi143 conmon[100258]: audit 2023-08-28T02:06:10.644945+0000 mon.a (mon.0) 88 : audit [INF] from='client.? 172.21.15.143:0/984358140' entity='client.admin' 2023-08-28T02:06:14.873 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2023-08-28T02:06:14.873 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph orch client-keyring set client.admin '*' --mode 0755 2023-08-28T02:06:15.337 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:15 smithi143 conmon[100258]: audit 2023-08-28T02:06:14.075174+0000 mon.a (mon.0) 89 : audit [INF] from='client.? 172.21.15.143:0/1471337095' entity='client.admin' 2023-08-28T02:06:16.312 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:16 smithi143 conmon[100258]: audit 2023-08-28T02:06:15.083936+0000 mon.a (mon.0) 90 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:16.313 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:16 smithi143 conmon[100258]: audit 2023-08-28T02:06:15.428581+0000 mon.a (mon.0) 91 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi143", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:06:16.313 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:16 smithi143 conmon[100258]: audit 2023-08-28T02:06:15.431738+0000 mon.a (mon.0) 92 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:16.313 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:16 smithi143 conmon[100258]: audit 2023-08-28T02:06:15.432979+0000 mon.a (mon.0) 93 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:06:16.313 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:16 smithi143 conmon[100258]: audit 2023-08-28T02:06:15.436249+0000 mon.a (mon.0) 94 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:17.189 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi169 2023-08-28T02:06:17.189 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2023-08-28T02:06:17.189 DEBUG:teuthology.orchestra.run.smithi169:> dd of=/etc/ceph/ceph.conf 2023-08-28T02:06:17.210 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2023-08-28T02:06:17.210 DEBUG:teuthology.orchestra.run.smithi169:> dd of=/etc/ceph/ceph.client.admin.keyring 2023-08-28T02:06:17.270 INFO:tasks.cephadm:Adding host smithi169 to orchestrator... 2023-08-28T02:06:17.270 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph orch host add smithi169 2023-08-28T02:06:17.949 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:17 smithi143 conmon[100258]: audit 2023-08-28T02:06:16.630848+0000 mon.a (mon.0) 95 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:17.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:17 smithi143 conmon[100258]: audit 2023-08-28T02:06:16.652900+0000 mon.a (mon.0) 96 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:06:17.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:17 smithi143 conmon[100258]: audit 2023-08-28T02:06:16.654123+0000 mon.a (mon.0) 97 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:17.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:17 smithi143 conmon[100258]: audit 2023-08-28T02:06:16.654781+0000 mon.a (mon.0) 98 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:06:17.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:17 smithi143 conmon[100258]: cephadm 2023-08-28T02:06:16.655918+0000 mgr.y (mgr.14146) 6 : cephadm [INF] Updating smithi143:/etc/ceph/ceph.conf 2023-08-28T02:06:17.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:17 smithi143 conmon[100258]: cephadm 2023-08-28T02:06:16.697053+0000 mgr.y (mgr.14146) 7 : cephadm [INF] Updating smithi143:/etc/ceph/ceph.client.admin.keyring 2023-08-28T02:06:17.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:17 smithi143 conmon[100258]: audit 2023-08-28T02:06:16.740564+0000 mon.a (mon.0) 99 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:17.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:17 smithi143 conmon[100258]: audit 2023-08-28T02:06:16.745405+0000 mon.a (mon.0) 100 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:19.450 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:06:19 smithi143 conmon[100708]: [90B blob data] 2023-08-28T02:06:19.693 INFO:teuthology.orchestra.run.smithi143.stdout:Added host 'smithi169' with addr '172.21.15.169' 2023-08-28T02:06:19.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:19 smithi143 conmon[100258]: audit 2023-08-28T02:06:19.014237+0000 mgr.y (mgr 2023-08-28T02:06:19.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:19 smithi143 conmon[100258]: .14146) 8 : audit [DBG] from='client.14170 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi169", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:06:19.950 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:06:19 smithi143 conmon[100708]: [80B blob data] 2023-08-28T02:06:20.297 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph orch host ls --format=json 2023-08-28T02:06:20.949 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:20 smithi143 conmon[100258]: audit 2023-08-28T02:06:19.694736+0000 mon.a (mon.0) 101 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:20.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:20 smithi143 conmon[100258]: cephadm 2023-08-28T02:06:19.695016+0000 mgr.y (mgr.14146) 9 : cephadm [INF] Added host smithi169 2023-08-28T02:06:20.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:20 smithi143 conmon[100258]: audit 2023-08-28T02:06:19.720751+0000 mon.a (mon.0) 102 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:06:20.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:20 smithi143 conmon[100258]: audit 2023-08-28T02:06:20.182905+0000 mon.a (mon.0) 103 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:20.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:20 smithi143 conmon[100258]: audit 2023-08-28T02:06:20.517193+0000 mon.a (mon.0) 104 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:21.998 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:06:21.998 INFO:teuthology.orchestra.run.smithi143.stdout:[{"addr": "172.21.15.143", "hostname": "smithi143", "labels": [], "status": ""}, {"addr": "172.21.15.169", "hostname": "smithi169", "labels": [], "status": ""}] 2023-08-28T02:06:22.547 INFO:tasks.cephadm:Setting crush tunables to default 2023-08-28T02:06:22.548 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd crush tunables default 2023-08-28T02:06:22.949 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:22 smithi143 conmon[100258]: audit 2023-08-28T02:06:21.999300+0000 mgr.y (mgr. 2023-08-28T02:06:22.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:22 smithi143 conmon[100258]: 14146) 10 : audit [DBG] from='client.14172 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:06:24.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:24 smithi143 conmon[100258]: cluster 2023-08-28T02:06:22.918101+0000 mgr.y (mgr.14146) 11 : cluster [DBG] pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:06:24.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:24 smithi143 conmon[100258]: audit 2023-08-28T02:06:23.422594+0000 mon.a (mon.0) 105 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:24.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:24 smithi143 conmon[100258]: audit 2023-08-28T02:06:23.755891+0000 mon.a (mon.0) 106 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:24.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:24 smithi143 conmon[100258]: audit 2023-08-28T02:06:23.756558+0000 mon.a (mon.0) 107 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi169", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:06:24.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:24 smithi143 conmon[100258]: audit 2023-08-28T02:06:23.757384+0000 mon.a (mon.0) 108 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:24.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:24 smithi143 conmon[100258]: audit 2023-08-28T02:06:23.757988+0000 mon.a (mon.0) 109 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:06:24.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:24 smithi143 conmon[100258]: audit 2023-08-28T02:06:23.807123+0000 mon.a (mon.0) 110 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:24.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:24 smithi143 conmon[100258]: audit 2023-08-28T02:06:23.811783+0000 mon.a (mon.0) 111 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:24.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:24 smithi143 conmon[100258]: audit 2023-08-28T02:06:24.372254+0000 mon.a (mon.0) 112 : audit [INF] from='client.? 172.21.15.143:0/1890648645' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-08-28T02:06:24.814 INFO:teuthology.orchestra.run.smithi143.stderr:adjusted tunables profile to default 2023-08-28T02:06:25.415 INFO:tasks.cephadm:Adding mon.a on smithi143 2023-08-28T02:06:25.415 INFO:tasks.cephadm:Adding mon.c on smithi143 2023-08-28T02:06:25.415 INFO:tasks.cephadm:Adding mon.b on smithi169 2023-08-28T02:06:25.416 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph orch apply mon '3;smithi143:172.21.15.143=a;smithi143:[v2:172.21.15.143:3301,v1:172.21.15.143:6790]=c;smithi169:172.21.15.169=b' 2023-08-28T02:06:25.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:25 smithi143 conmon[100258]: cephadm 2023-08-28T02:06:23.758974+0000 mgr.y (mgr.14146 2023-08-28T02:06:25.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:25 smithi143 conmon[100258]: ) 12 : cephadm [INF] Updating smithi169:/etc/ceph/ceph.conf 2023-08-28T02:06:25.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:25 smithi143 conmon[100258]: cephadm 2023-08-28T02:06:23.786979+0000 mgr.y (mgr.14146) 13 : cephadm 2023-08-28T02:06:25.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:25 smithi143 conmon[100258]: [INF] Updating smithi169:/etc/ceph/ceph.client.admin.keyring 2023-08-28T02:06:25.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:25 smithi143 conmon[100258]: audit 2023-08 2023-08-28T02:06:25.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:25 smithi143 conmon[100258]: -28T02:06:24.812574+0000 mon.a (mon. 2023-08-28T02:06:25.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:25 smithi143 conmon[100258]: 0) 113 : audit [INF] from='client.? 172.21.15.143:0/1890648645' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-08-28T02:06:25.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:25 smithi143 conmon[100258]: cluster 2023-08- 2023-08-28T02:06:25.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:25 smithi143 conmon[100258]: 28T02:06:24.812639+0000 mon.a 2023-08-28T02:06:25.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:25 smithi143 conmon[100258]: (mon.0) 114 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-08-28T02:06:26.699 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:26 smithi143 conmon[100258]: cluster 2023-08-28T02:06:24.918377+0000 mgr.y (mgr.14146) 14 : cluster [DBG] 2023-08-28T02:06:26.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:26 smithi143 conmon[100258]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:06:27.133 INFO:teuthology.orchestra.run.smithi169.stdout:Scheduled mon update... 2023-08-28T02:06:27.674 DEBUG:teuthology.orchestra.run.smithi143:mon.c> sudo journalctl -f -n 0 -u ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@mon.c.service 2023-08-28T02:06:27.677 DEBUG:teuthology.orchestra.run.smithi169:mon.b> sudo journalctl -f -n 0 -u ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@mon.b.service 2023-08-28T02:06:27.679 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-08-28T02:06:27.679 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph mon dump -f json 2023-08-28T02:06:27.710 INFO:journalctl@ceph.mon.b.smithi169.stdout:-- Logs begin at Mon 2023-08-28 01:49:52 UTC. -- 2023-08-28T02:06:27.713 INFO:journalctl@ceph.mon.c.smithi143.stdout:-- Logs begin at Mon 2023-08-28 01:49:41 UTC. -- 2023-08-28T02:06:28.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:28 smithi143 conmon[100258]: cluster 2023-08-28T02:06:26.918730+0000 mgr.y (mgr.14146) 15 : cluster [DBG] pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:06:28.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:28 smithi143 conmon[100258]: audit 2023-08-28T02:06:27.128633+0000 mgr.y (mgr.14146) 16 : audit [DBG] from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi143:172.21.15.143=a;smithi143:[v2:172.21.15.143:3301,v1:172.21.15.143:6790]=c;smithi169:172.21.15.169=b", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:06:28.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:28 smithi143 conmon[100258]: cephadm 2023-08-28T02:06:27.130842+0000 mgr.y (mgr.14146) 17 : cephadm [INF] Saving service mon spec with placement smithi143:172.21.15.143=a;smithi143:[v2:172.21.15.143:3301,v1:172.21.15.143:6790]=c;smithi169:172.21.15.169=b;count:3 2023-08-28T02:06:28.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:28 smithi143 conmon[100258]: audit 2023-08-28T02:06:27.133478+0000 mon.a (mon.0) 115 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:28.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:28 smithi143 conmon[100258]: audit 2023-08-28T02:06:27.184033+0000 mon.a (mon.0) 116 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:06:28.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:28 smithi143 conmon[100258]: audit 2023-08-28T02:06:27.185688+0000 mon.a (mon.0) 117 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:28.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:28 smithi143 conmon[100258]: audit 2023-08-28T02:06:27.186493+0000 mon.a (mon.0) 2023-08-28T02:06:28.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:28 smithi143 conmon[100258]: 118 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:06:28.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:28 smithi143 conmon[100258]: audit 2023-08-28T02:06:27.190309+0000 mon.a (mon.0) 119 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:28.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:28 smithi143 conmon[100258]: audit 2023-08-28T02:06:27.192132 2023-08-28T02:06:28.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:28 smithi143 conmon[100258]: +0000 mon.a (mon.0) 120 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-28T02:06:28.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:28 smithi143 conmon[100258]: audit 2023-08 2023-08-28T02:06:28.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:28 smithi143 conmon[100258]: -28T02:06:27.192919+0000 mon.a (mon.0) 121 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:28.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:28 smithi143 conmon[100258]: cephadm 2023 2023-08-28T02:06:28.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:28 smithi143 conmon[100258]: -08-28T02:06:27.193629+0000 mgr.y 2023-08-28T02:06:28.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:28 smithi143 conmon[100258]: (mgr.14146) 18 : cephadm [INF] Deploying daemon mon.b on smithi169 2023-08-28T02:06:30.043 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:06:30.043 INFO:teuthology.orchestra.run.smithi169.stdout:{"epoch":1,"fsid":"2fcb0bfa-4547-11ee-817c-1b7a5f1ba315","modified":"2023-08-28T02:05:17.823964Z","created":"2023-08-28T02:05:17.823964Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:3300","nonce":0},{"type":"v1","addr":"172.21.15.143:6789","nonce":0}]},"addr":"172.21.15.143:6789/0","public_addr":"172.21.15.143:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-08-28T02:06:30.045 INFO:teuthology.orchestra.run.smithi169.stderr:dumped monmap epoch 1 2023-08-28T02:06:30.449 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:30 smithi143 conmon[100258]: cluster 2023-08-28T02:06:28.919218+0000 mgr.y (mgr.14146) 19 : cluster [DBG] pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:06:30.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:30 smithi143 conmon[100258]: audit 2023-08-28T02:06:30.044661+0000 mon.a (mon.0) 122 : audit [DBG] from='client.? 172.21.15.169:0/2175612367' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-08-28T02:06:31.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 systemd[1]: Starting Ceph mon.b for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:06:31.918 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-08-28T02:06:31.919 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph mon dump -f json 2023-08-28T02:06:32.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.779+0000 7f2d5abda880 0 set uid:gid to 167:167 (ceph:ceph) 2023-08-28T02:06:32.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.779+0000 7f2d5abda880 0 ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable), process ceph-mon, pid 7 2023-08-28T02:06:32.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.779+0000 7f2d5abda880 0 pidfile_write: ignore empty --pid-file 2023-08-28T02:06:32.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.783+0000 7f2d5abda880 0 load: jerasure load: lrc load: isa 2023-08-28T02:06:32.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: RocksDB version: 6.8.1 2023-08-28T02:06:32.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: 2023-08-28T02:06:32.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-08-28T02:06:32.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Compile date Aug 22 2023 2023-08-28T02:06:32.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: DB SUMMARY 2023-08-28T02:06:32.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: 2023-08-28T02:06:32.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: CURRENT file: CURRENT 2023-08-28T02:06:32.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: 2023-08-28T02:06:32.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: IDENTITY file: IDENTITY 2023-08-28T02:06:32.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: 2023-08-28T02:06:32.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: MANIFEST file: MANIFEST-000001 size: 13 Bytes 2023-08-28T02:06:32.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: 2023-08-28T02:06:32.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2023-08-28T02:06:32.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: 2023-08-28T02:06:32.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000003.log size: 511 ; 2023-08-28T02:06:32.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: 2023-08-28T02:06:32.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.error_if_exists: 0 2023-08-28T02:06:32.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.create_if_missing: 0 2023-08-28T02:06:32.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.paranoid_checks: 1 2023-08-28T02:06:32.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.env: 0x559f6d03a080 2023-08-28T02:06:32.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.fs: Posix File System 2023-08-28T02:06:32.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.info_log: 0x559f6d953a80 2023-08-28T02:06:32.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.max_file_opening_threads: 16 2023-08-28T02:06:32.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.statistics: (nil) 2023-08-28T02:06:32.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.use_fsync: 0 2023-08-28T02:06:32.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.max_log_file_size: 0 2023-08-28T02:06:32.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-08-28T02:06:32.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.log_file_time_to_roll: 0 2023-08-28T02:06:32.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.keep_log_file_num: 1000 2023-08-28T02:06:32.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.recycle_log_file_num: 0 2023-08-28T02:06:32.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.allow_fallocate: 1 2023-08-28T02:06:32.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.allow_mmap_reads: 0 2023-08-28T02:06:32.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.allow_mmap_writes: 0 2023-08-28T02:06:32.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.use_direct_reads: 0 2023-08-28T02:06:32.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-08-28T02:06:32.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.create_missing_column_families: 0 2023-08-28T02:06:32.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.db_log_dir: 2023-08-28T02:06:32.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2023-08-28T02:06:32.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.table_cache_numshardbits: 6 2023-08-28T02:06:32.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.max_subcompactions: 1 2023-08-28T02:06:32.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.max_background_flushes: -1 2023-08-28T02:06:32.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-08-28T02:06:32.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-08-28T02:06:32.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-08-28T02:06:32.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-08-28T02:06:32.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-08-28T02:06:32.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.advise_random_on_open: 1 2023-08-28T02:06:32.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.db_write_buffer_size: 0 2023-08-28T02:06:32.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.write_buffer_manager: 0x559f6e64e2d0 2023-08-28T02:06:32.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-08-28T02:06:32.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-08-28T02:06:32.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-08-28T02:06:32.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.use_adaptive_mutex: 0 2023-08-28T02:06:32.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.rate_limiter: (nil) 2023-08-28T02:06:32.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-08-28T02:06:32.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.wal_recovery_mode: 2 2023-08-28T02:06:32.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.enable_thread_tracking: 0 2023-08-28T02:06:32.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.enable_pipelined_write: 0 2023-08-28T02:06:32.243 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.unordered_write: 0 2023-08-28T02:06:32.243 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-08-28T02:06:32.243 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-08-28T02:06:32.243 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-08-28T02:06:32.243 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-08-28T02:06:32.243 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.row_cache: None 2023-08-28T02:06:32.243 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.wal_filter: None 2023-08-28T02:06:32.243 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-08-28T02:06:32.243 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.allow_ingest_behind: 0 2023-08-28T02:06:32.243 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.preserve_deletes: 0 2023-08-28T02:06:32.243 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.two_write_queues: 0 2023-08-28T02:06:32.243 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.manual_wal_flush: 0 2023-08-28T02:06:32.244 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.atomic_flush: 0 2023-08-28T02:06:32.244 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-08-28T02:06:32.244 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.persist_stats_to_disk: 0 2023-08-28T02:06:32.244 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-08-28T02:06:32.244 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.log_readahead_size: 0 2023-08-28T02:06:32.244 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-08-28T02:06:32.244 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.max_background_jobs: 2 2023-08-28T02:06:32.244 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.max_background_compactions: -1 2023-08-28T02:06:32.244 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-08-28T02:06:32.244 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-08-28T02:06:32.245 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.delayed_write_rate : 16777216 2023-08-28T02:06:32.246 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.max_total_wal_size: 0 2023-08-28T02:06:32.246 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-08-28T02:06:32.246 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.stats_dump_period_sec: 600 2023-08-28T02:06:32.246 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.stats_persist_period_sec: 600 2023-08-28T02:06:32.246 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-08-28T02:06:32.246 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.max_open_files: -1 2023-08-28T02:06:32.246 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.bytes_per_sync: 0 2023-08-28T02:06:32.246 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-08-28T02:06:32.246 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-08-28T02:06:32.246 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.compaction_readahead_size: 0 2023-08-28T02:06:32.247 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Compression algorithms supported: 2023-08-28T02:06:32.247 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-08-28T02:06:32.247 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: kZSTD supported: 0 2023-08-28T02:06:32.247 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: kXpressCompression supported: 0 2023-08-28T02:06:32.247 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: kLZ4HCCompression supported: 1 2023-08-28T02:06:32.247 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: kLZ4Compression supported: 1 2023-08-28T02:06:32.247 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: kBZip2Compression supported: 0 2023-08-28T02:06:32.247 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: kZlibCompression supported: 1 2023-08-28T02:06:32.247 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: kSnappyCompression supported: 1 2023-08-28T02:06:32.247 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-08-28T02:06:32.247 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000001 2023-08-28T02:06:32.247 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: 2023-08-28T02:06:32.247 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-08-28T02:06:32.248 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: 2023-08-28T02:06:32.248 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-08-28T02:06:32.248 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.merge_operator: 2023-08-28T02:06:32.248 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.compaction_filter: None 2023-08-28T02:06:32.248 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.compaction_filter_factory: None 2023-08-28T02:06:32.248 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-08-28T02:06:32.248 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.table_factory: BlockBasedTable 2023-08-28T02:06:32.248 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x559f6d8fdd18) 2023-08-28T02:06:32.248 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: cache_index_and_filter_blocks: 1 2023-08-28T02:06:32.248 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: cache_index_and_filter_blocks_with_high_priority: 0 2023-08-28T02:06:32.248 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-08-28T02:06:32.248 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: pin_top_level_index_and_filter: 1 2023-08-28T02:06:32.249 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: index_type: 0 2023-08-28T02:06:32.249 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: data_block_index_type: 0 2023-08-28T02:06:32.249 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: index_shortening: 1 2023-08-28T02:06:32.249 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: data_block_hash_table_util_ratio: 0.750000 2023-08-28T02:06:32.249 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: hash_index_allow_collision: 1 2023-08-28T02:06:32.249 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: checksum: 1 2023-08-28T02:06:32.249 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: no_block_cache: 0 2023-08-28T02:06:32.249 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: block_cache: 0x559f6d936f10 2023-08-28T02:06:32.249 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: block_cache_name: BinnedLRUCache 2023-08-28T02:06:32.249 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: block_cache_options: 2023-08-28T02:06:32.249 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: capacity : 536870912 2023-08-28T02:06:32.249 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: num_shard_bits : 4 2023-08-28T02:06:32.249 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: strict_capacity_limit : 0 2023-08-28T02:06:32.250 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: high_pri_pool_ratio: 0.000 2023-08-28T02:06:32.250 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: block_cache_compressed: (nil) 2023-08-28T02:06:32.250 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: persistent_cache: (nil) 2023-08-28T02:06:32.250 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: block_size: 4096 2023-08-28T02:06:32.250 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: block_size_deviation: 10 2023-08-28T02:06:32.250 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: block_restart_interval: 16 2023-08-28T02:06:32.250 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: index_block_restart_interval: 1 2023-08-28T02:06:32.250 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: metadata_block_size: 4096 2023-08-28T02:06:32.250 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: partition_filters: 0 2023-08-28T02:06:32.250 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: use_delta_encoding: 1 2023-08-28T02:06:32.250 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: filter_policy: rocksdb.BuiltinBloomFilter 2023-08-28T02:06:32.250 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: whole_key_filtering: 1 2023-08-28T02:06:32.250 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: verify_compression: 0 2023-08-28T02:06:32.251 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: read_amp_bytes_per_bit: 0 2023-08-28T02:06:32.251 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: format_version: 2 2023-08-28T02:06:32.251 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: enable_index_compression: 1 2023-08-28T02:06:32.251 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: block_align: 0 2023-08-28T02:06:32.251 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: 2023-08-28T02:06:32.251 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.write_buffer_size: 33554432 2023-08-28T02:06:32.251 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.max_write_buffer_number: 2 2023-08-28T02:06:32.251 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.compression: NoCompression 2023-08-28T02:06:32.251 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.bottommost_compression: Disabled 2023-08-28T02:06:32.251 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.prefix_extractor: nullptr 2023-08-28T02:06:32.251 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-08-28T02:06:32.251 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.num_levels: 7 2023-08-28T02:06:32.252 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-08-28T02:06:32.252 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-08-28T02:06:32.252 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-08-28T02:06:32.252 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-08-28T02:06:32.252 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-08-28T02:06:32.252 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-08-28T02:06:32.252 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-08-28T02:06:32.252 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-08-28T02:06:32.252 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-08-28T02:06:32.252 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.compression_opts.window_bits: -14 2023-08-28T02:06:32.252 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.compression_opts.level: 32767 2023-08-28T02:06:32.252 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.compression_opts.strategy: 0 2023-08-28T02:06:32.252 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-08-28T02:06:32.253 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-08-28T02:06:32.253 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.compression_opts.enabled: false 2023-08-28T02:06:32.253 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-08-28T02:06:32.253 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-08-28T02:06:32.253 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-08-28T02:06:32.253 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.target_file_size_base: 67108864 2023-08-28T02:06:32.253 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.target_file_size_multiplier: 1 2023-08-28T02:06:32.253 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-08-28T02:06:32.253 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-08-28T02:06:32.253 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-08-28T02:06:32.253 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-08-28T02:06:32.253 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-08-28T02:06:32.253 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-08-28T02:06:32.254 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-08-28T02:06:32.254 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-08-28T02:06:32.254 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-08-28T02:06:32.254 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-08-28T02:06:32.254 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-08-28T02:06:32.254 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-08-28T02:06:32.254 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.arena_block_size: 4194304 2023-08-28T02:06:32.254 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-08-28T02:06:32.254 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-08-28T02:06:32.255 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-08-28T02:06:32.255 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.disable_auto_compactions: 0 2023-08-28T02:06:32.255 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-08-28T02:06:32.255 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-08-28T02:06:32.255 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-08-28T02:06:32.255 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-08-28T02:06:32.255 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-08-28T02:06:32.255 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-08-28T02:06:32.255 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-08-28T02:06:32.255 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-08-28T02:06:32.255 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-08-28T02:06:32.255 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-08-28T02:06:32.256 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.table_properties_collectors: 2023-08-28T02:06:32.256 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.inplace_update_support: 0 2023-08-28T02:06:32.256 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-08-28T02:06:32.256 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-08-28T02:06:32.256 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-08-28T02:06:32.256 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.memtable_huge_page_size: 0 2023-08-28T02:06:32.256 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.bloom_locality: 0 2023-08-28T02:06:32.256 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.max_successive_merges: 0 2023-08-28T02:06:32.256 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-08-28T02:06:32.256 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.paranoid_file_checks: 0 2023-08-28T02:06:32.256 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.force_consistency_checks: 0 2023-08-28T02:06:32.256 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.report_bg_io_stats: 0 2023-08-28T02:06:32.256 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.ttl: 2592000 2023-08-28T02:06:32.257 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.784+0000 7f2d5abda880 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-08-28T02:06:32.257 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.785+0000 7f2d5abda880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-08-28T02:06:32.257 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: 2023-08-28T02:06:32.257 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.785+0000 7f2d5abda880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 0 2023-08-28T02:06:32.257 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: 2023-08-28T02:06:32.257 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.785+0000 7f2d5abda880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1693188391786385, "job": 1, "event": "recovery_started", "log_files": [3]} 2023-08-28T02:06:32.257 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.785+0000 7f2d5abda880 4 rocksdb: [db_impl/db_impl_open.cc:760] Recovering log #3 mode 2 2023-08-28T02:06:32.257 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.785+0000 7f2d5abda880 3 rocksdb: [le/block_based/filter_policy.cc:584] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-08-28T02:06:32.257 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.785+0000 7f2d5abda880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1693188391786854, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 4, "file_size": 1437, "table_properties": {"data_size": 523, "index_size": 40, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 0, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1693188391, "oldest_key_time": 3, "file_creation_time": 0}} 2023-08-28T02:06:32.257 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.785+0000 7f2d5abda880 4 rocksdb: [version_set.cc:3826] Creating manifest 5 2023-08-28T02:06:32.257 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: 2023-08-28T02:06:32.257 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.786+0000 7f2d5abda880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1693188391787218, "job": 1, "event": "recovery_finished"} 2023-08-28T02:06:32.258 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.786+0000 7f2d5abda880 4 rocksdb: DB pointer 0x559f6da01800 2023-08-28T02:06:32.258 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.786+0000 7f2d419e1700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-08-28T02:06:32.258 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.786+0000 7f2d419e1700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-08-28T02:06:32.258 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: ** DB Stats ** 2023-08-28T02:06:32.258 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: Uptime(secs): 0.0 total, 0.0 interval 2023-08-28T02:06:32.258 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-08-28T02:06:32.258 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-08-28T02:06:32.258 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-08-28T02:06:32.258 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-08-28T02:06:32.258 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-08-28T02:06:32.258 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-08-28T02:06:32.258 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: 2023-08-28T02:06:32.258 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: ** Compaction Stats [default] ** 2023-08-28T02:06:32.259 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-08-28T02:06:32.259 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-08-28T02:06:32.259 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: L0 1/0 1.40 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.2 0.00 0.00 1 0.000 0 0 2023-08-28T02:06:32.259 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: Sum 1/0 1.40 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.2 0.00 0.00 1 0.000 0 0 2023-08-28T02:06:32.259 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.2 0.00 0.00 1 0.000 0 0 2023-08-28T02:06:32.259 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: 2023-08-28T02:06:32.259 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: ** Compaction Stats [default] ** 2023-08-28T02:06:32.259 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-08-28T02:06:32.259 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-08-28T02:06:32.259 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.2 0.00 0.00 1 0.000 0 0 2023-08-28T02:06:32.259 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: Uptime(secs): 0.0 total, 0.0 interval 2023-08-28T02:06:32.259 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: Flush(GB): cumulative 0.000, interval 0.000 2023-08-28T02:06:32.259 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: AddFile(GB): cumulative 0.000, interval 0.000 2023-08-28T02:06:32.260 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: AddFile(Total Files): cumulative 0, interval 0 2023-08-28T02:06:32.260 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: AddFile(L0 Files): cumulative 0, interval 0 2023-08-28T02:06:32.260 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: AddFile(Keys): cumulative 0, interval 0 2023-08-28T02:06:32.260 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: Cumulative compaction: 0.00 GB write, 0.63 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-08-28T02:06:32.260 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: Interval compaction: 0.00 GB write, 0.63 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-08-28T02:06:32.260 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-08-28T02:06:32.260 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: 2023-08-28T02:06:32.260 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: ** File Read Latency Histogram By Level [default] ** 2023-08-28T02:06:32.260 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: 2023-08-28T02:06:32.260 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: ** Compaction Stats [default] ** 2023-08-28T02:06:32.260 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-08-28T02:06:32.260 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-08-28T02:06:32.261 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: L0 1/0 1.40 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.2 0.00 0.00 1 0.000 0 0 2023-08-28T02:06:32.261 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: Sum 1/0 1.40 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.2 0.00 0.00 1 0.000 0 0 2023-08-28T02:06:32.261 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-08-28T02:06:32.261 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: 2023-08-28T02:06:32.261 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: ** Compaction Stats [default] ** 2023-08-28T02:06:32.261 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-08-28T02:06:32.261 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-08-28T02:06:32.261 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.2 0.00 0.00 1 0.000 0 0 2023-08-28T02:06:32.261 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: Uptime(secs): 0.0 total, 0.0 interval 2023-08-28T02:06:32.261 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: Flush(GB): cumulative 0.000, interval 0.000 2023-08-28T02:06:32.261 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: AddFile(GB): cumulative 0.000, interval 0.000 2023-08-28T02:06:32.261 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: AddFile(Total Files): cumulative 0, interval 0 2023-08-28T02:06:32.261 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: AddFile(L0 Files): cumulative 0, interval 0 2023-08-28T02:06:32.262 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: AddFile(Keys): cumulative 0, interval 0 2023-08-28T02:06:32.262 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: Cumulative compaction: 0.00 GB write, 0.62 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-08-28T02:06:32.262 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-08-28T02:06:32.262 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-08-28T02:06:32.262 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: 2023-08-28T02:06:32.262 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: ** File Read Latency Histogram By Level [default] ** 2023-08-28T02:06:32.262 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: 2023-08-28T02:06:32.262 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.786+0000 7f2d5abda880 0 mon.b does not exist in monmap, will attempt to join an existing cluster 2023-08-28T02:06:32.262 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.787+0000 7f2d5abda880 0 using public_addr v2:172.21.15.169:0/0 -> [v2:172.21.15.169:3300/0,v1:172.21.15.169:6789/0] 2023-08-28T02:06:32.262 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.787+0000 7f2d5abda880 0 starting mon.b rank -1 at public addrs [v2:172.21.15.169:3300/0,v1:172.21.15.169:6789/0] at bind addrs [v2:172.21.15.169:3300/0,v1:172.21.15.169:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 2023-08-28T02:06:32.262 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.788+0000 7f2d5abda880 1 mon.b@-1(???) e0 preinit fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 2023-08-28T02:06:32.262 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 bash[100459]: f70db2856b371aec7993513e6dcc7fead6faa126ca29a9ad317904181f61ff20 2023-08-28T02:06:32.262 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.803+0000 7f2d441e6700 0 mon.b@-1(synchronizing).mds e1 new map 2023-08-28T02:06:32.263 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.803+0000 7f2d441e6700 0 mon.b@-1(synchronizing).mds e1 print_map 2023-08-28T02:06:32.263 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: e1 2023-08-28T02:06:32.263 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: enable_multiple, ever_enabled_multiple: 1,1 2023-08-28T02:06:32.263 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-08-28T02:06:32.263 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: legacy client fscid: -1 2023-08-28T02:06:32.263 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: 2023-08-28T02:06:32.263 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: No filesystems configured 2023-08-28T02:06:32.263 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: 2023-08-28T02:06:32.263 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.803+0000 7f2d441e6700 1 mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-08-28T02:06:32.263 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.803+0000 7f2d441e6700 1 mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-08-28T02:06:32.263 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.803+0000 7f2d441e6700 1 mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-08-28T02:06:32.263 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.803+0000 7f2d441e6700 1 mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-08-28T02:06:32.263 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.803+0000 7f2d441e6700 1 mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-08-28T02:06:32.264 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.803+0000 7f2d441e6700 1 mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-08-28T02:06:32.264 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.803+0000 7f2d441e6700 0 mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-08-28T02:06:32.264 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.803+0000 7f2d441e6700 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-08-28T02:06:32.264 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.803+0000 7f2d441e6700 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-08-28T02:06:32.264 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.803+0000 7f2d441e6700 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-08-28T02:06:32.264 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: cluster 2023-08 2023-08-28T02:06:32.264 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: -28T02:06:28.919218+0000 mgr.y (mgr.14146) 19 : cluster [DBG] 2023-08-28T02:06:32.264 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:06:32.264 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: audit 2023-08- 2023-08-28T02:06:32.265 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: 28T02:06:30.044661+0000 mon.a (mon.0) 122 : audit [DBG] from='client.? 172.21.15.169:0/2175612367' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-08-28T02:06:32.265 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 conmon[100500]: debug 2023-08-28T02:06:31.803+0000 7f2d441e6700 1 mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-08-28T02:06:32.265 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:31 smithi169 systemd[1]: Started Ceph mon.b for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:06:37.098 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:06:37.099 INFO:teuthology.orchestra.run.smithi169.stdout:{"epoch":2,"fsid":"2fcb0bfa-4547-11ee-817c-1b7a5f1ba315","modified":"2023-08-28T02:06:31.808782Z","created":"2023-08-28T02:05:17.823964Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:3300","nonce":0},{"type":"v1","addr":"172.21.15.143:6789","nonce":0}]},"addr":"172.21.15.143:6789/0","public_addr":"172.21.15.143:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:3300","nonce":0},{"type":"v1","addr":"172.21.15.169:6789","nonce":0}]},"addr":"172.21.15.169:6789/0","public_addr":"172.21.15.169:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2023-08-28T02:06:37.100 INFO:teuthology.orchestra.run.smithi169.stderr:dumped monmap epoch 2 2023-08-28T02:06:37.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:36 smithi143 conmon[100258]: audit 2023-08-28T02:06:31.811401+0000 mon.a (mon.0) 124 : 2023-08-28T02:06:37.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:36 smithi143 conmon[100258]: audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T02:06:37.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:36 smithi143 conmon[100258]: cluster 2023-08-28T02:06:31.811569+0000 mon.a (mon.0) 125 : cluster [INF] mon.a calling monitor election 2023-08-28T02:06:37.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:36 smithi143 conmon[100258]: audit 2023-08-28T02:06:31.824994+0000 mon.a (mon.0) 126 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T02:06:37.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:36 smithi143 conmon[100258]: audit 2023-08-28T02:06:32.809208+0000 mon.a (mon.0) 127 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T02:06:37.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:36 smithi143 conmon[100258]: cluster 2023-08-28T02:06:32.919959+0000 mgr.y (mgr 2023-08-28T02:06:37.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:36 smithi143 conmon[100258]: .14146) 21 : cluster [DBG] pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:06:37.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:36 smithi143 conmon[100258]: audit 2023-08-28T02:06:37.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:36 smithi143 conmon[100258]: 2023-08-28T02:06:33.808831+0000 mon.a (mon.0) 128 : audit [DBG] 2023-08-28T02:06:37.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:36 smithi143 conmon[100258]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T02:06:37.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:36 smithi143 conmon[100258]: cluster 2023-08-28T02 2023-08-28T02:06:37.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:36 smithi143 conmon[100258]: :06:33.809611+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2023-08-28T02:06:37.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:36 smithi143 conmon[100258]: audit 2023-08-28T02:06:34.808652+0000 mon.a ( 2023-08-28T02:06:37.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:36 smithi143 conmon[100258]: mon.0) 129 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T02:06:37.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:36 smithi143 conmon[100258]: cluster 2023-08-28T02:06:34.920236+0000 mgr.y (mgr. 2023-08-28T02:06:37.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:36 smithi143 conmon[100258]: 14146) 22 : cluster [DBG] pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:06:37.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:36 smithi143 conmon[100258]: audit 2023-08-28T02:06:37.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:36 smithi143 conmon[100258]: 2023-08-28T02:06:35.808994+0000 mon.a (mon.0) 130 : audit [DBG] 2023-08-28T02:06:37.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:36 smithi143 conmon[100258]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T02:06:37.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:36 smithi143 conmon[100258]: audit 2023-08-28T02 2023-08-28T02:06:37.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:36 smithi143 conmon[100258]: :06:36.809082+0000 mon.a (mon.0) 131 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T02:06:37.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:36 smithi143 conmon[100258]: cluster 2023-08-28T02:06:36.838328+0000 mon.a (mon.0) 132 : cluster [INF] 2023-08-28T02:06:37.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:36 smithi143 conmon[100258]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-08-28T02:06:37.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:36 smithi143 conmon[100258]: cluster 2023-08-28T02: 2023-08-28T02:06:37.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:36 smithi143 conmon[100258]: 06:36.841320+0000 mon.a (mon.0) 133 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.143:3300/0,v1:172.21.15.143:6789/0],b=[v2:172.21.15.169:3300/0,v1:172.21.15.169:6789/0]} removed_ranks: {} 2023-08-28T02:06:37.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:36 smithi143 conmon[100258]: cluster 2023-08-28T02:06:36.853517+0000 mon.a (mon 2023-08-28T02:06:37.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:36 smithi143 conmon[100258]: .0) 134 : cluster [DBG] fsmap 2023-08-28T02:06:37.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:36 smithi143 conmon[100258]: cluster 2023-08-28T02:06:36.853566+0000 mon.a (mon. 2023-08-28T02:06:37.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:36 smithi143 conmon[100258]: 0) 135 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-08-28T02:06:37.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:36 smithi143 conmon[100258]: cluster 2023-08-28T02:06:37.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:36 smithi143 conmon[100258]: 2023-08-28T02:06:36.853772+0000 mon.a (mon.0) 136 : cluster [DBG] mgrmap e13: y(active, since 33s) 2023-08-28T02:06:37.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:36 smithi143 conmon[100258]: cluster 2023-08-28T02:06 2023-08-28T02:06:37.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:36 smithi143 conmon[100258]: :36.858295+0000 mon.a (mon.0) 137 : cluster [INF] overall HEALTH_OK 2023-08-28T02:06:37.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:36 smithi143 conmon[100258]: audit 2023-08-28T02:06: 2023-08-28T02:06:37.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:36 smithi143 conmon[100258]: 36.861722+0000 mon.a (mon.0) 138 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:37.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:36 smithi143 conmon[100258]: audit 2023-08-28T02:06:37.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:36 smithi143 conmon[100258]: 2023-08-28T02:06:36.866523+0000 mon.a (mon.0) 139 : audit [INF] 2023-08-28T02:06:37.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:36 smithi143 conmon[100258]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-28T02:06:37.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:36 smithi143 conmon[100258]: audit 2023-08- 2023-08-28T02:06:37.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:36 smithi143 conmon[100258]: 28T02:06:36.867337+0000 mon.a (mon.0) 140 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:37.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:36 smithi169 conmon[100500]: audit 2023-08-28T02:06:31.811401+0000 mon.a (mon.0) 124 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T02:06:37.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:36 smithi169 conmon[100500]: cluster 2023-08-28T02:06:31.811569+0000 mon.a (mon.0) 125 : cluster [INF] mon.a calling monitor election 2023-08-28T02:06:37.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:36 smithi169 conmon[100500]: audit 2023-08-28T02:06:31.824994+0000 mon.a (mon.0) 126 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T02:06:37.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:36 smithi169 conmon[100500]: audit 2023-08-28T02:06:32.809208+0000 mon.a (mon.0) 127 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T02:06:37.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:36 smithi169 conmon[100500]: cluster 2023-08-28T02:06:32.919959+0000 mgr.y (mgr.14146) 21 : cluster [DBG] pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:06:37.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:36 smithi169 conmon[100500]: audit 2023-08-28T02:06:33.808831+0000 mon.a (mon.0) 128 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T02:06:37.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:36 smithi169 conmon[100500]: cluster 2023-08-28T02:06:33.809611+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2023-08-28T02:06:37.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:36 smithi169 conmon[100500]: audit 2023-08-28T02:06:34.808652+0000 mon.a (mon.0) 129 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T02:06:37.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:36 smithi169 conmon[100500]: cluster 2023-08-28T02:06:34.920236+0000 mgr.y (mgr.14146) 22 : cluster [DBG] 2023-08-28T02:06:37.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:36 smithi169 conmon[100500]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:06:37.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:36 smithi169 conmon[100500]: audit 2023-08-28T02:06:35.808994+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T02:06:37.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:36 smithi169 conmon[100500]: audit 2023-08-28T02:06:36.809082+0000 mon.a (mon.0) 131 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T02:06:37.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:36 smithi169 conmon[100500]: cluster 2023-08-28T02:06:36.838328+0000 mon.a (mon.0) 132 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-08-28T02:06:37.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:36 smithi169 conmon[100500]: cluster 2023-08-28T02:06:36.841320+0000 mon.a (mon.0) 133 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.143:3300/0,v1:172.21.15.143:6789/0],b=[v2:172.21.15.169:3300/0,v1:172.21.15.169:6789/0]} removed_ranks: {} 2023-08-28T02:06:37.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:36 smithi169 conmon[100500]: cluster 2023-08-28T02:06:36.853517+0000 mon.a (mon.0) 134 : cluster [DBG] fsmap 2023-08-28T02:06:37.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:36 smithi169 conmon[100500]: cluster 2023-08-28T02:06:36.853566+0000 mon.a (mon.0) 135 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-08-28T02:06:37.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:36 smithi169 conmon[100500]: cluster 2023-08-28T02:06:36.853772+0000 mon.a (mon.0) 136 : cluster [DBG] mgrmap e13: y(active, since 33s) 2023-08-28T02:06:37.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:36 smithi169 conmon[100500]: cluster 2023-08-28T02:06:36.858295+0000 mon.a (mon.0) 137 : cluster [INF] overall HEALTH_OK 2023-08-28T02:06:37.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:36 smithi169 conmon[100500]: audit 2023-08-28T02:06:36.861722+0000 mon.a (mon.0) 138 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:37.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:36 smithi169 conmon[100500]: audit 2023-08-28T02:06:36.866523+0000 mon.a (mon.0) 139 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-28T02:06:37.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:36 smithi169 conmon[100500]: audit 2023-08-28T02:06:36.867337+0000 mon.a (mon.0) 140 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:37.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:37 smithi169 conmon[100500]: cephadm 2023-08-28T02:06:36.868002+0000 mgr.y (mgr.14146) 23 : cephadm [INF] Deploying daemon mon.c on smithi143 2023-08-28T02:06:37.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:37 smithi169 conmon[100500]: cluster 2023-08-28T02:06:37.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:37 smithi169 conmon[100500]: 2023-08-28T02:06:36.920531+0000 mgr.y (mgr.14146) 24 : cluster [DBG] pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:06:37.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:37 smithi169 conmon[100500]: audit 2023-08-28T02:06:37.100101+0000 mon.a (mon.0) 141 : audit [DBG] from='client.? 172.21.15.169:0/4289090380' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-08-28T02:06:37.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:37 smithi169 conmon[100500]: audit 2023-08-28T02:06:37.809515+0000 2023-08-28T02:06:37.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:37 smithi169 conmon[100500]: mon.a (mon.0) 142 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T02:06:38.102 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:37 smithi143 conmon[100258]: cephadm 2023-08-28T02:06:36.868002+0000 mgr.y (mgr.14146) 23 : cephadm [INF] Deploying daemon mon.c on smithi143 2023-08-28T02:06:38.102 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:37 smithi143 conmon[100258]: cluster 2023-08-28T02:06:36.920531+0000 mgr.y (mgr.14146) 24 : cluster [DBG] pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:06:38.102 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:37 smithi143 conmon[100258]: audit 2023-08-28T02:06:37.100101+0000 mon.a (mon.0) 141 : audit [DBG] from='client.? 172.21.15.169:0/4289090380' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-08-28T02:06:38.102 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:37 smithi143 conmon[100258]: audit 2023-08-28T02:06:37.809515+0000 mon.a (mon.0) 142 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T02:06:38.715 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-08-28T02:06:38.715 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph mon dump -f json 2023-08-28T02:06:39.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 systemd[1]: Starting Ceph mon.c for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:06:39.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.720+0000 7f1ba0e40880 0 set uid:gid to 167:167 (ceph:ceph) 2023-08-28T02:06:39.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.722+0000 7f1ba0e40880 0 ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable), process ceph-mon, pid 7 2023-08-28T02:06:39.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.722+0000 7f1ba0e40880 0 pidfile_write: ignore empty --pid-file 2023-08-28T02:06:39.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.725+0000 7f1ba0e40880 0 load: jerasure load: lrc load: isa 2023-08-28T02:06:39.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: RocksDB version: 6.8.1 2023-08-28T02:06:39.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: 2023-08-28T02:06:39.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-08-28T02:06:39.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Compile date Aug 22 2023 2023-08-28T02:06:39.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: DB SUMMARY 2023-08-28T02:06:39.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: 2023-08-28T02:06:39.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: CURRENT file: CURRENT 2023-08-28T02:06:39.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: 2023-08-28T02:06:39.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: IDENTITY file: IDENTITY 2023-08-28T02:06:39.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: 2023-08-28T02:06:39.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: MANIFEST file: MANIFEST-000001 size: 13 Bytes 2023-08-28T02:06:39.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: 2023-08-28T02:06:39.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2023-08-28T02:06:39.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: 2023-08-28T02:06:39.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000003.log size: 636 ; 2023-08-28T02:06:39.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: 2023-08-28T02:06:39.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.error_if_exists: 0 2023-08-28T02:06:39.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.create_if_missing: 0 2023-08-28T02:06:39.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.paranoid_checks: 1 2023-08-28T02:06:39.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.env: 0x55eddf8a0080 2023-08-28T02:06:39.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.fs: Posix File System 2023-08-28T02:06:39.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.info_log: 0x55ede1e71a80 2023-08-28T02:06:39.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.max_file_opening_threads: 16 2023-08-28T02:06:39.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.statistics: (nil) 2023-08-28T02:06:39.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.use_fsync: 0 2023-08-28T02:06:39.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.max_log_file_size: 0 2023-08-28T02:06:39.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-08-28T02:06:39.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.log_file_time_to_roll: 0 2023-08-28T02:06:39.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.keep_log_file_num: 1000 2023-08-28T02:06:39.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.recycle_log_file_num: 0 2023-08-28T02:06:39.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.allow_fallocate: 1 2023-08-28T02:06:39.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.allow_mmap_reads: 0 2023-08-28T02:06:39.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.allow_mmap_writes: 0 2023-08-28T02:06:39.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.use_direct_reads: 0 2023-08-28T02:06:39.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-08-28T02:06:39.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.create_missing_column_families: 0 2023-08-28T02:06:39.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.db_log_dir: 2023-08-28T02:06:39.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2023-08-28T02:06:39.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.table_cache_numshardbits: 6 2023-08-28T02:06:39.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.max_subcompactions: 1 2023-08-28T02:06:39.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.max_background_flushes: -1 2023-08-28T02:06:39.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-08-28T02:06:39.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-08-28T02:06:39.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-08-28T02:06:39.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-08-28T02:06:39.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-08-28T02:06:39.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.advise_random_on_open: 1 2023-08-28T02:06:39.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.db_write_buffer_size: 0 2023-08-28T02:06:39.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.write_buffer_manager: 0x55ede2b6c300 2023-08-28T02:06:39.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-08-28T02:06:39.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-08-28T02:06:39.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-08-28T02:06:39.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.use_adaptive_mutex: 0 2023-08-28T02:06:39.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.rate_limiter: (nil) 2023-08-28T02:06:39.957 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-08-28T02:06:39.957 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.wal_recovery_mode: 2 2023-08-28T02:06:39.957 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.enable_thread_tracking: 0 2023-08-28T02:06:39.957 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.enable_pipelined_write: 0 2023-08-28T02:06:39.957 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.unordered_write: 0 2023-08-28T02:06:39.957 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-08-28T02:06:39.957 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-08-28T02:06:39.957 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-08-28T02:06:39.957 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-08-28T02:06:39.957 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.row_cache: None 2023-08-28T02:06:39.957 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.wal_filter: None 2023-08-28T02:06:39.958 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-08-28T02:06:39.958 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.allow_ingest_behind: 0 2023-08-28T02:06:39.958 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.preserve_deletes: 0 2023-08-28T02:06:39.958 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.two_write_queues: 0 2023-08-28T02:06:39.958 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.manual_wal_flush: 0 2023-08-28T02:06:39.958 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.atomic_flush: 0 2023-08-28T02:06:39.958 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-08-28T02:06:39.958 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.persist_stats_to_disk: 0 2023-08-28T02:06:39.958 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-08-28T02:06:39.958 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.log_readahead_size: 0 2023-08-28T02:06:39.958 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-08-28T02:06:39.958 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.max_background_jobs: 2 2023-08-28T02:06:39.958 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.max_background_compactions: -1 2023-08-28T02:06:39.959 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-08-28T02:06:39.959 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-08-28T02:06:39.959 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.delayed_write_rate : 16777216 2023-08-28T02:06:39.959 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.max_total_wal_size: 0 2023-08-28T02:06:39.959 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-08-28T02:06:39.959 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.stats_dump_period_sec: 600 2023-08-28T02:06:39.959 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.stats_persist_period_sec: 600 2023-08-28T02:06:39.959 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-08-28T02:06:39.959 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.max_open_files: -1 2023-08-28T02:06:39.959 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.bytes_per_sync: 0 2023-08-28T02:06:39.959 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-08-28T02:06:39.959 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-08-28T02:06:39.959 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.compaction_readahead_size: 0 2023-08-28T02:06:39.960 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Compression algorithms supported: 2023-08-28T02:06:39.960 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-08-28T02:06:39.960 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 bash[109882]: 74b6ddae7a3fe1cab02b3d7211e0c901e58bbdc00cf4f2d40e2d06be40c2dd7b 2023-08-28T02:06:39.960 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: kZSTD supported: 0 2023-08-28T02:06:39.960 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: kXpressCompression supported: 0 2023-08-28T02:06:39.960 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: kLZ4HCCompression supported: 1 2023-08-28T02:06:39.960 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: kLZ4Compression supported: 1 2023-08-28T02:06:39.960 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: kBZip2Compression supported: 0 2023-08-28T02:06:39.960 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: kZlibCompression supported: 1 2023-08-28T02:06:39.960 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: kSnappyCompression supported: 1 2023-08-28T02:06:39.960 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-08-28T02:06:39.960 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000001 2023-08-28T02:06:39.960 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: 2023-08-28T02:06:39.961 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-08-28T02:06:39.961 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: 2023-08-28T02:06:39.961 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-08-28T02:06:39.961 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.merge_operator: 2023-08-28T02:06:39.961 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.compaction_filter: None 2023-08-28T02:06:39.961 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.compaction_filter_factory: None 2023-08-28T02:06:39.961 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-08-28T02:06:39.961 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.table_factory: BlockBasedTable 2023-08-28T02:06:39.961 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55ede1e1bd18) 2023-08-28T02:06:39.961 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: cache_index_and_filter_blocks: 1 2023-08-28T02:06:39.961 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: cache_index_and_filter_blocks_with_high_priority: 0 2023-08-28T02:06:39.961 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-08-28T02:06:39.961 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: pin_top_level_index_and_filter: 1 2023-08-28T02:06:39.962 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: index_type: 0 2023-08-28T02:06:39.962 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: data_block_index_type: 0 2023-08-28T02:06:39.962 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: index_shortening: 1 2023-08-28T02:06:39.962 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: data_block_hash_table_util_ratio: 0.750000 2023-08-28T02:06:39.962 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: hash_index_allow_collision: 1 2023-08-28T02:06:39.962 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: checksum: 1 2023-08-28T02:06:39.962 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: no_block_cache: 0 2023-08-28T02:06:39.962 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: block_cache: 0x55ede1e54f10 2023-08-28T02:06:39.962 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: block_cache_name: BinnedLRUCache 2023-08-28T02:06:39.962 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: block_cache_options: 2023-08-28T02:06:39.962 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: capacity : 536870912 2023-08-28T02:06:39.962 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: num_shard_bits : 4 2023-08-28T02:06:39.962 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: strict_capacity_limit : 0 2023-08-28T02:06:39.963 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: high_pri_pool_ratio: 0.000 2023-08-28T02:06:39.963 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: block_cache_compressed: (nil) 2023-08-28T02:06:39.963 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: persistent_cache: (nil) 2023-08-28T02:06:39.963 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: block_size: 4096 2023-08-28T02:06:39.963 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: block_size_deviation: 10 2023-08-28T02:06:39.963 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: block_restart_interval: 16 2023-08-28T02:06:39.963 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: index_block_restart_interval: 1 2023-08-28T02:06:39.963 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: metadata_block_size: 4096 2023-08-28T02:06:39.963 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: partition_filters: 0 2023-08-28T02:06:39.963 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: use_delta_encoding: 1 2023-08-28T02:06:39.963 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: filter_policy: rocksdb.BuiltinBloomFilter 2023-08-28T02:06:39.963 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: whole_key_filtering: 1 2023-08-28T02:06:39.964 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: verify_compression: 0 2023-08-28T02:06:39.964 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: read_amp_bytes_per_bit: 0 2023-08-28T02:06:39.964 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: format_version: 2 2023-08-28T02:06:39.964 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: enable_index_compression: 1 2023-08-28T02:06:39.964 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: block_align: 0 2023-08-28T02:06:39.964 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: 2023-08-28T02:06:39.964 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.write_buffer_size: 33554432 2023-08-28T02:06:39.964 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.max_write_buffer_number: 2 2023-08-28T02:06:39.964 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.compression: NoCompression 2023-08-28T02:06:39.964 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.bottommost_compression: Disabled 2023-08-28T02:06:39.964 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.prefix_extractor: nullptr 2023-08-28T02:06:39.964 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-08-28T02:06:39.964 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.num_levels: 7 2023-08-28T02:06:39.965 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-08-28T02:06:39.965 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-08-28T02:06:39.965 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-08-28T02:06:39.965 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-08-28T02:06:39.965 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-08-28T02:06:39.965 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-08-28T02:06:39.965 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-08-28T02:06:39.965 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-08-28T02:06:39.965 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-08-28T02:06:39.965 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.compression_opts.window_bits: -14 2023-08-28T02:06:39.965 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.compression_opts.level: 32767 2023-08-28T02:06:39.965 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.compression_opts.strategy: 0 2023-08-28T02:06:39.965 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-08-28T02:06:39.965 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-08-28T02:06:39.966 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.compression_opts.enabled: false 2023-08-28T02:06:39.966 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-08-28T02:06:39.966 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-08-28T02:06:39.966 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-08-28T02:06:39.966 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.target_file_size_base: 67108864 2023-08-28T02:06:39.966 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.target_file_size_multiplier: 1 2023-08-28T02:06:39.966 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-08-28T02:06:39.966 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-08-28T02:06:39.966 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-08-28T02:06:39.966 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-08-28T02:06:39.966 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-08-28T02:06:39.966 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-08-28T02:06:39.966 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-08-28T02:06:39.967 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-08-28T02:06:39.967 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-08-28T02:06:39.967 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-08-28T02:06:39.967 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-08-28T02:06:39.967 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-08-28T02:06:39.967 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.arena_block_size: 4194304 2023-08-28T02:06:39.967 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-08-28T02:06:39.967 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-08-28T02:06:39.967 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-08-28T02:06:39.967 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.disable_auto_compactions: 0 2023-08-28T02:06:39.967 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-08-28T02:06:39.967 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-08-28T02:06:39.967 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-08-28T02:06:39.967 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-08-28T02:06:39.968 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-08-28T02:06:39.968 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-08-28T02:06:39.968 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-08-28T02:06:39.968 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-08-28T02:06:39.968 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-08-28T02:06:39.968 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-08-28T02:06:39.968 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.table_properties_collectors: 2023-08-28T02:06:39.968 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.inplace_update_support: 0 2023-08-28T02:06:39.968 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-08-28T02:06:39.968 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-08-28T02:06:39.968 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-08-28T02:06:39.968 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.memtable_huge_page_size: 0 2023-08-28T02:06:39.968 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.bloom_locality: 0 2023-08-28T02:06:39.968 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.max_successive_merges: 0 2023-08-28T02:06:39.968 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-08-28T02:06:39.969 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.paranoid_file_checks: 0 2023-08-28T02:06:39.969 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.force_consistency_checks: 0 2023-08-28T02:06:39.969 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.report_bg_io_stats: 0 2023-08-28T02:06:39.969 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.ttl: 2592000 2023-08-28T02:06:39.969 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-08-28T02:06:39.969 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-08-28T02:06:39.969 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: 2023-08-28T02:06:39.969 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.726+0000 7f1ba0e40880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 0 2023-08-28T02:06:39.969 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: 2023-08-28T02:06:39.969 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.727+0000 7f1ba0e40880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1693188399728023, "job": 1, "event": "recovery_started", "log_files": [3]} 2023-08-28T02:06:39.969 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.727+0000 7f1ba0e40880 4 rocksdb: [db_impl/db_impl_open.cc:760] Recovering log #3 mode 2 2023-08-28T02:06:39.969 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.727+0000 7f1ba0e40880 3 rocksdb: [le/block_based/filter_policy.cc:584] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-08-28T02:06:39.970 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.727+0000 7f1ba0e40880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1693188399728487, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 4, "file_size": 1562, "table_properties": {"data_size": 648, "index_size": 40, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 0, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1693188399, "oldest_key_time": 3, "file_creation_time": 0}} 2023-08-28T02:06:39.970 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.727+0000 7f1ba0e40880 4 rocksdb: [version_set.cc:3826] Creating manifest 5 2023-08-28T02:06:39.970 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: 2023-08-28T02:06:39.970 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.727+0000 7f1ba0e40880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1693188399728856, "job": 1, "event": "recovery_finished"} 2023-08-28T02:06:39.970 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.728+0000 7f1ba0e40880 4 rocksdb: DB pointer 0x55ede1f1f800 2023-08-28T02:06:39.970 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.728+0000 7f1b87c47700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-08-28T02:06:39.970 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.728+0000 7f1b87c47700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-08-28T02:06:39.970 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: ** DB Stats ** 2023-08-28T02:06:39.970 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: Uptime(secs): 0.0 total, 0.0 interval 2023-08-28T02:06:39.970 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-08-28T02:06:39.970 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-08-28T02:06:39.970 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-08-28T02:06:39.970 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-08-28T02:06:39.971 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-08-28T02:06:39.971 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-08-28T02:06:39.971 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: 2023-08-28T02:06:39.971 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: ** Compaction Stats [default] ** 2023-08-28T02:06:39.971 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-08-28T02:06:39.971 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-08-28T02:06:39.971 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: L0 1/0 1.53 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.5 0.00 0.00 1 0.000 0 0 2023-08-28T02:06:39.971 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: Sum 1/0 1.53 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.5 0.00 0.00 1 0.000 0 0 2023-08-28T02:06:39.971 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.5 0.00 0.00 1 0.000 0 0 2023-08-28T02:06:39.971 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: 2023-08-28T02:06:39.971 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: ** Compaction Stats [default] ** 2023-08-28T02:06:39.971 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-08-28T02:06:39.971 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-08-28T02:06:39.972 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.5 0.00 0.00 1 0.000 0 0 2023-08-28T02:06:39.972 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: Uptime(secs): 0.0 total, 0.0 interval 2023-08-28T02:06:39.972 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: Flush(GB): cumulative 0.000, interval 0.000 2023-08-28T02:06:39.972 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: AddFile(GB): cumulative 0.000, interval 0.000 2023-08-28T02:06:39.972 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: AddFile(Total Files): cumulative 0, interval 0 2023-08-28T02:06:39.972 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: AddFile(L0 Files): cumulative 0, interval 0 2023-08-28T02:06:39.972 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: AddFile(Keys): cumulative 0, interval 0 2023-08-28T02:06:39.972 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: Cumulative compaction: 0.00 GB write, 0.65 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-08-28T02:06:39.972 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: Interval compaction: 0.00 GB write, 0.65 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-08-28T02:06:39.972 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-08-28T02:06:39.972 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: 2023-08-28T02:06:39.972 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: ** File Read Latency Histogram By Level [default] ** 2023-08-28T02:06:39.973 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: 2023-08-28T02:06:39.973 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: ** Compaction Stats [default] ** 2023-08-28T02:06:39.973 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-08-28T02:06:39.973 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-08-28T02:06:39.973 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: L0 1/0 1.53 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.5 0.00 0.00 1 0.000 0 0 2023-08-28T02:06:39.973 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: Sum 1/0 1.53 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.5 0.00 0.00 1 0.000 0 0 2023-08-28T02:06:39.973 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-08-28T02:06:39.973 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: 2023-08-28T02:06:39.973 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: ** Compaction Stats [default] ** 2023-08-28T02:06:39.973 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-08-28T02:06:39.973 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-08-28T02:06:39.973 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.5 0.00 0.00 1 0.000 0 0 2023-08-28T02:06:39.973 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: Uptime(secs): 0.0 total, 0.0 interval 2023-08-28T02:06:39.974 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: Flush(GB): cumulative 0.000, interval 0.000 2023-08-28T02:06:39.974 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: AddFile(GB): cumulative 0.000, interval 0.000 2023-08-28T02:06:39.974 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: AddFile(Total Files): cumulative 0, interval 0 2023-08-28T02:06:39.974 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: AddFile(L0 Files): cumulative 0, interval 0 2023-08-28T02:06:39.974 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: AddFile(Keys): cumulative 0, interval 0 2023-08-28T02:06:39.974 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: Cumulative compaction: 0.00 GB write, 0.65 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-08-28T02:06:39.974 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-08-28T02:06:39.974 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-08-28T02:06:39.974 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: 2023-08-28T02:06:39.974 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: ** File Read Latency Histogram By Level [default] ** 2023-08-28T02:06:39.974 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: 2023-08-28T02:06:39.975 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.728+0000 7f1ba0e40880 0 mon.c does not exist in monmap, will attempt to join an existing cluster 2023-08-28T02:06:39.975 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.728+0000 7f1ba0e40880 0 using public_addrv [v2:172.21.15.143:3301/0,v1:172.21.15.143:6790/0] 2023-08-28T02:06:39.975 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.729+0000 7f1ba0e40880 0 starting mon.c rank -1 at public addrs [v2:172.21.15.143:3301/0,v1:172.21.15.143:6790/0] at bind addrs [v2:172.21.15.143:3301/0,v1:172.21.15.143:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 2023-08-28T02:06:39.975 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.730+0000 7f1ba0e40880 1 mon.c@-1(???) e0 preinit fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 2023-08-28T02:06:39.975 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 systemd[1]: Started Ceph mon.c for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:06:39.975 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.748+0000 7f1b8a44c700 0 mon.c@-1(synchronizing).mds e1 new map 2023-08-28T02:06:39.975 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.748+0000 7f1b8a44c700 0 mon.c@-1(synchronizing).mds e1 print_map 2023-08-28T02:06:39.975 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: e1 2023-08-28T02:06:39.975 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: enable_multiple, ever_enabled_multiple: 1,1 2023-08-28T02:06:39.975 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-08-28T02:06:39.975 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: legacy client fscid: -1 2023-08-28T02:06:39.975 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: 2023-08-28T02:06:39.976 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: No filesystems configured 2023-08-28T02:06:39.976 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: 2023-08-28T02:06:39.976 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.748+0000 7f1b8a44c700 1 mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-08-28T02:06:39.976 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.748+0000 7f1b8a44c700 1 mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-08-28T02:06:39.976 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.748+0000 7f1b8a44c700 1 mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-08-28T02:06:39.976 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.748+0000 7f1b8a44c700 1 mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-08-28T02:06:39.976 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.748+0000 7f1b8a44c700 1 mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-08-28T02:06:39.976 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.748+0000 7f1b8a44c700 1 mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-08-28T02:06:39.976 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.750+0000 7f1b8a44c700 0 mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-08-28T02:06:39.976 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.750+0000 7f1b8a44c700 0 mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-08-28T02:06:39.976 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.750+0000 7f1b8a44c700 0 mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-08-28T02:06:39.976 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.750+0000 7f1b8a44c700 0 mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-08-28T02:06:39.976 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: cephadm 2023-08 2023-08-28T02:06:39.977 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: -28T02:06:36.868002+0000 mgr.y (mgr.14146) 23 : cephadm [INF] Deploying daemon mon.c on smithi143 2023-08-28T02:06:39.977 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: cluster 2023-08-28T02:06:36 2023-08-28T02:06:39.977 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: .920531+0000 mgr.y (mgr.14146) 24 : cluster [DBG] pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:06:39.977 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: audit 2023-08-28T02:06:37.100101+0000 mon.a 2023-08-28T02:06:39.977 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: (mon.0) 141 : audit [DBG] from='client.? 172.21.15.169:0/4289090380' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-08-28T02:06:39.977 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:06:39.977 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: 08-28T02:06:37.809515+0000 mon.a (mon.0) 142 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T02:06:39.977 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:39 smithi143 conmon[109923]: debug 2023-08-28T02:06:39.750+0000 7f1b8a44c700 1 mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-08-28T02:06:45.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:44 smithi143 conmon[100258]: audit 2023-08-28T02:06:39.759569+0000 mon.a (mon.0) 144 : audit [DBG] 2023-08-28T02:06:45.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:44 smithi143 conmon[100258]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T02:06:45.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:44 smithi143 conmon[100258]: audit 2023-08-28T02:06:39.759739+0000 mon.a (mon.0) 145 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T02:06:45.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:44 smithi143 conmon[100258]: audit 2023-08-28T02:06:39.759881+0000 mon.a (mon.0) 146 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T02:06:45.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:44 smithi143 conmon[100258]: cluster 2023-08-28T02:06:39.760190+0000 mon.a (mon.0) 147 : cluster [INF] mon.a calling monitor election 2023-08-28T02:06:45.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:44 smithi143 conmon[100258]: cluster 2023-08-28T02:06:39.760218+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2023-08-28T02:06:45.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:44 smithi143 conmon[100258]: audit 2023-08- 2023-08-28T02:06:45.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:44 smithi143 conmon[100258]: 28T02:06:40.755975+0000 mon.a (mon.0) 148 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T02:06:45.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:44 smithi143 conmon[100258]: cluster 2023-08-28T02:06:40.921176+0000 mgr.y (mgr.14146) 26 : cluster 2023-08-28T02:06:45.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:44 smithi143 conmon[100258]: [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:06:45.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:44 smithi143 conmon[100258]: audit 2023-08-28T02:06: 2023-08-28T02:06:45.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:44 smithi143 conmon[100258]: 41.756154+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T02:06:45.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:44 smithi143 conmon[100258]: audit 2023-08-28T02:06:42.756127+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T02:06:45.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:44 smithi143 conmon[100258]: cluster 2023-08-28T02:06 2023-08-28T02:06:45.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:44 smithi143 conmon[100258]: :42.921562+0000 mgr.y (mgr.14146) 27 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:06:45.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:44 smithi143 conmon[100258]: audit 2023-08-28T02:06: 2023-08-28T02:06:45.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:44 smithi143 conmon[100258]: 43.756264+0000 mon.a (mon.0) 151 : audit [DBG] 2023-08-28T02:06:45.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:44 smithi143 conmon[100258]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T02:06:45.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:44 smithi143 conmon[100258]: audit 2023- 2023-08-28T02:06:45.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:44 smithi143 conmon[100258]: 08-28T02:06:44.756351+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T02:06:45.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:44 smithi143 conmon[100258]: cluster 2023-08-28T 2023-08-28T02:06:45.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:44 smithi143 conmon[100258]: 02:06:44.773323+0000 mon.a (mon.0) 153 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-08-28T02:06:45.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:44 smithi143 conmon[100258]: cluster 2023-08-28T02: 2023-08-28T02:06:45.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:44 smithi143 conmon[100258]: 06:44.776357+0000 mon.a (mon.0) 154 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.143:3300/0,v1:172.21.15.143:6789/0],b=[v2:172.21.15.169:3300/0,v1:172.21.15.169:6789/0],c=[v2:172.21.15.143:3301/0,v1:172.21.15.143:6790/0]} removed_ranks: {} 2023-08-28T02:06:45.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:44 smithi143 conmon[100258]: cluster 2023-08-28T02: 2023-08-28T02:06:45.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:44 smithi143 conmon[100258]: 06:44.788694+0000 mon.a (mon.0) 155 : cluster [DBG] fsmap 2023-08-28T02:06:45.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:44 smithi143 conmon[100258]: cluster 2023-08-28T02 2023-08-28T02:06:45.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:44 smithi143 conmon[100258]: :06:44.788744+0000 mon.a (mon.0) 156 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-08-28T02:06:45.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:44 smithi143 conmon[100258]: cluster 2023-08-28 2023-08-28T02:06:45.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:44 smithi143 conmon[100258]: T02:06:44.788951+0000 mon.a (mon.0) 157 : cluster [DBG] 2023-08-28T02:06:45.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:44 smithi143 conmon[100258]: mgrmap e13: y(active, since 41s) 2023-08-28T02:06:45.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:44 smithi143 conmon[100258]: cluster 2023-08 2023-08-28T02:06:45.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:44 smithi143 conmon[100258]: -28T02:06:44.789136+0000 mon.a (mon.0) 158 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-08-28T02:06:45.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:44 smithi143 conmon[100258]: cluster 2023-08-28T02: 2023-08-28T02:06:45.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:44 smithi143 conmon[100258]: 06:44.793808+0000 mon.a (mon.0) 159 : cluster [WRN] Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-08-28T02:06:45.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:44 smithi143 conmon[100258]: cluster 2023-08-28T02:06 2023-08-28T02:06:45.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:44 smithi143 conmon[100258]: :44.793838+0000 mon.a (mon.0) 160 : cluster [WRN] [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-08-28T02:06:45.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:44 smithi143 conmon[100258]: cluster 2023-08-28T02 2023-08-28T02:06:45.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:44 smithi143 conmon[100258]: :06:44.793865+0000 mon.a (mon.0) 161 : cluster [WRN] mon.c (rank 2) addr [v2:172.21.15.143:3301/0,v1:172.21.15.143:6790/0] is down (out of quorum) 2023-08-28T02:06:45.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:44 smithi143 conmon[100258]: audit 2023-08-28T02 2023-08-28T02:06:45.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:44 smithi143 conmon[100258]: :06:44.797018+0000 mon.a (mon.0) 162 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:45.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:44 smithi143 conmon[100258]: audit 2023-08-28T02 2023-08-28T02:06:45.204 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:44 smithi143 conmon[100258]: :06:44.798263+0000 mon.a (mon.0) 163 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:06:45.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:44 smithi169 conmon[100500]: audit 2023-08-28T02:06:39.759569+0000 mon.a (mon.0) 144 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T02:06:45.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:44 smithi169 conmon[100500]: audit 2023-08-28T02:06:39.759739+0000 mon.a (mon.0) 145 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T02:06:45.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:44 smithi169 conmon[100500]: audit 2023-08-28T02:06:39.759881+0000 mon.a (mon.0) 146 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T02:06:45.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:44 smithi169 conmon[100500]: cluster 2023-08-28T02:06:39.760190+0000 mon.a (mon.0) 147 : cluster [INF] mon.a calling monitor election 2023-08-28T02:06:45.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:44 smithi169 conmon[100500]: cluster 2023-08-28T02:06:39.760218+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2023-08-28T02:06:45.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:44 smithi169 conmon[100500]: audit 2023-08-28T02:06:40.755975+0000 mon.a (mon.0) 148 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T02:06:45.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:44 smithi169 conmon[100500]: cluster 2023-08-28T02:06:40.921176+0000 mgr.y (mgr.14146) 26 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:06:45.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:44 smithi169 conmon[100500]: audit 2023-08-28T02:06:41.756154+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T02:06:45.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:44 smithi169 conmon[100500]: audit 2023-08-28T02:06:42.756127+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T02:06:45.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:44 smithi169 conmon[100500]: cluster 2023-08-28T02:06:42.921562+0000 mgr.y (mgr.14146) 27 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:06:45.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:44 smithi169 conmon[100500]: audit 2023-08-28T02:06:43.756264+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T02:06:45.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:44 smithi169 conmon[100500]: audit 2023-08-28T02:06:44.756351+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T02:06:45.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:44 smithi169 conmon[100500]: cluster 2023-08-28T02:06:44.773323+0000 mon.a (mon.0) 153 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-08-28T02:06:45.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:44 smithi169 conmon[100500]: cluster 2023-08-28T02:06:44.776357+0000 mon.a (mon.0) 154 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.143:3300/0,v1:172.21.15.143:6789/0],b=[v2:172.21.15.169:3300/0,v1:172.21.15.169:6789/0],c=[v2:172.21.15.143:3301/0,v1:172.21.15.143:6790/0]} removed_ranks: {} 2023-08-28T02:06:45.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:44 smithi169 conmon[100500]: cluster 2023-08-28T02:06:44.788694+0000 mon.a (mon.0) 155 : cluster [DBG] fsmap 2023-08-28T02:06:45.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:44 smithi169 conmon[100500]: cluster 2023-08-28T02:06:44.788744+0000 mon.a (mon.0) 156 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-08-28T02:06:45.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:44 smithi169 conmon[100500]: cluster 2023-08-28T02:06:44.788951+0000 mon.a (mon.0) 157 : cluster [DBG] mgrmap e13: y(active, since 41s) 2023-08-28T02:06:45.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:44 smithi169 conmon[100500]: cluster 2023-08-28T02:06:44.789136+0000 mon.a (mon.0) 158 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-08-28T02:06:45.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:44 smithi169 conmon[100500]: cluster 2023-08-28T02:06:44.793808+0000 mon.a (mon.0) 159 : cluster [WRN] Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-08-28T02:06:45.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:44 smithi169 conmon[100500]: cluster 2023-08-28T02:06:44.793838+0000 mon.a (mon.0) 160 : cluster [WRN] [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-08-28T02:06:45.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:44 smithi169 conmon[100500]: cluster 2023-08-28T02:06:44.793865+0000 mon.a (mon.0) 161 : cluster [WRN] mon.c (rank 2) addr [v2:172.21.15.143:3301/0,v1:172.21.15.143:6790/0] is down (out of quorum) 2023-08-28T02:06:45.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:44 smithi169 conmon[100500]: audit 2023-08-28T02:06:44.797018+0000 mon.a (mon.0) 162 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:45.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:44 smithi169 conmon[100500]: audit 2023-08-28T02:06:44.798263+0000 mon.a (mon.0) 163 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:06:46.394 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:46 smithi143 conmon[100258]: cluster 2023-08-28T02:06:44.921841+0000 mgr.y (mgr.14146) 28 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:06:46.395 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:46 smithi143 conmon[100258]: audit 2023-08-28T02:06:45.615698+0000 mon.a (mon.0) 164 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:46.395 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:46 smithi143 conmon[100258]: audit 2023-08-28T02:06:45.756480+0000 mon.a (mon.0) 165 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T02:06:46.395 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:46 smithi143 conmon[100258]: audit 2023-08-28T02:06:45.766185+0000 mon.a (mon.0) 166 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:46.395 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:46 smithi143 conmon[100258]: audit 2023-08-28T02:06:45.766750+0000 mon.a (mon.0) 167 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:46.395 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:46 smithi143 conmon[100258]: audit 2023-08-28T02:06:45.767330+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:06:46.395 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:46 smithi143 conmon[100258]: audit 2023-08-28T02:06:45.786376+0000 mon.a (mon.0) 169 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:46.395 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:46 smithi143 conmon[100258]: audit 2023-08-28T02:06 2023-08-28T02:06:46.395 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:46 smithi143 conmon[100258]: :45.792475+0000 mon.a (mon.0) 170 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:46.395 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:46 smithi143 conmon[100258]: audit 2023-08-28T02:06:45.794955+0000 mon.a (mon.0) 171 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:46.396 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:46 smithi143 conmon[100258]: audit 2023-08-28T02:06:45.796069+0000 mon.a (mon.0) 172 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-28T02:06:46.396 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:46 smithi143 conmon[100258]: audit 2023-08-28T02:06:45.796558+0000 mon.a (mon.0) 173 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-28T02:06:46.396 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:46 smithi143 conmon[100258]: audit 2023-08-28T02:06:45.797020+0000 mon.a (mon.0) 174 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:46.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:46 smithi169 conmon[100500]: cluster 2023-08-28T02:06:44.921841+0000 mgr.y (mgr.14146) 28 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:06:46.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:46 smithi169 conmon[100500]: audit 2023-08-28T02:06:45.615698+0000 mon.a (mon.0) 164 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:46.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:46 smithi169 conmon[100500]: audit 2023-08-28T02:06:45.756480+0000 mon.a (mon.0) 165 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T02:06:46.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:46 smithi169 conmon[100500]: audit 2023-08-28T02:06:45.766185+0000 mon.a (mon.0) 166 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:46.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:46 smithi169 conmon[100500]: audit 2023-08-28T02:06:45.766750+0000 mon.a (mon.0) 167 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:46.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:46 smithi169 conmon[100500]: audit 2023-08-28T02:06:45.767330+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:06:46.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:46 smithi169 conmon[100500]: audit 2023-08-28T02:06:45.786376+0000 2023-08-28T02:06:46.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:46 smithi169 conmon[100500]: mon.a (mon.0) 169 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:46.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:46 smithi169 conmon[100500]: audit 2023-08-28T02:06:45.792475+0000 mon.a (mon.0) 170 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:46.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:46 smithi169 conmon[100500]: audit 2023-08-28T02:06:45.794955+0000 mon.a (mon.0) 171 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:46.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:46 smithi169 conmon[100500]: audit 2023-08-28T02:06:45.796069+0000 mon.a (mon.0) 172 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-28T02:06:46.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:46 smithi169 conmon[100500]: audit 2023-08-28T02:06:45.796558+0000 mon.a (mon.0) 173 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-28T02:06:46.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:46 smithi169 conmon[100500]: audit 2023-08-28T02:06:45.797020+0000 mon.a (mon.0) 174 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:46.547 INFO:teuthology.orchestra.run.smithi169.stdout: 2023-08-28T02:06:46.547 INFO:teuthology.orchestra.run.smithi169.stdout:{"epoch":3,"fsid":"2fcb0bfa-4547-11ee-817c-1b7a5f1ba315","modified":"2023-08-28T02:06:39.756911Z","created":"2023-08-28T02:05:17.823964Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:3300","nonce":0},{"type":"v1","addr":"172.21.15.143:6789","nonce":0}]},"addr":"172.21.15.143:6789/0","public_addr":"172.21.15.143:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:3300","nonce":0},{"type":"v1","addr":"172.21.15.169:6789","nonce":0}]},"addr":"172.21.15.169:6789/0","public_addr":"172.21.15.169:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:3301","nonce":0},{"type":"v1","addr":"172.21.15.143:6790","nonce":0}]},"addr":"172.21.15.143:6790/0","public_addr":"172.21.15.143:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2023-08-28T02:06:46.549 INFO:teuthology.orchestra.run.smithi169.stderr:dumped monmap epoch 3 2023-08-28T02:06:47.036 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:46 smithi143 conmon[109923]: cluster 2023-08-28T02:06:44.921841+0000 mgr.y (mgr.14146) 2023-08-28T02:06:47.036 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:46 smithi143 conmon[109923]: 28 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:06:47.036 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:46 smithi143 conmon[109923]: audit 2023-08-28T02:06:45.615698+0000 mon.a (mon.0) 164 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:47.036 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:46 smithi143 conmon[109923]: audit 2023-08-28T02:06:45.756480+0000 mon.a (mon.0) 165 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T02:06:47.036 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:46 smithi143 conmon[109923]: audit 2023-08-28T02:06:45.766185+0000 mon.a (mon.0) 166 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:47.037 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:46 smithi143 conmon[109923]: audit 2023-08-28T02:06:45.766750+0000 mon.a (mon.0) 167 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:47.037 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:46 smithi143 conmon[109923]: audit 2023-08-28T02:06:45.767330+0000 2023-08-28T02:06:47.037 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:46 smithi143 conmon[109923]: mon.a (mon.0) 168 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:06:47.037 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:46 smithi143 conmon[109923]: audit 2023-08-28T02:06:45.786376+0000 mon.a (mon.0) 169 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:47.037 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:46 smithi143 conmon[109923]: audit 2023-08-28T02:06:47.037 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:46 smithi143 conmon[109923]: 2023-08-28T02:06:45.792475+0000 mon.a (mon.0) 170 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:47.037 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:46 smithi143 conmon[109923]: audit 2023-08 2023-08-28T02:06:47.037 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:46 smithi143 conmon[109923]: -28T02:06:45.794955+0000 mon.a (mon.0) 171 : 2023-08-28T02:06:47.037 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:46 smithi143 conmon[109923]: audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:47.037 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:46 smithi143 conmon[109923]: audit 2023-08-28T02:06:47.037 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:46 smithi143 conmon[109923]: 2023-08-28T02:06:45.796069+0000 mon.a (mon.0) 172 : audit [INF] 2023-08-28T02:06:47.037 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:46 smithi143 conmon[109923]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-28T02:06:47.037 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:46 smithi143 conmon[109923]: audit 2023-08-28T02:06:47.038 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:46 smithi143 conmon[109923]: 2023-08-28T02:06:45.796558+0000 mon.a (mon.0) 173 : audit [DBG] 2023-08-28T02:06:47.038 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:46 smithi143 conmon[109923]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-28T02:06:47.038 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:46 smithi143 conmon[109923]: audit 2023-08- 2023-08-28T02:06:47.038 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:46 smithi143 conmon[109923]: 28T02:06:45.797020+0000 mon.a (mon.0) 174 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:47.121 INFO:tasks.cephadm:Generating final ceph.conf file... 2023-08-28T02:06:47.122 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph config generate-minimal-conf 2023-08-28T02:06:47.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:47 smithi143 conmon[100258]: cluster 2023-08-28T02:06:41.758107+0000 2023-08-28T02:06:47.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:47 smithi143 conmon[100258]: mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-08-28T02:06:47.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:47 smithi143 conmon[100258]: cluster 2023-08-28T02:06:46.775279+0000 mon.c (mon.2) 2 : cluster [INF] mon.c calling monitor election 2023-08-28T02:06:47.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:47 smithi143 conmon[100258]: cluster 2023-08-28T02:06:46.775585+0000 mon.b (mon.1) 3 : cluster [INF] mon.b calling monitor election 2023-08-28T02:06:47.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:47 smithi143 conmon[100258]: cluster 2023-08-28T02:06:46.775618+0000 mon.a (mon.0) 177 : cluster [INF] mon.a calling monitor election 2023-08-28T02:06:47.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:47 smithi143 conmon[100258]: cluster 2023-08-28T02:06:46.788569+0000 mon.a (mon.0) 178 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-08-28T02:06:47.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:47 smithi143 conmon[100258]: cluster 2023-08-28T02:06:46.790938+0000 mon.a (mon.0) 179 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.143:3300/0,v1:172.21.15.143:6789/0],b=[v2:172.21.15.169:3300/0,v1:172.21.15.169:6789/0],c=[v2:172.21.15.143:3301/0,v1:172.21.15.143:6790/0]} removed_ranks: {} 2023-08-28T02:06:47.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:47 smithi143 conmon[100258]: cluster 2023-08-28T02:06:46.791341+0000 mon.a (mon.0) 180 : cluster [DBG] fsmap 2023-08-28T02:06:47.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:47 smithi143 conmon[100258]: cluster 2023-08-28T02:06:46.791392+0000 mon.a (mon.0) 181 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-08-28T02:06:47.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:47 smithi143 conmon[100258]: cluster 2023-08-28T02:06:46.791570+0000 mon.a (mon.0) 182 : cluster [DBG] mgrmap e13: y(active, since 43s) 2023-08-28T02:06:47.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:47 smithi143 conmon[100258]: cluster 2023-08-28T02:06:46.791735+0000 mon.a (mon.0) 183 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-08-28T02:06:47.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:47 smithi143 conmon[100258]: cluster 2023-08-28T02:06:46.791756+0000 mon.a (mon.0) 184 : cluster [INF] Cluster is now healthy 2023-08-28T02:06:47.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:47 smithi143 conmon[100258]: cluster 2023-08-28T02:06:46.794430+0000 mon.a (mon.0) 185 : cluster [INF] overall HEALTH_OK 2023-08-28T02:06:47.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:47 smithi143 conmon[100258]: audit 2023-08-28T02:06:47.074107+0000 mon.a (mon.0) 186 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:47.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:47 smithi143 conmon[100258]: audit 2023-08-28T02:06:47.074760+0000 mon.a (mon.0) 187 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-28T02:06:47.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:47 smithi143 conmon[100258]: audit 2023-08-28T02:06:47.075310+0000 mon.a (mon.0) 188 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-28T02:06:47.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:47 smithi143 conmon[100258]: audit 2023-08-28T02:06:47.075797+0000 mon.a (mon.0) 189 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:47.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:47 smithi143 conmon[109923]: cluster 2023-08-28T02:06:41.758107+0000 2023-08-28T02:06:47.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:47 smithi143 conmon[109923]: mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-08-28T02:06:47.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:47 smithi143 conmon[109923]: cluster 2023-08-28T02:06:46.775279+0000 mon.c (mon.2) 2 : cluster [INF] mon.c calling monitor election 2023-08-28T02:06:47.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:47 smithi143 conmon[109923]: cluster 2023-08-28T02:06:46.775585+0000 mon.b (mon.1) 3 : cluster [INF] mon.b calling monitor election 2023-08-28T02:06:47.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:47 smithi143 conmon[109923]: cluster 2023-08-28T02:06:46.775618+0000 mon.a (mon.0) 177 : cluster [INF] mon.a calling monitor election 2023-08-28T02:06:47.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:47 smithi143 conmon[109923]: cluster 2023-08-28T02:06:46.788569+0000 mon.a (mon.0) 178 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-08-28T02:06:47.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:47 smithi143 conmon[109923]: cluster 2023-08-28T02:06:46.790938+0000 mon.a (mon.0) 179 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.143:3300/0,v1:172.21.15.143:6789/0],b=[v2:172.21.15.169:3300/0,v1:172.21.15.169:6789/0],c=[v2:172.21.15.143:3301/0,v1:172.21.15.143:6790/0]} removed_ranks: {} 2023-08-28T02:06:47.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:47 smithi143 conmon[109923]: cluster 2023-08-28T02:06:46.791341+0000 mon.a (mon.0) 180 : cluster [DBG] fsmap 2023-08-28T02:06:47.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:47 smithi143 conmon[109923]: cluster 2023-08-28T02:06:46.791392+0000 mon.a (mon.0) 181 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-08-28T02:06:47.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:47 smithi143 conmon[109923]: cluster 2023-08-28T02: 2023-08-28T02:06:47.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:47 smithi143 conmon[109923]: 06:46.791570+0000 mon.a (mon.0) 182 : cluster [DBG] mgrmap e13: y(active, since 43s) 2023-08-28T02:06:47.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:47 smithi143 conmon[109923]: cluster 2023-08-28 2023-08-28T02:06:47.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:47 smithi143 conmon[109923]: T02:06:46.791735+0000 mon.a (mon.0 2023-08-28T02:06:47.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:47 smithi143 conmon[109923]: ) 183 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-08-28T02:06:47.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:47 smithi143 conmon[109923]: cluster 2023-08 2023-08-28T02:06:47.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:47 smithi143 conmon[109923]: -28T02:06:46.791756+0000 mon.a 2023-08-28T02:06:47.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:47 smithi143 conmon[109923]: (mon.0) 184 : cluster [INF] Cluster is now healthy 2023-08-28T02:06:47.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:47 smithi143 conmon[109923]: cluster 2023 2023-08-28T02:06:47.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:47 smithi143 conmon[109923]: -08-28T02:06:46.794430+0000 mon.a 2023-08-28T02:06:47.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:47 smithi143 conmon[109923]: (mon.0) 185 : cluster [INF] overall HEALTH_OK 2023-08-28T02:06:47.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:47 smithi143 conmon[109923]: audit 2023-08-28T 2023-08-28T02:06:47.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:47 smithi143 conmon[109923]: 02:06:47.074107+0000 mon.a (mon.0) 2023-08-28T02:06:47.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:47 smithi143 conmon[109923]: 186 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:47.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:47 smithi143 conmon[109923]: audit 2023-08-28T02:06:47.074760+0000 mon.a ( 2023-08-28T02:06:47.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:47 smithi143 conmon[109923]: mon.0) 187 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-28T02:06:47.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:47 smithi143 conmon[109923]: audit 2023-08-28T02:06:47.075310+0000 mon.a ( 2023-08-28T02:06:47.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:47 smithi143 conmon[109923]: mon.0) 188 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-28T02:06:47.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:47 smithi143 conmon[109923]: audit 2023-08-28T02:06:47.075797+0000 2023-08-28T02:06:47.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:47 smithi143 conmon[109923]: mon.a (mon.0) 189 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:47.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: cluster 2023-08-28T02:06:41.758107+0000 mon.c (mon. 2023-08-28T02:06:47.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: 2) 1 : cluster [INF] mon.c calling monitor election 2023-08-28T02:06:47.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: cluster 2023-08-28T02:06:46.775279+0000 mon.c (mon.2) 2 : cluster [INF] mon.c calling monitor election 2023-08-28T02:06:47.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: cluster 2023-08-28T02:06:46.775585+0000 mon.b (mon.1) 3 : cluster [INF] mon.b calling monitor election 2023-08-28T02:06:47.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: cluster 2023-08-28T02:06:47.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: 2023-08-28T02:06:46.775618+0000 mon.a (mon.0) 177 : cluster [INF] mon.a calling monitor election 2023-08-28T02:06:47.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: cluster 2023-08- 2023-08-28T02:06:47.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: 28T02:06:46.788569+0000 mon.a 2023-08-28T02:06:47.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: (mon.0) 178 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-08-28T02:06:47.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: cluster 2023-08-28T02:06:47.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: 2023-08-28T02:06: 2023-08-28T02:06:47.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: 46.790938+0000 mon.a (mon.0) 179 : 2023-08-28T02:06:47.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.143:3300/0,v1:172.21.15.143:6789/0],b=[v2:172.21.15.169:3300/0,v1:172.21.15.169:6789/0],c=[v2:172.21.15.143:3301/0,v1:172.21.15.143:6790/0]} removed_ranks: {} 2023-08-28T02:06:47.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: cluster 2023-08-28 2023-08-28T02:06:47.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: T02:06:46.791341+0000 2023-08-28T02:06:47.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: mon.a (mon.0) 180 : cluster [DBG] fsmap 2023-08-28T02:06:47.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: 2023-08-28T02:06:47.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: cluster 2023-08-28T02:06:47.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: 2023-08-28T02:06:46. 2023-08-28T02:06:47.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: 791392+0000 mon.a (mon.0) 181 : cluster [DBG] 2023-08-28T02:06:47.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: osdmap e4: 0 total, 0 up, 0 in 2023-08-28T02:06:47.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: cluster 2023-08-28T02:06:47.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: 2023-08-28T02:06:46.791570 2023-08-28T02:06:47.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: +0000 mon.a (mon.0) 182 : cluster [DBG] 2023-08-28T02:06:47.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: mgrmap e13: y(active, since 43s) 2023-08-28T02:06:47.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: cluster 2023-08-28T02 2023-08-28T02:06:47.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: :06:46.791735+0000 mon.a (mon.0 2023-08-28T02:06:47.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: ) 183 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-08-28T02:06:47.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: cluster 2023- 2023-08-28T02:06:47.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: 08-28T02:06:46. 2023-08-28T02:06:47.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: 791756+0000 mon.a (mon.0) 184 : cluster 2023-08-28T02:06:47.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: [INF] Cluster is now healthy 2023-08-28T02:06:47.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: cluster 2023-08-28T 2023-08-28T02:06:47.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: 02:06:46.794430+0000 mon.a (mon.0 2023-08-28T02:06:47.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: ) 185 : cluster [INF] overall HEALTH_OK 2023-08-28T02:06:47.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: audit 2023-08 2023-08-28T02:06:47.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: -28T02:06:47.074107+0000 2023-08-28T02:06:47.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: mon.a (mon.0) 186 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:47.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: 2023-08-28T02:06:47.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: audit 2023-08-28T02:06:47.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: 2023-08-28T02:06:47.074760+0000 2023-08-28T02:06:47.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: mon.a (mon.0) 187 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-28T02:06:47.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: audit 2023-08- 2023-08-28T02:06:47.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: 28T02:06:47.075310+0000 mon.a (mon.0) 188 2023-08-28T02:06:47.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-28T02:06:47.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: audit 2023-08-28T02:06:47.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: 2023-08-28T02:06:47.075797+0000 mon.a (mon.0) 189 : audit 2023-08-28T02:06:47.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:47 smithi169 conmon[100500]: [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:48.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:48 smithi143 conmon[109923]: cluster 2023-08-28T02:06:46.922158+0000 mgr.y (mgr.14146) 33 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:06:48.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:48 smithi143 conmon[109923]: cephadm 2023-08-28T02:06:47.074548+0000 mgr.y (mgr.14146) 34 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2023-08-28T02:06:48.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:48 smithi143 conmon[109923]: cephadm 2023-08-28T02:06:47.076288+0000 mgr.y (mgr.14146) 35 : cephadm [INF] Reconfiguring daemon mon.c on smithi143 2023-08-28T02:06:48.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:48 smithi143 conmon[109923]: audit 2023-08-28T02:06:47.756725+0000 mon.a (mon.0) 190 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T02:06:48.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:48 smithi143 conmon[109923]: audit 2023-08-28T02:06:48.051761+0000 mon.a (mon.0) 191 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:48.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:48 smithi143 conmon[100258]: cluster 2023-08-28T02:06:46.922158+0000 mgr.y (mgr.14146) 33 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:06:48.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:48 smithi143 conmon[100258]: cephadm 2023-08-28T02:06:47.074548+0000 mgr.y (mgr.14146) 34 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2023-08-28T02:06:48.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:48 smithi143 conmon[100258]: cephadm 2023-08-28T02:06:47.076288+0000 mgr.y (mgr.14146) 35 : cephadm [INF] Reconfiguring daemon mon.c on smithi143 2023-08-28T02:06:48.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:48 smithi143 conmon[100258]: audit 2023-08-28T02:06:47.756725+0000 mon.a (mon.0) 190 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T02:06:48.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:48 smithi143 conmon[100258]: audit 2023-08-28T02:06:48.051761+0000 mon.a (mon.0) 191 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:48.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:48 smithi169 conmon[100500]: cluster 2023-08-28T02:06:46.922158+0000 mgr.y (mgr.14146) 33 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:06:48.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:48 smithi169 conmon[100500]: cephadm 2023-08-28T02:06:47.074548+0000 mgr.y (mgr.14146) 34 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2023-08-28T02:06:48.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:48 smithi169 conmon[100500]: cephadm 2023-08-28T02:06:47.076288+0000 mgr.y (mgr.14146) 35 : cephadm [INF] Reconfiguring daemon mon.c on smithi143 2023-08-28T02:06:48.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:48 smithi169 conmon[100500]: audit 2023-08-28T02:06:47.756725+0000 2023-08-28T02:06:48.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:48 smithi169 conmon[100500]: mon.a (mon.0) 190 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T02:06:48.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:48 smithi169 conmon[100500]: audit 2023- 2023-08-28T02:06:48.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:48 smithi169 conmon[100500]: 08-28T02:06:48.051761+0000 mon.a ( 2023-08-28T02:06:48.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:48 smithi169 conmon[100500]: mon.0) 191 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:49.479 INFO:teuthology.orchestra.run.smithi143.stdout:# minimal ceph.conf for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 2023-08-28T02:06:49.479 INFO:teuthology.orchestra.run.smithi143.stdout:[global] 2023-08-28T02:06:49.479 INFO:teuthology.orchestra.run.smithi143.stdout: fsid = 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 2023-08-28T02:06:49.479 INFO:teuthology.orchestra.run.smithi143.stdout: mon_host = [v2:172.21.15.143:3300/0,v1:172.21.15.143:6789/0] [v2:172.21.15.169:3300/0,v1:172.21.15.169:6789/0] [v2:172.21.15.143:3301/0,v1:172.21.15.143:6790/0] 2023-08-28T02:06:49.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:49 smithi143 conmon[109923]: audit 2023-08-28T02:06:48.532556+0000 mon.a (mon 2023-08-28T02:06:49.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:49 smithi143 conmon[109923]: .0) 192 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:49.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:49 smithi143 conmon[109923]: cephadm 2023-08-28T02:06:48.533127+0000 mgr.y (mgr.14146) 36 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-08-28T02:06:49.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:49 smithi143 conmon[109923]: audit 2023-08-28T02:06:48.533376+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-28T02:06:49.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:49 smithi143 conmon[109923]: audit 2023-08-28T02:06:48.534138+0000 mon.a (mon.0) 194 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-28T02:06:49.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:49 smithi143 conmon[109923]: audit 2023-08-28T02:06:48.534779+0000 mon.a (mon.0) 195 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:49.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:49 smithi143 conmon[109923]: cephadm 2023-08-28T02:06:48.535395+0000 mgr.y (mgr.14146) 37 : cephadm [INF] Reconfiguring daemon mon.b on smithi169 2023-08-28T02:06:49.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:49 smithi143 conmon[109923]: audit 2023-08-28T02:06:49.480069+0000 mon.c (mon.2) 3 : audit [DBG] from='client.? 172.21.15.143:0/331788190' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:49.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:49 smithi143 conmon[100258]: audit 2023-08-28T02:06:48.532556+0000 mon.a (mon.0) 192 2023-08-28T02:06:49.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:49 smithi143 conmon[100258]: : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:49.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:49 smithi143 conmon[100258]: cephadm 2023-08-28T02:06:48.533127+0000 mgr.y (mgr.14146) 36 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-08-28T02:06:49.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:49 smithi143 conmon[100258]: audit 2023-08-28T02:06:48.533376+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-28T02:06:49.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:49 smithi143 conmon[100258]: audit 2023-08-28T02:06:48.534138+0000 mon.a (mon.0) 194 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-28T02:06:49.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:49 smithi143 conmon[100258]: audit 2023-08-28T02:06:48.534779+0000 mon.a (mon.0) 195 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:49.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:49 smithi143 conmon[100258]: cephadm 2023-08-28T02:06:48.535395+0000 mgr.y (mgr.14146) 37 : cephadm [INF] Reconfiguring daemon mon.b on smithi169 2023-08-28T02:06:49.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:49 smithi143 conmon[100258]: audit 2023-08-28T02:06:49.480069+0000 mon.c (mon.2) 3 : audit [DBG] from='client.? 172.21.15.143:0/331788190' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:49.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:49 smithi169 conmon[100500]: audit 2023-08-28T02:06:48.532556+0000 mon.a (mon.0) 2023-08-28T02:06:49.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:49 smithi169 conmon[100500]: 192 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:49.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:49 smithi169 conmon[100500]: cephadm 2023-08-28T02:06:48.533127+0000 mgr.y (mgr.14146) 36 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-08-28T02:06:49.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:49 smithi169 conmon[100500]: audit 2023-08-28T02:06:48.533376+0000 mon.a (mon.0) 193 : audit 2023-08-28T02:06:49.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:49 smithi169 conmon[100500]: [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-28T02:06:49.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:49 smithi169 conmon[100500]: audit 2023- 2023-08-28T02:06:49.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:49 smithi169 conmon[100500]: 08-28T02:06:48.534138+0000 mon.a (mon.0) 194 : 2023-08-28T02:06:49.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:49 smithi169 conmon[100500]: audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-28T02:06:49.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:49 smithi169 conmon[100500]: audit 2023-08-28T02: 2023-08-28T02:06:49.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:49 smithi169 conmon[100500]: 06:48.534779+0000 mon.a (mon.0) 195 : audit 2023-08-28T02:06:49.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:49 smithi169 conmon[100500]: [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:49.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:49 smithi169 conmon[100500]: cephadm 2023-08-28T02:06:49.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:49 smithi169 conmon[100500]: 2023-08-28T02:06:48.535395+0000 mgr.y (mgr.14146) 37 : 2023-08-28T02:06:49.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:49 smithi169 conmon[100500]: cephadm [INF] Reconfiguring daemon mon.b on smithi169 2023-08-28T02:06:49.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:49 smithi169 conmon[100500]: audit 2023-08-28T02:06:49.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:49 smithi169 conmon[100500]: 2023-08-28T02:06:49.480069+0000 mon.c (mon.2) 3 : 2023-08-28T02:06:49.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:49 smithi169 conmon[100500]: audit [DBG] from='client.? 172.21.15.143:0/331788190' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:50.135 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2023-08-28T02:06:50.136 DEBUG:teuthology.orchestra.run.smithi143:> set -ex 2023-08-28T02:06:50.136 DEBUG:teuthology.orchestra.run.smithi143:> sudo dd of=/etc/ceph/ceph.conf 2023-08-28T02:06:50.166 DEBUG:teuthology.orchestra.run.smithi143:> set -ex 2023-08-28T02:06:50.166 DEBUG:teuthology.orchestra.run.smithi143:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-08-28T02:06:50.233 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2023-08-28T02:06:50.233 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd of=/etc/ceph/ceph.conf 2023-08-28T02:06:50.266 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2023-08-28T02:06:50.266 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-08-28T02:06:50.334 INFO:tasks.cephadm:Adding mgr.y on smithi143 2023-08-28T02:06:50.334 INFO:tasks.cephadm:Adding mgr.x on smithi169 2023-08-28T02:06:50.334 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph orch apply mgr '2;smithi143=y;smithi169=x' 2023-08-28T02:06:50.813 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:50 smithi169 conmon[100500]: cluster 2023-08-28T02:06:48.922405+0000 mgr.y (mgr.14146) 38 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:06:50.813 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:50 smithi169 conmon[100500]: audit 2023-08-28T02:06:49.684699+0000 mon.a (mon.0) 196 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:50.814 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:50 smithi169 conmon[100500]: audit 2023-08-28T02:06:49.685470+0000 mon.a (mon.0) 197 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:06:50.814 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:50 smithi169 conmon[100500]: audit 2023-08-28T02:06:49.686605+0000 mon.a (mon.0) 198 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:50.814 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:50 smithi169 conmon[100500]: audit 2023-08-28T02:06:49.687156+0000 mon.a (mon.0) 199 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:06:50.814 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:50 smithi169 conmon[100500]: audit 2023-08-28T02:06:49.690837+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:50.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:50 smithi143 conmon[109923]: cluster 2023-08-28T02:06:48.922405+0000 mgr.y (mgr 2023-08-28T02:06:50.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:50 smithi143 conmon[109923]: .14146) 38 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:06:50.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:50 smithi143 conmon[109923]: audit 2023-08-28T02:06:49.684699+0000 mon.a (mon.0) 196 2023-08-28T02:06:50.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:50 smithi143 conmon[109923]: : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:50.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:50 smithi143 conmon[109923]: audit 2023-08-28T02:06:49.685470+0000 mon.a 2023-08-28T02:06:50.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:50 smithi143 conmon[109923]: (mon.0) 197 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:06:50.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:50 smithi143 conmon[109923]: audit 2023-08- 2023-08-28T02:06:50.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:50 smithi143 conmon[109923]: 28T02:06:49.686605+0000 mon.a (mon.0) 198 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:50.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:50 smithi143 conmon[109923]: audit 2023 2023-08-28T02:06:50.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:50 smithi143 conmon[109923]: -08-28T02:06:49.687156+0000 mon.a (mon.0) 199 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:06:50.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:50 smithi143 conmon[109923]: audit 2023-08-28T02:06:49 2023-08-28T02:06:50.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:50 smithi143 conmon[109923]: .690837+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:50.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:50 smithi143 conmon[100258]: cluster 2023-08-28T02:06:48.922405+0000 2023-08-28T02:06:50.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:50 smithi143 conmon[100258]: mgr.y (mgr.14146) 38 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:06:50.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:50 smithi143 conmon[100258]: audit 2023-08-28T02:06:49.684699+0000 mon.a (mon.0) 196 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:50.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:50 smithi143 conmon[100258]: audit 2023-08-28T02:06:49.685470+0000 mon.a (mon.0) 197 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:06:50.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:50 smithi143 conmon[100258]: audit 2023-08-28T02:06:49.686605+0000 mon.a (mon.0) 198 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:50.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:50 smithi143 conmon[100258]: audit 2023-08-28T02:06:49.687156+0000 mon.a (mon.0) 199 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:06:50.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:50 smithi143 conmon[100258]: audit 2023-08-28T02:06:49.690837+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:52.180 INFO:teuthology.orchestra.run.smithi169.stdout:Scheduled mgr update... 2023-08-28T02:06:52.778 DEBUG:teuthology.orchestra.run.smithi169:mgr.x> sudo journalctl -f -n 0 -u ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@mgr.x.service 2023-08-28T02:06:52.781 INFO:tasks.cephadm:Deploying OSDs... 2023-08-28T02:06:52.781 DEBUG:teuthology.orchestra.run.smithi143:> set -ex 2023-08-28T02:06:52.781 DEBUG:teuthology.orchestra.run.smithi143:> dd if=/scratch_devs of=/dev/stdout 2023-08-28T02:06:52.802 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-08-28T02:06:52.803 DEBUG:teuthology.orchestra.run.smithi143:> stat /dev/vg_nvme/lv_1 2023-08-28T02:06:52.805 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:52 smithi169 conmon[100500]: cluster 2023-08-28T02:06:50.922647+0000 mgr.y (mgr.14146) 39 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:06:52.805 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:52 smithi169 conmon[100500]: audit 2023-08-28T02:06:52.180545+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:52.805 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:52 smithi169 conmon[100500]: audit 2023-08-28T02:06:52.211331+0000 mon.a (mon.0) 202 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:06:52.805 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:52 smithi169 conmon[100500]: audit 2023-08-28T02:06:52.212945+0000 mon.a (mon.0) 203 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:52.805 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:52 smithi169 conmon[100500]: audit 2023-08-28T02:06:52.213733+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:06:52.805 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:52 smithi169 conmon[100500]: audit 2023-08-28T02:06:52.218231+0000 mon.a (mon.0) 205 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:52.805 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:52 smithi169 conmon[100500]: audit 2023-08-28T02:06:52.219549+0000 mon.a (mon.0) 206 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-28T02:06:52.805 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:52 smithi169 conmon[100500]: audit 2023-08-28T02:06:52.222548+0000 mon.a (mon.0) 207 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-08-28T02:06:52.806 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:52 smithi169 conmon[100500]: audit 2023-08-28T02:06:52.806 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:52 smithi169 conmon[100500]: 2023-08-28T02:06:52.223345+0000 mon.a (mon.0) 208 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-08-28T02:06:52.806 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:52 smithi169 conmon[100500]: audit 2023-08-28T02:06:52.224226+0000 mon.a (mon.0) 209 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:52.811 INFO:journalctl@ceph.mgr.x.smithi169.stdout:-- Logs begin at Mon 2023-08-28 01:49:52 UTC. -- 2023-08-28T02:06:52.864 INFO:teuthology.orchestra.run.smithi143.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-08-28T02:06:52.864 INFO:teuthology.orchestra.run.smithi143.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-08-28T02:06:52.865 INFO:teuthology.orchestra.run.smithi143.stdout:Device: 6h/6d Inode: 388857 Links: 1 2023-08-28T02:06:52.865 INFO:teuthology.orchestra.run.smithi143.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-08-28T02:06:52.865 INFO:teuthology.orchestra.run.smithi143.stdout:Context: system_u:object_r:device_t:s0 2023-08-28T02:06:52.865 INFO:teuthology.orchestra.run.smithi143.stdout:Access: 2023-08-28 02:06:39.171046448 +0000 2023-08-28T02:06:52.865 INFO:teuthology.orchestra.run.smithi143.stdout:Modify: 2023-08-28 01:58:57.426840010 +0000 2023-08-28T02:06:52.865 INFO:teuthology.orchestra.run.smithi143.stdout:Change: 2023-08-28 01:58:57.426840010 +0000 2023-08-28T02:06:52.865 INFO:teuthology.orchestra.run.smithi143.stdout: Birth: - 2023-08-28T02:06:52.865 DEBUG:teuthology.orchestra.run.smithi143:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-08-28T02:06:52.933 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[100258]: cluster 2023-08-28T02:06:50.922647+0000 mgr.y (mgr.14146) 39 : cluster [DBG] 2023-08-28T02:06:52.933 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[100258]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:06:52.933 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[100258]: audit 2023-08-28T02:06:52.180545+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:52.934 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[100258]: audit 2023-08-28T02:06:52.211331+0000 2023-08-28T02:06:52.934 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[100258]: mon.a (mon.0) 202 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:06:52.934 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[100258]: audit 2023-08-28T02:06:52.212945+0000 mon.a (mon.0) 203 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:52.934 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[100258]: audit 2023-08-28T02: 2023-08-28T02:06:52.934 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[100258]: 06:52.213733+0000 mon.a (mon.0) 204 : audit [INF] 2023-08-28T02:06:52.934 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[100258]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:06:52.934 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[100258]: audit 2023-08-28T02:06:52. 2023-08-28T02:06:52.934 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[100258]: 218231+0000 mon.a (mon.0) 205 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:52.934 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[100258]: 2023-08-28T02:06:52.934 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[100258]: audit 2023-08-28T02:06:52. 2023-08-28T02:06:52.934 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[100258]: 219549+0000 mon.a (mon.0) 206 : audit [INF] 2023-08-28T02:06:52.934 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[100258]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-28T02:06:52.935 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[100258]: audit 2023-08-28T02:06:52 2023-08-28T02:06:52.935 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[100258]: .222548+0000 mon.a (mon.0) 207 : audit [INF] 2023-08-28T02:06:52.935 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[100258]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-08-28T02:06:52.935 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[100258]: audit 2023-08-28T02:06 2023-08-28T02:06:52.935 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[100258]: :52.223345+0000 mon.a (mon.0) 208 : audit 2023-08-28T02:06:52.935 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[100258]: [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-08-28T02:06:52.935 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[100258]: audit 2023-08-28T02: 2023-08-28T02:06:52.935 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[100258]: 06:52.224226+0000 mon.a (mon.0) 209 : audit 2023-08-28T02:06:52.935 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[100258]: [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:52.936 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[109923]: cluster 2023-08-28T02:06:50.922647+0000 mgr.y (mgr.14146) 39 : 2023-08-28T02:06:52.936 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[109923]: cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:06:52.936 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[109923]: audit 2023-08-28T02:06:52.180545+0000 mon.a (mon 2023-08-28T02:06:52.936 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[109923]: .0) 201 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:52.936 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[109923]: audit 2023-08-28T02:06:52.936 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[109923]: 2023-08- 2023-08-28T02:06:52.936 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[109923]: 28T02:06:52.211331+0000 mon.a (mon. 2023-08-28T02:06:52.936 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[109923]: 0) 202 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:06:52.936 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[109923]: audit 2023-08-28T02:06:52.936 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[109923]: 2023-08-28T02:06:52.212945+0000 mon.a ( 2023-08-28T02:06:52.936 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[109923]: mon.0) 203 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:52.936 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[109923]: audit 2023 2023-08-28T02:06:52.937 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[109923]: -08-28T02:06:52.213733+0000 2023-08-28T02:06:52.937 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[109923]: mon.a (mon.0) 204 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:06:52.937 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[109923]: audit 2023-08-28T02:06:52.937 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[109923]: 2023-08-28T02:06:52. 2023-08-28T02:06:52.937 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[109923]: 218231+0000 mon.a (mon.0) 205 : audit 2023-08-28T02:06:52.937 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[109923]: [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:52.937 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[109923]: audit 2023-08 2023-08-28T02:06:52.937 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[109923]: -28T02:06:52.219549+0000 mon.a ( 2023-08-28T02:06:52.937 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[109923]: mon.0) 206 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-28T02:06:52.937 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[109923]: audit 2023-08-28T02:06:52.937 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[109923]: 2023-08-28T02:06:52.222548+0000 2023-08-28T02:06:52.937 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[109923]: mon.a (mon.0) 207 : audit [INF] 2023-08-28T02:06:52.938 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[109923]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-08-28T02:06:52.938 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[109923]: audit 2023-08-28 2023-08-28T02:06:52.938 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[109923]: T02:06:52.223345+0000 mon.a (mon.0 2023-08-28T02:06:52.938 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[109923]: ) 208 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-08-28T02:06:52.938 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[109923]: audit 2023-08-28T02:06:52.224226+0000 mon.a (mon.0) 209 2023-08-28T02:06:52.938 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:52 smithi143 conmon[109923]: : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:52.939 INFO:teuthology.orchestra.run.smithi143.stderr:1+0 records in 2023-08-28T02:06:52.939 INFO:teuthology.orchestra.run.smithi143.stderr:1+0 records out 2023-08-28T02:06:52.939 INFO:teuthology.orchestra.run.smithi143.stderr:512 bytes copied, 0.000247399 s, 2.1 MB/s 2023-08-28T02:06:52.939 DEBUG:teuthology.orchestra.run.smithi143:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-08-28T02:06:53.001 DEBUG:teuthology.orchestra.run.smithi143:> stat /dev/vg_nvme/lv_2 2023-08-28T02:06:53.062 INFO:teuthology.orchestra.run.smithi143.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-08-28T02:06:53.062 INFO:teuthology.orchestra.run.smithi143.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-08-28T02:06:53.062 INFO:teuthology.orchestra.run.smithi143.stdout:Device: 6h/6d Inode: 388928 Links: 1 2023-08-28T02:06:53.062 INFO:teuthology.orchestra.run.smithi143.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-08-28T02:06:53.062 INFO:teuthology.orchestra.run.smithi143.stdout:Context: system_u:object_r:device_t:s0 2023-08-28T02:06:53.063 INFO:teuthology.orchestra.run.smithi143.stdout:Access: 2023-08-28 02:06:39.171046448 +0000 2023-08-28T02:06:53.063 INFO:teuthology.orchestra.run.smithi143.stdout:Modify: 2023-08-28 01:58:57.946825711 +0000 2023-08-28T02:06:53.063 INFO:teuthology.orchestra.run.smithi143.stdout:Change: 2023-08-28 01:58:57.946825711 +0000 2023-08-28T02:06:53.063 INFO:teuthology.orchestra.run.smithi143.stdout: Birth: - 2023-08-28T02:06:53.063 DEBUG:teuthology.orchestra.run.smithi143:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-08-28T02:06:53.132 INFO:teuthology.orchestra.run.smithi143.stderr:1+0 records in 2023-08-28T02:06:53.133 INFO:teuthology.orchestra.run.smithi143.stderr:1+0 records out 2023-08-28T02:06:53.133 INFO:teuthology.orchestra.run.smithi143.stderr:512 bytes copied, 0.000254636 s, 2.0 MB/s 2023-08-28T02:06:53.134 DEBUG:teuthology.orchestra.run.smithi143:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-08-28T02:06:53.195 DEBUG:teuthology.orchestra.run.smithi143:> stat /dev/vg_nvme/lv_3 2023-08-28T02:06:53.253 INFO:teuthology.orchestra.run.smithi143.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-08-28T02:06:53.253 INFO:teuthology.orchestra.run.smithi143.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-08-28T02:06:53.253 INFO:teuthology.orchestra.run.smithi143.stdout:Device: 6h/6d Inode: 386945 Links: 1 2023-08-28T02:06:53.253 INFO:teuthology.orchestra.run.smithi143.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-08-28T02:06:53.254 INFO:teuthology.orchestra.run.smithi143.stdout:Context: system_u:object_r:device_t:s0 2023-08-28T02:06:53.254 INFO:teuthology.orchestra.run.smithi143.stdout:Access: 2023-08-28 02:06:39.171046448 +0000 2023-08-28T02:06:53.254 INFO:teuthology.orchestra.run.smithi143.stdout:Modify: 2023-08-28 01:58:58.456811688 +0000 2023-08-28T02:06:53.254 INFO:teuthology.orchestra.run.smithi143.stdout:Change: 2023-08-28 01:58:58.456811688 +0000 2023-08-28T02:06:53.254 INFO:teuthology.orchestra.run.smithi143.stdout: Birth: - 2023-08-28T02:06:53.254 DEBUG:teuthology.orchestra.run.smithi143:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-08-28T02:06:53.322 INFO:teuthology.orchestra.run.smithi143.stderr:1+0 records in 2023-08-28T02:06:53.323 INFO:teuthology.orchestra.run.smithi143.stderr:1+0 records out 2023-08-28T02:06:53.323 INFO:teuthology.orchestra.run.smithi143.stderr:512 bytes copied, 0.00029741 s, 1.7 MB/s 2023-08-28T02:06:53.323 DEBUG:teuthology.orchestra.run.smithi143:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-08-28T02:06:53.381 DEBUG:teuthology.orchestra.run.smithi143:> stat /dev/vg_nvme/lv_4 2023-08-28T02:06:53.438 INFO:teuthology.orchestra.run.smithi143.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-08-28T02:06:53.439 INFO:teuthology.orchestra.run.smithi143.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-08-28T02:06:53.439 INFO:teuthology.orchestra.run.smithi143.stdout:Device: 6h/6d Inode: 391850 Links: 1 2023-08-28T02:06:53.439 INFO:teuthology.orchestra.run.smithi143.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-08-28T02:06:53.439 INFO:teuthology.orchestra.run.smithi143.stdout:Context: system_u:object_r:device_t:s0 2023-08-28T02:06:53.439 INFO:teuthology.orchestra.run.smithi143.stdout:Access: 2023-08-28 02:06:39.171046448 +0000 2023-08-28T02:06:53.439 INFO:teuthology.orchestra.run.smithi143.stdout:Modify: 2023-08-28 01:58:58.967797636 +0000 2023-08-28T02:06:53.439 INFO:teuthology.orchestra.run.smithi143.stdout:Change: 2023-08-28 01:58:58.967797636 +0000 2023-08-28T02:06:53.439 INFO:teuthology.orchestra.run.smithi143.stdout: Birth: - 2023-08-28T02:06:53.439 DEBUG:teuthology.orchestra.run.smithi143:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-08-28T02:06:53.504 INFO:teuthology.orchestra.run.smithi143.stderr:1+0 records in 2023-08-28T02:06:53.505 INFO:teuthology.orchestra.run.smithi143.stderr:1+0 records out 2023-08-28T02:06:53.505 INFO:teuthology.orchestra.run.smithi143.stderr:512 bytes copied, 0.000219822 s, 2.3 MB/s 2023-08-28T02:06:53.506 DEBUG:teuthology.orchestra.run.smithi143:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-08-28T02:06:53.564 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2023-08-28T02:06:53.564 DEBUG:teuthology.orchestra.run.smithi169:> dd if=/scratch_devs of=/dev/stdout 2023-08-28T02:06:53.582 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-08-28T02:06:53.582 DEBUG:teuthology.orchestra.run.smithi169:> stat /dev/vg_nvme/lv_1 2023-08-28T02:06:53.639 INFO:teuthology.orchestra.run.smithi169.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-08-28T02:06:53.639 INFO:teuthology.orchestra.run.smithi169.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-08-28T02:06:53.639 INFO:teuthology.orchestra.run.smithi169.stdout:Device: 6h/6d Inode: 386915 Links: 1 2023-08-28T02:06:53.639 INFO:teuthology.orchestra.run.smithi169.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-08-28T02:06:53.640 INFO:teuthology.orchestra.run.smithi169.stdout:Context: system_u:object_r:device_t:s0 2023-08-28T02:06:53.640 INFO:teuthology.orchestra.run.smithi169.stdout:Access: 2023-08-28 02:06:31.182361455 +0000 2023-08-28T02:06:53.641 INFO:teuthology.orchestra.run.smithi169.stdout:Modify: 2023-08-28 01:58:57.400485830 +0000 2023-08-28T02:06:53.641 INFO:teuthology.orchestra.run.smithi169.stdout:Change: 2023-08-28 01:58:57.400485830 +0000 2023-08-28T02:06:53.643 INFO:teuthology.orchestra.run.smithi169.stdout: Birth: - 2023-08-28T02:06:53.644 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-08-28T02:06:53.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:53 smithi143 conmon[100258]: audit 2023-08-28T02:06:52.175579+0000 mgr.y (mgr. 2023-08-28T02:06:53.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:53 smithi143 conmon[100258]: 14146) 40 : audit [DBG] from='client.24106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi143=y;smithi169=x", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:06:53.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:53 smithi143 conmon[100258]: cephadm 2023-08-28T02:06:52.176962+0000 mgr.y (mgr.14146) 41 : cephadm [INF] Saving service mgr spec with placement smithi143=y;smithi169=x;count:2 2023-08-28T02:06:53.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:53 smithi143 conmon[100258]: cephadm 2023-08- 2023-08-28T02:06:53.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:53 smithi143 conmon[100258]: 28T02:06:52.224860+0000 mgr.y (mgr.14146) 42 : cephadm [INF] Deploying daemon mgr.x on smithi169 2023-08-28T02:06:53.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:53 smithi143 conmon[109923]: audit 2023-08-28T02:06:52 2023-08-28T02:06:53.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:53 smithi143 conmon[109923]: .175579+0000 mgr.y (mgr.14146) 40 : audit [DBG] from='client.24106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi143=y;smithi169=x", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:06:53.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:53 smithi143 conmon[109923]: cephadm 2023-08 2023-08-28T02:06:53.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:53 smithi143 conmon[109923]: -28T02:06:52.176962+0000 mgr.y (mgr.14146) 41 : cephadm [INF] Saving service mgr spec with placement smithi143=y;smithi169=x;count:2 2023-08-28T02:06:53.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:53 smithi143 conmon[109923]: cephadm 2023-08-28T02:06:53.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:53 smithi143 conmon[109923]: 2023-08-28T02:06: 2023-08-28T02:06:53.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:53 smithi143 conmon[109923]: 52.224860+0000 mgr.y (mgr. 2023-08-28T02:06:53.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:53 smithi143 conmon[109923]: 14146) 42 : cephadm [INF] Deploying daemon mgr.x on smithi169 2023-08-28T02:06:53.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:53 smithi143 conmon[109923]: 2023-08-28T02:06:53.707 INFO:teuthology.orchestra.run.smithi169.stderr:1+0 records in 2023-08-28T02:06:53.708 INFO:teuthology.orchestra.run.smithi169.stderr:1+0 records out 2023-08-28T02:06:53.708 INFO:teuthology.orchestra.run.smithi169.stderr:512 bytes copied, 0.000260404 s, 2.0 MB/s 2023-08-28T02:06:53.709 DEBUG:teuthology.orchestra.run.smithi169:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-08-28T02:06:53.768 DEBUG:teuthology.orchestra.run.smithi169:> stat /dev/vg_nvme/lv_2 2023-08-28T02:06:53.825 INFO:teuthology.orchestra.run.smithi169.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-08-28T02:06:53.825 INFO:teuthology.orchestra.run.smithi169.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-08-28T02:06:53.825 INFO:teuthology.orchestra.run.smithi169.stdout:Device: 6h/6d Inode: 383782 Links: 1 2023-08-28T02:06:53.825 INFO:teuthology.orchestra.run.smithi169.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-08-28T02:06:53.826 INFO:teuthology.orchestra.run.smithi169.stdout:Context: system_u:object_r:device_t:s0 2023-08-28T02:06:53.826 INFO:teuthology.orchestra.run.smithi169.stdout:Access: 2023-08-28 02:06:53.756893048 +0000 2023-08-28T02:06:53.826 INFO:teuthology.orchestra.run.smithi169.stdout:Modify: 2023-08-28 01:58:57.908474860 +0000 2023-08-28T02:06:53.826 INFO:teuthology.orchestra.run.smithi169.stdout:Change: 2023-08-28 01:58:57.908474860 +0000 2023-08-28T02:06:53.826 INFO:teuthology.orchestra.run.smithi169.stdout: Birth: - 2023-08-28T02:06:53.826 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-08-28T02:06:53.836 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:53 smithi169 conmon[100500]: audit 2023-08-28T02:06:52.175579+0000 mgr.y (mgr.14146) 40 : 2023-08-28T02:06:53.836 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:53 smithi169 conmon[100500]: audit [DBG] from='client.24106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi143=y;smithi169=x", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:06:53.836 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:53 smithi169 conmon[100500]: cephadm 2023-08-28T02:06:52.176962+0000 mgr.y (mgr.14146) 41 : cephadm [INF] Saving service mgr spec with placement smithi143=y;smithi169=x;count:2 2023-08-28T02:06:53.836 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:53 smithi169 conmon[100500]: cephadm 2023-08-28T02:06:52.224860+0000 mgr.y (mgr.14146) 42 : cephadm [INF] Deploying daemon mgr.x on smithi169 2023-08-28T02:06:53.931 INFO:teuthology.orchestra.run.smithi169.stderr:1+0 records in 2023-08-28T02:06:53.931 INFO:teuthology.orchestra.run.smithi169.stderr:1+0 records out 2023-08-28T02:06:53.931 INFO:teuthology.orchestra.run.smithi169.stderr:512 bytes copied, 0.000249631 s, 2.1 MB/s 2023-08-28T02:06:53.932 DEBUG:teuthology.orchestra.run.smithi169:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-08-28T02:06:53.951 DEBUG:teuthology.orchestra.run.smithi169:> stat /dev/vg_nvme/lv_3 2023-08-28T02:06:54.011 INFO:teuthology.orchestra.run.smithi169.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-08-28T02:06:54.011 INFO:teuthology.orchestra.run.smithi169.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-08-28T02:06:54.011 INFO:teuthology.orchestra.run.smithi169.stdout:Device: 6h/6d Inode: 391315 Links: 1 2023-08-28T02:06:54.011 INFO:teuthology.orchestra.run.smithi169.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-08-28T02:06:54.011 INFO:teuthology.orchestra.run.smithi169.stdout:Context: system_u:object_r:device_t:s0 2023-08-28T02:06:54.011 INFO:teuthology.orchestra.run.smithi169.stdout:Access: 2023-08-28 02:06:53.886890351 +0000 2023-08-28T02:06:54.011 INFO:teuthology.orchestra.run.smithi169.stdout:Modify: 2023-08-28 01:58:58.400464237 +0000 2023-08-28T02:06:54.011 INFO:teuthology.orchestra.run.smithi169.stdout:Change: 2023-08-28 01:58:58.400464237 +0000 2023-08-28T02:06:54.012 INFO:teuthology.orchestra.run.smithi169.stdout: Birth: - 2023-08-28T02:06:54.012 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-08-28T02:06:54.079 INFO:teuthology.orchestra.run.smithi169.stderr:1+0 records in 2023-08-28T02:06:54.079 INFO:teuthology.orchestra.run.smithi169.stderr:1+0 records out 2023-08-28T02:06:54.079 INFO:teuthology.orchestra.run.smithi169.stderr:512 bytes copied, 0.000213171 s, 2.4 MB/s 2023-08-28T02:06:54.080 DEBUG:teuthology.orchestra.run.smithi169:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-08-28T02:06:54.139 DEBUG:teuthology.orchestra.run.smithi169:> stat /dev/vg_nvme/lv_4 2023-08-28T02:06:54.197 INFO:teuthology.orchestra.run.smithi169.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-08-28T02:06:54.198 INFO:teuthology.orchestra.run.smithi169.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-08-28T02:06:54.198 INFO:teuthology.orchestra.run.smithi169.stdout:Device: 6h/6d Inode: 391374 Links: 1 2023-08-28T02:06:54.198 INFO:teuthology.orchestra.run.smithi169.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-08-28T02:06:54.198 INFO:teuthology.orchestra.run.smithi169.stdout:Context: system_u:object_r:device_t:s0 2023-08-28T02:06:54.198 INFO:teuthology.orchestra.run.smithi169.stdout:Access: 2023-08-28 02:06:53.887890330 +0000 2023-08-28T02:06:54.198 INFO:teuthology.orchestra.run.smithi169.stdout:Modify: 2023-08-28 01:58:58.931452771 +0000 2023-08-28T02:06:54.198 INFO:teuthology.orchestra.run.smithi169.stdout:Change: 2023-08-28 01:58:58.931452771 +0000 2023-08-28T02:06:54.198 INFO:teuthology.orchestra.run.smithi169.stdout: Birth: - 2023-08-28T02:06:54.198 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-08-28T02:06:54.237 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:06:53 smithi169 systemd[1]: Starting Ceph mgr.x for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:06:54.262 INFO:teuthology.orchestra.run.smithi169.stderr:1+0 records in 2023-08-28T02:06:54.262 INFO:teuthology.orchestra.run.smithi169.stderr:1+0 records out 2023-08-28T02:06:54.262 INFO:teuthology.orchestra.run.smithi169.stderr:512 bytes copied, 0.000205463 s, 2.5 MB/s 2023-08-28T02:06:54.263 DEBUG:teuthology.orchestra.run.smithi169:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-08-28T02:06:54.321 INFO:tasks.cephadm:Deploying osd.0 on smithi143 with /dev/vg_nvme/lv_4... 2023-08-28T02:06:54.322 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-08-28T02:06:54.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:54 smithi143 conmon[100258]: cluster 2023-08-28T02:06:52.922894+0000 2023-08-28T02:06:54.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:54 smithi143 conmon[100258]: mgr.y (mgr.14146) 43 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:06:54.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:54 smithi143 conmon[109923]: cluster 2023-08-28T02:06:52.922894+0000 mgr.y (mgr.14146) 2023-08-28T02:06:54.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:54 smithi143 conmon[109923]: 43 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:06:54.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:54 smithi169 conmon[100500]: cluster 2023-08-28T02:06:52.922894+0000 mgr.y (mgr.14146) 43 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:06:54.987 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:06:54 smithi169 conmon[102807]: debug 2023-08-28T02:06:54.533+0000 7f4decded700 1 -- 172.21.15.169:0/3397444734 <== mon.1 v2:172.21.15.169:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x562c37c8c4e0 con 0x562c37c62c00 2023-08-28T02:06:54.987 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:06:54 smithi169 bash[102692]: 96e04bb664893b082c58ecc0047815ffc0b222caebcfa5cf6320e19318e56b00 2023-08-28T02:06:54.987 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:06:54 smithi169 systemd[1]: Started Ceph mgr.x for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:06:54.987 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:06:54 smithi169 conmon[102807]: debug 2023-08-28T02:06:54.663+0000 7f4dfd3d4680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-08-28T02:06:54.987 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:06:54 smithi169 conmon[102807]: debug 2023-08-28T02:06:54.746+0000 7f4dfd3d4680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-08-28T02:06:55.486 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:06:55 smithi169 conmon[102807]: debug 2023-08-28T02:06:55.178+0000 7f4dfd3d4680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-08-28T02:06:55.772 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:55 smithi143 conmon[100258]: audit 2023-08-28T02:06:54.600729+0000 mon.a (mon.0) 210 : 2023-08-28T02:06:55.773 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:55 smithi143 conmon[100258]: audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:55.773 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:55 smithi143 conmon[100258]: audit 2023-08-28T02:06:54.602441+0000 mon.a (mon.0) 211 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:06:55.773 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:55 smithi143 conmon[100258]: cluster 2023-08-28T02:06:54.923242+0000 mgr.y (mgr.14146) 44 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:06:55.773 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:55 smithi143 conmon[100258]: audit 2023-08-28T02:06:55.507921+0000 mon.a (mon.0) 212 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:55.773 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:55 smithi143 conmon[100258]: audit 2023-08-28T02:06:55.508529+0000 mon.a (mon.0) 213 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:55.773 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:55 smithi143 conmon[100258]: audit 2023-08-28T02:06:55.509232+0000 mon.a (mon.0) 214 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:06:55.773 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:55 smithi143 conmon[100258]: audit 2023-08-28T02:06:55.513401+0000 mon.a (mon.0) 215 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:55.773 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:55 smithi143 conmon[100258]: cephadm 2023-08-28T02:06:55.515045+0000 mgr.y (mgr.14146) 45 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-08-28T02:06:55.773 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:55 smithi143 conmon[100258]: audit 2023-08-28T02:06:55.515297+0000 mon.a (mon.0) 216 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-28T02:06:55.774 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:55 smithi143 conmon[100258]: audit 2023-08-28T02:06:55.515965+0000 mon.a (mon.0) 217 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-08-28T02:06:55.774 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:55 smithi143 conmon[100258]: audit 2023-08-28T02:06:55.516546+0000 mon.a (mon.0) 218 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:55.774 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:55 smithi143 conmon[100258]: cephadm 2023-08-28T02:06:55.517129+0000 mgr.y (mgr.14146) 46 : cephadm [INF] Reconfiguring daemon mgr.y on smithi143 2023-08-28T02:06:55.774 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:55 smithi143 conmon[109923]: audit 2023-08-28T02:06:54.600729+0000 mon.a ( 2023-08-28T02:06:55.774 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:55 smithi143 conmon[109923]: mon.0) 210 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:55.774 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:55 smithi143 conmon[109923]: audit 2023-08-28T02:06:54.602441+0000 mon.a (mon.0) 211 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:06:55.774 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:55 smithi143 conmon[109923]: cluster 2023-08-28T02:06:54.923242+0000 mgr.y (mgr.14146) 44 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:06:55.774 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:55 smithi143 conmon[109923]: audit 2023-08-28T02:06:55.507921+0000 mon.a (mon.0) 212 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:55.774 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:55 smithi143 conmon[109923]: audit 2023-08-28T02:06:55.508529+0000 mon.a (mon.0) 213 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:55.774 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:55 smithi143 conmon[109923]: audit 2023-08-28T02:06:55.509232+0000 mon.a (mon.0) 214 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:06:55.775 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:55 smithi143 conmon[109923]: audit 2023-08-28T02:06:55.513401+0000 mon.a (mon.0) 215 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:55.775 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:55 smithi143 conmon[109923]: cephadm 2023-08-28T02:06 2023-08-28T02:06:55.775 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:55 smithi143 conmon[109923]: :55.515045+0000 mgr.y (mgr.14146) 45 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-08-28T02:06:55.775 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:55 smithi143 conmon[109923]: audit 2023-08-28T02:06:55.515297+0000 mon.a (mon.0) 216 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-28T02:06:55.775 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:55 smithi143 conmon[109923]: audit 2023-08-28T 2023-08-28T02:06:55.775 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:55 smithi143 conmon[109923]: 02:06:55.515965+0000 mon.a (mon 2023-08-28T02:06:55.775 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:55 smithi143 conmon[109923]: .0) 217 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-08-28T02:06:55.775 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:55 smithi143 conmon[109923]: audit 2023-08-28T02:06: 2023-08-28T02:06:55.775 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:55 smithi143 conmon[109923]: 55.516546+0000 mon.a (mon.0) 218 : audit 2023-08-28T02:06:55.775 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:55 smithi143 conmon[109923]: [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:55.775 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:55 smithi143 conmon[109923]: cephadm 2023 2023-08-28T02:06:55.775 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:55 smithi143 conmon[109923]: -08-28T02:06:55.517129+0000 mgr.y ( 2023-08-28T02:06:55.776 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:55 smithi143 conmon[109923]: mgr.14146) 46 : cephadm [INF] Reconfiguring daemon mgr.y on smithi143 2023-08-28T02:06:55.946 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:55 smithi169 conmon[100500]: audit 2023-08-28T02:06:54.600729+0000 mon.a (mon.0) 210 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:55.946 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:55 smithi169 conmon[100500]: audit 2023-08-28T02:06: 2023-08-28T02:06:55.946 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:55 smithi169 conmon[100500]: 54.602441+0000 mon.a (mon.0) 211 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:06:55.946 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:55 smithi169 conmon[100500]: cluster 2023-08-28T02:06:54.923242+0000 mgr.y (mgr.14146) 44 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:06:55.946 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:55 smithi169 conmon[100500]: audit 2023-08-28T02:06:55.507921+0000 mon.a (mon.0) 212 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:55.946 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:55 smithi169 conmon[100500]: audit 2023-08-28T02:06:55.508529+0000 mon.a (mon.0) 213 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:55.946 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:55 smithi169 conmon[100500]: audit 2023-08-28T02:06:55.509232+0000 mon.a ( 2023-08-28T02:06:55.946 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:55 smithi169 conmon[100500]: mon.0) 214 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:06:55.947 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:55 smithi169 conmon[100500]: audit 2023-08-28T02:06:55.513401+0000 mon.a (mon. 2023-08-28T02:06:55.947 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:55 smithi169 conmon[100500]: 0) 215 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:55.947 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:55 smithi169 conmon[100500]: cephadm 2023-08-28T02:06:55.947 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:55 smithi169 conmon[100500]: 2023-08-28T02:06:55.515045+0000 mgr.y ( 2023-08-28T02:06:55.947 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:55 smithi169 conmon[100500]: mgr.14146) 45 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-08-28T02:06:55.947 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:55 smithi169 conmon[100500]: audit 2023-08 2023-08-28T02:06:55.947 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:55 smithi169 conmon[100500]: -28T02:06:55.515297+0000 mon.a 2023-08-28T02:06:55.947 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:55 smithi169 conmon[100500]: (mon.0) 216 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-28T02:06:55.947 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:55 smithi169 conmon[100500]: audit 2023-08 2023-08-28T02:06:55.947 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:55 smithi169 conmon[100500]: -28T02:06:55.515965+0000 mon.a 2023-08-28T02:06:55.947 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:55 smithi169 conmon[100500]: (mon.0) 217 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-08-28T02:06:55.947 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:55 smithi169 conmon[100500]: audit 2023- 2023-08-28T02:06:55.948 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:55 smithi169 conmon[100500]: 08-28T02:06:55.516546+0000 mon.a 2023-08-28T02:06:55.948 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:55 smithi169 conmon[100500]: (mon.0) 218 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:55.948 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:55 smithi169 conmon[100500]: cephadm 2023- 2023-08-28T02:06:55.948 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:55 smithi169 conmon[100500]: 08-28T02:06:55.517129+0000 2023-08-28T02:06:55.948 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:55 smithi169 conmon[100500]: mgr.y (mgr.14146) 46 : cephadm [INF] Reconfiguring daemon mgr.y on smithi143 2023-08-28T02:06:55.948 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:55 smithi169 conmon[100500]: 2023-08-28T02:06:56.233 INFO:teuthology.orchestra.run.smithi143.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-08-28T02:06:56.233 INFO:teuthology.orchestra.run.smithi143.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-08-28T02:06:56.233 INFO:teuthology.orchestra.run.smithi143.stderr: stderr: 10+0 records in 2023-08-28T02:06:56.233 INFO:teuthology.orchestra.run.smithi143.stderr:10+0 records out 2023-08-28T02:06:56.233 INFO:teuthology.orchestra.run.smithi143.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0147277 s, 712 MB/s 2023-08-28T02:06:56.233 INFO:teuthology.orchestra.run.smithi143.stderr:--> Zapping successful for: 2023-08-28T02:06:56.236 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:06:55 smithi169 conmon[102807]: debug 2023-08-28T02:06:55.946+0000 7f4dfd3d4680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-08-28T02:06:56.237 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:06:56 smithi169 conmon[102807]: debug 2023-08-28T02:06:56.029+0000 7f4dfd3d4680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-08-28T02:06:56.736 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:06:56 smithi169 conmon[102807]: debug 2023-08-28T02:06:56.240+0000 7f4dfd3d4680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-08-28T02:06:57.250 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph orch daemon add osd smithi143:vg_nvme/lv_4 2023-08-28T02:06:57.471 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:06:57 smithi169 conmon[102807]: debug 2023-08-28T02:06:57.179+0000 7f4dfd3d4680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-08-28T02:06:57.472 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:06:57 smithi169 conmon[102807]: debug 2023-08-28T02:06:57.305+0000 7f4dfd3d4680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-08-28T02:06:57.472 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:06:57 smithi169 conmon[102807]: debug 2023-08-28T02:06:57.382+0000 7f4dfd3d4680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-08-28T02:06:57.736 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:06:57 smithi169 conmon[102807]: debug 2023-08-28T02:06:57.472+0000 7f4dfd3d4680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-08-28T02:06:57.737 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:06:57 smithi169 conmon[102807]: debug 2023-08-28T02:06:57.667+0000 7f4dfd3d4680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-08-28T02:06:57.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:57 smithi143 conmon[100258]: audit 2023-08-28T02:06:56.876246+0000 mon.a (mon.0) 219 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:57.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:57 smithi143 conmon[100258]: audit 2023-08-28T02:06:56.877057+0000 mon.a (mon.0) 220 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:06:57.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:57 smithi143 conmon[100258]: audit 2023-08-28T02:06:56.878303+0000 mon.a (mon.0) 221 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:57.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:57 smithi143 conmon[100258]: audit 2023-08-28T02:06:56.878968+0000 mon.a (mon.0) 222 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:06:57.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:57 smithi143 conmon[100258]: audit 2023-08-28T02:06:56.884533+0000 mon.a (mon.0) 223 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:57.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:57 smithi143 conmon[100258]: cluster 2023-08-28T02:06:56.923512+0000 mgr.y (mgr.14146) 47 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:06:57.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:57 smithi143 conmon[109923]: audit 2023-08-28T02:06:56.876246+0000 mon.a 2023-08-28T02:06:57.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:57 smithi143 conmon[109923]: (mon.0) 219 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:57.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:57 smithi143 conmon[109923]: audit 2023-08-28T02:06:56.877057+0000 mon.a (mon. 2023-08-28T02:06:57.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:57 smithi143 conmon[109923]: 0) 220 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:06:57.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:57 smithi143 conmon[109923]: audit 2023-08 2023-08-28T02:06:57.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:57 smithi143 conmon[109923]: -28T02:06:56.878303+0000 mon.a (mon.0) 221 : 2023-08-28T02:06:57.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:57 smithi143 conmon[109923]: audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:57.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:57 smithi143 conmon[109923]: audit 2023-08-28T02:06:57.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:57 smithi143 conmon[109923]: 2023-08-28T02:06:56.878968+0000 mon.a (mon. 2023-08-28T02:06:57.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:57 smithi143 conmon[109923]: 0) 222 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:06:57.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:57 smithi143 conmon[109923]: audit 2023-08-28T02:06:56.884533 2023-08-28T02:06:57.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:57 smithi143 conmon[109923]: +0000 mon.a (mon.0) 223 : audit [INF] 2023-08-28T02:06:57.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:57 smithi143 conmon[109923]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:57.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:57 smithi143 conmon[109923]: cluster 2023-08-28T02:06:57.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:57 smithi143 conmon[109923]: 2023-08-28T02:06:56.923512+0000 mgr.y 2023-08-28T02:06:57.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:57 smithi143 conmon[109923]: (mgr.14146) 47 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:06:58.237 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:06:57 smithi169 conmon[102807]: debug 2023-08-28T02:06:57.945+0000 7f4dfd3d4680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-08-28T02:06:58.237 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:06:58 smithi169 conmon[102807]: debug 2023-08-28T02:06:58.043+0000 7f4dfd3d4680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-08-28T02:06:58.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:57 smithi169 conmon[100500]: audit 2023-08-28T02:06:56.876246+0000 mon.a (mon.0) 219 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:58.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:57 smithi169 conmon[100500]: audit 2023-08-28T02:06:58.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:57 smithi169 conmon[100500]: 2023-08-28T02:06:56.877057+0000 mon.a (mon.0) 220 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:06:58.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:57 smithi169 conmon[100500]: audit 2023-08-28T02:06:56.878303+0000 mon.a (mon.0) 221 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:58.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:57 smithi169 conmon[100500]: audit 2023-08-28T02:06:58.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:57 smithi169 conmon[100500]: 2023-08-28T02:06:56.878968+0000 mon.a (mon.0) 222 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:06:58.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:57 smithi169 conmon[100500]: audit 2023-08 2023-08-28T02:06:58.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:57 smithi169 conmon[100500]: -28T02:06:56.884533+0000 mon.a ( 2023-08-28T02:06:58.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:57 smithi169 conmon[100500]: mon.0) 223 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:58.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:57 smithi169 conmon[100500]: cluster 2023-08 2023-08-28T02:06:58.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:57 smithi169 conmon[100500]: -28T02:06:56.923512+0000 mgr.y 2023-08-28T02:06:58.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:57 smithi169 conmon[100500]: (mgr.14146) 47 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:06:59.056 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:06:58 smithi169 conmon[102807]: debug 2023-08-28T02:06:58.764+0000 7f4dfd3d4680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-08-28T02:06:59.056 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:06:58 smithi169 conmon[102807]: debug 2023-08-28T02:06:58.845+0000 7f4dfd3d4680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-08-28T02:06:59.056 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:06:58 smithi169 conmon[102807]: debug 2023-08-28T02:06:58.936+0000 7f4dfd3d4680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-08-28T02:06:59.373 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:06:59 smithi169 conmon[102807]: debug 2023-08-28T02:06:59.111+0000 7f4dfd3d4680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-08-28T02:06:59.373 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:06:59 smithi169 conmon[102807]: debug 2023-08-28T02:06:59.373+0000 7f4dfd3d4680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-08-28T02:06:59.374 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:59 smithi169 conmon[100500]: audit 2023-08-28T02:06:58.055786+0000 mon.a (mon.0) 224 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:59.374 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:59 smithi169 conmon[100500]: audit 2023-08-28T02:06:58. 2023-08-28T02:06:59.374 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:59 smithi169 conmon[100500]: 936734+0000 mon.a (mon.0) 225 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-28T02:06:59.374 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:59 smithi169 conmon[100500]: audit 2023-08-28T02:06:58.938659+0000 mon.a (mon.0) 226 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-28T02:06:59.375 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:59 smithi169 conmon[100500]: audit 2023-08-28T02:06:58 2023-08-28T02:06:59.375 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:06:59 smithi169 conmon[100500]: .939199+0000 mon.a (mon.0) 227 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:59.433 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:59 smithi143 conmon[100258]: audit 2023-08-28T02:06:58.055786+0000 2023-08-28T02:06:59.434 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:59 smithi143 conmon[100258]: mon.a (mon.0) 224 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:59.434 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:59 smithi143 conmon[100258]: audit 2023-08-28T02:06:58.936734+0000 mon.a (mon.0) 225 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-28T02:06:59.434 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:59 smithi143 conmon[100258]: audit 2023-08-28T02:06:58.938659+0000 mon.a (mon.0) 226 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-28T02:06:59.434 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:06:59 smithi143 conmon[100258]: audit 2023-08-28T02:06:58.939199+0000 mon.a (mon.0) 227 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:59.434 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:59 smithi143 conmon[109923]: audit 2023-08-28T02:06:58.055786+0000 2023-08-28T02:06:59.434 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:59 smithi143 conmon[109923]: mon.a (mon.0) 224 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:06:59.434 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:59 smithi143 conmon[109923]: audit 2023-08-28T02:06:58.936734+0000 mon.a (mon.0) 225 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-28T02:06:59.434 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:59 smithi143 conmon[109923]: audit 2023-08-28T02:06:58.938659+0000 mon.a (mon.0) 226 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-28T02:06:59.434 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:06:59 smithi143 conmon[109923]: audit 2023-08-28T02:06:58.939199+0000 mon.a (mon.0) 227 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:59.635 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:06:59 smithi169 conmon[102807]: debug 2023-08-28T02:06:59.504+0000 7f4dfd3d4680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-08-28T02:06:59.889 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:06:59 smithi169 conmon[102807]: debug 2023-08-28T02:06:59.636+0000 7f4dfd3d4680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-08-28T02:06:59.889 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:06:59 smithi169 conmon[102807]: debug 2023-08-28T02:06:59.807+0000 7f4dfd3d4680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-08-28T02:07:00.237 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:06:59 smithi169 conmon[102807]: debug 2023-08-28T02:06:59.890+0000 7f4dfd3d4680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-08-28T02:07:00.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:00 smithi169 conmon[100500]: cluster 2023-08-28T02:06:58.923770+0000 mgr.y (mgr.14146) 48 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:07:00.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:00 smithi169 conmon[100500]: audit 2023-08-28T02:06 2023-08-28T02:07:00.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:00 smithi169 conmon[100500]: :58.934638+0000 mgr.y (mgr.14146) 49 : audit [DBG] from='client.24119 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi143:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:07:00.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:00 smithi169 conmon[100500]: cluster 2023-08-28T02:06:59.904908+0000 mon.a (mon.0) 228 : cluster [DBG] Standby manager daemon x started 2023-08-28T02:07:00.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:00 smithi169 conmon[100500]: audit 2023-08-28T02:07:00.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:00 smithi169 conmon[100500]: 2023-08-28T02:06:59.905927+0000 mon.b (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.169:0/2830030702' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-08-28T02:07:00.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:00 smithi169 conmon[100500]: audit 2023-08- 2023-08-28T02:07:00.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:00 smithi169 conmon[100500]: 28T02:06:59.906234+0000 mon.b (mon 2023-08-28T02:07:00.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:00 smithi169 conmon[100500]: .1) 5 : audit [DBG] from='mgr.? 172.21.15.169:0/2830030702' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-28T02:07:00.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:00 smithi169 conmon[100500]: audit 2023- 2023-08-28T02:07:00.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:00 smithi169 conmon[100500]: 08-28T02:06:59.907110+0000 mon.b 2023-08-28T02:07:00.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:00 smithi169 conmon[100500]: (mon.1) 6 : audit [DBG] from='mgr.? 172.21.15.169:0/2830030702' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-08-28T02:07:00.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:00 smithi169 conmon[100500]: audit 2023 2023-08-28T02:07:00.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:00 smithi169 conmon[100500]: -08-28T02:06:59.907404+0000 2023-08-28T02:07:00.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:00 smithi169 conmon[100500]: mon.b (mon.1) 7 : audit [DBG] from='mgr.? 172.21.15.169:0/2830030702' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-08-28T02:07:00.435 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:00 smithi143 conmon[100258]: cluster 2023-08-28T02:06:58.923770+0000 mgr.y (mgr.14146) 2023-08-28T02:07:00.435 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:00 smithi143 conmon[100258]: 48 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:07:00.435 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:00 smithi143 conmon[100258]: audit 2023-08-28T02:06:58.934638+0000 mgr.y (mgr.14146) 49 : audit [DBG] from='client.24119 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi143:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:07:00.435 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:00 smithi143 conmon[100258]: cluster 2023-08-28T02:06:59.904908+0000 mon.a (mon.0) 228 : cluster [DBG] Standby manager daemon x started 2023-08-28T02:07:00.436 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:00 smithi143 conmon[100258]: audit 2023-08-28T02:06:59.905927+0000 mon.b (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.169:0/2830030702' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-08-28T02:07:00.436 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:00 smithi143 conmon[100258]: audit 2023-08-28T02:06:59.906234+0000 mon.b (mon.1) 5 : audit [DBG] from='mgr.? 172.21.15.169:0/2830030702' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-28T02:07:00.436 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:00 smithi143 conmon[100258]: audit 2023-08-28T02:06:59.907110+0000 mon.b (mon.1) 6 : audit [DBG] from='mgr.? 172.21.15.169:0/2830030702' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-08-28T02:07:00.436 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:00 smithi143 conmon[100258]: audit 2023-08-28T02:06:59.907404+0000 mon.b (mon.1) 7 : audit [DBG] from='mgr.? 172.21.15.169:0/2830030702' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-08-28T02:07:00.436 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:00 smithi143 conmon[109923]: cluster 2023-08-28T02:06:58.923770+0000 2023-08-28T02:07:00.436 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:00 smithi143 conmon[109923]: mgr.y (mgr.14146) 48 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:07:00.436 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:00 smithi143 conmon[109923]: audit 2023-08-28T02:06:58.934638+0000 mgr.y (mgr.14146) 49 : audit [DBG] from='client.24119 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi143:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:07:00.436 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:00 smithi143 conmon[109923]: cluster 2023-08-28T02:06:59.904908+0000 mon.a (mon.0) 228 : cluster [DBG] Standby manager daemon x started 2023-08-28T02:07:00.436 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:00 smithi143 conmon[109923]: audit 2023-08-28T02:06:59.905927+0000 mon.b (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.169:0/2830030702' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-08-28T02:07:00.436 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:00 smithi143 conmon[109923]: audit 2023-08-28T02:06:59.906234+0000 mon.b (mon.1) 2023-08-28T02:07:00.437 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:00 smithi143 conmon[109923]: 5 : audit [DBG] from='mgr.? 172.21.15.169:0/2830030702' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-28T02:07:00.437 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:00 smithi143 conmon[109923]: audit 2023-08-28T 2023-08-28T02:07:00.437 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:00 smithi143 conmon[109923]: 02:06:59.907110+0000 mon.b (mon.1) 2023-08-28T02:07:00.437 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:00 smithi143 conmon[109923]: 6 : audit [DBG] from='mgr.? 172.21.15.169:0/2830030702' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-08-28T02:07:00.437 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:00 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:07:00.437 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:00 smithi143 conmon[109923]: 08-28T02:06:59.907404+0000 mon.b ( 2023-08-28T02:07:00.437 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:00 smithi143 conmon[109923]: mon.1) 7 : audit [DBG] from='mgr.? 172.21.15.169:0/2830030702' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-08-28T02:07:01.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:01 smithi143 conmon[100258]: cluster 2023-08-28T02:07:00.071382+0000 mon.a (mon.0) 229 : cluster [DBG] mgrmap e14: y(active, since 57s), standbys: x 2023-08-28T02:07:01.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:01 smithi143 conmon[100258]: audit 2023-08-28T02:07:00.071506+0000 mon.a (mon.0) 230 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-08-28T02:07:01.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:01 smithi143 conmon[109923]: cluster 2023-08-28T02:07:00.071382+0000 mon.a (mon.0) 229 : cluster [DBG] mgrmap e14: y(active, since 57s), standbys: x 2023-08-28T02:07:01.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:01 smithi143 conmon[109923]: audit 2023-08-28T02:07:00.071506+0000 mon.a (mon.0) 230 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-08-28T02:07:01.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:01 smithi169 conmon[100500]: cluster 2023-08-28T02:07:00.071382+0000 mon.a (mon.0) 229 : cluster [DBG] mgrmap e14: y(active, since 57s), standbys: x 2023-08-28T02:07:01.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:01 smithi169 conmon[100500]: 2023-08-28T02:07:01.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:01 smithi169 conmon[100500]: audit 2023-08-28T02:07:00.071506+0000 mon.a (mon.0) 230 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-08-28T02:07:02.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:02 smithi143 conmon[100258]: cluster 2023-08-28T02:07:00.924029+0000 mgr.y (mgr.14146) 50 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:07:02.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:02 smithi143 conmon[100258]: audit 2023-08-28T02:07:01.165090+0000 mon.a ( 2023-08-28T02:07:02.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:02 smithi143 conmon[100258]: mon.0) 231 : audit [INF] from='client.? 172.21.15.143:0/604468815' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "718c8f27-245b-4e9d-9e2e-d1fc65ca7472"}]: dispatch 2023-08-28T02:07:02.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:02 smithi143 conmon[100258]: audit 2023-08-28T02:07:01.168508+0000 mon.a (mon.0) 232 : audit [INF] from='client.? 172.21.15.143:0/604468815' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "718c8f27-245b-4e9d-9e2e-d1fc65ca7472"}]': finished 2023-08-28T02:07:02.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:02 smithi143 conmon[100258]: cluster 2023-08-28T02:07:01.168570+0000 mon.a (mon.0) 233 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-08-28T02:07:02.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:02 smithi143 conmon[100258]: audit 2023-08-28T02:07:01.168680+0000 mon.a (mon.0) 234 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-28T02:07:02.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:02 smithi143 conmon[100258]: audit 2023-08-28T02:07:01.733672+0000 mon.a (mon.0) 235 : audit [DBG] from='client.? 172.21.15.143:0/1606766697' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-28T02:07:02.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:02 smithi143 conmon[109923]: cluster 2023-08-28T02:07:00.924029+0000 mgr.y (mgr.14146) 50 2023-08-28T02:07:02.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:02 smithi143 conmon[109923]: : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:07:02.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:02 smithi143 conmon[109923]: audit 2023-08-28T02:07:01.165090+0000 mon.a (mon.0) 231 : audit [INF] from='client.? 172.21.15.143:0/604468815' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "718c8f27-245b-4e9d-9e2e-d1fc65ca7472"}]: dispatch 2023-08-28T02:07:02.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:02 smithi143 conmon[109923]: audit 2023-08-28T02:07:01.168508+0000 mon.a (mon.0) 232 : audit [INF] from='client.? 172.21.15.143:0/604468815' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "718c8f27-245b-4e9d-9e2e-d1fc65ca7472"}]': finished 2023-08-28T02:07:02.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:02 smithi143 conmon[109923]: 2023-08-28T02:07:02.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:02 smithi143 conmon[109923]: cluster 2023-08-28T02:07 2023-08-28T02:07:02.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:02 smithi143 conmon[109923]: :01.168570+0000 mon.a (mon.0) 233 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-08-28T02:07:02.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:02 smithi143 conmon[109923]: audit 2023-08-28T02:07:01.168680+0000 mon.a 2023-08-28T02:07:02.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:02 smithi143 conmon[109923]: (mon.0) 234 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-28T02:07:02.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:02 smithi143 conmon[109923]: audit 2023-08-28T02:07:01.733672+0000 mon.a 2023-08-28T02:07:02.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:02 smithi143 conmon[109923]: (mon.0) 235 : audit [DBG] from='client.? 172.21.15.143:0/1606766697' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-28T02:07:02.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:02 smithi169 conmon[100500]: cluster 2023-08-28T02:07:00.924029+0000 mgr.y (mgr.14146) 50 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:07:02.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:02 smithi169 conmon[100500]: audit 2023-08-28T02:07:01 2023-08-28T02:07:02.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:02 smithi169 conmon[100500]: .165090+0000 mon.a (mon.0) 231 : audit [INF] from='client.? 172.21.15.143:0/604468815' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "718c8f27-245b-4e9d-9e2e-d1fc65ca7472"}]: dispatch 2023-08-28T02:07:02.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:02 smithi169 conmon[100500]: audit 2023-08-28T02:07:01.168508+0000 mon.a (mon.0) 232 : audit [INF] from='client.? 172.21.15.143:0/604468815' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "718c8f27-245b-4e9d-9e2e-d1fc65ca7472"}]': finished 2023-08-28T02:07:02.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:02 smithi169 conmon[100500]: cluster 2023-08-28T02:07 2023-08-28T02:07:02.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:02 smithi169 conmon[100500]: :01.168570+0000 mon.a (mon.0) 233 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-08-28T02:07:02.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:02 smithi169 conmon[100500]: audit 2023-08-28T02:07:02.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:02 smithi169 conmon[100500]: 2023-08-28T02:07:01.168680+0000 mon.a (mon.0) 2023-08-28T02:07:02.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:02 smithi169 conmon[100500]: 234 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-28T02:07:02.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:02 smithi169 conmon[100500]: audit 2023-08-28T02:07 2023-08-28T02:07:02.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:02 smithi169 conmon[100500]: :01.733672+0000 mon.a (mon.0) 235 : audit 2023-08-28T02:07:02.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:02 smithi169 conmon[100500]: [DBG] from='client.? 172.21.15.143:0/1606766697' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-28T02:07:04.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:04 smithi143 conmon[100258]: cluster 2023-08-28T02:07:02.924298+0000 mgr.y 2023-08-28T02:07:04.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:04 smithi143 conmon[100258]: (mgr.14146) 51 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:07:04.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:04 smithi143 conmon[109923]: cluster 2023-08-28T02:07:02.924298+0000 2023-08-28T02:07:04.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:04 smithi143 conmon[109923]: mgr.y (mgr.14146) 51 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:07:04.486 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:04 smithi169 conmon[100500]: cluster 2023-08-28T02:07:02.924298+0000 mgr.y (mgr.14146) 51 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:07:06.237 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:06 smithi143 conmon[100258]: cluster 2023-08-28T02:07:04.924581+0000 mgr.y (mgr.14146) 52 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:07:06.237 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:06 smithi143 conmon[109923]: cluster 2023-08-28T02:07:04.924581+0000 mgr.y (mgr.14146) 52 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:07:06.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:06 smithi169 conmon[100500]: cluster 2023-08-28T02:07:04.924581+0000 mgr.y (mgr. 2023-08-28T02:07:06.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:06 smithi169 conmon[100500]: 14146) 52 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:07:08.449 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:08 smithi143 conmon[100258]: cluster 2023-08-28T02:07:06.924870+0000 mgr.y (mgr.14146) 53 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:07:08.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:08 smithi143 conmon[100258]: audit 2023-08-28T02:07:07.199793+0000 mon.a (mon.0) 236 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-08-28T02:07:08.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:08 smithi143 conmon[100258]: audit 2023-08-28T02:07:07.200448+0000 mon.a (mon.0) 237 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:07:08.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:08 smithi143 conmon[100258]: cephadm 2023-08-28T02:07:07.200940+0000 mgr.y (mgr.14146) 54 : cephadm [INF] Deploying daemon osd.0 on smithi143 2023-08-28T02:07:08.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:08 smithi143 conmon[109923]: cluster 2023-08-28T02:07:06.924870+0000 mgr.y (mgr.14146) 53 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:07:08.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:08 smithi143 conmon[109923]: audit 2023-08-28T02:07:07.199793+0000 mon.a (mon.0) 236 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-08-28T02:07:08.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:08 smithi143 conmon[109923]: audit 2023-08-28T02:07:07.200448+0000 mon.a (mon.0) 237 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:07:08.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:08 smithi143 conmon[109923]: cephadm 2023-08-28T02:07:07.200940+0000 mgr.y (mgr.14146) 54 : cephadm [INF] Deploying daemon osd.0 on smithi143 2023-08-28T02:07:08.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:08 smithi169 conmon[100500]: cluster 2023-08-28T02:07:06.924870+0000 mgr.y (mgr.14146) 53 : cluster 2023-08-28T02:07:08.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:08 smithi169 conmon[100500]: [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:07:08.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:08 smithi169 conmon[100500]: audit 2023-08-28T02:07:07.199793+0000 mon.a (mon.0) 236 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-08-28T02:07:08.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:08 smithi169 conmon[100500]: audit 2023-08-28T02:07:08.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:08 smithi169 conmon[100500]: 2023-08-28T02:07:07.200448+0000 mon.a (mon.0) 237 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:07:08.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:08 smithi169 conmon[100500]: cephadm 2023-08-28T02:07 2023-08-28T02:07:08.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:08 smithi169 conmon[100500]: :07.200940+0000 mgr.y (mgr.14146) 54 : cephadm 2023-08-28T02:07:08.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:08 smithi169 conmon[100500]: [INF] Deploying daemon osd.0 on smithi143 2023-08-28T02:07:10.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:10 smithi143 conmon[100258]: cluster 2023-08-28T02:07:08.925133+0000 mgr.y ( 2023-08-28T02:07:10.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:10 smithi143 conmon[100258]: mgr.14146) 55 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:07:10.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:10 smithi143 conmon[109923]: cluster 2023-08-28T02:07:08.925133+0000 mgr.y ( 2023-08-28T02:07:10.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:10 smithi143 conmon[109923]: mgr.14146) 55 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:07:10.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:10 smithi169 conmon[100500]: cluster 2023-08-28T02:07:08.925133+0000 mgr.y (mgr.14146) 55 : 2023-08-28T02:07:10.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:10 smithi169 conmon[100500]: cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:07:12.415 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:12 smithi143 conmon[100258]: cluster 2023-08-28T02:07:10.925354+0000 2023-08-28T02:07:12.415 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:12 smithi143 conmon[100258]: mgr.y (mgr.14146) 56 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:07:12.415 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:12 smithi143 conmon[100258]: audit 2023-08-28T02:07:11.942513+0000 mon.a (mon.0) 238 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:12.416 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:12 smithi143 conmon[100258]: audit 2023-08-28T02:07:11.978971+0000 mon.a (mon.0) 239 : 2023-08-28T02:07:12.416 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:12 smithi143 conmon[100258]: audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:07:12.416 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:12 smithi143 conmon[109923]: cluster 2023-08-28T02:07: 2023-08-28T02:07:12.416 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:12 smithi143 conmon[109923]: 10.925354+0000 mgr.y (mgr.14146) 56 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:07:12.416 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:12 smithi143 conmon[109923]: audit 2023-08 2023-08-28T02:07:12.416 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:12 smithi143 conmon[109923]: -28T02:07:11.942513 2023-08-28T02:07:12.416 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:12 smithi143 conmon[109923]: +0000 mon.a (mon.0) 238 2023-08-28T02:07:12.417 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:12 smithi143 conmon[109923]: : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:12.417 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:12 smithi143 conmon[109923]: audit 2023-08-28T02:07:12.417 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:12 smithi143 conmon[109923]: 2023-08-28T02:07: 2023-08-28T02:07:12.417 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:12 smithi143 conmon[109923]: 11.978971+0000 mon.a (mon 2023-08-28T02:07:12.417 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:12 smithi143 conmon[109923]: .0) 239 : audit [DBG] 2023-08-28T02:07:12.417 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:12 smithi143 conmon[109923]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:07:12.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:12 smithi169 conmon[100500]: cluster 2023-08-28T02:07:10.925354+0000 mgr.y (mgr.14146) 56 : 2023-08-28T02:07:12.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:12 smithi169 conmon[100500]: cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:07:12.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:12 smithi169 conmon[100500]: audit 2023-08-28T02:07:11.942513+0000 mon.a (mon.0) 238 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:12.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:12 smithi169 conmon[100500]: audit 2023-08-28T02:07:11.978971+0000 mon.a (mon.0) 239 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:07:12.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:12 smithi169 conmon[100500]: 2023-08-28T02:07:14.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:14 smithi143 conmon[100258]: cluster 2023-08-28T02:07:12.925587+0000 mgr.y (mgr.14146) 57 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:07:14.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:14 smithi143 conmon[100258]: audit 2023-08-28T02:07:13.721875+0000 mon.a (mon.0) 240 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:14.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:14 smithi143 conmon[100258]: audit 2023-08-28T02:07:13.722541+0000 mon.a (mon.0) 241 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:07:14.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:14 smithi143 conmon[100258]: audit 2023-08-28T02:07:13.723163+0000 mon.a (mon.0) 242 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:07:14.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:14 smithi143 conmon[100258]: audit 2023-08-28T02:07:13.728183+0000 mon.a (mon.0) 243 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:14.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:14 smithi143 conmon[109923]: cluster 2023-08-28T02:07:12.925587+0000 mgr.y (mgr.14146) 57 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:07:14.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:14 smithi143 conmon[109923]: audit 2023-08-28T02:07:13.721875+0000 mon.a (mon.0) 240 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:14.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:14 smithi143 conmon[109923]: audit 2023-08-28T02:07:13.722541+0000 mon.a (mon.0) 241 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:07:14.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:14 smithi143 conmon[109923]: audit 2023-08-28T02:07:13.723163+0000 mon.a (mon.0) 242 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:07:14.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:14 smithi143 conmon[109923]: audit 2023-08-28T02:07:13.728183+0000 mon.a (mon.0) 243 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:14.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:14 smithi169 conmon[100500]: cluster 2023-08-28T02:07:12.925587+0000 mgr.y (mgr.14146) 57 : cluster 2023-08-28T02:07:14.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:14 smithi169 conmon[100500]: [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:07:14.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:14 smithi169 conmon[100500]: audit 2023-08-28T02:07:13.721875+0000 mon.a (mon.0) 240 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:14.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:14 smithi169 conmon[100500]: audit 2023-08-28T02:07:13.722541+0000 mon.a (mon.0) 241 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:07:14.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:14 smithi169 conmon[100500]: 2023-08-28T02:07:14.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:14 smithi169 conmon[100500]: audit 2023-08-28T02:07: 2023-08-28T02:07:14.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:14 smithi169 conmon[100500]: 13.723163+0000 mon.a (mon.0) 242 : audit [INF] 2023-08-28T02:07:14.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:14 smithi169 conmon[100500]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:07:14.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:14 smithi169 conmon[100500]: audit 2023-08-28T02:07:13.728183+0000 2023-08-28T02:07:14.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:14 smithi169 conmon[100500]: mon.a (mon.0) 243 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:14.732 INFO:teuthology.orchestra.run.smithi143.stdout:Created osd(s) 0 on host 'smithi143' 2023-08-28T02:07:15.304 DEBUG:teuthology.orchestra.run.smithi143:osd.0> sudo journalctl -f -n 0 -u ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@osd.0.service 2023-08-28T02:07:15.306 INFO:tasks.cephadm:Deploying osd.1 on smithi143 with /dev/vg_nvme/lv_3... 2023-08-28T02:07:15.306 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-08-28T02:07:15.350 INFO:journalctl@ceph.osd.0.smithi143.stdout:-- Logs begin at Mon 2023-08-28 01:49:41 UTC. -- 2023-08-28T02:07:15.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:15 smithi143 conmon[100258]: audit 2023-08-28T02:07:14.733256+0000 mon.a (mon.0) 244 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:15.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:15 smithi143 conmon[100258]: audit 2023-08-28T02:07:14.745288+0000 mon.a (mon.0) 245 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:07:15.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:15 smithi143 conmon[100258]: cluster 2023-08-28T02:07:14.925838+0000 mgr.y (mgr.14146) 58 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:07:15.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:15 smithi143 conmon[100258]: audit 2023-08-28T02:07:15.132603+0000 mon.a (mon.0) 246 : audit [INF] from='osd.0 [v2:172.21.15.143:6802/1846477664,v1:172.21.15.143:6803/1846477664]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-08-28T02:07:15.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:15 smithi143 conmon[109923]: audit 2023-08-28T02:07:14.733256+0000 2023-08-28T02:07:15.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:15 smithi143 conmon[109923]: mon.a (mon.0) 244 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:15.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:15 smithi143 conmon[109923]: audit 2023-08-28T02:07:14.745288+0000 mon.a (mon.0) 245 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:07:15.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:15 smithi143 conmon[109923]: cluster 2023-08-28T02:07:14.925838+0000 mgr.y (mgr.14146) 58 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:07:15.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:15 smithi143 conmon[109923]: audit 2023-08-28T02:07:15.132603+0000 mon.a (mon.0) 246 : audit [INF] from='osd.0 [v2:172.21.15.143:6802/1846477664,v1:172.21.15.143:6803/1846477664]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-08-28T02:07:15.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:15 smithi169 conmon[100500]: audit 2023-08-28T02:07:14.733256+0000 mon.a (mon.0) 244 : audit [INF] 2023-08-28T02:07:15.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:15 smithi169 conmon[100500]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:15.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:15 smithi169 conmon[100500]: audit 2023-08-28T02:07:14.745288+0000 mon.a (mon.0) 245 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:07:15.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:15 smithi169 conmon[100500]: cluster 2023-08-28T02:07:14.925838+0000 mgr.y (mgr.14146) 58 : cluster 2023-08-28T02:07:15.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:15 smithi169 conmon[100500]: [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:07:15.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:15 smithi169 conmon[100500]: audit 2023-08-28T02:07:15 2023-08-28T02:07:15.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:15 smithi169 conmon[100500]: .132603+0000 mon.a (mon.0) 246 : audit [INF] 2023-08-28T02:07:15.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:15 smithi169 conmon[100500]: from='osd.0 [v2:172.21.15.143:6802/1846477664,v1:172.21.15.143:6803/1846477664]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-08-28T02:07:17.426 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:17 smithi143 conmon[100258]: audit 2023-08-28T02:07:16.112758+0000 mon.a (mon.0) 247 : audit [INF] from='osd.0 [v2:172.21.15.143:6802/1846477664,v1:172.21.15.143:6803/1846477664]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-08-28T02:07:17.426 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:17 smithi143 conmon[100258]: cluster 2023-08-28T02:07:16.112825+0000 mon.a (mon.0) 248 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-08-28T02:07:17.426 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:17 smithi143 conmon[100258]: audit 2023-08-28T02:07:16.112947+0000 mon.a (mon.0) 249 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-28T02:07:17.426 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:17 smithi143 conmon[100258]: audit 2023-08-28T02:07:16.113321+0000 mon.a (mon.0) 250 : audit [INF] from='osd.0 [v2:172.21.15.143:6802/1846477664,v1:172.21.15.143:6803/1846477664]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]: dispatch 2023-08-28T02:07:17.427 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:17 smithi143 conmon[109923]: audit 2023-08-28T02:07 2023-08-28T02:07:17.427 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:17 smithi143 conmon[109923]: :16.112758+0000 mon.a (mon.0) 247 : audit [INF] from='osd.0 [v2:172.21.15.143:6802/1846477664,v1:172.21.15.143:6803/1846477664]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-08-28T02:07:17.427 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:17 smithi143 conmon[109923]: cluster 2023- 2023-08-28T02:07:17.427 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:17 smithi143 conmon[109923]: 08-28T02:07:16.112825+0000 mon.a (mon.0) 248 2023-08-28T02:07:17.427 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:17 smithi143 conmon[109923]: : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-08-28T02:07:17.427 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:17 smithi143 conmon[109923]: audit 2023-08-28T02:07:17.427 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:17 smithi143 conmon[109923]: 2023-08-28T 2023-08-28T02:07:17.427 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:17 smithi143 conmon[109923]: 02:07:16.112947+0000 mon.a (mon 2023-08-28T02:07:17.427 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:17 smithi143 conmon[109923]: .0) 249 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-28T02:07:17.428 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:17 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:07:17.428 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:17 smithi143 conmon[109923]: 08-28T02:07:16.113321+0000 2023-08-28T02:07:17.428 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:17 smithi143 conmon[109923]: mon.a (mon.0) 250 : audit [INF] 2023-08-28T02:07:17.428 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:17 smithi143 conmon[109923]: from='osd.0 [v2:172.21.15.143:6802/1846477664,v1:172.21.15.143:6803/1846477664]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]: dispatch 2023-08-28T02:07:17.428 INFO:journalctl@ceph.osd.0.smithi143.stdout:Aug 28 02:07:17 smithi143 conmon[115063]: debug 2023-08-28T02:07:17.116+0000 7f21c8811700 -1 osd.0 0 waiting for initial osdmap 2023-08-28T02:07:17.428 INFO:journalctl@ceph.osd.0.smithi143.stdout:Aug 28 02:07:17 smithi143 conmon[115063]: debug 2023-08-28T02:07:17.122+0000 7f21c2f89700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-08-28T02:07:17.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:17 smithi169 conmon[100500]: audit 2023-08-28T02:07:16.112758+0000 mon.a (mon.0) 247 : 2023-08-28T02:07:17.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:17 smithi169 conmon[100500]: audit [INF] from='osd.0 [v2:172.21.15.143:6802/1846477664,v1:172.21.15.143:6803/1846477664]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-08-28T02:07:17.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:17 smithi169 conmon[100500]: cluster 2023-08-28T02:07:16.112825+0000 mon.a (mon.0) 248 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-08-28T02:07:17.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:17 smithi169 conmon[100500]: audit 2023-08-28T02:07:16.112947+0000 mon.a (mon.0) 249 : audit [DBG] 2023-08-28T02:07:17.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:17 smithi169 conmon[100500]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-28T02:07:17.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:17 smithi169 conmon[100500]: audit 2023-08-28T02:07:17.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:17 smithi169 conmon[100500]: 2023-08-28T02:07:16.113321+0000 mon.a (mon. 2023-08-28T02:07:17.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:17 smithi169 conmon[100500]: 0) 250 : audit [INF] from='osd.0 [v2:172.21.15.143:6802/1846477664,v1:172.21.15.143:6803/1846477664]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]: dispatch 2023-08-28T02:07:17.921 INFO:teuthology.orchestra.run.smithi143.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-08-28T02:07:17.921 INFO:teuthology.orchestra.run.smithi143.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-08-28T02:07:17.921 INFO:teuthology.orchestra.run.smithi143.stderr: stderr: 10+0 records in 2023-08-28T02:07:17.921 INFO:teuthology.orchestra.run.smithi143.stderr:10+0 records out 2023-08-28T02:07:17.921 INFO:teuthology.orchestra.run.smithi143.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0135609 s, 773 MB/s 2023-08-28T02:07:17.922 INFO:teuthology.orchestra.run.smithi143.stderr:--> Zapping successful for: 2023-08-28T02:07:18.415 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:18 smithi143 conmon[100258]: cluster 2023-08-28T02:07:16.926203+0000 mgr.y (mgr.14146) 59 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:07:18.415 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:18 smithi143 conmon[100258]: audit 2023-08-28T02:07:17.111122+0000 mon.a (mon.0) 251 : audit [INF] from='osd.0 [v2:172.21.15.143:6802/1846477664,v1:172.21.15.143:6803/1846477664]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]': finished 2023-08-28T02:07:18.415 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:18 smithi143 conmon[100258]: cluster 2023-08-28T02:07:17.111230+0000 mon.a (mon.0) 252 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-08-28T02:07:18.415 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:18 smithi143 conmon[100258]: audit 2023-08-28T02:07:17.113915+0000 mon.a (mon.0) 253 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-28T02:07:18.415 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:18 smithi143 conmon[100258]: audit 2023-08-28T02:07:17.117968+0000 mon.a (mon.0) 254 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-28T02:07:18.415 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:18 smithi143 conmon[109923]: cluster 2023-08-28T02:07:16.926203+0000 mgr.y (mgr.14146) 59 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:07:18.416 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:18 smithi143 conmon[109923]: audit 2023-08-28T02:07:17.111122+0000 mon.a (mon.0) 251 : audit [INF] from='osd.0 [v2:172.21.15.143:6802/1846477664,v1:172.21.15.143:6803/1846477664]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]': finished 2023-08-28T02:07:18.416 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:18 smithi143 conmon[109923]: cluster 2023-08-28T02:07:17.111230+0000 mon.a (mon.0) 252 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-08-28T02:07:18.416 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:18 smithi143 conmon[109923]: audit 2023-08-28T02:07:17.113915+0000 mon.a (mon.0) 253 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-28T02:07:18.416 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:18 smithi143 conmon[109923]: audit 2023-08-28T02:07:17.117968+0000 mon.a (mon.0) 254 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-28T02:07:18.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:18 smithi169 conmon[100500]: cluster 2023-08-28T02:07:16.926203+0000 mgr.y (mgr.14146) 59 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:07:18.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:18 smithi169 conmon[100500]: audit 2023-08-28T02:07:17.111122+0000 mon.a (mon.0) 251 : audit [INF] from='osd.0 [v2:172.21.15.143:6802/1846477664,v1:172.21.15.143:6803/1846477664]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]': finished 2023-08-28T02:07:18.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:18 smithi169 conmon[100500]: cluster 2023-08-28T02:07:17. 2023-08-28T02:07:18.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:18 smithi169 conmon[100500]: 111230+0000 mon.a (mon.0) 252 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-08-28T02:07:18.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:18 smithi169 conmon[100500]: audit 2023-08-28T02:07:17.113915 2023-08-28T02:07:18.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:18 smithi169 conmon[100500]: +0000 mon.a (mon.0) 253 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-28T02:07:18.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:18 smithi169 conmon[100500]: 2023-08-28T02:07:18.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:18 smithi169 conmon[100500]: audit 2023-08-28T02:07: 2023-08-28T02:07:18.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:18 smithi169 conmon[100500]: 17.117968+0000 mon.a (mon.0) 254 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-28T02:07:18.851 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph orch daemon add osd smithi143:vg_nvme/lv_3 2023-08-28T02:07:19.384 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:19 smithi143 conmon[100258]: cluster 2023-08-28T02:07:16.166729+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-08-28T02:07:19.384 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:19 smithi143 conmon[100258]: cluster 2023-08-28T02:07:16.166765+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-08-28T02:07:19.384 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:19 smithi143 conmon[100258]: audit 2023-08-28T02:07:18.120237+0000 mon.a (mon.0) 255 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-28T02:07:19.384 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:19 smithi143 conmon[100258]: cluster 2023-08-28T02:07:18.125455+0000 mon.a (mon.0) 256 : cluster [INF] osd.0 [v2:172.21.15.143:6802/1846477664,v1:172.21.15.143:6803/1846477664] boot 2023-08-28T02:07:19.385 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:19 smithi143 conmon[100258]: cluster 2023-08-28T02:07:18.125494+0000 mon.a (mon.0) 257 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-08-28T02:07:19.385 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:19 smithi143 conmon[100258]: audit 2023-08-28T02:07:18.125621+0000 mon.a (mon.0) 258 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-28T02:07:19.385 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:19 smithi143 conmon[100258]: audit 2023-08-28T02:07:18.895845+0000 mon.a (mon.0) 259 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:19.385 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:19 smithi143 conmon[109923]: cluster 2023-08-28T02:07:16.166729+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-08-28T02:07:19.385 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:19 smithi143 conmon[109923]: cluster 2023-08-28T02:07:16.166765+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-08-28T02:07:19.386 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:19 smithi143 conmon[109923]: audit 2023-08-28T02:07:18.120237+0000 mon.a (mon.0) 255 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-28T02:07:19.386 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:19 smithi143 conmon[109923]: cluster 2023-08-28T02:07:18.125455+0000 mon.a (mon.0) 256 : cluster [INF] osd.0 [v2:172.21.15.143:6802/1846477664,v1:172.21.15.143:6803/1846477664] boot 2023-08-28T02:07:19.386 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:19 smithi143 conmon[109923]: cluster 2023-08-28T02:07:18.125494+0000 mon.a (mon.0) 257 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-08-28T02:07:19.386 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:19 smithi143 conmon[109923]: audit 2023-08-28T02:07:18.125621+0000 mon.a (mon.0) 258 : audit 2023-08-28T02:07:19.386 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:19 smithi143 conmon[109923]: [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-28T02:07:19.386 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:19 smithi143 conmon[109923]: audit 2023-08-28T02:07:18.895845+0000 mon.a (mon.0) 259 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:19.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:19 smithi169 conmon[100500]: cluster 2023-08-28T02:07:16.166729+0000 osd.0 (osd. 2023-08-28T02:07:19.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:19 smithi169 conmon[100500]: 0) 1 : cluster [DBG] purged_snaps scrub starts 2023-08-28T02:07:19.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:19 smithi169 conmon[100500]: cluster 2023-08-28T02:07:16.166765+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-08-28T02:07:19.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:19 smithi169 conmon[100500]: audit 2023-08-28T02:07:18.120237+0000 2023-08-28T02:07:19.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:19 smithi169 conmon[100500]: mon.a (mon.0) 255 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-28T02:07:19.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:19 smithi169 conmon[100500]: cluster 2023-08- 2023-08-28T02:07:19.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:19 smithi169 conmon[100500]: 28T02:07:18.125455+0000 mon.a (mon.0 2023-08-28T02:07:19.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:19 smithi169 conmon[100500]: ) 256 : cluster [INF] osd.0 [v2:172.21.15.143:6802/1846477664,v1:172.21.15.143:6803/1846477664] boot 2023-08-28T02:07:19.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:19 smithi169 conmon[100500]: cluster 2023-08-28T02 2023-08-28T02:07:19.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:19 smithi169 conmon[100500]: :07:18.125494+0000 mon.a (mon.0) 257 2023-08-28T02:07:19.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:19 smithi169 conmon[100500]: : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-08-28T02:07:19.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:19 smithi169 conmon[100500]: audit 2023-08-28T02:07: 2023-08-28T02:07:19.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:19 smithi169 conmon[100500]: 18.125621+0000 mon.a (mon.0) 258 : audit [DBG] 2023-08-28T02:07:19.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:19 smithi169 conmon[100500]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-28T02:07:19.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:19 smithi169 conmon[100500]: audit 2023-08-28T02:07:18.895845 2023-08-28T02:07:19.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:19 smithi169 conmon[100500]: +0000 mon.a (mon.0) 259 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:19.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:19 smithi169 conmon[100500]: 2023-08-28T02:07:20.322 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:20 smithi143 conmon[109923]: cephadm 2023-08-28T02:07:18.891481+0000 mgr.y (mgr.14146) 60 : cephadm [INF] Detected new or changed devices on smithi143 2023-08-28T02:07:20.322 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:20 smithi143 conmon[109923]: cluster 2023-08-28T02:07:18.926472+0000 mgr.y (mgr.14146) 61 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:07:20.323 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:20 smithi143 conmon[109923]: cluster 2023-08-28T02:07:19.127480+0000 mon.a (mon.0) 260 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-08-28T02:07:20.323 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:20 smithi143 conmon[109923]: audit 2023-08-28T02:07:19.228698+0000 mon.a (mon.0) 261 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:07:20.323 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:20 smithi143 conmon[109923]: cephadm 2023-08-28T02:07:19.229252+0000 mgr.y (mgr.14146) 62 : cephadm [INF] Adjusting osd_memory_target on smithi143 to 16144M 2023-08-28T02:07:20.323 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:20 smithi143 conmon[109923]: audit 2023-08-28T02:07:19.233310+0000 mon.a (mon.0) 262 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:20.323 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:20 smithi143 conmon[109923]: audit 2023-08-28T02:07:19.234160+0000 mon.a (mon.0) 263 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:07:20.323 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:20 smithi143 conmon[109923]: audit 2023-08-28T02:07:19.234741+0000 mon.a (mon.0) 264 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:07:20.323 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:20 smithi143 conmon[109923]: audit 2023-08-28T02:07:19.238641+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:20.323 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:20 smithi143 conmon[100258]: cephadm 2023-08-28T02:07:18.891481+0000 mgr.y (mgr.14146) 60 : cephadm [INF] Detected new or changed devices on smithi143 2023-08-28T02:07:20.323 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:20 smithi143 conmon[100258]: cluster 2023-08-28T02:07:18.926472+0000 mgr.y (mgr.14146) 61 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:07:20.324 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:20 smithi143 conmon[100258]: cluster 2023-08-28T02:07:19.127480+0000 mon.a (mon.0) 260 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-08-28T02:07:20.324 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:20 smithi143 conmon[100258]: audit 2023-08-28T02:07:19.228698+0000 mon.a (mon.0) 261 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:07:20.324 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:20 smithi143 conmon[100258]: cephadm 2023-08-28T02:07:19.229252+0000 mgr.y (mgr.14146) 62 : cephadm [INF] Adjusting osd_memory_target on smithi143 to 16144M 2023-08-28T02:07:20.324 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:20 smithi143 conmon[100258]: audit 2023-08-28T02:07:19.233310+0000 mon.a (mon.0) 262 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:20.324 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:20 smithi143 conmon[100258]: audit 2023-08-28T02:07:19.234160+0000 mon.a (mon.0) 263 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:07:20.324 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:20 smithi143 conmon[100258]: audit 2023-08-28T02:07:19.234741+0000 mon.a (mon.0) 264 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:07:20.324 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:20 smithi143 conmon[100258]: audit 2023-08-28T02:07:19.238641+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:20.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:20 smithi169 conmon[100500]: cephadm 2023-08-28T02:07:18.891481+0000 mgr.y (mgr.14146) 60 : cephadm [INF] Detected new or changed devices on smithi143 2023-08-28T02:07:20.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:20 smithi169 conmon[100500]: cluster 2023-08-28T02:07:18.926472+0000 mgr.y (mgr.14146) 61 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-28T02:07:20.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:20 smithi169 conmon[100500]: cluster 2023-08-28T02:07:19.127480 2023-08-28T02:07:20.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:20 smithi169 conmon[100500]: +0000 mon.a (mon.0) 260 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-08-28T02:07:20.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:20 smithi169 conmon[100500]: audit 2023-08-28T02:07:19. 2023-08-28T02:07:20.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:20 smithi169 conmon[100500]: 228698+0000 mon.a (mon.0) 261 : audit [INF] 2023-08-28T02:07:20.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:20 smithi169 conmon[100500]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:07:20.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:20 smithi169 conmon[100500]: cephadm 2023-08-28T02:07:19.229252+0000 mgr.y (mgr.14146) 62 : 2023-08-28T02:07:20.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:20 smithi169 conmon[100500]: cephadm [INF] Adjusting osd_memory_target on smithi143 to 16144M 2023-08-28T02:07:20.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:20 smithi169 conmon[100500]: audit 2023-08-28T02:07 2023-08-28T02:07:20.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:20 smithi169 conmon[100500]: :19.233310+0000 mon.a (mon.0) 262 : 2023-08-28T02:07:20.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:20 smithi169 conmon[100500]: audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:20.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:20 smithi169 conmon[100500]: audit 2023-08-28T02: 2023-08-28T02:07:20.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:20 smithi169 conmon[100500]: 07:19.234160+0000 mon.a (mon.0) 263 : 2023-08-28T02:07:20.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:20 smithi169 conmon[100500]: audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:07:20.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:20 smithi169 conmon[100500]: audit 2023-08-28T02:07 2023-08-28T02:07:20.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:20 smithi169 conmon[100500]: :19.234741+0000 mon.a (mon.0) 264 : audit 2023-08-28T02:07:20.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:20 smithi169 conmon[100500]: [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:07:20.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:20 smithi169 conmon[100500]: audit 2023-08-28T02:07:19 2023-08-28T02:07:20.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:20 smithi169 conmon[100500]: .238641+0000 mon.a (mon.0) 265 : audit [INF] 2023-08-28T02:07:20.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:20 smithi169 conmon[100500]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:21.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:21 smithi143 conmon[109923]: audit 2023-08-28T02:07:20.676733+0000 mon.a (mon 2023-08-28T02:07:21.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:21 smithi143 conmon[109923]: .0) 266 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-28T02:07:21.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:21 smithi143 conmon[109923]: audit 2023-08-28T02:07:20.678530+0000 mon.a (mon.0) 267 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-28T02:07:21.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:21 smithi143 conmon[109923]: audit 2023-08-28T02:07:20.679088+0000 mon.a (mon.0) 268 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:07:21.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:21 smithi143 conmon[100258]: audit 2023-08-28T02:07:20.676733+0000 mon.a (mon.0) 266 2023-08-28T02:07:21.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:21 smithi143 conmon[100258]: : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-28T02:07:21.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:21 smithi143 conmon[100258]: audit 2023-08-28T02:07:20.678530+0000 mon.a (mon.0) 267 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-28T02:07:21.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:21 smithi143 conmon[100258]: audit 2023-08-28T02:07:20.679088+0000 mon.a (mon.0) 268 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:07:21.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:21 smithi169 conmon[100500]: audit 2023-08-28T02:07:20.676733+0000 mon.a (mon.0) 266 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-28T02:07:21.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:21 smithi169 conmon[100500]: audit 2023-08-28T02:07:20.678530+0000 mon.a (mon.0) 267 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-28T02:07:21.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:21 smithi169 conmon[100500]: audit 2023-08-28T02: 2023-08-28T02:07:21.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:21 smithi169 conmon[100500]: 07:20.679088+0000 mon.a (mon.0) 268 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:07:22.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:22 smithi143 conmon[100258]: audit 2023-08-28T02:07:20.674775+0000 mgr.y (mgr.14146) 63 : audit [DBG] from='client.14235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi143:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:07:22.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:22 smithi143 conmon[100258]: cluster 2023-08-28T02:07:20.926722+0000 mgr.y (mgr.14146) 64 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-28T02:07:22.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:22 smithi143 conmon[109923]: audit 2023-08-28T02:07:20.674775+0000 mgr.y (mgr.14146) 63 : audit [DBG] from='client.14235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi143:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:07:22.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:22 smithi143 conmon[109923]: cluster 2023-08-28T02:07:20.926722+0000 mgr.y (mgr.14146) 64 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-28T02:07:22.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:22 smithi169 conmon[100500]: audit 2023-08-28T02:07:20.674775+0000 mgr.y (mgr.14146) 63 : audit [DBG] from='client.14235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi143:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:07:22.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:22 smithi169 conmon[100500]: cluster 2023-08-28T02:07:20.926722+0000 mgr.y (mgr.14146) 64 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-28T02:07:23.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:23 smithi143 conmon[100258]: audit 2023-08-28T02:07:22.857108+0000 mon.a (mon 2023-08-28T02:07:23.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:23 smithi143 conmon[100258]: .0) 269 : audit [INF] from='client.? 172.21.15.143:0/1983805917' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cf24771f-3b0c-4a00-8a73-13e1fee242dc"}]: dispatch 2023-08-28T02:07:23.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:23 smithi143 conmon[100258]: audit 2023-08-28T02:07:22.861885+0000 mon.a (mon.0) 270 : audit [INF] from='client.? 172.21.15.143:0/1983805917' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cf24771f-3b0c-4a00-8a73-13e1fee242dc"}]': finished 2023-08-28T02:07:23.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:23 smithi143 conmon[100258]: cluster 2023-08-28T02:07:22.861928+0000 mon.a (mon.0) 271 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2023-08-28T02:07:23.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:23 smithi143 conmon[100258]: audit 2023-08-28T02:07:22.862089+0000 mon.a (mon.0) 272 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-28T02:07:23.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:23 smithi143 conmon[109923]: audit 2023-08-28T02:07 2023-08-28T02:07:23.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:23 smithi143 conmon[109923]: :22.857108+0000 mon.a (mon.0) 269 : audit [INF] from='client.? 172.21.15.143:0/1983805917' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cf24771f-3b0c-4a00-8a73-13e1fee242dc"}]: dispatch 2023-08-28T02:07:23.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:23 smithi143 conmon[109923]: audit 2023-08-28T02:07:23.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:23 smithi143 conmon[109923]: 2023-08-28T02:07:22.861885+0000 mon.a 2023-08-28T02:07:23.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:23 smithi143 conmon[109923]: (mon.0) 270 : audit [INF] 2023-08-28T02:07:23.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:23 smithi143 conmon[109923]: from='client.? 172.21.15.143:0/1983805917' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cf24771f-3b0c-4a00-8a73-13e1fee242dc"}]': finished 2023-08-28T02:07:23.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:23 smithi143 conmon[109923]: cluster 2023-08-28T02:07:23.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:23 smithi143 conmon[109923]: 2023-08-28T02:07 2023-08-28T02:07:23.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:23 smithi143 conmon[109923]: :22.861928+0000 mon.a (mon. 2023-08-28T02:07:23.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:23 smithi143 conmon[109923]: 0) 271 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2023-08-28T02:07:23.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:23 smithi143 conmon[109923]: 2023-08-28T02:07:23.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:23 smithi143 conmon[109923]: audit 2023-08-28T02:07:23.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:23 smithi143 conmon[109923]: 2023-08-28T02:07:22 2023-08-28T02:07:23.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:23 smithi143 conmon[109923]: .862089+0000 mon.a (mon.0 2023-08-28T02:07:23.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:23 smithi143 conmon[109923]: ) 272 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-28T02:07:23.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:23 smithi169 conmon[100500]: audit 2023-08-28T02:07:22.857108+0000 mon.a (mon.0) 269 : audit [INF] from='client.? 172.21.15.143:0/1983805917' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cf24771f-3b0c-4a00-8a73-13e1fee242dc"}]: dispatch 2023-08-28T02:07:23.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:23 smithi169 conmon[100500]: audit 2023-08-28T02:07:22.861885+0000 mon.a (mon.0) 270 : audit [INF] from='client.? 172.21.15.143:0/1983805917' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cf24771f-3b0c-4a00-8a73-13e1fee242dc"}]': finished 2023-08-28T02:07:23.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:23 smithi169 conmon[100500]: cluster 2023-08 2023-08-28T02:07:23.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:23 smithi169 conmon[100500]: -28T02:07:22.861928+0000 mon.a (mon.0) 271 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2023-08-28T02:07:23.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:23 smithi169 conmon[100500]: audit 2023-08 2023-08-28T02:07:23.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:23 smithi169 conmon[100500]: -28T02:07:22.862089+0000 mon.a ( 2023-08-28T02:07:23.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:23 smithi169 conmon[100500]: mon.0) 272 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-28T02:07:24.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:24 smithi143 conmon[100258]: cluster 2023-08-28T02:07:22.927014+0000 mgr.y (mgr.14146 2023-08-28T02:07:24.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:24 smithi143 conmon[100258]: ) 65 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-28T02:07:24.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:24 smithi143 conmon[100258]: audit 2023-08-28T02:07:23.437558+0000 mon.c (mon.2) 4 : audit [DBG] from='client.? 172.21.15.143:0/1387824476' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-28T02:07:24.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:24 smithi143 conmon[109923]: cluster 2023-08-28T02:07:22.927014+0000 mgr.y (mgr.14146) 65 2023-08-28T02:07:24.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:24 smithi143 conmon[109923]: : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-28T02:07:24.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:24 smithi143 conmon[109923]: audit 2023-08-28T02:07:23.437558+0000 mon.c (mon.2) 4 : audit [DBG] from='client.? 172.21.15.143:0/1387824476' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-28T02:07:24.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:24 smithi169 conmon[100500]: cluster 2023-08-28T02:07:22.927014+0000 mgr.y (mgr.14146) 65 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-28T02:07:24.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:24 smithi169 conmon[100500]: audit 2023-08-28T02:07 2023-08-28T02:07:24.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:24 smithi169 conmon[100500]: :23.437558+0000 mon.c (mon.2) 4 : audit [DBG] from='client.? 172.21.15.143:0/1387824476' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-28T02:07:26.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:26 smithi143 conmon[100258]: cluster 2023-08-28T02:07:24.927270+0000 mgr.y (mgr.14146) 66 2023-08-28T02:07:26.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:26 smithi143 conmon[100258]: : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-28T02:07:26.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:26 smithi143 conmon[109923]: cluster 2023-08-28T02:07:24 2023-08-28T02:07:26.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:26 smithi143 conmon[109923]: .927270+0000 mgr.y (mgr.14146) 66 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-28T02:07:26.486 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:26 smithi169 conmon[100500]: cluster 2023-08-28T02:07:24.927270+0000 mgr.y (mgr.14146) 66 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-28T02:07:28.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:28 smithi143 conmon[100258]: cluster 2023-08-28T02:07:26.927674+0000 mgr.y (mgr 2023-08-28T02:07:28.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:28 smithi143 conmon[100258]: .14146) 67 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-28T02:07:28.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:28 smithi143 conmon[109923]: cluster 2023-08-28T02:07:26.927674+0000 2023-08-28T02:07:28.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:28 smithi143 conmon[109923]: mgr.y (mgr.14146) 67 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-28T02:07:28.486 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:28 smithi169 conmon[100500]: cluster 2023-08-28T02:07:26.927674+0000 mgr.y (mgr.14146) 67 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-28T02:07:29.449 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:29 smithi143 conmon[100258]: audit 2023-08-28T02:07:28.787343+0000 mon.a (mon.0) 273 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-08-28T02:07:29.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:29 smithi143 conmon[100258]: audit 2023-08-28T02:07:28.787964+0000 mon.a (mon.0) 274 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:07:29.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:29 smithi143 conmon[109923]: audit 2023-08-28T02:07:28.787343+0000 mon.a (mon.0) 273 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-08-28T02:07:29.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:29 smithi143 conmon[109923]: audit 2023-08-28T02:07:28.787964+0000 mon.a (mon.0) 274 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:07:29.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:29 smithi169 conmon[100500]: audit 2023-08-28T02:07:28.787343+0000 mon.a (mon.0) 273 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-08-28T02:07:29.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:29 smithi169 conmon[100500]: audit 2023-08-28T02:07:28.787964+0000 mon.a (mon.0) 274 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:07:30.262 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:30 smithi143 conmon[100258]: cephadm 2023-08-28T02:07:28.788436+0000 mgr.y (mgr. 2023-08-28T02:07:30.262 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:30 smithi143 conmon[100258]: 14146) 68 : cephadm [INF] Deploying daemon osd.1 on smithi143 2023-08-28T02:07:30.262 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:30 smithi143 conmon[100258]: cluster 2023-08-28T02:07:28.928004+0000 mgr.y (mgr.14146) 69 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-28T02:07:30.262 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:30 smithi143 conmon[109923]: cephadm 2023-08-28T02:07:28.788436+0000 mgr.y (mgr. 2023-08-28T02:07:30.262 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:30 smithi143 conmon[109923]: 14146) 68 : cephadm [INF] Deploying daemon osd.1 on smithi143 2023-08-28T02:07:30.263 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:30 smithi143 conmon[109923]: cluster 2023-08-28T02:07:28.928004+0000 mgr.y (mgr.14146) 69 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-28T02:07:30.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:30 smithi169 conmon[100500]: cephadm 2023-08-28T02:07:28.788436+0000 mgr.y (mgr.14146) 68 : cephadm [INF] Deploying daemon osd.1 on smithi143 2023-08-28T02:07:30.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:30 smithi169 conmon[100500]: cluster 2023-08-28T02:07:28.928004+0000 mgr.y (mgr.14146) 69 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-28T02:07:32.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:32 smithi143 conmon[100258]: cluster 2023-08-28T02:07:30.928292+0000 mgr.y (mgr.14146) 70 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-28T02:07:32.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:32 smithi143 conmon[109923]: cluster 2023-08-28T02:07:30.928292+0000 mgr.y (mgr.14146) 70 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-28T02:07:32.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:32 smithi169 conmon[100500]: cluster 2023-08-28T02:07:30.928292+0000 mgr.y (mgr.14146) 70 2023-08-28T02:07:32.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:32 smithi169 conmon[100500]: : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-28T02:07:34.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:34 smithi143 conmon[100258]: cluster 2023-08-28T02:07:32 2023-08-28T02:07:34.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:34 smithi143 conmon[100258]: .928609+0000 mgr.y (mgr.14146) 71 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-28T02:07:34.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:34 smithi143 conmon[100258]: audit 2023-08-28T02:07:33.414472+0000 mon.a (mon.0) 275 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:34.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:34 smithi143 conmon[100258]: audit 2023-08-28T02:07:33.431425+0000 mon.a (mon.0) 276 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:07:34.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:34 smithi143 conmon[109923]: cluster 2023-08-28T02:07:32.928609+0000 mgr.y 2023-08-28T02:07:34.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:34 smithi143 conmon[109923]: (mgr.14146) 71 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-28T02:07:34.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:34 smithi143 conmon[109923]: audit 2023-08-28T02:07 2023-08-28T02:07:34.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:34 smithi143 conmon[109923]: :33.414472+0000 mon.a (mon.0) 275 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:34.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:34 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:07:34.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:34 smithi143 conmon[109923]: 08-28T02:07:33. 2023-08-28T02:07:34.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:34 smithi143 conmon[109923]: 431425+0000 mon.a (mon.0) 276 2023-08-28T02:07:34.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:34 smithi143 conmon[109923]: : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:07:34.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:34 smithi169 conmon[100500]: cluster 2023-08-28T02:07:32.928609+0000 mgr.y (mgr.14146) 71 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-28T02:07:34.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:34 smithi169 conmon[100500]: audit 2023-08-28T02:07:33.414472+0000 2023-08-28T02:07:34.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:34 smithi169 conmon[100500]: mon.a (mon.0) 275 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:34.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:34 smithi169 conmon[100500]: audit 2023-08-28T02:07:33.431425+0000 mon.a (mon.0) 276 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:07:36.389 INFO:teuthology.orchestra.run.smithi143.stdout:Created osd(s) 1 on host 'smithi143' 2023-08-28T02:07:36.652 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:36 smithi143 conmon[100258]: cluster 2023-08-28T02:07:34.928988+0000 mgr.y (mgr.14146) 72 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-28T02:07:36.652 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:36 smithi143 conmon[100258]: audit 2023-08-28T02:07:35.452725+0000 mon.a (mon.0) 277 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:36.652 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:36 smithi143 conmon[100258]: audit 2023-08-28T02:07:35.453528+0000 mon.a (mon.0) 278 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:07:36.652 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:36 smithi143 conmon[100258]: audit 2023-08-28T02:07:35.454103+0000 mon.a (mon.0) 279 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:07:36.653 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:36 smithi143 conmon[100258]: audit 2023-08-28T02:07:35.458953+0000 mon.a (mon.0) 280 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:36.653 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:36 smithi143 conmon[100258]: audit 2023-08-28T02:07:36.390533+0000 mon.a (mon.0) 281 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:36.653 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:36 smithi143 conmon[100258]: audit 2023-08-28T02:07:36.425853+0000 mon.a (mon.0) 282 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:07:36.653 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:36 smithi143 conmon[109923]: cluster 2023-08-28T02:07:34.928988+0000 mgr.y (mgr.14146) 72 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-28T02:07:36.653 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:36 smithi143 conmon[109923]: audit 2023-08-28T02:07:35.452725+0000 mon.a (mon.0) 277 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:36.653 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:36 smithi143 conmon[109923]: audit 2023-08-28T02:07:35.453528+0000 mon.a (mon.0) 278 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:07:36.653 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:36 smithi143 conmon[109923]: audit 2023-08-28T02:07:35.454103+0000 mon.a (mon.0) 279 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:07:36.653 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:36 smithi143 conmon[109923]: audit 2023-08-28T02:07:35.458953+0000 mon.a (mon.0) 280 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:36.653 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:36 smithi143 conmon[109923]: audit 2023-08-28T02:07:36.390533+0000 mon.a (mon.0) 281 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:36.653 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:36 smithi143 conmon[109923]: audit 2023-08-28T02:07:36.425853+0000 mon.a (mon.0) 282 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:07:36.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:36 smithi169 conmon[100500]: cluster 2023-08-28T02:07:34.928988+0000 mgr.y (mgr.14146) 72 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-28T02:07:36.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:36 smithi169 conmon[100500]: audit 2023-08-28T02:07:35. 2023-08-28T02:07:36.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:36 smithi169 conmon[100500]: 452725+0000 mon.a (mon.0) 277 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:36.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:36 smithi169 conmon[100500]: audit 2023-08-28T02:07:35.453528+0000 mon.a (mon.0) 278 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:07:36.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:36 smithi169 conmon[100500]: audit 2023-08-28T02:07:35.454103+0000 2023-08-28T02:07:36.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:36 smithi169 conmon[100500]: mon.a (mon.0) 279 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:07:36.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:36 smithi169 conmon[100500]: audit 2023-08 2023-08-28T02:07:36.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:36 smithi169 conmon[100500]: -28T02:07:35.458953+0000 mon.a (mon.0) 280 : audit [INF] 2023-08-28T02:07:36.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:36 smithi169 conmon[100500]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:36.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:36 smithi169 conmon[100500]: audit 2023-08-28T02:07:36.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:36 smithi169 conmon[100500]: 2023-08-28T02:07:36.390533+0000 mon.a (mon 2023-08-28T02:07:36.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:36 smithi169 conmon[100500]: .0) 281 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:36.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:36 smithi169 conmon[100500]: audit 2023-08-28T 2023-08-28T02:07:36.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:36 smithi169 conmon[100500]: 02:07:36.425853+0000 mon.a (mon.0 2023-08-28T02:07:36.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:36 smithi169 conmon[100500]: ) 282 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:07:36.968 DEBUG:teuthology.orchestra.run.smithi143:osd.1> sudo journalctl -f -n 0 -u ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@osd.1.service 2023-08-28T02:07:36.970 INFO:tasks.cephadm:Deploying osd.2 on smithi143 with /dev/vg_nvme/lv_2... 2023-08-28T02:07:36.970 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-08-28T02:07:36.999 INFO:journalctl@ceph.osd.1.smithi143.stdout:-- Logs begin at Mon 2023-08-28 01:49:41 UTC. -- 2023-08-28T02:07:37.570 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:37 smithi143 conmon[100258]: audit 2023-08-28T02:07:36. 2023-08-28T02:07:37.570 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:37 smithi143 conmon[100258]: 656623+0000 mon.a (mon.0) 283 : audit [INF] from='osd.1 [v2:172.21.15.143:6810/2341769896,v1:172.21.15.143:6811/2341769896]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-08-28T02:07:37.571 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:37 smithi143 conmon[109923]: audit 2023-08-28T02:07:36.656623+0000 2023-08-28T02:07:37.571 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:37 smithi143 conmon[109923]: mon.a (mon.0) 283 : audit [INF] from='osd.1 [v2:172.21.15.143:6810/2341769896,v1:172.21.15.143:6811/2341769896]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-08-28T02:07:37.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:37 smithi169 conmon[100500]: audit 2023-08-28T02:07:36.656623+0000 mon.a (mon.0) 283 : audit [INF] from='osd.1 [v2:172.21.15.143:6810/2341769896,v1:172.21.15.143:6811/2341769896]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-08-28T02:07:38.697 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:38 smithi143 conmon[100258]: cluster 2023-08-28T02:07:36. 2023-08-28T02:07:38.697 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:38 smithi143 conmon[100258]: 929236+0000 mgr.y (mgr.14146) 73 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-28T02:07:38.697 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:38 smithi143 conmon[100258]: audit 2023-08-28T02:07:37.458530+0000 mon.a (mon.0) 284 : audit [INF] from='osd.1 [v2:172.21.15.143:6810/2341769896,v1:172.21.15.143:6811/2341769896]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-08-28T02:07:38.698 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:38 smithi143 conmon[100258]: cluster 2023-08-28T02:07:37.458581+0000 mon.a (mon.0) 285 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-08-28T02:07:38.698 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:38 smithi143 conmon[100258]: audit 2023-08-28T02:07:37.458673+0000 mon.a (mon.0) 286 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-28T02:07:38.698 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:38 smithi143 conmon[100258]: audit 2023-08-28T02:07:37. 2023-08-28T02:07:38.698 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:38 smithi143 conmon[100258]: 459069+0000 mon.a (mon.0) 287 : audit [INF] from='osd.1 [v2:172.21.15.143:6810/2341769896,v1:172.21.15.143:6811/2341769896]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]: dispatch 2023-08-28T02:07:38.698 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:38 smithi143 conmon[109923]: cluster 2023-08-28T02:07:36.929236+0000 mgr.y (mgr.14146) 73 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-28T02:07:38.698 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:38 smithi143 conmon[109923]: audit 2023-08-28T02:07:37.458530+0000 mon.a (mon.0) 284 : audit [INF] from='osd.1 [v2:172.21.15.143:6810/2341769896,v1:172.21.15.143:6811/2341769896]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-08-28T02:07:38.698 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:38 smithi143 conmon[109923]: cluster 2023-08-28T02:07:37.458581+0000 mon.a (mon.0) 285 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-08-28T02:07:38.698 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:38 smithi143 conmon[109923]: audit 2023-08-28T02:07:37.458673+0000 mon.a (mon.0) 286 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-28T02:07:38.699 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:38 smithi143 conmon[109923]: audit 2023-08-28T02:07:37.459069+0000 mon.a (mon.0) 287 : audit [INF] from='osd.1 [v2:172.21.15.143:6810/2341769896,v1:172.21.15.143:6811/2341769896]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]: dispatch 2023-08-28T02:07:38.699 INFO:journalctl@ceph.osd.1.smithi143.stdout:Aug 28 02:07:38 smithi143 conmon[120884]: debug 2023-08-28T02:07:38.462+0000 7f6412206700 -1 osd.1 0 waiting for initial osdmap 2023-08-28T02:07:38.699 INFO:journalctl@ceph.osd.1.smithi143.stdout:Aug 28 02:07:38 smithi143 conmon[120884]: debug 2023-08-28T02:07:38.466+0000 7f640e982700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-08-28T02:07:38.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:38 smithi169 conmon[100500]: cluster 2023-08-28T02:07:36.929236+0000 mgr.y (mgr.14146) 73 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-28T02:07:38.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:38 smithi169 conmon[100500]: audit 2023-08-28T02:07:38.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:38 smithi169 conmon[100500]: 2023-08-28T02:07:37.458530+0000 mon.a (mon.0) 284 : audit [INF] from='osd.1 [v2:172.21.15.143:6810/2341769896,v1:172.21.15.143:6811/2341769896]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-08-28T02:07:38.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:38 smithi169 conmon[100500]: cluster 2023-08-28T02:07:37.458581+0000 mon.a (mon.0) 285 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-08-28T02:07:38.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:38 smithi169 conmon[100500]: audit 2023-08-28T02:07:37.458673 2023-08-28T02:07:38.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:38 smithi169 conmon[100500]: +0000 mon.a (mon.0) 286 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-28T02:07:38.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:38 smithi169 conmon[100500]: audit 2023- 2023-08-28T02:07:38.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:38 smithi169 conmon[100500]: 08-28T02:07:37.459069+0000 mon.a ( 2023-08-28T02:07:38.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:38 smithi169 conmon[100500]: mon.0) 287 : audit [INF] from='osd.1 [v2:172.21.15.143:6810/2341769896,v1:172.21.15.143:6811/2341769896]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]: dispatch 2023-08-28T02:07:39.518 INFO:teuthology.orchestra.run.smithi143.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-08-28T02:07:39.518 INFO:teuthology.orchestra.run.smithi143.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-08-28T02:07:39.518 INFO:teuthology.orchestra.run.smithi143.stderr: stderr: 10+0 records in 2023-08-28T02:07:39.518 INFO:teuthology.orchestra.run.smithi143.stderr:10+0 records out 2023-08-28T02:07:39.518 INFO:teuthology.orchestra.run.smithi143.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0108887 s, 963 MB/s 2023-08-28T02:07:39.518 INFO:teuthology.orchestra.run.smithi143.stderr:--> Zapping successful for: 2023-08-28T02:07:39.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:39 smithi143 conmon[100258]: audit 2023-08-28T02:07:38.460132+0000 mon.a (mon.0) 288 : audit [INF] from='osd.1 [v2:172.21.15.143:6810/2341769896,v1:172.21.15.143:6811/2341769896]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]': finished 2023-08-28T02:07:39.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:39 smithi143 conmon[100258]: cluster 2023-08-28T02:07:38.460199+0000 mon.a (mon.0) 289 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-08-28T02:07:39.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:39 smithi143 conmon[100258]: audit 2023-08-28T02:07:38.460556+0000 mon.a (mon.0) 290 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-28T02:07:39.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:39 smithi143 conmon[100258]: audit 2023-08-28T02:07:38.464906+0000 mon.a (mon.0) 291 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-28T02:07:39.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:39 smithi143 conmon[109923]: audit 2023-08-28T02:07:38.460132+0000 mon.a ( 2023-08-28T02:07:39.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:39 smithi143 conmon[109923]: mon.0) 288 : audit [INF] from='osd.1 [v2:172.21.15.143:6810/2341769896,v1:172.21.15.143:6811/2341769896]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]': finished 2023-08-28T02:07:39.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:39 smithi143 conmon[109923]: cluster 2023-08-28T02:07:38.460199+0000 mon.a (mon.0) 289 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-08-28T02:07:39.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:39 smithi143 conmon[109923]: audit 2023-08-28T02:07: 2023-08-28T02:07:39.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:39 smithi143 conmon[109923]: 38.460556+0000 mon.a (mon.0) 2023-08-28T02:07:39.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:39 smithi143 conmon[109923]: 290 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-28T02:07:39.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:39 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:07:39.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:39 smithi143 conmon[109923]: 08-28T02:07:38.464906 2023-08-28T02:07:39.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:39 smithi143 conmon[109923]: +0000 mon.a (mon.0) 2023-08-28T02:07:39.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:39 smithi143 conmon[109923]: 291 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-28T02:07:39.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:39 smithi169 conmon[100500]: audit 2023-08-28T02:07:38.460132+0000 mon.a (mon.0) 288 : audit [INF] from='osd.1 [v2:172.21.15.143:6810/2341769896,v1:172.21.15.143:6811/2341769896]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]': finished 2023-08-28T02:07:39.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:39 smithi169 conmon[100500]: cluster 2023-08- 2023-08-28T02:07:39.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:39 smithi169 conmon[100500]: 28T02:07:38.460199+0000 mon.a (mon.0) 289 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-08-28T02:07:39.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:39 smithi169 conmon[100500]: audit 2023-08-28T02:07:38.460556+0000 mon.a (mon.0) 290 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-28T02:07:39.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:39 smithi169 conmon[100500]: audit 2023-08-28T02:07:38.464906+0000 mon.a (mon 2023-08-28T02:07:39.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:39 smithi169 conmon[100500]: .0) 291 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-28T02:07:40.394 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph orch daemon add osd smithi143:vg_nvme/lv_2 2023-08-28T02:07:40.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:40 smithi143 conmon[109923]: cluster 2023-08-28T02:07: 2023-08-28T02:07:40.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:40 smithi143 conmon[109923]: 37.634799+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-08-28T02:07:40.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:40 smithi143 conmon[109923]: cluster 2023-08-28T02:07:37.634847+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-08-28T02:07:40.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:40 smithi143 conmon[109923]: cluster 2023-08-28T02:07:38.929461+0000 mgr.y (mgr.14146) 74 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-28T02:07:40.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:40 smithi143 conmon[109923]: cluster 2023-08-28T02:07:39.467047+0000 mon.a (mon.0) 292 : cluster [INF] osd.1 [v2:172.21.15.143:6810/2341769896,v1:172.21.15.143:6811/2341769896] boot 2023-08-28T02:07:40.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:40 smithi143 conmon[109923]: cluster 2023-08-28T02:07:39.467084+0000 mon.a (mon.0) 293 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-08-28T02:07:40.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:40 smithi143 conmon[109923]: audit 2023-08-28T02:07:39.470801+0000 mon.a (mon.0) 294 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-28T02:07:40.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:40 smithi143 conmon[100258]: cluster 2023-08-28T02:07:37.634799+0000 2023-08-28T02:07:40.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:40 smithi143 conmon[100258]: osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-08-28T02:07:40.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:40 smithi143 conmon[100258]: cluster 2023-08-28T02:07:37.634847+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-08-28T02:07:40.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:40 smithi143 conmon[100258]: cluster 2023-08-28T02:07:38.929461+0000 mgr.y (mgr.14146) 74 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-28T02:07:40.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:40 smithi143 conmon[100258]: cluster 2023-08-28T02:07:39.467047+0000 mon.a (mon.0) 292 : cluster [INF] osd.1 [v2:172.21.15.143:6810/2341769896,v1:172.21.15.143:6811/2341769896] boot 2023-08-28T02:07:40.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:40 smithi143 conmon[100258]: cluster 2023-08-28T02:07:39.467084+0000 mon.a (mon.0) 293 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-08-28T02:07:40.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:40 smithi143 conmon[100258]: audit 2023-08-28T02:07:39.470801+0000 mon.a (mon.0) 294 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-28T02:07:40.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:40 smithi169 conmon[100500]: cluster 2023-08-28T02:07:37.634799+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-08-28T02:07:40.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:40 smithi169 conmon[100500]: cluster 2023-08-28T02:07: 2023-08-28T02:07:40.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:40 smithi169 conmon[100500]: 37.634847+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-08-28T02:07:40.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:40 smithi169 conmon[100500]: cluster 2023-08-28T02:07:38.929461+0000 mgr.y (mgr.14146) 74 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-08-28T02:07:40.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:40 smithi169 conmon[100500]: cluster 2023-08-28T02:07:39.467047+0000 mon.a (mon.0) 292 : cluster [INF] osd.1 [v2:172.21.15.143:6810/2341769896,v1:172.21.15.143:6811/2341769896] boot 2023-08-28T02:07:40.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:40 smithi169 conmon[100500]: cluster 2023-08-28T 2023-08-28T02:07:40.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:40 smithi169 conmon[100500]: 02:07:39.467084+0000 mon.a (mon.0) 2023-08-28T02:07:40.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:40 smithi169 conmon[100500]: 293 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-08-28T02:07:40.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:40 smithi169 conmon[100500]: audit 2023-08-28T02 2023-08-28T02:07:40.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:40 smithi169 conmon[100500]: :07:39.470801+0000 mon.a (mon.0) 294 : 2023-08-28T02:07:40.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:40 smithi169 conmon[100500]: audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-28T02:07:41.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:41 smithi169 conmon[100500]: cluster 2023-08-28T02:07:40.479304+0000 mon.a (mon.0) 295 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-08-28T02:07:41.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:41 smithi169 conmon[100500]: audit 2023-08-28T02 2023-08-28T02:07:41.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:41 smithi169 conmon[100500]: :07:40.637331+0000 mon.a (mon.0) 296 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:41.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:41 smithi169 conmon[100500]: audit 2023-08-28T02:07:40.638215+0000 mon.a (mon.0) 297 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:07:41.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:41 smithi169 conmon[100500]: audit 2023-08-28T02:07:40.638894+0000 mon.a (mon.0) 298 : audit [INF] 2023-08-28T02:07:41.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:41 smithi169 conmon[100500]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:07:41.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:41 smithi169 conmon[100500]: audit 2023-08-28T02:07:40.643338 2023-08-28T02:07:41.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:41 smithi169 conmon[100500]: +0000 mon.a (mon.0) 299 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:41.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:41 smithi169 conmon[100500]: audit 2023 2023-08-28T02:07:41.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:41 smithi169 conmon[100500]: -08-28T02:07:40.644540+0000 mon.a ( 2023-08-28T02:07:41.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:41 smithi169 conmon[100500]: mon.0) 300 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:07:41.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:41 smithi169 conmon[100500]: audit 2023-08 2023-08-28T02:07:41.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:41 smithi169 conmon[100500]: -28T02:07:40.645338+0000 mon.a ( 2023-08-28T02:07:41.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:41 smithi169 conmon[100500]: mon.0) 301 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:07:41.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:41 smithi169 conmon[100500]: audit 2023-08- 2023-08-28T02:07:41.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:41 smithi169 conmon[100500]: 28T02:07:40.650030+0000 mon.a (mon. 2023-08-28T02:07:41.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:41 smithi169 conmon[100500]: 0) 302 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:41.787 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:41 smithi143 conmon[100258]: cluster 2023-08-28T02:07:40.479304+0000 mon.a (mon.0) 295 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-08-28T02:07:41.787 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:41 smithi143 conmon[100258]: audit 2023-08-28T02:07:40.637331+0000 mon.a (mon.0) 296 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:41.787 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:41 smithi143 conmon[100258]: audit 2023-08-28T02:07:40.638215+0000 mon.a (mon.0) 297 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:07:41.787 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:41 smithi143 conmon[100258]: audit 2023-08-28T02:07:40.638894+0000 mon.a (mon.0) 298 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:07:41.787 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:41 smithi143 conmon[100258]: audit 2023-08-28T02:07:40.643338+0000 mon.a (mon.0) 299 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:41.787 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:41 smithi143 conmon[100258]: audit 2023-08-28T02:07:40.644540+0000 mon.a (mon.0) 300 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:07:41.787 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:41 smithi143 conmon[100258]: audit 2023-08-28T02:07:40.645338+0000 mon.a (mon. 2023-08-28T02:07:41.787 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:41 smithi143 conmon[100258]: 0) 301 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:07:41.788 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:41 smithi143 conmon[100258]: audit 2023-08-28T02:07:40.650030+0000 mon.a (mon.0) 302 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:41.788 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:41 smithi143 conmon[109923]: cluster 2023-08-28T02:07:40.479304+0000 mon.a (mon.0) 295 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-08-28T02:07:41.788 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:41 smithi143 conmon[109923]: audit 2023-08-28T02:07:40.637331+0000 mon.a (mon.0) 296 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:41.788 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:41 smithi143 conmon[109923]: audit 2023-08-28T02:07:40.638215+0000 mon.a (mon.0) 297 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:07:41.788 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:41 smithi143 conmon[109923]: audit 2023-08-28T02:07:40.638894+0000 mon.a (mon.0) 298 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:07:41.788 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:41 smithi143 conmon[109923]: audit 2023-08-28T02:07:40.643338+0000 mon.a (mon.0) 299 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:41.788 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:41 smithi143 conmon[109923]: audit 2023-08-28T02:07:40.644540+0000 mon.a (mon.0) 300 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:07:41.788 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:41 smithi143 conmon[109923]: audit 2023-08-28T02:07:40.645338+0000 mon.a (mon.0) 301 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:07:41.788 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:41 smithi143 conmon[109923]: audit 2023-08-28T02:07:40.650030+0000 mon.a (mon.0) 302 : audit [INF] 2023-08-28T02:07:41.788 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:41 smithi143 conmon[109923]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:42.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:42 smithi169 conmon[100500]: cephadm 2023-08-28T02:07:40.632900+0000 mgr.y (mgr.14146) 75 : cephadm [INF] Detected new or changed devices on smithi143 2023-08-28T02:07:42.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:42 smithi169 conmon[100500]: cephadm 2023-08-28T02 2023-08-28T02:07:42.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:42 smithi169 conmon[100500]: :07:40.639399+0000 mgr.y (mgr.14146) 76 : cephadm [INF] Adjusting osd_memory_target on smithi143 to 8072M 2023-08-28T02:07:42.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:42 smithi169 conmon[100500]: cluster 2023-08-28T02:07:40.929722+0000 mgr.y (mgr.14146) 77 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-08-28T02:07:42.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:42 smithi169 conmon[100500]: audit 2023-08-28T02:07:42.160345+0000 mon.a (mon.0) 303 : audit 2023-08-28T02:07:42.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:42 smithi169 conmon[100500]: [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-28T02:07:42.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:42 smithi169 conmon[100500]: audit 2023-08-28T02:07:42.162267+0000 2023-08-28T02:07:42.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:42 smithi169 conmon[100500]: mon.a (mon.0) 304 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-28T02:07:42.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:42 smithi169 conmon[100500]: audit 2023- 2023-08-28T02:07:42.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:42 smithi169 conmon[100500]: 08-28T02:07:42.162763+0000 mon.a ( 2023-08-28T02:07:42.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:42 smithi169 conmon[100500]: mon.0) 305 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:07:42.820 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:42 smithi143 conmon[100258]: cephadm 2023-08-28T02:07:40.632900 2023-08-28T02:07:42.820 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:42 smithi143 conmon[100258]: +0000 mgr.y (mgr.14146) 75 : cephadm [INF] Detected new or changed devices on smithi143 2023-08-28T02:07:42.820 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:42 smithi143 conmon[100258]: cephadm 2023-08-28T02:07:40.639399+0000 mgr.y (mgr.14146) 76 : cephadm [INF] Adjusting osd_memory_target on smithi143 to 8072M 2023-08-28T02:07:42.820 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:42 smithi143 conmon[100258]: cluster 2023-08-28T02:07:40.929722+0000 mgr.y (mgr.14146) 77 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-08-28T02:07:42.820 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:42 smithi143 conmon[100258]: audit 2023-08-28T02:07:42.160345+0000 mon.a (mon.0) 303 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-28T02:07:42.820 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:42 smithi143 conmon[100258]: audit 2023-08-28T02:07:42.162267+0000 mon.a (mon.0) 304 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-28T02:07:42.820 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:42 smithi143 conmon[100258]: audit 2023-08-28T02:07:42.162763+0000 mon.a (mon.0) 305 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:07:42.820 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:42 smithi143 conmon[109923]: cephadm 2023-08-28T02:07:40.632900+0000 mgr.y (mgr.14146) 75 : cephadm [INF] Detected new or changed devices on smithi143 2023-08-28T02:07:42.821 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:42 smithi143 conmon[109923]: cephadm 2023-08-28T02:07:40.639399+0000 mgr.y (mgr.14146) 76 : cephadm [INF] Adjusting osd_memory_target on smithi143 to 8072M 2023-08-28T02:07:42.821 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:42 smithi143 conmon[109923]: cluster 2023-08-28T02:07:40.929722+0000 mgr.y (mgr.14146) 77 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-08-28T02:07:42.821 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:42 smithi143 conmon[109923]: audit 2023-08-28T02:07:42.160345+0000 mon.a (mon.0) 303 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-28T02:07:42.821 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:42 smithi143 conmon[109923]: audit 2023-08-28T02:07:42.162267+0000 mon.a (mon.0) 304 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-28T02:07:42.821 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:42 smithi143 conmon[109923]: audit 2023-08-28T02:07:42.162763+0000 mon.a (mon.0) 305 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:07:43.678 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:43 smithi143 conmon[100258]: audit 2023-08-28T02:07:42.158385+0000 mgr.y (mgr.14146) 78 : audit [DBG] from='client.14262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi143:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:07:43.679 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:43 smithi143 conmon[109923]: audit 2023-08-28T02:07:42.158385+0000 mgr.y (mgr.14146) 78 : audit [DBG] from='client.14262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi143:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:07:43.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:43 smithi169 conmon[100500]: audit 2023-08-28T02:07:42.158385+0000 mgr.y (mgr.14146) 78 : audit [DBG] from='client.14262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi143:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:07:44.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:44 smithi143 conmon[100258]: cluster 2023-08-28T02:07:42.930105+0000 mgr.y (mgr 2023-08-28T02:07:44.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:44 smithi143 conmon[100258]: .14146) 79 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-08-28T02:07:44.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:44 smithi143 conmon[100258]: audit 2023-08-28T02:07:44.331123+0000 mon.a (mon.0) 306 : audit [INF] from='client.? 172.21.15.143:0/3329703071' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4f8492d1-0e7a-49f8-b6a5-4f5e73842349"}]: dispatch 2023-08-28T02:07:44.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:44 smithi143 conmon[100258]: audit 2023-08-28T02:07:44.337347+0000 mon.a (mon.0) 307 : audit [INF] from='client.? 172.21.15.143:0/3329703071' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4f8492d1-0e7a-49f8-b6a5-4f5e73842349"}]': finished 2023-08-28T02:07:44.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:44 smithi143 conmon[100258]: cluster 2023-08-28T02:07:44.337422+0000 mon.a (mon.0) 308 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2023-08-28T02:07:44.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:44 smithi143 conmon[100258]: audit 2023-08-28T02:07:44.337537+0000 mon.a (mon.0) 309 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-28T02:07:44.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:44 smithi143 conmon[109923]: cluster 2023-08-28T02:07:42.930105+0000 mgr.y ( 2023-08-28T02:07:44.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:44 smithi143 conmon[109923]: mgr.14146) 79 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-08-28T02:07:44.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:44 smithi143 conmon[109923]: audit 2023-08-28T02:07:44.331123+0000 mon.a (mon.0) 306 : audit [INF] from='client.? 172.21.15.143:0/3329703071' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4f8492d1-0e7a-49f8-b6a5-4f5e73842349"}]: dispatch 2023-08-28T02:07:44.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:44 smithi143 conmon[109923]: audit 2023-08-28T02:07:44.337347+0000 mon.a (mon.0) 307 : audit [INF] from='client.? 172.21.15.143:0/3329703071' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4f8492d1-0e7a-49f8-b6a5-4f5e73842349"}]': finished 2023-08-28T02:07:44.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:44 smithi143 conmon[109923]: cluster 2023-08-28T02:07:44.337422+0000 mon.a (mon.0) 308 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2023-08-28T02:07:44.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:44 smithi143 conmon[109923]: audit 2023-08-28T02:07:44.337537+0000 mon.a (mon.0) 309 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-28T02:07:44.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:44 smithi169 conmon[100500]: cluster 2023-08-28T02:07:42.930105+0000 mgr.y (mgr.14146) 79 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-08-28T02:07:44.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:44 smithi169 conmon[100500]: audit 2023-08-28T02:07:44.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:44 smithi169 conmon[100500]: 2023-08-28T02:07:44.331123+0000 mon.a (mon.0) 306 : audit [INF] from='client.? 172.21.15.143:0/3329703071' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4f8492d1-0e7a-49f8-b6a5-4f5e73842349"}]: dispatch 2023-08-28T02:07:44.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:44 smithi169 conmon[100500]: audit 2023-08-28T02:07:44.337347+0000 mon.a (mon.0) 307 : audit [INF] from='client.? 172.21.15.143:0/3329703071' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4f8492d1-0e7a-49f8-b6a5-4f5e73842349"}]': finished 2023-08-28T02:07:44.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:44 smithi169 conmon[100500]: cluster 2023-08-28T02:07:44.337422+0000 mon.a 2023-08-28T02:07:44.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:44 smithi169 conmon[100500]: (mon.0) 308 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2023-08-28T02:07:44.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:44 smithi169 conmon[100500]: audit 2023-08- 2023-08-28T02:07:44.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:44 smithi169 conmon[100500]: 28T02:07:44.337537+0000 mon.a (mon 2023-08-28T02:07:44.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:44 smithi169 conmon[100500]: .0) 309 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-28T02:07:45.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:45 smithi143 conmon[109923]: audit 2023-08-28T02:07:44.895704 2023-08-28T02:07:45.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:45 smithi143 conmon[109923]: +0000 mon.a (mon.0) 310 : audit [DBG] from='client.? 172.21.15.143:0/3972350632' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-28T02:07:45.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:45 smithi143 conmon[100258]: audit 2023-08-28T02:07:44 2023-08-28T02:07:45.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:45 smithi143 conmon[100258]: .895704+0000 mon.a (mon.0) 310 : audit [DBG] from='client.? 172.21.15.143:0/3972350632' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-28T02:07:45.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:45 smithi169 conmon[100500]: audit 2023-08-28T02:07:44.895704+0000 mon.a (mon.0) 310 : audit [DBG] from='client.? 172.21.15.143:0/3972350632' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-28T02:07:46.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:46 smithi143 conmon[109923]: cluster 2023-08-28T02:07:44.930426+0000 mgr.y ( 2023-08-28T02:07:46.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:46 smithi143 conmon[109923]: mgr.14146) 80 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-08-28T02:07:46.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:46 smithi143 conmon[100258]: cluster 2023-08-28T02:07:44.930426+0000 mgr.y (mgr. 2023-08-28T02:07:46.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:46 smithi143 conmon[100258]: 14146) 80 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-08-28T02:07:46.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:46 smithi169 conmon[100500]: cluster 2023-08-28T02:07:44.930426+0000 mgr.y (mgr.14146) 80 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-08-28T02:07:48.635 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:48 smithi143 conmon[100258]: cluster 2023-08-28T02:07:46.930848+0000 mgr.y (mgr.14146) 81 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-08-28T02:07:48.635 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:48 smithi143 conmon[109923]: cluster 2023-08-28T02:07:46.930848+0000 mgr.y (mgr.14146) 81 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-08-28T02:07:48.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:48 smithi169 conmon[100500]: cluster 2023-08-28T02:07:46.930848+0000 mgr.y (mgr.14146) 2023-08-28T02:07:48.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:48 smithi169 conmon[100500]: 81 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-08-28T02:07:50.864 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:50 smithi143 conmon[100258]: cluster 2023-08-28T02:07:48.931282+0000 mgr.y (mgr.14146) 82 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-08-28T02:07:50.865 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:50 smithi143 conmon[100258]: audit 2023-08-28T02:07:50.166584+0000 mon.a (mon.0) 311 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-08-28T02:07:50.865 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:50 smithi143 conmon[100258]: audit 2023-08-28T02:07:50.167192+0000 mon.a (mon.0) 312 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:07:50.865 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:50 smithi143 conmon[109923]: cluster 2023-08-28T02:07:48.931282+0000 mgr.y (mgr.14146) 82 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-08-28T02:07:50.865 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:50 smithi143 conmon[109923]: audit 2023-08-28T02:07:50.166584+0000 mon.a (mon.0) 311 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-08-28T02:07:50.865 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:50 smithi143 conmon[109923]: audit 2023-08-28T02:07:50.167192+0000 mon.a (mon.0) 312 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:07:50.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:50 smithi169 conmon[100500]: cluster 2023-08-28T02:07:48.931282+0000 mgr.y ( 2023-08-28T02:07:50.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:50 smithi169 conmon[100500]: mgr.14146) 82 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-08-28T02:07:50.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:50 smithi169 conmon[100500]: audit 2023-08-28T02:07:50.166584+0000 mon.a (mon.0) 311 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-08-28T02:07:50.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:50 smithi169 conmon[100500]: audit 2023-08-28T02:07:50.167192+0000 2023-08-28T02:07:50.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:50 smithi169 conmon[100500]: mon.a (mon.0) 312 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:07:51.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:51 smithi143 conmon[100258]: cephadm 2023-08-28T02:07:50.167682+0000 mgr.y (mgr.14146 2023-08-28T02:07:51.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:51 smithi143 conmon[100258]: ) 83 : cephadm [INF] Deploying daemon osd.2 on smithi143 2023-08-28T02:07:51.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:51 smithi143 conmon[109923]: cephadm 2023-08-28T02:07:50.167682+0000 mgr.y ( 2023-08-28T02:07:51.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:51 smithi143 conmon[109923]: mgr.14146) 83 : cephadm [INF] Deploying daemon osd.2 on smithi143 2023-08-28T02:07:51.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:51 smithi169 conmon[100500]: cephadm 2023-08-28T02:07:50.167682+0000 mgr.y (mgr.14146) 83 : cephadm [INF] Deploying daemon osd.2 on smithi143 2023-08-28T02:07:52.822 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:52 smithi143 conmon[100258]: cluster 2023-08-28T02:07:50.931651+0000 mgr.y ( 2023-08-28T02:07:52.822 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:52 smithi143 conmon[100258]: mgr.14146) 84 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-28T02:07:52.823 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:52 smithi143 conmon[109923]: cluster 2023-08-28T02:07:50.931651+0000 2023-08-28T02:07:52.823 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:52 smithi143 conmon[109923]: mgr.y (mgr.14146) 84 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-28T02:07:52.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:52 smithi169 conmon[100500]: cluster 2023-08-28T02:07:50.931651+0000 2023-08-28T02:07:52.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:52 smithi169 conmon[100500]: mgr.y (mgr.14146) 84 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-28T02:07:54.806 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:54 smithi143 conmon[100258]: cluster 2023-08-28T02:07:52.932157+0000 2023-08-28T02:07:54.806 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:54 smithi143 conmon[100258]: mgr.y (mgr.14146) 85 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-28T02:07:54.807 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:54 smithi143 conmon[109923]: cluster 2023-08-28T02:07:52.932157+0000 mgr.y (mgr.14146) 2023-08-28T02:07:54.807 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:54 smithi143 conmon[109923]: 85 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-28T02:07:54.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:54 smithi169 conmon[100500]: cluster 2023-08-28T02:07:52.932157+0000 mgr.y ( 2023-08-28T02:07:54.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:54 smithi169 conmon[100500]: mgr.14146) 85 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-28T02:07:56.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:55 smithi143 conmon[100258]: audit 2023-08-28T02:07:54.872540 2023-08-28T02:07:56.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:55 smithi143 conmon[100258]: +0000 mon.a (mon.0) 313 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:56.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:55 smithi143 conmon[100258]: audit 2023-08-28T02:07:54.892071+0000 mon.a (mon.0) 314 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:07:56.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:55 smithi143 conmon[100258]: cluster 2023-08-28T02:07:54.932422+0000 mgr.y (mgr.14146) 86 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-28T02:07:56.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:55 smithi143 conmon[109923]: audit 2023-08-28T02:07:54.872540+0000 2023-08-28T02:07:56.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:55 smithi143 conmon[109923]: mon.a (mon.0) 313 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:56.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:55 smithi143 conmon[109923]: audit 2023-08 2023-08-28T02:07:56.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:55 smithi143 conmon[109923]: -28T02:07:54.892071+0000 mon.a (mon.0) 314 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:07:56.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:55 smithi143 conmon[109923]: cluster 2023-08-28T02:07:56.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:55 smithi143 conmon[109923]: 2023-08-28T02: 2023-08-28T02:07:56.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:55 smithi143 conmon[109923]: 07:54.932422+0000 mgr.y (mgr.14146) 86 2023-08-28T02:07:56.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:55 smithi143 conmon[109923]: : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-28T02:07:56.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:55 smithi169 conmon[100500]: audit 2023-08-28T02:07:54.872540+0000 mon.a (mon.0) 313 : 2023-08-28T02:07:56.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:55 smithi169 conmon[100500]: audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:56.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:55 smithi169 conmon[100500]: audit 2023-08-28T02:07:54.892071+0000 mon.a (mon.0) 314 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:07:56.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:55 smithi169 conmon[100500]: cluster 2023-08-28T02:07:54.932422+0000 mgr.y (mgr.14146) 86 : 2023-08-28T02:07:56.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:55 smithi169 conmon[100500]: cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-28T02:07:58.035 INFO:teuthology.orchestra.run.smithi143.stdout:Created osd(s) 2 on host 'smithi143' 2023-08-28T02:07:58.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[100258]: cluster 2023-08-28T02:07:56.932681+0000 mgr.y (mgr. 2023-08-28T02:07:58.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[100258]: 14146) 87 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-28T02:07:58.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[100258]: audit 2023-08-28T02:07:57.090909+0000 mon.a (mon.0) 315 : audit 2023-08-28T02:07:58.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[100258]: [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:58.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[100258]: audit 2023 2023-08-28T02:07:58.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[100258]: -08-28T02:07:57.091635+0000 mon.a (mon.0) 316 2023-08-28T02:07:58.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[100258]: : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:07:58.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[100258]: audit 2023-08-28T 2023-08-28T02:07:58.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[100258]: 02:07:57.092247+0000 mon.a (mon.0 2023-08-28T02:07:58.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[100258]: ) 317 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:07:58.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[100258]: audit 2023-08-28T02:07:58.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[100258]: 2023-08-28T02:07:57. 2023-08-28T02:07:58.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[100258]: 096373+0000 mon.a (mon.0) 318 : audit 2023-08-28T02:07:58.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[100258]: [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:58.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[100258]: audit 2023 2023-08-28T02:07:58.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[100258]: -08-28T02:07:58.035573+0000 2023-08-28T02:07:58.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[100258]: mon.a (mon.0) 319 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:58.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[100258]: audit 2023-08-28T02:07: 2023-08-28T02:07:58.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[100258]: 58.063624+0000 mon.a (mon.0) 320 : audit [DBG] 2023-08-28T02:07:58.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[100258]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:07:58.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[100258]: audit 2023-08- 2023-08-28T02:07:58.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[100258]: 28T02:07:58.079163+0000 mon.c (mon 2023-08-28T02:07:58.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[100258]: .2) 5 : audit [INF] from='osd.2 [v2:172.21.15.143:6818/2664999316,v1:172.21.15.143:6819/2664999316]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-08-28T02:07:58.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[100258]: audit 2023-08-28T02:07:58.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[100258]: 2023-08-28T02:07:58 2023-08-28T02:07:58.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[100258]: .079444+0000 mon.a (mon.0) 321 : audit 2023-08-28T02:07:58.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[100258]: [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-08-28T02:07:58.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[109923]: cluster 2023-08-28T02:07:56. 2023-08-28T02:07:58.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[109923]: 932681+0000 mgr.y (mgr.14146) 87 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-28T02:07:58.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[109923]: audit 2023-08-28T02:07:58.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[109923]: 2023-08-28T02:07:57.090909+0000 mon.a (mon.0) 315 2023-08-28T02:07:58.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[109923]: : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:58.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[109923]: audit 2023-08-28T02:07:58.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[109923]: 2023-08-28 2023-08-28T02:07:58.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[109923]: T02:07:57.091635+0000 mon.a 2023-08-28T02:07:58.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[109923]: (mon.0) 316 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:07:58.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[109923]: 2023-08-28T02:07:58.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[109923]: audit 2023-08-28T02 2023-08-28T02:07:58.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[109923]: :07:57.092247+0000 mon.a (mon.0) 2023-08-28T02:07:58.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[109923]: 317 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:07:58.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[109923]: audit 2023-08-28T02 2023-08-28T02:07:58.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[109923]: :07:57.096373+0000 mon.a (mon. 2023-08-28T02:07:58.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[109923]: 0) 318 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:58.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[109923]: audit 2023-08-28T02:07 2023-08-28T02:07:58.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[109923]: :58.035573+0000 mon.a (mon.0) 319 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:58.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[109923]: 2023-08-28T02:07:58.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[109923]: audit 2023-08-28T02:07:58.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[109923]: 2023-08-28T02:07:58.063624+0000 mon.a (mon.0 2023-08-28T02:07:58.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[109923]: ) 320 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:07:58.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:07:58.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[109923]: 08-28T02:07:58.079163+0000 2023-08-28T02:07:58.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[109923]: mon.c (mon.2) 5 : audit [INF] from='osd.2 [v2:172.21.15.143:6818/2664999316,v1:172.21.15.143:6819/2664999316]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-08-28T02:07:58.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[109923]: 2023-08-28T02:07:58.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[109923]: audit 2023-08-28T02:07:58.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[109923]: 2023-08-28T02:07:58.079444+0000 2023-08-28T02:07:58.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[109923]: mon.a (mon.0) 321 : audit [INF] 2023-08-28T02:07:58.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:07:58 smithi143 conmon[109923]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-08-28T02:07:58.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:58 smithi169 conmon[100500]: cluster 2023-08-28T02:07:56.932681+0000 mgr.y (mgr.14146) 87 : cluster [DBG] 2023-08-28T02:07:58.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:58 smithi169 conmon[100500]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-28T02:07:58.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:58 smithi169 conmon[100500]: audit 2023-08-28T02:07:57.090909+0000 mon.a (mon.0) 315 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:58.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:58 smithi169 conmon[100500]: audit 2023-08 2023-08-28T02:07:58.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:58 smithi169 conmon[100500]: -28T02:07:57.091635+0000 mon.a (mon.0) 316 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:07:58.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:58 smithi169 conmon[100500]: audit 2023-08-28T02:07:57 2023-08-28T02:07:58.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:58 smithi169 conmon[100500]: .092247+0000 mon.a (mon.0) 317 : audit [INF] 2023-08-28T02:07:58.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:58 smithi169 conmon[100500]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:07:58.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:58 smithi169 conmon[100500]: audit 2023-08-28T02:07:57.096373+0000 2023-08-28T02:07:58.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:58 smithi169 conmon[100500]: mon.a (mon.0) 318 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:58.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:58 smithi169 conmon[100500]: audit 2023-08-28 2023-08-28T02:07:58.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:58 smithi169 conmon[100500]: T02:07:58.035573+0000 mon.a (mon. 2023-08-28T02:07:58.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:58 smithi169 conmon[100500]: 0) 319 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:07:58.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:58 smithi169 conmon[100500]: audit 2023- 2023-08-28T02:07:58.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:58 smithi169 conmon[100500]: 08-28T02:07:58.063624+0000 mon.a (mon 2023-08-28T02:07:58.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:58 smithi169 conmon[100500]: .0) 320 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:07:58.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:58 smithi169 conmon[100500]: audit 2023-08-28T02:07:58.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:58 smithi169 conmon[100500]: 2023-08-28T02:07:58 2023-08-28T02:07:58.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:58 smithi169 conmon[100500]: .079163+0000 mon.c (mon.2) 5 : audit 2023-08-28T02:07:58.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:58 smithi169 conmon[100500]: [INF] from='osd.2 [v2:172.21.15.143:6818/2664999316,v1:172.21.15.143:6819/2664999316]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-08-28T02:07:58.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:58 smithi169 conmon[100500]: audit 2023- 2023-08-28T02:07:58.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:58 smithi169 conmon[100500]: 08-28T02:07:58.079444 2023-08-28T02:07:58.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:58 smithi169 conmon[100500]: +0000 mon.a (mon.0) 321 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-08-28T02:07:58.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:07:58 smithi169 conmon[100500]: 2023-08-28T02:07:58.717 DEBUG:teuthology.orchestra.run.smithi143:osd.2> sudo journalctl -f -n 0 -u ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@osd.2.service 2023-08-28T02:07:58.719 INFO:tasks.cephadm:Deploying osd.3 on smithi143 with /dev/vg_nvme/lv_1... 2023-08-28T02:07:58.719 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-08-28T02:07:58.750 INFO:journalctl@ceph.osd.2.smithi143.stdout:-- Logs begin at Mon 2023-08-28 01:49:41 UTC. -- 2023-08-28T02:08:00.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:00 smithi143 conmon[100258]: cluster 2023-08-28T02:07:58.932924+0000 mgr.y (mgr.14146) 88 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-28T02:08:00.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:00 smithi143 conmon[100258]: audit 2023-08-28T02:07:59.037528+0000 mon.a (mon.0) 322 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-08-28T02:08:00.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:00 smithi143 conmon[100258]: cluster 2023-08-28T02:07:59.037612+0000 mon.a (mon.0) 323 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-08-28T02:08:00.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:00 smithi143 conmon[100258]: audit 2023-08-28T02:07:59.037754+0000 mon.a (mon.0) 324 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-28T02:08:00.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:00 smithi143 conmon[100258]: audit 2023-08-28T 2023-08-28T02:08:00.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:00 smithi143 conmon[100258]: 02:07:59.038147+0000 mon.c (mon.2) 6 : audit [INF] from='osd.2 [v2:172.21.15.143:6818/2664999316,v1:172.21.15.143:6819/2664999316]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]: dispatch 2023-08-28T02:08:00.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:00 smithi143 conmon[100258]: audit 2023-08-28T02:07:59.038453+0000 mon.a (mon.0) 325 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]: dispatch 2023-08-28T02:08:00.201 INFO:journalctl@ceph.osd.2.smithi143.stdout:Aug 28 02:08:00 smithi143 conmon[126784]: debug 2023-08-28T02:08:00.042+0000 7f1445265700 -1 osd.2 0 waiting for initial osdmap 2023-08-28T02:08:00.201 INFO:journalctl@ceph.osd.2.smithi143.stdout:Aug 28 02:08:00 smithi143 conmon[126784]: debug 2023-08-28T02:08:00.045+0000 7f143d9d9700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-08-28T02:08:00.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:00 smithi143 conmon[109923]: cluster 2023-08-28T02:07:58.932924+0000 mgr.y (mgr.14146) 88 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-28T02:08:00.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:00 smithi143 conmon[109923]: audit 2023-08-28T02:07:59.037528+0000 mon.a (mon.0) 322 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-08-28T02:08:00.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:00 smithi143 conmon[109923]: cluster 2023-08-28T02:07:59.037612+0000 mon.a (mon.0) 323 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-08-28T02:08:00.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:00 smithi143 conmon[109923]: audit 2023-08-28T02:07:59.037754+0000 mon.a (mon.0) 324 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-28T02:08:00.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:00 smithi143 conmon[109923]: audit 2023-08-28T02:07:59.038147+0000 mon.c (mon.2) 6 : audit [INF] from='osd.2 [v2:172.21.15.143:6818/2664999316,v1:172.21.15.143:6819/2664999316]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]: dispatch 2023-08-28T02:08:00.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:00 smithi143 conmon[109923]: audit 2023-08-28T02:07:59.038453+0000 mon.a (mon.0) 325 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]: dispatch 2023-08-28T02:08:00.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:00 smithi169 conmon[100500]: cluster 2023-08-28T02:07:58.932924+0000 mgr.y (mgr.14146) 88 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-28T02:08:00.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:00 smithi169 conmon[100500]: audit 2023-08-28T02:07:59.037528+0000 mon.a (mon.0) 322 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-08-28T02:08:00.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:00 smithi169 conmon[100500]: cluster 2023-08-28T02:07:59.037612+0000 2023-08-28T02:08:00.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:00 smithi169 conmon[100500]: mon.a (mon.0) 323 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-08-28T02:08:00.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:00 smithi169 conmon[100500]: audit 2023-08-28T02:08:00.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:00 smithi169 conmon[100500]: 2023-08-28T02:07:59.037754+0000 2023-08-28T02:08:00.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:00 smithi169 conmon[100500]: mon.a (mon.0) 324 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-28T02:08:00.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:00 smithi169 conmon[100500]: audit 2023-08-28T02:08:00.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:00 smithi169 conmon[100500]: 2023-08-28T02:07:59 2023-08-28T02:08:00.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:00 smithi169 conmon[100500]: .038147+0000 mon.c (mon.2) 6 2023-08-28T02:08:00.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:00 smithi169 conmon[100500]: : audit [INF] from='osd.2 [v2:172.21.15.143:6818/2664999316,v1:172.21.15.143:6819/2664999316]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]: dispatch 2023-08-28T02:08:00.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:00 smithi169 conmon[100500]: audit 2023 2023-08-28T02:08:00.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:00 smithi169 conmon[100500]: -08-28T02:07:59. 2023-08-28T02:08:00.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:00 smithi169 conmon[100500]: 038453+0000 mon.a (mon.0) 325 : audit [INF] 2023-08-28T02:08:00.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:00 smithi169 conmon[100500]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]: dispatch 2023-08-28T02:08:01.241 INFO:teuthology.orchestra.run.smithi143.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-08-28T02:08:01.241 INFO:teuthology.orchestra.run.smithi143.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-08-28T02:08:01.242 INFO:teuthology.orchestra.run.smithi143.stderr: stderr: 10+0 records in 2023-08-28T02:08:01.242 INFO:teuthology.orchestra.run.smithi143.stderr:10+0 records out 2023-08-28T02:08:01.242 INFO:teuthology.orchestra.run.smithi143.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0117546 s, 892 MB/s 2023-08-28T02:08:01.242 INFO:teuthology.orchestra.run.smithi143.stderr:--> Zapping successful for: 2023-08-28T02:08:01.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:01 smithi143 conmon[100258]: audit 2023-08-28T02:08:00.038561+0000 mon.a (mon.0) 326 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]': finished 2023-08-28T02:08:01.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:01 smithi143 conmon[100258]: cluster 2023-08-28T02:08:00.038672+0000 mon.a (mon.0) 327 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-08-28T02:08:01.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:01 smithi143 conmon[100258]: audit 2023-08-28T02:08:00.039045+0000 mon.a (mon.0) 328 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-28T02:08:01.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:01 smithi143 conmon[100258]: audit 2023-08-28T02:08:00.043209+0000 mon.a (mon.0) 329 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-28T02:08:01.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:01 smithi143 conmon[109923]: audit 2023-08-28T02:08:00. 2023-08-28T02:08:01.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:01 smithi143 conmon[109923]: 038561+0000 mon.a (mon.0) 326 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]': finished 2023-08-28T02:08:01.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:01 smithi143 conmon[109923]: cluster 2023-08-28T02:08:00.038672+0000 mon.a (mon.0) 327 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-08-28T02:08:01.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:01 smithi143 conmon[109923]: audit 2023-08-28T02:08:00.039045+0000 mon.a (mon.0) 328 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-28T02:08:01.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:01 smithi143 conmon[109923]: audit 2023-08-28T02:08:00.043209+0000 mon.a (mon.0) 329 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-28T02:08:01.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:01 smithi169 conmon[100500]: audit 2023-08-28T02:08:00.038561+0000 mon.a (mon.0) 326 : 2023-08-28T02:08:01.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:01 smithi169 conmon[100500]: audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]': finished 2023-08-28T02:08:01.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:01 smithi169 conmon[100500]: cluster 2023-08-28T02:08:00.038672+0000 mon.a (mon.0) 327 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-08-28T02:08:01.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:01 smithi169 conmon[100500]: audit 2023-08-28T02:08:00. 2023-08-28T02:08:01.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:01 smithi169 conmon[100500]: 039045+0000 mon.a (mon.0) 328 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-28T02:08:01.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:01 smithi169 conmon[100500]: 2023-08-28T02:08:01.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:01 smithi169 conmon[100500]: audit 2023-08-28 2023-08-28T02:08:01.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:01 smithi169 conmon[100500]: T02:08:00.043209+0000 mon.a (mon 2023-08-28T02:08:01.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:01 smithi169 conmon[100500]: .0) 329 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-28T02:08:02.177 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph orch daemon add osd smithi143:vg_nvme/lv_1 2023-08-28T02:08:02.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:02 smithi143 conmon[100258]: cluster 2023-08-28T02:07:59.099531+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2023-08-28T02:08:02.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:02 smithi143 conmon[100258]: cluster 2023-08-28T02:07:59.099587+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-08-28T02:08:02.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:02 smithi143 conmon[100258]: cluster 2023-08-28T02:08:00.933231+0000 mgr.y (mgr.14146) 89 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-28T02:08:02.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:02 smithi143 conmon[100258]: audit 2023-08-28T02:08:01.041753+0000 mon.a (mon.0) 330 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-28T02:08:02.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:02 smithi143 conmon[100258]: cluster 2023-08-28T02:08:01.045808+0000 mon.a (mon.0) 331 : cluster [INF] osd.2 [v2:172.21.15.143:6818/2664999316,v1:172.21.15.143:6819/2664999316] boot 2023-08-28T02:08:02.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:02 smithi143 conmon[100258]: cluster 2023-08-28T02:08:01.045847+0000 mon.a (mon.0) 332 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2023-08-28T02:08:02.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:02 smithi143 conmon[100258]: audit 2023-08-28T02:08:01.053193+0000 mon.a (mon.0) 333 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-28T02:08:02.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:02 smithi143 conmon[100258]: audit 2023-08-28T02:08:01.053584+0000 mon.a (mon.0) 334 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-08-28T02:08:02.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:02 smithi143 conmon[100258]: audit 2023-08-28T02:08:02.049547+0000 mon.a (mon.0) 335 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-08-28T02:08:02.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:02 smithi143 conmon[100258]: cluster 2023-08-28T02:08:02.049610+0000 mon.a (mon.0) 336 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2023-08-28T02:08:02.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:02 smithi143 conmon[100258]: audit 2023-08-28T02:08:02.050503+0000 mon.a (mon.0) 337 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-08-28T02:08:02.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:02 smithi143 conmon[109923]: cluster 2023-08-28T02:07:59.099531+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2023-08-28T02:08:02.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:02 smithi143 conmon[109923]: cluster 2023-08-28T02:07:59.099587+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-08-28T02:08:02.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:02 smithi143 conmon[109923]: cluster 2023-08-28T02:08:00.933231+0000 mgr.y (mgr.14146) 89 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-28T02:08:02.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:02 smithi143 conmon[109923]: audit 2023-08-28T02:08:01.041753+0000 mon.a (mon.0) 330 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-28T02:08:02.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:02 smithi143 conmon[109923]: cluster 2023-08-28T02:08:01.045808+0000 mon.a (mon.0) 331 : cluster [INF] osd.2 [v2:172.21.15.143:6818/2664999316,v1:172.21.15.143:6819/2664999316] boot 2023-08-28T02:08:02.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:02 smithi143 conmon[109923]: cluster 2023-08-28T02:08:01.045847+0000 mon.a (mon.0) 332 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2023-08-28T02:08:02.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:02 smithi143 conmon[109923]: audit 2023-08-28T02:08:01.053193+0000 mon.a (mon.0) 333 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-28T02:08:02.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:02 smithi143 conmon[109923]: audit 2023-08-28T02:08:01.053584+0000 mon.a (mon.0) 334 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-08-28T02:08:02.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:02 smithi143 conmon[109923]: audit 2023-08-28T02:08:02.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:02 smithi143 conmon[109923]: 2023-08-28T02:08:02.049547+0000 mon.a (mon.0) 335 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-08-28T02:08:02.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:02 smithi143 conmon[109923]: cluster 2023-08-28T02:08:02.049610+0000 mon.a (mon.0) 336 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2023-08-28T02:08:02.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:02 smithi143 conmon[109923]: audit 2023-08-28T02:08:02.050503+0000 mon.a (mon.0) 337 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-08-28T02:08:02.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:02 smithi169 conmon[100500]: cluster 2023-08-28T02:07:59.099531+0000 osd.2 (osd.2 2023-08-28T02:08:02.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:02 smithi169 conmon[100500]: ) 1 : cluster [DBG] purged_snaps scrub starts 2023-08-28T02:08:02.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:02 smithi169 conmon[100500]: cluster 2023-08-28T02:07:59.099587+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-08-28T02:08:02.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:02 smithi169 conmon[100500]: 2023-08-28T02:08:02.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:02 smithi169 conmon[100500]: cluster 2023-08-28T02:08:02.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:02 smithi169 conmon[100500]: 2023-08-28T02:08:00.933231+0000 2023-08-28T02:08:02.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:02 smithi169 conmon[100500]: mgr.y (mgr.14146) 89 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-08-28T02:08:02.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:02 smithi169 conmon[100500]: audit 2023-08-28T02:08:02.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:02 smithi169 conmon[100500]: 2023-08-28T02:08:01. 2023-08-28T02:08:02.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:02 smithi169 conmon[100500]: 041753+0000 mon.a (mon.0) 330 : audit [DBG] 2023-08-28T02:08:02.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:02 smithi169 conmon[100500]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-28T02:08:02.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:02 smithi169 conmon[100500]: cluster 2023-08-28T02 2023-08-28T02:08:02.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:02 smithi169 conmon[100500]: :08:01.045808+0000 mon.a (mon. 2023-08-28T02:08:02.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:02 smithi169 conmon[100500]: 0) 331 : cluster [INF] osd.2 [v2:172.21.15.143:6818/2664999316,v1:172.21.15.143:6819/2664999316] boot 2023-08-28T02:08:02.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:02 smithi169 conmon[100500]: cluster 2023 2023-08-28T02:08:02.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:02 smithi169 conmon[100500]: -08-28T02:08:01. 2023-08-28T02:08:02.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:02 smithi169 conmon[100500]: 045847+0000 mon.a (mon.0) 332 : cluster [DBG] 2023-08-28T02:08:02.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:02 smithi169 conmon[100500]: osdmap e18: 3 total, 3 up, 3 in 2023-08-28T02:08:02.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:02 smithi169 conmon[100500]: audit 2023-08-28T02:08: 2023-08-28T02:08:02.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:02 smithi169 conmon[100500]: 01.053193+0000 mon.a (mon.0) 2023-08-28T02:08:02.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:02 smithi169 conmon[100500]: 333 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-28T02:08:02.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:02 smithi169 conmon[100500]: audit 2023 2023-08-28T02:08:02.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:02 smithi169 conmon[100500]: -08-28T02:08:01. 2023-08-28T02:08:02.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:02 smithi169 conmon[100500]: 053584+0000 mon.a (mon.0) 334 : audit [INF] 2023-08-28T02:08:02.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:02 smithi169 conmon[100500]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-08-28T02:08:02.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:02 smithi169 conmon[100500]: audit 2023-08-28T 2023-08-28T02:08:02.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:02 smithi169 conmon[100500]: 02:08:02.049547+0000 mon.a (mon. 2023-08-28T02:08:02.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:02 smithi169 conmon[100500]: 0) 335 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-08-28T02:08:02.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:02 smithi169 conmon[100500]: cluster 2023- 2023-08-28T02:08:02.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:02 smithi169 conmon[100500]: 08-28T02:08:02.049610 2023-08-28T02:08:02.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:02 smithi169 conmon[100500]: +0000 mon.a (mon.0) 336 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2023-08-28T02:08:02.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:02 smithi169 conmon[100500]: 2023-08-28T02:08:02.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:02 smithi169 conmon[100500]: audit 2023 2023-08-28T02:08:02.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:02 smithi169 conmon[100500]: -08-28T02:08:02 2023-08-28T02:08:02.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:02 smithi169 conmon[100500]: .050503+0000 mon.a (mon.0) 337 2023-08-28T02:08:02.741 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:02 smithi169 conmon[100500]: : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-08-28T02:08:03.694 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:03 smithi143 conmon[100258]: cephadm 2023-08-28T02:08:02.409748+0000 mgr.y (mgr.14146) 90 : cephadm [INF] Detected new or changed devices on smithi143 2023-08-28T02:08:03.694 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:03 smithi143 conmon[100258]: 2023-08-28T02:08:03.694 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:03 smithi143 conmon[100258]: audit 2023-08-28T02:08:02.414733+0000 mon.a (mon.0) 338 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:03.694 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:03 smithi143 conmon[100258]: audit 2023-08-28T02:08:02.415947+0000 mon.a (mon.0) 339 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:08:03.694 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:03 smithi143 conmon[100258]: audit 2023-08-28T02:08:02.416605+0000 mon.a (mon.0) 340 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:08:03.694 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:03 smithi143 conmon[100258]: audit 2023-08-28T02:08:02.417190+0000 mon.a (mon.0) 341 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:08:03.694 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:03 smithi143 conmon[100258]: cephadm 2023-08-28T02:08:02.417639+0000 mgr.y (mgr.14146) 91 : cephadm [INF] Adjusting osd_memory_target on smithi143 to 5381M 2023-08-28T02:08:03.695 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:03 smithi143 conmon[100258]: audit 2023-08-28T02:08:02.422209+0000 mon.a (mon.0) 342 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:03.695 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:03 smithi143 conmon[100258]: audit 2023-08-28T02:08:02.423210+0000 mon.a (mon.0) 343 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:08:03.695 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:03 smithi143 conmon[100258]: audit 2023-08-28T02:08:02.423873+0000 mon.a (mon.0) 344 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:08:03.695 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:03 smithi143 conmon[100258]: audit 2023-08-28T02:08:02.428898+0000 mon.a (mon.0) 345 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:03.695 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:03 smithi143 conmon[100258]: audit 2023-08-28T02:08:03.049137+0000 mon.a (mon.0) 346 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-08-28T02:08:03.695 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:03 smithi143 conmon[100258]: cluster 2023-08-28T02:08:03.049218+0000 mon.a (mon.0) 347 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2023-08-28T02:08:03.695 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:03 smithi143 conmon[109923]: cephadm 2023-08-28T02:08:02.409748+0000 mgr.y (mgr 2023-08-28T02:08:03.695 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:03 smithi143 conmon[109923]: .14146) 90 : cephadm [INF] Detected new or changed devices on smithi143 2023-08-28T02:08:03.695 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:03 smithi143 conmon[109923]: audit 2023-08-28T02:08:02.414733+0000 mon.a (mon.0) 338 : 2023-08-28T02:08:03.696 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:03 smithi143 conmon[109923]: audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:03.696 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:03 smithi143 conmon[109923]: audit 2023-08-28T02:08:02.415947+0000 mon.a 2023-08-28T02:08:03.696 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:03 smithi143 conmon[109923]: (mon.0) 339 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:08:03.696 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:03 smithi143 conmon[109923]: audit 2023-08- 2023-08-28T02:08:03.696 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:03 smithi143 conmon[109923]: 28T02:08:02.416605+0000 mon.a (mon.0) 340 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:08:03.696 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:03 smithi143 conmon[109923]: audit 2023-08-28T02:08:02.417190+0000 mon.a 2023-08-28T02:08:03.696 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:03 smithi143 conmon[109923]: (mon.0) 341 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:08:03.696 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:03 smithi143 conmon[109923]: cephadm 2023-08-28T02:08:02.417639+0000 mgr.y (mgr. 2023-08-28T02:08:03.696 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:03 smithi143 conmon[109923]: 14146) 91 : cephadm [INF] Adjusting osd_memory_target on smithi143 to 5381M 2023-08-28T02:08:03.696 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:03 smithi143 conmon[109923]: audit 2023-08-28T02:08:03.696 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:03 smithi143 conmon[109923]: 2023-08-28T02:08:02.422209+0000 mon.a (mon.0) 342 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:03.696 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:03 smithi143 conmon[109923]: 2023-08-28T02:08:03.696 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:03 smithi143 conmon[109923]: audit 2023-08-28T02:08 2023-08-28T02:08:03.697 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:03 smithi143 conmon[109923]: :02.423210+0000 mon.a (mon.0) 343 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:08:03.697 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:03 smithi143 conmon[109923]: audit 2023-08-28T02:08:02.423873+0000 mon.a 2023-08-28T02:08:03.697 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:03 smithi143 conmon[109923]: (mon.0) 344 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:08:03.697 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:03 smithi143 conmon[109923]: audit 2023-08-28T02:08:02.428898+0000 mon.a (mon 2023-08-28T02:08:03.697 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:03 smithi143 conmon[109923]: .0) 345 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:03.697 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:03 smithi143 conmon[109923]: audit 2023-08-28T02:08:03.697 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:03 smithi143 conmon[109923]: 2023-08-28T02:08:03.049137+0000 mon.a (mon.0) 346 : audit [INF] 2023-08-28T02:08:03.697 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:03 smithi143 conmon[109923]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-08-28T02:08:03.697 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:03 smithi143 conmon[109923]: cluster 2023-08-28T02:08:03.697 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:03 smithi143 conmon[109923]: 2023-08-28T02:08:03.049218+0000 mon.a (mon.0 2023-08-28T02:08:03.697 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:03 smithi143 conmon[109923]: ) 347 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2023-08-28T02:08:03.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:03 smithi169 conmon[100500]: cephadm 2023-08-28T02:08:02.409748+0000 mgr.y (mgr.14146) 2023-08-28T02:08:03.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:03 smithi169 conmon[100500]: 90 : cephadm [INF] Detected new or changed devices on smithi143 2023-08-28T02:08:03.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:03 smithi169 conmon[100500]: audit 2023-08-28T02:08:02.414733+0000 mon.a (mon.0) 338 : audit [INF] 2023-08-28T02:08:03.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:03 smithi169 conmon[100500]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:03.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:03 smithi169 conmon[100500]: audit 2023-08- 2023-08-28T02:08:03.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:03 smithi169 conmon[100500]: 28T02:08:02.415947+0000 mon.a (mon.0) 339 2023-08-28T02:08:03.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:03 smithi169 conmon[100500]: : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:08:03.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:03 smithi169 conmon[100500]: audit 2023-08-28T 2023-08-28T02:08:03.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:03 smithi169 conmon[100500]: 02:08:02.416605+0000 mon.a (mon 2023-08-28T02:08:03.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:03 smithi169 conmon[100500]: .0) 340 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:08:03.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:03 smithi169 conmon[100500]: audit 2023-08 2023-08-28T02:08:03.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:03 smithi169 conmon[100500]: -28T02:08:02.417190+0000 2023-08-28T02:08:03.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:03 smithi169 conmon[100500]: mon.a (mon.0) 341 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:08:03.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:03 smithi169 conmon[100500]: cephadm 2023 2023-08-28T02:08:03.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:03 smithi169 conmon[100500]: -08-28T02:08:02.417639 2023-08-28T02:08:03.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:03 smithi169 conmon[100500]: +0000 mgr.y (mgr.14146) 91 : cephadm [INF] Adjusting osd_memory_target on smithi143 to 5381M 2023-08-28T02:08:03.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:03 smithi169 conmon[100500]: audit 2023-08-28 2023-08-28T02:08:03.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:03 smithi169 conmon[100500]: T02:08:02.422209+0000 mon.a ( 2023-08-28T02:08:03.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:03 smithi169 conmon[100500]: mon.0) 342 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:03.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:03 smithi169 conmon[100500]: audit 2023-08-28T02:08:03.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:03 smithi169 conmon[100500]: 2023-08-28T02:08:02 2023-08-28T02:08:03.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:03 smithi169 conmon[100500]: .423210+0000 mon.a (mon.0) 343 : audit 2023-08-28T02:08:03.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:03 smithi169 conmon[100500]: [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:08:03.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:03 smithi169 conmon[100500]: audit 2023-08-28T02 2023-08-28T02:08:03.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:03 smithi169 conmon[100500]: :08:02.423873+0000 mon.a ( 2023-08-28T02:08:03.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:03 smithi169 conmon[100500]: mon.0) 344 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:08:03.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:03 smithi169 conmon[100500]: audit 2023-08-28T02:08:03.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:03 smithi169 conmon[100500]: 2023-08-28T02:08:02.428898 2023-08-28T02:08:03.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:03 smithi169 conmon[100500]: +0000 mon.a (mon.0) 345 : audit [INF] 2023-08-28T02:08:03.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:03 smithi169 conmon[100500]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:03.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:03 smithi169 conmon[100500]: audit 2023-08-28T02 2023-08-28T02:08:03.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:03 smithi169 conmon[100500]: :08:03.049137+0000 mon.a (mon.0 2023-08-28T02:08:03.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:03 smithi169 conmon[100500]: ) 346 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-08-28T02:08:03.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:03 smithi169 conmon[100500]: cluster 2023-08- 2023-08-28T02:08:03.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:03 smithi169 conmon[100500]: 28T02:08:03.049218+0000 mon.a 2023-08-28T02:08:03.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:03 smithi169 conmon[100500]: (mon.0) 347 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2023-08-28T02:08:04.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:04 smithi143 conmon[100258]: cluster 2023-08-28T02:08:02.933498+0000 mgr.y (mgr.14146) 92 : cluster [DBG] pgmap v69: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-28T02:08:04.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:04 smithi143 conmon[100258]: cluster 2023-08-28T02:08:04.054200+0000 mon.a (mon.0) 348 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2023-08-28T02:08:04.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:04 smithi143 conmon[100258]: audit 2023-08-28T02:08:04.054481+0000 mon.a (mon.0) 349 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-28T02:08:04.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:04 smithi143 conmon[100258]: audit 2023-08-28T02:08:04.056442+0000 mon.a (mon.0) 350 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-28T02:08:04.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:04 smithi143 conmon[100258]: audit 2023-08-28T02:08:04.057035+0000 mon.a (mon.0) 351 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:08:04.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:04 smithi143 conmon[109923]: cluster 2023-08-28T02:08:02.933498+0000 mgr.y (mgr.14146) 92 : cluster [DBG] pgmap v69: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-28T02:08:04.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:04 smithi143 conmon[109923]: cluster 2023-08-28T02:08:04.054200+0000 mon.a (mon.0) 348 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2023-08-28T02:08:04.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:04 smithi143 conmon[109923]: audit 2023-08-28T02:08:04.054481+0000 mon.a (mon.0) 349 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-28T02:08:04.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:04 smithi143 conmon[109923]: audit 2023-08-28T02:08:04.056442+0000 mon.a (mon.0) 350 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-28T02:08:04.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:04 smithi143 conmon[109923]: audit 2023-08-28T02:08:04.057035+0000 mon.a (mon.0) 351 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:08:04.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:04 smithi169 conmon[100500]: cluster 2023-08-28T02:08:02.933498+0000 mgr.y (mgr.14146) 92 2023-08-28T02:08:04.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:04 smithi169 conmon[100500]: : cluster [DBG] pgmap v69: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-28T02:08:04.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:04 smithi169 conmon[100500]: cluster 2023-08-28T02:08:04.054200+0000 mon.a (mon.0) 348 : 2023-08-28T02:08:04.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:04 smithi169 conmon[100500]: cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2023-08-28T02:08:04.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:04 smithi169 conmon[100500]: audit 2023-08-28T02:08:04.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:04 smithi169 conmon[100500]: 2023-08-28T02:08:04 2023-08-28T02:08:04.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:04 smithi169 conmon[100500]: .054481+0000 mon.a (mon.0) 349 : audit [DBG] 2023-08-28T02:08:04.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:04 smithi169 conmon[100500]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-28T02:08:04.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:04 smithi169 conmon[100500]: audit 2023 2023-08-28T02:08:04.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:04 smithi169 conmon[100500]: -08-28T02:08:04.056442+0000 2023-08-28T02:08:04.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:04 smithi169 conmon[100500]: mon.a (mon.0) 350 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-28T02:08:04.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:04 smithi169 conmon[100500]: audit 2023 2023-08-28T02:08:04.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:04 smithi169 conmon[100500]: -08-28T02:08:04. 2023-08-28T02:08:04.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:04 smithi169 conmon[100500]: 057035+0000 mon.a (mon.0) 351 : audit 2023-08-28T02:08:04.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:04 smithi169 conmon[100500]: [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:08:05.656 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:05 smithi143 conmon[100258]: audit 2023-08-28T02:08:04.051382+0000 mgr.y (mgr. 2023-08-28T02:08:05.656 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:05 smithi143 conmon[100258]: 14146) 93 : audit [DBG] from='client.14289 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi143:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:08:05.657 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:05 smithi143 conmon[109923]: audit 2023-08-28T02:08:04.051382 2023-08-28T02:08:05.657 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:05 smithi143 conmon[109923]: +0000 mgr.y (mgr.14146) 93 : audit [DBG] from='client.14289 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi143:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:08:05.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:05 smithi169 conmon[100500]: audit 2023-08-28T02:08:04.051382+0000 mgr.y ( 2023-08-28T02:08:05.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:05 smithi169 conmon[100500]: mgr.14146) 93 : audit [DBG] from='client.14289 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi143:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:08:06.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:06 smithi143 conmon[100258]: cluster 2023-08-28T02:08:04.933806+0000 mgr.y (mgr. 2023-08-28T02:08:06.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:06 smithi143 conmon[100258]: 14146) 94 : cluster [DBG] pgmap v72: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-28T02:08:06.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:06 smithi143 conmon[100258]: audit 2023-08-28T02:08:06.340215+0000 mon.a (mon.0) 352 : audit [INF] from='client.? 172.21.15.143:0/3609200390' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dd6abc8a-54a9-4e08-ba81-0b894f24d84a"}]: dispatch 2023-08-28T02:08:06.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:06 smithi143 conmon[100258]: audit 2023-08-28T02:08:06.349453+0000 mon.a (mon.0) 353 : audit [INF] from='client.? 172.21.15.143:0/3609200390' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "dd6abc8a-54a9-4e08-ba81-0b894f24d84a"}]': finished 2023-08-28T02:08:06.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:06 smithi143 conmon[100258]: cluster 2023-08-28T02:08:06.349514+0000 mon.a (mon.0) 354 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2023-08-28T02:08:06.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:06 smithi143 conmon[100258]: audit 2023-08-28T02:08:06.349632+0000 mon.a (mon.0) 355 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-28T02:08:06.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:06 smithi143 conmon[109923]: cluster 2023-08-28T02:08:04.933806+0000 mgr.y (mgr.14146 2023-08-28T02:08:06.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:06 smithi143 conmon[109923]: ) 94 : cluster [DBG] pgmap v72: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-28T02:08:06.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:06 smithi143 conmon[109923]: audit 2023-08-28T02:08:06.340215+0000 mon.a (mon.0) 352 : audit [INF] from='client.? 172.21.15.143:0/3609200390' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dd6abc8a-54a9-4e08-ba81-0b894f24d84a"}]: dispatch 2023-08-28T02:08:06.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:06 smithi143 conmon[109923]: audit 2023-08-28T02:08:06.349453+0000 mon.a (mon.0) 353 : audit [INF] from='client.? 172.21.15.143:0/3609200390' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "dd6abc8a-54a9-4e08-ba81-0b894f24d84a"}]': finished 2023-08-28T02:08:06.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:06 smithi143 conmon[109923]: cluster 2023-08-28T02:08:06.349514+0000 mon.a (mon.0) 354 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2023-08-28T02:08:06.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:06 smithi143 conmon[109923]: audit 2023-08-28T02:08:06.349632+0000 mon.a (mon.0) 355 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-28T02:08:06.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:06 smithi169 conmon[100500]: cluster 2023-08-28T02:08:04.933806+0000 mgr.y (mgr.14146) 94 : cluster 2023-08-28T02:08:06.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:06 smithi169 conmon[100500]: [DBG] pgmap v72: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-28T02:08:06.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:06 smithi169 conmon[100500]: audit 2023-08-28T02:08:06.340215+0000 mon.a (mon.0) 352 : audit [INF] from='client.? 172.21.15.143:0/3609200390' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dd6abc8a-54a9-4e08-ba81-0b894f24d84a"}]: dispatch 2023-08-28T02:08:06.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:06 smithi169 conmon[100500]: audit 2023-08-28T02:08 2023-08-28T02:08:06.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:06 smithi169 conmon[100500]: :06.349453+0000 mon.a (mon.0) 353 : audit 2023-08-28T02:08:06.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:06 smithi169 conmon[100500]: [INF] from='client.? 172.21.15.143:0/3609200390' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "dd6abc8a-54a9-4e08-ba81-0b894f24d84a"}]': finished 2023-08-28T02:08:06.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:06 smithi169 conmon[100500]: cluster 2023-08-28T 2023-08-28T02:08:06.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:06 smithi169 conmon[100500]: 02:08:06.349514+0000 mon.a (mon. 2023-08-28T02:08:06.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:06 smithi169 conmon[100500]: 0) 354 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2023-08-28T02:08:06.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:06 smithi169 conmon[100500]: audit 2023- 2023-08-28T02:08:06.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:06 smithi169 conmon[100500]: 08-28T02:08:06.349632+0000 2023-08-28T02:08:06.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:06 smithi169 conmon[100500]: mon.a (mon.0) 355 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-28T02:08:07.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:07 smithi143 conmon[100258]: audit 2023-08-28T02:08:06.936919+0000 mon.a (mon.0 2023-08-28T02:08:07.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:07 smithi143 conmon[100258]: ) 356 : audit [DBG] from='client.? 172.21.15.143:0/1831013488' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-28T02:08:07.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:07 smithi143 conmon[109923]: audit 2023-08-28T02:08:06.936919+0000 mon.a (mon.0) 356 : audit [DBG] from='client.? 172.21.15.143:0/1831013488' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-28T02:08:07.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:07 smithi143 conmon[109923]: 2023-08-28T02:08:07.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:07 smithi169 conmon[100500]: audit 2023-08-28T02:08:06.936919+0000 mon.a ( 2023-08-28T02:08:07.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:07 smithi169 conmon[100500]: mon.0) 356 : audit [DBG] from='client.? 172.21.15.143:0/1831013488' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-28T02:08:08.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:08 smithi143 conmon[109923]: cluster 2023-08-28T02:08:06.934150+0000 2023-08-28T02:08:08.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:08 smithi143 conmon[109923]: mgr.y (mgr.14146) 95 : cluster [DBG] pgmap v74: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-28T02:08:08.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:08 smithi143 conmon[100258]: cluster 2023-08-28T02:08:06.934150+0000 mgr.y (mgr.14146) 95 : cluster 2023-08-28T02:08:08.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:08 smithi143 conmon[100258]: [DBG] pgmap v74: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-28T02:08:08.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:08 smithi169 conmon[100500]: cluster 2023-08-28T02:08:06.934150+0000 mgr.y (mgr. 2023-08-28T02:08:08.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:08 smithi169 conmon[100500]: 14146) 95 : cluster [DBG] pgmap v74: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-28T02:08:10.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:10 smithi169 conmon[100500]: cluster 2023-08-28T02:08:08.934596+0000 mgr.y (mgr.14146) 96 2023-08-28T02:08:10.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:10 smithi169 conmon[100500]: : cluster [DBG] pgmap v75: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-28T02:08:10.873 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:10 smithi143 conmon[100258]: cluster 2023-08-28T02:08:08.934596+0000 mgr.y (mgr.14146) 96 : cluster [DBG] pgmap v75: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-28T02:08:10.873 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:10 smithi143 conmon[109923]: cluster 2023-08-28T02:08:08.934596+0000 mgr.y (mgr.14146) 96 : cluster [DBG] pgmap v75: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-28T02:08:12.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:12 smithi169 conmon[100500]: cluster 2023-08-28T02:08:10.935150+0000 mgr.y (mgr 2023-08-28T02:08:12.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:12 smithi169 conmon[100500]: .14146) 97 : cluster [DBG] pgmap v76: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-28T02:08:12.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:12 smithi143 conmon[100258]: cluster 2023-08-28T02:08:10.935150+0000 mgr.y (mgr.14146) 97 : cluster [DBG] pgmap v76: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-28T02:08:12.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:12 smithi143 conmon[109923]: cluster 2023-08-28T02:08:10.935150+0000 mgr.y (mgr.14146) 97 : cluster [DBG] pgmap v76: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-28T02:08:13.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:13 smithi143 conmon[100258]: audit 2023-08-28T02:08:12.564935+0000 mon.a (mon. 2023-08-28T02:08:13.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:13 smithi143 conmon[100258]: 0) 357 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-08-28T02:08:13.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:13 smithi143 conmon[100258]: audit 2023-08-28T02:08:12.565578+0000 mon.a (mon.0) 358 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:08:13.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:13 smithi143 conmon[100258]: cephadm 2023-08-28T02:08:12.566075+0000 mgr.y (mgr.14146) 98 : cephadm [INF] Deploying daemon osd.3 on smithi143 2023-08-28T02:08:13.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:13 smithi143 conmon[109923]: audit 2023-08-28T02:08:12.564935+0000 2023-08-28T02:08:13.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:13 smithi143 conmon[109923]: mon.a (mon.0) 357 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-08-28T02:08:13.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:13 smithi143 conmon[109923]: audit 2023-08-28T02:08:12.565578+0000 mon.a (mon.0) 358 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:08:13.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:13 smithi143 conmon[109923]: cephadm 2023-08- 2023-08-28T02:08:13.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:13 smithi143 conmon[109923]: 28T02:08:12.566075+0000 mgr.y (mgr.14146) 98 : cephadm [INF] Deploying daemon osd.3 on smithi143 2023-08-28T02:08:13.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:13 smithi169 conmon[100500]: audit 2023-08-28T02:08:12.564935+0000 mon.a (mon.0 2023-08-28T02:08:13.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:13 smithi169 conmon[100500]: ) 357 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-08-28T02:08:13.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:13 smithi169 conmon[100500]: audit 2023-08-28T02:08:12.565578+0000 mon.a (mon.0 2023-08-28T02:08:13.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:13 smithi169 conmon[100500]: ) 358 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:08:13.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:13 smithi169 conmon[100500]: cephadm 2023-08-28T02:08:13.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:13 smithi169 conmon[100500]: 2023-08-28T02:08:12.566075+0000 mgr.y ( 2023-08-28T02:08:13.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:13 smithi169 conmon[100500]: mgr.14146) 98 : cephadm [INF] Deploying daemon osd.3 on smithi143 2023-08-28T02:08:14.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:14 smithi143 conmon[100258]: cluster 2023-08-28T02:08:12.935436+0000 mgr.y (mgr.14146) 2023-08-28T02:08:14.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:14 smithi143 conmon[100258]: 99 : cluster [DBG] pgmap v77: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-28T02:08:14.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:14 smithi143 conmon[109923]: cluster 2023-08-28T02:08:12.935436+0000 2023-08-28T02:08:14.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:14 smithi143 conmon[109923]: mgr.y (mgr.14146) 99 : cluster [DBG] pgmap v77: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-28T02:08:14.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:14 smithi169 conmon[100500]: cluster 2023-08-28T02:08:12.935436+0000 mgr.y (mgr. 2023-08-28T02:08:14.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:14 smithi169 conmon[100500]: 14146) 99 : cluster [DBG] pgmap v77: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-28T02:08:16.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:16 smithi143 conmon[100258]: cluster 2023-08-28T02:08:14.935732+0000 mgr.y 2023-08-28T02:08:16.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:16 smithi143 conmon[100258]: (mgr.14146) 100 : cluster [DBG] pgmap v78: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-28T02:08:16.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:16 smithi143 conmon[109923]: cluster 2023-08-28T02:08:14.935732+0000 2023-08-28T02:08:16.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:16 smithi143 conmon[109923]: mgr.y (mgr.14146) 100 : cluster [DBG] pgmap v78: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-28T02:08:16.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:16 smithi169 conmon[100500]: cluster 2023-08-28T02:08:14.935732+0000 mgr.y (mgr.14146) 2023-08-28T02:08:16.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:16 smithi169 conmon[100500]: 100 : cluster [DBG] pgmap v78: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-28T02:08:18.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:18 smithi143 conmon[100258]: cluster 2023-08-28T02:08:16.936087+0000 2023-08-28T02:08:18.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:18 smithi143 conmon[100258]: mgr.y (mgr.14146) 101 : cluster [DBG] pgmap v79: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-28T02:08:18.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:18 smithi143 conmon[100258]: audit 2023-08-28T02:08:17.509318+0000 mon.a (mon.0) 359 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:18.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:18 smithi143 conmon[100258]: audit 2023-08-28T02:08:17.523931+0000 mon.a (mon.0) 360 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:08:18.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:18 smithi143 conmon[109923]: cluster 2023-08-28T02:08:16 2023-08-28T02:08:18.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:18 smithi143 conmon[109923]: .936087+0000 mgr.y (mgr.14146) 101 : cluster [DBG] pgmap v79: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-28T02:08:18.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:18 smithi143 conmon[109923]: audit 2023-08-28T02: 2023-08-28T02:08:18.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:18 smithi143 conmon[109923]: 08:17.509318+0000 mon.a (mon.0) 2023-08-28T02:08:18.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:18 smithi143 conmon[109923]: 359 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:18.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:18 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:08:18.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:18 smithi143 conmon[109923]: 08-28T02:08:17.523931 2023-08-28T02:08:18.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:18 smithi143 conmon[109923]: +0000 mon.a (mon.0) 360 2023-08-28T02:08:18.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:18 smithi143 conmon[109923]: : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:08:18.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:18 smithi169 conmon[100500]: cluster 2023-08-28T02:08:16.936087+0000 mgr.y (mgr.14146) 101 : 2023-08-28T02:08:18.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:18 smithi169 conmon[100500]: cluster [DBG] pgmap v79: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-28T02:08:18.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:18 smithi169 conmon[100500]: audit 2023-08-28T02:08:17.509318+0000 mon.a (mon.0) 359 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:18.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:18 smithi169 conmon[100500]: audit 2023-08-28T02 2023-08-28T02:08:18.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:18 smithi169 conmon[100500]: :08:17.523931+0000 mon.a (mon.0) 2023-08-28T02:08:18.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:18 smithi169 conmon[100500]: 360 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:08:20.534 INFO:teuthology.orchestra.run.smithi143.stdout:Created osd(s) 3 on host 'smithi143' 2023-08-28T02:08:20.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:20 smithi143 conmon[100258]: cluster 2023-08-28T02:08:18.936472+0000 mgr.y (mgr.14146) 102 : cluster [DBG] pgmap v80: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-28T02:08:20.956 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:20 smithi143 conmon[100258]: audit 2023-08-28T02:08:19.588469+0000 mon.a (mon.0) 361 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:20.956 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:20 smithi143 conmon[100258]: audit 2023-08-28T02:08:19.909178+0000 mon.a (mon.0) 362 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:08:20.956 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:20 smithi143 conmon[100258]: audit 2023-08-28T02:08:19.909855+0000 mon.a (mon.0) 363 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:08:20.956 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:20 smithi143 conmon[100258]: audit 2023-08-28T02:08:19.914726+0000 mon.a (mon.0) 364 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:20.956 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:20 smithi143 conmon[109923]: cluster 2023-08-28T02:08:18.936472+0000 mgr.y (mgr.14146) 102 : cluster [DBG] pgmap v80: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-28T02:08:20.956 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:20 smithi143 conmon[109923]: audit 2023-08-28T02:08:19.588469+0000 mon.a (mon.0) 361 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:20.956 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:20 smithi143 conmon[109923]: audit 2023-08-28T02:08:19.909178+0000 mon.a (mon.0) 362 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:08:20.956 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:20 smithi143 conmon[109923]: audit 2023-08-28T02:08:19.909855+0000 mon.a (mon.0) 363 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:08:20.957 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:20 smithi143 conmon[109923]: audit 2023-08-28T02:08:19.914726+0000 mon.a (mon.0) 364 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:20.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:20 smithi169 conmon[100500]: cluster 2023-08-28T02:08:18.936472+0000 mgr.y (mgr. 2023-08-28T02:08:20.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:20 smithi169 conmon[100500]: 14146) 102 : cluster [DBG] pgmap v80: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-28T02:08:20.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:20 smithi169 conmon[100500]: audit 2023-08-28T02:08:19.588469+0000 mon.a (mon.0) 361 : audit [INF] 2023-08-28T02:08:20.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:20 smithi169 conmon[100500]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:20.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:20 smithi169 conmon[100500]: audit 2023-08-28T 2023-08-28T02:08:20.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:20 smithi169 conmon[100500]: 02:08:19.909178+0000 mon.a (mon.0) 362 : audit [DBG] 2023-08-28T02:08:20.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:20 smithi169 conmon[100500]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:08:20.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:20 smithi169 conmon[100500]: audit 2023-08-28T02: 2023-08-28T02:08:20.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:20 smithi169 conmon[100500]: 08:19.909855+0000 mon.a (mon.0) 363 2023-08-28T02:08:20.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:20 smithi169 conmon[100500]: : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:08:20.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:20 smithi169 conmon[100500]: audit 2023-08-28 2023-08-28T02:08:20.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:20 smithi169 conmon[100500]: T02:08:19.914726+0000 mon.a ( 2023-08-28T02:08:20.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:20 smithi169 conmon[100500]: mon.0) 364 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:21.187 DEBUG:teuthology.orchestra.run.smithi143:osd.3> sudo journalctl -f -n 0 -u ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@osd.3.service 2023-08-28T02:08:21.328 INFO:tasks.cephadm:Deploying osd.4 on smithi169 with /dev/vg_nvme/lv_4... 2023-08-28T02:08:21.328 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-08-28T02:08:21.358 INFO:journalctl@ceph.osd.3.smithi143.stdout:-- Logs begin at Mon 2023-08-28 01:49:41 UTC. -- 2023-08-28T02:08:21.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:21 smithi143 conmon[100258]: audit 2023-08-28T02:08:20.531391+0000 mon.a (mon.0 2023-08-28T02:08:21.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:21 smithi143 conmon[100258]: ) 365 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:08:21.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:21 smithi143 conmon[100258]: audit 2023-08-28T02:08:20.534480+0000 mon.a (mon.0) 366 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:21.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:21 smithi143 conmon[100258]: audit 2023-08-28T02:08:20.722436+0000 mon.c (mon.2) 7 : audit [INF] from='osd.3 [v2:172.21.15.143:6826/1468263600,v1:172.21.15.143:6827/1468263600]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-08-28T02:08:21.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:21 smithi143 conmon[100258]: audit 2023-08-28T02:08:20.722802+0000 mon.a (mon.0) 367 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-08-28T02:08:21.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:21 smithi143 conmon[109923]: audit 2023-08-28T02:08: 2023-08-28T02:08:21.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:21 smithi143 conmon[109923]: 20.531391+0000 mon.a (mon.0) 365 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:08:21.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:21 smithi143 conmon[109923]: audit 2023-08-28T02:08:20.534480+0000 mon.a (mon.0) 366 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:21.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:21 smithi143 conmon[109923]: audit 2023-08-28T02:08:20.722436+0000 mon.c (mon.2) 7 : audit [INF] from='osd.3 [v2:172.21.15.143:6826/1468263600,v1:172.21.15.143:6827/1468263600]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-08-28T02:08:21.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:21 smithi143 conmon[109923]: audit 2023-08-28T02:08:20.722802+0000 mon.a (mon.0) 367 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-08-28T02:08:21.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:21 smithi169 conmon[100500]: audit 2023-08-28T02:08:20.531391+0000 mon.a (mon.0) 365 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:08:21.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:21 smithi169 conmon[100500]: audit 2023-08-28T02:08:20.534480+0000 mon.a (mon.0) 366 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:21.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:21 smithi169 conmon[100500]: audit 2023-08-28T02:08:20.722436+0000 mon.c (mon.2) 7 : audit [INF] from='osd.3 [v2:172.21.15.143:6826/1468263600,v1:172.21.15.143:6827/1468263600]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-08-28T02:08:21.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:21 smithi169 conmon[100500]: audit 2023-08-28T02:08:20.722802+0000 mon.a (mon.0) 367 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-08-28T02:08:22.874 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:22 smithi169 conmon[100500]: cluster 2023-08-28T02:08:20.936842+0000 mgr.y (mgr.14146) 103 : cluster [DBG] pgmap v81: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-28T02:08:22.874 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:22 smithi169 conmon[100500]: audit 2023-08-28T02:08:21.538972+0000 mon.a (mon.0) 368 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-08-28T02:08:22.874 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:22 smithi169 conmon[100500]: cluster 2023-08-28T02:08:21.539058+0000 mon.a (mon.0) 369 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-08-28T02:08:22.874 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:22 smithi169 conmon[100500]: audit 2023-08-28T02:08:21.539171+0000 mon.a (mon.0) 370 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-28T02:08:22.874 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:22 smithi169 conmon[100500]: audit 2023-08-28T02:08:21.539558+0000 mon.c (mon.2) 8 : audit [INF] from='osd.3 [v2:172.21.15.143:6826/1468263600,v1:172.21.15.143:6827/1468263600]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]: dispatch 2023-08-28T02:08:22.874 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:22 smithi169 conmon[100500]: audit 2023-08-28T02:08:21.539875+0000 mon.a (mon.0) 371 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]: dispatch 2023-08-28T02:08:22.950 INFO:journalctl@ceph.osd.3.smithi143.stdout:Aug 28 02:08:22 smithi143 conmon[132739]: debug 2023-08-28T02:08:22.545+0000 7fa1cf823700 -1 osd.3 0 waiting for initial osdmap 2023-08-28T02:08:22.950 INFO:journalctl@ceph.osd.3.smithi143.stdout:Aug 28 02:08:22 smithi143 conmon[132739]: debug 2023-08-28T02:08:22.551+0000 7fa1c9f9b700 -1 osd.3 24 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-08-28T02:08:22.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:22 smithi143 conmon[100258]: cluster 2023-08-28T02:08:20.936842+0000 mgr.y (mgr.14146) 103 : cluster [DBG] pgmap v81: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-28T02:08:22.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:22 smithi143 conmon[100258]: audit 2023-08-28T02:08:21.538972+0000 mon.a (mon.0) 368 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-08-28T02:08:22.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:22 smithi143 conmon[100258]: cluster 2023-08-28T02:08:21.539058+0000 mon.a (mon.0) 369 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-08-28T02:08:22.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:22 smithi143 conmon[100258]: audit 2023-08-28T02:08:21.539171+0000 mon.a (mon.0) 370 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-28T02:08:22.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:22 smithi143 conmon[100258]: audit 2023-08-28T02:08:21.539558+0000 mon.c (mon.2) 8 : audit [INF] from='osd.3 [v2:172.21.15.143:6826/1468263600,v1:172.21.15.143:6827/1468263600]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]: dispatch 2023-08-28T02:08:22.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:22 smithi143 conmon[100258]: audit 2023-08-28T02:08:21.539875+0000 mon.a (mon.0) 371 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]: dispatch 2023-08-28T02:08:22.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:22 smithi143 conmon[109923]: cluster 2023-08-28T02:08:20.936842+0000 mgr.y (mgr.14146) 103 : cluster [DBG] pgmap v81: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-28T02:08:22.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:22 smithi143 conmon[109923]: audit 2023-08-28T02:08:21.538972+0000 mon.a (mon.0) 368 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-08-28T02:08:22.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:22 smithi143 conmon[109923]: cluster 2023-08-28T02:08:21.539058+0000 mon.a (mon.0) 369 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-08-28T02:08:22.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:22 smithi143 conmon[109923]: audit 2023-08-28T02:08:21.539171+0000 mon.a (mon.0) 370 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-28T02:08:22.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:22 smithi143 conmon[109923]: audit 2023-08-28T02:08:21.539558+0000 mon.c (mon.2) 8 : audit [INF] from='osd.3 [v2:172.21.15.143:6826/1468263600,v1:172.21.15.143:6827/1468263600]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]: dispatch 2023-08-28T02:08:22.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:22 smithi143 conmon[109923]: audit 2023-08-28T02:08:21.539875+0000 mon.a (mon.0) 371 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]: dispatch 2023-08-28T02:08:23.358 INFO:teuthology.orchestra.run.smithi169.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-08-28T02:08:23.358 INFO:teuthology.orchestra.run.smithi169.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-08-28T02:08:23.358 INFO:teuthology.orchestra.run.smithi169.stderr: stderr: 10+0 records in 2023-08-28T02:08:23.358 INFO:teuthology.orchestra.run.smithi169.stderr:10+0 records out 2023-08-28T02:08:23.358 INFO:teuthology.orchestra.run.smithi169.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0127908 s, 820 MB/s 2023-08-28T02:08:23.358 INFO:teuthology.orchestra.run.smithi169.stderr:--> Zapping successful for: 2023-08-28T02:08:23.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:23 smithi169 conmon[100500]: audit 2023-08-28T02:08:22.542023+0000 mon.a (mon.0) 372 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]': finished 2023-08-28T02:08:23.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:23 smithi169 conmon[100500]: cluster 2023-08-28T02:08:22.542099+0000 mon.a (mon.0) 373 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-08-28T02:08:23.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:23 smithi169 conmon[100500]: audit 2023-08-28T02:08:22.542535+0000 mon.a (mon.0) 374 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-28T02:08:23.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:23 smithi169 conmon[100500]: audit 2023-08-28T02:08:22.546672+0000 mon.a (mon.0) 375 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-28T02:08:23.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:23 smithi143 conmon[100258]: audit 2023-08-28T02:08:22.542023+0000 mon.a (mon. 2023-08-28T02:08:23.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:23 smithi143 conmon[100258]: 0) 372 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]': finished 2023-08-28T02:08:23.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:23 smithi143 conmon[100258]: cluster 2023-08-28T02:08:22.542099+0000 mon.a (mon.0) 373 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-08-28T02:08:23.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:23 smithi143 conmon[100258]: audit 2023-08-28T02:08:22.542535+0000 mon.a (mon.0) 374 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-28T02:08:23.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:23 smithi143 conmon[100258]: audit 2023-08-28T02:08:22.546672+0000 mon.a (mon.0) 375 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-28T02:08:23.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:23 smithi143 conmon[109923]: audit 2023-08-28T02:08:22.542023+0000 mon.a (mon.0) 2023-08-28T02:08:23.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:23 smithi143 conmon[109923]: 372 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]': finished 2023-08-28T02:08:23.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:23 smithi143 conmon[109923]: cluster 2023-08-28T02:08:22.542099+0000 mon.a (mon.0) 373 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-08-28T02:08:23.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:23 smithi143 conmon[109923]: 2023-08-28T02:08:23.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:23 smithi143 conmon[109923]: audit 2023-08-28T02:08:22.542535+0000 mon.a (mon.0) 374 : audit 2023-08-28T02:08:23.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:23 smithi143 conmon[109923]: [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-28T02:08:23.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:23 smithi143 conmon[109923]: audit 2023-08- 2023-08-28T02:08:23.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:23 smithi143 conmon[109923]: 28T02:08:22.546672+0000 mon.a (mon.0) 375 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-28T02:08:24.055 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph orch daemon add osd smithi169:vg_nvme/lv_4 2023-08-28T02:08:24.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:24 smithi169 conmon[100500]: cluster 2023-08-28T02:08:21.737484+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2023-08-28T02:08:24.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:24 smithi169 conmon[100500]: cluster 2023-08-28T02:08 2023-08-28T02:08:24.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:24 smithi169 conmon[100500]: :21.737542+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2023-08-28T02:08:24.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:24 smithi169 conmon[100500]: cluster 2023-08-28T02:08:22.937127+0000 mgr.y (mgr.14146) 104 : cluster [DBG] pgmap v84: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-28T02:08:24.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:24 smithi169 conmon[100500]: audit 2023-08-28T02:08:23.546945+0000 mon.a (mon.0) 376 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-28T02:08:24.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:24 smithi169 conmon[100500]: cluster 2023-08-28T02:08:23.549940+0000 mon.a (mon.0) 377 : cluster [INF] osd.3 [v2:172.21.15.143:6826/1468263600,v1:172.21.15.143:6827/1468263600] boot 2023-08-28T02:08:24.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:24 smithi169 conmon[100500]: cluster 2023-08-28T02:08:23.549980+0000 mon.a (mon.0) 378 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-08-28T02:08:24.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:24 smithi169 conmon[100500]: audit 2023-08-28T02:08:23.550248+0000 mon.a (mon.0) 379 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-28T02:08:24.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:24 smithi169 conmon[100500]: audit 2023-08-28T02:08:23.843562+0000 mon.a (mon.0) 380 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:24.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:24 smithi169 conmon[100500]: audit 2023-08-28T02:08:23.844481+0000 mon.a (mon.0) 381 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:08:24.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:24 smithi169 conmon[100500]: audit 2023-08-28T02:08:23.845111+0000 mon.a (mon.0) 382 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:08:24.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:24 smithi169 conmon[100500]: audit 2023-08-28T02:08:23.845661+0000 mon.a (mon.0) 383 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:08:24.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:24 smithi169 conmon[100500]: audit 2023-08-28T02:08:23.846228+0000 mon.a (mon.0) 384 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:08:24.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:24 smithi169 conmon[100500]: audit 2023-08-28T02:08:23.853305+0000 mon.a (mon.0) 385 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:24.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:24 smithi169 conmon[100500]: audit 2023-08-28T02:08:23.854241+0000 mon.a (mon.0) 386 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:08:24.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:24 smithi169 conmon[100500]: audit 2023-08-28T02:08:23.854796+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:08:24.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:24 smithi169 conmon[100500]: audit 2023-08-28T02:08:23.862550+0000 mon.a (mon.0) 388 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:24.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[100258]: cluster 2023-08-28T02:08:21.737484+0000 osd.3 (osd.3 2023-08-28T02:08:24.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[100258]: ) 1 : cluster [DBG] purged_snaps scrub starts 2023-08-28T02:08:24.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[100258]: cluster 2023-08-28T02:08:21.737542+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2023-08-28T02:08:24.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[100258]: cluster 2023-08-28T02:08:22.937127+0000 mgr.y (mgr.14146) 104 : cluster [DBG] pgmap v84: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-28T02:08:24.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[100258]: audit 2023-08-28T02:08:23.546945+0000 mon.a (mon.0) 376 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-28T02:08:24.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[100258]: cluster 2023-08-28T02:08:23.549940+0000 mon.a (mon.0) 377 : cluster [INF] osd.3 [v2:172.21.15.143:6826/1468263600,v1:172.21.15.143:6827/1468263600] boot 2023-08-28T02:08:24.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[100258]: cluster 2023-08-28T02:08:23.549980+0000 mon.a (mon.0) 378 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-08-28T02:08:24.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[100258]: audit 2023-08-28T02:08:23.550248+0000 mon.a (mon.0) 379 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-28T02:08:24.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[100258]: audit 2023-08-28T02:08:23.843562+0000 mon.a 2023-08-28T02:08:24.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[100258]: (mon.0) 380 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:24.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[100258]: audit 2023-08-28T02:08:23.844481+0000 mon.a (mon.0) 381 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:08:24.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[100258]: audit 2023-08-28T02:08:23.845111+0000 mon.a (mon.0) 382 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:08:24.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[100258]: audit 2023-08-28T02:08:23.845661+0000 mon.a (mon.0) 383 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:08:24.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[100258]: audit 2023-08-28T02:08:23.846228+0000 2023-08-28T02:08:24.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[100258]: mon.a (mon.0) 384 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:08:24.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[100258]: audit 2023-08-28T02:08:23.853305+0000 mon.a (mon.0 2023-08-28T02:08:24.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[100258]: ) 385 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:24.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[100258]: audit 2023-08-28T02:08:23.854241+0000 mon.a (mon.0) 386 2023-08-28T02:08:24.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[100258]: : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:08:24.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[100258]: audit 2023-08-28T02:08:23.854796+0000 mon.a (mon.0) 387 2023-08-28T02:08:24.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[100258]: : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:08:24.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[100258]: audit 2023-08-28T02:08:23.862550+0000 mon.a (mon.0 2023-08-28T02:08:24.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[100258]: ) 388 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:24.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[109923]: cluster 2023-08-28T02:08:21.737484+0000 osd.3 (osd 2023-08-28T02:08:24.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[109923]: .3) 1 : cluster [DBG] purged_snaps scrub starts 2023-08-28T02:08:24.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[109923]: cluster 2023-08-28T02:08:21.737542+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2023-08-28T02:08:24.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[109923]: cluster 2023-08-28T02:08:22.937127+0000 mgr.y (mgr.14146) 104 : cluster 2023-08-28T02:08:24.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[109923]: [DBG] pgmap v84: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-08-28T02:08:24.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[109923]: audit 2023-08-28T02:08:23.546945+0000 mon.a (mon 2023-08-28T02:08:24.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[109923]: .0) 376 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-28T02:08:24.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[109923]: cluster 2023-08-28T02:08 2023-08-28T02:08:24.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[109923]: :23.549940+0000 mon.a (mon.0) 377 : cluster [INF] osd.3 [v2:172.21.15.143:6826/1468263600,v1:172.21.15.143:6827/1468263600] boot 2023-08-28T02:08:24.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[109923]: cluster 2023-08-28T02:08:24.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[109923]: 2023-08-28T02:08:23.549980+0000 mon.a (mon.0) 2023-08-28T02:08:24.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[109923]: 378 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-08-28T02:08:24.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[109923]: audit 2023-08-28T02 2023-08-28T02:08:24.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[109923]: :08:23.550248+0000 mon.a (mon.0) 379 : audit [DBG] 2023-08-28T02:08:24.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[109923]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-28T02:08:24.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[109923]: audit 2023-08-28T02:08:23. 2023-08-28T02:08:24.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[109923]: 843562+0000 mon.a (mon.0) 380 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:24.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[109923]: audit 2023-08-28T02:08:23.844481+0000 mon.a (mon.0) 381 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:08:24.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[109923]: audit 2023-08-28T02:08:23.845111+0000 mon.a (mon.0) 382 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:08:24.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[109923]: audit 2023-08-28T 2023-08-28T02:08:24.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[109923]: 02:08:23.845661+0000 mon.a ( 2023-08-28T02:08:24.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[109923]: mon.0) 383 : audit [INF] 2023-08-28T02:08:24.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[109923]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:08:24.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[109923]: audit 2023-08-28T02 2023-08-28T02:08:24.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[109923]: :08:23.846228+0000 mon.a (mon.0 2023-08-28T02:08:24.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[109923]: ) 384 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:08:24.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:08:24.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[109923]: 08-28T02:08:23.853305+0000 2023-08-28T02:08:24.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[109923]: mon.a (mon.0) 385 : audit 2023-08-28T02:08:24.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[109923]: [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:24.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[109923]: audit 2023-08 2023-08-28T02:08:24.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[109923]: -28T02:08:23.854241+0000 mon.a (mon.0) 386 2023-08-28T02:08:24.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[109923]: : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:08:24.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[109923]: audit 2023-08-28T02: 2023-08-28T02:08:24.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[109923]: 08:23.854796+0000 mon.a (mon.0) 2023-08-28T02:08:24.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[109923]: 387 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:08:24.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[109923]: audit 2023-08-28T 2023-08-28T02:08:24.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[109923]: 02:08:23.862550+0000 mon.a (mon.0) 388 : audit [INF] 2023-08-28T02:08:24.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:24 smithi143 conmon[109923]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:25.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:25 smithi143 conmon[100258]: cephadm 2023-08-28T02:08:23.836642+0000 mgr.y (mgr.14146) 105 : cephadm [INF] Detected new or changed devices on smithi143 2023-08-28T02:08:25.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:25 smithi143 conmon[100258]: cephadm 2023-08-28T02:08:23.846644+0000 mgr.y (mgr.14146) 106 : cephadm [INF] Adjusting osd_memory_target on smithi143 to 4036M 2023-08-28T02:08:25.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:25 smithi143 conmon[100258]: cluster 2023-08-28T02:08:24.563583+0000 mon.a (mon.0) 389 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2023-08-28T02:08:25.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:25 smithi143 conmon[109923]: cephadm 2023-08-28T02:08:23.836642+0000 mgr.y (mgr.14146) 105 : cephadm [INF] Detected new or changed devices on smithi143 2023-08-28T02:08:25.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:25 smithi143 conmon[109923]: 2023-08-28T02:08:25.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:25 smithi143 conmon[109923]: cephadm 2023-08-28T02:08:23.846644+0000 mgr.y (mgr.14146) 106 : cephadm [INF] 2023-08-28T02:08:25.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:25 smithi143 conmon[109923]: Adjusting osd_memory_target on smithi143 to 4036M 2023-08-28T02:08:25.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:25 smithi143 conmon[109923]: cluster 2023-08-28T02:08:24.563583+0000 2023-08-28T02:08:25.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:25 smithi143 conmon[109923]: mon.a (mon.0) 389 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2023-08-28T02:08:25.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:25 smithi169 conmon[100500]: cephadm 2023-08-28T02:08:23.836642+0000 mgr.y (mgr.14146) 105 : cephadm [INF] Detected new or changed devices on smithi143 2023-08-28T02:08:25.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:25 smithi169 conmon[100500]: cephadm 2023-08-28T02:08:23.846644+0000 mgr.y (mgr.14146) 106 : cephadm [INF] Adjusting osd_memory_target on smithi143 to 4036M 2023-08-28T02:08:25.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:25 smithi169 conmon[100500]: cluster 2023-08-28T02:08:24.563583+0000 mon.a (mon.0) 389 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2023-08-28T02:08:26.714 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:26 smithi169 conmon[100500]: cluster 2023-08-28T02:08:24.937581+0000 mgr.y (mgr.14146) 2023-08-28T02:08:26.714 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:26 smithi169 conmon[100500]: 107 : cluster [DBG] pgmap v87: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-28T02:08:26.715 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:26 smithi169 conmon[100500]: audit 2023-08-28T02:08:25.875080+0000 mon.a (mon.0) 390 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-28T02:08:26.715 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:26 smithi169 conmon[100500]: audit 2023-08-28T02:08:25.877543+0000 mon.a (mon.0) 391 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-28T02:08:26.715 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:26 smithi169 conmon[100500]: audit 2023-08-28T02:08:25.878306+0000 mon.a (mon.0) 392 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:08:26.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:26 smithi143 conmon[100258]: cluster 2023-08-28T02:08:24.937581+0000 mgr.y (mgr.14146) 107 : cluster [DBG] pgmap v87: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-28T02:08:26.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:26 smithi143 conmon[100258]: audit 2023-08-28T02:08:25.875080+0000 mon.a (mon.0) 390 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-28T02:08:26.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:26 smithi143 conmon[100258]: audit 2023-08-28T02:08:26.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:26 smithi143 conmon[100258]: 2023-08-28T02:08:25.877543+0000 mon.a (mon.0) 391 : audit 2023-08-28T02:08:26.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:26 smithi143 conmon[100258]: [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-28T02:08:26.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:26 smithi143 conmon[100258]: audit 2023- 2023-08-28T02:08:26.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:26 smithi143 conmon[100258]: 08-28T02:08:25.878306+0000 2023-08-28T02:08:26.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:26 smithi143 conmon[100258]: mon.a (mon.0) 392 : audit [DBG] 2023-08-28T02:08:26.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:26 smithi143 conmon[100258]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:08:26.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:26 smithi143 conmon[109923]: cluster 2023-08-28T02:08:24.937581 2023-08-28T02:08:26.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:26 smithi143 conmon[109923]: +0000 mgr.y (mgr.14146) 107 : cluster [DBG] pgmap v87: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-28T02:08:26.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:26 smithi143 conmon[109923]: audit 2023-08 2023-08-28T02:08:26.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:26 smithi143 conmon[109923]: -28T02:08:25.875080+0000 mon.a (mon.0) 390 : 2023-08-28T02:08:26.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:26 smithi143 conmon[109923]: audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-28T02:08:26.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:26 smithi143 conmon[109923]: audit 2023-08-28T02:08:26.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:26 smithi143 conmon[109923]: 2023-08-28T 2023-08-28T02:08:26.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:26 smithi143 conmon[109923]: 02:08:25.877543+0000 2023-08-28T02:08:26.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:26 smithi143 conmon[109923]: mon.a (mon.0) 391 : audit 2023-08-28T02:08:26.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:26 smithi143 conmon[109923]: [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-28T02:08:26.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:26 smithi143 conmon[109923]: audit 2023-08-28T02:08:26.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:26 smithi143 conmon[109923]: 2023-08-28T02: 2023-08-28T02:08:26.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:26 smithi143 conmon[109923]: 08:25.878306+0000 mon.a ( 2023-08-28T02:08:26.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:26 smithi143 conmon[109923]: mon.0) 392 : audit [DBG] 2023-08-28T02:08:26.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:26 smithi143 conmon[109923]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:08:27.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:27 smithi143 conmon[100258]: audit 2023-08-28T02:08:25.872486+0000 mgr.y (mgr.14146) 108 : audit [DBG] 2023-08-28T02:08:27.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:27 smithi143 conmon[100258]: from='client.24202 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi169:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:08:27.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:27 smithi143 conmon[109923]: audit 2023-08-28T02:08:25.872486+0000 2023-08-28T02:08:27.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:27 smithi143 conmon[109923]: mgr.y (mgr.14146) 108 : audit [DBG] from='client.24202 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi169:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:08:27.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:27 smithi169 conmon[100500]: audit 2023-08-28T02:08:25.872486+0000 mgr.y ( 2023-08-28T02:08:27.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:27 smithi169 conmon[100500]: mgr.14146) 108 : audit [DBG] from='client.24202 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi169:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:08:28.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:28 smithi143 conmon[100258]: cluster 2023-08-28T02:08:26.938062+0000 mgr.y (mgr 2023-08-28T02:08:28.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:28 smithi143 conmon[100258]: .14146) 109 : cluster [DBG] pgmap v88: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-28T02:08:28.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:28 smithi143 conmon[100258]: audit 2023-08-28T02:08:28.203104+0000 mon.b (mon.1) 8 : audit 2023-08-28T02:08:28.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:28 smithi143 conmon[100258]: [INF] from='client.? 172.21.15.169:0/3063822925' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "16122219-d985-46d2-bac1-f8ffcf48d8ad"}]: dispatch 2023-08-28T02:08:28.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:28 smithi143 conmon[100258]: audit 2023-08-28T02:08:28.203577+0000 mon.a ( 2023-08-28T02:08:28.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:28 smithi143 conmon[100258]: mon.0) 393 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "16122219-d985-46d2-bac1-f8ffcf48d8ad"}]: dispatch 2023-08-28T02:08:28.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:28 smithi143 conmon[100258]: audit 2023-08-28T02:08 2023-08-28T02:08:28.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:28 smithi143 conmon[100258]: :28.210642+0000 mon.a (mon.0) 394 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "16122219-d985-46d2-bac1-f8ffcf48d8ad"}]': finished 2023-08-28T02:08:28.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:28 smithi143 conmon[100258]: 2023-08-28T02:08:28.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:28 smithi143 conmon[100258]: cluster 2023-08-28T02:08:28.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:28 smithi143 conmon[100258]: 2023-08-28T02:08:28.210727+0000 mon.a (mon.0) 395 2023-08-28T02:08:28.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:28 smithi143 conmon[100258]: : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-08-28T02:08:28.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:28 smithi143 conmon[100258]: audit 2023-08-28T02:08 2023-08-28T02:08:28.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:28 smithi143 conmon[100258]: :28.210943+0000 mon.a (mon.0) 396 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-28T02:08:28.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:28 smithi143 conmon[109923]: cluster 2023-08-28T02:08:26.938062+0000 mgr.y (mgr.14146) 109 : cluster [DBG] pgmap v88: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-28T02:08:28.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:28 smithi143 conmon[109923]: audit 2023-08-28T02:08:28.203104+0000 mon.b (mon.1) 8 : audit [INF] from='client.? 172.21.15.169:0/3063822925' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "16122219-d985-46d2-bac1-f8ffcf48d8ad"}]: dispatch 2023-08-28T02:08:28.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:28 smithi143 conmon[109923]: audit 2023-08- 2023-08-28T02:08:28.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:28 smithi143 conmon[109923]: 28T02:08:28.203577+0000 mon.a (mon.0 2023-08-28T02:08:28.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:28 smithi143 conmon[109923]: ) 393 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "16122219-d985-46d2-bac1-f8ffcf48d8ad"}]: dispatch 2023-08-28T02:08:28.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:28 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:08:28.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:28 smithi143 conmon[109923]: 08-28T02:08:28.210642+0000 mon.a ( 2023-08-28T02:08:28.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:28 smithi143 conmon[109923]: mon.0) 394 : audit [INF] 2023-08-28T02:08:28.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:28 smithi143 conmon[109923]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "16122219-d985-46d2-bac1-f8ffcf48d8ad"}]': finished 2023-08-28T02:08:28.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:28 smithi143 conmon[109923]: cluster 2023-08-28T02:08:28.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:28 smithi143 conmon[109923]: 2023-08-28T02:08 2023-08-28T02:08:28.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:28 smithi143 conmon[109923]: :28.210727+0000 mon.a (mon. 2023-08-28T02:08:28.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:28 smithi143 conmon[109923]: 0) 395 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-08-28T02:08:28.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:28 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:08:28.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:28 smithi143 conmon[109923]: 08-28T02:08:28.210943+0000 2023-08-28T02:08:28.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:28 smithi143 conmon[109923]: mon.a (mon.0) 396 : audit 2023-08-28T02:08:28.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:28 smithi143 conmon[109923]: [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-28T02:08:28.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:28 smithi169 conmon[100500]: cluster 2023-08-28T02:08:26.938062+0000 mgr.y (mgr.14146) 109 : cluster [DBG] pgmap v88: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-28T02:08:28.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:28 smithi169 conmon[100500]: audit 2023-08-28T02:08:28.203104+0000 mon.b (mon.1) 8 : audit [INF] from='client.? 172.21.15.169:0/3063822925' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "16122219-d985-46d2-bac1-f8ffcf48d8ad"}]: dispatch 2023-08-28T02:08:28.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:28 smithi169 conmon[100500]: audit 2023-08-28T02:08:28.203577+0000 mon.a (mon.0) 393 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "16122219-d985-46d2-bac1-f8ffcf48d8ad"}]: dispatch 2023-08-28T02:08:28.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:28 smithi169 conmon[100500]: audit 2023-08-28T02:08:28.210642+0000 mon.a (mon.0) 394 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "16122219-d985-46d2-bac1-f8ffcf48d8ad"}]': finished 2023-08-28T02:08:28.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:28 smithi169 conmon[100500]: cluster 2023-08-28T02:08:28.210727+0000 mon.a (mon.0) 395 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-08-28T02:08:28.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:28 smithi169 conmon[100500]: audit 2023-08-28T02:08:28.210943+0000 mon.a (mon.0) 396 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-28T02:08:29.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:29 smithi143 conmon[100258]: audit 2023-08-28T02:08:28.795236+0000 mon.b (mon.1) 9 : audit [DBG] 2023-08-28T02:08:29.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:29 smithi143 conmon[100258]: from='client.? 172.21.15.169:0/1494425339' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-28T02:08:29.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:29 smithi143 conmon[109923]: audit 2023-08-28T02:08:28.795236+0000 mon.b (mon. 2023-08-28T02:08:29.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:29 smithi143 conmon[109923]: 1) 9 : audit [DBG] from='client.? 172.21.15.169:0/1494425339' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-28T02:08:29.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:29 smithi169 conmon[100500]: audit 2023-08-28T02:08:28.795236+0000 mon.b (mon.1) 9 : audit [DBG] from='client.? 172.21.15.169:0/1494425339' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-28T02:08:30.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:30 smithi143 conmon[100258]: cluster 2023-08-28T02:08:28.938502 2023-08-28T02:08:30.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:30 smithi143 conmon[100258]: +0000 mgr.y (mgr.14146) 110 : cluster [DBG] pgmap v90: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-28T02:08:30.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:30 smithi143 conmon[109923]: cluster 2023-08-28T02:08:28.938502+0000 mgr.y (mgr.14146) 110 2023-08-28T02:08:30.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:30 smithi143 conmon[109923]: : cluster [DBG] pgmap v90: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-28T02:08:30.986 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:30 smithi169 conmon[100500]: cluster 2023-08-28T02:08:28.938502+0000 mgr.y (mgr 2023-08-28T02:08:30.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:30 smithi169 conmon[100500]: .14146) 110 : cluster [DBG] pgmap v90: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-28T02:08:31.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:31 smithi143 conmon[100258]: cluster 2023-08-28T02:08:30.938973+0000 mgr.y (mgr.14146 2023-08-28T02:08:31.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:31 smithi143 conmon[100258]: ) 111 : cluster [DBG] pgmap v91: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-28T02:08:31.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:31 smithi143 conmon[109923]: cluster 2023-08-28T02:08:30.938973 2023-08-28T02:08:31.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:31 smithi143 conmon[109923]: +0000 mgr.y (mgr.14146) 111 : cluster [DBG] pgmap v91: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-28T02:08:31.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:31 smithi169 conmon[100500]: cluster 2023-08-28T02:08:30.938973+0000 mgr.y (mgr.14146) 2023-08-28T02:08:31.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:31 smithi169 conmon[100500]: 111 : cluster [DBG] pgmap v91: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-28T02:08:34.310 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:33 smithi169 conmon[100500]: cluster 2023-08-28T02:08:32.939279+0000 mgr.y (mgr.14146) 112 : cluster [DBG] pgmap v92: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-28T02:08:34.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:33 smithi143 conmon[100258]: cluster 2023-08-28T02:08:32.939279 2023-08-28T02:08:34.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:33 smithi143 conmon[100258]: +0000 mgr.y (mgr.14146) 112 : cluster [DBG] pgmap v92: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-28T02:08:34.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:33 smithi143 conmon[109923]: cluster 2023-08-28T02:08:32.939279+0000 mgr.y (mgr.14146) 112 : cluster [DBG] pgmap v92: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-28T02:08:35.230 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:34 smithi169 conmon[100500]: audit 2023-08-28T02:08:34.330604+0000 mon.a (mon 2023-08-28T02:08:35.230 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:34 smithi169 conmon[100500]: .0) 397 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-08-28T02:08:35.230 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:34 smithi169 conmon[100500]: audit 2023-08-28T02:08:34.331494+0000 mon.a (mon.0) 398 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:08:35.230 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:34 smithi169 conmon[100500]: cephadm 2023-08-28T02:08:34.332215+0000 mgr.y (mgr.14146) 113 : cephadm [INF] Deploying daemon osd.4 on smithi169 2023-08-28T02:08:35.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:34 smithi143 conmon[100258]: audit 2023-08-28T02:08:34.330604+0000 mon.a (mon.0) 397 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-08-28T02:08:35.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:34 smithi143 conmon[100258]: audit 2023-08-28T02:08:34.331494+0000 mon.a (mon.0) 398 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:08:35.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:34 smithi143 conmon[100258]: cephadm 2023-08-28T02:08:34.332215+0000 mgr.y (mgr.14146) 113 : cephadm [INF] Deploying daemon osd.4 on smithi169 2023-08-28T02:08:35.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:34 smithi143 conmon[109923]: audit 2023-08-28T02:08:34.330604+0000 mon.a (mon.0 2023-08-28T02:08:35.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:34 smithi143 conmon[109923]: ) 397 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-08-28T02:08:35.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:34 smithi143 conmon[109923]: audit 2023-08-28T02:08:34.331494+0000 mon.a (mon.0) 398 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:08:35.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:34 smithi143 conmon[109923]: cephadm 2023-08-28T02:08:34.332215+0000 mgr.y 2023-08-28T02:08:35.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:34 smithi143 conmon[109923]: (mgr.14146) 113 : cephadm [INF] Deploying daemon osd.4 on smithi169 2023-08-28T02:08:36.236 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:36 smithi169 conmon[100500]: cluster 2023-08-28T02:08:34.939600+0000 mgr.y (mgr.14146) 114 : cluster [DBG] pgmap v93: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-28T02:08:36.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:36 smithi143 conmon[100258]: cluster 2023-08-28T02:08:34.939600+0000 mgr.y (mgr.14146) 114 : cluster [DBG] pgmap v93: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-28T02:08:36.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:36 smithi143 conmon[109923]: cluster 2023-08-28T02:08:34.939600+0000 mgr.y (mgr.14146) 2023-08-28T02:08:36.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:36 smithi143 conmon[109923]: 114 : cluster [DBG] pgmap v93: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-28T02:08:38.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:38 smithi143 conmon[100258]: cluster 2023-08-28T02:08:36.939961+0000 mgr.y (mgr.14146) 115 : 2023-08-28T02:08:38.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:38 smithi143 conmon[100258]: cluster [DBG] pgmap v94: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-28T02:08:38.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:38 smithi143 conmon[109923]: cluster 2023-08-28T02:08:36.939961+0000 mgr.y (mgr.14146) 2023-08-28T02:08:38.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:38 smithi143 conmon[109923]: 115 : cluster [DBG] pgmap v94: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-28T02:08:38.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:38 smithi169 conmon[100500]: cluster 2023-08-28T02:08:36.939961+0000 mgr.y (mgr.14146) 115 : cluster [DBG] pgmap v94: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-28T02:08:40.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:40 smithi169 conmon[100500]: cluster 2023-08-28T02:08:38.940492+0000 mgr.y (mgr. 2023-08-28T02:08:40.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:40 smithi169 conmon[100500]: 14146) 116 : cluster [DBG] pgmap v95: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-28T02:08:40.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:40 smithi169 conmon[100500]: audit 2023-08-28T02:08:39.457189+0000 mon.a (mon.0) 399 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:40.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:40 smithi169 conmon[100500]: audit 2023-08-28T02:08:39.468947+0000 mon.a (mon.0) 400 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:08:40.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:40 smithi143 conmon[109923]: cluster 2023-08-28T02:08:38.940492+0000 mgr.y (mgr.14146) 116 : cluster [DBG] pgmap v95: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-28T02:08:40.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:40 smithi143 conmon[109923]: audit 2023-08-28T02:08:39.457189+0000 mon.a (mon.0) 399 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:40.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:40 smithi143 conmon[109923]: audit 2023-08-28T02:08:39 2023-08-28T02:08:40.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:40 smithi143 conmon[109923]: .468947+0000 mon.a (mon.0) 400 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:08:40.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:40 smithi143 conmon[100258]: cluster 2023-08-28T02:08:38.940492+0000 mgr.y (mgr.14146 2023-08-28T02:08:40.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:40 smithi143 conmon[100258]: ) 116 : cluster [DBG] pgmap v95: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-28T02:08:40.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:40 smithi143 conmon[100258]: audit 2023-08-28T02:08:39.457189+0000 mon.a (mon.0) 399 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:40.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:40 smithi143 conmon[100258]: audit 2023-08-28T02:08:40.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:40 smithi143 conmon[100258]: 2023-08- 2023-08-28T02:08:40.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:40 smithi143 conmon[100258]: 28T02:08:39.468947+0000 mon.a (mon. 2023-08-28T02:08:40.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:40 smithi143 conmon[100258]: 0) 400 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:08:42.514 INFO:teuthology.orchestra.run.smithi169.stdout:Created osd(s) 4 on host 'smithi169' 2023-08-28T02:08:42.542 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:42 smithi169 conmon[100500]: cluster 2023-08-28T02:08:40.941033+0000 mgr.y (mgr.14146) 117 : cluster [DBG] pgmap v96: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-28T02:08:42.543 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:42 smithi169 conmon[100500]: audit 2023-08-28T02:08:41.274403+0000 mon.a (mon.0) 401 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:42.543 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:42 smithi169 conmon[100500]: audit 2023-08-28T02:08:41.594450+0000 mon.a (mon.0) 402 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:08:42.543 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:42 smithi169 conmon[100500]: audit 2023-08-28T02:08:41.595346+0000 mon.a (mon.0) 403 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:08:42.543 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:42 smithi169 conmon[100500]: audit 2023-08-28T02:08:41.600180+0000 mon.a (mon.0) 404 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:42.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:42 smithi143 conmon[109923]: cluster 2023-08-28T02:08:40.941033+0000 mgr.y (mgr.14146) 117 : cluster [DBG] pgmap v96: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-28T02:08:42.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:42 smithi143 conmon[109923]: audit 2023-08-28T02:08:41.274403+0000 mon.a (mon.0) 401 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:42.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:42 smithi143 conmon[109923]: audit 2023-08-28T02:08:41.594450+0000 2023-08-28T02:08:42.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:42 smithi143 conmon[109923]: mon.a (mon.0) 402 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:08:42.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:42 smithi143 conmon[109923]: audit 2023-08-28T02:08:41.595346+0000 mon.a (mon 2023-08-28T02:08:42.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:42 smithi143 conmon[109923]: .0) 403 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:08:42.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:42 smithi143 conmon[109923]: audit 2023-08-28T02:08:41.600180+0000 mon.a (mon.0) 404 : audit 2023-08-28T02:08:42.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:42 smithi143 conmon[109923]: [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:42.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:42 smithi143 conmon[100258]: cluster 2023-08-28T02:08:40.941033+0000 mgr.y (mgr.14146) 117 : cluster [DBG] pgmap v96: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-28T02:08:42.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:42 smithi143 conmon[100258]: audit 2023-08-28T02:08:41 2023-08-28T02:08:42.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:42 smithi143 conmon[100258]: .274403+0000 mon.a (mon.0) 401 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:42.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:42 smithi143 conmon[100258]: audit 2023-08-28T02:08:41.594450+0000 mon.a (mon.0) 402 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:08:42.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:42 smithi143 conmon[100258]: audit 2023-08-28T02:08:41.595346 2023-08-28T02:08:42.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:42 smithi143 conmon[100258]: +0000 mon.a (mon.0) 403 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:08:42.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:42 smithi143 conmon[100258]: audit 2023-08-28 2023-08-28T02:08:42.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:42 smithi143 conmon[100258]: T02:08:41.600180+0000 mon.a (mon.0) 404 : audit [INF] 2023-08-28T02:08:42.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:42 smithi143 conmon[100258]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:43.134 DEBUG:teuthology.orchestra.run.smithi169:osd.4> sudo journalctl -f -n 0 -u ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@osd.4.service 2023-08-28T02:08:43.137 INFO:tasks.cephadm:Deploying osd.5 on smithi169 with /dev/vg_nvme/lv_3... 2023-08-28T02:08:43.137 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-08-28T02:08:43.166 INFO:journalctl@ceph.osd.4.smithi169.stdout:-- Logs begin at Mon 2023-08-28 01:49:52 UTC. -- 2023-08-28T02:08:43.455 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:43 smithi169 conmon[100500]: audit 2023-08-28T02:08:42.375757+0000 mon.b (mon.1) 10 : audit [INF] from='osd.4 [v2:172.21.15.169:6800/3960017153,v1:172.21.15.169:6801/3960017153]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-08-28T02:08:43.455 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:43 smithi169 conmon[100500]: audit 2023-08-28T02:08:42.376145+0000 mon.a (mon.0) 405 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-08-28T02:08:43.457 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:43 smithi169 conmon[100500]: audit 2023-08-28T02:08:42.515084+0000 mon.a (mon.0) 406 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:43.457 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:43 smithi169 conmon[100500]: audit 2023-08-28T02:08:42.518979+0000 mon.a (mon.0) 407 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:08:43.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:43 smithi143 conmon[100258]: audit 2023-08-28T02:08:42.375757+0000 mon.b (mon.1) 10 : audit [INF] 2023-08-28T02:08:43.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:43 smithi143 conmon[100258]: from='osd.4 [v2:172.21.15.169:6800/3960017153,v1:172.21.15.169:6801/3960017153]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-08-28T02:08:43.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:43 smithi143 conmon[100258]: audit 2023-08-28T02:08:42.376145+0000 mon.a (mon.0) 405 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-08-28T02:08:43.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:43 smithi143 conmon[100258]: audit 2023-08-28T02:08:42.515084+0000 mon.a (mon.0) 406 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:43.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:43 smithi143 conmon[100258]: audit 2023-08- 2023-08-28T02:08:43.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:43 smithi143 conmon[100258]: 28T02:08:42.518979+0000 mon.a (mon.0) 407 : 2023-08-28T02:08:43.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:43 smithi143 conmon[100258]: audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:08:43.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:43 smithi143 conmon[109923]: audit 2023-08-28T02:08:42.375757+0000 2023-08-28T02:08:43.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:43 smithi143 conmon[109923]: mon.b (mon.1) 10 : audit [INF] from='osd.4 [v2:172.21.15.169:6800/3960017153,v1:172.21.15.169:6801/3960017153]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-08-28T02:08:43.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:43 smithi143 conmon[109923]: audit 2023-08-28T 2023-08-28T02:08:43.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:43 smithi143 conmon[109923]: 02:08:42.376145+0000 mon.a (mon.0) 405 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-08-28T02:08:43.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:43 smithi143 conmon[109923]: audit 2023-08-28T02:08 2023-08-28T02:08:43.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:43 smithi143 conmon[109923]: :42.515084+0000 mon.a (mon.0) 406 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:43.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:43 smithi143 conmon[109923]: 2023-08-28T02:08:43.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:43 smithi143 conmon[109923]: audit 2023-08-28T02:08:43.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:43 smithi143 conmon[109923]: 2023-08-28T02:08 2023-08-28T02:08:43.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:43 smithi143 conmon[109923]: :42.518979+0000 mon.a (mon.0 2023-08-28T02:08:43.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:43 smithi143 conmon[109923]: ) 407 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:08:44.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:44 smithi143 conmon[100258]: cluster 2023-08-28T02:08:42.941403+0000 2023-08-28T02:08:44.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:44 smithi143 conmon[100258]: mgr.y (mgr.14146) 118 : cluster [DBG] pgmap v97: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-28T02:08:44.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:44 smithi143 conmon[100258]: audit 2023-08-28T02:08:43.289328+0000 mon.a (mon.0) 408 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-08-28T02:08:44.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:44 smithi143 conmon[100258]: cluster 2023-08-28T02:08:44.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:44 smithi143 conmon[100258]: 2023-08-28T02:08:43.289424+0000 mon.a (mon.0) 409 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-08-28T02:08:44.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:44 smithi143 conmon[100258]: audit 2023-08-28T02:08:43.289591+0000 mon.a (mon.0) 410 : audit 2023-08-28T02:08:44.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:44 smithi143 conmon[100258]: [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-28T02:08:44.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:44 smithi143 conmon[100258]: audit 2023-08-28T02:08 2023-08-28T02:08:44.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:44 smithi143 conmon[100258]: :43.290042+0000 mon.b (mon.1) 11 : audit [INF] 2023-08-28T02:08:44.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:44 smithi143 conmon[100258]: from='osd.4 [v2:172.21.15.169:6800/3960017153,v1:172.21.15.169:6801/3960017153]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2023-08-28T02:08:44.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:44 smithi143 conmon[100258]: audit 2023-08-28T02:08:43.290344+0000 mon.a (mon.0) 411 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2023-08-28T02:08:44.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:44 smithi143 conmon[109923]: cluster 2023-08-28T02:08:42.941403+0000 mgr.y ( 2023-08-28T02:08:44.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:44 smithi143 conmon[109923]: mgr.14146) 118 : cluster [DBG] pgmap v97: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-28T02:08:44.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:44 smithi143 conmon[109923]: audit 2023-08-28T02:08:43.289328+0000 mon.a 2023-08-28T02:08:44.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:44 smithi143 conmon[109923]: (mon.0) 408 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-08-28T02:08:44.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:44 smithi143 conmon[109923]: cluster 2023-08-28T02:08: 2023-08-28T02:08:44.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:44 smithi143 conmon[109923]: 43.289424+0000 mon.a (mon.0) 409 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-08-28T02:08:44.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:44 smithi143 conmon[109923]: audit 2023-08-28T02:08:44.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:44 smithi143 conmon[109923]: 2023-08-28T02:08:43.289591+0000 mon.a (mon.0) 410 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-28T02:08:44.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:44 smithi143 conmon[109923]: audit 2023-08-28T02:08: 2023-08-28T02:08:44.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:44 smithi143 conmon[109923]: 43.290042+0000 mon.b (mon.1) 11 : audit [INF] from='osd.4 [v2:172.21.15.169:6800/3960017153,v1:172.21.15.169:6801/3960017153]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2023-08-28T02:08:44.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:44 smithi143 conmon[109923]: audit 2023 2023-08-28T02:08:44.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:44 smithi143 conmon[109923]: -08-28T02:08:43 2023-08-28T02:08:44.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:44 smithi143 conmon[109923]: .290344+0000 mon.a (mon.0) 2023-08-28T02:08:44.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:44 smithi143 conmon[109923]: 411 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2023-08-28T02:08:44.737 INFO:journalctl@ceph.osd.4.smithi169.stdout:Aug 28 02:08:44 smithi169 conmon[106829]: debug 2023-08-28T02:08:44.295+0000 7faf8a5d5700 -1 osd.4 0 waiting for initial osdmap 2023-08-28T02:08:44.737 INFO:journalctl@ceph.osd.4.smithi169.stdout:Aug 28 02:08:44 smithi169 conmon[106829]: debug 2023-08-28T02:08:44.301+0000 7faf86550700 -1 osd.4 29 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-08-28T02:08:44.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:44 smithi169 conmon[100500]: cluster 2023-08-28T02:08:42.941403+0000 mgr.y (mgr.14146) 118 : 2023-08-28T02:08:44.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:44 smithi169 conmon[100500]: cluster [DBG] pgmap v97: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-28T02:08:44.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:44 smithi169 conmon[100500]: audit 2023-08-28T02:08:43.289328+0000 mon.a (mon.0) 408 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-08-28T02:08:44.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:44 smithi169 conmon[100500]: cluster 2023-08-28T02:08:43.289424+0000 mon.a (mon.0) 409 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-08-28T02:08:44.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:44 smithi169 conmon[100500]: audit 2023-08-28T02:08:43.289591+0000 mon.a (mon.0) 410 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-28T02:08:44.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:44 smithi169 conmon[100500]: audit 2023-08-28T02:08:43.290042+0000 mon.b (mon.1) 11 : audit [INF] from='osd.4 [v2:172.21.15.169:6800/3960017153,v1:172.21.15.169:6801/3960017153]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2023-08-28T02:08:44.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:44 smithi169 conmon[100500]: audit 2023-08-28T02:08:43.290344+0000 mon.a (mon.0) 411 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2023-08-28T02:08:45.600 INFO:teuthology.orchestra.run.smithi169.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-08-28T02:08:45.600 INFO:teuthology.orchestra.run.smithi169.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-08-28T02:08:45.600 INFO:teuthology.orchestra.run.smithi169.stderr: stderr: 10+0 records in 2023-08-28T02:08:45.601 INFO:teuthology.orchestra.run.smithi169.stderr:10+0 records out 2023-08-28T02:08:45.601 INFO:teuthology.orchestra.run.smithi169.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0113143 s, 927 MB/s 2023-08-28T02:08:45.601 INFO:teuthology.orchestra.run.smithi169.stderr:--> Zapping successful for: 2023-08-28T02:08:45.649 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:45 smithi169 conmon[100500]: audit 2023-08-28T02:08:44.291769+0000 mon.a (mon.0) 412 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]': finished 2023-08-28T02:08:45.649 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:45 smithi169 conmon[100500]: cluster 2023-08-28T02:08:44.291925+0000 mon.a (mon.0) 413 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-08-28T02:08:45.649 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:45 smithi169 conmon[100500]: audit 2023-08-28T02:08:44.292515+0000 mon.a (mon.0) 414 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-28T02:08:45.650 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:45 smithi169 conmon[100500]: audit 2023-08-28T02:08:44.296883+0000 mon.a (mon.0) 415 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-28T02:08:45.650 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:45 smithi169 conmon[100500]: cluster 2023-08-28T02:08:45.123853+0000 mon.a (mon.0) 416 : cluster [INF] osd.4 [v2:172.21.15.169:6800/3960017153,v1:172.21.15.169:6801/3960017153] boot 2023-08-28T02:08:45.650 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:45 smithi169 conmon[100500]: cluster 2023-08-28T02:08:45.123919+0000 mon.a (mon.0) 417 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2023-08-28T02:08:45.650 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:45 smithi169 conmon[100500]: audit 2023-08-28T02:08:45.124476+0000 mon.a (mon.0) 418 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-28T02:08:45.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:45 smithi143 conmon[100258]: audit 2023-08-28T02:08:44.291769+0000 mon.a (mon.0) 412 : audit [INF] 2023-08-28T02:08:45.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:45 smithi143 conmon[100258]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]': finished 2023-08-28T02:08:45.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:45 smithi143 conmon[100258]: cluster 2023-08-28T02:08:44.291925+0000 mon.a (mon.0) 413 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-08-28T02:08:45.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:45 smithi143 conmon[100258]: audit 2023-08-28T02:08:44.292515+0000 mon.a (mon.0) 414 : audit 2023-08-28T02:08:45.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:45 smithi143 conmon[100258]: [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-28T02:08:45.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:45 smithi143 conmon[100258]: audit 2023-08-28T02:08:44.296883+0000 mon.a (mon.0) 415 : audit 2023-08-28T02:08:45.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:45 smithi143 conmon[100258]: [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-28T02:08:45.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:45 smithi143 conmon[100258]: cluster 2023-08-28T02: 2023-08-28T02:08:45.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:45 smithi143 conmon[100258]: 08:45.123853+0000 mon.a (mon.0) 416 : cluster [INF] osd.4 [v2:172.21.15.169:6800/3960017153,v1:172.21.15.169:6801/3960017153] boot 2023-08-28T02:08:45.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:45 smithi143 conmon[100258]: cluster 2023-08-28T 2023-08-28T02:08:45.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:45 smithi143 conmon[100258]: 02:08:45.123919+0000 mon.a (mon.0) 417 2023-08-28T02:08:45.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:45 smithi143 conmon[100258]: : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2023-08-28T02:08:45.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:45 smithi143 conmon[100258]: audit 2023-08-28T02:08:45.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:45 smithi143 conmon[100258]: 2023-08-28T02:08:45.124476+0000 mon.a (mon.0) 2023-08-28T02:08:45.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:45 smithi143 conmon[100258]: 418 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-28T02:08:45.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:45 smithi143 conmon[109923]: audit 2023-08-28T02:08:44.291769+0000 mon.a (mon.0) 412 : 2023-08-28T02:08:45.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:45 smithi143 conmon[109923]: audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]': finished 2023-08-28T02:08:45.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:45 smithi143 conmon[109923]: cluster 2023-08-28T02:08:44.291925+0000 mon.a (mon.0) 413 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-08-28T02:08:45.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:45 smithi143 conmon[109923]: audit 2023-08-28 2023-08-28T02:08:45.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:45 smithi143 conmon[109923]: T02:08:44.292515+0000 mon.a (mon.0) 414 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-28T02:08:45.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:45 smithi143 conmon[109923]: audit 2023-08-28T02:08:45.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:45 smithi143 conmon[109923]: 2023-08-28 2023-08-28T02:08:45.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:45 smithi143 conmon[109923]: T02:08:44.296883+0000 mon.a 2023-08-28T02:08:45.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:45 smithi143 conmon[109923]: (mon.0) 415 : audit [DBG] 2023-08-28T02:08:45.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:45 smithi143 conmon[109923]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-28T02:08:45.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:45 smithi143 conmon[109923]: cluster 2023-08-28 2023-08-28T02:08:45.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:45 smithi143 conmon[109923]: T02:08:45.123853+0000 mon.a (mon.0) 416 : cluster 2023-08-28T02:08:45.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:45 smithi143 conmon[109923]: [INF] osd.4 [v2:172.21.15.169:6800/3960017153,v1:172.21.15.169:6801/3960017153] boot 2023-08-28T02:08:45.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:45 smithi143 conmon[109923]: cluster 2023-08- 2023-08-28T02:08:45.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:45 smithi143 conmon[109923]: 28T02:08:45.123919+0000 mon.a (mon.0) 417 : cluster 2023-08-28T02:08:45.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:45 smithi143 conmon[109923]: [DBG] osdmap e30: 5 total, 5 up, 5 in 2023-08-28T02:08:45.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:45 smithi143 conmon[109923]: audit 2023-08-28T02:08:45.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:45 smithi143 conmon[109923]: 2023-08-28T02:08:45.124476 2023-08-28T02:08:45.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:45 smithi143 conmon[109923]: +0000 mon.a (mon.0) 418 : audit 2023-08-28T02:08:45.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:45 smithi143 conmon[109923]: [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-28T02:08:46.635 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:46 smithi169 conmon[100500]: cluster 2023-08-28T02:08:43.377580+0000 osd.4 (osd.4) 1 : cluster 2023-08-28T02:08:46.636 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:46 smithi169 conmon[100500]: [DBG] purged_snaps scrub starts 2023-08-28T02:08:46.636 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:46 smithi169 conmon[100500]: cluster 2023-08-28T02:08:43.377644+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2023-08-28T02:08:46.636 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:46 smithi169 conmon[100500]: cluster 2023-08-28T02:08:44.941710+0000 mgr.y (mgr.14146) 119 : cluster [DBG] pgmap v100: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-28T02:08:46.636 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:46 smithi169 conmon[100500]: cluster 2023-08-28T02:08:46.128646+0000 mon.a (mon.0) 419 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2023-08-28T02:08:46.653 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph orch daemon add osd smithi169:vg_nvme/lv_3 2023-08-28T02:08:46.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:46 smithi143 conmon[100258]: cluster 2023-08-28T02:08:43.377580+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-08-28T02:08:46.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:46 smithi143 conmon[100258]: cluster 2023-08-28T02:08:46.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:46 smithi143 conmon[100258]: 2023-08-28T02:08:43.377644+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2023-08-28T02:08:46.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:46 smithi143 conmon[100258]: cluster 2023-08-28T02:08:44.941710+0000 mgr.y (mgr.14146) 119 : cluster [DBG] pgmap v100: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-28T02:08:46.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:46 smithi143 conmon[100258]: cluster 2023-08-28T02:08:46.128646+0000 mon.a 2023-08-28T02:08:46.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:46 smithi143 conmon[100258]: (mon.0) 419 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2023-08-28T02:08:46.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:46 smithi143 conmon[109923]: cluster 2023-08-28T02:08:43.377580+0000 2023-08-28T02:08:46.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:46 smithi143 conmon[109923]: osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-08-28T02:08:46.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:46 smithi143 conmon[109923]: cluster 2023-08-28T02:08:46.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:46 smithi143 conmon[109923]: 2023-08-28T02:08:43.377644+0000 osd.4 (osd.4) 2023-08-28T02:08:46.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:46 smithi143 conmon[109923]: 2 : cluster [DBG] purged_snaps scrub ok 2023-08-28T02:08:46.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:46 smithi143 conmon[109923]: cluster 2023-08- 2023-08-28T02:08:46.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:46 smithi143 conmon[109923]: 28T02:08:44.941710+0000 mgr.y (mgr. 2023-08-28T02:08:46.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:46 smithi143 conmon[109923]: 14146) 119 : cluster [DBG] pgmap v100: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-08-28T02:08:46.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:46 smithi143 conmon[109923]: cluster 2023- 2023-08-28T02:08:46.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:46 smithi143 conmon[109923]: 08-28T02:08:46.128646+0000 mon.a ( 2023-08-28T02:08:46.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:46 smithi143 conmon[109923]: mon.0) 419 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2023-08-28T02:08:46.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:46 smithi143 conmon[109923]: 2023-08-28T02:08:48.223 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:48 smithi169 conmon[100500]: cluster 2023-08-28T02:08:46.942082+0000 mgr.y (mgr.14146) 120 : cluster [DBG] pgmap v103: 1 pgs: 1 remapped+peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-28T02:08:48.223 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:48 smithi169 conmon[100500]: cluster 2023-08-28T02:08:47.126563+0000 mon.a (mon.0) 420 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2023-08-28T02:08:48.223 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:48 smithi169 conmon[100500]: cephadm 2023-08-28T02:08:47.166105+0000 mgr.y (mgr.14146) 121 : cephadm [INF] Detected new or changed devices on smithi169 2023-08-28T02:08:48.223 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:48 smithi169 conmon[100500]: audit 2023-08-28T02:08:47.171206+0000 mon.a (mon.0) 421 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:48.223 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:48 smithi169 conmon[100500]: audit 2023-08-28T02:08:47.172153+0000 mon.a (mon.0) 422 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:08:48.223 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:48 smithi169 conmon[100500]: cephadm 2023-08-28T02:08:47.172758+0000 mgr.y (mgr.14146) 122 : cephadm [INF] Adjusting osd_memory_target on smithi169 to 17168M 2023-08-28T02:08:48.223 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:48 smithi169 conmon[100500]: audit 2023-08-28T02:08:47.176592+0000 mon.a (mon.0) 423 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:48.223 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:48 smithi169 conmon[100500]: audit 2023-08-28T02:08:47.177582+0000 mon.a (mon.0) 424 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:08:48.223 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:48 smithi169 conmon[100500]: audit 2023-08-28T02:08:47.178429+0000 mon.a (mon.0) 425 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:08:48.223 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:48 smithi169 conmon[100500]: audit 2023-08-28T02:08:47. 2023-08-28T02:08:48.223 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:48 smithi169 conmon[100500]: 182909+0000 mon.a (mon.0) 426 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:48.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[100258]: cluster 2023-08-28T02:08:46.942082+0000 mgr.y (mgr.14146) 120 : cluster [DBG] 2023-08-28T02:08:48.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[100258]: pgmap v103: 1 pgs: 1 remapped+peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-28T02:08:48.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[100258]: cluster 2023-08-28T02:08:47.126563+0000 mon.a (mon.0) 420 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2023-08-28T02:08:48.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[100258]: cephadm 2023-08-28T02:08:47.166105+0000 mgr.y (mgr. 2023-08-28T02:08:48.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[100258]: 14146) 121 : cephadm [INF] Detected new or changed devices on smithi169 2023-08-28T02:08:48.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[100258]: audit 2023-08-28T02:08: 2023-08-28T02:08:48.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[100258]: 47.171206+0000 mon.a (mon.0) 421 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:48.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[100258]: audit 2023-08-28T02:08:48.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[100258]: 2023-08-28T02:08:47. 2023-08-28T02:08:48.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[100258]: 172153+0000 mon.a (mon.0) 422 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:08:48.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[100258]: 2023-08-28T02:08:48.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[100258]: cephadm 2023-08-28T02:08:47. 2023-08-28T02:08:48.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[100258]: 172758+0000 mgr.y (mgr.14146) 122 : cephadm [INF] 2023-08-28T02:08:48.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[100258]: Adjusting osd_memory_target on smithi169 to 17168M 2023-08-28T02:08:48.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[100258]: audit 2023-08-28T02:08:47. 2023-08-28T02:08:48.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[100258]: 176592+0000 mon.a (mon.0) 423 : audit [INF] 2023-08-28T02:08:48.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[100258]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:48.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[100258]: audit 2023-08-28T02:08:47. 2023-08-28T02:08:48.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[100258]: 177582+0000 mon.a (mon.0) 424 : audit [DBG] 2023-08-28T02:08:48.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[100258]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:08:48.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[100258]: audit 2023-08-28T02:08:47. 2023-08-28T02:08:48.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[100258]: 178429+0000 mon.a (mon.0) 425 : audit [INF] 2023-08-28T02:08:48.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[100258]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:08:48.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[100258]: audit 2023-08-28T02:08:47.182909+0000 2023-08-28T02:08:48.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[100258]: mon.a (mon.0) 426 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:48.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[109923]: cluster 2023-08-28T02:08:46.942082+0000 mgr.y (mgr.14146) 120 : cluster 2023-08-28T02:08:48.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[109923]: [DBG] pgmap v103: 1 pgs: 1 remapped+peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-28T02:08:48.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[109923]: cluster 2023-08-28T02:08:47.126563+0000 mon.a (mon.0) 420 2023-08-28T02:08:48.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[109923]: : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2023-08-28T02:08:48.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[109923]: cephadm 2023-08-28T02:08:48.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[109923]: 2023- 2023-08-28T02:08:48.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[109923]: 08-28T02:08:47.166105 2023-08-28T02:08:48.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[109923]: +0000 mgr.y (mgr.14146) 121 : cephadm [INF] Detected new or changed devices on smithi169 2023-08-28T02:08:48.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[109923]: audit 2023-08-28T02:08: 2023-08-28T02:08:48.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[109923]: 47.171206+0000 mon.a (mon.0) 421 : audit 2023-08-28T02:08:48.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[109923]: [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:48.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[109923]: audit 2023-08-28T02:08:48.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[109923]: 2023-08-28 2023-08-28T02:08:48.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[109923]: T02:08:47.172153+0000 mon.a (mon.0) 422 : 2023-08-28T02:08:48.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[109923]: audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:08:48.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[109923]: cephadm 2023-08-28T02:08:47.172758+0000 mgr.y 2023-08-28T02:08:48.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[109923]: (mgr.14146) 122 : cephadm [INF] Adjusting osd_memory_target on smithi169 to 17168M 2023-08-28T02:08:48.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[109923]: audit 2023-08-28T02:08:47.176592+0000 mon.a (mon.0) 2023-08-28T02:08:48.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[109923]: 423 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:48.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[109923]: audit 2023-08-28T02:08:47.177582+0000 2023-08-28T02:08:48.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[109923]: mon.a (mon.0) 424 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:08:48.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[109923]: audit 2023-08 2023-08-28T02:08:48.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[109923]: -28T02:08:47.178429+0000 mon.a (mon 2023-08-28T02:08:48.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[109923]: .0) 425 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:08:48.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[109923]: audit 2023-08-28T 2023-08-28T02:08:48.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[109923]: 02:08:47.182909+0000 mon.a (mon.0 2023-08-28T02:08:48.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:48 smithi143 conmon[109923]: ) 426 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:08:49.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:49 smithi143 conmon[100258]: audit 2023-08-28T02:08:48.568619+0000 mgr.y (mgr.14146) 123 : audit [DBG] from='client.24229 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi169:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:08:49.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:49 smithi143 conmon[100258]: audit 2023-08-28T02:08:48.571019+0000 mon.a (mon.0) 427 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-28T02:08:49.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:49 smithi143 conmon[100258]: audit 2023-08-28T02:08:48.573597+0000 mon.a (mon.0) 428 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-28T02:08:49.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:49 smithi143 conmon[100258]: audit 2023-08-28T02:08:48.574361+0000 mon.a (mon.0) 429 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:08:49.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:49 smithi143 conmon[109923]: audit 2023-08-28T02:08:48.568619+0000 mgr.y (mgr.14146 2023-08-28T02:08:49.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:49 smithi143 conmon[109923]: ) 123 : audit [DBG] from='client.24229 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi169:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:08:49.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:49 smithi143 conmon[109923]: audit 2023-08-28T02:08:48. 2023-08-28T02:08:49.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:49 smithi143 conmon[109923]: 571019+0000 mon.a (mon.0) 427 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-28T02:08:49.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:49 smithi143 conmon[109923]: audit 2023-08-28T02:08: 2023-08-28T02:08:49.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:49 smithi143 conmon[109923]: 48.573597+0000 mon.a (mon.0) 428 : audit [INF] 2023-08-28T02:08:49.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:49 smithi143 conmon[109923]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-28T02:08:49.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:49 smithi143 conmon[109923]: audit 2023-08-28T02:08:48.574361+0000 2023-08-28T02:08:49.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:49 smithi143 conmon[109923]: mon.a (mon.0) 429 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:08:49.461 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:49 smithi169 conmon[100500]: audit 2023-08-28T02:08:48.568619+0000 mgr.y (mgr.14146 2023-08-28T02:08:49.461 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:49 smithi169 conmon[100500]: ) 123 : audit [DBG] from='client.24229 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi169:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:08:49.461 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:49 smithi169 conmon[100500]: audit 2023-08-28T02:08:48.571019+0000 mon.a (mon.0) 427 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-28T02:08:49.461 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:49 smithi169 conmon[100500]: audit 2023-08-28T02:08:48.573597+0000 mon.a (mon.0) 428 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-28T02:08:49.461 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:49 smithi169 conmon[100500]: audit 2023-08-28T02:08:48.574361+0000 mon.a (mon.0) 429 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:08:50.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:50 smithi143 conmon[100258]: cluster 2023-08-28T02:08:48.942553+0000 mgr.y (mgr. 2023-08-28T02:08:50.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:50 smithi143 conmon[100258]: 14146) 124 : cluster [DBG] pgmap v105: 1 pgs: 1 remapped+peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-28T02:08:50.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:50 smithi143 conmon[109923]: cluster 2023-08-28T02:08:48.942553+0000 mgr.y (mgr.14146 2023-08-28T02:08:50.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:50 smithi143 conmon[109923]: ) 124 : cluster [DBG] pgmap v105: 1 pgs: 1 remapped+peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-28T02:08:50.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:50 smithi169 conmon[100500]: cluster 2023-08-28T02:08:48.942553+0000 mgr.y (mgr.14146) 124 : cluster [DBG] pgmap v105: 1 pgs: 1 remapped+peering; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-28T02:08:51.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:51 smithi143 conmon[100258]: audit 2023-08-28T02:08:50.861656+0000 mon.b (mon.1) 12 : audit 2023-08-28T02:08:51.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:51 smithi143 conmon[100258]: [INF] from='client.? 172.21.15.169:0/880401797' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "adbc3ea5-230f-419e-8f6f-99763378b296"}]: dispatch 2023-08-28T02:08:51.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:51 smithi143 conmon[100258]: audit 2023-08-28T02:08:50.862207+0000 mon.a (mon.0) 430 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "adbc3ea5-230f-419e-8f6f-99763378b296"}]: dispatch 2023-08-28T02:08:51.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:51 smithi143 conmon[100258]: audit 2023-08-28T02:08:50.867513+0000 mon.a (mon.0) 431 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "adbc3ea5-230f-419e-8f6f-99763378b296"}]': finished 2023-08-28T02:08:51.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:51 smithi143 conmon[100258]: cluster 2023-08-28T02 2023-08-28T02:08:51.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:51 smithi143 conmon[100258]: :08:50.867609+0000 mon.a (mon.0) 432 : 2023-08-28T02:08:51.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:51 smithi143 conmon[100258]: cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2023-08-28T02:08:51.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:51 smithi143 conmon[100258]: audit 2023-08-28T02:08 2023-08-28T02:08:51.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:51 smithi143 conmon[100258]: :50.867784+0000 mon.a (mon.0) 433 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-28T02:08:51.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:51 smithi143 conmon[109923]: audit 2023-08-28T02:08:50.861656+0000 2023-08-28T02:08:51.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:51 smithi143 conmon[109923]: mon.b (mon.1) 12 : audit [INF] from='client.? 172.21.15.169:0/880401797' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "adbc3ea5-230f-419e-8f6f-99763378b296"}]: dispatch 2023-08-28T02:08:51.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:51 smithi143 conmon[109923]: audit 2023-08 2023-08-28T02:08:51.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:51 smithi143 conmon[109923]: -28T02:08:50.862207+0000 mon.a 2023-08-28T02:08:51.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:51 smithi143 conmon[109923]: (mon.0) 430 : audit [INF] 2023-08-28T02:08:51.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:51 smithi143 conmon[109923]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "adbc3ea5-230f-419e-8f6f-99763378b296"}]: dispatch 2023-08-28T02:08:51.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:51 smithi143 conmon[109923]: audit 2023-08-28T02:08:51.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:51 smithi143 conmon[109923]: 2023-08-28T02:08:50 2023-08-28T02:08:51.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:51 smithi143 conmon[109923]: .867513+0000 mon.a (mon.0) 431 : audit 2023-08-28T02:08:51.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:51 smithi143 conmon[109923]: [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "adbc3ea5-230f-419e-8f6f-99763378b296"}]': finished 2023-08-28T02:08:51.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:51 smithi143 conmon[109923]: cluster 2023- 2023-08-28T02:08:51.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:51 smithi143 conmon[109923]: 08-28T02:08:50.867609+0000 2023-08-28T02:08:51.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:51 smithi143 conmon[109923]: mon.a (mon.0) 432 : cluster [DBG] 2023-08-28T02:08:51.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:51 smithi143 conmon[109923]: osdmap e33: 6 total, 5 up, 6 in 2023-08-28T02:08:51.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:51 smithi143 conmon[109923]: audit 2023 2023-08-28T02:08:51.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:51 smithi143 conmon[109923]: -08-28T02:08:50.867784 2023-08-28T02:08:51.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:51 smithi143 conmon[109923]: +0000 mon.a (mon.0) 433 : 2023-08-28T02:08:51.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:51 smithi143 conmon[109923]: audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-28T02:08:51.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:51 smithi169 conmon[100500]: audit 2023-08-28T02:08:50.861656+0000 mon.b (mon.1 2023-08-28T02:08:51.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:51 smithi169 conmon[100500]: ) 12 : audit [INF] from='client.? 172.21.15.169:0/880401797' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "adbc3ea5-230f-419e-8f6f-99763378b296"}]: dispatch 2023-08-28T02:08:51.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:51 smithi169 conmon[100500]: audit 2023-08-28T02:08:50.862207+0000 mon.a (mon.0) 430 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "adbc3ea5-230f-419e-8f6f-99763378b296"}]: dispatch 2023-08-28T02:08:51.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:51 smithi169 conmon[100500]: audit 2023-08-28T02:08:50.867513+0000 mon.a (mon.0) 431 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "adbc3ea5-230f-419e-8f6f-99763378b296"}]': finished 2023-08-28T02:08:51.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:51 smithi169 conmon[100500]: cluster 2023-08-28T02:08:50.867609+0000 mon.a (mon.0) 432 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2023-08-28T02:08:51.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:51 smithi169 conmon[100500]: audit 2023-08-28T02:08:50.867784+0000 mon.a (mon.0) 433 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-28T02:08:52.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:52 smithi143 conmon[100258]: cluster 2023-08-28T02:08:50.943050+0000 mgr.y (mgr.14146) 125 : cluster 2023-08-28T02:08:52.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:52 smithi143 conmon[100258]: [DBG] pgmap v107: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-28T02:08:52.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:52 smithi143 conmon[100258]: audit 2023-08-28T02:08:51.478320+0000 mon.b (mon.1) 13 : audit [DBG] from='client.? 172.21.15.169:0/334359914' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-28T02:08:52.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:52 smithi143 conmon[109923]: cluster 2023-08-28T02:08:50.943050+0000 mgr.y (mgr.14146 2023-08-28T02:08:52.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:52 smithi143 conmon[109923]: ) 125 : cluster [DBG] pgmap v107: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-28T02:08:52.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:52 smithi143 conmon[109923]: audit 2023-08-28T02:08:52.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:52 smithi143 conmon[109923]: 2023-08-28T02:08:51.478320+0000 mon.b (mon.1) 13 : audit [DBG] from='client.? 172.21.15.169:0/334359914' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-28T02:08:52.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:52 smithi169 conmon[100500]: cluster 2023-08-28T02:08:50.943050+0000 mgr.y (mgr 2023-08-28T02:08:52.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:52 smithi169 conmon[100500]: .14146) 125 : cluster [DBG] pgmap v107: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-28T02:08:52.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:52 smithi169 conmon[100500]: audit 2023-08-28T02:08:51.478320+0000 mon.b (mon.1) 13 : audit [DBG] from='client.? 172.21.15.169:0/334359914' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-28T02:08:54.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:54 smithi143 conmon[100258]: cluster 2023-08-28T02:08:52.943462+0000 mgr.y (mgr.14146) 126 : cluster [DBG] pgmap v108: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-28T02:08:54.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:54 smithi143 conmon[109923]: cluster 2023-08-28T02:08:52.943462+0000 mgr.y (mgr 2023-08-28T02:08:54.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:54 smithi143 conmon[109923]: .14146) 126 : cluster [DBG] pgmap v108: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-28T02:08:54.482 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:54 smithi169 conmon[100500]: cluster 2023-08-28T02:08:52.943462+0000 mgr.y (mgr.14146) 126 : cluster [DBG] pgmap v108: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-28T02:08:56.438 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:56 smithi169 conmon[100500]: cluster 2023-08-28T02:08:54.943825+0000 2023-08-28T02:08:56.438 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:56 smithi169 conmon[100500]: mgr.y (mgr.14146) 127 : cluster [DBG] pgmap v109: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-28T02:08:56.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:56 smithi143 conmon[100258]: cluster 2023-08-28T02:08:54.943825+0000 mgr.y (mgr.14146) 127 : cluster [DBG] pgmap v109: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-28T02:08:56.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:56 smithi143 conmon[100258]: 2023-08-28T02:08:56.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:56 smithi143 conmon[109923]: cluster 2023-08-28T02:08:54.943825+0000 mgr.y (mgr.14146 2023-08-28T02:08:56.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:56 smithi143 conmon[109923]: ) 127 : cluster [DBG] pgmap v109: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-28T02:08:57.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:57 smithi143 conmon[100258]: audit 2023-08-28T02:08:57.076584+0000 mon.a (mon.0) 434 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-08-28T02:08:57.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:57 smithi143 conmon[100258]: 2023-08-28T02:08:57.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:57 smithi143 conmon[100258]: audit 2023-08-28T02:08:57.077386+0000 mon.a (mon.0) 435 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:08:57.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:57 smithi143 conmon[109923]: audit 2023-08-28T02:08:57.076584+0000 mon.a (mon.0) 2023-08-28T02:08:57.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:57 smithi143 conmon[109923]: 434 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-08-28T02:08:57.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:57 smithi143 conmon[109923]: audit 2023-08-28T02:08:57.077386+0000 mon.a (mon. 2023-08-28T02:08:57.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:57 smithi143 conmon[109923]: 0) 435 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:08:57.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:57 smithi169 conmon[100500]: audit 2023-08-28T02:08:57.076584+0000 mon.a (mon.0) 434 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-08-28T02:08:57.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:57 smithi169 conmon[100500]: audit 2023-08-28T02:08:57.077386+0000 mon.a (mon.0) 435 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:08:58.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:58 smithi169 conmon[100500]: cluster 2023-08-28T02:08:56.944345+0000 mgr.y (mgr.14146) 128 : cluster [DBG] pgmap v110: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-28T02:08:58.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:58 smithi169 conmon[100500]: cephadm 2023-08-28T02:08 2023-08-28T02:08:58.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:08:58 smithi169 conmon[100500]: :57.078025+0000 mgr.y (mgr.14146) 129 : cephadm [INF] Deploying daemon osd.5 on smithi169 2023-08-28T02:08:58.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:58 smithi143 conmon[100258]: cluster 2023-08-28T02:08:56.944345+0000 mgr.y (mgr.14146) 128 : cluster [DBG] 2023-08-28T02:08:58.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:58 smithi143 conmon[100258]: pgmap v110: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-28T02:08:58.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:08:58 smithi143 conmon[100258]: cephadm 2023-08-28T02:08:57.078025+0000 mgr.y (mgr.14146) 129 : cephadm [INF] Deploying daemon osd.5 on smithi169 2023-08-28T02:08:58.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:58 smithi143 conmon[109923]: cluster 2023-08-28T02:08:56.944345+0000 mgr.y (mgr.14146 2023-08-28T02:08:58.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:58 smithi143 conmon[109923]: ) 128 : cluster [DBG] pgmap v110: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-28T02:08:58.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:58 smithi143 conmon[109923]: cephadm 2023-08-28T02:08:57.078025 2023-08-28T02:08:58.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:08:58 smithi143 conmon[109923]: +0000 mgr.y (mgr.14146) 129 : cephadm [INF] Deploying daemon osd.5 on smithi169 2023-08-28T02:09:00.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:00 smithi143 conmon[100258]: cluster 2023-08-28T02:08:58.944908+0000 mgr.y (mgr.14146) 2023-08-28T02:09:00.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:00 smithi143 conmon[100258]: 130 : cluster [DBG] pgmap v111: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-28T02:09:00.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:00 smithi143 conmon[109923]: cluster 2023-08-28T02:08:58.944908+0000 mgr.y (mgr.14146 2023-08-28T02:09:00.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:00 smithi143 conmon[109923]: ) 130 : cluster [DBG] pgmap v111: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-28T02:09:00.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:00 smithi169 conmon[100500]: cluster 2023-08-28T02:08:58.944908+0000 mgr.y (mgr.14146 2023-08-28T02:09:00.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:00 smithi169 conmon[100500]: ) 130 : cluster [DBG] pgmap v111: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-28T02:09:02.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:02 smithi143 conmon[100258]: cluster 2023-08-28T02:09:00.945427+0000 mgr.y (mgr.14146) 131 : cluster [DBG] 2023-08-28T02:09:02.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:02 smithi143 conmon[100258]: pgmap v112: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-28T02:09:02.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:02 smithi143 conmon[100258]: audit 2023-08-28T02:09:01.910986+0000 mon.a (mon.0) 436 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:02.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:02 smithi143 conmon[100258]: audit 2023- 2023-08-28T02:09:02.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:02 smithi143 conmon[100258]: 08-28T02:09:01.934992+0000 mon.a (mon.0) 437 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:09:02.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:02 smithi143 conmon[109923]: cluster 2023-08-28T02:09:00.945427+0000 mgr.y (mgr.14146) 2023-08-28T02:09:02.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:02 smithi143 conmon[109923]: 131 : cluster [DBG] pgmap v112: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-28T02:09:02.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:02 smithi143 conmon[109923]: audit 2023-08-28T02:09:01.910986+0000 mon.a (mon.0) 436 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:02.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:02 smithi143 conmon[109923]: audit 2023-08-28T02:09:01.934992+0000 mon.a (mon.0 2023-08-28T02:09:02.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:02 smithi143 conmon[109923]: ) 437 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:09:02.470 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:02 smithi169 conmon[100500]: cluster 2023-08-28T02:09:00.945427+0000 mgr.y (mgr 2023-08-28T02:09:02.470 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:02 smithi169 conmon[100500]: .14146) 131 : cluster [DBG] pgmap v112: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-28T02:09:02.470 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:02 smithi169 conmon[100500]: audit 2023-08-28T02:09:01.910986+0000 mon.a (mon.0) 436 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:02.470 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:02 smithi169 conmon[100500]: audit 2023-08-28T02:09:01.934992+0000 mon.a (mon.0) 437 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:09:04.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:04 smithi143 conmon[100258]: cluster 2023-08-28T02:09:02.945947+0000 mgr.y (mgr.14146) 132 : cluster [DBG] 2023-08-28T02:09:04.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:04 smithi143 conmon[100258]: pgmap v113: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-28T02:09:04.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:04 smithi143 conmon[100258]: audit 2023-08-28T02:09:03.596131+0000 mon.a (mon.0) 438 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:04.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:04 smithi143 conmon[100258]: audit 2023-08-28T02:09:03.596905+0000 mon.a (mon.0) 439 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:09:04.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:04 smithi143 conmon[100258]: audit 2023-08 2023-08-28T02:09:04.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:04 smithi143 conmon[100258]: -28T02:09:03.597769+0000 mon.a (mon.0) 440 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:09:04.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:04 smithi143 conmon[100258]: audit 2023-08-28T02:09:03.603888+0000 mon.a (mon.0) 441 : audit [INF] 2023-08-28T02:09:04.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:04 smithi143 conmon[100258]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:04.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:04 smithi143 conmon[109923]: cluster 2023-08-28T02:09:02.945947+0000 mgr.y (mgr.14146) 132 : 2023-08-28T02:09:04.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:04 smithi143 conmon[109923]: cluster [DBG] pgmap v113: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-28T02:09:04.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:04 smithi143 conmon[109923]: audit 2023-08-28T02:09:03.596131+0000 mon.a (mon.0) 438 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:04.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:04 smithi143 conmon[109923]: audit 2023-08-28T02:09: 2023-08-28T02:09:04.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:04 smithi143 conmon[109923]: 03.596905+0000 mon.a (mon.0) 439 : audit [DBG] 2023-08-28T02:09:04.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:04 smithi143 conmon[109923]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:09:04.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:04 smithi143 conmon[109923]: audit 2023-08-28T02:09 2023-08-28T02:09:04.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:04 smithi143 conmon[109923]: :03.597769+0000 mon.a (mon.0) 440 : audit 2023-08-28T02:09:04.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:04 smithi143 conmon[109923]: [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:09:04.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:04 smithi143 conmon[109923]: audit 2023-08-28T 2023-08-28T02:09:04.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:04 smithi143 conmon[109923]: 02:09:03.603888+0000 mon.a (mon 2023-08-28T02:09:04.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:04 smithi143 conmon[109923]: .0) 441 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:04.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:04 smithi169 conmon[100500]: cluster 2023-08-28T02:09:02.945947+0000 mgr.y (mgr.14146) 132 : cluster [DBG] pgmap v113: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-28T02:09:04.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:04 smithi169 conmon[100500]: audit 2023-08-28T02:09:03.596131+0000 mon.a (mon.0) 438 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:04.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:04 smithi169 conmon[100500]: audit 2023-08-28T02:09:03.596905+0000 mon.a (mon.0) 439 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:09:04.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:04 smithi169 conmon[100500]: audit 2023-08-28T02:09:03.597769+0000 mon.a (mon.0) 440 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:09:04.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:04 smithi169 conmon[100500]: audit 2023-08-28T02:09:03.603888+0000 mon.a (mon.0) 441 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:04.802 INFO:teuthology.orchestra.run.smithi169.stdout:Created osd(s) 5 on host 'smithi169' 2023-08-28T02:09:05.388 DEBUG:teuthology.orchestra.run.smithi169:osd.5> sudo journalctl -f -n 0 -u ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@osd.5.service 2023-08-28T02:09:05.391 INFO:tasks.cephadm:Deploying osd.6 on smithi169 with /dev/vg_nvme/lv_2... 2023-08-28T02:09:05.391 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-08-28T02:09:05.422 INFO:journalctl@ceph.osd.5.smithi169.stdout:-- Logs begin at Mon 2023-08-28 01:49:52 UTC. -- 2023-08-28T02:09:05.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:05 smithi169 conmon[100500]: audit 2023-08-28T02:09:04.803095+0000 mon.a (mon. 2023-08-28T02:09:05.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:05 smithi169 conmon[100500]: 0) 442 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:05.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:05 smithi169 conmon[100500]: audit 2023-08-28T02:09:04.823041+0000 mon.a (mon.0) 443 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:09:05.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:05 smithi169 conmon[100500]: cluster 2023-08-28T02:09:04.946220+0000 mgr.y (mgr.14146) 133 : cluster [DBG] pgmap v114: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-28T02:09:05.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:05 smithi169 conmon[100500]: audit 2023-08-28T02:09:05.125835+0000 mon.b (mon.1) 14 : audit [INF] from='osd.5 [v2:172.21.15.169:6808/4087672846,v1:172.21.15.169:6809/4087672846]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-08-28T02:09:05.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:05 smithi169 conmon[100500]: audit 2023-08-28T02:09:05.129438+0000 mon.a (mon.0) 444 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-08-28T02:09:06.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:05 smithi143 conmon[100258]: audit 2023-08-28T02:09:04.803095+0000 mon.a (mon.0) 442 : audit 2023-08-28T02:09:06.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:05 smithi143 conmon[100258]: [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:06.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:05 smithi143 conmon[100258]: audit 2023-08-28T02:09:04.823041+0000 mon.a (mon.0) 443 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:09:06.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:05 smithi143 conmon[100258]: cluster 2023-08-28 2023-08-28T02:09:06.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:05 smithi143 conmon[100258]: T02:09:04.946220+0000 mgr.y (mgr.14146) 133 : cluster [DBG] pgmap v114: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-28T02:09:06.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:05 smithi143 conmon[100258]: audit 2023-08-28T02:09:05.125835+0000 mon.b 2023-08-28T02:09:06.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:05 smithi143 conmon[100258]: (mon.1) 14 : audit [INF] from='osd.5 [v2:172.21.15.169:6808/4087672846,v1:172.21.15.169:6809/4087672846]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-08-28T02:09:06.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:05 smithi143 conmon[100258]: audit 2023 2023-08-28T02:09:06.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:05 smithi143 conmon[100258]: -08-28T02:09:05.129438+0000 mon.a (mon. 2023-08-28T02:09:06.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:05 smithi143 conmon[100258]: 0) 444 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-08-28T02:09:06.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:05 smithi143 conmon[109923]: audit 2023-08-28T02:09:04.803095+0000 2023-08-28T02:09:06.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:05 smithi143 conmon[109923]: mon.a (mon.0) 442 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:06.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:05 smithi143 conmon[109923]: audit 2023-08 2023-08-28T02:09:06.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:05 smithi143 conmon[109923]: -28T02:09:04.823041+0000 mon.a (mon.0) 443 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:09:06.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:05 smithi143 conmon[109923]: cluster 2023-08-28T02:09:06.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:05 smithi143 conmon[109923]: 2023-08-28T02:09:04.946220+0000 mgr.y (mgr.14146) 2023-08-28T02:09:06.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:05 smithi143 conmon[109923]: 133 : cluster [DBG] pgmap v114: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-28T02:09:06.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:05 smithi143 conmon[109923]: audit 2023-08- 2023-08-28T02:09:06.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:05 smithi143 conmon[109923]: 28T02:09:05.125835+0000 mon.b (mon.1) 14 : audit 2023-08-28T02:09:06.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:05 smithi143 conmon[109923]: [INF] from='osd.5 [v2:172.21.15.169:6808/4087672846,v1:172.21.15.169:6809/4087672846]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-08-28T02:09:06.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:05 smithi143 conmon[109923]: audit 2023-08-28T02:09:06.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:05 smithi143 conmon[109923]: 2023-08-28T02:09: 2023-08-28T02:09:06.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:05 smithi143 conmon[109923]: 05.129438+0000 mon.a (mon.0 2023-08-28T02:09:06.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:05 smithi143 conmon[109923]: ) 444 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-08-28T02:09:07.360 INFO:journalctl@ceph.osd.5.smithi169.stdout:Aug 28 02:09:07 smithi169 conmon[112660]: debug 2023-08-28T02:09:07.141+0000 7f2401736700 -1 osd.5 0 waiting for initial osdmap 2023-08-28T02:09:07.360 INFO:journalctl@ceph.osd.5.smithi169.stdout:Aug 28 02:09:07 smithi169 conmon[112660]: debug 2023-08-28T02:09:07.146+0000 7f23fdeb2700 -1 osd.5 35 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-08-28T02:09:07.360 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:07 smithi169 conmon[100500]: audit 2023-08-28T02:09:06.131068+0000 mon.a (mon.0) 445 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-08-28T02:09:07.360 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:07 smithi169 conmon[100500]: cluster 2023-08-28T02:09:06.131184+0000 mon.a (mon.0) 446 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2023-08-28T02:09:07.360 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:07 smithi169 conmon[100500]: audit 2023-08-28T02:09:06.131345+0000 mon.a (mon.0) 447 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-28T02:09:07.361 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:07 smithi169 conmon[100500]: audit 2023-08-28T02:09:06.131739+0000 mon.b (mon.1) 15 : audit [INF] from='osd.5 [v2:172.21.15.169:6808/4087672846,v1:172.21.15.169:6809/4087672846]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2023-08-28T02:09:07.361 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:07 smithi169 conmon[100500]: audit 2023-08-28T02:09:06.132033+0000 mon.a (mon.0) 448 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2023-08-28T02:09:07.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:07 smithi143 conmon[100258]: audit 2023-08-28T02:09:06.131068+0000 mon.a (mon.0) 445 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-08-28T02:09:07.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:07 smithi143 conmon[100258]: cluster 2023-08-28T02:09:06.131184+0000 mon.a (mon.0) 446 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2023-08-28T02:09:07.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:07 smithi143 conmon[100258]: audit 2023-08-28T02:09:06.131345+0000 mon.a (mon.0) 447 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-28T02:09:07.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:07 smithi143 conmon[100258]: audit 2023-08-28T02:09:06.131739+0000 mon.b (mon.1) 15 : audit [INF] from='osd.5 [v2:172.21.15.169:6808/4087672846,v1:172.21.15.169:6809/4087672846]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2023-08-28T02:09:07.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:07 smithi143 conmon[100258]: 2023-08-28T02:09:07.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:07 smithi143 conmon[100258]: audit 2023-08-28T02:09:06.132033+0000 2023-08-28T02:09:07.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:07 smithi143 conmon[100258]: mon.a (mon.0) 448 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2023-08-28T02:09:07.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:07 smithi143 conmon[109923]: audit 2023-08-28T02:09:06.131068+0000 mon.a (mon. 2023-08-28T02:09:07.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:07 smithi143 conmon[109923]: 0) 445 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-08-28T02:09:07.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:07 smithi143 conmon[109923]: cluster 2023-08-28T02:09:06.131184+0000 mon.a (mon.0) 2023-08-28T02:09:07.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:07 smithi143 conmon[109923]: 446 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2023-08-28T02:09:07.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:07 smithi143 conmon[109923]: audit 2023-08-28T02:09: 2023-08-28T02:09:07.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:07 smithi143 conmon[109923]: 06.131345+0000 mon.a (mon.0) 447 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-28T02:09:07.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:07 smithi143 conmon[109923]: audit 2023-08-28T02:09 2023-08-28T02:09:07.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:07 smithi143 conmon[109923]: :06.131739+0000 mon.b (mon.1) 15 : audit [INF] from='osd.5 [v2:172.21.15.169:6808/4087672846,v1:172.21.15.169:6809/4087672846]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2023-08-28T02:09:07.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:07 smithi143 conmon[109923]: audit 2023-08-28T02: 2023-08-28T02:09:07.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:07 smithi143 conmon[109923]: 09:06.132033+0000 mon.a (mon.0) 448 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2023-08-28T02:09:07.906 INFO:teuthology.orchestra.run.smithi169.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-08-28T02:09:07.906 INFO:teuthology.orchestra.run.smithi169.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-08-28T02:09:07.906 INFO:teuthology.orchestra.run.smithi169.stderr: stderr: 10+0 records in 2023-08-28T02:09:07.906 INFO:teuthology.orchestra.run.smithi169.stderr:10+0 records out 2023-08-28T02:09:07.907 INFO:teuthology.orchestra.run.smithi169.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0113726 s, 922 MB/s 2023-08-28T02:09:07.907 INFO:teuthology.orchestra.run.smithi169.stderr:--> Zapping successful for: 2023-08-28T02:09:08.227 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:08 smithi169 conmon[100500]: cluster 2023-08-28T02:09:06.116409+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2023-08-28T02:09:08.227 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:08 smithi169 conmon[100500]: cluster 2023-08-28T02:09:06.116510+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-08-28T02:09:08.227 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:08 smithi169 conmon[100500]: cluster 2023-08-28T02:09:06.946661+0000 mgr.y (mgr.14146) 134 : cluster [DBG] pgmap v116: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-28T02:09:08.227 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:08 smithi169 conmon[100500]: audit 2023-08-28T02:09:07.132917+0000 mon.a (mon.0) 449 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]': finished 2023-08-28T02:09:08.227 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:08 smithi169 conmon[100500]: cluster 2023-08-28T02:09:07.133061+0000 mon.a (mon.0) 450 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2023-08-28T02:09:08.227 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:08 smithi169 conmon[100500]: audit 2023-08-28T02:09:07.133509+0000 mon.a (mon.0) 451 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-28T02:09:08.228 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:08 smithi169 conmon[100500]: audit 2023-08-28T02:09:07.139411+0000 mon.a (mon.0) 452 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-28T02:09:08.228 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:08 smithi169 conmon[100500]: cluster 2023-08-28T02:09:08.134180+0000 mon.a (mon.0) 453 : cluster [INF] osd.5 [v2:172.21.15.169:6808/4087672846,v1:172.21.15.169:6809/4087672846] boot 2023-08-28T02:09:08.228 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:08 smithi169 conmon[100500]: cluster 2023-08-28T02:09:08.134238+0000 mon.a (mon.0) 454 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2023-08-28T02:09:08.228 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:08 smithi169 conmon[100500]: audit 2023-08-28T02:09:08.134406+0000 mon.a (mon.0) 455 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-28T02:09:08.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:08 smithi143 conmon[100258]: cluster 2023-08-28T02:09:06.116409+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2023-08-28T02:09:08.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:08 smithi143 conmon[100258]: cluster 2023-08-28T02:09:06.116510+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-08-28T02:09:08.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:08 smithi143 conmon[100258]: cluster 2023-08-28T02:09:06.946661+0000 mgr.y (mgr.14146) 134 : cluster [DBG] pgmap v116: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-28T02:09:08.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:08 smithi143 conmon[100258]: audit 2023-08-28T02:09:07.132917+0000 mon.a (mon.0) 449 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]': finished 2023-08-28T02:09:08.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:08 smithi143 conmon[100258]: cluster 2023-08-28T02:09:07.133061+0000 mon.a (mon.0) 450 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2023-08-28T02:09:08.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:08 smithi143 conmon[100258]: audit 2023-08-28T02:09:07.133509+0000 mon.a (mon.0) 451 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-28T02:09:08.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:08 smithi143 conmon[100258]: audit 2023-08-28T02: 2023-08-28T02:09:08.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:08 smithi143 conmon[100258]: 09:07.139411+0000 mon.a (mon.0) 452 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-28T02:09:08.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:08 smithi143 conmon[100258]: cluster 2023-08-28T02:09:08.134180+0000 mon.a (mon.0) 453 : cluster [INF] osd.5 [v2:172.21.15.169:6808/4087672846,v1:172.21.15.169:6809/4087672846] boot 2023-08-28T02:09:08.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:08 smithi143 conmon[100258]: cluster 2023-08-28T02:09:08.134238+0000 mon.a (mon.0) 454 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2023-08-28T02:09:08.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:08 smithi143 conmon[100258]: audit 2023-08-28T02:09:08.134406+0000 mon.a (mon.0) 455 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-28T02:09:08.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:08 smithi143 conmon[109923]: cluster 2023-08-28T02:09:06.116409+0000 osd.5 (osd.5) 1 : cluster 2023-08-28T02:09:08.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:08 smithi143 conmon[109923]: [DBG] purged_snaps scrub starts 2023-08-28T02:09:08.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:08 smithi143 conmon[109923]: cluster 2023-08-28T02:09:06.116510+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-08-28T02:09:08.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:08 smithi143 conmon[109923]: cluster 2023-08-28 2023-08-28T02:09:08.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:08 smithi143 conmon[109923]: T02:09:06.946661+0000 mgr.y (mgr.14146) 134 : cluster [DBG] pgmap v116: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-08-28T02:09:08.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:08 smithi143 conmon[109923]: audit 2023-08-28T02:09: 2023-08-28T02:09:08.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:08 smithi143 conmon[109923]: 07.132917+0000 mon.a (mon.0) 449 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]': finished 2023-08-28T02:09:08.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:08 smithi143 conmon[109923]: cluster 2023-08-28T02:09:07.133061+0000 mon.a (mon. 2023-08-28T02:09:08.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:08 smithi143 conmon[109923]: 0) 450 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2023-08-28T02:09:08.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:08 smithi143 conmon[109923]: audit 2023-08-28T02:09:07.133509+0000 mon.a (mon. 2023-08-28T02:09:08.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:08 smithi143 conmon[109923]: 0) 451 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-28T02:09:08.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:08 smithi143 conmon[109923]: audit 2023-08-28T02:09:07.139411+0000 mon.a (mon. 2023-08-28T02:09:08.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:08 smithi143 conmon[109923]: 0) 452 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-28T02:09:08.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:08 smithi143 conmon[109923]: cluster 2023-08-28T02:09:08.134180+0000 mon.a ( 2023-08-28T02:09:08.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:08 smithi143 conmon[109923]: mon.0) 453 : cluster [INF] osd.5 [v2:172.21.15.169:6808/4087672846,v1:172.21.15.169:6809/4087672846] boot 2023-08-28T02:09:08.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:08 smithi143 conmon[109923]: cluster 2023-08-28T02:09:08.134238+0000 mon.a 2023-08-28T02:09:08.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:08 smithi143 conmon[109923]: (mon.0) 454 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2023-08-28T02:09:08.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:08 smithi143 conmon[109923]: audit 2023-08-28T02:09:08.134406+0000 mon.a ( 2023-08-28T02:09:08.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:08 smithi143 conmon[109923]: mon.0) 455 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-28T02:09:09.008 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph orch daemon add osd smithi169:vg_nvme/lv_2 2023-08-28T02:09:10.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:10 smithi143 conmon[100258]: cluster 2023-08-28T02:09:08.947193+0000 mgr.y (mgr.14146) 135 : cluster [DBG] pgmap v119: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-28T02:09:10.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:10 smithi143 conmon[100258]: 2023-08-28T02:09:10.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:10 smithi143 conmon[100258]: cluster 2023-08-28T02:09:09.138690+0000 mon.a (mon.0) 456 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2023-08-28T02:09:10.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:10 smithi143 conmon[100258]: cephadm 2023-08-28T02:09:09.427423+0000 mgr.y (mgr.14146) 136 : cephadm [INF] Detected new or changed devices on smithi169 2023-08-28T02:09:10.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:10 smithi143 conmon[100258]: audit 2023-08-28T02:09:10.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:10 smithi143 conmon[100258]: 2023-08-28T02:09:09.433706+0000 mon.a (mon.0) 457 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:10.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:10 smithi143 conmon[100258]: audit 2023-08-28T02:09:10.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:10 smithi143 conmon[100258]: 2023-08-28T02:09:09.434804+0000 mon.a (mon.0) 458 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:09:10.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:10 smithi143 conmon[100258]: audit 2023-08-28T02:09:09 2023-08-28T02:09:10.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:10 smithi143 conmon[100258]: .435695+0000 mon.a (mon.0) 459 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:09:10.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:10 smithi143 conmon[100258]: cephadm 2023-08-28T02: 2023-08-28T02:09:10.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:10 smithi143 conmon[100258]: 09:09.436267+0000 mgr.y (mgr.14146) 137 : cephadm [INF] Adjusting osd_memory_target on smithi169 to 8584M 2023-08-28T02:09:10.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:10 smithi143 conmon[100258]: audit 2023-08-28T02:09:10.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:10 smithi143 conmon[100258]: 2023-08-28T02:09:09.442159+0000 mon.a (mon.0) 460 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:10.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:10 smithi143 conmon[100258]: audit 2023-08-28T02:09: 2023-08-28T02:09:10.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:10 smithi143 conmon[100258]: 09.443173+0000 mon.a (mon.0) 461 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:09:10.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:10 smithi143 conmon[100258]: audit 2023-08-28T02:09:09.444042+0000 2023-08-28T02:09:10.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:10 smithi143 conmon[100258]: mon.a (mon.0) 462 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:09:10.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:10 smithi143 conmon[100258]: audit 2023-08-28T02: 2023-08-28T02:09:10.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:10 smithi143 conmon[100258]: 09:09.450438+0000 mon.a (mon.0) 463 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:10.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:10 smithi143 conmon[109923]: cluster 2023-08-28T02:09:08.947193+0000 mgr.y (mgr.14146) 135 : 2023-08-28T02:09:10.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:10 smithi143 conmon[109923]: cluster [DBG] pgmap v119: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-28T02:09:10.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:10 smithi143 conmon[109923]: cluster 2023-08-28T 2023-08-28T02:09:10.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:10 smithi143 conmon[109923]: 02:09:09.138690+0000 mon.a (mon.0) 456 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2023-08-28T02:09:10.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:10 smithi143 conmon[109923]: cephadm 2023-08-28T02:09:10.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:10 smithi143 conmon[109923]: 2023-08-28T02:09:09.427423+0000 mgr.y (mgr.14146) 136 : 2023-08-28T02:09:10.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:10 smithi143 conmon[109923]: cephadm [INF] Detected new or changed devices on smithi169 2023-08-28T02:09:10.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:10 smithi143 conmon[109923]: audit 2023-08-28T02:09:10.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:10 smithi143 conmon[109923]: 2023-08-28T02:09:09.433706+0000 mon.a (mon.0) 457 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:10.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:10 smithi143 conmon[109923]: audit 2023-08-28T02:09:09.434804+0000 2023-08-28T02:09:10.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:10 smithi143 conmon[109923]: mon.a (mon.0) 458 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:09:10.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:10 smithi143 conmon[109923]: audit 2023-08-28T02:09:09.435695 2023-08-28T02:09:10.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:10 smithi143 conmon[109923]: +0000 mon.a (mon.0) 459 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:09:10.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:10 smithi143 conmon[109923]: cephadm 2023-08-28T02:09:09.436267+0000 mgr.y (mgr. 2023-08-28T02:09:10.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:10 smithi143 conmon[109923]: 14146) 137 : cephadm [INF] Adjusting osd_memory_target on smithi169 to 8584M 2023-08-28T02:09:10.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:10 smithi143 conmon[109923]: audit 2023-08-28T02:09:09.442159+0000 2023-08-28T02:09:10.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:10 smithi143 conmon[109923]: mon.a (mon.0) 460 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:10.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:10 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:09:10.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:10 smithi143 conmon[109923]: 08-28T02:09:09.443173+0000 mon.a (mon.0) 461 : audit [DBG] 2023-08-28T02:09:10.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:10 smithi143 conmon[109923]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:09:10.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:10 smithi143 conmon[109923]: audit 2023-08 2023-08-28T02:09:10.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:10 smithi143 conmon[109923]: -28T02:09:09.444042+0000 mon.a (mon.0) 462 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:09:10.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:10 smithi143 conmon[109923]: audit 2023-08-28T02:09:09. 2023-08-28T02:09:10.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:10 smithi143 conmon[109923]: 450438+0000 mon.a (mon.0) 463 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:10.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:10 smithi169 conmon[100500]: cluster 2023-08-28T02:09:08.947193+0000 mgr.y (mgr.14146) 135 : cluster [DBG] pgmap v119: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-28T02:09:10.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:10 smithi169 conmon[100500]: cluster 2023-08-28T02:09:09.138690+0000 mon.a (mon.0) 456 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2023-08-28T02:09:10.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:10 smithi169 conmon[100500]: cephadm 2023-08-28T02:09:09.427423+0000 mgr.y (mgr.14146) 136 : cephadm [INF] Detected new or changed devices on smithi169 2023-08-28T02:09:10.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:10 smithi169 conmon[100500]: audit 2023-08-28T02:09:09.433706+0000 mon.a (mon.0) 457 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:10.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:10 smithi169 conmon[100500]: audit 2023-08-28T02:09:09.434804+0000 mon.a (mon.0) 458 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:09:10.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:10 smithi169 conmon[100500]: audit 2023-08-28T02:09:09.435695+0000 mon.a (mon.0) 459 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:09:10.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:10 smithi169 conmon[100500]: cephadm 2023-08-28T02:09:09.436267+0000 mgr.y (mgr.14146) 137 : cephadm [INF] Adjusting osd_memory_target on smithi169 to 8584M 2023-08-28T02:09:10.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:10 smithi169 conmon[100500]: audit 2023-08-28T02:09:09.442159+0000 mon.a (mon.0) 460 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:10.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:10 smithi169 conmon[100500]: audit 2023-08-28T02:09:09.443173+0000 mon.a (mon.0) 461 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:09:10.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:10 smithi169 conmon[100500]: audit 2023-08-28T 2023-08-28T02:09:10.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:10 smithi169 conmon[100500]: 02:09:09.444042+0000 mon.a (mon.0) 462 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:09:10.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:10 smithi169 conmon[100500]: audit 2023-08-28T02:09:09.450438+0000 mon.a (mon.0) 463 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:11.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:11 smithi143 conmon[100258]: cluster 2023-08-28T02:09:10.148541+0000 mon.a (mon.0) 464 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-08-28T02:09:11.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:11 smithi143 conmon[100258]: audit 2023-08-28T02:09:10.923835+0000 mon.a (mon.0) 465 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-28T02:09:11.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:11 smithi143 conmon[100258]: audit 2023-08-28T02:09:10.926404+0000 mon.a (mon.0) 466 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-28T02:09:11.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:11 smithi143 conmon[100258]: audit 2023-08-28T02:09 2023-08-28T02:09:11.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:11 smithi143 conmon[100258]: :10.927189+0000 mon.a (mon.0) 467 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:09:11.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:11 smithi143 conmon[109923]: cluster 2023-08-28T02:09:10.148541+0000 mon.a (mon.0) 464 : 2023-08-28T02:09:11.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:11 smithi143 conmon[109923]: cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-08-28T02:09:11.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:11 smithi143 conmon[109923]: audit 2023-08-28T02:09:10.923835+0000 mon.a (mon.0) 465 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-28T02:09:11.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:11 smithi143 conmon[109923]: audit 2023-08-28T02:09:10.926404+0000 mon.a (mon.0) 466 : audit 2023-08-28T02:09:11.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:11 smithi143 conmon[109923]: [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-28T02:09:11.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:11 smithi143 conmon[109923]: audit 2023-08-28T02:09:10.927189 2023-08-28T02:09:11.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:11 smithi143 conmon[109923]: +0000 mon.a (mon.0) 467 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:09:11.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:11 smithi143 conmon[109923]: 2023-08-28T02:09:11.473 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:11 smithi169 conmon[100500]: cluster 2023-08-28T02:09:10.148541+0000 mon.a (mon.0) 2023-08-28T02:09:11.473 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:11 smithi169 conmon[100500]: 464 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-08-28T02:09:11.473 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:11 smithi169 conmon[100500]: audit 2023-08-28T02:09:10.923835+0000 mon.a (mon.0) 465 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-28T02:09:11.473 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:11 smithi169 conmon[100500]: audit 2023-08-28T02:09:10.926404+0000 mon.a (mon.0) 466 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-28T02:09:11.473 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:11 smithi169 conmon[100500]: audit 2023-08-28T02:09:10.927189+0000 mon.a (mon.0) 467 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:09:12.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:12 smithi143 conmon[100258]: audit 2023-08-28T02:09:10.921543+0000 mgr.y (mgr.14146) 138 : audit [DBG] from='client.24256 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi169:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:09:12.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:12 smithi143 conmon[100258]: cluster 2023-08-28T02:09:10.947514+0000 mgr.y (mgr.14146) 139 : cluster [DBG] pgmap v122: 1 pgs: 1 remapped+peering; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-28T02:09:12.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:12 smithi143 conmon[109923]: audit 2023-08-28T02:09:10.921543+0000 mgr.y (mgr.14146) 138 : audit [DBG] from='client.24256 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi169:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:09:12.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:12 smithi143 conmon[109923]: 2023-08-28T02:09:12.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:12 smithi143 conmon[109923]: cluster 2023-08-28T02:09:10.947514+0000 mgr.y (mgr.14146) 139 : cluster [DBG] pgmap v122: 1 pgs: 1 remapped+peering; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-28T02:09:12.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:12 smithi169 conmon[100500]: audit 2023-08-28T02:09:10.921543+0000 mgr.y (mgr.14146) 138 : 2023-08-28T02:09:12.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:12 smithi169 conmon[100500]: audit [DBG] from='client.24256 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi169:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:09:12.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:12 smithi169 conmon[100500]: cluster 2023-08-28T02:09:10.947514+0000 mgr.y (mgr.14146) 139 : cluster [DBG] pgmap v122: 1 pgs: 1 remapped+peering; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-28T02:09:14.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:14 smithi143 conmon[100258]: cluster 2023-08-28T02:09:12.947987+0000 mgr.y (mgr.14146) 140 : cluster [DBG] pgmap v123: 1 pgs: 1 remapped+peering; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-28T02:09:14.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:14 smithi143 conmon[100258]: audit 2023-08-28T02:09:13.194045+0000 mon.b (mon.1) 16 : audit [INF] from='client.? 172.21.15.169:0/2914552258' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2abe4d77-c12c-477f-a4ec-eabb753ca4dd"}]: dispatch 2023-08-28T02:09:14.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:14 smithi143 conmon[100258]: audit 2023-08-28T02:09:13.194552+0000 mon.a (mon.0) 468 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2abe4d77-c12c-477f-a4ec-eabb753ca4dd"}]: dispatch 2023-08-28T02:09:14.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:14 smithi143 conmon[100258]: audit 2023-08-28T02:09:13.198910+0000 mon.a 2023-08-28T02:09:14.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:14 smithi143 conmon[100258]: (mon.0) 469 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2abe4d77-c12c-477f-a4ec-eabb753ca4dd"}]': finished 2023-08-28T02:09:14.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:14 smithi143 conmon[100258]: cluster 2023-08-28T02:09:13.198997+0000 mon.a (mon.0) 470 : cluster [DBG] 2023-08-28T02:09:14.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:14 smithi143 conmon[100258]: osdmap e39: 7 total, 6 up, 7 in 2023-08-28T02:09:14.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:14 smithi143 conmon[100258]: audit 2023-08-28T02:09:13.199167 2023-08-28T02:09:14.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:14 smithi143 conmon[100258]: +0000 mon.a (mon.0) 471 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-28T02:09:14.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:14 smithi143 conmon[100258]: audit 2023-08-28T02:09:14.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:14 smithi143 conmon[100258]: 2023-08-28T02:09:13.772583+0000 mon.b (mon.1) 17 : audit [DBG] 2023-08-28T02:09:14.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:14 smithi143 conmon[100258]: from='client.? 172.21.15.169:0/528467546' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-28T02:09:14.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:14 smithi143 conmon[109923]: cluster 2023-08-28T02:09:12.947987+0000 mgr.y (mgr.14146) 140 : cluster [DBG] pgmap v123: 1 pgs: 1 remapped+peering; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-28T02:09:14.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:14 smithi143 conmon[109923]: audit 2023-08-28T02:09:13.194045+0000 mon.b (mon.1) 16 : audit [INF] from='client.? 172.21.15.169:0/2914552258' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2abe4d77-c12c-477f-a4ec-eabb753ca4dd"}]: dispatch 2023-08-28T02:09:14.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:14 smithi143 conmon[109923]: audit 2023-08-28T02:09:13.194552+0000 mon.a (mon. 2023-08-28T02:09:14.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:14 smithi143 conmon[109923]: 0) 468 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2abe4d77-c12c-477f-a4ec-eabb753ca4dd"}]: dispatch 2023-08-28T02:09:14.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:14 smithi143 conmon[109923]: audit 2023-08-28T02:09:13.198910+0000 mon.a (mon.0 2023-08-28T02:09:14.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:14 smithi143 conmon[109923]: ) 469 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2abe4d77-c12c-477f-a4ec-eabb753ca4dd"}]': finished 2023-08-28T02:09:14.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:14 smithi143 conmon[109923]: cluster 2023-08-28T02:09 2023-08-28T02:09:14.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:14 smithi143 conmon[109923]: :13.198997+0000 mon.a (mon.0) 470 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2023-08-28T02:09:14.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:14 smithi143 conmon[109923]: audit 2023-08-28T02:09:14.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:14 smithi143 conmon[109923]: 2023-08-28T02:09:13.199167+0000 mon.a (mon.0) 471 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-28T02:09:14.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:14 smithi143 conmon[109923]: audit 2023-08-28T02:09:13. 2023-08-28T02:09:14.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:14 smithi143 conmon[109923]: 772583+0000 mon.b (mon.1) 17 : audit [DBG] from='client.? 172.21.15.169:0/528467546' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-28T02:09:14.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:14 smithi169 conmon[100500]: cluster 2023-08-28T02:09:12.947987+0000 mgr.y (mgr.14146) 140 : cluster [DBG] pgmap v123: 1 pgs: 1 remapped+peering; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-28T02:09:14.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:14 smithi169 conmon[100500]: audit 2023-08-28T02:09:13.194045+0000 mon.b (mon.1) 16 : audit [INF] from='client.? 172.21.15.169:0/2914552258' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2abe4d77-c12c-477f-a4ec-eabb753ca4dd"}]: dispatch 2023-08-28T02:09:14.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:14 smithi169 conmon[100500]: audit 2023-08-28T02:09:13.194552+0000 mon.a (mon.0) 468 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2abe4d77-c12c-477f-a4ec-eabb753ca4dd"}]: dispatch 2023-08-28T02:09:14.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:14 smithi169 conmon[100500]: audit 2023-08-28T02:09:13.198910+0000 mon.a (mon.0) 469 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2abe4d77-c12c-477f-a4ec-eabb753ca4dd"}]': finished 2023-08-28T02:09:14.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:14 smithi169 conmon[100500]: cluster 2023-08-28T02:09:13.198997+0000 mon.a (mon.0) 470 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2023-08-28T02:09:14.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:14 smithi169 conmon[100500]: audit 2023-08-28T02:09:13.199167+0000 mon.a (mon.0) 471 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-28T02:09:14.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:14 smithi169 conmon[100500]: audit 2023-08-28T02:09:13.772583+0000 mon.b (mon.1) 17 : audit [DBG] from='client.? 172.21.15.169:0/528467546' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-28T02:09:16.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:16 smithi143 conmon[100258]: cluster 2023-08-28T02:09:14.948454+0000 mgr.y (mgr.14146) 141 : cluster [DBG] pgmap v125: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-28T02:09:16.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:16 smithi143 conmon[109923]: cluster 2023-08-28T02:09:14.948454+0000 mgr.y (mgr.14146) 141 : cluster 2023-08-28T02:09:16.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:16 smithi143 conmon[109923]: [DBG] pgmap v125: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-28T02:09:16.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:16 smithi169 conmon[100500]: cluster 2023-08-28T02:09:14.948454+0000 mgr.y (mgr.14146) 141 : cluster [DBG] pgmap v125: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-28T02:09:18.224 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:18 smithi169 conmon[100500]: cluster 2023-08-28T02:09:16.948813+0000 mgr.y (mgr.14146) 142 : cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-28T02:09:18.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:18 smithi143 conmon[109923]: cluster 2023-08-28T02:09:16.948813+0000 mgr.y (mgr.14146 2023-08-28T02:09:18.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:18 smithi143 conmon[109923]: ) 142 : cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-28T02:09:18.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:18 smithi143 conmon[100258]: cluster 2023-08-28T02:09:16.948813+0000 mgr.y (mgr.14146) 142 : cluster [DBG] 2023-08-28T02:09:18.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:18 smithi143 conmon[100258]: pgmap v126: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-28T02:09:20.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:20 smithi143 conmon[100258]: cluster 2023-08-28T02:09:18.949202+0000 mgr.y (mgr.14146) 143 : cluster [DBG] pgmap v127: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-28T02:09:20.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:20 smithi143 conmon[100258]: 2023-08-28T02:09:20.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:20 smithi143 conmon[100258]: audit 2023-08-28T02:09:19.298458+0000 mon.a (mon.0) 472 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-08-28T02:09:20.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:20 smithi143 conmon[100258]: audit 2023-08-28T02:09:19.299386+0000 mon.a (mon. 2023-08-28T02:09:20.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:20 smithi143 conmon[100258]: 0) 473 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:09:20.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:20 smithi143 conmon[100258]: cephadm 2023-08-28 2023-08-28T02:09:20.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:20 smithi143 conmon[100258]: T02:09:19.300085+0000 mgr.y (mgr.14146 2023-08-28T02:09:20.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:20 smithi143 conmon[100258]: ) 144 : cephadm [INF] Deploying daemon osd.6 on smithi169 2023-08-28T02:09:20.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:20 smithi143 conmon[109923]: cluster 2023-08-28T02:09:18.949202+0000 mgr.y (mgr. 2023-08-28T02:09:20.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:20 smithi143 conmon[109923]: 14146) 143 : cluster [DBG] pgmap v127: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-28T02:09:20.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:20 smithi143 conmon[109923]: audit 2023-08-28T02:09:19.298458+0000 mon.a (mon 2023-08-28T02:09:20.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:20 smithi143 conmon[109923]: .0) 472 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-08-28T02:09:20.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:20 smithi143 conmon[109923]: audit 2023 2023-08-28T02:09:20.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:20 smithi143 conmon[109923]: -08-28T02:09:19.299386+0000 mon.a (mon.0) 473 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:09:20.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:20 smithi143 conmon[109923]: 2023-08-28T02:09:20.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:20 smithi143 conmon[109923]: cephadm 2023-08- 2023-08-28T02:09:20.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:20 smithi143 conmon[109923]: 28T02:09:19.300085+0000 mgr.y (mgr.14146) 144 : cephadm [INF] Deploying daemon osd.6 on smithi169 2023-08-28T02:09:20.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:20 smithi169 conmon[100500]: cluster 2023-08-28T02:09:18.949202+0000 mgr.y (mgr.14146) 143 : cluster [DBG] pgmap v127: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-28T02:09:20.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:20 smithi169 conmon[100500]: audit 2023-08-28T02:09:19.298458+0000 mon.a (mon.0) 472 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-08-28T02:09:20.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:20 smithi169 conmon[100500]: audit 2023-08-28T02:09:19.299386+0000 mon.a (mon.0) 473 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:09:20.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:20 smithi169 conmon[100500]: cephadm 2023-08-28T02:09:19.300085+0000 mgr.y (mgr.14146) 144 : cephadm [INF] Deploying daemon osd.6 on smithi169 2023-08-28T02:09:22.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:22 smithi143 conmon[109923]: cluster 2023-08-28T02:09:20.949549+0000 mgr.y (mgr.14146) 145 : cluster [DBG] pgmap v128: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-28T02:09:22.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:22 smithi143 conmon[100258]: cluster 2023-08-28T02:09:20.949549+0000 mgr.y (mgr.14146) 145 : cluster 2023-08-28T02:09:22.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:22 smithi143 conmon[100258]: [DBG] pgmap v128: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-28T02:09:22.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:22 smithi169 conmon[100500]: cluster 2023-08-28T02:09 2023-08-28T02:09:22.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:22 smithi169 conmon[100500]: :20.949549+0000 mgr.y (mgr.14146) 145 : cluster [DBG] pgmap v128: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-28T02:09:24.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:24 smithi143 conmon[100258]: cluster 2023-08-28T02:09:22.950062+0000 mgr.y (mgr.14146) 146 : cluster [DBG] pgmap v129: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-28T02:09:24.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:24 smithi143 conmon[109923]: cluster 2023-08-28T02:09:22.950062+0000 mgr.y (mgr.14146) 146 : cluster [DBG] pgmap v129: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-28T02:09:24.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:24 smithi169 conmon[100500]: cluster 2023-08-28T02:09:22.950062+0000 2023-08-28T02:09:24.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:24 smithi169 conmon[100500]: mgr.y (mgr.14146) 146 : cluster [DBG] pgmap v129: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-28T02:09:25.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:25 smithi169 conmon[100500]: audit 2023-08-28T02:09:24.375614+0000 mon.a (mon.0) 474 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:25.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:25 smithi169 conmon[100500]: audit 2023-08-28T02:09:24.404763+0000 mon.a (mon.0) 475 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:09:25.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:25 smithi143 conmon[100258]: audit 2023-08-28T02:09:24.375614+0000 2023-08-28T02:09:25.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:25 smithi143 conmon[100258]: mon.a (mon.0) 474 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:25.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:25 smithi143 conmon[100258]: audit 2023-08-28T02:09:24.404763+0000 mon.a (mon.0) 475 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:09:25.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:25 smithi143 conmon[109923]: audit 2023-08-28T02:09:24.375614+0000 mon.a (mon 2023-08-28T02:09:25.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:25 smithi143 conmon[109923]: .0) 474 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:25.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:25 smithi143 conmon[109923]: audit 2023-08-28T02:09:24.404763+0000 mon.a (mon.0) 475 2023-08-28T02:09:25.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:25 smithi143 conmon[109923]: : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:09:26.640 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:26 smithi169 conmon[100500]: cluster 2023-08-28T02:09:24.950455+0000 mgr.y (mgr. 2023-08-28T02:09:26.640 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:26 smithi169 conmon[100500]: 14146) 147 : cluster [DBG] pgmap v130: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-28T02:09:26.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:26 smithi143 conmon[109923]: cluster 2023-08-28T02:09:24. 2023-08-28T02:09:26.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:26 smithi143 conmon[109923]: 950455+0000 mgr.y (mgr.14146) 147 : cluster [DBG] pgmap v130: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-28T02:09:26.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:26 smithi143 conmon[100258]: cluster 2023-08-28T02:09:24.950455+0000 mgr.y (mgr.14146) 147 2023-08-28T02:09:26.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:26 smithi143 conmon[100258]: : cluster [DBG] pgmap v130: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-28T02:09:27.242 INFO:teuthology.orchestra.run.smithi169.stdout:Created osd(s) 6 on host 'smithi169' 2023-08-28T02:09:27.474 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:27 smithi169 conmon[100500]: audit 2023-08-28T02:09:26.389536+0000 mon.a (mon.0) 476 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:27.474 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:27 smithi169 conmon[100500]: audit 2023-08-28T02:09:26.390408+0000 mon.a (mon.0) 477 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:09:27.474 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:27 smithi169 conmon[100500]: audit 2023-08-28T02:09:26.391281+0000 mon.a (mon.0) 478 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:09:27.474 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:27 smithi169 conmon[100500]: audit 2023-08-28T02:09:26.396488+0000 mon.a (mon.0) 479 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:27.474 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:27 smithi169 conmon[100500]: audit 2023-08-28T02:09:27.241531+0000 mon.a (mon.0) 480 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:27.474 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:27 smithi169 conmon[100500]: audit 2023-08-28T02:09:27.264656+0000 mon.a (mon.0) 481 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:09:27.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:27 smithi143 conmon[100258]: audit 2023-08-28T02:09:26.389536+0000 mon.a (mon.0) 476 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:27.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:27 smithi143 conmon[100258]: audit 2023-08-28T02:09:26.390408+0000 mon.a (mon.0) 477 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:09:27.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:27 smithi143 conmon[100258]: audit 2023-08-28T02:09:26.391281+0000 mon.a (mon.0) 478 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:09:27.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:27 smithi143 conmon[100258]: audit 2023-08-28T02:09:26.396488+0000 mon.a (mon.0) 479 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:27.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:27 smithi143 conmon[100258]: audit 2023-08-28T02 2023-08-28T02:09:27.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:27 smithi143 conmon[100258]: :09:27.241531+0000 mon.a (mon.0) 480 : audit 2023-08-28T02:09:27.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:27 smithi143 conmon[100258]: [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:27.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:27 smithi143 conmon[100258]: audit 2023-08-28T02:09:27. 2023-08-28T02:09:27.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:27 smithi143 conmon[100258]: 264656+0000 mon.a (mon.0) 481 : audit [DBG] 2023-08-28T02:09:27.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:27 smithi143 conmon[100258]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:09:27.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:27 smithi143 conmon[109923]: audit 2023-08-28T02:09:26.389536+0000 mon.a (mon.0) 476 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:27.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:27 smithi143 conmon[109923]: audit 2023-08-28T02:09:26.390408+0000 mon.a (mon.0) 477 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:09:27.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:27 smithi143 conmon[109923]: audit 2023-08-28T02:09:26.391281+0000 mon.a (mon.0) 478 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:09:27.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:27 smithi143 conmon[109923]: audit 2023-08-28T02 2023-08-28T02:09:27.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:27 smithi143 conmon[109923]: :09:26.396488+0000 mon.a (mon.0) 479 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:27.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:27 smithi143 conmon[109923]: audit 2023-08-28T02:09:27.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:27 smithi143 conmon[109923]: 2023-08-28 2023-08-28T02:09:27.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:27 smithi143 conmon[109923]: T02:09:27.241531+0000 2023-08-28T02:09:27.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:27 smithi143 conmon[109923]: mon.a (mon.0) 480 : audit 2023-08-28T02:09:27.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:27 smithi143 conmon[109923]: [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:27.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:27 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:09:27.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:27 smithi143 conmon[109923]: 08-28T02:09:27.264656+0000 2023-08-28T02:09:27.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:27 smithi143 conmon[109923]: mon.a (mon.0) 481 : audit 2023-08-28T02:09:27.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:27 smithi143 conmon[109923]: [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:09:27.819 DEBUG:teuthology.orchestra.run.smithi169:osd.6> sudo journalctl -f -n 0 -u ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@osd.6.service 2023-08-28T02:09:27.822 INFO:tasks.cephadm:Deploying osd.7 on smithi169 with /dev/vg_nvme/lv_1... 2023-08-28T02:09:27.822 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-08-28T02:09:27.855 INFO:journalctl@ceph.osd.6.smithi169.stdout:-- Logs begin at Mon 2023-08-28 01:49:52 UTC. -- 2023-08-28T02:09:28.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:28 smithi143 conmon[100258]: cluster 2023-08-28T02:09:26.950824+0000 mgr.y (mgr.14146) 148 : cluster 2023-08-28T02:09:28.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:28 smithi143 conmon[100258]: [DBG] pgmap v131: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-28T02:09:28.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:28 smithi143 conmon[100258]: audit 2023-08-28T02:09:27.589348+0000 mon.b (mon.1) 18 : audit [INF] from='osd.6 [v2:172.21.15.169:6816/3801227982,v1:172.21.15.169:6817/3801227982]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-08-28T02:09:28.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:28 smithi143 conmon[100258]: audit 2023-08-28T02:09:27.589702+0000 mon.a (mon.0) 482 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-08-28T02:09:28.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:28 smithi143 conmon[109923]: cluster 2023-08-28T02:09:26.950824+0000 mgr.y (mgr.14146 2023-08-28T02:09:28.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:28 smithi143 conmon[109923]: ) 148 : cluster [DBG] pgmap v131: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-28T02:09:28.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:28 smithi143 conmon[109923]: audit 2023-08-28T02:09:27.589348+0000 mon.b (mon. 2023-08-28T02:09:28.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:28 smithi143 conmon[109923]: 1) 18 : audit [INF] from='osd.6 [v2:172.21.15.169:6816/3801227982,v1:172.21.15.169:6817/3801227982]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-08-28T02:09:28.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:28 smithi143 conmon[109923]: audit 2023-08-28T02:09:28.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:28 smithi143 conmon[109923]: 2023-08-28T02:09:27.589702+0000 mon.a (mon.0) 482 : 2023-08-28T02:09:28.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:28 smithi143 conmon[109923]: audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-08-28T02:09:28.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:28 smithi169 conmon[100500]: cluster 2023-08-28T02:09:26.950824+0000 mgr.y (mgr.14146) 148 2023-08-28T02:09:28.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:28 smithi169 conmon[100500]: : cluster [DBG] pgmap v131: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-28T02:09:28.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:28 smithi169 conmon[100500]: audit 2023-08-28T02:09:27.589348+0000 mon.b (mon.1) 18 : audit [INF] from='osd.6 [v2:172.21.15.169:6816/3801227982,v1:172.21.15.169:6817/3801227982]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-08-28T02:09:28.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:28 smithi169 conmon[100500]: audit 2023-08-28T02:09:27.589702+0000 mon.a (mon.0) 482 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-08-28T02:09:29.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:29 smithi143 conmon[100258]: audit 2023-08-28T02:09:28.403891+0000 mon.a (mon.0) 483 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-08-28T02:09:29.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:29 smithi143 conmon[100258]: cluster 2023-08-28T02:09:28.403985+0000 mon.a (mon.0) 484 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2023-08-28T02:09:29.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:29 smithi143 conmon[100258]: audit 2023-08-28T02:09:28.404131+0000 mon.a (mon.0) 485 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-28T02:09:29.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:29 smithi143 conmon[100258]: audit 2023-08-28T02:09:28.404540+0000 mon.b (mon.1) 19 : audit [INF] from='osd.6 [v2:172.21.15.169:6816/3801227982,v1:172.21.15.169:6817/3801227982]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2023-08-28T02:09:29.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:29 smithi143 conmon[100258]: audit 2023-08-28T02:09:28.404815+0000 mon.a (mon.0) 486 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2023-08-28T02:09:29.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:29 smithi143 conmon[109923]: audit 2023-08-28T02:09:28.403891+0000 mon.a 2023-08-28T02:09:29.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:29 smithi143 conmon[109923]: (mon.0) 483 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-08-28T02:09:29.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:29 smithi143 conmon[109923]: cluster 2023-08-28T02:09:28.403985 2023-08-28T02:09:29.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:29 smithi143 conmon[109923]: +0000 mon.a (mon.0) 484 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2023-08-28T02:09:29.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:29 smithi143 conmon[109923]: audit 2023-08-28T02:09:29.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:29 smithi143 conmon[109923]: 2023-08-28T02:09:28.404131+0000 mon.a (mon.0) 485 : audit 2023-08-28T02:09:29.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:29 smithi143 conmon[109923]: [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-28T02:09:29.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:29 smithi143 conmon[109923]: audit 2023-08-28T02:09:28 2023-08-28T02:09:29.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:29 smithi143 conmon[109923]: .404540+0000 mon.b (mon.1) 19 : audit [INF] 2023-08-28T02:09:29.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:29 smithi143 conmon[109923]: from='osd.6 [v2:172.21.15.169:6816/3801227982,v1:172.21.15.169:6817/3801227982]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2023-08-28T02:09:29.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:29 smithi143 conmon[109923]: audit 2023-08-28T02:09:28.404815+0000 2023-08-28T02:09:29.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:29 smithi143 conmon[109923]: mon.a (mon.0) 486 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2023-08-28T02:09:29.737 INFO:journalctl@ceph.osd.6.smithi169.stdout:Aug 28 02:09:29 smithi169 conmon[118500]: debug 2023-08-28T02:09:29.408+0000 7f8d5e0ce700 -1 osd.6 0 waiting for initial osdmap 2023-08-28T02:09:29.737 INFO:journalctl@ceph.osd.6.smithi169.stdout:Aug 28 02:09:29 smithi169 conmon[118500]: debug 2023-08-28T02:09:29.415+0000 7f8d59047700 -1 osd.6 41 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-08-28T02:09:29.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:29 smithi169 conmon[100500]: audit 2023-08-28T02:09:28.403891+0000 mon.a 2023-08-28T02:09:29.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:29 smithi169 conmon[100500]: (mon.0) 483 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-08-28T02:09:29.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:29 smithi169 conmon[100500]: cluster 2023-08-28T02:09:28.403985+0000 mon.a (mon.0) 484 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2023-08-28T02:09:29.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:29 smithi169 conmon[100500]: audit 2023-08-28T02:09:28.404131+0000 mon.a (mon.0) 485 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-28T02:09:29.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:29 smithi169 conmon[100500]: audit 2023-08-28T02:09:28.404540+0000 mon.b (mon.1) 19 : audit [INF] from='osd.6 [v2:172.21.15.169:6816/3801227982,v1:172.21.15.169:6817/3801227982]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2023-08-28T02:09:29.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:29 smithi169 conmon[100500]: audit 2023-08-28T02:09:28.404815+0000 mon.a (mon.0) 486 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2023-08-28T02:09:30.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:30 smithi143 conmon[100258]: cluster 2023-08-28T02:09:28.951202+0000 2023-08-28T02:09:30.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:30 smithi143 conmon[100258]: mgr.y (mgr.14146) 149 : cluster [DBG] pgmap v133: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-28T02:09:30.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:30 smithi143 conmon[100258]: audit 2023-08-28T02:09:29.405266+0000 mon.a (mon.0) 487 : 2023-08-28T02:09:30.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:30 smithi143 conmon[100258]: audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]': finished 2023-08-28T02:09:30.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:30 smithi143 conmon[100258]: cluster 2023-08-28T02:09:29.405354+0000 mon.a (mon.0 2023-08-28T02:09:30.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:30 smithi143 conmon[100258]: ) 488 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2023-08-28T02:09:30.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:30 smithi143 conmon[100258]: audit 2023-08- 2023-08-28T02:09:30.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:30 smithi143 conmon[100258]: 28T02:09:29.405831+0000 mon.a (mon.0) 489 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-28T02:09:30.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:30 smithi143 conmon[100258]: audit 2023-08-28 2023-08-28T02:09:30.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:30 smithi143 conmon[100258]: T02:09:29.410046+0000 mon.a (mon.0) 490 : 2023-08-28T02:09:30.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:30 smithi143 conmon[100258]: audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-28T02:09:30.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:30 smithi143 conmon[100258]: cluster 2023-08- 2023-08-28T02:09:30.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:30 smithi143 conmon[100258]: 28T02:09:30.137747+0000 mon.a (mon.0) 491 2023-08-28T02:09:30.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:30 smithi143 conmon[100258]: : cluster [INF] osd.6 [v2:172.21.15.169:6816/3801227982,v1:172.21.15.169:6817/3801227982] boot 2023-08-28T02:09:30.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:30 smithi143 conmon[100258]: cluster 2023-08- 2023-08-28T02:09:30.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:30 smithi143 conmon[100258]: 28T02:09:30.137832+0000 mon.a (mon.0) 2023-08-28T02:09:30.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:30 smithi143 conmon[100258]: 492 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2023-08-28T02:09:30.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:30 smithi143 conmon[100258]: audit 2023-08- 2023-08-28T02:09:30.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:30 smithi143 conmon[100258]: 28T02:09:30.138291+0000 mon.a (mon.0) 2023-08-28T02:09:30.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:30 smithi143 conmon[100258]: 493 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-28T02:09:30.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:30 smithi143 conmon[109923]: cluster 2023-08-28T02:09:28.951202 2023-08-28T02:09:30.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:30 smithi143 conmon[109923]: +0000 mgr.y (mgr.14146) 149 : cluster [DBG] pgmap v133: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-28T02:09:30.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:30 smithi143 conmon[109923]: audit 2023-08-28T02:09:29.405266+0000 mon.a (mon.0) 2023-08-28T02:09:30.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:30 smithi143 conmon[109923]: 487 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]': finished 2023-08-28T02:09:30.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:30 smithi143 conmon[109923]: cluster 2023-08-28T02:09:30.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:30 smithi143 conmon[109923]: 2023-08-28T02:09:29.405354+0000 2023-08-28T02:09:30.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:30 smithi143 conmon[109923]: mon.a (mon.0) 488 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2023-08-28T02:09:30.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:30 smithi143 conmon[109923]: audit 2023-08-28T02:09:30.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:30 smithi143 conmon[109923]: 2023-08-28T02 2023-08-28T02:09:30.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:30 smithi143 conmon[109923]: :09:29.405831+0000 mon.a (mon. 2023-08-28T02:09:30.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:30 smithi143 conmon[109923]: 0) 489 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-28T02:09:30.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:30 smithi143 conmon[109923]: audit 2023-08-28T02:09:29.410046+0000 2023-08-28T02:09:30.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:30 smithi143 conmon[109923]: mon.a (mon.0) 490 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-28T02:09:30.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:30 smithi143 conmon[109923]: cluster 2023-08-28T02:09:30.137747+0000 mon.a (mon.0) 2023-08-28T02:09:30.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:30 smithi143 conmon[109923]: 491 : cluster [INF] osd.6 [v2:172.21.15.169:6816/3801227982,v1:172.21.15.169:6817/3801227982] boot 2023-08-28T02:09:30.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:30 smithi143 conmon[109923]: cluster 2023-08-28T02:09:30.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:30 smithi143 conmon[109923]: 2023-08-28T02:09:30.137832+0000 mon.a (mon.0) 492 : cluster 2023-08-28T02:09:30.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:30 smithi143 conmon[109923]: [DBG] osdmap e42: 7 total, 7 up, 7 in 2023-08-28T02:09:30.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:30 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:09:30.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:30 smithi143 conmon[109923]: 08-28T02:09:30.138291+0000 mon.a (mon.0 2023-08-28T02:09:30.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:30 smithi143 conmon[109923]: ) 493 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-28T02:09:30.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:30 smithi169 conmon[100500]: cluster 2023-08-28T02:09:28.951202+0000 mgr.y (mgr.14146) 149 : cluster [DBG] pgmap v133: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-08-28T02:09:30.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:30 smithi169 conmon[100500]: audit 2023-08-28T02:09:29.405266+0000 mon.a (mon.0) 487 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]': finished 2023-08-28T02:09:30.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:30 smithi169 conmon[100500]: cluster 2023-08-28T02:09:29.405354+0000 mon.a (mon.0) 488 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2023-08-28T02:09:30.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:30 smithi169 conmon[100500]: audit 2023-08-28T02:09:29.405831+0000 mon.a (mon.0) 489 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-28T02:09:30.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:30 smithi169 conmon[100500]: audit 2023-08-28T02:09:29.410046+0000 mon.a (mon.0) 490 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-28T02:09:30.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:30 smithi169 conmon[100500]: cluster 2023-08-28T02:09:30.137747+0000 mon.a (mon.0) 491 : cluster [INF] osd.6 [v2:172.21.15.169:6816/3801227982,v1:172.21.15.169:6817/3801227982] boot 2023-08-28T02:09:30.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:30 smithi169 conmon[100500]: cluster 2023-08-28T 2023-08-28T02:09:30.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:30 smithi169 conmon[100500]: 02:09:30.137832+0000 mon.a (mon.0) 492 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2023-08-28T02:09:30.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:30 smithi169 conmon[100500]: audit 2023-08-28T02:09:30.138291+0000 mon.a (mon.0) 493 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-28T02:09:30.748 INFO:teuthology.orchestra.run.smithi169.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-08-28T02:09:30.748 INFO:teuthology.orchestra.run.smithi169.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-08-28T02:09:30.748 INFO:teuthology.orchestra.run.smithi169.stderr: stderr: 10+0 records in 2023-08-28T02:09:30.748 INFO:teuthology.orchestra.run.smithi169.stderr:10+0 records out 2023-08-28T02:09:30.748 INFO:teuthology.orchestra.run.smithi169.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.011403 s, 920 MB/s 2023-08-28T02:09:30.748 INFO:teuthology.orchestra.run.smithi169.stderr:--> Zapping successful for: 2023-08-28T02:09:31.542 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph orch daemon add osd smithi169:vg_nvme/lv_1 2023-08-28T02:09:31.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:31 smithi143 conmon[100258]: cluster 2023-08-28T02:09:28.557276+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2023-08-28T02:09:31.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:31 smithi143 conmon[100258]: cluster 2023-08-28T02:09:28.557327+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2023-08-28T02:09:31.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:31 smithi143 conmon[100258]: cluster 2023-08-28T02:09:31.136116+0000 mon.a (mon.0) 494 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2023-08-28T02:09:31.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:31 smithi143 conmon[109923]: cluster 2023-08-28T02:09:28.557276+0000 osd.6 (osd.6) 2023-08-28T02:09:31.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:31 smithi143 conmon[109923]: 1 : cluster [DBG] purged_snaps scrub starts 2023-08-28T02:09:31.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:31 smithi143 conmon[109923]: cluster 2023-08-28T02:09:28.557327+0000 osd.6 (osd.6) 2 : 2023-08-28T02:09:31.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:31 smithi143 conmon[109923]: cluster [DBG] purged_snaps scrub ok 2023-08-28T02:09:31.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:31 smithi143 conmon[109923]: cluster 2023-08 2023-08-28T02:09:31.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:31 smithi143 conmon[109923]: -28T02:09:31.136116+0000 mon.a (mon.0) 494 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2023-08-28T02:09:31.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:31 smithi169 conmon[100500]: cluster 2023-08-28T02:09:28.557276+0000 osd.6 (osd.6) 2023-08-28T02:09:31.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:31 smithi169 conmon[100500]: 1 : cluster [DBG] purged_snaps scrub starts 2023-08-28T02:09:31.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:31 smithi169 conmon[100500]: cluster 2023-08-28T02:09:28.557327+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2023-08-28T02:09:31.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:31 smithi169 conmon[100500]: cluster 2023-08-28T02:09:31.136116+0000 mon.a (mon.0) 494 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2023-08-28T02:09:32.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[100258]: cluster 2023-08-28T02:09:30.951576+0000 mgr.y (mgr.14146) 150 : cluster [DBG] pgmap v136: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-28T02:09:32.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[100258]: audit 2023-08-28T02:09:31.609599+0000 mon.a (mon.0) 495 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:32.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[100258]: audit 2023-08-28T02:09:32.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[100258]: 2023-08-28T02:09:31. 2023-08-28T02:09:32.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[100258]: 610615+0000 mon.a (mon.0) 496 2023-08-28T02:09:32.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[100258]: : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:09:32.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[100258]: audit 2023-08- 2023-08-28T02:09:32.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[100258]: 28T02:09:31.611433+0000 mon.a (mon.0) 497 2023-08-28T02:09:32.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[100258]: : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:09:32.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[100258]: audit 2023- 2023-08-28T02:09:32.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[100258]: 08-28T02:09:31.612163+0000 2023-08-28T02:09:32.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[100258]: mon.a (mon.0) 498 : audit 2023-08-28T02:09:32.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[100258]: [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:09:32.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[100258]: audit 2023-08- 2023-08-28T02:09:32.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[100258]: 28T02:09:31.617664+0000 mon.a (mon.0) 499 2023-08-28T02:09:32.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[100258]: : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:32.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[100258]: audit 2023-08-28T02:09 2023-08-28T02:09:32.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[100258]: :31.618639+0000 mon.a (mon.0) 500 : 2023-08-28T02:09:32.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[100258]: audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:09:32.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[100258]: audit 2023-08-28T02:09 2023-08-28T02:09:32.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[100258]: :31.619461+0000 mon.a (mon.0) 501 2023-08-28T02:09:32.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[100258]: : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:09:32.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[100258]: audit 2023-08-28T02:09:31 2023-08-28T02:09:32.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[100258]: .623937+0000 mon.a (mon.0) 502 : audit [INF] 2023-08-28T02:09:32.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[100258]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:32.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[100258]: cluster 2023-08-28T02:09:32 2023-08-28T02:09:32.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[100258]: .137415+0000 mon.a (mon.0) 503 : cluster [DBG] 2023-08-28T02:09:32.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[100258]: osdmap e44: 7 total, 7 up, 7 in 2023-08-28T02:09:32.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[109923]: cluster 2023-08-28T02:09:30.951576+0000 2023-08-28T02:09:32.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[109923]: mgr.y (mgr.14146) 150 : cluster [DBG] pgmap v136: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-28T02:09:32.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[109923]: audit 2023-08- 2023-08-28T02:09:32.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[109923]: 28T02:09:31.609599+0000 mon.a (mon.0) 495 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:32.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[109923]: 2023-08-28T02:09:32.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[109923]: audit 2023 2023-08-28T02:09:32.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[109923]: -08-28T02:09:31.610615+0000 2023-08-28T02:09:32.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[109923]: mon.a (mon.0) 496 : audit [INF] 2023-08-28T02:09:32.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[109923]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:09:32.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[109923]: audit 2023-08 2023-08-28T02:09:32.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[109923]: -28T02:09:31.611433+0000 2023-08-28T02:09:32.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[109923]: mon.a (mon.0) 497 : audit [INF] 2023-08-28T02:09:32.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[109923]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:09:32.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:09:32.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[109923]: 08-28T02:09:31.612163 2023-08-28T02:09:32.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[109923]: +0000 mon.a (mon.0) 498 : audit 2023-08-28T02:09:32.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[109923]: [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:09:32.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[109923]: audit 2023-08-28T02:09:32.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[109923]: 2023-08-28T02:09:31. 2023-08-28T02:09:32.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[109923]: 617664+0000 mon.a (mon.0) 499 2023-08-28T02:09:32.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[109923]: : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:32.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[109923]: audit 2023-08- 2023-08-28T02:09:32.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[109923]: 28T02:09:31.618639+0000 mon.a ( 2023-08-28T02:09:32.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[109923]: mon.0) 500 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:09:32.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[109923]: 2023-08-28T02:09:32.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:09:32.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[109923]: 08-28T02:09:31.619461 2023-08-28T02:09:32.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[109923]: +0000 mon.a (mon.0) 501 : audit 2023-08-28T02:09:32.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[109923]: [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:09:32.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[109923]: audit 2023-08 2023-08-28T02:09:32.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[109923]: -28T02:09:31.623937+0000 mon.a (mon 2023-08-28T02:09:32.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[109923]: .0) 502 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:32.706 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[109923]: cluster 2023- 2023-08-28T02:09:32.706 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[109923]: 08-28T02:09:32.137415+0000 2023-08-28T02:09:32.706 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[109923]: mon.a (mon.0) 503 : cluster [DBG] 2023-08-28T02:09:32.706 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:32 smithi143 conmon[109923]: osdmap e44: 7 total, 7 up, 7 in 2023-08-28T02:09:32.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:32 smithi169 conmon[100500]: cluster 2023-08-28T02:09:30.951576+0000 mgr.y (mgr.14146) 150 : cluster [DBG] pgmap v136: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-28T02:09:32.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:32 smithi169 conmon[100500]: audit 2023-08-28T02:09:31.609599+0000 mon.a (mon.0) 495 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:32.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:32 smithi169 conmon[100500]: audit 2023-08-28T02:09:31.610615+0000 mon.a (mon.0) 496 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:09:32.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:32 smithi169 conmon[100500]: audit 2023-08-28T02:09:31.611433+0000 mon.a (mon.0) 497 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:09:32.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:32 smithi169 conmon[100500]: audit 2023-08-28T02:09:31.612163+0000 mon.a (mon.0) 498 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:09:32.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:32 smithi169 conmon[100500]: audit 2023-08-28T02:09:31.617664+0000 mon.a (mon.0) 499 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:32.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:32 smithi169 conmon[100500]: audit 2023-08-28T02:09:31.618639+0000 mon.a (mon.0) 500 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:09:32.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:32 smithi169 conmon[100500]: audit 2023-08-28T02:09:31.619461+0000 mon.a (mon.0) 501 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:09:32.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:32 smithi169 conmon[100500]: audit 2023-08-28T02:09:31.623937+0000 mon.a (mon.0) 502 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:32.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:32 smithi169 conmon[100500]: cluster 2023-08-28T02:09:32.137415+0000 mon.a (mon.0) 503 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2023-08-28T02:09:33.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:33 smithi143 conmon[109923]: cephadm 2023-08-28T02:09:31.603435 2023-08-28T02:09:33.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:33 smithi143 conmon[109923]: +0000 mgr.y (mgr.14146) 151 : cephadm [INF] Detected new or changed devices on smithi169 2023-08-28T02:09:33.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:33 smithi143 conmon[109923]: cephadm 2023-08 2023-08-28T02:09:33.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:33 smithi143 conmon[109923]: -28T02:09:31.612716+0000 mgr.y (mgr.14146) 152 : cephadm [INF] Adjusting osd_memory_target on smithi169 to 5722M 2023-08-28T02:09:33.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:33 smithi143 conmon[109923]: audit 2023-08-28T02:09:33.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:33 smithi143 conmon[109923]: 2023-08-28T02:09: 2023-08-28T02:09:33.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:33 smithi143 conmon[109923]: 33.395990+0000 mon.a (mon.0) 2023-08-28T02:09:33.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:33 smithi143 conmon[109923]: 504 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-28T02:09:33.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:33 smithi143 conmon[109923]: audit 2023-08- 2023-08-28T02:09:33.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:33 smithi143 conmon[109923]: 28T02:09:33.398480+0000 mon.a (mon.0) 505 2023-08-28T02:09:33.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:33 smithi143 conmon[109923]: : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-28T02:09:33.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:33 smithi143 conmon[109923]: audit 2023-08-28T02:09:33.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:33 smithi143 conmon[109923]: 2023-08-28T02:09:33.399278+0000 2023-08-28T02:09:33.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:33 smithi143 conmon[109923]: mon.a (mon.0) 506 : audit 2023-08-28T02:09:33.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:33 smithi143 conmon[109923]: [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:09:33.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:33 smithi143 conmon[100258]: cephadm 2023-08-28T02:09:31.603435+0000 mgr.y (mgr.14146) 151 : cephadm 2023-08-28T02:09:33.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:33 smithi143 conmon[100258]: [INF] Detected new or changed devices on smithi169 2023-08-28T02:09:33.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:33 smithi143 conmon[100258]: cephadm 2023-08-28T02:09:31.612716+0000 mgr.y (mgr.14146) 152 : cephadm [INF] Adjusting osd_memory_target on smithi169 to 5722M 2023-08-28T02:09:33.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:33 smithi143 conmon[100258]: audit 2023-08-28 2023-08-28T02:09:33.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:33 smithi143 conmon[100258]: T02:09:33.395990+0000 2023-08-28T02:09:33.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:33 smithi143 conmon[100258]: mon.a (mon.0) 504 : audit 2023-08-28T02:09:33.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:33 smithi143 conmon[100258]: [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-28T02:09:33.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:33 smithi143 conmon[100258]: audit 2023 2023-08-28T02:09:33.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:33 smithi143 conmon[100258]: -08-28T02:09:33.398480+0000 2023-08-28T02:09:33.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:33 smithi143 conmon[100258]: mon.a (mon.0) 505 : audit [INF] 2023-08-28T02:09:33.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:33 smithi143 conmon[100258]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-28T02:09:33.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:33 smithi143 conmon[100258]: audit 2023- 2023-08-28T02:09:33.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:33 smithi143 conmon[100258]: 08-28T02:09:33.399278 2023-08-28T02:09:33.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:33 smithi143 conmon[100258]: +0000 mon.a (mon.0) 506 : audit 2023-08-28T02:09:33.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:33 smithi143 conmon[100258]: [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:09:33.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:33 smithi169 conmon[100500]: cephadm 2023-08-28T02:09:31.603435+0000 mgr.y (mgr.14146) 151 : 2023-08-28T02:09:33.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:33 smithi169 conmon[100500]: cephadm [INF] Detected new or changed devices on smithi169 2023-08-28T02:09:33.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:33 smithi169 conmon[100500]: cephadm 2023-08-28T02:09:31.612716+0000 mgr.y (mgr.14146) 152 : cephadm [INF] Adjusting osd_memory_target on smithi169 to 5722M 2023-08-28T02:09:33.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:33 smithi169 conmon[100500]: audit 2023-08-28T02:09:33.395990+0000 mon.a (mon.0) 504 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-28T02:09:33.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:33 smithi169 conmon[100500]: audit 2023-08-28T02:09:33.398480+0000 mon.a (mon.0) 505 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-28T02:09:33.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:33 smithi169 conmon[100500]: audit 2023-08-28T02:09:33.399278+0000 mon.a (mon.0) 506 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:09:34.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:34 smithi143 conmon[109923]: cluster 2023-08-28T02:09:32.952106+0000 mgr.y (mgr.14146) 153 : 2023-08-28T02:09:34.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:34 smithi143 conmon[109923]: cluster [DBG] pgmap v139: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-28T02:09:34.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:34 smithi143 conmon[109923]: audit 2023-08-28T02:09:33.393636+0000 mgr.y (mgr.14146) 154 : audit [DBG] from='client.24283 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi169:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:09:34.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:34 smithi143 conmon[100258]: cluster 2023-08-28T02:09:32.952106+0000 mgr.y (mgr.14146) 153 : 2023-08-28T02:09:34.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:34 smithi143 conmon[100258]: cluster [DBG] pgmap v139: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-28T02:09:34.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:34 smithi143 conmon[100258]: audit 2023-08-28T02:09:33.393636+0000 mgr.y (mgr.14146) 154 : audit 2023-08-28T02:09:34.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:34 smithi143 conmon[100258]: [DBG] from='client.24283 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi169:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:09:34.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:34 smithi169 conmon[100500]: cluster 2023-08-28T02:09:32.952106+0000 mgr.y (mgr.14146) 153 : cluster [DBG] pgmap v139: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-28T02:09:34.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:34 smithi169 conmon[100500]: audit 2023-08-28T02:09:33.393636+0000 mgr.y (mgr.14146) 154 : audit [DBG] from='client.24283 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi169:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:09:36.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:36 smithi143 conmon[100258]: cluster 2023-08-28T02:09:34.952510+0000 mgr.y (mgr 2023-08-28T02:09:36.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:36 smithi143 conmon[100258]: .14146) 155 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-28T02:09:36.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:36 smithi143 conmon[100258]: audit 2023-08-28T02:09:35.716540+0000 mon.b (mon.1) 20 2023-08-28T02:09:36.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:36 smithi143 conmon[100258]: : audit [INF] from='client.? 172.21.15.169:0/1270286912' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fc5ed8fc-8ad7-4b54-a8ea-bc6a9e846f63"}]: dispatch 2023-08-28T02:09:36.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:36 smithi143 conmon[100258]: audit 2023-08-28T 2023-08-28T02:09:36.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:36 smithi143 conmon[100258]: 02:09:35.717148+0000 mon.a (mon.0) 507 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fc5ed8fc-8ad7-4b54-a8ea-bc6a9e846f63"}]: dispatch 2023-08-28T02:09:36.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:36 smithi143 conmon[100258]: audit 2023-08-28T02:09:35.721804+0000 mon.a (mon 2023-08-28T02:09:36.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:36 smithi143 conmon[100258]: .0) 508 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fc5ed8fc-8ad7-4b54-a8ea-bc6a9e846f63"}]': finished 2023-08-28T02:09:36.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:36 smithi143 conmon[100258]: cluster 2023-08-28T02:09:36.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:36 smithi143 conmon[100258]: 2023-08-28T02:09:35.721893+0000 mon.a (mon.0) 509 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2023-08-28T02:09:36.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:36 smithi143 conmon[100258]: audit 2023-08-28T02:09:35.722055+0000 mon.a (mon 2023-08-28T02:09:36.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:36 smithi143 conmon[100258]: .0) 510 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-28T02:09:36.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:36 smithi143 conmon[100258]: audit 2023-08-28T02:09:36.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:36 smithi143 conmon[100258]: 2023-08-28T02:09:36.319038+0000 mon.b (mon.1) 21 : audit [DBG] from='client.? 172.21.15.169:0/2029160439' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-28T02:09:36.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:36 smithi143 conmon[109923]: cluster 2023-08-28T02:09:34.952510+0000 mgr.y (mgr.14146) 155 2023-08-28T02:09:36.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:36 smithi143 conmon[109923]: : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-28T02:09:36.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:36 smithi143 conmon[109923]: audit 2023-08-28T02:09:35.716540+0000 mon.b (mon.1) 20 : audit [INF] from='client.? 172.21.15.169:0/1270286912' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fc5ed8fc-8ad7-4b54-a8ea-bc6a9e846f63"}]: dispatch 2023-08-28T02:09:36.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:36 smithi143 conmon[109923]: audit 2023-08-28T02:09:35. 2023-08-28T02:09:36.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:36 smithi143 conmon[109923]: 717148+0000 mon.a (mon.0) 507 : audit 2023-08-28T02:09:36.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:36 smithi143 conmon[109923]: [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fc5ed8fc-8ad7-4b54-a8ea-bc6a9e846f63"}]: dispatch 2023-08-28T02:09:36.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:36 smithi143 conmon[109923]: audit 2023 2023-08-28T02:09:36.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:36 smithi143 conmon[109923]: -08-28T02:09:35.721804+0000 2023-08-28T02:09:36.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:36 smithi143 conmon[109923]: mon.a (mon.0) 508 : audit 2023-08-28T02:09:36.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:36 smithi143 conmon[109923]: [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fc5ed8fc-8ad7-4b54-a8ea-bc6a9e846f63"}]': finished 2023-08-28T02:09:36.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:36 smithi143 conmon[109923]: cluster 2023-08 2023-08-28T02:09:36.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:36 smithi143 conmon[109923]: -28T02:09:35.721893+0000 2023-08-28T02:09:36.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:36 smithi143 conmon[109923]: mon.a (mon.0) 509 : cluster [DBG] 2023-08-28T02:09:36.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:36 smithi143 conmon[109923]: osdmap e45: 8 total, 7 up, 8 in 2023-08-28T02:09:36.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:36 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:09:36.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:36 smithi143 conmon[109923]: 08-28T02:09:35.722055+0000 2023-08-28T02:09:36.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:36 smithi143 conmon[109923]: mon.a (mon.0) 510 : audit 2023-08-28T02:09:36.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:36 smithi143 conmon[109923]: [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-28T02:09:36.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:36 smithi143 conmon[109923]: audit 2023 2023-08-28T02:09:36.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:36 smithi143 conmon[109923]: -08-28T02:09:36.319038 2023-08-28T02:09:36.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:36 smithi143 conmon[109923]: +0000 mon.b (mon.1) 21 2023-08-28T02:09:36.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:36 smithi143 conmon[109923]: : audit [DBG] from='client.? 172.21.15.169:0/2029160439' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-28T02:09:36.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:36 smithi169 conmon[100500]: cluster 2023-08-28T02:09:34.952510+0000 mgr.y (mgr.14146) 155 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-28T02:09:36.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:36 smithi169 conmon[100500]: audit 2023-08-28T02:09:35. 2023-08-28T02:09:36.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:36 smithi169 conmon[100500]: 716540+0000 mon.b (mon.1) 20 : audit [INF] from='client.? 172.21.15.169:0/1270286912' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fc5ed8fc-8ad7-4b54-a8ea-bc6a9e846f63"}]: dispatch 2023-08-28T02:09:36.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:36 smithi169 conmon[100500]: audit 2023-08-28T02:09:35.717148+0000 mon.a (mon.0) 507 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fc5ed8fc-8ad7-4b54-a8ea-bc6a9e846f63"}]: dispatch 2023-08-28T02:09:36.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:36 smithi169 conmon[100500]: audit 2023-08-28T02:09:35.721804+0000 mon.a (mon.0) 508 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fc5ed8fc-8ad7-4b54-a8ea-bc6a9e846f63"}]': finished 2023-08-28T02:09:36.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:36 smithi169 conmon[100500]: cluster 2023-08-28T02:09:35.721893+0000 mon.a (mon.0) 509 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2023-08-28T02:09:36.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:36 smithi169 conmon[100500]: audit 2023-08-28T02:09:35.722055+0000 mon.a (mon.0) 510 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-28T02:09:36.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:36 smithi169 conmon[100500]: audit 2023-08-28T02:09:36.319038+0000 mon.b (mon.1) 21 : audit [DBG] from='client.? 172.21.15.169:0/2029160439' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-28T02:09:38.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:38 smithi143 conmon[100258]: cluster 2023-08-28T02:09:36.952983+0000 mgr.y (mgr. 2023-08-28T02:09:38.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:38 smithi143 conmon[100258]: 14146) 156 : cluster [DBG] pgmap v142: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-28T02:09:38.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:38 smithi143 conmon[109923]: cluster 2023-08-28T02:09:36.952983+0000 mgr.y (mgr.14146 2023-08-28T02:09:38.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:38 smithi143 conmon[109923]: ) 156 : cluster [DBG] pgmap v142: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-28T02:09:38.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:38 smithi169 conmon[100500]: cluster 2023-08-28T02:09:36.952983+0000 mgr.y (mgr.14146) 156 : cluster [DBG] pgmap v142: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-28T02:09:38.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:38 smithi169 conmon[100500]: 2023-08-28T02:09:40.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:40 smithi169 conmon[100500]: cluster 2023-08-28T02:09:38.953444+0000 mgr.y (mgr.14146) 157 : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-28T02:09:40.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:40 smithi143 conmon[100258]: cluster 2023-08-28T02:09:38.953444+0000 mgr.y ( 2023-08-28T02:09:40.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:40 smithi143 conmon[100258]: mgr.14146) 157 : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-28T02:09:40.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:40 smithi143 conmon[109923]: cluster 2023-08-28T02:09:38.953444+0000 mgr.y (mgr.14146) 157 2023-08-28T02:09:40.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:40 smithi143 conmon[109923]: : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-28T02:09:42.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:42 smithi143 conmon[109923]: cluster 2023-08-28T02:09:40.953979+0000 mgr.y (mgr.14146) 158 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-28T02:09:42.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:42 smithi143 conmon[109923]: audit 2023-08-28T02:09:41.894558+0000 mon.a (mon.0) 511 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-08-28T02:09:42.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:42 smithi143 conmon[109923]: audit 2023-08-28T02:09:41 2023-08-28T02:09:42.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:42 smithi143 conmon[109923]: .895352+0000 mon.a (mon.0) 512 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:09:42.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:42 smithi143 conmon[100258]: cluster 2023-08-28T02:09:40.953979 2023-08-28T02:09:42.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:42 smithi143 conmon[100258]: +0000 mgr.y (mgr.14146) 158 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-28T02:09:42.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:42 smithi143 conmon[100258]: audit 2023-08-28T02:09:41.894558+0000 mon.a ( 2023-08-28T02:09:42.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:42 smithi143 conmon[100258]: mon.0) 511 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-08-28T02:09:42.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:42 smithi143 conmon[100258]: audit 2023-08-28T02:09 2023-08-28T02:09:42.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:42 smithi143 conmon[100258]: :41.895352+0000 mon.a (mon.0) 512 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:09:42.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:42 smithi169 conmon[100500]: cluster 2023-08-28T02:09:40.953979+0000 mgr.y (mgr.14146) 158 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-28T02:09:42.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:42 smithi169 conmon[100500]: audit 2023-08-28T02:09:41.894558+0000 mon.a (mon.0) 511 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-08-28T02:09:42.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:42 smithi169 conmon[100500]: audit 2023-08-28T02:09:41.895352+0000 mon.a (mon.0) 512 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:09:43.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:43 smithi143 conmon[100258]: cephadm 2023-08-28T02:09:41.895988+0000 mgr.y (mgr.14146) 159 : cephadm [INF] Deploying daemon osd.7 on smithi169 2023-08-28T02:09:43.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:43 smithi143 conmon[109923]: cephadm 2023-08-28T02:09:41.895988+0000 mgr.y (mgr.14146) 159 2023-08-28T02:09:43.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:43 smithi143 conmon[109923]: : cephadm [INF] Deploying daemon osd.7 on smithi169 2023-08-28T02:09:43.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:43 smithi169 conmon[100500]: cephadm 2023-08-28T02:09:41.895988+0000 mgr.y (mgr.14146) 159 : cephadm [INF] Deploying daemon osd.7 on smithi169 2023-08-28T02:09:44.679 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:44 smithi169 conmon[100500]: cluster 2023-08-28T02:09:42.954496+0000 mgr.y (mgr.14146 2023-08-28T02:09:44.679 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:44 smithi169 conmon[100500]: ) 160 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-28T02:09:44.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:44 smithi143 conmon[100258]: cluster 2023-08-28T02:09:42.954496+0000 mgr.y (mgr.14146) 160 : cluster 2023-08-28T02:09:44.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:44 smithi143 conmon[100258]: [DBG] pgmap v145: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-28T02:09:44.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:44 smithi143 conmon[109923]: cluster 2023-08-28T02:09:42.954496+0000 mgr.y (mgr.14146 2023-08-28T02:09:44.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:44 smithi143 conmon[109923]: ) 160 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-28T02:09:46.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:46 smithi169 conmon[100500]: cluster 2023-08-28T02:09:44.955087+0000 mgr.y (mgr.14146) 2023-08-28T02:09:46.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:46 smithi169 conmon[100500]: 161 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-28T02:09:46.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:46 smithi143 conmon[100258]: cluster 2023-08-28T02:09:44.955087+0000 mgr.y (mgr.14146) 161 : cluster [DBG] 2023-08-28T02:09:46.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:46 smithi143 conmon[100258]: pgmap v146: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-28T02:09:46.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:46 smithi143 conmon[109923]: cluster 2023-08-28T02:09:44.955087+0000 mgr.y (mgr.14146) 161 : 2023-08-28T02:09:46.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:46 smithi143 conmon[109923]: cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-28T02:09:48.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:47 smithi143 conmon[100258]: audit 2023-08-28T02:09:46.935900+0000 mon.a (mon.0) 2023-08-28T02:09:48.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:47 smithi143 conmon[100258]: 513 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:48.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:47 smithi143 conmon[100258]: cluster 2023-08-28T02:09:46.955588+0000 mgr.y (mgr.14146) 162 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-28T02:09:48.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:47 smithi143 conmon[100258]: audit 2023-08-28T02:09:46.978094+0000 mon.a (mon.0) 514 : audit [DBG] 2023-08-28T02:09:48.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:47 smithi143 conmon[100258]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:09:48.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:47 smithi143 conmon[109923]: audit 2023-08-28T02:09:46.935900+0000 mon.a (mon.0) 513 2023-08-28T02:09:48.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:47 smithi143 conmon[109923]: : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:48.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:47 smithi143 conmon[109923]: cluster 2023-08-28T02:09:46.955588+0000 mgr.y (mgr.14146) 162 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-28T02:09:48.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:47 smithi143 conmon[109923]: audit 2023-08-28T02:09:46.978094+0000 mon.a (mon.0) 514 2023-08-28T02:09:48.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:47 smithi143 conmon[109923]: : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:09:48.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:47 smithi169 conmon[100500]: audit 2023-08-28T02:09:46.935900+0000 mon.a (mon.0) 513 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:48.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:47 smithi169 conmon[100500]: cluster 2023-08-28T02:09:46.955588+0000 mgr.y (mgr.14146) 162 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-28T02:09:48.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:47 smithi169 conmon[100500]: audit 2023-08-28T02:09:46.978094+0000 mon.a (mon.0) 514 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:09:50.073 INFO:teuthology.orchestra.run.smithi169.stdout:Created osd(s) 7 on host 'smithi169' 2023-08-28T02:09:50.100 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:49 smithi169 conmon[100500]: audit 2023-08-28T02:09:48.804630+0000 mon.a (mon.0) 515 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:50.101 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:49 smithi169 conmon[100500]: cluster 2023-08-28T02:09:48.956049+0000 mgr.y (mgr.14146) 163 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-28T02:09:50.101 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:49 smithi169 conmon[100500]: audit 2023-08-28T02:09:49.219964+0000 mon.a (mon.0) 516 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:09:50.101 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:49 smithi169 conmon[100500]: audit 2023-08-28T02:09:49.220836+0000 mon.a (mon.0) 517 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:09:50.101 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:49 smithi169 conmon[100500]: audit 2023-08-28T02:09:49.228550+0000 mon.a (mon.0) 518 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:50.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:49 smithi143 conmon[100258]: audit 2023-08-28T02:09:48.804630+0000 mon.a (mon.0) 515 : audit 2023-08-28T02:09:50.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:49 smithi143 conmon[100258]: [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:50.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:49 smithi143 conmon[100258]: cluster 2023-08-28T02:09:48.956049+0000 mgr.y (mgr.14146) 163 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-28T02:09:50.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:49 smithi143 conmon[100258]: audit 2023-08-28T02:09:49.219964+0000 mon.a (mon.0) 516 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:09:50.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:49 smithi143 conmon[100258]: audit 2023-08-28T02:09:49.220836+0000 mon.a (mon.0) 517 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:09:50.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:49 smithi143 conmon[100258]: audit 2023-08-28T02:09:49.228550+0000 mon.a (mon.0) 518 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:50.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:49 smithi143 conmon[109923]: audit 2023-08-28T02:09:48.804630+0000 mon.a (mon. 2023-08-28T02:09:50.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:49 smithi143 conmon[109923]: 0) 515 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:50.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:49 smithi143 conmon[109923]: cluster 2023-08-28T02:09:48.956049+0000 mgr.y (mgr.14146) 163 : cluster 2023-08-28T02:09:50.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:49 smithi143 conmon[109923]: [DBG] pgmap v148: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-28T02:09:50.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:49 smithi143 conmon[109923]: audit 2023-08-28T02 2023-08-28T02:09:50.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:49 smithi143 conmon[109923]: :09:49.219964+0000 mon.a (mon.0) 516 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:09:50.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:49 smithi143 conmon[109923]: 2023-08-28T02:09:50.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:49 smithi143 conmon[109923]: audit 2023-08-28T02:09:50.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:49 smithi143 conmon[109923]: 2023-08-28T02:09:49.220836+0000 mon.a (mon.0) 2023-08-28T02:09:50.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:49 smithi143 conmon[109923]: 517 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:09:50.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:49 smithi143 conmon[109923]: audit 2023-08-28T02:09:49.228550+0000 2023-08-28T02:09:50.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:49 smithi143 conmon[109923]: mon.a (mon.0) 518 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:50.700 DEBUG:teuthology.orchestra.run.smithi169:osd.7> sudo journalctl -f -n 0 -u ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@osd.7.service 2023-08-28T02:09:50.702 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2023-08-28T02:09:50.703 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd stat -f json 2023-08-28T02:09:50.733 INFO:journalctl@ceph.osd.7.smithi169.stdout:-- Logs begin at Mon 2023-08-28 01:49:52 UTC. -- 2023-08-28T02:09:51.309 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:51 smithi169 conmon[100500]: audit 2023-08-28T02:09:50.073415+0000 mon.a (mon.0) 519 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:51.309 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:51 smithi169 conmon[100500]: audit 2023-08-28T02:09:50.097620+0000 mon.a (mon.0) 520 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:09:51.310 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:51 smithi169 conmon[100500]: audit 2023-08-28T02:09:50.113980+0000 mon.b (mon.1) 22 : audit [INF] from='osd.7 [v2:172.21.15.169:6824/3830851149,v1:172.21.15.169:6825/3830851149]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-08-28T02:09:51.310 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:51 smithi169 conmon[100500]: audit 2023-08-28T02:09:50.114326+0000 mon.a (mon.0) 521 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-08-28T02:09:51.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:51 smithi143 conmon[100258]: audit 2023-08-28T02:09:50.073415+0000 mon.a (mon.0) 519 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:51.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:51 smithi143 conmon[100258]: audit 2023-08-28T02:09:50.097620+0000 mon.a (mon.0) 520 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:09:51.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:51 smithi143 conmon[100258]: audit 2023-08-28T02:09:50.113980+0000 mon.b (mon.1) 22 : audit [INF] from='osd.7 [v2:172.21.15.169:6824/3830851149,v1:172.21.15.169:6825/3830851149]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-08-28T02:09:51.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:51 smithi143 conmon[100258]: audit 2023-08-28T02:09:50.114326+0000 mon.a (mon.0) 521 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-08-28T02:09:51.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:51 smithi143 conmon[109923]: audit 2023-08-28T02:09:50.073415+0000 2023-08-28T02:09:51.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:51 smithi143 conmon[109923]: mon.a (mon.0) 519 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:51.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:51 smithi143 conmon[109923]: audit 2023-08-28T02:09:50.097620+0000 2023-08-28T02:09:51.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:51 smithi143 conmon[109923]: mon.a (mon.0) 520 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:09:51.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:51 smithi143 conmon[109923]: audit 2023-08-28T02:09:50.113980 2023-08-28T02:09:51.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:51 smithi143 conmon[109923]: +0000 mon.b (mon.1) 22 : audit [INF] 2023-08-28T02:09:51.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:51 smithi143 conmon[109923]: from='osd.7 [v2:172.21.15.169:6824/3830851149,v1:172.21.15.169:6825/3830851149]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-08-28T02:09:51.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:51 smithi143 conmon[109923]: audit 2023-08-28T02:09:50. 2023-08-28T02:09:51.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:51 smithi143 conmon[109923]: 114326+0000 mon.a (mon.0) 521 : audit [INF] 2023-08-28T02:09:51.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:51 smithi143 conmon[109923]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-08-28T02:09:52.160 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:52 smithi143 conmon[100258]: cluster 2023-08-28T02:09:50.956422 2023-08-28T02:09:52.161 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:52 smithi143 conmon[100258]: +0000 mgr.y (mgr.14146) 164 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-28T02:09:52.161 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:52 smithi143 conmon[100258]: audit 2023-08-28T02:09:51.077219+0000 mon.a (mon.0) 522 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-08-28T02:09:52.161 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:52 smithi143 conmon[100258]: cluster 2023-08-28T02:09:51.077284+0000 mon.a (mon.0) 523 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2023-08-28T02:09:52.161 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:52 smithi143 conmon[100258]: audit 2023-08-28T02:09:51.078172+0000 mon.b (mon.1) 23 : audit [INF] from='osd.7 [v2:172.21.15.169:6824/3830851149,v1:172.21.15.169:6825/3830851149]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2023-08-28T02:09:52.161 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:52 smithi143 conmon[100258]: audit 2023-08-28T02:09:51.078706+0000 mon.a (mon.0) 524 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-28T02:09:52.161 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:52 smithi143 conmon[100258]: audit 2023-08-28T02:09:51.079087+0000 mon.a (mon.0) 525 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2023-08-28T02:09:52.161 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:52 smithi143 conmon[100258]: audit 2023-08-28T02:09:52.084324+0000 mon.a (mon.0) 526 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]': finished 2023-08-28T02:09:52.161 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:52 smithi143 conmon[100258]: cluster 2023-08-28T02:09:52.084579+0000 mon.a (mon.0) 527 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2023-08-28T02:09:52.161 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:52 smithi143 conmon[100258]: audit 2023-08-28T02:09:52.084706+0000 mon.a (mon.0) 528 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-28T02:09:52.161 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:52 smithi143 conmon[109923]: cluster 2023-08-28T02:09:50.956422+0000 mgr.y 2023-08-28T02:09:52.162 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:52 smithi143 conmon[109923]: (mgr.14146) 164 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-28T02:09:52.162 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:52 smithi143 conmon[109923]: audit 2023-08-28T02:09:51.077219+0000 mon.a (mon.0) 522 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-08-28T02:09:52.162 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:52 smithi143 conmon[109923]: cluster 2023-08-28T02:09:51.077284+0000 mon.a (mon.0) 523 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2023-08-28T02:09:52.162 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:52 smithi143 conmon[109923]: audit 2023-08-28T02:09:51.078172+0000 mon.b (mon.1) 23 : audit [INF] from='osd.7 [v2:172.21.15.169:6824/3830851149,v1:172.21.15.169:6825/3830851149]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2023-08-28T02:09:52.162 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:52 smithi143 conmon[109923]: audit 2023-08-28T02:09:51.078706+0000 mon.a (mon.0) 524 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-28T02:09:52.162 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:52 smithi143 conmon[109923]: audit 2023-08-28T02:09:51.079087+0000 mon.a (mon.0) 525 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2023-08-28T02:09:52.162 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:52 smithi143 conmon[109923]: audit 2023-08-28T02:09:52.084324+0000 mon.a (mon.0) 526 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]': finished 2023-08-28T02:09:52.162 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:52 smithi143 conmon[109923]: cluster 2023-08-28T02:09:52.084579+0000 mon.a (mon.0) 527 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2023-08-28T02:09:52.162 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:52 smithi143 conmon[109923]: audit 2023-08-28T02:09:52.084706+0000 mon.a (mon.0) 528 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-28T02:09:52.237 INFO:journalctl@ceph.osd.7.smithi169.stdout:Aug 28 02:09:52 smithi169 conmon[124434]: debug 2023-08-28T02:09:52.086+0000 7fa0e5a64700 -1 osd.7 0 waiting for initial osdmap 2023-08-28T02:09:52.237 INFO:journalctl@ceph.osd.7.smithi169.stdout:Aug 28 02:09:52 smithi169 conmon[124434]: debug 2023-08-28T02:09:52.095+0000 7fa0e01dc700 -1 osd.7 47 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-08-28T02:09:52.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:52 smithi169 conmon[100500]: cluster 2023-08-28T02:09:50.956422+0000 mgr.y (mgr.14146) 164 2023-08-28T02:09:52.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:52 smithi169 conmon[100500]: : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-28T02:09:52.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:52 smithi169 conmon[100500]: audit 2023-08-28T02:09:51.077219+0000 mon.a (mon.0) 522 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-08-28T02:09:52.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:52 smithi169 conmon[100500]: cluster 2023-08-28T02:09:51.077284+0000 mon.a (mon.0) 523 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2023-08-28T02:09:52.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:52 smithi169 conmon[100500]: audit 2023-08-28T02:09:51.078172+0000 mon.b (mon.1) 23 : audit [INF] from='osd.7 [v2:172.21.15.169:6824/3830851149,v1:172.21.15.169:6825/3830851149]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2023-08-28T02:09:52.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:52 smithi169 conmon[100500]: audit 2023-08-28T02:09:51.078706+0000 mon.a (mon.0) 524 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-28T02:09:52.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:52 smithi169 conmon[100500]: audit 2023-08-28T02:09:51.079087+0000 mon.a (mon.0) 525 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2023-08-28T02:09:52.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:52 smithi169 conmon[100500]: audit 2023-08-28T02:09:52.084324+0000 mon.a (mon.0) 526 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]': finished 2023-08-28T02:09:52.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:52 smithi169 conmon[100500]: cluster 2023 2023-08-28T02:09:52.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:52 smithi169 conmon[100500]: -08-28T02:09:52.084579+0000 mon.a (mon.0) 527 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2023-08-28T02:09:52.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:52 smithi169 conmon[100500]: audit 2023-08-28T02:09:52.084706 2023-08-28T02:09:52.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:52 smithi169 conmon[100500]: +0000 mon.a (mon.0) 528 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-28T02:09:52.474 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:09:53.103 INFO:teuthology.orchestra.run.smithi143.stdout:{"epoch":47,"num_osds":8,"num_up_osds":7,"osd_up_since":1693188570,"num_in_osds":8,"osd_in_since":1693188575,"num_remapped_pgs":0} 2023-08-28T02:09:53.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:53 smithi143 conmon[100258]: audit 2023-08-28T02:09:52.090333+0000 mon.a (mon.0 2023-08-28T02:09:53.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:53 smithi143 conmon[100258]: ) 529 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-28T02:09:53.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:53 smithi143 conmon[100258]: audit 2023-08-28T02:09:52.476109+0000 mon.c (mon.2) 9 : audit [DBG] from='client.? 172.21.15.143:0/2012752544' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-08-28T02:09:53.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:53 smithi143 conmon[100258]: cluster 2023-08-28T02:09:53.083376+0000 mon.a (mon.0) 530 : cluster [INF] osd.7 [v2:172.21.15.169:6824/3830851149,v1:172.21.15.169:6825/3830851149] boot 2023-08-28T02:09:53.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:53 smithi143 conmon[100258]: cluster 2023-08-28T02:09:53.083436+0000 mon.a (mon.0) 531 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2023-08-28T02:09:53.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:53 smithi143 conmon[100258]: audit 2023-08-28T02:09:53.083591+0000 mon.a (mon.0) 532 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-28T02:09:53.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:53 smithi143 conmon[109923]: audit 2023-08-28T02:09:52.090333+0000 mon.a (mon.0) 529 2023-08-28T02:09:53.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:53 smithi143 conmon[109923]: : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-28T02:09:53.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:53 smithi143 conmon[109923]: audit 2023-08-28T02:09:52.476109+0000 mon.c (mon.2) 9 : audit [DBG] from='client.? 172.21.15.143:0/2012752544' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-08-28T02:09:53.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:53 smithi143 conmon[109923]: cluster 2023-08-28T02:09:53.083376+0000 mon.a (mon.0) 530 : cluster [INF] osd.7 [v2:172.21.15.169:6824/3830851149,v1:172.21.15.169:6825/3830851149] boot 2023-08-28T02:09:53.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:53 smithi143 conmon[109923]: cluster 2023-08-28T02:09:53.083436+0000 mon.a (mon.0) 531 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2023-08-28T02:09:53.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:53 smithi143 conmon[109923]: audit 2023-08-28T02:09:53.083591+0000 mon.a (mon.0) 532 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-28T02:09:53.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:53 smithi169 conmon[100500]: audit 2023-08-28T02:09:52.090333+0000 mon.a (mon.0) 529 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-28T02:09:53.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:53 smithi169 conmon[100500]: audit 2023-08-28T02:09:52.476109+0000 mon.c (mon.2) 9 : audit [DBG] from='client.? 172.21.15.143:0/2012752544' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-08-28T02:09:53.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:53 smithi169 conmon[100500]: cluster 2023-08-28T02:09:53.083376+0000 mon.a (mon.0) 530 : cluster [INF] osd.7 [v2:172.21.15.169:6824/3830851149,v1:172.21.15.169:6825/3830851149] boot 2023-08-28T02:09:53.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:53 smithi169 conmon[100500]: cluster 2023-08-28T02:09:53.083436+0000 mon.a (mon.0) 531 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2023-08-28T02:09:53.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:53 smithi169 conmon[100500]: audit 2023-08-28T02:09:53.083591+0000 mon.a (mon.0) 532 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-28T02:09:54.104 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd stat -f json 2023-08-28T02:09:54.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[109923]: cluster 2023-08-28T02:09:51.159064+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2023-08-28T02:09:54.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[109923]: cluster 2023-08-28T02:09:51.159127+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-08-28T02:09:54.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[109923]: cluster 2023-08-28T02:09:52.956836+0000 mgr.y (mgr.14146) 165 : cluster [DBG] 2023-08-28T02:09:54.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[109923]: pgmap v152: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-28T02:09:54.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[109923]: cephadm 2023-08-28T02:09:53.489204+0000 mgr.y (mgr.14146) 166 : cephadm [INF] Detected new or changed devices on smithi169 2023-08-28T02:09:54.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[109923]: audit 2023-08-28T02:09:53.494128+0000 mon.a (mon.0) 533 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:54.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[109923]: 2023-08-28T02:09:54.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[109923]: audit 2023-08-28T02:09:53.495110+0000 2023-08-28T02:09:54.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[109923]: mon.a (mon.0) 534 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:09:54.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:09:54.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[109923]: 08-28T02:09:53.495777+0000 mon.a (mon.0) 535 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:09:54.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[109923]: audit 2023-08-28T02:09:54.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[109923]: 2023-08-28T02:09:53.496379+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:09:54.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[109923]: audit 2023-08-28T02:09:53.496964+0000 mon.a (mon.0) 537 2023-08-28T02:09:54.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[109923]: : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:09:54.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[109923]: cephadm 2023-08 2023-08-28T02:09:54.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[109923]: -28T02:09:53.497415+0000 mgr.y (mgr.14146) 167 : cephadm [INF] Adjusting osd_memory_target on smithi169 to 4292M 2023-08-28T02:09:54.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[109923]: audit 2023-08-28T02:09:54.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[109923]: 2023-08-28T02:09:53.501051+0000 mon.a (mon.0) 538 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:54.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[109923]: audit 2023-08-28T02:09:53.501888+0000 mon.a (mon 2023-08-28T02:09:54.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[109923]: .0) 539 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:09:54.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[109923]: audit 2023-08 2023-08-28T02:09:54.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[109923]: -28T02:09:53.502562+0000 mon.a (mon.0) 540 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:09:54.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[109923]: audit 2023-08-28T02:09:53.506618+0000 mon.a (mon.0) 541 : audit 2023-08-28T02:09:54.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[109923]: [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:54.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[109923]: cluster 2023 2023-08-28T02:09:54.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[109923]: -08-28T02:09:54.082534+0000 mon.a (mon.0) 542 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2023-08-28T02:09:54.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[100258]: cluster 2023-08-28T02:09:51.159064+0000 osd.7 (osd.7) 1 : 2023-08-28T02:09:54.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[100258]: cluster [DBG] purged_snaps scrub starts 2023-08-28T02:09:54.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[100258]: cluster 2023-08-28T02:09:51.159127+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-08-28T02:09:54.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[100258]: cluster 2023-08-28T02:09:52.956836+0000 mgr.y (mgr.14146) 165 : cluster 2023-08-28T02:09:54.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[100258]: [DBG] pgmap v152: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-28T02:09:54.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[100258]: cephadm 2023-08-28T02:09: 2023-08-28T02:09:54.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[100258]: 53.489204+0000 mgr.y (mgr.14146) 166 : cephadm [INF] 2023-08-28T02:09:54.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[100258]: Detected new or changed devices on smithi169 2023-08-28T02:09:54.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[100258]: audit 2023-08-28T02:09:54.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[100258]: 2023-08-28T02:09: 2023-08-28T02:09:54.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[100258]: 53.494128+0000 mon.a (mon.0) 533 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:54.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[100258]: audit 2023-08 2023-08-28T02:09:54.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[100258]: -28T02:09:53.495110+0000 mon.a (mon.0) 534 : 2023-08-28T02:09:54.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[100258]: audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:09:54.455 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[100258]: audit 2023- 2023-08-28T02:09:54.455 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[100258]: 08-28T02:09:53.495777+0000 mon.a (mon.0) 2023-08-28T02:09:54.455 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[100258]: 535 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:09:54.455 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[100258]: audit 2023-08-28T02:09:54.455 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[100258]: 2023-08 2023-08-28T02:09:54.455 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[100258]: -28T02:09:53.496379+0000 mon.a ( 2023-08-28T02:09:54.455 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[100258]: mon.0) 536 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:09:54.455 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[100258]: audit 2023-08-28T02: 2023-08-28T02:09:54.455 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[100258]: 09:53.496964+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:09:54.455 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[100258]: cephadm 2023-08-28T 2023-08-28T02:09:54.455 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[100258]: 02:09:53.497415+0000 mgr.y (mgr.14146) 167 : cephadm [INF] Adjusting osd_memory_target on smithi169 to 4292M 2023-08-28T02:09:54.455 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[100258]: audit 2023-08- 2023-08-28T02:09:54.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[100258]: 28T02:09:53.501051+0000 mon.a (mon.0) 2023-08-28T02:09:54.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[100258]: 538 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:54.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[100258]: audit 2023-08-28 2023-08-28T02:09:54.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[100258]: T02:09:53.501888+0000 mon.a (mon.0) 539 : audit 2023-08-28T02:09:54.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[100258]: [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:09:54.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[100258]: audit 2023-08-28T02:09:54.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[100258]: 2023-08-28T02:09:53. 2023-08-28T02:09:54.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[100258]: 502562+0000 mon.a (mon.0) 540 : audit 2023-08-28T02:09:54.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[100258]: [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:09:54.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[100258]: audit 2023-08-28T 2023-08-28T02:09:54.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[100258]: 02:09:53.506618+0000 mon.a (mon.0) 541 : audit 2023-08-28T02:09:54.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[100258]: [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:54.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[100258]: cluster 2023-08-28T02:09:54.457 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[100258]: 2023-08-28T02:09:54.082534 2023-08-28T02:09:54.457 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[100258]: +0000 mon.a (mon.0) 542 : 2023-08-28T02:09:54.457 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:54 smithi143 conmon[100258]: cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2023-08-28T02:09:54.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:54 smithi169 conmon[100500]: cluster 2023-08-28T02:09:51.159064+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2023-08-28T02:09:54.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:54 smithi169 conmon[100500]: cluster 2023-08-28T02:09:54.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:54 smithi169 conmon[100500]: 2023-08-28T02:09:51.159127+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-08-28T02:09:54.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:54 smithi169 conmon[100500]: cluster 2023-08-28T02:09:52.956836+0000 mgr.y (mgr.14146) 165 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-28T02:09:54.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:54 smithi169 conmon[100500]: cephadm 2023-08-28T02:09:53.489204+0000 2023-08-28T02:09:54.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:54 smithi169 conmon[100500]: mgr.y (mgr.14146) 166 : cephadm [INF] Detected new or changed devices on smithi169 2023-08-28T02:09:54.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:54 smithi169 conmon[100500]: audit 2023-08-28T02:09 2023-08-28T02:09:54.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:54 smithi169 conmon[100500]: :53.494128+0000 mon.a (mon.0) 533 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:54.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:54 smithi169 conmon[100500]: audit 2023-08- 2023-08-28T02:09:54.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:54 smithi169 conmon[100500]: 28T02:09:53.495110+0000 mon.a (mon.0 2023-08-28T02:09:54.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:54 smithi169 conmon[100500]: ) 534 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:09:54.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:54 smithi169 conmon[100500]: audit 2023-08-28T02:09:54.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:54 smithi169 conmon[100500]: 2023-08-28T02:09:53.495777+0000 mon.a (mon.0) 535 2023-08-28T02:09:54.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:54 smithi169 conmon[100500]: : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:09:54.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:54 smithi169 conmon[100500]: audit 2023-08-28T02:09:53.496379 2023-08-28T02:09:54.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:54 smithi169 conmon[100500]: +0000 mon.a (mon.0) 536 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:09:54.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:54 smithi169 conmon[100500]: audit 2023-08-28T02 2023-08-28T02:09:54.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:54 smithi169 conmon[100500]: :09:53.496964+0000 mon.a (mon.0) 537 2023-08-28T02:09:54.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:54 smithi169 conmon[100500]: : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:09:54.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:54 smithi169 conmon[100500]: cephadm 2023-08-28T02:09:53.497415+0000 2023-08-28T02:09:54.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:54 smithi169 conmon[100500]: mgr.y (mgr.14146) 167 : cephadm [INF] Adjusting osd_memory_target on smithi169 to 4292M 2023-08-28T02:09:54.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:54 smithi169 conmon[100500]: audit 2023-08-28T02:09 2023-08-28T02:09:54.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:54 smithi169 conmon[100500]: :53.501051+0000 mon.a (mon.0) 538 : audit [INF] 2023-08-28T02:09:54.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:54 smithi169 conmon[100500]: from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:54.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:54 smithi169 conmon[100500]: audit 2023- 2023-08-28T02:09:54.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:54 smithi169 conmon[100500]: 08-28T02:09:53.501888+0000 mon.a (mon.0) 539 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:09:54.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:54 smithi169 conmon[100500]: audit 2023-08-28T02:09 2023-08-28T02:09:54.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:54 smithi169 conmon[100500]: :53.502562+0000 mon.a (mon.0) 540 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:09:54.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:54 smithi169 conmon[100500]: audit 2023-08 2023-08-28T02:09:54.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:54 smithi169 conmon[100500]: -28T02:09:53.506618+0000 mon.a (mon. 2023-08-28T02:09:54.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:54 smithi169 conmon[100500]: 0) 541 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:09:54.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:54 smithi169 conmon[100500]: cluster 2023-08-28T02:09:54 2023-08-28T02:09:54.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:54 smithi169 conmon[100500]: .082534+0000 mon.a (mon.0) 542 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2023-08-28T02:09:55.892 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:09:56.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:56 smithi143 conmon[100258]: cluster 2023-08-28T02:09:54.957162+0000 mgr.y (mgr.14146) 168 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:09:56.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:56 smithi143 conmon[100258]: cluster 2023-08-28T02:09:55.083561+0000 mon.a (mon.0) 543 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2023-08-28T02:09:56.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:56 smithi143 conmon[100258]: audit 2023-08-28T02:09:55.893759+0000 mon.c (mon.2) 10 : audit [DBG] from='client.? 172.21.15.143:0/2909100300' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-08-28T02:09:56.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:56 smithi143 conmon[109923]: cluster 2023-08-28T02:09:54.957162+0000 mgr.y (mgr.14146) 168 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:09:56.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:56 smithi143 conmon[109923]: cluster 2023-08-28T02:09:55.083561+0000 mon.a (mon.0) 543 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2023-08-28T02:09:56.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:56 smithi143 conmon[109923]: audit 2023-08-28T02:09:55.893759+0000 mon.c (mon.2) 10 : audit [DBG] from='client.? 172.21.15.143:0/2909100300' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-08-28T02:09:56.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:56 smithi169 conmon[100500]: cluster 2023-08-28T02:09:54.957162+0000 mgr.y (mgr.14146) 168 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:09:56.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:56 smithi169 conmon[100500]: cluster 2023-08-28T02:09:55.083561+0000 mon.a (mon.0) 543 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2023-08-28T02:09:56.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:56 smithi169 conmon[100500]: audit 2023-08-28T02:09:55 2023-08-28T02:09:56.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:56 smithi169 conmon[100500]: .893759+0000 mon.c (mon.2) 10 : audit [DBG] from='client.? 172.21.15.143:0/2909100300' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-08-28T02:09:56.529 INFO:teuthology.orchestra.run.smithi143.stdout:{"epoch":50,"num_osds":8,"num_up_osds":8,"osd_up_since":1693188593,"num_in_osds":8,"osd_in_since":1693188575,"num_remapped_pgs":0} 2023-08-28T02:09:56.530 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd dump --format=json 2023-08-28T02:09:56.865 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:09:58.139 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:58 smithi143 conmon[100258]: cluster 2023-08-28T02:09:56.957461+0000 2023-08-28T02:09:58.139 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:58 smithi143 conmon[100258]: mgr.y (mgr.14146) 169 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:09:58.139 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:58 smithi143 conmon[109923]: cluster 2023-08-28T02:09:56.957461+0000 2023-08-28T02:09:58.139 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:58 smithi143 conmon[109923]: mgr.y (mgr.14146) 169 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:09:58.441 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:09:58.441 INFO:teuthology.orchestra.run.smithi143.stdout:{"epoch":50,"fsid":"2fcb0bfa-4547-11ee-817c-1b7a5f1ba315","created":"2023-08-28T02:05:20.056816+0000","modified":"2023-08-28T02:09:55.079291+0000","last_up_change":"2023-08-28T02:09:53.076490+0000","last_in_change":"2023-08-28T02:09:35.717712+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-08-28T02:08:01.318811+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"718c8f27-245b-4e9d-9e2e-d1fc65ca7472","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6802","nonce":1846477664},{"type":"v1","addr":"172.21.15.143:6803","nonce":1846477664}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6804","nonce":1846477664},{"type":"v1","addr":"172.21.15.143:6805","nonce":1846477664}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6808","nonce":1846477664},{"type":"v1","addr":"172.21.15.143:6809","nonce":1846477664}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6806","nonce":1846477664},{"type":"v1","addr":"172.21.15.143:6807","nonce":1846477664}]},"public_addr":"172.21.15.143:6803/1846477664","cluster_addr":"172.21.15.143:6805/1846477664","heartbeat_back_addr":"172.21.15.143:6809/1846477664","heartbeat_front_addr":"172.21.15.143:6807/1846477664","state":["exists","up"]},{"osd":1,"uuid":"cf24771f-3b0c-4a00-8a73-13e1fee242dc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6810","nonce":2341769896},{"type":"v1","addr":"172.21.15.143:6811","nonce":2341769896}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6812","nonce":2341769896},{"type":"v1","addr":"172.21.15.143:6813","nonce":2341769896}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6816","nonce":2341769896},{"type":"v1","addr":"172.21.15.143:6817","nonce":2341769896}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6814","nonce":2341769896},{"type":"v1","addr":"172.21.15.143:6815","nonce":2341769896}]},"public_addr":"172.21.15.143:6811/2341769896","cluster_addr":"172.21.15.143:6813/2341769896","heartbeat_back_addr":"172.21.15.143:6817/2341769896","heartbeat_front_addr":"172.21.15.143:6815/2341769896","state":["exists","up"]},{"osd":2,"uuid":"4f8492d1-0e7a-49f8-b6a5-4f5e73842349","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6818","nonce":2664999316},{"type":"v1","addr":"172.21.15.143:6819","nonce":2664999316}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6820","nonce":2664999316},{"type":"v1","addr":"172.21.15.143:6821","nonce":2664999316}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6824","nonce":2664999316},{"type":"v1","addr":"172.21.15.143:6825","nonce":2664999316}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6822","nonce":2664999316},{"type":"v1","addr":"172.21.15.143:6823","nonce":2664999316}]},"public_addr":"172.21.15.143:6819/2664999316","cluster_addr":"172.21.15.143:6821/2664999316","heartbeat_back_addr":"172.21.15.143:6825/2664999316","heartbeat_front_addr":"172.21.15.143:6823/2664999316","state":["exists","up"]},{"osd":3,"uuid":"dd6abc8a-54a9-4e08-ba81-0b894f24d84a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6826","nonce":1468263600},{"type":"v1","addr":"172.21.15.143:6827","nonce":1468263600}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6828","nonce":1468263600},{"type":"v1","addr":"172.21.15.143:6829","nonce":1468263600}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6832","nonce":1468263600},{"type":"v1","addr":"172.21.15.143:6833","nonce":1468263600}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6830","nonce":1468263600},{"type":"v1","addr":"172.21.15.143:6831","nonce":1468263600}]},"public_addr":"172.21.15.143:6827/1468263600","cluster_addr":"172.21.15.143:6829/1468263600","heartbeat_back_addr":"172.21.15.143:6833/1468263600","heartbeat_front_addr":"172.21.15.143:6831/1468263600","state":["exists","up"]},{"osd":4,"uuid":"16122219-d985-46d2-bac1-f8ffcf48d8ad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6800","nonce":3960017153},{"type":"v1","addr":"172.21.15.169:6801","nonce":3960017153}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6802","nonce":3960017153},{"type":"v1","addr":"172.21.15.169:6803","nonce":3960017153}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6806","nonce":3960017153},{"type":"v1","addr":"172.21.15.169:6807","nonce":3960017153}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6804","nonce":3960017153},{"type":"v1","addr":"172.21.15.169:6805","nonce":3960017153}]},"public_addr":"172.21.15.169:6801/3960017153","cluster_addr":"172.21.15.169:6803/3960017153","heartbeat_back_addr":"172.21.15.169:6807/3960017153","heartbeat_front_addr":"172.21.15.169:6805/3960017153","state":["exists","up"]},{"osd":5,"uuid":"adbc3ea5-230f-419e-8f6f-99763378b296","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":37,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6808","nonce":4087672846},{"type":"v1","addr":"172.21.15.169:6809","nonce":4087672846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6810","nonce":4087672846},{"type":"v1","addr":"172.21.15.169:6811","nonce":4087672846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6814","nonce":4087672846},{"type":"v1","addr":"172.21.15.169:6815","nonce":4087672846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6812","nonce":4087672846},{"type":"v1","addr":"172.21.15.169:6813","nonce":4087672846}]},"public_addr":"172.21.15.169:6809/4087672846","cluster_addr":"172.21.15.169:6811/4087672846","heartbeat_back_addr":"172.21.15.169:6815/4087672846","heartbeat_front_addr":"172.21.15.169:6813/4087672846","state":["exists","up"]},{"osd":6,"uuid":"2abe4d77-c12c-477f-a4ec-eabb753ca4dd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6816","nonce":3801227982},{"type":"v1","addr":"172.21.15.169:6817","nonce":3801227982}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6818","nonce":3801227982},{"type":"v1","addr":"172.21.15.169:6819","nonce":3801227982}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6822","nonce":3801227982},{"type":"v1","addr":"172.21.15.169:6823","nonce":3801227982}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6820","nonce":3801227982},{"type":"v1","addr":"172.21.15.169:6821","nonce":3801227982}]},"public_addr":"172.21.15.169:6817/3801227982","cluster_addr":"172.21.15.169:6819/3801227982","heartbeat_back_addr":"172.21.15.169:6823/3801227982","heartbeat_front_addr":"172.21.15.169:6821/3801227982","state":["exists","up"]},{"osd":7,"uuid":"fc5ed8fc-8ad7-4b54-a8ea-bc6a9e846f63","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6824","nonce":3830851149},{"type":"v1","addr":"172.21.15.169:6825","nonce":3830851149}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6826","nonce":3830851149},{"type":"v1","addr":"172.21.15.169:6827","nonce":3830851149}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6830","nonce":3830851149},{"type":"v1","addr":"172.21.15.169:6831","nonce":3830851149}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6828","nonce":3830851149},{"type":"v1","addr":"172.21.15.169:6829","nonce":3830851149}]},"public_addr":"172.21.15.169:6825/3830851149","cluster_addr":"172.21.15.169:6827/3830851149","heartbeat_back_addr":"172.21.15.169:6831/3830851149","heartbeat_front_addr":"172.21.15.169:6829/3830851149","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:07:16.166767+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:07:37.634849+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:07:59.099590+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:08:21.737546+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:08:43.377647+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:09:06.116513+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:09:28.557330+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:09:51.159131+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.143:6801/907474889":"2023-08-29T02:06:02.911068+0000","172.21.15.143:6800/907474889":"2023-08-29T02:06:02.911068+0000","172.21.15.143:0/2323551826":"2023-08-29T02:06:02.911068+0000","172.21.15.143:6800/1019944263":"2023-08-29T02:05:41.797747+0000","172.21.15.143:0/3606471716":"2023-08-29T02:06:02.911068+0000","172.21.15.143:6801/1019944263":"2023-08-29T02:05:41.797747+0000","172.21.15.143:0/11141072":"2023-08-29T02:05:41.797747+0000","172.21.15.143:0/1148418770":"2023-08-29T02:05:41.797747+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-08-28T02:09:58.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:58 smithi169 conmon[100500]: cluster 2023-08-28T02:09:56.957461+0000 mgr.y (mgr.14146) 169 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:09:59.046 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': 'device_health_metrics', 'create_time': '2023-08-28T02:08:01.318811+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr_devicehealth': {}}}] 2023-08-28T02:09:59.047 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd pool get device_health_metrics pg_num 2023-08-28T02:09:59.382 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:09:59.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:09:59 smithi143 conmon[100258]: audit 2023-08-28T02:09:58.441700+0000 mon.a (mon.0) 544 : audit [DBG] from='client.? 172.21.15.143:0/2244802248' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-28T02:09:59.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:59 smithi143 conmon[109923]: audit 2023-08-28T02:09:58.441700+0000 mon.a (mon.0) 544 : audit 2023-08-28T02:09:59.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:09:59 smithi143 conmon[109923]: [DBG] from='client.? 172.21.15.143:0/2244802248' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-28T02:09:59.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:09:59 smithi169 conmon[100500]: audit 2023-08-28T02:09:58.441700+0000 mon.a (mon.0) 544 : audit [DBG] from='client.? 172.21.15.143:0/2244802248' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-28T02:10:00.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:00 smithi143 conmon[100258]: cluster 2023-08-28T02:09:58.957818+0000 mgr.y (mgr.14146 2023-08-28T02:10:00.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:00 smithi143 conmon[100258]: ) 170 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:00.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:00 smithi143 conmon[100258]: cluster 2023-08-28T02:10:00.000121+0000 mon.a (mon.0) 545 : cluster [INF] overall HEALTH_OK 2023-08-28T02:10:00.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:00 smithi143 conmon[109923]: cluster 2023-08-28T02:09:58.957818+0000 2023-08-28T02:10:00.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:00 smithi143 conmon[109923]: mgr.y (mgr.14146) 170 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:00.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:00 smithi143 conmon[109923]: cluster 2023-08 2023-08-28T02:10:00.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:00 smithi143 conmon[109923]: -28T02:10:00.000121+0000 mon.a (mon.0) 545 : cluster [INF] overall HEALTH_OK 2023-08-28T02:10:00.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:00 smithi169 conmon[100500]: cluster 2023-08-28T02:09:58.957818+0000 mgr.y (mgr.14146) 170 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:00.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:00 smithi169 conmon[100500]: cluster 2023-08-28T02:10:00. 2023-08-28T02:10:00.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:00 smithi169 conmon[100500]: 000121+0000 mon.a (mon.0) 545 : cluster [INF] overall HEALTH_OK 2023-08-28T02:10:00.923 INFO:teuthology.orchestra.run.smithi143.stdout:pg_num: 1 2023-08-28T02:10:01.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:01 smithi143 conmon[109923]: audit 2023-08-28T02:10 2023-08-28T02:10:01.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:01 smithi143 conmon[109923]: :00.924253+0000 mon.a (mon.0) 546 : audit [DBG] from='client.? 172.21.15.143:0/4145802811' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2023-08-28T02:10:01.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:01 smithi143 conmon[100258]: audit 2023-08-28T02:10:00 2023-08-28T02:10:01.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:01 smithi143 conmon[100258]: .924253+0000 mon.a (mon.0) 546 : audit [DBG] from='client.? 172.21.15.143:0/4145802811' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2023-08-28T02:10:01.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:01 smithi169 conmon[100500]: audit 2023-08-28T02:10:00.924253+0000 mon.a (mon.0) 546 : audit [DBG] from='client.? 172.21.15.143:0/4145802811' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2023-08-28T02:10:01.587 INFO:tasks.cephadm:Adding prometheus.a on smithi169 2023-08-28T02:10:01.587 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph orch apply prometheus '1;smithi169=a' 2023-08-28T02:10:02.400 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:02 smithi169 conmon[100500]: cluster 2023-08-28T02:10:00.958389+0000 mgr.y (mgr.14146) 171 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:02.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:02 smithi143 conmon[100258]: cluster 2023-08-28T02:10:00.958389+0000 mgr.y (mgr.14146) 171 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:02.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:02 smithi143 conmon[109923]: cluster 2023-08-28T02:10:00.958389+0000 2023-08-28T02:10:02.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:02 smithi143 conmon[109923]: mgr.y (mgr.14146) 171 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:03.509 INFO:teuthology.orchestra.run.smithi169.stdout:Scheduled prometheus update... 2023-08-28T02:10:04.131 DEBUG:teuthology.orchestra.run.smithi169:prometheus.a> sudo journalctl -f -n 0 -u ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@prometheus.a.service 2023-08-28T02:10:04.134 INFO:tasks.cephadm:Adding node-exporter.a on smithi143 2023-08-28T02:10:04.134 INFO:tasks.cephadm:Adding node-exporter.b on smithi169 2023-08-28T02:10:04.134 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph orch apply node-exporter '2;smithi143=a;smithi169=b' 2023-08-28T02:10:04.163 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:-- Logs begin at Mon 2023-08-28 01:49:52 UTC. -- 2023-08-28T02:10:04.800 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[100708]: ignoring --setuser ceph since I am not root 2023-08-28T02:10:04.800 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[100708]: ignoring --setgroup ceph since I am not root 2023-08-28T02:10:04.800 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[100708]: debug 2023-08-28T02:10:04.597+0000 7f127d27f700 1 -- 172.21.15.143:0/4264254032 <== mon.2 v2:172.21.15.143:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55f1ec95e4e0 con 0x55f1ec937000 2023-08-28T02:10:04.800 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[100708]: debug 2023-08-28T02:10:04.719+0000 7f128d448680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-08-28T02:10:04.801 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[100708]: debug 2023-08-28T02:10:04.800+0000 7f128d448680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-08-28T02:10:04.801 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[100258]: cluster 2023-08-28T02:10:02.958855+0000 mgr.y (mgr.14146 2023-08-28T02:10:04.801 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[100258]: ) 172 : cluster [DBG] pgmap v160: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:04.801 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[100258]: audit 2023-08-28T02:10:03.503678+0000 mgr.y (mgr.14146) 173 : audit [DBG] from='client.24322 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi169=a", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:10:04.801 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[100258]: cephadm 2023-08-28T02:10:03.505078+0000 mgr.y 2023-08-28T02:10:04.801 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[100258]: (mgr.14146) 174 : cephadm [INF] Saving service prometheus spec with placement smithi169=a;count:1 2023-08-28T02:10:04.801 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[100258]: audit 2023-08-28T02:10:03.510213+0000 mon.a (mon.0) 547 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:10:04.801 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[100258]: audit 2023- 2023-08-28T02:10:04.802 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[100258]: 08-28T02:10:03.541622+0000 mon.a ( 2023-08-28T02:10:04.802 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[100258]: mon.0) 548 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:10:04.802 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[100258]: audit 2023-08-28T02 2023-08-28T02:10:04.802 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[100258]: :10:03.543198+0000 mon.a (mon.0) 549 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:10:04.802 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[100258]: audit 2023-08-28T 2023-08-28T02:10:04.802 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[100258]: 02:10:03.543977+0000 mon.a (mon.0) 2023-08-28T02:10:04.802 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[100258]: 550 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:10:04.802 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[100258]: audit 2023-08-28T02 2023-08-28T02:10:04.802 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[100258]: :10:03.551377+0000 mon.a (mon.0) 2023-08-28T02:10:04.802 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[100258]: 551 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:10:04.802 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[100258]: audit 2023-08-28T 2023-08-28T02:10:04.802 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[100258]: 02:10:03.557234+0000 mon.a (mon.0) 552 2023-08-28T02:10:04.803 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[100258]: : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-08-28T02:10:04.803 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[109923]: cluster 2023-08-28T02:10:02.958855+0000 mgr.y (mgr.14146 2023-08-28T02:10:04.803 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[109923]: ) 172 : cluster [DBG] pgmap v160: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:04.803 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[109923]: audit 2023-08-28T02:10:03 2023-08-28T02:10:04.803 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[109923]: .503678+0000 mgr.y (mgr.14146) 173 : audit [DBG] from='client.24322 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi169=a", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:10:04.803 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[109923]: cephadm 2023-08-28T02 2023-08-28T02:10:04.803 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[109923]: :10:03.505078+0000 mgr.y (mgr.14146) 174 : 2023-08-28T02:10:04.803 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[109923]: cephadm [INF] Saving service prometheus spec with placement smithi169=a;count:1 2023-08-28T02:10:04.803 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[109923]: audit 2023-08-28T02:10:04.803 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[109923]: 2023-08-28T02:10:03.510213+0000 2023-08-28T02:10:04.803 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[109923]: mon.a (mon.0) 547 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:10:04.804 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:10:04.804 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[109923]: 08-28T02:10:03.541622+0000 mon.a (mon.0) 548 : audit 2023-08-28T02:10:04.804 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[109923]: [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:10:04.804 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[109923]: audit 2023-08-28T02: 2023-08-28T02:10:04.804 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[109923]: 10:03.543198+0000 mon.a (mon.0) 549 : audit 2023-08-28T02:10:04.804 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[109923]: [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:10:04.804 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[109923]: audit 2023-08-28T02: 2023-08-28T02:10:04.804 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[109923]: 10:03.543977+0000 mon.a (mon.0) 550 : 2023-08-28T02:10:04.804 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[109923]: audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:10:04.804 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[109923]: audit 2023-08-28 2023-08-28T02:10:04.804 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[109923]: T02:10:03.551377+0000 mon.a (mon.0) 2023-08-28T02:10:04.804 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[109923]: 551 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:10:04.805 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:10:04.805 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[109923]: 08-28T02:10:03.557234+0000 mon.a (mon. 2023-08-28T02:10:04.805 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:04 smithi143 conmon[109923]: 0) 552 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-08-28T02:10:04.805 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:10:04 smithi169 conmon[102807]: ignoring --setuser ceph since I am not root 2023-08-28T02:10:04.805 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:10:04 smithi169 conmon[102807]: ignoring --setgroup ceph since I am not root 2023-08-28T02:10:04.805 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:10:04 smithi169 conmon[102807]: debug 2023-08-28T02:10:04.592+0000 7fe26f965700 1 -- 172.21.15.169:0/2089621138 <== mon.1 v2:172.21.15.169:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55f392aa84e0 con 0x55f392a80c00 2023-08-28T02:10:04.805 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:10:04 smithi169 conmon[102807]: debug 2023-08-28T02:10:04.718+0000 7fe27fb2e680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-08-28T02:10:04.805 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:10:04 smithi169 conmon[102807]: debug 2023-08-28T02:10:04.802+0000 7fe27fb2e680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-08-28T02:10:04.806 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:04 smithi169 conmon[100500]: cluster 2023-08-28T02:10:02.958855+0000 mgr.y (mgr.14146 2023-08-28T02:10:04.806 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:04 smithi169 conmon[100500]: ) 172 : cluster [DBG] pgmap v160: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:04.806 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:04 smithi169 conmon[100500]: audit 2023-08-28T02:10:03.503678+0000 mgr.y (mgr.14146) 173 : audit [DBG] from='client.24322 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi169=a", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:10:04.806 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:04 smithi169 conmon[100500]: cephadm 2023-08-28T02:10:03.505078+0000 mgr.y (mgr.14146) 174 : cephadm [INF] Saving service prometheus spec with placement smithi169=a;count:1 2023-08-28T02:10:04.806 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:04 smithi169 conmon[100500]: audit 2023-08-28T02:10:03.510213+0000 mon.a (mon.0) 547 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:10:04.806 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:04 smithi169 conmon[100500]: audit 2023-08-28T02:10:03.541622+0000 mon.a (mon.0) 548 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:10:04.806 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:04 smithi169 conmon[100500]: audit 2023-08-28T02:10:03.543198+0000 mon.a (mon.0) 549 : audit [DBG] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:10:04.806 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:04 smithi169 conmon[100500]: audit 2023-08-28T02:10:03.543977+0000 mon.a (mon.0) 550 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:10:04.806 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:04 smithi169 conmon[100500]: audit 2023-08-28T02:10:03.551377+0000 mon.a (mon.0) 551 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' 2023-08-28T02:10:04.806 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:04 smithi169 conmon[100500]: audit 2023-08-28T02:10:03.557234+0000 mon.a (mon.0) 552 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-08-28T02:10:05.450 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:10:05 smithi143 conmon[100708]: debug 2023-08-28T02:10:05.158+0000 7f128d448680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-08-28T02:10:05.487 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:10:05 smithi169 conmon[102807]: debug 2023-08-28T02:10:05.189+0000 7fe27fb2e680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-08-28T02:10:05.896 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:05 smithi143 conmon[100258]: audit 2023-08-28T02:10:04.558985+0000 mon.a (mon.0) 553 2023-08-28T02:10:05.896 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:05 smithi143 conmon[100258]: : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-08-28T02:10:05.896 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:05 smithi143 conmon[100258]: cluster 2023-08-28T02:10:04.559052+0000 mon.a (mon.0) 554 : cluster [DBG] mgrmap e15: y(active, since 4m), standbys: x 2023-08-28T02:10:05.897 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:05 smithi143 conmon[109923]: audit 2023-08-28T02:10:04.558985+0000 mon.a 2023-08-28T02:10:05.897 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:05 smithi143 conmon[109923]: (mon.0) 553 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-08-28T02:10:05.897 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:05 smithi143 conmon[109923]: cluster 2023-08-28T02:10:04.559052+0000 mon.a (mon.0 2023-08-28T02:10:05.897 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:05 smithi143 conmon[109923]: ) 554 : cluster [DBG] mgrmap e15: y(active, since 4m), standbys: x 2023-08-28T02:10:05.972 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:05 smithi169 conmon[100500]: audit 2023-08-28T02:10:04.558985+0000 mon.a (mon.0) 553 : audit [INF] from='mgr.14146 172.21.15.143:0/146591344' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-08-28T02:10:05.972 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:05 smithi169 conmon[100500]: cluster 2023-08-28T02: 2023-08-28T02:10:05.972 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:05 smithi169 conmon[100500]: 10:04.559052+0000 mon.a (mon.0) 554 : cluster [DBG] mgrmap e15: y(active, since 4m), standbys: x 2023-08-28T02:10:06.192 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:10:05 smithi143 conmon[100708]: debug 2023-08-28T02:10:05.896+0000 7f128d448680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-08-28T02:10:06.192 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:10:05 smithi143 conmon[100708]: debug 2023-08-28T02:10:05.982+0000 7f128d448680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-08-28T02:10:06.237 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:10:05 smithi169 conmon[102807]: debug 2023-08-28T02:10:05.972+0000 7fe27fb2e680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-08-28T02:10:06.237 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:10:06 smithi169 conmon[102807]: debug 2023-08-28T02:10:06.057+0000 7fe27fb2e680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-08-28T02:10:06.450 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:10:06 smithi143 conmon[100708]: debug 2023-08-28T02:10:06.193+0000 7f128d448680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-08-28T02:10:06.737 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:10:06 smithi169 conmon[102807]: debug 2023-08-28T02:10:06.270+0000 7fe27fb2e680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-08-28T02:10:07.418 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:10:07 smithi143 conmon[100708]: debug 2023-08-28T02:10:07.122+0000 7f128d448680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-08-28T02:10:07.418 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:10:07 smithi143 conmon[100708]: debug 2023-08-28T02:10:07.249+0000 7f128d448680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-08-28T02:10:07.418 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:10:07 smithi143 conmon[100708]: debug 2023-08-28T02:10:07.327+0000 7f128d448680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-08-28T02:10:07.487 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:10:07 smithi169 conmon[102807]: debug 2023-08-28T02:10:07.202+0000 7fe27fb2e680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-08-28T02:10:07.487 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:10:07 smithi169 conmon[102807]: debug 2023-08-28T02:10:07.329+0000 7fe27fb2e680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-08-28T02:10:07.487 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:10:07 smithi169 conmon[102807]: debug 2023-08-28T02:10:07.407+0000 7fe27fb2e680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-08-28T02:10:07.700 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:10:07 smithi143 conmon[100708]: debug 2023-08-28T02:10:07.418+0000 7f128d448680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-08-28T02:10:07.700 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:10:07 smithi143 conmon[100708]: debug 2023-08-28T02:10:07.614+0000 7f128d448680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-08-28T02:10:07.969 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:10:07 smithi169 conmon[102807]: debug 2023-08-28T02:10:07.497+0000 7fe27fb2e680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-08-28T02:10:07.969 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:10:07 smithi169 conmon[102807]: debug 2023-08-28T02:10:07.694+0000 7fe27fb2e680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-08-28T02:10:08.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:10:07 smithi143 conmon[100708]: debug 2023-08-28T02:10:07.891+0000 7f128d448680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-08-28T02:10:08.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:10:07 smithi143 conmon[100708]: debug 2023-08-28T02:10:07.989+0000 7f128d448680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-08-28T02:10:08.237 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:10:07 smithi169 conmon[102807]: debug 2023-08-28T02:10:07.969+0000 7fe27fb2e680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-08-28T02:10:08.237 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:10:08 smithi169 conmon[102807]: debug 2023-08-28T02:10:08.067+0000 7fe27fb2e680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-08-28T02:10:09.051 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:10:08 smithi143 conmon[100708]: debug 2023-08-28T02:10:08.705+0000 7f128d448680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-08-28T02:10:09.051 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:10:08 smithi143 conmon[100708]: debug 2023-08-28T02:10:08.787+0000 7f128d448680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-08-28T02:10:09.051 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:10:08 smithi143 conmon[100708]: debug 2023-08-28T02:10:08.878+0000 7f128d448680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-08-28T02:10:09.051 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100708]: debug 2023-08-28T02:10:09.051+0000 7f128d448680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-08-28T02:10:09.128 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:10:08 smithi169 conmon[102807]: debug 2023-08-28T02:10:08.785+0000 7fe27fb2e680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-08-28T02:10:09.128 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:10:08 smithi169 conmon[102807]: debug 2023-08-28T02:10:08.866+0000 7fe27fb2e680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-08-28T02:10:09.129 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:10:08 smithi169 conmon[102807]: debug 2023-08-28T02:10:08.957+0000 7fe27fb2e680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-08-28T02:10:09.575 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100708]: debug 2023-08-28T02:10:09.311+0000 7f128d448680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-08-28T02:10:09.575 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100708]: debug 2023-08-28T02:10:09.442+0000 7f128d448680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-08-28T02:10:09.650 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[102807]: debug 2023-08-28T02:10:09.128+0000 7fe27fb2e680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-08-28T02:10:09.650 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[102807]: debug 2023-08-28T02:10:09.388+0000 7fe27fb2e680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-08-28T02:10:09.650 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[102807]: debug 2023-08-28T02:10:09.518+0000 7fe27fb2e680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-08-28T02:10:09.831 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100708]: debug 2023-08-28T02:10:09.575+0000 7f128d448680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-08-28T02:10:09.832 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100708]: debug 2023-08-28T02:10:09.747+0000 7f128d448680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-08-28T02:10:09.900 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[102807]: debug 2023-08-28T02:10:09.650+0000 7fe27fb2e680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-08-28T02:10:09.900 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[102807]: debug 2023-08-28T02:10:09.820+0000 7fe27fb2e680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-08-28T02:10:09.901 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: cluster 2023-08-28T02:10:09.845796+0000 mon.a (mon.0) 555 : cluster [INF] Active manager daemon y restarted 2023-08-28T02:10:09.901 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: cluster 2023-08-28T02:10 2023-08-28T02:10:09.901 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: :09.846175+0000 mon.a (mon.0) 556 : cluster [INF] Activating manager daemon y 2023-08-28T02:10:09.901 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: cluster 2023-08-28T02:10:09.854142+0000 mon.a (mon.0) 557 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2023-08-28T02:10:09.901 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: cluster 2023- 2023-08-28T02:10:09.901 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: 08-28T02:10:09.854623+0000 mon.a (mon.0) 558 : cluster [DBG] mgrmap e16: y(active, starting, since 0.00857187s), standbys: x 2023-08-28T02:10:09.901 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: audit 2023-08 2023-08-28T02:10:09.901 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: -28T02:10:09.855672+0000 mon.a 2023-08-28T02:10:09.901 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: (mon.0) 559 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T02:10:10.083 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100708]: debug 2023-08-28T02:10:09.831+0000 7f128d448680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-08-28T02:10:10.083 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100708]: [28/Aug/2023:02:10:09] ENGINE Bus STARTING 2023-08-28T02:10:10.083 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[100708]: [80B blob data] 2023-08-28T02:10:10.083 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[100708]: [80B blob data] 2023-08-28T02:10:10.084 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: cluster 2023-08-28T02:10:09.845796+0000 2023-08-28T02:10:10.084 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: mon.a (mon.0) 555 : cluster [INF] Active manager daemon y restarted 2023-08-28T02:10:10.084 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: cluster 2023- 2023-08-28T02:10:10.084 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: 08-28T02:10:09.846175+0000 mon.a (mon.0 2023-08-28T02:10:10.084 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: ) 556 : cluster [INF] Activating manager daemon y 2023-08-28T02:10:10.084 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: cluster 2023-08-28T02:10:10.084 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: 2023-08-28T02 2023-08-28T02:10:10.084 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: :10:09.854142+0000 mon.a 2023-08-28T02:10:10.084 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: (mon.0) 557 : cluster [DBG] 2023-08-28T02:10:10.084 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: osdmap e51: 8 total, 8 up, 8 in 2023-08-28T02:10:10.084 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: cluster 2023-08-28T02:10:10.084 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: 2023-08-28 2023-08-28T02:10:10.085 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: T02:10:09.854623+0000 2023-08-28T02:10:10.085 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: mon.a (mon.0) 558 : 2023-08-28T02:10:10.085 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: cluster [DBG] mgrmap e16: y(active, starting, since 0.00857187s), standbys: x 2023-08-28T02:10:10.085 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: audit 2023 2023-08-28T02:10:10.085 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: -08-28T02:10:09. 2023-08-28T02:10:10.085 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: 855672+0000 mon.a (mon.0) 2023-08-28T02:10:10.085 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: 559 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T02:10:10.085 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: audit 2023 2023-08-28T02:10:10.085 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: -08-28T02:10:09 2023-08-28T02:10:10.085 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: .855795+0000 mon.a (mon.0 2023-08-28T02:10:10.086 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: ) 560 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T02:10:10.086 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: audit 2023 2023-08-28T02:10:10.086 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: -08-28T02:10:09 2023-08-28T02:10:10.086 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: .855907+0000 mon.a (mon.0) 2023-08-28T02:10:10.086 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: 561 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T02:10:10.086 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: audit 2023 2023-08-28T02:10:10.086 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: -08-28T02:10:09 2023-08-28T02:10:10.086 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: .856704+0000 mon.a (mon.0) 562 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-08-28T02:10:10.087 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: 2023-08-28T02:10:10.087 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: audit 2023-08-28T02:10:10.087 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: 2023-08-28T02 2023-08-28T02:10:10.087 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: :10:09.856865+0000 mon.a (mon.0) 563 : 2023-08-28T02:10:10.087 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-08-28T02:10:10.087 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: audit 2023- 2023-08-28T02:10:10.087 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: 08-28T02:10:09.857018+0000 2023-08-28T02:10:10.087 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: mon.a (mon.0) 564 : audit 2023-08-28T02:10:10.087 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-28T02:10:10.087 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: audit 2023-08-28 2023-08-28T02:10:10.088 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: T02:10:09.857198+0000 2023-08-28T02:10:10.088 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: mon.a (mon.0) 565 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-28T02:10:10.088 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: 2023-08-28T02:10:10.088 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: audit 2023-08-28T02:10 2023-08-28T02:10:10.088 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: :09.857358+0000 mon.a (mon.0) 566 2023-08-28T02:10:10.088 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-28T02:10:10.088 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: audit 2023 2023-08-28T02:10:10.088 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: -08-28T02:10:09.857519 2023-08-28T02:10:10.088 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: +0000 mon.a (mon.0) 567 : 2023-08-28T02:10:10.089 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: cluster 2023-08-28T02:10:09.845796+0000 mon.a (mon 2023-08-28T02:10:10.089 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: .0) 555 : cluster [INF] Active manager daemon y restarted 2023-08-28T02:10:10.089 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: cluster 2023-08-28T02:10:09. 2023-08-28T02:10:10.089 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: 846175+0000 mon.a (mon.0) 556 : cluster [INF] Activating manager daemon y 2023-08-28T02:10:10.089 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: cluster 2023-08-28T02 2023-08-28T02:10:10.089 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: :10:09.854142+0000 mon.a (mon.0) 557 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2023-08-28T02:10:10.090 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: cluster 2023-08-28 2023-08-28T02:10:10.090 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: T02:10:09.854623+0000 mon.a (mon.0) 558 : 2023-08-28T02:10:10.090 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: cluster [DBG] mgrmap e16: y(active, starting, since 0.00857187s), standbys: x 2023-08-28T02:10:10.090 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:10:10.090 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: 08-28T02:10:09.855672+0000 mon.a (mon.0) 559 : audit [DBG] 2023-08-28T02:10:10.090 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T02:10:10.090 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: audit 2023-08-28T 2023-08-28T02:10:10.090 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: 02:10:09.855795+0000 mon.a (mon.0) 560 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T02:10:10.091 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: audit 2023-08-28T02:10:09.855907 2023-08-28T02:10:10.091 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: +0000 mon.a (mon.0) 561 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T02:10:10.091 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: audit 2023-08-28T02:10:09.856704 2023-08-28T02:10:10.091 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: +0000 mon.a (mon.0) 562 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-08-28T02:10:10.091 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: audit 2023-08-28T02:10:09. 2023-08-28T02:10:10.091 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: 856865+0000 mon.a (mon.0) 563 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-08-28T02:10:10.091 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: audit 2023-08-28T02:10:10.091 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: 2023-08-28T02:10:09.857018+0000 mon.a (mon.0) 564 : 2023-08-28T02:10:10.091 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-28T02:10:10.092 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: audit 2023-08-28T02:10:10.092 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: 2023-08-28T02:10:09.857198+0000 mon.a (mon.0 2023-08-28T02:10:10.092 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: ) 565 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-28T02:10:10.092 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: audit 2023-08-28T02:10:10.092 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: 2023-08-28T02:10:09.857358+0000 mon.a (mon.0) 566 : audit 2023-08-28T02:10:10.092 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-28T02:10:10.092 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:10:10.092 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: 08-28T02:10:09.857519+0000 mon.a (mon.0) 567 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-28T02:10:10.092 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: 2023-08-28T02:10:10.092 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: audit 2023-08- 2023-08-28T02:10:10.092 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: 28T02:10:09.857697+0000 mon.a (mon.0) 568 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-28T02:10:10.093 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: audit 2023-08-28T02 2023-08-28T02:10:10.093 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: :10:09.857900+0000 mon.a (mon.0) 569 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-28T02:10:10.093 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: 2023-08-28T02:10:10.094 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-28T02:10:10.094 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: audit 2023- 2023-08-28T02:10:10.094 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: 08-28T02:10:09.857697 2023-08-28T02:10:10.094 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: +0000 mon.a (mon.0) 568 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-28T02:10:10.094 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: 2023-08-28T02:10:10.094 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: audit 2023-08- 2023-08-28T02:10:10.094 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: 28T02:10:09.857900+0000 mon.a ( 2023-08-28T02:10:10.094 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: mon.0) 569 : audit [DBG] 2023-08-28T02:10:10.094 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-28T02:10:10.095 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: audit 2023-08-28T02 2023-08-28T02:10:10.095 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: :10:09.858200+0000 mon.a 2023-08-28T02:10:10.095 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: (mon.0) 570 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-28T02:10:10.095 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: 2023-08-28T02:10:10.095 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: audit 2023-08-28T02:10:10.095 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: 2023-08-28T02:10:09.858447+0000 2023-08-28T02:10:10.095 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: mon.a (mon.0) 571 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-28T02:10:10.095 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: audit 2023-08-28 2023-08-28T02:10:10.095 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: T02:10:09.858673+0000 mon.a (mon.0) 2023-08-28T02:10:10.095 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: 572 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-08-28T02:10:10.095 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: audit 2023 2023-08-28T02:10:10.096 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: -08-28T02:10:09. 2023-08-28T02:10:10.096 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: 858837+0000 mon.a (mon.0) 573 2023-08-28T02:10:10.096 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-08-28T02:10:10.096 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: audit 2023-08-28T02:10:10.096 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: 2023-08-28T02:10:09. 2023-08-28T02:10:10.096 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: 859247+0000 mon.a (mon.0) 574 : audit [DBG] 2023-08-28T02:10:10.096 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-08-28T02:10:10.096 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: cluster 2023- 2023-08-28T02:10:10.096 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: 08-28T02:10:09.861754+0000 mon.a 2023-08-28T02:10:10.096 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: (mon.0) 575 : cluster [INF] 2023-08-28T02:10:10.097 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[100258]: Manager daemon y is now available 2023-08-28T02:10:10.097 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: audit 2023-08-28T02:10: 2023-08-28T02:10:10.097 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: 09.858200+0000 mon.a (mon.0) 570 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-28T02:10:10.097 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: audit 2023-08-28T02:10:09.858447+0000 mon.a 2023-08-28T02:10:10.098 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: (mon.0) 571 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-28T02:10:10.098 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: audit 2023-08-28T02:10:09. 2023-08-28T02:10:10.098 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: 858673+0000 mon.a (mon.0) 572 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-08-28T02:10:10.098 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: audit 2023-08-28T02:10:09.858837+0000 2023-08-28T02:10:10.098 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: mon.a (mon.0) 573 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-08-28T02:10:10.098 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: audit 2023-08 2023-08-28T02:10:10.098 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: -28T02:10:09.859247+0000 mon.a (mon.0) 574 : audit 2023-08-28T02:10:10.098 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-08-28T02:10:10.098 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: cluster 2023 2023-08-28T02:10:10.098 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: -08-28T02:10:09.861754+0000 2023-08-28T02:10:10.098 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:09 smithi143 conmon[109923]: mon.a (mon.0) 575 : cluster [INF] Manager daemon y is now available 2023-08-28T02:10:10.238 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[102807]: debug 2023-08-28T02:10:09.908+0000 7fe27fb2e680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-08-28T02:10:10.238 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[102807]: [28/Aug/2023:02:10:09] ENGINE Bus STARTING 2023-08-28T02:10:10.238 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:10:10 smithi169 conmon[102807]: CherryPy Checker: 2023-08-28T02:10:10.238 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:10:10 smithi169 conmon[102807]: The Application mounted at '' has an empty config. 2023-08-28T02:10:10.238 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:10:10 smithi169 conmon[102807]: 2023-08-28T02:10:10.238 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:10:10 smithi169 conmon[102807]: [28/Aug/2023:02:10:10] ENGINE Serving on http://:::9283 2023-08-28T02:10:10.238 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:10:10 smithi169 conmon[102807]: [28/Aug/2023:02:10:10] ENGINE Bus STARTED 2023-08-28T02:10:10.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: audit 2023 2023-08-28T02:10:10.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: -08-28T02:10:09. 2023-08-28T02:10:10.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: 855795+0000 mon.a (mon.0) 560 : audit [DBG] 2023-08-28T02:10:10.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T02:10:10.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: audit 2023-08-28T02:10 2023-08-28T02:10:10.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: :09.855907+0000 mon.a (mon.0) 561 2023-08-28T02:10:10.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T02:10:10.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: audit 2023-08-28T02:10 2023-08-28T02:10:10.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: :09.856704+0000 mon.a (mon.0) 562 : audit 2023-08-28T02:10:10.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-08-28T02:10:10.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: audit 2023-08-28T02:10:09 2023-08-28T02:10:10.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: .856865+0000 mon.a (mon.0) 563 : audit 2023-08-28T02:10:10.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-08-28T02:10:10.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: audit 2023-08-28T02:10: 2023-08-28T02:10:10.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: 09.857018+0000 mon.a (mon.0) 564 : 2023-08-28T02:10:10.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-28T02:10:10.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: audit 2023-08-28T02: 2023-08-28T02:10:10.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: 10:09.857198+0000 mon.a (mon.0) 2023-08-28T02:10:10.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: 565 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-28T02:10:10.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: audit 2023-08-28T02 2023-08-28T02:10:10.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: :10:09.857358+0000 mon.a (mon.0 2023-08-28T02:10:10.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: ) 566 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-28T02:10:10.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: audit 2023-08-28T 2023-08-28T02:10:10.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: 02:10:09.857519+0000 mon.a (mon. 2023-08-28T02:10:10.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: 0) 567 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-28T02:10:10.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: audit 2023-08-28T02:10 2023-08-28T02:10:10.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: :09.857697+0000 mon.a (mon.0) 568 : 2023-08-28T02:10:10.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-28T02:10:10.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: audit 2023-08-28T02: 2023-08-28T02:10:10.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: 10:09.857900+0000 mon.a (mon.0) 569 2023-08-28T02:10:10.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-28T02:10:10.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: audit 2023-08-28T02: 2023-08-28T02:10:10.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: 10:09.858200+0000 mon.a (mon.0) 2023-08-28T02:10:10.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: 570 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-28T02:10:10.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: audit 2023-08-28T02 2023-08-28T02:10:10.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: :10:09.858447+0000 mon.a (mon.0 2023-08-28T02:10:10.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: ) 571 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-28T02:10:10.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: audit 2023-08-28T02 2023-08-28T02:10:10.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: :10:09.858673+0000 mon.a (mon.0) 2023-08-28T02:10:10.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: 572 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-08-28T02:10:10.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: audit 2023-08-28T02: 2023-08-28T02:10:10.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: 10:09.858837+0000 mon.a (mon.0) 2023-08-28T02:10:10.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: 573 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-08-28T02:10:10.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: audit 2023-08-28T02 2023-08-28T02:10:10.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: :10:09.859247+0000 mon.a (mon.0 2023-08-28T02:10:10.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: ) 574 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-08-28T02:10:10.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: cluster 2023-08-28T02:10:10.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: 2023-08-28T02:10:09 2023-08-28T02:10:10.243 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: .861754+0000 mon.a (mon.0) 575 : cluster 2023-08-28T02:10:10.243 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:09 smithi169 conmon[100500]: [INF] Manager daemon y is now available 2023-08-28T02:10:10.450 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[100708]: CherryPy Checker: 2023-08-28T02:10:10.450 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[100708]: The Application mounted at '' has an empty config. 2023-08-28T02:10:10.450 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[100708]: 2023-08-28T02:10:10.450 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[100708]: [28/Aug/2023:02:10:10] ENGINE Serving on http://:::9283 2023-08-28T02:10:10.450 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[100708]: [28/Aug/2023:02:10:10] ENGINE Bus STARTED 2023-08-28T02:10:10.866 INFO:teuthology.orchestra.run.smithi169.stdout:Scheduled node-exporter update... 2023-08-28T02:10:11.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[100258]: cluster 2023-08-28T02:10:09.923474+0000 2023-08-28T02:10:11.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[100258]: mon.a (mon.0) 576 : cluster [DBG] Standby manager daemon x restarted 2023-08-28T02:10:11.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[100258]: cluster 2023-08-28T02:10:09.923571+0000 mon.a (mon.0) 577 : cluster [DBG] Standby manager daemon x started 2023-08-28T02:10:11.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[100258]: audit 2023-08-28T02:10:09.925204+0000 mon.b (mon.1) 24 : audit [DBG] from='mgr.? 172.21.15.169:0/2050427569' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-08-28T02:10:11.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[100258]: audit 2023-08-28T02:10:09.925726+0000 mon.b (mon.1) 25 : audit [DBG] from='mgr.? 172.21.15.169:0/2050427569' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-28T02:10:11.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[100258]: audit 2023-08-28T02:10:09.927561+0000 mon.b (mon.1) 26 : audit [DBG] from='mgr.? 172.21.15.169:0/2050427569' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-08-28T02:10:11.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[100258]: audit 2023-08-28T02:10:09.928514+0000 mon.b (mon.1) 27 : audit [DBG] from='mgr.? 172.21.15.169:0/2050427569' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-08-28T02:10:11.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[100258]: audit 2023-08-28T02:10:11.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[100258]: 2023-08-28T02:10:09.960174+0000 mon.a (mon.0) 578 2023-08-28T02:10:11.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[100258]: : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:10:11.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[100258]: audit 2023-08-28T02: 2023-08-28T02:10:11.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[100258]: 10:09.961565+0000 mon.a (mon.0) 579 : 2023-08-28T02:10:11.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[100258]: audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:10:11.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[100258]: audit 2023-08-28T02 2023-08-28T02:10:11.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[100258]: :10:09.980461+0000 mon.a (mon.0) 2023-08-28T02:10:11.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[100258]: 580 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:11.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[100258]: audit 2023-08- 2023-08-28T02:10:11.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[100258]: 28T02:10:10.004442+0000 mon.a ( 2023-08-28T02:10:11.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[100258]: mon.0) 581 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-08-28T02:10:11.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[100258]: audit 2023 2023-08-28T02:10:11.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[100258]: -08-28T02:10:10.010943+0000 2023-08-28T02:10:11.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[100258]: mon.a (mon.0) 582 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-08-28T02:10:11.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[100258]: cluster 2023 2023-08-28T02:10:11.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[100258]: -08-28T02:10:10.857603+0000 mon.a 2023-08-28T02:10:11.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[100258]: (mon.0) 583 : cluster [DBG] mgrmap e17: y(active, since 1.01155s), standbys: x 2023-08-28T02:10:11.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[100258]: audit 2023 2023-08-28T02:10:11.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[100258]: -08-28T02:10:10.867835+0000 mon.a 2023-08-28T02:10:11.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[100258]: (mon.0) 584 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:11.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: cluster 2023-08-28T02:10:09.923474+0000 mon.a (mon.0 2023-08-28T02:10:11.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: ) 576 : cluster [DBG] Standby manager daemon x restarted 2023-08-28T02:10:11.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: cluster 2023-08-28T02:10:09.923571+0000 mon.a (mon.0) 577 : cluster [DBG] Standby manager daemon x started 2023-08-28T02:10:11.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: audit 2023-08-28T02:10:09.925204+0000 mon.b (mon.1) 24 2023-08-28T02:10:11.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: : audit [DBG] from='mgr.? 172.21.15.169:0/2050427569' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-08-28T02:10:11.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: audit 2023-08-28T02 2023-08-28T02:10:11.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: :10:09.925726+0000 mon.b (mon 2023-08-28T02:10:11.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: .1) 25 : audit [DBG] from='mgr.? 172.21.15.169:0/2050427569' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-28T02:10:11.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: 2023-08-28T02:10:11.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: audit 2023-08 2023-08-28T02:10:11.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: -28T02:10:09. 2023-08-28T02:10:11.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: 927561+0000 mon.b (mon.1) 26 2023-08-28T02:10:11.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: : audit [DBG] from='mgr.? 172.21.15.169:0/2050427569' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-08-28T02:10:11.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: audit 2023-08-28T02 2023-08-28T02:10:11.205 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: :10:09.928514+0000 mon.b (mon 2023-08-28T02:10:11.205 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: .1) 27 : audit [DBG] 2023-08-28T02:10:11.205 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: from='mgr.? 172.21.15.169:0/2050427569' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-08-28T02:10:11.205 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: audit 2023 2023-08-28T02:10:11.205 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: -08-28T02:10: 2023-08-28T02:10:11.205 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: 09.960174+0000 mon.a (mon 2023-08-28T02:10:11.205 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: .0) 578 : audit [DBG] 2023-08-28T02:10:11.205 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:10:11.206 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:10:11.206 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: 08-28T02:10: 2023-08-28T02:10:11.206 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: 09.961565+0000 mon.a (mon. 2023-08-28T02:10:11.206 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: 0) 579 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:10:11.206 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: 2023-08-28T02:10:11.206 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:10:11.206 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: 08-28T02:10:09 2023-08-28T02:10:11.206 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: .980461+0000 mon.a (mon. 2023-08-28T02:10:11.207 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: 0) 580 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:11.207 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: 2023-08-28T02:10:11.207 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: audit 2023-08 2023-08-28T02:10:11.207 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: -28T02:10:10. 2023-08-28T02:10:11.207 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: 004442+0000 mon.a (mon.0) 2023-08-28T02:10:11.207 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: 581 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-08-28T02:10:11.207 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: audit 2023-08- 2023-08-28T02:10:11.207 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: 28T02:10:10.010943+0000 2023-08-28T02:10:11.208 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: mon.a (mon.0) 582 : audit [INF] 2023-08-28T02:10:11.208 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-08-28T02:10:11.208 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: cluster 2023 2023-08-28T02:10:11.208 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: -08-28T02:10: 2023-08-28T02:10:11.208 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: 10.857603+0000 mon.a (mon. 2023-08-28T02:10:11.208 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: 0) 583 : cluster [DBG] mgrmap e17: y(active, since 1.01155s), standbys: x 2023-08-28T02:10:11.208 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: 2023-08-28T02:10:11.208 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:10:11.208 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: 08-28T02:10: 2023-08-28T02:10:11.208 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: 10.867835+0000 mon.a (mon.0) 584 : audit 2023-08-28T02:10:11.209 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:10 smithi143 conmon[109923]: [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:11.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:10 smithi169 conmon[100500]: cluster 2023-08-28T02:10:09.923474+0000 mon.a (mon.0) 576 : cluster [DBG] Standby manager daemon x restarted 2023-08-28T02:10:11.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:10 smithi169 conmon[100500]: cluster 2023 2023-08-28T02:10:11.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:10 smithi169 conmon[100500]: -08-28T02:10:09.923571+0000 mon.a (mon.0) 577 : cluster [DBG] Standby manager daemon x started 2023-08-28T02:10:11.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:10 smithi169 conmon[100500]: audit 2023-08-28T02:10:09.925204+0000 mon.b (mon.1) 24 : audit [DBG] from='mgr.? 172.21.15.169:0/2050427569' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-08-28T02:10:11.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:10 smithi169 conmon[100500]: audit 2023-08-28T02:10:09.925726+0000 mon.b (mon.1) 25 : audit [DBG] from='mgr.? 172.21.15.169:0/2050427569' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-28T02:10:11.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:10 smithi169 conmon[100500]: audit 2023-08-28T02:10:09.927561+0000 mon.b (mon.1) 26 : audit [DBG] from='mgr.? 172.21.15.169:0/2050427569' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-08-28T02:10:11.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:10 smithi169 conmon[100500]: audit 2023-08-28T02:10:09.928514+0000 mon.b (mon.1) 27 : audit [DBG] from='mgr.? 172.21.15.169:0/2050427569' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-08-28T02:10:11.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:10 smithi169 conmon[100500]: audit 2023-08-28T02:10:09.960174+0000 mon.a (mon.0) 578 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:10:11.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:10 smithi169 conmon[100500]: audit 2023-08-28T02:10:09.961565+0000 mon.a (mon.0) 579 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:10:11.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:10 smithi169 conmon[100500]: audit 2023-08-28T02:10:09.980461+0000 mon.a (mon.0) 580 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:11.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:10 smithi169 conmon[100500]: audit 2023-08-28T02:10:10.004442+0000 mon.a (mon.0) 581 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-08-28T02:10:11.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:10 smithi169 conmon[100500]: audit 2023-08-28T02:10:10.010943+0000 mon.a (mon.0) 582 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-08-28T02:10:11.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:10 smithi169 conmon[100500]: cluster 2023-08-28T02:10:10.857603+0000 mon.a (mon.0) 583 : cluster [DBG] mgrmap e17: y(active, since 1.01155s), standbys: x 2023-08-28T02:10:11.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:10 smithi169 conmon[100500]: audit 2023-08-28T02: 2023-08-28T02:10:11.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:10 smithi169 conmon[100500]: 10:10.867835+0000 mon.a (mon.0) 584 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:11.725 DEBUG:teuthology.orchestra.run.smithi143:node-exporter.a> sudo journalctl -f -n 0 -u ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@node-exporter.a.service 2023-08-28T02:10:11.728 DEBUG:teuthology.orchestra.run.smithi169:node-exporter.b> sudo journalctl -f -n 0 -u ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@node-exporter.b.service 2023-08-28T02:10:11.730 INFO:tasks.cephadm:Adding alertmanager.a on smithi143 2023-08-28T02:10:11.730 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph orch apply alertmanager '1;smithi143=a' 2023-08-28T02:10:11.757 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:-- Logs begin at Mon 2023-08-28 01:49:41 UTC. -- 2023-08-28T02:10:11.761 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:-- Logs begin at Mon 2023-08-28 01:49:52 UTC. -- 2023-08-28T02:10:12.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:12 smithi169 conmon[100500]: cephadm 2023-08-28T02:10:10.864627+0000 mgr.y (mgr.14424) 3 : cephadm [INF] Saving service node-exporter spec with placement smithi143=a;smithi169=b;count:2 2023-08-28T02:10:12.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:12 smithi169 conmon[100500]: audit 2023-08-28T02:10:11.481726+0000 mon.a (mon.0) 585 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:12.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:12 smithi169 conmon[100500]: audit 2023-08-28T02:10:12.037995+0000 mon.a (mon.0) 586 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:12.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:12 smithi169 conmon[100500]: audit 2023-08-28T02:10:12.345964+0000 mon.a (mon.0) 587 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:10:12.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:12 smithi169 conmon[100500]: audit 2023-08-28T02:10:12.346777+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:10:12.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:12 smithi169 conmon[100500]: audit 2023-08-28T02:10:12.404680+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:12.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:12 smithi169 conmon[100500]: audit 2023-08-28T02:10:12.436034+0000 mon.a (mon.0) 590 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:12.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:12 smithi169 conmon[100500]: audit 2023-08-28T02:10:12.439461+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:12.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:12 smithi169 conmon[100500]: audit 2023-08-28T02:10:12.443667+0000 mon.a (mon.0) 592 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-08-28T02:10:12.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:12 smithi143 conmon[100258]: cephadm 2023-08-28T02:10:10.864627+0000 mgr.y (mgr.14424) 3 : cephadm [INF] Saving service node-exporter spec with placement smithi143=a;smithi169=b;count:2 2023-08-28T02:10:12.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:12 smithi143 conmon[100258]: audit 2023-08-28T02:10:11.481726+0000 mon.a (mon.0) 585 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:12.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:12 smithi143 conmon[100258]: audit 2023-08-28T02:10:12.037995+0000 mon.a (mon.0) 586 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:12.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:12 smithi143 conmon[100258]: audit 2023-08-28T02:10:12.345964+0000 mon.a (mon.0) 587 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:10:12.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:12 smithi143 conmon[100258]: audit 2023-08-28T02:10:12.346777+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:10:12.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:12 smithi143 conmon[100258]: audit 2023-08-28T02:10:12.404680+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:12.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:12 smithi143 conmon[100258]: audit 2023-08-28T02:10:12.436034+0000 mon.a (mon.0) 590 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:12.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:12 smithi143 conmon[100258]: audit 2023-08-28T02:10:12.439461+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:12.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:12 smithi143 conmon[100258]: audit 2023-08-28T02:10:12.443667+0000 mon.a (mon.0) 592 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-08-28T02:10:12.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:12 smithi143 conmon[109923]: cephadm 2023-08-28T02:10:10.864627+0000 mgr.y (mgr.14424) 3 : cephadm [INF] Saving service node-exporter spec with placement smithi143=a;smithi169=b;count:2 2023-08-28T02:10:12.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:12 smithi143 conmon[109923]: audit 2023-08-28T02:10:11.481726+0000 mon.a (mon.0) 585 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:12.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:12 smithi143 conmon[109923]: audit 2023-08-28T02:10:12.037995+0000 mon.a (mon.0) 586 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:12.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:12 smithi143 conmon[109923]: audit 2023-08-28T02:10:12.345964+0000 mon.a (mon.0) 587 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:10:12.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:12 smithi143 conmon[109923]: audit 2023-08-28T02:10:12.346777+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:10:12.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:12 smithi143 conmon[109923]: audit 2023-08-28T02:10:12.404680+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:12.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:12 smithi143 conmon[109923]: audit 2023-08-28T02:10:12.436034+0000 mon.a (mon.0) 590 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:12.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:12 smithi143 conmon[109923]: audit 2023-08-28T02:10:12.439461+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:12.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:12 smithi143 conmon[109923]: audit 2023-08-28T02:10:12.443667+0000 mon.a (mon.0) 592 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-08-28T02:10:13.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:13 smithi169 conmon[100500]: cluster 2023-08-28T02:10:11.857914+0000 2023-08-28T02:10:13.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:13 smithi169 conmon[100500]: mgr.y (mgr.14424) 4 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:13.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:13 smithi169 conmon[100500]: cephadm 2023-08-28T02:10:12.347925+0000 mgr.y (mgr.14424) 5 : cephadm [INF] Updating smithi143:/etc/ceph/ceph.conf 2023-08-28T02:10:13.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:13 smithi169 conmon[100500]: cephadm 2023-08-28T02:10:12.348170+0000 mgr.y (mgr.14424) 6 : cephadm [INF] Updating smithi169:/etc/ceph/ceph.conf 2023-08-28T02:10:13.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:13 smithi169 conmon[100500]: cephadm 2023-08-28T02:10:12.374500+0000 mgr.y (mgr.14424) 7 : cephadm [INF] Updating smithi143:/etc/ceph/ceph.client.admin.keyring 2023-08-28T02:10:13.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:13 smithi169 conmon[100500]: cephadm 2023-08-28T02:10:12.400500+0000 mgr.y (mgr.14424) 8 : cephadm [INF] Updating smithi169:/etc/ceph/ceph.client.admin.keyring 2023-08-28T02:10:13.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:13 smithi169 conmon[100500]: audit 2023-08-28T02:10:12.864696+0000 mon.a (mon.0) 593 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-08-28T02:10:13.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:13 smithi169 conmon[100500]: cluster 2023-08-28T02:10:12.864758+0000 mon.a (mon.0) 594 : cluster [DBG] mgrmap e18: y(active, since 3s), standbys: x 2023-08-28T02:10:13.752 INFO:teuthology.orchestra.run.smithi169.stdout:Scheduled alertmanager update... 2023-08-28T02:10:13.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:13 smithi143 conmon[100258]: cluster 2023-08-28T02:10:11.857914+0000 mgr.y (mgr.14424) 4 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:13.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:13 smithi143 conmon[100258]: 2023-08-28T02:10:13.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:13 smithi143 conmon[100258]: cephadm 2023-08-28T02:10:12.347925+0000 mgr.y (mgr.14424) 5 : cephadm [INF] Updating smithi143:/etc/ceph/ceph.conf 2023-08-28T02:10:13.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:13 smithi143 conmon[100258]: cephadm 2023-08-28T02:10:12.348170+0000 mgr.y (mgr.14424) 6 : cephadm [INF] 2023-08-28T02:10:13.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:13 smithi143 conmon[100258]: Updating smithi169:/etc/ceph/ceph.conf 2023-08-28T02:10:13.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:13 smithi143 conmon[100258]: cephadm 2023-08-28T02:10:12.374500+0000 mgr.y (mgr.14424) 7 : cephadm [INF] Updating smithi143:/etc/ceph/ceph.client.admin.keyring 2023-08-28T02:10:13.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:13 smithi143 conmon[100258]: cephadm 2023-08-28T02:10:12.400500+0000 mgr.y (mgr.14424) 8 : cephadm [INF] 2023-08-28T02:10:13.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:13 smithi143 conmon[100258]: Updating smithi169:/etc/ceph/ceph.client.admin.keyring 2023-08-28T02:10:13.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:13 smithi143 conmon[100258]: audit 2023-08-28 2023-08-28T02:10:13.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:13 smithi143 conmon[100258]: T02:10:12.864696+0000 mon.a (mon.0) 593 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-08-28T02:10:13.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:13 smithi143 conmon[100258]: 2023-08-28T02:10:13.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:13 smithi143 conmon[100258]: cluster 2023- 2023-08-28T02:10:13.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:13 smithi143 conmon[100258]: 08-28T02:10:12.864758+0000 mon.a (mon.0) 594 : cluster [DBG] mgrmap e18: y(active, since 3s), standbys: x 2023-08-28T02:10:13.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:13 smithi143 conmon[109923]: cluster 2023-08-28T02:10:11.857914+0000 mgr.y (mgr. 2023-08-28T02:10:13.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:13 smithi143 conmon[109923]: 14424) 4 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:13.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:13 smithi143 conmon[109923]: cephadm 2023-08-28T02:10 2023-08-28T02:10:13.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:13 smithi143 conmon[109923]: :12.347925+0000 mgr.y (mgr.14424) 5 : cephadm [INF] Updating smithi143:/etc/ceph/ceph.conf 2023-08-28T02:10:13.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:13 smithi143 conmon[109923]: cephadm 2023-08-28T 2023-08-28T02:10:13.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:13 smithi143 conmon[109923]: 02:10:12.348170+0000 mgr.y (mgr.14424) 6 : cephadm [INF] Updating smithi169:/etc/ceph/ceph.conf 2023-08-28T02:10:13.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:13 smithi143 conmon[109923]: cephadm 2023-08-28T02:10:12.374500+0000 mgr.y (mgr.14424) 7 : cephadm 2023-08-28T02:10:13.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:13 smithi143 conmon[109923]: [INF] Updating smithi143:/etc/ceph/ceph.client.admin.keyring 2023-08-28T02:10:13.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:13 smithi143 conmon[109923]: cephadm 2023-08-28 2023-08-28T02:10:13.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:13 smithi143 conmon[109923]: T02:10:12.400500+0000 mgr.y (mgr.14424) 8 : cephadm [INF] Updating smithi169:/etc/ceph/ceph.client.admin.keyring 2023-08-28T02:10:13.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:13 smithi143 conmon[109923]: audit 2023-08-28T02:10:12.864696+0000 mon.a (mon.0 2023-08-28T02:10:13.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:13 smithi143 conmon[109923]: ) 593 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-08-28T02:10:13.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:13 smithi143 conmon[109923]: cluster 2023-08-28T02:10:13.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:13 smithi143 conmon[109923]: 2023-08-28T02:10:12.864758+0000 mon.a (mon.0) 594 : cluster [DBG] mgrmap e18: y(active, since 3s), standbys: x 2023-08-28T02:10:14.557 DEBUG:teuthology.orchestra.run.smithi143:alertmanager.a> sudo journalctl -f -n 0 -u ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@alertmanager.a.service 2023-08-28T02:10:14.560 INFO:tasks.cephadm:Adding grafana.a on smithi169 2023-08-28T02:10:14.561 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph orch apply grafana '1;smithi169=a' 2023-08-28T02:10:14.593 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:-- Logs begin at Mon 2023-08-28 01:49:41 UTC. -- 2023-08-28T02:10:14.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:14 smithi169 conmon[100500]: cephadm 2023-08-28T02:10:12.876563+0000 mgr.y (mgr. 2023-08-28T02:10:14.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:14 smithi169 conmon[100500]: 14424) 9 : cephadm [INF] Deploying daemon prometheus.a on smithi169 2023-08-28T02:10:14.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:14 smithi169 conmon[100500]: audit 2023-08-28T02:10:13.753069+0000 mon.a (mon.0) 595 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:14.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:14 smithi143 conmon[100258]: cephadm 2023-08-28T02:10:12.876563+0000 mgr.y (mgr.14424) 9 : cephadm [INF] Deploying daemon prometheus.a on smithi169 2023-08-28T02:10:14.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:14 smithi143 conmon[100258]: 2023-08-28T02:10:14.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:14 smithi143 conmon[100258]: audit 2023-08-28T02:10:13.753069+0000 mon.a (mon.0) 595 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:14.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:14 smithi143 conmon[109923]: cephadm 2023-08-28T02:10:12.876563+0000 mgr.y (mgr.14424) 9 : cephadm 2023-08-28T02:10:14.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:14 smithi143 conmon[109923]: [INF] Deploying daemon prometheus.a on smithi169 2023-08-28T02:10:14.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:14 smithi143 conmon[109923]: audit 2023-08-28T02:10:13.753069+0000 mon.a (mon.0) 595 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:15.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:15 smithi169 conmon[100500]: audit 2023-08-28T02:10:13.747661+0000 mgr.y (mgr.14424) 10 : audit [DBG] from='client.24349 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi143=a", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:10:15.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:15 smithi169 conmon[100500]: cephadm 2023-08-28T02:10:13.749053+0000 mgr.y (mgr.14424) 11 : cephadm [INF] Saving service alertmanager spec with placement smithi143=a;count:1 2023-08-28T02:10:15.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:15 smithi169 conmon[100500]: cluster 2023-08-28T02:10:13.858250+0000 mgr.y (mgr.14424) 12 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:15.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:15 smithi143 conmon[100258]: audit 2023-08-28T02:10:13.747661+0000 mgr.y (mgr.14424) 10 : audit [DBG] from='client.24349 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi143=a", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:10:15.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:15 smithi143 conmon[100258]: cephadm 2023-08-28T02:10:13.749053+0000 mgr.y (mgr.14424) 11 : cephadm [INF] Saving service alertmanager spec with placement smithi143=a;count:1 2023-08-28T02:10:15.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:15 smithi143 conmon[100258]: cluster 2023-08-28T02:10:13.858250+0000 mgr.y (mgr.14424) 12 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:15.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:15 smithi143 conmon[109923]: audit 2023-08-28T02:10:13.747661+0000 mgr.y (mgr. 2023-08-28T02:10:15.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:15 smithi143 conmon[109923]: 14424) 10 : audit [DBG] from='client.24349 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi143=a", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:10:15.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:15 smithi143 conmon[109923]: cephadm 2023-08-28T02:10:13 2023-08-28T02:10:15.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:15 smithi143 conmon[109923]: .749053+0000 mgr.y (mgr.14424) 11 : cephadm [INF] Saving service alertmanager spec with placement smithi143=a;count:1 2023-08-28T02:10:15.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:15 smithi143 conmon[109923]: cluster 2023-08-28T02:10:13.858250+0000 mgr.y (mgr.14424) 12 : 2023-08-28T02:10:15.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:15 smithi143 conmon[109923]: cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:17.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:17 smithi143 conmon[100258]: cluster 2023-08-28T02:10:15.858573+0000 mgr.y (mgr.14424) 13 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:17.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:17 smithi143 conmon[109923]: cluster 2023-08-28T02:10:15.858573+0000 2023-08-28T02:10:17.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:17 smithi143 conmon[109923]: mgr.y (mgr.14424) 13 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:17.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:17 smithi169 conmon[100500]: cluster 2023-08-28T02:10:15.858573+0000 mgr.y (mgr.14424) 13 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:18.000 INFO:teuthology.orchestra.run.smithi169.stdout:Scheduled grafana update... 2023-08-28T02:10:19.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:19 smithi143 conmon[100258]: cluster 2023-08-28T02:10:17.859021+0000 mgr.y (mgr.14424) 14 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:19.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:19 smithi143 conmon[100258]: audit 2023-08-28T02:10:17.993604+0000 mgr.y (mgr.14424) 15 : audit [DBG] from='client.24355 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi169=a", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:10:19.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:19 smithi143 conmon[100258]: cephadm 2023-08-28T02:10:17.995118+0000 mgr.y (mgr.14424) 16 : cephadm [INF] Saving service grafana spec with placement smithi169=a;count:1 2023-08-28T02:10:19.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:19 smithi143 conmon[100258]: audit 2023-08-28T02:10 2023-08-28T02:10:19.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:19 smithi143 conmon[100258]: :18.001704+0000 mon.a (mon.0) 596 : 2023-08-28T02:10:19.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:19 smithi143 conmon[100258]: audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:19.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:19 smithi143 conmon[109923]: cluster 2023-08-28T02:10:17.859021+0000 mgr.y (mgr.14424) 14 : 2023-08-28T02:10:19.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:19 smithi143 conmon[109923]: cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:19.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:19 smithi143 conmon[109923]: audit 2023-08-28T02:10:17.993604+0000 mgr.y (mgr.14424) 15 : audit [DBG] from='client.24355 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi169=a", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:10:19.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:19 smithi143 conmon[109923]: cephadm 2023-08-28T02:10:17.995118+0000 mgr.y (mgr.14424) 16 2023-08-28T02:10:19.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:19 smithi143 conmon[109923]: : cephadm [INF] Saving service grafana spec with placement smithi169=a;count:1 2023-08-28T02:10:19.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:19 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:10:19.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:19 smithi143 conmon[109923]: 08-28T02:10:18.001704+0000 mon.a ( 2023-08-28T02:10:19.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:19 smithi143 conmon[109923]: mon.0) 596 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:19.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:19 smithi143 conmon[109923]: 2023-08-28T02:10:19.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:19 smithi169 conmon[100500]: cluster 2023-08-28T02:10:17.859021+0000 mgr.y (mgr 2023-08-28T02:10:19.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:19 smithi169 conmon[100500]: .14424) 14 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:19.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:19 smithi169 conmon[100500]: audit 2023-08-28T02:10:17.993604+0000 mgr.y (mgr.14424) 15 : audit [DBG] from='client.24355 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi169=a", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:10:19.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:19 smithi169 conmon[100500]: cephadm 2023-08-28T02:10:17.995118+0000 mgr.y (mgr.14424) 16 : cephadm [INF] Saving service grafana spec with placement smithi169=a;count:1 2023-08-28T02:10:19.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:19 smithi169 conmon[100500]: audit 2023-08-28T02:10:18.001704+0000 mon.a (mon.0) 596 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:21.090 DEBUG:teuthology.orchestra.run.smithi169:grafana.a> sudo journalctl -f -n 0 -u ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@grafana.a.service 2023-08-28T02:10:21.092 INFO:tasks.cephadm:Setting up client nodes... 2023-08-28T02:10:21.093 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-08-28T02:10:21.125 INFO:journalctl@ceph.grafana.a.smithi169.stdout:-- Logs begin at Mon 2023-08-28 01:49:52 UTC. -- 2023-08-28T02:10:21.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:21 smithi143 conmon[100258]: cluster 2023-08-28T02:10:19.859389+0000 mgr.y (mgr.14424) 17 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:21.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:21 smithi143 conmon[109923]: cluster 2023-08-28T02:10:19.859389+0000 2023-08-28T02:10:21.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:21 smithi143 conmon[109923]: mgr.y (mgr.14424) 17 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:21.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:21 smithi169 conmon[100500]: cluster 2023-08-28T02:10:19.859389+0000 mgr.y (mgr.14424) 17 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:22.948 INFO:teuthology.orchestra.run.smithi143.stdout:[client.0] 2023-08-28T02:10:22.948 INFO:teuthology.orchestra.run.smithi143.stdout: key = AQAOAuxkYhFdOBAAvTN+RoqmMnFWfLFKpgIv3g== 2023-08-28T02:10:23.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:23 smithi143 conmon[109923]: cluster 2023-08-28T02:10:21.859751+0000 mgr.y (mgr.14424 2023-08-28T02:10:23.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:23 smithi143 conmon[109923]: ) 18 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:23.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:23 smithi143 conmon[109923]: audit 2023-08-28T02:10:22.945139+0000 mon.c (mon.2) 11 : audit [INF] from='client.? 172.21.15.143:0/2380208710' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-08-28T02:10:23.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:23 smithi143 conmon[109923]: audit 2023-08-28T02:10:22.945481+0000 mon.a (mon.0) 597 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-08-28T02:10:23.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:23 smithi143 conmon[109923]: audit 2023-08-28T02:10:22.948447+0000 mon.a (mon.0) 598 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-08-28T02:10:23.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:23 smithi143 conmon[100258]: cluster 2023-08-28T02:10:21.859751+0000 2023-08-28T02:10:23.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:23 smithi143 conmon[100258]: mgr.y (mgr.14424) 18 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:23.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:23 smithi143 conmon[100258]: audit 2023-08-28T02:10:22.945139+0000 mon.c (mon.2) 11 : audit [INF] from='client.? 172.21.15.143:0/2380208710' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-08-28T02:10:23.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:23 smithi143 conmon[100258]: audit 2023-08-28T02:10:22.945481+0000 mon.a (mon.0) 597 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-08-28T02:10:23.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:23 smithi143 conmon[100258]: audit 2023-08-28T02:10:22.948447+0000 mon.a (mon.0) 598 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-08-28T02:10:23.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:23 smithi169 conmon[100500]: cluster 2023-08-28T02:10:21.859751+0000 mgr.y (mgr.14424) 18 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:23.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:23 smithi169 conmon[100500]: audit 2023-08-28T02:10:23.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:23 smithi169 conmon[100500]: 2023-08-28T02:10:22.945139+0000 mon.c (mon.2) 11 : audit [INF] from='client.? 172.21.15.143:0/2380208710' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-08-28T02:10:23.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:23 smithi169 conmon[100500]: audit 2023-08-28T02:10:22.945481+0000 mon.a (mon.0) 597 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-08-28T02:10:23.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:23 smithi169 conmon[100500]: audit 2023-08-28T02:10:22.948447+0000 mon.a (mon.0) 598 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-08-28T02:10:23.533 DEBUG:teuthology.orchestra.run.smithi143:> set -ex 2023-08-28T02:10:23.533 DEBUG:teuthology.orchestra.run.smithi143:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2023-08-28T02:10:23.534 DEBUG:teuthology.orchestra.run.smithi143:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2023-08-28T02:10:23.574 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-08-28T02:10:24.652 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:10:24 smithi169 systemd[1]: Starting Ceph prometheus.a for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:10:25.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:25 smithi169 conmon[100500]: cluster 2023-08-28T02:10:23.860106+0000 mgr.y (mgr.14424) 19 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:25.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:25 smithi143 conmon[100258]: cluster 2023-08-28T02:10:23.860106+0000 mgr.y (mgr.14424) 19 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:25.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:25 smithi143 conmon[109923]: cluster 2023-08-28T02:10:23.860106+0000 mgr.y (mgr.14424) 19 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:26.108 INFO:teuthology.orchestra.run.smithi169.stdout:[client.1] 2023-08-28T02:10:26.109 INFO:teuthology.orchestra.run.smithi169.stdout: key = AQASAuxk5cYzBhAAurYVbLEdD7qTdknFIy5TpA== 2023-08-28T02:10:26.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:26 smithi143 conmon[100258]: audit 2023-08-28T02:10:26.103260+0000 mon.b (mon.1) 28 : audit [INF] from='client.? 172.21.15.169:0/1021607470' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-08-28T02:10:26.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:26 smithi143 conmon[100258]: audit 2023-08-28T02:10:26.103856+0000 mon.a (mon.0) 599 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-08-28T02:10:26.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:26 smithi143 conmon[100258]: audit 2023-08-28T02:10:26 2023-08-28T02:10:26.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:26 smithi143 conmon[100258]: .108778+0000 mon.a (mon.0) 600 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-08-28T02:10:26.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:26 smithi143 conmon[100258]: audit 2023-08-28T02:10:26.141956+0000 mon.a (mon. 2023-08-28T02:10:26.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:26 smithi143 conmon[100258]: 0) 601 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:26.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:26 smithi143 conmon[109923]: audit 2023-08-28T02:10:26.103260+0000 mon.b ( 2023-08-28T02:10:26.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:26 smithi143 conmon[109923]: mon.1) 28 : audit [INF] from='client.? 172.21.15.169:0/1021607470' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-08-28T02:10:26.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:26 smithi143 conmon[109923]: audit 2023-08-28T02:10:26.103856+0000 mon.a ( 2023-08-28T02:10:26.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:26 smithi143 conmon[109923]: mon.0) 599 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-08-28T02:10:26.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:26 smithi143 conmon[109923]: audit 2023-08-28T02:10:26.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:26 smithi143 conmon[109923]: 2023-08-28T02:10:26.108778+0000 mon.a (mon.0) 600 : audit [INF] 2023-08-28T02:10:26.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:26 smithi143 conmon[109923]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-08-28T02:10:26.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:26 smithi143 conmon[109923]: audit 2023-08-28 2023-08-28T02:10:26.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:26 smithi143 conmon[109923]: T02:10:26.141956+0000 mon.a (mon.0) 601 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:26.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:26 smithi169 conmon[100500]: audit 2023-08-28T02:10:26.103260+0000 mon.b (mon.1) 28 : audit [INF] from='client.? 172.21.15.169:0/1021607470' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-08-28T02:10:26.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:26 smithi169 conmon[100500]: audit 2023-08-28T02:10: 2023-08-28T02:10:26.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:26 smithi169 conmon[100500]: 26.103856+0000 mon.a (mon.0) 599 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-08-28T02:10:26.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:26 smithi169 conmon[100500]: audit 2023-08-28T02:10:26.108778+0000 mon.a (mon.0) 600 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-08-28T02:10:26.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:26 smithi169 conmon[100500]: audit 2023-08-28T02:10:26.141956+0000 mon.a (mon.0) 601 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:26.488 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:10:26 smithi169 bash[129921]: 1d4ce707c8b732bb5a1c77c75d15366d3eb10255195dc1f46aeea7101a1f8444 2023-08-28T02:10:26.488 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:10:26 smithi169 systemd[1]: Started Ceph prometheus.a for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:10:26.488 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:10:26 smithi169 conmon[130010]: ts=2023-08-28T02:10:26.077Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-08-28T02:10:26.488 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:10:26 smithi169 conmon[130010]: ts=2023-08-28T02:10:26.077Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-08-28T02:10:26.488 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:10:26 smithi169 conmon[130010]: ts=2023-08-28T02:10:26.077Z caller=main.go:518 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi169 (none))" 2023-08-28T02:10:26.488 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:10:26 smithi169 conmon[130010]: ts=2023-08-28T02:10:26.077Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-08-28T02:10:26.488 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:10:26 smithi169 conmon[130010]: ts=2023-08-28T02:10:26.077Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-08-28T02:10:26.488 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:10:26 smithi169 conmon[130010]: ts=2023-08-28T02:10:26.078Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-08-28T02:10:26.488 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:10:26 smithi169 conmon[130010]: ts=2023-08-28T02:10:26.079Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-08-28T02:10:26.488 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:10:26 smithi169 conmon[130010]: ts=2023-08-28T02:10:26.079Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-08-28T02:10:26.489 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:10:26 smithi169 conmon[130010]: ts=2023-08-28T02:10:26.083Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-08-28T02:10:26.489 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:10:26 smithi169 conmon[130010]: ts=2023-08-28T02:10:26.083Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.875µs 2023-08-28T02:10:26.489 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:10:26 smithi169 conmon[130010]: ts=2023-08-28T02:10:26.083Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-08-28T02:10:26.489 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:10:26 smithi169 conmon[130010]: ts=2023-08-28T02:10:26.083Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2023-08-28T02:10:26.489 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:10:26 smithi169 conmon[130010]: ts=2023-08-28T02:10:26.083Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=37.064µs wal_replay_duration=191.82µs total_replay_duration=256.093µs 2023-08-28T02:10:26.489 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:10:26 smithi169 conmon[130010]: ts=2023-08-28T02:10:26.084Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-08-28T02:10:26.489 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:10:26 smithi169 conmon[130010]: ts=2023-08-28T02:10:26.084Z caller=main.go:947 level=info msg="TSDB started" 2023-08-28T02:10:26.489 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:10:26 smithi169 conmon[130010]: ts=2023-08-28T02:10:26.084Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-08-28T02:10:26.489 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:10:26 smithi169 conmon[130010]: ts=2023-08-28T02:10:26.106Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=22.173951ms db_storage=1.038µs remote_storage=1.728µs web_handler=353ns query_engine=741ns scrape=590.713µs scrape_sd=25.183µs notify=1.476µs notify_sd=2.469µs rules=21.31815ms 2023-08-28T02:10:26.489 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:10:26 smithi169 conmon[130010]: ts=2023-08-28T02:10:26.106Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-08-28T02:10:26.820 DEBUG:teuthology.orchestra.run.smithi169:> set -ex 2023-08-28T02:10:26.820 DEBUG:teuthology.orchestra.run.smithi169:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2023-08-28T02:10:26.820 DEBUG:teuthology.orchestra.run.smithi169:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2023-08-28T02:10:26.861 INFO:tasks.cephadm:Waiting for OSDs to come up 2023-08-28T02:10:26.862 DEBUG:teuthology.orchestra.run.smithi143:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2023-08-28T02:10:26.950 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:26 smithi143 systemd[1]: Starting Ceph node-exporter.a for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:10:27.190 INFO:teuthology.misc.health.smithi143.stdout: 2023-08-28T02:10:27.190 INFO:teuthology.misc.health.smithi143.stdout:{"epoch":51,"fsid":"2fcb0bfa-4547-11ee-817c-1b7a5f1ba315","created":"2023-08-28T02:05:20.056816+0000","modified":"2023-08-28T02:10:09.845871+0000","last_up_change":"2023-08-28T02:09:53.076490+0000","last_in_change":"2023-08-28T02:09:35.717712+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-08-28T02:08:01.318811+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"718c8f27-245b-4e9d-9e2e-d1fc65ca7472","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6802","nonce":1846477664},{"type":"v1","addr":"172.21.15.143:6803","nonce":1846477664}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6804","nonce":1846477664},{"type":"v1","addr":"172.21.15.143:6805","nonce":1846477664}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6808","nonce":1846477664},{"type":"v1","addr":"172.21.15.143:6809","nonce":1846477664}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6806","nonce":1846477664},{"type":"v1","addr":"172.21.15.143:6807","nonce":1846477664}]},"public_addr":"172.21.15.143:6803/1846477664","cluster_addr":"172.21.15.143:6805/1846477664","heartbeat_back_addr":"172.21.15.143:6809/1846477664","heartbeat_front_addr":"172.21.15.143:6807/1846477664","state":["exists","up"]},{"osd":1,"uuid":"cf24771f-3b0c-4a00-8a73-13e1fee242dc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6810","nonce":2341769896},{"type":"v1","addr":"172.21.15.143:6811","nonce":2341769896}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6812","nonce":2341769896},{"type":"v1","addr":"172.21.15.143:6813","nonce":2341769896}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6816","nonce":2341769896},{"type":"v1","addr":"172.21.15.143:6817","nonce":2341769896}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6814","nonce":2341769896},{"type":"v1","addr":"172.21.15.143:6815","nonce":2341769896}]},"public_addr":"172.21.15.143:6811/2341769896","cluster_addr":"172.21.15.143:6813/2341769896","heartbeat_back_addr":"172.21.15.143:6817/2341769896","heartbeat_front_addr":"172.21.15.143:6815/2341769896","state":["exists","up"]},{"osd":2,"uuid":"4f8492d1-0e7a-49f8-b6a5-4f5e73842349","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6818","nonce":2664999316},{"type":"v1","addr":"172.21.15.143:6819","nonce":2664999316}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6820","nonce":2664999316},{"type":"v1","addr":"172.21.15.143:6821","nonce":2664999316}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6824","nonce":2664999316},{"type":"v1","addr":"172.21.15.143:6825","nonce":2664999316}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6822","nonce":2664999316},{"type":"v1","addr":"172.21.15.143:6823","nonce":2664999316}]},"public_addr":"172.21.15.143:6819/2664999316","cluster_addr":"172.21.15.143:6821/2664999316","heartbeat_back_addr":"172.21.15.143:6825/2664999316","heartbeat_front_addr":"172.21.15.143:6823/2664999316","state":["exists","up"]},{"osd":3,"uuid":"dd6abc8a-54a9-4e08-ba81-0b894f24d84a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6826","nonce":1468263600},{"type":"v1","addr":"172.21.15.143:6827","nonce":1468263600}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6828","nonce":1468263600},{"type":"v1","addr":"172.21.15.143:6829","nonce":1468263600}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6832","nonce":1468263600},{"type":"v1","addr":"172.21.15.143:6833","nonce":1468263600}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6830","nonce":1468263600},{"type":"v1","addr":"172.21.15.143:6831","nonce":1468263600}]},"public_addr":"172.21.15.143:6827/1468263600","cluster_addr":"172.21.15.143:6829/1468263600","heartbeat_back_addr":"172.21.15.143:6833/1468263600","heartbeat_front_addr":"172.21.15.143:6831/1468263600","state":["exists","up"]},{"osd":4,"uuid":"16122219-d985-46d2-bac1-f8ffcf48d8ad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6800","nonce":3960017153},{"type":"v1","addr":"172.21.15.169:6801","nonce":3960017153}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6802","nonce":3960017153},{"type":"v1","addr":"172.21.15.169:6803","nonce":3960017153}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6806","nonce":3960017153},{"type":"v1","addr":"172.21.15.169:6807","nonce":3960017153}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6804","nonce":3960017153},{"type":"v1","addr":"172.21.15.169:6805","nonce":3960017153}]},"public_addr":"172.21.15.169:6801/3960017153","cluster_addr":"172.21.15.169:6803/3960017153","heartbeat_back_addr":"172.21.15.169:6807/3960017153","heartbeat_front_addr":"172.21.15.169:6805/3960017153","state":["exists","up"]},{"osd":5,"uuid":"adbc3ea5-230f-419e-8f6f-99763378b296","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":37,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6808","nonce":4087672846},{"type":"v1","addr":"172.21.15.169:6809","nonce":4087672846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6810","nonce":4087672846},{"type":"v1","addr":"172.21.15.169:6811","nonce":4087672846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6814","nonce":4087672846},{"type":"v1","addr":"172.21.15.169:6815","nonce":4087672846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6812","nonce":4087672846},{"type":"v1","addr":"172.21.15.169:6813","nonce":4087672846}]},"public_addr":"172.21.15.169:6809/4087672846","cluster_addr":"172.21.15.169:6811/4087672846","heartbeat_back_addr":"172.21.15.169:6815/4087672846","heartbeat_front_addr":"172.21.15.169:6813/4087672846","state":["exists","up"]},{"osd":6,"uuid":"2abe4d77-c12c-477f-a4ec-eabb753ca4dd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6816","nonce":3801227982},{"type":"v1","addr":"172.21.15.169:6817","nonce":3801227982}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6818","nonce":3801227982},{"type":"v1","addr":"172.21.15.169:6819","nonce":3801227982}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6822","nonce":3801227982},{"type":"v1","addr":"172.21.15.169:6823","nonce":3801227982}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6820","nonce":3801227982},{"type":"v1","addr":"172.21.15.169:6821","nonce":3801227982}]},"public_addr":"172.21.15.169:6817/3801227982","cluster_addr":"172.21.15.169:6819/3801227982","heartbeat_back_addr":"172.21.15.169:6823/3801227982","heartbeat_front_addr":"172.21.15.169:6821/3801227982","state":["exists","up"]},{"osd":7,"uuid":"fc5ed8fc-8ad7-4b54-a8ea-bc6a9e846f63","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6824","nonce":3830851149},{"type":"v1","addr":"172.21.15.169:6825","nonce":3830851149}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6826","nonce":3830851149},{"type":"v1","addr":"172.21.15.169:6827","nonce":3830851149}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6830","nonce":3830851149},{"type":"v1","addr":"172.21.15.169:6831","nonce":3830851149}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6828","nonce":3830851149},{"type":"v1","addr":"172.21.15.169:6829","nonce":3830851149}]},"public_addr":"172.21.15.169:6825/3830851149","cluster_addr":"172.21.15.169:6827/3830851149","heartbeat_back_addr":"172.21.15.169:6831/3830851149","heartbeat_front_addr":"172.21.15.169:6829/3830851149","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:07:16.166767+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:07:37.634849+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:07:59.099590+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:08:21.737546+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:08:43.377647+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:09:06.116513+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:09:28.557330+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:09:51.159131+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.143:0/1563442416":"2023-08-29T02:10:09.845824+0000","172.21.15.143:6801/1828180492":"2023-08-29T02:10:09.845824+0000","172.21.15.143:0/2157302459":"2023-08-29T02:10:09.845824+0000","172.21.15.143:6800/1828180492":"2023-08-29T02:10:09.845824+0000","172.21.15.143:6801/907474889":"2023-08-29T02:06:02.911068+0000","172.21.15.143:6800/907474889":"2023-08-29T02:06:02.911068+0000","172.21.15.143:0/2323551826":"2023-08-29T02:06:02.911068+0000","172.21.15.143:6800/1019944263":"2023-08-29T02:05:41.797747+0000","172.21.15.143:0/3606471716":"2023-08-29T02:06:02.911068+0000","172.21.15.143:6801/1019944263":"2023-08-29T02:05:41.797747+0000","172.21.15.143:0/11141072":"2023-08-29T02:05:41.797747+0000","172.21.15.143:0/1148418770":"2023-08-29T02:05:41.797747+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-08-28T02:10:27.202 DEBUG:teuthology.misc:8 of 8 OSDs are up 2023-08-28T02:10:27.202 INFO:tasks.cephadm:Creating RBD pool 2023-08-28T02:10:27.203 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- sudo ceph --cluster ceph osd pool create rbd 8 2023-08-28T02:10:27.418 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:26 smithi143 bash[138267]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2023-08-28T02:10:27.419 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:27 smithi143 conmon[100258]: cluster 2023-08-28T02:10:25.860471+0000 mgr.y (mgr.14424) 20 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:27.419 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:27 smithi143 conmon[100258]: cephadm 2023-08-28T02 2023-08-28T02:10:27.419 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:27 smithi143 conmon[100258]: :10:26.143950+0000 mgr.y (mgr.14424) 21 : cephadm [INF] Deploying daemon node-exporter.a on smithi143 2023-08-28T02:10:27.419 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:27 smithi143 conmon[109923]: cluster 2023-08-28T02:10:25.860471+0000 mgr.y 2023-08-28T02:10:27.419 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:27 smithi143 conmon[109923]: (mgr.14424) 20 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:27.419 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:27 smithi143 conmon[109923]: cephadm 2023-08-28T02:10:26.143950+0000 mgr.y (mgr. 2023-08-28T02:10:27.419 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:27 smithi143 conmon[109923]: 14424) 21 : cephadm [INF] Deploying daemon node-exporter.a on smithi143 2023-08-28T02:10:27.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:27 smithi169 conmon[100500]: cluster 2023-08-28T02:10:25.860471+0000 mgr.y (mgr.14424) 20 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:27.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:27 smithi169 conmon[100500]: cephadm 2023-08-28 2023-08-28T02:10:27.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:27 smithi169 conmon[100500]: T02:10:26.143950+0000 mgr.y (mgr.14424) 21 : cephadm [INF] Deploying daemon node-exporter.a on smithi143 2023-08-28T02:10:27.677 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:27 smithi143 bash[138267]: Getting image source signatures 2023-08-28T02:10:27.677 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:27 smithi143 bash[138267]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2023-08-28T02:10:27.950 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:27 smithi143 bash[138267]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2023-08-28T02:10:27.950 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:27 smithi143 bash[138267]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2023-08-28T02:10:28.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:28 smithi143 conmon[100258]: audit 2023-08-28T02:10:27.191047+0000 mon.a 2023-08-28T02:10:28.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:28 smithi143 conmon[100258]: (mon.0) 602 : audit [DBG] from='client.? 172.21.15.143:0/637805143' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-28T02:10:28.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:28 smithi143 conmon[109923]: audit 2023-08-28T02:10:27.191047+0000 mon.a (mon.0) 602 : 2023-08-28T02:10:28.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:28 smithi143 conmon[109923]: audit [DBG] from='client.? 172.21.15.143:0/637805143' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-28T02:10:28.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:28 smithi169 conmon[100500]: audit 2023-08-28T02:10:27.191047+0000 mon.a (mon.0) 602 : audit [DBG] from='client.? 172.21.15.143:0/637805143' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-28T02:10:28.950 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:28 smithi143 bash[138267]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2023-08-28T02:10:28.950 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:28 smithi143 bash[138267]: Writing manifest to image destination 2023-08-28T02:10:28.950 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:28 smithi143 bash[138267]: Storing signatures 2023-08-28T02:10:29.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:29 smithi143 conmon[100258]: cluster 2023-08-28T02:10:27.860811+0000 mgr.y (mgr. 2023-08-28T02:10:29.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:29 smithi143 conmon[100258]: 14424) 22 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:29.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:29 smithi143 conmon[109923]: cluster 2023-08-28T02:10:27.860811+0000 mgr.y (mgr.14424) 22 : cluster 2023-08-28T02:10:29.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:29 smithi143 conmon[109923]: [DBG] pgmap v12: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:29.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:29 smithi169 conmon[100500]: cluster 2023-08-28T02:10:27.860811+0000 mgr.y (mgr.14424) 22 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:30.450 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.043Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-08-28T02:10:30.450 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.043Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-08-28T02:10:30.450 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.043Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-08-28T02:10:30.450 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.043Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-08-28T02:10:30.451 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.044Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-08-28T02:10:30.451 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.044Z caller=node_exporter.go:115 level=info collector=arp 2023-08-28T02:10:30.451 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.044Z caller=node_exporter.go:115 level=info collector=bcache 2023-08-28T02:10:30.451 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.044Z caller=node_exporter.go:115 level=info collector=bonding 2023-08-28T02:10:30.451 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.044Z caller=node_exporter.go:115 level=info collector=btrfs 2023-08-28T02:10:30.451 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.044Z caller=node_exporter.go:115 level=info collector=conntrack 2023-08-28T02:10:30.451 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.044Z caller=node_exporter.go:115 level=info collector=cpu 2023-08-28T02:10:30.451 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.044Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-08-28T02:10:30.451 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.044Z caller=node_exporter.go:115 level=info collector=diskstats 2023-08-28T02:10:30.452 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.044Z caller=node_exporter.go:115 level=info collector=dmi 2023-08-28T02:10:30.452 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.044Z caller=node_exporter.go:115 level=info collector=edac 2023-08-28T02:10:30.452 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.044Z caller=node_exporter.go:115 level=info collector=entropy 2023-08-28T02:10:30.452 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.044Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-08-28T02:10:30.452 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.044Z caller=node_exporter.go:115 level=info collector=filefd 2023-08-28T02:10:30.452 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.044Z caller=node_exporter.go:115 level=info collector=filesystem 2023-08-28T02:10:30.452 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.044Z caller=node_exporter.go:115 level=info collector=hwmon 2023-08-28T02:10:30.452 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.044Z caller=node_exporter.go:115 level=info collector=infiniband 2023-08-28T02:10:30.452 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.044Z caller=node_exporter.go:115 level=info collector=ipvs 2023-08-28T02:10:30.452 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.044Z caller=node_exporter.go:115 level=info collector=loadavg 2023-08-28T02:10:30.453 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.044Z caller=node_exporter.go:115 level=info collector=mdadm 2023-08-28T02:10:30.453 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.044Z caller=node_exporter.go:115 level=info collector=meminfo 2023-08-28T02:10:30.453 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.044Z caller=node_exporter.go:115 level=info collector=netclass 2023-08-28T02:10:30.453 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.044Z caller=node_exporter.go:115 level=info collector=netdev 2023-08-28T02:10:30.453 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.044Z caller=node_exporter.go:115 level=info collector=netstat 2023-08-28T02:10:30.453 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.044Z caller=node_exporter.go:115 level=info collector=nfs 2023-08-28T02:10:30.453 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.044Z caller=node_exporter.go:115 level=info collector=nfsd 2023-08-28T02:10:30.453 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.044Z caller=node_exporter.go:115 level=info collector=nvme 2023-08-28T02:10:30.453 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.044Z caller=node_exporter.go:115 level=info collector=os 2023-08-28T02:10:30.453 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.044Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-08-28T02:10:30.453 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.044Z caller=node_exporter.go:115 level=info collector=pressure 2023-08-28T02:10:30.453 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.044Z caller=node_exporter.go:115 level=info collector=rapl 2023-08-28T02:10:30.454 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.044Z caller=node_exporter.go:115 level=info collector=schedstat 2023-08-28T02:10:30.454 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.044Z caller=node_exporter.go:115 level=info collector=sockstat 2023-08-28T02:10:30.454 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.044Z caller=node_exporter.go:115 level=info collector=softnet 2023-08-28T02:10:30.454 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.044Z caller=node_exporter.go:115 level=info collector=stat 2023-08-28T02:10:30.454 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.044Z caller=node_exporter.go:115 level=info collector=tapestats 2023-08-28T02:10:30.454 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.044Z caller=node_exporter.go:115 level=info collector=textfile 2023-08-28T02:10:30.454 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.044Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-08-28T02:10:30.454 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.044Z caller=node_exporter.go:115 level=info collector=time 2023-08-28T02:10:30.455 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.044Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-08-28T02:10:30.455 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.044Z caller=node_exporter.go:115 level=info collector=uname 2023-08-28T02:10:30.455 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.044Z caller=node_exporter.go:115 level=info collector=vmstat 2023-08-28T02:10:30.455 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.044Z caller=node_exporter.go:115 level=info collector=xfs 2023-08-28T02:10:30.455 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.044Z caller=node_exporter.go:115 level=info collector=zfs 2023-08-28T02:10:30.455 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.045Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-08-28T02:10:30.455 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[138592]: ts=2023-08-28T02:10:30.045Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-08-28T02:10:30.455 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 bash[138267]: 08a67263bdce604c5d878670d376f6912281809083c09ade24c38663249ad87d 2023-08-28T02:10:30.455 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:10:30 smithi143 systemd[1]: Started Ceph node-exporter.a for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:10:30.737 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:30 smithi169 systemd[1]: Starting Ceph node-exporter.b for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:10:31.178 INFO:teuthology.orchestra.run.smithi143.stderr:pool 'rbd' created 2023-08-28T02:10:31.237 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:30 smithi169 bash[130453]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2023-08-28T02:10:31.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:30 smithi169 conmon[100500]: cluster 2023-08-28T02:10:29.861297+0000 mgr.y (mgr.14424) 23 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:31.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:30 smithi169 conmon[100500]: audit 2023- 2023-08-28T02:10:31.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:30 smithi169 conmon[100500]: 08-28T02:10:29.995089+0000 mon.a (mon.0) 603 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:31.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:30 smithi169 conmon[100500]: audit 2023-08-28T02:10:30.130700+0000 mon.a (mon.0) 604 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:31.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:30 smithi169 conmon[100500]: cephadm 2023-08-28T02:10:30.131544+0000 mgr.y (mgr.14424) 24 : cephadm [INF] Deploying daemon node-exporter.b on smithi169 2023-08-28T02:10:31.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:30 smithi169 conmon[100500]: audit 2023-08-28T02:10:30.379917+0000 mon.c (mon.2) 12 : audit [INF] from='client.? 172.21.15.143:0/1560209343' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-08-28T02:10:31.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:30 smithi169 conmon[100500]: audit 2023-08-28T02:10:30.380258+0000 mon.a (mon.0) 605 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-08-28T02:10:31.399 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[109923]: cluster 2023-08-28T02:10:29.861297+0000 2023-08-28T02:10:31.400 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[109923]: mgr.y (mgr.14424) 23 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:31.400 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[109923]: audit 2023-08-28T02:10:31.400 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[109923]: 2023-08-28T02 2023-08-28T02:10:31.400 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[109923]: :10:29.995089+0000 mon.a (mon 2023-08-28T02:10:31.400 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[109923]: .0) 603 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:31.400 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[109923]: audit 2023-08 2023-08-28T02:10:31.400 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[109923]: -28T02:10:30.130700+0000 mon.a (mon.0) 604 : audit [INF] 2023-08-28T02:10:31.400 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[109923]: from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:31.400 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[109923]: cephadm 2023-08-28T02: 2023-08-28T02:10:31.400 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[109923]: 10:30.131544+0000 mgr.y (mgr.14424) 24 : cephadm [INF] Deploying daemon node-exporter.b on smithi169 2023-08-28T02:10:31.401 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[109923]: audit 2023-08-28T02:10:31.401 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[109923]: 2023-08-28T02:10:30.379917+0000 mon.c ( 2023-08-28T02:10:31.401 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[109923]: mon.2) 12 : audit [INF] from='client.? 172.21.15.143:0/1560209343' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-08-28T02:10:31.401 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[109923]: 2023-08-28T02:10:31.401 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[109923]: audit 2023 2023-08-28T02:10:31.401 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[109923]: -08-28T02:10:30.380258+0000 2023-08-28T02:10:31.401 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[109923]: mon.a (mon.0) 605 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-08-28T02:10:31.401 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[109923]: 2023-08-28T02:10:31.401 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[100258]: cluster 2023-08-28T02:10:29.861297+0000 2023-08-28T02:10:31.402 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[100258]: mgr.y (mgr.14424) 23 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:31.402 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[100258]: audit 2023-08-28T02:10:29.995089+0000 mon.a (mon.0) 603 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:31.402 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[100258]: audit 2023-08-28T02:10:30.130700+0000 mon.a (mon.0) 604 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:31.402 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[100258]: cephadm 2023-08-28T02:10: 2023-08-28T02:10:31.402 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[100258]: 30.131544+0000 mgr.y (mgr.14424) 24 : cephadm [INF] Deploying daemon node-exporter.b on smithi169 2023-08-28T02:10:31.402 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[100258]: audit 2023 2023-08-28T02:10:31.402 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[100258]: -08-28T02:10:30.379917+0000 mon.c 2023-08-28T02:10:31.402 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[100258]: (mon.2) 12 : audit [INF] from='client.? 172.21.15.143:0/1560209343' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-08-28T02:10:31.402 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[100258]: audit 2023 2023-08-28T02:10:31.402 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[100258]: -08-28T02:10:30 2023-08-28T02:10:31.403 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[100258]: .380258+0000 mon.a (mon.0) 605 : audit 2023-08-28T02:10:31.403 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:30 smithi143 conmon[100258]: [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-08-28T02:10:31.737 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:31 smithi169 bash[130453]: Getting image source signatures 2023-08-28T02:10:31.737 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:31 smithi169 bash[130453]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2023-08-28T02:10:31.737 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:31 smithi169 bash[130453]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2023-08-28T02:10:31.737 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:31 smithi169 bash[130453]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2023-08-28T02:10:32.129 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2023-08-28T02:10:32.177 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:31 smithi169 bash[130453]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2023-08-28T02:10:32.177 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:31 smithi169 bash[130453]: Writing manifest to image destination 2023-08-28T02:10:32.178 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:31 smithi169 bash[130453]: Storing signatures 2023-08-28T02:10:32.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:32 smithi143 conmon[100258]: audit 2023-08-28T02:10:31.175290+0000 mon.a (mon.0) 606 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-08-28T02:10:32.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:32 smithi143 conmon[100258]: cluster 2023-08-28T02:10:31.175355+0000 mon.a (mon.0) 607 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2023-08-28T02:10:32.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:32 smithi143 conmon[100258]: audit 2023-08-28T02:10:31.175677+0000 mon.a (mon.0) 608 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-08-28T02:10:32.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:32 smithi143 conmon[100258]: audit 2023-08-28T02:10:31.176449+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-08-28T02:10:32.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:32 smithi143 conmon[109923]: audit 2023-08-28T02:10:31.175290+0000 mon.a (mon.0) 606 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-08-28T02:10:32.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:32 smithi143 conmon[109923]: cluster 2023-08-28T02:10:31.175355+0000 mon.a (mon.0) 607 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2023-08-28T02:10:32.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:32 smithi143 conmon[109923]: audit 2023-08-28T02:10:31.175677+0000 mon.a (mon.0) 608 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-08-28T02:10:32.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:32 smithi143 conmon[109923]: audit 2023-08-28T02:10:31.176449+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-08-28T02:10:32.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:32 smithi169 conmon[100500]: audit 2023-08-28T02:10:31.175290+0000 2023-08-28T02:10:32.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:32 smithi169 conmon[100500]: mon.a (mon.0) 606 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-08-28T02:10:32.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:32 smithi169 conmon[100500]: cluster 2023-08-28T02:10:31.175355+0000 mon.a (mon.0) 607 : 2023-08-28T02:10:32.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:32 smithi169 conmon[100500]: cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2023-08-28T02:10:32.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:32 smithi169 conmon[100500]: audit 2023-08-28T02:10:31.175677 2023-08-28T02:10:32.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:32 smithi169 conmon[100500]: +0000 mon.a (mon.0) 608 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-08-28T02:10:32.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:32 smithi169 conmon[100500]: audit 2023-08-28T02: 2023-08-28T02:10:32.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:32 smithi169 conmon[100500]: 10:31.176449+0000 mon.a (mon.0) 609 : audit 2023-08-28T02:10:32.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:32 smithi169 conmon[100500]: [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-08-28T02:10:33.184 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:33 smithi143 conmon[100258]: cluster 2023-08-28T02:10:31.861731+0000 mgr.y 2023-08-28T02:10:33.185 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:33 smithi143 conmon[100258]: (mgr.14424) 25 : cluster [DBG] pgmap v15: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:33.185 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:33 smithi143 conmon[100258]: audit 2023-08-28T02:10:32.181354+0000 mon.a (mon.0) 610 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-08-28T02:10:33.185 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:33 smithi143 conmon[100258]: cluster 2023-08-28T02:10:32.181499+0000 mon.a (mon.0) 611 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2023-08-28T02:10:33.202 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:10:32 smithi169 conmon[102807]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:10:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-28T02:10:33.203 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[100500]: cluster 2023-08-28T02:10:31.861731+0000 mgr.y (mgr.14424) 25 : 2023-08-28T02:10:33.203 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[100500]: cluster [DBG] pgmap v15: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:33.203 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[100500]: audit 2023-08-28T02:10:32.181354+0000 mon.a (mon.0) 610 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-08-28T02:10:33.203 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[100500]: cluster 2023-08-28T02:10:32.181499 2023-08-28T02:10:33.203 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[100500]: +0000 mon.a (mon.0) 611 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2023-08-28T02:10:33.203 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[100500]: 2023-08-28T02:10:33.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:33 smithi143 conmon[109923]: cluster 2023-08-28T02:10:31.861731+0000 mgr.y 2023-08-28T02:10:33.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:33 smithi143 conmon[109923]: (mgr.14424) 25 : cluster [DBG] pgmap v15: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:33.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:33 smithi143 conmon[109923]: audit 2023-08-28T02:10:32.181354+0000 mon.a (mon.0) 610 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-08-28T02:10:33.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:33 smithi143 conmon[109923]: cluster 2023-08-28T02:10:32.181499+0000 mon.a (mon.0) 611 : cluster [DBG] 2023-08-28T02:10:33.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:33 smithi143 conmon[109923]: osdmap e53: 8 total, 8 up, 8 in 2023-08-28T02:10:33.737 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.315Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-08-28T02:10:33.737 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.316Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-08-28T02:10:33.738 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.316Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-08-28T02:10:33.738 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.316Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-08-28T02:10:33.738 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.317Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-08-28T02:10:33.738 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.317Z caller=node_exporter.go:115 level=info collector=arp 2023-08-28T02:10:33.738 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.317Z caller=node_exporter.go:115 level=info collector=bcache 2023-08-28T02:10:33.738 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.317Z caller=node_exporter.go:115 level=info collector=bonding 2023-08-28T02:10:33.738 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.317Z caller=node_exporter.go:115 level=info collector=btrfs 2023-08-28T02:10:33.738 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.317Z caller=node_exporter.go:115 level=info collector=conntrack 2023-08-28T02:10:33.738 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.317Z caller=node_exporter.go:115 level=info collector=cpu 2023-08-28T02:10:33.738 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.317Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-08-28T02:10:33.738 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.317Z caller=node_exporter.go:115 level=info collector=diskstats 2023-08-28T02:10:33.739 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.317Z caller=node_exporter.go:115 level=info collector=dmi 2023-08-28T02:10:33.739 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.317Z caller=node_exporter.go:115 level=info collector=edac 2023-08-28T02:10:33.739 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.317Z caller=node_exporter.go:115 level=info collector=entropy 2023-08-28T02:10:33.739 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.317Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-08-28T02:10:33.739 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.317Z caller=node_exporter.go:115 level=info collector=filefd 2023-08-28T02:10:33.739 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.317Z caller=node_exporter.go:115 level=info collector=filesystem 2023-08-28T02:10:33.739 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.317Z caller=node_exporter.go:115 level=info collector=hwmon 2023-08-28T02:10:33.739 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.317Z caller=node_exporter.go:115 level=info collector=infiniband 2023-08-28T02:10:33.739 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.317Z caller=node_exporter.go:115 level=info collector=ipvs 2023-08-28T02:10:33.739 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.317Z caller=node_exporter.go:115 level=info collector=loadavg 2023-08-28T02:10:33.739 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.317Z caller=node_exporter.go:115 level=info collector=mdadm 2023-08-28T02:10:33.739 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.317Z caller=node_exporter.go:115 level=info collector=meminfo 2023-08-28T02:10:33.739 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.317Z caller=node_exporter.go:115 level=info collector=netclass 2023-08-28T02:10:33.740 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.317Z caller=node_exporter.go:115 level=info collector=netdev 2023-08-28T02:10:33.740 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.317Z caller=node_exporter.go:115 level=info collector=netstat 2023-08-28T02:10:33.740 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.317Z caller=node_exporter.go:115 level=info collector=nfs 2023-08-28T02:10:33.740 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.317Z caller=node_exporter.go:115 level=info collector=nfsd 2023-08-28T02:10:33.740 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.317Z caller=node_exporter.go:115 level=info collector=nvme 2023-08-28T02:10:33.740 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.317Z caller=node_exporter.go:115 level=info collector=os 2023-08-28T02:10:33.740 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.317Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-08-28T02:10:33.740 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.317Z caller=node_exporter.go:115 level=info collector=pressure 2023-08-28T02:10:33.740 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.317Z caller=node_exporter.go:115 level=info collector=rapl 2023-08-28T02:10:33.740 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.317Z caller=node_exporter.go:115 level=info collector=schedstat 2023-08-28T02:10:33.740 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.317Z caller=node_exporter.go:115 level=info collector=sockstat 2023-08-28T02:10:33.740 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.317Z caller=node_exporter.go:115 level=info collector=softnet 2023-08-28T02:10:33.740 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.317Z caller=node_exporter.go:115 level=info collector=stat 2023-08-28T02:10:33.741 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.317Z caller=node_exporter.go:115 level=info collector=tapestats 2023-08-28T02:10:33.741 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.317Z caller=node_exporter.go:115 level=info collector=textfile 2023-08-28T02:10:33.741 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.317Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-08-28T02:10:33.741 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.317Z caller=node_exporter.go:115 level=info collector=time 2023-08-28T02:10:33.741 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.317Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-08-28T02:10:33.741 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.317Z caller=node_exporter.go:115 level=info collector=uname 2023-08-28T02:10:33.741 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.317Z caller=node_exporter.go:115 level=info collector=vmstat 2023-08-28T02:10:33.741 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.317Z caller=node_exporter.go:115 level=info collector=xfs 2023-08-28T02:10:33.741 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.317Z caller=node_exporter.go:115 level=info collector=zfs 2023-08-28T02:10:33.741 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.317Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-08-28T02:10:33.741 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 conmon[130530]: ts=2023-08-28T02:10:33.317Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-08-28T02:10:33.741 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 bash[130453]: b3659a52b45015144f30af8920d26185ebe63902c5f30051e7f5197bedd53aad 2023-08-28T02:10:33.741 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:10:33 smithi169 systemd[1]: Started Ceph node-exporter.b for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:10:34.418 INFO:teuthology.orchestra.run.smithi143.stderr:enabled application 'rbd' on pool 'rbd' 2023-08-28T02:10:34.443 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:34 smithi143 conmon[100258]: cluster 2023-08-28T02:10:33.181644+0000 2023-08-28T02:10:34.443 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:34 smithi143 conmon[100258]: mon.a (mon.0) 612 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-08-28T02:10:34.443 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:34 smithi143 conmon[100258]: audit 2023-08-28T02:10:33.404324+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:34.443 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:34 smithi143 conmon[100258]: audit 2023-08-28T02:10:33.405329+0000 mon.a (mon.0) 614 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:10:34.443 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:34 smithi143 conmon[100258]: audit 2023-08-28T02:10:33.953043+0000 mon.c (mon.2) 13 : audit [INF] from='client.? 172.21.15.143:0/806588443' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-08-28T02:10:34.443 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:34 smithi143 conmon[100258]: audit 2023-08-28T02:10:33.953421+0000 mon.a (mon.0) 615 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-08-28T02:10:34.443 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:34 smithi143 conmon[109923]: cluster 2023-08-28T02:10:33.181644+0000 2023-08-28T02:10:34.444 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:34 smithi143 conmon[109923]: mon.a (mon.0) 612 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-08-28T02:10:34.444 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:34 smithi143 conmon[109923]: audit 2023-08-28T02:10: 2023-08-28T02:10:34.444 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:34 smithi143 conmon[109923]: 33.404324+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:34.444 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:34 smithi143 conmon[109923]: audit 2023-08-28T 2023-08-28T02:10:34.444 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:34 smithi143 conmon[109923]: 02:10:33.405329+0000 mon.a (mon. 2023-08-28T02:10:34.444 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:34 smithi143 conmon[109923]: 0) 614 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:10:34.444 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:34 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:10:34.444 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:34 smithi143 conmon[109923]: 08-28T02:10:33.953043 2023-08-28T02:10:34.444 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:34 smithi143 conmon[109923]: +0000 mon.c (mon.2) 13 : audit [INF] 2023-08-28T02:10:34.445 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:34 smithi143 conmon[109923]: from='client.? 172.21.15.143:0/806588443' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-08-28T02:10:34.445 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:34 smithi143 conmon[109923]: audit 2023-08-28T02:10:34.445 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:34 smithi143 conmon[109923]: 2023-08-28T02:10:33.953421+0000 2023-08-28T02:10:34.445 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:34 smithi143 conmon[109923]: mon.a (mon.0) 615 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-08-28T02:10:34.445 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:34 smithi143 conmon[109923]: 2023-08-28T02:10:34.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:34 smithi169 conmon[100500]: cluster 2023-08-28T02:10:33.181644+0000 mon.a (mon.0) 612 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-08-28T02:10:34.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:34 smithi169 conmon[100500]: audit 2023-08-28T02:10:33.404324+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:34.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:34 smithi169 conmon[100500]: audit 2023-08-28T02:10:33.405329+0000 mon.a (mon.0) 614 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:10:34.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:34 smithi169 conmon[100500]: audit 2023-08-28T02:10:33.953043+0000 mon.c (mon.2) 13 : audit [INF] from='client.? 172.21.15.143:0/806588443' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-08-28T02:10:34.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:34 smithi169 conmon[100500]: audit 2023-08-28T02:10:33.953421+0000 mon.a (mon.0) 615 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-08-28T02:10:35.053 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-08-28T02:10:35.053 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-08-28T02:10:35.053 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph mgr dump --format=json 2023-08-28T02:10:35.618 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:10:35.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:35 smithi143 conmon[100258]: cluster 2023-08-28T02:10:33.862078+0000 mgr.y (mgr.14424) 26 : cluster [DBG] pgmap v18: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:35.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:35 smithi143 conmon[100258]: audit 2023-08-28T02:10:34.416215+0000 mon.a (mon.0) 616 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-08-28T02:10:35.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:35 smithi143 conmon[100258]: cluster 2023-08-28T02:10:34.416306+0000 mon.a (mon.0) 617 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-08-28T02:10:35.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:35 smithi143 conmon[100258]: audit 2023-08-28T02:10:35.000700+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:35.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:35 smithi143 conmon[109923]: cluster 2023-08-28T02:10:33.862078+0000 mgr.y (mgr.14424) 26 : cluster [DBG] pgmap v18: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:35.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:35 smithi143 conmon[109923]: audit 2023-08-28T02:10:34.416215+0000 mon.a (mon.0) 616 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-08-28T02:10:35.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:35 smithi143 conmon[109923]: cluster 2023-08-28T02:10:34.416306+0000 mon.a (mon.0) 617 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-08-28T02:10:35.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:35 smithi143 conmon[109923]: audit 2023-08-28T02:10:35.000700+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:35.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:35 smithi169 conmon[100500]: cluster 2023-08-28T02:10:33.862078+0000 mgr.y (mgr 2023-08-28T02:10:35.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:35 smithi169 conmon[100500]: .14424) 26 : cluster [DBG] pgmap v18: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:35.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:35 smithi169 conmon[100500]: audit 2023-08-28T02:10:34.416215+0000 mon.a (mon.0) 616 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-08-28T02:10:35.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:35 smithi169 conmon[100500]: cluster 2023-08-28T02:10:34.416306+0000 mon.a (mon.0) 617 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-08-28T02:10:35.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:35 smithi169 conmon[100500]: audit 2023-08-28T02:10:35.000700+0000 mon.a (mon.0) 618 2023-08-28T02:10:35.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:35 smithi169 conmon[100500]: : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:36.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:36 smithi169 conmon[100500]: audit 2023-08-28T02:10:35.475885+0000 mon.a (mon.0) 619 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:36.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:36 smithi169 conmon[100500]: audit 2023-08-28T02:10:35.587213+0000 mon.a (mon.0) 620 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:36.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:36 smithi169 conmon[100500]: audit 2023-08-28T02:10:35.587938+0000 mon.a (mon.0) 621 : 2023-08-28T02:10:36.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:36 smithi169 conmon[100500]: audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:10:36.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:36 smithi169 conmon[100500]: audit 2023-08-28T02:10:35.588546+0000 mon.a (mon.0) 622 2023-08-28T02:10:36.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:36 smithi169 conmon[100500]: : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:10:36.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:36 smithi169 conmon[100500]: audit 2023-08-28T02:10: 2023-08-28T02:10:36.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:36 smithi169 conmon[100500]: 35.592419+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:36.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:36 smithi169 conmon[100500]: cephadm 2023 2023-08-28T02:10:36.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:36 smithi169 conmon[100500]: -08-28T02:10:35.600092+0000 mgr.y (mgr.14424) 27 : cephadm [INF] Deploying daemon alertmanager.a on smithi143 2023-08-28T02:10:36.792 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:36 smithi143 conmon[100258]: audit 2023-08-28T02:10:35.475885+0000 mon.a (mon.0 2023-08-28T02:10:36.792 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:36 smithi143 conmon[100258]: ) 619 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:36.792 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:36 smithi143 conmon[100258]: audit 2023-08-28T02:10:35.587213+0000 mon.a (mon.0) 620 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:36.792 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:36 smithi143 conmon[100258]: audit 2023-08-28T02:10:35.587938+0000 mon.a (mon.0) 621 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:10:36.792 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:36 smithi143 conmon[100258]: audit 2023-08-28T02:10:35.588546+0000 mon.a (mon.0) 622 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:10:36.792 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:36 smithi143 conmon[100258]: audit 2023-08-28T02:10:35.592419+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:36.792 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:36 smithi143 conmon[100258]: cephadm 2023-08-28T02:10:35.600092+0000 mgr.y (mgr.14424) 27 : cephadm [INF] Deploying daemon alertmanager.a on smithi143 2023-08-28T02:10:36.792 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:36 smithi143 conmon[109923]: audit 2023-08-28T02:10:35.475885+0000 mon.a (mon 2023-08-28T02:10:36.793 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:36 smithi143 conmon[109923]: .0) 619 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:36.793 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:36 smithi143 conmon[109923]: audit 2023-08-28T02:10:35.587213+0000 mon.a (mon.0) 620 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:36.793 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:36 smithi143 conmon[109923]: audit 2023-08-28T02:10:35.587938+0000 mon.a (mon.0) 621 : audit [DBG] 2023-08-28T02:10:36.793 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:36 smithi143 conmon[109923]: from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:10:36.793 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:36 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:10:36.793 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:36 smithi143 conmon[109923]: 08-28T02:10:35.588546+0000 mon.a (mon.0) 622 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:10:36.793 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:36 smithi143 conmon[109923]: audit 2023-08-28T02:10: 2023-08-28T02:10:36.793 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:36 smithi143 conmon[109923]: 35.592419+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:36.793 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:36 smithi143 conmon[109923]: cephadm 2023-08-28T02:10: 2023-08-28T02:10:36.793 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:36 smithi143 conmon[109923]: 35.600092+0000 mgr.y (mgr.14424) 27 : cephadm [INF] Deploying daemon alertmanager.a on smithi143 2023-08-28T02:10:37.281 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:10:37.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:37 smithi143 conmon[100258]: cluster 2023-08-28T02:10:35.862363+0000 mgr.y (mgr.14424) 28 : cluster [DBG] pgmap v20: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:37.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:37 smithi143 conmon[100258]: audit 2023-08-28T02:10:37.277796+0000 mon.a (mon.0) 624 : audit [DBG] from='client.? 172.21.15.143:0/4261667059' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-08-28T02:10:37.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:37 smithi143 conmon[109923]: cluster 2023-08-28T02:10:35.862363+0000 mgr.y (mgr.14424 2023-08-28T02:10:37.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:37 smithi143 conmon[109923]: ) 28 : cluster [DBG] pgmap v20: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:37.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:37 smithi143 conmon[109923]: audit 2023-08-28T02:10:37.277796+0000 mon.a (mon.0) 2023-08-28T02:10:37.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:37 smithi143 conmon[109923]: 624 : audit [DBG] from='client.? 172.21.15.143:0/4261667059' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-08-28T02:10:37.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:37 smithi169 conmon[100500]: cluster 2023-08-28T02:10:35.862363+0000 mgr.y (mgr.14424) 28 : cluster [DBG] pgmap v20: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:37.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:37 smithi169 conmon[100500]: audit 2023-08-28T02:10:37.277796+0000 mon.a (mon.0) 624 : audit [DBG] from='client.? 172.21.15.143:0/4261667059' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-08-28T02:10:38.516 INFO:teuthology.orchestra.run.smithi143.stdout:{"epoch":18,"active_gid":14424,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6800","nonce":1118800746},{"type":"v1","addr":"172.21.15.143:6801","nonce":1118800746}]},"active_addr":"172.21.15.143:6801/1118800746","active_change":"2023-08-28T02:10:09.846041+0000","active_mgr_features":4540138314316775423,"available":true,"standbys":[{"gid":24328,"name":"x","mgr_features":4540138314316775423,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.143:8443/","prometheus":"http://172.21.15.143:9283/"},"always_on_modules":{"nautilus":["balancer","crash","devicehealth","orchestrator_cli","progress","rbd_support","status","volumes"],"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":51,"active_clients":[{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.143:0","nonce":1035910192}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.143:0","nonce":3251765285}]}]} 2023-08-28T02:10:38.521 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-08-28T02:10:38.521 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-08-28T02:10:38.521 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd dump --format=json 2023-08-28T02:10:39.191 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:10:39.916 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:39 smithi143 conmon[100258]: cluster 2023-08-28T02:10:37.862703 2023-08-28T02:10:39.942 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:39 smithi143 conmon[100258]: +0000 mgr.y (mgr.14424) 29 : cluster [DBG] pgmap v21: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:39.943 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:39 smithi143 conmon[109923]: cluster 2023-08-28T02:10:37.862703+0000 mgr.y (mgr.14424) 29 : cluster [DBG] pgmap v21: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:39.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:39 smithi169 conmon[100500]: cluster 2023-08-28T02:10:37.862703+0000 mgr.y (mgr.14424) 29 : cluster [DBG] pgmap v21: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:40.700 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:10:40 smithi143 conmon[100708]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:10:40] "GET /metrics HTTP/1.1" 200 155664 "" "Prometheus/2.33.4" 2023-08-28T02:10:41.792 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:41 smithi143 conmon[100258]: cluster 2023-08-28T02:10:39.862996+0000 2023-08-28T02:10:41.792 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:41 smithi143 conmon[100258]: mgr.y (mgr.14424) 30 : cluster [DBG] pgmap v22: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:41.792 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:41 smithi143 conmon[109923]: cluster 2023-08-28T02:10:39.862996+0000 2023-08-28T02:10:41.792 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:41 smithi143 conmon[109923]: mgr.y (mgr.14424) 30 : cluster [DBG] pgmap v22: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:41.892 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:10:41.892 INFO:teuthology.orchestra.run.smithi143.stdout:{"epoch":55,"fsid":"2fcb0bfa-4547-11ee-817c-1b7a5f1ba315","created":"2023-08-28T02:05:20.056816+0000","modified":"2023-08-28T02:10:34.402737+0000","last_up_change":"2023-08-28T02:09:53.076490+0000","last_in_change":"2023-08-28T02:09:35.717712+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-08-28T02:08:01.318811+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-08-28T02:10:30.618550+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"718c8f27-245b-4e9d-9e2e-d1fc65ca7472","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6802","nonce":1846477664},{"type":"v1","addr":"172.21.15.143:6803","nonce":1846477664}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6804","nonce":1846477664},{"type":"v1","addr":"172.21.15.143:6805","nonce":1846477664}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6808","nonce":1846477664},{"type":"v1","addr":"172.21.15.143:6809","nonce":1846477664}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6806","nonce":1846477664},{"type":"v1","addr":"172.21.15.143:6807","nonce":1846477664}]},"public_addr":"172.21.15.143:6803/1846477664","cluster_addr":"172.21.15.143:6805/1846477664","heartbeat_back_addr":"172.21.15.143:6809/1846477664","heartbeat_front_addr":"172.21.15.143:6807/1846477664","state":["exists","up"]},{"osd":1,"uuid":"cf24771f-3b0c-4a00-8a73-13e1fee242dc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6810","nonce":2341769896},{"type":"v1","addr":"172.21.15.143:6811","nonce":2341769896}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6812","nonce":2341769896},{"type":"v1","addr":"172.21.15.143:6813","nonce":2341769896}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6816","nonce":2341769896},{"type":"v1","addr":"172.21.15.143:6817","nonce":2341769896}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6814","nonce":2341769896},{"type":"v1","addr":"172.21.15.143:6815","nonce":2341769896}]},"public_addr":"172.21.15.143:6811/2341769896","cluster_addr":"172.21.15.143:6813/2341769896","heartbeat_back_addr":"172.21.15.143:6817/2341769896","heartbeat_front_addr":"172.21.15.143:6815/2341769896","state":["exists","up"]},{"osd":2,"uuid":"4f8492d1-0e7a-49f8-b6a5-4f5e73842349","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6818","nonce":2664999316},{"type":"v1","addr":"172.21.15.143:6819","nonce":2664999316}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6820","nonce":2664999316},{"type":"v1","addr":"172.21.15.143:6821","nonce":2664999316}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6824","nonce":2664999316},{"type":"v1","addr":"172.21.15.143:6825","nonce":2664999316}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6822","nonce":2664999316},{"type":"v1","addr":"172.21.15.143:6823","nonce":2664999316}]},"public_addr":"172.21.15.143:6819/2664999316","cluster_addr":"172.21.15.143:6821/2664999316","heartbeat_back_addr":"172.21.15.143:6825/2664999316","heartbeat_front_addr":"172.21.15.143:6823/2664999316","state":["exists","up"]},{"osd":3,"uuid":"dd6abc8a-54a9-4e08-ba81-0b894f24d84a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6826","nonce":1468263600},{"type":"v1","addr":"172.21.15.143:6827","nonce":1468263600}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6828","nonce":1468263600},{"type":"v1","addr":"172.21.15.143:6829","nonce":1468263600}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6832","nonce":1468263600},{"type":"v1","addr":"172.21.15.143:6833","nonce":1468263600}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6830","nonce":1468263600},{"type":"v1","addr":"172.21.15.143:6831","nonce":1468263600}]},"public_addr":"172.21.15.143:6827/1468263600","cluster_addr":"172.21.15.143:6829/1468263600","heartbeat_back_addr":"172.21.15.143:6833/1468263600","heartbeat_front_addr":"172.21.15.143:6831/1468263600","state":["exists","up"]},{"osd":4,"uuid":"16122219-d985-46d2-bac1-f8ffcf48d8ad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6800","nonce":3960017153},{"type":"v1","addr":"172.21.15.169:6801","nonce":3960017153}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6802","nonce":3960017153},{"type":"v1","addr":"172.21.15.169:6803","nonce":3960017153}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6806","nonce":3960017153},{"type":"v1","addr":"172.21.15.169:6807","nonce":3960017153}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6804","nonce":3960017153},{"type":"v1","addr":"172.21.15.169:6805","nonce":3960017153}]},"public_addr":"172.21.15.169:6801/3960017153","cluster_addr":"172.21.15.169:6803/3960017153","heartbeat_back_addr":"172.21.15.169:6807/3960017153","heartbeat_front_addr":"172.21.15.169:6805/3960017153","state":["exists","up"]},{"osd":5,"uuid":"adbc3ea5-230f-419e-8f6f-99763378b296","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6808","nonce":4087672846},{"type":"v1","addr":"172.21.15.169:6809","nonce":4087672846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6810","nonce":4087672846},{"type":"v1","addr":"172.21.15.169:6811","nonce":4087672846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6814","nonce":4087672846},{"type":"v1","addr":"172.21.15.169:6815","nonce":4087672846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6812","nonce":4087672846},{"type":"v1","addr":"172.21.15.169:6813","nonce":4087672846}]},"public_addr":"172.21.15.169:6809/4087672846","cluster_addr":"172.21.15.169:6811/4087672846","heartbeat_back_addr":"172.21.15.169:6815/4087672846","heartbeat_front_addr":"172.21.15.169:6813/4087672846","state":["exists","up"]},{"osd":6,"uuid":"2abe4d77-c12c-477f-a4ec-eabb753ca4dd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6816","nonce":3801227982},{"type":"v1","addr":"172.21.15.169:6817","nonce":3801227982}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6818","nonce":3801227982},{"type":"v1","addr":"172.21.15.169:6819","nonce":3801227982}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6822","nonce":3801227982},{"type":"v1","addr":"172.21.15.169:6823","nonce":3801227982}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6820","nonce":3801227982},{"type":"v1","addr":"172.21.15.169:6821","nonce":3801227982}]},"public_addr":"172.21.15.169:6817/3801227982","cluster_addr":"172.21.15.169:6819/3801227982","heartbeat_back_addr":"172.21.15.169:6823/3801227982","heartbeat_front_addr":"172.21.15.169:6821/3801227982","state":["exists","up"]},{"osd":7,"uuid":"fc5ed8fc-8ad7-4b54-a8ea-bc6a9e846f63","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6824","nonce":3830851149},{"type":"v1","addr":"172.21.15.169:6825","nonce":3830851149}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6826","nonce":3830851149},{"type":"v1","addr":"172.21.15.169:6827","nonce":3830851149}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6830","nonce":3830851149},{"type":"v1","addr":"172.21.15.169:6831","nonce":3830851149}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6828","nonce":3830851149},{"type":"v1","addr":"172.21.15.169:6829","nonce":3830851149}]},"public_addr":"172.21.15.169:6825/3830851149","cluster_addr":"172.21.15.169:6827/3830851149","heartbeat_back_addr":"172.21.15.169:6831/3830851149","heartbeat_front_addr":"172.21.15.169:6829/3830851149","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:07:16.166767+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:07:37.634849+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:07:59.099590+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:08:21.737546+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:08:43.377647+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:09:06.116513+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:09:28.557330+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:09:51.159131+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.143:0/1563442416":"2023-08-29T02:10:09.845824+0000","172.21.15.143:6801/1828180492":"2023-08-29T02:10:09.845824+0000","172.21.15.143:0/2157302459":"2023-08-29T02:10:09.845824+0000","172.21.15.143:6800/1828180492":"2023-08-29T02:10:09.845824+0000","172.21.15.143:6801/907474889":"2023-08-29T02:06:02.911068+0000","172.21.15.143:6800/907474889":"2023-08-29T02:06:02.911068+0000","172.21.15.143:0/2323551826":"2023-08-29T02:06:02.911068+0000","172.21.15.143:6800/1019944263":"2023-08-29T02:05:41.797747+0000","172.21.15.143:0/3606471716":"2023-08-29T02:06:02.911068+0000","172.21.15.143:6801/1019944263":"2023-08-29T02:05:41.797747+0000","172.21.15.143:0/11141072":"2023-08-29T02:05:41.797747+0000","172.21.15.143:0/1148418770":"2023-08-29T02:05:41.797747+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-08-28T02:10:41.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:41 smithi169 conmon[100500]: cluster 2023-08-28T02:10:39.862996+0000 mgr.y (mgr.14424) 30 : cluster [DBG] pgmap v22: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:42.837 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:42 smithi143 conmon[100258]: audit 2023-08-28T02:10:41.893047+0000 mon.a (mon.0) 625 : audit [DBG] from='client.? 172.21.15.143:0/2387694016' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-28T02:10:42.837 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:42 smithi143 conmon[109923]: audit 2023-08-28T02:10:41.893047+0000 mon.a (mon.0) 625 : audit [DBG] from='client.? 172.21.15.143:0/2387694016' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-28T02:10:42.921 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-08-28T02:10:42.921 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd dump --format=json 2023-08-28T02:10:42.937 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:42 smithi169 conmon[100500]: audit 2023-08-28T02:10:41.893047+0000 mon.a (mon.0) 625 : audit [DBG] from='client.? 172.21.15.143:0/2387694016' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-28T02:10:43.237 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:10:42 smithi169 conmon[102807]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:10:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-28T02:10:43.314 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:10:43.450 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:10:43 smithi143 systemd[1]: Starting Ceph alertmanager.a for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:10:43.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:43 smithi143 conmon[100258]: cluster 2023-08-28T02:10:41 2023-08-28T02:10:43.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:43 smithi143 conmon[100258]: .863326+0000 mgr.y (mgr.14424) 31 : cluster [DBG] pgmap v23: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:43.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:43 smithi143 conmon[109923]: cluster 2023-08-28T02:10: 2023-08-28T02:10:43.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:43 smithi143 conmon[109923]: 41.863326+0000 mgr.y (mgr.14424) 31 : cluster [DBG] pgmap v23: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:43.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:43 smithi169 conmon[100500]: cluster 2023-08-28T02:10:41.863326+0000 mgr.y (mgr.14424) 31 : cluster [DBG] pgmap v23: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:44.450 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:10:44 smithi143 conmon[141410]: level=info ts=2023-08-28T02:10:44.175Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2023-08-28T02:10:44.450 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:10:44 smithi143 conmon[141410]: level=info ts=2023-08-28T02:10:44.175Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2023-08-28T02:10:44.450 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:10:44 smithi143 conmon[141410]: level=info ts=2023-08-28T02:10:44.176Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.143 port=9094 2023-08-28T02:10:44.450 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:10:44 smithi143 conmon[141410]: level=info ts=2023-08-28T02:10:44.177Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-08-28T02:10:44.450 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:10:44 smithi143 conmon[141410]: level=info ts=2023-08-28T02:10:44.203Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-08-28T02:10:44.450 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:10:44 smithi143 conmon[141410]: level=info ts=2023-08-28T02:10:44.203Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-08-28T02:10:44.450 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:10:44 smithi143 conmon[141410]: level=info ts=2023-08-28T02:10:44.205Z caller=main.go:518 msg=Listening address=:9093 2023-08-28T02:10:44.451 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:10:44 smithi143 conmon[141410]: level=info ts=2023-08-28T02:10:44.205Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2023-08-28T02:10:44.451 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:10:44 smithi143 bash[141335]: f4162ae1de812d5b03a84d1911f2645d3a84369659a0ac5d3be1b60c03c840bf 2023-08-28T02:10:44.451 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:10:44 smithi143 systemd[1]: Started Ceph alertmanager.a for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:10:44.756 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:10:44 smithi143 conmon[100708]: [28/Aug/2023:02:10:44] ENGINE Bus STOPPING 2023-08-28T02:10:44.756 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:10:44 smithi143 conmon[100708]: [28/Aug/2023:02:10:44] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-08-28T02:10:44.756 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:10:44 smithi143 conmon[100708]: [28/Aug/2023:02:10:44] ENGINE Bus STOPPED 2023-08-28T02:10:44.756 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:10:44 smithi143 conmon[100708]: [28/Aug/2023:02:10:44] ENGINE Bus STARTING 2023-08-28T02:10:44.756 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:10:44 smithi143 conmon[100708]: [28/Aug/2023:02:10:44] ENGINE Serving on http://:::9283 2023-08-28T02:10:44.756 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:10:44 smithi143 conmon[100708]: [28/Aug/2023:02:10:44] ENGINE Bus STARTED 2023-08-28T02:10:45.069 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:10:45.069 INFO:teuthology.orchestra.run.smithi143.stdout:{"epoch":55,"fsid":"2fcb0bfa-4547-11ee-817c-1b7a5f1ba315","created":"2023-08-28T02:05:20.056816+0000","modified":"2023-08-28T02:10:34.402737+0000","last_up_change":"2023-08-28T02:09:53.076490+0000","last_in_change":"2023-08-28T02:09:35.717712+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-08-28T02:08:01.318811+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-08-28T02:10:30.618550+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"718c8f27-245b-4e9d-9e2e-d1fc65ca7472","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6802","nonce":1846477664},{"type":"v1","addr":"172.21.15.143:6803","nonce":1846477664}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6804","nonce":1846477664},{"type":"v1","addr":"172.21.15.143:6805","nonce":1846477664}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6808","nonce":1846477664},{"type":"v1","addr":"172.21.15.143:6809","nonce":1846477664}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6806","nonce":1846477664},{"type":"v1","addr":"172.21.15.143:6807","nonce":1846477664}]},"public_addr":"172.21.15.143:6803/1846477664","cluster_addr":"172.21.15.143:6805/1846477664","heartbeat_back_addr":"172.21.15.143:6809/1846477664","heartbeat_front_addr":"172.21.15.143:6807/1846477664","state":["exists","up"]},{"osd":1,"uuid":"cf24771f-3b0c-4a00-8a73-13e1fee242dc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6810","nonce":2341769896},{"type":"v1","addr":"172.21.15.143:6811","nonce":2341769896}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6812","nonce":2341769896},{"type":"v1","addr":"172.21.15.143:6813","nonce":2341769896}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6816","nonce":2341769896},{"type":"v1","addr":"172.21.15.143:6817","nonce":2341769896}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6814","nonce":2341769896},{"type":"v1","addr":"172.21.15.143:6815","nonce":2341769896}]},"public_addr":"172.21.15.143:6811/2341769896","cluster_addr":"172.21.15.143:6813/2341769896","heartbeat_back_addr":"172.21.15.143:6817/2341769896","heartbeat_front_addr":"172.21.15.143:6815/2341769896","state":["exists","up"]},{"osd":2,"uuid":"4f8492d1-0e7a-49f8-b6a5-4f5e73842349","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6818","nonce":2664999316},{"type":"v1","addr":"172.21.15.143:6819","nonce":2664999316}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6820","nonce":2664999316},{"type":"v1","addr":"172.21.15.143:6821","nonce":2664999316}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6824","nonce":2664999316},{"type":"v1","addr":"172.21.15.143:6825","nonce":2664999316}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6822","nonce":2664999316},{"type":"v1","addr":"172.21.15.143:6823","nonce":2664999316}]},"public_addr":"172.21.15.143:6819/2664999316","cluster_addr":"172.21.15.143:6821/2664999316","heartbeat_back_addr":"172.21.15.143:6825/2664999316","heartbeat_front_addr":"172.21.15.143:6823/2664999316","state":["exists","up"]},{"osd":3,"uuid":"dd6abc8a-54a9-4e08-ba81-0b894f24d84a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6826","nonce":1468263600},{"type":"v1","addr":"172.21.15.143:6827","nonce":1468263600}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6828","nonce":1468263600},{"type":"v1","addr":"172.21.15.143:6829","nonce":1468263600}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6832","nonce":1468263600},{"type":"v1","addr":"172.21.15.143:6833","nonce":1468263600}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6830","nonce":1468263600},{"type":"v1","addr":"172.21.15.143:6831","nonce":1468263600}]},"public_addr":"172.21.15.143:6827/1468263600","cluster_addr":"172.21.15.143:6829/1468263600","heartbeat_back_addr":"172.21.15.143:6833/1468263600","heartbeat_front_addr":"172.21.15.143:6831/1468263600","state":["exists","up"]},{"osd":4,"uuid":"16122219-d985-46d2-bac1-f8ffcf48d8ad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6800","nonce":3960017153},{"type":"v1","addr":"172.21.15.169:6801","nonce":3960017153}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6802","nonce":3960017153},{"type":"v1","addr":"172.21.15.169:6803","nonce":3960017153}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6806","nonce":3960017153},{"type":"v1","addr":"172.21.15.169:6807","nonce":3960017153}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6804","nonce":3960017153},{"type":"v1","addr":"172.21.15.169:6805","nonce":3960017153}]},"public_addr":"172.21.15.169:6801/3960017153","cluster_addr":"172.21.15.169:6803/3960017153","heartbeat_back_addr":"172.21.15.169:6807/3960017153","heartbeat_front_addr":"172.21.15.169:6805/3960017153","state":["exists","up"]},{"osd":5,"uuid":"adbc3ea5-230f-419e-8f6f-99763378b296","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6808","nonce":4087672846},{"type":"v1","addr":"172.21.15.169:6809","nonce":4087672846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6810","nonce":4087672846},{"type":"v1","addr":"172.21.15.169:6811","nonce":4087672846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6814","nonce":4087672846},{"type":"v1","addr":"172.21.15.169:6815","nonce":4087672846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6812","nonce":4087672846},{"type":"v1","addr":"172.21.15.169:6813","nonce":4087672846}]},"public_addr":"172.21.15.169:6809/4087672846","cluster_addr":"172.21.15.169:6811/4087672846","heartbeat_back_addr":"172.21.15.169:6815/4087672846","heartbeat_front_addr":"172.21.15.169:6813/4087672846","state":["exists","up"]},{"osd":6,"uuid":"2abe4d77-c12c-477f-a4ec-eabb753ca4dd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6816","nonce":3801227982},{"type":"v1","addr":"172.21.15.169:6817","nonce":3801227982}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6818","nonce":3801227982},{"type":"v1","addr":"172.21.15.169:6819","nonce":3801227982}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6822","nonce":3801227982},{"type":"v1","addr":"172.21.15.169:6823","nonce":3801227982}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6820","nonce":3801227982},{"type":"v1","addr":"172.21.15.169:6821","nonce":3801227982}]},"public_addr":"172.21.15.169:6817/3801227982","cluster_addr":"172.21.15.169:6819/3801227982","heartbeat_back_addr":"172.21.15.169:6823/3801227982","heartbeat_front_addr":"172.21.15.169:6821/3801227982","state":["exists","up"]},{"osd":7,"uuid":"fc5ed8fc-8ad7-4b54-a8ea-bc6a9e846f63","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6824","nonce":3830851149},{"type":"v1","addr":"172.21.15.169:6825","nonce":3830851149}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6826","nonce":3830851149},{"type":"v1","addr":"172.21.15.169:6827","nonce":3830851149}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6830","nonce":3830851149},{"type":"v1","addr":"172.21.15.169:6831","nonce":3830851149}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6828","nonce":3830851149},{"type":"v1","addr":"172.21.15.169:6829","nonce":3830851149}]},"public_addr":"172.21.15.169:6825/3830851149","cluster_addr":"172.21.15.169:6827/3830851149","heartbeat_back_addr":"172.21.15.169:6831/3830851149","heartbeat_front_addr":"172.21.15.169:6829/3830851149","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:07:16.166767+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:07:37.634849+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:07:59.099590+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:08:21.737546+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:08:43.377647+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:09:06.116513+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:09:28.557330+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:09:51.159131+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.143:0/1563442416":"2023-08-29T02:10:09.845824+0000","172.21.15.143:6801/1828180492":"2023-08-29T02:10:09.845824+0000","172.21.15.143:0/2157302459":"2023-08-29T02:10:09.845824+0000","172.21.15.143:6800/1828180492":"2023-08-29T02:10:09.845824+0000","172.21.15.143:6801/907474889":"2023-08-29T02:06:02.911068+0000","172.21.15.143:6800/907474889":"2023-08-29T02:06:02.911068+0000","172.21.15.143:0/2323551826":"2023-08-29T02:06:02.911068+0000","172.21.15.143:6800/1019944263":"2023-08-29T02:05:41.797747+0000","172.21.15.143:0/3606471716":"2023-08-29T02:06:02.911068+0000","172.21.15.143:6801/1019944263":"2023-08-29T02:05:41.797747+0000","172.21.15.143:0/11141072":"2023-08-29T02:05:41.797747+0000","172.21.15.143:0/1148418770":"2023-08-29T02:05:41.797747+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-08-28T02:10:45.363 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:45 smithi143 conmon[109923]: cluster 2023-08-28T02:10:43.863597+0000 mgr.y (mgr 2023-08-28T02:10:45.363 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:45 smithi143 conmon[109923]: .14424) 32 : cluster [DBG] pgmap v24: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:45.363 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:45 smithi143 conmon[109923]: audit 2023-08-28T02:10:44.362368+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:45.364 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:45 smithi143 conmon[109923]: audit 2023-08-28T02:10:44.452243+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:45.364 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:45 smithi143 conmon[109923]: audit 2023-08-28T02:10:44.455645+0000 mon.a (mon.0) 628 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:45.364 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:45 smithi143 conmon[109923]: audit 2023-08-28T02:10:45.364 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:45 smithi143 conmon[109923]: 2023-08-28T02:10:44.458141+0000 mon.a (mon.0) 629 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-08-28T02:10:45.364 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:45 smithi143 conmon[109923]: audit 2023-08-28T02:10:45.364 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:45 smithi143 conmon[100258]: cluster 2023-08-28T02:10:43. 2023-08-28T02:10:45.364 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:45 smithi143 conmon[100258]: 863597+0000 mgr.y (mgr.14424) 32 : cluster [DBG] pgmap v24: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:45.364 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:45 smithi143 conmon[100258]: audit 2023-08-28T02:10:44.362368+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:45.364 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:45 smithi143 conmon[100258]: audit 2023-08-28T02:10:44.452243+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:45.364 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:45 smithi143 conmon[100258]: audit 2023-08-28T02:10:44.455645+0000 mon.a (mon.0) 628 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:45.365 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:45 smithi143 conmon[100258]: audit 2023-08-28T02:10:44.458141+0000 mon.a (mon.0) 629 : audit 2023-08-28T02:10:45.365 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:45 smithi143 conmon[100258]: [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-08-28T02:10:45.365 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:45 smithi143 conmon[100258]: audit 2023-08-28T02:10:44.458471+0000 mgr.y (mgr.14424 2023-08-28T02:10:45.365 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:45 smithi143 conmon[100258]: ) 33 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-08-28T02:10:45.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:45 smithi143 conmon[100258]: audit 2023-08-28T02:10:44. 2023-08-28T02:10:45.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:45 smithi143 conmon[100258]: 462398+0000 mon.a (mon.0) 630 : audit [INF] 2023-08-28T02:10:45.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:45 smithi143 conmon[100258]: from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:45.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:45 smithi143 conmon[100258]: cephadm 2023-08-28T02:10:44 2023-08-28T02:10:45.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:45 smithi143 conmon[100258]: .466790+0000 mgr.y (mgr.14424) 34 : cephadm [INF] 2023-08-28T02:10:45.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:45 smithi143 conmon[100258]: Deploying daemon grafana.a on smithi169 2023-08-28T02:10:45.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:45 smithi143 conmon[100258]: audit 2023-08-28T02:10:45.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:45 smithi143 conmon[100258]: 2023-08-28T02:10:45.006890+0000 mon.a ( 2023-08-28T02:10:45.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:45 smithi143 conmon[100258]: mon.0) 631 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:45.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:45 smithi143 conmon[100258]: audit 2023-08-28T02:10:45.069573+0000 2023-08-28T02:10:45.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:45 smithi143 conmon[100258]: mon.a (mon.0) 632 : audit [DBG] from='client.? 172.21.15.143:0/1306751737' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-28T02:10:45.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:45 smithi143 conmon[109923]: 2023-08-28T02:10:44.458471+0000 mgr.y ( 2023-08-28T02:10:45.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:45 smithi143 conmon[109923]: mgr.14424) 33 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-08-28T02:10:45.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:45 smithi143 conmon[109923]: audit 2023-08-28T02:10:45.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:45 smithi143 conmon[109923]: 2023-08-28T02:10:44.462398+0000 2023-08-28T02:10:45.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:45 smithi143 conmon[109923]: mon.a (mon.0) 630 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:45.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:45 smithi143 conmon[109923]: cephadm 2023-08-28T02: 2023-08-28T02:10:45.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:45 smithi143 conmon[109923]: 10:44.466790+0000 mgr.y (mgr.14424) 34 : cephadm [INF] Deploying daemon grafana.a on smithi169 2023-08-28T02:10:45.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:45 smithi143 conmon[109923]: audit 2023 2023-08-28T02:10:45.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:45 smithi143 conmon[109923]: -08-28T02:10:45.006890+0000 mon.a 2023-08-28T02:10:45.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:45 smithi143 conmon[109923]: (mon.0) 631 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:45.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:45 smithi143 conmon[109923]: audit 2023-08-28T02:10:45.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:45 smithi143 conmon[109923]: 2023-08-28T02: 2023-08-28T02:10:45.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:45 smithi143 conmon[109923]: 10:45.069573+0000 mon.a (mon.0 2023-08-28T02:10:45.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:45 smithi143 conmon[109923]: ) 632 : audit [DBG] from='client.? 172.21.15.143:0/1306751737' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-28T02:10:45.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:45 smithi169 conmon[100500]: cluster 2023-08-28T02:10:43.863597+0000 mgr.y (mgr.14424) 2023-08-28T02:10:45.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:45 smithi169 conmon[100500]: 32 : cluster [DBG] pgmap v24: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:45.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:45 smithi169 conmon[100500]: audit 2023-08-28T02:10:44.362368+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:45.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:45 smithi169 conmon[100500]: audit 2023-08-28T02:10:44.452243+0000 mon.a (mon 2023-08-28T02:10:45.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:45 smithi169 conmon[100500]: .0) 627 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:45.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:45 smithi169 conmon[100500]: audit 2023-08-28T02:10:45.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:45 smithi169 conmon[100500]: 2023-08-28T02:10: 2023-08-28T02:10:45.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:45 smithi169 conmon[100500]: 44.455645+0000 mon.a (mon.0) 628 : audit 2023-08-28T02:10:45.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:45 smithi169 conmon[100500]: [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:45.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:45 smithi169 conmon[100500]: audit 2023-08- 2023-08-28T02:10:45.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:45 smithi169 conmon[100500]: 28T02:10:44.458141+0000 mon.a ( 2023-08-28T02:10:45.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:45 smithi169 conmon[100500]: mon.0) 629 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-08-28T02:10:45.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:45 smithi169 conmon[100500]: audit 2023-08-28T02:10:45.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:45 smithi169 conmon[100500]: 2023-08-28T02:10:44.458471 2023-08-28T02:10:45.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:45 smithi169 conmon[100500]: +0000 mgr.y (mgr.14424) 33 : audit [DBG] 2023-08-28T02:10:45.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:45 smithi169 conmon[100500]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-08-28T02:10:45.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:45 smithi169 conmon[100500]: audit 2023-08-28T02:10:45.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:45 smithi169 conmon[100500]: 2023-08-28T02:10:44.462398+0000 mon.a 2023-08-28T02:10:45.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:45 smithi169 conmon[100500]: (mon.0) 630 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:45.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:45 smithi169 conmon[100500]: cephadm 2023 2023-08-28T02:10:45.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:45 smithi169 conmon[100500]: -08-28T02:10:44.466790+0000 2023-08-28T02:10:45.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:45 smithi169 conmon[100500]: mgr.y (mgr.14424) 34 : cephadm [INF] Deploying daemon grafana.a on smithi169 2023-08-28T02:10:45.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:45 smithi169 conmon[100500]: 2023-08-28T02:10:45.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:45 smithi169 conmon[100500]: audit 2023-08-28T02:10:45.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:45 smithi169 conmon[100500]: 2023-08-28T02:10:45.006890 2023-08-28T02:10:45.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:45 smithi169 conmon[100500]: +0000 mon.a (mon.0) 631 : audit [INF] 2023-08-28T02:10:45.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:45 smithi169 conmon[100500]: from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:10:45.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:45 smithi169 conmon[100500]: audit 2023-08-28T02:10:45.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:45 smithi169 conmon[100500]: 2023-08-28T02:10:45.069573+0000 2023-08-28T02:10:45.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:45 smithi169 conmon[100500]: mon.a (mon.0) 632 : audit [DBG] from='client.? 172.21.15.143:0/1306751737' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-28T02:10:45.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:45 smithi169 conmon[100500]: 2023-08-28T02:10:45.750 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph tell osd.0 flush_pg_stats 2023-08-28T02:10:45.750 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph tell osd.1 flush_pg_stats 2023-08-28T02:10:45.751 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph tell osd.2 flush_pg_stats 2023-08-28T02:10:45.751 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph tell osd.3 flush_pg_stats 2023-08-28T02:10:45.752 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph tell osd.4 flush_pg_stats 2023-08-28T02:10:45.752 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph tell osd.5 flush_pg_stats 2023-08-28T02:10:45.752 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph tell osd.6 flush_pg_stats 2023-08-28T02:10:45.753 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph tell osd.7 flush_pg_stats 2023-08-28T02:10:46.450 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:10:46 smithi143 conmon[141410]: level=info ts=2023-08-28T02:10:46.178Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000478226s 2023-08-28T02:10:46.694 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:10:46.803 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:10:46.998 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:10:47.100 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:10:47.254 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:10:47.659 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:10:47.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:47 smithi143 conmon[100258]: cluster 2023-08-28T02:10:45.867313+0000 mgr.y (mgr.14424) 35 : cluster [DBG] pgmap v25: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:47.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:47 smithi143 conmon[109923]: cluster 2023-08-28T02:10:45.867313+0000 mgr.y (mgr.14424 2023-08-28T02:10:47.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:47 smithi143 conmon[109923]: ) 35 : cluster [DBG] pgmap v25: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:47.736 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:47 smithi169 conmon[100500]: cluster 2023-08-28T02:10:45.867313+0000 mgr.y (mgr.14424) 2023-08-28T02:10:47.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:47 smithi169 conmon[100500]: 35 : cluster [DBG] pgmap v25: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:48.201 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:10:48.565 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:10:49.413 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:49 smithi143 conmon[100258]: cluster 2023-08-28T02:10:47.867645+0000 mgr.y (mgr.14424) 36 : cluster [DBG] pgmap v26: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:49.414 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:49 smithi143 conmon[109923]: cluster 2023-08-28T02:10:47.867645+0000 mgr.y (mgr.14424) 36 : cluster [DBG] pgmap v26: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:49.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:49 smithi169 conmon[100500]: cluster 2023-08-28T02:10:47.867645+0000 mgr.y (mgr.14424) 36 : cluster [DBG] 2023-08-28T02:10:49.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:49 smithi169 conmon[100500]: pgmap v26: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:50.281 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:10:50 smithi143 conmon[100708]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:10:50] "GET /metrics HTTP/1.1" 200 155664 "" "Prometheus/2.33.4" 2023-08-28T02:10:51.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:51 smithi143 conmon[100258]: cluster 2023-08-28T02:10 2023-08-28T02:10:51.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:51 smithi143 conmon[100258]: :49.867933+0000 mgr.y (mgr.14424) 37 : cluster [DBG] pgmap v27: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:51.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:51 smithi143 conmon[109923]: cluster 2023-08-28T02:10:49 2023-08-28T02:10:51.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:51 smithi143 conmon[109923]: .867933+0000 mgr.y (mgr.14424) 37 : cluster [DBG] pgmap v27: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:51.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:51 smithi169 conmon[100500]: cluster 2023-08-28T02:10:49.867933+0000 mgr.y (mgr.14424) 37 : cluster [DBG] pgmap v27: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:52.963 INFO:teuthology.orchestra.run.smithi143.stdout:128849018907 2023-08-28T02:10:52.963 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd last-stat-seq osd.4 2023-08-28T02:10:53.237 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:10:52 smithi169 conmon[102807]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:10:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-28T02:10:53.580 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:53 smithi143 conmon[100258]: cluster 2023-08-28T02:10:51.868282+0000 mgr.y (mgr.14424) 38 : cluster [DBG] pgmap v28: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:53.580 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:53 smithi143 conmon[109923]: cluster 2023-08-28T02:10:51.868282+0000 mgr.y (mgr.14424) 38 : cluster [DBG] pgmap v28: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:53.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:53 smithi169 conmon[100500]: cluster 2023-08-28T02:10:51.868282+0000 mgr.y (mgr.14424) 2023-08-28T02:10:53.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:53 smithi169 conmon[100500]: 38 : cluster [DBG] pgmap v28: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:54.280 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:10:54 smithi143 conmon[141410]: level=info ts=2023-08-28T02:10:54.181Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.003648917s 2023-08-28T02:10:54.594 INFO:teuthology.orchestra.run.smithi143.stdout:154618822679 2023-08-28T02:10:54.595 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd last-stat-seq osd.5 2023-08-28T02:10:54.788 INFO:teuthology.orchestra.run.smithi143.stdout:107374182433 2023-08-28T02:10:54.788 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd last-stat-seq osd.3 2023-08-28T02:10:54.813 INFO:teuthology.orchestra.run.smithi143.stdout:55834574889 2023-08-28T02:10:54.813 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd last-stat-seq osd.1 2023-08-28T02:10:55.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:55 smithi143 conmon[100258]: cluster 2023-08-28T02:10:53.868782+0000 mgr.y (mgr.14424) 39 : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:55.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:55 smithi143 conmon[109923]: cluster 2023-08-28T02:10:53.868782+0000 mgr.y (mgr.14424) 39 : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:55.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:55 smithi169 conmon[100500]: cluster 2023-08-28T02:10:53.868782+0000 mgr.y (mgr.14424) 39 : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:56.402 INFO:teuthology.orchestra.run.smithi143.stdout:180388626449 2023-08-28T02:10:56.402 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd last-stat-seq osd.6 2023-08-28T02:10:56.620 INFO:teuthology.orchestra.run.smithi143.stdout:34359738414 2023-08-28T02:10:56.620 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd last-stat-seq osd.0 2023-08-28T02:10:56.759 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:10:56.808 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:10:57.610 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:10:57.636 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:57 smithi143 conmon[100258]: cluster 2023-08-28T02:10:55.869270+0000 mgr.y (mgr.14424) 40 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:57.636 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:57 smithi143 conmon[109923]: cluster 2023-08-28T02:10:55 2023-08-28T02:10:57.636 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:57 smithi143 conmon[109923]: .869270+0000 mgr.y (mgr.14424) 40 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:57.637 INFO:teuthology.orchestra.run.smithi143.stdout:206158430223 2023-08-28T02:10:57.638 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd last-stat-seq osd.7 2023-08-28T02:10:57.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:57 smithi169 conmon[100500]: cluster 2023-08-28T02:10:55.869270+0000 mgr.y (mgr.14424) 40 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:59.478 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:10:59.538 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:10:59 smithi143 conmon[100258]: cluster 2023-08-28T02:10:57.869592+0000 mgr.y (mgr.14424) 41 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:59.538 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:10:59 smithi143 conmon[109923]: cluster 2023-08-28T02:10:57.869592+0000 mgr.y (mgr.14424) 41 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:59.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:10:59 smithi169 conmon[100500]: cluster 2023-08-28T02:10:57.869592+0000 mgr.y (mgr.14424) 41 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:10:59.843 INFO:teuthology.orchestra.run.smithi143.stdout:77309411365 2023-08-28T02:10:59.844 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd last-stat-seq osd.2 2023-08-28T02:11:00.056 INFO:teuthology.orchestra.run.smithi143.stdout:154618822680 2023-08-28T02:11:00.177 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:11:00.508 INFO:teuthology.orchestra.run.smithi143.stdout:128849018908 2023-08-28T02:11:00.561 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:00 smithi143 conmon[100258]: audit 2023-08-28T02:11:00.057707+0000 mon.a (mon.0 2023-08-28T02:11:00.561 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:00 smithi143 conmon[100258]: ) 633 : audit [DBG] from='client.? 172.21.15.143:0/1097086593' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-08-28T02:11:00.561 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:00 smithi143 conmon[100258]: audit 2023-08-28T02:11:00.368562+0000 mon.a (mon.0) 634 : audit [DBG] from='client.? 172.21.15.143:0/2309017404' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-08-28T02:11:00.562 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:11:00 smithi143 conmon[100708]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:11:00] "GET /metrics HTTP/1.1" 200 155661 "" "Prometheus/2.33.4" 2023-08-28T02:11:00.562 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:00 smithi143 conmon[109923]: audit 2023-08-28T02:11: 2023-08-28T02:11:00.562 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:00 smithi143 conmon[109923]: 00.057707+0000 mon.a (mon.0) 633 : audit [DBG] from='client.? 172.21.15.143:0/1097086593' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-08-28T02:11:00.562 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:00 smithi143 conmon[109923]: audit 2023-08-28T02: 2023-08-28T02:11:00.562 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:00 smithi143 conmon[109923]: 11:00.368562+0000 mon.a (mon.0) 634 : audit [DBG] 2023-08-28T02:11:00.562 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:00 smithi143 conmon[109923]: from='client.? 172.21.15.143:0/2309017404' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-08-28T02:11:00.674 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:00 smithi169 conmon[100500]: audit 2023-08-28T02:11:00.057707+0000 mon.a (mon.0) 633 : audit [DBG] from='client.? 172.21.15.143:0/1097086593' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-08-28T02:11:00.674 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:00 smithi169 conmon[100500]: audit 2023-08-28T02:11:00.368562+0000 mon.a (mon.0) 634 : audit [DBG] from='client.? 172.21.15.143:0/2309017404' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-08-28T02:11:01.546 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:01 smithi143 conmon[100258]: cluster 2023-08-28T02:10:59.869890+0000 mgr.y (mgr.14424) 42 : 2023-08-28T02:11:01.546 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:01 smithi143 conmon[100258]: cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:01.546 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:01 smithi143 conmon[109923]: cluster 2023-08-28T02:10:59.869890+0000 2023-08-28T02:11:01.547 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:01 smithi143 conmon[109923]: mgr.y (mgr.14424) 42 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:01.665 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:01 smithi169 conmon[100500]: cluster 2023-08-28T02:10:59.869890+0000 mgr.y (mgr.14424) 42 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:01.987 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:01 smithi169 systemd[1]: Starting Ceph grafana.a for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:11:01.990 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:11:02.016 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822679 got 154618822680 for osd.5 2023-08-28T02:11:02.016 DEBUG:teuthology.parallel:result is None 2023-08-28T02:11:02.624 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:11:02.855 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:02 smithi169 conmon[131988]: t=2023-08-28T02:11:02+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2023-08-28T02:11:02.855 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:02 smithi169 conmon[131988]: t=2023-08-28T02:11:02+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2023-08-28T02:11:02.855 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:02 smithi169 conmon[131988]: t=2023-08-28T02:11:02+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2023-08-28T02:11:02.855 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:02 smithi169 conmon[131988]: t=2023-08-28T02:11:02+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2023-08-28T02:11:02.855 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:02 smithi169 conmon[131988]: t=2023-08-28T02:11:02+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2023-08-28T02:11:02.856 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:02 smithi169 conmon[131988]: t=2023-08-28T02:11:02+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2023-08-28T02:11:02.856 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:02 smithi169 conmon[131988]: t=2023-08-28T02:11:02+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-08-28T02:11:02.856 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:02 smithi169 conmon[131988]: t=2023-08-28T02:11:02+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-08-28T02:11:02.856 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:02 smithi169 conmon[131988]: t=2023-08-28T02:11:02+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2023-08-28T02:11:02.856 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:02 smithi169 conmon[131988]: t=2023-08-28T02:11:02+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2023-08-28T02:11:02.856 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:02 smithi169 conmon[131988]: t=2023-08-28T02:11:02+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2023-08-28T02:11:02.856 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:02 smithi169 conmon[131988]: t=2023-08-28T02:11:02+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2023-08-28T02:11:02.856 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:02 smithi169 conmon[131988]: t=2023-08-28T02:11:02+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2023-08-28T02:11:02.856 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:02 smithi169 conmon[131988]: t=2023-08-28T02:11:02+0000 lvl=info msg="App mode production" logger=settings 2023-08-28T02:11:02.856 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:02 smithi169 conmon[131988]: t=2023-08-28T02:11:02+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2023-08-28T02:11:02.856 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:02 smithi169 conmon[131988]: t=2023-08-28T02:11:02+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-08-28T02:11:02.856 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:02 smithi169 conmon[131988]: t=2023-08-28T02:11:02+0000 lvl=info msg="Starting DB migrations" logger=migrator 2023-08-28T02:11:02.856 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:02 smithi169 conmon[131988]: t=2023-08-28T02:11:02+0000 lvl=info msg="Executing migration" logger=migrator id="create migration_log table" 2023-08-28T02:11:02.857 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:02 smithi169 bash[131947]: dcf28c93da589384fd2d0cc878b0a1b35228fdba0b6d0cb53f4b090862f04312 2023-08-28T02:11:02.857 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:02 smithi169 systemd[1]: Started Ceph grafana.a for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:11:02.857 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:02 smithi169 conmon[131988]: t=2023-08-28T02:11:02+0000 lvl=info msg="Executing migration" logger=migrator id="create user table" 2023-08-28T02:11:02.857 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:02 smithi169 conmon[131988]: t=2023-08-28T02:11:02+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.login" 2023-08-28T02:11:02.857 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:02 smithi169 conmon[131988]: t=2023-08-28T02:11:02+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.email" 2023-08-28T02:11:02.857 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:02 smithi169 conmon[131988]: t=2023-08-28T02:11:02+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_login - v1" 2023-08-28T02:11:02.857 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:02 smithi169 conmon[131988]: t=2023-08-28T02:11:02+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_email - v1" 2023-08-28T02:11:02.857 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:02 smithi169 conmon[131988]: t=2023-08-28T02:11:02+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table user to user_v1 - v1" 2023-08-28T02:11:02.857 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:02 smithi169 conmon[131988]: t=2023-08-28T02:11:02+0000 lvl=info msg="Executing migration" logger=migrator id="create user table v2" 2023-08-28T02:11:03.012 INFO:teuthology.orchestra.run.smithi143.stdout:55834574891 2023-08-28T02:11:03.038 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018907 got 128849018908 for osd.4 2023-08-28T02:11:03.038 DEBUG:teuthology.parallel:result is None 2023-08-28T02:11:03.116 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:11:02 smithi169 conmon[102807]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:11:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-28T02:11:03.116 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:02 smithi169 conmon[131988]: t=2023-08-28T02:11:02+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_login - v2" 2023-08-28T02:11:03.116 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:02 smithi169 conmon[131988]: t=2023-08-28T02:11:02+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_email - v2" 2023-08-28T02:11:03.116 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:02 smithi169 conmon[131988]: t=2023-08-28T02:11:02+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2023-08-28T02:11:03.116 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:02 smithi169 conmon[131988]: t=2023-08-28T02:11:02+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table user_v1" 2023-08-28T02:11:03.116 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:03 smithi169 conmon[131988]: t=2023-08-28T02:11:03+0000 lvl=info msg="Executing migration" logger=migrator id="Add column help_flags1 to user table" 2023-08-28T02:11:03.116 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:03 smithi169 conmon[131988]: t=2023-08-28T02:11:03+0000 lvl=info msg="Executing migration" logger=migrator id="Update user table charset" 2023-08-28T02:11:03.116 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:03 smithi169 conmon[131988]: t=2023-08-28T02:11:03+0000 lvl=info msg="Executing migration" logger=migrator id="Add last_seen_at column to user" 2023-08-28T02:11:03.116 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:03 smithi169 conmon[131988]: t=2023-08-28T02:11:03+0000 lvl=info msg="Executing migration" logger=migrator id="Add missing user data" 2023-08-28T02:11:03.117 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:03 smithi169 conmon[131988]: t=2023-08-28T02:11:03+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_disabled column to user" 2023-08-28T02:11:03.367 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:03 smithi169 conmon[131988]: t=2023-08-28T02:11:03+0000 lvl=info msg="Executing migration" logger=migrator id="Add index user.login/user.email" 2023-08-28T02:11:03.368 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:03 smithi169 conmon[131988]: t=2023-08-28T02:11:03+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_service_account column to user" 2023-08-28T02:11:03.368 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:03 smithi169 conmon[131988]: t=2023-08-28T02:11:03+0000 lvl=info msg="Executing migration" logger=migrator id="create temp user table v1-7" 2023-08-28T02:11:03.375 INFO:teuthology.orchestra.run.smithi143.stdout:107374182435 2023-08-28T02:11:03.605 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:03 smithi143 conmon[100258]: cluster 2023-08-28T02:11:01.870286+0000 mgr.y (mgr.14424) 43 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:03.605 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:03 smithi143 conmon[100258]: audit 2023-08-28T02:11:02.641455+0000 mon.a (mon.0) 635 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:03.606 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:03 smithi143 conmon[100258]: audit 2023-08-28T02:11:02.643168+0000 mon.a (mon.0) 636 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:11:03.606 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:03 smithi143 conmon[100258]: audit 2023-08-28T02:11:03.013257+0000 mon.a (mon.0) 637 : audit [DBG] from='client.? 172.21.15.143:0/4283278731' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-08-28T02:11:03.606 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:03 smithi143 conmon[100258]: audit 2023-08-28T02:11:03.377027+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.143:0/3340541723' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-08-28T02:11:03.606 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:03 smithi143 conmon[109923]: cluster 2023-08-28T02:11:01.870286+0000 mgr.y (mgr.14424) 43 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:03.606 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:03 smithi143 conmon[109923]: audit 2023-08-28T02:11:02.641455+0000 mon.a (mon.0) 635 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:03.606 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:03 smithi143 conmon[109923]: audit 2023-08-28T02:11:02.643168+0000 mon.a (mon.0) 636 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:11:03.607 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:03 smithi143 conmon[109923]: audit 2023-08-28T02:11:03.013257+0000 mon.a (mon.0) 637 : audit [DBG] from='client.? 172.21.15.143:0/4283278731' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-08-28T02:11:03.607 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:03 smithi143 conmon[109923]: audit 2023-08-28T 2023-08-28T02:11:03.607 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:03 smithi143 conmon[109923]: 02:11:03.377027+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.143:0/3340541723' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-08-28T02:11:03.653 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:03 smithi169 conmon[131988]: t=2023-08-28T02:11:03+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v1-7" 2023-08-28T02:11:03.653 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:03 smithi169 conmon[131988]: t=2023-08-28T02:11:03+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v1-7" 2023-08-28T02:11:03.653 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:03 smithi169 conmon[131988]: t=2023-08-28T02:11:03+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v1-7" 2023-08-28T02:11:03.653 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:03 smithi169 conmon[131988]: t=2023-08-28T02:11:03+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v1-7" 2023-08-28T02:11:03.654 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:03 smithi169 conmon[131988]: t=2023-08-28T02:11:03+0000 lvl=info msg="Executing migration" logger=migrator id="Update temp_user table charset" 2023-08-28T02:11:03.654 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:03 smithi169 conmon[100500]: cluster 2023-08-28T02:11:01.870286+0000 mgr.y (mgr 2023-08-28T02:11:03.654 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:03 smithi169 conmon[100500]: .14424) 43 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:03.654 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:03 smithi169 conmon[100500]: audit 2023-08-28T02:11:02.641455+0000 mon.a (mon.0) 635 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:03.654 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:03 smithi169 conmon[100500]: audit 2023-08-28T02:11:02.643168+0000 mon.a (mon.0) 636 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:11:03.654 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:03 smithi169 conmon[100500]: 2023-08-28T02:11:03.654 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:03 smithi169 conmon[100500]: audit 2023-08-28T 2023-08-28T02:11:03.654 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:03 smithi169 conmon[100500]: 02:11:03.013257+0000 mon.a (mon.0) 637 : audit [DBG] from='client.? 172.21.15.143:0/4283278731' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-08-28T02:11:03.654 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:03 smithi169 conmon[100500]: audit 2023-08-28T02:11:03 2023-08-28T02:11:03.655 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:03 smithi169 conmon[100500]: .377027+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.143:0/3340541723' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-08-28T02:11:03.939 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:03 smithi169 conmon[131988]: t=2023-08-28T02:11:03+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_email - v1" 2023-08-28T02:11:03.939 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:03 smithi169 conmon[131988]: t=2023-08-28T02:11:03+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_org_id - v1" 2023-08-28T02:11:03.939 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:03 smithi169 conmon[131988]: t=2023-08-28T02:11:03+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_code - v1" 2023-08-28T02:11:03.939 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:03 smithi169 conmon[131988]: t=2023-08-28T02:11:03+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_status - v1" 2023-08-28T02:11:03.939 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:03 smithi169 conmon[131988]: t=2023-08-28T02:11:03+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2023-08-28T02:11:03.939 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:03 smithi169 conmon[131988]: t=2023-08-28T02:11:03+0000 lvl=info msg="Executing migration" logger=migrator id="create temp_user v2" 2023-08-28T02:11:03.939 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:03 smithi169 conmon[131988]: t=2023-08-28T02:11:03+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v2" 2023-08-28T02:11:03.939 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:03 smithi169 conmon[131988]: t=2023-08-28T02:11:03+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v2" 2023-08-28T02:11:04.207 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:03 smithi169 conmon[131988]: t=2023-08-28T02:11:03+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v2" 2023-08-28T02:11:04.208 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:04 smithi169 conmon[131988]: t=2023-08-28T02:11:04+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v2" 2023-08-28T02:11:04.208 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:04 smithi169 conmon[131988]: t=2023-08-28T02:11:04+0000 lvl=info msg="Executing migration" logger=migrator id="copy temp_user v1 to v2" 2023-08-28T02:11:04.487 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:04 smithi169 conmon[131988]: t=2023-08-28T02:11:04+0000 lvl=info msg="Executing migration" logger=migrator id="drop temp_user_tmp_qwerty" 2023-08-28T02:11:04.487 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:04 smithi169 conmon[131988]: t=2023-08-28T02:11:04+0000 lvl=info msg="Executing migration" logger=migrator id="Set created for temp users that will otherwise prematurely expire" 2023-08-28T02:11:04.487 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:04 smithi169 conmon[131988]: t=2023-08-28T02:11:04+0000 lvl=info msg="Executing migration" logger=migrator id="create star table" 2023-08-28T02:11:04.487 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:04 smithi169 conmon[131988]: t=2023-08-28T02:11:04+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index star.user_id_dashboard_id" 2023-08-28T02:11:04.623 INFO:teuthology.orchestra.run.smithi143.stdout:180388626451 2023-08-28T02:11:04.769 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:04 smithi169 conmon[131988]: t=2023-08-28T02:11:04+0000 lvl=info msg="Executing migration" logger=migrator id="create org table v1" 2023-08-28T02:11:04.769 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:04 smithi169 conmon[131988]: t=2023-08-28T02:11:04+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_name - v1" 2023-08-28T02:11:04.769 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:04 smithi169 conmon[131988]: t=2023-08-28T02:11:04+0000 lvl=info msg="Executing migration" logger=migrator id="create org_user table v1" 2023-08-28T02:11:04.769 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:04 smithi169 conmon[131988]: t=2023-08-28T02:11:04+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_org_id - v1" 2023-08-28T02:11:04.769 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:04 smithi169 conmon[131988]: t=2023-08-28T02:11:04+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_user_org_id_user_id - v1" 2023-08-28T02:11:04.769 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:04 smithi169 conmon[131988]: t=2023-08-28T02:11:04+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_user_id - v1" 2023-08-28T02:11:04.769 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:04 smithi169 conmon[131988]: t=2023-08-28T02:11:04+0000 lvl=info msg="Executing migration" logger=migrator id="Update org table charset" 2023-08-28T02:11:04.849 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:11:04.863 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182433 got 107374182435 for osd.3 2023-08-28T02:11:04.863 DEBUG:teuthology.parallel:result is None 2023-08-28T02:11:05.046 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:04 smithi169 conmon[131988]: t=2023-08-28T02:11:04+0000 lvl=info msg="Executing migration" logger=migrator id="Update org_user table charset" 2023-08-28T02:11:05.046 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:04 smithi169 conmon[131988]: t=2023-08-28T02:11:04+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate all Read Only Viewers to Viewers" 2023-08-28T02:11:05.046 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:04 smithi169 conmon[131988]: t=2023-08-28T02:11:04+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard table" 2023-08-28T02:11:05.297 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:05 smithi169 conmon[131988]: t=2023-08-28T02:11:05+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard.account_id" 2023-08-28T02:11:05.297 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:05 smithi169 conmon[131988]: t=2023-08-28T02:11:05+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_account_id_slug" 2023-08-28T02:11:05.297 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:05 smithi169 conmon[131988]: t=2023-08-28T02:11:05+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_tag table" 2023-08-28T02:11:05.298 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:05 smithi169 conmon[131988]: t=2023-08-28T02:11:05+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_tag.dasboard_id_term" 2023-08-28T02:11:05.298 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:05 smithi169 conmon[131988]: t=2023-08-28T02:11:05+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2023-08-28T02:11:05.607 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574889 got 55834574891 for osd.1 2023-08-28T02:11:05.607 DEBUG:teuthology.parallel:result is None 2023-08-28T02:11:05.608 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:05 smithi169 conmon[131988]: t=2023-08-28T02:11:05+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard to dashboard_v1 - v1" 2023-08-28T02:11:05.608 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:05 smithi169 conmon[131988]: t=2023-08-28T02:11:05+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard v2" 2023-08-28T02:11:05.608 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:05 smithi169 conmon[131988]: t=2023-08-28T02:11:05+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_org_id - v2" 2023-08-28T02:11:05.608 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:05 smithi169 conmon[131988]: t=2023-08-28T02:11:05+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_org_id_slug - v2" 2023-08-28T02:11:05.608 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:05 smithi169 conmon[100500]: cluster 2023-08-28T02:11:03.870657+0000 mgr.y (mgr.14424) 2023-08-28T02:11:05.608 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:05 smithi169 conmon[100500]: 44 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:05.609 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:05 smithi169 conmon[100500]: audit 2023-08-28T02:11:04.625214+0000 mon.a (mon.0) 638 : audit [DBG] from='client.? 172.21.15.143:0/832757883' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-08-28T02:11:05.609 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:05 smithi169 conmon[100500]: audit 2023-08-28T02:11:05 2023-08-28T02:11:05.609 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:05 smithi169 conmon[100500]: .023631+0000 mon.a (mon.0) 639 : audit 2023-08-28T02:11:05.609 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:05 smithi169 conmon[100500]: [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:05.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:05 smithi143 conmon[100258]: cluster 2023-08-28T02:11:03.870657+0000 mgr.y (mgr.14424) 44 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:05.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:05 smithi143 conmon[100258]: audit 2023-08-28T02:11:04.625214+0000 mon.a (mon.0) 638 : audit [DBG] from='client.? 172.21.15.143:0/832757883' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-08-28T02:11:05.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:05 smithi143 conmon[100258]: audit 2023-08-28T02:11:05.023631+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:05.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:05 smithi143 conmon[109923]: cluster 2023-08-28T02:11:03.870657+0000 mgr.y (mgr.14424) 44 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:05.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:05 smithi143 conmon[109923]: audit 2023-08-28T02:11:04.625214+0000 mon.a (mon.0) 638 : audit [DBG] from='client.? 172.21.15.143:0/832757883' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-08-28T02:11:05.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:05 smithi143 conmon[109923]: audit 2023-08-28T02:11:05.023631+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:05.884 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:05 smithi169 conmon[131988]: t=2023-08-28T02:11:05+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard v1 to v2" 2023-08-28T02:11:05.885 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:05 smithi169 conmon[131988]: t=2023-08-28T02:11:05+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_v1" 2023-08-28T02:11:05.885 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:05 smithi169 conmon[131988]: t=2023-08-28T02:11:05+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard.data to mediumtext v1" 2023-08-28T02:11:05.885 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:05 smithi169 conmon[131988]: t=2023-08-28T02:11:05+0000 lvl=info msg="Executing migration" logger=migrator id="Add column updated_by in dashboard - v2" 2023-08-28T02:11:05.885 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:05 smithi169 conmon[131988]: t=2023-08-28T02:11:05+0000 lvl=info msg="Executing migration" logger=migrator id="Add column created_by in dashboard - v2" 2023-08-28T02:11:06.145 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:05 smithi169 conmon[131988]: t=2023-08-28T02:11:05+0000 lvl=info msg="Executing migration" logger=migrator id="Add column gnetId in dashboard" 2023-08-28T02:11:06.145 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:05 smithi169 conmon[131988]: t=2023-08-28T02:11:05+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for gnetId in dashboard" 2023-08-28T02:11:06.145 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:05 smithi169 conmon[131988]: t=2023-08-28T02:11:05+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_id in dashboard" 2023-08-28T02:11:06.145 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:06 smithi169 conmon[131988]: t=2023-08-28T02:11:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for plugin_id in dashboard" 2023-08-28T02:11:06.145 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:06 smithi169 conmon[131988]: t=2023-08-28T02:11:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_id in dashboard_tag" 2023-08-28T02:11:06.145 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:06 smithi169 conmon[131988]: t=2023-08-28T02:11:06+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard table charset" 2023-08-28T02:11:06.146 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:06 smithi169 conmon[131988]: t=2023-08-28T02:11:06+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_tag table charset" 2023-08-28T02:11:06.146 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:06 smithi169 conmon[131988]: t=2023-08-28T02:11:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add column folder_id in dashboard" 2023-08-28T02:11:06.146 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:06 smithi169 conmon[131988]: t=2023-08-28T02:11:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add column isFolder in dashboard" 2023-08-28T02:11:06.226 INFO:teuthology.orchestra.run.smithi143.stdout:34359738416 2023-08-28T02:11:06.405 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:06 smithi169 conmon[131988]: t=2023-08-28T02:11:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add column has_acl in dashboard" 2023-08-28T02:11:06.405 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:06 smithi169 conmon[131988]: t=2023-08-28T02:11:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in dashboard" 2023-08-28T02:11:06.406 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:06 smithi169 conmon[131988]: t=2023-08-28T02:11:06+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in dashboard" 2023-08-28T02:11:06.406 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:06 smithi169 conmon[131988]: t=2023-08-28T02:11:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index dashboard_org_id_uid" 2023-08-28T02:11:06.406 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:06 smithi169 conmon[131988]: t=2023-08-28T02:11:06+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_slug" 2023-08-28T02:11:06.406 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:06 smithi169 conmon[131988]: t=2023-08-28T02:11:06+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard title length" 2023-08-28T02:11:06.406 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:06 smithi169 conmon[131988]: t=2023-08-28T02:11:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index for dashboard_org_id_title_folder_id" 2023-08-28T02:11:06.406 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:06 smithi169 conmon[131988]: t=2023-08-28T02:11:06+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning" 2023-08-28T02:11:06.406 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:06 smithi169 conmon[131988]: t=2023-08-28T02:11:06+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2023-08-28T02:11:06.674 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:06 smithi169 conmon[131988]: t=2023-08-28T02:11:06+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning v2" 2023-08-28T02:11:06.674 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:06 smithi169 conmon[131988]: t=2023-08-28T02:11:06+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2023-08-28T02:11:06.674 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:06 smithi169 conmon[131988]: t=2023-08-28T02:11:06+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2023-08-28T02:11:06.674 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:06 smithi169 conmon[131988]: t=2023-08-28T02:11:06+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard_provisioning v1 to v2" 2023-08-28T02:11:06.674 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:06 smithi169 conmon[131988]: t=2023-08-28T02:11:06+0000 lvl=info msg="Executing migration" logger=migrator id="drop dashboard_provisioning_tmp_qwerty" 2023-08-28T02:11:06.674 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:06 smithi169 conmon[131988]: t=2023-08-28T02:11:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add check_sum column" 2023-08-28T02:11:06.926 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:06 smithi169 conmon[100500]: audit 2023-08-28T02:11:05.837166+0000 mon.a (mon.0) 640 : audit [INF] 2023-08-28T02:11:06.926 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:06 smithi169 conmon[100500]: from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:06.926 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:06 smithi169 conmon[100500]: cluster 2023-08-28T02:11:05.870965+0000 mgr.y (mgr.14424) 45 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:06.926 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:06 smithi169 conmon[100500]: audit 2023-08-28T02:11:06.227537+0000 mon.a (mon.0) 641 : audit [DBG] from='client.? 172.21.15.143:0/1160773910' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-08-28T02:11:06.927 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:06 smithi169 conmon[131988]: t=2023-08-28T02:11:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_title" 2023-08-28T02:11:06.927 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:06 smithi169 conmon[131988]: t=2023-08-28T02:11:06+0000 lvl=info msg="Executing migration" logger=migrator id="delete tags for deleted dashboards" 2023-08-28T02:11:06.927 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:06 smithi169 conmon[131988]: t=2023-08-28T02:11:06+0000 lvl=info msg="Executing migration" logger=migrator id="delete stars for deleted dashboards" 2023-08-28T02:11:06.927 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:06 smithi169 conmon[131988]: t=2023-08-28T02:11:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_is_folder" 2023-08-28T02:11:06.927 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:06 smithi169 conmon[131988]: t=2023-08-28T02:11:06+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table" 2023-08-28T02:11:06.927 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:06 smithi169 conmon[131988]: t=2023-08-28T02:11:06+0000 lvl=info msg="Executing migration" logger=migrator id="add index data_source.account_id" 2023-08-28T02:11:06.927 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:06 smithi169 conmon[131988]: t=2023-08-28T02:11:06+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index data_source.account_id_name" 2023-08-28T02:11:06.927 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:06 smithi169 conmon[131988]: t=2023-08-28T02:11:06+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_data_source_account_id - v1" 2023-08-28T02:11:07.178 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:06 smithi169 conmon[131988]: t=2023-08-28T02:11:06+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_data_source_account_id_name - v1" 2023-08-28T02:11:07.179 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:06 smithi169 conmon[131988]: t=2023-08-28T02:11:06+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table data_source to data_source_v1 - v1" 2023-08-28T02:11:07.179 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:07 smithi169 conmon[131988]: t=2023-08-28T02:11:07+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table v2" 2023-08-28T02:11:07.179 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:07 smithi169 conmon[131988]: t=2023-08-28T02:11:07+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_data_source_org_id - v2" 2023-08-28T02:11:07.179 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:07 smithi169 conmon[131988]: t=2023-08-28T02:11:07+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_data_source_org_id_name - v2" 2023-08-28T02:11:07.179 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:07 smithi169 conmon[131988]: t=2023-08-28T02:11:07+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2023-08-28T02:11:07.179 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:07 smithi169 conmon[131988]: t=2023-08-28T02:11:07+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table data_source_v1 #2" 2023-08-28T02:11:07.180 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:07 smithi169 conmon[131988]: t=2023-08-28T02:11:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add column with_credentials" 2023-08-28T02:11:07.180 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:07 smithi169 conmon[131988]: t=2023-08-28T02:11:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add secure json data column" 2023-08-28T02:11:07.180 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:07 smithi169 conmon[131988]: t=2023-08-28T02:11:07+0000 lvl=info msg="Executing migration" logger=migrator id="Update data_source table charset" 2023-08-28T02:11:07.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:06 smithi143 conmon[100258]: audit 2023-08-28T02:11:05.837166+0000 mon.a (mon.0) 640 2023-08-28T02:11:07.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:06 smithi143 conmon[100258]: : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:07.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:06 smithi143 conmon[100258]: cluster 2023-08-28T02:11:05.870965+0000 mgr.y (mgr.14424) 45 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:07.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:06 smithi143 conmon[100258]: audit 2023-08-28T02:11:06.227537+0000 mon.a (mon.0) 641 : audit [DBG] from='client.? 172.21.15.143:0/1160773910' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-08-28T02:11:07.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:06 smithi143 conmon[109923]: audit 2023-08-28T02:11:05.837166+0000 mon.a (mon 2023-08-28T02:11:07.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:06 smithi143 conmon[109923]: .0) 640 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:07.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:06 smithi143 conmon[109923]: cluster 2023-08-28T02:11:05.870965+0000 mgr.y (mgr.14424) 45 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:07.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:06 smithi143 conmon[109923]: audit 2023-08-28T02:11:06.227537+0000 mon.a (mon.0) 641 : audit [DBG] from='client.? 172.21.15.143:0/1160773910' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-08-28T02:11:07.246 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626449 got 180388626451 for osd.6 2023-08-28T02:11:07.247 DEBUG:teuthology.parallel:result is None 2023-08-28T02:11:07.380 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738414 got 34359738416 for osd.0 2023-08-28T02:11:07.380 DEBUG:teuthology.parallel:result is None 2023-08-28T02:11:07.431 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:07 smithi169 conmon[131988]: t=2023-08-28T02:11:07+0000 lvl=info msg="Executing migration" logger=migrator id="Update initial version to 1" 2023-08-28T02:11:07.432 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:07 smithi169 conmon[131988]: t=2023-08-28T02:11:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add read_only data column" 2023-08-28T02:11:07.432 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:07 smithi169 conmon[131988]: t=2023-08-28T02:11:07+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate logging ds to loki ds" 2023-08-28T02:11:07.432 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:07 smithi169 conmon[131988]: t=2023-08-28T02:11:07+0000 lvl=info msg="Executing migration" logger=migrator id="Update json_data with nulls" 2023-08-28T02:11:07.432 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:07 smithi169 conmon[131988]: t=2023-08-28T02:11:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add uid column" 2023-08-28T02:11:07.433 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:07 smithi169 conmon[131988]: t=2023-08-28T02:11:07+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid value" 2023-08-28T02:11:07.433 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:07 smithi169 conmon[131988]: t=2023-08-28T02:11:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index datasource_org_id_uid" 2023-08-28T02:11:07.433 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:07 smithi169 conmon[131988]: t=2023-08-28T02:11:07+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index datasource_org_id_is_default" 2023-08-28T02:11:07.433 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:07 smithi169 conmon[131988]: t=2023-08-28T02:11:07+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table" 2023-08-28T02:11:07.434 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:07 smithi169 conmon[131988]: t=2023-08-28T02:11:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id" 2023-08-28T02:11:07.434 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:07 smithi169 conmon[131988]: t=2023-08-28T02:11:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.key" 2023-08-28T02:11:07.692 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:07 smithi169 conmon[131988]: t=2023-08-28T02:11:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id_name" 2023-08-28T02:11:07.692 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:07 smithi169 conmon[131988]: t=2023-08-28T02:11:07+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_api_key_account_id - v1" 2023-08-28T02:11:07.692 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:07 smithi169 conmon[131988]: t=2023-08-28T02:11:07+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_key - v1" 2023-08-28T02:11:07.692 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:07 smithi169 conmon[131988]: t=2023-08-28T02:11:07+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_account_id_name - v1" 2023-08-28T02:11:07.692 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:07 smithi169 conmon[131988]: t=2023-08-28T02:11:07+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table api_key to api_key_v1 - v1" 2023-08-28T02:11:07.692 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:07 smithi169 conmon[131988]: t=2023-08-28T02:11:07+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table v2" 2023-08-28T02:11:07.693 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:07 smithi169 conmon[131988]: t=2023-08-28T02:11:07+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_api_key_org_id - v2" 2023-08-28T02:11:07.717 INFO:teuthology.orchestra.run.smithi143.stdout:206158430225 2023-08-28T02:11:07.860 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:07 smithi143 conmon[100258]: audit 2023-08-28T02:11:07.718150+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.143:0/467622108' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-08-28T02:11:07.861 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:07 smithi143 conmon[109923]: audit 2023-08-28T02:11:07.718150+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.143:0/467622108' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-08-28T02:11:07.943 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:07 smithi169 conmon[100500]: audit 2023-08-28T02:11:07.718150+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.143:0/467622108' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-08-28T02:11:07.943 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:07 smithi169 conmon[131988]: t=2023-08-28T02:11:07+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_key - v2" 2023-08-28T02:11:07.943 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:07 smithi169 conmon[131988]: t=2023-08-28T02:11:07+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_org_id_name - v2" 2023-08-28T02:11:07.944 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:07 smithi169 conmon[131988]: t=2023-08-28T02:11:07+0000 lvl=info msg="Executing migration" logger=migrator id="copy api_key v1 to v2" 2023-08-28T02:11:07.944 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:07 smithi169 conmon[131988]: t=2023-08-28T02:11:07+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table api_key_v1" 2023-08-28T02:11:07.944 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:07 smithi169 conmon[131988]: t=2023-08-28T02:11:07+0000 lvl=info msg="Executing migration" logger=migrator id="Update api_key table charset" 2023-08-28T02:11:07.944 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:07 smithi169 conmon[131988]: t=2023-08-28T02:11:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add expires to api_key table" 2023-08-28T02:11:07.997 INFO:teuthology.orchestra.run.smithi143.stdout:77309411367 2023-08-28T02:11:08.196 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:08 smithi169 conmon[131988]: t=2023-08-28T02:11:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add service account foreign key" 2023-08-28T02:11:08.196 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:08 smithi169 conmon[131988]: t=2023-08-28T02:11:08+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v4" 2023-08-28T02:11:08.196 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:08 smithi169 conmon[131988]: t=2023-08-28T02:11:08+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_snapshot_v4 #1" 2023-08-28T02:11:08.196 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:08 smithi169 conmon[131988]: t=2023-08-28T02:11:08+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v5 #2" 2023-08-28T02:11:08.196 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:08 smithi169 conmon[131988]: t=2023-08-28T02:11:08+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_key - v5" 2023-08-28T02:11:08.196 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:08 smithi169 conmon[131988]: t=2023-08-28T02:11:08+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_delete_key - v5" 2023-08-28T02:11:08.196 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:08 smithi169 conmon[131988]: t=2023-08-28T02:11:08+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_snapshot_user_id - v5" 2023-08-28T02:11:08.196 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:08 smithi169 conmon[131988]: t=2023-08-28T02:11:08+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_snapshot to mediumtext v2" 2023-08-28T02:11:08.196 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:08 smithi169 conmon[131988]: t=2023-08-28T02:11:08+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_snapshot table charset" 2023-08-28T02:11:08.196 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:08 smithi169 conmon[131988]: t=2023-08-28T02:11:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external_delete_url to dashboard_snapshots table" 2023-08-28T02:11:08.197 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:08 smithi169 conmon[131988]: t=2023-08-28T02:11:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add encrypted dashboard json column" 2023-08-28T02:11:08.464 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:08 smithi169 conmon[131988]: t=2023-08-28T02:11:08+0000 lvl=info msg="Executing migration" logger=migrator id="Change dashboard_encrypted column to MEDIUMBLOB" 2023-08-28T02:11:08.465 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:08 smithi169 conmon[131988]: t=2023-08-28T02:11:08+0000 lvl=info msg="Executing migration" logger=migrator id="create quota table v1" 2023-08-28T02:11:08.465 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:08 smithi169 conmon[131988]: t=2023-08-28T02:11:08+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_quota_org_id_user_id_target - v1" 2023-08-28T02:11:08.465 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:08 smithi169 conmon[131988]: t=2023-08-28T02:11:08+0000 lvl=info msg="Executing migration" logger=migrator id="Update quota table charset" 2023-08-28T02:11:08.465 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:08 smithi169 conmon[131988]: t=2023-08-28T02:11:08+0000 lvl=info msg="Executing migration" logger=migrator id="create plugin_setting table" 2023-08-28T02:11:08.465 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:08 smithi169 conmon[131988]: t=2023-08-28T02:11:08+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2023-08-28T02:11:08.465 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:08 smithi169 conmon[131988]: t=2023-08-28T02:11:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_version to plugin_settings" 2023-08-28T02:11:08.734 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:08 smithi169 conmon[131988]: t=2023-08-28T02:11:08+0000 lvl=info msg="Executing migration" logger=migrator id="Update plugin_setting table charset" 2023-08-28T02:11:08.734 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:08 smithi169 conmon[131988]: t=2023-08-28T02:11:08+0000 lvl=info msg="Executing migration" logger=migrator id="create session table" 2023-08-28T02:11:08.735 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:08 smithi169 conmon[131988]: t=2023-08-28T02:11:08+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist table" 2023-08-28T02:11:08.735 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:08 smithi169 conmon[131988]: t=2023-08-28T02:11:08+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist_item table" 2023-08-28T02:11:08.735 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:08 smithi169 conmon[131988]: t=2023-08-28T02:11:08+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist table v2" 2023-08-28T02:11:08.735 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:08 smithi169 conmon[131988]: t=2023-08-28T02:11:08+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist item table v2" 2023-08-28T02:11:08.735 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:08 smithi169 conmon[131988]: t=2023-08-28T02:11:08+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist table charset" 2023-08-28T02:11:08.736 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:08 smithi169 conmon[131988]: t=2023-08-28T02:11:08+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist_item table charset" 2023-08-28T02:11:08.736 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:08 smithi169 conmon[131988]: t=2023-08-28T02:11:08+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v2" 2023-08-28T02:11:08.736 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:08 smithi169 conmon[131988]: t=2023-08-28T02:11:08+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v3" 2023-08-28T02:11:08.736 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:08 smithi169 conmon[131988]: t=2023-08-28T02:11:08+0000 lvl=info msg="Executing migration" logger=migrator id="create preferences table v3" 2023-08-28T02:11:08.736 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:08 smithi169 conmon[131988]: t=2023-08-28T02:11:08+0000 lvl=info msg="Executing migration" logger=migrator id="Update preferences table charset" 2023-08-28T02:11:08.831 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430223 got 206158430225 for osd.7 2023-08-28T02:11:08.831 DEBUG:teuthology.parallel:result is None 2023-08-28T02:11:08.873 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411365 got 77309411367 for osd.2 2023-08-28T02:11:08.874 DEBUG:teuthology.parallel:result is None 2023-08-28T02:11:08.874 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-08-28T02:11:08.874 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph pg dump --format=json 2023-08-28T02:11:08.986 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:08 smithi169 conmon[100500]: cluster 2023-08-28T02:11:07.871220+0000 mgr.y (mgr.14424) 46 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:08.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:08 smithi169 conmon[100500]: audit 2023-08-28T02:11:07 2023-08-28T02:11:08.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:08 smithi169 conmon[100500]: .998929+0000 mon.c (mon.2) 16 : audit [DBG] from='client.? 172.21.15.143:0/601145234' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-08-28T02:11:08.987 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:08 smithi169 conmon[131988]: t=2023-08-28T02:11:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add column team_id in preferences" 2023-08-28T02:11:08.987 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:08 smithi169 conmon[131988]: t=2023-08-28T02:11:08+0000 lvl=info msg="Executing migration" logger=migrator id="Update team_id column values in preferences" 2023-08-28T02:11:08.987 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:08 smithi169 conmon[131988]: t=2023-08-28T02:11:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add column week_start in preferences" 2023-08-28T02:11:08.987 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:08 smithi169 conmon[131988]: t=2023-08-28T02:11:08+0000 lvl=info msg="Executing migration" logger=migrator id="create alert table v1" 2023-08-28T02:11:08.987 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:08 smithi169 conmon[131988]: t=2023-08-28T02:11:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert org_id & id " 2023-08-28T02:11:08.987 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:08 smithi169 conmon[131988]: t=2023-08-28T02:11:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert state" 2023-08-28T02:11:08.988 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:08 smithi169 conmon[131988]: t=2023-08-28T02:11:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert dashboard_id" 2023-08-28T02:11:08.988 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:08 smithi169 conmon[131988]: t=2023-08-28T02:11:08+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v1" 2023-08-28T02:11:08.988 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:08 smithi169 conmon[131988]: t=2023-08-28T02:11:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_rule_tag.alert_id_tag_id" 2023-08-28T02:11:08.988 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:08 smithi169 conmon[131988]: t=2023-08-28T02:11:08+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2023-08-28T02:11:09.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:08 smithi143 conmon[109923]: cluster 2023-08-28T02:11:07.871220+0000 2023-08-28T02:11:09.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:08 smithi143 conmon[109923]: mgr.y (mgr.14424) 46 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:09.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:08 smithi143 conmon[109923]: audit 2023-08-28T02:11:07.998929+0000 mon.c (mon.2) 16 : audit [DBG] from='client.? 172.21.15.143:0/601145234' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-08-28T02:11:09.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:08 smithi143 conmon[100258]: cluster 2023-08-28T02:11:07.871220+0000 mgr.y (mgr.14424) 46 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:09.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:08 smithi143 conmon[100258]: audit 2023-08-28T02:11:07.998929+0000 mon.c (mon.2) 16 : audit [DBG] from='client.? 172.21.15.143:0/601145234' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-08-28T02:11:09.237 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:09 smithi169 conmon[131988]: t=2023-08-28T02:11:09+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2023-08-28T02:11:09.237 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:09 smithi169 conmon[131988]: t=2023-08-28T02:11:09+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v2" 2023-08-28T02:11:09.237 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:09 smithi169 conmon[131988]: t=2023-08-28T02:11:09+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2023-08-28T02:11:09.237 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:09 smithi169 conmon[131988]: t=2023-08-28T02:11:09+0000 lvl=info msg="Executing migration" logger=migrator id="copy alert_rule_tag v1 to v2" 2023-08-28T02:11:09.237 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:09 smithi169 conmon[131988]: t=2023-08-28T02:11:09+0000 lvl=info msg="Executing migration" logger=migrator id="drop table alert_rule_tag_v1" 2023-08-28T02:11:09.257 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:11:09.498 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:09 smithi169 conmon[131988]: t=2023-08-28T02:11:09+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification table v1" 2023-08-28T02:11:09.499 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:09 smithi169 conmon[131988]: t=2023-08-28T02:11:09+0000 lvl=info msg="Executing migration" logger=migrator id="Add column is_default" 2023-08-28T02:11:09.499 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:09 smithi169 conmon[131988]: t=2023-08-28T02:11:09+0000 lvl=info msg="Executing migration" logger=migrator id="Add column frequency" 2023-08-28T02:11:09.499 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:09 smithi169 conmon[131988]: t=2023-08-28T02:11:09+0000 lvl=info msg="Executing migration" logger=migrator id="Add column send_reminder" 2023-08-28T02:11:09.499 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:09 smithi169 conmon[131988]: t=2023-08-28T02:11:09+0000 lvl=info msg="Executing migration" logger=migrator id="Add column disable_resolve_message" 2023-08-28T02:11:09.499 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:09 smithi169 conmon[131988]: t=2023-08-28T02:11:09+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification org_id & name" 2023-08-28T02:11:09.499 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:09 smithi169 conmon[131988]: t=2023-08-28T02:11:09+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert table charset" 2023-08-28T02:11:09.500 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:09 smithi169 conmon[131988]: t=2023-08-28T02:11:09+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert_notification table charset" 2023-08-28T02:11:09.500 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:09 smithi169 conmon[131988]: t=2023-08-28T02:11:09+0000 lvl=info msg="Executing migration" logger=migrator id="create notification_journal table v1" 2023-08-28T02:11:09.759 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:09 smithi169 conmon[131988]: t=2023-08-28T02:11:09+0000 lvl=info msg="Executing migration" logger=migrator id="add index notification_journal org_id & alert_id & notifier_id" 2023-08-28T02:11:09.759 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:09 smithi169 conmon[131988]: t=2023-08-28T02:11:09+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_notification_journal" 2023-08-28T02:11:09.759 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:09 smithi169 conmon[131988]: t=2023-08-28T02:11:09+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification_state table v1" 2023-08-28T02:11:09.760 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:09 smithi169 conmon[131988]: t=2023-08-28T02:11:09+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification_state org_id & alert_id & notifier_id" 2023-08-28T02:11:09.760 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:09 smithi169 conmon[131988]: t=2023-08-28T02:11:09+0000 lvl=info msg="Executing migration" logger=migrator id="Add for to alert table" 2023-08-28T02:11:09.760 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:09 smithi169 conmon[131988]: t=2023-08-28T02:11:09+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in alert_notification" 2023-08-28T02:11:09.760 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:09 smithi169 conmon[131988]: t=2023-08-28T02:11:09+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in alert_notification" 2023-08-28T02:11:09.760 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:09 smithi169 conmon[131988]: t=2023-08-28T02:11:09+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_notification_org_id_uid" 2023-08-28T02:11:10.061 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:09 smithi169 conmon[131988]: t=2023-08-28T02:11:09+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_name" 2023-08-28T02:11:10.061 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:09 smithi169 conmon[131988]: t=2023-08-28T02:11:09+0000 lvl=info msg="Executing migration" logger=migrator id="Add column secure_settings in alert_notification" 2023-08-28T02:11:10.062 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:09 smithi169 conmon[131988]: t=2023-08-28T02:11:09+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert.settings to mediumtext" 2023-08-28T02:11:10.062 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:09 smithi169 conmon[131988]: t=2023-08-28T02:11:09+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_notification_state_alert_id" 2023-08-28T02:11:10.062 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:09 smithi169 conmon[131988]: t=2023-08-28T02:11:09+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_rule_tag_alert_id" 2023-08-28T02:11:10.062 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:09 smithi169 conmon[131988]: t=2023-08-28T02:11:09+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old annotation table v4" 2023-08-28T02:11:10.062 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:09 smithi169 conmon[131988]: t=2023-08-28T02:11:09+0000 lvl=info msg="Executing migration" logger=migrator id="create annotation table v5" 2023-08-28T02:11:10.062 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:09 smithi169 conmon[131988]: t=2023-08-28T02:11:09+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 0 v3" 2023-08-28T02:11:10.062 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:09 smithi169 conmon[131988]: t=2023-08-28T02:11:09+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 1 v3" 2023-08-28T02:11:10.062 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:09 smithi169 conmon[131988]: t=2023-08-28T02:11:09+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 2 v3" 2023-08-28T02:11:10.062 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:10 smithi169 conmon[131988]: t=2023-08-28T02:11:10+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 3 v3" 2023-08-28T02:11:10.339 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:10 smithi169 conmon[131988]: t=2023-08-28T02:11:10+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 4 v3" 2023-08-28T02:11:10.339 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:10 smithi169 conmon[131988]: t=2023-08-28T02:11:10+0000 lvl=info msg="Executing migration" logger=migrator id="Update annotation table charset" 2023-08-28T02:11:10.339 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:10 smithi169 conmon[131988]: t=2023-08-28T02:11:10+0000 lvl=info msg="Executing migration" logger=migrator id="Add column region_id to annotation table" 2023-08-28T02:11:10.339 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:10 smithi169 conmon[131988]: t=2023-08-28T02:11:10+0000 lvl=info msg="Executing migration" logger=migrator id="Drop category_id index" 2023-08-28T02:11:10.339 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:10 smithi169 conmon[131988]: t=2023-08-28T02:11:10+0000 lvl=info msg="Executing migration" logger=migrator id="Add column tags to annotation table" 2023-08-28T02:11:10.339 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:10 smithi169 conmon[131988]: t=2023-08-28T02:11:10+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v2" 2023-08-28T02:11:10.339 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:10 smithi169 conmon[131988]: t=2023-08-28T02:11:10+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index annotation_tag.annotation_id_tag_id" 2023-08-28T02:11:10.339 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:10 smithi169 conmon[131988]: t=2023-08-28T02:11:10+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2023-08-28T02:11:10.599 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:10 smithi169 conmon[131988]: t=2023-08-28T02:11:10+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table annotation_tag to annotation_tag_v2 - v2" 2023-08-28T02:11:10.599 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:10 smithi169 conmon[131988]: t=2023-08-28T02:11:10+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v3" 2023-08-28T02:11:10.599 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:10 smithi169 conmon[131988]: t=2023-08-28T02:11:10+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2023-08-28T02:11:10.599 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:10 smithi169 conmon[131988]: t=2023-08-28T02:11:10+0000 lvl=info msg="Executing migration" logger=migrator id="copy annotation_tag v2 to v3" 2023-08-28T02:11:10.599 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:10 smithi169 conmon[131988]: t=2023-08-28T02:11:10+0000 lvl=info msg="Executing migration" logger=migrator id="drop table annotation_tag_v2" 2023-08-28T02:11:10.599 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:10 smithi169 conmon[131988]: t=2023-08-28T02:11:10+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert annotations and set TEXT to empty" 2023-08-28T02:11:10.599 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:10 smithi169 conmon[131988]: t=2023-08-28T02:11:10+0000 lvl=info msg="Executing migration" logger=migrator id="Add created time to annotation table" 2023-08-28T02:11:10.600 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:10 smithi169 conmon[131988]: t=2023-08-28T02:11:10+0000 lvl=info msg="Executing migration" logger=migrator id="Add updated time to annotation table" 2023-08-28T02:11:10.670 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:11:10 smithi143 conmon[100708]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:11:10] "GET /metrics HTTP/1.1" 200 155662 "" "Prometheus/2.33.4" 2023-08-28T02:11:10.874 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:10 smithi169 conmon[131988]: t=2023-08-28T02:11:10+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for created in annotation table" 2023-08-28T02:11:10.894 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:10 smithi169 conmon[131988]: t=2023-08-28T02:11:10+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for updated in annotation table" 2023-08-28T02:11:10.894 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:10 smithi169 conmon[131988]: t=2023-08-28T02:11:10+0000 lvl=info msg="Executing migration" logger=migrator id="Convert existing annotations from seconds to milliseconds" 2023-08-28T02:11:10.894 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:10 smithi169 conmon[131988]: t=2023-08-28T02:11:10+0000 lvl=info msg="Executing migration" logger=migrator id="Add epoch_end column" 2023-08-28T02:11:10.894 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:10 smithi169 conmon[131988]: t=2023-08-28T02:11:10+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for epoch_end" 2023-08-28T02:11:10.894 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:10 smithi169 conmon[131988]: t=2023-08-28T02:11:10+0000 lvl=info msg="Executing migration" logger=migrator id="Make epoch_end the same as epoch" 2023-08-28T02:11:10.894 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:10 smithi169 conmon[131988]: t=2023-08-28T02:11:10+0000 lvl=info msg="Executing migration" logger=migrator id="Move region to single row" 2023-08-28T02:11:10.894 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:10 smithi169 conmon[131988]: t=2023-08-28T02:11:10+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch from annotation table" 2023-08-28T02:11:11.046 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:11:11.047 INFO:teuthology.orchestra.run.smithi143.stderr:dumped all 2023-08-28T02:11:11.161 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:10 smithi169 conmon[131988]: t=2023-08-28T02:11:10+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2023-08-28T02:11:11.162 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:10 smithi169 conmon[131988]: t=2023-08-28T02:11:10+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2023-08-28T02:11:11.162 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:10 smithi169 conmon[131988]: t=2023-08-28T02:11:10+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_epoch_end_epoch on annotation table" 2023-08-28T02:11:11.162 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:11 smithi169 conmon[131988]: t=2023-08-28T02:11:11+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch_epoch_end from annotation table" 2023-08-28T02:11:11.162 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:11 smithi169 conmon[131988]: t=2023-08-28T02:11:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for alert_id on annotation table" 2023-08-28T02:11:11.163 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:11 smithi169 conmon[131988]: t=2023-08-28T02:11:11+0000 lvl=info msg="Executing migration" logger=migrator id="create test_data table" 2023-08-28T02:11:11.163 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:11 smithi169 conmon[131988]: t=2023-08-28T02:11:11+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_version table v1" 2023-08-28T02:11:11.163 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:11 smithi169 conmon[131988]: t=2023-08-28T02:11:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_version.dashboard_id" 2023-08-28T02:11:11.163 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:11 smithi169 conmon[131988]: t=2023-08-28T02:11:11+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2023-08-28T02:11:11.422 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:11 smithi169 conmon[131988]: t=2023-08-28T02:11:11+0000 lvl=info msg="Executing migration" logger=migrator id="Set dashboard version to 1 where 0" 2023-08-28T02:11:11.422 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:11 smithi169 conmon[131988]: t=2023-08-28T02:11:11+0000 lvl=info msg="Executing migration" logger=migrator id="save existing dashboard data in dashboard_version table v1" 2023-08-28T02:11:11.422 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:11 smithi169 conmon[131988]: t=2023-08-28T02:11:11+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_version.data to mediumtext v1" 2023-08-28T02:11:11.422 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:11 smithi169 conmon[131988]: t=2023-08-28T02:11:11+0000 lvl=info msg="Executing migration" logger=migrator id="create team table" 2023-08-28T02:11:11.423 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:11 smithi169 conmon[131988]: t=2023-08-28T02:11:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index team.org_id" 2023-08-28T02:11:11.423 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:11 smithi169 conmon[131988]: t=2023-08-28T02:11:11+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_org_id_name" 2023-08-28T02:11:11.423 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:11 smithi169 conmon[131988]: t=2023-08-28T02:11:11+0000 lvl=info msg="Executing migration" logger=migrator id="create team member table" 2023-08-28T02:11:11.423 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:11 smithi169 conmon[131988]: t=2023-08-28T02:11:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.org_id" 2023-08-28T02:11:11.423 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:11 smithi169 conmon[131988]: t=2023-08-28T02:11:11+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_member_org_id_team_id_user_id" 2023-08-28T02:11:11.423 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:11 smithi169 conmon[131988]: t=2023-08-28T02:11:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.team_id" 2023-08-28T02:11:11.682 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:11 smithi169 conmon[100500]: cluster 2023-08-28T02:11:09.871495+0000 mgr.y (mgr.14424) 47 : 2023-08-28T02:11:11.683 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:11 smithi169 conmon[100500]: cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:11.683 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:11 smithi169 conmon[100500]: audit 2023-08-28T02:11:10.491335+0000 mon.a (mon.0) 642 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:11.683 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:11 smithi169 conmon[100500]: audit 2023- 2023-08-28T02:11:11.683 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:11 smithi169 conmon[100500]: 08-28T02:11:10.492091+0000 mon.a (mon.0) 643 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:11:11.683 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:11 smithi169 conmon[100500]: audit 2023-08-28T02:11:10.492687+0000 mon.a 2023-08-28T02:11:11.683 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:11 smithi169 conmon[100500]: (mon.0) 644 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:11:11.683 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:11 smithi169 conmon[100500]: audit 2023-08-28T02:11:10 2023-08-28T02:11:11.683 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:11 smithi169 conmon[100500]: .496222+0000 mon.a (mon.0) 645 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:11.684 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:11 smithi169 conmon[100500]: cephadm 2023- 2023-08-28T02:11:11.684 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:11 smithi169 conmon[100500]: 08-28T02:11:10.498976+0000 mgr.y (mgr.14424) 48 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-08-28T02:11:11.684 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:11 smithi169 conmon[100500]: cephadm 2023-08-28T02:11:10.501944+0000 mgr.y (mgr 2023-08-28T02:11:11.684 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:11 smithi169 conmon[100500]: .14424) 49 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi143 2023-08-28T02:11:11.684 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:11 smithi169 conmon[131988]: t=2023-08-28T02:11:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column email to team table" 2023-08-28T02:11:11.684 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:11 smithi169 conmon[131988]: t=2023-08-28T02:11:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external to team_member table" 2023-08-28T02:11:11.684 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:11 smithi169 conmon[131988]: t=2023-08-28T02:11:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column permission to team_member table" 2023-08-28T02:11:11.685 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:11 smithi169 conmon[131988]: t=2023-08-28T02:11:11+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard acl table" 2023-08-28T02:11:11.685 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:11 smithi169 conmon[131988]: t=2023-08-28T02:11:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_dashboard_id" 2023-08-28T02:11:11.685 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:11 smithi169 conmon[131988]: t=2023-08-28T02:11:11+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_user_id" 2023-08-28T02:11:11.685 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:11 smithi169 conmon[131988]: t=2023-08-28T02:11:11+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_team_id" 2023-08-28T02:11:11.685 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:11 smithi169 conmon[131988]: t=2023-08-28T02:11:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_user_id" 2023-08-28T02:11:11.685 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:11 smithi169 conmon[131988]: t=2023-08-28T02:11:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_team_id" 2023-08-28T02:11:11.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:11 smithi143 conmon[100258]: cluster 2023-08-28T02:11:09.871495+0000 mgr.y (mgr.14424) 47 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:11.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:11 smithi143 conmon[100258]: audit 2023-08-28T02:11:10.491335+0000 mon.a (mon.0) 642 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:11.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:11 smithi143 conmon[100258]: audit 2023-08-28T02:11:10.492091+0000 mon.a (mon.0) 643 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:11:11.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:11 smithi143 conmon[100258]: audit 2023-08-28T02:11:10.492687+0000 mon.a (mon.0) 644 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:11:11.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:11 smithi143 conmon[100258]: audit 2023-08-28T02:11:10.496222+0000 mon.a (mon.0) 645 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:11.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:11 smithi143 conmon[100258]: cephadm 2023-08-28T02:11:10.498976+0000 mgr.y (mgr.14424) 48 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-08-28T02:11:11.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:11 smithi143 conmon[100258]: cephadm 2023-08-28T02:11:10.501944+0000 mgr.y (mgr.14424) 49 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi143 2023-08-28T02:11:11.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:11 smithi143 conmon[109923]: cluster 2023-08-28T02:11:09.871495+0000 mgr.y (mgr.14424) 47 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:11.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:11 smithi143 conmon[109923]: audit 2023-08-28T02:11:10.491335+0000 mon.a (mon.0) 642 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:11.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:11 smithi143 conmon[109923]: audit 2023-08-28T02:11:10.492091+0000 mon.a (mon.0) 643 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:11:11.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:11 smithi143 conmon[109923]: audit 2023-08-28T02:11:10.492687+0000 mon.a (mon.0) 644 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:11:11.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:11 smithi143 conmon[109923]: audit 2023-08-28T02:11:10.496222+0000 mon.a (mon.0) 645 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:11.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:11 smithi143 conmon[109923]: cephadm 2023-08-28T02:11:10.498976+0000 mgr.y (mgr.14424) 48 : cephadm 2023-08-28T02:11:11.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:11 smithi143 conmon[109923]: [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-08-28T02:11:11.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:11 smithi143 conmon[109923]: cephadm 2023-08-28T02:11:10.501944+0000 mgr.y (mgr.14424) 49 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi143 2023-08-28T02:11:11.943 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:11 smithi169 conmon[131988]: t=2023-08-28T02:11:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_org_id_role" 2023-08-28T02:11:11.943 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:11 smithi169 conmon[131988]: t=2023-08-28T02:11:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_permission" 2023-08-28T02:11:11.943 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:11 smithi169 conmon[131988]: t=2023-08-28T02:11:11+0000 lvl=info msg="Executing migration" logger=migrator id="save default acl rules in dashboard_acl table" 2023-08-28T02:11:11.944 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:11 smithi169 conmon[131988]: t=2023-08-28T02:11:11+0000 lvl=info msg="Executing migration" logger=migrator id="delete acl rules for deleted dashboards and folders" 2023-08-28T02:11:11.944 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:11 smithi169 conmon[131988]: t=2023-08-28T02:11:11+0000 lvl=info msg="Executing migration" logger=migrator id="create tag table" 2023-08-28T02:11:11.944 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:11 smithi169 conmon[131988]: t=2023-08-28T02:11:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index tag.key_value" 2023-08-28T02:11:11.944 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:11 smithi169 conmon[131988]: t=2023-08-28T02:11:11+0000 lvl=info msg="Executing migration" logger=migrator id="create login attempt table" 2023-08-28T02:11:12.026 INFO:teuthology.orchestra.run.smithi143.stdout:{"pg_ready":true,"pg_map":{"version":37,"stamp":"2023-08-28T02:11:09.871373+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2380980,"kb_used_data":4212,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747580236,"statfs":{"total":767960285184,"available":765522161664,"internally_reserved":0,"allocated":4313088,"data_stored":1763175,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-08-28T02:10:33.840224+0000","last_change":"2023-08-28T02:10:32.184257+0000","last_active":"2023-08-28T02:10:33.840224+0000","last_peered":"2023-08-28T02:10:33.840224+0000","last_clean":"2023-08-28T02:10:33.840224+0000","last_became_active":"2023-08-28T02:10:32.183944+0000","last_became_peered":"2023-08-28T02:10:32.183944+0000","last_unstale":"2023-08-28T02:10:33.840224+0000","last_undegraded":"2023-08-28T02:10:33.840224+0000","last_fullsized":"2023-08-28T02:10:33.840224+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-08-28T02:10:33.839984+0000","last_change":"2023-08-28T02:10:32.197242+0000","last_active":"2023-08-28T02:10:33.839984+0000","last_peered":"2023-08-28T02:10:33.839984+0000","last_clean":"2023-08-28T02:10:33.839984+0000","last_became_active":"2023-08-28T02:10:32.196865+0000","last_became_peered":"2023-08-28T02:10:32.196865+0000","last_unstale":"2023-08-28T02:10:33.839984+0000","last_undegraded":"2023-08-28T02:10:33.839984+0000","last_fullsized":"2023-08-28T02:10:33.839984+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-08-28T02:10:33.840521+0000","last_change":"2023-08-28T02:10:32.198340+0000","last_active":"2023-08-28T02:10:33.840521+0000","last_peered":"2023-08-28T02:10:33.840521+0000","last_clean":"2023-08-28T02:10:33.840521+0000","last_became_active":"2023-08-28T02:10:32.198052+0000","last_became_peered":"2023-08-28T02:10:32.198052+0000","last_unstale":"2023-08-28T02:10:33.840521+0000","last_undegraded":"2023-08-28T02:10:33.840521+0000","last_fullsized":"2023-08-28T02:10:33.840521+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-08-28T02:10:33.840439+0000","last_change":"2023-08-28T02:10:32.198506+0000","last_active":"2023-08-28T02:10:33.840439+0000","last_peered":"2023-08-28T02:10:33.840439+0000","last_clean":"2023-08-28T02:10:33.840439+0000","last_became_active":"2023-08-28T02:10:32.198175+0000","last_became_peered":"2023-08-28T02:10:32.198175+0000","last_unstale":"2023-08-28T02:10:33.840439+0000","last_undegraded":"2023-08-28T02:10:33.840439+0000","last_fullsized":"2023-08-28T02:10:33.840439+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":15,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-08-28T02:10:33.840473+0000","last_change":"2023-08-28T02:09:55.609189+0000","last_active":"2023-08-28T02:10:33.840473+0000","last_peered":"2023-08-28T02:10:33.840473+0000","last_clean":"2023-08-28T02:10:33.840473+0000","last_became_active":"2023-08-28T02:09:55.608705+0000","last_became_peered":"2023-08-28T02:09:55.608705+0000","last_unstale":"2023-08-28T02:10:33.840473+0000","last_undegraded":"2023-08-28T02:10:33.840473+0000","last_fullsized":"2023-08-28T02:10:33.840473+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_clean_scrub_stamp":"2023-08-28T02:08:02.043718+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-08-28T02:10:33.840200+0000","last_change":"2023-08-28T02:10:32.184422+0000","last_active":"2023-08-28T02:10:33.840200+0000","last_peered":"2023-08-28T02:10:33.840200+0000","last_clean":"2023-08-28T02:10:33.840200+0000","last_became_active":"2023-08-28T02:10:32.184072+0000","last_became_peered":"2023-08-28T02:10:32.184072+0000","last_unstale":"2023-08-28T02:10:33.840200+0000","last_undegraded":"2023-08-28T02:10:33.840200+0000","last_fullsized":"2023-08-28T02:10:33.840200+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-08-28T02:10:33.180854+0000","last_change":"2023-08-28T02:10:32.197650+0000","last_active":"2023-08-28T02:10:33.180854+0000","last_peered":"2023-08-28T02:10:33.180854+0000","last_clean":"2023-08-28T02:10:33.180854+0000","last_became_active":"2023-08-28T02:10:32.197319+0000","last_became_peered":"2023-08-28T02:10:32.197319+0000","last_unstale":"2023-08-28T02:10:33.180854+0000","last_undegraded":"2023-08-28T02:10:33.180854+0000","last_fullsized":"2023-08-28T02:10:33.180854+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-08-28T02:10:33.840200+0000","last_change":"2023-08-28T02:10:32.184125+0000","last_active":"2023-08-28T02:10:33.840200+0000","last_peered":"2023-08-28T02:10:33.840200+0000","last_clean":"2023-08-28T02:10:33.840200+0000","last_became_active":"2023-08-28T02:10:32.183865+0000","last_became_peered":"2023-08-28T02:10:32.183865+0000","last_unstale":"2023-08-28T02:10:33.840200+0000","last_undegraded":"2023-08-28T02:10:33.840200+0000","last_fullsized":"2023-08-28T02:10:33.840200+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-08-28T02:10:33.839994+0000","last_change":"2023-08-28T02:10:32.197552+0000","last_active":"2023-08-28T02:10:33.839994+0000","last_peered":"2023-08-28T02:10:33.839994+0000","last_clean":"2023-08-28T02:10:33.839994+0000","last_became_active":"2023-08-28T02:10:32.197251+0000","last_became_peered":"2023-08-28T02:10:32.197251+0000","last_unstale":"2023-08-28T02:10:33.839994+0000","last_undegraded":"2023-08-28T02:10:33.839994+0000","last_fullsized":"2023-08-28T02:10:33.839994+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430225,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:10:54 2023","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.55900000000000005}]},{"osd":1,"last update":"Mon Aug 28 02:10:54 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.45800000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:10:54 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.60299999999999998}]},{"osd":3,"last update":"Mon Aug 28 02:10:54 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.58099999999999996}]},{"osd":4,"last update":"Mon Aug 28 02:10:54 2023","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.38},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.53600000000000003}]},{"osd":5,"last update":"Mon Aug 28 02:10:54 2023","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.34899999999999998}]},{"osd":6,"last update":"Mon Aug 28 02:10:54 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.43099999999999999}]}]},{"osd":6,"up_from":42,"seq":180388626452,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297620,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447532,"statfs":{"total":95995035648,"available":95690272768,"internally_reserved":0,"allocated":536576,"data_stored":220200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:10:30 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.38300000000000001}]},{"osd":1,"last update":"Mon Aug 28 02:10:30 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.501},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.51000000000000001}]},{"osd":2,"last update":"Mon Aug 28 02:10:30 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.41999999999999998}]},{"osd":3,"last update":"Mon Aug 28 02:10:30 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.48199999999999998}]},{"osd":4,"last update":"Mon Aug 28 02:10:30 2023","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.624,"5min":0.624,"15min":0.624},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.439}]},{"osd":5,"last update":"Mon Aug 28 02:10:30 2023","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.46800000000000003}]},{"osd":7,"last update":"Mon Aug 28 02:10:55 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.432},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.56599999999999995}]}]},{"osd":1,"up_from":13,"seq":55834574892,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:10:46 2023","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":0.34899999999999998,"15min":0.33300000000000002},"min":{"1min":0.26500000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"last":0.38},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.36799999999999999,"15min":0.34699999999999998},"min":{"1min":0.246,"5min":0.215,"15min":0.215},"max":{"1min":0.54100000000000004,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.36399999999999999}]},{"osd":2,"last update":"Mon Aug 28 02:10:04 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.375,"15min":0.36599999999999999},"min":{"1min":0.35699999999999998,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.55600000000000005,"5min":0.623,"15min":0.623},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.38700000000000001,"15min":0.38},"min":{"1min":0.32100000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.33200000000000002}]},{"osd":3,"last update":"Mon Aug 28 02:10:29 2023","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":0.436,"15min":0.437},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.56200000000000006,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.46800000000000003,"5min":0.47999999999999998,"15min":0.48199999999999998},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.41999999999999998}]},{"osd":4,"last update":"Mon Aug 28 02:10:46 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.55700000000000005,"15min":0.56000000000000005},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56899999999999995,"15min":0.56999999999999995},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.499}]},{"osd":5,"last update":"Mon Aug 28 02:10:08 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.42899999999999999}]},{"osd":6,"last update":"Mon Aug 28 02:10:34 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.45900000000000002}]},{"osd":7,"last update":"Mon Aug 28 02:10:53 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":0.94199999999999995,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.55100000000000005}]}]},{"osd":0,"up_from":8,"seq":34359738417,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Aug 28 02:10:49 2023","interfaces":[{"interface":"back","average":{"1min":0.34799999999999998,"5min":0.34899999999999998,"15min":0.34200000000000003},"min":{"1min":0.24299999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.35599999999999998,"5min":0.36099999999999999,"15min":0.34699999999999998},"min":{"1min":0.23400000000000001,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.55900000000000005,"5min":0.58499999999999996,"15min":0.58499999999999996},"last":0.29299999999999998}]},{"osd":2,"last update":"Mon Aug 28 02:11:04 2023","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.38400000000000001,"15min":0.38200000000000001},"min":{"1min":0.27100000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.371},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.41899999999999998,"15min":0.41299999999999998},"min":{"1min":0.28299999999999997,"5min":0.248,"15min":0.248},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.47799999999999998}]},{"osd":3,"last update":"Mon Aug 28 02:10:25 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.46100000000000002,"15min":0.46400000000000002},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":5.4509999999999996},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.499,"15min":0.505},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.60399999999999998,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.41499999999999998}]},{"osd":4,"last update":"Mon Aug 28 02:10:49 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.54900000000000004,"15min":0.54800000000000004},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58799999999999997,"15min":0.58899999999999997},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.71899999999999997,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.54800000000000004}]},{"osd":5,"last update":"Mon Aug 28 02:10:13 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.51700000000000002}]},{"osd":6,"last update":"Mon Aug 28 02:10:30 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.60899999999999999}]},{"osd":7,"last update":"Mon Aug 28 02:10:54 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.56699999999999995}]}]},{"osd":2,"up_from":18,"seq":77309411367,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:10:02 2023","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":0.36099999999999999,"15min":0.35899999999999999},"min":{"1min":0.22,"5min":0.216,"15min":0.216},"max":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.373,"5min":0.38,"15min":0.38100000000000001},"min":{"1min":0.255,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":0.47899999999999998,"5min":0.53500000000000003,"15min":0.53500000000000003},"last":0.33400000000000002}]},{"osd":1,"last update":"Mon Aug 28 02:10:02 2023","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.38100000000000001,"15min":0.379},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.5,"5min":0.56999999999999995,"15min":0.56999999999999995},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.38900000000000001,"15min":0.38500000000000001},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.58799999999999997,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.378}]},{"osd":3,"last update":"Mon Aug 28 02:10:26 2023","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.44900000000000001,"15min":0.45500000000000002},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.54200000000000004,"5min":0.55800000000000005,"15min":0.55800000000000005},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.47799999999999998,"15min":0.48799999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.51700000000000002,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.39600000000000002}]},{"osd":4,"last update":"Mon Aug 28 02:10:47 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54900000000000004,"15min":0.55100000000000005},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.5},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.56999999999999995,"15min":0.57299999999999995},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.73799999999999999,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.52100000000000002}]},{"osd":5,"last update":"Mon Aug 28 02:10:10 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.53000000000000003}]},{"osd":6,"last update":"Mon Aug 28 02:10:32 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.625,"5min":0.625,"15min":0.625},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.496}]},{"osd":7,"last update":"Mon Aug 28 02:10:54 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.55700000000000005}]}]},{"osd":3,"up_from":25,"seq":107374182436,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:10:28 2023","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":0.39400000000000002,"15min":0.39800000000000002},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.41999999999999998,"15min":0.42399999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.32200000000000001}]},{"osd":1,"last update":"Mon Aug 28 02:10:28 2023","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.42899999999999999,"15min":0.43099999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.27600000000000002},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.433,"15min":0.435},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.70399999999999996,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.33800000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:10:28 2023","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.46700000000000003,"15min":0.46999999999999997},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.44600000000000001,"15min":0.45100000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.35199999999999998}]},{"osd":4,"last update":"Mon Aug 28 02:10:48 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.56799999999999995,"15min":0.57299999999999995},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.59199999999999997,"15min":0.59599999999999997},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.79000000000000004}]},{"osd":5,"last update":"Mon Aug 28 02:10:11 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.70699999999999996},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.54800000000000004}]},{"osd":6,"last update":"Mon Aug 28 02:10:33 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.878,"5min":0.878,"15min":0.878},"last":0.73599999999999999},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.52800000000000002}]},{"osd":7,"last update":"Mon Aug 28 02:10:54 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":1.4179999999999999,"5min":1.4179999999999999,"15min":1.4179999999999999},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.77000000000000002}]}]},{"osd":4,"up_from":30,"seq":128849018910,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:10:48 2023","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.54200000000000004,"15min":0.54800000000000004},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.66400000000000003,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.55100000000000005,"15min":0.56100000000000005},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.64900000000000002,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.48799999999999999}]},{"osd":1,"last update":"Mon Aug 28 02:10:48 2023","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.52900000000000003,"15min":0.53500000000000003},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.63700000000000001,"5min":0.98899999999999999,"15min":0.98899999999999999},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.53900000000000003,"15min":0.54100000000000004},"min":{"1min":0.42199999999999999,"5min":0.374,"15min":0.374},"max":{"1min":0.77000000000000002,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.50900000000000001}]},{"osd":2,"last update":"Mon Aug 28 02:10:48 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.55700000000000005,"15min":0.56599999999999995},"min":{"1min":0.35099999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.73599999999999999,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.437},{"interface":"front","average":{"1min":0.498,"5min":0.54600000000000004,"15min":0.55400000000000005},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.58299999999999996,"5min":0.873,"15min":0.873},"last":0.45800000000000002}]},{"osd":3,"last update":"Mon Aug 28 02:10:48 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.54900000000000004,"15min":0.55500000000000005},"min":{"1min":0.375,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.54600000000000004,"15min":0.54700000000000004},"min":{"1min":0.45000000000000001,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.72899999999999998,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.37}]},{"osd":5,"last update":"Mon Aug 28 02:10:10 2023","interfaces":[{"interface":"back","average":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"last":0.40400000000000003}]},{"osd":6,"last update":"Mon Aug 28 02:10:30 2023","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":0.434,"15min":0.434},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.46899999999999997}]},{"osd":7,"last update":"Mon Aug 28 02:10:57 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.2110000000000001,"5min":1.2110000000000001,"15min":1.2110000000000001},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.44900000000000001}]}]},{"osd":5,"up_from":36,"seq":154618822682,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297616,"kb_used_data":520,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447536,"statfs":{"total":95995035648,"available":95690276864,"internally_reserved":0,"allocated":532480,"data_stored":219885,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:10:10 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.41999999999999998}]},{"osd":1,"last update":"Mon Aug 28 02:10:10 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.46200000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:10:10 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.438},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.39400000000000002}]},{"osd":3,"last update":"Mon Aug 28 02:10:10 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.41499999999999998}]},{"osd":4,"last update":"Mon Aug 28 02:10:10 2023","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.438,"5min":0.438,"15min":0.438},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.30499999999999999}]},{"osd":6,"last update":"Mon Aug 28 02:10:30 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.40400000000000003}]},{"osd":7,"last update":"Mon Aug 28 02:10:53 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.33500000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-08-28T02:11:12.029 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph pg dump --format=json 2023-08-28T02:11:12.220 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:11 smithi169 conmon[131988]: t=2023-08-28T02:11:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index login_attempt.username" 2023-08-28T02:11:12.220 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:12 smithi169 conmon[131988]: t=2023-08-28T02:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_login_attempt_username - v1" 2023-08-28T02:11:12.220 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:12 smithi169 conmon[131988]: t=2023-08-28T02:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2023-08-28T02:11:12.220 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:12 smithi169 conmon[131988]: t=2023-08-28T02:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create login_attempt v2" 2023-08-28T02:11:12.220 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:12 smithi169 conmon[131988]: t=2023-08-28T02:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_login_attempt_username - v2" 2023-08-28T02:11:12.220 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:12 smithi169 conmon[131988]: t=2023-08-28T02:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="copy login_attempt v1 to v2" 2023-08-28T02:11:12.220 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:12 smithi169 conmon[131988]: t=2023-08-28T02:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="drop login_attempt_tmp_qwerty" 2023-08-28T02:11:12.487 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:12 smithi169 conmon[131988]: t=2023-08-28T02:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth table" 2023-08-28T02:11:12.487 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:12 smithi169 conmon[131988]: t=2023-08-28T02:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_user_auth_auth_module_auth_id - v1" 2023-08-28T02:11:12.487 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:12 smithi169 conmon[131988]: t=2023-08-28T02:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="alter user_auth.auth_id to length 190" 2023-08-28T02:11:12.487 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:12 smithi169 conmon[131988]: t=2023-08-28T02:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth access token to user_auth" 2023-08-28T02:11:12.487 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:12 smithi169 conmon[131988]: t=2023-08-28T02:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth refresh token to user_auth" 2023-08-28T02:11:12.487 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:12 smithi169 conmon[131988]: t=2023-08-28T02:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth token type to user_auth" 2023-08-28T02:11:12.488 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:12 smithi169 conmon[131988]: t=2023-08-28T02:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth expiry to user_auth" 2023-08-28T02:11:12.488 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:12 smithi169 conmon[131988]: t=2023-08-28T02:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add index to user_id column in user_auth" 2023-08-28T02:11:12.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:12 smithi143 conmon[100258]: audit 2023-08-28T02:11:11.043215+0000 mgr.y (mgr. 2023-08-28T02:11:12.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:12 smithi143 conmon[100258]: 14424) 50 : audit [DBG] from='client.24509 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:11:12.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:12 smithi143 conmon[109923]: audit 2023-08-28T02:11:11.043215+0000 2023-08-28T02:11:12.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:12 smithi143 conmon[109923]: mgr.y (mgr.14424) 50 : audit [DBG] from='client.24509 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:11:12.757 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:12 smithi169 conmon[100500]: audit 2023-08-28T02:11:11.043215+0000 mgr.y (mgr. 2023-08-28T02:11:12.757 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:12 smithi169 conmon[100500]: 14424) 50 : audit [DBG] from='client.24509 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:11:12.757 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:12 smithi169 conmon[131988]: t=2023-08-28T02:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create server_lock table" 2023-08-28T02:11:12.757 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:12 smithi169 conmon[131988]: t=2023-08-28T02:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="add index server_lock.operation_uid" 2023-08-28T02:11:12.757 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:12 smithi169 conmon[131988]: t=2023-08-28T02:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth token table" 2023-08-28T02:11:12.757 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:12 smithi169 conmon[131988]: t=2023-08-28T02:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.auth_token" 2023-08-28T02:11:12.758 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:12 smithi169 conmon[131988]: t=2023-08-28T02:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.prev_auth_token" 2023-08-28T02:11:12.758 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:12 smithi169 conmon[131988]: t=2023-08-28T02:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_auth_token.user_id" 2023-08-28T02:11:12.758 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:12 smithi169 conmon[131988]: t=2023-08-28T02:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add revoked_at to the user auth token" 2023-08-28T02:11:12.758 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:12 smithi169 conmon[131988]: t=2023-08-28T02:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create cache_data table" 2023-08-28T02:11:13.017 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:11:12 smithi169 conmon[102807]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:11:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-28T02:11:13.017 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:12 smithi169 conmon[131988]: t=2023-08-28T02:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index cache_data.cache_key" 2023-08-28T02:11:13.018 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:12 smithi169 conmon[131988]: t=2023-08-28T02:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="create short_url table v1" 2023-08-28T02:11:13.018 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:12 smithi169 conmon[131988]: t=2023-08-28T02:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="add index short_url.org_id-uid" 2023-08-28T02:11:13.018 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:12 smithi169 conmon[131988]: t=2023-08-28T02:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition table" 2023-08-28T02:11:13.018 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:12 smithi169 conmon[131988]: t=2023-08-28T02:11:12+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition table" 2023-08-28T02:11:13.018 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:13 smithi169 conmon[131988]: t=2023-08-28T02:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and title columns" 2023-08-28T02:11:13.021 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:11:13.269 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:13 smithi169 conmon[131988]: t=2023-08-28T02:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and uid columns" 2023-08-28T02:11:13.269 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:13 smithi169 conmon[131988]: t=2023-08-28T02:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition table data column to mediumtext in mysql" 2023-08-28T02:11:13.269 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:13 smithi169 conmon[131988]: t=2023-08-28T02:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and title columns" 2023-08-28T02:11:13.269 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:13 smithi169 conmon[131988]: t=2023-08-28T02:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and uid columns" 2023-08-28T02:11:13.269 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:13 smithi169 conmon[131988]: t=2023-08-28T02:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and title columns" 2023-08-28T02:11:13.270 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:13 smithi169 conmon[131988]: t=2023-08-28T02:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and uid columns" 2023-08-28T02:11:13.270 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:13 smithi169 conmon[131988]: t=2023-08-28T02:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="Add column paused in alert_definition" 2023-08-28T02:11:13.270 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:13 smithi169 conmon[131988]: t=2023-08-28T02:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition table" 2023-08-28T02:11:13.579 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:13 smithi169 conmon[100500]: cluster 2023-08-28T02:11:11.871959+0000 mgr.y (mgr.14424) 51 2023-08-28T02:11:13.579 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:13 smithi169 conmon[100500]: : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:13.579 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:13 smithi169 conmon[131988]: t=2023-08-28T02:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition_version table" 2023-08-28T02:11:13.580 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:13 smithi169 conmon[131988]: t=2023-08-28T02:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition_version table" 2023-08-28T02:11:13.580 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:13 smithi169 conmon[131988]: t=2023-08-28T02:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_id and version columns" 2023-08-28T02:11:13.580 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:13 smithi169 conmon[131988]: t=2023-08-28T02:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_uid and version columns" 2023-08-28T02:11:13.580 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:13 smithi169 conmon[131988]: t=2023-08-28T02:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition_version table data column to mediumtext in mysql" 2023-08-28T02:11:13.580 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:13 smithi169 conmon[131988]: t=2023-08-28T02:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition_version table" 2023-08-28T02:11:13.585 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:13 smithi143 conmon[100258]: cluster 2023-08-28T02:11:11.871959+0000 mgr.y (mgr.14424 2023-08-28T02:11:13.585 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:13 smithi143 conmon[100258]: ) 51 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:13.585 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:13 smithi143 conmon[109923]: cluster 2023-08-28T02:11:11.871959 2023-08-28T02:11:13.585 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:13 smithi143 conmon[109923]: +0000 mgr.y (mgr.14424) 51 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:13.586 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:11:13 smithi143 systemd[1]: Stopping Ceph alertmanager.a for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:11:13.586 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:11:13 smithi143 bash[149837]: Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-alertmanager.a found: no such container 2023-08-28T02:11:13.856 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:13 smithi169 conmon[131988]: t=2023-08-28T02:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_instance table" 2023-08-28T02:11:13.856 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:13 smithi169 conmon[131988]: t=2023-08-28T02:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2023-08-28T02:11:13.856 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:13 smithi169 conmon[131988]: t=2023-08-28T02:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, current_state columns" 2023-08-28T02:11:13.856 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:13 smithi169 conmon[131988]: t=2023-08-28T02:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add column current_state_end to alert_instance" 2023-08-28T02:11:13.856 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:13 smithi169 conmon[131988]: t=2023-08-28T02:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, def_uid, current_state on alert_instance" 2023-08-28T02:11:13.856 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:13 smithi169 conmon[131988]: t=2023-08-28T02:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, current_state on alert_instance" 2023-08-28T02:11:14.125 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:13 smithi169 conmon[131988]: t=2023-08-28T02:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_org_id to rule_org_id in alert_instance" 2023-08-28T02:11:14.125 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:13 smithi169 conmon[131988]: t=2023-08-28T02:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_uid to rule_uid in alert_instance" 2023-08-28T02:11:14.125 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:13 smithi169 conmon[131988]: t=2023-08-28T02:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, rule_uid, current_state on alert_instance" 2023-08-28T02:11:14.125 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:13 smithi169 conmon[131988]: t=2023-08-28T02:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, current_state on alert_instance" 2023-08-28T02:11:14.125 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:13 smithi169 conmon[131988]: t=2023-08-28T02:11:13+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule table" 2023-08-28T02:11:14.126 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:14 smithi169 conmon[131988]: t=2023-08-28T02:11:14+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and title columns" 2023-08-28T02:11:14.126 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:14 smithi169 conmon[131988]: t=2023-08-28T02:11:14+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and uid columns" 2023-08-28T02:11:14.126 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:14 smithi169 conmon[131988]: t=2023-08-28T02:11:14+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2023-08-28T02:11:14.126 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:14 smithi169 conmon[131988]: t=2023-08-28T02:11:14+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule table data column to mediumtext in mysql" 2023-08-28T02:11:14.126 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:14 smithi169 conmon[131988]: t=2023-08-28T02:11:14+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule" 2023-08-28T02:11:14.200 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:11:13 smithi143 conmon[141410]: level=info ts=2023-08-28T02:11:13.950Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2023-08-28T02:11:14.394 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:14 smithi169 conmon[131988]: t=2023-08-28T02:11:14+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule" 2023-08-28T02:11:14.394 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:14 smithi169 conmon[131988]: t=2023-08-28T02:11:14+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule" 2023-08-28T02:11:14.394 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:14 smithi169 conmon[131988]: t=2023-08-28T02:11:14+0000 lvl=info msg="Executing migration" logger=migrator id="remove unique index from alert_rule on org_id, title columns" 2023-08-28T02:11:14.394 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:14 smithi169 conmon[131988]: t=2023-08-28T02:11:14+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespase_uid and title columns" 2023-08-28T02:11:14.394 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:14 smithi169 conmon[131988]: t=2023-08-28T02:11:14+0000 lvl=info msg="Executing migration" logger=migrator id="add dashboard_uid column to alert_rule" 2023-08-28T02:11:14.394 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:14 smithi169 conmon[131988]: t=2023-08-28T02:11:14+0000 lvl=info msg="Executing migration" logger=migrator id="add panel_id column to alert_rule" 2023-08-28T02:11:14.394 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:14 smithi169 conmon[131988]: t=2023-08-28T02:11:14+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2023-08-28T02:11:14.394 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:14 smithi169 conmon[131988]: t=2023-08-28T02:11:14+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule_version table" 2023-08-28T02:11:14.668 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:11:14 smithi143 bash[149970]: f4162ae1de812d5b03a84d1911f2645d3a84369659a0ac5d3be1b60c03c840bf 2023-08-28T02:11:14.679 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:14 smithi169 conmon[131988]: t=2023-08-28T02:11:14+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2023-08-28T02:11:14.679 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:14 smithi169 conmon[131988]: t=2023-08-28T02:11:14+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2023-08-28T02:11:14.679 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:14 smithi169 conmon[131988]: t=2023-08-28T02:11:14+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule_version table data column to mediumtext in mysql" 2023-08-28T02:11:14.679 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:14 smithi169 conmon[131988]: t=2023-08-28T02:11:14+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule_version" 2023-08-28T02:11:14.679 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:14 smithi169 conmon[131988]: t=2023-08-28T02:11:14+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule_version" 2023-08-28T02:11:14.679 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:14 smithi169 conmon[131988]: t=2023-08-28T02:11:14+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule_version" 2023-08-28T02:11:14.679 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:14 smithi169 conmon[131988]: t=2023-08-28T02:11:14+0000 lvl=info msg="Executing migration" logger=migrator id=create_alert_configuration_table 2023-08-28T02:11:14.680 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:14 smithi169 conmon[131988]: t=2023-08-28T02:11:14+0000 lvl=info msg="Executing migration" logger=migrator id="Add column default in alert_configuration" 2023-08-28T02:11:14.680 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:14 smithi169 conmon[131988]: t=2023-08-28T02:11:14+0000 lvl=info msg="Executing migration" logger=migrator id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2023-08-28T02:11:14.680 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:14 smithi169 conmon[131988]: t=2023-08-28T02:11:14+0000 lvl=info msg="Executing migration" logger=migrator id="add column org_id in alert_configuration" 2023-08-28T02:11:14.932 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:11:14 smithi143 systemd[1]: ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@alertmanager.a.service: Succeeded. 2023-08-28T02:11:14.933 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:11:14 smithi143 systemd[1]: Stopped Ceph alertmanager.a for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:11:14.933 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:11:14 smithi143 systemd[1]: Starting Ceph alertmanager.a for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:11:14.939 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:14 smithi169 conmon[131988]: t=2023-08-28T02:11:14+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_configuration table on org_id column" 2023-08-28T02:11:14.939 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:14 smithi169 conmon[131988]: t=2023-08-28T02:11:14+0000 lvl=info msg="Executing migration" logger=migrator id=create_ngalert_configuration_table 2023-08-28T02:11:14.939 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:14 smithi169 conmon[131988]: t=2023-08-28T02:11:14+0000 lvl=info msg="Executing migration" logger=migrator id="add index in ngalert_configuration on org_id column" 2023-08-28T02:11:14.940 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:14 smithi169 conmon[131988]: t=2023-08-28T02:11:14+0000 lvl=info msg="Executing migration" logger=migrator id="clear migration entry \"remove unified alerting data\"" 2023-08-28T02:11:14.940 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:14 smithi169 conmon[131988]: t=2023-08-28T02:11:14+0000 lvl=info msg="Executing migration" logger=migrator id="move dashboard alerts to unified alerting" 2023-08-28T02:11:14.940 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:14 smithi169 conmon[131988]: t=2023-08-28T02:11:14+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element table v1" 2023-08-28T02:11:14.940 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:14 smithi169 conmon[131988]: t=2023-08-28T02:11:14+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element org_id-folder_id-name-kind" 2023-08-28T02:11:14.940 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:14 smithi169 conmon[131988]: t=2023-08-28T02:11:14+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element_connection table v1" 2023-08-28T02:11:15.208 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:14 smithi169 conmon[131988]: t=2023-08-28T02:11:14+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element_connection element_id-kind-connection_id" 2023-08-28T02:11:15.208 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:15 smithi169 conmon[131988]: t=2023-08-28T02:11:15+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index library_element org_id_uid" 2023-08-28T02:11:15.208 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:15 smithi169 conmon[131988]: t=2023-08-28T02:11:15+0000 lvl=info msg="Executing migration" logger=migrator id="clone move dashboard alerts to unified alerting" 2023-08-28T02:11:15.208 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:15 smithi169 conmon[131988]: t=2023-08-28T02:11:15+0000 lvl=info msg="Executing migration" logger=migrator id="create data_keys table" 2023-08-28T02:11:15.208 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:15 smithi169 conmon[131988]: t=2023-08-28T02:11:15+0000 lvl=info msg="Executing migration" logger=migrator id="create kv_store table v1" 2023-08-28T02:11:15.208 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:15 smithi169 conmon[131988]: t=2023-08-28T02:11:15+0000 lvl=info msg="Executing migration" logger=migrator id="add index kv_store.org_id-namespace-key" 2023-08-28T02:11:15.208 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:15 smithi169 conmon[131988]: t=2023-08-28T02:11:15+0000 lvl=info msg="Executing migration" logger=migrator id="update dashboard_uid and panel_id from existing annotations" 2023-08-28T02:11:15.281 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:11:15.283 INFO:teuthology.orchestra.run.smithi143.stderr:dumped all 2023-08-28T02:11:15.477 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:15 smithi169 conmon[131988]: t=2023-08-28T02:11:15+0000 lvl=info msg="Executing migration" logger=migrator id="create permission table" 2023-08-28T02:11:15.477 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:15 smithi169 conmon[131988]: t=2023-08-28T02:11:15+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index permission.role_id" 2023-08-28T02:11:15.477 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:15 smithi169 conmon[131988]: t=2023-08-28T02:11:15+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_id_action_scope" 2023-08-28T02:11:15.477 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:15 smithi169 conmon[131988]: t=2023-08-28T02:11:15+0000 lvl=info msg="Executing migration" logger=migrator id="create role table" 2023-08-28T02:11:15.477 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:15 smithi169 conmon[131988]: t=2023-08-28T02:11:15+0000 lvl=info msg="Executing migration" logger=migrator id="add column display_name" 2023-08-28T02:11:15.477 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:15 smithi169 conmon[131988]: t=2023-08-28T02:11:15+0000 lvl=info msg="Executing migration" logger=migrator id="add column group_name" 2023-08-28T02:11:15.478 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:15 smithi169 conmon[131988]: t=2023-08-28T02:11:15+0000 lvl=info msg="Executing migration" logger=migrator id="add index role.org_id" 2023-08-28T02:11:15.478 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:15 smithi169 conmon[131988]: t=2023-08-28T02:11:15+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_org_id_name" 2023-08-28T02:11:15.478 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:15 smithi169 conmon[131988]: t=2023-08-28T02:11:15+0000 lvl=info msg="Executing migration" logger=migrator id="add index role_org_id_uid" 2023-08-28T02:11:15.559 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:15 smithi143 conmon[100258]: cluster 2023-08-28T02:11:13.872266+0000 2023-08-28T02:11:15.560 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:15 smithi143 conmon[100258]: mgr.y (mgr.14424) 52 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:15.560 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:15 smithi143 conmon[109923]: cluster 2023-08-28T02:11:13. 2023-08-28T02:11:15.560 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:15 smithi143 conmon[109923]: 872266+0000 mgr.y (mgr.14424) 52 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:15.560 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:11:15 smithi143 conmon[150510]: level=info ts=2023-08-28T02:11:15.543Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2023-08-28T02:11:15.560 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:11:15 smithi143 conmon[150510]: level=info ts=2023-08-28T02:11:15.543Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2023-08-28T02:11:15.560 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:11:15 smithi143 conmon[150510]: level=info ts=2023-08-28T02:11:15.544Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.143 port=9094 2023-08-28T02:11:15.561 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:11:15 smithi143 conmon[150510]: level=info ts=2023-08-28T02:11:15.545Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-08-28T02:11:15.561 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:11:15 smithi143 bash[150452]: 24d74df782bd39fafc2f94bfeca17e010e4db5855995ca0feefbd734d5a21798 2023-08-28T02:11:15.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:15 smithi169 conmon[100500]: cluster 2023-08-28T02:11:13.872266+0000 mgr.y (mgr.14424) 52 : cluster 2023-08-28T02:11:15.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:15 smithi169 conmon[100500]: [DBG] pgmap v39: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:15.737 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:15 smithi169 conmon[131988]: t=2023-08-28T02:11:15+0000 lvl=info msg="Executing migration" logger=migrator id="create team role table" 2023-08-28T02:11:15.737 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:15 smithi169 conmon[131988]: t=2023-08-28T02:11:15+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.org_id" 2023-08-28T02:11:15.737 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:15 smithi169 conmon[131988]: t=2023-08-28T02:11:15+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_role_org_id_team_id_role_id" 2023-08-28T02:11:15.737 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:15 smithi169 conmon[131988]: t=2023-08-28T02:11:15+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.team_id" 2023-08-28T02:11:15.738 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:15 smithi169 conmon[131988]: t=2023-08-28T02:11:15+0000 lvl=info msg="Executing migration" logger=migrator id="create user role table" 2023-08-28T02:11:15.738 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:15 smithi169 conmon[131988]: t=2023-08-28T02:11:15+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.org_id" 2023-08-28T02:11:15.738 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:15 smithi169 conmon[131988]: t=2023-08-28T02:11:15+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_role_org_id_user_id_role_id" 2023-08-28T02:11:15.950 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:11:15 smithi143 systemd[1]: Started Ceph alertmanager.a for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:11:15.950 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:11:15 smithi143 conmon[150510]: level=info ts=2023-08-28T02:11:15.572Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-08-28T02:11:15.950 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:11:15 smithi143 conmon[150510]: level=info ts=2023-08-28T02:11:15.572Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-08-28T02:11:15.950 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:11:15 smithi143 conmon[150510]: level=info ts=2023-08-28T02:11:15.574Z caller=main.go:518 msg=Listening address=:9093 2023-08-28T02:11:15.950 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:11:15 smithi143 conmon[150510]: level=info ts=2023-08-28T02:11:15.574Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2023-08-28T02:11:16.023 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:15 smithi169 conmon[131988]: t=2023-08-28T02:11:15+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.user_id" 2023-08-28T02:11:16.023 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:15 smithi169 conmon[131988]: t=2023-08-28T02:11:15+0000 lvl=info msg="Executing migration" logger=migrator id="create builtin role table" 2023-08-28T02:11:16.024 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:15 smithi169 conmon[131988]: t=2023-08-28T02:11:15+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.role_id" 2023-08-28T02:11:16.024 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:15 smithi169 conmon[131988]: t=2023-08-28T02:11:15+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.name" 2023-08-28T02:11:16.024 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:15 smithi169 conmon[131988]: t=2023-08-28T02:11:15+0000 lvl=info msg="Executing migration" logger=migrator id="Add column org_id to builtin_role table" 2023-08-28T02:11:16.024 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:16 smithi169 conmon[131988]: t=2023-08-28T02:11:16+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.org_id" 2023-08-28T02:11:16.067 INFO:teuthology.orchestra.run.smithi143.stdout:{"pg_ready":true,"pg_map":{"version":39,"stamp":"2023-08-28T02:11:13.872143+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2380980,"kb_used_data":4212,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747580236,"statfs":{"total":767960285184,"available":765522161664,"internally_reserved":0,"allocated":4313088,"data_stored":1763175,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-08-28T02:10:33.840224+0000","last_change":"2023-08-28T02:10:32.184257+0000","last_active":"2023-08-28T02:10:33.840224+0000","last_peered":"2023-08-28T02:10:33.840224+0000","last_clean":"2023-08-28T02:10:33.840224+0000","last_became_active":"2023-08-28T02:10:32.183944+0000","last_became_peered":"2023-08-28T02:10:32.183944+0000","last_unstale":"2023-08-28T02:10:33.840224+0000","last_undegraded":"2023-08-28T02:10:33.840224+0000","last_fullsized":"2023-08-28T02:10:33.840224+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-08-28T02:10:33.839984+0000","last_change":"2023-08-28T02:10:32.197242+0000","last_active":"2023-08-28T02:10:33.839984+0000","last_peered":"2023-08-28T02:10:33.839984+0000","last_clean":"2023-08-28T02:10:33.839984+0000","last_became_active":"2023-08-28T02:10:32.196865+0000","last_became_peered":"2023-08-28T02:10:32.196865+0000","last_unstale":"2023-08-28T02:10:33.839984+0000","last_undegraded":"2023-08-28T02:10:33.839984+0000","last_fullsized":"2023-08-28T02:10:33.839984+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-08-28T02:10:33.840521+0000","last_change":"2023-08-28T02:10:32.198340+0000","last_active":"2023-08-28T02:10:33.840521+0000","last_peered":"2023-08-28T02:10:33.840521+0000","last_clean":"2023-08-28T02:10:33.840521+0000","last_became_active":"2023-08-28T02:10:32.198052+0000","last_became_peered":"2023-08-28T02:10:32.198052+0000","last_unstale":"2023-08-28T02:10:33.840521+0000","last_undegraded":"2023-08-28T02:10:33.840521+0000","last_fullsized":"2023-08-28T02:10:33.840521+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-08-28T02:10:33.840439+0000","last_change":"2023-08-28T02:10:32.198506+0000","last_active":"2023-08-28T02:10:33.840439+0000","last_peered":"2023-08-28T02:10:33.840439+0000","last_clean":"2023-08-28T02:10:33.840439+0000","last_became_active":"2023-08-28T02:10:32.198175+0000","last_became_peered":"2023-08-28T02:10:32.198175+0000","last_unstale":"2023-08-28T02:10:33.840439+0000","last_undegraded":"2023-08-28T02:10:33.840439+0000","last_fullsized":"2023-08-28T02:10:33.840439+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":15,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-08-28T02:10:33.840473+0000","last_change":"2023-08-28T02:09:55.609189+0000","last_active":"2023-08-28T02:10:33.840473+0000","last_peered":"2023-08-28T02:10:33.840473+0000","last_clean":"2023-08-28T02:10:33.840473+0000","last_became_active":"2023-08-28T02:09:55.608705+0000","last_became_peered":"2023-08-28T02:09:55.608705+0000","last_unstale":"2023-08-28T02:10:33.840473+0000","last_undegraded":"2023-08-28T02:10:33.840473+0000","last_fullsized":"2023-08-28T02:10:33.840473+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_clean_scrub_stamp":"2023-08-28T02:08:02.043718+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-08-28T02:10:33.840200+0000","last_change":"2023-08-28T02:10:32.184422+0000","last_active":"2023-08-28T02:10:33.840200+0000","last_peered":"2023-08-28T02:10:33.840200+0000","last_clean":"2023-08-28T02:10:33.840200+0000","last_became_active":"2023-08-28T02:10:32.184072+0000","last_became_peered":"2023-08-28T02:10:32.184072+0000","last_unstale":"2023-08-28T02:10:33.840200+0000","last_undegraded":"2023-08-28T02:10:33.840200+0000","last_fullsized":"2023-08-28T02:10:33.840200+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-08-28T02:10:33.180854+0000","last_change":"2023-08-28T02:10:32.197650+0000","last_active":"2023-08-28T02:10:33.180854+0000","last_peered":"2023-08-28T02:10:33.180854+0000","last_clean":"2023-08-28T02:10:33.180854+0000","last_became_active":"2023-08-28T02:10:32.197319+0000","last_became_peered":"2023-08-28T02:10:32.197319+0000","last_unstale":"2023-08-28T02:10:33.180854+0000","last_undegraded":"2023-08-28T02:10:33.180854+0000","last_fullsized":"2023-08-28T02:10:33.180854+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-08-28T02:10:33.840200+0000","last_change":"2023-08-28T02:10:32.184125+0000","last_active":"2023-08-28T02:10:33.840200+0000","last_peered":"2023-08-28T02:10:33.840200+0000","last_clean":"2023-08-28T02:10:33.840200+0000","last_became_active":"2023-08-28T02:10:32.183865+0000","last_became_peered":"2023-08-28T02:10:32.183865+0000","last_unstale":"2023-08-28T02:10:33.840200+0000","last_undegraded":"2023-08-28T02:10:33.840200+0000","last_fullsized":"2023-08-28T02:10:33.840200+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-08-28T02:10:33.839994+0000","last_change":"2023-08-28T02:10:32.197552+0000","last_active":"2023-08-28T02:10:33.839994+0000","last_peered":"2023-08-28T02:10:33.839994+0000","last_clean":"2023-08-28T02:10:33.839994+0000","last_became_active":"2023-08-28T02:10:32.197251+0000","last_became_peered":"2023-08-28T02:10:32.197251+0000","last_unstale":"2023-08-28T02:10:33.839994+0000","last_undegraded":"2023-08-28T02:10:33.839994+0000","last_fullsized":"2023-08-28T02:10:33.839994+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430226,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:10:54 2023","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.59799999999999998}]},{"osd":1,"last update":"Mon Aug 28 02:10:54 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.40600000000000003}]},{"osd":2,"last update":"Mon Aug 28 02:10:54 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.37},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.503}]},{"osd":3,"last update":"Mon Aug 28 02:10:54 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.497},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.56100000000000005}]},{"osd":4,"last update":"Mon Aug 28 02:10:54 2023","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.63700000000000001}]},{"osd":5,"last update":"Mon Aug 28 02:10:54 2023","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.54800000000000004}]},{"osd":6,"last update":"Mon Aug 28 02:10:54 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.61899999999999999}]}]},{"osd":6,"up_from":42,"seq":180388626453,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297620,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447532,"statfs":{"total":95995035648,"available":95690272768,"internally_reserved":0,"allocated":536576,"data_stored":220200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:10:30 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.72199999999999998}]},{"osd":1,"last update":"Mon Aug 28 02:10:30 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.625}]},{"osd":2,"last update":"Mon Aug 28 02:10:30 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.501}]},{"osd":3,"last update":"Mon Aug 28 02:10:30 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.67100000000000004},{"interface":"front","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.54700000000000004}]},{"osd":4,"last update":"Mon Aug 28 02:10:30 2023","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.624,"5min":0.624,"15min":0.624},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.79200000000000004}]},{"osd":5,"last update":"Mon Aug 28 02:10:30 2023","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.65200000000000002}]},{"osd":7,"last update":"Mon Aug 28 02:10:55 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.88400000000000001}]}]},{"osd":1,"up_from":13,"seq":55834574893,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:10:46 2023","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":0.34899999999999998,"15min":0.33300000000000002},"min":{"1min":0.26500000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.36799999999999999,"15min":0.34699999999999998},"min":{"1min":0.246,"5min":0.215,"15min":0.215},"max":{"1min":0.54100000000000004,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.27400000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:11:09 2023","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":0.375,"15min":0.36599999999999999},"min":{"1min":0.26200000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.51600000000000001,"5min":0.623,"15min":0.623},"last":0.373},{"interface":"front","average":{"1min":0.377,"5min":0.38700000000000001,"15min":0.38},"min":{"1min":0.27700000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.52400000000000002,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.34799999999999998}]},{"osd":3,"last update":"Mon Aug 28 02:10:29 2023","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":0.436,"15min":0.437},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.56200000000000006,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.46800000000000003,"5min":0.47999999999999998,"15min":0.48199999999999998},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.33800000000000002}]},{"osd":4,"last update":"Mon Aug 28 02:10:46 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.55700000000000005,"15min":0.56000000000000005},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56899999999999995,"15min":0.56999999999999995},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.53600000000000003}]},{"osd":5,"last update":"Mon Aug 28 02:11:09 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.55900000000000005,"15min":0.56100000000000005},"min":{"1min":0.42799999999999999,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.65900000000000003,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.42799999999999999}]},{"osd":6,"last update":"Mon Aug 28 02:10:34 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.55200000000000005}]},{"osd":7,"last update":"Mon Aug 28 02:10:53 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":0.94199999999999995,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.52000000000000002}]}]},{"osd":0,"up_from":8,"seq":34359738418,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Aug 28 02:10:49 2023","interfaces":[{"interface":"back","average":{"1min":0.34799999999999998,"5min":0.34899999999999998,"15min":0.34200000000000003},"min":{"1min":0.24299999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.35599999999999998,"5min":0.36099999999999999,"15min":0.34699999999999998},"min":{"1min":0.23400000000000001,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.55900000000000005,"5min":0.58499999999999996,"15min":0.58499999999999996},"last":0.23400000000000001}]},{"osd":2,"last update":"Mon Aug 28 02:11:04 2023","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.38400000000000001,"15min":0.38200000000000001},"min":{"1min":0.27100000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.41899999999999998,"15min":0.41299999999999998},"min":{"1min":0.28299999999999997,"5min":0.248,"15min":0.248},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.28499999999999998}]},{"osd":3,"last update":"Mon Aug 28 02:10:25 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.46100000000000002,"15min":0.46400000000000002},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.37},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.499,"15min":0.505},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.60399999999999998,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.39200000000000002}]},{"osd":4,"last update":"Mon Aug 28 02:10:49 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.54900000000000004,"15min":0.54800000000000004},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58799999999999997,"15min":0.58899999999999997},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.71899999999999997,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.56499999999999995}]},{"osd":5,"last update":"Mon Aug 28 02:10:13 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.497},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.51200000000000001}]},{"osd":6,"last update":"Mon Aug 28 02:10:30 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.54400000000000004}]},{"osd":7,"last update":"Mon Aug 28 02:10:54 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.53600000000000003}]}]},{"osd":2,"up_from":18,"seq":77309411368,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:11:06 2023","interfaces":[{"interface":"back","average":{"1min":0.34899999999999998,"5min":0.35899999999999999,"15min":0.35899999999999999},"min":{"1min":0.23100000000000001,"5min":0.216,"15min":0.216},"max":{"1min":0.48999999999999999,"5min":0.54400000000000004,"15min":0.54400000000000004},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.34499999999999997,"5min":0.372,"15min":0.378},"min":{"1min":0.26300000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":0.48899999999999999,"5min":0.53500000000000003,"15min":0.53500000000000003},"last":0.35599999999999998}]},{"osd":1,"last update":"Mon Aug 28 02:11:06 2023","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.377,"15min":0.378},"min":{"1min":0.23599999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.54100000000000004,"5min":0.56999999999999995,"15min":0.56999999999999995},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.38200000000000001,"5min":0.38900000000000001,"15min":0.38500000000000001},"min":{"1min":0.249,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.39100000000000001}]},{"osd":3,"last update":"Mon Aug 28 02:10:26 2023","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.44900000000000001,"15min":0.45500000000000002},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.54200000000000004,"5min":0.55800000000000005,"15min":0.55800000000000005},"last":0.379},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.47799999999999998,"15min":0.48799999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.51700000000000002,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.432}]},{"osd":4,"last update":"Mon Aug 28 02:10:47 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54900000000000004,"15min":0.55100000000000005},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.56999999999999995,"15min":0.57299999999999995},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.73799999999999999,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.66900000000000004}]},{"osd":5,"last update":"Mon Aug 28 02:10:10 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.65900000000000003}]},{"osd":6,"last update":"Mon Aug 28 02:10:32 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.625,"5min":0.625,"15min":0.625},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.68899999999999995}]},{"osd":7,"last update":"Mon Aug 28 02:10:54 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.70699999999999996},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.62}]}]},{"osd":3,"up_from":25,"seq":107374182437,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:10:28 2023","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":0.39400000000000002,"15min":0.39800000000000002},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.443},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.41999999999999998,"15min":0.42399999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.54400000000000004}]},{"osd":1,"last update":"Mon Aug 28 02:10:28 2023","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.42899999999999999,"15min":0.43099999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.629},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.433,"15min":0.435},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.70399999999999996,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.66800000000000004}]},{"osd":2,"last update":"Mon Aug 28 02:10:28 2023","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.46700000000000003,"15min":0.46999999999999997},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.44600000000000001,"15min":0.45100000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.52600000000000002}]},{"osd":4,"last update":"Mon Aug 28 02:10:48 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.56799999999999995,"15min":0.57299999999999995},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.59199999999999997,"15min":0.59599999999999997},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.64600000000000002}]},{"osd":5,"last update":"Mon Aug 28 02:10:11 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.58799999999999997}]},{"osd":6,"last update":"Mon Aug 28 02:10:33 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.878,"5min":0.878,"15min":0.878},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.59899999999999998}]},{"osd":7,"last update":"Mon Aug 28 02:10:54 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":1.4179999999999999,"5min":1.4179999999999999,"15min":1.4179999999999999},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.56000000000000005}]}]},{"osd":4,"up_from":30,"seq":128849018911,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:10:48 2023","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.54200000000000004,"15min":0.54800000000000004},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.66400000000000003,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.55100000000000005,"15min":0.56100000000000005},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.64900000000000002,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.71099999999999997}]},{"osd":1,"last update":"Mon Aug 28 02:10:48 2023","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.52900000000000003,"15min":0.53500000000000003},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.63700000000000001,"5min":0.98899999999999999,"15min":0.98899999999999999},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.53900000000000003,"15min":0.54100000000000004},"min":{"1min":0.42199999999999999,"5min":0.374,"15min":0.374},"max":{"1min":0.77000000000000002,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.59499999999999997}]},{"osd":2,"last update":"Mon Aug 28 02:10:48 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.55700000000000005,"15min":0.56599999999999995},"min":{"1min":0.35099999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.73599999999999999,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.498,"5min":0.54600000000000004,"15min":0.55400000000000005},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.58299999999999996,"5min":0.873,"15min":0.873},"last":0.57199999999999995}]},{"osd":3,"last update":"Mon Aug 28 02:10:48 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.54900000000000004,"15min":0.55500000000000005},"min":{"1min":0.375,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.54600000000000004,"15min":0.54700000000000004},"min":{"1min":0.45000000000000001,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.72899999999999998,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.63900000000000001}]},{"osd":5,"last update":"Mon Aug 28 02:10:10 2023","interfaces":[{"interface":"back","average":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"last":0.73299999999999998}]},{"osd":6,"last update":"Mon Aug 28 02:10:30 2023","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":0.434,"15min":0.434},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.65300000000000002}]},{"osd":7,"last update":"Mon Aug 28 02:10:57 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.2110000000000001,"5min":1.2110000000000001,"15min":1.2110000000000001},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.67000000000000004}]}]},{"osd":5,"up_from":36,"seq":154618822683,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297616,"kb_used_data":520,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447536,"statfs":{"total":95995035648,"available":95690276864,"internally_reserved":0,"allocated":532480,"data_stored":219885,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:10:10 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.47699999999999998}]},{"osd":1,"last update":"Mon Aug 28 02:10:10 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.33700000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:10:10 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.42899999999999999}]},{"osd":3,"last update":"Mon Aug 28 02:10:10 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.50700000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:10:10 2023","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.438,"5min":0.438,"15min":0.438},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.64600000000000002}]},{"osd":6,"last update":"Mon Aug 28 02:10:30 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.624}]},{"osd":7,"last update":"Mon Aug 28 02:10:53 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.53700000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-08-28T02:11:16.069 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-08-28T02:11:16.070 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-08-28T02:11:16.070 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-08-28T02:11:16.070 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph health --format=json 2023-08-28T02:11:16.284 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:16 smithi169 conmon[131988]: t=2023-08-28T02:11:16+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_org_id_role_id_role" 2023-08-28T02:11:16.284 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:16 smithi169 conmon[131988]: t=2023-08-28T02:11:16+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index role_org_id_uid" 2023-08-28T02:11:16.284 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:16 smithi169 conmon[131988]: t=2023-08-28T02:11:16+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role.uid" 2023-08-28T02:11:16.421 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:11:16.594 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:16 smithi169 conmon[131988]: t=2023-08-28T02:11:16+0000 lvl=info msg="Executing migration" logger=migrator id="create seed assignment table" 2023-08-28T02:11:16.594 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:16 smithi169 conmon[131988]: t=2023-08-28T02:11:16+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_role_name" 2023-08-28T02:11:16.594 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:16 smithi169 conmon[131988]: t=2023-08-28T02:11:16+0000 lvl=info msg="migrations completed" logger=migrator performed=381 skipped=0 duration=13.832449927s 2023-08-28T02:11:16.594 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:16 smithi169 conmon[131988]: t=2023-08-28T02:11:16+0000 lvl=info msg="Created default organization" logger=sqlstore 2023-08-28T02:11:16.595 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:16 smithi169 conmon[131988]: t=2023-08-28T02:11:16+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2023-08-28T02:11:16.595 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:16 smithi169 conmon[131988]: t=2023-08-28T02:11:16+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2023-08-28T02:11:16.595 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:16 smithi169 conmon[131988]: t=2023-08-28T02:11:16+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2023-08-28T02:11:16.595 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:16 smithi169 conmon[131988]: t=2023-08-28T02:11:16+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2023-08-28T02:11:16.595 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:16 smithi169 conmon[131988]: t=2023-08-28T02:11:16+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2023-08-28T02:11:16.917 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:16 smithi143 conmon[100258]: audit 2023-08-28T02:11:15.279547+0000 mgr.y ( 2023-08-28T02:11:16.917 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:16 smithi143 conmon[100258]: mgr.14424) 53 : audit [DBG] from='client.24515 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:11:16.917 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:16 smithi143 conmon[100258]: audit 2023-08-28T02:11:15.590809+0000 mon.a (mon.0) 646 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:16.917 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:16 smithi143 conmon[100258]: cephadm 2023-08-28T02:11:15.591740+0000 mgr.y (mgr.14424) 54 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-08-28T02:11:16.917 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:16 smithi143 conmon[100258]: cephadm 2023-08-28T02:11:15.595053+0000 mgr.y (mgr.14424) 55 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi169 2023-08-28T02:11:16.918 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:16 smithi143 conmon[109923]: audit 2023-08-28T02:11:15.279547+0000 mgr.y (mgr 2023-08-28T02:11:16.918 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:16 smithi143 conmon[109923]: .14424) 53 : audit [DBG] from='client.24515 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:11:16.918 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:16 smithi143 conmon[109923]: audit 2023-08-28T02:11:15.590809+0000 mon.a (mon 2023-08-28T02:11:16.918 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:16 smithi143 conmon[109923]: .0) 646 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:16.918 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:16 smithi143 conmon[109923]: cephadm 2023-08-28T 2023-08-28T02:11:16.918 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:16 smithi143 conmon[109923]: 02:11:15.591740+0000 mgr.y (mgr.14424) 54 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-08-28T02:11:16.918 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:16 smithi143 conmon[109923]: cephadm 2023-08-28T02:11:16.918 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:16 smithi143 conmon[109923]: 2023-08-28T02: 2023-08-28T02:11:16.918 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:16 smithi143 conmon[109923]: 11:15.595053+0000 mgr.y (mgr.14424) 55 : 2023-08-28T02:11:16.918 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:16 smithi143 conmon[109923]: cephadm [INF] Reconfiguring daemon prometheus.a on smithi169 2023-08-28T02:11:16.987 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:16 smithi169 conmon[131988]: t=2023-08-28T02:11:16+0000 lvl=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" logger=provisioning.datasources filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2023-08-28T02:11:16.987 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:16 smithi169 conmon[131988]: t=2023-08-28T02:11:16+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2023-08-28T02:11:16.987 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:16 smithi169 conmon[131988]: t=2023-08-28T02:11:16+0000 lvl=info msg="warming cache for startup" logger=ngalert 2023-08-28T02:11:16.988 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:16 smithi169 conmon[131988]: t=2023-08-28T02:11:16+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2023-08-28T02:11:16.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:16 smithi169 conmon[100500]: audit 2023-08-28T02:11:15.279547+0000 mgr.y (mgr.14424) 53 2023-08-28T02:11:16.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:16 smithi169 conmon[100500]: : audit [DBG] from='client.24515 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:11:16.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:16 smithi169 conmon[100500]: audit 2023-08-28T02:11:15.590809+0000 mon.a (mon.0) 646 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:16.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:16 smithi169 conmon[100500]: cephadm 2023-08-28T02:11 2023-08-28T02:11:16.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:16 smithi169 conmon[100500]: :15.591740+0000 mgr.y (mgr.14424) 54 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-08-28T02:11:16.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:16 smithi169 conmon[100500]: cephadm 2023-08-28T02:11:15 2023-08-28T02:11:16.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:16 smithi169 conmon[100500]: .595053+0000 mgr.y (mgr.14424) 55 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi169 2023-08-28T02:11:17.595 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:11:17 smithi143 conmon[150510]: level=info ts=2023-08-28T02:11:17.546Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000205388s 2023-08-28T02:11:17.595 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:11:17 smithi169 conmon[131988]: t=2023-08-28T02:11:17+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2023-08-28T02:11:17.941 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:17 smithi169 conmon[100500]: cluster 2023-08-28T02:11:15.872626+0000 mgr.y (mgr.14424) 56 : cluster 2023-08-28T02:11:17.941 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:17 smithi169 conmon[100500]: [DBG] pgmap v40: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:17.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:17 smithi143 conmon[100258]: cluster 2023-08-28T02:11:15.872626+0000 mgr.y (mgr.14424 2023-08-28T02:11:17.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:17 smithi143 conmon[100258]: ) 56 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:17.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:17 smithi143 conmon[109923]: cluster 2023-08-28T02:11:15.872626+0000 2023-08-28T02:11:17.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:17 smithi143 conmon[109923]: mgr.y (mgr.14424) 56 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:17.958 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:11:18.626 INFO:teuthology.orchestra.run.smithi143.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-08-28T02:11:18.626 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-08-28T02:11:18.626 INFO:tasks.cephadm:Setup complete, yielding 2023-08-28T02:11:18.627 INFO:teuthology.run_tasks:Running task cephadm.shell... 2023-08-28T02:11:18.635 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi143.front.sepia.ceph.com 2023-08-28T02:11:18.636 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- bash -c 'ceph fs volume create foo' 2023-08-28T02:11:18.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:18 smithi169 conmon[100500]: audit 2023-08-28T02:11:17.959579+0000 mon.c (mon.2) 17 : 2023-08-28T02:11:18.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:18 smithi169 conmon[100500]: audit [DBG] from='client.? 172.21.15.143:0/147539761' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-08-28T02:11:18.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:18 smithi143 conmon[100258]: audit 2023-08-28T02:11:17.959579+0000 mon.c (mon.2) 17 : audit 2023-08-28T02:11:18.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:18 smithi143 conmon[100258]: [DBG] from='client.? 172.21.15.143:0/147539761' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-08-28T02:11:18.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:18 smithi143 conmon[109923]: audit 2023-08-28T02:11:17.959579 2023-08-28T02:11:18.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:18 smithi143 conmon[109923]: +0000 mon.c (mon.2) 17 : audit [DBG] from='client.? 172.21.15.143:0/147539761' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-08-28T02:11:19.183 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:11:18 smithi169 systemd[1]: Stopping Ceph prometheus.a for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:11:19.184 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:11:18 smithi169 bash[133274]: Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-prometheus.a found: no such container 2023-08-28T02:11:19.487 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:11:19 smithi169 conmon[130010]: ts=2023-08-28T02:11:19.184Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-08-28T02:11:19.488 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:11:19 smithi169 conmon[130010]: ts=2023-08-28T02:11:19.185Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-08-28T02:11:19.488 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:11:19 smithi169 conmon[130010]: ts=2023-08-28T02:11:19.185Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-08-28T02:11:19.488 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:11:19 smithi169 conmon[130010]: ts=2023-08-28T02:11:19.185Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-08-28T02:11:19.488 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:11:19 smithi169 conmon[130010]: ts=2023-08-28T02:11:19.185Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-08-28T02:11:19.488 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:11:19 smithi169 conmon[130010]: ts=2023-08-28T02:11:19.185Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-08-28T02:11:19.488 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:11:19 smithi169 conmon[130010]: ts=2023-08-28T02:11:19.185Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-08-28T02:11:19.488 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:11:19 smithi169 conmon[130010]: ts=2023-08-28T02:11:19.185Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-08-28T02:11:19.488 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:11:19 smithi169 conmon[130010]: ts=2023-08-28T02:11:19.185Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-08-28T02:11:19.488 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:11:19 smithi169 conmon[130010]: ts=2023-08-28T02:11:19.185Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-08-28T02:11:19.488 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:11:19 smithi169 conmon[130010]: ts=2023-08-28T02:11:19.185Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-08-28T02:11:19.488 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:11:19 smithi169 conmon[130010]: ts=2023-08-28T02:11:19.185Z caller=main.go:1066 level=info msg="See you next time!" 2023-08-28T02:11:19.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:19 smithi143 conmon[100258]: cluster 2023-08-28T02:11:17.872924+0000 mgr.y (mgr.14424) 57 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:19.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:19 smithi143 conmon[109923]: cluster 2023-08-28T02:11:17.872924+0000 mgr.y (mgr.14424) 57 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:19.888 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:19 smithi169 conmon[100500]: cluster 2023-08-28T02:11:17.872924+0000 mgr.y (mgr.14424) 57 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:19.888 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:11:19 smithi169 bash[133390]: 1d4ce707c8b732bb5a1c77c75d15366d3eb10255195dc1f46aeea7101a1f8444 2023-08-28T02:11:20.237 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:11:19 smithi169 systemd[1]: ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@prometheus.a.service: Succeeded. 2023-08-28T02:11:20.237 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:11:19 smithi169 systemd[1]: Stopped Ceph prometheus.a for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:11:20.237 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:11:19 smithi169 systemd[1]: Starting Ceph prometheus.a for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:11:20.737 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:11:20 smithi169 conmon[133764]: ts=2023-08-28T02:11:20.486Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-08-28T02:11:20.737 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:11:20 smithi169 conmon[133764]: ts=2023-08-28T02:11:20.486Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-08-28T02:11:20.737 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:11:20 smithi169 bash[133723]: 3ceb1bb30c93983c012eabe6ff07715e588ed09c03dffad2bfde6999f396e3e3 2023-08-28T02:11:20.738 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:11:20 smithi169 conmon[133764]: ts=2023-08-28T02:11:20.487Z caller=main.go:518 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi169 (none))" 2023-08-28T02:11:20.738 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:11:20 smithi169 conmon[133764]: ts=2023-08-28T02:11:20.487Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-08-28T02:11:20.738 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:11:20 smithi169 conmon[133764]: ts=2023-08-28T02:11:20.487Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-08-28T02:11:20.738 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:11:20 smithi169 conmon[133764]: ts=2023-08-28T02:11:20.489Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-08-28T02:11:20.738 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:11:20 smithi169 conmon[133764]: ts=2023-08-28T02:11:20.489Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-08-28T02:11:20.738 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:11:20 smithi169 conmon[133764]: ts=2023-08-28T02:11:20.490Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-08-28T02:11:20.738 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:11:20 smithi169 systemd[1]: Started Ceph prometheus.a for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:11:20.738 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:11:20 smithi169 conmon[133764]: ts=2023-08-28T02:11:20.493Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-08-28T02:11:20.738 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:11:20 smithi169 conmon[133764]: ts=2023-08-28T02:11:20.493Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.718µs 2023-08-28T02:11:20.739 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:11:20 smithi169 conmon[133764]: ts=2023-08-28T02:11:20.493Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-08-28T02:11:20.739 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:11:20 smithi169 conmon[133764]: ts=2023-08-28T02:11:20.691Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2023-08-28T02:11:20.739 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:11:20 smithi169 conmon[133764]: ts=2023-08-28T02:11:20.691Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2023-08-28T02:11:20.739 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:11:20 smithi169 conmon[133764]: ts=2023-08-28T02:11:20.691Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=27.759µs wal_replay_duration=197.933637ms total_replay_duration=197.976019ms 2023-08-28T02:11:20.739 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:11:20 smithi169 conmon[133764]: ts=2023-08-28T02:11:20.693Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-08-28T02:11:20.739 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:11:20 smithi169 conmon[133764]: ts=2023-08-28T02:11:20.693Z caller=main.go:947 level=info msg="TSDB started" 2023-08-28T02:11:20.739 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:11:20 smithi169 conmon[133764]: ts=2023-08-28T02:11:20.693Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-08-28T02:11:20.739 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:11:20 smithi169 conmon[133764]: ts=2023-08-28T02:11:20.710Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=16.91864ms db_storage=910ns remote_storage=1.641µs web_handler=342ns query_engine=752ns scrape=406.795µs scrape_sd=26.595µs notify=21.728µs notify_sd=7.879µs rules=16.211769ms 2023-08-28T02:11:20.739 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:11:20 smithi169 conmon[133764]: ts=2023-08-28T02:11:20.710Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-08-28T02:11:21.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:21 smithi143 conmon[109923]: cluster 2023-08-28T02:11:19.873194+0000 mgr.y (mgr.14424) 2023-08-28T02:11:21.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:21 smithi143 conmon[109923]: 58 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:21.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:21 smithi143 conmon[109923]: audit 2023-08-28T02:11:20.443472+0000 mgr.y (mgr.14424) 59 : audit [DBG] from='client.24527 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:11:21.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:21 smithi143 conmon[109923]: audit 2023-08-28T02:11:20.444055+0000 mon.a (mon.0) 647 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-08-28T02:11:21.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:21 smithi143 conmon[109923]: audit 2023-08-28T02:11:20.687768+0000 mon.a (mon.0) 648 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:21.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:21 smithi143 conmon[109923]: audit 2023-08-28T02:11:20.690439+0000 mon.a (mon.0) 649 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-08-28T02:11:21.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:21 smithi143 conmon[100258]: cluster 2023-08-28T02:11:19.873194+0000 mgr.y (mgr.14424) 2023-08-28T02:11:21.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:21 smithi143 conmon[100258]: 58 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:21.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:21 smithi143 conmon[100258]: audit 2023-08-28T02:11:20.443472+0000 mgr.y (mgr.14424) 59 : audit [DBG] from='client.24527 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:11:21.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:21 smithi143 conmon[100258]: audit 2023-08-28T02:11:20.444055+0000 mon.a (mon.0) 647 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-08-28T02:11:21.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:21 smithi143 conmon[100258]: audit 2023-08-28T02:11:20.687768+0000 mon.a (mon.0) 648 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:21.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:21 smithi143 conmon[100258]: audit 2023-08-28T02:11:20.690439+0000 mon.a (mon.0) 649 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-08-28T02:11:21.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:21 smithi169 conmon[100500]: cluster 2023-08-28T02:11:19.873194+0000 mgr.y (mgr.14424) 58 : cluster [DBG] 2023-08-28T02:11:21.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:21 smithi169 conmon[100500]: pgmap v42: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:21.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:21 smithi169 conmon[100500]: audit 2023-08-28T02:11:20.443472+0000 mgr.y (mgr.14424) 59 : audit [DBG] from='client.24527 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:11:21.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:21 smithi169 conmon[100500]: audit 2023-08-28T02:11:20 2023-08-28T02:11:21.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:21 smithi169 conmon[100500]: .444055+0000 mon.a (mon.0) 647 : audit [INF] 2023-08-28T02:11:21.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:21 smithi169 conmon[100500]: from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-08-28T02:11:21.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:21 smithi169 conmon[100500]: audit 2023-08-28T02:11:20.687768 2023-08-28T02:11:21.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:21 smithi169 conmon[100500]: +0000 mon.a (mon.0) 648 : audit [INF] 2023-08-28T02:11:21.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:21 smithi169 conmon[100500]: from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:21.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:21 smithi169 conmon[100500]: audit 2023-08-28T02:11:21.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:21 smithi169 conmon[100500]: 2023-08-28T02:11:20. 2023-08-28T02:11:21.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:21 smithi169 conmon[100500]: 690439+0000 mon.a (mon.0) 649 : audit [DBG] 2023-08-28T02:11:21.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:21 smithi169 conmon[100500]: from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-08-28T02:11:22.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:22 smithi143 conmon[100258]: audit 2023-08-28T02:11:20.690766+0000 mgr.y (mgr 2023-08-28T02:11:22.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:22 smithi143 conmon[100258]: .14424) 60 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-08-28T02:11:22.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:22 smithi143 conmon[100258]: audit 2023-08-28T02:11:21.164332+0000 mon.a (mon.0) 650 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-08-28T02:11:22.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:22 smithi143 conmon[100258]: cluster 2023-08-28T02:11:21.164389+0000 mon.a (mon.0) 651 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-08-28T02:11:22.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:22 smithi143 conmon[100258]: audit 2023-08-28T02:11:22.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:22 smithi143 conmon[100258]: 2023-08-28T02 2023-08-28T02:11:22.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:22 smithi143 conmon[100258]: :11:21.165481+0000 mon.a (mon.0) 652 2023-08-28T02:11:22.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:22 smithi143 conmon[100258]: : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-08-28T02:11:22.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:22 smithi143 conmon[100258]: debug 2023-08-28T02:11:22.177+0000 7f0a52cc1700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-08-28T02:11:22.452 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:11:22 smithi143 conmon[100708]: [28/Aug/2023:02:11:22] ENGINE Bus STOPPING 2023-08-28T02:11:22.453 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:11:22 smithi143 conmon[100708]: [28/Aug/2023:02:11:22] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-08-28T02:11:22.453 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:11:22 smithi143 conmon[100708]: [28/Aug/2023:02:11:22] ENGINE Bus STOPPED 2023-08-28T02:11:22.453 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:11:22 smithi143 conmon[100708]: [28/Aug/2023:02:11:22] ENGINE Bus STARTING 2023-08-28T02:11:22.453 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:11:22 smithi143 conmon[100708]: [28/Aug/2023:02:11:22] ENGINE Serving on http://:::9283 2023-08-28T02:11:22.453 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:11:22 smithi143 conmon[100708]: [28/Aug/2023:02:11:22] ENGINE Bus STARTED 2023-08-28T02:11:22.453 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:11:22 smithi143 conmon[100708]: [28/Aug/2023:02:11:22] ENGINE Bus STOPPING 2023-08-28T02:11:22.453 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:11:22 smithi143 conmon[100708]: [28/Aug/2023:02:11:22] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-08-28T02:11:22.453 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:11:22 smithi143 conmon[100708]: [28/Aug/2023:02:11:22] ENGINE Bus STOPPED 2023-08-28T02:11:22.453 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:11:22 smithi143 conmon[100708]: [28/Aug/2023:02:11:22] ENGINE Bus STARTING 2023-08-28T02:11:22.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:22 smithi143 conmon[109923]: audit 2023-08-28T02:11:20.690766+0000 mgr.y 2023-08-28T02:11:22.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:22 smithi143 conmon[109923]: (mgr.14424) 60 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-08-28T02:11:22.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:22 smithi143 conmon[109923]: audit 2023-08-28T 2023-08-28T02:11:22.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:22 smithi143 conmon[109923]: 02:11:21.164332+0000 mon.a (mon.0) 650 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-08-28T02:11:22.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:22 smithi143 conmon[109923]: cluster 2023-08- 2023-08-28T02:11:22.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:22 smithi143 conmon[109923]: 28T02:11:21.164389+0000 mon.a (mon. 2023-08-28T02:11:22.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:22 smithi143 conmon[109923]: 0) 651 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-08-28T02:11:22.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:22 smithi143 conmon[109923]: audit 2023-08-28 2023-08-28T02:11:22.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:22 smithi143 conmon[109923]: T02:11:21.165481+0000 mon.a (mon.0) 652 : audit 2023-08-28T02:11:22.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:22 smithi143 conmon[109923]: [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-08-28T02:11:22.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:22 smithi169 conmon[100500]: audit 2023-08-28T02:11:20.690766+0000 mgr.y (mgr.14424) 60 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-08-28T02:11:22.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:22 smithi169 conmon[100500]: audit 2023-08-28T02:11:21.164332+0000 mon.a (mon.0) 650 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-08-28T02:11:22.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:22 smithi169 conmon[100500]: cluster 2023-08-28T02:11:21.164389+0000 mon.a (mon.0) 651 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-08-28T02:11:22.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:22 smithi169 conmon[100500]: audit 2023-08-28T02:11:21.165481+0000 mon.a (mon. 2023-08-28T02:11:22.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:22 smithi169 conmon[100500]: 0) 652 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-08-28T02:11:22.818 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:11:22 smithi143 conmon[100708]: [28/Aug/2023:02:11:22] ENGINE Serving on http://:::9283 2023-08-28T02:11:22.818 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:11:22 smithi143 conmon[100708]: [28/Aug/2023:02:11:22] ENGINE Bus STARTED 2023-08-28T02:11:22.818 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:11:22 smithi143 conmon[100708]: [28/Aug/2023:02:11:22] ENGINE Bus STOPPING 2023-08-28T02:11:22.818 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:11:22 smithi143 conmon[100708]: [28/Aug/2023:02:11:22] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-08-28T02:11:22.818 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:11:22 smithi143 conmon[100708]: [28/Aug/2023:02:11:22] ENGINE Bus STOPPED 2023-08-28T02:11:22.818 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:11:22 smithi143 conmon[100708]: [28/Aug/2023:02:11:22] ENGINE Bus STARTING 2023-08-28T02:11:22.819 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:11:22 smithi143 conmon[100708]: [28/Aug/2023:02:11:22] ENGINE Serving on http://:::9283 2023-08-28T02:11:22.819 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:11:22 smithi143 conmon[100708]: [28/Aug/2023:02:11:22] ENGINE Bus STARTED 2023-08-28T02:11:22.837 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2023-08-28T02:11:23.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[100258]: cluster 2023-08-28T02:11:21.873699+0000 mgr.y (mgr.14424) 61 : cluster [DBG] pgmap v44: 41 pgs: 32 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:23.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[100258]: audit 2023-08-28T02:11:22.171905+0000 mon.a (mon.0) 653 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-08-28T02:11:23.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[100258]: cluster 2023-08-28T02:11:22.172079+0000 mon.a (mon.0) 654 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-08-28T02:11:23.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[100258]: audit 2023-08-28T02:11:22.177331+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-08-28T02:11:23.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[100258]: cluster 2023-08-28T02:11:22.178184+0000 mon.a (mon.0) 656 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-08-28T02:11:23.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[100258]: cluster 2023-08-28T02:11:22.178209+0000 mon.a (mon.0) 657 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-08-28T02:11:23.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[100258]: cluster 2023-08-28T02:11:22.191850+0000 mon.a (mon.0) 658 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-08-28T02:11:23.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[100258]: audit 2023-08-28T02:11:22.191947+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-08-28T02:11:23.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[100258]: cluster 2023-08-28T02:11:22.191999+0000 mon.a (mon.0) 660 : cluster [DBG] fsmap foo:0 2023-08-28T02:11:23.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[100258]: cephadm 2023-08-28T02:11:22.195130+0000 mgr.y (mgr.14424) 62 : 2023-08-28T02:11:23.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[100258]: cephadm [INF] Saving service mds.foo spec with placement count:2 2023-08-28T02:11:23.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[100258]: audit 2023-08-28T02:11:22.203414+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:23.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[100258]: audit 2023-08-28T02:11:22.205053+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi143:9093"}]: dispatch 2023-08-28T02:11:23.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[100258]: audit 2023-08-28T02:11:22.205426+0000 mgr.y (mgr.14424) 63 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi143:9093"}]: dispatch 2023-08-28T02:11:23.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[100258]: audit 2023-08-28T02:11:22.211119+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:23.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[100258]: audit 2023-08-28T02:11:22.215791+0000 mon.a (mon.0) 664 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-08-28T02:11:23.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[100258]: audit 2023-08-28T02:11:22.216218+0000 mgr.y (mgr.14424) 64 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-08-28T02:11:23.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[100258]: audit 2023-08-28T02:11:22.334855+0000 mon.a (mon.0) 665 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi169.front.sepia.ceph.com:9095"}]: dispatch 2023-08-28T02:11:23.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[100258]: audit 2023-08-28T02:11:22.335267+0000 mgr.y (mgr.14424) 65 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi169.front.sepia.ceph.com:9095"}]: dispatch 2023-08-28T02:11:23.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[100258]: audit 2023-08-28T02:11:22.342384+0000 mon.a (mon.0) 666 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:23.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[109923]: cluster 2023-08-28T02:11:21.873699+0000 mgr.y (mgr.14424) 61 : cluster [DBG] pgmap v44: 41 pgs: 32 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:23.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[109923]: audit 2023-08-28T02:11:22.171905+0000 mon.a (mon.0) 653 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-08-28T02:11:23.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[109923]: cluster 2023-08-28T02:11:22.172079+0000 mon.a (mon.0) 654 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-08-28T02:11:23.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[109923]: audit 2023-08-28T02:11:22.177331+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-08-28T02:11:23.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[109923]: cluster 2023-08-28T02:11:22.178184+0000 mon.a (mon.0) 656 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-08-28T02:11:23.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[109923]: cluster 2023-08-28T02:11:22.178209+0000 mon.a (mon.0) 657 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-08-28T02:11:23.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[109923]: cluster 2023-08-28T02:11:22.191850+0000 mon.a (mon.0) 658 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-08-28T02:11:23.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[109923]: audit 2023-08-28T02:11:22.191947+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-08-28T02:11:23.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[109923]: cluster 2023-08-28T02:11:22.191999+0000 mon.a (mon.0) 660 : cluster [DBG] fsmap foo:0 2023-08-28T02:11:23.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[109923]: cephadm 2023-08-28T02:11:22.195130+0000 mgr.y (mgr.14424) 62 : cephadm [INF] Saving service mds.foo spec with placement count:2 2023-08-28T02:11:23.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[109923]: audit 2023-08-28T02:11:22.203414+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:23.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[109923]: audit 2023-08-28T02:11:22.205053+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi143:9093"}]: dispatch 2023-08-28T02:11:23.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[109923]: audit 2023-08-28T02:11:22.205426+0000 mgr.y (mgr.14424) 63 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi143:9093"}]: dispatch 2023-08-28T02:11:23.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[109923]: audit 2023-08-28T02:11:22.211119+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:23.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[109923]: audit 2023-08-28T02:11:22.215791+0000 mon.a (mon.0) 664 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-08-28T02:11:23.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[109923]: audit 2023-08-28T02:11:22.216218+0000 mgr.y (mgr.14424) 64 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-08-28T02:11:23.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[109923]: audit 2023-08-28T02:11:22.334855+0000 mon.a (mon.0) 665 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi169.front.sepia.ceph.com:9095"}]: dispatch 2023-08-28T02:11:23.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[109923]: audit 2023-08-28T02:11:22.335267+0000 mgr.y (mgr.14424 2023-08-28T02:11:23.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[109923]: ) 65 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi169.front.sepia.ceph.com:9095"}]: dispatch 2023-08-28T02:11:23.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[109923]: audit 2023-08-28T02:11:23.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[109923]: 2023-08-28T02:11:22.342384+0000 mon.a (mon.0) 2023-08-28T02:11:23.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[100258]: audit 2023-08-28T02:11:22.347073+0000 mon.a (mon.0) 667 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-08-28T02:11:23.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[100258]: audit 2023-08-28T02:11:22.347527+0000 mgr.y (mgr.14424) 66 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-08-28T02:11:23.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[100258]: audit 2023-08-28T02:11:22.465553+0000 mon.a (mon.0) 668 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi169.front.sepia.ceph.com:3000"}]: dispatch 2023-08-28T02:11:23.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[100258]: audit 2023-08-28T02:11:23.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[100258]: 2023-08-28T02:11:22.465904+0000 mgr.y (mgr.14424) 67 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi169.front.sepia.ceph.com:3000"}]: dispatch 2023-08-28T02:11:23.455 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[100258]: audit 2023-08-28T02:11:22.469865+0000 mon.a (mon.0) 669 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:23.455 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[100258]: audit 2023- 2023-08-28T02:11:23.455 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[100258]: 08-28T02:11:22.471914+0000 mon.a (mon.0) 670 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:11:23.455 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[100258]: 2023-08-28T02:11:23.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[109923]: 666 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:23.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[109923]: audit 2023-08-28T02:11: 2023-08-28T02:11:23.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[109923]: 22.347073+0000 mon.a (mon.0) 667 : audit 2023-08-28T02:11:23.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[109923]: [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-08-28T02:11:23.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[109923]: audit 2023-08-28T02:11 2023-08-28T02:11:23.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[109923]: :22.347527+0000 mgr.y (mgr.14424) 66 2023-08-28T02:11:23.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[109923]: : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-08-28T02:11:23.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[109923]: audit 2023-08-28 2023-08-28T02:11:23.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[109923]: T02:11:22.465553+0000 mon.a (mon.0 2023-08-28T02:11:23.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[109923]: ) 668 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi169.front.sepia.ceph.com:3000"}]: dispatch 2023-08-28T02:11:23.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[109923]: audit 2023-08-28T02: 2023-08-28T02:11:23.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[109923]: 11:22.465904+0000 mgr.y (mgr.14424) 67 2023-08-28T02:11:23.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[109923]: : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi169.front.sepia.ceph.com:3000"}]: dispatch 2023-08-28T02:11:23.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[109923]: audit 2023-08-28T 2023-08-28T02:11:23.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[109923]: 02:11:22.469865+0000 mon.a (mon.0 2023-08-28T02:11:23.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[109923]: ) 669 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:23.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[109923]: audit 2023-08-28T 2023-08-28T02:11:23.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[109923]: 02:11:22.471914+0000 mon.a (mon. 2023-08-28T02:11:23.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:23 smithi143 conmon[109923]: 0) 670 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:11:23.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: cluster 2023-08-28T02 2023-08-28T02:11:23.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: :11:21.873699+0000 mgr.y (mgr.14424) 61 : cluster [DBG] pgmap v44: 41 pgs: 32 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:23.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: audit 2023-08-28T02:11:22.171905+0000 mon.a (mon.0) 653 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-08-28T02:11:23.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: cluster 2023-08-28T02:11:22.172079+0000 mon.a (mon.0) 654 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-08-28T02:11:23.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: audit 2023-08-28T02:11:22.177331+0000 mon.a (mon.0) 655 : audit 2023-08-28T02:11:23.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-08-28T02:11:23.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: cluster 2023-08-28T02:11:22.178184+0000 mon.a (mon.0) 656 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-08-28T02:11:23.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: cluster 2023-08-28T02:11:23.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: 2023-08-28T02:11:22.178209+0000 mon.a (mon.0) 657 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-08-28T02:11:23.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: cluster 2023-08-28T02:11:22.191850+0000 2023-08-28T02:11:23.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: mon.a (mon.0) 658 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-08-28T02:11:23.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: audit 2023-08-28T02: 2023-08-28T02:11:23.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: 11:22.191947+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-08-28T02:11:23.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: 2023-08-28T02:11:23.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: cluster 2023-08-28T 2023-08-28T02:11:23.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: 02:11:22.191999+0000 mon.a (mon.0) 660 : cluster [DBG] fsmap foo:0 2023-08-28T02:11:23.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: 2023-08-28T02:11:23.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: cephadm 2023-08-28T02:11:23.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: 2023-08-28T02:11:22.195130+0000 mgr.y 2023-08-28T02:11:23.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: (mgr.14424) 62 : cephadm [INF] Saving service mds.foo spec with placement count:2 2023-08-28T02:11:23.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: audit 2023-08-28T02:11 2023-08-28T02:11:23.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: :22.203414+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:23.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: 2023-08-28T02:11:23.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: audit 2023-08-28T02:11:23.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: 2023-08-28T02:11:22.205053+0000 mon.a (mon 2023-08-28T02:11:23.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: .0) 662 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi143:9093"}]: dispatch 2023-08-28T02:11:23.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: audit 2023-08-28T 2023-08-28T02:11:23.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: 02:11:22.205426+0000 mgr.y (mgr.14424) 63 2023-08-28T02:11:23.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi143:9093"}]: dispatch 2023-08-28T02:11:23.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: audit 2023-08-28T02:11:23.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: 2023-08-28T02:11:22.211119+0000 mon.a (mon.0) 663 2023-08-28T02:11:23.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:23.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: audit 2023-08-28T02:11:22. 2023-08-28T02:11:23.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: 215791+0000 mon.a (mon.0) 664 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-08-28T02:11:23.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: audit 2023-08-28T 2023-08-28T02:11:23.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: 02:11:22.216218+0000 mgr.y (mgr.14424) 64 2023-08-28T02:11:23.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-08-28T02:11:23.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: audit 2023-08-28T02:11:23.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: 2023-08-28T02:11:22.334855+0000 mon.a (mon.0 2023-08-28T02:11:23.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: ) 665 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi169.front.sepia.ceph.com:9095"}]: dispatch 2023-08-28T02:11:23.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: audit 2023-08-28T02:11:22. 2023-08-28T02:11:23.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: 335267+0000 mgr.y (mgr.14424) 65 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi169.front.sepia.ceph.com:9095"}]: dispatch 2023-08-28T02:11:23.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: audit 2023-08-28 2023-08-28T02:11:23.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: T02:11:22.342384+0000 mon.a (mon.0) 666 : audit 2023-08-28T02:11:23.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:23.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: audit 2023-08-28T02:11:23.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: 2023-08-28T02:11:22.347073+0000 mon.a (mon.0) 667 : 2023-08-28T02:11:23.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-08-28T02:11:23.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: audit 2023-08-28T02:11:22. 2023-08-28T02:11:23.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: 347527+0000 mgr.y (mgr.14424) 66 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-08-28T02:11:23.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: audit 2023-08-28 2023-08-28T02:11:23.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: T02:11:22.465553+0000 mon.a (mon.0 2023-08-28T02:11:23.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: ) 668 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi169.front.sepia.ceph.com:3000"}]: dispatch 2023-08-28T02:11:23.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: audit 2023-08-28T02:11 2023-08-28T02:11:23.492 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: :22.465904+0000 mgr.y (mgr.14424) 67 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi169.front.sepia.ceph.com:3000"}]: dispatch 2023-08-28T02:11:23.492 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: audit 2023-08-28 2023-08-28T02:11:23.492 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: T02:11:22.469865+0000 mon.a (mon.0) 2023-08-28T02:11:23.492 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: 669 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:23.492 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: audit 2023-08-28T02:11 2023-08-28T02:11:23.492 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:23 smithi169 conmon[100500]: :22.471914+0000 mon.a (mon.0) 670 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:11:24.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:24 smithi143 conmon[109923]: cluster 2023-08-28T02:11:23.191644+0000 mon.a (mon.0) 671 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-08-28T02:11:24.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:24 smithi143 conmon[100258]: cluster 2023-08-28T02:11:23.191644+0000 mon.a 2023-08-28T02:11:24.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:24 smithi143 conmon[100258]: (mon.0) 671 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-08-28T02:11:24.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:24 smithi169 conmon[100500]: cluster 2023-08-28T02:11:23.191644+0000 2023-08-28T02:11:24.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:24 smithi169 conmon[100500]: mon.a (mon.0) 671 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-08-28T02:11:25.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:25 smithi143 conmon[109923]: cluster 2023-08-28T02:11:23.874135+0000 mgr.y (mgr.14424) 68 : cluster [DBG] pgmap v48: 73 pgs: 64 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:25.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:25 smithi143 conmon[109923]: cluster 2023-08-28T02:11:24.195216+0000 mon.a (mon.0) 672 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-08-28T02:11:25.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:25 smithi143 conmon[109923]: audit 2023-08-28T02:11:24.963043+0000 mon.a (mon.0) 673 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:25.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:25 smithi143 conmon[109923]: audit 2023-08-28T02:11:24.999794+0000 mon.a (mon.0) 674 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:25.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:25 smithi143 conmon[100258]: cluster 2023-08-28T02:11:23.874135+0000 mgr.y (mgr.14424) 68 : cluster [DBG] pgmap v48: 73 pgs: 64 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:25.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:25 smithi143 conmon[100258]: cluster 2023-08-28T02:11:24.195216+0000 mon.a (mon.0) 672 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-08-28T02:11:25.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:25 smithi143 conmon[100258]: audit 2023-08-28T02:11:24.963043+0000 mon.a (mon.0) 673 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:25.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:25 smithi143 conmon[100258]: audit 2023-08-28T02:11:24.999794+0000 mon.a (mon.0) 674 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:25.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:25 smithi169 conmon[100500]: cluster 2023-08-28T02:11:23.874135+0000 mgr.y (mgr 2023-08-28T02:11:25.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:25 smithi169 conmon[100500]: .14424) 68 : cluster [DBG] pgmap v48: 73 pgs: 64 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:25.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:25 smithi169 conmon[100500]: cluster 2023-08-28T02:11:24.195216+0000 mon.a (mon.0) 672 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-08-28T02:11:25.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:25 smithi169 conmon[100500]: audit 2023-08-28T02:11:24.963043+0000 mon.a (mon.0) 673 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:25.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:25 smithi169 conmon[100500]: audit 2023-08-28T02:11:24.999794+0000 mon.a (mon.0) 674 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:25.920 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:11:25 smithi143 conmon[150510]: level=info ts=2023-08-28T02:11:25.548Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.002780387s 2023-08-28T02:11:25.942 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2023-08-28T02:11:26.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:26 smithi143 conmon[100258]: audit 2023-08-28T02:11:25.812093+0000 mon.a (mon.0) 2023-08-28T02:11:26.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:26 smithi143 conmon[100258]: 675 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:26.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:26 smithi143 conmon[100258]: cluster 2023-08-28T02:11:25.874510+0000 mgr.y (mgr.14424) 69 : cluster [DBG] pgmap v50: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:26.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:26 smithi143 conmon[100258]: audit 2023-08-28T02:11:26.128294+0000 mon.a (mon.0) 676 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:11:26.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:26 smithi143 conmon[100258]: audit 2023-08-28T02:11:26.128912+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:11:26.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:26 smithi143 conmon[100258]: audit 2023-08-28T02:11:26.133047+0000 mon.a (mon.0) 678 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:26.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:26 smithi143 conmon[100258]: audit 2023-08-28T02:11:26.139777+0000 mon.a (mon.0) 679 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:26.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:26 smithi143 conmon[100258]: audit 2023-08-28T02:11:26.140597+0000 mon.a (mon.0) 680 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi143.spifrj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-08-28T02:11:26.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:26 smithi143 conmon[100258]: audit 2023-08-28T02:11:26.143655+0000 mon.a (mon.0) 681 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi143.spifrj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-08-28T02:11:26.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:26 smithi143 conmon[100258]: audit 2023-08-28T02:11:26.144482+0000 mon.a (mon.0) 682 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:11:26.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:26 smithi143 conmon[100258]: cephadm 2023-08-28T02:11:26.145081+0000 mgr.y (mgr.14424) 70 : cephadm [INF] Deploying daemon mds.foo.smithi143.spifrj on smithi143 2023-08-28T02:11:26.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:26 smithi143 conmon[109923]: audit 2023-08-28T02:11:25.812093+0000 2023-08-28T02:11:26.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:26 smithi143 conmon[109923]: mon.a (mon.0) 675 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:26.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:26 smithi143 conmon[109923]: cluster 2023-08-28T02:11:25 2023-08-28T02:11:26.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:26 smithi143 conmon[109923]: .874510+0000 mgr.y (mgr.14424) 69 : cluster [DBG] pgmap v50: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:26.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:26 smithi143 conmon[109923]: audit 2023-08-28T02:11:26.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:26 smithi143 conmon[109923]: 2023-08-28T02:11: 2023-08-28T02:11:26.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:26 smithi143 conmon[109923]: 26.128294+0000 mon.a (mon. 2023-08-28T02:11:26.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:26 smithi143 conmon[109923]: 0) 676 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:11:26.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:26 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:11:26.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:26 smithi143 conmon[109923]: 08-28T02:11:26.128912 2023-08-28T02:11:26.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:26 smithi143 conmon[109923]: +0000 mon.a (mon.0) 677 : 2023-08-28T02:11:26.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:26 smithi143 conmon[109923]: audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:11:26.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:26 smithi143 conmon[109923]: audit 2023-08-28T02:11:26.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:26 smithi143 conmon[109923]: 2023-08- 2023-08-28T02:11:26.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:26 smithi143 conmon[109923]: 28T02:11:26.133047+0000 mon.a (mon 2023-08-28T02:11:26.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:26 smithi143 conmon[109923]: .0) 678 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:26.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:26 smithi143 conmon[109923]: audit 2023-08 2023-08-28T02:11:26.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:26 smithi143 conmon[109923]: -28T02:11:26.139777+0000 mon.a (mon 2023-08-28T02:11:26.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:26 smithi143 conmon[109923]: .0) 679 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:26.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:26 smithi143 conmon[109923]: audit 2023-08-28T02 2023-08-28T02:11:26.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:26 smithi143 conmon[109923]: :11:26.140597+0000 mon.a (mon.0) 2023-08-28T02:11:26.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:26 smithi143 conmon[109923]: 680 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi143.spifrj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-08-28T02:11:26.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:26 smithi143 conmon[109923]: audit 2023-08-28T02:11: 2023-08-28T02:11:26.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:26 smithi143 conmon[109923]: 26.143655+0000 mon.a (mon.0) 681 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi143.spifrj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-08-28T02:11:26.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:26 smithi143 conmon[109923]: audit 2023-08-28T02:11 2023-08-28T02:11:26.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:26 smithi143 conmon[109923]: :26.144482+0000 mon.a (mon.0) 682 : audit [DBG] 2023-08-28T02:11:26.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:26 smithi143 conmon[109923]: from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:11:26.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:26 smithi143 conmon[109923]: cephadm 2023- 2023-08-28T02:11:26.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:26 smithi143 conmon[109923]: 08-28T02:11:26.145081+0000 mgr.y (mgr.14424) 70 : cephadm [INF] Deploying daemon mds.foo.smithi143.spifrj on smithi143 2023-08-28T02:11:27.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:26 smithi169 conmon[100500]: audit 2023-08-28T02:11:25.812093+0000 mon.a (mon.0) 675 : 2023-08-28T02:11:27.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:26 smithi169 conmon[100500]: audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:27.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:26 smithi169 conmon[100500]: cluster 2023-08-28T02:11:25.874510+0000 mgr.y (mgr.14424) 69 : cluster [DBG] pgmap v50: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:27.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:26 smithi169 conmon[100500]: audit 2023-08-28T02:11:26.128294+0000 mon.a (mon.0) 676 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:11:27.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:26 smithi169 conmon[100500]: audit 2023- 2023-08-28T02:11:27.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:26 smithi169 conmon[100500]: 08-28T02:11:26.128912+0000 mon.a (mon.0) 2023-08-28T02:11:27.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:26 smithi169 conmon[100500]: 677 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:11:27.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:26 smithi169 conmon[100500]: audit 2023-08-28T02:11:26.133047+0000 mon.a 2023-08-28T02:11:27.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:26 smithi169 conmon[100500]: (mon.0) 678 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:27.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:26 smithi169 conmon[100500]: audit 2023-08 2023-08-28T02:11:27.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:26 smithi169 conmon[100500]: -28T02:11:26.139777+0000 mon.a (mon.0 2023-08-28T02:11:27.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:26 smithi169 conmon[100500]: ) 679 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:27.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:26 smithi169 conmon[100500]: audit 2023-08-28 2023-08-28T02:11:27.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:26 smithi169 conmon[100500]: T02:11:26.140597+0000 mon.a (mon.0) 680 2023-08-28T02:11:27.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:26 smithi169 conmon[100500]: : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi143.spifrj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-08-28T02:11:27.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:26 smithi169 conmon[100500]: audit 2023-08-28T02: 2023-08-28T02:11:27.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:26 smithi169 conmon[100500]: 11:26.143655+0000 mon.a (mon.0) 681 : 2023-08-28T02:11:27.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:26 smithi169 conmon[100500]: audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi143.spifrj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-08-28T02:11:27.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:26 smithi169 conmon[100500]: audit 2023-08-28T02:11: 2023-08-28T02:11:27.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:26 smithi169 conmon[100500]: 26.144482+0000 mon.a (mon.0) 682 : audit [DBG] 2023-08-28T02:11:27.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:26 smithi169 conmon[100500]: from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:11:27.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:26 smithi169 conmon[100500]: cephadm 2023-08-28T02:11:26. 2023-08-28T02:11:27.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:26 smithi169 conmon[100500]: 145081+0000 mgr.y (mgr.14424) 70 : cephadm [INF] Deploying daemon mds.foo.smithi143.spifrj on smithi143 2023-08-28T02:11:27.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:26 smithi169 conmon[100500]: 2023-08-28T02:11:28.638 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-08-28T02:11:28.648 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-08-28T02:11:28.648 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-08-28T02:11:28.648 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph mgr dump --format=json 2023-08-28T02:11:29.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:28 smithi143 conmon[100258]: cluster 2023-08-28T02:11 2023-08-28T02:11:29.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:28 smithi143 conmon[100258]: :27.874837+0000 mgr.y (mgr.14424) 71 : cluster [DBG] pgmap v51: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:29.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:28 smithi143 conmon[109923]: cluster 2023-08-28T02:11:27.874837 2023-08-28T02:11:29.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:28 smithi143 conmon[109923]: +0000 mgr.y (mgr.14424) 71 : cluster [DBG] pgmap v51: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:29.230 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:11:29.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:28 smithi169 conmon[100500]: cluster 2023-08-28T02:11:27.874837+0000 mgr.y (mgr.14424) 71 : 2023-08-28T02:11:29.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:28 smithi169 conmon[100500]: cluster [DBG] pgmap v51: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:30.524 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[100708]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:11:30] "GET /metrics HTTP/1.1" 200 160686 "" "Prometheus/2.33.4" 2023-08-28T02:11:30.524 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[109923]: audit 2023-08-28T02:11:29 2023-08-28T02:11:30.524 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[109923]: .278187+0000 mon.a (mon.0) 683 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:30.524 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[109923]: audit 2023-08-28T02:11:30.524 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[109923]: 2023-08-28T02:11:29.279256+0000 mon.a (mon.0) 684 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi169.yeisur", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-08-28T02:11:30.524 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[109923]: 2023-08-28T02:11:30.525 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[109923]: audit 2023-08- 2023-08-28T02:11:30.525 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[109923]: 28T02:11:29.283155+0000 mon.a (mon.0) 685 2023-08-28T02:11:30.525 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[109923]: : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi169.yeisur", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-08-28T02:11:30.525 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[109923]: audit 2023-08-28T02:11:30.525 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[109923]: 2023-08-28T02:11 2023-08-28T02:11:30.525 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[109923]: :29.284136+0000 mon.a (mon.0) 686 2023-08-28T02:11:30.525 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[109923]: : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:11:30.525 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[109923]: cephadm 2023-08-28T 2023-08-28T02:11:30.525 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[109923]: 02:11:29.284897+0000 mgr.y (mgr.14424) 72 2023-08-28T02:11:30.525 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[109923]: : cephadm [INF] Deploying daemon mds.foo.smithi169.yeisur on smithi169 2023-08-28T02:11:30.525 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[109923]: cluster 2023-08-28T02:11:30.525 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[109923]: 2023-08-28T02:11:29.931862+0000 2023-08-28T02:11:30.526 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[109923]: mon.a (mon.0) 687 : cluster [DBG] mds.? [v2:172.21.15.143:6834/2211284212,v1:172.21.15.143:6835/2211284212] up:boot 2023-08-28T02:11:30.526 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[109923]: cluster 2023 2023-08-28T02:11:30.526 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[109923]: -08-28T02:11:29.931915+0000 mon.a ( 2023-08-28T02:11:30.526 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[109923]: mon.0) 688 : cluster [INF] daemon mds.foo.smithi143.spifrj assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-08-28T02:11:30.526 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[109923]: cluster 2023-08- 2023-08-28T02:11:30.526 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[109923]: 28T02:11:29.931976+0000 2023-08-28T02:11:30.526 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[109923]: mon.a (mon.0) 689 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-08-28T02:11:30.526 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[109923]: 2023-08-28T02:11:30.526 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[109923]: cluster 2023-08-28T02:11:30.526 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[109923]: 2023-08-28T02:11:29.931998+0000 mon.a (mon.0 2023-08-28T02:11:30.526 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[109923]: ) 690 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-08-28T02:11:30.526 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[109923]: cluster 2023-08-28T02:11:30.526 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[109923]: 2023-08-28T02: 2023-08-28T02:11:30.527 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[109923]: 11:29.932022+0000 mon.a (mon. 2023-08-28T02:11:30.527 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[109923]: 0) 691 : cluster [INF] Cluster is now healthy 2023-08-28T02:11:30.527 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[109923]: cluster 2023-08-28T02:11:30.527 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[109923]: 2023-08-28T02:11:29.932475+0000 mon.a ( 2023-08-28T02:11:30.527 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[109923]: mon.0) 692 : cluster [DBG] fsmap foo:0 1 up:standby 2023-08-28T02:11:30.527 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[109923]: audit 2023-08-28 2023-08-28T02:11:30.527 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[109923]: T02:11:29.932601+0000 mon.a (mon.0) 693 : audit 2023-08-28T02:11:30.527 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[109923]: [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi143.spifrj"}]: dispatch 2023-08-28T02:11:30.527 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[109923]: cluster 2023-08-28 2023-08-28T02:11:30.527 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[109923]: T02:11:29.937706+0000 mon.a (mon.0) 694 : cluster [DBG] fsmap foo:1 {0=foo.smithi143.spifrj=up:creating} 2023-08-28T02:11:30.527 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[109923]: cluster 2023-08-28T02:11:29.946953+0000 mon.a (mon.0 2023-08-28T02:11:30.527 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[109923]: ) 695 : cluster [INF] daemon mds.foo.smithi143.spifrj is now active in filesystem foo as rank 0 2023-08-28T02:11:30.528 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[100258]: audit 2023-08-28T02:11:29.278187+0000 mon.a (mon.0) 683 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:30.528 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[100258]: audit 2023-08-28T02:11:30.528 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[100258]: 2023-08-28T02:11:29.279256+0000 mon.a (mon.0) 684 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi169.yeisur", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-08-28T02:11:30.528 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[100258]: audit 2023-08-28T02:11:29.283155+0000 mon.a (mon.0) 685 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi169.yeisur", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-08-28T02:11:30.528 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[100258]: audit 2023-08-28T02:11:29 2023-08-28T02:11:30.528 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[100258]: .284136+0000 mon.a (mon.0) 686 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:11:30.528 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[100258]: cephadm 2023 2023-08-28T02:11:30.528 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[100258]: -08-28T02:11:29.284897+0000 mgr.y 2023-08-28T02:11:30.528 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[100258]: (mgr.14424) 72 : cephadm [INF] Deploying daemon mds.foo.smithi169.yeisur on smithi169 2023-08-28T02:11:30.528 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[100258]: cluster 2023-08-28T02:11:30.528 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[100258]: 2023-08-28T02:11:29.931862+0000 mon.a 2023-08-28T02:11:30.529 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[100258]: (mon.0) 687 : cluster [DBG] mds.? [v2:172.21.15.143:6834/2211284212,v1:172.21.15.143:6835/2211284212] up:boot 2023-08-28T02:11:30.529 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[100258]: cluster 2023-08- 2023-08-28T02:11:30.529 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[100258]: 28T02:11:29.931915+0000 mon.a (mon.0 2023-08-28T02:11:30.529 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[100258]: ) 688 : cluster [INF] daemon mds.foo.smithi143.spifrj assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-08-28T02:11:30.529 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[100258]: cluster 2023-08-28T02:11:30.529 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[100258]: 2023-08-28T02:11:29.931976+0000 2023-08-28T02:11:30.529 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[100258]: mon.a (mon.0) 689 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-08-28T02:11:30.529 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[100258]: cluster 2023-08-28T02:11:30.529 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[100258]: 2023-08-28T02:11:29.931998 2023-08-28T02:11:30.529 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[100258]: +0000 mon.a (mon.0) 690 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-08-28T02:11:30.529 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[100258]: 2023-08-28T02:11:30.529 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[100258]: cluster 2023-08-28T02:11:30.529 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[100258]: 2023-08-28T02:11:29.932022+0000 mon.a (mon.0 2023-08-28T02:11:30.530 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[100258]: ) 691 : cluster [INF] Cluster is now healthy 2023-08-28T02:11:30.530 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[100258]: cluster 2023-08-28T02 2023-08-28T02:11:30.530 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[100258]: :11:29.932475+0000 mon.a (mon.0) 692 2023-08-28T02:11:30.530 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[100258]: : cluster [DBG] fsmap foo:0 1 up:standby 2023-08-28T02:11:30.530 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[100258]: audit 2023-08-28T 2023-08-28T02:11:30.530 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[100258]: 02:11:29.932601+0000 mon.a (mon.0) 693 2023-08-28T02:11:30.530 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[100258]: : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi143.spifrj"}]: dispatch 2023-08-28T02:11:30.530 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[100258]: cluster 2023-08-28T 2023-08-28T02:11:30.530 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[100258]: 02:11:29.937706+0000 mon.a (mon.0) 2023-08-28T02:11:30.530 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[100258]: 694 : cluster [DBG] fsmap foo:1 {0=foo.smithi143.spifrj=up:creating} 2023-08-28T02:11:30.530 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[100258]: cluster 2023-08-28T02:11:29.946953+0000 2023-08-28T02:11:30.530 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:30 smithi143 conmon[100258]: mon.a (mon.0) 695 : cluster [INF] daemon mds.foo.smithi143.spifrj is now active in filesystem foo as rank 0 2023-08-28T02:11:30.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:30 smithi169 conmon[100500]: audit 2023-08-28T02:11:29.278187+0000 mon.a (mon.0) 683 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:30.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:30 smithi169 conmon[100500]: audit 2023-08-28T02:11:29.279256+0000 mon.a (mon.0) 684 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi169.yeisur", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-08-28T02:11:30.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:30 smithi169 conmon[100500]: audit 2023-08-28T02:11:29.283155+0000 mon.a (mon.0) 685 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi169.yeisur", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-08-28T02:11:30.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:30 smithi169 conmon[100500]: audit 2023-08-28T02:11:29.284136+0000 mon.a (mon.0) 686 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:11:30.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:30 smithi169 conmon[100500]: cephadm 2023-08-28T02:11:29.284897+0000 mgr.y (mgr.14424) 72 : cephadm [INF] Deploying daemon mds.foo.smithi169.yeisur on smithi169 2023-08-28T02:11:30.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:30 smithi169 conmon[100500]: cluster 2023-08-28T02:11:29.931862+0000 mon.a (mon.0) 687 : cluster [DBG] mds.? [v2:172.21.15.143:6834/2211284212,v1:172.21.15.143:6835/2211284212] up:boot 2023-08-28T02:11:30.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:30 smithi169 conmon[100500]: cluster 2023-08-28T02:11:30.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:30 smithi169 conmon[100500]: 2023-08-28T02:11:29.931915+0000 mon.a (mon.0) 688 : cluster [INF] daemon mds.foo.smithi143.spifrj assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-08-28T02:11:30.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:30 smithi169 conmon[100500]: cluster 2023-08-28T02:11:29.931976+0000 mon.a (mon.0) 689 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-08-28T02:11:30.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:30 smithi169 conmon[100500]: cluster 2023-08-28T02:11:29.931998+0000 mon.a (mon.0) 690 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-08-28T02:11:30.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:30 smithi169 conmon[100500]: cluster 2023-08-28T02:11:29.932022+0000 mon.a (mon.0) 691 : cluster [INF] Cluster is now healthy 2023-08-28T02:11:30.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:30 smithi169 conmon[100500]: cluster 2023-08-28T02:11:29.932475+0000 mon.a (mon.0) 692 : cluster [DBG] fsmap foo:0 1 up:standby 2023-08-28T02:11:30.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:30 smithi169 conmon[100500]: audit 2023-08-28T02:11:29.932601+0000 mon.a (mon.0) 693 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi143.spifrj"}]: dispatch 2023-08-28T02:11:30.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:30 smithi169 conmon[100500]: cluster 2023-08-28T02:11:29.937706+0000 mon.a (mon.0) 694 : cluster [DBG] fsmap foo:1 {0=foo.smithi143.spifrj=up:creating} 2023-08-28T02:11:30.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:30 smithi169 conmon[100500]: cluster 2023-08-28T02:11:29.946953+0000 mon.a (mon.0) 695 : cluster [INF] daemon mds.foo.smithi143.spifrj is now active in filesystem foo as rank 0 2023-08-28T02:11:30.909 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:11:31.541 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:31 smithi143 conmon[100258]: cluster 2023-08-28T02:11:29.875194+0000 mgr.y (mgr.14424) 2023-08-28T02:11:31.541 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:31 smithi143 conmon[100258]: 73 : cluster [DBG] pgmap v52: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:31.541 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:31 smithi143 conmon[100258]: audit 2023-08-28T02:11:30.905681+0000 mon.a (mon.0) 696 : audit [DBG] from='client.? 172.21.15.143:0/3153728912' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-08-28T02:11:31.541 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:31 smithi143 conmon[100258]: cluster 2023-08-28T02:11:30.938159+0000 mon.a (mon.0) 697 : cluster [DBG] mds.? [v2:172.21.15.143:6834/2211284212,v1:172.21.15.143:6835/2211284212] up:active 2023-08-28T02:11:31.541 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:31 smithi143 conmon[100258]: cluster 2023-08-28T02:11:30.938211+0000 mon.a (mon.0) 698 : cluster [DBG] fsmap foo:1 {0=foo.smithi143.spifrj=up:active} 2023-08-28T02:11:31.542 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:31 smithi143 conmon[109923]: cluster 2023-08-28T02:11:29.875194+0000 mgr.y 2023-08-28T02:11:31.542 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:31 smithi143 conmon[109923]: (mgr.14424) 73 : cluster [DBG] pgmap v52: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:31.542 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:31 smithi143 conmon[109923]: audit 2023-08-28T02:11:30.905681+0000 mon.a (mon.0) 696 : audit [DBG] from='client.? 172.21.15.143:0/3153728912' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-08-28T02:11:31.542 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:31 smithi143 conmon[109923]: cluster 2023-08-28T02:11:30.938159+0000 mon.a (mon.0) 697 : cluster [DBG] mds.? [v2:172.21.15.143:6834/2211284212,v1:172.21.15.143:6835/2211284212] up:active 2023-08-28T02:11:31.542 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:31 smithi143 conmon[109923]: cluster 2023-08-28T02:11:30.938211+0000 mon.a (mon.0) 698 : cluster [DBG] fsmap foo:1 {0=foo.smithi143.spifrj=up:active} 2023-08-28T02:11:31.558 INFO:teuthology.orchestra.run.smithi143.stdout:{"epoch":18,"active_gid":14424,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6800","nonce":1118800746},{"type":"v1","addr":"172.21.15.143:6801","nonce":1118800746}]},"active_addr":"172.21.15.143:6801/1118800746","active_change":"2023-08-28T02:10:09.846041+0000","active_mgr_features":4540138314316775423,"available":true,"standbys":[{"gid":24328,"name":"x","mgr_features":4540138314316775423,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.143:8443/","prometheus":"http://172.21.15.143:9283/"},"always_on_modules":{"nautilus":["balancer","crash","devicehealth","orchestrator_cli","progress","rbd_support","status","volumes"],"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":51,"active_clients":[{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.143:0","nonce":1035910192}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.143:0","nonce":3251765285}]}]} 2023-08-28T02:11:31.564 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-08-28T02:11:31.564 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-08-28T02:11:31.564 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd dump --format=json 2023-08-28T02:11:31.595 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:31 smithi169 conmon[100500]: cluster 2023-08-28T02:11:29.875194+0000 mgr.y (mgr.14424) 73 : cluster [DBG] pgmap v52: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:31.595 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:31 smithi169 conmon[100500]: audit 2023-08-28T02:11:30.905681+0000 mon.a (mon.0) 696 : audit [DBG] from='client.? 172.21.15.143:0/3153728912' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-08-28T02:11:31.595 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:31 smithi169 conmon[100500]: cluster 2023-08-28T02:11:30.938159+0000 mon.a (mon.0) 697 : cluster [DBG] mds.? [v2:172.21.15.143:6834/2211284212,v1:172.21.15.143:6835/2211284212] up:active 2023-08-28T02:11:31.595 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:31 smithi169 conmon[100500]: cluster 2023-08-28T02:11:30.938211+0000 mon.a (mon.0) 698 : cluster [DBG] fsmap foo:1 {0=foo.smithi143.spifrj=up:active} 2023-08-28T02:11:31.922 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:11:33.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:32 smithi143 conmon[100258]: audit 2023-08-28T02:11:31.788868+0000 mon.a (mon.0) 699 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:33.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:32 smithi143 conmon[100258]: audit 2023-08-28T02:11:31.790052+0000 mon.a (mon.0) 700 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:11:33.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:32 smithi143 conmon[100258]: cluster 2023-08-28T02:11:31.875639+0000 mgr.y (mgr.14424) 74 : cluster [DBG] pgmap v53: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:33.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:32 smithi143 conmon[100258]: cluster 2023-08-28T02:11:32.287280+0000 mon.a (mon.0) 701 : cluster [DBG] mds.? [v2:172.21.15.169:6832/3278371000,v1:172.21.15.169:6833/3278371000] up:boot 2023-08-28T02:11:33.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:32 smithi143 conmon[100258]: cluster 2023-08-28T02:11:32.287563+0000 mon.a (mon.0) 702 : cluster [DBG] fsmap foo:1 {0=foo.smithi143.spifrj=up:active} 1 up:standby 2023-08-28T02:11:33.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:32 smithi143 conmon[100258]: audit 2023-08-28T02:11:32.287654+0000 mon.a (mon.0) 703 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi169.yeisur"}]: dispatch 2023-08-28T02:11:33.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:32 smithi143 conmon[100258]: cluster 2023-08-28T02:11:32.292492+0000 mon.a (mon.0) 704 : cluster [DBG] fsmap foo:1 {0=foo.smithi143.spifrj=up:active} 1 up:standby 2023-08-28T02:11:33.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:32 smithi143 conmon[109923]: audit 2023-08-28T02:11:31.788868+0000 mon.a (mon.0) 699 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:33.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:32 smithi143 conmon[109923]: audit 2023-08-28T02:11:31.790052+0000 mon.a (mon.0) 700 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:11:33.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:32 smithi143 conmon[109923]: cluster 2023-08-28T02:11:31.875639+0000 mgr.y (mgr.14424) 74 : cluster [DBG] pgmap v53: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:33.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:32 smithi143 conmon[109923]: cluster 2023-08-28T02:11:32.287280+0000 mon.a (mon.0) 701 : cluster [DBG] mds.? [v2:172.21.15.169:6832/3278371000,v1:172.21.15.169:6833/3278371000] up:boot 2023-08-28T02:11:33.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:32 smithi143 conmon[109923]: cluster 2023-08-28T02:11:32.287563+0000 mon.a (mon.0) 702 : cluster [DBG] fsmap foo:1 {0=foo.smithi143.spifrj=up:active} 1 up:standby 2023-08-28T02:11:33.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:32 smithi143 conmon[109923]: audit 2023-08-28T02:11 2023-08-28T02:11:33.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:32 smithi143 conmon[109923]: :32.287654+0000 mon.a (mon.0) 703 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi169.yeisur"}]: dispatch 2023-08-28T02:11:33.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:32 smithi143 conmon[109923]: cluster 2023-08-28T02:11:32.292492+0000 mon.a (mon.0) 704 : cluster [DBG] fsmap foo:1 {0=foo.smithi143.spifrj=up:active} 1 up:standby 2023-08-28T02:11:33.237 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:11:32 smithi169 conmon[102807]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:11:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-28T02:11:33.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:32 smithi169 conmon[100500]: audit 2023-08-28T02:11:31.788868 2023-08-28T02:11:33.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:32 smithi169 conmon[100500]: +0000 mon.a (mon.0) 699 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:33.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:32 smithi169 conmon[100500]: audit 2023-08-28T02:11:31.790052+0000 mon.a (mon.0) 700 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:11:33.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:32 smithi169 conmon[100500]: cluster 2023-08-28T02:11:31.875639+0000 mgr.y (mgr.14424) 74 : cluster [DBG] pgmap v53: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:33.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:32 smithi169 conmon[100500]: cluster 2023-08-28T02:11:32.287280+0000 mon.a (mon.0) 701 : cluster [DBG] mds.? [v2:172.21.15.169:6832/3278371000,v1:172.21.15.169:6833/3278371000] up:boot 2023-08-28T02:11:33.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:32 smithi169 conmon[100500]: cluster 2023-08-28T02:11:32.287563+0000 mon.a (mon.0) 702 : cluster [DBG] fsmap foo:1 {0=foo.smithi143.spifrj=up:active} 1 up:standby 2023-08-28T02:11:33.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:32 smithi169 conmon[100500]: audit 2023-08-28T02:11:32.287654+0000 mon.a (mon.0) 703 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi169.yeisur"}]: dispatch 2023-08-28T02:11:33.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:32 smithi169 conmon[100500]: cluster 2023-08-28T02:11:32.292492+0000 mon.a (mon.0) 704 : cluster [DBG] fsmap foo:1 {0=foo.smithi143.spifrj=up:active} 1 up:standby 2023-08-28T02:11:33.348 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:11:33.348 INFO:teuthology.orchestra.run.smithi143.stdout:{"epoch":60,"fsid":"2fcb0bfa-4547-11ee-817c-1b7a5f1ba315","created":"2023-08-28T02:05:20.056816+0000","modified":"2023-08-28T02:11:24.189318+0000","last_up_change":"2023-08-28T02:09:53.076490+0000","last_in_change":"2023-08-28T02:09:35.717712+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-08-28T02:08:01.318811+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-08-28T02:10:30.618550+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-08-28T02:11:20.682059+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-08-28T02:11:21.407431+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"718c8f27-245b-4e9d-9e2e-d1fc65ca7472","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6802","nonce":1846477664},{"type":"v1","addr":"172.21.15.143:6803","nonce":1846477664}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6804","nonce":1846477664},{"type":"v1","addr":"172.21.15.143:6805","nonce":1846477664}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6808","nonce":1846477664},{"type":"v1","addr":"172.21.15.143:6809","nonce":1846477664}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6806","nonce":1846477664},{"type":"v1","addr":"172.21.15.143:6807","nonce":1846477664}]},"public_addr":"172.21.15.143:6803/1846477664","cluster_addr":"172.21.15.143:6805/1846477664","heartbeat_back_addr":"172.21.15.143:6809/1846477664","heartbeat_front_addr":"172.21.15.143:6807/1846477664","state":["exists","up"]},{"osd":1,"uuid":"cf24771f-3b0c-4a00-8a73-13e1fee242dc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6810","nonce":2341769896},{"type":"v1","addr":"172.21.15.143:6811","nonce":2341769896}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6812","nonce":2341769896},{"type":"v1","addr":"172.21.15.143:6813","nonce":2341769896}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6816","nonce":2341769896},{"type":"v1","addr":"172.21.15.143:6817","nonce":2341769896}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6814","nonce":2341769896},{"type":"v1","addr":"172.21.15.143:6815","nonce":2341769896}]},"public_addr":"172.21.15.143:6811/2341769896","cluster_addr":"172.21.15.143:6813/2341769896","heartbeat_back_addr":"172.21.15.143:6817/2341769896","heartbeat_front_addr":"172.21.15.143:6815/2341769896","state":["exists","up"]},{"osd":2,"uuid":"4f8492d1-0e7a-49f8-b6a5-4f5e73842349","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6818","nonce":2664999316},{"type":"v1","addr":"172.21.15.143:6819","nonce":2664999316}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6820","nonce":2664999316},{"type":"v1","addr":"172.21.15.143:6821","nonce":2664999316}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6824","nonce":2664999316},{"type":"v1","addr":"172.21.15.143:6825","nonce":2664999316}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6822","nonce":2664999316},{"type":"v1","addr":"172.21.15.143:6823","nonce":2664999316}]},"public_addr":"172.21.15.143:6819/2664999316","cluster_addr":"172.21.15.143:6821/2664999316","heartbeat_back_addr":"172.21.15.143:6825/2664999316","heartbeat_front_addr":"172.21.15.143:6823/2664999316","state":["exists","up"]},{"osd":3,"uuid":"dd6abc8a-54a9-4e08-ba81-0b894f24d84a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6826","nonce":1468263600},{"type":"v1","addr":"172.21.15.143:6827","nonce":1468263600}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6828","nonce":1468263600},{"type":"v1","addr":"172.21.15.143:6829","nonce":1468263600}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6832","nonce":1468263600},{"type":"v1","addr":"172.21.15.143:6833","nonce":1468263600}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6830","nonce":1468263600},{"type":"v1","addr":"172.21.15.143:6831","nonce":1468263600}]},"public_addr":"172.21.15.143:6827/1468263600","cluster_addr":"172.21.15.143:6829/1468263600","heartbeat_back_addr":"172.21.15.143:6833/1468263600","heartbeat_front_addr":"172.21.15.143:6831/1468263600","state":["exists","up"]},{"osd":4,"uuid":"16122219-d985-46d2-bac1-f8ffcf48d8ad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6800","nonce":3960017153},{"type":"v1","addr":"172.21.15.169:6801","nonce":3960017153}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6802","nonce":3960017153},{"type":"v1","addr":"172.21.15.169:6803","nonce":3960017153}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6806","nonce":3960017153},{"type":"v1","addr":"172.21.15.169:6807","nonce":3960017153}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6804","nonce":3960017153},{"type":"v1","addr":"172.21.15.169:6805","nonce":3960017153}]},"public_addr":"172.21.15.169:6801/3960017153","cluster_addr":"172.21.15.169:6803/3960017153","heartbeat_back_addr":"172.21.15.169:6807/3960017153","heartbeat_front_addr":"172.21.15.169:6805/3960017153","state":["exists","up"]},{"osd":5,"uuid":"adbc3ea5-230f-419e-8f6f-99763378b296","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6808","nonce":4087672846},{"type":"v1","addr":"172.21.15.169:6809","nonce":4087672846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6810","nonce":4087672846},{"type":"v1","addr":"172.21.15.169:6811","nonce":4087672846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6814","nonce":4087672846},{"type":"v1","addr":"172.21.15.169:6815","nonce":4087672846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6812","nonce":4087672846},{"type":"v1","addr":"172.21.15.169:6813","nonce":4087672846}]},"public_addr":"172.21.15.169:6809/4087672846","cluster_addr":"172.21.15.169:6811/4087672846","heartbeat_back_addr":"172.21.15.169:6815/4087672846","heartbeat_front_addr":"172.21.15.169:6813/4087672846","state":["exists","up"]},{"osd":6,"uuid":"2abe4d77-c12c-477f-a4ec-eabb753ca4dd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6816","nonce":3801227982},{"type":"v1","addr":"172.21.15.169:6817","nonce":3801227982}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6818","nonce":3801227982},{"type":"v1","addr":"172.21.15.169:6819","nonce":3801227982}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6822","nonce":3801227982},{"type":"v1","addr":"172.21.15.169:6823","nonce":3801227982}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6820","nonce":3801227982},{"type":"v1","addr":"172.21.15.169:6821","nonce":3801227982}]},"public_addr":"172.21.15.169:6817/3801227982","cluster_addr":"172.21.15.169:6819/3801227982","heartbeat_back_addr":"172.21.15.169:6823/3801227982","heartbeat_front_addr":"172.21.15.169:6821/3801227982","state":["exists","up"]},{"osd":7,"uuid":"fc5ed8fc-8ad7-4b54-a8ea-bc6a9e846f63","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6824","nonce":3830851149},{"type":"v1","addr":"172.21.15.169:6825","nonce":3830851149}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6826","nonce":3830851149},{"type":"v1","addr":"172.21.15.169:6827","nonce":3830851149}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6830","nonce":3830851149},{"type":"v1","addr":"172.21.15.169:6831","nonce":3830851149}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6828","nonce":3830851149},{"type":"v1","addr":"172.21.15.169:6829","nonce":3830851149}]},"public_addr":"172.21.15.169:6825/3830851149","cluster_addr":"172.21.15.169:6827/3830851149","heartbeat_back_addr":"172.21.15.169:6831/3830851149","heartbeat_front_addr":"172.21.15.169:6829/3830851149","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:07:16.166767+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:07:37.634849+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:07:59.099590+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:08:21.737546+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:08:43.377647+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:09:06.116513+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:09:28.557330+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:09:51.159131+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.143:0/1563442416":"2023-08-29T02:10:09.845824+0000","172.21.15.143:6801/1828180492":"2023-08-29T02:10:09.845824+0000","172.21.15.143:0/2157302459":"2023-08-29T02:10:09.845824+0000","172.21.15.143:6800/1828180492":"2023-08-29T02:10:09.845824+0000","172.21.15.143:6801/907474889":"2023-08-29T02:06:02.911068+0000","172.21.15.143:6800/907474889":"2023-08-29T02:06:02.911068+0000","172.21.15.143:0/2323551826":"2023-08-29T02:06:02.911068+0000","172.21.15.143:6800/1019944263":"2023-08-29T02:05:41.797747+0000","172.21.15.143:0/3606471716":"2023-08-29T02:06:02.911068+0000","172.21.15.143:6801/1019944263":"2023-08-29T02:05:41.797747+0000","172.21.15.143:0/11141072":"2023-08-29T02:05:41.797747+0000","172.21.15.143:0/1148418770":"2023-08-29T02:05:41.797747+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-08-28T02:11:33.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:33 smithi143 conmon[100258]: audit 2023-08-28T02:11:33.348454+0000 mon.a ( 2023-08-28T02:11:33.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:33 smithi143 conmon[100258]: mon.0) 705 : audit [DBG] from='client.? 172.21.15.143:0/4290571182' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-28T02:11:33.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:33 smithi143 conmon[109923]: audit 2023-08-28T02:11: 2023-08-28T02:11:33.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:33 smithi143 conmon[109923]: 33.348454+0000 mon.a (mon.0) 705 : audit [DBG] from='client.? 172.21.15.143:0/4290571182' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-28T02:11:34.076 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-08-28T02:11:34.076 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd dump --format=json 2023-08-28T02:11:34.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:33 smithi169 conmon[100500]: audit 2023-08-28T02:11:33.348454+0000 mon.a (mon.0) 705 : audit [DBG] from='client.? 172.21.15.143:0/4290571182' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-28T02:11:34.564 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:11:35.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:35 smithi143 conmon[100258]: cluster 2023-08-28T02:11:33.875954+0000 mgr.y (mgr.14424) 75 : cluster [DBG] pgmap v54: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:35.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:35 smithi143 conmon[100258]: audit 2023-08-28T02:11:34.384950+0000 mon.a (mon.0) 706 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:35.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:35 smithi143 conmon[100258]: audit 2023-08-28T02:11:35.035776+0000 mon.a (mon.0) 707 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:35.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:35 smithi143 conmon[109923]: cluster 2023-08-28T02:11:33.875954+0000 mgr.y (mgr.14424) 75 : cluster [DBG] pgmap v54: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:35.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:35 smithi143 conmon[109923]: audit 2023-08- 2023-08-28T02:11:35.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:35 smithi143 conmon[109923]: 28T02:11:34.384950+0000 mon.a (mon.0) 706 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:35.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:35 smithi143 conmon[109923]: audit 2023-08-28T02:11:35 2023-08-28T02:11:35.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:35 smithi143 conmon[109923]: .035776+0000 mon.a (mon.0) 707 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:35.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:35 smithi169 conmon[100500]: cluster 2023-08-28T02:11:33.875954+0000 mgr.y (mgr.14424) 75 : cluster [DBG] pgmap v54: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:35.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:35 smithi169 conmon[100500]: audit 2023-08-28T02:11:34.384950+0000 mon.a (mon.0) 706 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:35.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:35 smithi169 conmon[100500]: audit 2023-08-28T02:11:35.035776+0000 mon.a (mon.0) 707 : audit 2023-08-28T02:11:35.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:35 smithi169 conmon[100500]: [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:36.473 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:11:36.473 INFO:teuthology.orchestra.run.smithi143.stdout:{"epoch":60,"fsid":"2fcb0bfa-4547-11ee-817c-1b7a5f1ba315","created":"2023-08-28T02:05:20.056816+0000","modified":"2023-08-28T02:11:24.189318+0000","last_up_change":"2023-08-28T02:09:53.076490+0000","last_in_change":"2023-08-28T02:09:35.717712+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-08-28T02:08:01.318811+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-08-28T02:10:30.618550+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-08-28T02:11:20.682059+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-08-28T02:11:21.407431+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"718c8f27-245b-4e9d-9e2e-d1fc65ca7472","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6802","nonce":1846477664},{"type":"v1","addr":"172.21.15.143:6803","nonce":1846477664}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6804","nonce":1846477664},{"type":"v1","addr":"172.21.15.143:6805","nonce":1846477664}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6808","nonce":1846477664},{"type":"v1","addr":"172.21.15.143:6809","nonce":1846477664}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6806","nonce":1846477664},{"type":"v1","addr":"172.21.15.143:6807","nonce":1846477664}]},"public_addr":"172.21.15.143:6803/1846477664","cluster_addr":"172.21.15.143:6805/1846477664","heartbeat_back_addr":"172.21.15.143:6809/1846477664","heartbeat_front_addr":"172.21.15.143:6807/1846477664","state":["exists","up"]},{"osd":1,"uuid":"cf24771f-3b0c-4a00-8a73-13e1fee242dc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6810","nonce":2341769896},{"type":"v1","addr":"172.21.15.143:6811","nonce":2341769896}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6812","nonce":2341769896},{"type":"v1","addr":"172.21.15.143:6813","nonce":2341769896}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6816","nonce":2341769896},{"type":"v1","addr":"172.21.15.143:6817","nonce":2341769896}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6814","nonce":2341769896},{"type":"v1","addr":"172.21.15.143:6815","nonce":2341769896}]},"public_addr":"172.21.15.143:6811/2341769896","cluster_addr":"172.21.15.143:6813/2341769896","heartbeat_back_addr":"172.21.15.143:6817/2341769896","heartbeat_front_addr":"172.21.15.143:6815/2341769896","state":["exists","up"]},{"osd":2,"uuid":"4f8492d1-0e7a-49f8-b6a5-4f5e73842349","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6818","nonce":2664999316},{"type":"v1","addr":"172.21.15.143:6819","nonce":2664999316}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6820","nonce":2664999316},{"type":"v1","addr":"172.21.15.143:6821","nonce":2664999316}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6824","nonce":2664999316},{"type":"v1","addr":"172.21.15.143:6825","nonce":2664999316}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6822","nonce":2664999316},{"type":"v1","addr":"172.21.15.143:6823","nonce":2664999316}]},"public_addr":"172.21.15.143:6819/2664999316","cluster_addr":"172.21.15.143:6821/2664999316","heartbeat_back_addr":"172.21.15.143:6825/2664999316","heartbeat_front_addr":"172.21.15.143:6823/2664999316","state":["exists","up"]},{"osd":3,"uuid":"dd6abc8a-54a9-4e08-ba81-0b894f24d84a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6826","nonce":1468263600},{"type":"v1","addr":"172.21.15.143:6827","nonce":1468263600}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6828","nonce":1468263600},{"type":"v1","addr":"172.21.15.143:6829","nonce":1468263600}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6832","nonce":1468263600},{"type":"v1","addr":"172.21.15.143:6833","nonce":1468263600}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6830","nonce":1468263600},{"type":"v1","addr":"172.21.15.143:6831","nonce":1468263600}]},"public_addr":"172.21.15.143:6827/1468263600","cluster_addr":"172.21.15.143:6829/1468263600","heartbeat_back_addr":"172.21.15.143:6833/1468263600","heartbeat_front_addr":"172.21.15.143:6831/1468263600","state":["exists","up"]},{"osd":4,"uuid":"16122219-d985-46d2-bac1-f8ffcf48d8ad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6800","nonce":3960017153},{"type":"v1","addr":"172.21.15.169:6801","nonce":3960017153}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6802","nonce":3960017153},{"type":"v1","addr":"172.21.15.169:6803","nonce":3960017153}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6806","nonce":3960017153},{"type":"v1","addr":"172.21.15.169:6807","nonce":3960017153}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6804","nonce":3960017153},{"type":"v1","addr":"172.21.15.169:6805","nonce":3960017153}]},"public_addr":"172.21.15.169:6801/3960017153","cluster_addr":"172.21.15.169:6803/3960017153","heartbeat_back_addr":"172.21.15.169:6807/3960017153","heartbeat_front_addr":"172.21.15.169:6805/3960017153","state":["exists","up"]},{"osd":5,"uuid":"adbc3ea5-230f-419e-8f6f-99763378b296","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6808","nonce":4087672846},{"type":"v1","addr":"172.21.15.169:6809","nonce":4087672846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6810","nonce":4087672846},{"type":"v1","addr":"172.21.15.169:6811","nonce":4087672846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6814","nonce":4087672846},{"type":"v1","addr":"172.21.15.169:6815","nonce":4087672846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6812","nonce":4087672846},{"type":"v1","addr":"172.21.15.169:6813","nonce":4087672846}]},"public_addr":"172.21.15.169:6809/4087672846","cluster_addr":"172.21.15.169:6811/4087672846","heartbeat_back_addr":"172.21.15.169:6815/4087672846","heartbeat_front_addr":"172.21.15.169:6813/4087672846","state":["exists","up"]},{"osd":6,"uuid":"2abe4d77-c12c-477f-a4ec-eabb753ca4dd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6816","nonce":3801227982},{"type":"v1","addr":"172.21.15.169:6817","nonce":3801227982}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6818","nonce":3801227982},{"type":"v1","addr":"172.21.15.169:6819","nonce":3801227982}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6822","nonce":3801227982},{"type":"v1","addr":"172.21.15.169:6823","nonce":3801227982}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6820","nonce":3801227982},{"type":"v1","addr":"172.21.15.169:6821","nonce":3801227982}]},"public_addr":"172.21.15.169:6817/3801227982","cluster_addr":"172.21.15.169:6819/3801227982","heartbeat_back_addr":"172.21.15.169:6823/3801227982","heartbeat_front_addr":"172.21.15.169:6821/3801227982","state":["exists","up"]},{"osd":7,"uuid":"fc5ed8fc-8ad7-4b54-a8ea-bc6a9e846f63","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6824","nonce":3830851149},{"type":"v1","addr":"172.21.15.169:6825","nonce":3830851149}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6826","nonce":3830851149},{"type":"v1","addr":"172.21.15.169:6827","nonce":3830851149}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6830","nonce":3830851149},{"type":"v1","addr":"172.21.15.169:6831","nonce":3830851149}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6828","nonce":3830851149},{"type":"v1","addr":"172.21.15.169:6829","nonce":3830851149}]},"public_addr":"172.21.15.169:6825/3830851149","cluster_addr":"172.21.15.169:6827/3830851149","heartbeat_back_addr":"172.21.15.169:6831/3830851149","heartbeat_front_addr":"172.21.15.169:6829/3830851149","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:07:16.166767+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:07:37.634849+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:07:59.099590+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:08:21.737546+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:08:43.377647+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:09:06.116513+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:09:28.557330+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:09:51.159131+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.143:0/1563442416":"2023-08-29T02:10:09.845824+0000","172.21.15.143:6801/1828180492":"2023-08-29T02:10:09.845824+0000","172.21.15.143:0/2157302459":"2023-08-29T02:10:09.845824+0000","172.21.15.143:6800/1828180492":"2023-08-29T02:10:09.845824+0000","172.21.15.143:6801/907474889":"2023-08-29T02:06:02.911068+0000","172.21.15.143:6800/907474889":"2023-08-29T02:06:02.911068+0000","172.21.15.143:0/2323551826":"2023-08-29T02:06:02.911068+0000","172.21.15.143:6800/1019944263":"2023-08-29T02:05:41.797747+0000","172.21.15.143:0/3606471716":"2023-08-29T02:06:02.911068+0000","172.21.15.143:6801/1019944263":"2023-08-29T02:05:41.797747+0000","172.21.15.143:0/11141072":"2023-08-29T02:05:41.797747+0000","172.21.15.143:0/1148418770":"2023-08-29T02:05:41.797747+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-08-28T02:11:36.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:36 smithi143 conmon[100258]: audit 2023-08-28T02:11:35.437982+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:36.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:36 smithi143 conmon[100258]: audit 2023-08-28T02:11:35.438763+0000 mon.a (mon.0) 709 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:11:36.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:36 smithi143 conmon[100258]: audit 2023-08-28T02:11:35.439412+0000 mon.a (mon.0) 710 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:11:36.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:36 smithi143 conmon[100258]: audit 2023-08-28T02:11:35.443707+0000 mon.a (mon.0) 711 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:36.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:36 smithi143 conmon[100258]: audit 2023-08-28T02:11:35.447609+0000 mon.a (mon.0) 712 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:11:36.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:36 smithi143 conmon[109923]: audit 2023-08-28T02:11:35.437982+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:36.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:36 smithi143 conmon[109923]: audit 2023-08-28T02:11:35.438763+0000 mon.a (mon.0) 709 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:11:36.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:36 smithi143 conmon[109923]: audit 2023-08-28T02:11:35.439412+0000 mon.a (mon.0) 710 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:11:36.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:36 smithi143 conmon[109923]: audit 2023-08-28T02:11:35.443707+0000 mon.a (mon.0) 711 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:36.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:36 smithi143 conmon[109923]: audit 2023-08-28T02:11:35.447609+0000 mon.a (mon.0) 712 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:11:36.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:36 smithi169 conmon[100500]: audit 2023-08-28T02:11:35.437982+0000 mon.a (mon. 2023-08-28T02:11:36.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:36 smithi169 conmon[100500]: 0) 708 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:36.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:36 smithi169 conmon[100500]: audit 2023-08-28T02:11:35.438763+0000 mon.a (mon.0) 709 : audit 2023-08-28T02:11:36.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:36 smithi169 conmon[100500]: [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:11:36.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:36 smithi169 conmon[100500]: audit 2023-08-28T02: 2023-08-28T02:11:36.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:36 smithi169 conmon[100500]: 11:35.439412+0000 mon.a (mon.0) 710 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:11:36.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:36 smithi169 conmon[100500]: audit 2023-08- 2023-08-28T02:11:36.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:36 smithi169 conmon[100500]: 28T02:11:35.443707+0000 mon.a (mon.0) 711 2023-08-28T02:11:36.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:36 smithi169 conmon[100500]: : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:36.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:36 smithi169 conmon[100500]: audit 2023-08-28T02:11:36.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:36 smithi169 conmon[100500]: 2023-08-28T02:11:35.447609+0000 mon.a (mon.0) 712 2023-08-28T02:11:36.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:36 smithi169 conmon[100500]: : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:11:37.029 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph tell osd.0 flush_pg_stats 2023-08-28T02:11:37.030 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph tell osd.1 flush_pg_stats 2023-08-28T02:11:37.030 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph tell osd.2 flush_pg_stats 2023-08-28T02:11:37.031 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph tell osd.3 flush_pg_stats 2023-08-28T02:11:37.031 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph tell osd.4 flush_pg_stats 2023-08-28T02:11:37.031 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph tell osd.5 flush_pg_stats 2023-08-28T02:11:37.032 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph tell osd.6 flush_pg_stats 2023-08-28T02:11:37.032 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph tell osd.7 flush_pg_stats 2023-08-28T02:11:37.449 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:37 smithi143 conmon[100258]: cluster 2023-08-28T02:11:35.876662+0000 2023-08-28T02:11:37.449 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:37 smithi143 conmon[100258]: mgr.y (mgr.14424) 76 : cluster [DBG] pgmap v55: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-08-28T02:11:37.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:37 smithi143 conmon[100258]: audit 2023-08-28T02:11:36.473512+0000 mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.143:0/2089953537' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-28T02:11:37.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:37 smithi143 conmon[109923]: cluster 2023-08-28T02:11:35.876662+0000 mgr.y (mgr.14424) 76 : cluster [DBG] 2023-08-28T02:11:37.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:37 smithi143 conmon[109923]: pgmap v55: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-08-28T02:11:37.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:37 smithi143 conmon[109923]: audit 2023-08-28T02:11:36.473512+0000 mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.143:0/2089953537' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-28T02:11:37.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:37 smithi169 conmon[100500]: cluster 2023-08-28T02:11:35.876662+0000 2023-08-28T02:11:37.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:37 smithi169 conmon[100500]: mgr.y (mgr.14424) 76 : cluster [DBG] pgmap v55: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-08-28T02:11:37.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:37 smithi169 conmon[100500]: audit 2023-08-28T02:11:36.473512+0000 mon.c (mon 2023-08-28T02:11:37.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:37 smithi169 conmon[100500]: .2) 18 : audit [DBG] from='client.? 172.21.15.143:0/2089953537' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-28T02:11:38.464 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:11:38.565 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:11:38.766 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:11:38.767 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:11:38.786 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:11:39.329 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:11:39.330 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:39 smithi143 conmon[100258]: cluster 2023-08-28T02:11:37.877128+0000 mgr.y (mgr.14424) 77 : cluster 2023-08-28T02:11:39.330 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:39 smithi143 conmon[100258]: [DBG] pgmap v56: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-08-28T02:11:39.330 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:39 smithi143 conmon[100258]: audit 2023-08-28T02:11:38.108513+0000 mon.a (mon.0) 713 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:39.330 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:39 smithi143 conmon[100258]: audit 2023-08-28T02:11:38.109394+0000 mon.a (mon.0) 714 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:11:39.330 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:39 smithi143 conmon[100258]: audit 2023-08-28T02:11:38.110070+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:11:39.331 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:39 smithi143 conmon[100258]: audit 2023-08-28T02:11:38.118193+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:39.331 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:39 smithi143 conmon[109923]: cluster 2023-08-28T02:11:37. 2023-08-28T02:11:39.331 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:39 smithi143 conmon[109923]: 877128+0000 mgr.y (mgr.14424) 77 : cluster [DBG] pgmap v56: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-08-28T02:11:39.331 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:39 smithi143 conmon[109923]: audit 2023-08-28T02:11:38.108513+0000 mon.a (mon.0) 713 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:39.331 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:39 smithi143 conmon[109923]: audit 2023-08-28T02:11:38.109394+0000 mon.a (mon.0) 714 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:11:39.331 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:39 smithi143 conmon[109923]: audit 2023-08-28T02:11:38.110070+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:11:39.331 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:39 smithi143 conmon[109923]: audit 2023-08-28T02:11:38.118193+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:39.378 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:11:39.481 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:11:39.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:39 smithi169 conmon[100500]: cluster 2023-08-28T02:11:37.877128+0000 mgr.y (mgr. 2023-08-28T02:11:39.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:39 smithi169 conmon[100500]: 14424) 77 : cluster [DBG] pgmap v56: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-08-28T02:11:39.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:39 smithi169 conmon[100500]: audit 2023-08-28T02:11:38.108513+0000 mon.a (mon.0) 713 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:39.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:39 smithi169 conmon[100500]: audit 2023-08-28T02:11:38.109394+0000 2023-08-28T02:11:39.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:39 smithi169 conmon[100500]: mon.a (mon.0) 714 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:11:39.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:39 smithi169 conmon[100500]: audit 2023- 2023-08-28T02:11:39.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:39 smithi169 conmon[100500]: 08-28T02:11:38.110070+0000 mon.a (mon. 2023-08-28T02:11:39.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:39 smithi169 conmon[100500]: 0) 715 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:11:39.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:39 smithi169 conmon[100500]: audit 2023-08 2023-08-28T02:11:39.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:39 smithi169 conmon[100500]: -28T02:11:38.118193+0000 mon.a (mon. 2023-08-28T02:11:39.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:39 smithi169 conmon[100500]: 0) 716 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:40.335 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:11:40 smithi143 conmon[100708]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:11:40] "GET /metrics HTTP/1.1" 200 191475 "" "Prometheus/2.33.4" 2023-08-28T02:11:41.260 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:40 smithi143 conmon[100258]: cluster 2023-08-28T02:11:39 2023-08-28T02:11:41.260 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:40 smithi143 conmon[100258]: .877446+0000 mgr.y (mgr.14424) 78 : cluster [DBG] pgmap v57: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-08-28T02:11:41.260 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:40 smithi143 conmon[100258]: audit 2023-08-28T02:11:39.999444+0000 mon.a (mon.0) 717 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:41.260 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:40 smithi143 conmon[109923]: cluster 2023-08-28T02:11:39.877446+0000 mgr.y 2023-08-28T02:11:41.260 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:40 smithi143 conmon[109923]: (mgr.14424) 78 : cluster [DBG] pgmap v57: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-08-28T02:11:41.260 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:40 smithi143 conmon[109923]: audit 2023-08-28T02 2023-08-28T02:11:41.260 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:40 smithi143 conmon[109923]: :11:39.999444+0000 mon.a (mon.0) 717 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:41.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:40 smithi169 conmon[100500]: cluster 2023-08-28T02:11:39.877446+0000 mgr.y (mgr.14424) 78 : cluster [DBG] pgmap v57: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-08-28T02:11:41.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:40 smithi169 conmon[100500]: audit 2023-08-28T02:11:39.999444+0000 2023-08-28T02:11:41.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:40 smithi169 conmon[100500]: mon.a (mon.0) 717 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:11:43.237 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:11:42 smithi169 conmon[102807]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:11:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-28T02:11:43.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:43 smithi169 conmon[100500]: cluster 2023-08-28T02:11:41.877925+0000 mgr.y (mgr.14424) 79 : cluster 2023-08-28T02:11:43.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:43 smithi169 conmon[100500]: [DBG] pgmap v58: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-08-28T02:11:43.328 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:43 smithi143 conmon[100258]: cluster 2023-08-28T02:11:41.877925+0000 mgr.y (mgr.14424) 79 : cluster [DBG] pgmap v58: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-08-28T02:11:43.329 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:43 smithi143 conmon[109923]: cluster 2023-08-28T02:11:41.877925+0000 mgr.y (mgr.14424) 79 : cluster [DBG] pgmap v58: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-08-28T02:11:43.348 INFO:teuthology.orchestra.run.smithi143.stdout:206158430233 2023-08-28T02:11:43.348 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd last-stat-seq osd.7 2023-08-28T02:11:45.012 INFO:teuthology.orchestra.run.smithi143.stdout:34359738425 2023-08-28T02:11:45.012 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd last-stat-seq osd.0 2023-08-28T02:11:45.269 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:45 smithi143 conmon[109923]: cluster 2023-08-28T02:11:43.878403+0000 mgr.y (mgr.14424) 80 : cluster [DBG] pgmap v59: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-08-28T02:11:45.270 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:45 smithi143 conmon[100258]: cluster 2023-08-28T02:11:43.878403+0000 mgr.y (mgr.14424 2023-08-28T02:11:45.270 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:45 smithi143 conmon[100258]: ) 80 : cluster [DBG] pgmap v59: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-08-28T02:11:45.301 INFO:teuthology.orchestra.run.smithi143.stdout:107374182444 2023-08-28T02:11:45.302 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd last-stat-seq osd.3 2023-08-28T02:11:45.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:45 smithi169 conmon[100500]: cluster 2023-08-28T02:11:43.878403+0000 mgr.y (mgr.14424) 80 : cluster [DBG] pgmap v59: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-08-28T02:11:46.103 INFO:teuthology.orchestra.run.smithi143.stdout:55834574900 2023-08-28T02:11:46.104 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd last-stat-seq osd.1 2023-08-28T02:11:47.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:47 smithi143 conmon[100258]: cluster 2023-08-28T02:11:45.879187+0000 mgr.y (mgr.14424) 81 : cluster [DBG] pgmap v60: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-08-28T02:11:47.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:47 smithi143 conmon[109923]: cluster 2023-08-28T02:11:45.879187+0000 mgr.y (mgr.14424) 81 : cluster [DBG] pgmap v60: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-08-28T02:11:47.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:47 smithi169 conmon[100500]: cluster 2023-08-28T02:11:45.879187+0000 mgr.y (mgr.14424) 81 : cluster [DBG] pgmap v60: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-08-28T02:11:47.908 INFO:teuthology.orchestra.run.smithi143.stdout:128849018919 2023-08-28T02:11:47.908 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd last-stat-seq osd.4 2023-08-28T02:11:48.127 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:11:48.214 INFO:teuthology.orchestra.run.smithi143.stdout:154618822691 2023-08-28T02:11:48.214 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd last-stat-seq osd.5 2023-08-28T02:11:48.242 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:11:48.258 INFO:teuthology.orchestra.run.smithi143.stdout:77309411375 2023-08-28T02:11:48.259 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd last-stat-seq osd.2 2023-08-28T02:11:48.958 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:11:48.990 INFO:teuthology.orchestra.run.smithi143.stdout:180388626461 2023-08-28T02:11:48.990 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd last-stat-seq osd.6 2023-08-28T02:11:49.314 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:11:49.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:49 smithi143 conmon[100258]: cluster 2023-08-28T02:11:47.879752+0000 mgr.y (mgr.14424) 82 : cluster [DBG] pgmap v61: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:49.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:49 smithi143 conmon[109923]: cluster 2023-08-28T02:11:47.879752+0000 mgr.y (mgr.14424) 82 : cluster [DBG] pgmap v61: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:49.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:49 smithi169 conmon[100500]: cluster 2023-08-28T02:11:47.879752+0000 mgr.y (mgr.14424) 82 : cluster [DBG] pgmap v61: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:50.253 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:11:50 smithi143 conmon[100708]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:11:50] "GET /metrics HTTP/1.1" 200 191475 "" "Prometheus/2.33.4" 2023-08-28T02:11:51.068 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:11:51.233 INFO:teuthology.orchestra.run.smithi143.stdout:107374182445 2023-08-28T02:11:51.383 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:51 smithi143 conmon[100258]: cluster 2023-08-28T02:11:49.880220+0000 mgr.y (mgr.14424) 83 : cluster [DBG] pgmap v62: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:51.383 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:51 smithi143 conmon[109923]: cluster 2023-08-28T02:11:49.880220+0000 mgr.y (mgr.14424) 83 : cluster [DBG] pgmap v62: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:51.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:51 smithi169 conmon[100500]: cluster 2023-08-28T02:11:49.880220+0000 mgr.y (mgr.14424) 83 : cluster [DBG] pgmap v62: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:52.378 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:11:52.385 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:11:52.411 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:11:52.415 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182444 got 107374182445 for osd.3 2023-08-28T02:11:52.416 DEBUG:teuthology.parallel:result is None 2023-08-28T02:11:52.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:52 smithi143 conmon[109923]: audit 2023-08-28T02:11:51.234967+0000 2023-08-28T02:11:52.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:52 smithi143 conmon[109923]: mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.143:0/881230262' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-08-28T02:11:52.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:52 smithi143 conmon[100258]: audit 2023-08-28T02:11:51.234967+0000 2023-08-28T02:11:52.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:52 smithi143 conmon[100258]: mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.143:0/881230262' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-08-28T02:11:52.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:52 smithi169 conmon[100500]: audit 2023-08-28T02:11:51.234967+0000 mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.143:0/881230262' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-08-28T02:11:53.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:53 smithi169 conmon[100500]: cluster 2023-08-28T02:11:51.880894+0000 mgr.y (mgr.14424) 84 : cluster 2023-08-28T02:11:53.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:53 smithi169 conmon[100500]: [DBG] pgmap v63: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:53.238 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:11:52 smithi169 conmon[102807]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:11:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-28T02:11:53.367 INFO:teuthology.orchestra.run.smithi143.stdout:206158430235 2023-08-28T02:11:53.446 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:53 smithi143 conmon[100258]: cluster 2023-08-28T02:11:51.880894+0000 mgr.y (mgr. 2023-08-28T02:11:53.446 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:53 smithi143 conmon[100258]: 14424) 84 : cluster [DBG] pgmap v63: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:53.447 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:53 smithi143 conmon[109923]: cluster 2023-08-28T02:11:51.880894 2023-08-28T02:11:53.447 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:53 smithi143 conmon[109923]: +0000 mgr.y (mgr.14424) 84 : cluster [DBG] pgmap v63: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:54.046 INFO:teuthology.orchestra.run.smithi143.stdout:34359738427 2023-08-28T02:11:54.112 INFO:teuthology.orchestra.run.smithi143.stdout:55834574902 2023-08-28T02:11:54.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:54 smithi143 conmon[100258]: audit 2023-08-28T02:11:53.368142+0000 mon.c (mon.2) 20 : audit [DBG] from='client.? 172.21.15.143:0/1337518140' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-08-28T02:11:54.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:54 smithi143 conmon[100258]: audit 2023-08-28T02:11:54.047327+0000 mon.a (mon.0) 718 : audit [DBG] from='client.? 172.21.15.143:0/3698136298' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-08-28T02:11:54.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:54 smithi143 conmon[100258]: audit 2023-08-28T02:11:54.113223+0000 mon.a (mon.0) 719 : audit [DBG] from='client.? 172.21.15.143:0/2541089154' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-08-28T02:11:54.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:54 smithi143 conmon[109923]: audit 2023-08-28T02:11:53.368142 2023-08-28T02:11:54.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:54 smithi143 conmon[109923]: +0000 mon.c (mon.2) 20 : audit [DBG] from='client.? 172.21.15.143:0/1337518140' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-08-28T02:11:54.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:54 smithi143 conmon[109923]: audit 2023-08-28T02:11:54.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:54 smithi143 conmon[109923]: 2023-08-28T02:11:54.047327+0000 mon.a (mon.0) 718 : audit [DBG] from='client.? 172.21.15.143:0/3698136298' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-08-28T02:11:54.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:54 smithi143 conmon[109923]: audit 2023-08-28T02:11:54.113223+0000 mon.a (mon.0 2023-08-28T02:11:54.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:54 smithi143 conmon[109923]: ) 719 : audit [DBG] from='client.? 172.21.15.143:0/2541089154' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-08-28T02:11:54.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:54 smithi169 conmon[100500]: audit 2023-08-28T02:11:53.368142+0000 mon.c (mon.2) 20 : audit [DBG] 2023-08-28T02:11:54.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:54 smithi169 conmon[100500]: from='client.? 172.21.15.143:0/1337518140' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-08-28T02:11:54.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:54 smithi169 conmon[100500]: audit 2023-08-28T02:11:54.047327+0000 mon.a (mon.0) 718 : audit [DBG] from='client.? 172.21.15.143:0/3698136298' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-08-28T02:11:54.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:54 smithi169 conmon[100500]: audit 2023-08-28T02:11:54.113223+0000 mon.a (mon. 2023-08-28T02:11:54.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:54 smithi169 conmon[100500]: 0) 719 : audit [DBG] from='client.? 172.21.15.143:0/2541089154' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-08-28T02:11:55.331 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:55 smithi143 conmon[100258]: cluster 2023-08-28T02:11:53 2023-08-28T02:11:55.331 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:55 smithi143 conmon[100258]: .881191+0000 mgr.y (mgr.14424) 85 : cluster [DBG] pgmap v64: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:55.331 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:55 smithi143 conmon[109923]: cluster 2023-08-28T02:11:53.881191+0000 mgr.y (mgr.14424 2023-08-28T02:11:55.331 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:55 smithi143 conmon[109923]: ) 85 : cluster [DBG] pgmap v64: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:55.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:55 smithi169 conmon[100500]: cluster 2023-08-28T02:11:53.881191+0000 mgr.y (mgr.14424) 85 : cluster [DBG] pgmap v64: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:55.838 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574900 got 55834574902 for osd.1 2023-08-28T02:11:55.838 DEBUG:teuthology.parallel:result is None 2023-08-28T02:11:55.972 INFO:teuthology.orchestra.run.smithi143.stdout:128849018920 2023-08-28T02:11:56.019 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430233 got 206158430235 for osd.7 2023-08-28T02:11:56.019 DEBUG:teuthology.parallel:result is None 2023-08-28T02:11:56.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:56 smithi143 conmon[100258]: audit 2023-08-28T02:11:55.896732+0000 mon.c (mon.2) 21 : audit [DBG] from='client.? 172.21.15.143:0/105529062' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-08-28T02:11:56.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:56 smithi143 conmon[109923]: audit 2023-08-28T02:11:55.896732+0000 mon.c (mon.2) 21 : audit [DBG] from='client.? 172.21.15.143:0/105529062' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-08-28T02:11:56.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:56 smithi169 conmon[100500]: audit 2023-08-28T02:11:55.896732+0000 mon.c (mon.2) 21 : audit [DBG] from='client.? 172.21.15.143:0/105529062' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-08-28T02:11:56.719 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018919 got 128849018920 for osd.4 2023-08-28T02:11:56.719 DEBUG:teuthology.parallel:result is None 2023-08-28T02:11:57.355 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738425 got 34359738427 for osd.0 2023-08-28T02:11:57.355 DEBUG:teuthology.parallel:result is None 2023-08-28T02:11:57.450 INFO:teuthology.orchestra.run.smithi143.stdout:154618822693 2023-08-28T02:11:57.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:57 smithi143 conmon[100258]: cluster 2023-08-28T02:11:55 2023-08-28T02:11:57.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:57 smithi143 conmon[100258]: .881594+0000 mgr.y (mgr.14424) 86 : cluster [DBG] pgmap v65: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:57.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:57 smithi143 conmon[109923]: cluster 2023-08-28T02:11:55.881594+0000 mgr.y (mgr.14424 2023-08-28T02:11:57.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:57 smithi143 conmon[109923]: ) 86 : cluster [DBG] pgmap v65: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:57.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:57 smithi169 conmon[100500]: cluster 2023-08-28T02:11:55.881594+0000 mgr.y (mgr.14424) 86 : cluster [DBG] 2023-08-28T02:11:57.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:57 smithi169 conmon[100500]: pgmap v65: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:57.728 INFO:teuthology.orchestra.run.smithi143.stdout:180388626463 2023-08-28T02:11:58.444 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822691 got 154618822693 for osd.5 2023-08-28T02:11:58.444 DEBUG:teuthology.parallel:result is None 2023-08-28T02:11:58.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:58 smithi169 conmon[100500]: audit 2023-08-28T02:11:57.450994+0000 mon.c (mon.2) 22 : audit [DBG] from='client.? 172.21.15.143:0/2187839028' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-08-28T02:11:58.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:58 smithi169 conmon[100500]: audit 2023-08-28T02:11:57.728879+0000 mon.a (mon.0) 720 : audit [DBG] from='client.? 172.21.15.143:0/2251548878' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-08-28T02:11:58.688 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:58 smithi143 conmon[100258]: audit 2023-08-28T02:11:57.450994+0000 mon.c (mon.2) 22 : audit 2023-08-28T02:11:58.688 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:58 smithi143 conmon[100258]: [DBG] from='client.? 172.21.15.143:0/2187839028' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-08-28T02:11:58.688 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:58 smithi143 conmon[100258]: audit 2023-08-28T02:11:57.728879+0000 mon.a (mon.0) 720 : audit [DBG] from='client.? 172.21.15.143:0/2251548878' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-08-28T02:11:58.688 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:58 smithi143 conmon[109923]: audit 2023-08-28T02:11:57.450994+0000 2023-08-28T02:11:58.688 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:58 smithi143 conmon[109923]: mon.c (mon.2) 22 : audit [DBG] from='client.? 172.21.15.143:0/2187839028' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-08-28T02:11:58.689 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:58 smithi143 conmon[109923]: audit 2023-08-28T02:11:57 2023-08-28T02:11:58.689 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:58 smithi143 conmon[109923]: .728879+0000 mon.a (mon.0) 720 : audit [DBG] from='client.? 172.21.15.143:0/2251548878' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-08-28T02:11:58.767 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626461 got 180388626463 for osd.6 2023-08-28T02:11:58.767 DEBUG:teuthology.parallel:result is None 2023-08-28T02:11:59.087 INFO:teuthology.orchestra.run.smithi143.stdout:77309411378 2023-08-28T02:11:59.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:59 smithi143 conmon[100258]: cluster 2023-08-28T02:11:57.881867+0000 mgr.y (mgr.14424) 87 : cluster [DBG] pgmap v66: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:59.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:11:59 smithi143 conmon[100258]: audit 2023-08-28T02:11:59.088562+0000 mon.a (mon.0) 721 : audit [DBG] from='client.? 172.21.15.143:0/3207482922' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-08-28T02:11:59.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:59 smithi143 conmon[109923]: cluster 2023-08-28T02:11:57.881867+0000 mgr.y (mgr.14424) 87 : cluster [DBG] pgmap v66: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:59.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:11:59 smithi143 conmon[109923]: audit 2023-08-28T02:11:59.088562+0000 mon.a (mon.0) 721 : audit [DBG] from='client.? 172.21.15.143:0/3207482922' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-08-28T02:11:59.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:59 smithi169 conmon[100500]: cluster 2023-08-28T02:11:57.881867+0000 mgr.y (mgr.14424) 87 : cluster [DBG] pgmap v66: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:11:59.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:11:59 smithi169 conmon[100500]: audit 2023-08-28T02:11:59.088562+0000 mon.a (mon.0) 721 : audit [DBG] from='client.? 172.21.15.143:0/3207482922' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-08-28T02:11:59.726 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411375 got 77309411378 for osd.2 2023-08-28T02:11:59.727 DEBUG:teuthology.parallel:result is None 2023-08-28T02:11:59.727 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-08-28T02:11:59.727 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph pg dump --format=json 2023-08-28T02:12:00.076 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:12:00.593 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:12:00 smithi143 conmon[100708]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:12:00] "GET /metrics HTTP/1.1" 200 191485 "" "Prometheus/2.33.4" 2023-08-28T02:12:01.222 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:01 smithi143 conmon[100258]: cluster 2023-08-28T02:11:59.882280+0000 mgr.y 2023-08-28T02:12:01.222 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:01 smithi143 conmon[100258]: (mgr.14424) 88 : cluster [DBG] pgmap v67: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:12:01.222 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:01 smithi143 conmon[109923]: cluster 2023-08-28T02:11:59.882280+0000 2023-08-28T02:12:01.223 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:01 smithi143 conmon[109923]: mgr.y (mgr.14424) 88 : cluster [DBG] pgmap v67: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:12:01.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:01 smithi169 conmon[100500]: cluster 2023-08-28T02:11:59.882280+0000 mgr.y (mgr.14424) 88 : cluster [DBG] pgmap v67: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:12:01.583 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:12:01.584 INFO:teuthology.orchestra.run.smithi143.stderr:dumped all 2023-08-28T02:12:02.210 INFO:teuthology.orchestra.run.smithi143.stdout:{"pg_ready":true,"pg_map":{"version":67,"stamp":"2023-08-28T02:11:59.882061+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2381556,"kb_used_data":4788,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747579660,"statfs":{"total":767960285184,"available":765521571840,"internally_reserved":0,"allocated":4902912,"data_stored":2055085,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002702"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.246926+0000","last_change":"2023-08-28T02:11:22.438750+0000","last_active":"2023-08-28T02:11:23.246926+0000","last_peered":"2023-08-28T02:11:23.246926+0000","last_clean":"2023-08-28T02:11:23.246926+0000","last_became_active":"2023-08-28T02:11:22.438438+0000","last_became_peered":"2023-08-28T02:11:22.438438+0000","last_unstale":"2023-08-28T02:11:23.246926+0000","last_undegraded":"2023-08-28T02:11:23.246926+0000","last_fullsized":"2023-08-28T02:11:23.246926+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":16,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-08-28T02:11:29.944829+0000","last_change":"2023-08-28T02:11:22.183512+0000","last_active":"2023-08-28T02:11:29.944829+0000","last_peered":"2023-08-28T02:11:29.944829+0000","last_clean":"2023-08-28T02:11:29.944829+0000","last_became_active":"2023-08-28T02:11:22.182755+0000","last_became_peered":"2023-08-28T02:11:22.182755+0000","last_unstale":"2023-08-28T02:11:29.944829+0000","last_undegraded":"2023-08-28T02:11:29.944829+0000","last_fullsized":"2023-08-28T02:11:29.944829+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.247534+0000","last_change":"2023-08-28T02:11:22.688074+0000","last_active":"2023-08-28T02:11:23.247534+0000","last_peered":"2023-08-28T02:11:23.247534+0000","last_clean":"2023-08-28T02:11:23.247534+0000","last_became_active":"2023-08-28T02:11:22.687745+0000","last_became_peered":"2023-08-28T02:11:22.687745+0000","last_unstale":"2023-08-28T02:11:23.247534+0000","last_undegraded":"2023-08-28T02:11:23.247534+0000","last_fullsized":"2023-08-28T02:11:23.247534+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-08-28T02:11:24.397783+0000","last_change":"2023-08-28T02:11:22.178584+0000","last_active":"2023-08-28T02:11:24.397783+0000","last_peered":"2023-08-28T02:11:24.397783+0000","last_clean":"2023-08-28T02:11:24.397783+0000","last_became_active":"2023-08-28T02:11:22.178247+0000","last_became_peered":"2023-08-28T02:11:22.178247+0000","last_unstale":"2023-08-28T02:11:24.397783+0000","last_undegraded":"2023-08-28T02:11:24.397783+0000","last_fullsized":"2023-08-28T02:11:24.397783+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.247113+0000","last_change":"2023-08-28T02:11:22.688542+0000","last_active":"2023-08-28T02:11:23.247113+0000","last_peered":"2023-08-28T02:11:23.247113+0000","last_clean":"2023-08-28T02:11:23.247113+0000","last_became_active":"2023-08-28T02:11:22.688239+0000","last_became_peered":"2023-08-28T02:11:22.688239+0000","last_unstale":"2023-08-28T02:11:23.247113+0000","last_undegraded":"2023-08-28T02:11:23.247113+0000","last_fullsized":"2023-08-28T02:11:23.247113+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-08-28T02:11:29.945472+0000","last_change":"2023-08-28T02:11:22.186024+0000","last_active":"2023-08-28T02:11:29.945472+0000","last_peered":"2023-08-28T02:11:29.945472+0000","last_clean":"2023-08-28T02:11:29.945472+0000","last_became_active":"2023-08-28T02:11:22.185500+0000","last_became_peered":"2023-08-28T02:11:22.185500+0000","last_unstale":"2023-08-28T02:11:29.945472+0000","last_undegraded":"2023-08-28T02:11:29.945472+0000","last_fullsized":"2023-08-28T02:11:29.945472+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.247478+0000","last_change":"2023-08-28T02:11:22.199354+0000","last_active":"2023-08-28T02:11:23.247478+0000","last_peered":"2023-08-28T02:11:23.247478+0000","last_clean":"2023-08-28T02:11:23.247478+0000","last_became_active":"2023-08-28T02:11:22.199098+0000","last_became_peered":"2023-08-28T02:11:22.199098+0000","last_unstale":"2023-08-28T02:11:23.247478+0000","last_undegraded":"2023-08-28T02:11:23.247478+0000","last_fullsized":"2023-08-28T02:11:23.247478+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-08-28T02:11:29.945272+0000","last_change":"2023-08-28T02:11:22.193215+0000","last_active":"2023-08-28T02:11:29.945272+0000","last_peered":"2023-08-28T02:11:29.945272+0000","last_clean":"2023-08-28T02:11:29.945272+0000","last_became_active":"2023-08-28T02:11:22.192780+0000","last_became_peered":"2023-08-28T02:11:22.192780+0000","last_unstale":"2023-08-28T02:11:29.945272+0000","last_undegraded":"2023-08-28T02:11:29.945272+0000","last_fullsized":"2023-08-28T02:11:29.945272+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.248695+0000","last_change":"2023-08-28T02:11:23.248065+0000","last_active":"2023-08-28T02:11:23.248695+0000","last_peered":"2023-08-28T02:11:23.248695+0000","last_clean":"2023-08-28T02:11:23.248695+0000","last_became_active":"2023-08-28T02:11:23.247741+0000","last_became_peered":"2023-08-28T02:11:23.247741+0000","last_unstale":"2023-08-28T02:11:23.248695+0000","last_undegraded":"2023-08-28T02:11:23.248695+0000","last_fullsized":"2023-08-28T02:11:23.248695+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-08-28T02:11:24.397815+0000","last_change":"2023-08-28T02:11:22.188952+0000","last_active":"2023-08-28T02:11:24.397815+0000","last_peered":"2023-08-28T02:11:24.397815+0000","last_clean":"2023-08-28T02:11:24.397815+0000","last_became_active":"2023-08-28T02:11:22.188583+0000","last_became_peered":"2023-08-28T02:11:22.188583+0000","last_unstale":"2023-08-28T02:11:24.397815+0000","last_undegraded":"2023-08-28T02:11:24.397815+0000","last_fullsized":"2023-08-28T02:11:24.397815+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.247427+0000","last_change":"2023-08-28T02:11:22.437146+0000","last_active":"2023-08-28T02:11:23.247427+0000","last_peered":"2023-08-28T02:11:23.247427+0000","last_clean":"2023-08-28T02:11:23.247427+0000","last_became_active":"2023-08-28T02:11:22.436798+0000","last_became_peered":"2023-08-28T02:11:22.436798+0000","last_unstale":"2023-08-28T02:11:23.247427+0000","last_undegraded":"2023-08-28T02:11:23.247427+0000","last_fullsized":"2023-08-28T02:11:23.247427+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-08-28T02:11:24.398113+0000","last_change":"2023-08-28T02:11:22.182980+0000","last_active":"2023-08-28T02:11:24.398113+0000","last_peered":"2023-08-28T02:11:24.398113+0000","last_clean":"2023-08-28T02:11:24.398113+0000","last_became_active":"2023-08-28T02:11:22.182484+0000","last_became_peered":"2023-08-28T02:11:22.182484+0000","last_unstale":"2023-08-28T02:11:24.398113+0000","last_undegraded":"2023-08-28T02:11:24.398113+0000","last_fullsized":"2023-08-28T02:11:24.398113+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.192062+0000","last_change":"2023-08-28T02:10:32.184257+0000","last_active":"2023-08-28T02:11:23.192062+0000","last_peered":"2023-08-28T02:11:23.192062+0000","last_clean":"2023-08-28T02:11:23.192062+0000","last_became_active":"2023-08-28T02:10:32.183944+0000","last_became_peered":"2023-08-28T02:10:32.183944+0000","last_unstale":"2023-08-28T02:11:23.192062+0000","last_undegraded":"2023-08-28T02:11:23.192062+0000","last_fullsized":"2023-08-28T02:11:23.192062+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.248362+0000","last_change":"2023-08-28T02:11:22.795960+0000","last_active":"2023-08-28T02:11:23.248362+0000","last_peered":"2023-08-28T02:11:23.248362+0000","last_clean":"2023-08-28T02:11:23.248362+0000","last_became_active":"2023-08-28T02:11:22.795379+0000","last_became_peered":"2023-08-28T02:11:22.795379+0000","last_unstale":"2023-08-28T02:11:23.248362+0000","last_undegraded":"2023-08-28T02:11:23.248362+0000","last_fullsized":"2023-08-28T02:11:23.248362+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.194979+0000","last_change":"2023-08-28T02:10:32.197242+0000","last_active":"2023-08-28T02:11:23.194979+0000","last_peered":"2023-08-28T02:11:23.194979+0000","last_clean":"2023-08-28T02:11:23.194979+0000","last_became_active":"2023-08-28T02:10:32.196865+0000","last_became_peered":"2023-08-28T02:10:32.196865+0000","last_unstale":"2023-08-28T02:11:23.194979+0000","last_undegraded":"2023-08-28T02:11:23.194979+0000","last_fullsized":"2023-08-28T02:11:23.194979+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-08-28T02:11:29.942467+0000","last_change":"2023-08-28T02:11:22.183063+0000","last_active":"2023-08-28T02:11:29.942467+0000","last_peered":"2023-08-28T02:11:29.942467+0000","last_clean":"2023-08-28T02:11:29.942467+0000","last_became_active":"2023-08-28T02:11:22.182709+0000","last_became_peered":"2023-08-28T02:11:22.182709+0000","last_unstale":"2023-08-28T02:11:29.942467+0000","last_undegraded":"2023-08-28T02:11:29.942467+0000","last_fullsized":"2023-08-28T02:11:29.942467+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.250261+0000","last_change":"2023-08-28T02:11:23.248976+0000","last_active":"2023-08-28T02:11:23.250261+0000","last_peered":"2023-08-28T02:11:23.250261+0000","last_clean":"2023-08-28T02:11:23.250261+0000","last_became_active":"2023-08-28T02:11:23.248676+0000","last_became_peered":"2023-08-28T02:11:23.248676+0000","last_unstale":"2023-08-28T02:11:23.250261+0000","last_undegraded":"2023-08-28T02:11:23.250261+0000","last_fullsized":"2023-08-28T02:11:23.250261+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.194264+0000","last_change":"2023-08-28T02:11:22.187297+0000","last_active":"2023-08-28T02:11:23.194264+0000","last_peered":"2023-08-28T02:11:23.194264+0000","last_clean":"2023-08-28T02:11:23.194264+0000","last_became_active":"2023-08-28T02:11:22.186947+0000","last_became_peered":"2023-08-28T02:11:22.186947+0000","last_unstale":"2023-08-28T02:11:23.194264+0000","last_undegraded":"2023-08-28T02:11:23.194264+0000","last_fullsized":"2023-08-28T02:11:23.194264+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.249223+0000","last_change":"2023-08-28T02:10:32.198340+0000","last_active":"2023-08-28T02:11:23.249223+0000","last_peered":"2023-08-28T02:11:23.249223+0000","last_clean":"2023-08-28T02:11:23.249223+0000","last_became_active":"2023-08-28T02:10:32.198052+0000","last_became_peered":"2023-08-28T02:10:32.198052+0000","last_unstale":"2023-08-28T02:11:23.249223+0000","last_undegraded":"2023-08-28T02:11:23.249223+0000","last_fullsized":"2023-08-28T02:11:23.249223+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.196689+0000","last_change":"2023-08-28T02:11:23.195766+0000","last_active":"2023-08-28T02:11:23.196689+0000","last_peered":"2023-08-28T02:11:23.196689+0000","last_clean":"2023-08-28T02:11:23.196689+0000","last_became_active":"2023-08-28T02:11:23.195454+0000","last_became_peered":"2023-08-28T02:11:23.195454+0000","last_unstale":"2023-08-28T02:11:23.196689+0000","last_undegraded":"2023-08-28T02:11:23.196689+0000","last_fullsized":"2023-08-28T02:11:23.196689+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.190332+0000","last_change":"2023-08-28T02:11:22.188624+0000","last_active":"2023-08-28T02:11:23.190332+0000","last_peered":"2023-08-28T02:11:23.190332+0000","last_clean":"2023-08-28T02:11:23.190332+0000","last_became_active":"2023-08-28T02:11:22.187351+0000","last_became_peered":"2023-08-28T02:11:22.187351+0000","last_unstale":"2023-08-28T02:11:23.190332+0000","last_undegraded":"2023-08-28T02:11:23.190332+0000","last_fullsized":"2023-08-28T02:11:23.190332+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.248295+0000","last_change":"2023-08-28T02:10:32.198506+0000","last_active":"2023-08-28T02:11:23.248295+0000","last_peered":"2023-08-28T02:11:23.248295+0000","last_clean":"2023-08-28T02:11:23.248295+0000","last_became_active":"2023-08-28T02:10:32.198175+0000","last_became_peered":"2023-08-28T02:10:32.198175+0000","last_unstale":"2023-08-28T02:11:23.248295+0000","last_undegraded":"2023-08-28T02:11:23.248295+0000","last_fullsized":"2023-08-28T02:11:23.248295+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.250383+0000","last_change":"2023-08-28T02:11:23.249640+0000","last_active":"2023-08-28T02:11:23.250383+0000","last_peered":"2023-08-28T02:11:23.250383+0000","last_clean":"2023-08-28T02:11:23.250383+0000","last_became_active":"2023-08-28T02:11:23.249355+0000","last_became_peered":"2023-08-28T02:11:23.249355+0000","last_unstale":"2023-08-28T02:11:23.250383+0000","last_undegraded":"2023-08-28T02:11:23.250383+0000","last_fullsized":"2023-08-28T02:11:23.250383+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.247532+0000","last_change":"2023-08-28T02:11:22.184384+0000","last_active":"2023-08-28T02:11:23.247532+0000","last_peered":"2023-08-28T02:11:23.247532+0000","last_clean":"2023-08-28T02:11:23.247532+0000","last_became_active":"2023-08-28T02:11:22.184123+0000","last_became_peered":"2023-08-28T02:11:22.184123+0000","last_unstale":"2023-08-28T02:11:23.247532+0000","last_undegraded":"2023-08-28T02:11:23.247532+0000","last_fullsized":"2023-08-28T02:11:23.247532+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":20,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.248205+0000","last_change":"2023-08-28T02:09:55.609189+0000","last_active":"2023-08-28T02:11:23.248205+0000","last_peered":"2023-08-28T02:11:23.248205+0000","last_clean":"2023-08-28T02:11:23.248205+0000","last_became_active":"2023-08-28T02:09:55.608705+0000","last_became_peered":"2023-08-28T02:09:55.608705+0000","last_unstale":"2023-08-28T02:11:23.248205+0000","last_undegraded":"2023-08-28T02:11:23.248205+0000","last_fullsized":"2023-08-28T02:11:23.248205+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_clean_scrub_stamp":"2023-08-28T02:08:02.043718+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.193836+0000","last_change":"2023-08-28T02:10:32.184422+0000","last_active":"2023-08-28T02:11:23.193836+0000","last_peered":"2023-08-28T02:11:23.193836+0000","last_clean":"2023-08-28T02:11:23.193836+0000","last_became_active":"2023-08-28T02:10:32.184072+0000","last_became_peered":"2023-08-28T02:10:32.184072+0000","last_unstale":"2023-08-28T02:11:23.193836+0000","last_undegraded":"2023-08-28T02:11:23.193836+0000","last_fullsized":"2023-08-28T02:11:23.193836+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.196457+0000","last_change":"2023-08-28T02:11:23.195860+0000","last_active":"2023-08-28T02:11:23.196457+0000","last_peered":"2023-08-28T02:11:23.196457+0000","last_clean":"2023-08-28T02:11:23.196457+0000","last_became_active":"2023-08-28T02:11:23.195574+0000","last_became_peered":"2023-08-28T02:11:23.195574+0000","last_unstale":"2023-08-28T02:11:23.196457+0000","last_undegraded":"2023-08-28T02:11:23.196457+0000","last_fullsized":"2023-08-28T02:11:23.196457+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.194687+0000","last_change":"2023-08-28T02:11:22.184399+0000","last_active":"2023-08-28T02:11:23.194687+0000","last_peered":"2023-08-28T02:11:23.194687+0000","last_clean":"2023-08-28T02:11:23.194687+0000","last_became_active":"2023-08-28T02:11:22.184024+0000","last_became_peered":"2023-08-28T02:11:22.184024+0000","last_unstale":"2023-08-28T02:11:23.194687+0000","last_undegraded":"2023-08-28T02:11:23.194687+0000","last_fullsized":"2023-08-28T02:11:23.194687+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.192330+0000","last_change":"2023-08-28T02:10:32.197650+0000","last_active":"2023-08-28T02:11:23.192330+0000","last_peered":"2023-08-28T02:11:23.192330+0000","last_clean":"2023-08-28T02:11:23.192330+0000","last_became_active":"2023-08-28T02:10:32.197319+0000","last_became_peered":"2023-08-28T02:10:32.197319+0000","last_unstale":"2023-08-28T02:11:23.192330+0000","last_undegraded":"2023-08-28T02:11:23.192330+0000","last_fullsized":"2023-08-28T02:11:23.192330+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.251921+0000","last_change":"2023-08-28T02:11:23.250778+0000","last_active":"2023-08-28T02:11:23.251921+0000","last_peered":"2023-08-28T02:11:23.251921+0000","last_clean":"2023-08-28T02:11:23.251921+0000","last_became_active":"2023-08-28T02:11:23.250402+0000","last_became_peered":"2023-08-28T02:11:23.250402+0000","last_unstale":"2023-08-28T02:11:23.251921+0000","last_undegraded":"2023-08-28T02:11:23.251921+0000","last_fullsized":"2023-08-28T02:11:23.251921+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.247277+0000","last_change":"2023-08-28T02:11:22.188913+0000","last_active":"2023-08-28T02:11:23.247277+0000","last_peered":"2023-08-28T02:11:23.247277+0000","last_clean":"2023-08-28T02:11:23.247277+0000","last_became_active":"2023-08-28T02:11:22.188508+0000","last_became_peered":"2023-08-28T02:11:22.188508+0000","last_unstale":"2023-08-28T02:11:23.247277+0000","last_undegraded":"2023-08-28T02:11:23.247277+0000","last_fullsized":"2023-08-28T02:11:23.247277+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.193852+0000","last_change":"2023-08-28T02:10:32.184125+0000","last_active":"2023-08-28T02:11:23.193852+0000","last_peered":"2023-08-28T02:11:23.193852+0000","last_clean":"2023-08-28T02:11:23.193852+0000","last_became_active":"2023-08-28T02:10:32.183865+0000","last_became_peered":"2023-08-28T02:10:32.183865+0000","last_unstale":"2023-08-28T02:11:23.193852+0000","last_undegraded":"2023-08-28T02:11:23.193852+0000","last_fullsized":"2023-08-28T02:11:23.193852+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.249798+0000","last_change":"2023-08-28T02:11:23.249476+0000","last_active":"2023-08-28T02:11:23.249798+0000","last_peered":"2023-08-28T02:11:23.249798+0000","last_clean":"2023-08-28T02:11:23.249798+0000","last_became_active":"2023-08-28T02:11:23.249246+0000","last_became_peered":"2023-08-28T02:11:23.249246+0000","last_unstale":"2023-08-28T02:11:23.249798+0000","last_undegraded":"2023-08-28T02:11:23.249798+0000","last_fullsized":"2023-08-28T02:11:23.249798+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.194008+0000","last_change":"2023-08-28T02:11:22.193844+0000","last_active":"2023-08-28T02:11:23.194008+0000","last_peered":"2023-08-28T02:11:23.194008+0000","last_clean":"2023-08-28T02:11:23.194008+0000","last_became_active":"2023-08-28T02:11:22.193583+0000","last_became_peered":"2023-08-28T02:11:22.193583+0000","last_unstale":"2023-08-28T02:11:23.194008+0000","last_undegraded":"2023-08-28T02:11:23.194008+0000","last_fullsized":"2023-08-28T02:11:23.194008+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.194163+0000","last_change":"2023-08-28T02:10:32.197552+0000","last_active":"2023-08-28T02:11:23.194163+0000","last_peered":"2023-08-28T02:11:23.194163+0000","last_clean":"2023-08-28T02:11:23.194163+0000","last_became_active":"2023-08-28T02:10:32.197251+0000","last_became_peered":"2023-08-28T02:10:32.197251+0000","last_unstale":"2023-08-28T02:11:23.194163+0000","last_undegraded":"2023-08-28T02:11:23.194163+0000","last_fullsized":"2023-08-28T02:11:23.194163+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.252207+0000","last_change":"2023-08-28T02:11:23.251282+0000","last_active":"2023-08-28T02:11:23.252207+0000","last_peered":"2023-08-28T02:11:23.252207+0000","last_clean":"2023-08-28T02:11:23.252207+0000","last_became_active":"2023-08-28T02:11:23.250975+0000","last_became_peered":"2023-08-28T02:11:23.250975+0000","last_unstale":"2023-08-28T02:11:23.252207+0000","last_undegraded":"2023-08-28T02:11:23.252207+0000","last_fullsized":"2023-08-28T02:11:23.252207+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.248465+0000","last_change":"2023-08-28T02:11:22.184709+0000","last_active":"2023-08-28T02:11:23.248465+0000","last_peered":"2023-08-28T02:11:23.248465+0000","last_clean":"2023-08-28T02:11:23.248465+0000","last_became_active":"2023-08-28T02:11:22.183968+0000","last_became_peered":"2023-08-28T02:11:22.183968+0000","last_unstale":"2023-08-28T02:11:23.248465+0000","last_undegraded":"2023-08-28T02:11:23.248465+0000","last_fullsized":"2023-08-28T02:11:23.248465+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.247271+0000","last_change":"2023-08-28T02:11:22.437038+0000","last_active":"2023-08-28T02:11:23.247271+0000","last_peered":"2023-08-28T02:11:23.247271+0000","last_clean":"2023-08-28T02:11:23.247271+0000","last_became_active":"2023-08-28T02:11:22.436628+0000","last_became_peered":"2023-08-28T02:11:22.436628+0000","last_unstale":"2023-08-28T02:11:23.247271+0000","last_undegraded":"2023-08-28T02:11:23.247271+0000","last_fullsized":"2023-08-28T02:11:23.247271+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.247278+0000","last_change":"2023-08-28T02:11:22.182003+0000","last_active":"2023-08-28T02:11:23.247278+0000","last_peered":"2023-08-28T02:11:23.247278+0000","last_clean":"2023-08-28T02:11:23.247278+0000","last_became_active":"2023-08-28T02:11:22.181682+0000","last_became_peered":"2023-08-28T02:11:22.181682+0000","last_unstale":"2023-08-28T02:11:23.247278+0000","last_undegraded":"2023-08-28T02:11:23.247278+0000","last_fullsized":"2023-08-28T02:11:23.247278+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.247745+0000","last_change":"2023-08-28T02:11:22.198232+0000","last_active":"2023-08-28T02:11:23.247745+0000","last_peered":"2023-08-28T02:11:23.247745+0000","last_clean":"2023-08-28T02:11:23.247745+0000","last_became_active":"2023-08-28T02:11:22.197847+0000","last_became_peered":"2023-08-28T02:11:22.197847+0000","last_unstale":"2023-08-28T02:11:23.247745+0000","last_undegraded":"2023-08-28T02:11:23.247745+0000","last_fullsized":"2023-08-28T02:11:23.247745+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.192181+0000","last_change":"2023-08-28T02:11:22.185928+0000","last_active":"2023-08-28T02:11:23.192181+0000","last_peered":"2023-08-28T02:11:23.192181+0000","last_clean":"2023-08-28T02:11:23.192181+0000","last_became_active":"2023-08-28T02:11:22.185594+0000","last_became_peered":"2023-08-28T02:11:22.185594+0000","last_unstale":"2023-08-28T02:11:23.192181+0000","last_undegraded":"2023-08-28T02:11:23.192181+0000","last_fullsized":"2023-08-28T02:11:23.192181+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.247316+0000","last_change":"2023-08-28T02:11:22.437303+0000","last_active":"2023-08-28T02:11:23.247316+0000","last_peered":"2023-08-28T02:11:23.247316+0000","last_clean":"2023-08-28T02:11:23.247316+0000","last_became_active":"2023-08-28T02:11:22.436825+0000","last_became_peered":"2023-08-28T02:11:22.436825+0000","last_unstale":"2023-08-28T02:11:23.247316+0000","last_undegraded":"2023-08-28T02:11:23.247316+0000","last_fullsized":"2023-08-28T02:11:23.247316+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-08-28T02:11:29.945097+0000","last_change":"2023-08-28T02:11:22.181752+0000","last_active":"2023-08-28T02:11:29.945097+0000","last_peered":"2023-08-28T02:11:29.945097+0000","last_clean":"2023-08-28T02:11:29.945097+0000","last_became_active":"2023-08-28T02:11:22.181420+0000","last_became_peered":"2023-08-28T02:11:22.181420+0000","last_unstale":"2023-08-28T02:11:29.945097+0000","last_undegraded":"2023-08-28T02:11:29.945097+0000","last_fullsized":"2023-08-28T02:11:29.945097+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.249951+0000","last_change":"2023-08-28T02:11:23.248381+0000","last_active":"2023-08-28T02:11:23.249951+0000","last_peered":"2023-08-28T02:11:23.249951+0000","last_clean":"2023-08-28T02:11:23.249951+0000","last_became_active":"2023-08-28T02:11:23.248045+0000","last_became_peered":"2023-08-28T02:11:23.248045+0000","last_unstale":"2023-08-28T02:11:23.249951+0000","last_undegraded":"2023-08-28T02:11:23.249951+0000","last_fullsized":"2023-08-28T02:11:23.249951+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.194009+0000","last_change":"2023-08-28T02:11:22.186907+0000","last_active":"2023-08-28T02:11:23.194009+0000","last_peered":"2023-08-28T02:11:23.194009+0000","last_clean":"2023-08-28T02:11:23.194009+0000","last_became_active":"2023-08-28T02:11:22.186524+0000","last_became_peered":"2023-08-28T02:11:22.186524+0000","last_unstale":"2023-08-28T02:11:23.194009+0000","last_undegraded":"2023-08-28T02:11:23.194009+0000","last_fullsized":"2023-08-28T02:11:23.194009+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.249825+0000","last_change":"2023-08-28T02:11:23.249127+0000","last_active":"2023-08-28T02:11:23.249825+0000","last_peered":"2023-08-28T02:11:23.249825+0000","last_clean":"2023-08-28T02:11:23.249825+0000","last_became_active":"2023-08-28T02:11:23.248816+0000","last_became_peered":"2023-08-28T02:11:23.248816+0000","last_unstale":"2023-08-28T02:11:23.249825+0000","last_undegraded":"2023-08-28T02:11:23.249825+0000","last_fullsized":"2023-08-28T02:11:23.249825+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-08-28T02:11:24.398759+0000","last_change":"2023-08-28T02:11:22.188982+0000","last_active":"2023-08-28T02:11:24.398759+0000","last_peered":"2023-08-28T02:11:24.398759+0000","last_clean":"2023-08-28T02:11:24.398759+0000","last_became_active":"2023-08-28T02:11:22.188663+0000","last_became_peered":"2023-08-28T02:11:22.188663+0000","last_unstale":"2023-08-28T02:11:24.398759+0000","last_undegraded":"2023-08-28T02:11:24.398759+0000","last_fullsized":"2023-08-28T02:11:24.398759+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.247099+0000","last_change":"2023-08-28T02:11:22.202705+0000","last_active":"2023-08-28T02:11:23.247099+0000","last_peered":"2023-08-28T02:11:23.247099+0000","last_clean":"2023-08-28T02:11:23.247099+0000","last_became_active":"2023-08-28T02:11:22.202439+0000","last_became_peered":"2023-08-28T02:11:22.202439+0000","last_unstale":"2023-08-28T02:11:23.247099+0000","last_undegraded":"2023-08-28T02:11:23.247099+0000","last_fullsized":"2023-08-28T02:11:23.247099+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-08-28T02:11:29.943584+0000","last_change":"2023-08-28T02:11:22.191353+0000","last_active":"2023-08-28T02:11:29.943584+0000","last_peered":"2023-08-28T02:11:29.943584+0000","last_clean":"2023-08-28T02:11:29.943584+0000","last_became_active":"2023-08-28T02:11:22.191014+0000","last_became_peered":"2023-08-28T02:11:22.191014+0000","last_unstale":"2023-08-28T02:11:29.943584+0000","last_undegraded":"2023-08-28T02:11:29.943584+0000","last_fullsized":"2023-08-28T02:11:29.943584+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.249690+0000","last_change":"2023-08-28T02:11:23.249061+0000","last_active":"2023-08-28T02:11:23.249690+0000","last_peered":"2023-08-28T02:11:23.249690+0000","last_clean":"2023-08-28T02:11:23.249690+0000","last_became_active":"2023-08-28T02:11:23.248739+0000","last_became_peered":"2023-08-28T02:11:23.248739+0000","last_unstale":"2023-08-28T02:11:23.249690+0000","last_undegraded":"2023-08-28T02:11:23.249690+0000","last_fullsized":"2023-08-28T02:11:23.249690+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-08-28T02:11:24.398114+0000","last_change":"2023-08-28T02:11:22.190892+0000","last_active":"2023-08-28T02:11:24.398114+0000","last_peered":"2023-08-28T02:11:24.398114+0000","last_clean":"2023-08-28T02:11:24.398114+0000","last_became_active":"2023-08-28T02:11:22.190587+0000","last_became_peered":"2023-08-28T02:11:22.190587+0000","last_unstale":"2023-08-28T02:11:24.398114+0000","last_undegraded":"2023-08-28T02:11:24.398114+0000","last_fullsized":"2023-08-28T02:11:24.398114+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.247873+0000","last_change":"2023-08-28T02:11:22.795766+0000","last_active":"2023-08-28T02:11:23.247873+0000","last_peered":"2023-08-28T02:11:23.247873+0000","last_clean":"2023-08-28T02:11:23.247873+0000","last_became_active":"2023-08-28T02:11:22.795518+0000","last_became_peered":"2023-08-28T02:11:22.795518+0000","last_unstale":"2023-08-28T02:11:23.247873+0000","last_undegraded":"2023-08-28T02:11:23.247873+0000","last_fullsized":"2023-08-28T02:11:23.247873+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":20,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-08-28T02:11:29.946223+0000","last_change":"2023-08-28T02:11:22.188874+0000","last_active":"2023-08-28T02:11:29.946223+0000","last_peered":"2023-08-28T02:11:29.946223+0000","last_clean":"2023-08-28T02:11:29.946223+0000","last_became_active":"2023-08-28T02:11:22.188574+0000","last_became_peered":"2023-08-28T02:11:22.188574+0000","last_unstale":"2023-08-28T02:11:29.946223+0000","last_undegraded":"2023-08-28T02:11:29.946223+0000","last_fullsized":"2023-08-28T02:11:29.946223+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.252023+0000","last_change":"2023-08-28T02:11:23.251287+0000","last_active":"2023-08-28T02:11:23.252023+0000","last_peered":"2023-08-28T02:11:23.252023+0000","last_clean":"2023-08-28T02:11:23.252023+0000","last_became_active":"2023-08-28T02:11:23.250514+0000","last_became_peered":"2023-08-28T02:11:23.250514+0000","last_unstale":"2023-08-28T02:11:23.252023+0000","last_undegraded":"2023-08-28T02:11:23.252023+0000","last_fullsized":"2023-08-28T02:11:23.252023+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.248297+0000","last_change":"2023-08-28T02:11:22.188327+0000","last_active":"2023-08-28T02:11:23.248297+0000","last_peered":"2023-08-28T02:11:23.248297+0000","last_clean":"2023-08-28T02:11:23.248297+0000","last_became_active":"2023-08-28T02:11:22.187870+0000","last_became_peered":"2023-08-28T02:11:22.187870+0000","last_unstale":"2023-08-28T02:11:23.248297+0000","last_undegraded":"2023-08-28T02:11:23.248297+0000","last_fullsized":"2023-08-28T02:11:23.248297+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.251136+0000","last_change":"2023-08-28T02:11:23.250801+0000","last_active":"2023-08-28T02:11:23.251136+0000","last_peered":"2023-08-28T02:11:23.251136+0000","last_clean":"2023-08-28T02:11:23.251136+0000","last_became_active":"2023-08-28T02:11:23.250472+0000","last_became_peered":"2023-08-28T02:11:23.250472+0000","last_unstale":"2023-08-28T02:11:23.251136+0000","last_undegraded":"2023-08-28T02:11:23.251136+0000","last_fullsized":"2023-08-28T02:11:23.251136+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.190463+0000","last_change":"2023-08-28T02:11:22.186018+0000","last_active":"2023-08-28T02:11:23.190463+0000","last_peered":"2023-08-28T02:11:23.190463+0000","last_clean":"2023-08-28T02:11:23.190463+0000","last_became_active":"2023-08-28T02:11:22.185463+0000","last_became_peered":"2023-08-28T02:11:22.185463+0000","last_unstale":"2023-08-28T02:11:23.190463+0000","last_undegraded":"2023-08-28T02:11:23.190463+0000","last_fullsized":"2023-08-28T02:11:23.190463+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.248259+0000","last_change":"2023-08-28T02:11:22.795824+0000","last_active":"2023-08-28T02:11:23.248259+0000","last_peered":"2023-08-28T02:11:23.248259+0000","last_clean":"2023-08-28T02:11:23.248259+0000","last_became_active":"2023-08-28T02:11:22.795520+0000","last_became_peered":"2023-08-28T02:11:22.795520+0000","last_unstale":"2023-08-28T02:11:23.248259+0000","last_undegraded":"2023-08-28T02:11:23.248259+0000","last_fullsized":"2023-08-28T02:11:23.248259+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-08-28T02:11:29.945335+0000","last_change":"2023-08-28T02:11:22.188321+0000","last_active":"2023-08-28T02:11:29.945335+0000","last_peered":"2023-08-28T02:11:29.945335+0000","last_clean":"2023-08-28T02:11:29.945335+0000","last_became_active":"2023-08-28T02:11:22.187964+0000","last_became_peered":"2023-08-28T02:11:22.187964+0000","last_unstale":"2023-08-28T02:11:29.945335+0000","last_undegraded":"2023-08-28T02:11:29.945335+0000","last_fullsized":"2023-08-28T02:11:29.945335+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.247081+0000","last_change":"2023-08-28T02:11:22.437260+0000","last_active":"2023-08-28T02:11:23.247081+0000","last_peered":"2023-08-28T02:11:23.247081+0000","last_clean":"2023-08-28T02:11:23.247081+0000","last_became_active":"2023-08-28T02:11:22.436540+0000","last_became_peered":"2023-08-28T02:11:22.436540+0000","last_unstale":"2023-08-28T02:11:23.247081+0000","last_undegraded":"2023-08-28T02:11:23.247081+0000","last_fullsized":"2023-08-28T02:11:23.247081+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"60'6","reported_seq":18,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-08-28T02:11:29.945752+0000","last_change":"2023-08-28T02:11:22.184497+0000","last_active":"2023-08-28T02:11:29.945752+0000","last_peered":"2023-08-28T02:11:29.945752+0000","last_clean":"2023-08-28T02:11:29.945752+0000","last_became_active":"2023-08-28T02:11:22.184173+0000","last_became_peered":"2023-08-28T02:11:22.184173+0000","last_unstale":"2023-08-28T02:11:29.945752+0000","last_undegraded":"2023-08-28T02:11:29.945752+0000","last_fullsized":"2023-08-28T02:11:29.945752+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.196145+0000","last_change":"2023-08-28T02:11:23.195565+0000","last_active":"2023-08-28T02:11:23.196145+0000","last_peered":"2023-08-28T02:11:23.196145+0000","last_clean":"2023-08-28T02:11:23.196145+0000","last_became_active":"2023-08-28T02:11:23.195257+0000","last_became_peered":"2023-08-28T02:11:23.195257+0000","last_unstale":"2023-08-28T02:11:23.196145+0000","last_undegraded":"2023-08-28T02:11:23.196145+0000","last_fullsized":"2023-08-28T02:11:23.196145+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":15,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-08-28T02:11:29.943646+0000","last_change":"2023-08-28T02:11:22.188562+0000","last_active":"2023-08-28T02:11:29.943646+0000","last_peered":"2023-08-28T02:11:29.943646+0000","last_clean":"2023-08-28T02:11:29.943646+0000","last_became_active":"2023-08-28T02:11:22.188202+0000","last_became_peered":"2023-08-28T02:11:22.188202+0000","last_unstale":"2023-08-28T02:11:29.943646+0000","last_undegraded":"2023-08-28T02:11:29.943646+0000","last_fullsized":"2023-08-28T02:11:29.943646+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.247611+0000","last_change":"2023-08-28T02:11:22.795818+0000","last_active":"2023-08-28T02:11:23.247611+0000","last_peered":"2023-08-28T02:11:23.247611+0000","last_clean":"2023-08-28T02:11:23.247611+0000","last_became_active":"2023-08-28T02:11:22.795403+0000","last_became_peered":"2023-08-28T02:11:22.795403+0000","last_unstale":"2023-08-28T02:11:23.247611+0000","last_undegraded":"2023-08-28T02:11:23.247611+0000","last_fullsized":"2023-08-28T02:11:23.247611+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.193895+0000","last_change":"2023-08-28T02:11:22.193129+0000","last_active":"2023-08-28T02:11:23.193895+0000","last_peered":"2023-08-28T02:11:23.193895+0000","last_clean":"2023-08-28T02:11:23.193895+0000","last_became_active":"2023-08-28T02:11:22.192809+0000","last_became_peered":"2023-08-28T02:11:22.192809+0000","last_unstale":"2023-08-28T02:11:23.193895+0000","last_undegraded":"2023-08-28T02:11:23.193895+0000","last_fullsized":"2023-08-28T02:11:23.193895+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.248543+0000","last_change":"2023-08-28T02:11:22.436987+0000","last_active":"2023-08-28T02:11:23.248543+0000","last_peered":"2023-08-28T02:11:23.248543+0000","last_clean":"2023-08-28T02:11:23.248543+0000","last_became_active":"2023-08-28T02:11:22.436674+0000","last_became_peered":"2023-08-28T02:11:22.436674+0000","last_unstale":"2023-08-28T02:11:23.248543+0000","last_undegraded":"2023-08-28T02:11:23.248543+0000","last_fullsized":"2023-08-28T02:11:23.248543+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.190506+0000","last_change":"2023-08-28T02:11:22.185140+0000","last_active":"2023-08-28T02:11:23.190506+0000","last_peered":"2023-08-28T02:11:23.190506+0000","last_clean":"2023-08-28T02:11:23.190506+0000","last_became_active":"2023-08-28T02:11:22.184758+0000","last_became_peered":"2023-08-28T02:11:22.184758+0000","last_unstale":"2023-08-28T02:11:23.190506+0000","last_undegraded":"2023-08-28T02:11:23.190506+0000","last_fullsized":"2023-08-28T02:11:23.190506+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.196034+0000","last_change":"2023-08-28T02:11:23.195697+0000","last_active":"2023-08-28T02:11:23.196034+0000","last_peered":"2023-08-28T02:11:23.196034+0000","last_clean":"2023-08-28T02:11:23.196034+0000","last_became_active":"2023-08-28T02:11:23.195410+0000","last_became_peered":"2023-08-28T02:11:23.195410+0000","last_unstale":"2023-08-28T02:11:23.196034+0000","last_undegraded":"2023-08-28T02:11:23.196034+0000","last_fullsized":"2023-08-28T02:11:23.196034+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.248648+0000","last_change":"2023-08-28T02:11:22.184927+0000","last_active":"2023-08-28T02:11:23.248648+0000","last_peered":"2023-08-28T02:11:23.248648+0000","last_clean":"2023-08-28T02:11:23.248648+0000","last_became_active":"2023-08-28T02:11:22.184380+0000","last_became_peered":"2023-08-28T02:11:22.184380+0000","last_unstale":"2023-08-28T02:11:23.248648+0000","last_undegraded":"2023-08-28T02:11:23.248648+0000","last_fullsized":"2023-08-28T02:11:23.248648+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.252044+0000","last_change":"2023-08-28T02:11:23.251470+0000","last_active":"2023-08-28T02:11:23.252044+0000","last_peered":"2023-08-28T02:11:23.252044+0000","last_clean":"2023-08-28T02:11:23.252044+0000","last_became_active":"2023-08-28T02:11:23.250981+0000","last_became_peered":"2023-08-28T02:11:23.250981+0000","last_unstale":"2023-08-28T02:11:23.252044+0000","last_undegraded":"2023-08-28T02:11:23.252044+0000","last_fullsized":"2023-08-28T02:11:23.252044+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.194456+0000","last_change":"2023-08-28T02:11:22.188237+0000","last_active":"2023-08-28T02:11:23.194456+0000","last_peered":"2023-08-28T02:11:23.194456+0000","last_clean":"2023-08-28T02:11:23.194456+0000","last_became_active":"2023-08-28T02:11:22.187915+0000","last_became_peered":"2023-08-28T02:11:22.187915+0000","last_unstale":"2023-08-28T02:11:23.194456+0000","last_undegraded":"2023-08-28T02:11:23.194456+0000","last_fullsized":"2023-08-28T02:11:23.194456+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.248504+0000","last_change":"2023-08-28T02:11:22.201830+0000","last_active":"2023-08-28T02:11:23.248504+0000","last_peered":"2023-08-28T02:11:23.248504+0000","last_clean":"2023-08-28T02:11:23.248504+0000","last_became_active":"2023-08-28T02:11:22.201503+0000","last_became_peered":"2023-08-28T02:11:22.201503+0000","last_unstale":"2023-08-28T02:11:23.248504+0000","last_undegraded":"2023-08-28T02:11:23.248504+0000","last_fullsized":"2023-08-28T02:11:23.248504+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-08-28T02:11:24.194481+0000","last_change":"2023-08-28T02:11:22.190852+0000","last_active":"2023-08-28T02:11:24.194481+0000","last_peered":"2023-08-28T02:11:24.194481+0000","last_clean":"2023-08-28T02:11:24.194481+0000","last_became_active":"2023-08-28T02:11:22.190569+0000","last_became_peered":"2023-08-28T02:11:22.190569+0000","last_unstale":"2023-08-28T02:11:24.194481+0000","last_undegraded":"2023-08-28T02:11:24.194481+0000","last_fullsized":"2023-08-28T02:11:24.194481+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430236,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297712,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447440,"statfs":{"total":95995035648,"available":95690178560,"internally_reserved":0,"allocated":630784,"data_stored":257891,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:10:54 2023","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.74199999999999999}]},{"osd":1,"last update":"Mon Aug 28 02:10:54 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.56000000000000005}]},{"osd":2,"last update":"Mon Aug 28 02:10:54 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.77800000000000002}]},{"osd":3,"last update":"Mon Aug 28 02:10:54 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.75900000000000001},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.82099999999999995}]},{"osd":4,"last update":"Mon Aug 28 02:10:54 2023","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.64300000000000002}]},{"osd":5,"last update":"Mon Aug 28 02:10:54 2023","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.80600000000000005},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.71699999999999997}]},{"osd":6,"last update":"Mon Aug 28 02:10:54 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.77100000000000002},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.79300000000000004}]}]},{"osd":6,"up_from":42,"seq":180388626463,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297696,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447456,"statfs":{"total":95995035648,"available":95690194944,"internally_reserved":0,"allocated":614400,"data_stored":257430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:11:36 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.497,"15min":0.501},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.65200000000000002,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.52800000000000002,"15min":0.53200000000000003},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.73199999999999998,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.59499999999999997}]},{"osd":1,"last update":"Mon Aug 28 02:11:36 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.51000000000000001,"15min":0.51200000000000001},"min":{"1min":0.373,"5min":0.312,"15min":0.312},"max":{"1min":0.69099999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.64500000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:11:36 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.499,"15min":0.503},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.622,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.54400000000000004}]},{"osd":3,"last update":"Mon Aug 28 02:11:36 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.55100000000000005,"15min":0.55200000000000005},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.67500000000000004},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.502,"15min":0.498},"min":{"1min":0.36699999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.65600000000000003}]},{"osd":4,"last update":"Mon Aug 28 02:11:36 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.42099999999999999,"15min":0.40699999999999997},"min":{"1min":0.31900000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.434,"15min":0.41599999999999998},"min":{"1min":0.33000000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.73899999999999999}]},{"osd":5,"last update":"Mon Aug 28 02:11:36 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.435,"15min":0.42299999999999999},"min":{"1min":0.35499999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.68999999999999995,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.629,"5min":0.502,"15min":0.48099999999999998},"min":{"1min":0.36599999999999999,"5min":0.316,"15min":0.316},"max":{"1min":2.0609999999999999,"5min":2.0609999999999999,"15min":2.0609999999999999},"last":0.56000000000000005}]},{"osd":7,"last update":"Mon Aug 28 02:10:55 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.629},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.71899999999999997}]}]},{"osd":1,"up_from":13,"seq":55834574903,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297684,"kb_used_data":588,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447468,"statfs":{"total":95995035648,"available":95690207232,"internally_reserved":0,"allocated":602112,"data_stored":256139,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:11:51 2023","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.36099999999999999,"15min":0.33700000000000002},"min":{"1min":0.251,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.54000000000000004,"5min":0.59799999999999998,"15min":0.59799999999999998},"last":0.33400000000000002},{"interface":"front","average":{"1min":0.371,"5min":0.375,"15min":0.34899999999999998},"min":{"1min":0.22600000000000001,"5min":0.215,"15min":0.215},"max":{"1min":0.56699999999999995,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.28100000000000003}]},{"osd":2,"last update":"Mon Aug 28 02:11:09 2023","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":0.375,"15min":0.36599999999999999},"min":{"1min":0.26200000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.51600000000000001,"5min":0.623,"15min":0.623},"last":0.246},{"interface":"front","average":{"1min":0.377,"5min":0.38700000000000001,"15min":0.38},"min":{"1min":0.27700000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.52400000000000002,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.32100000000000001}]},{"osd":3,"last update":"Mon Aug 28 02:11:31 2023","interfaces":[{"interface":"back","average":{"1min":0.38900000000000001,"5min":0.42599999999999999,"15min":0.434},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.48499999999999999,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.311},{"interface":"front","average":{"1min":0.432,"5min":0.46899999999999997,"15min":0.47799999999999998},"min":{"1min":0.33600000000000002,"5min":0.317,"15min":0.317},"max":{"1min":0.66900000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.29399999999999998}]},{"osd":4,"last update":"Mon Aug 28 02:11:51 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.55800000000000005,"15min":0.56000000000000005},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.629},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.57599999999999996,"15min":0.57199999999999995},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.97099999999999997,"5min":0.97099999999999997,"15min":0.97099999999999997},"last":0.66500000000000004}]},{"osd":5,"last update":"Mon Aug 28 02:11:09 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.55900000000000005,"15min":0.56100000000000005},"min":{"1min":0.42799999999999999,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.65900000000000003,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.59999999999999998}]},{"osd":6,"last update":"Mon Aug 28 02:11:35 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.58699999999999997,"15min":0.58899999999999997},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.997,"5min":0.997,"15min":0.997},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.58399999999999996,"15min":0.58099999999999996},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.95399999999999996,"5min":0.95399999999999996,"15min":0.95399999999999996},"last":0.57299999999999995}]},{"osd":7,"last update":"Mon Aug 28 02:11:54 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.61199999999999999,"15min":0.61499999999999999},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.90400000000000003,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.57899999999999996,"15min":0.57299999999999995},"min":{"1min":0.46600000000000003,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.65400000000000003}]}]},{"osd":0,"up_from":8,"seq":34359738428,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297692,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447460,"statfs":{"total":95995035648,"available":95690199040,"internally_reserved":0,"allocated":610304,"data_stored":256715,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Aug 28 02:11:53 2023","interfaces":[{"interface":"back","average":{"1min":0.36099999999999999,"5min":0.35299999999999998,"15min":0.34300000000000003},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.34499999999999997,"5min":0.36199999999999999,"15min":0.34699999999999998},"min":{"1min":0.215,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.62,"5min":0.62,"15min":0.62},"last":0.25}]},{"osd":2,"last update":"Mon Aug 28 02:11:04 2023","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.38400000000000001,"15min":0.38200000000000001},"min":{"1min":0.27100000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.41899999999999998,"15min":0.41299999999999998},"min":{"1min":0.28299999999999997,"5min":0.248,"15min":0.248},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.29899999999999999}]},{"osd":3,"last update":"Mon Aug 28 02:11:27 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.48899999999999999,"15min":0.47299999999999998},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":5.4509999999999996,"5min":5.4509999999999996,"15min":5.4509999999999996},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.36199999999999999,"5min":0.46999999999999997,"15min":0.495},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.46300000000000002,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.38700000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:11:53 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.55100000000000005,"15min":0.54900000000000004},"min":{"1min":0.42199999999999999,"5min":0.37,"15min":0.37},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.58499999999999996,"15min":0.58799999999999997},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.55800000000000005}]},{"osd":5,"last update":"Mon Aug 28 02:11:13 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.76500000000000001,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.58599999999999997,"15min":0.58999999999999997},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.72499999999999998,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.57999999999999996}]},{"osd":6,"last update":"Mon Aug 28 02:11:33 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.57099999999999995,"15min":0.57699999999999996},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1.0469999999999999,"5min":1.0469999999999999,"15min":1.0469999999999999},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.57899999999999996,"15min":0.58099999999999996},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.501}]},{"osd":7,"last update":"Mon Aug 28 02:10:54 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.54400000000000004}]}]},{"osd":2,"up_from":18,"seq":77309411378,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297692,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447460,"statfs":{"total":95995035648,"available":95690199040,"internally_reserved":0,"allocated":610304,"data_stored":256727,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:11:06 2023","interfaces":[{"interface":"back","average":{"1min":0.34899999999999998,"5min":0.35899999999999999,"15min":0.35899999999999999},"min":{"1min":0.23100000000000001,"5min":0.216,"15min":0.216},"max":{"1min":0.48999999999999999,"5min":0.54400000000000004,"15min":0.54400000000000004},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.34499999999999997,"5min":0.372,"15min":0.378},"min":{"1min":0.26300000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":0.48899999999999999,"5min":0.53500000000000003,"15min":0.53500000000000003},"last":0.29499999999999998}]},{"osd":1,"last update":"Mon Aug 28 02:11:06 2023","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.377,"15min":0.378},"min":{"1min":0.23599999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.54100000000000004,"5min":0.56999999999999995,"15min":0.56999999999999995},"last":0.309},{"interface":"front","average":{"1min":0.38200000000000001,"5min":0.38900000000000001,"15min":0.38500000000000001},"min":{"1min":0.249,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.442}]},{"osd":3,"last update":"Mon Aug 28 02:11:29 2023","interfaces":[{"interface":"back","average":{"1min":0.34699999999999998,"5min":0.42599999999999999,"15min":0.44800000000000001},"min":{"1min":0.23799999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.498,"5min":0.55800000000000005,"15min":0.55800000000000005},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.45500000000000002,"15min":0.47999999999999998},"min":{"1min":0.28199999999999997,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.53700000000000003,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.32200000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:11:51 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.55000000000000004,"15min":0.55100000000000005},"min":{"1min":0.38,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.56899999999999995,"15min":0.57299999999999995},"min":{"1min":0.41399999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.497}]},{"osd":5,"last update":"Mon Aug 28 02:11:11 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.53100000000000003,"15min":0.53700000000000003},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.71499999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.53300000000000003,"15min":0.53200000000000003},"min":{"1min":0.38800000000000001,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.505}]},{"osd":6,"last update":"Mon Aug 28 02:11:35 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52800000000000002,"15min":0.52900000000000003},"min":{"1min":0.36599999999999999,"5min":0.316,"15min":0.316},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.432},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.56100000000000005,"15min":0.56499999999999995},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.52400000000000002}]},{"osd":7,"last update":"Mon Aug 28 02:10:54 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.42399999999999999}]}]},{"osd":3,"up_from":25,"seq":107374182447,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297688,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447464,"statfs":{"total":95995035648,"available":95690203136,"internally_reserved":0,"allocated":606208,"data_stored":256229,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:11:28 2023","interfaces":[{"interface":"back","average":{"1min":0.33400000000000002,"5min":0.38100000000000001,"15min":0.39300000000000002},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.443,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.34599999999999997,"5min":0.40400000000000003,"15min":0.41899999999999998},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.54400000000000004,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.26500000000000001}]},{"osd":1,"last update":"Mon Aug 28 02:11:28 2023","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.42699999999999999,"15min":0.43099999999999999},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":1.0580000000000001,"5min":1.0580000000000001,"15min":1.0580000000000001},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.41899999999999998,"15min":0.42999999999999999},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.66800000000000004,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.39200000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:11:28 2023","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.45300000000000001,"15min":0.46500000000000002},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.53300000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.436,"15min":0.44800000000000001},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.57099999999999995,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.40799999999999997}]},{"osd":4,"last update":"Mon Aug 28 02:11:49 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.55800000000000005,"15min":0.56999999999999995},"min":{"1min":0.378,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.68400000000000005,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.58399999999999996,"15min":0.59399999999999997},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.81299999999999994,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.60099999999999998}]},{"osd":5,"last update":"Mon Aug 28 02:11:13 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.57299999999999995,"15min":0.57399999999999995},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.59199999999999997,"15min":0.58799999999999997},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":1.8460000000000001,"5min":1.8460000000000001,"15min":1.8460000000000001},"last":0.54300000000000004}]},{"osd":6,"last update":"Mon Aug 28 02:11:34 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.59699999999999998,"15min":0.60699999999999998},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.73599999999999999,"5min":0.878,"15min":0.878},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.59499999999999997,"15min":0.60099999999999998},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.77100000000000002,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.67800000000000005}]},{"osd":7,"last update":"Mon Aug 28 02:11:55 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.61499999999999999,"15min":0.61899999999999999},"min":{"1min":0.40300000000000002,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.80100000000000005,"5min":1.4179999999999999,"15min":1.4179999999999999},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.623,"15min":0.61899999999999999},"min":{"1min":0.42999999999999999,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":1.2829999999999999,"5min":1.2829999999999999,"15min":1.2829999999999999},"last":0.66400000000000003}]}]},{"osd":4,"up_from":30,"seq":128849018921,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297708,"kb_used_data":612,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447444,"statfs":{"total":95995035648,"available":95690182656,"internally_reserved":0,"allocated":626688,"data_stored":257821,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:11:49 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.53400000000000003,"15min":0.54500000000000004},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.83999999999999997,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.55500000000000005,"15min":0.56200000000000006},"min":{"1min":0.38900000000000001,"5min":0.375,"15min":0.375},"max":{"1min":1.1339999999999999,"5min":1.1339999999999999,"15min":1.1339999999999999},"last":0.29399999999999998}]},{"osd":1,"last update":"Mon Aug 28 02:11:49 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.52400000000000002,"15min":0.53300000000000003},"min":{"1min":0.33000000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.81899999999999995,"5min":0.98899999999999999,"15min":0.98899999999999999},"last":0.439},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.53400000000000003,"15min":0.54000000000000004},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.77000000000000002,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.27400000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:11:49 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.54600000000000004,"15min":0.56200000000000006},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.79300000000000004,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.54500000000000004,"15min":0.55300000000000005},"min":{"1min":0.33600000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.1020000000000001,"5min":1.1020000000000001,"15min":1.1020000000000001},"last":0.32700000000000001}]},{"osd":3,"last update":"Mon Aug 28 02:11:49 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.55400000000000005,"15min":0.55600000000000005},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.1599999999999999,"5min":1.1599999999999999,"15min":1.1599999999999999},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.36399999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.50600000000000001}]},{"osd":5,"last update":"Mon Aug 28 02:11:13 2023","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.39400000000000002,"15min":0.39000000000000001},"min":{"1min":0.32100000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.55800000000000005,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.40999999999999998,"15min":0.39900000000000002},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":1.1839999999999999,"5min":1.1839999999999999,"15min":1.1839999999999999},"last":0.48499999999999999}]},{"osd":6,"last update":"Mon Aug 28 02:11:34 2023","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.442,"15min":0.436},"min":{"1min":0.34999999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.60899999999999999,"5min":0.63,"15min":0.63},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.46800000000000003,"15min":0.46200000000000002},"min":{"1min":0.39400000000000002,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.68899999999999995,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.45600000000000002}]},{"osd":7,"last update":"Mon Aug 28 02:10:57 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.2110000000000001,"5min":1.2110000000000001,"15min":1.2110000000000001},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.56599999999999995}]}]},{"osd":5,"up_from":36,"seq":154618822693,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297684,"kb_used_data":588,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447468,"statfs":{"total":95995035648,"available":95690207232,"internally_reserved":0,"allocated":602112,"data_stored":256133,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:11:12 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.55000000000000004,"15min":0.55500000000000005},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.27400000000000002},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.52700000000000002,"15min":0.52500000000000002},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.56599999999999995}]},{"osd":1,"last update":"Mon Aug 28 02:11:12 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.56899999999999995,"15min":0.57099999999999995},"min":{"1min":0.35599999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.97499999999999998,"5min":0.97499999999999998,"15min":0.97499999999999998},"last":0.436},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.53000000000000003,"15min":0.53300000000000003},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.61299999999999999}]},{"osd":2,"last update":"Mon Aug 28 02:11:12 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.54300000000000004,"15min":0.54400000000000004},"min":{"1min":0.316,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.31},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.56699999999999995,"15min":0.57399999999999995},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.49099999999999999}]},{"osd":3,"last update":"Mon Aug 28 02:11:12 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.53900000000000003,"15min":0.53700000000000003},"min":{"1min":0.38,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.57099999999999995,"15min":0.57399999999999995},"min":{"1min":0.39200000000000002,"5min":0.38,"15min":0.38},"max":{"1min":0.94799999999999995,"5min":0.94799999999999995,"15min":0.94799999999999995},"last":0.54500000000000004}]},{"osd":4,"last update":"Mon Aug 28 02:11:12 2023","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.43099999999999999,"15min":0.42799999999999999},"min":{"1min":0.29399999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.745},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.44800000000000001,"15min":0.441},"min":{"1min":0.30499999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.45900000000000002}]},{"osd":6,"last update":"Mon Aug 28 02:11:31 2023","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.48899999999999999,"15min":0.49199999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.71299999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.80100000000000005},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.49099999999999999,"15min":0.48799999999999999},"min":{"1min":0.35699999999999998,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.63600000000000001,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.52000000000000002}]},{"osd":7,"last update":"Mon Aug 28 02:11:54 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.54200000000000004,"15min":0.54400000000000004},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.67900000000000005,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.55500000000000005,"15min":0.55700000000000005},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.76000000000000001,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.76400000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-08-28T02:12:02.215 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph pg dump --format=json 2023-08-28T02:12:02.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:02 smithi143 conmon[100258]: audit 2023-08-28T02:12:01.575640+0000 mgr.y (mgr.14424) 89 : audit [DBG] from='client.14760 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:12:02.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:02 smithi143 conmon[109923]: audit 2023-08-28T02:12:01.575640+0000 mgr.y (mgr.14424) 89 : audit [DBG] from='client.14760 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:12:02.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:02 smithi169 conmon[100500]: audit 2023-08-28T02:12:01.575640+0000 mgr.y (mgr.14424) 89 : audit [DBG] from='client.14760 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:12:02.566 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:12:03.215 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:12:02 smithi169 conmon[102807]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:12:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-28T02:12:03.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:03 smithi143 conmon[100258]: cluster 2023-08-28T02:12:01.883011+0000 mgr.y (mgr.14424) 90 : cluster [DBG] 2023-08-28T02:12:03.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:03 smithi143 conmon[100258]: pgmap v68: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:12:03.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:03 smithi143 conmon[109923]: cluster 2023-08-28T02:12:01.883011+0000 mgr.y ( 2023-08-28T02:12:03.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:03 smithi143 conmon[109923]: mgr.14424) 90 : cluster [DBG] pgmap v68: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:12:03.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:03 smithi169 conmon[100500]: cluster 2023-08-28T02:12:01.883011+0000 mgr.y (mgr.14424) 90 : cluster [DBG] pgmap v68: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:12:04.033 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:12:04.034 INFO:teuthology.orchestra.run.smithi143.stderr:dumped all 2023-08-28T02:12:04.611 INFO:teuthology.orchestra.run.smithi143.stdout:{"pg_ready":true,"pg_map":{"version":69,"stamp":"2023-08-28T02:12:03.883165+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2381556,"kb_used_data":4788,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747579660,"statfs":{"total":767960285184,"available":765521571840,"internally_reserved":0,"allocated":4902912,"data_stored":2055085,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002729"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.246926+0000","last_change":"2023-08-28T02:11:22.438750+0000","last_active":"2023-08-28T02:11:23.246926+0000","last_peered":"2023-08-28T02:11:23.246926+0000","last_clean":"2023-08-28T02:11:23.246926+0000","last_became_active":"2023-08-28T02:11:22.438438+0000","last_became_peered":"2023-08-28T02:11:22.438438+0000","last_unstale":"2023-08-28T02:11:23.246926+0000","last_undegraded":"2023-08-28T02:11:23.246926+0000","last_fullsized":"2023-08-28T02:11:23.246926+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":16,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-08-28T02:11:29.944829+0000","last_change":"2023-08-28T02:11:22.183512+0000","last_active":"2023-08-28T02:11:29.944829+0000","last_peered":"2023-08-28T02:11:29.944829+0000","last_clean":"2023-08-28T02:11:29.944829+0000","last_became_active":"2023-08-28T02:11:22.182755+0000","last_became_peered":"2023-08-28T02:11:22.182755+0000","last_unstale":"2023-08-28T02:11:29.944829+0000","last_undegraded":"2023-08-28T02:11:29.944829+0000","last_fullsized":"2023-08-28T02:11:29.944829+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.247534+0000","last_change":"2023-08-28T02:11:22.688074+0000","last_active":"2023-08-28T02:11:23.247534+0000","last_peered":"2023-08-28T02:11:23.247534+0000","last_clean":"2023-08-28T02:11:23.247534+0000","last_became_active":"2023-08-28T02:11:22.687745+0000","last_became_peered":"2023-08-28T02:11:22.687745+0000","last_unstale":"2023-08-28T02:11:23.247534+0000","last_undegraded":"2023-08-28T02:11:23.247534+0000","last_fullsized":"2023-08-28T02:11:23.247534+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-08-28T02:11:24.397783+0000","last_change":"2023-08-28T02:11:22.178584+0000","last_active":"2023-08-28T02:11:24.397783+0000","last_peered":"2023-08-28T02:11:24.397783+0000","last_clean":"2023-08-28T02:11:24.397783+0000","last_became_active":"2023-08-28T02:11:22.178247+0000","last_became_peered":"2023-08-28T02:11:22.178247+0000","last_unstale":"2023-08-28T02:11:24.397783+0000","last_undegraded":"2023-08-28T02:11:24.397783+0000","last_fullsized":"2023-08-28T02:11:24.397783+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.247113+0000","last_change":"2023-08-28T02:11:22.688542+0000","last_active":"2023-08-28T02:11:23.247113+0000","last_peered":"2023-08-28T02:11:23.247113+0000","last_clean":"2023-08-28T02:11:23.247113+0000","last_became_active":"2023-08-28T02:11:22.688239+0000","last_became_peered":"2023-08-28T02:11:22.688239+0000","last_unstale":"2023-08-28T02:11:23.247113+0000","last_undegraded":"2023-08-28T02:11:23.247113+0000","last_fullsized":"2023-08-28T02:11:23.247113+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-08-28T02:11:29.945472+0000","last_change":"2023-08-28T02:11:22.186024+0000","last_active":"2023-08-28T02:11:29.945472+0000","last_peered":"2023-08-28T02:11:29.945472+0000","last_clean":"2023-08-28T02:11:29.945472+0000","last_became_active":"2023-08-28T02:11:22.185500+0000","last_became_peered":"2023-08-28T02:11:22.185500+0000","last_unstale":"2023-08-28T02:11:29.945472+0000","last_undegraded":"2023-08-28T02:11:29.945472+0000","last_fullsized":"2023-08-28T02:11:29.945472+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.247478+0000","last_change":"2023-08-28T02:11:22.199354+0000","last_active":"2023-08-28T02:11:23.247478+0000","last_peered":"2023-08-28T02:11:23.247478+0000","last_clean":"2023-08-28T02:11:23.247478+0000","last_became_active":"2023-08-28T02:11:22.199098+0000","last_became_peered":"2023-08-28T02:11:22.199098+0000","last_unstale":"2023-08-28T02:11:23.247478+0000","last_undegraded":"2023-08-28T02:11:23.247478+0000","last_fullsized":"2023-08-28T02:11:23.247478+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-08-28T02:11:29.945272+0000","last_change":"2023-08-28T02:11:22.193215+0000","last_active":"2023-08-28T02:11:29.945272+0000","last_peered":"2023-08-28T02:11:29.945272+0000","last_clean":"2023-08-28T02:11:29.945272+0000","last_became_active":"2023-08-28T02:11:22.192780+0000","last_became_peered":"2023-08-28T02:11:22.192780+0000","last_unstale":"2023-08-28T02:11:29.945272+0000","last_undegraded":"2023-08-28T02:11:29.945272+0000","last_fullsized":"2023-08-28T02:11:29.945272+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.248695+0000","last_change":"2023-08-28T02:11:23.248065+0000","last_active":"2023-08-28T02:11:23.248695+0000","last_peered":"2023-08-28T02:11:23.248695+0000","last_clean":"2023-08-28T02:11:23.248695+0000","last_became_active":"2023-08-28T02:11:23.247741+0000","last_became_peered":"2023-08-28T02:11:23.247741+0000","last_unstale":"2023-08-28T02:11:23.248695+0000","last_undegraded":"2023-08-28T02:11:23.248695+0000","last_fullsized":"2023-08-28T02:11:23.248695+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-08-28T02:11:24.397815+0000","last_change":"2023-08-28T02:11:22.188952+0000","last_active":"2023-08-28T02:11:24.397815+0000","last_peered":"2023-08-28T02:11:24.397815+0000","last_clean":"2023-08-28T02:11:24.397815+0000","last_became_active":"2023-08-28T02:11:22.188583+0000","last_became_peered":"2023-08-28T02:11:22.188583+0000","last_unstale":"2023-08-28T02:11:24.397815+0000","last_undegraded":"2023-08-28T02:11:24.397815+0000","last_fullsized":"2023-08-28T02:11:24.397815+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.247427+0000","last_change":"2023-08-28T02:11:22.437146+0000","last_active":"2023-08-28T02:11:23.247427+0000","last_peered":"2023-08-28T02:11:23.247427+0000","last_clean":"2023-08-28T02:11:23.247427+0000","last_became_active":"2023-08-28T02:11:22.436798+0000","last_became_peered":"2023-08-28T02:11:22.436798+0000","last_unstale":"2023-08-28T02:11:23.247427+0000","last_undegraded":"2023-08-28T02:11:23.247427+0000","last_fullsized":"2023-08-28T02:11:23.247427+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-08-28T02:11:24.398113+0000","last_change":"2023-08-28T02:11:22.182980+0000","last_active":"2023-08-28T02:11:24.398113+0000","last_peered":"2023-08-28T02:11:24.398113+0000","last_clean":"2023-08-28T02:11:24.398113+0000","last_became_active":"2023-08-28T02:11:22.182484+0000","last_became_peered":"2023-08-28T02:11:22.182484+0000","last_unstale":"2023-08-28T02:11:24.398113+0000","last_undegraded":"2023-08-28T02:11:24.398113+0000","last_fullsized":"2023-08-28T02:11:24.398113+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.192062+0000","last_change":"2023-08-28T02:10:32.184257+0000","last_active":"2023-08-28T02:11:23.192062+0000","last_peered":"2023-08-28T02:11:23.192062+0000","last_clean":"2023-08-28T02:11:23.192062+0000","last_became_active":"2023-08-28T02:10:32.183944+0000","last_became_peered":"2023-08-28T02:10:32.183944+0000","last_unstale":"2023-08-28T02:11:23.192062+0000","last_undegraded":"2023-08-28T02:11:23.192062+0000","last_fullsized":"2023-08-28T02:11:23.192062+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.248362+0000","last_change":"2023-08-28T02:11:22.795960+0000","last_active":"2023-08-28T02:11:23.248362+0000","last_peered":"2023-08-28T02:11:23.248362+0000","last_clean":"2023-08-28T02:11:23.248362+0000","last_became_active":"2023-08-28T02:11:22.795379+0000","last_became_peered":"2023-08-28T02:11:22.795379+0000","last_unstale":"2023-08-28T02:11:23.248362+0000","last_undegraded":"2023-08-28T02:11:23.248362+0000","last_fullsized":"2023-08-28T02:11:23.248362+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.194979+0000","last_change":"2023-08-28T02:10:32.197242+0000","last_active":"2023-08-28T02:11:23.194979+0000","last_peered":"2023-08-28T02:11:23.194979+0000","last_clean":"2023-08-28T02:11:23.194979+0000","last_became_active":"2023-08-28T02:10:32.196865+0000","last_became_peered":"2023-08-28T02:10:32.196865+0000","last_unstale":"2023-08-28T02:11:23.194979+0000","last_undegraded":"2023-08-28T02:11:23.194979+0000","last_fullsized":"2023-08-28T02:11:23.194979+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-08-28T02:11:29.942467+0000","last_change":"2023-08-28T02:11:22.183063+0000","last_active":"2023-08-28T02:11:29.942467+0000","last_peered":"2023-08-28T02:11:29.942467+0000","last_clean":"2023-08-28T02:11:29.942467+0000","last_became_active":"2023-08-28T02:11:22.182709+0000","last_became_peered":"2023-08-28T02:11:22.182709+0000","last_unstale":"2023-08-28T02:11:29.942467+0000","last_undegraded":"2023-08-28T02:11:29.942467+0000","last_fullsized":"2023-08-28T02:11:29.942467+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.250261+0000","last_change":"2023-08-28T02:11:23.248976+0000","last_active":"2023-08-28T02:11:23.250261+0000","last_peered":"2023-08-28T02:11:23.250261+0000","last_clean":"2023-08-28T02:11:23.250261+0000","last_became_active":"2023-08-28T02:11:23.248676+0000","last_became_peered":"2023-08-28T02:11:23.248676+0000","last_unstale":"2023-08-28T02:11:23.250261+0000","last_undegraded":"2023-08-28T02:11:23.250261+0000","last_fullsized":"2023-08-28T02:11:23.250261+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.194264+0000","last_change":"2023-08-28T02:11:22.187297+0000","last_active":"2023-08-28T02:11:23.194264+0000","last_peered":"2023-08-28T02:11:23.194264+0000","last_clean":"2023-08-28T02:11:23.194264+0000","last_became_active":"2023-08-28T02:11:22.186947+0000","last_became_peered":"2023-08-28T02:11:22.186947+0000","last_unstale":"2023-08-28T02:11:23.194264+0000","last_undegraded":"2023-08-28T02:11:23.194264+0000","last_fullsized":"2023-08-28T02:11:23.194264+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.249223+0000","last_change":"2023-08-28T02:10:32.198340+0000","last_active":"2023-08-28T02:11:23.249223+0000","last_peered":"2023-08-28T02:11:23.249223+0000","last_clean":"2023-08-28T02:11:23.249223+0000","last_became_active":"2023-08-28T02:10:32.198052+0000","last_became_peered":"2023-08-28T02:10:32.198052+0000","last_unstale":"2023-08-28T02:11:23.249223+0000","last_undegraded":"2023-08-28T02:11:23.249223+0000","last_fullsized":"2023-08-28T02:11:23.249223+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.196689+0000","last_change":"2023-08-28T02:11:23.195766+0000","last_active":"2023-08-28T02:11:23.196689+0000","last_peered":"2023-08-28T02:11:23.196689+0000","last_clean":"2023-08-28T02:11:23.196689+0000","last_became_active":"2023-08-28T02:11:23.195454+0000","last_became_peered":"2023-08-28T02:11:23.195454+0000","last_unstale":"2023-08-28T02:11:23.196689+0000","last_undegraded":"2023-08-28T02:11:23.196689+0000","last_fullsized":"2023-08-28T02:11:23.196689+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.190332+0000","last_change":"2023-08-28T02:11:22.188624+0000","last_active":"2023-08-28T02:11:23.190332+0000","last_peered":"2023-08-28T02:11:23.190332+0000","last_clean":"2023-08-28T02:11:23.190332+0000","last_became_active":"2023-08-28T02:11:22.187351+0000","last_became_peered":"2023-08-28T02:11:22.187351+0000","last_unstale":"2023-08-28T02:11:23.190332+0000","last_undegraded":"2023-08-28T02:11:23.190332+0000","last_fullsized":"2023-08-28T02:11:23.190332+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.248295+0000","last_change":"2023-08-28T02:10:32.198506+0000","last_active":"2023-08-28T02:11:23.248295+0000","last_peered":"2023-08-28T02:11:23.248295+0000","last_clean":"2023-08-28T02:11:23.248295+0000","last_became_active":"2023-08-28T02:10:32.198175+0000","last_became_peered":"2023-08-28T02:10:32.198175+0000","last_unstale":"2023-08-28T02:11:23.248295+0000","last_undegraded":"2023-08-28T02:11:23.248295+0000","last_fullsized":"2023-08-28T02:11:23.248295+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.250383+0000","last_change":"2023-08-28T02:11:23.249640+0000","last_active":"2023-08-28T02:11:23.250383+0000","last_peered":"2023-08-28T02:11:23.250383+0000","last_clean":"2023-08-28T02:11:23.250383+0000","last_became_active":"2023-08-28T02:11:23.249355+0000","last_became_peered":"2023-08-28T02:11:23.249355+0000","last_unstale":"2023-08-28T02:11:23.250383+0000","last_undegraded":"2023-08-28T02:11:23.250383+0000","last_fullsized":"2023-08-28T02:11:23.250383+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.247532+0000","last_change":"2023-08-28T02:11:22.184384+0000","last_active":"2023-08-28T02:11:23.247532+0000","last_peered":"2023-08-28T02:11:23.247532+0000","last_clean":"2023-08-28T02:11:23.247532+0000","last_became_active":"2023-08-28T02:11:22.184123+0000","last_became_peered":"2023-08-28T02:11:22.184123+0000","last_unstale":"2023-08-28T02:11:23.247532+0000","last_undegraded":"2023-08-28T02:11:23.247532+0000","last_fullsized":"2023-08-28T02:11:23.247532+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":20,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.248205+0000","last_change":"2023-08-28T02:09:55.609189+0000","last_active":"2023-08-28T02:11:23.248205+0000","last_peered":"2023-08-28T02:11:23.248205+0000","last_clean":"2023-08-28T02:11:23.248205+0000","last_became_active":"2023-08-28T02:09:55.608705+0000","last_became_peered":"2023-08-28T02:09:55.608705+0000","last_unstale":"2023-08-28T02:11:23.248205+0000","last_undegraded":"2023-08-28T02:11:23.248205+0000","last_fullsized":"2023-08-28T02:11:23.248205+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_clean_scrub_stamp":"2023-08-28T02:08:02.043718+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.193836+0000","last_change":"2023-08-28T02:10:32.184422+0000","last_active":"2023-08-28T02:11:23.193836+0000","last_peered":"2023-08-28T02:11:23.193836+0000","last_clean":"2023-08-28T02:11:23.193836+0000","last_became_active":"2023-08-28T02:10:32.184072+0000","last_became_peered":"2023-08-28T02:10:32.184072+0000","last_unstale":"2023-08-28T02:11:23.193836+0000","last_undegraded":"2023-08-28T02:11:23.193836+0000","last_fullsized":"2023-08-28T02:11:23.193836+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.196457+0000","last_change":"2023-08-28T02:11:23.195860+0000","last_active":"2023-08-28T02:11:23.196457+0000","last_peered":"2023-08-28T02:11:23.196457+0000","last_clean":"2023-08-28T02:11:23.196457+0000","last_became_active":"2023-08-28T02:11:23.195574+0000","last_became_peered":"2023-08-28T02:11:23.195574+0000","last_unstale":"2023-08-28T02:11:23.196457+0000","last_undegraded":"2023-08-28T02:11:23.196457+0000","last_fullsized":"2023-08-28T02:11:23.196457+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.194687+0000","last_change":"2023-08-28T02:11:22.184399+0000","last_active":"2023-08-28T02:11:23.194687+0000","last_peered":"2023-08-28T02:11:23.194687+0000","last_clean":"2023-08-28T02:11:23.194687+0000","last_became_active":"2023-08-28T02:11:22.184024+0000","last_became_peered":"2023-08-28T02:11:22.184024+0000","last_unstale":"2023-08-28T02:11:23.194687+0000","last_undegraded":"2023-08-28T02:11:23.194687+0000","last_fullsized":"2023-08-28T02:11:23.194687+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.192330+0000","last_change":"2023-08-28T02:10:32.197650+0000","last_active":"2023-08-28T02:11:23.192330+0000","last_peered":"2023-08-28T02:11:23.192330+0000","last_clean":"2023-08-28T02:11:23.192330+0000","last_became_active":"2023-08-28T02:10:32.197319+0000","last_became_peered":"2023-08-28T02:10:32.197319+0000","last_unstale":"2023-08-28T02:11:23.192330+0000","last_undegraded":"2023-08-28T02:11:23.192330+0000","last_fullsized":"2023-08-28T02:11:23.192330+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.251921+0000","last_change":"2023-08-28T02:11:23.250778+0000","last_active":"2023-08-28T02:11:23.251921+0000","last_peered":"2023-08-28T02:11:23.251921+0000","last_clean":"2023-08-28T02:11:23.251921+0000","last_became_active":"2023-08-28T02:11:23.250402+0000","last_became_peered":"2023-08-28T02:11:23.250402+0000","last_unstale":"2023-08-28T02:11:23.251921+0000","last_undegraded":"2023-08-28T02:11:23.251921+0000","last_fullsized":"2023-08-28T02:11:23.251921+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.247277+0000","last_change":"2023-08-28T02:11:22.188913+0000","last_active":"2023-08-28T02:11:23.247277+0000","last_peered":"2023-08-28T02:11:23.247277+0000","last_clean":"2023-08-28T02:11:23.247277+0000","last_became_active":"2023-08-28T02:11:22.188508+0000","last_became_peered":"2023-08-28T02:11:22.188508+0000","last_unstale":"2023-08-28T02:11:23.247277+0000","last_undegraded":"2023-08-28T02:11:23.247277+0000","last_fullsized":"2023-08-28T02:11:23.247277+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.193852+0000","last_change":"2023-08-28T02:10:32.184125+0000","last_active":"2023-08-28T02:11:23.193852+0000","last_peered":"2023-08-28T02:11:23.193852+0000","last_clean":"2023-08-28T02:11:23.193852+0000","last_became_active":"2023-08-28T02:10:32.183865+0000","last_became_peered":"2023-08-28T02:10:32.183865+0000","last_unstale":"2023-08-28T02:11:23.193852+0000","last_undegraded":"2023-08-28T02:11:23.193852+0000","last_fullsized":"2023-08-28T02:11:23.193852+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.249798+0000","last_change":"2023-08-28T02:11:23.249476+0000","last_active":"2023-08-28T02:11:23.249798+0000","last_peered":"2023-08-28T02:11:23.249798+0000","last_clean":"2023-08-28T02:11:23.249798+0000","last_became_active":"2023-08-28T02:11:23.249246+0000","last_became_peered":"2023-08-28T02:11:23.249246+0000","last_unstale":"2023-08-28T02:11:23.249798+0000","last_undegraded":"2023-08-28T02:11:23.249798+0000","last_fullsized":"2023-08-28T02:11:23.249798+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.194008+0000","last_change":"2023-08-28T02:11:22.193844+0000","last_active":"2023-08-28T02:11:23.194008+0000","last_peered":"2023-08-28T02:11:23.194008+0000","last_clean":"2023-08-28T02:11:23.194008+0000","last_became_active":"2023-08-28T02:11:22.193583+0000","last_became_peered":"2023-08-28T02:11:22.193583+0000","last_unstale":"2023-08-28T02:11:23.194008+0000","last_undegraded":"2023-08-28T02:11:23.194008+0000","last_fullsized":"2023-08-28T02:11:23.194008+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.194163+0000","last_change":"2023-08-28T02:10:32.197552+0000","last_active":"2023-08-28T02:11:23.194163+0000","last_peered":"2023-08-28T02:11:23.194163+0000","last_clean":"2023-08-28T02:11:23.194163+0000","last_became_active":"2023-08-28T02:10:32.197251+0000","last_became_peered":"2023-08-28T02:10:32.197251+0000","last_unstale":"2023-08-28T02:11:23.194163+0000","last_undegraded":"2023-08-28T02:11:23.194163+0000","last_fullsized":"2023-08-28T02:11:23.194163+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.252207+0000","last_change":"2023-08-28T02:11:23.251282+0000","last_active":"2023-08-28T02:11:23.252207+0000","last_peered":"2023-08-28T02:11:23.252207+0000","last_clean":"2023-08-28T02:11:23.252207+0000","last_became_active":"2023-08-28T02:11:23.250975+0000","last_became_peered":"2023-08-28T02:11:23.250975+0000","last_unstale":"2023-08-28T02:11:23.252207+0000","last_undegraded":"2023-08-28T02:11:23.252207+0000","last_fullsized":"2023-08-28T02:11:23.252207+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.248465+0000","last_change":"2023-08-28T02:11:22.184709+0000","last_active":"2023-08-28T02:11:23.248465+0000","last_peered":"2023-08-28T02:11:23.248465+0000","last_clean":"2023-08-28T02:11:23.248465+0000","last_became_active":"2023-08-28T02:11:22.183968+0000","last_became_peered":"2023-08-28T02:11:22.183968+0000","last_unstale":"2023-08-28T02:11:23.248465+0000","last_undegraded":"2023-08-28T02:11:23.248465+0000","last_fullsized":"2023-08-28T02:11:23.248465+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.247271+0000","last_change":"2023-08-28T02:11:22.437038+0000","last_active":"2023-08-28T02:11:23.247271+0000","last_peered":"2023-08-28T02:11:23.247271+0000","last_clean":"2023-08-28T02:11:23.247271+0000","last_became_active":"2023-08-28T02:11:22.436628+0000","last_became_peered":"2023-08-28T02:11:22.436628+0000","last_unstale":"2023-08-28T02:11:23.247271+0000","last_undegraded":"2023-08-28T02:11:23.247271+0000","last_fullsized":"2023-08-28T02:11:23.247271+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.247278+0000","last_change":"2023-08-28T02:11:22.182003+0000","last_active":"2023-08-28T02:11:23.247278+0000","last_peered":"2023-08-28T02:11:23.247278+0000","last_clean":"2023-08-28T02:11:23.247278+0000","last_became_active":"2023-08-28T02:11:22.181682+0000","last_became_peered":"2023-08-28T02:11:22.181682+0000","last_unstale":"2023-08-28T02:11:23.247278+0000","last_undegraded":"2023-08-28T02:11:23.247278+0000","last_fullsized":"2023-08-28T02:11:23.247278+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.247745+0000","last_change":"2023-08-28T02:11:22.198232+0000","last_active":"2023-08-28T02:11:23.247745+0000","last_peered":"2023-08-28T02:11:23.247745+0000","last_clean":"2023-08-28T02:11:23.247745+0000","last_became_active":"2023-08-28T02:11:22.197847+0000","last_became_peered":"2023-08-28T02:11:22.197847+0000","last_unstale":"2023-08-28T02:11:23.247745+0000","last_undegraded":"2023-08-28T02:11:23.247745+0000","last_fullsized":"2023-08-28T02:11:23.247745+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.192181+0000","last_change":"2023-08-28T02:11:22.185928+0000","last_active":"2023-08-28T02:11:23.192181+0000","last_peered":"2023-08-28T02:11:23.192181+0000","last_clean":"2023-08-28T02:11:23.192181+0000","last_became_active":"2023-08-28T02:11:22.185594+0000","last_became_peered":"2023-08-28T02:11:22.185594+0000","last_unstale":"2023-08-28T02:11:23.192181+0000","last_undegraded":"2023-08-28T02:11:23.192181+0000","last_fullsized":"2023-08-28T02:11:23.192181+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.247316+0000","last_change":"2023-08-28T02:11:22.437303+0000","last_active":"2023-08-28T02:11:23.247316+0000","last_peered":"2023-08-28T02:11:23.247316+0000","last_clean":"2023-08-28T02:11:23.247316+0000","last_became_active":"2023-08-28T02:11:22.436825+0000","last_became_peered":"2023-08-28T02:11:22.436825+0000","last_unstale":"2023-08-28T02:11:23.247316+0000","last_undegraded":"2023-08-28T02:11:23.247316+0000","last_fullsized":"2023-08-28T02:11:23.247316+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-08-28T02:11:29.945097+0000","last_change":"2023-08-28T02:11:22.181752+0000","last_active":"2023-08-28T02:11:29.945097+0000","last_peered":"2023-08-28T02:11:29.945097+0000","last_clean":"2023-08-28T02:11:29.945097+0000","last_became_active":"2023-08-28T02:11:22.181420+0000","last_became_peered":"2023-08-28T02:11:22.181420+0000","last_unstale":"2023-08-28T02:11:29.945097+0000","last_undegraded":"2023-08-28T02:11:29.945097+0000","last_fullsized":"2023-08-28T02:11:29.945097+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.249951+0000","last_change":"2023-08-28T02:11:23.248381+0000","last_active":"2023-08-28T02:11:23.249951+0000","last_peered":"2023-08-28T02:11:23.249951+0000","last_clean":"2023-08-28T02:11:23.249951+0000","last_became_active":"2023-08-28T02:11:23.248045+0000","last_became_peered":"2023-08-28T02:11:23.248045+0000","last_unstale":"2023-08-28T02:11:23.249951+0000","last_undegraded":"2023-08-28T02:11:23.249951+0000","last_fullsized":"2023-08-28T02:11:23.249951+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.194009+0000","last_change":"2023-08-28T02:11:22.186907+0000","last_active":"2023-08-28T02:11:23.194009+0000","last_peered":"2023-08-28T02:11:23.194009+0000","last_clean":"2023-08-28T02:11:23.194009+0000","last_became_active":"2023-08-28T02:11:22.186524+0000","last_became_peered":"2023-08-28T02:11:22.186524+0000","last_unstale":"2023-08-28T02:11:23.194009+0000","last_undegraded":"2023-08-28T02:11:23.194009+0000","last_fullsized":"2023-08-28T02:11:23.194009+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.249825+0000","last_change":"2023-08-28T02:11:23.249127+0000","last_active":"2023-08-28T02:11:23.249825+0000","last_peered":"2023-08-28T02:11:23.249825+0000","last_clean":"2023-08-28T02:11:23.249825+0000","last_became_active":"2023-08-28T02:11:23.248816+0000","last_became_peered":"2023-08-28T02:11:23.248816+0000","last_unstale":"2023-08-28T02:11:23.249825+0000","last_undegraded":"2023-08-28T02:11:23.249825+0000","last_fullsized":"2023-08-28T02:11:23.249825+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-08-28T02:11:24.398759+0000","last_change":"2023-08-28T02:11:22.188982+0000","last_active":"2023-08-28T02:11:24.398759+0000","last_peered":"2023-08-28T02:11:24.398759+0000","last_clean":"2023-08-28T02:11:24.398759+0000","last_became_active":"2023-08-28T02:11:22.188663+0000","last_became_peered":"2023-08-28T02:11:22.188663+0000","last_unstale":"2023-08-28T02:11:24.398759+0000","last_undegraded":"2023-08-28T02:11:24.398759+0000","last_fullsized":"2023-08-28T02:11:24.398759+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.247099+0000","last_change":"2023-08-28T02:11:22.202705+0000","last_active":"2023-08-28T02:11:23.247099+0000","last_peered":"2023-08-28T02:11:23.247099+0000","last_clean":"2023-08-28T02:11:23.247099+0000","last_became_active":"2023-08-28T02:11:22.202439+0000","last_became_peered":"2023-08-28T02:11:22.202439+0000","last_unstale":"2023-08-28T02:11:23.247099+0000","last_undegraded":"2023-08-28T02:11:23.247099+0000","last_fullsized":"2023-08-28T02:11:23.247099+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-08-28T02:11:29.943584+0000","last_change":"2023-08-28T02:11:22.191353+0000","last_active":"2023-08-28T02:11:29.943584+0000","last_peered":"2023-08-28T02:11:29.943584+0000","last_clean":"2023-08-28T02:11:29.943584+0000","last_became_active":"2023-08-28T02:11:22.191014+0000","last_became_peered":"2023-08-28T02:11:22.191014+0000","last_unstale":"2023-08-28T02:11:29.943584+0000","last_undegraded":"2023-08-28T02:11:29.943584+0000","last_fullsized":"2023-08-28T02:11:29.943584+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.249690+0000","last_change":"2023-08-28T02:11:23.249061+0000","last_active":"2023-08-28T02:11:23.249690+0000","last_peered":"2023-08-28T02:11:23.249690+0000","last_clean":"2023-08-28T02:11:23.249690+0000","last_became_active":"2023-08-28T02:11:23.248739+0000","last_became_peered":"2023-08-28T02:11:23.248739+0000","last_unstale":"2023-08-28T02:11:23.249690+0000","last_undegraded":"2023-08-28T02:11:23.249690+0000","last_fullsized":"2023-08-28T02:11:23.249690+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-08-28T02:11:24.398114+0000","last_change":"2023-08-28T02:11:22.190892+0000","last_active":"2023-08-28T02:11:24.398114+0000","last_peered":"2023-08-28T02:11:24.398114+0000","last_clean":"2023-08-28T02:11:24.398114+0000","last_became_active":"2023-08-28T02:11:22.190587+0000","last_became_peered":"2023-08-28T02:11:22.190587+0000","last_unstale":"2023-08-28T02:11:24.398114+0000","last_undegraded":"2023-08-28T02:11:24.398114+0000","last_fullsized":"2023-08-28T02:11:24.398114+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.247873+0000","last_change":"2023-08-28T02:11:22.795766+0000","last_active":"2023-08-28T02:11:23.247873+0000","last_peered":"2023-08-28T02:11:23.247873+0000","last_clean":"2023-08-28T02:11:23.247873+0000","last_became_active":"2023-08-28T02:11:22.795518+0000","last_became_peered":"2023-08-28T02:11:22.795518+0000","last_unstale":"2023-08-28T02:11:23.247873+0000","last_undegraded":"2023-08-28T02:11:23.247873+0000","last_fullsized":"2023-08-28T02:11:23.247873+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":20,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-08-28T02:11:29.946223+0000","last_change":"2023-08-28T02:11:22.188874+0000","last_active":"2023-08-28T02:11:29.946223+0000","last_peered":"2023-08-28T02:11:29.946223+0000","last_clean":"2023-08-28T02:11:29.946223+0000","last_became_active":"2023-08-28T02:11:22.188574+0000","last_became_peered":"2023-08-28T02:11:22.188574+0000","last_unstale":"2023-08-28T02:11:29.946223+0000","last_undegraded":"2023-08-28T02:11:29.946223+0000","last_fullsized":"2023-08-28T02:11:29.946223+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.252023+0000","last_change":"2023-08-28T02:11:23.251287+0000","last_active":"2023-08-28T02:11:23.252023+0000","last_peered":"2023-08-28T02:11:23.252023+0000","last_clean":"2023-08-28T02:11:23.252023+0000","last_became_active":"2023-08-28T02:11:23.250514+0000","last_became_peered":"2023-08-28T02:11:23.250514+0000","last_unstale":"2023-08-28T02:11:23.252023+0000","last_undegraded":"2023-08-28T02:11:23.252023+0000","last_fullsized":"2023-08-28T02:11:23.252023+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.248297+0000","last_change":"2023-08-28T02:11:22.188327+0000","last_active":"2023-08-28T02:11:23.248297+0000","last_peered":"2023-08-28T02:11:23.248297+0000","last_clean":"2023-08-28T02:11:23.248297+0000","last_became_active":"2023-08-28T02:11:22.187870+0000","last_became_peered":"2023-08-28T02:11:22.187870+0000","last_unstale":"2023-08-28T02:11:23.248297+0000","last_undegraded":"2023-08-28T02:11:23.248297+0000","last_fullsized":"2023-08-28T02:11:23.248297+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.251136+0000","last_change":"2023-08-28T02:11:23.250801+0000","last_active":"2023-08-28T02:11:23.251136+0000","last_peered":"2023-08-28T02:11:23.251136+0000","last_clean":"2023-08-28T02:11:23.251136+0000","last_became_active":"2023-08-28T02:11:23.250472+0000","last_became_peered":"2023-08-28T02:11:23.250472+0000","last_unstale":"2023-08-28T02:11:23.251136+0000","last_undegraded":"2023-08-28T02:11:23.251136+0000","last_fullsized":"2023-08-28T02:11:23.251136+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.190463+0000","last_change":"2023-08-28T02:11:22.186018+0000","last_active":"2023-08-28T02:11:23.190463+0000","last_peered":"2023-08-28T02:11:23.190463+0000","last_clean":"2023-08-28T02:11:23.190463+0000","last_became_active":"2023-08-28T02:11:22.185463+0000","last_became_peered":"2023-08-28T02:11:22.185463+0000","last_unstale":"2023-08-28T02:11:23.190463+0000","last_undegraded":"2023-08-28T02:11:23.190463+0000","last_fullsized":"2023-08-28T02:11:23.190463+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.248259+0000","last_change":"2023-08-28T02:11:22.795824+0000","last_active":"2023-08-28T02:11:23.248259+0000","last_peered":"2023-08-28T02:11:23.248259+0000","last_clean":"2023-08-28T02:11:23.248259+0000","last_became_active":"2023-08-28T02:11:22.795520+0000","last_became_peered":"2023-08-28T02:11:22.795520+0000","last_unstale":"2023-08-28T02:11:23.248259+0000","last_undegraded":"2023-08-28T02:11:23.248259+0000","last_fullsized":"2023-08-28T02:11:23.248259+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-08-28T02:11:29.945335+0000","last_change":"2023-08-28T02:11:22.188321+0000","last_active":"2023-08-28T02:11:29.945335+0000","last_peered":"2023-08-28T02:11:29.945335+0000","last_clean":"2023-08-28T02:11:29.945335+0000","last_became_active":"2023-08-28T02:11:22.187964+0000","last_became_peered":"2023-08-28T02:11:22.187964+0000","last_unstale":"2023-08-28T02:11:29.945335+0000","last_undegraded":"2023-08-28T02:11:29.945335+0000","last_fullsized":"2023-08-28T02:11:29.945335+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.247081+0000","last_change":"2023-08-28T02:11:22.437260+0000","last_active":"2023-08-28T02:11:23.247081+0000","last_peered":"2023-08-28T02:11:23.247081+0000","last_clean":"2023-08-28T02:11:23.247081+0000","last_became_active":"2023-08-28T02:11:22.436540+0000","last_became_peered":"2023-08-28T02:11:22.436540+0000","last_unstale":"2023-08-28T02:11:23.247081+0000","last_undegraded":"2023-08-28T02:11:23.247081+0000","last_fullsized":"2023-08-28T02:11:23.247081+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"60'6","reported_seq":18,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-08-28T02:11:29.945752+0000","last_change":"2023-08-28T02:11:22.184497+0000","last_active":"2023-08-28T02:11:29.945752+0000","last_peered":"2023-08-28T02:11:29.945752+0000","last_clean":"2023-08-28T02:11:29.945752+0000","last_became_active":"2023-08-28T02:11:22.184173+0000","last_became_peered":"2023-08-28T02:11:22.184173+0000","last_unstale":"2023-08-28T02:11:29.945752+0000","last_undegraded":"2023-08-28T02:11:29.945752+0000","last_fullsized":"2023-08-28T02:11:29.945752+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.196145+0000","last_change":"2023-08-28T02:11:23.195565+0000","last_active":"2023-08-28T02:11:23.196145+0000","last_peered":"2023-08-28T02:11:23.196145+0000","last_clean":"2023-08-28T02:11:23.196145+0000","last_became_active":"2023-08-28T02:11:23.195257+0000","last_became_peered":"2023-08-28T02:11:23.195257+0000","last_unstale":"2023-08-28T02:11:23.196145+0000","last_undegraded":"2023-08-28T02:11:23.196145+0000","last_fullsized":"2023-08-28T02:11:23.196145+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":15,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-08-28T02:11:29.943646+0000","last_change":"2023-08-28T02:11:22.188562+0000","last_active":"2023-08-28T02:11:29.943646+0000","last_peered":"2023-08-28T02:11:29.943646+0000","last_clean":"2023-08-28T02:11:29.943646+0000","last_became_active":"2023-08-28T02:11:22.188202+0000","last_became_peered":"2023-08-28T02:11:22.188202+0000","last_unstale":"2023-08-28T02:11:29.943646+0000","last_undegraded":"2023-08-28T02:11:29.943646+0000","last_fullsized":"2023-08-28T02:11:29.943646+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.247611+0000","last_change":"2023-08-28T02:11:22.795818+0000","last_active":"2023-08-28T02:11:23.247611+0000","last_peered":"2023-08-28T02:11:23.247611+0000","last_clean":"2023-08-28T02:11:23.247611+0000","last_became_active":"2023-08-28T02:11:22.795403+0000","last_became_peered":"2023-08-28T02:11:22.795403+0000","last_unstale":"2023-08-28T02:11:23.247611+0000","last_undegraded":"2023-08-28T02:11:23.247611+0000","last_fullsized":"2023-08-28T02:11:23.247611+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.193895+0000","last_change":"2023-08-28T02:11:22.193129+0000","last_active":"2023-08-28T02:11:23.193895+0000","last_peered":"2023-08-28T02:11:23.193895+0000","last_clean":"2023-08-28T02:11:23.193895+0000","last_became_active":"2023-08-28T02:11:22.192809+0000","last_became_peered":"2023-08-28T02:11:22.192809+0000","last_unstale":"2023-08-28T02:11:23.193895+0000","last_undegraded":"2023-08-28T02:11:23.193895+0000","last_fullsized":"2023-08-28T02:11:23.193895+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.248543+0000","last_change":"2023-08-28T02:11:22.436987+0000","last_active":"2023-08-28T02:11:23.248543+0000","last_peered":"2023-08-28T02:11:23.248543+0000","last_clean":"2023-08-28T02:11:23.248543+0000","last_became_active":"2023-08-28T02:11:22.436674+0000","last_became_peered":"2023-08-28T02:11:22.436674+0000","last_unstale":"2023-08-28T02:11:23.248543+0000","last_undegraded":"2023-08-28T02:11:23.248543+0000","last_fullsized":"2023-08-28T02:11:23.248543+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.190506+0000","last_change":"2023-08-28T02:11:22.185140+0000","last_active":"2023-08-28T02:11:23.190506+0000","last_peered":"2023-08-28T02:11:23.190506+0000","last_clean":"2023-08-28T02:11:23.190506+0000","last_became_active":"2023-08-28T02:11:22.184758+0000","last_became_peered":"2023-08-28T02:11:22.184758+0000","last_unstale":"2023-08-28T02:11:23.190506+0000","last_undegraded":"2023-08-28T02:11:23.190506+0000","last_fullsized":"2023-08-28T02:11:23.190506+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.196034+0000","last_change":"2023-08-28T02:11:23.195697+0000","last_active":"2023-08-28T02:11:23.196034+0000","last_peered":"2023-08-28T02:11:23.196034+0000","last_clean":"2023-08-28T02:11:23.196034+0000","last_became_active":"2023-08-28T02:11:23.195410+0000","last_became_peered":"2023-08-28T02:11:23.195410+0000","last_unstale":"2023-08-28T02:11:23.196034+0000","last_undegraded":"2023-08-28T02:11:23.196034+0000","last_fullsized":"2023-08-28T02:11:23.196034+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.248648+0000","last_change":"2023-08-28T02:11:22.184927+0000","last_active":"2023-08-28T02:11:23.248648+0000","last_peered":"2023-08-28T02:11:23.248648+0000","last_clean":"2023-08-28T02:11:23.248648+0000","last_became_active":"2023-08-28T02:11:22.184380+0000","last_became_peered":"2023-08-28T02:11:22.184380+0000","last_unstale":"2023-08-28T02:11:23.248648+0000","last_undegraded":"2023-08-28T02:11:23.248648+0000","last_fullsized":"2023-08-28T02:11:23.248648+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.252044+0000","last_change":"2023-08-28T02:11:23.251470+0000","last_active":"2023-08-28T02:11:23.252044+0000","last_peered":"2023-08-28T02:11:23.252044+0000","last_clean":"2023-08-28T02:11:23.252044+0000","last_became_active":"2023-08-28T02:11:23.250981+0000","last_became_peered":"2023-08-28T02:11:23.250981+0000","last_unstale":"2023-08-28T02:11:23.252044+0000","last_undegraded":"2023-08-28T02:11:23.252044+0000","last_fullsized":"2023-08-28T02:11:23.252044+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.194456+0000","last_change":"2023-08-28T02:11:22.188237+0000","last_active":"2023-08-28T02:11:23.194456+0000","last_peered":"2023-08-28T02:11:23.194456+0000","last_clean":"2023-08-28T02:11:23.194456+0000","last_became_active":"2023-08-28T02:11:22.187915+0000","last_became_peered":"2023-08-28T02:11:22.187915+0000","last_unstale":"2023-08-28T02:11:23.194456+0000","last_undegraded":"2023-08-28T02:11:23.194456+0000","last_fullsized":"2023-08-28T02:11:23.194456+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-08-28T02:11:23.248504+0000","last_change":"2023-08-28T02:11:22.201830+0000","last_active":"2023-08-28T02:11:23.248504+0000","last_peered":"2023-08-28T02:11:23.248504+0000","last_clean":"2023-08-28T02:11:23.248504+0000","last_became_active":"2023-08-28T02:11:22.201503+0000","last_became_peered":"2023-08-28T02:11:22.201503+0000","last_unstale":"2023-08-28T02:11:23.248504+0000","last_undegraded":"2023-08-28T02:11:23.248504+0000","last_fullsized":"2023-08-28T02:11:23.248504+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-08-28T02:11:24.194481+0000","last_change":"2023-08-28T02:11:22.190852+0000","last_active":"2023-08-28T02:11:24.194481+0000","last_peered":"2023-08-28T02:11:24.194481+0000","last_clean":"2023-08-28T02:11:24.194481+0000","last_became_active":"2023-08-28T02:11:22.190569+0000","last_became_peered":"2023-08-28T02:11:22.190569+0000","last_unstale":"2023-08-28T02:11:24.194481+0000","last_undegraded":"2023-08-28T02:11:24.194481+0000","last_fullsized":"2023-08-28T02:11:24.194481+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430237,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297712,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447440,"statfs":{"total":95995035648,"available":95690178560,"internally_reserved":0,"allocated":630784,"data_stored":257891,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:11:58 2023","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.499,"15min":0.503},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.56399999999999995,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.495,"5min":0.51500000000000001,"15min":0.51800000000000002},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.65200000000000002}]},{"osd":1,"last update":"Mon Aug 28 02:11:58 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.54100000000000004,"15min":0.54700000000000004},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.69699999999999995,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.621},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.58199999999999996}]},{"osd":2,"last update":"Mon Aug 28 02:11:58 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.53400000000000003,"15min":0.53800000000000003},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.68999999999999995,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.41599999999999998,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.69999999999999996}]},{"osd":3,"last update":"Mon Aug 28 02:11:58 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.53500000000000003,"15min":0.53100000000000003},"min":{"1min":0.433,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.75900000000000001,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.54800000000000004,"15min":0.54400000000000004},"min":{"1min":0.39200000000000002,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.67400000000000004}]},{"osd":4,"last update":"Mon Aug 28 02:11:58 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.47099999999999997,"15min":0.46400000000000002},"min":{"1min":0.33400000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.435},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.49199999999999999,"15min":0.48399999999999999},"min":{"1min":0.35599999999999998,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.56100000000000005}]},{"osd":5,"last update":"Mon Aug 28 02:11:58 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.48299999999999998,"15min":0.47699999999999998},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.53100000000000003,"15min":0.52700000000000002},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.497}]},{"osd":6,"last update":"Mon Aug 28 02:11:58 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.53000000000000003,"15min":0.52600000000000002},"min":{"1min":0.39800000000000002,"5min":0.374,"15min":0.374},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.56299999999999994,"15min":0.55800000000000005},"min":{"1min":0.41099999999999998,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.79300000000000004,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.54500000000000004}]}]},{"osd":6,"up_from":42,"seq":180388626464,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297696,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447456,"statfs":{"total":95995035648,"available":95690194944,"internally_reserved":0,"allocated":614400,"data_stored":257430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:11:36 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.497,"15min":0.501},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.65200000000000002,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.52800000000000002,"15min":0.53200000000000003},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.73199999999999998,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.57899999999999996}]},{"osd":1,"last update":"Mon Aug 28 02:11:36 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.51000000000000001,"15min":0.51200000000000001},"min":{"1min":0.373,"5min":0.312,"15min":0.312},"max":{"1min":0.69099999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.73499999999999999}]},{"osd":2,"last update":"Mon Aug 28 02:11:36 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.81799999999999995},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.499,"15min":0.503},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.622,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.59699999999999998}]},{"osd":3,"last update":"Mon Aug 28 02:11:36 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.55100000000000005,"15min":0.55200000000000005},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.502,"15min":0.498},"min":{"1min":0.36699999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.63800000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:11:36 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.42099999999999999,"15min":0.40699999999999997},"min":{"1min":0.31900000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.75},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.434,"15min":0.41599999999999998},"min":{"1min":0.33000000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.78400000000000003}]},{"osd":5,"last update":"Mon Aug 28 02:11:36 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.435,"15min":0.42299999999999999},"min":{"1min":0.35499999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.68999999999999995,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.629,"5min":0.502,"15min":0.48099999999999998},"min":{"1min":0.36599999999999999,"5min":0.316,"15min":0.316},"max":{"1min":2.0609999999999999,"5min":2.0609999999999999,"15min":2.0609999999999999},"last":0.60999999999999999}]},{"osd":7,"last update":"Mon Aug 28 02:11:56 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.55100000000000005,"15min":0.54400000000000004},"min":{"1min":0.39200000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.70399999999999996},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.57499999999999996,"15min":0.56100000000000005},"min":{"1min":0.42299999999999999,"5min":0.309,"15min":0.309},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.84699999999999998}]}]},{"osd":1,"up_from":13,"seq":55834574904,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297684,"kb_used_data":588,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447468,"statfs":{"total":95995035648,"available":95690207232,"internally_reserved":0,"allocated":602112,"data_stored":256139,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:11:51 2023","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.36099999999999999,"15min":0.33700000000000002},"min":{"1min":0.251,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.54000000000000004,"5min":0.59799999999999998,"15min":0.59799999999999998},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.371,"5min":0.375,"15min":0.34899999999999998},"min":{"1min":0.22600000000000001,"5min":0.215,"15min":0.215},"max":{"1min":0.56699999999999995,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.21099999999999999}]},{"osd":2,"last update":"Mon Aug 28 02:11:09 2023","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":0.375,"15min":0.36599999999999999},"min":{"1min":0.26200000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.51600000000000001,"5min":0.623,"15min":0.623},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.377,"5min":0.38700000000000001,"15min":0.38},"min":{"1min":0.27700000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.52400000000000002,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.23899999999999999}]},{"osd":3,"last update":"Mon Aug 28 02:11:31 2023","interfaces":[{"interface":"back","average":{"1min":0.38900000000000001,"5min":0.42599999999999999,"15min":0.434},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.48499999999999999,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.432,"5min":0.46899999999999997,"15min":0.47799999999999998},"min":{"1min":0.33600000000000002,"5min":0.317,"15min":0.317},"max":{"1min":0.66900000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.313}]},{"osd":4,"last update":"Mon Aug 28 02:11:51 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.55800000000000005,"15min":0.56000000000000005},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.57599999999999996,"15min":0.57199999999999995},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.97099999999999997,"5min":0.97099999999999997,"15min":0.97099999999999997},"last":0.45000000000000001}]},{"osd":5,"last update":"Mon Aug 28 02:11:09 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.497},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.55900000000000005,"15min":0.56100000000000005},"min":{"1min":0.42799999999999999,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.65900000000000003,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.44500000000000001}]},{"osd":6,"last update":"Mon Aug 28 02:11:35 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.58699999999999997,"15min":0.58899999999999997},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.997,"5min":0.997,"15min":0.997},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.58399999999999996,"15min":0.58099999999999996},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.95399999999999996,"5min":0.95399999999999996,"15min":0.95399999999999996},"last":0.48199999999999998}]},{"osd":7,"last update":"Mon Aug 28 02:11:54 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.61199999999999999,"15min":0.61499999999999999},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.90400000000000003,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.57899999999999996,"15min":0.57299999999999995},"min":{"1min":0.46600000000000003,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.55000000000000004}]}]},{"osd":0,"up_from":8,"seq":34359738429,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297692,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447460,"statfs":{"total":95995035648,"available":95690199040,"internally_reserved":0,"allocated":610304,"data_stored":256715,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Aug 28 02:11:53 2023","interfaces":[{"interface":"back","average":{"1min":0.36099999999999999,"5min":0.35299999999999998,"15min":0.34300000000000003},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.34499999999999997,"5min":0.36199999999999999,"15min":0.34699999999999998},"min":{"1min":0.215,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.62,"5min":0.62,"15min":0.62},"last":0.26300000000000001}]},{"osd":2,"last update":"Mon Aug 28 02:11:04 2023","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.38400000000000001,"15min":0.38200000000000001},"min":{"1min":0.27100000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.41899999999999998,"15min":0.41299999999999998},"min":{"1min":0.28299999999999997,"5min":0.248,"15min":0.248},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.36399999999999999}]},{"osd":3,"last update":"Mon Aug 28 02:11:27 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.48899999999999999,"15min":0.47299999999999998},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":5.4509999999999996,"5min":5.4509999999999996,"15min":5.4509999999999996},"last":0.317},{"interface":"front","average":{"1min":0.36199999999999999,"5min":0.46999999999999997,"15min":0.495},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.46300000000000002,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.44800000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:11:53 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.55100000000000005,"15min":0.54900000000000004},"min":{"1min":0.42199999999999999,"5min":0.37,"15min":0.37},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.58499999999999996,"15min":0.58799999999999997},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.68600000000000005}]},{"osd":5,"last update":"Mon Aug 28 02:11:13 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.76500000000000001,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.58599999999999997,"15min":0.58999999999999997},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.72499999999999998,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.65400000000000003}]},{"osd":6,"last update":"Mon Aug 28 02:11:33 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.57099999999999995,"15min":0.57699999999999996},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1.0469999999999999,"5min":1.0469999999999999,"15min":1.0469999999999999},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.57899999999999996,"15min":0.58099999999999996},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.67100000000000004}]},{"osd":7,"last update":"Mon Aug 28 02:11:58 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.56200000000000006,"15min":0.56599999999999995},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.57599999999999996,"15min":0.57799999999999996},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.95399999999999996,"5min":0.95399999999999996,"15min":0.95399999999999996},"last":0.63300000000000001}]}]},{"osd":2,"up_from":18,"seq":77309411379,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297692,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447460,"statfs":{"total":95995035648,"available":95690199040,"internally_reserved":0,"allocated":610304,"data_stored":256727,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:11:06 2023","interfaces":[{"interface":"back","average":{"1min":0.34899999999999998,"5min":0.35899999999999999,"15min":0.35899999999999999},"min":{"1min":0.23100000000000001,"5min":0.216,"15min":0.216},"max":{"1min":0.48999999999999999,"5min":0.54400000000000004,"15min":0.54400000000000004},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.34499999999999997,"5min":0.372,"15min":0.378},"min":{"1min":0.26300000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":0.48899999999999999,"5min":0.53500000000000003,"15min":0.53500000000000003},"last":0.33700000000000002}]},{"osd":1,"last update":"Mon Aug 28 02:11:06 2023","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.377,"15min":0.378},"min":{"1min":0.23599999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.54100000000000004,"5min":0.56999999999999995,"15min":0.56999999999999995},"last":0.28100000000000003},{"interface":"front","average":{"1min":0.38200000000000001,"5min":0.38900000000000001,"15min":0.38500000000000001},"min":{"1min":0.249,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.38400000000000001}]},{"osd":3,"last update":"Mon Aug 28 02:11:29 2023","interfaces":[{"interface":"back","average":{"1min":0.34699999999999998,"5min":0.42599999999999999,"15min":0.44800000000000001},"min":{"1min":0.23799999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.498,"5min":0.55800000000000005,"15min":0.55800000000000005},"last":0.316},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.45500000000000002,"15min":0.47999999999999998},"min":{"1min":0.28199999999999997,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.53700000000000003,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.35999999999999999}]},{"osd":4,"last update":"Mon Aug 28 02:11:51 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.55000000000000004,"15min":0.55100000000000005},"min":{"1min":0.38,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.56899999999999995,"15min":0.57299999999999995},"min":{"1min":0.41399999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.66700000000000004}]},{"osd":5,"last update":"Mon Aug 28 02:11:11 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.53100000000000003,"15min":0.53700000000000003},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.71499999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.53300000000000003,"15min":0.53200000000000003},"min":{"1min":0.38800000000000001,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.71399999999999997}]},{"osd":6,"last update":"Mon Aug 28 02:11:35 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52800000000000002,"15min":0.52900000000000003},"min":{"1min":0.36599999999999999,"5min":0.316,"15min":0.316},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.56100000000000005,"15min":0.56499999999999995},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.69099999999999995}]},{"osd":7,"last update":"Mon Aug 28 02:11:56 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.54400000000000004,"15min":0.54000000000000004},"min":{"1min":0.41399999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.56399999999999995,"15min":0.56200000000000006},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.76700000000000002,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.67600000000000005}]}]},{"osd":3,"up_from":25,"seq":107374182448,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297688,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447464,"statfs":{"total":95995035648,"available":95690203136,"internally_reserved":0,"allocated":606208,"data_stored":256229,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:11:28 2023","interfaces":[{"interface":"back","average":{"1min":0.33400000000000002,"5min":0.38100000000000001,"15min":0.39300000000000002},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.443,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.25700000000000001},{"interface":"front","average":{"1min":0.34599999999999997,"5min":0.40400000000000003,"15min":0.41899999999999998},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.54400000000000004,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.33300000000000002}]},{"osd":1,"last update":"Mon Aug 28 02:11:28 2023","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.42699999999999999,"15min":0.43099999999999999},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":1.0580000000000001,"5min":1.0580000000000001,"15min":1.0580000000000001},"last":0.29099999999999998},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.41899999999999998,"15min":0.42999999999999999},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.66800000000000004,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.30399999999999999}]},{"osd":2,"last update":"Mon Aug 28 02:11:28 2023","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.45300000000000001,"15min":0.46500000000000002},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.53300000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.37},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.436,"15min":0.44800000000000001},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.57099999999999995,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.34499999999999997}]},{"osd":4,"last update":"Mon Aug 28 02:11:49 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.55800000000000005,"15min":0.56999999999999995},"min":{"1min":0.378,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.68400000000000005,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.58399999999999996,"15min":0.59399999999999997},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.81299999999999994,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.67900000000000005}]},{"osd":5,"last update":"Mon Aug 28 02:11:13 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.57299999999999995,"15min":0.57399999999999995},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.59199999999999997,"15min":0.58799999999999997},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":1.8460000000000001,"5min":1.8460000000000001,"15min":1.8460000000000001},"last":0.69899999999999995}]},{"osd":6,"last update":"Mon Aug 28 02:11:34 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.59699999999999998,"15min":0.60699999999999998},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.73599999999999999,"5min":0.878,"15min":0.878},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.59499999999999997,"15min":0.60099999999999998},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.77100000000000002,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.68899999999999995}]},{"osd":7,"last update":"Mon Aug 28 02:11:55 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.61499999999999999,"15min":0.61899999999999999},"min":{"1min":0.40300000000000002,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.80100000000000005,"5min":1.4179999999999999,"15min":1.4179999999999999},"last":0.66900000000000004},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.623,"15min":0.61899999999999999},"min":{"1min":0.42999999999999999,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":1.2829999999999999,"5min":1.2829999999999999,"15min":1.2829999999999999},"last":0.64800000000000002}]}]},{"osd":4,"up_from":30,"seq":128849018922,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297708,"kb_used_data":612,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447444,"statfs":{"total":95995035648,"available":95690182656,"internally_reserved":0,"allocated":626688,"data_stored":257821,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:11:49 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.53400000000000003,"15min":0.54500000000000004},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.83999999999999997,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.377},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.55500000000000005,"15min":0.56200000000000006},"min":{"1min":0.38900000000000001,"5min":0.375,"15min":0.375},"max":{"1min":1.1339999999999999,"5min":1.1339999999999999,"15min":1.1339999999999999},"last":0.45700000000000002}]},{"osd":1,"last update":"Mon Aug 28 02:11:49 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.52400000000000002,"15min":0.53300000000000003},"min":{"1min":0.33000000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.81899999999999995,"5min":0.98899999999999999,"15min":0.98899999999999999},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.53400000000000003,"15min":0.54000000000000004},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.77000000000000002,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.39600000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:11:49 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.54600000000000004,"15min":0.56200000000000006},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.79300000000000004,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.54500000000000004,"15min":0.55300000000000005},"min":{"1min":0.33600000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.1020000000000001,"5min":1.1020000000000001,"15min":1.1020000000000001},"last":0.36299999999999999}]},{"osd":3,"last update":"Mon Aug 28 02:11:49 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.55400000000000005,"15min":0.55600000000000005},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.1599999999999999,"5min":1.1599999999999999,"15min":1.1599999999999999},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.36399999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.51900000000000002}]},{"osd":5,"last update":"Mon Aug 28 02:11:13 2023","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.39400000000000002,"15min":0.39000000000000001},"min":{"1min":0.32100000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.55800000000000005,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.495},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.40999999999999998,"15min":0.39900000000000002},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":1.1839999999999999,"5min":1.1839999999999999,"15min":1.1839999999999999},"last":0.55900000000000005}]},{"osd":6,"last update":"Mon Aug 28 02:11:34 2023","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.442,"15min":0.436},"min":{"1min":0.34999999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.60899999999999999,"5min":0.63,"15min":0.63},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.46800000000000003,"15min":0.46200000000000002},"min":{"1min":0.39400000000000002,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.68899999999999995,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.57799999999999996}]},{"osd":7,"last update":"Mon Aug 28 02:11:59 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.70199999999999996,"5min":1.2110000000000001,"15min":1.2110000000000001},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.51700000000000002,"15min":0.51100000000000001},"min":{"1min":0.36899999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.59499999999999997}]}]},{"osd":5,"up_from":36,"seq":154618822694,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297684,"kb_used_data":588,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447468,"statfs":{"total":95995035648,"available":95690207232,"internally_reserved":0,"allocated":602112,"data_stored":256133,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:11:12 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.55000000000000004,"15min":0.55500000000000005},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.52700000000000002,"15min":0.52500000000000002},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.57799999999999996}]},{"osd":1,"last update":"Mon Aug 28 02:11:12 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.56899999999999995,"15min":0.57099999999999995},"min":{"1min":0.35599999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.97499999999999998,"5min":0.97499999999999998,"15min":0.97499999999999998},"last":0.629},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.53000000000000003,"15min":0.53300000000000003},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.52500000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:11:12 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.54300000000000004,"15min":0.54400000000000004},"min":{"1min":0.316,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.56699999999999995,"15min":0.57399999999999995},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.54700000000000004}]},{"osd":3,"last update":"Mon Aug 28 02:11:12 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.53900000000000003,"15min":0.53700000000000003},"min":{"1min":0.38,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.57099999999999995,"15min":0.57399999999999995},"min":{"1min":0.39200000000000002,"5min":0.38,"15min":0.38},"max":{"1min":0.94799999999999995,"5min":0.94799999999999995,"15min":0.94799999999999995},"last":0.50900000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:11:12 2023","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.43099999999999999,"15min":0.42799999999999999},"min":{"1min":0.29399999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.44800000000000001,"15min":0.441},"min":{"1min":0.30499999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.373}]},{"osd":6,"last update":"Mon Aug 28 02:11:31 2023","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.48899999999999999,"15min":0.49199999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.71299999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.49099999999999999,"15min":0.48799999999999999},"min":{"1min":0.35699999999999998,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.63600000000000001,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.44700000000000001}]},{"osd":7,"last update":"Mon Aug 28 02:11:54 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.54200000000000004,"15min":0.54400000000000004},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.67900000000000005,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.55500000000000005,"15min":0.55700000000000005},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.76000000000000001,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.41399999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-08-28T02:12:04.618 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-08-28T02:12:04.618 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-08-28T02:12:04.618 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-08-28T02:12:04.618 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph health --format=json 2023-08-28T02:12:04.962 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:12:05.426 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:05 smithi143 conmon[100258]: cluster 2023-08-28T02:12:03.883306+0000 mgr.y (mgr.14424) 91 : cluster [DBG] pgmap v69: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:12:05.426 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:05 smithi143 conmon[100258]: audit 2023-08-28T02:12:04.025698+0000 mgr.y (mgr.14424) 92 : audit [DBG] from='client.24677 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:12:05.426 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:05 smithi143 conmon[109923]: cluster 2023-08-28T02:12:03.883306+0000 mgr.y (mgr.14424) 91 : cluster [DBG] pgmap v69: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:12:05.426 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:05 smithi143 conmon[109923]: audit 2023-08-28T02:12:04.025698+0000 mgr.y (mgr.14424) 92 : audit [DBG] from='client.24677 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:12:05.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:05 smithi169 conmon[100500]: cluster 2023-08-28T02:12:03.883306+0000 mgr.y (mgr.14424) 91 : cluster [DBG] pgmap v69: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:12:05.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:05 smithi169 conmon[100500]: audit 2023-08-28T02:12:04.025698 2023-08-28T02:12:05.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:05 smithi169 conmon[100500]: +0000 mgr.y (mgr.14424) 92 : audit [DBG] from='client.24677 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:12:06.404 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:12:07.007 INFO:teuthology.orchestra.run.smithi143.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-08-28T02:12:07.007 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-08-28T02:12:07.008 INFO:teuthology.run_tasks:Running task print... 2023-08-28T02:12:07.017 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2023-08-28T02:12:07.017 INFO:teuthology.run_tasks:Running task parallel... 2023-08-28T02:12:07.035 INFO:teuthology.task.parallel:starting parallel... 2023-08-28T02:12:07.035 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-08-28T02:12:07.035 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-08-28T02:12:07.039 INFO:tasks.workunit:Pulling workunits from ref pacific 2023-08-28T02:12:07.039 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-08-28T02:12:07.039 DEBUG:teuthology.orchestra.run.smithi143:> stat -- /home/ubuntu/cephtest/mnt.0 2023-08-28T02:12:07.040 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-08-28T02:12:07.040 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2023-08-28T02:12:07.040 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi143.front.sepia.ceph.com 2023-08-28T02:12:07.040 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -e sha1=9cf208207dfe23ffccb33308b22e51570cb45e51 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2023-08-28T02:12:07.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-08-28T02:12:07.058 INFO:teuthology.orchestra.run.smithi143.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-08-28T02:12:07.058 DEBUG:teuthology.orchestra.run.smithi143:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-08-28T02:12:07.115 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-08-28T02:12:07.115 DEBUG:teuthology.orchestra.run.smithi143:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-08-28T02:12:07.173 INFO:tasks.workunit:timeout=3h 2023-08-28T02:12:07.173 INFO:tasks.workunit:cleanup=True 2023-08-28T02:12:07.174 DEBUG:teuthology.orchestra.run.smithi143:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch pacific https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-08-28T02:12:07.231 INFO:tasks.workunit.client.0.smithi143.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-08-28T02:12:07.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:07 smithi143 conmon[109923]: cluster 2023-08-28T02:12:05.883736+0000 mgr.y (mgr.14424) 93 : cluster [DBG] pgmap v70: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:12:07.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:07 smithi143 conmon[109923]: audit 2023-08-28T02:12:06.405964+0000 mon.a (mon.0) 722 : audit [DBG] from='client.? 172.21.15.143:0/849317145' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-08-28T02:12:07.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:07 smithi143 conmon[100258]: cluster 2023-08-28T02:12:05.883736+0000 mgr.y (mgr.14424) 93 : cluster [DBG] pgmap v70: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:12:07.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:07 smithi143 conmon[100258]: audit 2023-08-28T02:12:06.405964+0000 mon.a (mon.0) 722 : audit [DBG] from='client.? 172.21.15.143:0/849317145' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-08-28T02:12:07.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:07 smithi169 conmon[100500]: cluster 2023-08-28T02:12:05.883736+0000 mgr.y (mgr.14424) 93 : cluster [DBG] pgmap v70: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:12:07.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:07 smithi169 conmon[100500]: audit 2023-08-28T02:12: 2023-08-28T02:12:07.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:07 smithi169 conmon[100500]: 06.405964+0000 mon.a (mon.0) 722 : audit [DBG] from='client.? 172.21.15.143:0/849317145' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-08-28T02:12:09.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:12:08 smithi143 conmon[100708]: [28/Aug/2023:02:12:08] ENGINE Bus STOPPING 2023-08-28T02:12:09.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:12:08 smithi143 conmon[100708]: [28/Aug/2023:02:12:08] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-08-28T02:12:09.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:12:08 smithi143 conmon[100708]: [28/Aug/2023:02:12:08] ENGINE Bus STOPPED 2023-08-28T02:12:09.201 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:12:08 smithi143 conmon[100708]: [28/Aug/2023:02:12:08] ENGINE Bus STARTING 2023-08-28T02:12:09.201 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:12:08 smithi143 conmon[100708]: [28/Aug/2023:02:12:08] ENGINE Serving on http://:::9283 2023-08-28T02:12:09.201 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:12:08 smithi143 conmon[100708]: [28/Aug/2023:02:12:08] ENGINE Bus STARTED 2023-08-28T02:12:09.426 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -e sha1=9cf208207dfe23ffccb33308b22e51570cb45e51 -- bash -c 'ceph config set global log_to_journald false --force' 2023-08-28T02:12:09.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:09 smithi169 conmon[100500]: cluster 2023-08-28T02:12:07.884196+0000 mgr.y (mgr.14424) 94 : cluster 2023-08-28T02:12:09.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:09 smithi169 conmon[100500]: [DBG] pgmap v71: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:12:09.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:09 smithi169 conmon[100500]: audit 2023-08-28T02:12:08.809810+0000 mon.a (mon.0) 723 : audit [INF] from='client.? 172.21.15.143:0/1754896581' entity='client.admin' 2023-08-28T02:12:09.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:09 smithi169 conmon[100500]: audit 2023-08-28T02:12:08. 2023-08-28T02:12:09.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:09 smithi169 conmon[100500]: 843609+0000 mon.a (mon.0) 724 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:12:09.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:09 smithi169 conmon[100500]: audit 2023-08- 2023-08-28T02:12:09.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:09 smithi169 conmon[100500]: 28T02:12:08.844870+0000 mon.a 2023-08-28T02:12:09.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:09 smithi169 conmon[100500]: (mon.0) 725 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:12:09.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:09 smithi169 conmon[100500]: 2023-08-28T02:12:09.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:09 smithi169 conmon[100500]: audit 2023-08- 2023-08-28T02:12:09.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:09 smithi169 conmon[100500]: 28T02:12:08.845379+0000 2023-08-28T02:12:09.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:09 smithi169 conmon[100500]: mon.a (mon.0) 726 : audit 2023-08-28T02:12:09.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:09 smithi169 conmon[100500]: [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:12:09.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:09 smithi169 conmon[100500]: audit 2023-08-28T02: 2023-08-28T02:12:09.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:09 smithi169 conmon[100500]: 12:08.852709+0000 mon.a (mon.0) 727 2023-08-28T02:12:09.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:09 smithi169 conmon[100500]: : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:12:09.492 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:09 smithi143 conmon[100258]: cluster 2023-08-28T02:12:07.884196+0000 2023-08-28T02:12:09.492 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:09 smithi143 conmon[100258]: mgr.y (mgr.14424) 94 : cluster [DBG] pgmap v71: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:12:09.492 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:09 smithi143 conmon[100258]: audit 2023-08-28T02:12:08.809810+0000 mon.a (mon.0) 723 : audit [INF] 2023-08-28T02:12:09.492 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:09 smithi143 conmon[100258]: from='client.? 172.21.15.143:0/1754896581' entity='client.admin' 2023-08-28T02:12:09.493 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:09 smithi143 conmon[100258]: audit 2023-08-28T02:12:08.843609+0000 mon.a ( 2023-08-28T02:12:09.493 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:09 smithi143 conmon[100258]: mon.0) 724 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:12:09.493 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:09 smithi143 conmon[100258]: audit 2023 2023-08-28T02:12:09.493 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:09 smithi143 conmon[100258]: -08-28T02:12:08.844870+0000 mon.a (mon.0) 725 : audit [DBG] 2023-08-28T02:12:09.493 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:09 smithi143 conmon[100258]: from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:12:09.493 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:09 smithi143 conmon[100258]: audit 2023-08 2023-08-28T02:12:09.493 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:09 smithi143 conmon[100258]: -28T02:12:08.845379+0000 mon.a (mon.0) 726 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:12:09.494 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:09 smithi143 conmon[100258]: audit 2023-08-28T02:12:08 2023-08-28T02:12:09.494 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:09 smithi143 conmon[100258]: .852709+0000 mon.a (mon.0) 727 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:12:09.494 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:09 smithi143 conmon[109923]: cluster 2023-08-28T02:12:07.884196+0000 2023-08-28T02:12:09.494 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:09 smithi143 conmon[109923]: mgr.y (mgr.14424) 94 : cluster [DBG] pgmap v71: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:12:09.495 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:09 smithi143 conmon[109923]: audit 2023-08-28T02:12:08.809810+0000 mon.a 2023-08-28T02:12:09.495 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:09 smithi143 conmon[109923]: (mon.0) 723 : audit [INF] from='client.? 172.21.15.143:0/1754896581' entity='client.admin' 2023-08-28T02:12:09.495 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:09 smithi143 conmon[109923]: audit 2023-08-28T02:12:09.495 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:09 smithi143 conmon[109923]: 2023-08-28T02:12:08.843609+0000 mon.a (mon 2023-08-28T02:12:09.495 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:09 smithi143 conmon[109923]: .0) 724 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:12:09.495 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:09 smithi143 conmon[109923]: audit 2023-08-28T02: 2023-08-28T02:12:09.495 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:09 smithi143 conmon[109923]: 12:08.844870+0000 mon.a (mon.0) 725 2023-08-28T02:12:09.495 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:09 smithi143 conmon[109923]: : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:12:09.496 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:09 smithi143 conmon[109923]: audit 2023-08 2023-08-28T02:12:09.496 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:09 smithi143 conmon[109923]: -28T02:12:08.845379+0000 2023-08-28T02:12:09.496 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:09 smithi143 conmon[109923]: mon.a (mon.0) 726 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:12:09.496 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:09 smithi143 conmon[109923]: 2023-08-28T02:12:09.496 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:09 smithi143 conmon[109923]: audit 2023-08-28T02:12:09.496 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:09 smithi143 conmon[109923]: 2023-08-28T02:12:08.852709+0000 2023-08-28T02:12:09.496 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:09 smithi143 conmon[109923]: mon.a (mon.0) 727 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:12:09.496 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:09 smithi143 conmon[109923]: 2023-08-28T02:12:10.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:10 smithi169 conmon[100500]: audit 2023-08-28T02:12:09.867988+0000 mon.a (mon.0) 728 : audit [INF] 2023-08-28T02:12:10.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:10 smithi169 conmon[100500]: from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-08-28T02:12:10.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:10 smithi169 conmon[100500]: audit 2023-08-28T02:12:09.868224+0000 mon.a (mon.0) 729 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2023-08-28T02:12:10.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:10 smithi169 conmon[100500]: audit 2023-08-28T02:12:10.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:10 smithi169 conmon[100500]: 2023-08-28T02:12:09.868392+0000 mon.a (mon.0) 730 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2023-08-28T02:12:10.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:10 smithi143 conmon[100258]: audit 2023-08-28T02:12:09.867988+0000 mon.a (mon.0) 728 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-08-28T02:12:11.710 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:10 smithi143 conmon[100258]: audit 2023-08-28T02:12:09.868224+0000 mon.a (mon.0) 729 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2023-08-28T02:12:11.710 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:10 smithi143 conmon[100258]: audit 2023-08-28T02:12:09.868392+0000 mon.a (mon.0) 730 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2023-08-28T02:12:11.710 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:10 smithi143 conmon[109923]: audit 2023-08-28T02:12:09.867988+0000 mon.a (mon.0) 728 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-08-28T02:12:11.710 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:10 smithi143 conmon[109923]: audit 2023-08-28T02:12:09.868224+0000 mon.a (mon.0) 729 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2023-08-28T02:12:11.710 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:10 smithi143 conmon[109923]: audit 2023-08-28T02:12:09.868392+0000 mon.a (mon.0) 730 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2023-08-28T02:12:11.711 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:12:10 smithi143 conmon[100708]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:12:10] "GET /metrics HTTP/1.1" 200 191492 "" "Prometheus/2.33.4" 2023-08-28T02:12:11.742 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:11 smithi143 conmon[100258]: cluster 2023-08-28T02: 2023-08-28T02:12:11.742 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:11 smithi143 conmon[100258]: 12:09.884469+0000 mgr.y (mgr.14424) 95 : cluster [DBG] pgmap v72: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:12:11.742 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:11 smithi143 conmon[100258]: audit 2023-08-28T02:12:10.235040+0000 mon.a (mon.0) 731 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2023-08-28T02:12:11.742 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:11 smithi143 conmon[100258]: audit 2023-08-28T02:12:10.235144+0000 mon.a (mon.0) 732 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2023-08-28T02:12:11.742 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:11 smithi143 conmon[100258]: audit 2023-08-28T02:12:10.235197+0000 mon.a (mon.0) 733 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2023-08-28T02:12:11.742 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:11 smithi143 conmon[100258]: cluster 2023-08-28T02:12:10.235244+0000 mon.a (mon.0) 734 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-08-28T02:12:11.746 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:11 smithi143 conmon[109923]: cluster 2023-08-28T02:12:09.884469 2023-08-28T02:12:11.747 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:11 smithi143 conmon[109923]: +0000 mgr.y (mgr.14424) 95 : cluster [DBG] pgmap v72: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:12:11.751 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:11 smithi143 conmon[109923]: audit 2023-08-28T02:12:11.751 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:11 smithi143 conmon[109923]: 2023-08-28T02:12:10.235040+0000 mon.a (mon.0) 2023-08-28T02:12:11.751 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:11 smithi143 conmon[109923]: 731 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2023-08-28T02:12:11.751 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:11 smithi143 conmon[109923]: audit 2023-08- 2023-08-28T02:12:11.754 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:11 smithi143 conmon[109923]: 28T02:12:10.235144+0000 mon.a (mon 2023-08-28T02:12:11.755 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:11 smithi143 conmon[109923]: .0) 732 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2023-08-28T02:12:11.755 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:11 smithi143 conmon[109923]: audit 2023 2023-08-28T02:12:11.755 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:11 smithi143 conmon[109923]: -08-28T02:12:10.235197+0000 mon.a ( 2023-08-28T02:12:11.755 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:11 smithi143 conmon[109923]: mon.0) 733 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2023-08-28T02:12:11.755 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:11 smithi143 conmon[109923]: cluster 2023 2023-08-28T02:12:11.755 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:11 smithi143 conmon[109923]: -08-28T02:12:10.235244+0000 mon.a 2023-08-28T02:12:11.755 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:11 smithi143 conmon[109923]: (mon.0) 734 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-08-28T02:12:11.755 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:11 smithi143 conmon[109923]: 2023-08-28T02:12:11.756 DEBUG:teuthology.orchestra.run.smithi143:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-08-28T02:12:11.757 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:11 smithi169 conmon[100500]: cluster 2023-08-28T02:12:09.884469+0000 mgr.y (mgr.14424) 95 : cluster [DBG] pgmap v72: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:12:11.757 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:11 smithi169 conmon[100500]: 2023-08-28T02:12:11.757 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:11 smithi169 conmon[100500]: audit 2023-08-28T02:12:10.235040+0000 mon.a (mon.0) 731 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2023-08-28T02:12:11.757 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:11 smithi169 conmon[100500]: audit 2023-08-28T02:12:10.235144+0000 mon.a (mon.0) 732 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2023-08-28T02:12:11.757 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:11 smithi169 conmon[100500]: audit 2023-08-28T02:12:11.757 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:11 smithi169 conmon[100500]: 2023-08-28T02:12:10.235197+0000 mon.a (mon.0) 733 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2023-08-28T02:12:11.758 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:11 smithi169 conmon[100500]: cluster 2023-08-28T02:12:10.235244+0000 2023-08-28T02:12:11.758 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:11 smithi169 conmon[100500]: mon.a (mon.0) 734 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-08-28T02:12:11.833 INFO:tasks.workunit.client.0.smithi143.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-08-28T02:12:11.835 INFO:tasks.workunit.client.0.smithi143.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-08-28T02:12:11.835 INFO:tasks.workunit.client.0.smithi143.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-08-28T02:12:11.950 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:12:11 smithi143 conmon[100708]: [28/Aug/2023:02:12:11] ENGINE Bus STOPPING 2023-08-28T02:12:11.950 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:12:11 smithi143 conmon[100708]: [28/Aug/2023:02:12:11] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-08-28T02:12:11.950 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:12:11 smithi143 conmon[100708]: [28/Aug/2023:02:12:11] ENGINE Bus STOPPED 2023-08-28T02:12:11.950 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:12:11 smithi143 conmon[100708]: [28/Aug/2023:02:12:11] ENGINE Bus STARTING 2023-08-28T02:12:11.950 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:12:11 smithi143 conmon[100708]: [28/Aug/2023:02:12:11] ENGINE Serving on http://:::9283 2023-08-28T02:12:11.950 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:12:11 smithi143 conmon[100708]: [28/Aug/2023:02:12:11] ENGINE Bus STARTED 2023-08-28T02:12:12.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:12 smithi143 conmon[109923]: cluster 2023-08-28T02:12:11.240774+0000 mon.a 2023-08-28T02:12:12.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:12 smithi143 conmon[109923]: (mon.0) 735 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-08-28T02:12:12.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:12 smithi143 conmon[109923]: audit 2023-08-28 2023-08-28T02:12:12.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:12 smithi143 conmon[109923]: T02:12:11.659325+0000 mon.a (mon.0) 736 : audit [INF] from='client.? ' entity='client.admin' 2023-08-28T02:12:12.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:12 smithi143 conmon[109923]: audit 2023-08- 2023-08-28T02:12:12.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:12 smithi143 conmon[109923]: 28T02:12:11.676571+0000 mon.a (mon.0) 737 : audit 2023-08-28T02:12:12.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:12 smithi143 conmon[109923]: [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:12:12.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:12 smithi143 conmon[109923]: audit 2023-08-28T02:12:12.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:12 smithi143 conmon[109923]: 2023-08-28T02:12:11.677862+0000 mon.a (mon.0 2023-08-28T02:12:12.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:12 smithi143 conmon[109923]: ) 738 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:12:12.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:12 smithi143 conmon[109923]: audit 2023-08-28T02:12:11. 2023-08-28T02:12:12.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:12 smithi143 conmon[109923]: 678422+0000 mon.a (mon.0) 739 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:12:12.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:12 smithi143 conmon[109923]: audit 2023-08-28T02:12:12.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:12 smithi143 conmon[109923]: 2023-08-28T02:12:11.684785+0000 mon.a (mon.0 2023-08-28T02:12:12.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:12 smithi143 conmon[109923]: ) 740 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:12:12.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:12 smithi143 conmon[100258]: cluster 2023-08-28T02:12:11.240774+0000 mon.a 2023-08-28T02:12:12.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:12 smithi143 conmon[100258]: (mon.0) 735 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-08-28T02:12:12.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:12 smithi143 conmon[100258]: audit 2023-08-28T02:12:11.659325+0000 mon.a (mon.0) 736 : audit [INF] from='client.? ' entity='client.admin' 2023-08-28T02:12:12.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:12 smithi143 conmon[100258]: 2023-08-28T02:12:12.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:12 smithi143 conmon[100258]: audit 2023-08-28T 2023-08-28T02:12:12.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:12 smithi143 conmon[100258]: 02:12:11.676571+0000 mon.a (mon.0) 737 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:12:12.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:12 smithi143 conmon[100258]: audit 2023-08-28T02:12:12.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:12 smithi143 conmon[100258]: 2023-08-28T02 2023-08-28T02:12:12.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:12 smithi143 conmon[100258]: :12:11.677862+0000 mon.a (mon.0) 2023-08-28T02:12:12.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:12 smithi143 conmon[100258]: 738 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:12:12.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:12 smithi143 conmon[100258]: audit 2023-08- 2023-08-28T02:12:12.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:12 smithi143 conmon[100258]: 28T02:12:11.678422+0000 mon.a (mon 2023-08-28T02:12:12.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:12 smithi143 conmon[100258]: .0) 739 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:12:12.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:12 smithi143 conmon[100258]: audit 2023 2023-08-28T02:12:12.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:12 smithi143 conmon[100258]: -08-28T02:12:11.684785+0000 mon.a ( 2023-08-28T02:12:12.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:12 smithi143 conmon[100258]: mon.0) 740 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:12:12.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:12 smithi169 conmon[100500]: cluster 2023-08-28T02:12:11.240774+0000 mon.a (mon.0) 735 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-08-28T02:12:12.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:12 smithi169 conmon[100500]: audit 2023-08-28T02:12:11.659325+0000 mon.a (mon.0) 736 : audit [INF] from='client.? ' entity='client.admin' 2023-08-28T02:12:12.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:12 smithi169 conmon[100500]: audit 2023-08-28T02:12:11.676571+0000 mon.a (mon.0) 737 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:12:12.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:12 smithi169 conmon[100500]: audit 2023-08- 2023-08-28T02:12:12.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:12 smithi169 conmon[100500]: 28T02:12:11.677862+0000 mon.a (mon.0) 738 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:12:12.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:12 smithi169 conmon[100500]: audit 2023-08-28T02:12:11. 2023-08-28T02:12:12.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:12 smithi169 conmon[100500]: 678422+0000 mon.a (mon.0) 739 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:12:12.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:12 smithi169 conmon[100500]: audit 2023 2023-08-28T02:12:12.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:12 smithi169 conmon[100500]: -08-28T02:12:11.684785+0000 mon.a (mon.0) 740 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:12:13.237 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:12:12 smithi169 conmon[102807]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:12:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-28T02:12:13.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:13 smithi143 conmon[100258]: cluster 2023-08-28T02:12:11.884780+0000 mgr.y (mgr. 2023-08-28T02:12:13.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:13 smithi143 conmon[100258]: 14424) 96 : cluster [DBG] pgmap v75: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:12:13.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:13 smithi143 conmon[109923]: cluster 2023-08-28T02:12:11.884780+0000 mgr.y (mgr 2023-08-28T02:12:13.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:13 smithi143 conmon[109923]: .14424) 96 : cluster [DBG] pgmap v75: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:12:13.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:13 smithi169 conmon[100500]: cluster 2023-08-28T02:12:11.884780+0000 mgr.y (mgr.14424) 96 : cluster [DBG] 2023-08-28T02:12:13.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:13 smithi169 conmon[100500]: pgmap v75: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:12:14.240 INFO:tasks.workunit.client.0.smithi143.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-08-28T02:12:15.377 INFO:tasks.workunit.client.0.smithi143.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-08-28T02:12:15.417 INFO:tasks.workunit.client.0.smithi143.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-08-28T02:12:15.419 INFO:tasks.workunit.client.0.smithi143.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-08-28T02:12:15.419 INFO:tasks.workunit.client.0.smithi143.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-08-28T02:12:15.456 INFO:tasks.workunit.client.0.smithi143.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-08-28T02:12:15.459 DEBUG:teuthology.orchestra.run.smithi143:> set -ex 2023-08-28T02:12:15.459 DEBUG:teuthology.orchestra.run.smithi143:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-08-28T02:12:15.505 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:15 smithi143 conmon[100258]: cluster 2023-08-28T02:12:13.885144+0000 mgr.y (mgr.14424) 97 : cluster [DBG] 2023-08-28T02:12:15.505 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:15 smithi143 conmon[100258]: pgmap v76: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:12:15.505 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:15 smithi143 conmon[109923]: cluster 2023-08-28T02:12:13.885144+0000 mgr.y (mgr.14424) 97 : cluster 2023-08-28T02:12:15.506 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:15 smithi143 conmon[109923]: [DBG] pgmap v76: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:12:15.518 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.0... 2023-08-28T02:12:15.519 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2023-08-28T02:12:15.519 DEBUG:teuthology.orchestra.run.smithi143:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=pacific TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/import_export.sh 2023-08-28T02:12:15.527 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -e sha1=9cf208207dfe23ffccb33308b22e51570cb45e51 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2023-08-28T02:12:15.588 INFO:tasks.workunit.client.0.smithi143.stderr:+ export RBD_FORCE_ALLOW_V1=1 2023-08-28T02:12:15.588 INFO:tasks.workunit.client.0.smithi143.stderr:+ RBD_FORCE_ALLOW_V1=1 2023-08-28T02:12:15.588 INFO:tasks.workunit.client.0.smithi143.stderr:+ TMPDIR=/tmp/rbd_import_export_164148 2023-08-28T02:12:15.588 INFO:tasks.workunit.client.0.smithi143.stderr:+ rm -rf /tmp/rbd_import_export_164148 2023-08-28T02:12:15.589 INFO:tasks.workunit.client.0.smithi143.stderr:+ mkdir /tmp/rbd_import_export_164148 2023-08-28T02:12:15.589 INFO:tasks.workunit.client.0.smithi143.stderr:+ trap 'rm -rf /tmp/rbd_import_export_164148' INT TERM EXIT 2023-08-28T02:12:15.589 INFO:tasks.workunit.client.0.smithi143.stderr:+ mkdir foo.164148 2023-08-28T02:12:15.590 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd import foo.164148 foo.dir 2023-08-28T02:12:15.627 INFO:tasks.workunit.client.0.smithi143.stderr:rbd: cannot import a directory 2023-08-28T02:12:15.627 INFO:tasks.workunit.client.0.smithi143.stderr: Importing image: 0% complete...failed. 2023-08-28T02:12:15.628 INFO:tasks.workunit.client.0.smithi143.stderr:rbd: import failed: (21) Is a directory 2023-08-28T02:12:15.630 INFO:tasks.workunit.client.0.smithi143.stderr:+ true 2023-08-28T02:12:15.630 INFO:tasks.workunit.client.0.smithi143.stderr:+ rmdir foo.164148 2023-08-28T02:12:15.639 INFO:tasks.workunit.client.0.smithi143.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_164148/img bs=1k count=1 seek=10 2023-08-28T02:12:15.640 INFO:tasks.workunit.client.0.smithi143.stderr:1+0 records in 2023-08-28T02:12:15.640 INFO:tasks.workunit.client.0.smithi143.stderr:1+0 records out 2023-08-28T02:12:15.640 INFO:tasks.workunit.client.0.smithi143.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 4.5559e-05 s, 22.5 MB/s 2023-08-28T02:12:15.640 INFO:tasks.workunit.client.0.smithi143.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_164148/img bs=1k count=10 seek=100 2023-08-28T02:12:15.640 INFO:tasks.workunit.client.0.smithi143.stderr:10+0 records in 2023-08-28T02:12:15.640 INFO:tasks.workunit.client.0.smithi143.stderr:10+0 records out 2023-08-28T02:12:15.641 INFO:tasks.workunit.client.0.smithi143.stderr:10240 bytes (10 kB, 10 KiB) copied, 5.299e-05 s, 193 MB/s 2023-08-28T02:12:15.641 INFO:tasks.workunit.client.0.smithi143.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_164148/img bs=1k count=100 seek=1000 2023-08-28T02:12:15.641 INFO:tasks.workunit.client.0.smithi143.stderr:70+1 records in 2023-08-28T02:12:15.641 INFO:tasks.workunit.client.0.smithi143.stderr:70+1 records out 2023-08-28T02:12:15.641 INFO:tasks.workunit.client.0.smithi143.stderr:72048 bytes (72 kB, 70 KiB) copied, 0.000180193 s, 400 MB/s 2023-08-28T02:12:15.642 INFO:tasks.workunit.client.0.smithi143.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_164148/img bs=1k seek=10000 2023-08-28T02:12:15.643 INFO:tasks.workunit.client.0.smithi143.stderr:139+1 records in 2023-08-28T02:12:15.643 INFO:tasks.workunit.client.0.smithi143.stderr:139+1 records out 2023-08-28T02:12:15.643 INFO:tasks.workunit.client.0.smithi143.stderr:143224 bytes (143 kB, 140 KiB) copied, 0.000315116 s, 455 MB/s 2023-08-28T02:12:15.643 INFO:tasks.workunit.client.0.smithi143.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_164148/img bs=1k seek=100000 2023-08-28T02:12:15.647 INFO:tasks.workunit.client.0.smithi143.stderr:70+1 records in 2023-08-28T02:12:15.647 INFO:tasks.workunit.client.0.smithi143.stderr:70+1 records out 2023-08-28T02:12:15.647 INFO:tasks.workunit.client.0.smithi143.stderr:72168 bytes (72 kB, 70 KiB) copied, 0.00402672 s, 17.9 MB/s 2023-08-28T02:12:15.647 INFO:tasks.workunit.client.0.smithi143.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_164148/img bs=1k seek=1000000 2023-08-28T02:12:15.648 INFO:tasks.workunit.client.0.smithi143.stderr:189+1 records in 2023-08-28T02:12:15.649 INFO:tasks.workunit.client.0.smithi143.stderr:189+1 records out 2023-08-28T02:12:15.649 INFO:tasks.workunit.client.0.smithi143.stderr:193768 bytes (194 kB, 189 KiB) copied, 0.000404241 s, 479 MB/s 2023-08-28T02:12:15.649 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd rm testimg 2023-08-28T02:12:15.684 INFO:tasks.workunit.client.0.smithi143.stderr: Removing image: 0% complete...failed. 2023-08-28T02:12:15.684 INFO:tasks.workunit.client.0.smithi143.stderr:rbd: delete error: (2) No such file or directory 2023-08-28T02:12:15.687 INFO:tasks.workunit.client.0.smithi143.stderr:+ true 2023-08-28T02:12:15.687 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd import --new-format /tmp/rbd_import_export_164148/img testimg 2023-08-28T02:12:15.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:15 smithi169 conmon[100500]: cluster 2023-08-28T02:12:13.885144+0000 mgr.y (mgr.14424) 97 : cluster [DBG] pgmap v76: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:12:17.536 INFO:teuthology.orchestra.run.smithi143.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:9cf208207dfe23ffccb33308b22e51570cb45e51 2023-08-28T02:12:17.687 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:17 smithi143 conmon[100258]: cluster 2023-08-28T02:12:15.885594+0000 mgr.y (mgr.14424) 98 : cluster [DBG] pgmap v77: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:12:17.887 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:17 smithi143 conmon[100258]: cluster 2023-08-28T02:12:16.259548+0000 mon.a (mon.0) 741 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-08-28T02:12:17.888 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:17 smithi143 conmon[109923]: cluster 2023-08-28T02:12: 2023-08-28T02:12:17.888 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:17 smithi143 conmon[109923]: 15.885594+0000 mgr.y (mgr.14424) 98 : cluster [DBG] pgmap v77: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:12:17.888 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:17 smithi143 conmon[109923]: cluster 2023-08-28T02:12:16.259548+0000 mon.a (mon.0) 741 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-08-28T02:12:17.891 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:17 smithi169 conmon[100500]: cluster 2023-08-28T02:12:15.885594+0000 mgr.y (mgr.14424) 98 : cluster [DBG] pgmap v77: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:12:17.892 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:17 smithi169 conmon[100500]: 2023-08-28T02:12:17.897 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:17 smithi169 conmon[100500]: cluster 2023-08-28T02:12:16.259548+0000 mon.a (mon.0) 741 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-08-28T02:12:17.900 INFO:tasks.workunit.client.0.smithi143.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2023-08-28T02:12:18.091 INFO:tasks.workunit.client.0.smithi143.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-08-28T02:12:18.091 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd export testimg /tmp/rbd_import_export_164148/img2 2023-08-28T02:12:18.098 INFO:tasks.workunit.client.0.smithi143.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-08-28T02:12:18.664 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd export testimg - 2023-08-28T02:12:18.668 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -e sha1=9cf208207dfe23ffccb33308b22e51570cb45e51 -- bash -c 'ceph orch ps' 2023-08-28T02:12:18.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[100258]: cluster 2023-08-28T02:12:17.262517+0000 mon.a (mon.0 2023-08-28T02:12:18.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[100258]: ) 742 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-08-28T02:12:18.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[100258]: audit 2023-08-28T02:12:17.532671+0000 mgr.y (mgr.14424) 99 : audit [DBG] from='client.14808 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:9cf208207dfe23ffccb33308b22e51570cb45e51", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:12:18.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[100258]: cephadm 2023-08-28T02:12:17.533426+0000 mgr.y (mgr.14424) 100 : cephadm 2023-08-28T02:12:18.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[100258]: [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:9cf208207dfe23ffccb33308b22e51570cb45e51 2023-08-28T02:12:18.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[100258]: audit 2023-08-28T02:12:17.537463+0000 mon.a (mon.0) 743 : audit [INF] 2023-08-28T02:12:18.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[100258]: from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:12:18.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[100258]: audit 2023-08-28T02:12:17 2023-08-28T02:12:18.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[100258]: .582511+0000 mon.a (mon.0) 744 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:12:18.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[100258]: audit 2023- 2023-08-28T02:12:18.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[100258]: 08-28T02:12:17.584247+0000 mon.a (mon.0) 745 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:12:18.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[100258]: audit 2023-08-28T02:12:18.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[100258]: 2023-08-28T02 2023-08-28T02:12:18.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[100258]: :12:17.584934+0000 mon.a (mon.0) 2023-08-28T02:12:18.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[100258]: 746 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:12:18.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[100258]: audit 2023-08-28T02:12:17 2023-08-28T02:12:18.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[100258]: .589903+0000 mon.a (mon.0) 747 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:12:18.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[100258]: cephadm 2023-08-28T02:12:17. 2023-08-28T02:12:18.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[100258]: 593935+0000 mgr.y (mgr.14424) 101 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:9cf208207dfe23ffccb33308b22e51570cb45e51 2023-08-28T02:12:18.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[109923]: cluster 2023-08-28T02:12:17. 2023-08-28T02:12:18.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[109923]: 262517+0000 mon.a (mon.0) 742 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-08-28T02:12:18.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[109923]: audit 2023-08-28 2023-08-28T02:12:18.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[109923]: T02:12:17.532671+0000 mgr.y (mgr.14424) 99 : audit 2023-08-28T02:12:18.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[109923]: [DBG] from='client.14808 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:9cf208207dfe23ffccb33308b22e51570cb45e51", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:12:18.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[109923]: cephadm 2023-08-28T02:12:18.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[109923]: 2023-08-28T02:12:17 2023-08-28T02:12:18.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[109923]: .533426+0000 mgr.y (mgr.14424) 2023-08-28T02:12:18.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[109923]: 100 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:9cf208207dfe23ffccb33308b22e51570cb45e51 2023-08-28T02:12:18.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[109923]: audit 2023 2023-08-28T02:12:18.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[109923]: -08-28T02:12:17.537463+0000 2023-08-28T02:12:18.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[109923]: mon.a (mon.0) 743 : audit [INF] 2023-08-28T02:12:18.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[109923]: from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:12:18.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:12:18.822 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[109923]: 08-28T02:12:17. 2023-08-28T02:12:18.822 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[109923]: 582511+0000 mon.a (mon.0 2023-08-28T02:12:18.823 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[109923]: ) 744 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:12:18.823 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:12:18.823 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[109923]: 08-28T02:12:17 2023-08-28T02:12:18.823 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[109923]: .584247+0000 mon.a (mon.0) 745 : 2023-08-28T02:12:18.823 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[109923]: audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:12:18.823 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[109923]: audit 2023-08-28T02:12:18.823 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[109923]: 2023-08-28 2023-08-28T02:12:18.823 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[109923]: T02:12:17.584934+0000 mon.a ( 2023-08-28T02:12:18.824 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[109923]: mon.0) 746 : audit [INF] 2023-08-28T02:12:18.824 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[109923]: from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:12:18.824 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[109923]: audit 2023-08-28T02:12:18.824 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[109923]: 2023-08-28T02:12:17.589903 2023-08-28T02:12:18.824 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[109923]: +0000 mon.a (mon.0) 747 : audit 2023-08-28T02:12:18.824 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[109923]: [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:12:18.824 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[109923]: cephadm 2023-08-28T02 2023-08-28T02:12:18.824 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[109923]: :12:17.593935+0000 mgr.y (mgr.14424 2023-08-28T02:12:18.824 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:18 smithi143 conmon[109923]: ) 101 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:9cf208207dfe23ffccb33308b22e51570cb45e51 2023-08-28T02:12:18.825 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:18 smithi169 conmon[100500]: cluster 2023-08-28T02:12:17.262517+0000 mon.a (mon.0) 742 : cluster [DBG] 2023-08-28T02:12:18.825 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:18 smithi169 conmon[100500]: osdmap e64: 8 total, 8 up, 8 in 2023-08-28T02:12:18.825 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:18 smithi169 conmon[100500]: audit 2023-08-28T02:12:17.532671+0000 mgr.y (mgr.14424) 99 : audit [DBG] from='client.14808 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:9cf208207dfe23ffccb33308b22e51570cb45e51", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:12:18.826 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:18 smithi169 conmon[100500]: cephadm 2023-08-28T02:12:17.533426+0000 mgr.y (mgr.14424) 100 2023-08-28T02:12:18.826 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:18 smithi169 conmon[100500]: : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:9cf208207dfe23ffccb33308b22e51570cb45e51 2023-08-28T02:12:18.826 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:18 smithi169 conmon[100500]: audit 2023-08-28T02:12 2023-08-28T02:12:18.826 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:18 smithi169 conmon[100500]: :17.537463+0000 mon.a (mon.0) 743 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:12:18.828 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:18 smithi169 conmon[100500]: audit 2023-08- 2023-08-28T02:12:18.828 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:18 smithi169 conmon[100500]: 28T02:12:17.582511+0000 mon.a (mon.0) 744 2023-08-28T02:12:18.829 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:18 smithi169 conmon[100500]: : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:12:18.829 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:18 smithi169 conmon[100500]: audit 2023-08- 2023-08-28T02:12:18.829 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:18 smithi169 conmon[100500]: 28T02:12:17.584247+0000 mon.a (mon.0) 745 2023-08-28T02:12:18.829 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:18 smithi169 conmon[100500]: : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:12:18.829 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:18 smithi169 conmon[100500]: audit 2023 2023-08-28T02:12:18.829 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:18 smithi169 conmon[100500]: -08-28T02:12:17.584934+0000 mon.a (mon. 2023-08-28T02:12:18.829 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:18 smithi169 conmon[100500]: 0) 746 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:12:18.829 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:18 smithi169 conmon[100500]: audit 2023-08-28T02:12:18.829 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:18 smithi169 conmon[100500]: 2023-08-28 2023-08-28T02:12:18.829 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:18 smithi169 conmon[100500]: T02:12:17.589903+0000 mon.a ( 2023-08-28T02:12:18.829 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:18 smithi169 conmon[100500]: mon.0) 747 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:12:18.829 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:18 smithi169 conmon[100500]: 2023-08-28T02:12:18.830 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:18 smithi169 conmon[100500]: cephadm 2023-08-28T02:12:18.830 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:18 smithi169 conmon[100500]: 2023-08-28T02:12:17 2023-08-28T02:12:18.830 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:18 smithi169 conmon[100500]: .593935+0000 mgr.y (mgr.14424) 101 2023-08-28T02:12:18.830 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:18 smithi169 conmon[100500]: : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:9cf208207dfe23ffccb33308b22e51570cb45e51 2023-08-28T02:12:18.846 INFO:tasks.workunit.client.0.smithi143.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-08-28T02:12:18.850 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd rm testimg 2023-08-28T02:12:18.945 INFO:tasks.workunit.client.0.smithi143.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-08-28T02:12:18.949 INFO:tasks.workunit.client.0.smithi143.stderr:+ cmp /tmp/rbd_import_export_164148/img /tmp/rbd_import_export_164148/img2 2023-08-28T02:12:19.643 INFO:tasks.workunit.client.0.smithi143.stderr:+ cmp /tmp/rbd_import_export_164148/img /tmp/rbd_import_export_164148/img3 2023-08-28T02:12:19.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:19 smithi143 conmon[100258]: cluster 2023-08-28T02:12:17.885924+0000 mgr.y (mgr.14424) 102 : cluster [DBG] pgmap v80: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:12:19.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:19 smithi143 conmon[109923]: cluster 2023-08-28T02:12:17.885924+0000 2023-08-28T02:12:19.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:19 smithi143 conmon[109923]: mgr.y (mgr.14424) 102 : cluster [DBG] pgmap v80: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:12:19.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:19 smithi169 conmon[100500]: cluster 2023-08-28T02:12:17.885924+0000 mgr.y (mgr.14424) 102 : cluster 2023-08-28T02:12:19.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:19 smithi169 conmon[100500]: [DBG] pgmap v80: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:12:20.101 INFO:tasks.workunit.client.0.smithi143.stderr:+ rm /tmp/rbd_import_export_164148/img2 /tmp/rbd_import_export_164148/img3 2023-08-28T02:12:20.700 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:12:20 smithi143 conmon[100708]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:12:20] "GET /metrics HTTP/1.1" 200 191492 "" "Prometheus/2.33.4" 2023-08-28T02:12:21.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:21 smithi143 conmon[109923]: cluster 2023-08-28T02:12:19.886236+0000 mgr.y (mgr.14424) 103 : 2023-08-28T02:12:21.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:21 smithi143 conmon[109923]: cluster [DBG] pgmap v81: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:12:21.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:21 smithi143 conmon[100258]: cluster 2023-08-28T02:12:19.886236+0000 mgr.y (mgr.14424) 103 : cluster 2023-08-28T02:12:21.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:21 smithi143 conmon[100258]: [DBG] pgmap v81: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:12:21.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:21 smithi169 conmon[100500]: cluster 2023-08-28T02:12:19.886236+0000 mgr.y (mgr.14424) 103 : cluster [DBG] 2023-08-28T02:12:21.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:21 smithi169 conmon[100500]: pgmap v81: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:12:23.237 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:12:22 smithi169 conmon[102807]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:12:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-28T02:12:23.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:23 smithi143 conmon[100258]: cluster 2023-08-28T02:12:21. 2023-08-28T02:12:23.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:23 smithi143 conmon[100258]: 886728+0000 mgr.y (mgr.14424) 104 : cluster [DBG] pgmap v82: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 148 KiB/s rd, 64 KiB/s wr, 38 op/s 2023-08-28T02:12:23.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:23 smithi143 conmon[109923]: cluster 2023-08-28T02:12:21.886728 2023-08-28T02:12:23.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:23 smithi143 conmon[109923]: +0000 mgr.y (mgr.14424) 104 : cluster [DBG] pgmap v82: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 148 KiB/s rd, 64 KiB/s wr, 38 op/s 2023-08-28T02:12:23.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:23 smithi169 conmon[100500]: cluster 2023-08-28T02:12:21.886728+0000 mgr.y (mgr.14424) 104 : cluster 2023-08-28T02:12:23.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:23 smithi169 conmon[100500]: [DBG] pgmap v82: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 148 KiB/s rd, 64 KiB/s wr, 38 op/s 2023-08-28T02:12:25.550 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd import --new-format - testimg 2023-08-28T02:12:25.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:25 smithi143 conmon[100258]: cluster 2023-08-28T02:12:23.887079+0000 mgr.y (mgr.14424) 105 : cluster 2023-08-28T02:12:25.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:25 smithi143 conmon[100258]: [DBG] pgmap v83: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 148 KiB/s rd, 64 KiB/s wr, 38 op/s 2023-08-28T02:12:25.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:25 smithi143 conmon[100258]: cluster 2023-08-28T02:12:25.174351+0000 mon.a (mon.0) 748 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-08-28T02:12:25.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:25 smithi143 conmon[109923]: cluster 2023-08-28T02:12:23.887079 2023-08-28T02:12:25.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:25 smithi143 conmon[109923]: +0000 mgr.y (mgr.14424) 105 : cluster [DBG] pgmap v83: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 148 KiB/s rd, 64 KiB/s wr, 38 op/s 2023-08-28T02:12:25.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:25 smithi143 conmon[109923]: cluster 2023-08-28T02:12:25.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:25 smithi143 conmon[109923]: 2023-08-28T02:12:25.174351+0000 mon.a (mon.0) 748 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-08-28T02:12:25.981 INFO:tasks.workunit.client.0.smithi143.stderr: Importing image: 100% complete...done. 2023-08-28T02:12:25.981 INFO:tasks.workunit.client.0.smithi143.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-08-28T02:12:25.984 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd export testimg /tmp/rbd_import_export_164148/img2 2023-08-28T02:12:25.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:25 smithi169 conmon[100500]: cluster 2023-08-28T02:12:23.887079+0000 mgr.y (mgr.14424) 105 : cluster 2023-08-28T02:12:25.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:25 smithi169 conmon[100500]: [DBG] pgmap v83: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 148 KiB/s rd, 64 KiB/s wr, 38 op/s 2023-08-28T02:12:25.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:25 smithi169 conmon[100500]: cluster 2023-08-28T02:12:25.174351+0000 mon.a (mon.0) 748 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-08-28T02:12:26.264 INFO:tasks.workunit.client.0.smithi143.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-08-28T02:12:26.269 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd export testimg - 2023-08-28T02:12:27.099 INFO:tasks.workunit.client.0.smithi143.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-08-28T02:12:27.102 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd rm testimg 2023-08-28T02:12:27.194 INFO:tasks.workunit.client.0.smithi143.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-08-28T02:12:27.197 INFO:tasks.workunit.client.0.smithi143.stderr:+ cmp /tmp/rbd_import_export_164148/img /tmp/rbd_import_export_164148/img2 2023-08-28T02:12:27.850 INFO:tasks.workunit.client.0.smithi143.stderr:+ cmp /tmp/rbd_import_export_164148/img /tmp/rbd_import_export_164148/img3 2023-08-28T02:12:27.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:27 smithi143 conmon[100258]: cluster 2023-08-28T02:12:25.887570+0000 mgr.y (mgr.14424) 106 : cluster [DBG] pgmap v85: 73 pgs: 73 active+clean; 4.7 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 158 KiB/s rd, 97 KiB/s wr, 63 op/s 2023-08-28T02:12:27.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:27 smithi143 conmon[109923]: cluster 2023-08-28T02:12:25.887570+0000 mgr.y 2023-08-28T02:12:27.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:27 smithi143 conmon[109923]: (mgr.14424) 106 : cluster [DBG] pgmap v85: 73 pgs: 73 active+clean; 4.7 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 158 KiB/s rd, 97 KiB/s wr, 63 op/s 2023-08-28T02:12:27.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:27 smithi169 conmon[100500]: cluster 2023-08-28T02:12:25.887570+0000 mgr.y (mgr.14424) 106 : cluster [DBG] 2023-08-28T02:12:27.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:27 smithi169 conmon[100500]: pgmap v85: 73 pgs: 73 active+clean; 4.7 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 158 KiB/s rd, 97 KiB/s wr, 63 op/s 2023-08-28T02:12:28.304 INFO:tasks.workunit.client.0.smithi143.stderr:+ rm /tmp/rbd_import_export_164148/img /tmp/rbd_import_export_164148/img2 /tmp/rbd_import_export_164148/img3 2023-08-28T02:12:29.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:29 smithi143 conmon[109923]: cluster 2023-08-28T02:12:27 2023-08-28T02:12:29.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:29 smithi143 conmon[109923]: .887921+0000 mgr.y (mgr.14424) 107 : cluster [DBG] pgmap v86: 73 pgs: 73 active+clean; 4.7 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 136 KiB/s rd, 84 KiB/s wr, 54 op/s 2023-08-28T02:12:29.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:29 smithi143 conmon[100258]: cluster 2023-08-28T02:12:27.887921+0000 mgr.y (mgr. 2023-08-28T02:12:29.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:29 smithi143 conmon[100258]: 14424) 107 : cluster [DBG] pgmap v86: 73 pgs: 73 active+clean; 4.7 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 136 KiB/s rd, 84 KiB/s wr, 54 op/s 2023-08-28T02:12:29.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:29 smithi169 conmon[100500]: cluster 2023-08-28T02:12:27.887921+0000 mgr.y (mgr.14424 2023-08-28T02:12:29.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:29 smithi169 conmon[100500]: ) 107 : cluster [DBG] pgmap v86: 73 pgs: 73 active+clean; 4.7 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 136 KiB/s rd, 84 KiB/s wr, 54 op/s 2023-08-28T02:12:30.700 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:12:30 smithi143 conmon[100708]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:12:30] "GET /metrics HTTP/1.1" 200 192002 "" "Prometheus/2.33.4" 2023-08-28T02:12:31.345 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd help export 2023-08-28T02:12:31.346 INFO:tasks.workunit.client.0.smithi143.stderr:+ grep -q export-format 2023-08-28T02:12:31.374 INFO:tasks.workunit.client.0.smithi143.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_164148/img bs=1k count=10 seek=100 2023-08-28T02:12:31.375 INFO:tasks.workunit.client.0.smithi143.stderr:10+0 records in 2023-08-28T02:12:31.375 INFO:tasks.workunit.client.0.smithi143.stderr:10+0 records out 2023-08-28T02:12:31.375 INFO:tasks.workunit.client.0.smithi143.stderr:10240 bytes (10 kB, 10 KiB) copied, 6.5523e-05 s, 156 MB/s 2023-08-28T02:12:31.375 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd import --new-format /tmp/rbd_import_export_164148/img testimg 2023-08-28T02:12:31.438 INFO:tasks.workunit.client.0.smithi143.stderr: Importing image: 100% complete...done. 2023-08-28T02:12:31.439 INFO:tasks.workunit.client.0.smithi143.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-08-28T02:12:31.441 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd snap create testimg@snap 2023-08-28T02:12:31.628 INFO:tasks.workunit.client.0.smithi143.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-08-28T02:12:31.636 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd image-meta set testimg key1 value1 2023-08-28T02:12:31.684 INFO:tasks.workunit.client.0.smithi143.stderr:++ rbd image-meta list testimg 2023-08-28T02:12:31.724 INFO:tasks.workunit.client.0.smithi143.stderr:+ IMAGEMETA_BEFORE='There is 1 metadatum on this image: 2023-08-28T02:12:31.724 INFO:tasks.workunit.client.0.smithi143.stderr: 2023-08-28T02:12:31.724 INFO:tasks.workunit.client.0.smithi143.stderr:Key Value 2023-08-28T02:12:31.724 INFO:tasks.workunit.client.0.smithi143.stderr:key1 value1' 2023-08-28T02:12:31.724 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_164148/img_v2 2023-08-28T02:12:31.761 INFO:tasks.workunit.client.0.smithi143.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-08-28T02:12:31.764 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_164148/img_v2 testimg_import 2023-08-28T02:12:31.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:31 smithi143 conmon[100258]: cluster 2023-08-28T02:12:29.888251+0000 mgr.y (mgr.14424) 108 : cluster [DBG] pgmap v87: 73 pgs: 73 active+clean; 4.7 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 136 KiB/s rd, 84 KiB/s wr, 54 op/s 2023-08-28T02:12:31.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:31 smithi143 conmon[109923]: cluster 2023-08-28T02:12:29.888251+0000 mgr.y (mgr.14424 2023-08-28T02:12:31.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:31 smithi143 conmon[109923]: ) 108 : cluster [DBG] pgmap v87: 73 pgs: 73 active+clean; 4.7 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 136 KiB/s rd, 84 KiB/s wr, 54 op/s 2023-08-28T02:12:31.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:31 smithi169 conmon[100500]: cluster 2023-08-28T02:12:29.888251+0000 mgr.y (mgr.14424) 108 : cluster 2023-08-28T02:12:31.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:31 smithi169 conmon[100500]: [DBG] pgmap v87: 73 pgs: 73 active+clean; 4.7 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 136 KiB/s rd, 84 KiB/s wr, 54 op/s 2023-08-28T02:12:32.637 INFO:tasks.workunit.client.0.smithi143.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-08-28T02:12:32.640 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd info testimg_import 2023-08-28T02:12:32.673 INFO:tasks.workunit.client.0.smithi143.stdout:rbd image 'testimg_import': 2023-08-28T02:12:32.673 INFO:tasks.workunit.client.0.smithi143.stdout: size 110 KiB in 1 objects 2023-08-28T02:12:32.673 INFO:tasks.workunit.client.0.smithi143.stdout: order 22 (4 MiB objects) 2023-08-28T02:12:32.673 INFO:tasks.workunit.client.0.smithi143.stdout: snapshot_count: 1 2023-08-28T02:12:32.673 INFO:tasks.workunit.client.0.smithi143.stdout: id: 3a264182e114 2023-08-28T02:12:32.673 INFO:tasks.workunit.client.0.smithi143.stdout: block_name_prefix: rbd_data.3a264182e114 2023-08-28T02:12:32.673 INFO:tasks.workunit.client.0.smithi143.stdout: format: 2 2023-08-28T02:12:32.674 INFO:tasks.workunit.client.0.smithi143.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-08-28T02:12:32.674 INFO:tasks.workunit.client.0.smithi143.stdout: op_features: 2023-08-28T02:12:32.674 INFO:tasks.workunit.client.0.smithi143.stdout: flags: 2023-08-28T02:12:32.674 INFO:tasks.workunit.client.0.smithi143.stdout: create_timestamp: Mon Aug 28 02:12:31 2023 2023-08-28T02:12:32.674 INFO:tasks.workunit.client.0.smithi143.stdout: access_timestamp: Mon Aug 28 02:12:31 2023 2023-08-28T02:12:32.674 INFO:tasks.workunit.client.0.smithi143.stdout: modify_timestamp: Mon Aug 28 02:12:31 2023 2023-08-28T02:12:32.676 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd info testimg_import@snap 2023-08-28T02:12:32.710 INFO:tasks.workunit.client.0.smithi143.stdout:rbd image 'testimg_import': 2023-08-28T02:12:32.710 INFO:tasks.workunit.client.0.smithi143.stdout: size 110 KiB in 1 objects 2023-08-28T02:12:32.710 INFO:tasks.workunit.client.0.smithi143.stdout: order 22 (4 MiB objects) 2023-08-28T02:12:32.710 INFO:tasks.workunit.client.0.smithi143.stdout: snapshot_count: 1 2023-08-28T02:12:32.710 INFO:tasks.workunit.client.0.smithi143.stdout: id: 3a264182e114 2023-08-28T02:12:32.710 INFO:tasks.workunit.client.0.smithi143.stdout: block_name_prefix: rbd_data.3a264182e114 2023-08-28T02:12:32.710 INFO:tasks.workunit.client.0.smithi143.stdout: format: 2 2023-08-28T02:12:32.710 INFO:tasks.workunit.client.0.smithi143.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-08-28T02:12:32.710 INFO:tasks.workunit.client.0.smithi143.stdout: op_features: 2023-08-28T02:12:32.711 INFO:tasks.workunit.client.0.smithi143.stdout: flags: 2023-08-28T02:12:32.711 INFO:tasks.workunit.client.0.smithi143.stdout: create_timestamp: Mon Aug 28 02:12:31 2023 2023-08-28T02:12:32.711 INFO:tasks.workunit.client.0.smithi143.stdout: access_timestamp: Mon Aug 28 02:12:31 2023 2023-08-28T02:12:32.711 INFO:tasks.workunit.client.0.smithi143.stdout: modify_timestamp: Mon Aug 28 02:12:31 2023 2023-08-28T02:12:32.711 INFO:tasks.workunit.client.0.smithi143.stdout: protected: False 2023-08-28T02:12:32.713 INFO:tasks.workunit.client.0.smithi143.stderr:++ rbd image-meta list testimg_import 2023-08-28T02:12:32.756 INFO:tasks.workunit.client.0.smithi143.stderr:+ IMAGEMETA_AFTER='There is 1 metadatum on this image: 2023-08-28T02:12:32.756 INFO:tasks.workunit.client.0.smithi143.stderr: 2023-08-28T02:12:32.756 INFO:tasks.workunit.client.0.smithi143.stderr:Key Value 2023-08-28T02:12:32.756 INFO:tasks.workunit.client.0.smithi143.stderr:key1 value1' 2023-08-28T02:12:32.756 INFO:tasks.workunit.client.0.smithi143.stderr:+ '[' 'There is 1 metadatum on this image: 2023-08-28T02:12:32.756 INFO:tasks.workunit.client.0.smithi143.stderr: 2023-08-28T02:12:32.756 INFO:tasks.workunit.client.0.smithi143.stderr:Key Value 2023-08-28T02:12:32.756 INFO:tasks.workunit.client.0.smithi143.stderr:key1 value1' = 'There is 1 metadatum on this image: 2023-08-28T02:12:32.756 INFO:tasks.workunit.client.0.smithi143.stderr: 2023-08-28T02:12:32.757 INFO:tasks.workunit.client.0.smithi143.stderr:Key Value 2023-08-28T02:12:32.757 INFO:tasks.workunit.client.0.smithi143.stderr:key1 value1' ']' 2023-08-28T02:12:32.757 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd export testimg_import /tmp/rbd_import_export_164148/img_import 2023-08-28T02:12:32.761 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:32 smithi143 conmon[100258]: cluster 2023-08-28T02:12:31.619456+0000 mon.a ( 2023-08-28T02:12:32.762 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:32 smithi143 conmon[100258]: mon.0) 749 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-08-28T02:12:32.762 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:32 smithi143 conmon[109923]: cluster 2023-08-28T02:12:31.619456+0000 mon.a (mon.0) 749 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-08-28T02:12:32.793 INFO:tasks.workunit.client.0.smithi143.stderr: Exporting image: 100% complete...done. 2023-08-28T02:12:32.795 INFO:tasks.workunit.client.0.smithi143.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_164148/img /tmp/rbd_import_export_164148/img_import 2023-08-28T02:12:32.795 INFO:tasks.workunit.client.0.smithi143.stderr:+ cmp -l /tmp/rbd_import_export_164148/img /tmp/rbd_import_export_164148/img_import 2023-08-28T02:12:32.796 INFO:tasks.workunit.client.0.smithi143.stderr:++ stat /tmp/rbd_import_export_164148/img --format %b 2023-08-28T02:12:32.797 INFO:tasks.workunit.client.0.smithi143.stderr:+ origsize=24 2023-08-28T02:12:32.797 INFO:tasks.workunit.client.0.smithi143.stderr:++ stat /tmp/rbd_import_export_164148/img_import --format %b 2023-08-28T02:12:32.798 INFO:tasks.workunit.client.0.smithi143.stderr:+ exportsize=224 2023-08-28T02:12:32.798 INFO:tasks.workunit.client.0.smithi143.stderr:+ difference=200 2023-08-28T02:12:32.798 INFO:tasks.workunit.client.0.smithi143.stderr:+ difference=200 2023-08-28T02:12:32.799 INFO:tasks.workunit.client.0.smithi143.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2023-08-28T02:12:32.799 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_164148/img_snap 2023-08-28T02:12:32.832 INFO:tasks.workunit.client.0.smithi143.stderr: Exporting image: 100% complete...done. 2023-08-28T02:12:32.835 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_164148/img_snap_import 2023-08-28T02:12:32.881 INFO:tasks.workunit.client.0.smithi143.stderr: Exporting image: 100% complete...done. 2023-08-28T02:12:32.886 INFO:tasks.workunit.client.0.smithi143.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_164148/img_snap /tmp/rbd_import_export_164148/img_snap_import 2023-08-28T02:12:32.886 INFO:tasks.workunit.client.0.smithi143.stderr:+ cmp -l /tmp/rbd_import_export_164148/img_snap /tmp/rbd_import_export_164148/img_snap_import 2023-08-28T02:12:32.887 INFO:tasks.workunit.client.0.smithi143.stderr:++ stat /tmp/rbd_import_export_164148/img_snap --format %b 2023-08-28T02:12:32.888 INFO:tasks.workunit.client.0.smithi143.stderr:+ origsize=224 2023-08-28T02:12:32.888 INFO:tasks.workunit.client.0.smithi143.stderr:++ stat /tmp/rbd_import_export_164148/img_snap_import --format %b 2023-08-28T02:12:32.889 INFO:tasks.workunit.client.0.smithi143.stderr:+ exportsize=224 2023-08-28T02:12:32.889 INFO:tasks.workunit.client.0.smithi143.stderr:+ difference=0 2023-08-28T02:12:32.889 INFO:tasks.workunit.client.0.smithi143.stderr:+ difference=0 2023-08-28T02:12:32.889 INFO:tasks.workunit.client.0.smithi143.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-08-28T02:12:32.889 INFO:tasks.workunit.client.0.smithi143.stderr:+ rm /tmp/rbd_import_export_164148/img_v2 2023-08-28T02:12:32.890 INFO:tasks.workunit.client.0.smithi143.stderr:+ rm /tmp/rbd_import_export_164148/img_import 2023-08-28T02:12:32.890 INFO:tasks.workunit.client.0.smithi143.stderr:+ rm /tmp/rbd_import_export_164148/img_snap 2023-08-28T02:12:32.891 INFO:tasks.workunit.client.0.smithi143.stderr:+ rm /tmp/rbd_import_export_164148/img_snap_import 2023-08-28T02:12:32.892 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd snap rm testimg_import@snap 2023-08-28T02:12:32.937 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:32 smithi169 conmon[100500]: cluster 2023-08-28T02:12:31.619456+0000 mon.a (mon.0 2023-08-28T02:12:32.937 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:32 smithi169 conmon[100500]: ) 749 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-08-28T02:12:33.194 INFO:teuthology.orchestra.run.smithi143.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-08-28T02:12:33.194 INFO:teuthology.orchestra.run.smithi143.stdout:alertmanager.a smithi143 *:9093,9094 running (78s) 57s ago 108s - - 0.23.0 ba2b418f427c 24d74df782bd 2023-08-28T02:12:33.194 INFO:teuthology.orchestra.run.smithi143.stdout:grafana.a smithi169 *:3000 running (91s) 55s ago 90s 49.2M - 8.3.5 dad864ee21e9 dcf28c93da58 2023-08-28T02:12:33.194 INFO:teuthology.orchestra.run.smithi143.stdout:mds.foo.smithi143.spifrj smithi143 running (64s) 57s ago 63s - - 16.2.13-565-g595355f4 4099d2e00414 4eeedd3bc917 2023-08-28T02:12:33.194 INFO:teuthology.orchestra.run.smithi143.stdout:mds.foo.smithi169.yeisur smithi169 running (61s) 55s ago 61s 15.6M - 16.2.13-565-g595355f4 4099d2e00414 fed9013c1357 2023-08-28T02:12:33.194 INFO:teuthology.orchestra.run.smithi143.stdout:mgr.x smithi169 *:8443 running (5m) 55s ago 5m 392M - 16.2.13-565-g595355f4 4099d2e00414 96e04bb66489 2023-08-28T02:12:33.194 INFO:teuthology.orchestra.run.smithi143.stdout:mgr.y smithi143 *:9283 running (7m) 57s ago 7m - - 16.2.13-565-g595355f4 4099d2e00414 48d3608092c2 2023-08-28T02:12:33.195 INFO:teuthology.orchestra.run.smithi143.stdout:mon.a smithi143 running (7m) 57s ago 7m - 2048M 16.2.13-565-g595355f4 4099d2e00414 3e714a619127 2023-08-28T02:12:33.195 INFO:teuthology.orchestra.run.smithi143.stdout:mon.b smithi169 running (6m) 55s ago 6m 99.5M 2048M 16.2.13-565-g595355f4 4099d2e00414 f70db2856b37 2023-08-28T02:12:33.195 INFO:teuthology.orchestra.run.smithi143.stdout:mon.c smithi143 running (5m) 57s ago 5m - 2048M 16.2.13-565-g595355f4 4099d2e00414 74b6ddae7a3f 2023-08-28T02:12:33.195 INFO:teuthology.orchestra.run.smithi143.stdout:node-exporter.a smithi143 *:9100 running (2m) 57s ago 2m - - 1.3.1 1dbe0e931976 08a67263bdce 2023-08-28T02:12:33.195 INFO:teuthology.orchestra.run.smithi143.stdout:node-exporter.b smithi169 *:9100 running (2m) 55s ago 119s 10.8M - 1.3.1 1dbe0e931976 b3659a52b450 2023-08-28T02:12:33.195 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0 smithi143 running (5m) 57s ago 5m - 4036M 16.2.13-565-g595355f4 4099d2e00414 ce7c7eb91333 2023-08-28T02:12:33.195 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1 smithi143 running (5m) 57s ago 4m - 4036M 16.2.13-565-g595355f4 4099d2e00414 f27082b59650 2023-08-28T02:12:33.195 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2 smithi143 running (4m) 57s ago 4m - 4036M 16.2.13-565-g595355f4 4099d2e00414 e20c9920166c 2023-08-28T02:12:33.195 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3 smithi143 running (4m) 57s ago 4m - 4036M 16.2.13-565-g595355f4 4099d2e00414 ab95599724c4 2023-08-28T02:12:33.195 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4 smithi169 running (3m) 55s ago 3m 48.1M 4292M 16.2.13-565-g595355f4 4099d2e00414 e6ed776def27 2023-08-28T02:12:33.195 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5 smithi169 running (3m) 55s ago 3m 44.7M 4292M 16.2.13-565-g595355f4 4099d2e00414 6fe1195fca7e 2023-08-28T02:12:33.196 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6 smithi169 running (3m) 55s ago 3m 45.0M 4292M 16.2.13-565-g595355f4 4099d2e00414 12854ca48e76 2023-08-28T02:12:33.196 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7 smithi169 running (2m) 55s ago 2m 45.5M 4292M 16.2.13-565-g595355f4 4099d2e00414 64fa1422c85a 2023-08-28T02:12:33.196 INFO:teuthology.orchestra.run.smithi143.stdout:prometheus.a smithi169 *:9095 running (73s) 55s ago 2m 42.6M - 2.33.4 514e6a882f6e 3ceb1bb30c93 2023-08-28T02:12:33.237 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:12:32 smithi169 conmon[102807]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:12:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-28T02:12:33.638 INFO:tasks.workunit.client.0.smithi143.stderr: Removing snap: 100% complete...done. 2023-08-28T02:12:33.645 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd remove testimg_import 2023-08-28T02:12:33.733 INFO:tasks.workunit.client.0.smithi143.stderr: Removing image: 100% complete...done. 2023-08-28T02:12:33.736 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd snap rm testimg@snap 2023-08-28T02:12:33.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:33 smithi143 conmon[100258]: cluster 2023-08-28T02:12:31.888569+0000 mgr.y (mgr.14424) 109 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 167 KiB/s rd, 66 KiB/s wr, 62 op/s 2023-08-28T02:12:33.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:33 smithi143 conmon[100258]: cluster 2023-08-28T02:12:32.626574+0000 mon.a (mon.0) 750 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-08-28T02:12:33.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:33 smithi143 conmon[109923]: cluster 2023-08-28T02:12:31.888569+0000 mgr.y (mgr.14424) 109 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 167 KiB/s rd, 66 KiB/s wr, 62 op/s 2023-08-28T02:12:33.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:33 smithi143 conmon[109923]: cluster 2023-08-28T02:12:32.626574+0000 mon.a (mon.0) 750 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-08-28T02:12:33.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:33 smithi169 conmon[100500]: cluster 2023-08-28T02:12:31.888569+0000 mgr.y (mgr.14424) 109 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 167 KiB/s rd, 66 KiB/s wr, 62 op/s 2023-08-28T02:12:33.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:33 smithi169 conmon[100500]: 2023-08-28T02:12:33.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:33 smithi169 conmon[100500]: cluster 2023-08-28T02:12:32.626574+0000 mon.a (mon.0) 750 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-08-28T02:12:34.322 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -e sha1=9cf208207dfe23ffccb33308b22e51570cb45e51 -- bash -c 'echo wait for minority of mons to upgrade' 2023-08-28T02:12:34.646 INFO:tasks.workunit.client.0.smithi143.stderr: Removing snap: 100% complete...done. 2023-08-28T02:12:34.661 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd rm testimg 2023-08-28T02:12:34.760 INFO:tasks.workunit.client.0.smithi143.stderr: Removing image: 100% complete...done. 2023-08-28T02:12:34.766 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd import --order 20 /tmp/rbd_import_export_164148/img testimg 2023-08-28T02:12:34.822 INFO:tasks.workunit.client.0.smithi143.stderr: Importing image: 100% complete...done. 2023-08-28T02:12:34.822 INFO:tasks.workunit.client.0.smithi143.stderr:rbd: --order is deprecated, use --object-size 2023-08-28T02:12:34.825 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_164148/img_v2 2023-08-28T02:12:34.877 INFO:tasks.workunit.client.0.smithi143.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-08-28T02:12:34.879 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:34 smithi143 conmon[100258]: audit 2023-08-28T02:12:33 2023-08-28T02:12:34.879 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:34 smithi143 conmon[100258]: .185873+0000 mgr.y (mgr.14424) 110 : audit [DBG] from='client.14934 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:12:34.879 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:34 smithi143 conmon[100258]: cluster 2023-08-28T02:12:33.636689+0000 mon.a (mon.0) 751 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2023-08-28T02:12:34.879 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:34 smithi143 conmon[100258]: cluster 2023-08-28T02:12:34.879 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:34 smithi143 conmon[100258]: 2023-08-28T02:12: 2023-08-28T02:12:34.879 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:34 smithi143 conmon[100258]: 33.888910+0000 mgr.y (mgr.14424) 111 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 194 KiB/s rd, 33 KiB/s wr, 42 op/s 2023-08-28T02:12:34.879 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:34 smithi143 conmon[109923]: audit 2023-08-28T02:12: 2023-08-28T02:12:34.879 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:34 smithi143 conmon[109923]: 33.185873+0000 mgr.y (mgr.14424) 110 : audit [DBG] from='client.14934 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:12:34.879 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:34 smithi143 conmon[109923]: cluster 2023-08-28T02:12:33.636689+0000 mon.a (mon.0) 751 : cluster [DBG] 2023-08-28T02:12:34.879 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:34 smithi143 conmon[109923]: osdmap e68: 8 total, 8 up, 8 in 2023-08-28T02:12:34.880 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:34 smithi143 conmon[109923]: cluster 2023-08-28 2023-08-28T02:12:34.880 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:34 smithi143 conmon[109923]: T02:12:33.888910+0000 mgr.y (mgr.14424) 111 : 2023-08-28T02:12:34.880 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:34 smithi143 conmon[109923]: cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 194 KiB/s rd, 33 KiB/s wr, 42 op/s 2023-08-28T02:12:34.881 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_164148/img_v2 testimg_import 2023-08-28T02:12:34.946 INFO:tasks.workunit.client.0.smithi143.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-08-28T02:12:34.949 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd info testimg_import 2023-08-28T02:12:34.949 INFO:tasks.workunit.client.0.smithi143.stderr:+ grep order 2023-08-28T02:12:34.950 INFO:tasks.workunit.client.0.smithi143.stderr:+ awk '{print $2}' 2023-08-28T02:12:34.950 INFO:tasks.workunit.client.0.smithi143.stderr:+ grep 20 2023-08-28T02:12:34.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:34 smithi169 conmon[100500]: audit 2023-08-28T02:12:33.185873+0000 mgr.y (mgr.14424) 110 : audit [DBG] from='client.14934 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:12:34.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:34 smithi169 conmon[100500]: cluster 2023-08-28T02:12:33.636689+0000 mon.a (mon.0) 751 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2023-08-28T02:12:34.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:34 smithi169 conmon[100500]: cluster 2023-08-28T02:12:33.888910+0000 mgr.y (mgr.14424) 111 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 194 KiB/s rd, 33 KiB/s wr, 42 op/s 2023-08-28T02:12:34.993 INFO:tasks.workunit.client.0.smithi143.stdout:20 2023-08-28T02:12:34.993 INFO:tasks.workunit.client.0.smithi143.stderr:+ rm /tmp/rbd_import_export_164148/img_v2 2023-08-28T02:12:34.994 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd remove testimg_import 2023-08-28T02:12:35.088 INFO:tasks.workunit.client.0.smithi143.stderr: Removing image: 100% complete...done. 2023-08-28T02:12:35.091 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd remove testimg 2023-08-28T02:12:35.183 INFO:tasks.workunit.client.0.smithi143.stderr: Removing image: 100% complete...done. 2023-08-28T02:12:35.185 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_164148/img testimg 2023-08-28T02:12:35.234 INFO:tasks.workunit.client.0.smithi143.stderr: Importing image: 100% complete...done. 2023-08-28T02:12:35.237 INFO:tasks.workunit.client.0.smithi143.stderr:++ rbd info testimg 2023-08-28T02:12:35.237 INFO:tasks.workunit.client.0.smithi143.stderr:++ grep features 2023-08-28T02:12:35.275 INFO:tasks.workunit.client.0.smithi143.stderr:+ FEATURES_BEFORE=' features: layering 2023-08-28T02:12:35.275 INFO:tasks.workunit.client.0.smithi143.stderr: op_features: ' 2023-08-28T02:12:35.275 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_164148/img_v2 2023-08-28T02:12:35.311 INFO:tasks.workunit.client.0.smithi143.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-08-28T02:12:35.314 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_164148/img_v2 testimg_import 2023-08-28T02:12:35.368 INFO:tasks.workunit.client.0.smithi143.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-08-28T02:12:35.371 INFO:tasks.workunit.client.0.smithi143.stderr:++ rbd info testimg_import 2023-08-28T02:12:35.371 INFO:tasks.workunit.client.0.smithi143.stderr:++ grep features 2023-08-28T02:12:35.408 INFO:tasks.workunit.client.0.smithi143.stderr:+ FEATURES_AFTER=' features: layering 2023-08-28T02:12:35.408 INFO:tasks.workunit.client.0.smithi143.stderr: op_features: ' 2023-08-28T02:12:35.408 INFO:tasks.workunit.client.0.smithi143.stderr:+ '[' ' features: layering 2023-08-28T02:12:35.408 INFO:tasks.workunit.client.0.smithi143.stderr: op_features: ' '!=' ' features: layering 2023-08-28T02:12:35.409 INFO:tasks.workunit.client.0.smithi143.stderr: op_features: ' ']' 2023-08-28T02:12:35.409 INFO:tasks.workunit.client.0.smithi143.stderr:+ rm /tmp/rbd_import_export_164148/img_v2 2023-08-28T02:12:35.626 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd remove testimg_import 2023-08-28T02:12:35.697 INFO:tasks.workunit.client.0.smithi143.stderr: Removing image: 100% complete...done. 2023-08-28T02:12:35.699 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd remove testimg 2023-08-28T02:12:35.767 INFO:tasks.workunit.client.0.smithi143.stderr: Removing image: 100% complete...done. 2023-08-28T02:12:35.770 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_164148/img testimg 2023-08-28T02:12:35.830 INFO:tasks.workunit.client.0.smithi143.stderr: Importing image: 100% complete...done. 2023-08-28T02:12:35.833 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_164148/img_v2 2023-08-28T02:12:35.875 INFO:tasks.workunit.client.0.smithi143.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-08-28T02:12:35.877 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_164148/img_v2 testimg_import 2023-08-28T02:12:35.941 INFO:tasks.workunit.client.0.smithi143.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-08-28T02:12:35.944 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd info testimg_import 2023-08-28T02:12:35.944 INFO:tasks.workunit.client.0.smithi143.stderr:+ grep 'stripe unit' 2023-08-28T02:12:35.944 INFO:tasks.workunit.client.0.smithi143.stderr:+ grep -Ei '(4 KiB|4096)' 2023-08-28T02:12:35.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:35 smithi143 conmon[100258]: cluster 2023-08-28T02:12:34.642718+0000 mon.a (mon.0) 752 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-08-28T02:12:35.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:35 smithi143 conmon[109923]: cluster 2023-08-28T02:12:34.642718+0000 mon.a 2023-08-28T02:12:35.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:35 smithi143 conmon[109923]: (mon.0) 752 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-08-28T02:12:35.980 INFO:tasks.workunit.client.0.smithi143.stdout: stripe unit: 4 KiB 2023-08-28T02:12:35.981 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd info testimg_import 2023-08-28T02:12:35.981 INFO:tasks.workunit.client.0.smithi143.stderr:+ grep 'stripe count' 2023-08-28T02:12:35.981 INFO:tasks.workunit.client.0.smithi143.stderr:+ awk '{print $3}' 2023-08-28T02:12:35.981 INFO:tasks.workunit.client.0.smithi143.stderr:+ grep 1000 2023-08-28T02:12:35.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:35 smithi169 conmon[100500]: cluster 2023-08-28T02:12:34.642718+0000 mon.a (mon.0) 752 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-08-28T02:12:36.019 INFO:tasks.workunit.client.0.smithi143.stdout:1000 2023-08-28T02:12:36.019 INFO:tasks.workunit.client.0.smithi143.stderr:+ rm /tmp/rbd_import_export_164148/img_v2 2023-08-28T02:12:36.019 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd remove testimg_import 2023-08-28T02:12:36.126 INFO:tasks.workunit.client.0.smithi143.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-08-28T02:12:36.129 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd remove testimg 2023-08-28T02:12:36.219 INFO:tasks.workunit.client.0.smithi143.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-08-28T02:12:36.222 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_164148/img testimg 2023-08-28T02:12:36.279 INFO:tasks.workunit.client.0.smithi143.stderr: Importing image: 100% complete...done. 2023-08-28T02:12:36.284 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd snap create testimg@snap1 2023-08-28T02:12:36.386 INFO:teuthology.orchestra.run.smithi143.stdout:wait for minority of mons to upgrade 2023-08-28T02:12:36.654 INFO:tasks.workunit.client.0.smithi143.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-08-28T02:12:36.660 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd snap create testimg@snap2 2023-08-28T02:12:36.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:36 smithi143 conmon[100258]: cluster 2023-08-28T02:12:35.889386+0000 mgr.y (mgr.14424) 112 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 12 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 366 KiB/s rd, 40 KiB/s wr, 476 op/s 2023-08-28T02:12:36.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:36 smithi143 conmon[109923]: cluster 2023-08-28T02:12:35.889386+0000 2023-08-28T02:12:36.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:36 smithi143 conmon[109923]: mgr.y (mgr.14424) 112 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 12 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 366 KiB/s rd, 40 KiB/s wr, 476 op/s 2023-08-28T02:12:36.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:36 smithi169 conmon[100500]: cluster 2023-08-28T02:12:35.889386+0000 mgr.y ( 2023-08-28T02:12:36.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:36 smithi169 conmon[100500]: mgr.14424) 112 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 12 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 366 KiB/s rd, 40 KiB/s wr, 476 op/s 2023-08-28T02:12:37.270 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -e sha1=9cf208207dfe23ffccb33308b22e51570cb45e51 -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2023-08-28T02:12:37.664 INFO:tasks.workunit.client.0.smithi143.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-08-28T02:12:37.671 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd snap protect testimg@snap2 2023-08-28T02:12:37.725 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_164148/snap_protect 2023-08-28T02:12:37.767 INFO:tasks.workunit.client.0.smithi143.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-08-28T02:12:37.771 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_164148/snap_protect testimg_import 2023-08-28T02:12:37.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:37 smithi143 conmon[100258]: cluster 2023-08-28T02:12:36 2023-08-28T02:12:37.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:37 smithi143 conmon[100258]: .648588+0000 mon.a (mon.0) 753 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2023-08-28T02:12:37.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:37 smithi143 conmon[109923]: cluster 2023-08-28T02:12:36.648588+0000 2023-08-28T02:12:37.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:37 smithi143 conmon[109923]: mon.a (mon.0) 753 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2023-08-28T02:12:37.991 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:37 smithi169 conmon[100500]: cluster 2023-08-28T02:12:36.648588+0000 mon.a (mon.0) 753 : cluster [DBG] 2023-08-28T02:12:37.991 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:37 smithi169 conmon[100500]: osdmap e70: 8 total, 8 up, 8 in 2023-08-28T02:12:38.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:38 smithi143 conmon[100258]: cluster 2023-08-28T02:12:37.653311+0000 mon.a 2023-08-28T02:12:38.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:38 smithi143 conmon[100258]: (mon.0) 754 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-08-28T02:12:38.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:38 smithi143 conmon[100258]: cluster 2023-08-28T02:12:37.889697+0000 mgr.y (mgr.14424) 113 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 12 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 367 KiB/s rd, 40 KiB/s wr, 478 op/s 2023-08-28T02:12:38.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:38 smithi143 conmon[109923]: cluster 2023-08-28T02:12:37.653311 2023-08-28T02:12:38.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:38 smithi143 conmon[109923]: +0000 mon.a (mon.0) 754 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-08-28T02:12:38.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:38 smithi143 conmon[109923]: cluster 2023-08-28T02:12:37.889697+0000 mgr.y (mgr.14424) 113 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 12 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 367 KiB/s rd, 40 KiB/s wr, 478 op/s 2023-08-28T02:12:38.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:38 smithi169 conmon[100500]: cluster 2023-08-28T02:12 2023-08-28T02:12:38.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:38 smithi169 conmon[100500]: :37.653311+0000 mon.a (mon.0) 754 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-08-28T02:12:38.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:38 smithi169 conmon[100500]: cluster 2023-08-28T02 2023-08-28T02:12:38.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:38 smithi169 conmon[100500]: :12:37.889697+0000 mgr.y (mgr.14424) 113 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 12 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 367 KiB/s rd, 40 KiB/s wr, 478 op/s 2023-08-28T02:12:39.692 INFO:tasks.workunit.client.0.smithi143.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-08-28T02:12:39.696 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd info testimg_import@snap1 2023-08-28T02:12:39.696 INFO:tasks.workunit.client.0.smithi143.stderr:+ grep 'protected: False' 2023-08-28T02:12:39.735 INFO:tasks.workunit.client.0.smithi143.stdout: protected: False 2023-08-28T02:12:39.736 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd info testimg_import@snap2 2023-08-28T02:12:39.736 INFO:tasks.workunit.client.0.smithi143.stderr:+ grep 'protected: True' 2023-08-28T02:12:39.777 INFO:tasks.workunit.client.0.smithi143.stdout: protected: True 2023-08-28T02:12:39.777 INFO:tasks.workunit.client.0.smithi143.stderr:+ rm /tmp/rbd_import_export_164148/snap_protect 2023-08-28T02:12:39.778 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd snap unprotect testimg@snap2 2023-08-28T02:12:39.822 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd snap unprotect testimg_import@snap2 2023-08-28T02:12:39.869 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd snap purge testimg 2023-08-28T02:12:39.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:39 smithi143 conmon[100258]: cluster 2023-08-28T02:12:38.661081 2023-08-28T02:12:39.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:39 smithi143 conmon[100258]: +0000 mon.a (mon.0) 755 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-08-28T02:12:39.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:39 smithi143 conmon[109923]: cluster 2023-08-28T02:12:38.661081+0000 2023-08-28T02:12:39.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:39 smithi143 conmon[109923]: mon.a (mon.0) 755 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-08-28T02:12:39.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:39 smithi169 conmon[100500]: cluster 2023-08-28T02:12:38.661081+0000 mon.a (mon.0) 755 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-08-28T02:12:40.666 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:40 smithi143 conmon[100258]: cluster 2023-08-28T02:12:39.667643+0000 mon.a 2023-08-28T02:12:40.666 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:40 smithi143 conmon[100258]: (mon.0) 756 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2023-08-28T02:12:40.667 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:40 smithi143 conmon[100258]: cluster 2023-08-28T02:12:39.890027+0000 mgr.y (mgr.14424) 114 : cluster [DBG] 2023-08-28T02:12:40.667 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:40 smithi143 conmon[100258]: pgmap v100: 73 pgs: 73 active+clean; 12 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:12:40.667 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:40 smithi143 conmon[100258]: cluster 2023 2023-08-28T02:12:40.667 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:40 smithi143 conmon[100258]: -08-28T02:12:40.176827+0000 mon.a (mon.0) 757 : cluster 2023-08-28T02:12:40.667 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:40 smithi143 conmon[100258]: [DBG] osdmap e74: 8 total, 8 up, 8 in 2023-08-28T02:12:40.667 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:12:40 smithi143 conmon[100708]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:12:40] "GET /metrics HTTP/1.1" 200 192139 "" "Prometheus/2.33.4" 2023-08-28T02:12:40.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:40 smithi143 conmon[109923]: cluster 2023-08-28T02:12:39.667643+0000 mon.a (mon.0 2023-08-28T02:12:40.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:40 smithi143 conmon[109923]: ) 756 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2023-08-28T02:12:40.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:40 smithi143 conmon[109923]: cluster 2023-08-28T02:12:39.890027+0000 mgr.y (mgr.14424) 114 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 12 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:12:40.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:40 smithi143 conmon[109923]: cluster 2023-08-28T02:12:40.176827+0000 mon.a (mon.0) 757 : cluster [DBG] 2023-08-28T02:12:40.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:40 smithi143 conmon[109923]: osdmap e74: 8 total, 8 up, 8 in 2023-08-28T02:12:40.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:40 smithi169 conmon[100500]: cluster 2023-08-28T02:12:39.667643+0000 mon.a (mon.0) 756 : cluster [DBG] 2023-08-28T02:12:40.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:40 smithi169 conmon[100500]: osdmap e73: 8 total, 8 up, 8 in 2023-08-28T02:12:40.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:40 smithi169 conmon[100500]: cluster 2023-08-28T02:12:39.890027+0000 mgr.y (mgr.14424) 114 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 12 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:12:40.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:40 smithi169 conmon[100500]: cluster 2023-08-28T02:12:40.176827+0000 mon.a ( 2023-08-28T02:12:40.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:40 smithi169 conmon[100500]: mon.0) 757 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2023-08-28T02:12:41.195 INFO:tasks.workunit.client.0.smithi143.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-08-28T02:12:41.202 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd snap purge testimg_import 2023-08-28T02:12:42.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:42 smithi143 conmon[100258]: cluster 2023-08-28T02:12:41.190827+0000 mon.a (mon.0) 758 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2023-08-28T02:12:42.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:42 smithi143 conmon[109923]: cluster 2023-08-28T02:12:41.190827+0000 mon.a (mon.0) 758 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2023-08-28T02:12:42.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:42 smithi169 conmon[100500]: cluster 2023-08-28T02:12:41.190827+0000 mon.a (mon.0) 2023-08-28T02:12:42.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:42 smithi169 conmon[100500]: 758 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2023-08-28T02:12:43.192 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:12:42 smithi169 conmon[102807]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:12:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-28T02:12:43.206 INFO:tasks.workunit.client.0.smithi143.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-08-28T02:12:43.216 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd remove testimg 2023-08-28T02:12:43.327 INFO:tasks.workunit.client.0.smithi143.stderr: Removing image: 100% complete...done. 2023-08-28T02:12:43.330 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd remove testimg_import 2023-08-28T02:12:43.429 INFO:tasks.workunit.client.0.smithi143.stderr: Removing image: 100% complete...done. 2023-08-28T02:12:43.433 INFO:tasks.workunit.client.0.smithi143.stderr:+ tiered=0 2023-08-28T02:12:43.434 INFO:tasks.workunit.client.0.smithi143.stderr:+ ceph osd dump 2023-08-28T02:12:43.434 INFO:tasks.workunit.client.0.smithi143.stderr:+ grep '^pool' 2023-08-28T02:12:43.434 INFO:tasks.workunit.client.0.smithi143.stderr:+ grep ''\''rbd'\''' 2023-08-28T02:12:43.434 INFO:tasks.workunit.client.0.smithi143.stderr:+ grep tier 2023-08-28T02:12:43.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:43 smithi143 conmon[100258]: cluster 2023-08-28T02:12:41.890374+0000 mgr.y (mgr.14424) 115 : cluster [DBG] pgmap v103: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 239 KiB/s rd, 25 KiB/s wr, 322 op/s 2023-08-28T02:12:43.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:43 smithi143 conmon[100258]: cluster 2023-08-28 2023-08-28T02:12:43.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:43 smithi143 conmon[100258]: T02:12:42.194758+0000 mon.a (mon.0) 759 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2023-08-28T02:12:43.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:43 smithi143 conmon[109923]: cluster 2023-08-28T02:12:41.890374+0000 2023-08-28T02:12:43.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:43 smithi143 conmon[109923]: mgr.y (mgr.14424) 115 : cluster [DBG] pgmap v103: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 239 KiB/s rd, 25 KiB/s wr, 322 op/s 2023-08-28T02:12:43.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:43 smithi143 conmon[109923]: cluster 2023 2023-08-28T02:12:43.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:43 smithi143 conmon[109923]: -08-28T02:12:42.194758+0000 mon.a (mon.0) 759 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2023-08-28T02:12:43.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:43 smithi143 conmon[109923]: 2023-08-28T02:12:43.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:43 smithi169 conmon[100500]: cluster 2023-08-28T02:12:41.890374+0000 mgr.y (mgr.14424) 115 : cluster [DBG] pgmap v103: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 239 KiB/s rd, 25 KiB/s wr, 322 op/s 2023-08-28T02:12:43.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:43 smithi169 conmon[100500]: cluster 2023-08-28T02:12:42.194758+0000 mon.a (mon.0) 759 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2023-08-28T02:12:43.995 INFO:tasks.workunit.client.0.smithi143.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_164148/sparse1 2023-08-28T02:12:44.002 INFO:tasks.workunit.client.0.smithi143.stderr:1+0 records in 2023-08-28T02:12:44.003 INFO:tasks.workunit.client.0.smithi143.stderr:1+0 records out 2023-08-28T02:12:44.003 INFO:tasks.workunit.client.0.smithi143.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.0071177 s, 147 MB/s 2023-08-28T02:12:44.003 INFO:tasks.workunit.client.0.smithi143.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_164148/sparse2 2023-08-28T02:12:44.008 INFO:tasks.workunit.client.0.smithi143.stderr:1+0 records in 2023-08-28T02:12:44.008 INFO:tasks.workunit.client.0.smithi143.stderr:1+0 records out 2023-08-28T02:12:44.009 INFO:tasks.workunit.client.0.smithi143.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00497433 s, 211 MB/s 2023-08-28T02:12:44.009 INFO:tasks.workunit.client.0.smithi143.stderr:+ truncate /tmp/rbd_import_export_164148/sparse2 -s 2M 2023-08-28T02:12:44.022 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd rm sparse1 2023-08-28T02:12:44.065 INFO:tasks.workunit.client.0.smithi143.stderr: Removing image: 0% complete...failed. 2023-08-28T02:12:44.066 INFO:tasks.workunit.client.0.smithi143.stderr:rbd: delete error: (2) No such file or directory 2023-08-28T02:12:44.069 INFO:tasks.workunit.client.0.smithi143.stderr:+ true 2023-08-28T02:12:44.069 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_164148/sparse1 2023-08-28T02:12:44.147 INFO:tasks.workunit.client.0.smithi143.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-08-28T02:12:44.147 INFO:tasks.workunit.client.0.smithi143.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-08-28T02:12:44.147 INFO:tasks.workunit.client.0.smithi143.stderr:rbd: --order is deprecated, use --object-size 2023-08-28T02:12:44.150 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd ls -l 2023-08-28T02:12:44.150 INFO:tasks.workunit.client.0.smithi143.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-08-28T02:12:44.154 INFO:tasks.workunit.client.0.smithi143.stderr:+ grep sparse1 2023-08-28T02:12:44.192 INFO:tasks.workunit.client.0.smithi143.stdout:sparse1 2 MiB 2 2023-08-28T02:12:44.192 INFO:tasks.workunit.client.0.smithi143.stderr:++ objects sparse1 2023-08-28T02:12:44.193 INFO:tasks.workunit.client.0.smithi143.stderr:++ image=sparse1 2023-08-28T02:12:44.193 INFO:tasks.workunit.client.0.smithi143.stderr:+++ rbd info sparse1 2023-08-28T02:12:44.193 INFO:tasks.workunit.client.0.smithi143.stderr:+++ grep block_name_prefix 2023-08-28T02:12:44.193 INFO:tasks.workunit.client.0.smithi143.stderr:+++ awk '{print $NF;}' 2023-08-28T02:12:44.240 INFO:tasks.workunit.client.0.smithi143.stderr:++ prefix=rbd_data.3b4061d8dc70 2023-08-28T02:12:44.242 INFO:tasks.workunit.client.0.smithi143.stderr:+++ grep rbd_data.3b4061d8dc70 2023-08-28T02:12:44.242 INFO:tasks.workunit.client.0.smithi143.stderr:++++ get_image_data_pool sparse1 2023-08-28T02:12:44.242 INFO:tasks.workunit.client.0.smithi143.stderr:++++ image=sparse1 2023-08-28T02:12:44.242 INFO:tasks.workunit.client.0.smithi143.stderr:+++ sort -u 2023-08-28T02:12:44.242 INFO:tasks.workunit.client.0.smithi143.stderr:+++ sed -e 's/rbd_data.3b4061d8dc70\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-08-28T02:12:44.242 INFO:tasks.workunit.client.0.smithi143.stderr:+++++ rbd info sparse1 2023-08-28T02:12:44.242 INFO:tasks.workunit.client.0.smithi143.stderr:+++++ awk -F: '{ print $NF }' 2023-08-28T02:12:44.243 INFO:tasks.workunit.client.0.smithi143.stderr:+++++ grep 'data_pool: ' 2023-08-28T02:12:44.287 INFO:tasks.workunit.client.0.smithi143.stderr:++++ data_pool= 2023-08-28T02:12:44.287 INFO:tasks.workunit.client.0.smithi143.stderr:++++ '[' -z ']' 2023-08-28T02:12:44.287 INFO:tasks.workunit.client.0.smithi143.stderr:++++ data_pool=rbd 2023-08-28T02:12:44.287 INFO:tasks.workunit.client.0.smithi143.stderr:++++ echo rbd 2023-08-28T02:12:44.287 INFO:tasks.workunit.client.0.smithi143.stderr:+++ rados ls -p rbd 2023-08-28T02:12:44.319 INFO:tasks.workunit.client.0.smithi143.stderr:++ objects=1 2023-08-28T02:12:44.320 INFO:tasks.workunit.client.0.smithi143.stderr:++ echo 1 2023-08-28T02:12:44.320 INFO:tasks.workunit.client.0.smithi143.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-08-28T02:12:44.320 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd export sparse1 /tmp/rbd_import_export_164148/sparse1.out 2023-08-28T02:12:44.358 INFO:tasks.workunit.client.0.smithi143.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-08-28T02:12:44.362 INFO:tasks.workunit.client.0.smithi143.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_164148/sparse1 /tmp/rbd_import_export_164148/sparse1.out 2023-08-28T02:12:44.362 INFO:tasks.workunit.client.0.smithi143.stderr:+ cmp -l /tmp/rbd_import_export_164148/sparse1 /tmp/rbd_import_export_164148/sparse1.out 2023-08-28T02:12:44.364 INFO:tasks.workunit.client.0.smithi143.stderr:++ stat /tmp/rbd_import_export_164148/sparse1 --format %b 2023-08-28T02:12:44.365 INFO:tasks.workunit.client.0.smithi143.stderr:+ origsize=2048 2023-08-28T02:12:44.365 INFO:tasks.workunit.client.0.smithi143.stderr:++ stat /tmp/rbd_import_export_164148/sparse1.out --format %b 2023-08-28T02:12:44.366 INFO:tasks.workunit.client.0.smithi143.stderr:+ exportsize=2048 2023-08-28T02:12:44.367 INFO:tasks.workunit.client.0.smithi143.stderr:+ difference=0 2023-08-28T02:12:44.367 INFO:tasks.workunit.client.0.smithi143.stderr:+ difference=0 2023-08-28T02:12:44.367 INFO:tasks.workunit.client.0.smithi143.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-08-28T02:12:44.367 INFO:tasks.workunit.client.0.smithi143.stderr:+ rm /tmp/rbd_import_export_164148/sparse1.out 2023-08-28T02:12:44.367 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd rm sparse1 2023-08-28T02:12:44.456 INFO:tasks.workunit.client.0.smithi143.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-08-28T02:12:44.459 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd rm sparse2 2023-08-28T02:12:44.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:44 smithi169 conmon[100500]: cluster 2023-08-28T02:12:43.200478+0000 mon.a (mon.0) 760 : 2023-08-28T02:12:44.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:44 smithi169 conmon[100500]: cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2023-08-28T02:12:44.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:44 smithi169 conmon[100500]: audit 2023-08-28T02:12:43.815725+0000 mon.c (mon.2) 23 : audit [DBG] from='client.? 172.21.15.143:0/2067772242' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-08-28T02:12:44.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:44 smithi169 conmon[100500]: audit 2023 2023-08-28T02:12:44.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:44 smithi169 conmon[100500]: -08-28T02:12:43.978404+0000 mon.c (mon.2) 24 : audit [DBG] from='client.? 172.21.15.143:0/2067772242' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-08-28T02:12:44.495 INFO:tasks.workunit.client.0.smithi143.stderr: Removing image: 0% complete...failed. 2023-08-28T02:12:44.495 INFO:tasks.workunit.client.0.smithi143.stderr:rbd: delete error: (2) No such file or directory 2023-08-28T02:12:44.498 INFO:tasks.workunit.client.0.smithi143.stderr:+ true 2023-08-28T02:12:44.498 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_164148/sparse2 2023-08-28T02:12:44.558 INFO:tasks.workunit.client.0.smithi143.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-08-28T02:12:44.559 INFO:tasks.workunit.client.0.smithi143.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-08-28T02:12:44.559 INFO:tasks.workunit.client.0.smithi143.stderr:rbd: --order is deprecated, use --object-size 2023-08-28T02:12:44.563 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd ls -l 2023-08-28T02:12:44.563 INFO:tasks.workunit.client.0.smithi143.stderr:+ grep sparse2 2023-08-28T02:12:44.563 INFO:tasks.workunit.client.0.smithi143.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-08-28T02:12:44.599 INFO:tasks.workunit.client.0.smithi143.stdout:sparse2 2 MiB 2 2023-08-28T02:12:44.600 INFO:tasks.workunit.client.0.smithi143.stderr:++ objects sparse2 2023-08-28T02:12:44.600 INFO:tasks.workunit.client.0.smithi143.stderr:++ image=sparse2 2023-08-28T02:12:44.600 INFO:tasks.workunit.client.0.smithi143.stderr:+++ rbd info sparse2 2023-08-28T02:12:44.600 INFO:tasks.workunit.client.0.smithi143.stderr:+++ grep block_name_prefix 2023-08-28T02:12:44.600 INFO:tasks.workunit.client.0.smithi143.stderr:+++ awk '{print $NF;}' 2023-08-28T02:12:44.637 INFO:tasks.workunit.client.0.smithi143.stderr:++ prefix=rbd_data.3b6d9b8aea75 2023-08-28T02:12:44.638 INFO:tasks.workunit.client.0.smithi143.stderr:+++ grep rbd_data.3b6d9b8aea75 2023-08-28T02:12:44.638 INFO:tasks.workunit.client.0.smithi143.stderr:++++ get_image_data_pool sparse2 2023-08-28T02:12:44.638 INFO:tasks.workunit.client.0.smithi143.stderr:+++ sed -e 's/rbd_data.3b6d9b8aea75\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-08-28T02:12:44.638 INFO:tasks.workunit.client.0.smithi143.stderr:++++ image=sparse2 2023-08-28T02:12:44.638 INFO:tasks.workunit.client.0.smithi143.stderr:+++ sort -u 2023-08-28T02:12:44.639 INFO:tasks.workunit.client.0.smithi143.stderr:+++++ rbd info sparse2 2023-08-28T02:12:44.639 INFO:tasks.workunit.client.0.smithi143.stderr:+++++ grep 'data_pool: ' 2023-08-28T02:12:44.639 INFO:tasks.workunit.client.0.smithi143.stderr:+++++ awk -F: '{ print $NF }' 2023-08-28T02:12:44.673 INFO:tasks.workunit.client.0.smithi143.stderr:++++ data_pool= 2023-08-28T02:12:44.674 INFO:tasks.workunit.client.0.smithi143.stderr:++++ '[' -z ']' 2023-08-28T02:12:44.674 INFO:tasks.workunit.client.0.smithi143.stderr:++++ data_pool=rbd 2023-08-28T02:12:44.674 INFO:tasks.workunit.client.0.smithi143.stderr:++++ echo rbd 2023-08-28T02:12:44.674 INFO:tasks.workunit.client.0.smithi143.stderr:+++ rados ls -p rbd 2023-08-28T02:12:44.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:44 smithi143 conmon[100258]: cluster 2023-08-28T02:12:43.200478+0000 mon.a (mon.0) 760 : 2023-08-28T02:12:44.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:44 smithi143 conmon[100258]: cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2023-08-28T02:12:44.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:44 smithi143 conmon[100258]: audit 2023-08-28T02:12:43.815725+0000 mon.c (mon.2) 23 : audit [DBG] from='client.? 172.21.15.143:0/2067772242' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-08-28T02:12:44.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:44 smithi143 conmon[100258]: audit 2023-08-28T02:12:43.978404+0000 mon.c (mon.2) 24 : audit [DBG] from='client.? 172.21.15.143:0/2067772242' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-08-28T02:12:44.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:44 smithi143 conmon[109923]: cluster 2023-08-28T02:12:43.200478 2023-08-28T02:12:44.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:44 smithi143 conmon[109923]: +0000 mon.a (mon.0) 760 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2023-08-28T02:12:44.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:44 smithi143 conmon[109923]: audit 2023-08-28T 2023-08-28T02:12:44.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:44 smithi143 conmon[109923]: 02:12:43.815725+0000 mon.c (mon.2) 23 : audit [DBG] from='client.? 172.21.15.143:0/2067772242' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-08-28T02:12:44.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:44 smithi143 conmon[109923]: audit 2023-08-28 2023-08-28T02:12:44.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:44 smithi143 conmon[109923]: T02:12:43.978404+0000 mon.c (mon.2 2023-08-28T02:12:44.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:44 smithi143 conmon[109923]: ) 24 : audit [DBG] from='client.? 172.21.15.143:0/2067772242' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-08-28T02:12:44.705 INFO:tasks.workunit.client.0.smithi143.stderr:++ objects=0 2023-08-28T02:12:44.705 INFO:tasks.workunit.client.0.smithi143.stderr:++ echo 0 2023-08-28T02:12:44.705 INFO:tasks.workunit.client.0.smithi143.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-08-28T02:12:44.705 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd export sparse2 /tmp/rbd_import_export_164148/sparse2.out 2023-08-28T02:12:44.743 INFO:tasks.workunit.client.0.smithi143.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-08-28T02:12:44.746 INFO:tasks.workunit.client.0.smithi143.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_164148/sparse2 /tmp/rbd_import_export_164148/sparse2.out 2023-08-28T02:12:44.746 INFO:tasks.workunit.client.0.smithi143.stderr:+ cmp -l /tmp/rbd_import_export_164148/sparse2 /tmp/rbd_import_export_164148/sparse2.out 2023-08-28T02:12:44.748 INFO:tasks.workunit.client.0.smithi143.stderr:++ stat /tmp/rbd_import_export_164148/sparse2 --format %b 2023-08-28T02:12:44.749 INFO:tasks.workunit.client.0.smithi143.stderr:+ origsize=2048 2023-08-28T02:12:44.749 INFO:tasks.workunit.client.0.smithi143.stderr:++ stat /tmp/rbd_import_export_164148/sparse2.out --format %b 2023-08-28T02:12:44.750 INFO:tasks.workunit.client.0.smithi143.stderr:+ exportsize=2048 2023-08-28T02:12:44.751 INFO:tasks.workunit.client.0.smithi143.stderr:+ difference=0 2023-08-28T02:12:44.751 INFO:tasks.workunit.client.0.smithi143.stderr:+ difference=0 2023-08-28T02:12:44.751 INFO:tasks.workunit.client.0.smithi143.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-08-28T02:12:44.752 INFO:tasks.workunit.client.0.smithi143.stderr:+ rm /tmp/rbd_import_export_164148/sparse2.out 2023-08-28T02:12:44.752 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd rm sparse2 2023-08-28T02:12:44.842 INFO:tasks.workunit.client.0.smithi143.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-08-28T02:12:44.845 INFO:tasks.workunit.client.0.smithi143.stderr:+ truncate /tmp/rbd_import_export_164148/sparse1 -s 10M 2023-08-28T02:12:44.846 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd import --new-format --order 20 - sparse1 2023-08-28T02:12:44.930 INFO:tasks.workunit.client.0.smithi143.stderr: Importing image: 100% complete...done. 2023-08-28T02:12:44.931 INFO:tasks.workunit.client.0.smithi143.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-08-28T02:12:44.931 INFO:tasks.workunit.client.0.smithi143.stderr:rbd: --order is deprecated, use --object-size 2023-08-28T02:12:44.933 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd ls -l 2023-08-28T02:12:44.933 INFO:tasks.workunit.client.0.smithi143.stderr:+ grep sparse1 2023-08-28T02:12:44.933 INFO:tasks.workunit.client.0.smithi143.stderr:+ grep -Ei '(10 MiB|10240k)' 2023-08-28T02:12:44.974 INFO:tasks.workunit.client.0.smithi143.stdout:sparse1 10 MiB 2 2023-08-28T02:12:44.975 INFO:tasks.workunit.client.0.smithi143.stderr:++ objects sparse1 2023-08-28T02:12:44.975 INFO:tasks.workunit.client.0.smithi143.stderr:++ image=sparse1 2023-08-28T02:12:44.975 INFO:tasks.workunit.client.0.smithi143.stderr:+++ rbd info sparse1 2023-08-28T02:12:44.975 INFO:tasks.workunit.client.0.smithi143.stderr:+++ grep block_name_prefix 2023-08-28T02:12:44.975 INFO:tasks.workunit.client.0.smithi143.stderr:+++ awk '{print $NF;}' 2023-08-28T02:12:45.017 INFO:tasks.workunit.client.0.smithi143.stderr:++ prefix=rbd_data.3b9a5e3e7577 2023-08-28T02:12:45.018 INFO:tasks.workunit.client.0.smithi143.stderr:+++ grep rbd_data.3b9a5e3e7577 2023-08-28T02:12:45.018 INFO:tasks.workunit.client.0.smithi143.stderr:++++ get_image_data_pool sparse1 2023-08-28T02:12:45.018 INFO:tasks.workunit.client.0.smithi143.stderr:++++ image=sparse1 2023-08-28T02:12:45.018 INFO:tasks.workunit.client.0.smithi143.stderr:+++ sed -e 's/rbd_data.3b9a5e3e7577\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-08-28T02:12:45.018 INFO:tasks.workunit.client.0.smithi143.stderr:+++ sort -u 2023-08-28T02:12:45.019 INFO:tasks.workunit.client.0.smithi143.stderr:+++++ rbd info sparse1 2023-08-28T02:12:45.020 INFO:tasks.workunit.client.0.smithi143.stderr:+++++ grep 'data_pool: ' 2023-08-28T02:12:45.020 INFO:tasks.workunit.client.0.smithi143.stderr:+++++ awk -F: '{ print $NF }' 2023-08-28T02:12:45.064 INFO:tasks.workunit.client.0.smithi143.stderr:++++ data_pool= 2023-08-28T02:12:45.065 INFO:tasks.workunit.client.0.smithi143.stderr:++++ '[' -z ']' 2023-08-28T02:12:45.065 INFO:tasks.workunit.client.0.smithi143.stderr:++++ data_pool=rbd 2023-08-28T02:12:45.065 INFO:tasks.workunit.client.0.smithi143.stderr:++++ echo rbd 2023-08-28T02:12:45.065 INFO:tasks.workunit.client.0.smithi143.stderr:+++ rados ls -p rbd 2023-08-28T02:12:45.098 INFO:tasks.workunit.client.0.smithi143.stderr:++ objects=1 2023-08-28T02:12:45.098 INFO:tasks.workunit.client.0.smithi143.stderr:++ echo 1 2023-08-28T02:12:45.098 INFO:tasks.workunit.client.0.smithi143.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-08-28T02:12:45.098 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd export sparse1 /tmp/rbd_import_export_164148/sparse1.out 2023-08-28T02:12:45.137 INFO:tasks.workunit.client.0.smithi143.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2023-08-28T02:12:45.141 INFO:tasks.workunit.client.0.smithi143.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_164148/sparse1 /tmp/rbd_import_export_164148/sparse1.out 2023-08-28T02:12:45.142 INFO:tasks.workunit.client.0.smithi143.stderr:+ cmp -l /tmp/rbd_import_export_164148/sparse1 /tmp/rbd_import_export_164148/sparse1.out 2023-08-28T02:12:45.148 INFO:tasks.workunit.client.0.smithi143.stderr:++ stat /tmp/rbd_import_export_164148/sparse1 --format %b 2023-08-28T02:12:45.149 INFO:tasks.workunit.client.0.smithi143.stderr:+ origsize=2048 2023-08-28T02:12:45.149 INFO:tasks.workunit.client.0.smithi143.stderr:++ stat /tmp/rbd_import_export_164148/sparse1.out --format %b 2023-08-28T02:12:45.150 INFO:tasks.workunit.client.0.smithi143.stderr:+ exportsize=2048 2023-08-28T02:12:45.150 INFO:tasks.workunit.client.0.smithi143.stderr:+ difference=0 2023-08-28T02:12:45.151 INFO:tasks.workunit.client.0.smithi143.stderr:+ difference=0 2023-08-28T02:12:45.151 INFO:tasks.workunit.client.0.smithi143.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-08-28T02:12:45.151 INFO:tasks.workunit.client.0.smithi143.stderr:+ rm /tmp/rbd_import_export_164148/sparse1.out 2023-08-28T02:12:45.151 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd rm sparse1 2023-08-28T02:12:45.249 INFO:tasks.workunit.client.0.smithi143.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-08-28T02:12:45.252 INFO:tasks.workunit.client.0.smithi143.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_164148/sparse2 oflag=append conv=notrunc 2023-08-28T02:12:45.264 INFO:tasks.workunit.client.0.smithi143.stderr:1+0 records in 2023-08-28T02:12:45.264 INFO:tasks.workunit.client.0.smithi143.stderr:1+0 records out 2023-08-28T02:12:45.264 INFO:tasks.workunit.client.0.smithi143.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.0108738 s, 193 MB/s 2023-08-28T02:12:45.264 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd import --new-format --order 20 - sparse2 2023-08-28T02:12:45.352 INFO:tasks.workunit.client.0.smithi143.stderr: Importing image: 100% complete...done. 2023-08-28T02:12:45.352 INFO:tasks.workunit.client.0.smithi143.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-08-28T02:12:45.352 INFO:tasks.workunit.client.0.smithi143.stderr:rbd: --order is deprecated, use --object-size 2023-08-28T02:12:45.354 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd ls -l 2023-08-28T02:12:45.355 INFO:tasks.workunit.client.0.smithi143.stderr:+ grep sparse2 2023-08-28T02:12:45.355 INFO:tasks.workunit.client.0.smithi143.stderr:+ grep -Ei '(4 MiB|4096k)' 2023-08-28T02:12:45.391 INFO:tasks.workunit.client.0.smithi143.stdout:sparse2 4 MiB 2 2023-08-28T02:12:45.393 INFO:tasks.workunit.client.0.smithi143.stderr:++ objects sparse2 2023-08-28T02:12:45.393 INFO:tasks.workunit.client.0.smithi143.stderr:++ image=sparse2 2023-08-28T02:12:45.393 INFO:tasks.workunit.client.0.smithi143.stderr:+++ rbd info sparse2 2023-08-28T02:12:45.394 INFO:tasks.workunit.client.0.smithi143.stderr:+++ grep block_name_prefix 2023-08-28T02:12:45.394 INFO:tasks.workunit.client.0.smithi143.stderr:+++ awk '{print $NF;}' 2023-08-28T02:12:45.428 INFO:tasks.workunit.client.0.smithi143.stderr:++ prefix=rbd_data.3bc4c65da574 2023-08-28T02:12:45.429 INFO:tasks.workunit.client.0.smithi143.stderr:+++ grep rbd_data.3bc4c65da574 2023-08-28T02:12:45.429 INFO:tasks.workunit.client.0.smithi143.stderr:++++ get_image_data_pool sparse2 2023-08-28T02:12:45.430 INFO:tasks.workunit.client.0.smithi143.stderr:++++ image=sparse2 2023-08-28T02:12:45.430 INFO:tasks.workunit.client.0.smithi143.stderr:+++ sed -e 's/rbd_data.3bc4c65da574\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-08-28T02:12:45.430 INFO:tasks.workunit.client.0.smithi143.stderr:+++ sort -u 2023-08-28T02:12:45.430 INFO:tasks.workunit.client.0.smithi143.stderr:+++++ rbd info sparse2 2023-08-28T02:12:45.431 INFO:tasks.workunit.client.0.smithi143.stderr:+++++ awk -F: '{ print $NF }' 2023-08-28T02:12:45.432 INFO:tasks.workunit.client.0.smithi143.stderr:+++++ grep 'data_pool: ' 2023-08-28T02:12:45.466 INFO:tasks.workunit.client.0.smithi143.stderr:++++ data_pool= 2023-08-28T02:12:45.466 INFO:tasks.workunit.client.0.smithi143.stderr:++++ '[' -z ']' 2023-08-28T02:12:45.467 INFO:tasks.workunit.client.0.smithi143.stderr:++++ data_pool=rbd 2023-08-28T02:12:45.467 INFO:tasks.workunit.client.0.smithi143.stderr:++++ echo rbd 2023-08-28T02:12:45.467 INFO:tasks.workunit.client.0.smithi143.stderr:+++ rados ls -p rbd 2023-08-28T02:12:45.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:45 smithi169 conmon[100500]: cluster 2023-08-28T02:12:43.890714+0000 mgr.y (mgr 2023-08-28T02:12:45.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:45 smithi169 conmon[100500]: .14424) 116 : cluster [DBG] pgmap v106: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 239 KiB/s rd, 25 KiB/s wr, 322 op/s 2023-08-28T02:12:45.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:45 smithi169 conmon[100500]: cluster 2023-08-28T02:12:45.176700+0000 mon.a (mon.0) 761 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2023-08-28T02:12:45.498 INFO:tasks.workunit.client.0.smithi143.stderr:++ objects='0 2023-08-28T02:12:45.499 INFO:tasks.workunit.client.0.smithi143.stderr:2 2023-08-28T02:12:45.499 INFO:tasks.workunit.client.0.smithi143.stderr:3' 2023-08-28T02:12:45.499 INFO:tasks.workunit.client.0.smithi143.stderr:++ echo 0 2 3 2023-08-28T02:12:45.499 INFO:tasks.workunit.client.0.smithi143.stderr:+ '[' 0 -eq 1 -o '0 2 3' = '0 2 3' ']' 2023-08-28T02:12:45.499 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd export sparse2 /tmp/rbd_import_export_164148/sparse2.out 2023-08-28T02:12:45.540 INFO:tasks.workunit.client.0.smithi143.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2023-08-28T02:12:45.543 INFO:tasks.workunit.client.0.smithi143.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_164148/sparse2 /tmp/rbd_import_export_164148/sparse2.out 2023-08-28T02:12:45.543 INFO:tasks.workunit.client.0.smithi143.stderr:+ cmp -l /tmp/rbd_import_export_164148/sparse2 /tmp/rbd_import_export_164148/sparse2.out 2023-08-28T02:12:45.546 INFO:tasks.workunit.client.0.smithi143.stderr:++ stat /tmp/rbd_import_export_164148/sparse2 --format %b 2023-08-28T02:12:45.547 INFO:tasks.workunit.client.0.smithi143.stderr:+ origsize=6144 2023-08-28T02:12:45.548 INFO:tasks.workunit.client.0.smithi143.stderr:++ stat /tmp/rbd_import_export_164148/sparse2.out --format %b 2023-08-28T02:12:45.549 INFO:tasks.workunit.client.0.smithi143.stderr:+ exportsize=6144 2023-08-28T02:12:45.549 INFO:tasks.workunit.client.0.smithi143.stderr:+ difference=0 2023-08-28T02:12:45.549 INFO:tasks.workunit.client.0.smithi143.stderr:+ difference=0 2023-08-28T02:12:45.549 INFO:tasks.workunit.client.0.smithi143.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-08-28T02:12:45.549 INFO:tasks.workunit.client.0.smithi143.stderr:+ rm /tmp/rbd_import_export_164148/sparse2.out 2023-08-28T02:12:45.550 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd rm sparse2 2023-08-28T02:12:45.642 INFO:tasks.workunit.client.0.smithi143.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2023-08-28T02:12:45.645 INFO:tasks.workunit.client.0.smithi143.stderr:+ echo 'partially-sparse file imports to partially-sparse image' 2023-08-28T02:12:45.646 INFO:tasks.workunit.client.0.smithi143.stdout:partially-sparse file imports to partially-sparse image 2023-08-28T02:12:45.646 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_164148/sparse1 sparse 2023-08-28T02:12:45.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:45 smithi143 conmon[100258]: cluster 2023-08-28T02:12:43.890714+0000 mgr.y (mgr.14424) 116 : cluster 2023-08-28T02:12:45.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:45 smithi143 conmon[100258]: [DBG] pgmap v106: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 239 KiB/s rd, 25 KiB/s wr, 322 op/s 2023-08-28T02:12:45.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:45 smithi143 conmon[100258]: cluster 2023-08-28T02:12:45.176700+0000 mon.a (mon.0) 761 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2023-08-28T02:12:45.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:45 smithi143 conmon[109923]: cluster 2023-08-28T02:12:43.890714+0000 2023-08-28T02:12:45.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:45 smithi143 conmon[109923]: mgr.y (mgr.14424) 116 : cluster [DBG] pgmap v106: 73 pgs: 73 active+clean; 222 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 239 KiB/s rd, 25 KiB/s wr, 322 op/s 2023-08-28T02:12:45.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:45 smithi143 conmon[109923]: cluster 2023-08-28T02:12:45.176700+0000 mon.a (mon 2023-08-28T02:12:45.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:45 smithi143 conmon[109923]: .0) 761 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2023-08-28T02:12:45.710 INFO:tasks.workunit.client.0.smithi143.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2023-08-28T02:12:45.711 INFO:tasks.workunit.client.0.smithi143.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-08-28T02:12:45.711 INFO:tasks.workunit.client.0.smithi143.stderr:rbd: --order is deprecated, use --object-size 2023-08-28T02:12:45.713 INFO:tasks.workunit.client.0.smithi143.stderr:++ objects sparse 2023-08-28T02:12:45.713 INFO:tasks.workunit.client.0.smithi143.stderr:++ image=sparse 2023-08-28T02:12:45.713 INFO:tasks.workunit.client.0.smithi143.stderr:+++ rbd info sparse 2023-08-28T02:12:45.713 INFO:tasks.workunit.client.0.smithi143.stderr:+++ grep block_name_prefix 2023-08-28T02:12:45.714 INFO:tasks.workunit.client.0.smithi143.stderr:+++ awk '{print $NF;}' 2023-08-28T02:12:45.750 INFO:tasks.workunit.client.0.smithi143.stderr:++ prefix=rbd_data.3bf1262d550a 2023-08-28T02:12:45.751 INFO:tasks.workunit.client.0.smithi143.stderr:+++ grep rbd_data.3bf1262d550a 2023-08-28T02:12:45.751 INFO:tasks.workunit.client.0.smithi143.stderr:++++ get_image_data_pool sparse 2023-08-28T02:12:45.751 INFO:tasks.workunit.client.0.smithi143.stderr:++++ image=sparse 2023-08-28T02:12:45.751 INFO:tasks.workunit.client.0.smithi143.stderr:+++ sed -e 's/rbd_data.3bf1262d550a\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-08-28T02:12:45.751 INFO:tasks.workunit.client.0.smithi143.stderr:+++ sort -u 2023-08-28T02:12:45.753 INFO:tasks.workunit.client.0.smithi143.stderr:+++++ rbd info sparse 2023-08-28T02:12:45.753 INFO:tasks.workunit.client.0.smithi143.stderr:+++++ grep 'data_pool: ' 2023-08-28T02:12:45.753 INFO:tasks.workunit.client.0.smithi143.stderr:+++++ awk -F: '{ print $NF }' 2023-08-28T02:12:45.790 INFO:tasks.workunit.client.0.smithi143.stderr:++++ data_pool= 2023-08-28T02:12:45.790 INFO:tasks.workunit.client.0.smithi143.stderr:++++ '[' -z ']' 2023-08-28T02:12:45.790 INFO:tasks.workunit.client.0.smithi143.stderr:++++ data_pool=rbd 2023-08-28T02:12:45.790 INFO:tasks.workunit.client.0.smithi143.stderr:++++ echo rbd 2023-08-28T02:12:45.790 INFO:tasks.workunit.client.0.smithi143.stderr:+++ rados ls -p rbd 2023-08-28T02:12:45.825 INFO:tasks.workunit.client.0.smithi143.stderr:++ objects=1 2023-08-28T02:12:45.825 INFO:tasks.workunit.client.0.smithi143.stderr:++ echo 1 2023-08-28T02:12:45.825 INFO:tasks.workunit.client.0.smithi143.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-08-28T02:12:45.825 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd rm sparse 2023-08-28T02:12:45.921 INFO:tasks.workunit.client.0.smithi143.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-08-28T02:12:45.923 INFO:tasks.workunit.client.0.smithi143.stdout:zeros import through stdin to sparse image 2023-08-28T02:12:45.924 INFO:tasks.workunit.client.0.smithi143.stderr:+ echo 'zeros import through stdin to sparse image' 2023-08-28T02:12:45.924 INFO:tasks.workunit.client.0.smithi143.stderr:+ dd if=/dev/zero bs=1M count=4 2023-08-28T02:12:45.924 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd import --new-format - sparse 2023-08-28T02:12:45.973 INFO:tasks.workunit.client.0.smithi143.stderr:4+0 records in 2023-08-28T02:12:45.974 INFO:tasks.workunit.client.0.smithi143.stderr:4+0 records out 2023-08-28T02:12:45.974 INFO:tasks.workunit.client.0.smithi143.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0490412 s, 85.5 MB/s 2023-08-28T02:12:45.985 INFO:tasks.workunit.client.0.smithi143.stderr: Importing image: 100% complete...done. 2023-08-28T02:12:45.985 INFO:tasks.workunit.client.0.smithi143.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-08-28T02:12:45.987 INFO:tasks.workunit.client.0.smithi143.stderr:++ objects sparse 2023-08-28T02:12:45.988 INFO:tasks.workunit.client.0.smithi143.stderr:++ image=sparse 2023-08-28T02:12:45.988 INFO:tasks.workunit.client.0.smithi143.stderr:+++ rbd info sparse 2023-08-28T02:12:45.988 INFO:tasks.workunit.client.0.smithi143.stderr:+++ grep block_name_prefix 2023-08-28T02:12:45.989 INFO:tasks.workunit.client.0.smithi143.stderr:+++ awk '{print $NF;}' 2023-08-28T02:12:46.026 INFO:tasks.workunit.client.0.smithi143.stderr:++ prefix=rbd_data.62cf158e31bc 2023-08-28T02:12:46.026 INFO:tasks.workunit.client.0.smithi143.stderr:+++ grep rbd_data.62cf158e31bc 2023-08-28T02:12:46.027 INFO:tasks.workunit.client.0.smithi143.stderr:++++ get_image_data_pool sparse 2023-08-28T02:12:46.027 INFO:tasks.workunit.client.0.smithi143.stderr:+++ sed -e 's/rbd_data.62cf158e31bc\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-08-28T02:12:46.027 INFO:tasks.workunit.client.0.smithi143.stderr:++++ image=sparse 2023-08-28T02:12:46.027 INFO:tasks.workunit.client.0.smithi143.stderr:+++ sort -u 2023-08-28T02:12:46.028 INFO:tasks.workunit.client.0.smithi143.stderr:+++++ rbd info sparse 2023-08-28T02:12:46.028 INFO:tasks.workunit.client.0.smithi143.stderr:+++++ grep 'data_pool: ' 2023-08-28T02:12:46.028 INFO:tasks.workunit.client.0.smithi143.stderr:+++++ awk -F: '{ print $NF }' 2023-08-28T02:12:46.064 INFO:tasks.workunit.client.0.smithi143.stderr:++++ data_pool= 2023-08-28T02:12:46.064 INFO:tasks.workunit.client.0.smithi143.stderr:++++ '[' -z ']' 2023-08-28T02:12:46.064 INFO:tasks.workunit.client.0.smithi143.stderr:++++ data_pool=rbd 2023-08-28T02:12:46.065 INFO:tasks.workunit.client.0.smithi143.stderr:++++ echo rbd 2023-08-28T02:12:46.065 INFO:tasks.workunit.client.0.smithi143.stderr:+++ rados ls -p rbd 2023-08-28T02:12:46.097 INFO:tasks.workunit.client.0.smithi143.stderr:++ objects= 2023-08-28T02:12:46.098 INFO:tasks.workunit.client.0.smithi143.stderr:++ echo 2023-08-28T02:12:46.098 INFO:tasks.workunit.client.0.smithi143.stderr:+ '[' 0 -eq 1 -o '' = '' ']' 2023-08-28T02:12:46.098 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd rm sparse 2023-08-28T02:12:46.184 INFO:tasks.workunit.client.0.smithi143.stderr: Removing image: 100% complete...done. 2023-08-28T02:12:46.187 INFO:tasks.workunit.client.0.smithi143.stdout:zeros export to sparse file 2023-08-28T02:12:46.188 INFO:tasks.workunit.client.0.smithi143.stderr:+ echo 'zeros export to sparse file' 2023-08-28T02:12:46.188 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd create --new-format sparse --size 4 2023-08-28T02:12:46.233 INFO:tasks.workunit.client.0.smithi143.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-08-28T02:12:46.235 INFO:tasks.workunit.client.0.smithi143.stderr:++ rbd info sparse 2023-08-28T02:12:46.236 INFO:tasks.workunit.client.0.smithi143.stderr:++ grep block_name_prefix 2023-08-28T02:12:46.236 INFO:tasks.workunit.client.0.smithi143.stderr:++ awk '{print $NF;}' 2023-08-28T02:12:46.271 INFO:tasks.workunit.client.0.smithi143.stderr:+ prefix=rbd_data.3c332c051b7c 2023-08-28T02:12:46.271 INFO:tasks.workunit.client.0.smithi143.stderr:+ dd if=/dev/zero bs=4M count=1 2023-08-28T02:12:46.271 INFO:tasks.workunit.client.0.smithi143.stderr:++ get_image_data_pool sparse 2023-08-28T02:12:46.272 INFO:tasks.workunit.client.0.smithi143.stderr:++ image=sparse 2023-08-28T02:12:46.272 INFO:tasks.workunit.client.0.smithi143.stderr:+++ rbd info sparse 2023-08-28T02:12:46.272 INFO:tasks.workunit.client.0.smithi143.stderr:+++ grep 'data_pool: ' 2023-08-28T02:12:46.272 INFO:tasks.workunit.client.0.smithi143.stderr:+++ awk -F: '{ print $NF }' 2023-08-28T02:12:46.307 INFO:tasks.workunit.client.0.smithi143.stderr:++ data_pool= 2023-08-28T02:12:46.307 INFO:tasks.workunit.client.0.smithi143.stderr:++ '[' -z ']' 2023-08-28T02:12:46.307 INFO:tasks.workunit.client.0.smithi143.stderr:++ data_pool=rbd 2023-08-28T02:12:46.307 INFO:tasks.workunit.client.0.smithi143.stderr:++ echo rbd 2023-08-28T02:12:46.307 INFO:tasks.workunit.client.0.smithi143.stderr:+ rados -p rbd put rbd_data.3c332c051b7c.000000000000 - 2023-08-28T02:12:46.337 INFO:tasks.workunit.client.0.smithi143.stderr:1+0 records in 2023-08-28T02:12:46.338 INFO:tasks.workunit.client.0.smithi143.stderr:1+0 records out 2023-08-28T02:12:46.338 INFO:tasks.workunit.client.0.smithi143.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0657162 s, 63.8 MB/s 2023-08-28T02:12:46.365 INFO:tasks.workunit.client.0.smithi143.stderr:++ objects sparse 2023-08-28T02:12:46.365 INFO:tasks.workunit.client.0.smithi143.stderr:++ image=sparse 2023-08-28T02:12:46.365 INFO:tasks.workunit.client.0.smithi143.stderr:+++ rbd info sparse 2023-08-28T02:12:46.366 INFO:tasks.workunit.client.0.smithi143.stderr:+++ grep block_name_prefix 2023-08-28T02:12:46.366 INFO:tasks.workunit.client.0.smithi143.stderr:+++ awk '{print $NF;}' 2023-08-28T02:12:46.403 INFO:tasks.workunit.client.0.smithi143.stderr:++ prefix=rbd_data.3c332c051b7c 2023-08-28T02:12:46.404 INFO:tasks.workunit.client.0.smithi143.stderr:+++ grep rbd_data.3c332c051b7c 2023-08-28T02:12:46.404 INFO:tasks.workunit.client.0.smithi143.stderr:++++ get_image_data_pool sparse 2023-08-28T02:12:46.404 INFO:tasks.workunit.client.0.smithi143.stderr:++++ image=sparse 2023-08-28T02:12:46.404 INFO:tasks.workunit.client.0.smithi143.stderr:+++ sed -e 's/rbd_data.3c332c051b7c\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-08-28T02:12:46.405 INFO:tasks.workunit.client.0.smithi143.stderr:+++ sort -u 2023-08-28T02:12:46.405 INFO:tasks.workunit.client.0.smithi143.stderr:+++++ rbd info sparse 2023-08-28T02:12:46.405 INFO:tasks.workunit.client.0.smithi143.stderr:+++++ awk -F: '{ print $NF }' 2023-08-28T02:12:46.405 INFO:tasks.workunit.client.0.smithi143.stderr:+++++ grep 'data_pool: ' 2023-08-28T02:12:46.441 INFO:tasks.workunit.client.0.smithi143.stderr:++++ data_pool= 2023-08-28T02:12:46.441 INFO:tasks.workunit.client.0.smithi143.stderr:++++ '[' -z ']' 2023-08-28T02:12:46.441 INFO:tasks.workunit.client.0.smithi143.stderr:++++ data_pool=rbd 2023-08-28T02:12:46.442 INFO:tasks.workunit.client.0.smithi143.stderr:++++ echo rbd 2023-08-28T02:12:46.442 INFO:tasks.workunit.client.0.smithi143.stderr:+++ rados ls -p rbd 2023-08-28T02:12:46.477 INFO:tasks.workunit.client.0.smithi143.stderr:++ objects=0 2023-08-28T02:12:46.477 INFO:tasks.workunit.client.0.smithi143.stderr:++ echo 0 2023-08-28T02:12:46.477 INFO:tasks.workunit.client.0.smithi143.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-08-28T02:12:46.477 INFO:tasks.workunit.client.0.smithi143.stderr:+ rm /tmp/rbd_import_export_164148/sparse 2023-08-28T02:12:46.478 INFO:tasks.workunit.client.0.smithi143.stderr:rm: cannot remove '/tmp/rbd_import_export_164148/sparse': No such file or directory 2023-08-28T02:12:46.478 INFO:tasks.workunit.client.0.smithi143.stderr:+ true 2023-08-28T02:12:46.478 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd export sparse /tmp/rbd_import_export_164148/sparse 2023-08-28T02:12:46.522 INFO:tasks.workunit.client.0.smithi143.stderr: Exporting image: 100% complete...done. 2023-08-28T02:12:46.526 INFO:tasks.workunit.client.0.smithi143.stderr:++ stat /tmp/rbd_import_export_164148/sparse --format=%b 2023-08-28T02:12:46.527 INFO:tasks.workunit.client.0.smithi143.stderr:+ '[' 0 = 0 ']' 2023-08-28T02:12:46.527 INFO:tasks.workunit.client.0.smithi143.stderr:+ rbd rm sparse 2023-08-28T02:12:46.620 INFO:tasks.workunit.client.0.smithi143.stderr: Removing image: 100% complete...done. 2023-08-28T02:12:46.623 INFO:tasks.workunit.client.0.smithi143.stderr:+ rm /tmp/rbd_import_export_164148/sparse /tmp/rbd_import_export_164148/sparse1 /tmp/rbd_import_export_164148/sparse2 /tmp/rbd_import_export_164148/sparse3 2023-08-28T02:12:46.625 INFO:tasks.workunit.client.0.smithi143.stderr:rm: cannot remove '/tmp/rbd_import_export_164148/sparse3': No such file or directory 2023-08-28T02:12:46.625 INFO:tasks.workunit.client.0.smithi143.stderr:+ true 2023-08-28T02:12:46.626 INFO:tasks.workunit.client.0.smithi143.stdout:OK 2023-08-28T02:12:46.627 INFO:tasks.workunit.client.0.smithi143.stderr:+ echo OK 2023-08-28T02:12:46.627 INFO:tasks.workunit.client.0.smithi143.stderr:+ rm -rf /tmp/rbd_import_export_164148 2023-08-28T02:12:46.627 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-08-28T02:12:46.627 DEBUG:teuthology.orchestra.run.smithi143:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-08-28T02:12:46.661 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.0... 2023-08-28T02:12:46.661 DEBUG:teuthology.orchestra.run.smithi143:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-08-28T02:12:46.950 DEBUG:teuthology.parallel:result is None 2023-08-28T02:12:46.951 DEBUG:teuthology.orchestra.run.smithi143:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-08-28T02:12:46.979 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-08-28T02:12:46.980 DEBUG:teuthology.orchestra.run.smithi143:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-08-28T02:12:47.037 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-08-28T02:12:47.037 INFO:teuthology.task.sequential:In sequential, running task print... 2023-08-28T02:12:47.037 INFO:teuthology.task.print:**** done rbd/import_export.sh 5-workload 2023-08-28T02:12:47.037 DEBUG:teuthology.parallel:result is None 2023-08-28T02:12:47.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:47 smithi169 conmon[100500]: cluster 2023-08-28T02:12:45.891305+0000 mgr.y (mgr.14424) 117 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 1.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.5 MiB/s wr, 439 op/s 2023-08-28T02:12:47.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:47 smithi143 conmon[109923]: cluster 2023-08-28T02:12:45.891305 2023-08-28T02:12:47.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:47 smithi143 conmon[109923]: +0000 mgr.y (mgr.14424) 117 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 1.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.5 MiB/s wr, 439 op/s 2023-08-28T02:12:47.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:47 smithi143 conmon[100258]: cluster 2023-08-28T02:12:45.891305+0000 mgr.y (mgr.14424) 117 : cluster [DBG] 2023-08-28T02:12:47.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:47 smithi143 conmon[100258]: pgmap v108: 73 pgs: 73 active+clean; 1.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.5 MiB/s wr, 439 op/s 2023-08-28T02:12:49.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:49 smithi169 conmon[100500]: cluster 2023-08-28T02:12:47.891635+0000 mgr.y (mgr.14424) 118 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 1.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.2 MiB/s wr, 344 op/s 2023-08-28T02:12:49.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:49 smithi143 conmon[100258]: cluster 2023-08-28T02:12:47.891635+0000 mgr.y (mgr.14424) 118 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 1.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.2 MiB/s wr, 344 op/s 2023-08-28T02:12:49.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:49 smithi143 conmon[109923]: cluster 2023-08-28T02:12:47.891635+0000 mgr.y (mgr.14424) 118 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 1.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.2 MiB/s wr, 344 op/s 2023-08-28T02:12:50.343 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:12:50 smithi143 conmon[100708]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:12:50] "GET /metrics HTTP/1.1" 200 192139 "" "Prometheus/2.33.4" 2023-08-28T02:12:51.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:51 smithi143 conmon[100258]: cluster 2023-08-28T02:12:49.892057+0000 2023-08-28T02:12:51.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:51 smithi143 conmon[100258]: mgr.y (mgr.14424) 119 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 1.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 998 KiB/s rd, 944 KiB/s wr, 268 op/s 2023-08-28T02:12:51.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:51 smithi143 conmon[100258]: cluster 2023-08-28T02:12:50.177249+0000 mon.a (mon.0) 762 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2023-08-28T02:12:51.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:51 smithi143 conmon[100258]: audit 2023-08-28T02:12:50.242775+0000 mon.c (mon.2) 25 : audit [DBG] from='client.? 172.21.15.143:0/117591133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:51.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:51 smithi143 conmon[109923]: cluster 2023-08-28T02:12:49.892057+0000 2023-08-28T02:12:51.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:51 smithi143 conmon[109923]: mgr.y (mgr.14424) 119 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 1.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 998 KiB/s rd, 944 KiB/s wr, 268 op/s 2023-08-28T02:12:51.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:51 smithi143 conmon[109923]: cluster 2023- 2023-08-28T02:12:51.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:51 smithi143 conmon[109923]: 08-28T02:12:50.177249+0000 mon.a (mon.0) 762 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2023-08-28T02:12:51.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:51 smithi143 conmon[109923]: audit 2023-08- 2023-08-28T02:12:51.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:51 smithi143 conmon[109923]: 28T02:12:50.242775+0000 mon.c (mon.2) 2023-08-28T02:12:51.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:51 smithi143 conmon[109923]: 25 : audit [DBG] from='client.? 172.21.15.143:0/117591133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:51.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:51 smithi169 conmon[100500]: cluster 2023-08-28T02:12:49.892057+0000 mgr.y (mgr.14424) 119 : cluster [DBG] 2023-08-28T02:12:51.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:51 smithi169 conmon[100500]: pgmap v110: 73 pgs: 73 active+clean; 1.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 998 KiB/s rd, 944 KiB/s wr, 268 op/s 2023-08-28T02:12:51.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:51 smithi169 conmon[100500]: cluster 2023-08-28T02:12:50.177249+0000 mon.a (mon.0) 762 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2023-08-28T02:12:51.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:51 smithi169 conmon[100500]: audit 2023-08-28T02:12: 2023-08-28T02:12:51.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:51 smithi169 conmon[100500]: 50.242775+0000 mon.c (mon.2) 25 : audit [DBG] from='client.? 172.21.15.143:0/117591133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:53.237 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:12:52 smithi169 conmon[102807]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:12:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-28T02:12:53.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:53 smithi169 conmon[100500]: cluster 2023-08-28T02:12:51.892725+0000 mgr.y (mgr.14424) 120 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1021 KiB/s rd, 1.4 MiB/s wr, 339 op/s 2023-08-28T02:12:53.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:53 smithi169 conmon[100500]: audit 2023-08-28T02:12:52.600155+0000 mon.c (mon.2) 26 : audit [DBG] from='client.? 172.21.15.143:0/1040267505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:53.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:53 smithi143 conmon[100258]: cluster 2023-08-28T02:12:51.892725+0000 mgr.y (mgr.14424) 120 : cluster [DBG] 2023-08-28T02:12:53.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:53 smithi143 conmon[100258]: pgmap v112: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1021 KiB/s rd, 1.4 MiB/s wr, 339 op/s 2023-08-28T02:12:53.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:53 smithi143 conmon[100258]: audit 2023-08-28T02:12:52.600155+0000 mon.c (mon.2) 26 : audit [DBG] from='client.? 172.21.15.143:0/1040267505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:53.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:53 smithi143 conmon[109923]: cluster 2023-08-28T02:12:51.892725+0000 mgr.y (mgr.14424) 120 2023-08-28T02:12:53.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:53 smithi143 conmon[109923]: : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1021 KiB/s rd, 1.4 MiB/s wr, 339 op/s 2023-08-28T02:12:53.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:53 smithi143 conmon[109923]: audit 2023-08-28T02:12:52.600155+0000 mon.c (mon.2) 26 : audit [DBG] from='client.? 172.21.15.143:0/1040267505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:55.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:55 smithi143 conmon[100258]: cluster 2023-08-28T02:12:53.893169+0000 mgr.y (mgr.14424 2023-08-28T02:12:55.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:55 smithi143 conmon[100258]: ) 121 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 937 KiB/s rd, 1.3 MiB/s wr, 311 op/s 2023-08-28T02:12:55.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:55 smithi143 conmon[100258]: audit 2023-08-28T02:12:54.945808+0000 mon.a (mon.0) 763 : audit [DBG] from='client.? 172.21.15.143:0/1187438181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:55.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:55 smithi143 conmon[109923]: cluster 2023-08-28T02:12:53. 2023-08-28T02:12:55.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:55 smithi143 conmon[109923]: 893169+0000 mgr.y (mgr.14424) 121 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 937 KiB/s rd, 1.3 MiB/s wr, 311 op/s 2023-08-28T02:12:55.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:55 smithi143 conmon[109923]: audit 2023-08-28T02:12:55.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:55 smithi143 conmon[109923]: 2023-08-28T02:12:54.945808+0000 mon.a (mon.0) 763 : audit [DBG] from='client.? 172.21.15.143:0/1187438181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:55.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:55 smithi143 conmon[109923]: 2023-08-28T02:12:55.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:55 smithi169 conmon[100500]: cluster 2023-08-28T02:12:53.893169+0000 mgr.y (mgr.14424) 121 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 937 KiB/s rd, 1.3 MiB/s wr, 311 op/s 2023-08-28T02:12:55.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:55 smithi169 conmon[100500]: audit 2023-08-28T02:12:54.945808+0000 mon.a (mon.0) 763 : audit [DBG] from='client.? 172.21.15.143:0/1187438181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:57.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:57 smithi143 conmon[100258]: cluster 2023-08-28T02:12:55.893651+0000 mgr.y (mgr 2023-08-28T02:12:57.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:57 smithi143 conmon[100258]: .14424) 122 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s rd, 412 KiB/s wr, 64 op/s 2023-08-28T02:12:57.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:57 smithi143 conmon[109923]: cluster 2023-08-28T02:12:55.893651+0000 2023-08-28T02:12:57.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:57 smithi143 conmon[109923]: mgr.y (mgr.14424) 122 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s rd, 412 KiB/s wr, 64 op/s 2023-08-28T02:12:57.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:57 smithi169 conmon[100500]: cluster 2023-08-28T02:12:55.893651+0000 mgr.y (mgr.14424) 122 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s rd, 412 KiB/s wr, 64 op/s 2023-08-28T02:12:58.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:58 smithi143 conmon[100258]: audit 2023-08-28T02:12:57.298495+0000 mon.c (mon.2) 27 : audit [DBG] 2023-08-28T02:12:58.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:58 smithi143 conmon[100258]: from='client.? 172.21.15.143:0/1522608809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:58.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:58 smithi143 conmon[109923]: audit 2023-08-28T02:12:57.298495+0000 mon.c (mon.2) 2023-08-28T02:12:58.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:58 smithi143 conmon[109923]: 27 : audit [DBG] from='client.? 172.21.15.143:0/1522608809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:58.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:58 smithi169 conmon[100500]: audit 2023-08-28T02:12:57.298495+0000 mon.c (mon.2) 27 : audit [DBG] 2023-08-28T02:12:58.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:58 smithi169 conmon[100500]: from='client.? 172.21.15.143:0/1522608809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:59.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:59 smithi143 conmon[100258]: cluster 2023-08-28T02:12: 2023-08-28T02:12:59.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:12:59 smithi143 conmon[100258]: 57.894212+0000 mgr.y (mgr.14424) 123 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s rd, 412 KiB/s wr, 64 op/s 2023-08-28T02:12:59.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:59 smithi143 conmon[109923]: cluster 2023-08-28T02:12:57.894212+0000 2023-08-28T02:12:59.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:12:59 smithi143 conmon[109923]: mgr.y (mgr.14424) 123 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s rd, 412 KiB/s wr, 64 op/s 2023-08-28T02:12:59.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:59 smithi169 conmon[100500]: cluster 2023-08-28T02:12:57.894212+0000 mgr.y (mgr.14424) 2023-08-28T02:12:59.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:12:59 smithi169 conmon[100500]: 123 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s rd, 412 KiB/s wr, 64 op/s 2023-08-28T02:13:00.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:00 smithi143 conmon[100258]: audit 2023-08-28T02:12:59.648256+0000 mon.c (mon.2) 28 : audit 2023-08-28T02:13:00.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:00 smithi143 conmon[100258]: [DBG] from='client.? 172.21.15.143:0/741714643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:00.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:00 smithi143 conmon[109923]: audit 2023-08-28T02:12:59.648256+0000 2023-08-28T02:13:00.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:00 smithi143 conmon[109923]: mon.c (mon.2) 28 : audit [DBG] from='client.? 172.21.15.143:0/741714643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:00.451 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:13:00 smithi143 conmon[100708]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:13:00] "GET /metrics HTTP/1.1" 200 192212 "" "Prometheus/2.33.4" 2023-08-28T02:13:00.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:00 smithi169 conmon[100500]: audit 2023-08-28T02:12:59.648256+0000 mon.c (mon.2) 28 : audit [DBG] from='client.? 172.21.15.143:0/741714643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:01.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:01 smithi143 conmon[100258]: cluster 2023-08-28T02:12:59.894707+0000 mgr.y (mgr.14424) 124 : cluster [DBG] 2023-08-28T02:13:01.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:01 smithi143 conmon[100258]: pgmap v116: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s rd, 412 KiB/s wr, 64 op/s 2023-08-28T02:13:01.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:01 smithi143 conmon[109923]: cluster 2023-08-28T02:12:59.894707+0000 2023-08-28T02:13:01.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:01 smithi143 conmon[109923]: mgr.y (mgr.14424) 124 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s rd, 412 KiB/s wr, 64 op/s 2023-08-28T02:13:01.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:01 smithi169 conmon[100500]: cluster 2023-08-28T02:12:59.894707+0000 mgr.y (mgr.14424) 124 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s rd, 412 KiB/s wr, 64 op/s 2023-08-28T02:13:02.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:02 smithi143 conmon[100258]: audit 2023-08-28T02:13:01.998432+0000 mon.c (mon.2) 29 : audit [DBG] 2023-08-28T02:13:02.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:02 smithi143 conmon[100258]: from='client.? 172.21.15.143:0/3995203728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:02.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:02 smithi143 conmon[100258]: audit 2023-08-28T02:13:02.193345+0000 mon.a (mon.0) 764 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:13:02.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:02 smithi143 conmon[100258]: audit 2023-08-28T02: 2023-08-28T02:13:02.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:02 smithi143 conmon[100258]: 13:02.194526+0000 mon.a (mon.0) 765 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:13:02.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:02 smithi143 conmon[100258]: audit 2023-08-28T02:13:02.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:02 smithi143 conmon[100258]: 2023-08-28T02:13:02.198361+0000 mon.a ( 2023-08-28T02:13:02.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:02 smithi143 conmon[100258]: mon.0) 766 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:13:02.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:02 smithi143 conmon[109923]: audit 2023-08-28 2023-08-28T02:13:02.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:02 smithi143 conmon[109923]: T02:13:01.998432+0000 mon.c (mon.2) 29 : audit [DBG] from='client.? 172.21.15.143:0/3995203728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:02.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:02 smithi143 conmon[109923]: audit 2023-08-28T02:13:02.193345 2023-08-28T02:13:02.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:02 smithi143 conmon[109923]: +0000 mon.a (mon.0) 764 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:13:02.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:02 smithi143 conmon[109923]: 2023-08-28T02:13:02.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:02 smithi143 conmon[109923]: audit 2023-08-28T02:13:02.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:02 smithi143 conmon[109923]: 2023-08-28T02:13:02.194526+0000 mon.a (mon.0) 2023-08-28T02:13:02.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:02 smithi143 conmon[109923]: 765 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:13:02.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:02 smithi143 conmon[109923]: audit 2023-08-28T02: 2023-08-28T02:13:02.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:02 smithi143 conmon[109923]: 13:02.198361+0000 mon.a ( 2023-08-28T02:13:02.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:02 smithi143 conmon[109923]: mon.0) 766 : audit [INF] 2023-08-28T02:13:02.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:02 smithi143 conmon[109923]: from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:13:02.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:02 smithi169 conmon[100500]: audit 2023-08-28T02:13:01.998432+0000 mon.c (mon.2) 29 : audit 2023-08-28T02:13:02.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:02 smithi169 conmon[100500]: [DBG] from='client.? 172.21.15.143:0/3995203728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:02.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:02 smithi169 conmon[100500]: audit 2023-08-28T02:13:02.193345+0000 mon.a (mon.0) 764 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:13:02.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:02 smithi169 conmon[100500]: audit 2023-08-28T02:13:02 2023-08-28T02:13:02.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:02 smithi169 conmon[100500]: .194526+0000 mon.a (mon.0) 765 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:13:02.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:02 smithi169 conmon[100500]: audit 2023-08-28 2023-08-28T02:13:02.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:02 smithi169 conmon[100500]: T02:13:02.198361+0000 mon.a (mon.0) 766 : audit [INF] 2023-08-28T02:13:02.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:02 smithi169 conmon[100500]: from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:13:03.208 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:02 smithi169 conmon[102807]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:13:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-28T02:13:03.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:03 smithi169 conmon[100500]: cluster 2023-08-28T02:13:01.895223+0000 mgr.y (mgr.14424) 125 : cluster 2023-08-28T02:13:03.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:03 smithi169 conmon[100500]: [DBG] pgmap v117: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 351 KiB/s wr, 55 op/s 2023-08-28T02:13:03.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:03 smithi169 conmon[100500]: cephadm 2023-08-28T02:13:02.193662+0000 mgr.y (mgr.14424) 126 : cephadm [INF] Upgrade: Target is version 18.2.0-187-g9cf20820 (unknown) 2023-08-28T02:13:03.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:03 smithi169 conmon[100500]: cephadm 2023-08-28T02:13:02.193722+0000 mgr.y 2023-08-28T02:13:03.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:03 smithi169 conmon[100500]: (mgr.14424) 127 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8, digests ['quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8'] 2023-08-28T02:13:03.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:03 smithi169 conmon[100500]: cephadm 2023-08 2023-08-28T02:13:03.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:03 smithi169 conmon[100500]: -28T02:13:02.198836+0000 mgr.y (mgr.14424) 128 : cephadm 2023-08-28T02:13:03.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:03 smithi169 conmon[100500]: [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-08-28T02:13:03.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:03 smithi169 conmon[100500]: cephadm 2023-08-28T02: 2023-08-28T02:13:03.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:03 smithi169 conmon[100500]: 13:02.488728+0000 mgr.y (mgr.14424) 129 : cephadm 2023-08-28T02:13:03.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:03 smithi169 conmon[100500]: [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8 on smithi169 2023-08-28T02:13:03.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:03 smithi143 conmon[100258]: cluster 2023-08-28T02:13:01.895223+0000 mgr.y (mgr.14424) 2023-08-28T02:13:03.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:03 smithi143 conmon[100258]: 125 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 351 KiB/s wr, 55 op/s 2023-08-28T02:13:03.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:03 smithi143 conmon[100258]: cephadm 2023-08-28T02:13:02.193662+0000 mgr.y (mgr.14424) 126 : cephadm [INF] Upgrade: Target is version 18.2.0-187-g9cf20820 (unknown) 2023-08-28T02:13:03.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:03 smithi143 conmon[100258]: cephadm 2023-08-28T02:13:03.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:03 smithi143 conmon[100258]: 2023-08-28T02:13:02.193722+0000 mgr.y (mgr.14424) 127 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8, digests ['quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8'] 2023-08-28T02:13:03.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:03 smithi143 conmon[100258]: 2023-08-28T02:13:03.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:03 smithi143 conmon[100258]: cephadm 2023-08- 2023-08-28T02:13:03.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:03 smithi143 conmon[100258]: 28T02:13:02.198836+0000 mgr.y 2023-08-28T02:13:03.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:03 smithi143 conmon[100258]: (mgr.14424) 128 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-08-28T02:13:03.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:03 smithi143 conmon[100258]: cephadm 2023 2023-08-28T02:13:03.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:03 smithi143 conmon[100258]: -08-28T02:13:02. 2023-08-28T02:13:03.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:03 smithi143 conmon[100258]: 488728+0000 mgr.y (mgr.14424) 129 : cephadm 2023-08-28T02:13:03.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:03 smithi143 conmon[100258]: [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8 on smithi169 2023-08-28T02:13:03.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:03 smithi143 conmon[109923]: cluster 2023-08-28T02:13:01.895223+0000 mgr.y ( 2023-08-28T02:13:03.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:03 smithi143 conmon[109923]: mgr.14424) 125 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 351 KiB/s wr, 55 op/s 2023-08-28T02:13:03.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:03 smithi143 conmon[109923]: cephadm 2023-08-28T02:13:02.193662+0000 mgr.y ( 2023-08-28T02:13:03.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:03 smithi143 conmon[109923]: mgr.14424) 126 : cephadm [INF] Upgrade: Target is version 18.2.0-187-g9cf20820 (unknown) 2023-08-28T02:13:03.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:03 smithi143 conmon[109923]: cephadm 2023-08-28 2023-08-28T02:13:03.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:03 smithi143 conmon[109923]: T02:13:02.193722+0000 mgr.y (mgr.14424) 127 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8, digests ['quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8'] 2023-08-28T02:13:03.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:03 smithi143 conmon[109923]: 2023-08-28T02:13:03.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:03 smithi143 conmon[109923]: cephadm 2023-08-28T02:13: 2023-08-28T02:13:03.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:03 smithi143 conmon[109923]: 02.198836+0000 mgr.y (mgr.14424) 128 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-08-28T02:13:03.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:03 smithi143 conmon[109923]: cephadm 2023-08-28T02:13:02.488728 2023-08-28T02:13:03.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:03 smithi143 conmon[109923]: +0000 mgr.y (mgr.14424) 129 : cephadm [INF] 2023-08-28T02:13:03.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:03 smithi143 conmon[109923]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8 on smithi169 2023-08-28T02:13:05.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:05 smithi169 conmon[100500]: cluster 2023-08-28T02:13:03.895761+0000 mgr.y (mgr.14424 2023-08-28T02:13:05.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:05 smithi169 conmon[100500]: ) 130 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:05.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:05 smithi169 conmon[100500]: audit 2023-08-28T02:13:04.347464+0000 mon.a (mon.0) 767 : audit [DBG] from='client.? 172.21.15.143:0/2725532029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:05.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:05 smithi143 conmon[100258]: cluster 2023-08-28T02:13:03.895761+0000 mgr.y (mgr.14424) 130 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:05.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:05 smithi143 conmon[100258]: audit 2023-08-28T02:13:04.347464+0000 mon.a (mon.0) 767 : audit [DBG] from='client.? 172.21.15.143:0/2725532029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:05.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:05 smithi143 conmon[109923]: cluster 2023-08-28T02:13:03.895761+0000 mgr.y (mgr.14424) 130 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:05.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:05 smithi143 conmon[109923]: audit 2023-08-28T02:13:04.347464+0000 mon.a (mon.0) 767 : audit [DBG] from='client.? 172.21.15.143:0/2725532029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:07.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:07 smithi169 conmon[100500]: cluster 2023-08-28T02:13:05.896421+0000 mgr.y (mgr.14424) 2023-08-28T02:13:07.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:07 smithi169 conmon[100500]: 131 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:07.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:07 smithi169 conmon[100500]: audit 2023-08-28T02:13:06.694942+0000 mon.c (mon.2) 30 : audit [DBG] from='client.? 172.21.15.143:0/298762749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:07.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:07 smithi143 conmon[100258]: cluster 2023-08-28T02:13:05.896421+0000 mgr.y (mgr.14424) 131 : cluster 2023-08-28T02:13:07.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:07 smithi143 conmon[100258]: [DBG] pgmap v119: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:07.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:07 smithi143 conmon[100258]: audit 2023-08-28T02:13:06.694942+0000 mon.c (mon.2) 30 : audit [DBG] from='client.? 172.21.15.143:0/298762749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:07.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:07 smithi143 conmon[109923]: cluster 2023-08-28T02:13:05.896421+0000 mgr.y ( 2023-08-28T02:13:07.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:07 smithi143 conmon[109923]: mgr.14424) 131 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:07.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:07 smithi143 conmon[109923]: audit 2023-08-28T02:13:06.694942+0000 mon.c ( 2023-08-28T02:13:07.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:07 smithi143 conmon[109923]: mon.2) 30 : audit [DBG] from='client.? 172.21.15.143:0/298762749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:09.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:09 smithi169 conmon[100500]: cluster 2023-08-28T02:13:07.896837+0000 mgr.y ( 2023-08-28T02:13:09.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:09 smithi169 conmon[100500]: mgr.14424) 132 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:09.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:09 smithi169 conmon[100500]: audit 2023-08-28T02:13:09.045706+0000 mon.a (mon.0) 768 : audit [DBG] from='client.? 172.21.15.143:0/2122927740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:09.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:09 smithi143 conmon[100258]: cluster 2023-08-28T02:13:07.896837+0000 mgr.y (mgr.14424) 132 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:09.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:09 smithi143 conmon[100258]: 2023-08-28T02:13:09.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:09 smithi143 conmon[100258]: audit 2023-08-28T02:13:09.045706+0000 mon.a (mon.0) 768 : audit [DBG] from='client.? 172.21.15.143:0/2122927740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:09.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:09 smithi143 conmon[109923]: cluster 2023-08-28T02:13:07.896837+0000 mgr.y (mgr.14424) 2023-08-28T02:13:09.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:09 smithi143 conmon[109923]: 132 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:09.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:09 smithi143 conmon[109923]: audit 2023-08-28T02:13:09.045706+0000 mon.a (mon.0) 768 : audit [DBG] from='client.? 172.21.15.143:0/2122927740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:09.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:09 smithi143 conmon[109923]: 2023-08-28T02:13:10.700 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:13:10 smithi143 conmon[100708]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:13:10] "GET /metrics HTTP/1.1" 200 192207 "" "Prometheus/2.33.4" 2023-08-28T02:13:11.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:11 smithi169 conmon[100500]: cluster 2023-08-28T02:13:09.897311+0000 mgr.y (mgr.14424) 133 : cluster [DBG] 2023-08-28T02:13:11.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:11 smithi169 conmon[100500]: pgmap v121: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:11.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:11 smithi143 conmon[109923]: cluster 2023-08-28T02:13:09.897311+0000 mgr.y (mgr.14424) 133 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:11.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:11 smithi143 conmon[100258]: cluster 2023-08-28T02:13:09.897311+0000 mgr.y 2023-08-28T02:13:11.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:11 smithi143 conmon[100258]: (mgr.14424) 133 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:12.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:12 smithi169 conmon[100500]: audit 2023-08-28T02:13:11.393069+0000 mon.a (mon.0) 769 : audit 2023-08-28T02:13:12.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:12 smithi169 conmon[100500]: [DBG] from='client.? 172.21.15.143:0/1701214544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:12.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:12 smithi143 conmon[100258]: audit 2023-08-28T02:13:11.393069+0000 mon.a (mon.0 2023-08-28T02:13:12.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:12 smithi143 conmon[100258]: ) 769 : audit [DBG] from='client.? 172.21.15.143:0/1701214544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:12.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:12 smithi143 conmon[109923]: audit 2023-08-28T02:13:11.393069+0000 mon.a (mon.0) 2023-08-28T02:13:12.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:12 smithi143 conmon[109923]: 769 : audit [DBG] from='client.? 172.21.15.143:0/1701214544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:13.237 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:12 smithi169 conmon[102807]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:13:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-28T02:13:13.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:13 smithi143 conmon[100258]: cluster 2023-08-28T02:13:11.898075+0000 mgr.y (mgr.14424) 134 2023-08-28T02:13:13.869 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:13 smithi143 conmon[100258]: : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:13.870 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:13 smithi143 conmon[109923]: cluster 2023-08-28T02:13:11.898075+0000 mgr.y (mgr.14424) 134 : cluster 2023-08-28T02:13:13.870 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:13 smithi143 conmon[109923]: [DBG] pgmap v122: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:13.870 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:13 smithi169 conmon[100500]: cluster 2023-08-28T02:13:11.898075+0000 mgr.y (mgr. 2023-08-28T02:13:13.870 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:13 smithi169 conmon[100500]: 14424) 134 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:14.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:14 smithi143 conmon[109923]: audit 2023-08-28T02:13:13.739476+0000 mon.a (mon 2023-08-28T02:13:14.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:14 smithi143 conmon[109923]: .0) 770 : audit [DBG] from='client.? 172.21.15.143:0/1001901469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:14.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:14 smithi143 conmon[100258]: audit 2023-08-28T02:13:13.739476+0000 2023-08-28T02:13:14.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:14 smithi143 conmon[100258]: mon.a (mon.0) 770 : audit [DBG] from='client.? 172.21.15.143:0/1001901469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:14.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:14 smithi169 conmon[100500]: audit 2023-08-28T02:13:13.739476+0000 mon.a ( 2023-08-28T02:13:14.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:14 smithi169 conmon[100500]: mon.0) 770 : audit [DBG] from='client.? 172.21.15.143:0/1001901469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:15.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:15 smithi143 conmon[100258]: cluster 2023-08-28T02:13:13.898371+0000 mgr.y (mgr.14424) 2023-08-28T02:13:15.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:15 smithi143 conmon[100258]: 135 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:15.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:15 smithi143 conmon[109923]: cluster 2023-08-28T02:13:13.898371+0000 mgr.y (mgr.14424 2023-08-28T02:13:15.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:15 smithi143 conmon[109923]: ) 135 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:15.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:15 smithi169 conmon[100500]: cluster 2023-08-28T02:13:13.898371 2023-08-28T02:13:15.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:15 smithi169 conmon[100500]: +0000 mgr.y (mgr.14424) 135 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:16.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:16 smithi143 conmon[100258]: audit 2023-08-28T02:13:16.091309+0000 2023-08-28T02:13:16.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:16 smithi143 conmon[100258]: mon.b (mon.1) 29 : audit [DBG] from='client.? 172.21.15.143:0/1172037220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:16.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:16 smithi143 conmon[109923]: audit 2023-08-28T02:13:16.091309+0000 mon.b 2023-08-28T02:13:16.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:16 smithi143 conmon[109923]: (mon.1) 29 : audit [DBG] from='client.? 172.21.15.143:0/1172037220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:16.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:16 smithi169 conmon[100500]: audit 2023-08-28T02:13:16.091309+0000 mon.b ( 2023-08-28T02:13:16.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:16 smithi169 conmon[100500]: mon.1) 29 : audit [DBG] from='client.? 172.21.15.143:0/1172037220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:17.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:17 smithi143 conmon[100258]: cluster 2023-08-28T02:13:15.898850+0000 mgr.y (mgr.14424) 136 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:17.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:17 smithi143 conmon[109923]: cluster 2023-08-28T02:13:15.898850+0000 mgr.y (mgr.14424) 136 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:17.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:17 smithi169 conmon[100500]: cluster 2023-08-28T02:13:15.898850+0000 2023-08-28T02:13:17.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:17 smithi169 conmon[100500]: mgr.y (mgr.14424) 136 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:19.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:19 smithi143 conmon[100258]: cluster 2023-08-28T02:13:17.899347+0000 mgr.y (mgr.14424) 2023-08-28T02:13:19.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:19 smithi143 conmon[100258]: 137 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:19.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:19 smithi143 conmon[100258]: audit 2023-08-28T02:13:18.441822+0000 mon.a (mon.0) 771 : audit [DBG] from='client.? 172.21.15.143:0/1839163783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:19.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:19 smithi143 conmon[109923]: cluster 2023-08-28T02:13:17.899347+0000 mgr.y (mgr.14424) 2023-08-28T02:13:19.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:19 smithi143 conmon[109923]: 137 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:19.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:19 smithi143 conmon[109923]: audit 2023-08-28T02:13:18.441822+0000 mon.a (mon.0) 771 : audit 2023-08-28T02:13:19.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:19 smithi143 conmon[109923]: [DBG] from='client.? 172.21.15.143:0/1839163783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:19.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:19 smithi169 conmon[100500]: cluster 2023-08-28T02:13:17.899347+0000 mgr.y (mgr 2023-08-28T02:13:19.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:19 smithi169 conmon[100500]: .14424) 137 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:19.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:19 smithi169 conmon[100500]: audit 2023-08-28T02:13:18.441822+0000 mon.a (mon.0) 771 : audit [DBG] from='client.? 172.21.15.143:0/1839163783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:20.700 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:13:20 smithi143 conmon[100708]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:13:20] "GET /metrics HTTP/1.1" 200 192207 "" "Prometheus/2.33.4" 2023-08-28T02:13:21.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:21 smithi143 conmon[109923]: cluster 2023-08-28T02:13:19.899835+0000 mgr.y (mgr.14424) 138 : 2023-08-28T02:13:21.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:21 smithi143 conmon[109923]: cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:21.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:21 smithi143 conmon[109923]: audit 2023-08-28T02:13:20.789834+0000 mon.a (mon.0) 772 : audit [DBG] from='client.? 172.21.15.143:0/19128575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:21.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:21 smithi143 conmon[100258]: cluster 2023-08-28T02:13:19.899835+0000 mgr.y (mgr.14424) 138 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:21.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:21 smithi143 conmon[100258]: audit 2023-08-28T02:13:20.789834+0000 mon.a (mon.0) 772 : audit [DBG] from='client.? 172.21.15.143:0/19128575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:21.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:21 smithi169 conmon[100500]: cluster 2023-08-28T02:13:19.899835+0000 2023-08-28T02:13:21.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:21 smithi169 conmon[100500]: mgr.y (mgr.14424) 138 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:21.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:21 smithi169 conmon[100500]: audit 2023-08-28T02:13:20. 2023-08-28T02:13:21.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:21 smithi169 conmon[100500]: 789834+0000 mon.a (mon.0) 772 : audit [DBG] from='client.? 172.21.15.143:0/19128575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:23.237 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:22 smithi169 conmon[102807]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:13:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-28T02:13:23.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:23 smithi143 conmon[109923]: cluster 2023-08-28T02:13:21 2023-08-28T02:13:23.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:23 smithi143 conmon[109923]: .900484+0000 mgr.y (mgr.14424) 139 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:23.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:23 smithi143 conmon[109923]: audit 2023-08-28T02:13:23.134137+0000 mon.a 2023-08-28T02:13:23.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:23 smithi143 conmon[109923]: (mon.0) 773 : audit [DBG] from='client.? 172.21.15.143:0/2958481323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:23.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:23 smithi143 conmon[100258]: cluster 2023-08-28T02:13:21.900484 2023-08-28T02:13:23.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:23 smithi143 conmon[100258]: +0000 mgr.y (mgr.14424) 139 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:23.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:23 smithi143 conmon[100258]: audit 2023-08-28T02:13:23.134137 2023-08-28T02:13:23.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:23 smithi143 conmon[100258]: +0000 mon.a (mon.0) 773 : audit [DBG] from='client.? 172.21.15.143:0/2958481323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:23.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:23 smithi169 conmon[100500]: cluster 2023-08-28T02:13:21 2023-08-28T02:13:23.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:23 smithi169 conmon[100500]: .900484+0000 mgr.y (mgr.14424) 139 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:23.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:23 smithi169 conmon[100500]: audit 2023-08-28T02:13:23.134137+0000 2023-08-28T02:13:23.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:23 smithi169 conmon[100500]: mon.a (mon.0) 773 : audit [DBG] from='client.? 172.21.15.143:0/2958481323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:25.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:25 smithi143 conmon[109923]: cluster 2023-08-28T02:13 2023-08-28T02:13:25.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:25 smithi143 conmon[109923]: :23.900960+0000 mgr.y (mgr.14424) 140 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:25.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:25 smithi143 conmon[100258]: cluster 2023-08-28T02:13:23.900960+0000 mgr.y (mgr.14424 2023-08-28T02:13:25.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:25 smithi143 conmon[100258]: ) 140 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:25.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:25 smithi169 conmon[100500]: cluster 2023-08-28T02:13:23.900960+0000 mgr.y (mgr. 2023-08-28T02:13:25.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:25 smithi169 conmon[100500]: 14424) 140 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:26.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:26 smithi143 conmon[100258]: audit 2023-08-28T02:13:25.483781+0000 mon.c (mon.2) 31 : audit [DBG] from='client.? 172.21.15.143:0/2226187210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:26.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:26 smithi143 conmon[109923]: audit 2023-08-28T02:13:25.483781+0000 mon.c (mon.2) 31 2023-08-28T02:13:26.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:26 smithi143 conmon[109923]: : audit [DBG] from='client.? 172.21.15.143:0/2226187210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:26.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:26 smithi169 conmon[100500]: audit 2023-08-28T02:13:25.483781+0000 mon.c (mon.2) 31 : 2023-08-28T02:13:26.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:26 smithi169 conmon[100500]: audit [DBG] from='client.? 172.21.15.143:0/2226187210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:27.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:27 smithi143 conmon[100258]: cluster 2023-08-28T02:13:25.901450+0000 mgr.y (mgr.14424) 141 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:27.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:27 smithi143 conmon[109923]: cluster 2023-08-28T02:13:25.901450+0000 mgr.y (mgr.14424) 141 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:27.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:27 smithi169 conmon[100500]: cluster 2023-08-28T02:13:25.901450+0000 mgr.y (mgr.14424) 141 : cluster 2023-08-28T02:13:27.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:27 smithi169 conmon[100500]: [DBG] pgmap v129: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:28.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:28 smithi143 conmon[100258]: audit 2023-08-28T02:13:27.834560+0000 mon.a (mon.0) 774 : 2023-08-28T02:13:28.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:28 smithi143 conmon[100258]: audit [DBG] from='client.? 172.21.15.143:0/1571019220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:28.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:28 smithi143 conmon[109923]: audit 2023-08-28T02:13: 2023-08-28T02:13:28.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:28 smithi143 conmon[109923]: 27.834560+0000 mon.a (mon.0) 774 : audit [DBG] from='client.? 172.21.15.143:0/1571019220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:28.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:28 smithi169 conmon[100500]: audit 2023-08-28T02:13:27.834560+0000 mon.a (mon.0) 774 : audit 2023-08-28T02:13:28.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:28 smithi169 conmon[100500]: [DBG] from='client.? 172.21.15.143:0/1571019220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:29.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:29 smithi143 conmon[109923]: cluster 2023-08-28T02:13:27.901863+0000 mgr.y (mgr.14424) 142 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:29.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:29 smithi143 conmon[100258]: cluster 2023-08-28T02:13:27.901863+0000 mgr.y (mgr.14424) 142 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:29.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:29 smithi143 conmon[100258]: 2023-08-28T02:13:29.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:29 smithi169 conmon[100500]: cluster 2023-08-28T02:13:27.901863+0000 mgr.y (mgr.14424 2023-08-28T02:13:29.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:29 smithi169 conmon[100500]: ) 142 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:30.700 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:13:30 smithi143 conmon[100708]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:13:30] "GET /metrics HTTP/1.1" 200 192210 "" "Prometheus/2.33.4" 2023-08-28T02:13:30.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:30 smithi143 conmon[109923]: audit 2023-08-28T02:13:30.180287+0000 mon.a (mon.0) 775 : audit [DBG] from='client.? 172.21.15.143:0/957224591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:30.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:30 smithi143 conmon[100258]: audit 2023-08-28T02:13:30.180287+0000 mon.a (mon. 2023-08-28T02:13:30.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:30 smithi143 conmon[100258]: 0) 775 : audit [DBG] from='client.? 172.21.15.143:0/957224591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:30.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:30 smithi169 conmon[100500]: audit 2023-08-28T02:13:30.180287 2023-08-28T02:13:30.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:30 smithi169 conmon[100500]: +0000 mon.a (mon.0) 775 : audit [DBG] from='client.? 172.21.15.143:0/957224591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:31.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:31 smithi143 conmon[100258]: cluster 2023-08-28T02:13:29.902245+0000 mgr.y (mgr.14424) 143 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:31.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:31 smithi143 conmon[109923]: cluster 2023-08-28T02:13:29.902245+0000 2023-08-28T02:13:31.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:31 smithi143 conmon[109923]: mgr.y (mgr.14424) 143 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:31.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:31 smithi169 conmon[100500]: cluster 2023-08-28T02:13:29.902245+0000 mgr.y (mgr.14424 2023-08-28T02:13:31.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:31 smithi169 conmon[100500]: ) 143 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:33.237 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:32 smithi169 conmon[102807]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:13:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-28T02:13:33.648 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:33 smithi169 conmon[100500]: cluster 2023-08-28T02:13:31.902782+0000 mgr.y (mgr.14424 2023-08-28T02:13:33.648 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:33 smithi169 conmon[100500]: ) 144 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:33.648 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:33 smithi169 conmon[100500]: audit 2023-08-28T02:13:32.537361+0000 mon.c (mon.2) 32 : audit [DBG] from='client.? 172.21.15.143:0/2661487009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:33.648 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:33 smithi169 conmon[100500]: 2023-08-28T02:13:33.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:33 smithi143 conmon[100258]: cluster 2023-08-28T02:13:31.902782+0000 mgr.y (mgr.14424) 144 : 2023-08-28T02:13:33.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:33 smithi143 conmon[100258]: cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:33.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:33 smithi143 conmon[100258]: audit 2023-08-28T02:13:32.537361+0000 2023-08-28T02:13:33.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:33 smithi143 conmon[100258]: mon.c (mon.2) 32 : audit [DBG] from='client.? 172.21.15.143:0/2661487009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:33.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:33 smithi143 conmon[109923]: cluster 2023-08-28T02:13:31.902782 2023-08-28T02:13:33.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:33 smithi143 conmon[109923]: +0000 mgr.y (mgr.14424) 144 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:33.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:33 smithi143 conmon[109923]: audit 2023-08-28T02 2023-08-28T02:13:33.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:33 smithi143 conmon[109923]: :13:32.537361+0000 mon.c (mon.2) 32 : audit [DBG] from='client.? 172.21.15.143:0/2661487009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:35.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:35 smithi143 conmon[100258]: cluster 2023-08-28T02:13:33.903187+0000 mgr.y (mgr.14424) 145 : cluster 2023-08-28T02:13:35.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:35 smithi143 conmon[100258]: [DBG] pgmap v133: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:35.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:35 smithi143 conmon[100258]: audit 2023-08-28T02:13:34.885955+0000 mon.c (mon.2) 33 : audit [DBG] from='client.? 172.21.15.143:0/3698358034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:35.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:35 smithi143 conmon[109923]: cluster 2023-08-28T02:13:33.903187+0000 mgr.y (mgr.14424) 2023-08-28T02:13:35.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:35 smithi143 conmon[109923]: 145 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:35.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:35 smithi143 conmon[109923]: audit 2023-08-28T02:13:34.885955+0000 mon.c (mon. 2023-08-28T02:13:35.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:35 smithi143 conmon[109923]: 2) 33 : audit [DBG] from='client.? 172.21.15.143:0/3698358034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:35.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:35 smithi169 conmon[100500]: cluster 2023-08-28T02:13:33.903187+0000 mgr.y (mgr.14424) 145 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:35.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:35 smithi169 conmon[100500]: audit 2023-08-28T02:13:34.885955+0000 mon.c (mon.2) 33 : audit [DBG] from='client.? 172.21.15.143:0/3698358034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:37.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:37 smithi143 conmon[100258]: cluster 2023-08-28T02:13:35.903772+0000 mgr.y (mgr.14424) 2023-08-28T02:13:37.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:37 smithi143 conmon[100258]: 146 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:37.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:37 smithi143 conmon[100258]: audit 2023-08-28T02:13:37.231889+0000 mon.a (mon.0) 2023-08-28T02:13:37.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:37 smithi143 conmon[100258]: 776 : audit [DBG] from='client.? 172.21.15.143:0/510512904' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:37.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:37 smithi143 conmon[109923]: cluster 2023-08-28T02:13:35.903772+0000 mgr.y 2023-08-28T02:13:37.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:37 smithi143 conmon[109923]: (mgr.14424) 146 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:37.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:37 smithi143 conmon[109923]: audit 2023-08- 2023-08-28T02:13:37.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:37 smithi143 conmon[109923]: 28T02:13:37.231889+0000 mon.a (mon.0) 776 : audit [DBG] 2023-08-28T02:13:37.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:37 smithi143 conmon[109923]: from='client.? 172.21.15.143:0/510512904' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:37.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:37 smithi169 conmon[100500]: cluster 2023-08-28T02:13:35.903772+0000 mgr.y (mgr.14424) 146 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:37.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:37 smithi169 conmon[100500]: audit 2023-08-28T02:13:37.231889+0000 mon.a (mon.0) 776 : audit [DBG] from='client.? 172.21.15.143:0/510512904' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:39.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:39 smithi143 conmon[109923]: cluster 2023-08-28T02:13:37.904330+0000 mgr.y ( 2023-08-28T02:13:39.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:39 smithi143 conmon[109923]: mgr.14424) 147 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:39.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:39 smithi143 conmon[100258]: cluster 2023-08-28T02:13:37.904330+0000 mgr.y 2023-08-28T02:13:39.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:39 smithi143 conmon[100258]: (mgr.14424) 147 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:39.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:39 smithi169 conmon[100500]: cluster 2023-08-28T02:13:37.904330+0000 mgr.y (mgr.14424) 147 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:40.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:40 smithi143 conmon[100258]: audit 2023-08-28T02:13:39.581060+0000 mon.c (mon.2) 2023-08-28T02:13:40.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:40 smithi143 conmon[100258]: 34 : audit [DBG] from='client.? 172.21.15.143:0/1421669982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:40.702 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:13:40 smithi143 conmon[100708]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:13:40] "GET /metrics HTTP/1.1" 200 192208 "" "Prometheus/2.33.4" 2023-08-28T02:13:40.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:40 smithi143 conmon[109923]: audit 2023-08-28T02:13:39.581060+0000 mon.c (mon.2) 2023-08-28T02:13:40.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:40 smithi143 conmon[109923]: 34 : audit [DBG] from='client.? 172.21.15.143:0/1421669982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:40.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:40 smithi169 conmon[100500]: audit 2023-08-28T02:13:39.581060+0000 mon.c (mon.2) 34 : audit [DBG] from='client.? 172.21.15.143:0/1421669982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:41.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:41 smithi143 conmon[100258]: cluster 2023-08-28T02:13:39.904843+0000 mgr.y (mgr.14424) 148 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:41.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:41 smithi143 conmon[100258]: 2023-08-28T02:13:41.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:41 smithi143 conmon[109923]: cluster 2023-08-28T02:13:39.904843+0000 mgr.y (mgr.14424) 148 : cluster 2023-08-28T02:13:41.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:41 smithi143 conmon[109923]: [DBG] pgmap v136: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:41.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:41 smithi169 conmon[100500]: cluster 2023-08-28T02:13:39.904843+0000 mgr.y (mgr.14424) 148 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:42.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:42 smithi143 conmon[100258]: audit 2023-08-28T02:13:41.928420+0000 2023-08-28T02:13:42.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:42 smithi143 conmon[100258]: mon.c (mon.2) 35 : audit [DBG] from='client.? 172.21.15.143:0/2492284459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:42.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:42 smithi143 conmon[109923]: audit 2023-08-28T02:13:41.928420+0000 mon.c (mon.2) 2023-08-28T02:13:42.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:42 smithi143 conmon[109923]: 35 : audit [DBG] from='client.? 172.21.15.143:0/2492284459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:42.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:42 smithi169 conmon[100500]: audit 2023-08-28T02:13:41.928420+0000 2023-08-28T02:13:42.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:42 smithi169 conmon[100500]: mon.c (mon.2) 35 : audit [DBG] from='client.? 172.21.15.143:0/2492284459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:43.237 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:42 smithi169 conmon[102807]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:13:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-28T02:13:43.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:43 smithi143 conmon[100258]: cluster 2023-08-28T02:13:41.905346+0000 2023-08-28T02:13:43.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:43 smithi143 conmon[100258]: mgr.y (mgr.14424) 149 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:43.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:43 smithi143 conmon[109923]: cluster 2023-08-28T02:13:41.905346+0000 mgr.y (mgr.14424) 149 : cluster 2023-08-28T02:13:43.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:43 smithi143 conmon[109923]: [DBG] pgmap v137: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:43.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:43 smithi169 conmon[100500]: cluster 2023-08-28T02:13:41.905346+0000 mgr.y (mgr.14424) 149 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:44.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:44 smithi143 conmon[100258]: audit 2023-08-28T02:13:44.278080 2023-08-28T02:13:44.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:44 smithi143 conmon[100258]: +0000 mon.c (mon.2) 36 : audit [DBG] from='client.? 172.21.15.143:0/2676004526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:44.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:44 smithi143 conmon[109923]: audit 2023-08-28T02:13:44.278080+0000 mon.c (mon. 2023-08-28T02:13:44.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:44 smithi143 conmon[109923]: 2) 36 : audit [DBG] from='client.? 172.21.15.143:0/2676004526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:44.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:44 smithi169 conmon[100500]: audit 2023-08-28T02:13:44.278080+0000 mon.c (mon.2) 36 : audit [DBG] from='client.? 172.21.15.143:0/2676004526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:45.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:45 smithi143 conmon[100258]: cluster 2023-08-28T02:13:43.905771+0000 mgr.y ( 2023-08-28T02:13:45.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:45 smithi143 conmon[100258]: mgr.14424) 150 : cluster [DBG] pgmap v138: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:45.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:45 smithi143 conmon[109923]: cluster 2023-08-28T02:13:43.905771+0000 mgr.y (mgr.14424) 2023-08-28T02:13:45.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:45 smithi143 conmon[109923]: 150 : cluster [DBG] pgmap v138: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:45.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:45 smithi169 conmon[100500]: cluster 2023-08-28T02:13:43.905771+0000 mgr.y (mgr.14424) 150 : cluster [DBG] pgmap v138: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:46.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:46 smithi169 conmon[100500]: cephadm 2023-08-28T02:13:45.575221+0000 mgr.y (mgr.14424) 2023-08-28T02:13:46.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:46 smithi169 conmon[100500]: 151 : cephadm [INF] Upgrade: Updating mgr.x 2023-08-28T02:13:46.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:46 smithi169 conmon[100500]: audit 2023-08-28T02:13:45.580158+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:13:46.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:46 smithi169 conmon[100500]: audit 2023-08-28T02:13:45.581159+0000 mon.a (mon.0) 778 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-28T02:13:46.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:46 smithi169 conmon[100500]: audit 2023-08-28T02:13:45.581999+0000 mon.a (mon.0) 779 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-08-28T02:13:46.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:46 smithi169 conmon[100500]: audit 2023-08-28T02:13:45.582913+0000 mon.a (mon.0) 780 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:13:46.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:46 smithi169 conmon[100500]: cephadm 2023-08-28T02:13:45.583635+0000 mgr.y (mgr.14424) 152 : cephadm [INF] Deploying daemon mgr.x on smithi169 2023-08-28T02:13:46.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:46 smithi143 conmon[100258]: cephadm 2023-08-28T02:13:45.575221+0000 mgr.y (mgr.14424 2023-08-28T02:13:46.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:46 smithi143 conmon[100258]: ) 151 : cephadm [INF] Upgrade: Updating mgr.x 2023-08-28T02:13:46.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:46 smithi143 conmon[100258]: audit 2023-08-28T02:13:45.580158+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:13:46.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:46 smithi143 conmon[100258]: audit 2023-08-28T02:13:45.581159+0000 mon.a (mon.0) 778 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-28T02:13:46.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:46 smithi143 conmon[100258]: audit 2023-08-28T02:13:45.581999+0000 mon.a (mon.0) 779 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-08-28T02:13:46.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:46 smithi143 conmon[100258]: audit 2023-08-28T02 2023-08-28T02:13:46.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:46 smithi143 conmon[100258]: :13:45.582913+0000 mon.a (mon.0) 780 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:13:46.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:46 smithi143 conmon[100258]: cephadm 2023-08-28T02:13:45 2023-08-28T02:13:46.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:46 smithi143 conmon[100258]: .583635+0000 mgr.y (mgr.14424) 152 : cephadm [INF] Deploying daemon mgr.x on smithi169 2023-08-28T02:13:46.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:46 smithi143 conmon[109923]: cephadm 2023-08-28T02:13:45.575221+0000 mgr.y (mgr.14424 2023-08-28T02:13:46.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:46 smithi143 conmon[109923]: ) 151 : cephadm [INF] Upgrade: Updating mgr.x 2023-08-28T02:13:46.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:46 smithi143 conmon[109923]: audit 2023-08-28T02:13:45.580158+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:13:46.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:46 smithi143 conmon[109923]: audit 2023-08-28T02:13:45.581159+0000 mon.a (mon.0) 778 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-28T02:13:46.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:46 smithi143 conmon[109923]: 2023-08-28T02:13:46.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:46 smithi143 conmon[109923]: audit 2023-08-28T02:13 2023-08-28T02:13:46.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:46 smithi143 conmon[109923]: :45.581999+0000 mon.a (mon.0) 779 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-08-28T02:13:46.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:46 smithi143 conmon[109923]: audit 2023-08-28T02:13:45.582913+0000 mon.a (mon.0) 780 2023-08-28T02:13:46.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:46 smithi143 conmon[109923]: : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:13:46.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:46 smithi143 conmon[109923]: cephadm 2023-08-28T02:13:46.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:46 smithi143 conmon[109923]: 2023-08-28T02:13:45.583635+0000 mgr.y (mgr.14424) 152 : cephadm [INF] Deploying daemon mgr.x on smithi169 2023-08-28T02:13:47.238 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:47 smithi169 systemd[1]: Stopping Ceph mgr.x for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:13:47.279 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:47 smithi169 bash[137889]: Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-mgr.x found: no such container 2023-08-28T02:13:47.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:47 smithi169 conmon[100500]: cluster 2023-08-28T02:13:45.906524+0000 mgr.y ( 2023-08-28T02:13:47.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:47 smithi169 conmon[100500]: mgr.14424) 153 : cluster [DBG] pgmap v139: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:47.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:47 smithi169 conmon[100500]: audit 2023-08-28T02:13:46.630744+0000 mon.a (mon.0) 781 : audit [DBG] from='client.? 172.21.15.143:0/386062172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:47.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:47 smithi143 conmon[100258]: cluster 2023-08-28T02:13:45.906524+0000 mgr.y (mgr. 2023-08-28T02:13:47.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:47 smithi143 conmon[100258]: 14424) 153 : cluster [DBG] pgmap v139: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:47.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:47 smithi143 conmon[100258]: audit 2023-08-28T02:13:46.630744+0000 mon.a (mon.0) 781 : audit [DBG] from='client.? 172.21.15.143:0/386062172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:47.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:47 smithi143 conmon[109923]: cluster 2023-08-28T02:13:45.906524+0000 mgr.y (mgr.14424) 153 : cluster 2023-08-28T02:13:47.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:47 smithi143 conmon[109923]: [DBG] pgmap v139: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:47.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:47 smithi143 conmon[109923]: audit 2023-08-28T02:13:46.630744+0000 mon.a (mon.0) 781 : audit [DBG] from='client.? 172.21.15.143:0/386062172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:48.161 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:47 smithi169 systemd[1]: ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-08-28T02:13:48.162 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:47 smithi169 bash[138004]: 96e04bb664893b082c58ecc0047815ffc0b222caebcfa5cf6320e19318e56b00 2023-08-28T02:13:48.162 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:48 smithi169 systemd[1]: ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@mgr.x.service: Failed with result 'exit-code'. 2023-08-28T02:13:48.162 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:48 smithi169 systemd[1]: Stopped Ceph mgr.x for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:13:48.487 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:48 smithi169 systemd[1]: Starting Ceph mgr.x for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:13:48.878 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:48 smithi169 bash[138359]: f3ab5a3cce46a59891c5410e2ba7596c1525daedda40887ccfe2d0956f52bfc7 2023-08-28T02:13:48.878 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:48 smithi169 conmon[138400]: debug 2023-08-28T02:13:48.753+0000 7f624aab2700 1 -- 172.21.15.169:0/3859907109 <== mon.1 v2:172.21.15.169:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55f6f2e7f1e0 con 0x55f6f2e56c00 2023-08-28T02:13:48.878 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:48 smithi169 systemd[1]: Started Ceph mgr.x for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:13:49.171 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:48 smithi169 conmon[138400]: debug 2023-08-28T02:13:48.878+0000 7f6254c00200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-08-28T02:13:49.171 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:48 smithi169 conmon[138400]: debug 2023-08-28T02:13:48.962+0000 7f6254c00200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-08-28T02:13:49.171 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:49 smithi169 conmon[138400]: debug 2023-08-28T02:13:49.063+0000 7f6254c00200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-08-28T02:13:49.487 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:49 smithi169 conmon[138400]: debug 2023-08-28T02:13:49.171+0000 7f6254c00200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-08-28T02:13:49.487 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:49 smithi169 conmon[138400]: debug 2023-08-28T02:13:49.366+0000 7f6254c00200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-08-28T02:13:49.922 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:49 smithi169 conmon[138400]: debug 2023-08-28T02:13:49.770+0000 7f6254c00200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-08-28T02:13:49.923 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:49 smithi169 conmon[100500]: cluster 2023-08-28T02:13:47.906998+0000 mgr.y (mgr.14424 2023-08-28T02:13:49.923 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:49 smithi169 conmon[100500]: ) 154 : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:49.923 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:49 smithi169 conmon[100500]: audit 2023-08-28T02:13:48.803409+0000 mon.a (mon.0) 782 : audit [INF] 2023-08-28T02:13:49.923 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:49 smithi169 conmon[100500]: from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:13:49.923 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:49 smithi169 conmon[100500]: audit 2023-08-28T 2023-08-28T02:13:49.923 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:49 smithi169 conmon[100500]: 02:13:48.804226+0000 mon.a (mon.0) 783 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:13:49.924 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:49 smithi169 conmon[100500]: audit 2023-08-28T02:13: 2023-08-28T02:13:49.924 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:49 smithi169 conmon[100500]: 48.993292+0000 mon.c (mon.2) 37 : audit [DBG] 2023-08-28T02:13:49.924 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:49 smithi169 conmon[100500]: from='client.? 172.21.15.143:0/1472908846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:49.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:49 smithi143 conmon[100258]: cluster 2023-08-28T02:13:47.906998+0000 mgr.y (mgr.14424) 154 : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:49.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:49 smithi143 conmon[100258]: audit 2023-08-28T02:13:48.803409+0000 mon.a (mon.0) 782 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:13:49.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:49 smithi143 conmon[100258]: audit 2023-08-28T02:13:48.804226+0000 mon.a (mon.0) 783 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:13:49.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:49 smithi143 conmon[100258]: audit 2023-08-28T02:13:48.993292+0000 mon.c (mon.2) 37 : audit [DBG] from='client.? 172.21.15.143:0/1472908846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:49.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:49 smithi143 conmon[109923]: cluster 2023-08-28T02:13:47.906998+0000 mgr.y (mgr.14424) 154 : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:49.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:49 smithi143 conmon[109923]: audit 2023-08-28T02:13:48.803409+0000 mon.a (mon.0) 782 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:13:49.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:49 smithi143 conmon[109923]: audit 2023-08-28T02:13:48.804226+0000 mon.a (mon.0) 783 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:13:49.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:49 smithi143 conmon[109923]: audit 2023-08-28T02:13:48.993292+0000 mon.c (mon.2) 37 : audit [DBG] from='client.? 172.21.15.143:0/1472908846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:50.237 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:49 smithi169 conmon[138400]: debug 2023-08-28T02:13:49.922+0000 7f6254c00200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-08-28T02:13:50.237 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:50 smithi169 conmon[138400]: debug 2023-08-28T02:13:50.022+0000 7f6254c00200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-08-28T02:13:50.237 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:50 smithi169 conmon[138400]: debug 2023-08-28T02:13:50.122+0000 7f6254c00200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-08-28T02:13:50.700 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:13:50 smithi143 conmon[100708]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:13:50] "GET /metrics HTTP/1.1" 200 192208 "" "Prometheus/2.33.4" 2023-08-28T02:13:50.987 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:50 smithi169 conmon[138400]: debug 2023-08-28T02:13:50.622+0000 7f6254c00200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-08-28T02:13:51.737 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:51 smithi169 conmon[138400]: debug 2023-08-28T02:13:51.356+0000 7f6254c00200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-08-28T02:13:51.737 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:51 smithi169 conmon[138400]: debug 2023-08-28T02:13:51.439+0000 7f6254c00200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-08-28T02:13:51.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:51 smithi169 conmon[100500]: cluster 2023-08-28T02:13:49.907335+0000 mgr.y (mgr.14424) 155 2023-08-28T02:13:51.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:51 smithi169 conmon[100500]: : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:51.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:51 smithi169 conmon[100500]: audit 2023-08-28T02:13:50.878726+0000 mon.a (mon.0) 784 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:13:51.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:51 smithi169 conmon[100500]: audit 2023-08-28T 2023-08-28T02:13:51.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:51 smithi169 conmon[100500]: 02:13:51.347255+0000 mon.a (mon.0) 785 : audit [DBG] from='client.? 172.21.15.143:0/1020202895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:51.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:51 smithi143 conmon[100258]: cluster 2023-08-28T02:13:49.907335 2023-08-28T02:13:51.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:51 smithi143 conmon[100258]: +0000 mgr.y (mgr.14424) 155 : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:51.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:51 smithi143 conmon[100258]: audit 2023-08-28T02:13:50.878726+0000 mon.a (mon.0) 784 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:13:51.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:51 smithi143 conmon[100258]: audit 2023-08-28T02:13:51.347255+0000 mon.a (mon.0) 785 : audit [DBG] from='client.? 172.21.15.143:0/1020202895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:51.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:51 smithi143 conmon[109923]: cluster 2023-08-28T02:13:49.907335+0000 mgr.y ( 2023-08-28T02:13:51.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:51 smithi143 conmon[109923]: mgr.14424) 155 : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:51.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:51 smithi143 conmon[109923]: audit 2023-08-28T02:13:50.878726+0000 mon.a (mon.0) 784 2023-08-28T02:13:51.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:51 smithi143 conmon[109923]: : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:13:51.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:51 smithi143 conmon[109923]: audit 2023-08-28 2023-08-28T02:13:51.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:51 smithi143 conmon[109923]: T02:13:51.347255+0000 mon.a (mon.0) 785 : audit 2023-08-28T02:13:51.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:51 smithi143 conmon[109923]: [DBG] from='client.? 172.21.15.143:0/1020202895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:52.587 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:52 smithi169 conmon[138400]: debug 2023-08-28T02:13:52.307+0000 7f6254c00200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-08-28T02:13:52.587 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:52 smithi169 conmon[138400]: debug 2023-08-28T02:13:52.406+0000 7f6254c00200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-08-28T02:13:52.700 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:13:52 smithi143 conmon[100708]: debug 2023-08-28T02:13:52.448+0000 7f127aa7a700 -1 mgr handle_mgr_map I was active but no longer am 2023-08-28T02:13:52.700 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:13:52 smithi143 conmon[100708]: ignoring --setuser ceph since I am not root 2023-08-28T02:13:52.700 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:13:52 smithi143 conmon[100708]: ignoring --setgroup ceph since I am not root 2023-08-28T02:13:52.700 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:13:52 smithi143 conmon[100708]: debug 2023-08-28T02:13:52.493+0000 7feae9523700 1 -- 172.21.15.143:0/702303383 <== mon.2 v2:172.21.15.143:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55c51f4a64e0 con 0x55c51f47f000 2023-08-28T02:13:52.700 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:13:52 smithi143 conmon[100708]: debug 2023-08-28T02:13:52.615+0000 7feaf96ec680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-08-28T02:13:52.701 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:13:52 smithi143 conmon[100708]: debug 2023-08-28T02:13:52.695+0000 7feaf96ec680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-08-28T02:13:52.866 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:52 smithi169 conmon[138400]: debug 2023-08-28T02:13:52.588+0000 7f6254c00200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-08-28T02:13:53.237 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:52 smithi169 conmon[138400]: debug 2023-08-28T02:13:52.867+0000 7f6254c00200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-08-28T02:13:53.237 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:53 smithi169 conmon[138400]: debug 2023-08-28T02:13:53.099+0000 7f6254c00200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-08-28T02:13:53.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:53 smithi169 conmon[100500]: audit 2023-08-28T02:13:52.096837+0000 mon.a (mon.0) 786 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:13:53.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:53 smithi169 conmon[100500]: audit 2023-08-28T02:13:52.422679+0000 mon.a (mon.0) 787 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:13:53.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:53 smithi169 conmon[100500]: audit 2023-08-28T02:13:52.423596+0000 mon.a (mon.0) 788 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:13:53.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:53 smithi169 conmon[100500]: audit 2023-08-28T02:13:52.431403+0000 mon.a (mon.0) 789 : audit [INF] 2023-08-28T02:13:53.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:53 smithi169 conmon[100500]: from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:13:53.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:53 smithi169 conmon[100500]: audit 2023-08-28T02 2023-08-28T02:13:53.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:53 smithi169 conmon[100500]: :13:52.438038+0000 mon.a (mon.0) 790 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:13:53.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:53 smithi169 conmon[100500]: audit 2023-08-28T02:13:52 2023-08-28T02:13:53.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:53 smithi169 conmon[100500]: .442840+0000 mon.a (mon.0) 791 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-08-28T02:13:53.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:53 smithi169 conmon[100500]: cluster 2023-08-28T02:13:52.451461+0000 mon.a (mon. 2023-08-28T02:13:53.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:53 smithi169 conmon[100500]: 0) 792 : cluster [DBG] osdmap e80: 8 total, 8 up, 8 in 2023-08-28T02:13:53.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:53 smithi169 conmon[100500]: audit 2023-08-28T02:13:53.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:53 smithi169 conmon[100500]: 2023-08-28T02:13:52.451712+0000 mon.a (mon.0) 793 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-08-28T02:13:53.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:53 smithi169 conmon[100500]: 2023-08-28T02:13:53.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:53 smithi169 conmon[100500]: cluster 2023-08-28 2023-08-28T02:13:53.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:53 smithi169 conmon[100500]: T02:13:52.451765+0000 mon.a (mon.0) 794 : cluster [DBG] mgrmap e19: x(active, starting, since 0.00841329s) 2023-08-28T02:13:53.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[109923]: audit 2023-08-28T02:13:52.096837+0000 mon.a (mon. 2023-08-28T02:13:53.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[109923]: 0) 786 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:13:53.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[109923]: audit 2023-08-28T02:13:52.422679+0000 mon.a (mon.0) 787 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:13:53.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[109923]: audit 2023-08-28T02:13:52.423596+0000 mon.a (mon.0) 788 2023-08-28T02:13:53.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[109923]: : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:13:53.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[109923]: audit 2023-08-28T02:13:53.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[109923]: 2023-08-28T02:13:52.431403+0000 2023-08-28T02:13:53.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[109923]: mon.a (mon.0) 789 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:13:53.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[109923]: 2023-08-28T02:13:53.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[109923]: audit 2023-08-28T02:13:53.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[109923]: 2023-08-28T02:13:52.438038 2023-08-28T02:13:53.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[109923]: +0000 mon.a (mon.0) 790 : audit [DBG] 2023-08-28T02:13:53.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[109923]: from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:13:53.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[109923]: audit 2023-08-28T02:13:53.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[109923]: 2023-08-28T02:13:52.442840 2023-08-28T02:13:53.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[109923]: +0000 mon.a (mon.0) 791 : audit [INF] 2023-08-28T02:13:53.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[109923]: from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-08-28T02:13:53.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[109923]: cluster 2023-08-28T02 2023-08-28T02:13:53.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[109923]: :13:52.451461+0000 mon.a (mon.0 2023-08-28T02:13:53.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[109923]: ) 792 : cluster [DBG] osdmap e80: 8 total, 8 up, 8 in 2023-08-28T02:13:53.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:13:53.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[109923]: 08-28T02:13:52.451712 2023-08-28T02:13:53.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[109923]: +0000 mon.a (mon.0) 793 : audit [INF] 2023-08-28T02:13:53.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[109923]: from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-08-28T02:13:53.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[109923]: cluster 2023-08-28T02 2023-08-28T02:13:53.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[109923]: :13:52.451765+0000 mon.a (mon. 2023-08-28T02:13:53.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[109923]: 0) 794 : cluster [DBG] mgrmap e19: x(active, starting, since 0.00841329s) 2023-08-28T02:13:53.455 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[100258]: audit 2023-08-28T02:13:52.096837+0000 mon.a (mon.0 2023-08-28T02:13:53.455 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[100258]: ) 786 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:13:53.455 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[100258]: audit 2023-08-28T02:13:52.422679+0000 mon.a (mon.0) 787 : audit [DBG] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:13:53.455 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[100258]: audit 2023-08-28T02 2023-08-28T02:13:53.455 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[100258]: :13:52.423596+0000 mon.a (mon.0) 788 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:13:53.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[100258]: audit 2023-08-28T02:13:52.431403+0000 2023-08-28T02:13:53.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[100258]: mon.a (mon.0) 789 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' 2023-08-28T02:13:53.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[100258]: 2023-08-28T02:13:53.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[100258]: audit 2023-08-28T02:13 2023-08-28T02:13:53.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[100258]: :52.438038+0000 mon.a (mon.0) 790 : audit [DBG] 2023-08-28T02:13:53.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[100258]: from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:13:53.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[100258]: audit 2023-08-28T02:13:52.442840+0000 2023-08-28T02:13:53.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[100258]: mon.a (mon.0) 791 : audit [INF] from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-08-28T02:13:53.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[100258]: cluster 2023-08-28T02:13:52.451461+0000 2023-08-28T02:13:53.457 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[100258]: mon.a (mon.0) 792 : cluster [DBG] osdmap e80: 8 total, 8 up, 8 in 2023-08-28T02:13:53.457 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[100258]: 2023-08-28T02:13:53.457 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[100258]: audit 2023-08-28T02:13:52.451712 2023-08-28T02:13:53.457 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[100258]: +0000 mon.a (mon.0) 793 : audit [INF] 2023-08-28T02:13:53.457 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[100258]: from='mgr.14424 172.21.15.143:0/1440630390' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-08-28T02:13:53.457 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[100258]: cluster 2023-08-28T02: 2023-08-28T02:13:53.457 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[100258]: 13:52.451765+0000 mon.a (mon.0) 794 : 2023-08-28T02:13:53.457 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[100258]: cluster [DBG] mgrmap e19: x(active, starting, since 0.00841329s) 2023-08-28T02:13:53.457 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[100708]: debug 2023-08-28T02:13:53.052+0000 7feaf96ec680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-08-28T02:13:53.512 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:53 smithi169 conmon[138400]: debug 2023-08-28T02:13:53.249+0000 7f6254c00200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-08-28T02:13:53.987 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:53 smithi169 conmon[138400]: debug 2023-08-28T02:13:53.513+0000 7f6254c00200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-08-28T02:13:54.098 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[100708]: debug 2023-08-28T02:13:53.806+0000 7feaf96ec680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-08-28T02:13:54.098 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:13:53 smithi143 conmon[100708]: debug 2023-08-28T02:13:53.890+0000 7feaf96ec680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-08-28T02:13:54.356 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:54 smithi169 conmon[100500]: audit 2023-08-28T02:13:53.695195+0000 mon.c (mon.2) 38 : audit [DBG] from='client.? 172.21.15.143:0/2608145259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:54.357 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:54 smithi169 conmon[138400]: debug 2023-08-28T02:13:54.224+0000 7f6254c00200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-08-28T02:13:54.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:54 smithi143 conmon[100258]: audit 2023-08-28T02:13:53.695195+0000 mon.c (mon. 2023-08-28T02:13:54.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:54 smithi143 conmon[100258]: 2) 38 : audit [DBG] from='client.? 172.21.15.143:0/2608145259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:54.450 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:13:54 smithi143 conmon[100708]: debug 2023-08-28T02:13:54.103+0000 7feaf96ec680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-08-28T02:13:54.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:54 smithi143 conmon[109923]: audit 2023-08-28T02:13: 2023-08-28T02:13:54.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:54 smithi143 conmon[109923]: 53.695195+0000 mon.c (mon.2) 38 : audit [DBG] from='client.? 172.21.15.143:0/2608145259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:54.664 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:54 smithi169 conmon[138400]: debug 2023-08-28T02:13:54.358+0000 7f6254c00200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-08-28T02:13:54.664 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:54 smithi169 conmon[138400]: debug 2023-08-28T02:13:54.551+0000 7f6254c00200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-08-28T02:13:54.987 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:54 smithi169 conmon[138400]: debug 2023-08-28T02:13:54.665+0000 7f6254c00200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-08-28T02:13:54.987 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:54 smithi169 conmon[138400]: debug 2023-08-28T02:13:54.882+0000 7f6254c00200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-08-28T02:13:55.338 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:13:55 smithi143 conmon[100708]: debug 2023-08-28T02:13:55.040+0000 7feaf96ec680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-08-28T02:13:55.994 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:13:55 smithi143 conmon[100708]: debug 2023-08-28T02:13:55.167+0000 7feaf96ec680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-08-28T02:13:55.995 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:13:55 smithi143 conmon[100708]: debug 2023-08-28T02:13:55.247+0000 7feaf96ec680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-08-28T02:13:55.996 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:13:55 smithi143 conmon[100708]: debug 2023-08-28T02:13:55.338+0000 7feaf96ec680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-08-28T02:13:55.996 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:13:55 smithi143 conmon[100708]: debug 2023-08-28T02:13:55.534+0000 7feaf96ec680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-08-28T02:13:56.029 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:55 smithi169 conmon[138400]: debug 2023-08-28T02:13:55.748+0000 7f6254c00200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-08-28T02:13:56.030 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:55 smithi169 conmon[138400]: [28/Aug/2023:02:13:55] ENGINE Bus STARTING 2023-08-28T02:13:56.030 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:55 smithi169 conmon[138400]: CherryPy Checker: 2023-08-28T02:13:56.030 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:55 smithi169 conmon[138400]: The Application mounted at '' has an empty config. 2023-08-28T02:13:56.030 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:55 smithi169 conmon[138400]: 2023-08-28T02:13:56.030 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:55 smithi169 conmon[138400]: [28/Aug/2023:02:13:55] ENGINE Serving on http://:::9283 2023-08-28T02:13:56.030 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:55 smithi169 conmon[138400]: [28/Aug/2023:02:13:55] ENGINE Bus STARTED 2023-08-28T02:13:56.030 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:55 smithi169 conmon[138400]: [28/Aug/2023:02:13:55] ENGINE Bus STOPPING 2023-08-28T02:13:56.030 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:55 smithi169 conmon[138400]: [28/Aug/2023:02:13:55] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-08-28T02:13:56.030 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:55 smithi169 conmon[138400]: [28/Aug/2023:02:13:55] ENGINE Bus STOPPED 2023-08-28T02:13:56.030 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:55 smithi169 conmon[138400]: [28/Aug/2023:02:13:55] ENGINE Bus STARTING 2023-08-28T02:13:56.030 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[138400]: debug 2023-08-28T02:13:56.029+0000 7f61b3133700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-28T02:13:56.089 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:13:55 smithi143 conmon[100708]: debug 2023-08-28T02:13:55.825+0000 7feaf96ec680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-08-28T02:13:56.089 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:13:55 smithi143 conmon[100708]: debug 2023-08-28T02:13:55.928+0000 7feaf96ec680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-08-28T02:13:56.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: cluster 2023-08-28T02:13:55.761869+0000 mon.a (mon.0) 2023-08-28T02:13:56.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: 795 : cluster [INF] Active manager daemon x restarted 2023-08-28T02:13:56.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: cluster 2023-08-28T02:13:55.762459+0000 mon.a (mon.0) 796 : cluster [INF] Activating manager daemon x 2023-08-28T02:13:56.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: audit 2023-08-28T02:13:55.763390+0000 mon.b (mon.1) 30 : audit [DBG] from='mgr.? 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-08-28T02:13:56.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: audit 2023-08-28T02:13:55.763901+0000 mon.b (mon.1) 31 : audit [DBG] from='mgr.? 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-28T02:13:56.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: cluster 2023-08-28T02:13:55.768082+0000 mon.a (mon.0) 797 : cluster 2023-08-28T02:13:56.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: [DBG] osdmap e81: 8 total, 8 up, 8 in 2023-08-28T02:13:56.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: cluster 2023-08-28T02:13:55.769062+0000 mon.a (mon 2023-08-28T02:13:56.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: .0) 798 : cluster [DBG] mgrmap e20: x(active, starting, since 0.00686482s) 2023-08-28T02:13:56.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: audit 2023 2023-08-28T02:13:56.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: -08-28T02:13:55 2023-08-28T02:13:56.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: .769267+0000 mon.b (mon.1) 32 2023-08-28T02:13:56.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-28T02:13:56.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: audit 2023-08-28T02:13:56.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: 2023-08-28T02:13:55 2023-08-28T02:13:56.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: .769418+0000 mon.b (mon.1) 2023-08-28T02:13:56.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: 33 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T02:13:56.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: audit 2023-08 2023-08-28T02:13:56.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: -28T02:13:55.769559+0000 2023-08-28T02:13:56.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: mon.b (mon.1) 34 : audit 2023-08-28T02:13:56.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T02:13:56.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: audit 2023-08-28T02:13:56.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: 2023-08-28T02:13: 2023-08-28T02:13:56.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: 55.769693+0000 mon.b (mon. 2023-08-28T02:13:56.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: 1) 35 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T02:13:56.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: audit 2023 2023-08-28T02:13:56.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: -08-28T02:13:55. 2023-08-28T02:13:56.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: 769880+0000 mon.b (mon.1) 36 : 2023-08-28T02:13:56.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi143.spifrj"}]: dispatch 2023-08-28T02:13:56.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: audit 2023-08-28T02:13:56.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: 2023-08-28T02:13: 2023-08-28T02:13:56.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: 55.770176+0000 mon.b (mon.1 2023-08-28T02:13:56.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: ) 37 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi169.yeisur"}]: dispatch 2023-08-28T02:13:56.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: audit 2023- 2023-08-28T02:13:56.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: 08-28T02:13:55.770443+0000 2023-08-28T02:13:56.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: mon.b (mon.1) 38 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-08-28T02:13:56.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: audit 2023- 2023-08-28T02:13:56.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: 08-28T02:13:55.770679+0000 mon.b (mon. 2023-08-28T02:13:56.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: 1) 39 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-28T02:13:56.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: audit 2023-08-28T02: 2023-08-28T02:13:56.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: cluster 2023-08-28T02:13:55 2023-08-28T02:13:56.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: .761869+0000 mon.a (mon.0) 795 : cluster [INF] Active manager daemon x restarted 2023-08-28T02:13:56.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: cluster 2023-08-28T02:13 2023-08-28T02:13:56.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: :55.762459+0000 mon.a (mon.0) 796 : cluster 2023-08-28T02:13:56.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: [INF] Activating manager daemon x 2023-08-28T02:13:56.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: audit 2023 2023-08-28T02:13:56.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: -08-28T02:13:55 2023-08-28T02:13:56.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: .763390+0000 mon.b (mon.1) 30 2023-08-28T02:13:56.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: : audit [DBG] from='mgr.? 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-08-28T02:13:56.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: audit 2023-08-28T02:13:56.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: 2023-08 2023-08-28T02:13:56.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: -28T02:13:55.763901+0000 2023-08-28T02:13:56.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: mon.b (mon.1) 31 : audit [DBG] from='mgr.? 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-28T02:13:56.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: 2023-08-28T02:13:56.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: cluster 2023-08-28T02:13:56.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: 2023-08-28T02:13:55 2023-08-28T02:13:56.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: .768082+0000 mon.a (mon.0) 797 : cluster 2023-08-28T02:13:56.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: [DBG] osdmap e81: 8 total, 8 up, 8 in 2023-08-28T02:13:56.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: cluster 2023- 2023-08-28T02:13:56.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: 08-28T02:13:55.769062+0000 mon.a (mon 2023-08-28T02:13:56.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: .0) 798 : cluster [DBG] mgrmap e20: x(active, starting, since 0.00686482s) 2023-08-28T02:13:56.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: audit 2023 2023-08-28T02:13:56.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: -08-28T02:13:55.769267+0000 mon.b 2023-08-28T02:13:56.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: (mon.1) 32 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-28T02:13:56.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: 2023-08-28T02:13:56.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: audit 2023-08-28T02:13:56.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: 2023-08-28T02:13 2023-08-28T02:13:56.457 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: :55.769418+0000 mon.b (mon.1) 2023-08-28T02:13:56.457 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: 33 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T02:13:56.457 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:13:56.457 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: 08-28T02:13:55.769559+0000 mon.b 2023-08-28T02:13:56.457 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: (mon.1) 34 : audit [DBG] 2023-08-28T02:13:56.457 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T02:13:56.457 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: audit 2023-08-28T02:13:56.457 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: 2023-08-28T02:13:55.769693 2023-08-28T02:13:56.457 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: +0000 mon.b (mon.1) 35 : audit 2023-08-28T02:13:56.457 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T02:13:56.457 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: audit 2023-08-28T02:13:56.457 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: 2023-08-28T02: 2023-08-28T02:13:56.457 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: 13:55.769880+0000 mon.b ( 2023-08-28T02:13:56.458 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: mon.1) 36 : audit [DBG] 2023-08-28T02:13:56.458 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi143.spifrj"}]: dispatch 2023-08-28T02:13:56.458 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: audit 2023-08-28T02:13:56.458 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: 2023-08-28T02:13: 2023-08-28T02:13:56.458 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: 55.770176+0000 mon.b (mon. 2023-08-28T02:13:56.458 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: 1) 37 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi169.yeisur"}]: dispatch 2023-08-28T02:13:56.458 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: audit 2023-08- 2023-08-28T02:13:56.458 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: 28T02:13:55.770443+0000 mon.b 2023-08-28T02:13:56.459 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: 13:55.770950+0000 mon.b (mon.1) 40 : audit 2023-08-28T02:13:56.459 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-28T02:13:56.459 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: audit 2023-08-28T02:13:55.771222+0000 2023-08-28T02:13:56.459 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: mon.b (mon.1) 41 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-28T02:13:56.459 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: audit 2023- 2023-08-28T02:13:56.459 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: 08-28T02:13:55.771490+0000 mon.b (mon 2023-08-28T02:13:56.459 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: .1) 42 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-28T02:13:56.459 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: audit 2023-08-28 2023-08-28T02:13:56.459 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: T02:13:55.771748+0000 mon.b (mon.1) 43 2023-08-28T02:13:56.459 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-28T02:13:56.460 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: audit 2023-08 2023-08-28T02:13:56.460 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: -28T02:13:55.772015+0000 mon.b (mon 2023-08-28T02:13:56.460 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: .1) 44 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-28T02:13:56.460 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: audit 2023-08-28T02:13:56.460 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: 2023-08- 2023-08-28T02:13:56.460 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: 28T02:13:55.772287+0000 mon.b ( 2023-08-28T02:13:56.460 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: mon.1) 45 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-28T02:13:56.460 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: audit 2023-08-28T02:13:56.460 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: 2023-08-28T02 2023-08-28T02:13:56.460 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: :13:55.772547+0000 mon.b (mon.1) 46 : 2023-08-28T02:13:56.460 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-28T02:13:56.461 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: audit 2023-08-28T02:13:56.461 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: 2023-08-28T02:13:55.772855+0000 2023-08-28T02:13:56.461 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: mon.b (mon.1) 47 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-08-28T02:13:56.461 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: audit 2023-08-28T02:13:56.461 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: 2023-08-28T02:13:55.773188+0000 2023-08-28T02:13:56.461 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: mon.b (mon.1) 48 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-08-28T02:13:56.461 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: audit 2023 2023-08-28T02:13:56.461 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: -08-28T02:13:55.773851+0000 mon.b 2023-08-28T02:13:56.461 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: (mon.1) 49 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-08-28T02:13:56.461 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: audit 2023-08-28T02:13:56.461 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: 2023-08-28T02:13: 2023-08-28T02:13:56.462 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: 55.774022+0000 mon.b (mon.1) 50 : audit 2023-08-28T02:13:56.462 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-08-28T02:13:56.462 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: audit 2023-08-28T02: 2023-08-28T02:13:56.462 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: 13:55.774303+0000 mon.b (mon.1) 51 : 2023-08-28T02:13:56.462 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-08-28T02:13:56.462 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: cluster 2023-08-28T02:13 2023-08-28T02:13:56.462 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: :55.929783+0000 mon.a (mon.0) 799 : 2023-08-28T02:13:56.462 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: cluster [INF] Manager daemon x is now available 2023-08-28T02:13:56.462 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: audit 2023-08-28T02 2023-08-28T02:13:56.462 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: :13:55.951039+0000 mon.a (mon.0) 800 2023-08-28T02:13:56.462 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:13:56.462 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: audit 2023-08-28 2023-08-28T02:13:56.462 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: T02:13:55.955419+0000 mon.b (mon.1) 2023-08-28T02:13:56.463 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: 52 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:13:56.463 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: audit 2023-08 2023-08-28T02:13:56.463 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: -28T02:13:55.957813+0000 mon.b (mon.1 2023-08-28T02:13:56.463 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: ) 53 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:13:56.463 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: audit 2023-08-28T02:13:56.463 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: 2023-08-28T02:13:55 2023-08-28T02:13:56.463 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: .965039+0000 mon.b (mon.1) 54 : audit [INF] 2023-08-28T02:13:56.463 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-08-28T02:13:56.463 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: audit 2023-08-28T02:13: 2023-08-28T02:13:56.463 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: 55.965418+0000 mon.a (mon.0) 801 : audit [INF] 2023-08-28T02:13:56.463 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: from='mgr.25369 ' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-08-28T02:13:56.463 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: audit 2023-08-28T02:13:55 2023-08-28T02:13:56.463 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: .979306+0000 mon.b (mon.1) 55 : audit [DBG] 2023-08-28T02:13:56.464 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:13:56.464 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: audit 2023-08-28T02:13:55. 2023-08-28T02:13:56.464 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: 985585+0000 mon.b (mon.1) 56 : audit [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-08-28T02:13:56.464 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: 2023-08-28T02:13:56.464 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: audit 2023-08-28T02:13: 2023-08-28T02:13:56.464 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: 55.985948+0000 mon.a (mon.0) 802 : audit [INF] 2023-08-28T02:13:56.464 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: from='mgr.25369 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-08-28T02:13:56.464 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: audit 2023-08-28T02:13 2023-08-28T02:13:56.464 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: :56.005280+0000 mon.b (mon.1) 57 : audit 2023-08-28T02:13:56.464 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-08-28T02:13:56.464 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: audit 2023-08-28T02 2023-08-28T02:13:56.464 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: :13:56.005740+0000 mon.a (mon.0) 803 : 2023-08-28T02:13:56.464 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: audit [INF] from='mgr.25369 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-08-28T02:13:56.465 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: audit 2023-08 2023-08-28T02:13:56.465 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: -28T02:13:56.011037+0000 mon.a (mon 2023-08-28T02:13:56.465 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: .0) 804 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:13:56.465 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: audit 2023-08-28T02:13:56.465 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: 2023-08-28T02: 2023-08-28T02:13:56.465 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: 13:56.054803+0000 mon.c (mon.2) 39 : audit 2023-08-28T02:13:56.465 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100258]: [DBG] from='client.? 172.21.15.143:0/1326038915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:56.465 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: (mon.1) 38 : audit [DBG] 2023-08-28T02:13:56.466 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-08-28T02:13:56.466 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: audit 2023-08-28T 2023-08-28T02:13:56.466 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: 02:13:55.770679+0000 mon.b 2023-08-28T02:13:56.466 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: (mon.1) 39 : audit 2023-08-28T02:13:56.466 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-28T02:13:56.466 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: audit 2023-08-28T02:13:56.466 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: 2023-08-28T02:13: 2023-08-28T02:13:56.466 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: 55.770950+0000 mon.b (mon. 2023-08-28T02:13:56.466 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: 1) 40 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-28T02:13:56.466 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: audit 2023-08-28T02:13:56.466 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: 2023-08-28T02: 2023-08-28T02:13:56.466 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: 13:55.771222+0000 mon.b ( 2023-08-28T02:13:56.467 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: mon.1) 41 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-28T02:13:56.467 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: 2023-08-28T02:13:56.467 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: audit 2023-08-28T 2023-08-28T02:13:56.467 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: 02:13:55.771490+0000 mon.b 2023-08-28T02:13:56.467 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: (mon.1) 42 : audit [DBG] 2023-08-28T02:13:56.467 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-28T02:13:56.467 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: audit 2023-08-28T02:13:56.467 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: 2023-08-28T02: 2023-08-28T02:13:56.467 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: 13:55.771748+0000 mon.b (mon.1 2023-08-28T02:13:56.467 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: ) 43 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-28T02:13:56.467 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: audit 2023-08-28T02:13:56.467 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: 2023-08-28T02:13:55.772015+0000 2023-08-28T02:13:56.468 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: mon.b (mon.1) 44 : audit [DBG] 2023-08-28T02:13:56.468 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-28T02:13:56.468 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: audit 2023-08-28T02:13:56.468 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: 2023-08-28T02:13:55 2023-08-28T02:13:56.468 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: .772287+0000 mon.b (mon.1 2023-08-28T02:13:56.468 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: ) 45 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-28T02:13:56.468 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: audit 2023-08-28 2023-08-28T02:13:56.468 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: T02:13:55.772547+0000 mon.b (mon.1) 46 : audit 2023-08-28T02:13:56.468 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-28T02:13:56.468 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: audit 2023-08-28T02:13:55.772855+0000 mon.b ( 2023-08-28T02:13:56.468 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: mon.1) 47 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-08-28T02:13:56.468 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: audit 2023-08-28T02:13:56.469 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: 2023-08-28T02:13:55.773188+0000 mon.b (mon.1) 48 : audit [DBG] 2023-08-28T02:13:56.469 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-08-28T02:13:56.469 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: audit 2023-08-28T02:13:55.773851+0000 mon.b 2023-08-28T02:13:56.469 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: (mon.1) 49 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-08-28T02:13:56.469 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: audit 2023-08-28T02:13: 2023-08-28T02:13:56.469 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: 55.774022+0000 mon.b (mon.1) 50 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-08-28T02:13:56.469 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: audit 2023-08-28T02 2023-08-28T02:13:56.469 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: :13:55.774303+0000 mon.b (mon.1) 51 : audit [DBG] 2023-08-28T02:13:56.469 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-08-28T02:13:56.469 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: cluster 2023-08-28T02:13:56.469 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: 2023-08-28T02:13:55.929783+0000 mon.a (mon.0) 799 : cluster [INF] 2023-08-28T02:13:56.470 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: Manager daemon x is now available 2023-08-28T02:13:56.470 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:13:56.470 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: 08-28T02:13:55.951039+0000 mon.a (mon.0) 800 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:13:56.470 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: 2023-08-28T02:13:56.470 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: audit 2023-08 2023-08-28T02:13:56.470 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: -28T02:13:55.955419+0000 mon.b (mon.1) 52 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:13:56.470 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: audit 2023-08-28T 2023-08-28T02:13:56.470 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: 02:13:55.957813+0000 mon.b (mon.1) 53 : audit 2023-08-28T02:13:56.470 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:13:56.470 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: audit 2023-08-28T02:13:56.471 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: 2023-08-28T02:13:55.965039+0000 mon.b (mon.1) 54 : audit [INF] 2023-08-28T02:13:56.471 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-08-28T02:13:56.471 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: audit 2023 2023-08-28T02:13:56.471 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: -08-28T02:13:55.965418+0000 mon.a (mon.0) 801 : audit [INF] from='mgr.25369 ' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-08-28T02:13:56.471 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: 2023-08-28T02:13:56.471 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: audit 2023-08-28 2023-08-28T02:13:56.471 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: T02:13:55.979306+0000 mon.b (mon.1) 55 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:13:56.471 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: audit 2023-08-28T02 2023-08-28T02:13:56.471 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: :13:55.985585+0000 mon.b (mon.1) 56 : audit [INF] 2023-08-28T02:13:56.471 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-08-28T02:13:56.472 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: audit 2023-08-28T02:13:56.472 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: 2023-08-28T02:13:55.985948+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.25369 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-08-28T02:13:56.472 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: 2023-08-28T02:13:56.472 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: audit 2023-08- 2023-08-28T02:13:56.472 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: 28T02:13:56.005280+0000 mon.b (mon.1) 57 : audit [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-08-28T02:13:56.472 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: audit 2023-08-28T02:13:56. 2023-08-28T02:13:56.472 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: 005740+0000 mon.a (mon.0) 803 : audit [INF] from='mgr.25369 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-08-28T02:13:56.472 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: audit 2023-08-28 2023-08-28T02:13:56.472 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: T02:13:56.011037+0000 mon.a (mon.0) 804 : audit 2023-08-28T02:13:56.472 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:13:56.473 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: audit 2023-08-28T02:13:56.473 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: 2023-08-28T02:13:56.054803+0000 mon.c (mon.2) 39 : audit [DBG] 2023-08-28T02:13:56.473 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[109923]: from='client.? 172.21.15.143:0/1326038915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:56.487 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[138400]: debug 2023-08-28T02:13:56.029+0000 7f61b3133700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-28T02:13:56.488 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[138400]: debug 2023-08-28T02:13:56.029+0000 7f61b3133700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-28T02:13:56.488 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[138400]: debug 2023-08-28T02:13:56.029+0000 7f61b3133700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-28T02:13:56.488 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[138400]: debug 2023-08-28T02:13:56.029+0000 7f61b3133700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-28T02:13:56.488 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[138400]: debug 2023-08-28T02:13:56.029+0000 7f61b6139700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-28T02:13:56.488 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[138400]: debug 2023-08-28T02:13:56.029+0000 7f61b6139700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-28T02:13:56.488 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[138400]: debug 2023-08-28T02:13:56.029+0000 7f61b6139700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-28T02:13:56.488 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[138400]: debug 2023-08-28T02:13:56.029+0000 7f61b6139700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-28T02:13:56.488 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[138400]: debug 2023-08-28T02:13:56.029+0000 7f61b6139700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-28T02:13:56.488 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[138400]: [28/Aug/2023:02:13:56] ENGINE Serving on http://:::9283 2023-08-28T02:13:56.488 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[138400]: [28/Aug/2023:02:13:56] ENGINE Bus STARTED 2023-08-28T02:13:56.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: cluster 2023-08-28T02:13:55.761869+0000 mon.a (mon.0) 795 : 2023-08-28T02:13:56.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: cluster [INF] Active manager daemon x restarted 2023-08-28T02:13:56.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: cluster 2023-08-28T02:13:55.762459+0000 mon.a (mon.0) 796 : cluster [INF] Activating manager daemon x 2023-08-28T02:13:56.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: audit 2023-08-28T02:13:55.763390+0000 mon.b (mon.1) 30 : audit [DBG] from='mgr.? 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-08-28T02:13:56.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: audit 2023-08-28T02:13:55.763901+0000 mon.b (mon.1) 31 : 2023-08-28T02:13:56.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: audit [DBG] from='mgr.? 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-28T02:13:56.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: cluster 2023-08-28T02:13:55. 2023-08-28T02:13:56.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: 768082+0000 mon.a (mon.0) 797 : cluster [DBG] 2023-08-28T02:13:56.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: osdmap e81: 8 total, 8 up, 8 in 2023-08-28T02:13:56.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: cluster 2023-08-28T02:13:55. 2023-08-28T02:13:56.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: 769062+0000 mon.a (mon.0) 798 : cluster [DBG] 2023-08-28T02:13:56.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: mgrmap e20: x(active, starting, since 0.00686482s) 2023-08-28T02:13:56.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: audit 2023-08-28T02:13:56.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: 2023-08-28T02:13:55.769267+0000 mon.b (mon 2023-08-28T02:13:56.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: .1) 32 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-28T02:13:56.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: audit 2023-08-28 2023-08-28T02:13:56.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: T02:13:55.769418+0000 mon.b (mon 2023-08-28T02:13:56.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: .1) 33 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T02:13:56.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: audit 2023-08-28 2023-08-28T02:13:56.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: T02:13:55.769559+0000 mon.b (mon 2023-08-28T02:13:56.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: .1) 34 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T02:13:56.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: audit 2023-08- 2023-08-28T02:13:56.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: 28T02:13:55.769693+0000 mon.b (mon. 2023-08-28T02:13:56.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: 1) 35 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T02:13:56.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: audit 2023-08- 2023-08-28T02:13:56.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: 28T02:13:55.769880+0000 mon.b ( 2023-08-28T02:13:56.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: mon.1) 36 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi143.spifrj"}]: dispatch 2023-08-28T02:13:56.492 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: audit 2023-08- 2023-08-28T02:13:56.492 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: 28T02:13:55.770176+0000 mon.b ( 2023-08-28T02:13:56.492 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: mon.1) 37 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi169.yeisur"}]: dispatch 2023-08-28T02:13:56.492 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: audit 2023-08- 2023-08-28T02:13:56.492 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: 28T02:13:55.770443+0000 mon.b ( 2023-08-28T02:13:56.492 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: mon.1) 38 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-08-28T02:13:56.492 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: audit 2023-08- 2023-08-28T02:13:56.492 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: 28T02:13:55.770679+0000 mon.b (mon.1) 39 2023-08-28T02:13:56.492 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-28T02:13:56.492 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: audit 2023-08-28T02:13: 2023-08-28T02:13:56.493 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: 55.770950+0000 mon.b (mon.1) 40 : audit 2023-08-28T02:13:56.493 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-28T02:13:56.493 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: audit 2023-08-28T02:13: 2023-08-28T02:13:56.493 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: 55.771222+0000 mon.b (mon.1) 41 : 2023-08-28T02:13:56.493 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-28T02:13:56.493 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: audit 2023-08-28T02: 2023-08-28T02:13:56.493 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: 13:55.771490+0000 mon.b (mon.1) 2023-08-28T02:13:56.493 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: 42 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-28T02:13:56.493 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: audit 2023-08-28T02:13:56.493 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: 2023-08-28T02: 2023-08-28T02:13:56.493 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: 13:55.771748+0000 mon.b (mon.1) 43 2023-08-28T02:13:56.494 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-28T02:13:56.494 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: audit 2023-08-28T 2023-08-28T02:13:56.494 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: 02:13:55.772015+0000 mon.b (mon. 2023-08-28T02:13:56.494 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: 1) 44 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-28T02:13:56.494 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: audit 2023-08-28 2023-08-28T02:13:56.494 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: T02:13:55.772287+0000 mon.b (mon. 2023-08-28T02:13:56.494 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: 1) 45 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-28T02:13:56.494 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: audit 2023-08- 2023-08-28T02:13:56.494 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: 28T02:13:55.772547+0000 mon.b ( 2023-08-28T02:13:56.494 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: mon.1) 46 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-28T02:13:56.494 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: audit 2023- 2023-08-28T02:13:56.494 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: 08-28T02:13:55.772855+0000 2023-08-28T02:13:56.495 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: mon.b (mon.1) 47 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-08-28T02:13:56.495 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: audit 2023 2023-08-28T02:13:56.495 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: -08-28T02:13:55.773188+0000 2023-08-28T02:13:56.495 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: mon.b (mon.1) 48 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-08-28T02:13:56.495 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: audit 2023- 2023-08-28T02:13:56.495 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: 08-28T02:13:55.773851+0000 2023-08-28T02:13:56.495 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: mon.b (mon.1) 49 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-08-28T02:13:56.495 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: audit 2023-08-28T02:13 2023-08-28T02:13:56.496 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: :55.774022+0000 mon.b (mon.1) 50 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-08-28T02:13:56.496 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: audit 2023-08-28T02:13:56.496 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: 2023-08-28T02:13:55.774303+0000 mon.b (mon. 2023-08-28T02:13:56.496 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: 1) 51 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-08-28T02:13:56.496 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: cluster 2023-08-28 2023-08-28T02:13:56.496 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: T02:13:55.929783+0000 mon.a (mon.0) 2023-08-28T02:13:56.496 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: 799 : cluster [INF] Manager daemon x is now available 2023-08-28T02:13:56.496 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: audit 2023-08-28T 2023-08-28T02:13:56.496 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: 02:13:55.951039+0000 mon.a (mon.0) 800 : audit 2023-08-28T02:13:56.496 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:13:56.496 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: audit 2023-08-28T02: 2023-08-28T02:13:56.496 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: 13:55.955419+0000 mon.b (mon.1) 52 2023-08-28T02:13:56.497 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:13:56.497 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: audit 2023-08-28 2023-08-28T02:13:56.497 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: T02:13:55.957813+0000 mon.b (mon. 2023-08-28T02:13:56.497 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: 1) 53 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:13:56.497 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: audit 2023-08 2023-08-28T02:13:56.497 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: -28T02:13:55.965039+0000 mon.b ( 2023-08-28T02:13:56.497 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: mon.1) 54 : audit [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-08-28T02:13:56.497 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: audit 2023- 2023-08-28T02:13:56.497 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: 08-28T02:13:55.965418+0000 mon.a (mon 2023-08-28T02:13:56.497 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: .0) 801 : audit [INF] from='mgr.25369 ' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-08-28T02:13:56.497 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: audit 2023- 2023-08-28T02:13:56.498 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: 08-28T02:13:55.979306+0000 mon.b 2023-08-28T02:13:56.498 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: (mon.1) 55 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:13:56.498 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: audit 2023- 2023-08-28T02:13:56.498 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: 08-28T02:13:55.985585+0000 mon.b ( 2023-08-28T02:13:56.498 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: mon.1) 56 : audit [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-08-28T02:13:56.498 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: audit 2023-08-28T02:13:56.498 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: 2023-08-28T02:13:55.985948+0000 2023-08-28T02:13:56.498 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: mon.a (mon.0) 802 : audit [INF] from='mgr.25369 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-08-28T02:13:56.498 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: audit 2023- 2023-08-28T02:13:56.498 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: 08-28T02:13:56.005280+0000 mon.b (mon.1 2023-08-28T02:13:56.498 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: ) 57 : audit [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-08-28T02:13:56.499 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: audit 2023-08 2023-08-28T02:13:56.499 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: -28T02:13:56.005740+0000 mon.a ( 2023-08-28T02:13:56.499 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: mon.0) 803 : audit [INF] from='mgr.25369 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-08-28T02:13:56.499 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: audit 2023- 2023-08-28T02:13:56.499 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: 08-28T02:13:56.011037+0000 mon.a 2023-08-28T02:13:56.499 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: (mon.0) 804 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:13:56.499 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: audit 2023 2023-08-28T02:13:56.499 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: -08-28T02:13:56.054803+0000 2023-08-28T02:13:56.499 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: mon.c (mon.2) 39 : audit [DBG] from='client.? 172.21.15.143:0/1326038915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:56.499 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:56 smithi169 conmon[100500]: 2023-08-28T02:13:56.951 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100708]: debug 2023-08-28T02:13:56.691+0000 7feaf96ec680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-08-28T02:13:56.952 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100708]: debug 2023-08-28T02:13:56.777+0000 7feaf96ec680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-08-28T02:13:56.952 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:13:56 smithi143 conmon[100708]: debug 2023-08-28T02:13:56.870+0000 7feaf96ec680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-08-28T02:13:57.311 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[100708]: debug 2023-08-28T02:13:57.049+0000 7feaf96ec680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-08-28T02:13:57.574 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[100708]: debug 2023-08-28T02:13:57.310+0000 7feaf96ec680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-08-28T02:13:57.575 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[100708]: debug 2023-08-28T02:13:57.442+0000 7feaf96ec680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-08-28T02:13:57.850 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[100708]: debug 2023-08-28T02:13:57.582+0000 7feaf96ec680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-08-28T02:13:57.850 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[100708]: debug 2023-08-28T02:13:57.758+0000 7feaf96ec680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-08-28T02:13:57.850 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[100708]: debug 2023-08-28T02:13:57.850+0000 7feaf96ec680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-08-28T02:13:57.850 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[100258]: cephadm 2023-08-28T02:13:56.353062+0000 mgr.x (mgr.25369) 1 : cephadm [INF] Deploying cephadm binary to smithi169 2023-08-28T02:13:57.850 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[100258]: 2023-08-28T02:13:57.851 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[100258]: cephadm 2023-08-28T02:13:56.353821+0000 mgr.x (mgr.25369) 2 : cephadm [INF] Deploying cephadm binary to smithi143 2023-08-28T02:13:57.851 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[100258]: audit 2023-08-28T02:13:56.576404 2023-08-28T02:13:57.851 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[100258]: +0000 mon.a (mon.0) 805 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:13:57.851 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[100258]: audit 2023-08-28T02:13:56.593014+0000 2023-08-28T02:13:57.851 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[100258]: mon.a (mon.0) 806 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:13:57.851 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[100258]: 2023-08-28T02:13:57.851 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[100258]: cluster 2023-08-28T02:13:56 2023-08-28T02:13:57.851 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[100258]: .770000+0000 mon.a (mon.0) 807 : cluster 2023-08-28T02:13:57.851 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[100258]: [DBG] mgrmap e21: x(active, since 1.00781s) 2023-08-28T02:13:57.851 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[100258]: audit 2023-08-28T02: 2023-08-28T02:13:57.852 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[100258]: 13:56.953050+0000 mon.a (mon.0) 808 : 2023-08-28T02:13:57.852 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[100258]: audit [INF] from='mgr.25369 ' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2023-08-28T02:13:57.852 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[100258]: cluster 2023-08-28T02 2023-08-28T02:13:57.852 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[100258]: :13:56.953135+0000 mon.a (mon.0) 2023-08-28T02:13:57.852 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[100258]: 809 : cluster [DBG] osdmap e82: 8 total, 8 up, 8 in 2023-08-28T02:13:57.852 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[100258]: audit 2023-08-28T02:13:57.852 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[100258]: 2023-08-28T02: 2023-08-28T02:13:57.852 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[100258]: 13:56.963272+0000 mon.b (mon.1) 2023-08-28T02:13:57.852 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[100258]: 58 : audit [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-08-28T02:13:57.852 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[100258]: audit 2023- 2023-08-28T02:13:57.853 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[100258]: 08-28T02:13:56.963595+0000 mon.a ( 2023-08-28T02:13:57.853 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[100258]: mon.0) 810 : audit [INF] from='mgr.25369 ' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-08-28T02:13:57.853 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[109923]: cephadm 2023-08-28T02:13: 2023-08-28T02:13:57.853 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[109923]: 56.353062+0000 mgr.x (mgr.25369) 1 : cephadm [INF] Deploying cephadm binary to smithi169 2023-08-28T02:13:57.853 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[109923]: cephadm 2023-08 2023-08-28T02:13:57.853 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[109923]: -28T02:13:56.353821+0000 2023-08-28T02:13:57.853 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[109923]: mgr.x (mgr.25369) 2 2023-08-28T02:13:57.853 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[109923]: : cephadm [INF] Deploying cephadm binary to smithi143 2023-08-28T02:13:57.853 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:13:57.854 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[109923]: 08-28T02:13:56. 2023-08-28T02:13:57.854 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[109923]: 576404+0000 mon.a (mon.0) 2023-08-28T02:13:57.854 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[109923]: 805 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:13:57.854 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[109923]: audit 2023 2023-08-28T02:13:57.854 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[109923]: -08-28T02:13:56 2023-08-28T02:13:57.854 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[109923]: .593014+0000 mon.a (mon.0 2023-08-28T02:13:57.854 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[109923]: ) 806 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:13:57.854 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[109923]: cluster 2023 2023-08-28T02:13:57.854 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[109923]: -08-28T02:13:56. 2023-08-28T02:13:57.854 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[109923]: 770000+0000 mon.a (mon.0) 2023-08-28T02:13:57.855 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[109923]: 807 : cluster [DBG] mgrmap e21: x(active, since 1.00781s) 2023-08-28T02:13:57.855 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[109923]: audit 2023 2023-08-28T02:13:57.855 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[109923]: -08-28T02:13:56 2023-08-28T02:13:57.855 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[109923]: .953050+0000 mon.a (mon. 2023-08-28T02:13:57.855 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[109923]: 0) 808 : audit [INF] from='mgr.25369 ' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2023-08-28T02:13:57.855 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[109923]: 2023-08-28T02:13:57.855 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[109923]: cluster 2023-08-28T02:13:57.855 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[109923]: 2023-08- 2023-08-28T02:13:57.855 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[109923]: 28T02:13:56.953135 2023-08-28T02:13:57.855 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[109923]: +0000 mon.a (mon.0) 2023-08-28T02:13:57.855 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[109923]: 809 : cluster [DBG] osdmap e82: 8 total, 8 up, 8 in 2023-08-28T02:13:57.856 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:13:57.856 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[109923]: 08-28T02:13:56.963272+0000 mon.b (mon 2023-08-28T02:13:57.856 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[109923]: .1) 58 : audit [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-08-28T02:13:57.856 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[109923]: 2023-08-28T02:13:57.856 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[109923]: audit 2023-08-28T02:13:57.856 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[109923]: 2023- 2023-08-28T02:13:57.856 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[109923]: 08-28T02:13:56 2023-08-28T02:13:57.856 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[109923]: .963595+0000 mon.a (mon.0 2023-08-28T02:13:57.856 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[109923]: ) 810 : audit [INF] from='mgr.25369 ' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-08-28T02:13:57.856 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[109923]: 2023-08-28T02:13:57.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:57 smithi169 conmon[100500]: cephadm 2023-08-28T02:13:56.353062+0000 mgr.x (mgr.25369) 1 : cephadm [INF] Deploying cephadm binary to smithi169 2023-08-28T02:13:57.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:57 smithi169 conmon[100500]: cephadm 2023-08-28T02:13:56.353821+0000 mgr.x (mgr.25369) 2 : cephadm [INF] Deploying cephadm binary to smithi143 2023-08-28T02:13:57.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:57 smithi169 conmon[100500]: audit 2023-08-28T02:13:56.576404+0000 mon.a (mon.0) 805 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:13:57.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:57 smithi169 conmon[100500]: audit 2023-08-28T02:13:56.593014+0000 mon.a (mon.0) 806 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:13:57.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:57 smithi169 conmon[100500]: cluster 2023-08-28T02:13:56.770000+0000 mon.a (mon.0) 807 : cluster [DBG] mgrmap e21: x(active, since 1.00781s) 2023-08-28T02:13:57.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:57 smithi169 conmon[100500]: audit 2023-08-28T02:13:56.953050+0000 mon.a (mon.0) 808 : audit [INF] from='mgr.25369 ' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2023-08-28T02:13:57.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:57 smithi169 conmon[100500]: cluster 2023-08-28T02:13:56. 2023-08-28T02:13:57.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:57 smithi169 conmon[100500]: 953135+0000 mon.a (mon.0) 809 : cluster [DBG] osdmap e82: 8 total, 8 up, 8 in 2023-08-28T02:13:57.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:57 smithi169 conmon[100500]: 2023-08-28T02:13:57.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:57 smithi169 conmon[100500]: audit 2023- 2023-08-28T02:13:57.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:57 smithi169 conmon[100500]: 08-28T02:13:56 2023-08-28T02:13:57.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:57 smithi169 conmon[100500]: .963272+0000 mon.b (mon. 2023-08-28T02:13:57.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:57 smithi169 conmon[100500]: 1) 58 : audit [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-08-28T02:13:57.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:57 smithi169 conmon[100500]: 2023-08-28T02:13:57.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:57 smithi169 conmon[100500]: audit 2023-08 2023-08-28T02:13:57.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:57 smithi169 conmon[100500]: -28T02:13:56. 2023-08-28T02:13:57.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:57 smithi169 conmon[100500]: 963595+0000 mon.a (mon.0 2023-08-28T02:13:57.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:57 smithi169 conmon[100500]: ) 810 : audit [INF] from='mgr.25369 ' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-08-28T02:13:58.201 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[100708]: [28/Aug/2023:02:13:57] ENGINE Bus STARTING 2023-08-28T02:13:58.201 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[100708]: CherryPy Checker: 2023-08-28T02:13:58.201 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[100708]: The Application mounted at '' has an empty config. 2023-08-28T02:13:58.201 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[100708]: 2023-08-28T02:13:58.201 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[100708]: [28/Aug/2023:02:13:57] ENGINE Serving on http://:::9283 2023-08-28T02:13:58.202 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:13:57 smithi143 conmon[100708]: [28/Aug/2023:02:13:57] ENGINE Bus STARTED 2023-08-28T02:13:58.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[100258]: cluster 2023-08-28T02:13:56.780652+0000 mgr.x (mgr.25369 2023-08-28T02:13:58.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[100258]: ) 3 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:58.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[100258]: cephadm 2023-08-28T02:13:57.500019+0000 mgr.x (mgr.25369) 4 : cephadm [INF] [28/Aug/2023:02:13:57] ENGINE Bus STARTING 2023-08-28T02:13:58.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[100258]: cephadm 2023-08-28T02:13:57.614258+0000 mgr.x (mgr.25369) 5 : cephadm [INF] [28/Aug/2023:02:13:57] ENGINE Serving on http://172.21.15.169:8765 2023-08-28T02:13:58.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[100258]: cephadm 2023-08-28T02:13:57.754770+0000 mgr.x (mgr.25369) 6 : cephadm [INF] [28/Aug/2023:02:13:57] ENGINE Serving on https://172.21.15.169:7150 2023-08-28T02:13:58.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[100258]: cephadm 2023-08-28T02:13:57.754930+0000 mgr.x (mgr.25369) 7 : cephadm [INF] [28/Aug/2023:02:13:57] ENGINE Bus STARTED 2023-08-28T02:13:58.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[100258]: cluster 2023-08-28T02:13:57.864971+0000 mon.a (mon.0) 811 : cluster [DBG] Standby manager daemon y started 2023-08-28T02:13:58.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[100258]: audit 2023-08-28T02:13:57.867676+0000 mon.a (mon.0) 2023-08-28T02:13:58.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[100258]: 812 : audit [DBG] from='mgr.? 172.21.15.143:0/2603502258' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-08-28T02:13:58.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[100258]: audit 2023-08-28T02:13:57.868467+0000 mon.a (mon.0) 813 : audit [DBG] from='mgr.? 172.21.15.143:0/2603502258' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-28T02:13:58.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[100258]: audit 2023-08-28T02:13:57 2023-08-28T02:13:58.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[100258]: .870862+0000 mon.a (mon.0) 814 : audit [DBG] from='mgr.? 172.21.15.143:0/2603502258' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-08-28T02:13:58.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[100258]: audit 2023-08- 2023-08-28T02:13:58.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[100258]: 28T02:13:57.871215+0000 mon.a (mon.0) 815 : audit [DBG] from='mgr.? 172.21.15.143:0/2603502258' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-08-28T02:13:58.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[100258]: audit 2023-08-28T02: 2023-08-28T02:13:58.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[100258]: 13:57.953933+0000 mon.a (mon.0) 816 : audit 2023-08-28T02:13:58.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[100258]: [INF] from='mgr.25369 ' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-08-28T02:13:58.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[100258]: cluster 2023 2023-08-28T02:13:58.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[100258]: -08-28T02:13:57.954047+0000 mon.a (mon.0) 817 : cluster [DBG] 2023-08-28T02:13:58.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[100258]: osdmap e83: 8 total, 8 up, 8 in 2023-08-28T02:13:58.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[100258]: audit 2023- 2023-08-28T02:13:58.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[100258]: 08-28T02:13:58.483692+0000 mon.c (mon.2) 40 : audit [DBG] from='client.? 172.21.15.143:0/534499021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:58.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[109923]: cluster 2023-08-28T02:13:56.780652+0000 2023-08-28T02:13:58.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[109923]: mgr.x (mgr.25369) 3 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:58.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[109923]: cephadm 2023-08-28T02:13:57.500019+0000 mgr.x 2023-08-28T02:13:58.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[109923]: (mgr.25369) 4 : cephadm [INF] [28/Aug/2023:02:13:57] ENGINE Bus STARTING 2023-08-28T02:13:58.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[109923]: cephadm 2023-08-28T02:13:58.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[109923]: 2023-08-28T02:13:57.614258+0000 mgr.x 2023-08-28T02:13:58.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[109923]: (mgr.25369) 5 : cephadm [INF] [28/Aug/2023:02:13:57] ENGINE Serving on http://172.21.15.169:8765 2023-08-28T02:13:58.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[109923]: cephadm 2023 2023-08-28T02:13:58.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[109923]: -08-28T02:13: 2023-08-28T02:13:58.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[109923]: 57.754770+0000 mgr.x (mgr.25369) 6 : 2023-08-28T02:13:58.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[109923]: cephadm [INF] [28/Aug/2023:02:13:57] ENGINE Serving on https://172.21.15.169:7150 2023-08-28T02:13:58.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[109923]: cephadm 2023-08-28 2023-08-28T02:13:58.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[109923]: T02:13:57.754930+0000 mgr.x ( 2023-08-28T02:13:58.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[109923]: mgr.25369) 7 : cephadm [INF] [28/Aug/2023:02:13:57] ENGINE Bus STARTED 2023-08-28T02:13:58.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[109923]: cluster 2023-08-28T02:13:58.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[109923]: 2023-08-28T02:13: 2023-08-28T02:13:58.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[109923]: 57.864971+0000 mon.a (mon.0) 811 : 2023-08-28T02:13:58.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[109923]: cluster [DBG] Standby manager daemon y started 2023-08-28T02:13:58.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[109923]: audit 2023-08- 2023-08-28T02:13:58.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[109923]: 28T02:13:57.867676+0000 mon.a 2023-08-28T02:13:58.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[109923]: (mon.0) 812 : audit [DBG] from='mgr.? 172.21.15.143:0/2603502258' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-08-28T02:13:58.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[109923]: audit 2023-08-28 2023-08-28T02:13:58.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[109923]: T02:13:57.868467+0000 mon.a (mon 2023-08-28T02:13:58.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[109923]: .0) 813 : audit [DBG] from='mgr.? 172.21.15.143:0/2603502258' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-28T02:13:58.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:13:58.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[109923]: 08-28T02:13:57.870862 2023-08-28T02:13:58.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[109923]: +0000 mon.a (mon.0) 814 : audit [DBG] 2023-08-28T02:13:58.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[109923]: from='mgr.? 172.21.15.143:0/2603502258' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-08-28T02:13:58.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[109923]: audit 2023-08-28T02:13:58.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[109923]: 2023-08-28T02:13:57 2023-08-28T02:13:58.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[109923]: .871215+0000 mon.a (mon.0) 2023-08-28T02:13:58.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[109923]: 815 : audit [DBG] from='mgr.? 172.21.15.143:0/2603502258' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-08-28T02:13:58.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:13:58.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[109923]: 08-28T02:13:57. 2023-08-28T02:13:58.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[109923]: 953933+0000 mon.a (mon.0) 816 : audit 2023-08-28T02:13:58.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[109923]: [INF] from='mgr.25369 ' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-08-28T02:13:58.956 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[109923]: cluster 2023-08-28T 2023-08-28T02:13:58.956 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[109923]: 02:13:57.954047+0000 mon.a (mon. 2023-08-28T02:13:58.956 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[109923]: 0) 817 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2023-08-28T02:13:58.956 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:13:58.956 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[109923]: 08-28T02:13:58.483692 2023-08-28T02:13:58.956 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[109923]: +0000 mon.c (mon.2) 40 : audit [DBG] 2023-08-28T02:13:58.956 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:58 smithi143 conmon[109923]: from='client.? 172.21.15.143:0/534499021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:58.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:58 smithi169 conmon[100500]: cluster 2023-08-28T02:13:56.780652+0000 mgr.x (mgr.25369) 3 : cluster [DBG] 2023-08-28T02:13:58.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:58 smithi169 conmon[100500]: pgmap v3: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:58.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:58 smithi169 conmon[100500]: cephadm 2023-08-28T02:13:57.500019+0000 mgr.x (mgr.25369) 4 : cephadm [INF] [28/Aug/2023:02:13:57] ENGINE Bus STARTING 2023-08-28T02:13:58.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:58 smithi169 conmon[100500]: cephadm 2023-08-28T02:13:57.614258+0000 mgr.x (mgr.25369) 5 : cephadm [INF] [28/Aug/2023:02:13:57] ENGINE Serving on http://172.21.15.169:8765 2023-08-28T02:13:58.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:58 smithi169 conmon[100500]: cephadm 2023-08-28T02:13:57.754770+0000 mgr.x (mgr.25369) 6 : cephadm [INF] [28/Aug/2023:02:13:57] ENGINE Serving on https://172.21.15.169:7150 2023-08-28T02:13:58.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:58 smithi169 conmon[100500]: cephadm 2023-08-28T02:13:57.754930+0000 mgr.x (mgr.25369) 7 : cephadm [INF] [28/Aug/2023:02:13:57] ENGINE Bus STARTED 2023-08-28T02:13:58.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:58 smithi169 conmon[100500]: cluster 2023-08-28T02:13:57.864971+0000 2023-08-28T02:13:58.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:58 smithi169 conmon[100500]: mon.a (mon.0) 811 : cluster [DBG] Standby manager daemon y started 2023-08-28T02:13:58.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:58 smithi169 conmon[100500]: audit 2023-08-28T02:13:57.867676+0000 mon.a (mon.0) 2023-08-28T02:13:58.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:58 smithi169 conmon[100500]: 812 : audit [DBG] from='mgr.? 172.21.15.143:0/2603502258' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-08-28T02:13:58.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:58 smithi169 conmon[100500]: audit 2023-08 2023-08-28T02:13:58.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:58 smithi169 conmon[100500]: -28T02:13:57.868467+0000 mon.a ( 2023-08-28T02:13:58.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:58 smithi169 conmon[100500]: mon.0) 813 : audit [DBG] from='mgr.? 172.21.15.143:0/2603502258' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-28T02:13:58.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:58 smithi169 conmon[100500]: audit 2023- 2023-08-28T02:13:58.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:58 smithi169 conmon[100500]: 08-28T02:13:57.870862+0000 mon.a ( 2023-08-28T02:13:58.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:58 smithi169 conmon[100500]: mon.0) 814 : audit [DBG] from='mgr.? 172.21.15.143:0/2603502258' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-08-28T02:13:58.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:58 smithi169 conmon[100500]: audit 2023-08-28T02:13:58.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:58 smithi169 conmon[100500]: 2023-08-28T02:13:57 2023-08-28T02:13:58.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:58 smithi169 conmon[100500]: .871215+0000 mon.a (mon.0) 815 : audit 2023-08-28T02:13:58.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:58 smithi169 conmon[100500]: [DBG] from='mgr.? 172.21.15.143:0/2603502258' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-08-28T02:13:58.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:58 smithi169 conmon[100500]: audit 2023-08-28 2023-08-28T02:13:58.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:58 smithi169 conmon[100500]: T02:13:57.953933+0000 mon.a (mon 2023-08-28T02:13:58.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:58 smithi169 conmon[100500]: .0) 816 : audit [INF] from='mgr.25369 ' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-08-28T02:13:58.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:58 smithi169 conmon[100500]: cluster 2023-08-28T02:13:58.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:58 smithi169 conmon[100500]: 2023-08-28T02:13:57.954047 2023-08-28T02:13:58.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:58 smithi169 conmon[100500]: +0000 mon.a (mon.0) 817 : cluster [DBG] 2023-08-28T02:13:58.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:58 smithi169 conmon[100500]: osdmap e83: 8 total, 8 up, 8 in 2023-08-28T02:13:58.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:58 smithi169 conmon[100500]: audit 2023-08-28T02:13:58 2023-08-28T02:13:58.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:58 smithi169 conmon[100500]: .483692+0000 mon.c (mon.2) 40 : audit 2023-08-28T02:13:58.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:58 smithi169 conmon[100500]: [DBG] from='client.? 172.21.15.143:0/534499021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:59.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:59 smithi143 conmon[100258]: cluster 2023-08-28T02:13:57.772779+0000 mgr.x 2023-08-28T02:13:59.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:59 smithi143 conmon[100258]: (mgr.25369) 8 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:59.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:59 smithi143 conmon[100258]: audit 2023-08-28T02:13:58.631521+0000 mon.a (mon.0) 818 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:13:59.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:59 smithi143 conmon[100258]: cluster 2023-08-28T02:13: 2023-08-28T02:13:59.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:59 smithi143 conmon[100258]: 58.777176+0000 mon.a (mon.0) 2023-08-28T02:13:59.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:59 smithi143 conmon[100258]: 819 : cluster [DBG] mgrmap e22: x(active, since 3s), standbys: y 2023-08-28T02:13:59.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:59 smithi143 conmon[100258]: audit 2023-08-28T02:13:59.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:59 smithi143 conmon[100258]: 2023-08-28T02:13: 2023-08-28T02:13:59.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:59 smithi143 conmon[100258]: 58.780257+0000 mon.b (mon.1 2023-08-28T02:13:59.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:13:59 smithi143 conmon[100258]: ) 59 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-08-28T02:13:59.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:59 smithi143 conmon[109923]: cluster 2023-08-28T02:13: 2023-08-28T02:13:59.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:59 smithi143 conmon[109923]: 57.772779+0000 mgr.x (mgr.25369) 8 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:59.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:59 smithi143 conmon[109923]: audit 2023-08-28T02:13:59.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:59 smithi143 conmon[109923]: 2023-08- 2023-08-28T02:13:59.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:59 smithi143 conmon[109923]: 28T02:13:58.631521 2023-08-28T02:13:59.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:59 smithi143 conmon[109923]: +0000 mon.a (mon.0) 818 2023-08-28T02:13:59.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:59 smithi143 conmon[109923]: : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:13:59.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:59 smithi143 conmon[109923]: cluster 2023 2023-08-28T02:13:59.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:59 smithi143 conmon[109923]: -08-28T02:13:58.777176+0000 2023-08-28T02:13:59.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:59 smithi143 conmon[109923]: mon.a (mon.0) 819 : cluster 2023-08-28T02:13:59.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:59 smithi143 conmon[109923]: [DBG] mgrmap e22: x(active, since 3s), standbys: y 2023-08-28T02:13:59.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:59 smithi143 conmon[109923]: audit 2023 2023-08-28T02:13:59.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:59 smithi143 conmon[109923]: -08-28T02:13:58 2023-08-28T02:13:59.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:59 smithi143 conmon[109923]: .780257+0000 mon.b (mon.1) 59 : audit [DBG] 2023-08-28T02:13:59.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:13:59 smithi143 conmon[109923]: from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-08-28T02:13:59.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:59 smithi169 conmon[100500]: cluster 2023-08-28T02:13:57.772779+0000 mgr.x (mgr.25369) 8 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:13:59.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:59 smithi169 conmon[100500]: audit 2023-08-28T02 2023-08-28T02:13:59.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:59 smithi169 conmon[100500]: :13:58.631521+0000 mon.a (mon.0) 818 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:13:59.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:59 smithi169 conmon[100500]: cluster 2023-08-28T02:13:58.777176+0000 mon.a (mon.0) 819 : cluster [DBG] mgrmap e22: x(active, since 3s), standbys: y 2023-08-28T02:13:59.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:13:59 smithi169 conmon[100500]: audit 2023-08-28T02:13:58.780257+0000 mon.b (mon.1) 59 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-08-28T02:14:00.701 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:14:00 smithi143 conmon[100708]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:14:00] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-28T02:14:01.396 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[100258]: cluster 2023-08-28T02:13:59.773061+0000 mgr.x (mgr. 2023-08-28T02:14:01.396 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[100258]: 25369) 9 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:01.396 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[100258]: audit 2023-08-28T02:14:00.029698+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:01.396 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[100258]: audit 2023-08-28T02:14:00.035906+0000 mon.a (mon.0) 821 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:01.396 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[100258]: cluster 2023-08-28T02:14:00.641560+0000 mon.a (mon.0) 822 : cluster [DBG] mgrmap e23: x(active, since 4s), standbys: y 2023-08-28T02:14:01.396 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[100258]: audit 2023-08-28T02:14:00.676213+0000 2023-08-28T02:14:01.397 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[100258]: mon.a (mon.0) 823 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:01.397 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[100258]: audit 2023-08-28T02:14:00.682424+0000 mon.a (mon.0) 824 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:01.397 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[100258]: audit 2023-08-28T02:14:00.817982+0000 mon.a ( 2023-08-28T02:14:01.397 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[100258]: mon.0) 825 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:01.397 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[100258]: audit 2023-08-28T02:14:01.397 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[100258]: 2023-08-28T02:14:00 2023-08-28T02:14:01.398 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[100258]: .892027+0000 mon.c (mon.2) 41 : audit 2023-08-28T02:14:01.398 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[100258]: [DBG] from='client.? 172.21.15.143:0/3503789358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:01.398 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[100258]: audit 2023-08-28T 2023-08-28T02:14:01.398 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[100258]: 02:14:00.950779+0000 mon.a 2023-08-28T02:14:01.398 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[100258]: (mon.0) 826 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:01.398 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[100258]: audit 2023-08-28T02: 2023-08-28T02:14:01.398 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[100258]: 14:00.956053+0000 mon.a (mon.0) 827 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:01.399 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[109923]: cluster 2023-08-28T02:13:59.773061+0000 2023-08-28T02:14:01.399 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[109923]: mgr.x (mgr.25369) 9 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:01.399 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[109923]: audit 2023-08-28T02:14:00 2023-08-28T02:14:01.399 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[109923]: .029698+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:01.399 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[109923]: audit 2023-08-28T02:14: 2023-08-28T02:14:01.400 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[109923]: 00.035906+0000 mon.a (mon.0) 821 : audit [INF] 2023-08-28T02:14:01.400 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[109923]: from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:01.400 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[109923]: cluster 2023-08-28T02:14: 2023-08-28T02:14:01.400 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[109923]: 00.641560+0000 mon.a (mon.0) 822 : cluster 2023-08-28T02:14:01.400 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[109923]: [DBG] mgrmap e23: x(active, since 4s), standbys: y 2023-08-28T02:14:01.400 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[109923]: audit 2023-08-28T02: 2023-08-28T02:14:01.400 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[109923]: 14:00.676213+0000 mon.a (mon.0) 823 : 2023-08-28T02:14:01.401 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[109923]: audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:01.401 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[109923]: audit 2023-08-28T02:14:01.401 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[109923]: 2023-08-28T02:14:00.682424 2023-08-28T02:14:01.401 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[109923]: +0000 mon.a (mon.0) 824 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:01.401 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[109923]: 2023-08-28T02:14:01.401 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[109923]: audit 2023-08-28T02:14:01.402 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[109923]: 2023-08-28T02:14:00.817982+0000 mon.a (mon. 2023-08-28T02:14:01.402 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[109923]: 0) 825 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:01.402 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[109923]: audit 2023-08-28T02: 2023-08-28T02:14:01.402 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[109923]: 14:00.892027+0000 mon.c (mon.2) 41 : audit 2023-08-28T02:14:01.402 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[109923]: [DBG] from='client.? 172.21.15.143:0/3503789358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:01.402 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[109923]: audit 2023-08-28T02:14:00.950779 2023-08-28T02:14:01.403 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[109923]: +0000 mon.a (mon.0) 826 : audit [INF] 2023-08-28T02:14:01.403 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[109923]: from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:01.403 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[109923]: audit 2023-08-28T02: 2023-08-28T02:14:01.403 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[109923]: 14:00.956053+0000 mon.a (mon 2023-08-28T02:14:01.403 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:01 smithi143 conmon[109923]: .0) 827 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:01.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:01 smithi169 conmon[100500]: cluster 2023-08-28T02:13:59.773061+0000 mgr.x (mgr.25369) 9 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:01.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:01 smithi169 conmon[100500]: audit 2023-08-28T02:14:00.029698+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:01.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:01 smithi169 conmon[100500]: audit 2023-08-28T02:14:00.035906+0000 mon.a (mon.0) 821 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:01.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:01 smithi169 conmon[100500]: cluster 2023-08-28T02:14:00.641560+0000 mon.a (mon.0) 822 : cluster [DBG] mgrmap e23: x(active, since 4s), standbys: y 2023-08-28T02:14:01.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:01 smithi169 conmon[100500]: audit 2023-08-28T02:14:00.676213+0000 mon.a (mon.0) 823 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:01.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:01 smithi169 conmon[100500]: audit 2023-08-28T02:14:00.682424+0000 mon.a (mon.0) 824 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:01.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:01 smithi169 conmon[100500]: audit 2023-08-28T02:14:00.817982+0000 mon.a (mon.0) 825 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:01.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:01 smithi169 conmon[100500]: audit 2023-08-28T02:14:00.892027+0000 mon.c (mon.2) 41 : audit [DBG] from='client.? 172.21.15.143:0/3503789358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:01.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:01 smithi169 conmon[100500]: audit 2023 2023-08-28T02:14:01.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:01 smithi169 conmon[100500]: -08-28T02:14:00.950779+0000 mon.a (mon.0) 826 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:01.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:01 smithi169 conmon[100500]: audit 2023-08-28T02:14:00.956053+0000 mon.a (mon.0) 827 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:03.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:02 smithi143 conmon[109923]: audit 2023-08-28T02:14:01.589836+0000 mon.a (mon.0) 828 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:03.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:02 smithi143 conmon[109923]: audit 2023-08-28T02:14:01.595404+0000 mon.a (mon.0) 829 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:03.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:02 smithi143 conmon[109923]: audit 2023-08-28T02:14:02.411953+0000 mon.a (mon.0) 830 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-08-28T02:14:03.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:02 smithi143 conmon[100258]: audit 2023-08-28T02:14:01.589836+0000 mon.a (mon.0) 828 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:03.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:02 smithi143 conmon[100258]: audit 2023-08-28T02:14:01.595404+0000 mon.a (mon.0) 829 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:03.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:02 smithi143 conmon[100258]: audit 2023-08-28T02:14:02.411953+0000 mon.a (mon.0) 830 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-08-28T02:14:03.237 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:14:02 smithi169 conmon[138400]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:14:02] "GET /metrics HTTP/1.1" 200 31057 "" "Prometheus/2.33.4" 2023-08-28T02:14:03.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:02 smithi169 conmon[100500]: audit 2023-08-28T02:14:01.589836+0000 mon.a 2023-08-28T02:14:03.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:02 smithi169 conmon[100500]: (mon.0) 828 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:03.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:02 smithi169 conmon[100500]: audit 2023-08-28T02:14:01.595404+0000 mon.a (mon.0) 829 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:03.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:02 smithi169 conmon[100500]: audit 2023-08-28T02:14:02.411953+0000 mon.a (mon.0) 830 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-08-28T02:14:03.969 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:03 smithi169 conmon[100500]: cluster 2023-08-28T02:14:01.773451+0000 mgr.x (mgr.25369) 10 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:03.969 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:03 smithi169 conmon[100500]: audit 2023-08-28T02:14:02.412307+0000 mon.b (mon.1) 60 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T02:14:03.969 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:03 smithi169 conmon[100500]: audit 2023-08-28T02:14:02.412518+0000 mon.b (mon.1) 61 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T02:14:03.969 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:03 smithi169 conmon[100500]: audit 2023-08-28T02:14:02.412707+0000 mon.b (mon.1) 62 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T02:14:03.969 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:03 smithi169 conmon[100500]: audit 2023-08-28T02:14:02.932729+0000 mon.a (mon.0) 831 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-08-28T02:14:03.969 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:03 smithi169 conmon[100500]: audit 2023-08-28T02:14:02.936979+0000 mon.b (mon.1) 63 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-08-28T02:14:03.969 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:03 smithi169 conmon[100500]: audit 2023-08-28T02:14:03.286649+0000 mon.c (mon.2) 42 : audit [DBG] from='client.? 172.21.15.143:0/1400290217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:03.969 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:03 smithi169 conmon[100500]: audit 2023-08-28T02:14:03.456233+0000 mon.b (mon.1) 64 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-08-28T02:14:03.969 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:03 smithi169 conmon[100500]: audit 2023-08-28T02:14:03.456848+0000 mon.b (mon.1) 65 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T02:14:03.969 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:03 smithi169 conmon[100500]: audit 2023-08-28T02:14:03.456982+0000 mon.b (mon.1) 66 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T02:14:03.970 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:03 smithi169 conmon[100500]: audit 2023-08-28T02:14:03.457118+0000 mon.b (mon.1) 67 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T02:14:03.970 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:03 smithi169 conmon[100500]: audit 2023-08-28T02: 2023-08-28T02:14:03.970 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:03 smithi169 conmon[100500]: 14:03.461250+0000 mon.b (mon.1) 68 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T02:14:03.970 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:03 smithi169 conmon[100500]: audit 2023-08-28T02:14:03. 2023-08-28T02:14:03.970 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:03 smithi169 conmon[100500]: 461436+0000 mon.b (mon.1) 69 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T02:14:03.970 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:03 smithi169 conmon[100500]: audit 2023-08-28T02:14: 2023-08-28T02:14:03.970 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:03 smithi169 conmon[100500]: 03.461593+0000 mon.b (mon.1) 70 : audit [DBG] 2023-08-28T02:14:03.970 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:03 smithi169 conmon[100500]: from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T02:14:04.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:03 smithi143 conmon[109923]: cluster 2023-08-28T02:14:01.773451+0000 mgr.x 2023-08-28T02:14:04.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:03 smithi143 conmon[109923]: (mgr.25369) 10 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:04.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:03 smithi143 conmon[109923]: audit 2023-08-28T02:14:02.412307+0000 mon.b (mon.1) 60 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T02:14:04.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:03 smithi143 conmon[109923]: audit 2023-08-28T02:14:02.412518+0000 mon.b (mon.1) 61 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T02:14:04.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:03 smithi143 conmon[109923]: audit 2023-08-28T02:14:02.412707+0000 mon.b (mon.1) 62 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T02:14:04.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:03 smithi143 conmon[109923]: audit 2023-08-28T02:14:02.932729+0000 mon.a (mon.0) 831 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-08-28T02:14:04.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:03 smithi143 conmon[109923]: audit 2023-08-28T02:14:02.936979+0000 mon.b (mon.1) 63 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-08-28T02:14:04.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:03 smithi143 conmon[109923]: audit 2023-08-28T02:14:03.286649+0000 mon.c (mon.2) 42 : audit [DBG] from='client.? 172.21.15.143:0/1400290217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:04.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:03 smithi143 conmon[109923]: audit 2023-08-28T02:14:03.456233+0000 mon.b (mon.1) 64 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-08-28T02:14:04.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:03 smithi143 conmon[109923]: audit 2023-08-28T02:14:03.456848+0000 mon.b (mon.1 2023-08-28T02:14:04.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:03 smithi143 conmon[109923]: ) 65 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T02:14:04.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:03 smithi143 conmon[109923]: audit 2023-08-28T02:14:03.456982+0000 mon.b (mon.1) 66 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T02:14:04.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:03 smithi143 conmon[109923]: audit 2023-08-28T02:14: 2023-08-28T02:14:04.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:03 smithi143 conmon[109923]: 03.457118+0000 mon.b (mon.1) 67 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T02:14:04.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:03 smithi143 conmon[109923]: audit 2023-08-28T02 2023-08-28T02:14:04.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:03 smithi143 conmon[109923]: :14:03.461250+0000 mon.b (mon 2023-08-28T02:14:04.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:03 smithi143 conmon[109923]: .1) 68 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T02:14:04.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:03 smithi143 conmon[109923]: 2023-08-28T02:14:04.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:03 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:14:04.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:03 smithi143 conmon[109923]: 08-28T02:14:03.461436+0000 2023-08-28T02:14:04.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:03 smithi143 conmon[109923]: mon.b (mon.1) 69 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T02:14:04.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:03 smithi143 conmon[109923]: audit 2023-08-28T02:14:04.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:03 smithi143 conmon[109923]: 2023-08-28T02:14:03.461593 2023-08-28T02:14:04.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:03 smithi143 conmon[109923]: +0000 mon.b (mon.1) 70 : audit [DBG] 2023-08-28T02:14:04.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:03 smithi143 conmon[109923]: from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T02:14:04.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:03 smithi143 conmon[100258]: cluster 2023-08-28T02:14:01.773451+0000 mgr.x (mgr.25369) 2023-08-28T02:14:04.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:03 smithi143 conmon[100258]: 10 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 4.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:04.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:03 smithi143 conmon[100258]: audit 2023-08-28T02:14:02.412307+0000 mon.b (mon.1) 60 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T02:14:04.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:03 smithi143 conmon[100258]: audit 2023-08-28T02:14:02.412518+0000 mon.b (mon.1) 61 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T02:14:04.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:03 smithi143 conmon[100258]: audit 2023-08-28T02:14:02.412707+0000 mon.b (mon.1) 62 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T02:14:04.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:03 smithi143 conmon[100258]: audit 2023-08-28T02:14:02.932729+0000 mon.a (mon.0) 831 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-08-28T02:14:04.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:03 smithi143 conmon[100258]: audit 2023-08-28T02:14:02.936979+0000 mon.b (mon.1) 63 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-08-28T02:14:04.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:03 smithi143 conmon[100258]: audit 2023-08-28T02:14:03.286649+0000 mon.c (mon.2) 42 : audit [DBG] from='client.? 172.21.15.143:0/1400290217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:04.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:03 smithi143 conmon[100258]: audit 2023-08-28T02:14:03.456233+0000 mon.b (mon.1) 64 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-08-28T02:14:04.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:03 smithi143 conmon[100258]: audit 2023-08-28T02:14:03.456848+0000 mon.b (mon.1) 65 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T02:14:04.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:03 smithi143 conmon[100258]: audit 2023-08-28T02:14:03.456982+0000 mon.b (mon.1) 66 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T02:14:04.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:03 smithi143 conmon[100258]: audit 2023-08-28T02:14:03.457118+0000 mon.b (mon.1) 67 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T02:14:04.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:03 smithi143 conmon[100258]: audit 2023-08-28T02:14:03.461250+0000 mon.b (mon.1) 68 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T02:14:04.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:03 smithi143 conmon[100258]: audit 2023-08-28T02:14:03.461436+0000 mon.b (mon.1) 69 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T02:14:04.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:03 smithi143 conmon[100258]: audit 2023-08-28T02:14:03.461593+0000 mon.b (mon.1) 70 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T02:14:05.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: audit 2023-08-28T02:14:03.461095+0000 mon.c (mon.2 2023-08-28T02:14:05.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: ) 43 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-08-28T02:14:05.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: cluster 2023-08-28T02:14:03.773926+0000 mgr.x (mgr.25369) 11 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 308 KiB/s wr, 35 op/s 2023-08-28T02:14:05.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: audit 2023-08-28T02:14:03.959994+0000 mon.c (mon.2) 44 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-08-28T02:14:05.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: audit 2023-08-28T02:14:03.962822+0000 mon.a (mon.0) 832 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:05.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: audit 2023-08-28T02:14:03.974486+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:05.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: audit 2023-08-28T02:14:03.976969+0000 mon.b (mon.1) 71 : audit [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: audit 2023-08-28T02:14:03.977212+0000 mon.a (mon.0) 834 : audit [INF] from='mgr.25369 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: audit 2023-08-28T02:14:03.978198+0000 mon.b (mon.1) 72 : audit [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: audit 2023-08-28T02:14:03.978403+0000 mon.a (mon.0) 835 : audit [INF] from='mgr.25369 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: audit 2023-08-28T02:14:03.979338+0000 mon.b (mon.1) 73 : audit [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: audit 2023-08-28T02:14:03.979503+0000 mon.a (mon.0) 836 : audit [INF] from='mgr.25369 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: audit 2023-08-28T02:14:03.980232+0000 mon.b (mon.1) 74 : audit [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: audit 2023-08-28T02:14:03.980410+0000 mon.a (mon.0) 837 : audit [INF] from='mgr.25369 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: cephadm 2023-08-28T02:14:03.980998+0000 mgr.x (mgr.25369) 12 : cephadm [INF] Adjusting osd_memory_target on smithi143 to 2500M 2023-08-28T02:14:05.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: audit 2023-08-28T02:14:05.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: 2023-08-28T02:14:03.986645+0000 mon.a (mon.0) 838 2023-08-28T02:14:05.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:05.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: audit 2023-08-28 2023-08-28T02:14:05.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: T02:14:04.774123+0000 mon.a ( 2023-08-28T02:14:05.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: mon.0) 839 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:05.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: audit 2023 2023-08-28T02:14:05.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: -08-28T02:14:04. 2023-08-28T02:14:05.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: 779868+0000 mon.a (mon.0) 840 : audit [INF] 2023-08-28T02:14:05.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:05.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: audit 2023-08-28T02:14:04.781270+0000 2023-08-28T02:14:05.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: mon.b (mon.1) 75 : audit [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: audit 2023-08-28T02:14:04.781615+0000 mon.a 2023-08-28T02:14:05.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: (mon.0) 841 : audit [INF] from='mgr.25369 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: audit 2023-08-28T02:14:05.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: 2023-08-28T02:14:04.782450+0000 mon.b (mon.1) 76 : audit [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: audit 2023-08-28T02:14:04 2023-08-28T02:14:05.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: .782750+0000 mon.a (mon.0) 842 : audit [INF] from='mgr.25369 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: audit 2023-08-28T02:14:04 2023-08-28T02:14:05.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: .783480+0000 mon.b (mon.1) 77 : audit [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: audit 2023-08-28T02:14:04 2023-08-28T02:14:05.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: .783755+0000 mon.a (mon.0) 843 : audit [INF] from='mgr.25369 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: audit 2023-08-28T02:14 2023-08-28T02:14:05.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: :04.784477+0000 mon.b (mon.1) 78 : audit [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: audit 2023-08-28T 2023-08-28T02:14:05.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: 02:14:04.784771 2023-08-28T02:14:05.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: +0000 mon.a (mon.0) 844 : 2023-08-28T02:14:05.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: audit [INF] from='mgr.25369 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: audit 2023-08-28 2023-08-28T02:14:05.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: T02:14:04.793434+0000 mon.a (mon. 2023-08-28T02:14:05.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: 0) 845 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:05.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: audit 2023-08-28 2023-08-28T02:14:05.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: T02:14:04.794386+0000 mon.b (mon 2023-08-28T02:14:05.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: .1) 79 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:14:05.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: audit 2023-08- 2023-08-28T02:14:05.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: 28T02:14:04.794968+0000 mon.b ( 2023-08-28T02:14:05.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:04 smithi169 conmon[100500]: mon.1) 80 : audit [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:14:05.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: audit 2023-08-28T02:14:03.461095+0000 mon.c (mon.2) 43 : audit [INF] 2023-08-28T02:14:05.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-08-28T02:14:05.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: cluster 2023-08-28T02:14:03.773926+0000 mgr.x (mgr.25369) 11 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 308 KiB/s wr, 35 op/s 2023-08-28T02:14:05.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: audit 2023-08-28T02:14:03.959994+0000 mon.c (mon.2) 44 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-08-28T02:14:05.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: audit 2023-08-28T02:14:03.962822+0000 mon.a (mon.0) 832 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:05.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: audit 2023-08-28T02:14:03.974486+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:05.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: audit 2023-08-28T02:14:03.976969+0000 mon.b (mon.1) 71 : audit [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: audit 2023-08-28T02:14:05.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: 2023-08-28T02:14:03.977212+0000 mon.a (mon.0) 834 : audit [INF] 2023-08-28T02:14:05.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: from='mgr.25369 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: audit 2023 2023-08-28T02:14:05.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: -08-28T02:14:03.978198+0000 mon.b (mon.1) 72 : audit [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: 2023-08-28T02:14:05.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: audit 2023-08- 2023-08-28T02:14:05.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: 28T02:14:03.978403+0000 mon.a (mon.0) 835 : audit [INF] from='mgr.25369 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: audit 2023 2023-08-28T02:14:05.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: -08-28T02:14:03.979338+0000 mon.b (mon. 2023-08-28T02:14:05.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: 1) 73 : audit [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: audit 2023-08-28T02:14:03 2023-08-28T02:14:05.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: .979503+0000 mon.a (mon.0) 836 : audit [INF] from='mgr.25369 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: 2023-08-28T02:14:05.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: audit 2023-08- 2023-08-28T02:14:05.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: 28T02:14:03.980232+0000 mon.b (mon.1) 74 : audit [INF] 2023-08-28T02:14:05.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: audit 2023-08-28T02:14:05.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: 2023-08-28T02:14:03.980410+0000 mon.a (mon.0) 2023-08-28T02:14:05.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: 837 : audit [INF] from='mgr.25369 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: cephadm 2023-08-28T02:14:03.980998+0000 2023-08-28T02:14:05.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: mgr.x (mgr.25369) 12 : cephadm [INF] Adjusting osd_memory_target on smithi143 to 2500M 2023-08-28T02:14:05.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: audit 2023-08 2023-08-28T02:14:05.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: -28T02:14:03.986645+0000 mon.a (mon.0) 2023-08-28T02:14:05.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: 838 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:05.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: audit 2023-08-28T02:14 2023-08-28T02:14:05.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: :04.774123+0000 mon.a (mon.0) 839 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:05.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: audit 2023-08-28T02:14:03 2023-08-28T02:14:05.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: .461095+0000 mon.c (mon.2) 43 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-08-28T02:14:05.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: cluster 2023-08-28T02:14:05.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: 2023-08-28T02:14:03.773926 2023-08-28T02:14:05.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: +0000 mgr.x (mgr.25369) 11 2023-08-28T02:14:05.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 308 KiB/s wr, 35 op/s 2023-08-28T02:14:05.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: audit 2023-08 2023-08-28T02:14:05.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: -28T02:14:03.959994+0000 mon.c 2023-08-28T02:14:05.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: (mon.2) 44 : audit 2023-08-28T02:14:05.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-08-28T02:14:05.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: audit 2023-08-28T02:14:05.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: 2023-08-28T02:14:03.962822+0000 2023-08-28T02:14:05.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: mon.a (mon.0) 832 : audit 2023-08-28T02:14:05.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:05.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: audit 2023-08-28T02:14:05.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: 2023-08-28T02: 2023-08-28T02:14:05.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: 14:03.974486+0000 mon.a (mon 2023-08-28T02:14:05.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: .0) 833 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:05.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: 2023-08-28T02:14:05.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: audit 2023-08-28T02:14:05.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: 2023-08-28T02:14:03 2023-08-28T02:14:05.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: .976969+0000 mon.b (mon.1) 2023-08-28T02:14:05.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: 71 : audit [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: audit 2023-08 2023-08-28T02:14:05.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: -28T02:14:03.977212+0000 mon.a 2023-08-28T02:14:05.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: (mon.0) 834 : audit 2023-08-28T02:14:05.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: [INF] from='mgr.25369 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: audit 2023-08-28T02:14:05.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: 2023-08-28T02:14:03.978198 2023-08-28T02:14:05.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: +0000 mon.b (mon.1) 72 : 2023-08-28T02:14:05.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: audit [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: audit 2023-08- 2023-08-28T02:14:05.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: 28T02:14:03.978403+0000 mon.a ( 2023-08-28T02:14:05.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: mon.0) 835 : audit [INF] 2023-08-28T02:14:05.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: from='mgr.25369 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: audit 2023-08-28T02:14:05.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: 2023-08-28T02:14 2023-08-28T02:14:05.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: :03.979338+0000 mon.b (mon 2023-08-28T02:14:05.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: .1) 73 : audit [INF] 2023-08-28T02:14:05.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.457 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: audit 2023 2023-08-28T02:14:05.457 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: -08-28T02:14:03.979503+0000 2023-08-28T02:14:05.457 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: mon.a (mon.0) 836 : audit 2023-08-28T02:14:05.457 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: [INF] from='mgr.25369 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.457 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: audit 2023-08-28T02:14:05.457 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: 2023-08-28T02:14:03.980232+0000 2023-08-28T02:14:05.457 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: mon.b (mon.1) 74 : audit 2023-08-28T02:14:05.457 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: 2023-08-28T02:14:05.457 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: audit 2023-08-28 2023-08-28T02:14:05.457 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: T02:14:04.779868+0000 mon.a (mon.0) 840 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:05.458 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: 2023-08-28T02:14:05.458 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: audit 2023-08 2023-08-28T02:14:05.458 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: -28T02:14:04.781270+0000 mon.b (mon.1) 75 : audit [INF] 2023-08-28T02:14:05.458 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.458 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: audit 2023-08-28T02:14:05.458 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: 2023-08-28T02:14:04.781615+0000 mon.a (mon.0) 841 2023-08-28T02:14:05.458 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: : audit [INF] from='mgr.25369 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.458 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: audit 2023-08-28T02:14:05.458 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: 2023-08-28T02:14:04.782450+0000 mon.b (mon.1) 76 : 2023-08-28T02:14:05.458 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: audit [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.458 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: audit 2023-08-28T02:14:04.782750 2023-08-28T02:14:05.459 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: +0000 mon.a (mon.0) 842 : audit [INF] from='mgr.25369 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.459 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: 2023-08-28T02:14:05.459 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: audit 2023 2023-08-28T02:14:05.459 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: -08-28T02:14:04.783480+0000 mon.b (mon.1) 77 2023-08-28T02:14:05.459 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: : audit [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.459 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: audit 2023-08-28T02:14:04 2023-08-28T02:14:05.459 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: .783755+0000 mon.a (mon.0) 843 : audit [INF] from='mgr.25369 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.459 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: 2023-08-28T02:14:05.459 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: audit 2023-08-28T02:14:05.459 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: 2023-08-28T02:14:04.784477+0000 mon.b (mon.1) 2023-08-28T02:14:05.459 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: 78 : audit [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.459 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: audit 2023-08-28T02:14: 2023-08-28T02:14:05.460 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: 04.784771+0000 mon.a (mon.0) 844 : audit 2023-08-28T02:14:05.460 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: [INF] from='mgr.25369 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.460 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: audit 2023-08-28T02:14:05.460 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: 2023-08-28T02:14:04.793434+0000 mon.a (mon.0 2023-08-28T02:14:05.460 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: ) 845 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:05.460 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: audit 2023-08-28T02: 2023-08-28T02:14:05.460 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: 14:04.794386+0000 mon.b (mon.1) 79 : audit 2023-08-28T02:14:05.460 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:14:05.460 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: audit 2023-08-28T02:14:04.794968+0000 2023-08-28T02:14:05.460 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[100258]: mon.b (mon.1) 80 : audit [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:14:05.461 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.461 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: audit 2023-08 2023-08-28T02:14:05.461 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: -28T02:14:03.980410+0000 mon.a (mon 2023-08-28T02:14:05.461 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: .0) 837 : audit [INF] from='mgr.25369 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.461 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: cephadm 2023-08-28T 2023-08-28T02:14:05.461 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: 02:14:03.980998+0000 mgr.x (mgr.25369) 12 : cephadm [INF] 2023-08-28T02:14:05.461 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: Adjusting osd_memory_target on smithi143 to 2500M 2023-08-28T02:14:05.461 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: audit 2023-08-28T 2023-08-28T02:14:05.461 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: 02:14:03.986645+0000 mon.a (mon.0) 838 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:05.461 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: audit 2023-08-28T02:14:04.774123+0000 2023-08-28T02:14:05.461 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: mon.a (mon.0) 839 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:05.462 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: audit 2023-08-28T02:14:04.779868+0000 mon.a ( 2023-08-28T02:14:05.462 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: mon.0) 840 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:05.462 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: audit 2023-08-28T02:14 2023-08-28T02:14:05.462 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: :04.781270+0000 mon.b (mon. 2023-08-28T02:14:05.462 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: 1) 75 : audit [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.462 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: audit 2023 2023-08-28T02:14:05.462 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: -08-28T02:14: 2023-08-28T02:14:05.462 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: 04.781615+0000 mon.a (mon.0) 841 : audit 2023-08-28T02:14:05.462 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: [INF] from='mgr.25369 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.462 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: audit 2023-08-28 2023-08-28T02:14:05.462 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: T02:14:04.782450+0000 mon.b 2023-08-28T02:14:05.462 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: (mon.1) 76 : audit [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.462 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: audit 2023-08-28T02:14:05.463 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: 2023-08-28T02:14:04 2023-08-28T02:14:05.463 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: .782750+0000 mon.a (mon.0) 842 : audit 2023-08-28T02:14:05.463 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: [INF] from='mgr.25369 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.463 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: audit 2023-08-28 2023-08-28T02:14:05.463 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: T02:14:04.783480+0000 mon.b ( 2023-08-28T02:14:05.463 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: mon.1) 77 : audit [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.463 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: audit 2023-08-28T02:14:05.463 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: 2023-08-28T02:14 2023-08-28T02:14:05.463 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: :04.783755+0000 mon.a (mon.0) 843 2023-08-28T02:14:05.463 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: : audit [INF] from='mgr.25369 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.463 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:14:05.463 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: 08-28T02:14:04. 2023-08-28T02:14:05.464 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: 784477+0000 mon.b (mon.1) 78 : audit 2023-08-28T02:14:05.464 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.464 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: audit 2023-08-28T02:14 2023-08-28T02:14:05.464 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: :04.784771+0000 mon.a (mon.0) 844 2023-08-28T02:14:05.464 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: : audit [INF] from='mgr.25369 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:14:05.464 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: audit 2023-08- 2023-08-28T02:14:05.464 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: 28T02:14:04.793434+0000 2023-08-28T02:14:05.464 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: mon.a (mon.0) 845 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:05.464 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: 2023-08-28T02:14:05.464 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: audit 2023-08- 2023-08-28T02:14:05.464 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: 28T02:14:04.794386+0000 mon.b 2023-08-28T02:14:05.464 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: (mon.1) 79 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:14:05.464 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: audit 2023 2023-08-28T02:14:05.465 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: -08-28T02:14:04 2023-08-28T02:14:05.465 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: .794968+0000 mon.b (mon.1) 2023-08-28T02:14:05.465 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:04 smithi143 conmon[109923]: 80 : audit [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:14:06.450 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:06 smithi143 systemd[1]: Stopping Ceph node-exporter.a for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:14:06.450 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:06 smithi143 bash[173086]: Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-node-exporter.a found: no such container 2023-08-28T02:14:06.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: cephadm 2023-08-28T02:14:04.785386+0000 mgr.x 2023-08-28T02:14:06.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: (mgr.25369) 13 : cephadm [INF] Adjusting osd_memory_target on smithi169 to 2500M 2023-08-28T02:14:06.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: cephadm 2023-08-28T02:14 2023-08-28T02:14:06.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: :04.795862+0000 mgr.x (mgr.25369) 14 : cephadm [INF] Updating smithi143:/etc/ceph/ceph.conf 2023-08-28T02:14:06.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: cephadm 2023-08- 2023-08-28T02:14:06.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: 28T02:14:04.796005+0000 mgr.x (mgr.25369 2023-08-28T02:14:06.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: ) 15 : cephadm [INF] Updating smithi169:/etc/ceph/ceph.conf 2023-08-28T02:14:06.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: cephadm 2023-08-28T02:14:06.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: 2023-08-28T02: 2023-08-28T02:14:06.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: 14:05.060076+0000 mgr.x (mgr.25369) 16 : cephadm 2023-08-28T02:14:06.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: [INF] Updating smithi143:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.conf 2023-08-28T02:14:06.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: cephadm 2023-08- 2023-08-28T02:14:06.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: 28T02:14:05.060247+0000 2023-08-28T02:14:06.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: mgr.x (mgr.25369) 17 : cephadm [INF] Updating smithi169:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.conf 2023-08-28T02:14:06.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: cephadm 2023-08 2023-08-28T02:14:06.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: -28T02:14:05.324768+0000 2023-08-28T02:14:06.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: mgr.x (mgr.25369) 18 : cephadm [INF] 2023-08-28T02:14:06.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: Updating smithi143:/etc/ceph/ceph.client.admin.keyring 2023-08-28T02:14:06.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: cephadm 2023-08-28T02:14:06.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: 2023-08-28T02:14:05.324896+0000 mgr.x (mgr. 2023-08-28T02:14:06.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: 25369) 19 : cephadm [INF] Updating smithi169:/etc/ceph/ceph.client.admin.keyring 2023-08-28T02:14:06.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: cephadm 2023-08-28T02:14:06.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: 2023-08-28T02:14:05.589471 2023-08-28T02:14:06.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: +0000 mgr.x (mgr.25369) 20 : cephadm [INF] Updating smithi169:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.client.admin.keyring 2023-08-28T02:14:06.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: cephadm 2023 2023-08-28T02:14:06.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: -08-28T02:14:05.589612 2023-08-28T02:14:06.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: +0000 mgr.x (mgr.25369) 21 : cephadm [INF] 2023-08-28T02:14:06.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: Updating smithi143:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.client.admin.keyring 2023-08-28T02:14:06.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: audit 2023-08-28T02:14:06.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: 2023-08-28T02:14 2023-08-28T02:14:06.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: :05.675376+0000 mon.c (mon.2) 45 : 2023-08-28T02:14:06.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: audit [DBG] from='client.? 172.21.15.143:0/2316730831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:06.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: audit 2023-08-28T02:14:06.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: 2023-08-28T02:14:05 2023-08-28T02:14:06.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: .858841+0000 mon.a (mon.0) 846 2023-08-28T02:14:06.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:06.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: audit 2023-08-28T02:14:05.861934 2023-08-28T02:14:06.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: +0000 mon.a (mon. 2023-08-28T02:14:06.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: 0) 847 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:06.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: audit 2023-08-28T02:14:06.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: 2023-08-28T02:14:05.868440 2023-08-28T02:14:06.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: +0000 mon.a (mon.0) 848 2023-08-28T02:14:06.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:06.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: audit 2023-08-28T02:14:06.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: 2023-08-28T02 2023-08-28T02:14:06.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: :14:05.873581+0000 mon.a 2023-08-28T02:14:06.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: (mon.0) 849 : audit [INF] 2023-08-28T02:14:06.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:06.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: audit 2023-08-28 2023-08-28T02:14:06.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: T02:14:05.880090+0000 2023-08-28T02:14:06.455 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: cephadm 2023-08-28T02:14: 2023-08-28T02:14:06.455 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: 04.785386+0000 mgr.x (mgr.25369) 13 : cephadm [INF] Adjusting osd_memory_target on smithi169 to 2500M 2023-08-28T02:14:06.455 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: cephadm 2023-08-28T02:14:04.795862+0000 mgr.x (mgr.25369) 14 : cephadm [INF] Updating smithi143:/etc/ceph/ceph.conf 2023-08-28T02:14:06.455 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: cephadm 2023-08-28T02:14:06.455 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: 2023-08-28T02:14:04.796005+0000 mgr.x (mgr.25369) 15 : cephadm [INF] Updating smithi169:/etc/ceph/ceph.conf 2023-08-28T02:14:06.455 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: cephadm 2023-08-28T02:14:06.455 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: 2023-08-28T02:14:05.060076+0000 2023-08-28T02:14:06.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: mgr.x (mgr.25369) 16 : cephadm 2023-08-28T02:14:06.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: [INF] Updating smithi143:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.conf 2023-08-28T02:14:06.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: cephadm 2023-08-28T02:14:06.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: 2023-08- 2023-08-28T02:14:06.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: 28T02:14:05.060247 2023-08-28T02:14:06.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: +0000 mgr.x (mgr.25369) 17 : 2023-08-28T02:14:06.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: cephadm [INF] Updating smithi169:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.conf 2023-08-28T02:14:06.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: cephadm 2023- 2023-08-28T02:14:06.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: 08-28T02:14:05. 2023-08-28T02:14:06.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: 324768+0000 mgr.x (mgr.25369) 18 : cephadm [INF] 2023-08-28T02:14:06.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: Updating smithi143:/etc/ceph/ceph.client.admin.keyring 2023-08-28T02:14:06.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: cephadm 2023-08-28T02:14:06.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: 2023-08-28T02:14:05. 2023-08-28T02:14:06.457 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: 324896+0000 mgr.x (mgr.25369) 19 : cephadm 2023-08-28T02:14:06.457 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: [INF] Updating smithi169:/etc/ceph/ceph.client.admin.keyring 2023-08-28T02:14:06.457 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: cephadm 2023 2023-08-28T02:14:06.457 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: -08-28T02:14:05.589471+0000 2023-08-28T02:14:06.457 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: mgr.x (mgr.25369) 20 : cephadm 2023-08-28T02:14:06.457 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: [INF] Updating smithi169:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.client.admin.keyring 2023-08-28T02:14:06.457 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: cephadm 2023-08- 2023-08-28T02:14:06.457 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: 28T02:14:05.589612+0000 2023-08-28T02:14:06.457 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: mgr.x (mgr.25369) 21 : cephadm [INF] Updating smithi143:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.client.admin.keyring 2023-08-28T02:14:06.457 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: audit 2023 2023-08-28T02:14:06.457 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: -08-28T02:14:05. 2023-08-28T02:14:06.457 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: 675376+0000 mon.c (mon.2) 45 : audit [DBG] 2023-08-28T02:14:06.457 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: from='client.? 172.21.15.143:0/2316730831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:06.458 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: audit 2023- 2023-08-28T02:14:06.458 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: 08-28T02: 2023-08-28T02:14:06.458 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: 14:05.858841+0000 mon.a (mon.0) 846 : audit 2023-08-28T02:14:06.458 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:06.458 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: audit 2023-08- 2023-08-28T02:14:06.458 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: 28T02:14:05.861934+0000 mon.a (mon. 2023-08-28T02:14:06.458 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: 0) 847 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:06.458 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: audit 2023-08 2023-08-28T02:14:06.458 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: -28T02:14:05.868440+0000 mon.a (mon.0 2023-08-28T02:14:06.458 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: ) 848 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:06.458 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: audit 2023-08-28T02:14:06.458 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: 2023-08 2023-08-28T02:14:06.458 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: -28T02:14:05.873581+0000 mon.a ( 2023-08-28T02:14:06.459 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: mon.0) 849 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:06.459 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: audit 2023- 2023-08-28T02:14:06.459 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: 08-28T02:14:05.880090+0000 mon.a 2023-08-28T02:14:06.459 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[100258]: (mon.0) 850 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:06.459 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: mon.a (mon.0) 850 : audit 2023-08-28T02:14:06.459 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:06 smithi143 conmon[109923]: [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:06.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:06 smithi169 conmon[100500]: cephadm 2023-08-28T02:14:04.785386+0000 mgr.x (mgr.25369) 13 : cephadm [INF] Adjusting osd_memory_target on smithi169 to 2500M 2023-08-28T02:14:06.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:06 smithi169 conmon[100500]: cephadm 2023-08-28T02:14:04 2023-08-28T02:14:06.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:06 smithi169 conmon[100500]: .795862+0000 mgr.x (mgr.25369) 14 : cephadm [INF] Updating smithi143:/etc/ceph/ceph.conf 2023-08-28T02:14:06.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:06 smithi169 conmon[100500]: cephadm 2023-08-28T02:14:04.796005+0000 mgr.x (mgr.25369) 15 : cephadm [INF] Updating smithi169:/etc/ceph/ceph.conf 2023-08-28T02:14:06.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:06 smithi169 conmon[100500]: cephadm 2023-08-28T02:14 2023-08-28T02:14:06.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:06 smithi169 conmon[100500]: :05.060076+0000 mgr.x (mgr.25369) 16 : cephadm [INF] Updating smithi143:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.conf 2023-08-28T02:14:06.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:06 smithi169 conmon[100500]: cephadm 2023-08-28T02: 2023-08-28T02:14:06.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:06 smithi169 conmon[100500]: 14:05.060247+0000 mgr.x (mgr.25369) 2023-08-28T02:14:06.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:06 smithi169 conmon[100500]: 17 : cephadm [INF] Updating smithi169:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.conf 2023-08-28T02:14:06.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:06 smithi169 conmon[100500]: cephadm 2023-08-28T02:14:06.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:06 smithi169 conmon[100500]: 2023-08-28T02:14:05. 2023-08-28T02:14:06.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:06 smithi169 conmon[100500]: 324768+0000 mgr.x (mgr.25369) 18 : cephadm [INF] 2023-08-28T02:14:06.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:06 smithi169 conmon[100500]: Updating smithi143:/etc/ceph/ceph.client.admin.keyring 2023-08-28T02:14:06.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:06 smithi169 conmon[100500]: cephadm 2023-08-28 2023-08-28T02:14:06.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:06 smithi169 conmon[100500]: T02:14:05.324896+0000 mgr.x (mgr. 2023-08-28T02:14:06.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:06 smithi169 conmon[100500]: 25369) 19 : cephadm [INF] Updating smithi169:/etc/ceph/ceph.client.admin.keyring 2023-08-28T02:14:06.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:06 smithi169 conmon[100500]: cephadm 2023-08-28T02:14:06.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:06 smithi169 conmon[100500]: 2023-08-28T02:14:05.589471 2023-08-28T02:14:06.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:06 smithi169 conmon[100500]: +0000 mgr.x (mgr.25369) 20 : cephadm [INF] 2023-08-28T02:14:06.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:06 smithi169 conmon[100500]: Updating smithi169:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.client.admin.keyring 2023-08-28T02:14:06.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:06 smithi169 conmon[100500]: cephadm 2023-08-28T02:14:06.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:06 smithi169 conmon[100500]: 2023-08-28T02:14:05.589612 2023-08-28T02:14:06.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:06 smithi169 conmon[100500]: +0000 mgr.x (mgr.25369) 21 : cephadm [INF] 2023-08-28T02:14:06.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:06 smithi169 conmon[100500]: Updating smithi143:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.client.admin.keyring 2023-08-28T02:14:06.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:06 smithi169 conmon[100500]: audit 2023-08-28T02:14:06.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:06 smithi169 conmon[100500]: 2023-08-28T02:14:05 2023-08-28T02:14:06.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:06 smithi169 conmon[100500]: .675376+0000 mon.c (mon.2) 45 : audit 2023-08-28T02:14:06.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:06 smithi169 conmon[100500]: [DBG] from='client.? 172.21.15.143:0/2316730831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:06.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:06 smithi169 conmon[100500]: audit 2023-08-28 2023-08-28T02:14:06.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:06 smithi169 conmon[100500]: T02:14:05.858841+0000 mon.a (mon 2023-08-28T02:14:06.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:06 smithi169 conmon[100500]: .0) 846 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:06.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:06 smithi169 conmon[100500]: audit 2023-08-28T02:14:06.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:06 smithi169 conmon[100500]: 2023-08-28T02:14:05.861934 2023-08-28T02:14:06.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:06 smithi169 conmon[100500]: +0000 mon.a (mon.0) 847 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:06.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:06 smithi169 conmon[100500]: 2023-08-28T02:14:06.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:06 smithi169 conmon[100500]: audit 2023 2023-08-28T02:14:06.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:06 smithi169 conmon[100500]: -08-28T02:14:05.868440+0000 mon.a (mon. 2023-08-28T02:14:06.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:06 smithi169 conmon[100500]: 0) 848 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:06.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:06 smithi169 conmon[100500]: audit 2023-08 2023-08-28T02:14:06.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:06 smithi169 conmon[100500]: -28T02:14:05.873581+0000 mon.a ( 2023-08-28T02:14:06.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:06 smithi169 conmon[100500]: mon.0) 849 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:06.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:06 smithi169 conmon[100500]: audit 2023-08-28T02:14:06.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:06 smithi169 conmon[100500]: 2023-08-28T02:14 2023-08-28T02:14:06.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:06 smithi169 conmon[100500]: :05.880090+0000 mon.a (mon.0) 850 2023-08-28T02:14:06.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:06 smithi169 conmon[100500]: : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:07.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[100258]: cluster 2023-08-28T02:14:05.774249+0000 mgr.x ( 2023-08-28T02:14:07.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[100258]: mgr.25369) 22 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 244 KiB/s wr, 28 op/s 2023-08-28T02:14:07.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[100258]: cephadm 2023-08-28T02:14:05.946782+0000 mgr.x (mgr.25369) 23 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2023-08-28T02:14:07.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[100258]: cephadm 2023-08-28T02:14:05.947092+0000 mgr.x (mgr.25369) 24 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi143 2023-08-28T02:14:07.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[109923]: cluster 2023-08-28T02:14:05.774249+0000 mgr.x ( 2023-08-28T02:14:07.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[109923]: mgr.25369) 22 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 244 KiB/s wr, 28 op/s 2023-08-28T02:14:07.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[109923]: cephadm 2023-08-28T02:14:05.946782+0000 mgr.x (mgr.25369) 23 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2023-08-28T02:14:07.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[109923]: cephadm 2023-08-28T02:14:05.947092+0000 mgr.x (mgr.25369) 24 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi143 2023-08-28T02:14:07.451 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:06 smithi143 bash[173205]: 08a67263bdce604c5d878670d376f6912281809083c09ade24c38663249ad87d 2023-08-28T02:14:07.451 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:06 smithi143 systemd[1]: ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-08-28T02:14:07.451 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 systemd[1]: ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@node-exporter.a.service: Failed with result 'exit-code'. 2023-08-28T02:14:07.451 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 systemd[1]: Stopped Ceph node-exporter.a for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:14:07.452 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 systemd[1]: Starting Ceph node-exporter.a for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:14:07.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:07 smithi169 conmon[100500]: cluster 2023-08-28T02:14:05.774249+0000 mgr.x (mgr.25369) 22 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 244 KiB/s wr, 28 op/s 2023-08-28T02:14:07.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:07 smithi169 conmon[100500]: 2023-08-28T02:14:07.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:07 smithi169 conmon[100500]: cephadm 2023-08-28T02:14:05.946782+0000 mgr.x (mgr.25369) 23 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2023-08-28T02:14:07.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:07 smithi169 conmon[100500]: cephadm 2023-08-28T02:14:05.947092+0000 mgr.x (mgr.25369) 24 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi143 2023-08-28T02:14:07.950 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.677Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-08-28T02:14:07.951 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.677Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-08-28T02:14:07.951 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.677Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-08-28T02:14:07.951 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.677Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-08-28T02:14:07.951 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.678Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-08-28T02:14:07.951 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.678Z caller=node_exporter.go:115 level=info collector=arp 2023-08-28T02:14:07.951 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.678Z caller=node_exporter.go:115 level=info collector=bcache 2023-08-28T02:14:07.951 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.678Z caller=node_exporter.go:115 level=info collector=bonding 2023-08-28T02:14:07.951 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.678Z caller=node_exporter.go:115 level=info collector=btrfs 2023-08-28T02:14:07.951 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.678Z caller=node_exporter.go:115 level=info collector=conntrack 2023-08-28T02:14:07.951 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.678Z caller=node_exporter.go:115 level=info collector=cpu 2023-08-28T02:14:07.952 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.678Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-08-28T02:14:07.952 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.678Z caller=node_exporter.go:115 level=info collector=diskstats 2023-08-28T02:14:07.952 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.678Z caller=node_exporter.go:115 level=info collector=dmi 2023-08-28T02:14:07.952 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.678Z caller=node_exporter.go:115 level=info collector=edac 2023-08-28T02:14:07.952 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.678Z caller=node_exporter.go:115 level=info collector=entropy 2023-08-28T02:14:07.952 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.678Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-08-28T02:14:07.952 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.678Z caller=node_exporter.go:115 level=info collector=filefd 2023-08-28T02:14:07.952 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.678Z caller=node_exporter.go:115 level=info collector=filesystem 2023-08-28T02:14:07.952 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.678Z caller=node_exporter.go:115 level=info collector=hwmon 2023-08-28T02:14:07.952 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.678Z caller=node_exporter.go:115 level=info collector=infiniband 2023-08-28T02:14:07.952 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.678Z caller=node_exporter.go:115 level=info collector=ipvs 2023-08-28T02:14:07.952 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.678Z caller=node_exporter.go:115 level=info collector=loadavg 2023-08-28T02:14:07.952 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.678Z caller=node_exporter.go:115 level=info collector=mdadm 2023-08-28T02:14:07.953 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.678Z caller=node_exporter.go:115 level=info collector=meminfo 2023-08-28T02:14:07.953 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.678Z caller=node_exporter.go:115 level=info collector=netclass 2023-08-28T02:14:07.953 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.678Z caller=node_exporter.go:115 level=info collector=netdev 2023-08-28T02:14:07.953 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.678Z caller=node_exporter.go:115 level=info collector=netstat 2023-08-28T02:14:07.953 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.678Z caller=node_exporter.go:115 level=info collector=nfs 2023-08-28T02:14:07.953 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.678Z caller=node_exporter.go:115 level=info collector=nfsd 2023-08-28T02:14:07.953 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.678Z caller=node_exporter.go:115 level=info collector=nvme 2023-08-28T02:14:07.953 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.678Z caller=node_exporter.go:115 level=info collector=os 2023-08-28T02:14:07.953 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.678Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-08-28T02:14:07.953 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.678Z caller=node_exporter.go:115 level=info collector=pressure 2023-08-28T02:14:07.953 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.678Z caller=node_exporter.go:115 level=info collector=rapl 2023-08-28T02:14:07.953 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.678Z caller=node_exporter.go:115 level=info collector=schedstat 2023-08-28T02:14:07.954 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.678Z caller=node_exporter.go:115 level=info collector=sockstat 2023-08-28T02:14:07.954 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.678Z caller=node_exporter.go:115 level=info collector=softnet 2023-08-28T02:14:07.954 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.678Z caller=node_exporter.go:115 level=info collector=stat 2023-08-28T02:14:07.954 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.678Z caller=node_exporter.go:115 level=info collector=tapestats 2023-08-28T02:14:07.954 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.678Z caller=node_exporter.go:115 level=info collector=textfile 2023-08-28T02:14:07.954 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.678Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-08-28T02:14:07.954 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.678Z caller=node_exporter.go:115 level=info collector=time 2023-08-28T02:14:07.954 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.678Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-08-28T02:14:07.954 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.678Z caller=node_exporter.go:115 level=info collector=uname 2023-08-28T02:14:07.954 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.678Z caller=node_exporter.go:115 level=info collector=vmstat 2023-08-28T02:14:07.954 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.678Z caller=node_exporter.go:115 level=info collector=xfs 2023-08-28T02:14:07.954 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.678Z caller=node_exporter.go:115 level=info collector=zfs 2023-08-28T02:14:07.954 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.678Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-08-28T02:14:07.955 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 conmon[173571]: ts=2023-08-28T02:14:07.678Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-08-28T02:14:07.955 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 bash[173530]: e389cc167c3986b475b5e681a0ea31b47baf25761688c53c36b732829a13a24a 2023-08-28T02:14:07.955 INFO:journalctl@ceph.node-exporter.a.smithi143.stdout:Aug 28 02:14:07 smithi143 systemd[1]: Started Ceph node-exporter.a for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:14:08.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:08 smithi169 conmon[100500]: audit 2023-08-28T02:14:07.719346+0000 mon.a (mon.0) 851 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:08.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:08 smithi169 conmon[100500]: audit 2023-08-28T02:14:07.725276+0000 mon.a ( 2023-08-28T02:14:08.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:08 smithi169 conmon[100500]: mon.0) 852 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:08.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:08 smithi169 conmon[100500]: audit 2023-08-28T02:14:08.070339+0000 mon.a (mon.0) 853 : audit [DBG] from='client.? 172.21.15.143:0/3872141279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:09.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:08 smithi143 conmon[100258]: audit 2023-08-28T02:14:07.719346+0000 mon.a (mon.0 2023-08-28T02:14:09.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:08 smithi143 conmon[100258]: ) 851 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:09.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:08 smithi143 conmon[100258]: audit 2023-08-28T02:14:07.725276+0000 mon.a (mon.0) 852 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:09.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:08 smithi143 conmon[100258]: audit 2023-08-28T02:14:08.070339+0000 mon.a (mon.0) 853 : audit [DBG] from='client.? 172.21.15.143:0/3872141279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:09.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:08 smithi143 conmon[109923]: audit 2023-08-28T02:14:07.719346+0000 mon.a ( 2023-08-28T02:14:09.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:08 smithi143 conmon[109923]: mon.0) 851 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:09.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:08 smithi143 conmon[109923]: audit 2023-08-28T02:14:07.725276+0000 mon.a (mon.0) 852 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:09.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:08 smithi143 conmon[109923]: audit 2023-08-28T02:14:08.070339+0000 mon.a (mon.0) 853 : audit [DBG] from='client.? 172.21.15.143:0/3872141279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:09.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:09 smithi169 conmon[100500]: cephadm 2023-08-28T02:14:07.765734+0000 mgr.x (mgr.25369) 25 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-08-28T02:14:09.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:09 smithi169 conmon[100500]: cluster 2023-08-28T02:14:07.774823 2023-08-28T02:14:09.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:09 smithi169 conmon[100500]: +0000 mgr.x (mgr.25369) 26 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 242 KiB/s wr, 27 op/s 2023-08-28T02:14:09.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:09 smithi169 conmon[100500]: cephadm 2023-08-28T02:14:07.778276+0000 mgr.x (mgr.25369) 27 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi143 2023-08-28T02:14:10.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:09 smithi143 conmon[109923]: cephadm 2023-08-28T02:14:07.765734 2023-08-28T02:14:10.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:09 smithi143 conmon[109923]: +0000 mgr.x (mgr.25369) 25 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-08-28T02:14:10.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:09 smithi143 conmon[109923]: cluster 2023 2023-08-28T02:14:10.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:09 smithi143 conmon[109923]: -08-28T02:14:07.774823+0000 mgr.x (mgr.25369) 26 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 242 KiB/s wr, 27 op/s 2023-08-28T02:14:10.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:09 smithi143 conmon[109923]: 2023-08-28T02:14:10.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:09 smithi143 conmon[109923]: cephadm 2023-08-28T02 2023-08-28T02:14:10.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:09 smithi143 conmon[109923]: :14:07.778276+0000 mgr.x (mgr.25369) 27 2023-08-28T02:14:10.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:09 smithi143 conmon[109923]: : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi143 2023-08-28T02:14:10.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:09 smithi143 conmon[100258]: cephadm 2023-08-28T02:14:07.765734+0000 mgr.x (mgr. 2023-08-28T02:14:10.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:09 smithi143 conmon[100258]: 25369) 25 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-08-28T02:14:10.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:09 smithi143 conmon[100258]: cluster 2023-08-28T02:14:07.774823+0000 mgr.x (mgr.25369) 26 : cluster 2023-08-28T02:14:10.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:09 smithi143 conmon[100258]: [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 242 KiB/s wr, 27 op/s 2023-08-28T02:14:10.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:09 smithi143 conmon[100258]: cephadm 2023-08-28T02:14:10.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:09 smithi143 conmon[100258]: 2023-08-28T02:14:07.778276+0000 mgr.x ( 2023-08-28T02:14:10.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:09 smithi143 conmon[100258]: mgr.25369) 27 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi143 2023-08-28T02:14:10.700 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:14:10 smithi143 conmon[100708]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:14:10] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-28T02:14:11.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:11 smithi143 conmon[100258]: cluster 2023-08-28T02:14:09.775367+0000 mgr.x (mgr.25369) 28 2023-08-28T02:14:11.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:11 smithi143 conmon[100258]: : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 204 KiB/s wr, 23 op/s 2023-08-28T02:14:11.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:11 smithi143 conmon[100258]: audit 2023-08-28T02:14:10.489029+0000 mon.c (mon.2) 46 : audit [DBG] 2023-08-28T02:14:11.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:11 smithi143 conmon[100258]: from='client.? 172.21.15.143:0/2318255532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:11.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:11 smithi143 conmon[100258]: audit 2023-08-28 2023-08-28T02:14:11.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:11 smithi143 conmon[100258]: T02:14:10.979692+0000 mon.b (mon.1) 81 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:14:11.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:11 smithi143 conmon[109923]: cluster 2023-08-28T02:14:09.775367+0000 mgr.x (mgr. 2023-08-28T02:14:11.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:11 smithi143 conmon[109923]: 25369) 28 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 204 KiB/s wr, 23 op/s 2023-08-28T02:14:11.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:11 smithi143 conmon[109923]: audit 2023-08-28T02:14:10.489029+0000 2023-08-28T02:14:11.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:11 smithi143 conmon[109923]: mon.c (mon.2) 46 : audit [DBG] from='client.? 172.21.15.143:0/2318255532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:11.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:11 smithi143 conmon[109923]: audit 2023-08-28 2023-08-28T02:14:11.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:11 smithi143 conmon[109923]: T02:14:10.979692+0000 mon.b ( 2023-08-28T02:14:11.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:11 smithi143 conmon[109923]: mon.1) 81 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:14:11.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:11 smithi169 conmon[100500]: cluster 2023-08-28T02:14:09.775367+0000 mgr.x (mgr.25369) 28 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 204 KiB/s wr, 23 op/s 2023-08-28T02:14:11.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:11 smithi169 conmon[100500]: audit 2023-08-28T02:14:10.489029+0000 mon.c (mon.2) 46 : audit [DBG] from='client.? 172.21.15.143:0/2318255532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:11.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:11 smithi169 conmon[100500]: audit 2023-08-28T02:14:10.979692+0000 mon.b (mon.1) 81 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:14:13.199 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:14:12 smithi169 conmon[138400]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:14:12] "GET /metrics HTTP/1.1" 200 33949 "" "Prometheus/2.33.4" 2023-08-28T02:14:13.372 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:13 smithi143 conmon[100258]: cluster 2023-08-28T02:14:11.775906+0000 mgr.x (mgr.25369) 29 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 201 KiB/s wr, 22 op/s 2023-08-28T02:14:13.372 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:13 smithi143 conmon[100258]: audit 2023-08-28T02:14:12.871904+0000 mon.a (mon.0) 854 : audit [DBG] from='client.? 172.21.15.143:0/2524153127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:13.372 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:13 smithi143 conmon[109923]: cluster 2023-08-28T02:14:11.775906+0000 mgr.x (mgr.25369) 29 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 201 KiB/s wr, 22 op/s 2023-08-28T02:14:13.372 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:13 smithi143 conmon[109923]: audit 2023-08-28T02:14:13.372 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:13 smithi143 conmon[109923]: 2023-08-28T02:14:12.871904+0000 mon.a (mon.0) 854 : audit [DBG] from='client.? 172.21.15.143:0/2524153127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:13.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:13 smithi169 conmon[100500]: cluster 2023-08-28T02:14:11.775906+0000 mgr.x (mgr.25369) 29 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 201 KiB/s wr, 22 op/s 2023-08-28T02:14:13.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:13 smithi169 conmon[100500]: audit 2023-08-28T02:14:12.871904+0000 mon.a (mon.0) 854 : audit [DBG] from='client.? 172.21.15.143:0/2524153127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:14.110 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:14:13 smithi143 systemd[1]: Stopping Ceph alertmanager.a for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:14:14.110 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:14:13 smithi143 bash[174107]: Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-alertmanager.a found: no such container 2023-08-28T02:14:14.450 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:14:14 smithi143 conmon[150510]: level=info ts=2023-08-28T02:14:14.111Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2023-08-28T02:14:14.950 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:14:14 smithi143 bash[174223]: 24d74df782bd39fafc2f94bfeca17e010e4db5855995ca0feefbd734d5a21798 2023-08-28T02:14:14.950 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:14:14 smithi143 systemd[1]: ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@alertmanager.a.service: Succeeded. 2023-08-28T02:14:14.950 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:14:14 smithi143 systemd[1]: Stopped Ceph alertmanager.a for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:14:14.950 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:14:14 smithi143 systemd[1]: Starting Ceph alertmanager.a for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:14:15.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:15 smithi169 conmon[100500]: cluster 2023-08-28T02:14:13.776501+0000 mgr.x (mgr.25369) 30 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 201 KiB/s wr, 22 op/s 2023-08-28T02:14:15.490 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:15 smithi143 conmon[100258]: cluster 2023-08-28T02:14:13.776501+0000 mgr.x (mgr 2023-08-28T02:14:15.490 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:15 smithi143 conmon[100258]: .25369) 30 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 201 KiB/s wr, 22 op/s 2023-08-28T02:14:15.490 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:15 smithi143 conmon[109923]: cluster 2023-08-28T02:14: 2023-08-28T02:14:15.490 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:15 smithi143 conmon[109923]: 13.776501+0000 mgr.x (mgr.25369) 30 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 201 KiB/s wr, 22 op/s 2023-08-28T02:14:15.950 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:14:15 smithi143 conmon[174616]: level=info ts=2023-08-28T02:14:15.491Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2023-08-28T02:14:15.950 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:14:15 smithi143 conmon[174616]: level=info ts=2023-08-28T02:14:15.491Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2023-08-28T02:14:15.950 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:14:15 smithi143 conmon[174616]: level=info ts=2023-08-28T02:14:15.492Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.143 port=9094 2023-08-28T02:14:15.951 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:14:15 smithi143 conmon[174616]: level=info ts=2023-08-28T02:14:15.494Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-08-28T02:14:15.951 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:14:15 smithi143 bash[174572]: f216a6d30bb2d43c584fabd5fe043e1410119aceae4b889513f2b59d87e85c64 2023-08-28T02:14:15.951 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:14:15 smithi143 systemd[1]: Started Ceph alertmanager.a for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:14:15.951 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:14:15 smithi143 conmon[174616]: level=info ts=2023-08-28T02:14:15.524Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-08-28T02:14:15.951 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:14:15 smithi143 conmon[174616]: level=info ts=2023-08-28T02:14:15.525Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-08-28T02:14:15.951 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:14:15 smithi143 conmon[174616]: level=info ts=2023-08-28T02:14:15.527Z caller=main.go:518 msg=Listening address=:9093 2023-08-28T02:14:15.951 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:14:15 smithi143 conmon[174616]: level=info ts=2023-08-28T02:14:15.527Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2023-08-28T02:14:16.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:16 smithi169 conmon[100500]: audit 2023-08-28T02:14:15.287032+0000 mon.c (mon.2) 2023-08-28T02:14:16.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:16 smithi169 conmon[100500]: 47 : audit [DBG] from='client.? 172.21.15.143:0/1744560909' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:16.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:16 smithi169 conmon[100500]: audit 2023-08-28T02:14:15.530791+0000 mon.a (mon.0) 855 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:16.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:16 smithi169 conmon[100500]: audit 2023-08-28T02:14:15.540964+0000 mon.a (mon.0) 856 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:16.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:16 smithi169 conmon[100500]: cephadm 2023-08-28T02:14:15.560291+0000 mgr.x (mgr.25369) 31 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-08-28T02:14:16.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:16 smithi143 conmon[100258]: audit 2023-08-28T02:14:15.287032+0000 mon.c (mon.2) 47 : audit [DBG] from='client.? 172.21.15.143:0/1744560909' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:16.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:16 smithi143 conmon[100258]: 2023-08-28T02:14:16.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:16 smithi143 conmon[100258]: audit 2023-08-28T02:14:15.530791+0000 mon.a (mon.0) 855 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:16.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:16 smithi143 conmon[100258]: audit 2023-08-28T02 2023-08-28T02:14:16.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:16 smithi143 conmon[100258]: :14:15.540964+0000 mon.a (mon.0) 856 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:16.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:16 smithi143 conmon[100258]: cephadm 2023-08-28T02:14 2023-08-28T02:14:16.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:16 smithi143 conmon[100258]: :15.560291+0000 mgr.x (mgr.25369) 31 : 2023-08-28T02:14:16.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:16 smithi143 conmon[100258]: cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-08-28T02:14:16.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:16 smithi143 conmon[109923]: audit 2023-08-28T02:14:15. 2023-08-28T02:14:16.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:16 smithi143 conmon[109923]: 287032+0000 mon.c (mon.2) 47 : audit [DBG] from='client.? 172.21.15.143:0/1744560909' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:16.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:16 smithi143 conmon[109923]: audit 2023-08-28T02:14:16.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:16 smithi143 conmon[109923]: 2023-08-28T02:14:15.530791+0000 mon.a (mon.0) 855 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:16.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:16 smithi143 conmon[109923]: audit 2023-08-28T02:14:16.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:16 smithi143 conmon[109923]: 2023-08-28T02:14: 2023-08-28T02:14:16.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:16 smithi143 conmon[109923]: 15.540964+0000 mon.a (mon.0) 2023-08-28T02:14:16.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:16 smithi143 conmon[109923]: 856 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:16.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:16 smithi143 conmon[109923]: cephadm 2023-08-28T 2023-08-28T02:14:16.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:16 smithi143 conmon[109923]: 02:14:15.560291+0000 mgr.x (mgr.25369) 31 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-08-28T02:14:17.950 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:14:17 smithi143 conmon[174616]: level=info ts=2023-08-28T02:14:17.495Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000912268s 2023-08-28T02:14:17.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:17 smithi143 conmon[109923]: cluster 2023-08-28T02:14:15 2023-08-28T02:14:17.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:17 smithi143 conmon[109923]: .776802+0000 mgr.x (mgr.25369) 32 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-08-28T02:14:17.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:17 smithi143 conmon[109923]: cephadm 2023-08-28T02 2023-08-28T02:14:17.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:17 smithi143 conmon[109923]: :14:15.780678+0000 mgr.x (mgr. 2023-08-28T02:14:17.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:17 smithi143 conmon[109923]: 25369) 33 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi169 2023-08-28T02:14:17.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:17 smithi143 conmon[100258]: cluster 2023-08-28T02:14:15 2023-08-28T02:14:17.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:17 smithi143 conmon[100258]: .776802+0000 mgr.x (mgr.25369) 32 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-08-28T02:14:17.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:17 smithi143 conmon[100258]: cephadm 2023-08-28T02:14:17.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:17 smithi143 conmon[100258]: 2023-08-28T02:14:15.780678+0000 mgr.x 2023-08-28T02:14:17.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:17 smithi143 conmon[100258]: (mgr.25369) 33 : cephadm 2023-08-28T02:14:17.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:17 smithi143 conmon[100258]: [INF] Reconfiguring daemon prometheus.a on smithi169 2023-08-28T02:14:17.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:17 smithi169 conmon[100500]: cluster 2023-08-28T02:14:15. 2023-08-28T02:14:17.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:17 smithi169 conmon[100500]: 776802+0000 mgr.x (mgr.25369) 32 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-08-28T02:14:17.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:17 smithi169 conmon[100500]: cephadm 2023-08-28T02:14: 2023-08-28T02:14:17.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:17 smithi169 conmon[100500]: 15.780678+0000 mgr.x (mgr.25369) 33 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi169 2023-08-28T02:14:18.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:18 smithi143 conmon[100258]: audit 2023-08-28T02:14:17.668101+0000 mon.a (mon.0) 857 : audit [DBG] from='client.? 172.21.15.143:0/2031436420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:18.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:18 smithi143 conmon[109923]: audit 2023-08-28T02:14:17. 2023-08-28T02:14:18.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:18 smithi143 conmon[109923]: 668101+0000 mon.a (mon.0) 857 : audit [DBG] from='client.? 172.21.15.143:0/2031436420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:18.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:18 smithi169 conmon[100500]: audit 2023-08-28T02:14:17.668101+0000 mon.a (mon.0) 2023-08-28T02:14:18.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:18 smithi169 conmon[100500]: 857 : audit [DBG] from='client.? 172.21.15.143:0/2031436420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:19.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:19 smithi143 conmon[100258]: cluster 2023-08-28T02:14:17.777252+0000 mgr.x (mgr.25369) 34 : cluster 2023-08-28T02:14:19.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:19 smithi143 conmon[100258]: [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-08-28T02:14:19.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:19 smithi143 conmon[109923]: cluster 2023-08-28T02:14:17.777252+0000 mgr.x (mgr.25369) 34 : cluster 2023-08-28T02:14:19.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:19 smithi143 conmon[109923]: [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-08-28T02:14:19.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:19 smithi169 conmon[100500]: cluster 2023-08-28T02:14:17.777252+0000 mgr.x (mgr.25369) 2023-08-28T02:14:19.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:19 smithi169 conmon[100500]: 34 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 1 op/s 2023-08-28T02:14:20.563 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:14:20 smithi143 conmon[100708]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:14:20] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-08-28T02:14:20.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:20 smithi143 conmon[100258]: audit 2023-08-28T02:14:20.043082+0000 mon.a (mon.0) 858 : audit [DBG] from='client.? 172.21.15.143:0/4126930074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:20.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:20 smithi143 conmon[109923]: audit 2023-08-28T02:14:20.043082+0000 mon.a (mon. 2023-08-28T02:14:20.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:20 smithi143 conmon[109923]: 0) 858 : audit [DBG] from='client.? 172.21.15.143:0/4126930074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:20.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:20 smithi169 conmon[100500]: audit 2023-08-28T02:14:20.043082+0000 mon.a (mon.0) 2023-08-28T02:14:20.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:20 smithi169 conmon[100500]: 858 : audit [DBG] from='client.? 172.21.15.143:0/4126930074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:21.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:21 smithi143 conmon[100258]: cluster 2023-08-28T02:14:19.777636+0000 mgr.x (mgr.25369) 35 : cluster [DBG] 2023-08-28T02:14:21.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:21 smithi143 conmon[100258]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:21.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:21 smithi143 conmon[109923]: cluster 2023-08-28T02:14:19.777636+0000 mgr.x (mgr.25369) 35 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:21.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:21 smithi169 conmon[100500]: cluster 2023-08-28T02:14:19.777636+0000 mgr.x (mgr.25369) 35 2023-08-28T02:14:21.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:21 smithi169 conmon[100500]: : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:22.573 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:14:22 smithi169 conmon[133764]: ts=2023-08-28T02:14:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi169.front.sepia.ceph.com:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.143:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-08-28T02:14:22.865 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:22 smithi169 conmon[100500]: audit 2023-08-28T02:14:22.428121+0000 mon.c (mon.2) 48 : audit [DBG] from='client.? 172.21.15.143:0/2121460501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:22.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:22 smithi143 conmon[100258]: audit 2023-08-28T02:14:22.428121+0000 mon.c (mon.2) 48 : audit 2023-08-28T02:14:22.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:22 smithi143 conmon[100258]: [DBG] from='client.? 172.21.15.143:0/2121460501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:22.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:22 smithi143 conmon[109923]: audit 2023-08-28T02:14: 2023-08-28T02:14:22.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:22 smithi143 conmon[109923]: 22.428121+0000 mon.c (mon.2) 48 : audit [DBG] from='client.? 172.21.15.143:0/2121460501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:23.237 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:14:22 smithi169 conmon[138400]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:14:22] "GET /metrics HTTP/1.1" 200 33949 "" "Prometheus/2.33.4" 2023-08-28T02:14:23.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:23 smithi143 conmon[100258]: cluster 2023-08-28T02:14:21.778073+0000 mgr.x (mgr.25369) 36 2023-08-28T02:14:23.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:23 smithi143 conmon[100258]: : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:23.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:23 smithi143 conmon[109923]: cluster 2023-08-28T02:14:21.778073+0000 mgr.x (mgr.25369) 2023-08-28T02:14:23.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:23 smithi143 conmon[109923]: 36 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:23.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:23 smithi169 conmon[100500]: cluster 2023-08-28T02:14:21.778073+0000 mgr.x (mgr.25369) 36 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:25.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:25 smithi169 conmon[100500]: cluster 2023-08-28T02:14:23.778814+0000 mgr.x (mgr.25369) 37 : cluster 2023-08-28T02:14:25.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:25 smithi169 conmon[100500]: [DBG] pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:25.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:25 smithi169 conmon[100500]: audit 2023-08-28T02:14:24.804419+0000 mon.c (mon.2) 49 : audit [DBG] from='client.? 172.21.15.143:0/2753445491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:25.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:25 smithi143 conmon[109923]: cluster 2023-08-28T02:14:23.778814+0000 mgr.x (mgr.25369) 37 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:25.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:25 smithi143 conmon[109923]: audit 2023-08-28T02:14:24.804419+0000 mon.c (mon.2) 49 : audit [DBG] from='client.? 172.21.15.143:0/2753445491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:25.951 INFO:journalctl@ceph.alertmanager.a.smithi143.stdout:Aug 28 02:14:25 smithi143 conmon[174616]: level=info ts=2023-08-28T02:14:25.498Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.004129224s 2023-08-28T02:14:25.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:25 smithi143 conmon[100258]: cluster 2023-08-28T02:14:23.778814+0000 mgr.x (mgr.25369 2023-08-28T02:14:25.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:25 smithi143 conmon[100258]: ) 37 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:25.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:25 smithi143 conmon[100258]: audit 2023-08-28T02:14:24.804419+0000 mon.c (mon.2) 49 : audit [DBG] from='client.? 172.21.15.143:0/2753445491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:26.488 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:14:26 smithi169 systemd[1]: Stopping Ceph prometheus.a for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:14:26.488 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:14:26 smithi169 bash[143694]: Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-prometheus.a found: no such container 2023-08-28T02:14:26.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:26 smithi143 conmon[100258]: audit 2023-08-28T02:14:25.979714+0000 mon.b (mon.1) 82 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:14:26.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:26 smithi143 conmon[109923]: audit 2023-08-28T02:14:25.979714+0000 mon.b (mon.1) 82 : 2023-08-28T02:14:26.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:26 smithi143 conmon[109923]: audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:14:26.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:26 smithi169 conmon[100500]: audit 2023-08-28T02:14:25.979714+0000 mon.b (mon.1) 2023-08-28T02:14:26.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:26 smithi169 conmon[100500]: 82 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:14:26.988 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:14:26 smithi169 conmon[133764]: ts=2023-08-28T02:14:26.549Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-08-28T02:14:26.988 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:14:26 smithi169 conmon[133764]: ts=2023-08-28T02:14:26.549Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-08-28T02:14:26.988 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:14:26 smithi169 conmon[133764]: ts=2023-08-28T02:14:26.549Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-08-28T02:14:26.988 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:14:26 smithi169 conmon[133764]: ts=2023-08-28T02:14:26.549Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-08-28T02:14:26.988 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:14:26 smithi169 conmon[133764]: ts=2023-08-28T02:14:26.549Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-08-28T02:14:26.988 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:14:26 smithi169 conmon[133764]: ts=2023-08-28T02:14:26.549Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-08-28T02:14:26.988 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:14:26 smithi169 conmon[133764]: ts=2023-08-28T02:14:26.549Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-08-28T02:14:26.988 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:14:26 smithi169 conmon[133764]: ts=2023-08-28T02:14:26.549Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-08-28T02:14:26.988 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:14:26 smithi169 conmon[133764]: ts=2023-08-28T02:14:26.549Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-08-28T02:14:26.988 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:14:26 smithi169 conmon[133764]: ts=2023-08-28T02:14:26.549Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-08-28T02:14:26.988 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:14:26 smithi169 conmon[133764]: ts=2023-08-28T02:14:26.549Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-08-28T02:14:26.988 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:14:26 smithi169 conmon[133764]: ts=2023-08-28T02:14:26.549Z caller=main.go:1066 level=info msg="See you next time!" 2023-08-28T02:14:27.487 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:14:27 smithi169 bash[143810]: 3ceb1bb30c93983c012eabe6ff07715e588ed09c03dffad2bfde6999f396e3e3 2023-08-28T02:14:27.487 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:14:27 smithi169 systemd[1]: ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@prometheus.a.service: Succeeded. 2023-08-28T02:14:27.487 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:14:27 smithi169 systemd[1]: Stopped Ceph prometheus.a for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:14:27.487 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:14:27 smithi169 systemd[1]: Starting Ceph prometheus.a for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:14:27.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:27 smithi143 conmon[100258]: cluster 2023-08-28T02:14:25.779139+0000 mgr.x (mgr.25369) 38 2023-08-28T02:14:27.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:27 smithi143 conmon[100258]: : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:27.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:27 smithi143 conmon[100258]: audit 2023-08-28T02:14:27.181541+0000 mon.c (mon.2) 50 : audit [DBG] from='client.? 172.21.15.143:0/4159186291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:27.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:27 smithi143 conmon[109923]: cluster 2023-08-28T02:14:25.779139+0000 mgr.x ( 2023-08-28T02:14:27.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:27 smithi143 conmon[109923]: mgr.25369) 38 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:27.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:27 smithi143 conmon[109923]: audit 2023-08-28T02:14:27.181541+0000 mon.c (mon.2) 50 : audit 2023-08-28T02:14:27.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:27 smithi143 conmon[109923]: [DBG] from='client.? 172.21.15.143:0/4159186291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:27.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:27 smithi169 conmon[100500]: cluster 2023-08-28T02:14:25. 2023-08-28T02:14:27.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:27 smithi169 conmon[100500]: 779139+0000 mgr.x (mgr.25369) 38 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:27.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:27 smithi169 conmon[100500]: audit 2023-08-28T02:14:27.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:27 smithi169 conmon[100500]: 2023-08-28T02:14:27.181541+0000 mon.c ( 2023-08-28T02:14:27.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:27 smithi169 conmon[100500]: mon.2) 50 : audit [DBG] 2023-08-28T02:14:27.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:27 smithi169 conmon[100500]: from='client.? 172.21.15.143:0/4159186291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:27.979 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:14:27 smithi169 conmon[144179]: ts=2023-08-28T02:14:27.815Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-08-28T02:14:27.979 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:14:27 smithi169 conmon[144179]: ts=2023-08-28T02:14:27.815Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-08-28T02:14:27.979 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:14:27 smithi169 conmon[144179]: ts=2023-08-28T02:14:27.815Z caller=main.go:518 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi169 (none))" 2023-08-28T02:14:27.979 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:14:27 smithi169 conmon[144179]: ts=2023-08-28T02:14:27.815Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-08-28T02:14:27.980 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:14:27 smithi169 conmon[144179]: ts=2023-08-28T02:14:27.815Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-08-28T02:14:27.980 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:14:27 smithi169 conmon[144179]: ts=2023-08-28T02:14:27.817Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-08-28T02:14:27.980 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:14:27 smithi169 conmon[144179]: ts=2023-08-28T02:14:27.817Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-08-28T02:14:27.980 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:14:27 smithi169 conmon[144179]: ts=2023-08-28T02:14:27.818Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-08-28T02:14:27.980 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:14:27 smithi169 conmon[144179]: ts=2023-08-28T02:14:27.820Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-08-28T02:14:27.980 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:14:27 smithi169 conmon[144179]: ts=2023-08-28T02:14:27.821Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.07µs 2023-08-28T02:14:27.980 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:14:27 smithi169 conmon[144179]: ts=2023-08-28T02:14:27.821Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-08-28T02:14:27.981 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:14:27 smithi169 bash[144137]: b370b47c85c80d837f76262930915f003371f5a22085d0e9235ca3fd913e35b1 2023-08-28T02:14:27.981 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:14:27 smithi169 systemd[1]: Started Ceph prometheus.a for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:14:27.981 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:14:27 smithi169 conmon[144179]: ts=2023-08-28T02:14:27.978Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2023-08-28T02:14:28.237 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:28 smithi169 systemd[1]: Stopping Ceph node-exporter.b for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:14:28.237 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:28 smithi169 bash[144330]: Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-node-exporter.b found: no such container 2023-08-28T02:14:28.237 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:14:28 smithi169 conmon[144179]: ts=2023-08-28T02:14:28.203Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2023-08-28T02:14:28.237 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:14:28 smithi169 conmon[144179]: ts=2023-08-28T02:14:28.203Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2023-08-28T02:14:28.238 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:14:28 smithi169 conmon[144179]: ts=2023-08-28T02:14:28.203Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=37.144µs wal_replay_duration=382.823226ms total_replay_duration=382.873736ms 2023-08-28T02:14:28.238 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:14:28 smithi169 conmon[144179]: ts=2023-08-28T02:14:28.207Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-08-28T02:14:28.238 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:14:28 smithi169 conmon[144179]: ts=2023-08-28T02:14:28.207Z caller=main.go:947 level=info msg="TSDB started" 2023-08-28T02:14:28.238 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:14:28 smithi169 conmon[144179]: ts=2023-08-28T02:14:28.207Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-08-28T02:14:28.238 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:14:28 smithi169 conmon[144179]: ts=2023-08-28T02:14:28.224Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=17.410085ms db_storage=967ns remote_storage=1.808µs web_handler=384ns query_engine=751ns scrape=482.906µs scrape_sd=80.569µs notify=16.557µs notify_sd=15.427µs rules=16.381481ms 2023-08-28T02:14:28.238 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:14:28 smithi169 conmon[144179]: ts=2023-08-28T02:14:28.224Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-08-28T02:14:29.153 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:28 smithi169 bash[144446]: b3659a52b45015144f30af8920d26185ebe63902c5f30051e7f5197bedd53aad 2023-08-28T02:14:29.153 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:28 smithi169 systemd[1]: ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-08-28T02:14:29.153 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:28 smithi169 conmon[100500]: cluster 2023-08-28T02:14:27.779583+0000 mgr.x (mgr.25369) 39 : 2023-08-28T02:14:29.153 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:28 smithi169 conmon[100500]: cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:29.153 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:28 smithi169 conmon[100500]: audit 2023-08-28T02:14:27.856279+0000 mon.a (mon.0) 859 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:29.153 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:28 smithi169 conmon[100500]: audit 2023-08-28T02: 2023-08-28T02:14:29.154 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:28 smithi169 conmon[100500]: 14:27.862018+0000 mon.a (mon.0) 860 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:29.154 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:28 smithi169 conmon[100500]: cephadm 2023-08-28T02 2023-08-28T02:14:29.154 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:28 smithi169 conmon[100500]: :14:27.883382+0000 mgr.x (mgr.25369) 40 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2023-08-28T02:14:29.154 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:28 smithi169 conmon[100500]: cephadm 2023- 2023-08-28T02:14:29.154 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:28 smithi169 conmon[100500]: 08-28T02:14:27.883859+0000 mgr.x ( 2023-08-28T02:14:29.154 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:28 smithi169 conmon[100500]: mgr.25369) 41 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi169 2023-08-28T02:14:29.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:28 smithi143 conmon[109923]: cluster 2023-08-28T02:14:27.779583+0000 mgr.x (mgr.25369) 39 2023-08-28T02:14:29.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:28 smithi143 conmon[109923]: : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:29.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:28 smithi143 conmon[109923]: audit 2023-08-28T02:14:27.856279+0000 mon.a (mon.0) 859 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:29.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:28 smithi143 conmon[109923]: audit 2023-08-28T02:14:27.862018+0000 mon.a (mon.0) 860 : audit [INF] 2023-08-28T02:14:29.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:28 smithi143 conmon[109923]: from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:29.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:28 smithi143 conmon[109923]: cephadm 2023-08-28T02:14:29.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:28 smithi143 conmon[109923]: 2023-08-28T02:14:27.883382+0000 mgr.x (mgr. 2023-08-28T02:14:29.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:28 smithi143 conmon[109923]: 25369) 40 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2023-08-28T02:14:29.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:28 smithi143 conmon[109923]: cephadm 2023-08-28 2023-08-28T02:14:29.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:28 smithi143 conmon[109923]: T02:14:27.883859+0000 mgr.x (mgr.25369 2023-08-28T02:14:29.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:28 smithi143 conmon[109923]: ) 41 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi169 2023-08-28T02:14:29.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:28 smithi143 conmon[100258]: cluster 2023-08-28T02:14:27.779583+0000 mgr.x ( 2023-08-28T02:14:29.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:28 smithi143 conmon[100258]: mgr.25369) 39 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:29.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:28 smithi143 conmon[100258]: audit 2023-08-28T02:14:27.856279+0000 mon.a (mon.0) 859 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:29.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:28 smithi143 conmon[100258]: 2023-08-28T02:14:29.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:28 smithi143 conmon[100258]: audit 2023-08- 2023-08-28T02:14:29.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:28 smithi143 conmon[100258]: 28T02:14:27.862018+0000 mon.a (mon.0) 860 : audit [INF] 2023-08-28T02:14:29.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:28 smithi143 conmon[100258]: from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:29.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:28 smithi143 conmon[100258]: cephadm 2023-08-28 2023-08-28T02:14:29.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:28 smithi143 conmon[100258]: T02:14:27.883382+0000 mgr.x (mgr.25369) 40 : cephadm [INF] 2023-08-28T02:14:29.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:28 smithi143 conmon[100258]: Reconfiguring node-exporter.b (dependencies changed)... 2023-08-28T02:14:29.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:28 smithi143 conmon[100258]: cephadm 2023- 2023-08-28T02:14:29.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:28 smithi143 conmon[100258]: 08-28T02:14:27.883859+0000 mgr.x (mgr.25369) 41 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi169 2023-08-28T02:14:29.487 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 systemd[1]: ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@node-exporter.b.service: Failed with result 'exit-code'. 2023-08-28T02:14:29.487 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 systemd[1]: Stopped Ceph node-exporter.b for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:14:29.488 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 systemd[1]: Starting Ceph node-exporter.b for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:14:29.860 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.743Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-08-28T02:14:29.860 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.743Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-08-28T02:14:29.860 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.743Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-08-28T02:14:29.861 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.743Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-08-28T02:14:29.861 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.744Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-08-28T02:14:29.861 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.744Z caller=node_exporter.go:115 level=info collector=arp 2023-08-28T02:14:29.861 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.744Z caller=node_exporter.go:115 level=info collector=bcache 2023-08-28T02:14:29.861 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.744Z caller=node_exporter.go:115 level=info collector=bonding 2023-08-28T02:14:29.861 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.744Z caller=node_exporter.go:115 level=info collector=btrfs 2023-08-28T02:14:29.861 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.744Z caller=node_exporter.go:115 level=info collector=conntrack 2023-08-28T02:14:29.861 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.744Z caller=node_exporter.go:115 level=info collector=cpu 2023-08-28T02:14:29.861 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.744Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-08-28T02:14:29.861 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.744Z caller=node_exporter.go:115 level=info collector=diskstats 2023-08-28T02:14:29.862 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.744Z caller=node_exporter.go:115 level=info collector=dmi 2023-08-28T02:14:29.862 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.744Z caller=node_exporter.go:115 level=info collector=edac 2023-08-28T02:14:29.862 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.744Z caller=node_exporter.go:115 level=info collector=entropy 2023-08-28T02:14:29.862 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.744Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-08-28T02:14:29.862 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.744Z caller=node_exporter.go:115 level=info collector=filefd 2023-08-28T02:14:29.862 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.744Z caller=node_exporter.go:115 level=info collector=filesystem 2023-08-28T02:14:29.862 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.744Z caller=node_exporter.go:115 level=info collector=hwmon 2023-08-28T02:14:29.862 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.744Z caller=node_exporter.go:115 level=info collector=infiniband 2023-08-28T02:14:29.862 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.744Z caller=node_exporter.go:115 level=info collector=ipvs 2023-08-28T02:14:29.862 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.744Z caller=node_exporter.go:115 level=info collector=loadavg 2023-08-28T02:14:29.862 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.744Z caller=node_exporter.go:115 level=info collector=mdadm 2023-08-28T02:14:29.863 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.744Z caller=node_exporter.go:115 level=info collector=meminfo 2023-08-28T02:14:29.863 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.744Z caller=node_exporter.go:115 level=info collector=netclass 2023-08-28T02:14:29.863 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.744Z caller=node_exporter.go:115 level=info collector=netdev 2023-08-28T02:14:29.863 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.744Z caller=node_exporter.go:115 level=info collector=netstat 2023-08-28T02:14:29.863 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.744Z caller=node_exporter.go:115 level=info collector=nfs 2023-08-28T02:14:29.863 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.744Z caller=node_exporter.go:115 level=info collector=nfsd 2023-08-28T02:14:29.863 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.744Z caller=node_exporter.go:115 level=info collector=nvme 2023-08-28T02:14:29.863 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.744Z caller=node_exporter.go:115 level=info collector=os 2023-08-28T02:14:29.863 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.744Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-08-28T02:14:29.864 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.744Z caller=node_exporter.go:115 level=info collector=pressure 2023-08-28T02:14:29.864 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.744Z caller=node_exporter.go:115 level=info collector=rapl 2023-08-28T02:14:29.864 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.744Z caller=node_exporter.go:115 level=info collector=schedstat 2023-08-28T02:14:29.864 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.744Z caller=node_exporter.go:115 level=info collector=sockstat 2023-08-28T02:14:29.864 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.744Z caller=node_exporter.go:115 level=info collector=softnet 2023-08-28T02:14:29.864 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.744Z caller=node_exporter.go:115 level=info collector=stat 2023-08-28T02:14:29.864 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.744Z caller=node_exporter.go:115 level=info collector=tapestats 2023-08-28T02:14:29.864 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.744Z caller=node_exporter.go:115 level=info collector=textfile 2023-08-28T02:14:29.864 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.744Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-08-28T02:14:29.865 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.744Z caller=node_exporter.go:115 level=info collector=time 2023-08-28T02:14:29.865 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.744Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-08-28T02:14:29.865 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.744Z caller=node_exporter.go:115 level=info collector=uname 2023-08-28T02:14:29.865 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.744Z caller=node_exporter.go:115 level=info collector=vmstat 2023-08-28T02:14:29.865 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.744Z caller=node_exporter.go:115 level=info collector=xfs 2023-08-28T02:14:29.865 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.744Z caller=node_exporter.go:115 level=info collector=zfs 2023-08-28T02:14:29.865 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.744Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-08-28T02:14:29.865 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[144819]: ts=2023-08-28T02:14:29.744Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-08-28T02:14:29.866 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 bash[144778]: 5e5215d11ac88dffe918b35efb79f8bc4355a3d098160a79dbab312dedc24553 2023-08-28T02:14:29.866 INFO:journalctl@ceph.node-exporter.b.smithi169.stdout:Aug 28 02:14:29 smithi169 systemd[1]: Started Ceph node-exporter.b for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:14:30.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:29 smithi143 conmon[100258]: audit 2023-08-28T02:14:29.557331+0000 mon.c 2023-08-28T02:14:30.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:29 smithi143 conmon[100258]: (mon.2) 51 : audit [DBG] from='client.? 172.21.15.143:0/3201130108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:30.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:29 smithi143 conmon[100258]: audit 2023-08-28T02:14:29.795141+0000 mon.a (mon.0) 861 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:30.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:29 smithi143 conmon[100258]: audit 2023-08-28T02:14:29.799861+0000 mon.a ( 2023-08-28T02:14:30.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:29 smithi143 conmon[100258]: mon.0) 862 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:30.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:29 smithi143 conmon[109923]: audit 2023-08-28T02:14:29.557331+0000 mon.c ( 2023-08-28T02:14:30.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:29 smithi143 conmon[109923]: mon.2) 51 : audit [DBG] from='client.? 172.21.15.143:0/3201130108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:30.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:29 smithi143 conmon[109923]: audit 2023-08-28T02:14:29.795141+0000 mon.a (mon.0) 861 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:30.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:29 smithi143 conmon[109923]: audit 2023-08-28T02:14:29.799861+0000 mon.a (mon.0) 862 : audit [INF] 2023-08-28T02:14:30.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:29 smithi143 conmon[109923]: from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:30.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[100500]: audit 2023-08-28T02:14:29.557331+0000 mon.c (mon.2) 51 2023-08-28T02:14:30.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[100500]: : audit [DBG] from='client.? 172.21.15.143:0/3201130108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:30.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[100500]: audit 2023-08-28T02:14:29.795141+0000 mon.a (mon.0) 861 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:30.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[100500]: audit 2023-08- 2023-08-28T02:14:30.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:29 smithi169 conmon[100500]: 28T02:14:29.799861+0000 mon.a (mon.0) 862 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:31.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:31 smithi143 conmon[100258]: cluster 2023-08-28T02:14:29.779866+0000 mgr.x (mgr.25369) 42 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:31.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:31 smithi143 conmon[100258]: cephadm 2023-08-28T02:14:29.850479+0000 mgr.x (mgr.25369) 43 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2023-08-28T02:14:31.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:31 smithi143 conmon[100258]: cephadm 2023-08-28T02:14:29.862170+0000 mgr.x (mgr.25369) 2023-08-28T02:14:31.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:31 smithi143 conmon[100258]: 44 : cephadm [INF] Reconfiguring daemon grafana.a on smithi169 2023-08-28T02:14:31.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:31 smithi143 conmon[109923]: cluster 2023-08-28T02:14:29.779866+0000 mgr.x (mgr.25369) 42 : 2023-08-28T02:14:31.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:31 smithi143 conmon[109923]: cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:31.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:31 smithi143 conmon[109923]: cephadm 2023-08-28T02:14:29.850479+0000 mgr.x (mgr.25369) 43 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2023-08-28T02:14:31.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:31 smithi143 conmon[109923]: cephadm 2023-08-28T02:14:29.862170+0000 mgr.x (mgr.25369) 44 : cephadm 2023-08-28T02:14:31.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:31 smithi143 conmon[109923]: [INF] Reconfiguring daemon grafana.a on smithi169 2023-08-28T02:14:31.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:31 smithi169 conmon[100500]: cluster 2023-08-28T02:14:29.779866+0000 mgr.x (mgr.25369) 42 2023-08-28T02:14:31.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:31 smithi169 conmon[100500]: : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:31.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:31 smithi169 conmon[100500]: cephadm 2023-08-28T02:14:29.850479+0000 mgr.x (mgr.25369) 43 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2023-08-28T02:14:31.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:31 smithi169 conmon[100500]: cephadm 2023-08-28T02:14:29.862170+0000 mgr.x (mgr.25369) 44 : cephadm [INF] Reconfiguring daemon grafana.a on smithi169 2023-08-28T02:14:32.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:32 smithi169 conmon[100500]: audit 2023-08-28T02:14:31.938413+0000 mon.a (mon.0 2023-08-28T02:14:32.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:32 smithi169 conmon[100500]: ) 863 : audit [DBG] from='client.? 172.21.15.143:0/182876203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:32.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:32 smithi143 conmon[100258]: audit 2023-08-28T02:14:31.938413+0000 mon.a (mon.0) 863 2023-08-28T02:14:32.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:32 smithi143 conmon[100258]: : audit [DBG] from='client.? 172.21.15.143:0/182876203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:32.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:32 smithi143 conmon[109923]: audit 2023-08-28T02:14:31.938413+0000 mon.a ( 2023-08-28T02:14:32.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:32 smithi143 conmon[109923]: mon.0) 863 : audit [DBG] from='client.? 172.21.15.143:0/182876203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:33.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:33 smithi169 conmon[100500]: cluster 2023-08-28T02:14:31.780298+0000 mgr.x (mgr 2023-08-28T02:14:33.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:33 smithi169 conmon[100500]: .25369) 45 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:33.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:33 smithi143 conmon[100258]: cluster 2023-08-28T02:14:31.780298+0000 mgr.x (mgr.25369) 45 : cluster [DBG] 2023-08-28T02:14:33.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:33 smithi143 conmon[100258]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:33.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:33 smithi143 conmon[109923]: cluster 2023-08-28T02:14:31.780298+0000 mgr.x (mgr.25369) 2023-08-28T02:14:33.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:33 smithi143 conmon[109923]: 45 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:35.480 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:35 smithi169 conmon[100500]: cluster 2023-08-28T02:14:33.780726+0000 mgr.x (mgr.25369) 46 : 2023-08-28T02:14:35.480 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:35 smithi169 conmon[100500]: cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:35.480 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:35 smithi169 conmon[100500]: audit 2023-08-28T02:14:34.321833+0000 mon.a (mon.0) 864 : audit [DBG] from='client.? 172.21.15.143:0/4159089354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:35.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:35 smithi143 conmon[100258]: cluster 2023-08-28T02:14:33.780726+0000 mgr.x (mgr.25369) 46 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:35.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:35 smithi143 conmon[100258]: audit 2023-08-28T02:14:34.321833+0000 mon.a (mon.0) 864 : audit [DBG] from='client.? 172.21.15.143:0/4159089354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:35.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:35 smithi143 conmon[109923]: cluster 2023-08-28T02:14:33.780726+0000 mgr.x (mgr. 2023-08-28T02:14:35.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:35 smithi143 conmon[109923]: 25369) 46 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:35.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:35 smithi143 conmon[109923]: audit 2023-08-28T02:14:34.321833+0000 mon.a (mon.0) 864 : audit [DBG] from='client.? 172.21.15.143:0/4159089354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:35.737 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:14:35 smithi169 conmon[138400]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:14:35] "GET /metrics HTTP/1.1" 200 33945 "" "Prometheus/2.33.4" 2023-08-28T02:14:37.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:37 smithi169 conmon[100500]: cluster 2023-08-28T02:14:35.781016+0000 mgr.x (mgr.25369 2023-08-28T02:14:37.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:37 smithi169 conmon[100500]: ) 47 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:37.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:37 smithi169 conmon[100500]: audit 2023-08-28T02:14:36.700044+0000 mon.a (mon.0) 865 : audit [DBG] from='client.? 172.21.15.143:0/654891606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:37.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:37 smithi143 conmon[100258]: cluster 2023-08-28T02:14:35.781016+0000 mgr.x (mgr.25369) 47 : cluster 2023-08-28T02:14:37.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:37 smithi143 conmon[100258]: [DBG] pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:37.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:37 smithi143 conmon[100258]: audit 2023-08-28T02:14:36.700044+0000 mon.a (mon.0) 865 : audit [DBG] from='client.? 172.21.15.143:0/654891606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:37.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:37 smithi143 conmon[109923]: cluster 2023-08-28T02:14:35.781016+0000 mgr.x ( 2023-08-28T02:14:37.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:37 smithi143 conmon[109923]: mgr.25369) 47 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:37.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:37 smithi143 conmon[109923]: audit 2023-08-28T02:14:36.700044+0000 2023-08-28T02:14:37.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:37 smithi143 conmon[109923]: mon.a (mon.0) 865 : audit [DBG] from='client.? 172.21.15.143:0/654891606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:38.987 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:14:38 smithi169 conmon[144179]: ts=2023-08-28T02:14:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi169:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi169.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-08-28T02:14:39.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:39 smithi169 conmon[100500]: cluster 2023-08-28T02:14:37.781510+0000 mgr.x (mgr.25369) 2023-08-28T02:14:39.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:39 smithi169 conmon[100500]: 48 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:39.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:39 smithi169 conmon[100500]: audit 2023-08-28T02:14:39.084247+0000 mon.a (mon.0) 866 : audit [DBG] from='client.? 172.21.15.143:0/4140048748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:39.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:39 smithi143 conmon[100258]: cluster 2023-08-28T02:14:37.781510+0000 mgr.x (mgr. 2023-08-28T02:14:39.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:39 smithi143 conmon[100258]: 25369) 48 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:39.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:39 smithi143 conmon[100258]: audit 2023-08-28T02:14:39.084247+0000 mon.a ( 2023-08-28T02:14:39.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:39 smithi143 conmon[100258]: mon.0) 866 : audit [DBG] from='client.? 172.21.15.143:0/4140048748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:39.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:39 smithi143 conmon[109923]: cluster 2023-08-28T02:14:37.781510+0000 2023-08-28T02:14:39.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:39 smithi143 conmon[109923]: mgr.x (mgr.25369) 48 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:39.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:39 smithi143 conmon[109923]: audit 2023-08-28T02:14:39 2023-08-28T02:14:39.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:39 smithi143 conmon[109923]: .084247+0000 mon.a (mon.0) 866 : audit [DBG] from='client.? 172.21.15.143:0/4140048748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:41.479 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:41 smithi169 conmon[100500]: cluster 2023-08-28T02:14:39.781880+0000 mgr.x (mgr.25369) 49 : cluster 2023-08-28T02:14:41.479 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:41 smithi169 conmon[100500]: [DBG] pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:41.479 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:41 smithi169 conmon[100500]: audit 2023-08-28T02:14:40.980010+0000 mon.b (mon.1) 83 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:14:41.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:41 smithi143 conmon[100258]: cluster 2023-08-28T02:14:39.781880+0000 mgr.x (mgr.25369) 49 : cluster 2023-08-28T02:14:41.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:41 smithi143 conmon[100258]: [DBG] pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:41.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:41 smithi143 conmon[100258]: audit 2023-08-28T02:14:40.980010+0000 mon.b (mon.1) 83 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:14:41.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:41 smithi143 conmon[109923]: cluster 2023-08-28T02:14:39.781880+0000 2023-08-28T02:14:41.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:41 smithi143 conmon[109923]: mgr.x (mgr.25369) 49 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:41.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:41 smithi143 conmon[109923]: audit 2023-08-28T02:14:41.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:41 smithi143 conmon[109923]: 2023-08-28T02:14:40.980010+0000 mon.b (mon.1) 83 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:14:41.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:41 smithi143 conmon[109923]: 2023-08-28T02:14:42.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:42 smithi169 conmon[100500]: audit 2023-08-28T02:14:41.461357+0000 mon.c (mon.2) 52 : audit [DBG] from='client.? 172.21.15.143:0/3857963801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:42.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:42 smithi143 conmon[100258]: audit 2023-08-28T02:14:41.461357+0000 mon.c (mon.2) 52 : audit [DBG] from='client.? 172.21.15.143:0/3857963801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:42.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:42 smithi143 conmon[109923]: audit 2023-08-28T02:14:41.461357+0000 mon.c ( 2023-08-28T02:14:42.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:42 smithi143 conmon[109923]: mon.2) 52 : audit [DBG] from='client.? 172.21.15.143:0/3857963801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:43.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:43 smithi169 conmon[100500]: cluster 2023-08-28T02:14:41.782265+0000 mgr.x (mgr.25369) 50 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:43.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:43 smithi143 conmon[100258]: cluster 2023-08-28T02:14:41.782265+0000 mgr.x (mgr.25369) 50 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:43.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:43 smithi143 conmon[109923]: cluster 2023-08-28T02:14:41.782265+0000 mgr.x (mgr 2023-08-28T02:14:43.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:43 smithi143 conmon[109923]: .25369) 50 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:44.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:44 smithi169 conmon[100500]: audit 2023-08-28T02:14:43.838148+0000 mon.a (mon.0) 867 : audit [DBG] from='client.? 172.21.15.143:0/1223694182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:44.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:44 smithi143 conmon[100258]: audit 2023-08-28T02:14:43.838148+0000 mon.a (mon.0) 867 : audit [DBG] from='client.? 172.21.15.143:0/1223694182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:44.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:44 smithi143 conmon[100258]: 2023-08-28T02:14:44.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:44 smithi143 conmon[109923]: audit 2023-08-28T02:14:43.838148+0000 mon.a ( 2023-08-28T02:14:44.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:44 smithi143 conmon[109923]: mon.0) 867 : audit [DBG] from='client.? 172.21.15.143:0/1223694182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:45.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:45 smithi143 conmon[100258]: cluster 2023-08-28T02:14:43.782890+0000 mgr.x (mgr.25369) 51 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:45.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:45 smithi143 conmon[100258]: 2023-08-28T02:14:45.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:45 smithi143 conmon[109923]: cluster 2023-08-28T02:14:43.782890+0000 mgr.x (mgr.25369) 2023-08-28T02:14:45.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:45 smithi143 conmon[109923]: 51 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:45.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:45 smithi169 conmon[100500]: cluster 2023-08-28T02:14:43.782890+0000 mgr.x (mgr.25369) 51 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:45.737 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:14:45 smithi169 conmon[138400]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:14:45] "GET /metrics HTTP/1.1" 200 33947 "" "Prometheus/2.33.4" 2023-08-28T02:14:46.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:46 smithi143 conmon[100258]: audit 2023-08-28T02:14:46.214334+0000 mon.a (mon.0) 868 : audit [DBG] from='client.? 172.21.15.143:0/1785781604' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:46.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:46 smithi143 conmon[109923]: audit 2023-08-28T02:14:46.214334+0000 2023-08-28T02:14:46.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:46 smithi143 conmon[109923]: mon.a (mon.0) 868 : audit [DBG] from='client.? 172.21.15.143:0/1785781604' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:46.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:46 smithi169 conmon[100500]: audit 2023-08-28T02:14:46.214334+0000 mon.a (mon.0) 868 : audit [DBG] from='client.? 172.21.15.143:0/1785781604' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:47.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:47 smithi143 conmon[100258]: cluster 2023-08-28T02:14:45.783496+0000 mgr.x (mgr.25369) 52 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:47.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:47 smithi143 conmon[109923]: cluster 2023-08-28T02:14:45.783496+0000 mgr.x (mgr.25369) 52 2023-08-28T02:14:47.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:47 smithi143 conmon[109923]: : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:47.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:47 smithi169 conmon[100500]: cluster 2023-08-28T02:14:45.783496+0000 mgr.x (mgr.25369) 52 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:48.936 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:14:48 smithi169 systemd[1]: Stopping Ceph grafana.a for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:14:48.936 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:14:48 smithi169 conmon[144179]: ts=2023-08-28T02:14:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi169:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi169.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-08-28T02:14:49.237 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:14:48 smithi169 bash[145234]: Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-grafana.a found: no such container 2023-08-28T02:14:49.237 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:14:49 smithi169 conmon[131988]: t=2023-08-28T02:14:49+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2023-08-28T02:14:49.237 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:14:49 smithi169 conmon[131988]: t=2023-08-28T02:14:49+0000 lvl=info msg="Database locked, sleeping then retrying" logger=sqlstore error="database is locked" retry=0 2023-08-28T02:14:49.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:49 smithi143 conmon[100258]: cluster 2023-08-28T02:14:47.784063+0000 mgr.x ( 2023-08-28T02:14:49.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:49 smithi143 conmon[100258]: mgr.25369) 53 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:49.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:49 smithi143 conmon[100258]: audit 2023-08-28T02:14:48.601473+0000 mon.c (mon.2) 53 : audit [DBG] from='client.? 172.21.15.143:0/17364430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:49.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:49 smithi143 conmon[109923]: cluster 2023-08-28T02:14:47.784063+0000 mgr.x (mgr. 2023-08-28T02:14:49.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:49 smithi143 conmon[109923]: 25369) 53 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:49.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:49 smithi143 conmon[109923]: audit 2023-08-28T02:14:48.601473+0000 mon.c (mon.2) 53 : audit [DBG] from='client.? 172.21.15.143:0/17364430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:49.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:49 smithi169 conmon[100500]: cluster 2023-08-28T02:14:47.784063+0000 mgr.x (mgr.25369) 53 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:49.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:49 smithi169 conmon[100500]: audit 2023-08- 2023-08-28T02:14:49.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:49 smithi169 conmon[100500]: 28T02:14:48.601473+0000 mon.c (mon.2) 53 : audit [DBG] from='client.? 172.21.15.143:0/17364430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:50.237 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:14:49 smithi169 bash[145350]: dcf28c93da589384fd2d0cc878b0a1b35228fdba0b6d0cb53f4b090862f04312 2023-08-28T02:14:50.238 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:14:49 smithi169 systemd[1]: ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@grafana.a.service: Succeeded. 2023-08-28T02:14:50.238 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:14:49 smithi169 systemd[1]: Stopped Ceph grafana.a for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:14:50.238 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:14:49 smithi169 systemd[1]: Starting Ceph grafana.a for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:14:50.870 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:14:50 smithi169 conmon[138400]: [28/Aug/2023:02:14:50] ENGINE Bus STOPPING 2023-08-28T02:14:50.870 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:14:50 smithi169 conmon[138400]: [28/Aug/2023:02:14:50] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-08-28T02:14:50.870 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:14:50 smithi169 conmon[138400]: [28/Aug/2023:02:14:50] ENGINE Bus STOPPED 2023-08-28T02:14:50.870 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:14:50 smithi169 conmon[138400]: [28/Aug/2023:02:14:50] ENGINE Bus STARTING 2023-08-28T02:14:50.870 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:14:50 smithi169 conmon[138400]: [28/Aug/2023:02:14:50] ENGINE Serving on http://:::9283 2023-08-28T02:14:50.871 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:14:50 smithi169 conmon[138400]: [28/Aug/2023:02:14:50] ENGINE Bus STARTED 2023-08-28T02:14:50.871 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:14:50 smithi169 conmon[138400]: [28/Aug/2023:02:14:50] ENGINE Bus STOPPING 2023-08-28T02:14:50.871 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:14:50 smithi169 conmon[138400]: [28/Aug/2023:02:14:50] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-08-28T02:14:50.871 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:14:50 smithi169 conmon[138400]: [28/Aug/2023:02:14:50] ENGINE Bus STOPPED 2023-08-28T02:14:50.871 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:14:50 smithi169 conmon[138400]: [28/Aug/2023:02:14:50] ENGINE Bus STARTING 2023-08-28T02:14:50.872 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:14:50 smithi169 conmon[145727]: t=2023-08-28T02:14:50+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2023-08-28T02:14:50.872 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:14:50 smithi169 conmon[145727]: t=2023-08-28T02:14:50+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2023-08-28T02:14:50.872 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:14:50 smithi169 conmon[145727]: t=2023-08-28T02:14:50+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2023-08-28T02:14:50.872 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:14:50 smithi169 conmon[145727]: t=2023-08-28T02:14:50+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2023-08-28T02:14:50.872 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:14:50 smithi169 conmon[145727]: t=2023-08-28T02:14:50+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2023-08-28T02:14:50.872 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:14:50 smithi169 conmon[145727]: t=2023-08-28T02:14:50+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2023-08-28T02:14:50.872 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:14:50 smithi169 conmon[145727]: t=2023-08-28T02:14:50+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-08-28T02:14:50.872 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:14:50 smithi169 conmon[145727]: t=2023-08-28T02:14:50+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-08-28T02:14:50.872 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:14:50 smithi169 conmon[145727]: t=2023-08-28T02:14:50+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2023-08-28T02:14:50.872 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:14:50 smithi169 conmon[145727]: t=2023-08-28T02:14:50+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2023-08-28T02:14:50.872 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:14:50 smithi169 conmon[145727]: t=2023-08-28T02:14:50+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2023-08-28T02:14:50.872 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:14:50 smithi169 conmon[145727]: t=2023-08-28T02:14:50+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2023-08-28T02:14:50.873 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:14:50 smithi169 conmon[145727]: t=2023-08-28T02:14:50+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2023-08-28T02:14:50.873 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:14:50 smithi169 conmon[145727]: t=2023-08-28T02:14:50+0000 lvl=info msg="App mode production" logger=settings 2023-08-28T02:14:50.873 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:14:50 smithi169 conmon[145727]: t=2023-08-28T02:14:50+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2023-08-28T02:14:50.873 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:14:50 smithi169 conmon[145727]: t=2023-08-28T02:14:50+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-08-28T02:14:50.873 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:14:50 smithi169 bash[145686]: 4c9eb6ad00e1e14a2e2a818785bbabe838aad85ccd2edcfe65d3b60ae03657f8 2023-08-28T02:14:50.873 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:14:50 smithi169 systemd[1]: Started Ceph grafana.a for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:14:50.873 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:14:50 smithi169 conmon[145727]: t=2023-08-28T02:14:50+0000 lvl=info msg="Starting DB migrations" logger=migrator 2023-08-28T02:14:50.873 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:14:50 smithi169 conmon[145727]: t=2023-08-28T02:14:50+0000 lvl=info msg="migrations completed" logger=migrator performed=0 skipped=377 duration=1.147958ms 2023-08-28T02:14:50.873 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:14:50 smithi169 conmon[145727]: t=2023-08-28T02:14:50+0000 lvl=info msg="Created default organization" logger=sqlstore 2023-08-28T02:14:50.873 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:14:50 smithi169 conmon[145727]: t=2023-08-28T02:14:50+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2023-08-28T02:14:50.873 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:14:50 smithi169 conmon[145727]: t=2023-08-28T02:14:50+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2023-08-28T02:14:50.873 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:14:50 smithi169 conmon[145727]: t=2023-08-28T02:14:50+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2023-08-28T02:14:50.874 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:14:50 smithi169 conmon[145727]: t=2023-08-28T02:14:50+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2023-08-28T02:14:50.874 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:14:50 smithi169 conmon[145727]: t=2023-08-28T02:14:50+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2023-08-28T02:14:50.874 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:14:50 smithi169 conmon[145727]: t=2023-08-28T02:14:50+0000 lvl=info msg="deleted datasource based on configuration" logger=provisioning.datasources name=Dashboard1 2023-08-28T02:14:50.874 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:14:50 smithi169 conmon[145727]: t=2023-08-28T02:14:50+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2023-08-28T02:14:50.874 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:14:50 smithi169 conmon[145727]: t=2023-08-28T02:14:50+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Loki uid=P8E80F9AEF21F6940 2023-08-28T02:14:50.874 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:14:50 smithi169 conmon[145727]: t=2023-08-28T02:14:50+0000 lvl=info msg="warming cache for startup" logger=ngalert 2023-08-28T02:14:51.237 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:14:50 smithi169 conmon[145727]: t=2023-08-28T02:14:50+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2023-08-28T02:14:51.237 INFO:journalctl@ceph.grafana.a.smithi169.stdout:Aug 28 02:14:50 smithi169 conmon[145727]: t=2023-08-28T02:14:50+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2023-08-28T02:14:51.237 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:14:50 smithi169 conmon[138400]: [28/Aug/2023:02:14:50] ENGINE Serving on http://:::9283 2023-08-28T02:14:51.238 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:14:50 smithi169 conmon[138400]: [28/Aug/2023:02:14:50] ENGINE Bus STARTED 2023-08-28T02:14:51.238 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:14:50 smithi169 conmon[138400]: [28/Aug/2023:02:14:50] ENGINE Bus STOPPING 2023-08-28T02:14:51.238 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:14:50 smithi169 conmon[138400]: [28/Aug/2023:02:14:50] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-08-28T02:14:51.238 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:14:50 smithi169 conmon[138400]: [28/Aug/2023:02:14:50] ENGINE Bus STOPPED 2023-08-28T02:14:51.238 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:14:50 smithi169 conmon[138400]: [28/Aug/2023:02:14:50] ENGINE Bus STARTING 2023-08-28T02:14:51.238 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:14:51 smithi169 conmon[138400]: [28/Aug/2023:02:14:51] ENGINE Serving on http://:::9283 2023-08-28T02:14:51.238 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:14:51 smithi169 conmon[138400]: [28/Aug/2023:02:14:51] ENGINE Bus STARTED 2023-08-28T02:14:51.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[109923]: cluster 2023-08-28T02:14:49.784381+0000 2023-08-28T02:14:51.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[109923]: mgr.x (mgr.25369) 54 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:51.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[109923]: audit 2023-08-28T02:14:50.632056+0000 mon.a (mon.0) 869 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:51.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[109923]: audit 2023-08-28T02:14:50.639114+0000 mon.a (mon.0) 870 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:51.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[109923]: audit 2023-08-28T02:14:50.666799+0000 mon.b (mon.1) 84 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-08-28T02:14:51.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[109923]: audit 2023-08-28T02:14:50.667228+0000 mgr.x (mgr.25369) 55 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-08-28T02:14:51.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[109923]: audit 2023-08-28T02:14:50.668127+0000 mon.b (mon.1) 85 : audit [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi143.front.sepia.ceph.com:9093"}]: dispatch 2023-08-28T02:14:51.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[109923]: audit 2023-08-28T02:14 2023-08-28T02:14:51.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[109923]: :50.668402+0000 mgr.x (mgr.25369) 56 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi143.front.sepia.ceph.com:9093"}]: dispatch 2023-08-28T02:14:51.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[109923]: audit 2023-08-28T02:14:50.674474+0000 mon.a (mon.0) 871 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:51.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[109923]: audit 2023-08-28T02:14:50.681504+0000 mon.b (mon.1) 86 : 2023-08-28T02:14:51.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[109923]: audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-08-28T02:14:51.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[109923]: audit 2023-08-28T02:14:51.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[109923]: 2023-08-28T02:14:50.682112+0000 mgr.x (mgr.25369) 57 : audit [DBG] 2023-08-28T02:14:51.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[109923]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-08-28T02:14:51.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[109923]: audit 2023-08 2023-08-28T02:14:51.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[109923]: -28T02:14:50.685602+0000 mon.b (mon.1) 87 : 2023-08-28T02:14:51.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[109923]: audit [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi169:9095"}]: dispatch 2023-08-28T02:14:51.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[109923]: audit 2023-08-28T02:14:50.685957+0000 mgr.x 2023-08-28T02:14:51.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[109923]: (mgr.25369) 58 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi169:9095"}]: dispatch 2023-08-28T02:14:51.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[109923]: audit 2023-08-28T02:14:50.692093+0000 2023-08-28T02:14:51.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[109923]: mon.a (mon.0) 872 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:51.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[109923]: audit 2023-08-28T02:14 2023-08-28T02:14:51.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[109923]: :50.699303+0000 mon.b (mon.1) 88 : 2023-08-28T02:14:51.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[109923]: audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-08-28T02:14:51.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[109923]: audit 2023-08-28T02:14:50.699685+0000 2023-08-28T02:14:51.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[109923]: mgr.x (mgr.25369) 59 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-08-28T02:14:51.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[109923]: audit 2023-08-28T02: 2023-08-28T02:14:51.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[109923]: 14:50.703989+0000 mon.b (mon.1) 89 : audit 2023-08-28T02:14:51.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[109923]: [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi169:3000"}]: dispatch 2023-08-28T02:14:51.953 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[100258]: cluster 2023-08-28T02:14:49.784381+0000 mgr.x (mgr.25369) 54 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:51.953 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[100258]: audit 2023-08-28T02:14:50.632056+0000 mon.a (mon.0) 869 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:51.953 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[100258]: audit 2023-08-28T02:14:50.639114+0000 mon.a (mon.0) 870 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:51.953 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[100258]: audit 2023-08-28T02:14:50.666799+0000 mon.b (mon.1) 84 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-08-28T02:14:51.953 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[100258]: audit 2023-08-28T02:14:50.667228+0000 mgr.x (mgr.25369) 55 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-08-28T02:14:51.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[100258]: audit 2023-08-28T02:14:50.668127+0000 mon.b (mon.1) 85 : audit [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi143.front.sepia.ceph.com:9093"}]: dispatch 2023-08-28T02:14:51.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[100258]: audit 2023-08-28T02:14:50.668402+0000 mgr.x (mgr.25369) 56 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi143.front.sepia.ceph.com:9093"}]: dispatch 2023-08-28T02:14:51.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[100258]: audit 2023-08-28T02:14:50.674474+0000 mon.a (mon.0) 871 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:51.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[100258]: audit 2023-08-28T02:14:50.681504+0000 mon.b (mon.1) 86 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-08-28T02:14:51.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[100258]: audit 2023-08-28T02:14:50.682112+0000 mgr.x (mgr.25369) 57 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-08-28T02:14:51.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[100258]: audit 2023-08-28T02:14:50.685602+0000 mon.b (mon.1) 87 : audit [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi169:9095"}]: dispatch 2023-08-28T02:14:51.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[100258]: audit 2023-08-28T02:14:50.685957+0000 mgr.x (mgr.25369) 58 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi169:9095"}]: dispatch 2023-08-28T02:14:51.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[100258]: audit 2023-08-28T02:14:50.692093+0000 mon.a (mon.0) 872 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:51.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[100258]: audit 2023-08-28T02:14:50.699303+0000 mon.b (mon.1) 88 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-08-28T02:14:51.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[100258]: audit 2023-08-28T02:14:50.699685+0000 mgr.x (mgr.25369) 59 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-08-28T02:14:51.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[100258]: audit 2023-08-28T02:14:50.703989+0000 mon.b (mon.1) 89 : audit [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi169:3000"}]: dispatch 2023-08-28T02:14:51.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[100258]: audit 2023-08-28T02:14:50.704319+0000 mgr.x (mgr.25369) 60 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi169:3000"}]: dispatch 2023-08-28T02:14:51.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[100258]: audit 2023-08-28T02:14:50.710625+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:51.955 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[100258]: audit 2023-08-28T02:14:50.722432+0000 mon.b (mon.1) 90 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:14:51.955 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[100258]: cephadm 2023-08-28T02:14:50.724681+0000 mgr.x (mgr.25369) 61 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-08-28T02:14:51.955 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[100258]: cephadm 2023-08-28T02:14:50.725049+0000 mgr.x (mgr.25369) 62 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-08-28T02:14:51.955 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[100258]: audit 2023-08-28T02:14:51.010286+0000 mon.c (mon.2) 54 : audit [DBG] from='client.? 172.21.15.143:0/1803548656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:51.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:14:51.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[109923]: 08-28T02:14:50.704319+0000 mgr.x (mgr.25369) 60 : audit [DBG] 2023-08-28T02:14:51.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[109923]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi169:3000"}]: dispatch 2023-08-28T02:14:51.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[109923]: audit 2023-08 2023-08-28T02:14:51.956 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[109923]: -28T02:14:50.710625+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:51.956 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[109923]: audit 2023-08-28T02 2023-08-28T02:14:51.956 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[109923]: :14:50.722432+0000 mon.b (mon.1) 90 : audit 2023-08-28T02:14:51.956 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[109923]: [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:14:51.956 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[109923]: cephadm 2023-08-28T02:14:51.956 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[109923]: 2023-08-28T02:14:50.724681+0000 mgr.x (mgr.25369 2023-08-28T02:14:51.956 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[109923]: ) 61 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-08-28T02:14:51.956 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[109923]: cephadm 2023-08-28T02:14:50. 2023-08-28T02:14:51.956 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[109923]: 725049+0000 mgr.x (mgr.25369) 62 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-08-28T02:14:51.956 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[109923]: audit 2023-08-28T02:14: 2023-08-28T02:14:51.956 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:51 smithi143 conmon[109923]: 51.010286+0000 mon.c (mon.2) 54 : audit [DBG] from='client.? 172.21.15.143:0/1803548656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:51.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:51 smithi169 conmon[100500]: cluster 2023-08-28T02:14:49.784381+0000 mgr.x (mgr.25369) 54 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:51.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:51 smithi169 conmon[100500]: audit 2023-08-28T02:14:50.632056+0000 mon.a (mon.0) 869 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:51.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:51 smithi169 conmon[100500]: audit 2023-08-28T02:14:50.639114+0000 mon.a (mon.0) 870 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:51.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:51 smithi169 conmon[100500]: audit 2023-08-28T02:14:50.666799+0000 mon.b (mon.1) 84 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-08-28T02:14:51.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:51 smithi169 conmon[100500]: audit 2023-08-28T02:14:50 2023-08-28T02:14:51.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:51 smithi169 conmon[100500]: .667228+0000 mgr.x (mgr.25369) 55 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-08-28T02:14:51.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:51 smithi169 conmon[100500]: audit 2023-08-28T02:14:50.668127+0000 mon.b (mon.1) 85 : audit 2023-08-28T02:14:51.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:51 smithi169 conmon[100500]: [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi143.front.sepia.ceph.com:9093"}]: dispatch 2023-08-28T02:14:51.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:51 smithi169 conmon[100500]: audit 2023-08-28 2023-08-28T02:14:51.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:51 smithi169 conmon[100500]: T02:14:50.668402+0000 mgr.x (mgr.25369) 56 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi143.front.sepia.ceph.com:9093"}]: dispatch 2023-08-28T02:14:51.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:51 smithi169 conmon[100500]: audit 2023-08-28T02:14:50.674474+0000 mon.a 2023-08-28T02:14:51.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:51 smithi169 conmon[100500]: (mon.0) 871 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:51.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:51 smithi169 conmon[100500]: audit 2023 2023-08-28T02:14:51.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:51 smithi169 conmon[100500]: -08-28T02:14:50.681504+0000 mon.b (mon.1) 86 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-08-28T02:14:51.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:51 smithi169 conmon[100500]: audit 2023-08-28T02:14:50.682112+0000 mgr.x (mgr.25369 2023-08-28T02:14:51.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:51 smithi169 conmon[100500]: ) 57 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-08-28T02:14:51.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:51 smithi169 conmon[100500]: audit 2023-08-28T02:14:51.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:51 smithi169 conmon[100500]: 2023-08-28T02:14:50.685602+0000 mon.b (mon.1) 87 : audit [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi169:9095"}]: dispatch 2023-08-28T02:14:51.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:51 smithi169 conmon[100500]: audit 2023-08-28T02:14:50 2023-08-28T02:14:51.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:51 smithi169 conmon[100500]: .685957+0000 mgr.x (mgr.25369) 58 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi169:9095"}]: dispatch 2023-08-28T02:14:51.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:51 smithi169 conmon[100500]: audit 2023-08-28T02:14:50.692093 2023-08-28T02:14:51.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:51 smithi169 conmon[100500]: +0000 mon.a (mon.0) 872 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:51.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:51 smithi169 conmon[100500]: audit 2023-08-28T02:14:50.699303+0000 mon.b (mon 2023-08-28T02:14:51.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:51 smithi169 conmon[100500]: .1) 88 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-08-28T02:14:51.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:51 smithi169 conmon[100500]: audit 2023-08-28T02:14:51.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:51 smithi169 conmon[100500]: 2023-08-28T02:14:50.699685+0000 mgr.x (mgr.25369) 59 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-08-28T02:14:51.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:51 smithi169 conmon[100500]: audit 2023-08-28T02:14:50 2023-08-28T02:14:51.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:51 smithi169 conmon[100500]: .703989+0000 mon.b (mon.1) 89 : audit [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi169:3000"}]: dispatch 2023-08-28T02:14:51.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:51 smithi169 conmon[100500]: audit 2023-08-28T02:14:50.704319 2023-08-28T02:14:51.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:51 smithi169 conmon[100500]: +0000 mgr.x (mgr.25369) 60 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi169:3000"}]: dispatch 2023-08-28T02:14:51.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:51 smithi169 conmon[100500]: audit 2023-08-28T02: 2023-08-28T02:14:51.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:51 smithi169 conmon[100500]: 14:50.710625+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:51.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:51 smithi169 conmon[100500]: audit 2023-08-28T02:14: 2023-08-28T02:14:51.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:51 smithi169 conmon[100500]: 50.722432+0000 mon.b (mon.1) 90 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:14:51.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:51 smithi169 conmon[100500]: cephadm 2023-08-28T02:14:50.724681+0000 mgr.x (mgr.25369 2023-08-28T02:14:51.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:51 smithi169 conmon[100500]: ) 61 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-08-28T02:14:51.991 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:51 smithi169 conmon[100500]: cephadm 2023-08-28 2023-08-28T02:14:51.991 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:51 smithi169 conmon[100500]: T02:14:50.725049+0000 mgr.x (mgr.25369) 62 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-08-28T02:14:51.991 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:51 smithi169 conmon[100500]: audit 2023-08-28T02:14:51.010286+0000 mon.c (mon.2 2023-08-28T02:14:51.991 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:51 smithi169 conmon[100500]: ) 54 : audit [DBG] from='client.? 172.21.15.143:0/1803548656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:53.395 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:53 smithi143 conmon[100258]: cluster 2023-08-28T02:14:51.784875+0000 mgr.x (mgr.25369) 63 : cluster 2023-08-28T02:14:53.395 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:53 smithi143 conmon[100258]: [DBG] pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:53.395 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:53 smithi143 conmon[100258]: cephadm 2023-08-28T02:14:52.192423+0000 mgr.x (mgr.25369) 64 : cephadm [INF] Upgrade: Updating mgr.y 2023-08-28T02:14:53.396 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:53 smithi143 conmon[100258]: audit 2023-08-28T02:14:52.197133+0000 mon.a (mon.0) 874 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:53.396 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:53 smithi143 conmon[100258]: audit 2023-08-28T02:14:52.199633+0000 mon.b (mon.1) 91 : audit [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-28T02:14:53.396 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:53 smithi143 conmon[100258]: audit 2023-08-28T02:14:52.200052+0000 mon.a (mon.0) 875 : audit [INF] from='mgr.25369 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-28T02:14:53.396 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:53 smithi143 conmon[100258]: audit 2023-08-28T02:14:52.201219+0000 mon.b (mon.1) 92 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-08-28T02:14:53.396 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:53 smithi143 conmon[100258]: audit 2023-08-28T02:14:52.202139+0000 mon.b (mon.1) 93 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:14:53.396 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:53 smithi143 conmon[100258]: cephadm 2023-08-28T02:14:52.203083+0000 mgr.x (mgr.25369) 65 : cephadm [INF] Deploying daemon mgr.y on smithi143 2023-08-28T02:14:53.396 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:53 smithi143 conmon[109923]: cluster 2023-08-28T02:14:51.784875+0000 mgr.x (mgr.25369) 63 2023-08-28T02:14:53.396 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:53 smithi143 conmon[109923]: : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:53.397 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:53 smithi143 conmon[109923]: cephadm 2023-08-28T02:14:52.192423+0000 mgr.x (mgr.25369) 64 : cephadm [INF] Upgrade: Updating mgr.y 2023-08-28T02:14:53.397 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:53 smithi143 conmon[109923]: audit 2023-08-28T02:14:52.197133+0000 mon.a (mon.0) 874 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:53.397 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:53 smithi143 conmon[109923]: audit 2023-08-28T 2023-08-28T02:14:53.397 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:53 smithi143 conmon[109923]: 02:14:52.199633+0000 mon.b (mon.1) 2023-08-28T02:14:53.397 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:53 smithi143 conmon[109923]: 91 : audit [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-28T02:14:53.397 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:53 smithi143 conmon[109923]: audit 2023-08- 2023-08-28T02:14:53.398 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:53 smithi143 conmon[109923]: 28T02:14:52.200052+0000 mon.a (mon.0) 2023-08-28T02:14:53.398 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:53 smithi143 conmon[109923]: 875 : audit [INF] from='mgr.25369 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-28T02:14:53.398 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:53 smithi143 conmon[109923]: audit 2023-08-28T02:14:53.398 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:53 smithi143 conmon[109923]: 2023-08-28T02:14:52.201219+0000 mon.b (mon.1) 92 : audit 2023-08-28T02:14:53.398 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:53 smithi143 conmon[109923]: [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-08-28T02:14:53.398 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:53 smithi143 conmon[109923]: audit 2023 2023-08-28T02:14:53.398 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:53 smithi143 conmon[109923]: -08-28T02:14:52.202139+0000 mon.b (mon.1) 93 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:14:53.399 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:53 smithi143 conmon[109923]: cephadm 2023-08-28 2023-08-28T02:14:53.399 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:53 smithi143 conmon[109923]: T02:14:52.203083+0000 mgr.x (mgr.25369) 65 : 2023-08-28T02:14:53.399 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:53 smithi143 conmon[109923]: cephadm [INF] Deploying daemon mgr.y on smithi143 2023-08-28T02:14:53.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:53 smithi169 conmon[100500]: cluster 2023-08-28T02:14:51.784875+0000 mgr.x (mgr.25369) 63 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:53.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:53 smithi169 conmon[100500]: cephadm 2023-08-28T02:14:52.192423+0000 mgr.x (mgr.25369) 64 : cephadm [INF] Upgrade: Updating mgr.y 2023-08-28T02:14:53.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:53 smithi169 conmon[100500]: audit 2023-08-28T02:14:52.197133+0000 mon.a (mon.0) 874 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:53.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:53 smithi169 conmon[100500]: audit 2023-08-28T02:14:52.199633+0000 mon.b (mon.1) 91 : audit [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-28T02:14:53.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:53 smithi169 conmon[100500]: audit 2023-08-28T02:14:52.200052+0000 mon.a 2023-08-28T02:14:53.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:53 smithi169 conmon[100500]: (mon.0) 875 : audit [INF] from='mgr.25369 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-28T02:14:53.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:53 smithi169 conmon[100500]: audit 2023-08-28T02:14:52.201219+0000 mon.b (mon.1) 92 2023-08-28T02:14:53.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:53 smithi169 conmon[100500]: : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-08-28T02:14:53.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:53 smithi169 conmon[100500]: audit 2023-08-28T02:14:53.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:53 smithi169 conmon[100500]: 2023-08-28T02:14:52.202139+0000 mon.b (mon.1) 93 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:14:53.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:53 smithi169 conmon[100500]: cephadm 2023-08-28T02:14:52.203083+0000 mgr.x (mgr. 2023-08-28T02:14:53.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:53 smithi169 conmon[100500]: 25369) 65 : cephadm [INF] Deploying daemon mgr.y on smithi143 2023-08-28T02:14:53.700 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:14:53 smithi143 systemd[1]: Stopping Ceph mgr.y for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:14:54.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:54 smithi143 conmon[100258]: audit 2023-08-28T02:14:53.389206+0000 mon.c (mon.2) 2023-08-28T02:14:54.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:54 smithi143 conmon[100258]: 55 : audit [DBG] from='client.? 172.21.15.143:0/3678768556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:54.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:54 smithi143 conmon[109923]: audit 2023-08-28T02:14 2023-08-28T02:14:54.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:54 smithi143 conmon[109923]: :53.389206+0000 mon.c (mon.2) 55 : audit [DBG] from='client.? 172.21.15.143:0/3678768556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:54.451 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:14:54 smithi143 systemd[1]: ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@mgr.y.service: Main process exited, code=exited, status=143/n/a 2023-08-28T02:14:54.451 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:14:54 smithi143 bash[175658]: 48d3608092c2cdae1485f3415b4bdef79931b86d7d292111cb235bc15fd048c8 2023-08-28T02:14:54.451 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:14:54 smithi143 systemd[1]: ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@mgr.y.service: Failed with result 'exit-code'. 2023-08-28T02:14:54.451 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:14:54 smithi143 systemd[1]: Stopped Ceph mgr.y for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:14:54.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:54 smithi169 conmon[100500]: audit 2023-08-28T02:14:53.389206+0000 mon.c (mon.2) 55 : audit [DBG] from='client.? 172.21.15.143:0/3678768556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:54.951 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:14:54 smithi143 systemd[1]: Starting Ceph mgr.y for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:14:55.283 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:55 smithi143 conmon[100258]: cluster 2023-08-28T02:14:53.785319+0000 mgr.x (mgr 2023-08-28T02:14:55.284 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:55 smithi143 conmon[100258]: .25369) 66 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:55.284 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:55 smithi143 conmon[109923]: cluster 2023-08-28T02:14:53. 2023-08-28T02:14:55.284 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:55 smithi143 conmon[109923]: 785319+0000 mgr.x (mgr.25369) 66 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:55.284 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:14:55 smithi143 bash[176012]: 8de2036a51e32dd5799079e3b6a456773f827f24a85249d51c5748deb0a3a6e4 2023-08-28T02:14:55.285 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:14:55 smithi143 ceph-mgr[176077]: -- 172.21.15.143:0/2844174691 <== mon.2 v2:172.21.15.143:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5591a79b31e0 con 0x5591a798b000 2023-08-28T02:14:55.285 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:14:55 smithi143 systemd[1]: Started Ceph mgr.y for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:14:55.285 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:14:55 smithi143 conmon[176053]: 2023-08-28T02:14:55.283+0000 7f834f565200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-08-28T02:14:55.481 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:55 smithi169 conmon[100500]: cluster 2023-08-28T02:14:53.785319+0000 mgr.x (mgr.25369) 66 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:55.583 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:14:55 smithi143 conmon[176053]: 2023-08-28T02:14:55.367+0000 7f834f565200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-08-28T02:14:55.583 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:14:55 smithi143 conmon[176053]: 2023-08-28T02:14:55.465+0000 7f834f565200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-08-28T02:14:55.584 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:14:55 smithi143 conmon[176053]: 2023-08-28T02:14:55.583+0000 7f834f565200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-08-28T02:14:55.737 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:14:55 smithi169 conmon[138400]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:14:55] "GET /metrics HTTP/1.1" 200 33947 "" "Prometheus/2.33.4" 2023-08-28T02:14:55.950 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:14:55 smithi143 conmon[176053]: 2023-08-28T02:14:55.787+0000 7f834f565200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-08-28T02:14:56.450 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:14:56 smithi143 conmon[176053]: 2023-08-28T02:14:56.187+0000 7f834f565200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-08-28T02:14:56.451 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:14:56 smithi143 conmon[176053]: 2023-08-28T02:14:56.334+0000 7f834f565200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-08-28T02:14:56.451 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:14:56 smithi143 conmon[176053]: 2023-08-28T02:14:56.428+0000 7f834f565200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-08-28T02:14:56.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:56 smithi143 conmon[109923]: audit 2023-08-28T02:14:55.213095+0000 mon.a ( 2023-08-28T02:14:56.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:56 smithi143 conmon[109923]: mon.0) 876 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:56.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:56 smithi143 conmon[109923]: audit 2023-08-28T02:14:55. 2023-08-28T02:14:56.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:56 smithi143 conmon[109923]: 220576+0000 mon.a (mon.0) 877 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:56.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:56 smithi143 conmon[109923]: audit 2023-08- 2023-08-28T02:14:56.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:56 smithi143 conmon[109923]: 28T02:14:55.223961+0000 mon.b ( 2023-08-28T02:14:56.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:56 smithi143 conmon[109923]: mon.1) 94 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:14:56.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:56 smithi143 conmon[109923]: audit 2023-08-28T02:14:55.830351 2023-08-28T02:14:56.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:56 smithi143 conmon[109923]: +0000 mon.a (mon.0) 878 : audit [DBG] from='client.? 172.21.15.143:0/1430422329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:56.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:56 smithi143 conmon[109923]: 2023-08-28T02:14:56.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:56 smithi143 conmon[109923]: audit 2023-08-28T02:14:56.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:56 smithi143 conmon[109923]: 2023-08-28T02:14:55.983549+0000 mon.b (mon.1) 2023-08-28T02:14:56.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:56 smithi143 conmon[109923]: 95 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:14:56.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:56 smithi143 conmon[100258]: audit 2023-08-28T02:14:55. 2023-08-28T02:14:56.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:56 smithi143 conmon[100258]: 213095+0000 mon.a (mon.0) 876 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:56.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:56 smithi143 conmon[100258]: audit 2023-08-28T02:14:55.220576+0000 mon.a (mon.0) 877 2023-08-28T02:14:56.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:56 smithi143 conmon[100258]: : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:56.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:56 smithi143 conmon[100258]: audit 2023 2023-08-28T02:14:56.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:56 smithi143 conmon[100258]: -08-28T02:14:55.223961+0000 mon.b (mon.1 2023-08-28T02:14:56.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:56 smithi143 conmon[100258]: ) 94 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:14:56.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:56 smithi143 conmon[100258]: audit 2023-08-28T02:14:56.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:56 smithi143 conmon[100258]: 2023-08 2023-08-28T02:14:56.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:56 smithi143 conmon[100258]: -28T02:14:55.830351+0000 2023-08-28T02:14:56.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:56 smithi143 conmon[100258]: mon.a (mon.0) 878 : audit [DBG] from='client.? 172.21.15.143:0/1430422329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:56.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:56 smithi143 conmon[100258]: 2023-08-28T02:14:56.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:56 smithi143 conmon[100258]: audit 2023- 2023-08-28T02:14:56.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:56 smithi143 conmon[100258]: 08-28T02:14:55.983549 2023-08-28T02:14:56.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:56 smithi143 conmon[100258]: +0000 mon.b (mon.1) 95 : audit 2023-08-28T02:14:56.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:56 smithi143 conmon[100258]: [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:14:56.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:56 smithi169 conmon[100500]: audit 2023-08-28T02:14:55. 2023-08-28T02:14:56.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:56 smithi169 conmon[100500]: 213095+0000 mon.a (mon.0) 876 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:56.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:56 smithi169 conmon[100500]: audit 2023-08-28T02:14:55.220576+0000 mon.a (mon.0) 877 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:56.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:56 smithi169 conmon[100500]: audit 2023-08-28T02:14:55.223961+0000 mon.b (mon.1) 94 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:14:56.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:56 smithi169 conmon[100500]: audit 2023-08-28T02:14:55.830351+0000 mon.a (mon.0) 878 : audit [DBG] from='client.? 172.21.15.143:0/1430422329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:56.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:56 smithi169 conmon[100500]: audit 2023-08-28T02:14:55.983549+0000 mon.b (mon.1) 95 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:14:56.951 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:14:56 smithi143 conmon[176053]: 2023-08-28T02:14:56.528+0000 7f834f565200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-08-28T02:14:57.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:57 smithi143 conmon[100258]: cluster 2023-08-28T02:14:55.785800+0000 mgr.x (mgr.25369) 67 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:57.450 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:14:57 smithi143 conmon[176053]: 2023-08-28T02:14:57.019+0000 7f834f565200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-08-28T02:14:57.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:57 smithi143 conmon[109923]: cluster 2023-08-28T02:14:55.785800+0000 mgr.x (mgr.25369) 67 2023-08-28T02:14:57.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:57 smithi143 conmon[109923]: : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:57.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:57 smithi169 conmon[100500]: cluster 2023-08-28T02:14:55.785800+0000 mgr.x (mgr.25369) 67 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:58.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:14:57 smithi143 conmon[176053]: 2023-08-28T02:14:57.729+0000 7f834f565200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-08-28T02:14:58.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:14:57 smithi143 conmon[176053]: 2023-08-28T02:14:57.815+0000 7f834f565200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-08-28T02:14:58.950 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:14:58 smithi143 conmon[176053]: 2023-08-28T02:14:58.676+0000 7f834f565200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-08-28T02:14:58.951 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:14:58 smithi143 conmon[176053]: 2023-08-28T02:14:58.783+0000 7f834f565200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-08-28T02:14:58.987 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:14:58 smithi169 conmon[144179]: ts=2023-08-28T02:14:58.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi169:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi169.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-08-28T02:14:59.249 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[100258]: cluster 2023-08-28T02:14:57.786416+0000 mgr.x 2023-08-28T02:14:59.250 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[100258]: (mgr.25369) 68 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:59.250 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[100258]: audit 2023-08-28T02:14:58.001156+0000 mon.a (mon.0) 879 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:59.250 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[100258]: audit 2023-08-28T02:14:58.007256+0000 mon.a (mon.0) 880 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:59.250 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[100258]: audit 2023-08-28T02:14 2023-08-28T02:14:59.250 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[100258]: :58.245208+0000 mon.c (mon.2) 56 : audit [DBG] 2023-08-28T02:14:59.250 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[100258]: from='client.? 172.21.15.143:0/2204006529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:59.250 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[100258]: audit 2023-08-28T02:14:59.250 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[100258]: 2023-08-28T02:14:58.353386+0000 mon.a (mon.0) 881 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:59.250 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[100258]: audit 2023-08-28T02:14:58. 2023-08-28T02:14:59.251 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[100258]: 366675+0000 mon.a (mon.0) 882 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:59.251 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[100258]: audit 2023-08-28T02: 2023-08-28T02:14:59.251 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[100258]: 14:58.700670+0000 mon.a (mon.0) 883 : audit 2023-08-28T02:14:59.251 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[100258]: [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:59.251 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[100258]: audit 2023-08-28T02:14:58.708288+0000 mon.a (mon 2023-08-28T02:14:59.251 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[100258]: .0) 884 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:59.251 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[109923]: cluster 2023-08-28T02:14:57.786416+0000 2023-08-28T02:14:59.251 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[109923]: mgr.x (mgr.25369) 68 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:59.251 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[109923]: audit 2023-08-28T02:14:59.252 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[109923]: 2023-08-28T 2023-08-28T02:14:59.252 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[109923]: 02:14:58.001156+0000 2023-08-28T02:14:59.252 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[109923]: mon.a (mon.0) 879 : audit 2023-08-28T02:14:59.252 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[109923]: [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:59.252 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[109923]: audit 2023-08-28T02:14:59.252 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[109923]: 2023-08-28T02 2023-08-28T02:14:59.252 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[109923]: :14:58.007256+0000 mon.a (mon.0) 880 : 2023-08-28T02:14:59.252 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[109923]: audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:59.252 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[109923]: audit 2023-08-28T02:14:59.252 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[109923]: 2023-08-28T02:14:58 2023-08-28T02:14:59.252 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[109923]: .245208+0000 mon.c (mon.2 2023-08-28T02:14:59.253 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[109923]: ) 56 : audit [DBG] from='client.? 172.21.15.143:0/2204006529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:59.253 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[109923]: audit 2023 2023-08-28T02:14:59.253 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[109923]: -08-28T02:14:58 2023-08-28T02:14:59.253 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[109923]: .353386+0000 mon.a (mon.0 2023-08-28T02:14:59.253 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[109923]: ) 881 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:59.253 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[109923]: audit 2023 2023-08-28T02:14:59.253 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[109923]: -08-28T02:14:58 2023-08-28T02:14:59.253 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[109923]: .366675+0000 mon.a (mon.0 2023-08-28T02:14:59.253 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[109923]: ) 882 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:59.253 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[109923]: audit 2023-08-28T02:14:59.253 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[109923]: 2023-08-28T02:14:58 2023-08-28T02:14:59.253 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[109923]: .700670+0000 mon.a (mon. 2023-08-28T02:14:59.254 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[109923]: 0) 883 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:59.254 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[109923]: 2023-08-28T02:14:59.254 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[109923]: audit 2023-08-28T02:14:59.254 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[109923]: 2023-08- 2023-08-28T02:14:59.254 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[109923]: 28T02:14:58.708288+0000 2023-08-28T02:14:59.254 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[109923]: mon.a (mon.0) 884 : audit 2023-08-28T02:14:59.254 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[109923]: [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:59.254 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:14:58 smithi143 conmon[176053]: 2023-08-28T02:14:58.978+0000 7f834f565200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-08-28T02:14:59.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:59 smithi169 conmon[100500]: cluster 2023-08-28T02:14:57.786416+0000 mgr.x (mgr.25369) 68 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:14:59.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:59 smithi169 conmon[100500]: audit 2023-08-28T 2023-08-28T02:14:59.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:59 smithi169 conmon[100500]: 02:14:58.001156+0000 mon.a (mon.0) 879 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:59.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:59 smithi169 conmon[100500]: audit 2023-08-28T02:14:58.007256+0000 mon.a (mon.0) 880 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:59.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:59 smithi169 conmon[100500]: audit 2023-08-28T02 2023-08-28T02:14:59.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:59 smithi169 conmon[100500]: :14:58.245208+0000 mon.c (mon.2) 2023-08-28T02:14:59.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:59 smithi169 conmon[100500]: 56 : audit [DBG] from='client.? 172.21.15.143:0/2204006529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:59.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:59 smithi169 conmon[100500]: audit 2023-08-28T02:14:59.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:59 smithi169 conmon[100500]: 2023-08-28T02:14:58.353386+0000 2023-08-28T02:14:59.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:59 smithi169 conmon[100500]: mon.a (mon.0) 881 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:59.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:59 smithi169 conmon[100500]: 2023-08-28T02:14:59.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:59 smithi169 conmon[100500]: audit 2023-08-28T02:14:59.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:59 smithi169 conmon[100500]: 2023-08-28T02:14:58.366675 2023-08-28T02:14:59.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:59 smithi169 conmon[100500]: +0000 mon.a (mon.0) 882 : audit [INF] 2023-08-28T02:14:59.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:59 smithi169 conmon[100500]: from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:59.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:59 smithi169 conmon[100500]: audit 2023-08-28T02:14: 2023-08-28T02:14:59.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:59 smithi169 conmon[100500]: 58.700670+0000 mon.a (mon.0) 883 : 2023-08-28T02:14:59.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:59 smithi169 conmon[100500]: audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:59.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:59 smithi169 conmon[100500]: audit 2023-08-28T02:14:59.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:14:59 smithi169 conmon[100500]: 2023-08-28T02:14:58.708288+0000 mon.a (mon.0) 884 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:14:59.564 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[176053]: 2023-08-28T02:14:59.320+0000 7f834f565200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-08-28T02:14:59.950 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[176053]: 2023-08-28T02:14:59.572+0000 7f834f565200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-08-28T02:14:59.950 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[176053]: 2023-08-28T02:14:59.727+0000 7f834f565200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-08-28T02:15:00.312 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:14:59 smithi143 conmon[176053]: 2023-08-28T02:14:59.997+0000 7f834f565200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-08-28T02:15:01.072 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:00 smithi143 conmon[176053]: 2023-08-28T02:15:00.776+0000 7f834f565200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-08-28T02:15:01.072 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:00 smithi143 conmon[176053]: 2023-08-28T02:15:00.942+0000 7f834f565200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-08-28T02:15:01.450 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:01 smithi143 conmon[176053]: 2023-08-28T02:15:01.152+0000 7f834f565200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-08-28T02:15:01.451 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:01 smithi143 conmon[176053]: 2023-08-28T02:15:01.273+0000 7f834f565200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-08-28T02:15:01.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:01 smithi143 conmon[109923]: cluster 2023-08-28T02:14 2023-08-28T02:15:01.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:01 smithi143 conmon[109923]: :59.786880+0000 mgr.x (mgr.25369) 69 : cluster [DBG] pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:01.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:01 smithi143 conmon[109923]: audit 2023-08-28T02:15:01.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:01 smithi143 conmon[109923]: 2023-08-28T02:15: 2023-08-28T02:15:01.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:01 smithi143 conmon[109923]: 00.632953+0000 mon.a (mon 2023-08-28T02:15:01.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:01 smithi143 conmon[109923]: .0) 885 : audit [DBG] 2023-08-28T02:15:01.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:01 smithi143 conmon[109923]: from='client.? 172.21.15.143:0/826816168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:01.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:01 smithi143 conmon[100258]: cluster 2023-08-28T02:14:59.786880+0000 mgr.x (mgr.25369) 69 : cluster 2023-08-28T02:15:01.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:01 smithi143 conmon[100258]: [DBG] pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:01.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:01 smithi143 conmon[100258]: audit 2023-08-28T02:15:00.632953+0000 mon.a (mon.0) 885 : audit [DBG] from='client.? 172.21.15.143:0/826816168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:01.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:01 smithi169 conmon[100500]: cluster 2023-08-28T02:14:59.786880+0000 mgr.x (mgr.25369) 69 : cluster [DBG] pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:01.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:01 smithi169 conmon[100500]: audit 2023-08-28T02:15:00.632953+0000 mon.a (mon.0) 885 : audit [DBG] from='client.? 172.21.15.143:0/826816168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:01.782 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:01 smithi143 conmon[176053]: 2023-08-28T02:15:01.500+0000 7f834f565200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-08-28T02:15:02.088 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:01 smithi169 conmon[138400]: debug 2023-08-28T02:15:01.755+0000 7f62482ad700 -1 mgr handle_mgr_map I was active but no longer am 2023-08-28T02:15:02.088 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:01 smithi169 conmon[138400]: ignoring --setuser ceph since I am not root 2023-08-28T02:15:02.088 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:01 smithi169 conmon[138400]: ignoring --setgroup ceph since I am not root 2023-08-28T02:15:02.088 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:01 smithi169 conmon[138400]: debug 2023-08-28T02:15:01.805+0000 7fb99a0f8700 1 -- 172.21.15.169:0/3323932978 <== mon.1 v2:172.21.15.169:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5620b985b1e0 con 0x5620b9833000 2023-08-28T02:15:02.088 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:01 smithi169 conmon[138400]: debug 2023-08-28T02:15:01.921+0000 7fb9a3e28200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-08-28T02:15:02.089 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[138400]: debug 2023-08-28T02:15:01.999+0000 7fb9a3e28200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-08-28T02:15:02.089 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[138400]: debug 2023-08-28T02:15:02.087+0000 7fb9a3e28200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-08-28T02:15:02.350 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[138400]: debug 2023-08-28T02:15:02.180+0000 7fb9a3e28200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-08-28T02:15:02.672 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[176053]: 2023-08-28T02:15:02.398+0000 7f834f565200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-08-28T02:15:02.672 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[176053]: [28/Aug/2023:02:15:02] ENGINE Bus STARTING 2023-08-28T02:15:02.672 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[176053]: CherryPy Checker: 2023-08-28T02:15:02.673 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[176053]: The Application mounted at '' has an empty config. 2023-08-28T02:15:02.673 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[176053]: 2023-08-28T02:15:02.673 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[176053]: [28/Aug/2023:02:15:02] ENGINE Serving on http://:::9283 2023-08-28T02:15:02.673 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[176053]: [28/Aug/2023:02:15:02] ENGINE Bus STARTED 2023-08-28T02:15:02.673 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[176053]: [28/Aug/2023:02:15:02] ENGINE Bus STOPPING 2023-08-28T02:15:02.673 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[176053]: [28/Aug/2023:02:15:02] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-08-28T02:15:02.673 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[176053]: [28/Aug/2023:02:15:02] ENGINE Bus STOPPED 2023-08-28T02:15:02.673 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[176053]: [28/Aug/2023:02:15:02] ENGINE Bus STARTING 2023-08-28T02:15:02.674 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: audit 2023-08-28T02:15:01.585000+0000 mon.a (mon. 2023-08-28T02:15:02.674 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: 0) 886 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:15:02.674 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: audit 2023-08-28T02:15:01.591425+0000 mon.a (mon.0) 887 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:15:02.674 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: audit 2023-08-28T02:15:01 2023-08-28T02:15:02.674 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: .592920+0000 mon.b (mon.1) 96 : audit [DBG] 2023-08-28T02:15:02.674 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:15:02.674 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: audit 2023-08-28T02:15 2023-08-28T02:15:02.674 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: :01.593814+0000 mon.b (mon.1) 97 : 2023-08-28T02:15:02.674 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: audit [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:15:02.674 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: audit 2023-08-28T02:15:02.674 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: 2023-08-28T02:15:01.598708+0000 2023-08-28T02:15:02.675 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: mon.a (mon.0) 888 : audit [INF] 2023-08-28T02:15:02.675 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:15:02.675 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: audit 2023-08 2023-08-28T02:15:02.675 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: -28T02:15:01.740030+0000 mon.b ( 2023-08-28T02:15:02.675 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: mon.1) 98 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:15:02.675 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: audit 2023-08-28T02:15:02.675 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: 2023-08-28T02:15:01.745599+0000 2023-08-28T02:15:02.675 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: mon.b (mon.1) 99 : audit [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-08-28T02:15:02.675 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: audit 2023-08-28T02:15:02.675 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: 2023-08-28T02:15:01.745930+0000 2023-08-28T02:15:02.675 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: mon.a (mon.0) 889 : audit [INF] from='mgr.25369 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-08-28T02:15:02.675 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: cluster 2023-08-28T02:15:02.676 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: 2023-08-28T02:15:01 2023-08-28T02:15:02.676 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: .752075+0000 mon.a (mon.0) 890 : cluster 2023-08-28T02:15:02.676 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: [DBG] osdmap e84: 8 total, 8 up, 8 in 2023-08-28T02:15:02.676 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: audit 2023-08-28T02 2023-08-28T02:15:02.676 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: :15:01.752334+0000 mon.a (mon.0 2023-08-28T02:15:02.676 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: ) 891 : audit [INF] from='mgr.25369 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-08-28T02:15:02.676 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: cluster 2023-08- 2023-08-28T02:15:02.676 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: 28T02:15:01.752389+0000 mon.a (mon 2023-08-28T02:15:02.676 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: .0) 892 : cluster [DBG] mgrmap e24: y(active, starting, since 0.00576803s) 2023-08-28T02:15:02.676 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: cluster 2023 2023-08-28T02:15:02.676 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: -08-28T02:15:02.401406+0000 2023-08-28T02:15:02.676 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: mon.a (mon.0) 893 : cluster [INF] Active manager daemon y restarted 2023-08-28T02:15:02.676 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: cluster 2023-08-28T02:15:02.677 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: 2023-08-28T02:15: 2023-08-28T02:15:02.677 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: 02.401880+0000 mon.a (mon.0) 894 : 2023-08-28T02:15:02.677 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: cluster [INF] Activating manager daemon y 2023-08-28T02:15:02.677 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: cluster 2023- 2023-08-28T02:15:02.677 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: 08-28T02:15:02.407798+0000 mon.a ( 2023-08-28T02:15:02.677 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: mon.0) 895 : cluster [DBG] osdmap e85: 8 total, 8 up, 8 in 2023-08-28T02:15:02.677 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: cluster 2023-08-28T02:15:02.677 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: 2023-08-28T02:15:02 2023-08-28T02:15:02.677 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: .408037+0000 mon.a (mon.0) 896 : cluster 2023-08-28T02:15:02.677 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: [DBG] mgrmap e25: y(active, starting, since 0.00634238s) 2023-08-28T02:15:02.677 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: audit 2023-08-28T02 2023-08-28T02:15:02.678 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: audit 2023-08-28T02:15 2023-08-28T02:15:02.678 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: :01.585000+0000 mon.a (mon.0) 886 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:15:02.678 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: audit 2023 2023-08-28T02:15:02.678 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: -08-28T02:15:01.591425+0000 mon.a (mon. 2023-08-28T02:15:02.678 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: 0) 887 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:15:02.678 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:15:02.678 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: 08-28T02:15:01.592920+0000 mon.b (mon. 2023-08-28T02:15:02.678 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: 1) 96 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:15:02.678 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:15:02.679 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: 08-28T02:15:01.593814+0000 mon.b (mon. 2023-08-28T02:15:02.679 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: 1) 97 : audit [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:15:02.679 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: audit 2023-08 2023-08-28T02:15:02.679 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: -28T02:15:01.598708+0000 mon.a ( 2023-08-28T02:15:02.679 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: mon.0) 888 : audit [INF] 2023-08-28T02:15:02.679 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:15:02.679 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: audit 2023 2023-08-28T02:15:02.679 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: -08-28T02:15:01.740030+0000 2023-08-28T02:15:02.679 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: mon.b (mon.1) 98 : audit [DBG] 2023-08-28T02:15:02.679 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:15:02.679 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: audit 2023-08-28T02:15:02.679 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: 2023-08-28T02:15:01.745599+0000 2023-08-28T02:15:02.680 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: mon.b (mon.1) 99 : audit 2023-08-28T02:15:02.680 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-08-28T02:15:02.680 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: audit 2023-08-28T02:15:02.680 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: 2023-08-28T02:15:01.745930+0000 2023-08-28T02:15:02.680 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: mon.a (mon.0) 889 : audit 2023-08-28T02:15:02.680 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: [INF] from='mgr.25369 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-08-28T02:15:02.680 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: cluster 2023-08-28T02:15:02.680 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: 2023-08-28T02:15:01.752075 2023-08-28T02:15:02.680 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: +0000 mon.a (mon.0) 890 : cluster 2023-08-28T02:15:02.680 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: [DBG] osdmap e84: 8 total, 8 up, 8 in 2023-08-28T02:15:02.680 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: audit 2023-08 2023-08-28T02:15:02.680 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: -28T02:15:01.752334+0000 mon.a ( 2023-08-28T02:15:02.681 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: mon.0) 891 : audit [INF] from='mgr.25369 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-08-28T02:15:02.681 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: 2023-08-28T02:15:02.681 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: cluster 2023- 2023-08-28T02:15:02.681 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: 08-28T02:15:01.752389+0000 2023-08-28T02:15:02.681 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: mon.a (mon.0) 892 : cluster [DBG] 2023-08-28T02:15:02.681 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: mgrmap e24: y(active, starting, since 0.00576803s) 2023-08-28T02:15:02.681 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: cluster 2023-08-28T02:15:02.681 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: 2023-08-28T02:15:02.401406+0000 2023-08-28T02:15:02.681 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: mon.a (mon.0) 893 : cluster 2023-08-28T02:15:02.681 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: [INF] Active manager daemon y restarted 2023-08-28T02:15:02.681 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: cluster 2023-08-28T02:15:02.681 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: 2023-08-28T02:15:02.401880 2023-08-28T02:15:02.682 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: +0000 mon.a (mon.0) 894 : 2023-08-28T02:15:02.682 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: cluster [INF] Activating manager daemon y 2023-08-28T02:15:02.682 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: cluster 2023-08- 2023-08-28T02:15:02.682 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: 28T02:15:02.407798+0000 mon.a (mon. 2023-08-28T02:15:02.682 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: 0) 895 : cluster [DBG] osdmap e85: 8 total, 8 up, 8 in 2023-08-28T02:15:02.682 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: 2023-08-28T02:15:02.682 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: :15:02.408197+0000 mon.a (mon.0) 897 2023-08-28T02:15:02.682 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: : audit [DBG] from='mgr.? 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-08-28T02:15:02.683 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: audit 2023-08-28T 2023-08-28T02:15:02.683 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: 02:15:02.410438+0000 mon.c (mon.2) 2023-08-28T02:15:02.683 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: 57 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-08-28T02:15:02.683 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: audit 2023-08-28T02:15:02.683 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: 2023-08-28T02:15:02.410697+0000 2023-08-28T02:15:02.683 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: mon.c (mon.2) 58 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T02:15:02.683 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: 2023-08-28T02:15:02.683 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: audit 2023-08-28T02:15:02.683 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: 2023-08-28T02:15:02.410907+0000 mon.c (mon 2023-08-28T02:15:02.683 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: .2) 59 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T02:15:02.683 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: audit 2023-08-28T02:15:02.684 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: 2023-08-28T02:15:02.411064 2023-08-28T02:15:02.684 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: +0000 mon.c (mon.2) 60 : audit [DBG] 2023-08-28T02:15:02.684 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T02:15:02.684 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: audit 2023-08-28T02:15: 2023-08-28T02:15:02.684 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: 02.411277+0000 mon.c (mon.2) 61 : 2023-08-28T02:15:02.684 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi143.spifrj"}]: dispatch 2023-08-28T02:15:02.684 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: audit 2023-08-28T02: 2023-08-28T02:15:02.684 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: 15:02.411556+0000 mon.c (mon.2) 62 2023-08-28T02:15:02.684 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi169.yeisur"}]: dispatch 2023-08-28T02:15:02.684 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: audit 2023 2023-08-28T02:15:02.684 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: -08-28T02:15:02.411806+0000 2023-08-28T02:15:02.685 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: mon.c (mon.2) 63 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-08-28T02:15:02.685 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: audit 2023-08-28T02:15:02.685 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: 2023-08-28T02:15:02.412060+0000 2023-08-28T02:15:02.685 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: mon.c (mon.2) 64 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-28T02:15:02.685 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: audit 2023 2023-08-28T02:15:02.685 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: -08-28T02:15:02.412333+0000 mon.c 2023-08-28T02:15:02.685 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: (mon.2) 65 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-28T02:15:02.685 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: audit 2023-08-28T02:15:02.685 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: 2023-08-28T02:15:02.412578+0000 2023-08-28T02:15:02.685 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: mon.c (mon.2) 66 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-28T02:15:02.685 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: 2023-08-28T02:15:02.686 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: audit 2023-08-28T02:15:02.686 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: 2023-08-28T02:15:02.412854+0000 mon.c (mon. 2023-08-28T02:15:02.686 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: 2) 67 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-28T02:15:02.686 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: audit 2023-08- 2023-08-28T02:15:02.686 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: 28T02:15:02.413171+0000 mon.c (mon 2023-08-28T02:15:02.686 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: cluster 2023 2023-08-28T02:15:02.686 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: -08-28T02:15:02.408037+0000 2023-08-28T02:15:02.686 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: mon.a (mon.0) 896 : cluster [DBG] 2023-08-28T02:15:02.686 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: mgrmap e25: y(active, starting, since 0.00634238s) 2023-08-28T02:15:02.687 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: audit 2023-08-28T02:15:02.687 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: 2023-08-28T02:15 2023-08-28T02:15:02.687 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: :02.408197+0000 mon.a (mon 2023-08-28T02:15:02.687 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: .0) 897 : audit [DBG] from='mgr.? 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-08-28T02:15:02.687 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: 2023-08-28T02:15:02.687 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:15:02.687 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: 08-28T02:15:02.410438+0000 mon.c 2023-08-28T02:15:02.687 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: (mon.2) 57 : audit 2023-08-28T02:15:02.687 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-08-28T02:15:02.687 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: audit 2023 2023-08-28T02:15:02.687 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: -08-28T02:15:02.410697+0000 2023-08-28T02:15:02.687 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: mon.c (mon.2) 58 : audit [DBG] 2023-08-28T02:15:02.688 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T02:15:02.688 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: audit 2023 2023-08-28T02:15:02.688 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: -08-28T02:15:02.410907+0000 2023-08-28T02:15:02.688 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: mon.c (mon.2) 59 : audit 2023-08-28T02:15:02.688 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T02:15:02.688 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: audit 2023 2023-08-28T02:15:02.688 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: -08-28T02:15:02.411064+0000 mon.c 2023-08-28T02:15:02.688 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: (mon.2) 60 : audit [DBG] 2023-08-28T02:15:02.688 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T02:15:02.688 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: audit 2023-08-28T02:15:02.688 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: 2023-08-28T02:15:02.411277+0000 2023-08-28T02:15:02.688 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: mon.c (mon.2) 61 : audit 2023-08-28T02:15:02.688 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi143.spifrj"}]: dispatch 2023-08-28T02:15:02.688 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: audit 2023-08- 2023-08-28T02:15:02.689 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: 28T02:15:02.411556+0000 mon.c ( 2023-08-28T02:15:02.689 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: mon.2) 62 : audit [DBG] 2023-08-28T02:15:02.689 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi169.yeisur"}]: dispatch 2023-08-28T02:15:02.689 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: audit 2023 2023-08-28T02:15:02.689 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: -08-28T02:15:02.411806+0000 2023-08-28T02:15:02.689 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: mon.c (mon.2) 63 : audit 2023-08-28T02:15:02.689 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-08-28T02:15:02.689 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: audit 2023-08-28T02:15:02.689 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: 2023-08 2023-08-28T02:15:02.689 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: -28T02:15:02.412060 2023-08-28T02:15:02.689 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: +0000 mon.c (mon.2) 2023-08-28T02:15:02.689 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: 64 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-28T02:15:02.689 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: audit 2023 2023-08-28T02:15:02.690 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: -08-28T02:15:02.412333 2023-08-28T02:15:02.690 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: +0000 mon.c (mon.2) 2023-08-28T02:15:02.690 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: 65 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-28T02:15:02.690 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:15:02.690 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: .2) 68 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-28T02:15:02.690 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: audit 2023-08-28T02:15:02.690 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: 2023-08-28T02:15:02.413437 2023-08-28T02:15:02.690 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: +0000 mon.c (mon.2) 69 : audit [DBG] 2023-08-28T02:15:02.690 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-28T02:15:02.690 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: audit 2023-08-28T02:15:02 2023-08-28T02:15:02.691 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: .413771+0000 mon.c (mon.2) 70 : audit 2023-08-28T02:15:02.691 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-28T02:15:02.691 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: audit 2023-08-28T02: 2023-08-28T02:15:02.691 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: 15:02.413985+0000 mon.c (mon.2) 71 2023-08-28T02:15:02.691 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-28T02:15:02.691 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: audit 2023-08-28T02 2023-08-28T02:15:02.691 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: :15:02.414228+0000 mon.c (mon.2) 2023-08-28T02:15:02.691 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: 72 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-08-28T02:15:02.691 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: audit 2023-08-28T02:15:02.691 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: 2023-08-28T02:15:02. 2023-08-28T02:15:02.691 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: 414473+0000 mon.c (mon.2) 73 : audit [DBG] 2023-08-28T02:15:02.691 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-08-28T02:15:02.691 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: audit 2023-08-28T02:15:02.692 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: 2023-08 2023-08-28T02:15:02.692 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: -28T02:15:02.414897+0000 mon.c ( 2023-08-28T02:15:02.692 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: mon.2) 74 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-08-28T02:15:02.692 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: audit 2023-08-28T02:15:02.692 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: 2023-08-28T02:15:02.415124+0000 2023-08-28T02:15:02.692 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: mon.c (mon.2) 75 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-28T02:15:02.692 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: 2023-08-28T02:15:02.692 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: audit 2023-08-28T02:15:02.692 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: 2023-08-28T02:15:02.416472+0000 mon.c ( 2023-08-28T02:15:02.692 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: mon.2) 76 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-08-28T02:15:02.692 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: audit 2023 2023-08-28T02:15:02.692 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: -08-28T02:15:02.416734+0000 mon.c 2023-08-28T02:15:02.692 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: (mon.2) 77 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-08-28T02:15:02.692 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: cluster 2023-08-28T02:15:02.692 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: 2023-08-28T02:15: 2023-08-28T02:15:02.693 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: 02.573313+0000 mon.a (mon.0) 898 : cluster 2023-08-28T02:15:02.693 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[100258]: [INF] Manager daemon y is now available 2023-08-28T02:15:02.693 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: 08-28T02:15:02.412578+0000 2023-08-28T02:15:02.693 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: mon.c (mon.2) 66 : audit 2023-08-28T02:15:02.693 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-28T02:15:02.693 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:15:02.693 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: 08-28T02:15:02.412854+0000 2023-08-28T02:15:02.693 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: mon.c (mon.2) 67 : 2023-08-28T02:15:02.693 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-28T02:15:02.694 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:15:02.694 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: 08-28T02:15:02.413171+0000 2023-08-28T02:15:02.694 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: mon.c (mon.2) 68 : 2023-08-28T02:15:02.694 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-28T02:15:02.694 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:15:02.694 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: 08-28T02:15:02.413437 2023-08-28T02:15:02.694 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: +0000 mon.c (mon.2) 69 2023-08-28T02:15:02.694 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-28T02:15:02.694 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:15:02.694 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: 08-28T02:15:02.413771 2023-08-28T02:15:02.694 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: +0000 mon.c (mon.2) 70 2023-08-28T02:15:02.694 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-28T02:15:02.694 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:15:02.695 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: 08-28T02:15:02.413985 2023-08-28T02:15:02.695 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: +0000 mon.c (mon.2) 71 : 2023-08-28T02:15:02.695 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-28T02:15:02.695 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:15:02.695 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: 08-28T02:15:02.414228 2023-08-28T02:15:02.695 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: +0000 mon.c (mon.2) 2023-08-28T02:15:02.695 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: 72 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-08-28T02:15:02.695 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:15:02.695 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: 08-28T02:15:02.414473 2023-08-28T02:15:02.695 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: +0000 mon.c (mon.2) 2023-08-28T02:15:02.695 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: 73 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-08-28T02:15:02.695 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:15:02.695 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: 08-28T02:15:02.414897+0000 mon.c 2023-08-28T02:15:02.696 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: (mon.2) 74 : audit 2023-08-28T02:15:02.696 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-08-28T02:15:02.696 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:15:02.696 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: 08-28T02:15:02.415124+0000 2023-08-28T02:15:02.696 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: mon.c (mon.2) 75 : 2023-08-28T02:15:02.696 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-28T02:15:02.696 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:15:02.696 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: 08-28T02:15:02.416472 2023-08-28T02:15:02.696 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: +0000 mon.c (mon.2) 76 2023-08-28T02:15:02.696 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-08-28T02:15:02.696 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:15:02.696 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: 08-28T02:15:02.416734+0000 2023-08-28T02:15:02.696 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: mon.c (mon.2) 77 2023-08-28T02:15:02.696 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-08-28T02:15:02.697 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: cluster 2023- 2023-08-28T02:15:02.697 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: 08-28T02:15:02.573313+0000 2023-08-28T02:15:02.697 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: mon.a (mon.0) 898 : 2023-08-28T02:15:02.697 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[109923]: cluster [INF] Manager daemon y is now available 2023-08-28T02:15:02.697 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[138400]: debug 2023-08-28T02:15:02.350+0000 7fb9a3e28200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-08-28T02:15:02.698 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: audit 2023-08-28T02:15:01.585000+0000 mon.a (mon.0) 886 : 2023-08-28T02:15:02.698 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:15:02.698 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: audit 2023-08-28T02:15:01.591425+0000 mon.a (mon.0) 887 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:15:02.698 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: audit 2023-08-28T02:15:01.592920 2023-08-28T02:15:02.698 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: +0000 mon.b (mon.1) 96 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:15:02.698 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: audit 2023-08-28T02:15:01.593814+0000 2023-08-28T02:15:02.698 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: mon.b (mon.1) 97 : audit [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:15:02.698 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: audit 2023-08-28T02:15:01.598708 2023-08-28T02:15:02.698 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: +0000 mon.a (mon.0) 888 : audit [INF] from='mgr.25369 ' entity='mgr.x' 2023-08-28T02:15:02.698 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: audit 2023-08-28T02:15:01.740030+0000 mon.b ( 2023-08-28T02:15:02.699 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: mon.1) 98 : audit [DBG] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:15:02.699 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: audit 2023-08-28T02:15:01.745599 2023-08-28T02:15:02.699 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: +0000 mon.b (mon.1) 99 : audit [INF] from='mgr.25369 172.21.15.169:0/1538506574' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-08-28T02:15:02.699 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: audit 2023-08-28T02:15 2023-08-28T02:15:02.699 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: :01.745930+0000 mon.a (mon.0) 889 : audit [INF] from='mgr.25369 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-08-28T02:15:02.699 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: cluster 2023-08-28T 2023-08-28T02:15:02.699 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: 02:15:01.752075+0000 mon.a (mon.0) 890 : cluster [DBG] 2023-08-28T02:15:02.699 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: osdmap e84: 8 total, 8 up, 8 in 2023-08-28T02:15:02.699 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: audit 2023 2023-08-28T02:15:02.699 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: -08-28T02:15:01.752334+0000 mon.a (mon.0) 891 : audit [INF] from='mgr.25369 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-08-28T02:15:02.699 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: 2023-08-28T02:15:02.699 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: cluster 2023-08 2023-08-28T02:15:02.699 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: -28T02:15:01.752389+0000 mon.a (mon.0) 892 : cluster [DBG] mgrmap e24: y(active, starting, since 0.00576803s) 2023-08-28T02:15:02.700 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: cluster 2023-08-28 2023-08-28T02:15:02.700 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: T02:15:02.401406+0000 mon.a (mon.0) 893 : cluster 2023-08-28T02:15:02.700 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: [INF] Active manager daemon y restarted 2023-08-28T02:15:02.700 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: cluster 2023-08-28T02:15:02.700 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: 2023-08-28T02:15:02.401880+0000 mon.a (mon.0) 894 : cluster [INF] 2023-08-28T02:15:02.700 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: Activating manager daemon y 2023-08-28T02:15:02.700 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: cluster 2023- 2023-08-28T02:15:02.700 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: 08-28T02:15:02.407798+0000 mon.a (mon.0) 895 : cluster [DBG] osdmap e85: 8 total, 8 up, 8 in 2023-08-28T02:15:02.700 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: 2023-08-28T02:15:02.700 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: cluster 2023-08 2023-08-28T02:15:02.700 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: -28T02:15:02.408037+0000 mon.a (mon.0) 896 : cluster [DBG] mgrmap e25: y(active, starting, since 0.00634238s) 2023-08-28T02:15:02.700 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: audit 2023-08-28T02 2023-08-28T02:15:02.700 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: :15:02.408197+0000 mon.a (mon.0) 897 : audit [DBG] from='mgr.? 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-08-28T02:15:02.701 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: audit 2023-08-28T02:15:02 2023-08-28T02:15:02.701 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: .410438+0000 mon.c (mon.2) 57 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-08-28T02:15:02.701 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: audit 2023-08-28T 2023-08-28T02:15:02.701 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: 02:15:02.410697+0000 mon.c (mon.2) 58 : audit [DBG] 2023-08-28T02:15:02.701 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T02:15:02.701 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: audit 2023-08-28T02:15:02.701 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: 2023-08-28T02:15:02.410907+0000 mon.c (mon.2) 59 : audit [DBG] 2023-08-28T02:15:02.701 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T02:15:02.701 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: audit 2023 2023-08-28T02:15:02.701 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: -08-28T02:15:02.411064+0000 mon.c (mon.2) 60 : audit [DBG] 2023-08-28T02:15:02.701 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T02:15:02.701 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: audit 2023- 2023-08-28T02:15:02.701 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: 08-28T02:15:02.411277+0000 mon.c (mon.2) 61 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi143.spifrj"}]: dispatch 2023-08-28T02:15:02.702 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: 2023-08-28T02:15:02.702 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: audit 2023-08- 2023-08-28T02:15:02.702 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: 28T02:15:02.411556+0000 mon.c (mon.2) 62 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi169.yeisur"}]: dispatch 2023-08-28T02:15:02.702 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: audit 2023-08-28T 2023-08-28T02:15:02.702 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: 02:15:02.411806+0000 mon.c (mon.2) 63 : audit [DBG] 2023-08-28T02:15:02.702 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-08-28T02:15:02.702 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: audit 2023 2023-08-28T02:15:02.702 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: -08-28T02:15:02.412060+0000 mon.c (mon.2) 64 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-28T02:15:02.702 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: 2023-08-28T02:15:02.702 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: audit 2023-08- 2023-08-28T02:15:02.702 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: 28T02:15:02.412333+0000 mon.c (mon.2) 65 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-28T02:15:02.702 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: audit 2023-08-28T 2023-08-28T02:15:02.703 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: 02:15:02.412578+0000 mon.c (mon.2) 66 : audit 2023-08-28T02:15:02.703 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-28T02:15:02.703 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: audit 2023-08-28T02:15:02.703 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: 2023-08-28T02:15:02.412854+0000 mon.c (mon.2) 67 : audit [DBG] 2023-08-28T02:15:02.703 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-28T02:15:02.703 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: audit 2023 2023-08-28T02:15:02.703 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: -08-28T02:15:02.413171+0000 mon.c (mon.2) 68 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-28T02:15:02.703 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: audit 2023-08-28 2023-08-28T02:15:02.703 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: T02:15:02.413437+0000 mon.c (mon.2) 69 : audit [DBG] 2023-08-28T02:15:02.703 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-28T02:15:02.703 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: audit 2023 2023-08-28T02:15:02.703 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: -08-28T02:15:02.413771+0000 mon.c (mon.2) 70 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-28T02:15:02.703 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: audit 2023-08-28T02:15: 2023-08-28T02:15:02.704 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: 02.413985+0000 mon.c (mon.2) 71 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-28T02:15:02.704 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: audit 2023-08-28 2023-08-28T02:15:02.704 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: T02:15:02.414228+0000 mon.c (mon.2) 72 : audit 2023-08-28T02:15:02.704 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-08-28T02:15:02.704 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: audit 2023-08-28T02:15:02.704 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: 2023-08-28T02:15:02.414473+0000 mon.c (mon.2) 73 : audit [DBG] 2023-08-28T02:15:02.704 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-08-28T02:15:02.704 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: audit 2023-08 2023-08-28T02:15:02.704 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: -28T02:15:02.414897+0000 mon.c (mon.2) 74 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-08-28T02:15:02.704 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: audit 2023-08-28 2023-08-28T02:15:02.704 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: T02:15:02.415124+0000 mon.c (mon.2) 75 : audit 2023-08-28T02:15:02.704 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-28T02:15:02.704 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: audit 2023-08-28T02:15:02.705 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: 2023-08-28T02:15:02.416472+0000 mon.c (mon.2) 76 : audit [DBG] 2023-08-28T02:15:02.705 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-08-28T02:15:02.705 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: audit 2023 2023-08-28T02:15:02.705 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: -08-28T02:15:02.416734+0000 mon.c (mon.2) 77 : audit [DBG] 2023-08-28T02:15:02.705 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-08-28T02:15:02.705 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: cluster 2023-08 2023-08-28T02:15:02.705 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[100500]: -28T02:15:02.573313+0000 mon.a (mon.0) 898 : cluster [INF] Manager daemon y is now available 2023-08-28T02:15:02.950 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[176053]: 2023-08-28T02:15:02.679+0000 7f82a92b9700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-28T02:15:02.950 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[176053]: 2023-08-28T02:15:02.679+0000 7f82a92b9700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-28T02:15:02.950 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[176053]: 2023-08-28T02:15:02.679+0000 7f82a92b9700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-28T02:15:02.950 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[176053]: 2023-08-28T02:15:02.679+0000 7f82a92b9700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-28T02:15:02.951 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[176053]: 2023-08-28T02:15:02.679+0000 7f82a92b9700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-28T02:15:02.951 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[176053]: 2023-08-28T02:15:02.688+0000 7f82ababe700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-28T02:15:02.951 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[176053]: 2023-08-28T02:15:02.688+0000 7f82ababe700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-28T02:15:02.951 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[176053]: 2023-08-28T02:15:02.688+0000 7f82ababe700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-28T02:15:02.951 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[176053]: 2023-08-28T02:15:02.688+0000 7f82ababe700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-28T02:15:02.951 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[176053]: 2023-08-28T02:15:02.688+0000 7f82ababe700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-28T02:15:02.951 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[176053]: [28/Aug/2023:02:15:02] ENGINE Serving on http://:::9283 2023-08-28T02:15:02.951 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:02 smithi143 conmon[176053]: [28/Aug/2023:02:15:02] ENGINE Bus STARTED 2023-08-28T02:15:02.958 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[138400]: debug 2023-08-28T02:15:02.728+0000 7fb9a3e28200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-08-28T02:15:02.958 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[138400]: debug 2023-08-28T02:15:02.864+0000 7fb9a3e28200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-08-28T02:15:03.237 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:02 smithi169 conmon[138400]: debug 2023-08-28T02:15:02.958+0000 7fb9a3e28200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-08-28T02:15:03.237 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:03 smithi169 conmon[138400]: debug 2023-08-28T02:15:03.058+0000 7fb9a3e28200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-08-28T02:15:03.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:03 smithi143 conmon[100258]: audit 2023-08-28T02:15:02.595188+0000 mon.c (mon.2) 78 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:15:03.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:03 smithi143 conmon[100258]: audit 2023-08-28T02:15:02.597805+0000 mon.c (mon.2) 79 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:15:03.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:03 smithi143 conmon[100258]: audit 2023-08-28T02:15:02.616954+0000 mon.c (mon.2) 80 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:15:03.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:03 smithi143 conmon[100258]: audit 2023-08-28T02:15:02.626456+0000 mon.c (mon.2) 81 : audit [INF] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-08-28T02:15:03.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:03 smithi143 conmon[100258]: audit 2023-08-28T02:15:02.626846+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.15837 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-08-28T02:15:03.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:03 smithi143 conmon[100258]: audit 2023-08-28T02:15:02.650259+0000 mon.c (mon.2) 82 : audit [INF] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-08-28T02:15:03.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:03 smithi143 conmon[100258]: audit 2023-08-28T02:15:02.650775+0000 mon.a (mon.0) 900 : audit [INF] from='mgr.15837 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-08-28T02:15:03.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:03 smithi143 conmon[100258]: audit 2023-08-28T02:15:03.038081+0000 mon.a (mon.0) 901 : audit [DBG] from='client.? 172.21.15.143:0/1090625058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:03.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:03 smithi143 conmon[100258]: cluster 2023-08-28T02:15:03.410559+0000 mon.a (mon.0) 902 : cluster [DBG] mgrmap e26: y(active, since 1.00882s) 2023-08-28T02:15:03.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:03 smithi143 conmon[109923]: audit 2023-08-28T02:15:02.595188+0000 mon.c (mon.2) 78 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:15:03.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:03 smithi143 conmon[109923]: audit 2023-08-28T02:15:02.597805+0000 mon.c (mon.2) 79 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:15:03.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:03 smithi143 conmon[109923]: audit 2023-08-28T02:15:02.616954+0000 mon.c (mon.2) 80 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:15:03.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:03 smithi143 conmon[109923]: audit 2023-08-28T02:15:02.626456+0000 mon.c (mon.2) 81 : audit [INF] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-08-28T02:15:03.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:03 smithi143 conmon[109923]: audit 2023-08-28T02:15:02.626846+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.15837 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-08-28T02:15:03.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:03 smithi143 conmon[109923]: audit 2023-08-28T02:15:02.650259+0000 mon.c (mon.2) 82 : audit [INF] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-08-28T02:15:03.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:03 smithi143 conmon[109923]: 2023-08-28T02:15:03.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:03 smithi143 conmon[109923]: audit 2023-08-28T02:15:02.650775+0000 mon.a (mon.0) 900 : audit [INF] from='mgr.15837 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-08-28T02:15:03.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:03 smithi143 conmon[109923]: audit 2023-08-28T02:15:03.038081+0000 mon.a (mon.0) 901 : audit [DBG] from='client.? 172.21.15.143:0/1090625058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:03.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:03 smithi143 conmon[109923]: cluster 2023-08-28T02:15:03.410559+0000 mon.a (mon.0) 902 : cluster [DBG] mgrmap e26: y(active, since 1.00882s) 2023-08-28T02:15:03.987 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:03 smithi169 conmon[138400]: debug 2023-08-28T02:15:03.548+0000 7fb9a3e28200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-08-28T02:15:03.993 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:03 smithi169 conmon[100500]: audit 2023-08-28T02:15:02.595188+0000 mon.c (mon.2) 78 2023-08-28T02:15:03.994 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:03 smithi169 conmon[100500]: : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:15:03.994 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:03 smithi169 conmon[100500]: audit 2023-08-28T02:15:02.597805+0000 mon.c (mon.2) 79 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:15:03.994 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:03 smithi169 conmon[100500]: audit 2023-08-28T02:15:02.616954+0000 mon.c (mon.2) 80 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:15:03.994 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:03 smithi169 conmon[100500]: audit 2023-08-28T02:15:02.626456+0000 mon.c (mon.2) 81 : audit 2023-08-28T02:15:03.994 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:03 smithi169 conmon[100500]: [INF] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-08-28T02:15:03.994 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:03 smithi169 conmon[100500]: audit 2023-08-28T02:15:02.626846+0000 mon.a (mon.0) 899 2023-08-28T02:15:03.994 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:03 smithi169 conmon[100500]: : audit [INF] from='mgr.15837 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-08-28T02:15:03.994 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:03 smithi169 conmon[100500]: audit 2023-08-28T02:15:03.994 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:03 smithi169 conmon[100500]: 2023-08-28T02:15:02.650259+0000 mon.c (mon.2 2023-08-28T02:15:03.994 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:03 smithi169 conmon[100500]: ) 82 : audit [INF] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-08-28T02:15:03.994 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:03 smithi169 conmon[100500]: audit 2023-08-28T 2023-08-28T02:15:03.994 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:03 smithi169 conmon[100500]: 02:15:02.650775+0000 mon.a (mon.0) 2023-08-28T02:15:03.995 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:03 smithi169 conmon[100500]: 900 : audit [INF] from='mgr.15837 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-08-28T02:15:03.995 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:03 smithi169 conmon[100500]: audit 2023-08-28T02: 2023-08-28T02:15:03.995 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:03 smithi169 conmon[100500]: 15:03.038081+0000 mon.a (mon.0) 2023-08-28T02:15:03.995 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:03 smithi169 conmon[100500]: 901 : audit [DBG] from='client.? 172.21.15.143:0/1090625058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:03.995 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:03 smithi169 conmon[100500]: cluster 2023-08-28T02: 2023-08-28T02:15:03.995 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:03 smithi169 conmon[100500]: 15:03.410559+0000 mon.a (mon.0) 902 2023-08-28T02:15:03.995 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:03 smithi169 conmon[100500]: : cluster [DBG] mgrmap e26: y(active, since 1.00882s) 2023-08-28T02:15:04.588 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:04 smithi169 conmon[138400]: debug 2023-08-28T02:15:04.260+0000 7fb9a3e28200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-08-28T02:15:04.589 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:04 smithi169 conmon[138400]: debug 2023-08-28T02:15:04.355+0000 7fb9a3e28200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-08-28T02:15:04.589 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:04 smithi169 conmon[100500]: cephadm 2023-08-28T02:15:03.287776+0000 mgr.y (mgr. 2023-08-28T02:15:04.589 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:04 smithi169 conmon[100500]: 15837) 1 : cephadm [INF] [28/Aug/2023:02:15:03] ENGINE Bus STARTING 2023-08-28T02:15:04.589 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:04 smithi169 conmon[100500]: cephadm 2023-08-28T02:15:03.403453+0000 mgr.y (mgr.15837) 2 : cephadm [INF] [28/Aug/2023:02:15:03] ENGINE Serving on https://172.21.15.143:7150 2023-08-28T02:15:04.589 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:04 smithi169 conmon[100500]: cluster 2023-08-28T02:15:03.425708+0000 2023-08-28T02:15:04.590 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:04 smithi169 conmon[100500]: mgr.y (mgr.15837) 3 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:04.590 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:04 smithi169 conmon[100500]: cephadm 2023-08-28T 2023-08-28T02:15:04.590 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:04 smithi169 conmon[100500]: 02:15:03.504853+0000 mgr.y (mgr.15837) 4 : 2023-08-28T02:15:04.590 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:04 smithi169 conmon[100500]: cephadm [INF] [28/Aug/2023:02:15:03] ENGINE Serving on http://172.21.15.143:8765 2023-08-28T02:15:04.590 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:04 smithi169 conmon[100500]: cephadm 2023-08-28T02:15: 2023-08-28T02:15:04.590 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:04 smithi169 conmon[100500]: 03.505003+0000 mgr.y (mgr.15837) 5 2023-08-28T02:15:04.590 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:04 smithi169 conmon[100500]: : cephadm [INF] [28/Aug/2023:02:15:03] ENGINE Bus STARTED 2023-08-28T02:15:04.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:04 smithi143 conmon[100258]: cephadm 2023-08-28T02:15:03 2023-08-28T02:15:04.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:04 smithi143 conmon[100258]: .287776+0000 mgr.y (mgr.15837) 1 : cephadm [INF] [28/Aug/2023:02:15:03] ENGINE Bus STARTING 2023-08-28T02:15:04.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:04 smithi143 conmon[100258]: cephadm 2023-08-28T02:15:03.403453+0000 mgr.y (mgr.15837) 2 : cephadm [INF] [28/Aug/2023:02:15:03] ENGINE Serving on https://172.21.15.143:7150 2023-08-28T02:15:04.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:04 smithi143 conmon[100258]: cluster 2023-08-28T02:15:03.425708+0000 mgr.y (mgr.15837) 3 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:04.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:04 smithi143 conmon[100258]: cephadm 2023-08-28T02:15:03.504853+0000 mgr.y (mgr. 2023-08-28T02:15:04.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:04 smithi143 conmon[100258]: 15837) 4 : cephadm [INF] [28/Aug/2023:02:15:03] ENGINE Serving on http://172.21.15.143:8765 2023-08-28T02:15:04.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:04 smithi143 conmon[100258]: cephadm 2023-08-28T02:15:03. 2023-08-28T02:15:04.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:04 smithi143 conmon[100258]: 505003+0000 mgr.y (mgr.15837) 5 : cephadm [INF] [28/Aug/2023:02:15:03] ENGINE Bus STARTED 2023-08-28T02:15:04.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:04 smithi143 conmon[109923]: cephadm 2023- 2023-08-28T02:15:04.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:04 smithi143 conmon[109923]: 08-28T02:15:03.287776+0000 mgr.y (mgr.15837) 1 : cephadm [INF] [28/Aug/2023:02:15:03] ENGINE Bus STARTING 2023-08-28T02:15:04.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:04 smithi143 conmon[109923]: cephadm 2023 2023-08-28T02:15:04.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:04 smithi143 conmon[109923]: -08-28T02:15:03.403453+0000 mgr.y 2023-08-28T02:15:04.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:04 smithi143 conmon[109923]: (mgr.15837) 2 : cephadm [INF] [28/Aug/2023:02:15:03] ENGINE Serving on https://172.21.15.143:7150 2023-08-28T02:15:04.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:04 smithi143 conmon[109923]: 2023-08-28T02:15:04.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:04 smithi143 conmon[109923]: cluster 2023- 2023-08-28T02:15:04.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:04 smithi143 conmon[109923]: 08-28T02:15:03.425708+0000 2023-08-28T02:15:04.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:04 smithi143 conmon[109923]: mgr.y (mgr.15837) 3 : cluster [DBG] 2023-08-28T02:15:04.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:04 smithi143 conmon[109923]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:04.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:04 smithi143 conmon[109923]: cephadm 2023-08-28T02:15: 2023-08-28T02:15:04.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:04 smithi143 conmon[109923]: 03.504853+0000 mgr.y (mgr.15837) 4 : 2023-08-28T02:15:04.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:04 smithi143 conmon[109923]: cephadm [INF] [28/Aug/2023:02:15:03] ENGINE Serving on http://172.21.15.143:8765 2023-08-28T02:15:04.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:04 smithi143 conmon[109923]: cephadm 2023-08-28T02: 2023-08-28T02:15:04.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:04 smithi143 conmon[109923]: 15:03.505003+0000 mgr.y (mgr.15837) 2023-08-28T02:15:04.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:04 smithi143 conmon[109923]: 5 : cephadm [INF] [28/Aug/2023:02:15:03] ENGINE Bus STARTED 2023-08-28T02:15:05.487 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:05 smithi169 conmon[138400]: debug 2023-08-28T02:15:05.230+0000 7fb9a3e28200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-08-28T02:15:05.487 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:05 smithi169 conmon[138400]: debug 2023-08-28T02:15:05.329+0000 7fb9a3e28200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-08-28T02:15:05.828 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:05 smithi169 conmon[138400]: debug 2023-08-28T02:15:05.510+0000 7fb9a3e28200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-08-28T02:15:05.829 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:05 smithi169 conmon[100500]: cluster 2023-08-28T02:15:04.413231+0000 mgr.y (mgr.15837) 6 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:05.829 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:05 smithi169 conmon[100500]: audit 2023-08-28T02:15:05.353788+0000 mon.a (mon.0) 903 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:05.829 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:05 smithi169 conmon[100500]: audit 2023-08-28T02:15:05.360559+0000 mon.a (mon.0) 904 : audit 2023-08-28T02:15:05.829 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:05 smithi169 conmon[100500]: [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:05.829 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:05 smithi169 conmon[100500]: cluster 2023- 2023-08-28T02:15:05.829 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:05 smithi169 conmon[100500]: 08-28T02:15:05.416049+0000 mon.a ( 2023-08-28T02:15:05.829 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:05 smithi169 conmon[100500]: mon.0) 905 : cluster [DBG] mgrmap e27: y(active, since 3s) 2023-08-28T02:15:05.829 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:05 smithi169 conmon[100500]: 2023-08-28T02:15:05.829 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:05 smithi169 conmon[100500]: audit 2023- 2023-08-28T02:15:05.829 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:05 smithi169 conmon[100500]: 08-28T02:15:05.466774 2023-08-28T02:15:05.829 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:05 smithi169 conmon[100500]: +0000 mon.a (mon.0) 906 : audit 2023-08-28T02:15:05.830 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:05 smithi169 conmon[100500]: [DBG] from='client.? 172.21.15.143:0/867799329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:05.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:05 smithi143 conmon[100258]: cluster 2023-08-28T02:15:04.413231+0000 mgr.y (mgr.15837) 6 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:05.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:05 smithi143 conmon[100258]: audit 2023-08-28T02:15:05.353788+0000 mon.a (mon.0) 903 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:05.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:05 smithi143 conmon[100258]: audit 2023-08-28T02:15:05.360559+0000 mon.a (mon.0) 904 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:05.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:05 smithi143 conmon[100258]: cluster 2023-08-28T02:15:05.416049+0000 mon.a (mon.0) 905 : cluster [DBG] mgrmap e27: y(active, since 3s) 2023-08-28T02:15:05.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:05 smithi143 conmon[100258]: audit 2023-08-28T02:15:05.466774+0000 mon.a (mon.0) 906 : audit [DBG] from='client.? 172.21.15.143:0/867799329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:05.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:05 smithi143 conmon[109923]: cluster 2023-08-28T02:15:04.413231 2023-08-28T02:15:05.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:05 smithi143 conmon[109923]: +0000 mgr.y (mgr.15837) 6 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:05.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:05 smithi143 conmon[109923]: audit 2023-08-28T02:15:05.353788+0000 2023-08-28T02:15:05.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:05 smithi143 conmon[109923]: mon.a (mon.0) 903 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:05.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:05 smithi143 conmon[109923]: audit 2023-08-28T02:15:05.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:05 smithi143 conmon[109923]: 2023-08-28T02:15:05.360559+0000 2023-08-28T02:15:05.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:05 smithi143 conmon[109923]: mon.a (mon.0) 904 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:05.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:05 smithi143 conmon[109923]: cluster 2023-08-28T 2023-08-28T02:15:05.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:05 smithi143 conmon[109923]: 02:15:05.416049+0000 mon.a (mon.0) 905 2023-08-28T02:15:05.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:05 smithi143 conmon[109923]: : cluster [DBG] mgrmap e27: y(active, since 3s) 2023-08-28T02:15:05.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:05 smithi143 conmon[109923]: audit 2023-08-28 2023-08-28T02:15:05.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:05 smithi143 conmon[109923]: T02:15:05.466774+0000 mon.a (mon. 2023-08-28T02:15:05.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:05 smithi143 conmon[109923]: 0) 906 : audit [DBG] from='client.? 172.21.15.143:0/867799329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:06.082 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:05 smithi169 conmon[138400]: debug 2023-08-28T02:15:05.828+0000 7fb9a3e28200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-08-28T02:15:06.487 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:06 smithi169 conmon[138400]: debug 2023-08-28T02:15:06.082+0000 7fb9a3e28200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-08-28T02:15:06.487 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:06 smithi169 conmon[138400]: debug 2023-08-28T02:15:06.245+0000 7fb9a3e28200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-08-28T02:15:06.987 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:06 smithi169 conmon[138400]: debug 2023-08-28T02:15:06.534+0000 7fb9a3e28200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-08-28T02:15:07.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:07 smithi143 conmon[100258]: audit 2023-08-28T02:15:06.044505+0000 mon.a (mon.0) 907 : audit 2023-08-28T02:15:07.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:07 smithi143 conmon[100258]: [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:07.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:07 smithi143 conmon[100258]: audit 2023-08-28T02:15:06.049122+0000 mon.a (mon.0) 908 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:07.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:07 smithi143 conmon[100258]: audit 2023-08-28T02:15:06.114158+0000 mon.a (mon.0) 909 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:07.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:07 smithi143 conmon[100258]: audit 2023-08-28T02:15:06.120344+0000 mon.a (mon.0) 910 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:07.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:07 smithi143 conmon[100258]: audit 2023-08-28T02:15:06.804997+0000 mon.a (mon.0) 911 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:07.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:07 smithi143 conmon[100258]: audit 2023-08-28T02:15:06.809829+0000 mon.a (mon.0) 912 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:07.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:07 smithi143 conmon[109923]: audit 2023-08-28T02:15:06.044505+0000 mon.a (mon. 2023-08-28T02:15:07.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:07 smithi143 conmon[109923]: 0) 907 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:07.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:07 smithi143 conmon[109923]: audit 2023-08-28T02:15:06.049122+0000 mon.a 2023-08-28T02:15:07.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:07 smithi143 conmon[109923]: (mon.0) 908 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:07.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:07 smithi143 conmon[109923]: audit 2023-08-28T02: 2023-08-28T02:15:07.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:07 smithi143 conmon[109923]: 15:06.114158+0000 mon.a (mon.0) 909 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:07.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:07 smithi143 conmon[109923]: audit 2023-08-28T02:15:06.120344+0000 mon.a (mon. 2023-08-28T02:15:07.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:07 smithi143 conmon[109923]: 0) 910 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:07.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:07 smithi143 conmon[109923]: audit 2023-08-28T02:15: 2023-08-28T02:15:07.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:07 smithi143 conmon[109923]: 06.804997+0000 mon.a (mon.0) 911 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:07.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:07 smithi143 conmon[109923]: audit 2023-08-28T 2023-08-28T02:15:07.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:07 smithi143 conmon[109923]: 02:15:06.809829+0000 mon.a (mon.0) 912 : audit [INF] 2023-08-28T02:15:07.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:07 smithi143 conmon[109923]: from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:07.408 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:07 smithi169 conmon[138400]: debug 2023-08-28T02:15:07.276+0000 7fb9a3e28200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-08-28T02:15:07.409 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:07 smithi169 conmon[100500]: audit 2023-08-28T02:15:06.044505+0000 mon.a (mon.0) 907 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:07.409 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:07 smithi169 conmon[100500]: audit 2023-08-28T02:15:06. 2023-08-28T02:15:07.409 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:07 smithi169 conmon[100500]: 049122+0000 mon.a (mon.0) 908 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:07.409 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:07 smithi169 conmon[100500]: audit 2023-08-28T02:15:06.114158+0000 mon.a (mon.0) 909 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:07.409 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:07 smithi169 conmon[100500]: audit 2023-08-28T02:15:06.120344+0000 mon.a (mon.0 2023-08-28T02:15:07.409 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:07 smithi169 conmon[100500]: ) 910 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:07.409 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:07 smithi169 conmon[100500]: audit 2023-08-28T02 2023-08-28T02:15:07.409 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:07 smithi169 conmon[100500]: :15:06.804997+0000 mon.a (mon.0) 911 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:07.409 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:07 smithi169 conmon[100500]: 2023-08-28T02:15:07.410 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:07 smithi169 conmon[100500]: audit 2023-08-28T02:15:07.410 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:07 smithi169 conmon[100500]: 2023-08-28T02:15:06.809829+0000 mon.a (mon 2023-08-28T02:15:07.410 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:07 smithi169 conmon[100500]: .0) 912 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:07.728 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:07 smithi169 conmon[138400]: debug 2023-08-28T02:15:07.408+0000 7fb9a3e28200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-08-28T02:15:07.728 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:07 smithi169 conmon[138400]: debug 2023-08-28T02:15:07.607+0000 7fb9a3e28200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-08-28T02:15:07.987 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:07 smithi169 conmon[138400]: debug 2023-08-28T02:15:07.728+0000 7fb9a3e28200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-08-28T02:15:07.987 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:07 smithi169 conmon[138400]: debug 2023-08-28T02:15:07.954+0000 7fb9a3e28200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-08-28T02:15:08.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:08 smithi143 conmon[100258]: cluster 2023-08-28T02:15:06.413568+0000 mgr.y (mgr.15837) 7 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:08.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:08 smithi143 conmon[100258]: audit 2023-08-28T02:15:07.859145+0000 mon.c (mon.2) 83 : audit [DBG] from='client.? 172.21.15.143:0/1016592928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:08.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:08 smithi143 conmon[109923]: cluster 2023-08-28T02:15:06.413568+0000 mgr.y (mgr.15837) 7 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:08.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:08 smithi143 conmon[109923]: audit 2023-08-28T02:15:07.859145+0000 mon.c (mon.2) 83 : audit [DBG] from='client.? 172.21.15.143:0/1016592928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:08.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:08 smithi169 conmon[100500]: cluster 2023-08-28T02:15:06.413568+0000 mgr.y (mgr.15837) 7 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:08.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:08 smithi169 conmon[100500]: audit 2023-08-28T02:15:07.859145+0000 mon.c (mon.2) 83 : audit [DBG] from='client.? 172.21.15.143:0/1016592928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:09.237 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:08 smithi169 conmon[138400]: debug 2023-08-28T02:15:08.836+0000 7fb9a3e28200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-08-28T02:15:09.237 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:08 smithi169 conmon[138400]: [28/Aug/2023:02:15:08] ENGINE Bus STARTING 2023-08-28T02:15:09.237 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:08 smithi169 conmon[138400]: CherryPy Checker: 2023-08-28T02:15:09.237 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:08 smithi169 conmon[138400]: The Application mounted at '' has an empty config. 2023-08-28T02:15:09.237 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:08 smithi169 conmon[138400]: 2023-08-28T02:15:09.238 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:08 smithi169 conmon[138400]: [28/Aug/2023:02:15:08] ENGINE Serving on http://:::9283 2023-08-28T02:15:09.238 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:08 smithi169 conmon[138400]: [28/Aug/2023:02:15:08] ENGINE Bus STARTED 2023-08-28T02:15:09.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:09 smithi169 conmon[100500]: cluster 2023-08-28T02:15:08.413980+0000 mgr.y (mgr.15837) 8 2023-08-28T02:15:09.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:09 smithi169 conmon[100500]: : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:09.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:09 smithi169 conmon[100500]: audit 2023-08-28T02:15:08.728940+0000 mon.a (mon.0) 913 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:09.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:09 smithi169 conmon[100500]: audit 2023-08-28T02:15:08.733572+0000 2023-08-28T02:15:09.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:09 smithi169 conmon[100500]: mon.a (mon.0) 914 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:09.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:09 smithi169 conmon[100500]: audit 2023- 2023-08-28T02:15:09.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:09 smithi169 conmon[100500]: 08-28T02:15:08.735242+0000 mon.c 2023-08-28T02:15:09.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:09 smithi169 conmon[100500]: (mon.2) 84 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:15:09.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:09 smithi169 conmon[100500]: audit 2023-08-28T02:15:08. 2023-08-28T02:15:09.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:09 smithi169 conmon[100500]: 736089+0000 mon.c (mon.2) 85 : audit [INF] 2023-08-28T02:15:09.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:09 smithi169 conmon[100500]: from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:15:09.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:09 smithi169 conmon[100500]: cephadm 2023-08-28T02:15:09.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:09 smithi169 conmon[100500]: 2023-08-28T02:15:08.737027+0000 mgr.y (mgr 2023-08-28T02:15:09.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:09 smithi169 conmon[100500]: .15837) 9 : cephadm [INF] Updating smithi143:/etc/ceph/ceph.conf 2023-08-28T02:15:09.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:09 smithi169 conmon[100500]: cephadm 2023-08-28 2023-08-28T02:15:09.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:09 smithi169 conmon[100500]: T02:15:08.737185+0000 mgr.y ( 2023-08-28T02:15:09.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:09 smithi169 conmon[100500]: mgr.15837) 10 : cephadm [INF] Updating smithi169:/etc/ceph/ceph.conf 2023-08-28T02:15:09.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:09 smithi169 conmon[100500]: audit 2023- 2023-08-28T02:15:09.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:09 smithi169 conmon[100500]: 08-28T02:15:08.853222+0000 2023-08-28T02:15:09.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:09 smithi169 conmon[100500]: mon.b (mon.1) 100 : audit [DBG] from='mgr.? 172.21.15.169:0/4082786681' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-08-28T02:15:09.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:09 smithi169 conmon[100500]: audit 2023- 2023-08-28T02:15:09.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:09 smithi169 conmon[100500]: 08-28T02:15:08.854104+0000 2023-08-28T02:15:09.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:09 smithi169 conmon[100500]: mon.b (mon.1) 101 : audit [DBG] from='mgr.? 172.21.15.169:0/4082786681' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-28T02:15:09.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:09 smithi169 conmon[100500]: cluster 2023-08 2023-08-28T02:15:09.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:09 smithi169 conmon[100500]: -28T02:15:08.854106+0000 mon.a (mon 2023-08-28T02:15:09.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:09 smithi169 conmon[100500]: .0) 915 : cluster [DBG] Standby manager daemon x started 2023-08-28T02:15:09.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:09 smithi169 conmon[100500]: audit 2023-08- 2023-08-28T02:15:09.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:09 smithi169 conmon[100500]: 28T02:15:08.856488+0000 mon.b (mon.1) 102 2023-08-28T02:15:09.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:09 smithi169 conmon[100500]: : audit [DBG] from='mgr.? 172.21.15.169:0/4082786681' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-08-28T02:15:09.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:09 smithi169 conmon[100500]: audit 2023- 2023-08-28T02:15:09.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:09 smithi169 conmon[100500]: 08-28T02:15:08.856882+0000 mon.b (mon.1) 103 : audit [DBG] 2023-08-28T02:15:09.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:09 smithi169 conmon[100500]: from='mgr.? 172.21.15.169:0/4082786681' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-08-28T02:15:09.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:09 smithi169 conmon[100500]: cephadm 2023-08-28T02:15:09.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:09 smithi169 conmon[100500]: 2023-08-28T02:15:09.001372+0000 mgr.y (mgr 2023-08-28T02:15:09.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:09 smithi169 conmon[100500]: .15837) 11 : cephadm [INF] Updating smithi169:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.conf 2023-08-28T02:15:09.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:09 smithi169 conmon[100500]: cephadm 2023-08-28 2023-08-28T02:15:09.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:09 smithi169 conmon[100500]: T02:15:09.001524+0000 mgr.y (mgr 2023-08-28T02:15:09.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:09 smithi169 conmon[100500]: .15837) 12 : cephadm [INF] Updating smithi143:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.conf 2023-08-28T02:15:10.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[109923]: cluster 2023-08-28T02:15:08.413980+0000 2023-08-28T02:15:10.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[109923]: mgr.y (mgr.15837) 8 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:10.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[109923]: audit 2023-08-28T02:15:08.728940+0000 mon.a (mon.0) 913 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:10.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[109923]: audit 2023-08-28T02:15:08.733572+0000 mon.a (mon.0) 914 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:10.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[109923]: audit 2023-08-28T02:15:08.735242+0000 mon.c (mon.2) 84 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:15:10.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[109923]: audit 2023-08-28T02:15:08.736089+0000 2023-08-28T02:15:10.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[109923]: mon.c (mon.2) 85 : audit [INF] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:15:10.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[109923]: cephadm 2023-08-28T02:15:08.737027+0000 mgr.y (mgr.15837) 9 : 2023-08-28T02:15:10.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[109923]: cephadm [INF] Updating smithi143:/etc/ceph/ceph.conf 2023-08-28T02:15:10.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[109923]: cephadm 2023-08-28T 2023-08-28T02:15:10.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[109923]: 02:15:08.737185+0000 mgr.y (mgr.15837) 10 : cephadm [INF] Updating smithi169:/etc/ceph/ceph.conf 2023-08-28T02:15:10.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[109923]: audit 2023-08- 2023-08-28T02:15:10.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[109923]: 28T02:15:08.853222+0000 mon.b 2023-08-28T02:15:10.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[109923]: (mon.1) 100 : audit [DBG] from='mgr.? 172.21.15.169:0/4082786681' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-08-28T02:15:10.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[109923]: audit 2023-08-28 2023-08-28T02:15:10.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[109923]: T02:15:08.854104+0000 mon.b (mon.1 2023-08-28T02:15:10.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[109923]: ) 101 : audit [DBG] from='mgr.? 172.21.15.169:0/4082786681' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-28T02:15:10.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[109923]: cluster 2023-08 2023-08-28T02:15:10.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[109923]: -28T02:15:08.854106+0000 mon.a ( 2023-08-28T02:15:10.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[109923]: mon.0) 915 : cluster [DBG] Standby manager daemon x started 2023-08-28T02:15:10.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[109923]: audit 2023-08-28T02 2023-08-28T02:15:10.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[109923]: :15:08.856488+0000 mon.b (mon.1) 2023-08-28T02:15:10.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[109923]: 102 : audit [DBG] from='mgr.? 172.21.15.169:0/4082786681' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-08-28T02:15:10.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[109923]: audit 2023-08-28T02:15: 2023-08-28T02:15:10.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[109923]: 08.856882+0000 mon.b (mon.1) 103 : audit [DBG] 2023-08-28T02:15:10.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[109923]: from='mgr.? 172.21.15.169:0/4082786681' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-08-28T02:15:10.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[109923]: cephadm 2023-08-28T02:15:10.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[109923]: 2023-08-28T02:15:09.001372+0000 mgr.y (mgr 2023-08-28T02:15:10.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[109923]: .15837) 11 : cephadm [INF] Updating smithi169:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.conf 2023-08-28T02:15:10.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[109923]: cephadm 2023-08 2023-08-28T02:15:10.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[109923]: -28T02:15:09.001524+0000 2023-08-28T02:15:10.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[109923]: mgr.y (mgr.15837) 12 : cephadm [INF] 2023-08-28T02:15:10.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[109923]: Updating smithi143:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.conf 2023-08-28T02:15:10.204 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[100258]: cluster 2023-08-28T02:15: 2023-08-28T02:15:10.204 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[100258]: 08.413980+0000 mgr.y (mgr.15837) 8 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:10.204 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[100258]: audit 2023-08-28T02:15:08.728940+0000 mon.a (mon.0) 913 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:10.204 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[100258]: audit 2023-08-28T02:15:08.733572+0000 mon.a (mon.0) 914 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:10.204 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[100258]: audit 2023-08-28T02:15:08.735242+0000 mon.c (mon.2) 84 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:15:10.204 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[100258]: audit 2023-08-28T02:15:08.736089+0000 mon.c (mon.2) 85 : audit 2023-08-28T02:15:10.204 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[100258]: [INF] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:15:10.204 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[100258]: cephadm 2023-08-28T02:15:10.204 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[100258]: 2023-08-28T02:15 2023-08-28T02:15:10.205 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[100258]: :08.737027+0000 mgr.y (mgr.15837) 2023-08-28T02:15:10.205 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[100258]: 9 : cephadm [INF] Updating smithi143:/etc/ceph/ceph.conf 2023-08-28T02:15:10.205 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[100258]: cephadm 2023-08-28T02: 2023-08-28T02:15:10.205 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[100258]: 15:08.737185+0000 mgr.y (mgr.15837) 10 2023-08-28T02:15:10.205 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[100258]: : cephadm [INF] Updating smithi169:/etc/ceph/ceph.conf 2023-08-28T02:15:10.205 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[100258]: audit 2023-08-28T02:15:10.205 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[100258]: 2023-08-28T02:15:08.853222 2023-08-28T02:15:10.205 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[100258]: +0000 mon.b (mon.1) 100 : audit [DBG] 2023-08-28T02:15:10.205 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[100258]: from='mgr.? 172.21.15.169:0/4082786681' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-08-28T02:15:10.205 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[100258]: audit 2023- 2023-08-28T02:15:10.205 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[100258]: 08-28T02:15:08.854104 2023-08-28T02:15:10.205 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[100258]: +0000 mon.b (mon.1) 101 : audit 2023-08-28T02:15:10.206 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[100258]: [DBG] from='mgr.? 172.21.15.169:0/4082786681' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-28T02:15:10.206 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[100258]: cluster 2023 2023-08-28T02:15:10.206 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[100258]: -08-28T02:15:08.854106 2023-08-28T02:15:10.206 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[100258]: +0000 mon.a (mon.0) 915 : cluster [DBG] 2023-08-28T02:15:10.206 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[100258]: Standby manager daemon x started 2023-08-28T02:15:10.206 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[100258]: audit 2023-08-28T 2023-08-28T02:15:10.206 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[100258]: 02:15:08.856488+0000 mon.b (mon. 2023-08-28T02:15:10.206 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[100258]: 1) 102 : audit [DBG] from='mgr.? 172.21.15.169:0/4082786681' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-08-28T02:15:10.206 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[100258]: audit 2023-08-28T02:15:10.206 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[100258]: 2023-08- 2023-08-28T02:15:10.206 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[100258]: 28T02:15:08.856882+0000 2023-08-28T02:15:10.207 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[100258]: mon.b (mon.1) 103 : audit [DBG] 2023-08-28T02:15:10.207 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[100258]: from='mgr.? 172.21.15.169:0/4082786681' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-08-28T02:15:10.207 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[100258]: cephadm 2023-08- 2023-08-28T02:15:10.207 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[100258]: 28T02:15:09.001372+0000 mgr.y 2023-08-28T02:15:10.207 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[100258]: (mgr.15837) 11 : cephadm [INF] Updating smithi169:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.conf 2023-08-28T02:15:10.207 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[100258]: cephadm 2023- 2023-08-28T02:15:10.207 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[100258]: 08-28T02:15:09.001524+0000 mgr.y 2023-08-28T02:15:10.207 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:09 smithi143 conmon[100258]: (mgr.15837) 12 : cephadm [INF] Updating smithi143:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.conf 2023-08-28T02:15:10.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:10 smithi169 conmon[100500]: cephadm 2023-08-28T02:15:09.266209+0000 mgr.y (mgr.15837) 2023-08-28T02:15:10.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:10 smithi169 conmon[100500]: 13 : cephadm [INF] Updating smithi169:/etc/ceph/ceph.client.admin.keyring 2023-08-28T02:15:10.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:10 smithi169 conmon[100500]: cephadm 2023-08-28T02:15:09.266438+0000 mgr.y (mgr.15837) 14 : cephadm [INF] Updating smithi143:/etc/ceph/ceph.client.admin.keyring 2023-08-28T02:15:10.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:10 smithi169 conmon[100500]: cephadm 2023-08-28T02:15:09.530679+0000 mgr.y (mgr.15837) 15 : cephadm [INF] Updating smithi169:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.client.admin.keyring 2023-08-28T02:15:10.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:10 smithi169 conmon[100500]: cephadm 2023-08-28T02:15:09.531167+0000 mgr.y (mgr.15837) 16 : cephadm [INF] Updating smithi143:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.client.admin.keyring 2023-08-28T02:15:10.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:10 smithi169 conmon[100500]: cluster 2023-08-28T02:15:09.740469+0000 mon.a (mon.0) 916 : cluster [DBG] mgrmap e28: y(active, since 7s), standbys: x 2023-08-28T02:15:10.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:10 smithi169 conmon[100500]: audit 2023-08-28T02:15:09.740853+0000 mon.c (mon.2) 86 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-08-28T02:15:10.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:10 smithi169 conmon[100500]: audit 2023-08-28T02:15:09.800541+0000 mon.a (mon.0) 917 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:10.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:10 smithi169 conmon[100500]: audit 2023-08-28T02:15:09.806128+0000 mon.a (mon.0) 918 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:10.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:10 smithi169 conmon[100500]: audit 2023-08-28T02:15:09.811815+0000 mon.a (mon.0) 919 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:10.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:10 smithi169 conmon[100500]: audit 2023-08-28T02:15:09.816793+0000 mon.a (mon.0) 920 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:10.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:10 smithi169 conmon[100500]: audit 2023-08-28T02:15:09. 2023-08-28T02:15:10.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:10 smithi169 conmon[100500]: 821812+0000 mon.a (mon.0) 921 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:10.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:10 smithi169 conmon[100500]: cephadm 2023-08-28T02:15:09.889252+0000 mgr.y (mgr. 2023-08-28T02:15:10.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:10 smithi169 conmon[100500]: 15837) 17 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-08-28T02:15:10.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:10 smithi169 conmon[100500]: cephadm 2023-08 2023-08-28T02:15:10.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:10 smithi169 conmon[100500]: -28T02:15:10.121109 2023-08-28T02:15:10.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:10 smithi169 conmon[100500]: +0000 mgr.y (mgr.15837) 18 2023-08-28T02:15:10.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:10 smithi169 conmon[100500]: : cephadm [INF] Reconfiguring daemon prometheus.a on smithi169 2023-08-28T02:15:10.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:10 smithi169 conmon[100500]: audit 2023 2023-08-28T02:15:10.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:10 smithi169 conmon[100500]: -08-28T02:15:10.240126 2023-08-28T02:15:10.991 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:10 smithi169 conmon[100500]: +0000 mon.c (mon.2) 87 2023-08-28T02:15:10.991 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:10 smithi169 conmon[100500]: : audit [DBG] from='client.? 172.21.15.143:0/1615636507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:11.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[100258]: cephadm 2023-08-28T02:15:09.266209+0000 mgr.y (mgr.15837 2023-08-28T02:15:11.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[100258]: ) 13 : cephadm [INF] Updating smithi169:/etc/ceph/ceph.client.admin.keyring 2023-08-28T02:15:11.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[100258]: cephadm 2023-08-28T02:15:09.266438+0000 mgr.y (mgr.15837) 14 : cephadm [INF] Updating smithi143:/etc/ceph/ceph.client.admin.keyring 2023-08-28T02:15:11.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[100258]: cephadm 2023-08 2023-08-28T02:15:11.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[100258]: -28T02:15:09.530679+0000 mgr.y (mgr.15837) 15 : cephadm [INF] Updating smithi169:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.client.admin.keyring 2023-08-28T02:15:11.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[100258]: cephadm 2023-08-28T 2023-08-28T02:15:11.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[100258]: 02:15:09.531167+0000 mgr.y (mgr.15837) 16 : cephadm [INF] Updating smithi143:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.client.admin.keyring 2023-08-28T02:15:11.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[100258]: cluster 2023-08-28T02:15:09.740469+0000 2023-08-28T02:15:11.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[100258]: mon.a (mon.0) 916 : cluster [DBG] mgrmap e28: y(active, since 7s), standbys: x 2023-08-28T02:15:11.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[100258]: audit 2023-08-28T02:15:09 2023-08-28T02:15:11.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[100258]: .740853+0000 mon.c (mon.2) 86 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-08-28T02:15:11.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[100258]: audit 2023-08-28T02:15:09.800541+0000 mon.a (mon. 2023-08-28T02:15:11.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[100258]: 0) 917 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:11.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[100258]: audit 2023-08-28T02:15:11.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[100258]: 2023-08-28T02:15:09.806128+0000 mon.a (mon.0) 918 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:11.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[100258]: audit 2023-08-28T02:15 2023-08-28T02:15:11.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[100258]: :09.811815+0000 mon.a (mon.0) 919 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:11.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[100258]: audit 2023-08-28T02:15:09. 2023-08-28T02:15:11.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[100258]: 816793+0000 mon.a (mon.0) 920 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:11.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[100258]: audit 2023-08-28T02:15:09.821812 2023-08-28T02:15:11.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[100258]: +0000 mon.a (mon.0) 921 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:11.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[100258]: cephadm 2023-08-28T02:15:09 2023-08-28T02:15:11.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[100258]: .889252+0000 mgr.y (mgr.15837) 17 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-08-28T02:15:11.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[100258]: cephadm 2023-08-28T02:15: 2023-08-28T02:15:11.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[100258]: 10.121109+0000 mgr.y (mgr.15837) 18 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi169 2023-08-28T02:15:11.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[100258]: audit 2023-08-28T02:15: 2023-08-28T02:15:11.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[100258]: 10.240126+0000 mon.c (mon.2) 87 : audit [DBG] from='client.? 172.21.15.143:0/1615636507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:11.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[109923]: cephadm 2023-08-28T02:15:09.266209+0000 2023-08-28T02:15:11.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[109923]: mgr.y (mgr.15837) 13 : cephadm [INF] Updating smithi169:/etc/ceph/ceph.client.admin.keyring 2023-08-28T02:15:11.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[109923]: cephadm 2023-08-28T02:15:09.266438+0000 mgr.y (mgr. 2023-08-28T02:15:11.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[109923]: 15837) 14 : cephadm [INF] Updating smithi143:/etc/ceph/ceph.client.admin.keyring 2023-08-28T02:15:11.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[109923]: cephadm 2023 2023-08-28T02:15:11.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[109923]: -08-28T02:15:09.530679+0000 mgr.y 2023-08-28T02:15:11.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[109923]: (mgr.15837) 15 : cephadm [INF] Updating smithi169:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.client.admin.keyring 2023-08-28T02:15:11.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[109923]: cephadm 2023-08 2023-08-28T02:15:11.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[109923]: -28T02:15:09.531167+0000 mgr.y (mgr.15837) 16 : cephadm 2023-08-28T02:15:11.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[109923]: [INF] Updating smithi143:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.client.admin.keyring 2023-08-28T02:15:11.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[109923]: cluster 2023-08-28T02:15:11.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[109923]: 2023-08-28T02:15:09.740469+0000 mon.a (mon.0) 916 : cluster [DBG] mgrmap e28: y(active, since 7s), standbys: x 2023-08-28T02:15:11.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[109923]: audit 2023-08-28T02 2023-08-28T02:15:11.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[109923]: :15:09.740853+0000 mon.c (mon.2) 86 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-08-28T02:15:11.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[109923]: audit 2023-08-28T02:15:09.800541+0000 2023-08-28T02:15:11.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[109923]: mon.a (mon.0) 917 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:11.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[109923]: audit 2023-08-28T02:15:09. 2023-08-28T02:15:11.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[109923]: 806128+0000 mon.a (mon.0) 918 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:11.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[109923]: audit 2023-08-28T02:15 2023-08-28T02:15:11.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[109923]: :09.811815+0000 mon.a (mon.0) 919 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:11.205 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[109923]: audit 2023-08-28 2023-08-28T02:15:11.205 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[109923]: T02:15:09.816793+0000 mon.a (mon.0) 920 : audit [INF] 2023-08-28T02:15:11.205 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[109923]: from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:11.205 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[109923]: audit 2023-08 2023-08-28T02:15:11.205 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[109923]: -28T02:15:09.821812+0000 mon.a (mon.0) 921 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:11.205 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[109923]: cephadm 2023-08-28T02:15 2023-08-28T02:15:11.205 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[109923]: :09.889252+0000 mgr.y (mgr.15837) 17 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-08-28T02:15:11.205 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[109923]: cephadm 2023-08-28T02 2023-08-28T02:15:11.206 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[109923]: :15:10.121109+0000 mgr.y (mgr.15837) 18 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi169 2023-08-28T02:15:11.206 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[109923]: audit 2023-08- 2023-08-28T02:15:11.206 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[109923]: 28T02:15:10.240126+0000 mon.c (mon.2) 87 : audit [DBG] 2023-08-28T02:15:11.206 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:10 smithi143 conmon[109923]: from='client.? 172.21.15.143:0/1615636507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:11.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:11 smithi169 conmon[100500]: cluster 2023-08-28T02:15:10.414851+0000 mgr.y (mgr. 2023-08-28T02:15:11.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:11 smithi169 conmon[100500]: 15837) 19 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:12.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:11 smithi143 conmon[100258]: cluster 2023-08-28T02:15:10.414851+0000 mgr.y (mgr.15837) 19 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:12.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:11 smithi143 conmon[109923]: cluster 2023-08-28T02:15:10.414851+0000 mgr.y (mgr.15837) 19 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:12.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:12 smithi169 conmon[100500]: audit 2023-08-28T02:15:12.622692+0000 mon.c (mon.2) 88 : audit [DBG] from='client.? 172.21.15.143:0/4249179887' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:13.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:12 smithi143 conmon[100258]: audit 2023-08-28T02:15:12.622692+0000 mon.c ( 2023-08-28T02:15:13.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:12 smithi143 conmon[100258]: mon.2) 88 : audit [DBG] from='client.? 172.21.15.143:0/4249179887' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:13.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:12 smithi143 conmon[109923]: audit 2023-08-28T02:15:12.622692+0000 2023-08-28T02:15:13.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:12 smithi143 conmon[109923]: mon.c (mon.2) 88 : audit [DBG] from='client.? 172.21.15.143:0/4249179887' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:13.337 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:15:13 smithi169 systemd[1]: Stopping Ceph prometheus.a for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:15:13.337 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:15:13 smithi169 bash[149640]: Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-prometheus.a found: no such container 2023-08-28T02:15:13.737 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:15:13 smithi169 conmon[144179]: ts=2023-08-28T02:15:13.338Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-08-28T02:15:13.737 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:15:13 smithi169 conmon[144179]: ts=2023-08-28T02:15:13.338Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-08-28T02:15:13.737 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:15:13 smithi169 conmon[144179]: ts=2023-08-28T02:15:13.338Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-08-28T02:15:13.737 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:15:13 smithi169 conmon[144179]: ts=2023-08-28T02:15:13.338Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-08-28T02:15:13.737 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:15:13 smithi169 conmon[144179]: ts=2023-08-28T02:15:13.338Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-08-28T02:15:13.737 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:15:13 smithi169 conmon[144179]: ts=2023-08-28T02:15:13.338Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-08-28T02:15:13.737 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:15:13 smithi169 conmon[144179]: ts=2023-08-28T02:15:13.339Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-08-28T02:15:13.738 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:15:13 smithi169 conmon[144179]: ts=2023-08-28T02:15:13.339Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-08-28T02:15:13.738 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:15:13 smithi169 conmon[144179]: ts=2023-08-28T02:15:13.339Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-08-28T02:15:13.738 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:15:13 smithi169 conmon[144179]: ts=2023-08-28T02:15:13.339Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-08-28T02:15:13.738 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:15:13 smithi169 conmon[144179]: ts=2023-08-28T02:15:13.339Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-08-28T02:15:13.738 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:15:13 smithi169 conmon[144179]: ts=2023-08-28T02:15:13.339Z caller=main.go:1066 level=info msg="See you next time!" 2023-08-28T02:15:14.042 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:13 smithi169 conmon[100500]: cluster 2023-08-28T02:15:12.415285+0000 mgr.y (mgr.15837) 20 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:14.042 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:15:13 smithi169 bash[149755]: b370b47c85c80d837f76262930915f003371f5a22085d0e9235ca3fd913e35b1 2023-08-28T02:15:14.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:13 smithi143 conmon[109923]: cluster 2023-08-28T02:15:12.415285+0000 mgr.y (mgr.15837) 20 : cluster 2023-08-28T02:15:14.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:13 smithi143 conmon[109923]: [DBG] pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:14.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:13 smithi143 conmon[100258]: cluster 2023-08-28T02:15:12.415285+0000 mgr.y (mgr.15837) 20 : cluster [DBG] 2023-08-28T02:15:14.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:13 smithi143 conmon[100258]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:14.487 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:15:14 smithi169 systemd[1]: ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@prometheus.a.service: Succeeded. 2023-08-28T02:15:14.487 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:15:14 smithi169 systemd[1]: Stopped Ceph prometheus.a for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:15:14.487 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:15:14 smithi169 systemd[1]: Starting Ceph prometheus.a for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:15:14.808 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:15:14 smithi169 conmon[150127]: ts=2023-08-28T02:15:14.648Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-08-28T02:15:14.808 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:15:14 smithi169 conmon[150127]: ts=2023-08-28T02:15:14.648Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-08-28T02:15:14.808 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:15:14 smithi169 conmon[150127]: ts=2023-08-28T02:15:14.648Z caller=main.go:518 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi169 (none))" 2023-08-28T02:15:14.808 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:15:14 smithi169 conmon[150127]: ts=2023-08-28T02:15:14.648Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-08-28T02:15:14.808 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:15:14 smithi169 conmon[150127]: ts=2023-08-28T02:15:14.648Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-08-28T02:15:14.808 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:15:14 smithi169 conmon[150127]: ts=2023-08-28T02:15:14.650Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-08-28T02:15:14.809 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:15:14 smithi169 conmon[150127]: ts=2023-08-28T02:15:14.651Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-08-28T02:15:14.809 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:15:14 smithi169 conmon[150127]: ts=2023-08-28T02:15:14.651Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-08-28T02:15:14.809 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:15:14 smithi169 conmon[150127]: ts=2023-08-28T02:15:14.654Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-08-28T02:15:14.809 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:15:14 smithi169 conmon[150127]: ts=2023-08-28T02:15:14.654Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.901µs 2023-08-28T02:15:14.809 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:15:14 smithi169 conmon[150127]: ts=2023-08-28T02:15:14.654Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-08-28T02:15:14.809 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:15:14 smithi169 bash[150086]: cd763ce7d28253c1b0f877b1e2939cdba212a4dab10715b46eca8942effd1eeb 2023-08-28T02:15:14.809 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:15:14 smithi169 systemd[1]: Started Ceph prometheus.a for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:15:14.809 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:15:14 smithi169 conmon[150127]: ts=2023-08-28T02:15:14.809Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2023-08-28T02:15:15.062 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:15:15 smithi169 conmon[150127]: ts=2023-08-28T02:15:15.030Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2023-08-28T02:15:15.063 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:15:15 smithi169 conmon[150127]: ts=2023-08-28T02:15:15.063Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2023-08-28T02:15:15.063 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:15:15 smithi169 conmon[150127]: ts=2023-08-28T02:15:15.063Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2023-08-28T02:15:15.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:14 smithi143 conmon[176053]: [28/Aug/2023:02:15:14] ENGINE Bus STOPPING 2023-08-28T02:15:15.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:14 smithi143 conmon[176053]: [28/Aug/2023:02:15:14] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-08-28T02:15:15.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:14 smithi143 conmon[176053]: [28/Aug/2023:02:15:14] ENGINE Bus STOPPED 2023-08-28T02:15:15.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:14 smithi143 conmon[176053]: [28/Aug/2023:02:15:14] ENGINE Bus STARTING 2023-08-28T02:15:15.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:14 smithi143 conmon[176053]: [28/Aug/2023:02:15:14] ENGINE Serving on http://:::9283 2023-08-28T02:15:15.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:14 smithi143 conmon[176053]: [28/Aug/2023:02:15:14] ENGINE Bus STARTED 2023-08-28T02:15:15.323 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:15:15 smithi169 conmon[150127]: ts=2023-08-28T02:15:15.063Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=35.684µs wal_replay_duration=409.423946ms total_replay_duration=409.473299ms 2023-08-28T02:15:15.323 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:15:15 smithi169 conmon[150127]: ts=2023-08-28T02:15:15.067Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-08-28T02:15:15.323 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:15:15 smithi169 conmon[150127]: ts=2023-08-28T02:15:15.067Z caller=main.go:947 level=info msg="TSDB started" 2023-08-28T02:15:15.323 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:15:15 smithi169 conmon[150127]: ts=2023-08-28T02:15:15.067Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-08-28T02:15:15.323 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:15:15 smithi169 conmon[150127]: ts=2023-08-28T02:15:15.086Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=18.737915ms db_storage=703ns remote_storage=1.286µs web_handler=400ns query_engine=680ns scrape=520.082µs scrape_sd=94.451µs notify=27.109µs notify_sd=16.585µs rules=17.801665ms 2023-08-28T02:15:15.324 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:15:15 smithi169 conmon[150127]: ts=2023-08-28T02:15:15.086Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-08-28T02:15:15.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[109923]: cluster 2023-08-28T02:15:14.415886+0000 mgr.y 2023-08-28T02:15:15.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[109923]: (mgr.15837) 21 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:15.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[109923]: audit 2023-08-28T02:15:14.697919+0000 mon.a (mon 2023-08-28T02:15:15.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[109923]: .0) 922 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:15.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[109923]: audit 2023-08-28T02:15:15.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[109923]: 2023-08-28T02:15:14.705368+0000 mon.a (mon.0) 923 : audit 2023-08-28T02:15:15.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[109923]: [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:15.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[109923]: audit 2023-08-28T 2023-08-28T02:15:15.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[109923]: 02:15:14.752585+0000 mon.c (mon.2) 89 2023-08-28T02:15:15.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[109923]: : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-08-28T02:15:15.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[109923]: audit 2023-08-28 2023-08-28T02:15:15.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[109923]: T02:15:14.753066+0000 mgr.y (mgr.15837) 22 2023-08-28T02:15:15.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[109923]: : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-08-28T02:15:15.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[109923]: audit 2023-08-28 2023-08-28T02:15:15.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[109923]: T02:15:14.753974+0000 mon.c (mon.2) 90 2023-08-28T02:15:15.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[109923]: : audit [INF] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi169.front.sepia.ceph.com:9095"}]: dispatch 2023-08-28T02:15:15.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[109923]: audit 2023-08-28T02:15 2023-08-28T02:15:15.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[109923]: :14.754303+0000 mgr.y (mgr.15837) 23 : audit 2023-08-28T02:15:15.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[109923]: [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi169.front.sepia.ceph.com:9095"}]: dispatch 2023-08-28T02:15:15.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[109923]: audit 2023-08-28T02:15:14 2023-08-28T02:15:15.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[109923]: .758738+0000 mon.a (mon.0) 924 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:15.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[109923]: audit 2023-08-28T02: 2023-08-28T02:15:15.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[109923]: 15:14.773290+0000 mon.c (mon.2) 91 : audit 2023-08-28T02:15:15.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[109923]: [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:15:15.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[109923]: audit 2023-08-28T02:15:15.013709 2023-08-28T02:15:15.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[109923]: +0000 mon.c (mon.2) 92 : audit [DBG] from='client.? 172.21.15.143:0/3304576593' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:15.953 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[100258]: cluster 2023-08-28T02:15:14.415886+0000 mgr.y (mgr 2023-08-28T02:15:15.953 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[100258]: .15837) 21 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:15.953 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[100258]: audit 2023-08-28T02:15:14.697919+0000 2023-08-28T02:15:15.953 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[100258]: mon.a (mon.0) 922 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:15.953 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[100258]: audit 2023-08-28T 2023-08-28T02:15:15.953 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[100258]: 02:15:14.705368+0000 mon.a (mon.0) 923 : audit [INF] 2023-08-28T02:15:15.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[100258]: from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:15.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[100258]: audit 2023-08-28T02: 2023-08-28T02:15:15.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[100258]: 15:14.752585+0000 mon.c (mon.2) 2023-08-28T02:15:15.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[100258]: 89 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-08-28T02:15:15.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[100258]: audit 2023- 2023-08-28T02:15:15.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[100258]: 08-28T02:15:14.753066+0000 mgr.y 2023-08-28T02:15:15.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[100258]: (mgr.15837) 22 : audit [DBG] 2023-08-28T02:15:15.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[100258]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-08-28T02:15:15.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[100258]: audit 2023- 2023-08-28T02:15:15.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[100258]: 08-28T02:15:14.753974+0000 2023-08-28T02:15:15.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[100258]: mon.c (mon.2) 90 : audit [INF] 2023-08-28T02:15:15.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[100258]: from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi169.front.sepia.ceph.com:9095"}]: dispatch 2023-08-28T02:15:15.955 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[100258]: audit 2023 2023-08-28T02:15:15.955 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[100258]: -08-28T02:15:14.754303 2023-08-28T02:15:15.955 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[100258]: +0000 mgr.y (mgr.15837) 23 : 2023-08-28T02:15:15.955 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[100258]: audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi169.front.sepia.ceph.com:9095"}]: dispatch 2023-08-28T02:15:15.955 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[100258]: audit 2023- 2023-08-28T02:15:15.955 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[100258]: 08-28T02:15:14.758738+0000 2023-08-28T02:15:15.955 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[100258]: mon.a (mon.0) 924 : audit 2023-08-28T02:15:15.955 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[100258]: [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:15.955 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[100258]: audit 2023- 2023-08-28T02:15:15.955 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[100258]: 08-28T02:15:14.773290 2023-08-28T02:15:15.955 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[100258]: +0000 mon.c (mon.2) 91 : 2023-08-28T02:15:15.955 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[100258]: audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:15:15.956 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[100258]: audit 2023- 2023-08-28T02:15:15.956 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[100258]: 08-28T02:15:15.013709+0000 2023-08-28T02:15:15.956 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[100258]: mon.c (mon.2) 92 : audit 2023-08-28T02:15:15.956 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:15 smithi143 conmon[100258]: [DBG] from='client.? 172.21.15.143:0/3304576593' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:15.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:15 smithi169 conmon[100500]: cluster 2023-08-28T02:15:14.415886+0000 mgr.y (mgr.15837) 21 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:15.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:15 smithi169 conmon[100500]: audit 2023-08-28T02:15:14.697919+0000 mon.a (mon.0) 922 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:15.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:15 smithi169 conmon[100500]: audit 2023-08-28T02:15:14.705368+0000 mon.a (mon.0) 923 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:15.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:15 smithi169 conmon[100500]: audit 2023-08-28T02:15:14.752585+0000 mon.c (mon.2) 89 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-08-28T02:15:15.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:15 smithi169 conmon[100500]: audit 2023-08-28T02:15:14.753066+0000 mgr.y (mgr.15837) 22 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-08-28T02:15:15.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:15 smithi169 conmon[100500]: audit 2023-08-28T02:15:14.753974+0000 mon.c (mon.2) 90 : audit [INF] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi169.front.sepia.ceph.com:9095"}]: dispatch 2023-08-28T02:15:15.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:15 smithi169 conmon[100500]: audit 2023-08-28T02:15:14.754303+0000 mgr.y (mgr.15837) 23 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi169.front.sepia.ceph.com:9095"}]: dispatch 2023-08-28T02:15:15.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:15 smithi169 conmon[100500]: audit 2023-08-28T02: 2023-08-28T02:15:15.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:15 smithi169 conmon[100500]: 15:14.758738+0000 mon.a (mon.0) 924 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:15.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:15 smithi169 conmon[100500]: audit 2023-08-28T02:15:14.773290+0000 mon.c (mon.2) 91 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:15:15.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:15 smithi169 conmon[100500]: audit 2023-08-28T02:15:15.013709+0000 mon.c (mon.2) 92 : audit [DBG] from='client.? 172.21.15.143:0/3304576593' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:16.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:16 smithi169 conmon[100500]: audit 2023-08-28T02:15:16.293734+0000 mon.c (mon.2 2023-08-28T02:15:16.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:16 smithi169 conmon[100500]: ) 93 : audit [INF] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-28T02:15:16.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:16 smithi169 conmon[100500]: audit 2023-08-28T02:15:16.294210+0000 mon.a (mon.0) 925 : audit [INF] from='mgr.15837 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-28T02:15:16.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:16 smithi169 conmon[100500]: audit 2023-08-28T02:15:16.295363+0000 mon.c (mon.2) 94 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-08-28T02:15:16.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:16 smithi169 conmon[100500]: audit 2023-08-28T02:15:16 2023-08-28T02:15:16.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:16 smithi169 conmon[100500]: .296309+0000 mon.c (mon.2) 95 : audit [DBG] 2023-08-28T02:15:16.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:16 smithi169 conmon[100500]: from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:15:17.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:16 smithi143 conmon[109923]: audit 2023-08-28T02:15:16.293734+0000 mon.c (mon. 2023-08-28T02:15:17.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:16 smithi143 conmon[109923]: 2) 93 : audit [INF] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-28T02:15:17.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:16 smithi143 conmon[109923]: audit 2023-08-28T02:15:16.294210+0000 2023-08-28T02:15:17.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:16 smithi143 conmon[109923]: mon.a (mon.0) 925 : audit [INF] from='mgr.15837 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-28T02:15:17.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:16 smithi143 conmon[109923]: audit 2023-08-28T02:15: 2023-08-28T02:15:17.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:16 smithi143 conmon[109923]: 16.295363+0000 mon.c (mon.2) 94 : audit 2023-08-28T02:15:17.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:16 smithi143 conmon[109923]: [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-08-28T02:15:17.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:16 smithi143 conmon[109923]: audit 2023-08-28T02:15:16.296309 2023-08-28T02:15:17.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:16 smithi143 conmon[109923]: +0000 mon.c (mon.2) 95 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:15:17.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:16 smithi143 conmon[100258]: audit 2023-08-28T02:15:16.293734+0000 mon.c (mon.2) 2023-08-28T02:15:17.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:16 smithi143 conmon[100258]: 93 : audit [INF] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-28T02:15:17.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:16 smithi143 conmon[100258]: audit 2023-08-28T02:15:16.294210+0000 mon.a (mon.0) 925 : audit [INF] from='mgr.15837 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-28T02:15:17.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:16 smithi143 conmon[100258]: audit 2023-08-28 2023-08-28T02:15:17.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:16 smithi143 conmon[100258]: T02:15:16.295363+0000 mon.c 2023-08-28T02:15:17.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:16 smithi143 conmon[100258]: (mon.2) 94 : audit 2023-08-28T02:15:17.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:16 smithi143 conmon[100258]: [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-08-28T02:15:17.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:16 smithi143 conmon[100258]: audit 2023-08 2023-08-28T02:15:17.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:16 smithi143 conmon[100258]: -28T02:15:16.296309+0000 mon.c (mon.2) 2023-08-28T02:15:17.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:16 smithi143 conmon[100258]: 95 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:15:17.988 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:17 smithi169 systemd[1]: Stopping Ceph mgr.x for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:15:17.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:17 smithi169 conmon[100500]: cephadm 2023-08-28T02:15:16.293289+0000 mgr.y (mgr.15837) 24 2023-08-28T02:15:17.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:17 smithi169 conmon[100500]: : cephadm [INF] Upgrade: Updating mgr.x 2023-08-28T02:15:17.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:17 smithi169 conmon[100500]: cephadm 2023-08-28T02:15:16.297287+0000 mgr.y (mgr.15837) 25 : cephadm [INF] Deploying daemon mgr.x on smithi169 2023-08-28T02:15:17.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:17 smithi169 conmon[100500]: cluster 2023-08-28T02:15:16.416372+0000 mgr.y (mgr.15837) 26 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:17.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:17 smithi169 conmon[100500]: audit 2023-08-28T02:15:17.393508+0000 mon.a (mon.0) 926 : audit [DBG] from='client.? 172.21.15.143:0/3126924546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:17.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:17 smithi169 conmon[100500]: audit 2023-08-28T02:15:17.617352+0000 mon.c (mon.2) 96 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:15:18.204 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:17 smithi143 conmon[100258]: cephadm 2023-08-28T02:15:16.293289 2023-08-28T02:15:18.204 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:17 smithi143 conmon[100258]: +0000 mgr.y (mgr.15837) 24 : cephadm [INF] Upgrade: Updating mgr.x 2023-08-28T02:15:18.205 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:17 smithi143 conmon[100258]: cephadm 2023-08-28T02:15:16.297287+0000 mgr.y (mgr.15837) 25 : cephadm [INF] Deploying daemon mgr.x on smithi169 2023-08-28T02:15:18.205 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:17 smithi143 conmon[100258]: cluster 2023-08-28T02:15:16.416372+0000 mgr.y (mgr.15837) 26 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:18.205 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:17 smithi143 conmon[100258]: audit 2023-08-28T02:15:17. 2023-08-28T02:15:18.205 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:17 smithi143 conmon[100258]: 393508+0000 mon.a (mon.0) 926 : audit [DBG] from='client.? 172.21.15.143:0/3126924546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:18.205 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:17 smithi143 conmon[100258]: audit 2023-08-28T02:15:17 2023-08-28T02:15:18.205 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:17 smithi143 conmon[100258]: .617352+0000 mon.c (mon.2) 96 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:15:18.205 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:17 smithi143 conmon[109923]: cephadm 2023-08-28T02:15:16.293289+0000 2023-08-28T02:15:18.205 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:17 smithi143 conmon[109923]: mgr.y (mgr.15837) 24 : cephadm [INF] Upgrade: Updating mgr.x 2023-08-28T02:15:18.205 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:17 smithi143 conmon[109923]: cephadm 2023-08-28T02:15:16. 2023-08-28T02:15:18.205 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:17 smithi143 conmon[109923]: 297287+0000 mgr.y (mgr.15837) 25 : cephadm [INF] Deploying daemon mgr.x on smithi169 2023-08-28T02:15:18.206 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:17 smithi143 conmon[109923]: cluster 2023-08- 2023-08-28T02:15:18.206 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:17 smithi143 conmon[109923]: 28T02:15:16.416372+0000 mgr.y 2023-08-28T02:15:18.206 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:17 smithi143 conmon[109923]: (mgr.15837) 26 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:18.206 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:17 smithi143 conmon[109923]: audit 2023 2023-08-28T02:15:18.206 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:17 smithi143 conmon[109923]: -08-28T02:15:17. 2023-08-28T02:15:18.206 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:17 smithi143 conmon[109923]: 393508+0000 mon.a (mon.0) 926 : audit 2023-08-28T02:15:18.206 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:17 smithi143 conmon[109923]: [DBG] from='client.? 172.21.15.143:0/3126924546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:18.206 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:17 smithi143 conmon[109923]: audit 2023-08-28 2023-08-28T02:15:18.206 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:17 smithi143 conmon[109923]: T02:15:17.617352+0000 mon.c (mon.2) 96 2023-08-28T02:15:18.206 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:17 smithi143 conmon[109923]: : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:15:18.718 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:18 smithi169 bash[150773]: f3ab5a3cce46a59891c5410e2ba7596c1525daedda40887ccfe2d0956f52bfc7 2023-08-28T02:15:18.719 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:18 smithi169 systemd[1]: ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-08-28T02:15:18.719 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:18 smithi169 systemd[1]: ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@mgr.x.service: Failed with result 'exit-code'. 2023-08-28T02:15:18.719 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:18 smithi169 systemd[1]: Stopped Ceph mgr.x for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:15:18.987 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:18 smithi169 systemd[1]: Starting Ceph mgr.x for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:15:19.661 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:19 smithi169 bash[151140]: 73a560edcde93db80d8838e6b20742e7ef7bd729472e05537405bbf25c65595a 2023-08-28T02:15:19.661 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:19 smithi169 systemd[1]: Started Ceph mgr.x for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:15:19.662 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:19 smithi169 ceph-mgr[151205]: -- 172.21.15.169:0/2607589793 <== mon.1 v2:172.21.15.169:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55ea82c331e0 con 0x55ea82c0b000 2023-08-28T02:15:19.662 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:19 smithi169 conmon[151181]: 2023-08-28T02:15:19.477+0000 7f739ced5200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-08-28T02:15:19.662 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:19 smithi169 conmon[151181]: 2023-08-28T02:15:19.561+0000 7f739ced5200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-08-28T02:15:19.662 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:19 smithi169 conmon[151181]: 2023-08-28T02:15:19.661+0000 7f739ced5200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-08-28T02:15:19.969 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:19 smithi169 conmon[151181]: 2023-08-28T02:15:19.773+0000 7f739ced5200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-08-28T02:15:19.970 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:19 smithi169 conmon[100500]: cluster 2023-08-28T02:15:18.416761+0000 mgr.y (mgr.15837) 2023-08-28T02:15:19.970 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:19 smithi169 conmon[100500]: 27 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:19.970 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:19 smithi169 conmon[100500]: audit 2023-08-28T02:15:19.396618+0000 mon.a (mon.0) 927 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:19.970 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:19 smithi169 conmon[100500]: audit 2023-08-28T02:15:19.403852+0000 mon.a (mon.0) 928 : audit 2023-08-28T02:15:19.970 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:19 smithi169 conmon[100500]: [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:19.970 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:19 smithi169 conmon[100500]: audit 2023-08-28T02:15:19. 2023-08-28T02:15:19.970 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:19 smithi169 conmon[100500]: 418802+0000 mon.c (mon.2) 97 : audit [DBG] 2023-08-28T02:15:19.970 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:19 smithi169 conmon[100500]: from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:15:20.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:19 smithi143 conmon[109923]: cluster 2023-08-28T02:15:18.416761 2023-08-28T02:15:20.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:19 smithi143 conmon[109923]: +0000 mgr.y (mgr.15837) 27 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:20.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:19 smithi143 conmon[109923]: audit 2023-08-28T02:15:19.396618 2023-08-28T02:15:20.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:19 smithi143 conmon[109923]: +0000 mon.a (mon.0) 927 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:20.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:19 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:15:20.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:19 smithi143 conmon[109923]: 08-28T02:15:19.403852+0000 mon.a 2023-08-28T02:15:20.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:19 smithi143 conmon[109923]: (mon.0) 928 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:20.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:19 smithi143 conmon[109923]: audit 2023 2023-08-28T02:15:20.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:19 smithi143 conmon[109923]: -08-28T02:15:19.418802+0000 mon.c 2023-08-28T02:15:20.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:19 smithi143 conmon[109923]: (mon.2) 97 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:15:20.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:19 smithi143 conmon[100258]: cluster 2023-08-28T02:15:18.416761+0000 2023-08-28T02:15:20.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:19 smithi143 conmon[100258]: mgr.y (mgr.15837) 27 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:20.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:19 smithi143 conmon[100258]: audit 2023-08-28 2023-08-28T02:15:20.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:19 smithi143 conmon[100258]: T02:15:19.396618+0000 mon.a (mon.0) 927 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:20.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:19 smithi143 conmon[100258]: audit 2023 2023-08-28T02:15:20.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:19 smithi143 conmon[100258]: -08-28T02:15:19.403852+0000 2023-08-28T02:15:20.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:19 smithi143 conmon[100258]: mon.a (mon.0) 928 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:20.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:19 smithi143 conmon[100258]: audit 2023-08- 2023-08-28T02:15:20.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:19 smithi143 conmon[100258]: 28T02:15:19.418802+0000 mon.c (mon.2 2023-08-28T02:15:20.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:19 smithi143 conmon[100258]: ) 97 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:15:20.237 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:19 smithi169 conmon[151181]: 2023-08-28T02:15:19.969+0000 7f739ced5200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-08-28T02:15:20.707 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:20 smithi169 conmon[151181]: 2023-08-28T02:15:20.369+0000 7f739ced5200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-08-28T02:15:20.707 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:20 smithi169 conmon[151181]: 2023-08-28T02:15:20.518+0000 7f739ced5200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-08-28T02:15:20.707 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:20 smithi169 conmon[151181]: 2023-08-28T02:15:20.613+0000 7f739ced5200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-08-28T02:15:20.987 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:20 smithi169 conmon[151181]: 2023-08-28T02:15:20.715+0000 7f739ced5200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-08-28T02:15:20.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:20 smithi169 conmon[100500]: audit 2023-08-28T02:15:19.770867+0000 mon.a (mon.0) 929 2023-08-28T02:15:20.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:20 smithi169 conmon[100500]: : audit [DBG] from='client.? 172.21.15.143:0/279949520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:21.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:20 smithi143 conmon[109923]: audit 2023-08-28T02:15:19.770867+0000 mon.a (mon.0) 929 : audit [DBG] from='client.? 172.21.15.143:0/279949520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:21.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:20 smithi143 conmon[100258]: audit 2023-08-28T02:15:19.770867+0000 mon.a (mon.0) 929 : audit [DBG] 2023-08-28T02:15:21.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:20 smithi143 conmon[100258]: from='client.? 172.21.15.143:0/279949520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:21.487 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:21 smithi169 conmon[151181]: 2023-08-28T02:15:21.212+0000 7f739ced5200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-08-28T02:15:21.987 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:21 smithi169 conmon[151181]: 2023-08-28T02:15:21.937+0000 7f739ced5200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-08-28T02:15:21.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:21 smithi169 conmon[100500]: cluster 2023-08-28T02:15:20. 2023-08-28T02:15:21.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:21 smithi169 conmon[100500]: 417497+0000 mgr.y (mgr.15837) 28 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:22.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:21 smithi143 conmon[100258]: cluster 2023-08-28T02:15:20.417497+0000 mgr.y (mgr.15837) 2023-08-28T02:15:22.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:21 smithi143 conmon[100258]: 28 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:22.201 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:21 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:15:21] "GET /metrics HTTP/1.1" 200 33948 "" "Prometheus/2.33.4" 2023-08-28T02:15:22.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:21 smithi143 conmon[109923]: cluster 2023-08-28T02:15:20.417497+0000 mgr.y (mgr.15837) 28 : cluster [DBG] 2023-08-28T02:15:22.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:21 smithi143 conmon[109923]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:22.487 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:22 smithi169 conmon[151181]: 2023-08-28T02:15:22.033+0000 7f739ced5200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-08-28T02:15:22.987 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:22 smithi169 conmon[151181]: 2023-08-28T02:15:22.888+0000 7f739ced5200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-08-28T02:15:22.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:22 smithi169 conmon[100500]: audit 2023-08-28T02:15:22.151526+0000 mon.a (mon.0 2023-08-28T02:15:22.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:22 smithi169 conmon[100500]: ) 930 : audit [DBG] from='client.? 172.21.15.143:0/3774979717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:22.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:22 smithi169 conmon[100500]: audit 2023-08-28T02:15:22.546661+0000 mon.a (mon.0) 931 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:22.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:22 smithi169 conmon[100500]: audit 2023-08-28T02:15:22.551982+0000 mon.a (mon.0) 932 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:23.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:22 smithi143 conmon[109923]: audit 2023-08-28T02:15:22.151526+0000 mon.a (mon.0) 930 : audit 2023-08-28T02:15:23.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:22 smithi143 conmon[109923]: [DBG] from='client.? 172.21.15.143:0/3774979717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:23.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:22 smithi143 conmon[109923]: audit 2023-08-28T02:15:22.546661+0000 mon.a (mon.0) 931 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:23.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:22 smithi143 conmon[109923]: audit 2023-08-28T02:15:22.551982+0000 mon.a (mon.0) 932 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:23.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:22 smithi143 conmon[100258]: audit 2023-08-28T02:15:22. 2023-08-28T02:15:23.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:22 smithi143 conmon[100258]: 151526+0000 mon.a (mon.0) 930 : audit [DBG] from='client.? 172.21.15.143:0/3774979717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:23.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:22 smithi143 conmon[100258]: audit 2023-08-28T02:15:22. 2023-08-28T02:15:23.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:22 smithi143 conmon[100258]: 546661+0000 mon.a (mon.0) 931 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:23.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:22 smithi143 conmon[100258]: audit 2023-08-28 2023-08-28T02:15:23.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:22 smithi143 conmon[100258]: T02:15:22.551982+0000 mon.a (mon.0) 2023-08-28T02:15:23.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:22 smithi143 conmon[100258]: 932 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:23.486 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:22 smithi169 conmon[151181]: 2023-08-28T02:15:22.989+0000 7f739ced5200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-08-28T02:15:23.486 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:23 smithi169 conmon[151181]: 2023-08-28T02:15:23.177+0000 7f739ced5200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-08-28T02:15:23.486 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:23 smithi169 conmon[151181]: 2023-08-28T02:15:23.486+0000 7f739ced5200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-08-28T02:15:23.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:23 smithi169 conmon[100500]: cluster 2023-08-28T02:15:22.417864+0000 mgr.y (mgr.15837) 2023-08-28T02:15:23.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:23 smithi169 conmon[100500]: 29 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:23.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:23 smithi169 conmon[100500]: audit 2023-08-28T02:15:23.240141+0000 mon.a (mon.0) 933 : audit [INF] 2023-08-28T02:15:23.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:23 smithi169 conmon[100500]: from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:23.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:23 smithi169 conmon[100500]: audit 2023 2023-08-28T02:15:23.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:23 smithi169 conmon[100500]: -08-28T02:15:23.248231+0000 mon.a 2023-08-28T02:15:23.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:23 smithi169 conmon[100500]: (mon.0) 934 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:24.005 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:23 smithi169 conmon[151181]: 2023-08-28T02:15:23.739+0000 7f739ced5200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-08-28T02:15:24.006 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:23 smithi169 conmon[151181]: 2023-08-28T02:15:23.901+0000 7f739ced5200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-08-28T02:15:24.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:23 smithi143 conmon[100258]: cluster 2023-08-28T02:15:22.417864+0000 mgr.y (mgr.15837 2023-08-28T02:15:24.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:23 smithi143 conmon[100258]: ) 29 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:24.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:23 smithi143 conmon[100258]: audit 2023-08-28T02:15:23.240141+0000 mon.a (mon.0) 933 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:24.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:23 smithi143 conmon[100258]: audit 2023-08-28T 2023-08-28T02:15:24.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:23 smithi143 conmon[100258]: 02:15:23.248231+0000 mon.a (mon.0) 934 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:24.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:23 smithi143 conmon[109923]: cluster 2023-08-28T02:15:22.417864+0000 mgr.y (mgr.15837) 2023-08-28T02:15:24.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:23 smithi143 conmon[109923]: 29 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:24.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:23 smithi143 conmon[109923]: audit 2023-08-28T02:15:23.240141+0000 mon.a (mon.0) 933 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:24.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:23 smithi143 conmon[109923]: audit 2023-08-28T02:15:23.248231+0000 mon.a (mon.0) 934 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:24.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:23 smithi143 conmon[109923]: 2023-08-28T02:15:24.487 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:24 smithi169 conmon[151181]: 2023-08-28T02:15:24.181+0000 7f739ced5200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-08-28T02:15:24.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:24 smithi169 conmon[100500]: audit 2023-08-28T02:15:24.527138+0000 mon.c (mon.2) 98 : audit [DBG] from='client.? 172.21.15.143:0/2061181827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:24.987 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:24 smithi169 conmon[151181]: 2023-08-28T02:15:24.919+0000 7f739ced5200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-08-28T02:15:25.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:24 smithi143 conmon[100258]: audit 2023-08-28T02:15:24.527138+0000 mon.c (mon.2) 98 : audit [DBG] from='client.? 172.21.15.143:0/2061181827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:25.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:24 smithi143 conmon[109923]: audit 2023-08-28T02:15:24.527138+0000 mon.c (mon.2) 98 2023-08-28T02:15:25.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:24 smithi143 conmon[109923]: : audit [DBG] from='client.? 172.21.15.143:0/2061181827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:25.389 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:25 smithi169 conmon[151181]: 2023-08-28T02:15:25.056+0000 7f739ced5200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-08-28T02:15:25.389 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:25 smithi169 conmon[151181]: 2023-08-28T02:15:25.266+0000 7f739ced5200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-08-28T02:15:25.389 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:25 smithi169 conmon[151181]: 2023-08-28T02:15:25.389+0000 7f739ced5200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-08-28T02:15:25.735 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:25 smithi169 conmon[151181]: 2023-08-28T02:15:25.618+0000 7f739ced5200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-08-28T02:15:25.735 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:25 smithi169 conmon[100500]: cluster 2023-08-28T02:15:24. 2023-08-28T02:15:25.735 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:25 smithi169 conmon[100500]: 418364+0000 mgr.y (mgr.15837) 30 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:26.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:25 smithi143 conmon[100258]: cluster 2023-08-28T02:15:24.418364+0000 mgr.y (mgr.15837) 30 : cluster 2023-08-28T02:15:26.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:25 smithi143 conmon[100258]: [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:26.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:25 smithi143 conmon[109923]: cluster 2023-08-28T02:15:24.418364+0000 mgr.y (mgr.15837) 30 : 2023-08-28T02:15:26.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:25 smithi143 conmon[109923]: cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:26.737 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:26 smithi169 conmon[151181]: 2023-08-28T02:15:26.534+0000 7f739ced5200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-08-28T02:15:26.737 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:26 smithi169 conmon[151181]: [28/Aug/2023:02:15:26] ENGINE Bus STARTING 2023-08-28T02:15:26.737 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:26 smithi169 conmon[151181]: CherryPy Checker: 2023-08-28T02:15:26.737 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:26 smithi169 conmon[151181]: The Application mounted at '' has an empty config. 2023-08-28T02:15:26.737 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:26 smithi169 conmon[151181]: 2023-08-28T02:15:26.738 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:26 smithi169 conmon[151181]: [28/Aug/2023:02:15:26] ENGINE Serving on http://:::9283 2023-08-28T02:15:26.738 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:26 smithi169 conmon[151181]: [28/Aug/2023:02:15:26] ENGINE Bus STARTED 2023-08-28T02:15:27.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: cluster 2023-08-28T02:15:26.418904+0000 mgr.y ( 2023-08-28T02:15:27.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: mgr.15837) 31 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:27.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: audit 2023-08-28T02:15:26.529366+0000 mon.a (mon.0) 935 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:27.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: audit 2023-08-28T02:15:26.535536+0000 mon.a (mon.0) 936 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:27.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: audit 2023-08-28T02:15:26.548161+0000 mon.c (mon.2) 99 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:15:27.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: audit 2023-08-28T02:15:26.549225+0000 mon.c (mon.2) 100 2023-08-28T02:15:27.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: : audit [INF] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:15:27.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: cluster 2023-08-28T02:15:26.549799+0000 mon.a (mon.0) 937 : cluster [DBG] Standby manager daemon x restarted 2023-08-28T02:15:27.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: cluster 2023-08-28T02:15:27.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: 2023-08-28T02:15:26.549989+0000 mon.a (mon.0) 938 : cluster [DBG] Standby manager daemon x started 2023-08-28T02:15:27.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: audit 2023- 2023-08-28T02:15:27.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: 08-28T02:15:26.551947+0000 mon.b (mon.1) 2023-08-28T02:15:27.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: 104 : audit [DBG] from='mgr.? 172.21.15.169:0/3417525951' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-08-28T02:15:27.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: audit 2023-08-28T02:15:26.552290 2023-08-28T02:15:27.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: +0000 mon.b (mon.1) 105 : audit [DBG] from='mgr.? 172.21.15.169:0/3417525951' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-28T02:15:27.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: audit 2023-08-28T02:15:26. 2023-08-28T02:15:27.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: 554982+0000 mon.b (mon.1) 106 : audit [DBG] from='mgr.? 172.21.15.169:0/3417525951' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-08-28T02:15:27.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: audit 2023 2023-08-28T02:15:27.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: -08-28T02:15:26.559102+0000 mon.a ( 2023-08-28T02:15:27.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: mon.0) 939 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:27.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: audit 2023-08-28T02:15:26.559568 2023-08-28T02:15:27.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: +0000 mon.b (mon.1) 107 : audit [DBG] from='mgr.? 172.21.15.169:0/3417525951' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-08-28T02:15:27.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: audit 2023-08-28T02:15:26.701476+0000 2023-08-28T02:15:27.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: mon.c (mon.2) 101 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:15:27.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: audit 2023-08-28T02:15:26.704257+0000 mon.c (mon.2 2023-08-28T02:15:27.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: ) 102 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:15:27.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: cephadm 2023-08-28T02:15:27.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: 2023-08-28T02:15:26.705099+0000 mgr.y (mgr.15837) 32 2023-08-28T02:15:27.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: : cephadm [INF] Upgrade: Setting container_image for all mgr 2023-08-28T02:15:27.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: audit 2023-08-28T02:15:27.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: 2023-08-28T02:15:26.713027+0000 mon.a (mon.0) 940 : audit 2023-08-28T02:15:27.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:27.704 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: audit 2023-08-28T02:15:27.704 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: 2023-08-28T02:15:26.713936+0000 mon.c (mon.2) 103 : audit [INF] 2023-08-28T02:15:27.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: cluster 2023-08-28T02:15:26.418904+0000 mgr.y 2023-08-28T02:15:27.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: (mgr.15837) 31 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:27.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: audit 2023-08-28T02:15:26.529366+0000 mon.a (mon.0) 935 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:27.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: 2023-08-28T02:15:27.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: audit 2023-08-28T02:15:26.535536+0000 mon.a (mon.0 2023-08-28T02:15:27.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: ) 936 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:27.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: audit 2023-08-28 2023-08-28T02:15:27.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: T02:15:26.548161+0000 mon.c (mon.2) 99 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:15:27.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: audit 2023-08-28T02:15:26 2023-08-28T02:15:27.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: .549225+0000 mon.c (mon.2) 100 : audit [INF] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:15:27.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: cluster 2023-08-28T02:15: 2023-08-28T02:15:27.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: 26.549799+0000 mon.a (mon.0) 937 : cluster [DBG] Standby manager daemon x restarted 2023-08-28T02:15:27.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: cluster 2023-08-28T02 2023-08-28T02:15:27.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: :15:26.549989+0000 mon.a (mon.0) 938 : cluster [DBG] 2023-08-28T02:15:27.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: Standby manager daemon x started 2023-08-28T02:15:27.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: audit 2023-08-28T02:15:27.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: 2023-08-28T02:15:26 2023-08-28T02:15:27.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: .551947+0000 mon.b (mon.1) 104 : audit [DBG] from='mgr.? 172.21.15.169:0/3417525951' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-08-28T02:15:27.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: audit 2023-08- 2023-08-28T02:15:27.706 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: 28T02:15:26.552290+0000 mon.b (mon.1) 2023-08-28T02:15:27.706 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: 105 : audit [DBG] from='mgr.? 172.21.15.169:0/3417525951' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-28T02:15:27.706 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: audit 2023-08-28T02:15:26.554982+0000 mon.b (mon.1 2023-08-28T02:15:27.706 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: ) 106 : audit [DBG] from='mgr.? 172.21.15.169:0/3417525951' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-08-28T02:15:27.706 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: audit 2023-08-28T02:15:27.706 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: 2023-08-28T02:15:26.559102+0000 mon.a 2023-08-28T02:15:27.706 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: (mon.0) 939 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:27.706 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: audit 2023-08-28T02:15 2023-08-28T02:15:27.706 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: :26.559568+0000 mon.b (mon.1) 107 : 2023-08-28T02:15:27.706 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: audit [DBG] from='mgr.? 172.21.15.169:0/3417525951' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-08-28T02:15:27.706 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: audit 2023-08-28T02:15:26.701476+0000 mon.c ( 2023-08-28T02:15:27.707 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: mon.2) 101 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:15:27.707 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: audit 2023-08-28T02:15 2023-08-28T02:15:27.707 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: :26.704257+0000 mon.c (mon.2) 102 : audit [DBG] 2023-08-28T02:15:27.707 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:15:27.707 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: cephadm 2023-08- 2023-08-28T02:15:27.707 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: 28T02:15:26.705099+0000 mgr.y (mgr.15837) 32 : cephadm [INF] Upgrade: Setting container_image for all mgr 2023-08-28T02:15:27.707 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: audit 2023-08-28T 2023-08-28T02:15:27.707 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: 02:15:26.713027+0000 mon.a (mon.0) 940 : audit 2023-08-28T02:15:27.707 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-08-28T02:15:27.708 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: audit 2023-08- 2023-08-28T02:15:27.708 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: 28T02:15:26.714287+0000 mon.a (mon.0) 941 : audit [INF] from='mgr.15837 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-08-28T02:15:27.708 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: audit 2023-08-28 2023-08-28T02:15:27.708 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: T02:15:26.721773+0000 mon.a (mon.0) 942 2023-08-28T02:15:27.708 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: : audit [INF] from='mgr.15837 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-08-28T02:15:27.708 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: audit 2023-08-28T02:15:27.708 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: 2023-08-28T02:15:26.722682+0000 mon.c (mon.2) 104 : audit [INF] 2023-08-28T02:15:27.708 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-08-28T02:15:27.708 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: audit 2023-08-28T 2023-08-28T02:15:27.708 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: 02:15:26.722989+0000 mon.a (mon.0) 943 : audit [INF] from='mgr.15837 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-08-28T02:15:27.708 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: audit 2023-08-28T02:15:26.729935+0000 mon.a 2023-08-28T02:15:27.708 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: (mon.0) 944 : audit [INF] from='mgr.15837 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-08-28T02:15:27.708 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: audit 2023-08-28T02:15:26.731124+0000 mon.c (mon. 2023-08-28T02:15:27.709 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: 2) 105 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-08-28T02:15:27.709 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: audit 2023-08-28T02:15:27.709 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: 2023-08-28T02:15:26.731793+0000 mon.c (mon.2) 106 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-08-28T02:15:27.709 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: cephadm 2023-08 2023-08-28T02:15:27.709 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: -28T02:15:26.732264+0000 mgr.y (mgr.15837) 33 : 2023-08-28T02:15:27.709 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: cephadm [INF] Upgrade: It appears safe to stop mon.a 2023-08-28T02:15:27.709 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: audit 2023- 2023-08-28T02:15:27.709 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[100258]: 08-28T02:15:26.937966+0000 mon.a (mon.0) 945 : audit [DBG] from='client.? 172.21.15.143:0/579660853' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:27.709 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:27.709 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: audit 2023 2023-08-28T02:15:27.709 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: -08-28T02:15:26.713936+0000 mon.c (mon.2) 103 : audit [INF] 2023-08-28T02:15:27.710 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-08-28T02:15:27.710 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: audit 2023-08- 2023-08-28T02:15:27.710 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: 28T02:15:26.714287+0000 mon.a (mon.0) 941 : audit [INF] from='mgr.15837 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-08-28T02:15:27.710 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: audit 2023-08-28T02 2023-08-28T02:15:27.710 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: :15:26.721773+0000 mon.a (mon.0) 942 : audit 2023-08-28T02:15:27.710 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: [INF] from='mgr.15837 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-08-28T02:15:27.710 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:15:27.710 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: 08-28T02:15:26.722682+0000 mon.c (mon.2) 104 : audit [INF] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-08-28T02:15:27.710 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: audit 2023-08-28T02: 2023-08-28T02:15:27.710 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: 15:26.722989+0000 mon.a (mon.0) 943 : audit [INF] 2023-08-28T02:15:27.710 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: from='mgr.15837 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-08-28T02:15:27.710 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: audit 2023-08 2023-08-28T02:15:27.711 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: -28T02:15:26.729935+0000 mon.a (mon.0) 944 : audit [INF] from='mgr.15837 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-08-28T02:15:27.711 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: audit 2023-08-28T02 2023-08-28T02:15:27.711 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: :15:26.731124+0000 mon.c (mon.2) 105 : audit 2023-08-28T02:15:27.711 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-08-28T02:15:27.711 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: audit 2023-08 2023-08-28T02:15:27.711 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: -28T02:15:26.731793+0000 mon.c (mon.2) 106 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-08-28T02:15:27.711 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: cephadm 2023-08-28T02: 2023-08-28T02:15:27.711 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: 15:26.732264+0000 mgr.y (mgr.15837) 33 : cephadm [INF] 2023-08-28T02:15:27.711 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: Upgrade: It appears safe to stop mon.a 2023-08-28T02:15:27.711 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: audit 2023-08- 2023-08-28T02:15:27.711 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:27 smithi143 conmon[109923]: 28T02:15:26.937966+0000 mon.a (mon.0) 945 : audit [DBG] from='client.? 172.21.15.143:0/579660853' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:27.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: cluster 2023-08-28T02:15:26.418904+0000 mgr.y (mgr.15837) 31 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:27.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: audit 2023-08-28T02:15:26.529366 2023-08-28T02:15:27.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: +0000 mon.a (mon.0) 935 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:27.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: audit 2023-08-28T02:15:26.535536+0000 mon.a (mon.0) 936 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:27.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: audit 2023-08-28T02:15:26 2023-08-28T02:15:27.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: .548161+0000 mon.c (mon.2) 99 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:15:27.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: audit 2023-08-28T02:15:27.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: 2023-08-28T02:15:26. 2023-08-28T02:15:27.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: 549225+0000 mon.c (mon.2) 100 2023-08-28T02:15:27.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: : audit [INF] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:15:27.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: cluster 2023-08- 2023-08-28T02:15:27.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: 28T02:15:26.549799+0000 mon.a ( 2023-08-28T02:15:27.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: mon.0) 937 : cluster [DBG] Standby manager daemon x restarted 2023-08-28T02:15:27.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: 2023-08-28T02:15:27.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: cluster 2023- 2023-08-28T02:15:27.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: 08-28T02:15:26.549989 2023-08-28T02:15:27.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: +0000 mon.a (mon.0) 938 : 2023-08-28T02:15:27.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: cluster [DBG] Standby manager daemon x started 2023-08-28T02:15:27.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: audit 2023-08 2023-08-28T02:15:27.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: -28T02:15:26.551947+0000 mon.b (mon 2023-08-28T02:15:27.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: .1) 104 : audit [DBG] from='mgr.? 172.21.15.169:0/3417525951' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-08-28T02:15:27.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: audit 2023- 2023-08-28T02:15:27.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: 08-28T02:15:26.552290+0000 2023-08-28T02:15:27.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: mon.b (mon.1) 105 : audit 2023-08-28T02:15:27.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: [DBG] from='mgr.? 172.21.15.169:0/3417525951' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-28T02:15:27.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: audit 2023-08 2023-08-28T02:15:27.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: -28T02:15:26.554982+0000 mon.b 2023-08-28T02:15:27.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: (mon.1) 106 : audit [DBG] from='mgr.? 172.21.15.169:0/3417525951' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-08-28T02:15:27.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: 2023-08-28T02:15:27.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: audit 2023 2023-08-28T02:15:27.991 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: -08-28T02:15:26. 2023-08-28T02:15:27.991 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: 559102+0000 mon.a (mon.0) 939 2023-08-28T02:15:27.991 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:27.991 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: audit 2023-08-28T02:15:27.991 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: 2023-08-28T02:15:26. 2023-08-28T02:15:27.991 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: 559568+0000 mon.b (mon.1) 107 2023-08-28T02:15:27.991 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: : audit [DBG] from='mgr.? 172.21.15.169:0/3417525951' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-08-28T02:15:27.991 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: audit 2023- 2023-08-28T02:15:27.991 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: 08-28T02:15:26.701476+0000 mon.c (mon.2 2023-08-28T02:15:27.991 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: ) 101 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:15:27.991 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: audit 2023- 2023-08-28T02:15:27.991 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: 08-28T02:15:26.704257+0000 2023-08-28T02:15:27.992 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: mon.c (mon.2) 102 : audit [DBG] 2023-08-28T02:15:27.992 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:15:27.992 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: cephadm 2023-08-28 2023-08-28T02:15:27.992 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: T02:15:26.705099+0000 mgr.y (mgr. 2023-08-28T02:15:27.992 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: 15837) 32 : cephadm [INF] Upgrade: Setting container_image for all mgr 2023-08-28T02:15:27.992 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: audit 2023- 2023-08-28T02:15:27.992 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: 08-28T02:15:26.713027+0000 2023-08-28T02:15:27.992 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: mon.a (mon.0) 940 : audit 2023-08-28T02:15:27.992 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:27.992 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: audit 2023 2023-08-28T02:15:27.992 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: -08-28T02:15:26.713936 2023-08-28T02:15:27.992 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: +0000 mon.c (mon.2) 103 : 2023-08-28T02:15:27.993 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: audit [INF] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-08-28T02:15:27.993 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: audit 2023 2023-08-28T02:15:27.993 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: -08-28T02:15:26.714287 2023-08-28T02:15:27.993 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: +0000 mon.a (mon.0) 941 : 2023-08-28T02:15:27.993 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: audit [INF] from='mgr.15837 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-08-28T02:15:27.993 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: audit 2023- 2023-08-28T02:15:27.993 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: 08-28T02:15:26.721773+0000 2023-08-28T02:15:27.993 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: mon.a (mon.0) 942 : audit 2023-08-28T02:15:27.993 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: [INF] from='mgr.15837 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-08-28T02:15:27.993 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: audit 2023- 2023-08-28T02:15:27.993 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: 08-28T02:15:26.722682+0000 2023-08-28T02:15:27.993 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: mon.c (mon.2) 104 : audit [INF] 2023-08-28T02:15:27.993 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-08-28T02:15:27.994 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: audit 2023-08-28T02:15:27.994 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: 2023-08-28T02:15: 2023-08-28T02:15:27.994 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: 26.722989+0000 mon.a (mon.0 2023-08-28T02:15:27.994 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: ) 943 : audit [INF] from='mgr.15837 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-08-28T02:15:27.994 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: audit 2023- 2023-08-28T02:15:27.994 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: 08-28T02:15:26.729935+0000 2023-08-28T02:15:27.994 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: mon.a (mon.0) 944 : audit 2023-08-28T02:15:27.994 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: [INF] from='mgr.15837 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-08-28T02:15:27.994 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: audit 2023 2023-08-28T02:15:27.994 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: -08-28T02:15:26.731124 2023-08-28T02:15:27.994 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: +0000 mon.c (mon.2) 105 2023-08-28T02:15:27.994 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-08-28T02:15:27.994 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: audit 2023- 2023-08-28T02:15:27.995 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: 08-28T02:15:26.731793+0000 2023-08-28T02:15:27.995 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: mon.c (mon.2) 106 : audit 2023-08-28T02:15:27.995 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-08-28T02:15:27.995 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: cephadm 2023 2023-08-28T02:15:27.995 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: -08-28T02:15:26.732264+0000 2023-08-28T02:15:27.995 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: mgr.y (mgr.15837) 33 : cephadm 2023-08-28T02:15:27.995 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: [INF] Upgrade: It appears safe to stop mon.a 2023-08-28T02:15:27.995 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: audit 2023- 2023-08-28T02:15:27.995 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: 08-28T02:15:26.937966+0000 2023-08-28T02:15:27.995 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: mon.a (mon.0) 945 : 2023-08-28T02:15:27.996 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:27 smithi169 conmon[100500]: audit [DBG] from='client.? 172.21.15.143:0/579660853' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:28.884 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:28 smithi143 conmon[100258]: cluster 2023-08-28T02:15:27.542577+0000 mon.a ( 2023-08-28T02:15:28.884 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:28 smithi143 conmon[100258]: mon.0) 946 : cluster [DBG] mgrmap e29: y(active, since 25s), standbys: x 2023-08-28T02:15:28.884 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:28 smithi143 conmon[100258]: audit 2023-08-28T02:15:28.203979+0000 mon.a (mon.0) 947 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:28.885 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:28 smithi143 conmon[100258]: audit 2023-08-28T02:15:28.204862+0000 mon.c (mon.2) 107 : audit [INF] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-28T02:15:28.885 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:28 smithi143 conmon[100258]: audit 2023-08-28T02:15:28.205439+0000 mon.c (mon.2) 108 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-28T02:15:28.885 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:28 smithi143 conmon[100258]: audit 2023-08-28T02:15:28.206193+0000 mon.c (mon.2) 109 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:15:28.885 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:28 smithi143 conmon[109923]: cluster 2023-08-28T02:15:27.542577+0000 mon.a ( 2023-08-28T02:15:28.885 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:28 smithi143 conmon[109923]: mon.0) 946 : cluster [DBG] mgrmap e29: y(active, since 25s), standbys: x 2023-08-28T02:15:28.885 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:28 smithi143 conmon[109923]: audit 2023-08-28T02:15:28.203979+0000 mon.a (mon.0) 947 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:28.885 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:28 smithi143 conmon[109923]: audit 2023-08-28T02:15:28.204862+0000 mon.c (mon.2) 107 : audit [INF] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-28T02:15:28.885 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:28 smithi143 conmon[109923]: audit 2023-08-28T02:15:28.205439+0000 mon.c (mon.2) 108 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-28T02:15:28.885 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:28 smithi143 conmon[109923]: audit 2023-08-28T02:15:28.206193+0000 mon.c (mon.2) 109 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:15:28.987 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:15:28 smithi169 conmon[150127]: ts=2023-08-28T02:15:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi169.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi143:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-08-28T02:15:28.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:28 smithi169 conmon[100500]: cluster 2023-08-28T02:15:27.542577+0000 mon.a (mon.0) 946 : cluster [DBG] mgrmap e29: y(active, since 25s), standbys: x 2023-08-28T02:15:28.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:28 smithi169 conmon[100500]: audit 2023-08-28T02:15:28.203979+0000 mon.a (mon.0) 947 : audit [INF] from='mgr.15837 ' entity='mgr.y' 2023-08-28T02:15:28.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:28 smithi169 conmon[100500]: audit 2023-08-28T02:15:28.204862+0000 mon.c (mon.2) 107 : audit [INF] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-28T02:15:28.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:28 smithi169 conmon[100500]: audit 2023-08-28T02:15 2023-08-28T02:15:28.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:28 smithi169 conmon[100500]: :28.205439+0000 mon.c (mon.2) 2023-08-28T02:15:28.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:28 smithi169 conmon[100500]: 108 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-28T02:15:28.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:28 smithi169 conmon[100500]: audit 2023-08-28 2023-08-28T02:15:28.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:28 smithi169 conmon[100500]: T02:15:28.206193+0000 mon.c (mon 2023-08-28T02:15:28.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:28 smithi169 conmon[100500]: .2) 109 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:15:29.542 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:29 smithi143 systemd[1]: Stopping Ceph mon.a for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:15:29.802 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:29 smithi143 conmon[109923]: cephadm 2023-08-28T02:15 2023-08-28T02:15:29.802 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:29 smithi143 conmon[109923]: :28.198149+0000 mgr.y (mgr.15837) 34 : cephadm [INF] Upgrade: Updating mon.a 2023-08-28T02:15:29.802 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:29 smithi143 conmon[109923]: cephadm 2023-08-28T02:15:28.207315+0000 mgr.y (mgr.15837) 35 : cephadm [INF] Deploying daemon mon.a on smithi143 2023-08-28T02:15:29.802 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:29 smithi143 conmon[109923]: cluster 2023-08-28T02:15:28.419240+0000 mgr.y (mgr.15837) 36 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:29.802 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:29 smithi143 conmon[109923]: audit 2023-08-28T02:15:29.330450+0000 mon.a (mon.0) 948 : audit [DBG] from='client.? 172.21.15.143:0/249395138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:29.802 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:29 smithi143 conmon[100258]: cephadm 2023-08-28T02:15:28.198149 2023-08-28T02:15:29.802 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:29 smithi143 conmon[100258]: +0000 mgr.y (mgr.15837) 34 : cephadm [INF] Upgrade: Updating mon.a 2023-08-28T02:15:29.803 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:29 smithi143 conmon[100258]: cephadm 2023-08-28T02:15:28.207315+0000 mgr.y (mgr.15837) 35 : cephadm [INF] Deploying daemon mon.a on smithi143 2023-08-28T02:15:29.803 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:29 smithi143 conmon[100258]: cluster 2023-08-28T02:15:28.419240+0000 mgr.y (mgr.15837) 36 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:29.803 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:29 smithi143 conmon[100258]: audit 2023-08-28T02:15:29.330450+0000 mon.a (mon.0) 948 : audit [DBG] from='client.? 172.21.15.143:0/249395138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:29.803 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:29 smithi143 conmon[100258]: debug 2023-08-28T02:15:29.589+0000 7f0a594ce700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-08-28T02:15:29.803 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:29 smithi143 conmon[100258]: debug 2023-08-28T02:15:29.589+0000 7f0a594ce700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2023-08-28T02:15:29.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:29 smithi169 conmon[100500]: cephadm 2023-08-28T02:15:28.198149+0000 mgr.y (mgr.15837) 34 : cephadm [INF] Upgrade: Updating mon.a 2023-08-28T02:15:29.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:29 smithi169 conmon[100500]: cephadm 2023-08-28T02:15:28.207315+0000 mgr.y (mgr.15837) 35 : cephadm [INF] Deploying daemon mon.a on smithi143 2023-08-28T02:15:29.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:29 smithi169 conmon[100500]: cluster 2023-08-28T02:15:28.419240+0000 mgr.y (mgr.15837) 36 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:29.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:29 smithi169 conmon[100500]: audit 2023-08- 2023-08-28T02:15:29.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:29 smithi169 conmon[100500]: 28T02:15:29.330450+0000 2023-08-28T02:15:29.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:29 smithi169 conmon[100500]: mon.a (mon.0) 948 : audit [DBG] 2023-08-28T02:15:29.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:29 smithi169 conmon[100500]: from='client.? 172.21.15.143:0/249395138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:30.498 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:30 smithi143 bash[182638]: 3e714a6191278b03ddf36d79b64cfeb2b3f674cf0fdfe867c9e30c7501d13506 2023-08-28T02:15:30.498 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:30 smithi143 systemd[1]: ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@mon.a.service: Succeeded. 2023-08-28T02:15:30.498 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:30 smithi143 systemd[1]: Stopped Ceph mon.a for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:15:30.931 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:30 smithi143 systemd[1]: Starting Ceph mon.a for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:15:31.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: set uid:gid to 167:167 (ceph:ceph) 2023-08-28T02:15:31.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: ceph version 18.2.0-187-g9cf20820 (9cf208207dfe23ffccb33308b22e51570cb45e51) reef (stable), process ceph-mon, pid 7 2023-08-28T02:15:31.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: pidfile_write: ignore empty --pid-file 2023-08-28T02:15:31.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: load: jerasure load: lrc 2023-08-28T02:15:31.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: RocksDB version: 7.9.2 2023-08-28T02:15:31.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Git sha 0 2023-08-28T02:15:31.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Compile date 2023-08-25 14:59:12 2023-08-28T02:15:31.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: DB SUMMARY 2023-08-28T02:15:31.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: DB Session ID: ZXXR4BREXB46HP2YBZ09 2023-08-28T02:15:31.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: CURRENT file: CURRENT 2023-08-28T02:15:31.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: IDENTITY file: IDENTITY 2023-08-28T02:15:31.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: MANIFEST file: MANIFEST-000008 size: 780 Bytes 2023-08-28T02:15:31.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000020.sst 2023-08-28T02:15:31.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000018.log size: 2499216 ; 2023-08-28T02:15:31.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.error_if_exists: 0 2023-08-28T02:15:31.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.create_if_missing: 0 2023-08-28T02:15:31.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.paranoid_checks: 1 2023-08-28T02:15:31.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.flush_verify_memtable_count: 1 2023-08-28T02:15:31.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-08-28T02:15:31.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-08-28T02:15:31.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.env: 0x557139faa720 2023-08-28T02:15:31.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.fs: PosixFileSystem 2023-08-28T02:15:31.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.info_log: 0x55713bc9f420 2023-08-28T02:15:31.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.max_file_opening_threads: 16 2023-08-28T02:15:31.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.statistics: (nil) 2023-08-28T02:15:31.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.use_fsync: 0 2023-08-28T02:15:31.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.max_log_file_size: 0 2023-08-28T02:15:31.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-08-28T02:15:31.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.log_file_time_to_roll: 0 2023-08-28T02:15:31.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.keep_log_file_num: 1000 2023-08-28T02:15:31.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.recycle_log_file_num: 0 2023-08-28T02:15:31.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.allow_fallocate: 1 2023-08-28T02:15:31.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.allow_mmap_reads: 0 2023-08-28T02:15:31.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.allow_mmap_writes: 0 2023-08-28T02:15:31.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.use_direct_reads: 0 2023-08-28T02:15:31.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-08-28T02:15:31.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.create_missing_column_families: 0 2023-08-28T02:15:31.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.db_log_dir: 2023-08-28T02:15:31.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.wal_dir: 2023-08-28T02:15:31.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.table_cache_numshardbits: 6 2023-08-28T02:15:31.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.WAL_ttl_seconds: 0 2023-08-28T02:15:31.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.WAL_size_limit_MB: 0 2023-08-28T02:15:31.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-08-28T02:15:31.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-08-28T02:15:31.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.is_fd_close_on_exec: 1 2023-08-28T02:15:31.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.advise_random_on_open: 1 2023-08-28T02:15:31.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.db_write_buffer_size: 0 2023-08-28T02:15:31.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.write_buffer_manager: 0x55713af2ec80 2023-08-28T02:15:31.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-08-28T02:15:31.457 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-08-28T02:15:31.457 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.use_adaptive_mutex: 0 2023-08-28T02:15:31.457 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.rate_limiter: (nil) 2023-08-28T02:15:31.457 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-08-28T02:15:31.457 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.wal_recovery_mode: 2 2023-08-28T02:15:31.457 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.enable_thread_tracking: 0 2023-08-28T02:15:31.457 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.enable_pipelined_write: 0 2023-08-28T02:15:31.457 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.unordered_write: 0 2023-08-28T02:15:31.457 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-08-28T02:15:31.457 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-08-28T02:15:31.457 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-08-28T02:15:31.457 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-08-28T02:15:31.458 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.row_cache: None 2023-08-28T02:15:31.458 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.wal_filter: None 2023-08-28T02:15:31.458 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-08-28T02:15:31.458 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.allow_ingest_behind: 0 2023-08-28T02:15:31.458 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.two_write_queues: 0 2023-08-28T02:15:31.458 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.manual_wal_flush: 0 2023-08-28T02:15:31.458 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.wal_compression: 0 2023-08-28T02:15:31.458 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.atomic_flush: 0 2023-08-28T02:15:31.458 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-08-28T02:15:31.458 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.persist_stats_to_disk: 0 2023-08-28T02:15:31.458 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.write_dbid_to_manifest: 0 2023-08-28T02:15:31.458 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.log_readahead_size: 0 2023-08-28T02:15:31.459 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-08-28T02:15:31.459 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.best_efforts_recovery: 0 2023-08-28T02:15:31.459 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-08-28T02:15:31.459 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-08-28T02:15:31.459 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.allow_data_in_errors: 0 2023-08-28T02:15:31.459 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.db_host_id: __hostname__ 2023-08-28T02:15:31.459 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.enforce_single_del_contracts: true 2023-08-28T02:15:31.459 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.max_background_jobs: 2 2023-08-28T02:15:31.459 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.max_background_compactions: -1 2023-08-28T02:15:31.459 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.max_subcompactions: 1 2023-08-28T02:15:31.459 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-08-28T02:15:31.460 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-08-28T02:15:31.460 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.delayed_write_rate : 16777216 2023-08-28T02:15:31.460 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.max_total_wal_size: 0 2023-08-28T02:15:31.460 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-08-28T02:15:31.460 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.stats_dump_period_sec: 600 2023-08-28T02:15:31.460 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.stats_persist_period_sec: 600 2023-08-28T02:15:31.460 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-08-28T02:15:31.460 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.max_open_files: -1 2023-08-28T02:15:31.460 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.bytes_per_sync: 0 2023-08-28T02:15:31.460 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.wal_bytes_per_sync: 0 2023-08-28T02:15:31.460 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.strict_bytes_per_sync: 0 2023-08-28T02:15:31.460 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.compaction_readahead_size: 0 2023-08-28T02:15:31.460 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.max_background_flushes: -1 2023-08-28T02:15:31.461 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Compression algorithms supported: 2023-08-28T02:15:31.461 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-08-28T02:15:31.461 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: kZSTD supported: 0 2023-08-28T02:15:31.461 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: kXpressCompression supported: 0 2023-08-28T02:15:31.461 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: kLZ4HCCompression supported: 1 2023-08-28T02:15:31.461 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: kZlibCompression supported: 1 2023-08-28T02:15:31.461 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: kSnappyCompression supported: 1 2023-08-28T02:15:31.461 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: kLZ4Compression supported: 1 2023-08-28T02:15:31.461 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: kBZip2Compression supported: 0 2023-08-28T02:15:31.461 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Fast CRC32 supported: Supported on x86 2023-08-28T02:15:31.461 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: DMutex implementation: pthread_mutex_t 2023-08-28T02:15:31.461 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000008 2023-08-28T02:15:31.461 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-08-28T02:15:31.462 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-08-28T02:15:31.462 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.merge_operator: 2023-08-28T02:15:31.462 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.compaction_filter: None 2023-08-28T02:15:31.462 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.compaction_filter_factory: None 2023-08-28T02:15:31.462 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.sst_partitioner_factory: None 2023-08-28T02:15:31.462 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.memtable_factory: SkipListFactory 2023-08-28T02:15:31.462 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.table_factory: BlockBasedTable 2023-08-28T02:15:31.462 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55713bc9f520) 2023-08-28T02:15:31.462 INFO:journalctl@ceph.mon.a.smithi143.stdout: cache_index_and_filter_blocks: 1 2023-08-28T02:15:31.462 INFO:journalctl@ceph.mon.a.smithi143.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-08-28T02:15:31.462 INFO:journalctl@ceph.mon.a.smithi143.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-08-28T02:15:31.462 INFO:journalctl@ceph.mon.a.smithi143.stdout: pin_top_level_index_and_filter: 1 2023-08-28T02:15:31.463 INFO:journalctl@ceph.mon.a.smithi143.stdout: index_type: 0 2023-08-28T02:15:31.463 INFO:journalctl@ceph.mon.a.smithi143.stdout: data_block_index_type: 0 2023-08-28T02:15:31.463 INFO:journalctl@ceph.mon.a.smithi143.stdout: index_shortening: 1 2023-08-28T02:15:31.463 INFO:journalctl@ceph.mon.a.smithi143.stdout: data_block_hash_table_util_ratio: 0.750000 2023-08-28T02:15:31.463 INFO:journalctl@ceph.mon.a.smithi143.stdout: checksum: 4 2023-08-28T02:15:31.463 INFO:journalctl@ceph.mon.a.smithi143.stdout: no_block_cache: 0 2023-08-28T02:15:31.463 INFO:journalctl@ceph.mon.a.smithi143.stdout: block_cache: 0x55713afb11f0 2023-08-28T02:15:31.463 INFO:journalctl@ceph.mon.a.smithi143.stdout: block_cache_name: BinnedLRUCache 2023-08-28T02:15:31.463 INFO:journalctl@ceph.mon.a.smithi143.stdout: block_cache_options: 2023-08-28T02:15:31.463 INFO:journalctl@ceph.mon.a.smithi143.stdout: capacity : 536870912 2023-08-28T02:15:31.463 INFO:journalctl@ceph.mon.a.smithi143.stdout: num_shard_bits : 4 2023-08-28T02:15:31.463 INFO:journalctl@ceph.mon.a.smithi143.stdout: strict_capacity_limit : 0 2023-08-28T02:15:31.464 INFO:journalctl@ceph.mon.a.smithi143.stdout: high_pri_pool_ratio: 0.000 2023-08-28T02:15:31.464 INFO:journalctl@ceph.mon.a.smithi143.stdout: block_cache_compressed: (nil) 2023-08-28T02:15:31.464 INFO:journalctl@ceph.mon.a.smithi143.stdout: persistent_cache: (nil) 2023-08-28T02:15:31.464 INFO:journalctl@ceph.mon.a.smithi143.stdout: block_size: 4096 2023-08-28T02:15:31.464 INFO:journalctl@ceph.mon.a.smithi143.stdout: block_size_deviation: 10 2023-08-28T02:15:31.464 INFO:journalctl@ceph.mon.a.smithi143.stdout: block_restart_interval: 16 2023-08-28T02:15:31.464 INFO:journalctl@ceph.mon.a.smithi143.stdout: index_block_restart_interval: 1 2023-08-28T02:15:31.464 INFO:journalctl@ceph.mon.a.smithi143.stdout: metadata_block_size: 4096 2023-08-28T02:15:31.464 INFO:journalctl@ceph.mon.a.smithi143.stdout: partition_filters: 0 2023-08-28T02:15:31.464 INFO:journalctl@ceph.mon.a.smithi143.stdout: use_delta_encoding: 1 2023-08-28T02:15:31.464 INFO:journalctl@ceph.mon.a.smithi143.stdout: filter_policy: bloomfilter 2023-08-28T02:15:31.464 INFO:journalctl@ceph.mon.a.smithi143.stdout: whole_key_filtering: 1 2023-08-28T02:15:31.465 INFO:journalctl@ceph.mon.a.smithi143.stdout: verify_compression: 0 2023-08-28T02:15:31.465 INFO:journalctl@ceph.mon.a.smithi143.stdout: read_amp_bytes_per_bit: 0 2023-08-28T02:15:31.465 INFO:journalctl@ceph.mon.a.smithi143.stdout: format_version: 5 2023-08-28T02:15:31.465 INFO:journalctl@ceph.mon.a.smithi143.stdout: enable_index_compression: 1 2023-08-28T02:15:31.465 INFO:journalctl@ceph.mon.a.smithi143.stdout: block_align: 0 2023-08-28T02:15:31.465 INFO:journalctl@ceph.mon.a.smithi143.stdout: max_auto_readahead_size: 262144 2023-08-28T02:15:31.465 INFO:journalctl@ceph.mon.a.smithi143.stdout: prepopulate_block_cache: 0 2023-08-28T02:15:31.465 INFO:journalctl@ceph.mon.a.smithi143.stdout: initial_auto_readahead_size: 8192 2023-08-28T02:15:31.465 INFO:journalctl@ceph.mon.a.smithi143.stdout: num_file_reads_for_auto_readahead: 2 2023-08-28T02:15:31.465 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.write_buffer_size: 33554432 2023-08-28T02:15:31.465 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.max_write_buffer_number: 2 2023-08-28T02:15:31.465 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.compression: NoCompression 2023-08-28T02:15:31.465 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.bottommost_compression: Disabled 2023-08-28T02:15:31.466 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.prefix_extractor: nullptr 2023-08-28T02:15:31.466 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-08-28T02:15:31.466 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.num_levels: 7 2023-08-28T02:15:31.466 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-08-28T02:15:31.466 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-08-28T02:15:31.466 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-08-28T02:15:31.466 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-08-28T02:15:31.466 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-08-28T02:15:31.466 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-08-28T02:15:31.466 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-08-28T02:15:31.466 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-08-28T02:15:31.466 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-08-28T02:15:31.467 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-08-28T02:15:31.467 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-08-28T02:15:31.467 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-08-28T02:15:31.467 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.compression_opts.window_bits: -14 2023-08-28T02:15:31.467 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.compression_opts.level: 32767 2023-08-28T02:15:31.467 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.compression_opts.strategy: 0 2023-08-28T02:15:31.467 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-08-28T02:15:31.467 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-08-28T02:15:31.467 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-08-28T02:15:31.467 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-08-28T02:15:31.467 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.compression_opts.enabled: false 2023-08-28T02:15:31.467 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-08-28T02:15:31.468 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-08-28T02:15:31.468 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-08-28T02:15:31.468 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-08-28T02:15:31.468 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.target_file_size_base: 67108864 2023-08-28T02:15:31.468 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.target_file_size_multiplier: 1 2023-08-28T02:15:31.468 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-08-28T02:15:31.468 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-08-28T02:15:31.468 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-08-28T02:15:31.468 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-08-28T02:15:31.468 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-08-28T02:15:31.468 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-08-28T02:15:31.468 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-08-28T02:15:31.468 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-08-28T02:15:31.469 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-08-28T02:15:31.469 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-08-28T02:15:31.469 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-08-28T02:15:31.469 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-08-28T02:15:31.469 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-08-28T02:15:31.469 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.arena_block_size: 1048576 2023-08-28T02:15:31.469 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-08-28T02:15:31.469 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-08-28T02:15:31.469 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.disable_auto_compactions: 0 2023-08-28T02:15:31.469 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-08-28T02:15:31.469 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-08-28T02:15:31.469 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-08-28T02:15:31.470 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-08-28T02:15:31.470 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-08-28T02:15:31.470 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-08-28T02:15:31.470 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-08-28T02:15:31.470 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-08-28T02:15:31.470 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-08-28T02:15:31.470 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-08-28T02:15:31.470 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.table_properties_collectors: 2023-08-28T02:15:31.470 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.inplace_update_support: 0 2023-08-28T02:15:31.470 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.inplace_update_num_locks: 10000 2023-08-28T02:15:31.470 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-08-28T02:15:31.470 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-08-28T02:15:31.471 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.memtable_huge_page_size: 0 2023-08-28T02:15:31.471 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.bloom_locality: 0 2023-08-28T02:15:31.471 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.max_successive_merges: 0 2023-08-28T02:15:31.471 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.optimize_filters_for_hits: 0 2023-08-28T02:15:31.471 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.paranoid_file_checks: 0 2023-08-28T02:15:31.471 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.force_consistency_checks: 1 2023-08-28T02:15:31.471 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.report_bg_io_stats: 0 2023-08-28T02:15:31.471 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.ttl: 2592000 2023-08-28T02:15:31.471 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.periodic_compaction_seconds: 0 2023-08-28T02:15:31.471 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-08-28T02:15:31.471 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-08-28T02:15:31.472 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.enable_blob_files: false 2023-08-28T02:15:31.472 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.min_blob_size: 0 2023-08-28T02:15:31.472 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.blob_file_size: 268435456 2023-08-28T02:15:31.472 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.blob_compression_type: NoCompression 2023-08-28T02:15:31.472 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.enable_blob_garbage_collection: false 2023-08-28T02:15:31.472 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-08-28T02:15:31.472 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-08-28T02:15:31.472 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-08-28T02:15:31.472 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.blob_file_starting_level: 0 2023-08-28T02:15:31.472 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-08-28T02:15:31.472 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 20.sst 2023-08-28T02:15:31.472 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-08-28T02:15:31.473 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000008 succeeded,manifest_file_number is 8, next_file_number is 22, last_sequence is 10064, log_number is 18,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-08-28T02:15:31.473 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 18 2023-08-28T02:15:31.473 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 7a93f734-b23b-40c3-9e85-f3f143b9a64a 2023-08-28T02:15:31.473 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: EVENT_LOG_v1 {"time_micros": 1693188931031617, "job": 1, "event": "recovery_started", "wal_files": [18]} 2023-08-28T02:15:31.473 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #18 mode 2 2023-08-28T02:15:31.473 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: EVENT_LOG_v1 {"time_micros": 1693188931038089, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 23, "file_size": 2370663, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 10056, "largest_seqno": 10623, "table_properties": {"data_size": 2366526, "index_size": 2607, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 517, "raw_key_size": 4439, "raw_average_key_size": 21, "raw_value_size": 2360611, "raw_average_value_size": 11686, "num_data_blocks": 118, "num_entries": 202, "num_filter_entries": 202, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1693188931, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "7a93f734-b23b-40c3-9e85-f3f143b9a64a", "db_session_id": "ZXXR4BREXB46HP2YBZ09", "orig_file_number": 23, "seqno_to_time_mapping": "N/A"}} 2023-08-28T02:15:31.473 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: EVENT_LOG_v1 {"time_micros": 1693188931038183, "job": 1, "event": "recovery_finished"} 2023-08-28T02:15:31.473 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: [db/version_set.cc:5047] Creating manifest 25 2023-08-28T02:15:31.473 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-08-28T02:15:31.473 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000018.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-08-28T02:15:31.473 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55713b04c000 2023-08-28T02:15:31.473 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: DB pointer 0x55713b03a000 2023-08-28T02:15:31.473 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-08-28T02:15:31.474 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-08-28T02:15:31.474 INFO:journalctl@ceph.mon.a.smithi143.stdout: ** DB Stats ** 2023-08-28T02:15:31.474 INFO:journalctl@ceph.mon.a.smithi143.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-08-28T02:15:31.474 INFO:journalctl@ceph.mon.a.smithi143.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-08-28T02:15:31.474 INFO:journalctl@ceph.mon.a.smithi143.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-08-28T02:15:31.474 INFO:journalctl@ceph.mon.a.smithi143.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-08-28T02:15:31.474 INFO:journalctl@ceph.mon.a.smithi143.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-08-28T02:15:31.474 INFO:journalctl@ceph.mon.a.smithi143.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-08-28T02:15:31.474 INFO:journalctl@ceph.mon.a.smithi143.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-08-28T02:15:31.474 INFO:journalctl@ceph.mon.a.smithi143.stdout: 2023-08-28T02:15:31.474 INFO:journalctl@ceph.mon.a.smithi143.stdout: ** Compaction Stats [default] ** 2023-08-28T02:15:31.474 INFO:journalctl@ceph.mon.a.smithi143.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-08-28T02:15:31.475 INFO:journalctl@ceph.mon.a.smithi143.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-08-28T02:15:31.475 INFO:journalctl@ceph.mon.a.smithi143.stdout: L0 1/0 2.26 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 660.9 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-08-28T02:15:31.475 INFO:journalctl@ceph.mon.a.smithi143.stdout: L6 1/0 23.00 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-08-28T02:15:31.475 INFO:journalctl@ceph.mon.a.smithi143.stdout: Sum 2/0 25.27 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 660.9 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-08-28T02:15:31.475 INFO:journalctl@ceph.mon.a.smithi143.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 660.9 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-08-28T02:15:31.475 INFO:journalctl@ceph.mon.a.smithi143.stdout: 2023-08-28T02:15:31.475 INFO:journalctl@ceph.mon.a.smithi143.stdout: ** Compaction Stats [default] ** 2023-08-28T02:15:31.475 INFO:journalctl@ceph.mon.a.smithi143.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-08-28T02:15:31.475 INFO:journalctl@ceph.mon.a.smithi143.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-08-28T02:15:31.475 INFO:journalctl@ceph.mon.a.smithi143.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 660.9 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-08-28T02:15:31.475 INFO:journalctl@ceph.mon.a.smithi143.stdout: 2023-08-28T02:15:31.476 INFO:journalctl@ceph.mon.a.smithi143.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-08-28T02:15:31.476 INFO:journalctl@ceph.mon.a.smithi143.stdout: 2023-08-28T02:15:31.476 INFO:journalctl@ceph.mon.a.smithi143.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-08-28T02:15:31.476 INFO:journalctl@ceph.mon.a.smithi143.stdout: Flush(GB): cumulative 0.002, interval 0.002 2023-08-28T02:15:31.476 INFO:journalctl@ceph.mon.a.smithi143.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-08-28T02:15:31.476 INFO:journalctl@ceph.mon.a.smithi143.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-08-28T02:15:31.476 INFO:journalctl@ceph.mon.a.smithi143.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-08-28T02:15:31.476 INFO:journalctl@ceph.mon.a.smithi143.stdout: AddFile(Keys): cumulative 0, interval 0 2023-08-28T02:15:31.477 INFO:journalctl@ceph.mon.a.smithi143.stdout: Cumulative compaction: 0.00 GB write, 207.61 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-08-28T02:15:31.477 INFO:journalctl@ceph.mon.a.smithi143.stdout: Interval compaction: 0.00 GB write, 207.61 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-08-28T02:15:31.477 INFO:journalctl@ceph.mon.a.smithi143.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-08-28T02:15:31.477 INFO:journalctl@ceph.mon.a.smithi143.stdout: Block cache BinnedLRUCache@0x55713afb11f0#7 capacity: 512.00 MB usage: 3.42 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2023-08-28T02:15:31.477 INFO:journalctl@ceph.mon.a.smithi143.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.59 KB,0.000113249%) IndexBlock(1,2.83 KB,0.000539422%) Misc(1,0.00 KB,0%) 2023-08-28T02:15:31.477 INFO:journalctl@ceph.mon.a.smithi143.stdout: 2023-08-28T02:15:31.477 INFO:journalctl@ceph.mon.a.smithi143.stdout: ** File Read Latency Histogram By Level [default] ** 2023-08-28T02:15:31.477 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: starting mon.a rank 0 at public addrs [v2:172.21.15.143:3300/0,v1:172.21.15.143:6789/0] at bind addrs [v2:172.21.15.143:3300/0,v1:172.21.15.143:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 2023-08-28T02:15:31.477 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: mon.a@-1(???) e3 preinit fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 2023-08-28T02:15:31.477 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: mon.a@-1(???).mds e7 new map 2023-08-28T02:15:31.478 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: mon.a@-1(???).mds e7 print_map 2023-08-28T02:15:31.478 INFO:journalctl@ceph.mon.a.smithi143.stdout: e7 2023-08-28T02:15:31.478 INFO:journalctl@ceph.mon.a.smithi143.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-08-28T02:15:31.478 INFO:journalctl@ceph.mon.a.smithi143.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-08-28T02:15:31.478 INFO:journalctl@ceph.mon.a.smithi143.stdout: legacy client fscid: 1 2023-08-28T02:15:31.478 INFO:journalctl@ceph.mon.a.smithi143.stdout: 2023-08-28T02:15:31.478 INFO:journalctl@ceph.mon.a.smithi143.stdout: Filesystem 'foo' (1) 2023-08-28T02:15:31.478 INFO:journalctl@ceph.mon.a.smithi143.stdout: fs_name foo 2023-08-28T02:15:31.478 INFO:journalctl@ceph.mon.a.smithi143.stdout: epoch 5 2023-08-28T02:15:31.478 INFO:journalctl@ceph.mon.a.smithi143.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-08-28T02:15:31.478 INFO:journalctl@ceph.mon.a.smithi143.stdout: created 2023-08-28T02:11:22.178022+0000 2023-08-28T02:15:31.478 INFO:journalctl@ceph.mon.a.smithi143.stdout: modified 2023-08-28T02:11:30.935128+0000 2023-08-28T02:15:31.479 INFO:journalctl@ceph.mon.a.smithi143.stdout: tableserver 0 2023-08-28T02:15:31.479 INFO:journalctl@ceph.mon.a.smithi143.stdout: root 0 2023-08-28T02:15:31.479 INFO:journalctl@ceph.mon.a.smithi143.stdout: session_timeout 60 2023-08-28T02:15:31.479 INFO:journalctl@ceph.mon.a.smithi143.stdout: session_autoclose 300 2023-08-28T02:15:31.479 INFO:journalctl@ceph.mon.a.smithi143.stdout: max_file_size 1099511627776 2023-08-28T02:15:31.479 INFO:journalctl@ceph.mon.a.smithi143.stdout: required_client_features {} 2023-08-28T02:15:31.479 INFO:journalctl@ceph.mon.a.smithi143.stdout: last_failure 0 2023-08-28T02:15:31.479 INFO:journalctl@ceph.mon.a.smithi143.stdout: last_failure_osd_epoch 0 2023-08-28T02:15:31.480 INFO:journalctl@ceph.mon.a.smithi143.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-08-28T02:15:31.480 INFO:journalctl@ceph.mon.a.smithi143.stdout: max_mds 1 2023-08-28T02:15:31.480 INFO:journalctl@ceph.mon.a.smithi143.stdout: in 0 2023-08-28T02:15:31.480 INFO:journalctl@ceph.mon.a.smithi143.stdout: up {0=14637} 2023-08-28T02:15:31.480 INFO:journalctl@ceph.mon.a.smithi143.stdout: failed 2023-08-28T02:15:31.481 INFO:journalctl@ceph.mon.a.smithi143.stdout: damaged 2023-08-28T02:15:31.481 INFO:journalctl@ceph.mon.a.smithi143.stdout: stopped 2023-08-28T02:15:31.481 INFO:journalctl@ceph.mon.a.smithi143.stdout: data_pools [4] 2023-08-28T02:15:31.481 INFO:journalctl@ceph.mon.a.smithi143.stdout: metadata_pool 3 2023-08-28T02:15:31.481 INFO:journalctl@ceph.mon.a.smithi143.stdout: inline_data disabled 2023-08-28T02:15:31.481 INFO:journalctl@ceph.mon.a.smithi143.stdout: balancer 2023-08-28T02:15:31.481 INFO:journalctl@ceph.mon.a.smithi143.stdout: bal_rank_mask -1 2023-08-28T02:15:31.481 INFO:journalctl@ceph.mon.a.smithi143.stdout: standby_count_wanted 1 2023-08-28T02:15:31.481 INFO:journalctl@ceph.mon.a.smithi143.stdout: [mds.foo.smithi143.spifrj{0:14637} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.143:6834/2211284212,v1:172.21.15.143:6835/2211284212] compat {c=[1],r=[1],i=[7ff]}] 2023-08-28T02:15:31.481 INFO:journalctl@ceph.mon.a.smithi143.stdout: 2023-08-28T02:15:31.482 INFO:journalctl@ceph.mon.a.smithi143.stdout: 2023-08-28T02:15:31.482 INFO:journalctl@ceph.mon.a.smithi143.stdout: Standby daemons: 2023-08-28T02:15:31.482 INFO:journalctl@ceph.mon.a.smithi143.stdout: 2023-08-28T02:15:31.482 INFO:journalctl@ceph.mon.a.smithi143.stdout: [mds.foo.smithi169.yeisur{-1:24520} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.169:6832/3278371000,v1:172.21.15.169:6833/3278371000] compat {c=[1],r=[1],i=[7ff]}] 2023-08-28T02:15:31.482 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: mon.a@-1(???).osd e85 crush map has features 3314933000854323200, adjusting msgr requires 2023-08-28T02:15:31.482 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: mon.a@-1(???).osd e85 crush map has features 432629239337189376, adjusting msgr requires 2023-08-28T02:15:31.482 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: mon.a@-1(???).osd e85 crush map has features 432629239337189376, adjusting msgr requires 2023-08-28T02:15:31.482 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: mon.a@-1(???).osd e85 crush map has features 432629239337189376, adjusting msgr requires 2023-08-28T02:15:31.482 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 ceph-mon[183059]: mon.a@-1(???).paxosservice(auth 1..18) refresh upgraded, format 0 -> 3 2023-08-28T02:15:31.482 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 bash[182994]: 072272663d4f3d43e322332b24cdeb65ec2e8608661d17cd874c4b65d66ffde6 2023-08-28T02:15:31.482 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:15:31 smithi143 systemd[1]: Started Ceph mon.a for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:15:32.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:31 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:15:31] "GET /metrics HTTP/1.1" 200 33948 "" "Prometheus/2.33.4" 2023-08-28T02:15:34.429 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[176053]: ignoring --setuser ceph since I am not root 2023-08-28T02:15:34.429 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[176053]: ignoring --setgroup ceph since I am not root 2023-08-28T02:15:34.429 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:34 smithi143 ceph-mgr[176077]: -- 172.21.15.143:0/3884333013 <== mon.2 v2:172.21.15.143:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5647cb6e51e0 con 0x5647cb6bc800 2023-08-28T02:15:34.429 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[176053]: 2023-08-28T02:15:34.262+0000 7f00ce0cc200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-08-28T02:15:34.430 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[176053]: 2023-08-28T02:15:34.340+0000 7f00ce0cc200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-08-28T02:15:34.430 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[176053]: 2023-08-28T02:15:34.428+0000 7f00ce0cc200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-08-28T02:15:34.430 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: cluster 2023-08-28T02:15:30 2023-08-28T02:15:34.430 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: .419727+0000 mgr.y (mgr.15837) 37 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:34.430 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: audit 2023-08-28T02:15:31.701838+0000 mon.c (mon.2) 110 : audit [DBG] from='client.? 172.21.15.143:0/1220731376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:34.430 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: cluster 2023-08-28T02:15:34.430 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: 2023-08-28T02:15:32.420415+0000 mgr.y 2023-08-28T02:15:34.430 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: (mgr.15837) 38 : cluster [DBG] 2023-08-28T02:15:34.430 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:34.430 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: audit 2023-08-28T02:15:34.431 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: 2023-08-28T02:15 2023-08-28T02:15:34.431 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: :32.617796+0000 mon.c (mon. 2023-08-28T02:15:34.431 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: 2) 111 : audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:15:34.431 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: cluster 2023-08-28T02:15:34.431 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: 2023-08- 2023-08-28T02:15:34.431 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: 28T02:15:34.054802+0000 2023-08-28T02:15:34.431 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: mon.a (mon.0) 1 : cluster 2023-08-28T02:15:34.431 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: [INF] mon.a calling monitor election 2023-08-28T02:15:34.431 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: cluster 2023-08-28T02:15:34.431 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: 2023-08-28T02 2023-08-28T02:15:34.431 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: :15:34.068411+0000 mon.a (mon.0) 2 : cluster 2023-08-28T02:15:34.431 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-08-28T02:15:34.432 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: cluster 2023-08-28T 2023-08-28T02:15:34.432 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: 02:15:34.074670+0000 mon.a (mon.0) 2023-08-28T02:15:34.432 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: 3 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.143:3300/0,v1:172.21.15.143:6789/0],b=[v2:172.21.15.169:3300/0,v1:172.21.15.169:6789/0],c=[v2:172.21.15.143:3301/0,v1:172.21.15.143:6790/0]} removed_ranks: {} 2023-08-28T02:15:34.432 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: cluster 2023-08-28T02:15:34.432 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: 2023-08-28T02:15:34.075224 2023-08-28T02:15:34.432 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: +0000 mon.a (mon.0) 4 : cluster [DBG] fsmap foo:1 {0=foo.smithi143.spifrj=up:active} 1 up:standby 2023-08-28T02:15:34.432 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: cluster 2023-08-28T02:15:34.432 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: 2023-08-28T02: 2023-08-28T02:15:34.432 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: 15:34.075254+0000 mon.a (mon.0) 5 : 2023-08-28T02:15:34.432 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: cluster [DBG] osdmap e85: 8 total, 8 up, 8 in 2023-08-28T02:15:34.432 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: cluster 2023- 2023-08-28T02:15:34.433 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: 08-28T02:15:34.076038+0000 mon.a ( 2023-08-28T02:15:34.433 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: mon.0) 6 : cluster [DBG] mgrmap e29: y(active, since 31s), standbys: x 2023-08-28T02:15:34.433 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: audit 2023-08-28T02:15:34.433 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: 2023-08-28T02:15:34.086734+0000 2023-08-28T02:15:34.433 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: mon.c (mon.2) 112 : audit [DBG] from='client.? 172.21.15.143:0/3095192885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:34.433 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: cluster 2023-08- 2023-08-28T02:15:34.433 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: 28T02:15:34.095975+0000 2023-08-28T02:15:34.433 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: mon.a (mon.0) 7 : cluster 2023-08-28T02:15:34.433 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: [DBG] mgrmap e30: y(active, since 31s), standbys: x 2023-08-28T02:15:34.433 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: cluster 2023-08-28T02:15:34.433 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: 2023-08 2023-08-28T02:15:34.433 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: -28T02:15:34.096025 2023-08-28T02:15:34.433 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: +0000 mon.a (mon.0) 8 2023-08-28T02:15:34.434 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: : cluster [INF] overall HEALTH_OK 2023-08-28T02:15:34.434 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: audit 2023-08 2023-08-28T02:15:34.434 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: -28T02:15:34.096121+0000 mon.a (mon. 2023-08-28T02:15:34.434 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[109923]: 0) 9 : audit [INF] from='mgr.15837 ' entity='' 2023-08-28T02:15:34.434 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[151181]: ignoring --setuser ceph since I am not root 2023-08-28T02:15:34.434 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[151181]: ignoring --setgroup ceph since I am not root 2023-08-28T02:15:34.435 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:34 smithi169 ceph-mgr[151205]: -- 172.21.15.169:0/2993506162 <== mon.0 v2:172.21.15.143:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55f7f3f1d1e0 con 0x55f7f3ef5000 2023-08-28T02:15:34.435 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[151181]: 2023-08-28T02:15:34.262+0000 7f26737fd200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-08-28T02:15:34.435 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[151181]: 2023-08-28T02:15:34.342+0000 7f26737fd200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-08-28T02:15:34.435 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[151181]: 2023-08-28T02:15:34.430+0000 7f26737fd200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-08-28T02:15:34.435 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[100500]: cluster 2023-08-28T02:15:30.419727+0000 mgr.y (mgr. 2023-08-28T02:15:34.435 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[100500]: 15837) 37 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:34.435 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[100500]: audit 2023-08-28T02:15:31.701838+0000 mon.c (mon. 2023-08-28T02:15:34.435 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[100500]: 2) 110 : audit [DBG] from='client.? 172.21.15.143:0/1220731376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:34.435 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[100500]: cluster 2023-08-28T02:15: 2023-08-28T02:15:34.435 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[100500]: 32.420415+0000 mgr.y (mgr.15837) 38 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:34.436 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[100500]: audit 2023-08 2023-08-28T02:15:34.436 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[100500]: -28T02:15:32.617796+0000 mon.c (mon.2) 111 : 2023-08-28T02:15:34.436 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[100500]: audit [DBG] from='mgr.15837 172.21.15.143:0/3087940273' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:15:34.436 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[100500]: cluster 2023-08-28T 2023-08-28T02:15:34.436 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[100500]: 02:15:34.054802+0000 mon.a (mon.0) 1 : cluster 2023-08-28T02:15:34.436 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[100500]: [INF] mon.a calling monitor election 2023-08-28T02:15:34.436 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[100500]: cluster 2023-08-28T02 2023-08-28T02:15:34.436 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[100500]: :15:34.068411+0000 mon.a (mon.0) 2 2023-08-28T02:15:34.436 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[100500]: : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-08-28T02:15:34.436 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[100500]: cluster 2023- 2023-08-28T02:15:34.436 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[100500]: 08-28T02:15:34.074670+0000 mon.a 2023-08-28T02:15:34.436 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[100500]: (mon.0) 3 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.143:3300/0,v1:172.21.15.143:6789/0],b=[v2:172.21.15.169:3300/0,v1:172.21.15.169:6789/0],c=[v2:172.21.15.143:3301/0,v1:172.21.15.143:6790/0]} removed_ranks: {} 2023-08-28T02:15:34.437 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[100500]: cluster 2023 2023-08-28T02:15:34.437 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[100500]: -08-28T02:15:34.075224+0000 2023-08-28T02:15:34.437 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[100500]: mon.a (mon.0) 4 : cluster [DBG] fsmap foo:1 {0=foo.smithi143.spifrj=up:active} 1 up:standby 2023-08-28T02:15:34.437 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[100500]: cluster 2023- 2023-08-28T02:15:34.437 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[100500]: 08-28T02:15:34.075254+0000 mon.a (mon 2023-08-28T02:15:34.437 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[100500]: .0) 5 : cluster [DBG] osdmap e85: 8 total, 8 up, 8 in 2023-08-28T02:15:34.437 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[100500]: cluster 2023-08-28T02:15:34.437 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[100500]: 2023-08-28T02 2023-08-28T02:15:34.437 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[100500]: :15:34.076038+0000 mon.a (mon.0) 6 : 2023-08-28T02:15:34.437 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[100500]: cluster [DBG] mgrmap e29: y(active, since 31s), standbys: x 2023-08-28T02:15:34.437 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[100500]: audit 2023-08 2023-08-28T02:15:34.438 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[100500]: -28T02:15:34.086734+0000 mon.c (mon. 2023-08-28T02:15:34.438 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[100500]: 2) 112 : audit [DBG] from='client.? 172.21.15.143:0/3095192885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:34.438 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[100500]: cluster 2023-08-28T02:15:34.438 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[100500]: 2023-08-28T02: 2023-08-28T02:15:34.438 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[100500]: 15:34.095975+0000 mon.a (mon.0) 7 : cluster 2023-08-28T02:15:34.438 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[100500]: [DBG] mgrmap e30: y(active, since 31s), standbys: x 2023-08-28T02:15:34.438 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[100500]: cluster 2023-08-28T02:15:34.439 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[100500]: 2023-08-28T02:15:34.096025+0000 mon.a ( 2023-08-28T02:15:34.439 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[100500]: mon.0) 8 : cluster [INF] overall HEALTH_OK 2023-08-28T02:15:34.439 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[100500]: audit 2023-08 2023-08-28T02:15:34.439 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[100500]: -28T02:15:34.096121+0000 mon.a (mon.0) 9 2023-08-28T02:15:34.439 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[100500]: : audit [INF] from='mgr.15837 ' entity='' 2023-08-28T02:15:34.690 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[176053]: 2023-08-28T02:15:34.522+0000 7f00ce0cc200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-08-28T02:15:34.691 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[151181]: 2023-08-28T02:15:34.524+0000 7f26737fd200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-08-28T02:15:34.950 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:34 smithi143 conmon[176053]: 2023-08-28T02:15:34.690+0000 7f00ce0cc200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-08-28T02:15:34.987 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:34 smithi169 conmon[151181]: 2023-08-28T02:15:34.692+0000 7f26737fd200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-08-28T02:15:35.364 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:35 smithi143 conmon[176053]: 2023-08-28T02:15:35.050+0000 7f00ce0cc200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-08-28T02:15:35.364 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:35 smithi143 conmon[176053]: 2023-08-28T02:15:35.182+0000 7f00ce0cc200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-08-28T02:15:35.364 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:35 smithi143 conmon[176053]: 2023-08-28T02:15:35.269+0000 7f00ce0cc200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-08-28T02:15:35.368 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:35 smithi169 conmon[151181]: 2023-08-28T02:15:35.053+0000 7f26737fd200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-08-28T02:15:35.368 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:35 smithi169 conmon[151181]: 2023-08-28T02:15:35.187+0000 7f26737fd200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-08-28T02:15:35.368 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:35 smithi169 conmon[151181]: 2023-08-28T02:15:35.273+0000 7f26737fd200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-08-28T02:15:35.737 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:35 smithi169 conmon[151181]: 2023-08-28T02:15:35.368+0000 7f26737fd200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-08-28T02:15:36.203 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:35 smithi143 conmon[176053]: 2023-08-28T02:15:35.364+0000 7f00ce0cc200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-08-28T02:15:36.204 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:35 smithi143 conmon[176053]: 2023-08-28T02:15:35.787+0000 7f00ce0cc200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-08-28T02:15:36.237 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:35 smithi169 conmon[151181]: 2023-08-28T02:15:35.792+0000 7f26737fd200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-08-28T02:15:36.526 INFO:teuthology.orchestra.run.smithi143.stdout: "ceph version 18.2.0-187-g9cf20820 (9cf208207dfe23ffccb33308b22e51570cb45e51) reef (stable)": 1 2023-08-28T02:15:36.708 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:36 smithi143 conmon[176053]: 2023-08-28T02:15:36.454+0000 7f00ce0cc200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-08-28T02:15:36.709 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:36 smithi143 conmon[176053]: 2023-08-28T02:15:36.541+0000 7f00ce0cc200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-08-28T02:15:36.737 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:36 smithi169 conmon[151181]: 2023-08-28T02:15:36.450+0000 7f26737fd200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-08-28T02:15:36.737 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:36 smithi169 conmon[151181]: 2023-08-28T02:15:36.533+0000 7f26737fd200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-08-28T02:15:37.233 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -e sha1=9cf208207dfe23ffccb33308b22e51570cb45e51 -- bash -c 'ceph orch ps' 2023-08-28T02:15:37.303 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:37 smithi169 conmon[100500]: audit 2023-08-28T02:15:36.512247+0000 mon.a (mon.0) 10 : audit [DBG] 2023-08-28T02:15:37.303 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:37 smithi169 conmon[100500]: from='client.? 172.21.15.143:0/481141155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:37.349 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:37 smithi143 conmon[109923]: audit 2023-08-28T02:15:36.512247+0000 mon.a (mon.0) 10 : audit [DBG] from='client.? 172.21.15.143:0/481141155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:37.569 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:37 smithi169 conmon[151181]: 2023-08-28T02:15:37.303+0000 7f26737fd200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-08-28T02:15:37.570 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:37 smithi169 conmon[151181]: 2023-08-28T02:15:37.396+0000 7f26737fd200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-08-28T02:15:37.644 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:37 smithi143 conmon[176053]: 2023-08-28T02:15:37.349+0000 7f00ce0cc200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-08-28T02:15:37.644 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:37 smithi143 conmon[176053]: 2023-08-28T02:15:37.451+0000 7f00ce0cc200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-08-28T02:15:37.858 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:37 smithi169 conmon[151181]: 2023-08-28T02:15:37.569+0000 7f26737fd200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-08-28T02:15:37.947 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:37 smithi143 conmon[176053]: 2023-08-28T02:15:37.644+0000 7f00ce0cc200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-08-28T02:15:38.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:37 smithi143 conmon[176053]: 2023-08-28T02:15:37.947+0000 7f00ce0cc200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-08-28T02:15:38.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:38 smithi143 conmon[176053]: 2023-08-28T02:15:38.187+0000 7f00ce0cc200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-08-28T02:15:38.237 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:37 smithi169 conmon[151181]: 2023-08-28T02:15:37.858+0000 7f26737fd200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-08-28T02:15:38.237 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:38 smithi169 conmon[151181]: 2023-08-28T02:15:38.094+0000 7f26737fd200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-08-28T02:15:38.515 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:38 smithi169 conmon[151181]: 2023-08-28T02:15:38.248+0000 7f26737fd200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-08-28T02:15:38.632 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:38 smithi143 conmon[176053]: 2023-08-28T02:15:38.341+0000 7f00ce0cc200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-08-28T02:15:38.632 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:38 smithi143 conmon[176053]: 2023-08-28T02:15:38.632+0000 7f00ce0cc200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-08-28T02:15:38.987 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:38 smithi169 conmon[151181]: 2023-08-28T02:15:38.515+0000 7f26737fd200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-08-28T02:15:38.987 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:15:38 smithi169 conmon[150127]: ts=2023-08-28T02:15:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi169.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi143:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-08-28T02:15:39.487 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:39 smithi169 conmon[151181]: 2023-08-28T02:15:39.231+0000 7f26737fd200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-08-28T02:15:39.487 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:39 smithi169 conmon[151181]: 2023-08-28T02:15:39.367+0000 7f26737fd200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-08-28T02:15:39.700 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:39 smithi143 conmon[176053]: 2023-08-28T02:15:39.372+0000 7f00ce0cc200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-08-28T02:15:39.700 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:39 smithi143 conmon[176053]: 2023-08-28T02:15:39.507+0000 7f00ce0cc200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-08-28T02:15:39.900 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:39 smithi169 conmon[151181]: 2023-08-28T02:15:39.564+0000 7f26737fd200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-08-28T02:15:39.900 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:39 smithi169 conmon[151181]: 2023-08-28T02:15:39.679+0000 7f26737fd200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-08-28T02:15:40.040 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:39 smithi143 conmon[176053]: 2023-08-28T02:15:39.701+0000 7f00ce0cc200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-08-28T02:15:40.040 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:39 smithi143 conmon[176053]: 2023-08-28T02:15:39.819+0000 7f00ce0cc200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-08-28T02:15:40.237 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:39 smithi169 conmon[151181]: 2023-08-28T02:15:39.900+0000 7f26737fd200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-08-28T02:15:40.450 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:40 smithi143 conmon[176053]: 2023-08-28T02:15:40.040+0000 7f00ce0cc200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-08-28T02:15:41.079 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:40 smithi169 conmon[151181]: 2023-08-28T02:15:40.776+0000 7f26737fd200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-08-28T02:15:41.079 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:40 smithi169 conmon[151181]: [28/Aug/2023:02:15:40] ENGINE Bus STARTING 2023-08-28T02:15:41.079 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:40 smithi169 conmon[151181]: CherryPy Checker: 2023-08-28T02:15:41.080 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:40 smithi169 conmon[151181]: The Application mounted at '' has an empty config. 2023-08-28T02:15:41.080 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:40 smithi169 conmon[151181]: 2023-08-28T02:15:41.080 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:40 smithi169 conmon[151181]: [28/Aug/2023:02:15:40] ENGINE Serving on http://:::9283 2023-08-28T02:15:41.080 INFO:journalctl@ceph.mgr.x.smithi169.stdout:Aug 28 02:15:40 smithi169 conmon[151181]: [28/Aug/2023:02:15:40] ENGINE Bus STARTED 2023-08-28T02:15:41.080 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:40 smithi169 conmon[100500]: cluster 2023-08-28T02:15:40.791153+0000 mon.a (mon.0) 11 : 2023-08-28T02:15:41.080 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:40 smithi169 conmon[100500]: cluster [DBG] Standby manager daemon x restarted 2023-08-28T02:15:41.080 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:40 smithi169 conmon[100500]: cluster 2023-08-28T02:15:40.791249+0000 mon.a (mon.0) 12 : cluster [DBG] Standby manager daemon x started 2023-08-28T02:15:41.080 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:40 smithi169 conmon[100500]: audit 2023-08-28T02:15:40.793088+0000 mon.b 2023-08-28T02:15:41.080 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:40 smithi169 conmon[100500]: (mon.1) 108 : audit [DBG] from='mgr.? 172.21.15.169:0/2165535497' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-08-28T02:15:41.080 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:40 smithi169 conmon[100500]: audit 2023-08-28T02:15:40.793521+0000 mon.b (mon.1 2023-08-28T02:15:41.081 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:40 smithi169 conmon[100500]: ) 109 : audit [DBG] from='mgr.? 172.21.15.169:0/2165535497' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-28T02:15:41.081 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:40 smithi169 conmon[100500]: audit 2023-08-28T02 2023-08-28T02:15:41.081 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:40 smithi169 conmon[100500]: :15:40.795911+0000 mon.b (mon.1) 110 : audit [DBG] from='mgr.? 172.21.15.169:0/2165535497' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-08-28T02:15:41.081 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:40 smithi169 conmon[100500]: audit 2023-08-28T02:15:40.796432 2023-08-28T02:15:41.081 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:40 smithi169 conmon[100500]: +0000 mon.b (mon.1) 111 : audit [DBG] from='mgr.? 172.21.15.169:0/2165535497' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-08-28T02:15:41.081 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:40 smithi169 conmon[100500]: debug 2023-08-28T02:15:40.932+0000 7f2d441e6700 -1 mon.b@1(peon).osd e86 update_from_paxos full map CRC mismatch, resetting to canonical 2023-08-28T02:15:41.149 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:40 smithi143 conmon[109923]: cluster 2023-08-28T02:15:40.791153+0000 mon.a (mon.0) 11 : cluster [DBG] Standby manager daemon x restarted 2023-08-28T02:15:41.149 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:40 smithi143 conmon[109923]: 2023-08-28T02:15:41.149 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:40 smithi143 conmon[109923]: cluster 2023-08-28T02:15:40.791249+0000 mon.a (mon.0) 12 : cluster [DBG] Standby manager daemon x started 2023-08-28T02:15:41.149 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:40 smithi143 conmon[109923]: audit 2023-08-28T02:15:40.793088+0000 2023-08-28T02:15:41.149 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:40 smithi143 conmon[109923]: mon.b (mon.1) 108 : audit [DBG] from='mgr.? 172.21.15.169:0/2165535497' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-08-28T02:15:41.149 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:40 smithi143 conmon[109923]: 2023-08-28T02:15:41.149 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:40 smithi143 conmon[109923]: audit 2023-08 2023-08-28T02:15:41.149 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:40 smithi143 conmon[109923]: -28T02:15:40.793521+0000 mon.b (mon. 2023-08-28T02:15:41.149 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:40 smithi143 conmon[109923]: 1) 109 : audit [DBG] from='mgr.? 172.21.15.169:0/2165535497' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-28T02:15:41.150 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:40 smithi143 conmon[109923]: audit 2023-08 2023-08-28T02:15:41.150 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:40 smithi143 conmon[109923]: -28T02:15:40.795911+0000 mon.b ( 2023-08-28T02:15:41.150 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:40 smithi143 conmon[109923]: mon.1) 110 : audit [DBG] 2023-08-28T02:15:41.150 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:40 smithi143 conmon[109923]: from='mgr.? 172.21.15.169:0/2165535497' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-08-28T02:15:41.150 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:40 smithi143 conmon[109923]: audit 2023-08-28T02:15:41.150 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:40 smithi143 conmon[109923]: 2023-08-28T02:15:40.796432 2023-08-28T02:15:41.150 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:40 smithi143 conmon[109923]: +0000 mon.b (mon.1) 111 : 2023-08-28T02:15:41.150 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:40 smithi143 conmon[109923]: audit [DBG] from='mgr.? 172.21.15.169:0/2165535497' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-08-28T02:15:41.150 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:40 smithi143 conmon[109923]: debug 2023-08-28T02:15:40.932+0000 7f1b8a44c700 -1 mon.c@2(peon).osd e86 update_from_paxos full map CRC mismatch, resetting to canonical 2023-08-28T02:15:41.150 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:40 smithi143 conmon[176053]: 2023-08-28T02:15:40.916+0000 7f00ce0cc200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-08-28T02:15:41.150 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[176053]: [28/Aug/2023:02:15:41] ENGINE Bus STARTING 2023-08-28T02:15:41.151 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[176053]: 2023-08-28T02:15:41.065+0000 7f00284f7700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-28T02:15:41.151 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[176053]: 2023-08-28T02:15:41.065+0000 7f00284f7700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-28T02:15:41.151 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[176053]: 2023-08-28T02:15:41.065+0000 7f00284f7700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-28T02:15:41.151 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[176053]: 2023-08-28T02:15:41.065+0000 7f00284f7700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-28T02:15:41.151 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[176053]: 2023-08-28T02:15:41.065+0000 7f00284f7700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-28T02:15:41.151 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[176053]: 2023-08-28T02:15:41.065+0000 7f002e503700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-28T02:15:41.151 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[176053]: 2023-08-28T02:15:41.065+0000 7f002e503700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-28T02:15:41.151 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[176053]: 2023-08-28T02:15:41.065+0000 7f002e503700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-28T02:15:41.151 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[176053]: 2023-08-28T02:15:41.065+0000 7f002e503700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-28T02:15:41.151 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[176053]: 2023-08-28T02:15:41.065+0000 7f002e503700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-28T02:15:41.151 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[176053]: CherryPy Checker: 2023-08-28T02:15:41.151 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[176053]: The Application mounted at '' has an empty config. 2023-08-28T02:15:41.152 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[176053]: 2023-08-28T02:15:41.152 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[176053]: [28/Aug/2023:02:15:41] ENGINE Serving on http://:::9283 2023-08-28T02:15:41.152 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[176053]: [28/Aug/2023:02:15:41] ENGINE Bus STARTED 2023-08-28T02:15:41.966 INFO:teuthology.orchestra.run.smithi143.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-08-28T02:15:41.966 INFO:teuthology.orchestra.run.smithi143.stdout:alertmanager.a smithi143 *:9093,9094 running (86s) 36s ago 4m 18.4M - 0.23.0 ba2b418f427c f216a6d30bb2 2023-08-28T02:15:41.966 INFO:teuthology.orchestra.run.smithi143.stdout:grafana.a smithi169 *:3000 running (51s) 19s ago 4m 45.5M - 8.3.5 dad864ee21e9 4c9eb6ad00e1 2023-08-28T02:15:41.966 INFO:teuthology.orchestra.run.smithi143.stdout:mds.foo.smithi143.spifrj smithi143 running (4m) 36s ago 4m 16.6M - 16.2.13-565-g595355f4 4099d2e00414 4eeedd3bc917 2023-08-28T02:15:41.966 INFO:teuthology.orchestra.run.smithi143.stdout:mds.foo.smithi169.yeisur smithi169 running (4m) 19s ago 4m 17.4M - 16.2.13-565-g595355f4 4099d2e00414 fed9013c1357 2023-08-28T02:15:41.966 INFO:teuthology.orchestra.run.smithi143.stdout:mgr.x smithi169 *:8443,9283,8765 running (22s) 19s ago 8m 44.1M - 18.2.0-187-g9cf20820 62b871dea10e 73a560edcde9 2023-08-28T02:15:41.966 INFO:teuthology.orchestra.run.smithi143.stdout:mgr.y smithi143 *:8443,9283,8765 running (47s) 36s ago 10m 475M - 18.2.0-187-g9cf20820 62b871dea10e 8de2036a51e3 2023-08-28T02:15:41.967 INFO:teuthology.orchestra.run.smithi143.stdout:mon.a smithi143 running (10m) 36s ago 10m 119M 2048M 16.2.13-565-g595355f4 4099d2e00414 3e714a619127 2023-08-28T02:15:41.967 INFO:teuthology.orchestra.run.smithi143.stdout:mon.b smithi169 running (9m) 19s ago 9m 90.7M 2048M 16.2.13-565-g595355f4 4099d2e00414 f70db2856b37 2023-08-28T02:15:41.967 INFO:teuthology.orchestra.run.smithi143.stdout:mon.c smithi143 running (9m) 36s ago 9m 86.7M 2048M 16.2.13-565-g595355f4 4099d2e00414 74b6ddae7a3f 2023-08-28T02:15:41.967 INFO:teuthology.orchestra.run.smithi143.stdout:node-exporter.a smithi143 *:9100 running (94s) 36s ago 5m 22.3M - 1.3.1 1dbe0e931976 e389cc167c39 2023-08-28T02:15:41.967 INFO:teuthology.orchestra.run.smithi143.stdout:node-exporter.b smithi169 *:9100 running (72s) 19s ago 5m 18.5M - 1.3.1 1dbe0e931976 5e5215d11ac8 2023-08-28T02:15:41.967 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0 smithi143 running (8m) 36s ago 8m 68.8M 2500M 16.2.13-565-g595355f4 4099d2e00414 ce7c7eb91333 2023-08-28T02:15:41.967 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1 smithi143 running (8m) 36s ago 8m 85.6M 2500M 16.2.13-565-g595355f4 4099d2e00414 f27082b59650 2023-08-28T02:15:41.967 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2 smithi143 running (7m) 36s ago 7m 64.4M 2500M 16.2.13-565-g595355f4 4099d2e00414 e20c9920166c 2023-08-28T02:15:41.967 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3 smithi143 running (7m) 36s ago 7m 61.8M 2500M 16.2.13-565-g595355f4 4099d2e00414 ab95599724c4 2023-08-28T02:15:41.967 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4 smithi169 running (7m) 19s ago 7m 67.0M 2500M 16.2.13-565-g595355f4 4099d2e00414 e6ed776def27 2023-08-28T02:15:41.967 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5 smithi169 running (6m) 19s ago 6m 80.9M 2500M 16.2.13-565-g595355f4 4099d2e00414 6fe1195fca7e 2023-08-28T02:15:41.967 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6 smithi169 running (6m) 19s ago 6m 75.5M 2500M 16.2.13-565-g595355f4 4099d2e00414 12854ca48e76 2023-08-28T02:15:41.967 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7 smithi169 running (5m) 19s ago 5m 91.2M 2500M 16.2.13-565-g595355f4 4099d2e00414 64fa1422c85a 2023-08-28T02:15:41.968 INFO:teuthology.orchestra.run.smithi143.stdout:prometheus.a smithi169 *:9095 running (27s) 19s ago 5m 44.2M - 2.33.4 514e6a882f6e cd763ce7d282 2023-08-28T02:15:42.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:15:41] "GET /metrics HTTP/1.1" 200 31245 "" "Prometheus/2.33.4" 2023-08-28T02:15:42.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[176053]: 2023-08-28T02:15:41.943+0000 7f008786b700 -1 mgr.server handle_report got status from non-daemon mon.a 2023-08-28T02:15:42.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: cluster 2023-08-28T02:15:40.847041+0000 mon.a (mon.0 2023-08-28T02:15:42.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: ) 13 : cluster [DBG] mgrmap e31: y(active, since 38s), standbys: x 2023-08-28T02:15:42.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: cluster 2023-08-28T02:15:40.929743+0000 mon.a (mon.0) 14 : cluster [INF] Active manager daemon y restarted 2023-08-28T02:15:42.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: cluster 2023-08-28T02:15:40.930087+0000 mon.a (mon.0) 15 : cluster [INF] Activating manager daemon y 2023-08-28T02:15:42.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: cluster 2023-08-28T02:15:40.936605+0000 mon.a (mon.0) 16 : cluster [DBG] osdmap e86: 8 total, 8 up, 8 in 2023-08-28T02:15:42.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: cluster 2023-08-28T02:15:40.936929+0000 osd.1 (osd.1) 3 : cluster [WRN] failed to encode map e86 with expected crc 2023-08-28T02:15:42.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: cluster 2023-08-28T02:15:40.936996+0000 mon.a (mon.0) 17 : cluster [DBG] mgrmap e32: y(active, starting, since 0.00701262s), standbys: x 2023-08-28T02:15:42.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: cluster 2023-08-28T02:15:40.938503+0000 osd.2 (osd.2) 3 : cluster [WRN] failed to encode map e86 with expected crc 2023-08-28T02:15:42.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: cluster 2023- 2023-08-28T02:15:42.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: 08-28T02:15:40.941711+0000 2023-08-28T02:15:42.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: osd.4 (osd.4) 3 : cluster [WRN] 2023-08-28T02:15:42.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: failed to encode map e86 with expected crc 2023-08-28T02:15:42.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: audit 2023-08- 2023-08-28T02:15:42.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: 28T02:15:40.943154+0000 2023-08-28T02:15:42.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: mon.c (mon.2) 113 : audit [DBG] 2023-08-28T02:15:42.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T02:15:42.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: audit 2023-08-28 2023-08-28T02:15:42.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: T02:15:40.943377+0000 2023-08-28T02:15:42.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: mon.c (mon.2) 114 : audit [DBG] 2023-08-28T02:15:42.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T02:15:42.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:15:42.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: 08-28T02:15:40.943596 2023-08-28T02:15:42.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: +0000 mon.c (mon.2) 115 : audit 2023-08-28T02:15:42.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T02:15:42.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:15:42.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: 08-28T02:15:40.943827 2023-08-28T02:15:42.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: +0000 mon.c (mon.2) 116 : audit 2023-08-28T02:15:42.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi143.spifrj"}]: dispatch 2023-08-28T02:15:42.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: audit 2023-08 2023-08-28T02:15:42.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: -28T02:15:40.944060+0000 2023-08-28T02:15:42.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: mon.c (mon.2) 117 : audit [DBG] 2023-08-28T02:15:42.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi169.yeisur"}]: dispatch 2023-08-28T02:15:42.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: audit 2023-08-28T02:15:42.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: 2023-08- 2023-08-28T02:15:42.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: 28T02:15:40.945325+0000 2023-08-28T02:15:42.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: mon.c (mon.2) 118 : audit 2023-08-28T02:15:42.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-08-28T02:15:42.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: audit 2023-08 2023-08-28T02:15:42.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: -28T02:15:40.945569+0000 2023-08-28T02:15:42.205 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: mon.c (mon.2) 119 : 2023-08-28T02:15:42.205 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-08-28T02:15:42.205 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:15:42.205 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: 08-28T02:15:40. 2023-08-28T02:15:42.205 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: 945812+0000 mon.c (mon.2) 120 2023-08-28T02:15:42.205 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-28T02:15:42.205 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: audit 2023 2023-08-28T02:15:42.205 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: -08-28T02:15:40. 2023-08-28T02:15:42.205 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: 946142+0000 mon.c (mon.2) 121 2023-08-28T02:15:42.205 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-28T02:15:42.205 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: audit 2023 2023-08-28T02:15:42.205 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: -08-28T02:15:40 2023-08-28T02:15:42.205 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: .946436+0000 mon.c (mon.2) 2023-08-28T02:15:42.206 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: 122 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-28T02:15:42.206 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: audit 2023 2023-08-28T02:15:42.206 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: -08-28T02:15:40.946718 2023-08-28T02:15:42.206 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: +0000 mon.c (mon.2) 123 : 2023-08-28T02:15:42.206 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-28T02:15:42.206 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: audit 2023 2023-08-28T02:15:42.206 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: -08-28T02:15:40.947030 2023-08-28T02:15:42.206 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: +0000 mon.c (mon.2) 124 : 2023-08-28T02:15:42.206 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-28T02:15:42.206 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: audit 2023-08-28T02:15:42.206 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: 2023-08-28T02:15 2023-08-28T02:15:42.207 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: :40.947288+0000 mon.c (mon. 2023-08-28T02:15:42.207 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: 2) 125 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-28T02:15:42.207 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:15:42.207 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: 08-28T02:15:40.947537 2023-08-28T02:15:42.207 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: +0000 mon.c (mon.2) 126 : 2023-08-28T02:15:42.207 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-28T02:15:42.207 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: audit 2023-08-28T02:15:42.207 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: 2023-08-28T02:15:40. 2023-08-28T02:15:42.207 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: 947783+0000 mon.c (mon.2) 127 2023-08-28T02:15:42.207 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-28T02:15:42.207 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: audit 2023- 2023-08-28T02:15:42.208 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: 08-28T02:15:40. 2023-08-28T02:15:42.208 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: 947993+0000 mon.c (mon.2) 2023-08-28T02:15:42.208 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: 128 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-08-28T02:15:42.208 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: audit 2023 2023-08-28T02:15:42.208 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: -08-28T02:15:40. 2023-08-28T02:15:42.208 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: 948321+0000 mon.c (mon.2) 2023-08-28T02:15:42.208 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: 129 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-08-28T02:15:42.208 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: audit 2023 2023-08-28T02:15:42.208 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: -08-28T02:15:40.948755 2023-08-28T02:15:42.208 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: +0000 mon.c (mon.2) 130 2023-08-28T02:15:42.208 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-08-28T02:15:42.208 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: cluster 2023 2023-08-28T02:15:42.209 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: -08-28T02:15:40. 2023-08-28T02:15:42.209 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: 967442+0000 mon.a (mon.0) 2023-08-28T02:15:42.209 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: 18 : cluster [INF] Manager daemon y is now available 2023-08-28T02:15:42.209 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: audit 2023-08-28T02:15:42.209 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: 2023-08-28T02:15 2023-08-28T02:15:42.209 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: :40.988778+0000 mon.c (mon 2023-08-28T02:15:42.209 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: .2) 131 : audit [DBG] 2023-08-28T02:15:42.209 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:15:42.209 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: audit 2023-08-28T02:15:42.209 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: 2023-08-28T02: 2023-08-28T02:15:42.209 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: 15:40.991195+0000 mon.c (mon 2023-08-28T02:15:42.210 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: .2) 132 : audit [DBG] 2023-08-28T02:15:42.210 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:15:42.210 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: audit 2023-08-28T02:15:42.210 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: 2023-08-28T02:15: 2023-08-28T02:15:42.210 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: 41.009687+0000 mon.c (mon. 2023-08-28T02:15:42.210 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: 2) 133 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:15:42.210 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: audit 2023 2023-08-28T02:15:42.210 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: -08-28T02:15:41 2023-08-28T02:15:42.210 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: .020239+0000 mon.c (mon.2 2023-08-28T02:15:42.210 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: ) 134 : audit [INF] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-08-28T02:15:42.210 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: audit 2023 2023-08-28T02:15:42.211 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: -08-28T02:15:41 2023-08-28T02:15:42.211 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: .020618+0000 mon.a (mon.0 2023-08-28T02:15:42.211 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: ) 19 : audit [INF] from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-08-28T02:15:42.211 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: cluster 2023 2023-08-28T02:15:42.211 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: -08-28T02:15:41.030993 2023-08-28T02:15:42.211 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: +0000 osd.5 (osd.5) 2023-08-28T02:15:42.211 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: 3 : cluster [WRN] failed to encode map e86 with expected crc 2023-08-28T02:15:42.211 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: audit 2023 2023-08-28T02:15:42.211 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: -08-28T02:15: 2023-08-28T02:15:42.211 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: 41.041049+0000 mon.c (mon. 2023-08-28T02:15:42.211 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: 2) 135 : audit [INF] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-08-28T02:15:42.211 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: audit 2023 2023-08-28T02:15:42.212 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: -08-28T02:15:41. 2023-08-28T02:15:42.212 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: 041536+0000 mon.a (mon.0 2023-08-28T02:15:42.212 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:41 smithi143 conmon[109923]: ) 20 : audit [INF] from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-08-28T02:15:42.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: cluster 2023-08-28T02:15:40.847041+0000 mon.a (mon.0) 13 : cluster 2023-08-28T02:15:42.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: [DBG] mgrmap e31: y(active, since 38s), standbys: x 2023-08-28T02:15:42.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: cluster 2023-08-28T02:15:40.929743+0000 mon.a (mon.0) 14 : cluster [INF] Active manager daemon y restarted 2023-08-28T02:15:42.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: cluster 2023-08-28T02:15:40.930087+0000 mon.a (mon.0) 15 : cluster [INF] Activating manager daemon y 2023-08-28T02:15:42.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: cluster 2023-08-28T02:15:40.936605+0000 mon.a (mon.0) 16 : cluster [DBG] osdmap e86: 8 total, 8 up, 8 in 2023-08-28T02:15:42.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: cluster 2023-08-28T02:15:40.936929+0000 osd.1 (osd.1) 3 : cluster [WRN] failed to encode map e86 with expected crc 2023-08-28T02:15:42.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: cluster 2023-08-28T02:15:40.936996+0000 mon.a (mon.0) 17 : cluster [DBG] mgrmap e32: y(active, starting, since 0.00701262s), standbys: x 2023-08-28T02:15:42.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: cluster 2023-08-28T02:15:40.938503+0000 osd.2 (osd.2) 3 : cluster [WRN] failed to encode map e86 with expected crc 2023-08-28T02:15:42.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: cluster 2023-08-28T02:15:40.941711+0000 osd.4 (osd.4) 3 : cluster [WRN] failed to encode map e86 with expected crc 2023-08-28T02:15:42.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: audit 2023-08-28T02:15:40.943154+0000 mon.c (mon.2) 113 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T02:15:42.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: audit 2023-08-28T02:15:40.943377+0000 mon.c (mon.2) 114 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T02:15:42.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: audit 2023-08-28T02:15:40.943596+0000 mon.c (mon.2) 115 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T02:15:42.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: audit 2023-08-28T02:15:40.943827+0000 mon.c (mon.2) 116 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi143.spifrj"}]: dispatch 2023-08-28T02:15:42.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: audit 2023-08-28T02:15:40.944060+0000 mon.c (mon.2) 117 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi169.yeisur"}]: dispatch 2023-08-28T02:15:42.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: audit 2023-08-28T02:15:40.945325+0000 mon.c (mon.2) 118 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-08-28T02:15:42.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: audit 2023-08-28T02:15:40.945569+0000 mon.c (mon.2) 119 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-08-28T02:15:42.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: audit 2023-08-28T02:15:40.945812+0000 mon.c (mon.2) 120 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-28T02:15:42.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: audit 2023-08-28T02:15:40.946142+0000 mon.c (mon.2) 121 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-28T02:15:42.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: audit 2023-08-28T02:15:40.946436+0000 mon.c (mon.2) 122 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-28T02:15:42.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: audit 2023-08-28T02:15:40.946718+0000 mon.c (mon.2) 123 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-28T02:15:42.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: audit 2023-08-28 2023-08-28T02:15:42.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: T02:15:40.947030+0000 mon.c (mon.2) 124 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-28T02:15:42.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: audit 2023-08-28T02:15:40.947288+0000 mon.c (mon.2) 125 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-28T02:15:42.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: audit 2023-08-28T02:15:40.947537+0000 mon.c (mon.2) 126 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-28T02:15:42.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: audit 2023 2023-08-28T02:15:42.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: -08-28T02:15:40.947783+0000 mon.c (mon.2) 127 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-28T02:15:42.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: audit 2023-08- 2023-08-28T02:15:42.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: 28T02:15:40.947993+0000 mon.c (mon 2023-08-28T02:15:42.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: .2) 128 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-08-28T02:15:42.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: audit 2023-08-28T02:15:42.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: 2023-08-28T02:15:40 2023-08-28T02:15:42.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: .948321+0000 mon.c (mon.2) 129 : audit [DBG] 2023-08-28T02:15:42.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-08-28T02:15:42.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: audit 2023-08-28T02 2023-08-28T02:15:42.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: :15:40.948755+0000 mon.c (mon.2) 2023-08-28T02:15:42.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: 130 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-08-28T02:15:42.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: cluster 2023-08-28T02:15:42.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: 2023-08-28T02:15:40.967442 2023-08-28T02:15:42.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: +0000 mon.a (mon.0) 18 : cluster [INF] 2023-08-28T02:15:42.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: Manager daemon y is now available 2023-08-28T02:15:42.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: audit 2023-08-28T02:15:40 2023-08-28T02:15:42.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: .988778+0000 mon.c (mon.2) 131 : audit 2023-08-28T02:15:42.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:15:42.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: audit 2023-08-28T02:15 2023-08-28T02:15:42.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: :40.991195+0000 mon.c (mon.2) 132 2023-08-28T02:15:42.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:15:42.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: audit 2023- 2023-08-28T02:15:42.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: 08-28T02:15:41.009687+0000 mon.c 2023-08-28T02:15:42.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: (mon.2) 133 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:15:42.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: audit 2023-08-28T02:15:42.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: 2023-08-28T02:15:41.020239 2023-08-28T02:15:42.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: +0000 mon.c (mon.2) 134 : audit [INF] 2023-08-28T02:15:42.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-08-28T02:15:42.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: audit 2023-08-28T02:15:42.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: 2023-08-28T02:15:41.020618+0000 mon.a ( 2023-08-28T02:15:42.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: mon.0) 19 : audit [INF] from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-08-28T02:15:42.243 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: cluster 2023-08-28T02:15:42.243 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: 2023-08-28T02:15:41.030993 2023-08-28T02:15:42.243 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: +0000 osd.5 (osd.5) 3 : cluster [WRN] 2023-08-28T02:15:42.243 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: failed to encode map e86 with expected crc 2023-08-28T02:15:42.243 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: audit 2023-08-28T02:15: 2023-08-28T02:15:42.243 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: 41.041049+0000 mon.c (mon.2) 135 : 2023-08-28T02:15:42.243 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: audit [INF] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-08-28T02:15:42.243 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: audit 2023-08-28T02: 2023-08-28T02:15:42.243 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: 15:41.041536+0000 mon.a (mon.0) 2023-08-28T02:15:42.243 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:41 smithi169 conmon[100500]: 20 : audit [INF] from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-08-28T02:15:42.789 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -e sha1=9cf208207dfe23ffccb33308b22e51570cb45e51 -- bash -c 'ceph orch upgrade pause' 2023-08-28T02:15:43.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:42 smithi143 conmon[109923]: cephadm 2023-08-28T02:15:41.674615+0000 mgr.y (mgr.34107) 1 : cephadm [INF] [28/Aug/2023:02:15:41] ENGINE Bus STARTING 2023-08-28T02:15:43.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:42 smithi143 conmon[109923]: cephadm 2023-08-28T02:15:41.776467+0000 mgr.y (mgr.34107) 2 : cephadm [INF] [28/Aug/2023:02:15:41] ENGINE Serving on http://172.21.15.143:8765 2023-08-28T02:15:43.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:42 smithi143 conmon[109923]: cephadm 2023-08-28T02:15:41.890537+0000 mgr.y (mgr.34107) 3 : cephadm [INF] [28/Aug/2023:02:15:41] ENGINE Serving on https://172.21.15.143:7150 2023-08-28T02:15:43.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:42 smithi143 conmon[109923]: cephadm 2023-08-28T02:15:41.890659+0000 mgr.y (mgr.34107) 4 : cephadm [INF] [28/Aug/2023:02:15:41] ENGINE Bus STARTED 2023-08-28T02:15:43.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:42 smithi143 conmon[109923]: cluster 2023-08-28T02:15:41.942683+0000 mon.a (mon.0) 21 : cluster [DBG] mgrmap e33: y(active, since 1.0127s), standbys: x 2023-08-28T02:15:43.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:42 smithi143 conmon[109923]: audit 2023-08-28T02:15:41.947885+0000 mgr.y (mgr.34107) 5 : audit [DBG] from='client.25919 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:15:43.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:42 smithi143 conmon[109923]: cluster 2023-08-28T02:15:41.950461+0000 mgr.y (mgr.34107) 6 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:43.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:42 smithi169 conmon[100500]: cephadm 2023-08-28T02:15:41.674615+0000 mgr.y (mgr.34107) 1 : cephadm [INF] [28/Aug/2023:02:15:41] ENGINE Bus STARTING 2023-08-28T02:15:43.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:42 smithi169 conmon[100500]: cephadm 2023-08-28T02:15:41.776467+0000 mgr.y (mgr.34107) 2 : cephadm [INF] [28/Aug/2023:02:15:41] ENGINE Serving on http://172.21.15.143:8765 2023-08-28T02:15:43.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:42 smithi169 conmon[100500]: cephadm 2023-08-28T02:15:41.890537+0000 mgr.y (mgr.34107) 3 : cephadm [INF] [28/Aug/2023:02:15:41] ENGINE Serving on https://172.21.15.143:7150 2023-08-28T02:15:43.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:42 smithi169 conmon[100500]: cephadm 2023-08-28T02:15:41.890659+0000 mgr.y (mgr.34107) 4 : cephadm [INF] [28/Aug/2023:02:15:41] ENGINE Bus STARTED 2023-08-28T02:15:43.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:42 smithi169 conmon[100500]: cluster 2023-08-28T02:15:41.942683+0000 mon.a (mon.0) 21 : cluster [DBG] mgrmap e33: y(active, since 1.0127s), standbys: x 2023-08-28T02:15:43.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:42 smithi169 conmon[100500]: audit 2023-08-28T02:15:41.947885+0000 mgr.y (mgr.34107) 5 : audit [DBG] from='client.25919 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:15:43.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:42 smithi169 conmon[100500]: cluster 2023-08-28T02:15:41.950461+0000 mgr.y (mgr.34107) 6 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:43.949 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:43 smithi143 conmon[109923]: cluster 2023-08-28T02:15:42.947054+0000 mgr.y (mgr.34107) 7 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:44.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:43 smithi169 conmon[100500]: cluster 2023-08-28T02:15:42.947054+0000 mgr.y (mgr.34107) 7 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:44.886 INFO:teuthology.orchestra.run.smithi143.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8 2023-08-28T02:15:45.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:44 smithi143 conmon[109923]: cluster 2023-08-28T02:15:43.953472+0000 mon.a (mon.0) 22 : cluster [DBG] mgrmap e34: y(active, since 3s), standbys: x 2023-08-28T02:15:45.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:44 smithi143 conmon[109923]: audit 2023-08-28T02:15:44.603663+0000 mon.a (mon.0) 23 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:15:45.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:44 smithi143 conmon[109923]: audit 2023-08-28T02:15:44.611674+0000 mon.a (mon.0) 24 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:15:45.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:44 smithi143 conmon[109923]: audit 2023-08-28T02:15:44.886728+0000 mon.a (mon.0) 25 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:15:45.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:44 smithi169 conmon[100500]: cluster 2023-08-28T02:15:43.953472+0000 mon.a ( 2023-08-28T02:15:45.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:44 smithi169 conmon[100500]: mon.0) 22 : cluster [DBG] mgrmap e34: y(active, since 3s), standbys: x 2023-08-28T02:15:45.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:44 smithi169 conmon[100500]: audit 2023-08-28T02:15:44.603663+0000 mon.a (mon.0) 23 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:15:45.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:44 smithi169 conmon[100500]: audit 2023-08-28T02:15:44.611674+0000 mon.a (mon 2023-08-28T02:15:45.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:44 smithi169 conmon[100500]: .0) 24 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:15:45.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:44 smithi169 conmon[100500]: audit 2023- 2023-08-28T02:15:45.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:44 smithi169 conmon[100500]: 08-28T02:15:44.886728+0000 mon.a ( 2023-08-28T02:15:45.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:44 smithi169 conmon[100500]: mon.0) 25 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:15:45.698 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -e sha1=9cf208207dfe23ffccb33308b22e51570cb45e51 -- bash -c 'sleep 60' 2023-08-28T02:15:46.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:46 smithi143 conmon[109923]: audit 2023-08-28T02:15:44.882873+0000 mgr.y (mgr.34107) 8 : audit [DBG] from='client.25955 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:15:46.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:46 smithi143 conmon[109923]: cephadm 2023-08-28T02:15:44.883296+0000 mgr.y (mgr.34107) 9 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8 2023-08-28T02:15:46.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:46 smithi143 conmon[109923]: cluster 2023-08-28T02:15:44.947332+0000 mgr.y (mgr.34107) 10 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:46.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:46 smithi143 conmon[109923]: audit 2023-08-28T02:15:45.129329+0000 mon.a (mon.0) 26 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:15:46.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:46 smithi143 conmon[109923]: audit 2023-08-28T02:15:45.135200+0000 mon.a (mon.0) 27 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:15:46.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:46 smithi143 conmon[109923]: audit 2023-08-28T02:15:45.247256+0000 mon.a (mon.0) 28 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:15:46.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:46 smithi143 conmon[109923]: audit 2023-08-28T02:15:45.253134+0000 mon.a (mon.0) 29 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:15:46.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:46 smithi143 conmon[109923]: audit 2023-08-28T02:15:45.871635+0000 mon.a (mon.0) 30 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:15:46.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:46 smithi143 conmon[109923]: audit 2023-08-28T02:15:45.878929+0000 mon.a (mon.0) 31 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:15:46.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:46 smithi169 conmon[100500]: audit 2023-08-28T02:15:44.882873+0000 mgr.y (mgr 2023-08-28T02:15:46.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:46 smithi169 conmon[100500]: .34107) 8 : audit [DBG] from='client.25955 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:15:46.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:46 smithi169 conmon[100500]: cephadm 2023-08-28T02:15:44.883296+0000 mgr.y (mgr.34107) 9 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8 2023-08-28T02:15:46.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:46 smithi169 conmon[100500]: cluster 2023-08-28T02:15:44.947332+0000 mgr.y (mgr.34107) 10 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:46.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:46 smithi169 conmon[100500]: audit 2023-08-28T02:15:45.129329+0000 mon.a (mon.0) 26 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:15:46.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:46 smithi169 conmon[100500]: audit 2023-08-28T02:15:45.135200+0000 mon.a (mon.0) 27 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:15:46.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:46 smithi169 conmon[100500]: audit 2023-08-28T02:15:45.247256+0000 mon.a (mon.0) 28 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:15:46.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:46 smithi169 conmon[100500]: audit 2023-08-28T02:15:45.253134+0000 mon.a (mon.0) 29 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:15:46.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:46 smithi169 conmon[100500]: audit 2023-08-28T02:15:45.871635+0000 mon.a (mon.0) 30 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:15:46.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:46 smithi169 conmon[100500]: audit 2023-08-28T02:15:45.878929+0000 mon.a (mon.0) 31 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:15:48.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:48 smithi143 conmon[109923]: cluster 2023-08-28T02:15:46.947794+0000 mgr.y (mgr.34107) 11 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:48.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:48 smithi143 conmon[109923]: audit 2023-08-28T02:15:48.130229+0000 mon.a (mon.0) 32 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:15:48.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:48 smithi169 conmon[100500]: cluster 2023-08-28T02:15:46.947794+0000 mgr.y (mgr.34107) 11 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:48.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:48 smithi169 conmon[100500]: audit 2023-08-28T02:15:48.130229+0000 mon.a (mon.0) 32 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:15:48.987 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:15:48 smithi169 conmon[150127]: ts=2023-08-28T02:15:48.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi169.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi143:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-08-28T02:15:49.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:49 smithi143 conmon[109923]: audit 2023-08-28T02:15:48.140208 2023-08-28T02:15:49.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:49 smithi143 conmon[109923]: +0000 mon.a (mon.0) 33 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:15:49.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:49 smithi143 conmon[109923]: audit 2023-08-28T02:15:48.142183+0000 mon.c (mon.2) 136 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:15:49.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:49 smithi143 conmon[109923]: audit 2023-08-28T02:15:48.143110+0000 mon.c (mon.2) 137 : audit [INF] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:15:49.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:49 smithi143 conmon[109923]: cephadm 2023-08-28T02:15:48.144254+0000 mgr.y (mgr.34107) 12 : cephadm [INF] Updating smithi143:/etc/ceph/ceph.conf 2023-08-28T02:15:49.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:49 smithi143 conmon[109923]: cephadm 2023-08-28T02:15:48.144435+0000 mgr.y (mgr.34107) 13 : cephadm [INF] 2023-08-28T02:15:49.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:49 smithi143 conmon[109923]: Updating smithi169:/etc/ceph/ceph.conf 2023-08-28T02:15:49.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:49 smithi169 conmon[100500]: audit 2023-08-28T02:15:48.140208+0000 mon.a (mon 2023-08-28T02:15:49.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:49 smithi169 conmon[100500]: .0) 33 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:15:49.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:49 smithi169 conmon[100500]: audit 2023-08-28T02:15:48.142183+0000 mon.c (mon.2) 136 : audit 2023-08-28T02:15:49.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:49 smithi169 conmon[100500]: [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:15:49.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:49 smithi169 conmon[100500]: audit 2023-08 2023-08-28T02:15:49.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:49 smithi169 conmon[100500]: -28T02:15:48.143110+0000 mon.c (mon.2) 137 : audit [INF] 2023-08-28T02:15:49.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:49 smithi169 conmon[100500]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:15:49.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:49 smithi169 conmon[100500]: cephadm 2023-08-28T02:15: 2023-08-28T02:15:49.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:49 smithi169 conmon[100500]: 48.144254+0000 mgr.y (mgr.34107) 12 : cephadm 2023-08-28T02:15:49.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:49 smithi169 conmon[100500]: [INF] Updating smithi143:/etc/ceph/ceph.conf 2023-08-28T02:15:49.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:49 smithi169 conmon[100500]: cephadm 2023-08-28T02: 2023-08-28T02:15:49.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:49 smithi169 conmon[100500]: 15:48.144435+0000 mgr.y (mgr.34107) 13 2023-08-28T02:15:49.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:49 smithi169 conmon[100500]: : cephadm [INF] Updating smithi169:/etc/ceph/ceph.conf 2023-08-28T02:15:50.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:50 smithi143 conmon[109923]: cephadm 2023-08-28T02:15:48.409054+0000 mgr.y (mgr.34107) 14 : 2023-08-28T02:15:50.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:50 smithi143 conmon[109923]: cephadm [INF] Updating smithi169:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.conf 2023-08-28T02:15:50.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:50 smithi143 conmon[109923]: cephadm 2023-08-28T02:15:48.409208+0000 mgr.y (mgr.34107) 15 : cephadm [INF] Updating smithi143:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.conf 2023-08-28T02:15:50.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:50 smithi143 conmon[109923]: cephadm 2023-08-28T02:15:48.673891+0000 mgr.y (mgr.34107) 16 : cephadm [INF] Updating smithi143:/etc/ceph/ceph.client.admin.keyring 2023-08-28T02:15:50.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:50 smithi143 conmon[109923]: 2023-08-28T02:15:50.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:50 smithi143 conmon[109923]: cephadm 2023-08-28T02:15:50.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:50 smithi143 conmon[109923]: 2023-08-28T02:15:48.674028+0000 mgr.y (mgr.34107) 17 : 2023-08-28T02:15:50.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:50 smithi143 conmon[109923]: cephadm [INF] Updating smithi169:/etc/ceph/ceph.client.admin.keyring 2023-08-28T02:15:50.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:50 smithi143 conmon[109923]: cephadm 2023-08-28T02 2023-08-28T02:15:50.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:50 smithi143 conmon[109923]: :15:48.938020+0000 mgr.y (mgr.34107) 18 : 2023-08-28T02:15:50.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:50 smithi143 conmon[109923]: cephadm [INF] Updating smithi143:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.client.admin.keyring 2023-08-28T02:15:50.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:50 smithi143 conmon[109923]: cephadm 2023-08-28T02:15 2023-08-28T02:15:50.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:50 smithi143 conmon[109923]: :48.938486+0000 mgr.y (mgr.34107) 19 : cephadm 2023-08-28T02:15:50.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:50 smithi143 conmon[109923]: [INF] Updating smithi169:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.client.admin.keyring 2023-08-28T02:15:50.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:50 smithi143 conmon[109923]: cluster 2023-08-28T02:15: 2023-08-28T02:15:50.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:50 smithi143 conmon[109923]: 48.948207+0000 mgr.y (mgr.34107) 20 : cluster [DBG] 2023-08-28T02:15:50.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:50 smithi143 conmon[109923]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:50.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:50 smithi143 conmon[109923]: audit 2023-08-28T02:15:49. 2023-08-28T02:15:50.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:50 smithi143 conmon[109923]: 206891+0000 mon.a (mon.0) 34 : audit [INF] 2023-08-28T02:15:50.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:50 smithi143 conmon[109923]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:15:50.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:50 smithi143 conmon[109923]: audit 2023-08-28T02:15:49.209517 2023-08-28T02:15:50.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:50 smithi143 conmon[109923]: +0000 mon.a (mon.0) 35 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:15:50.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:50 smithi143 conmon[109923]: 2023-08-28T02:15:50.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:50 smithi143 conmon[109923]: audit 2023-08-28T02:15:49.214706 2023-08-28T02:15:50.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:50 smithi143 conmon[109923]: +0000 mon.a (mon.0) 36 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:15:50.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:50 smithi143 conmon[109923]: audit 2023-08-28T02: 2023-08-28T02:15:50.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:50 smithi143 conmon[109923]: 15:49.218399+0000 mon.a (mon.0) 37 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:15:50.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:50 smithi143 conmon[109923]: audit 2023-08- 2023-08-28T02:15:50.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:50 smithi143 conmon[109923]: 28T02:15:49.223903+0000 mon.a (mon.0) 38 : audit 2023-08-28T02:15:50.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:50 smithi143 conmon[109923]: [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:15:50.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:50 smithi143 conmon[109923]: audit 2023-08-28T 2023-08-28T02:15:50.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:50 smithi143 conmon[109923]: 02:15:49.296001+0000 mon.c (mon.2) 138 : audit 2023-08-28T02:15:50.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:50 smithi143 conmon[109923]: [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:15:50.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:50 smithi143 conmon[109923]: audit 2023-08-28T02: 2023-08-28T02:15:50.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:50 smithi143 conmon[109923]: 15:49.297465+0000 mon.c (mon.2) 139 : audit 2023-08-28T02:15:50.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:50 smithi143 conmon[109923]: [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:15:50.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:50 smithi143 conmon[109923]: audit 2023-08-28T02:15: 2023-08-28T02:15:50.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:50 smithi143 conmon[109923]: 49.298046+0000 mon.c (mon.2) 140 : audit 2023-08-28T02:15:50.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:50 smithi143 conmon[109923]: [INF] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:15:50.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:50 smithi143 conmon[109923]: audit 2023-08-28T02 2023-08-28T02:15:50.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:50 smithi143 conmon[109923]: :15:49.302863+0000 mon.a (mon.0) 39 : 2023-08-28T02:15:50.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:50 smithi143 conmon[109923]: audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:15:50.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:50 smithi169 conmon[100500]: cephadm 2023-08-28T02:15:48.409054+0000 mgr.y (mgr.34107) 14 : cephadm [INF] 2023-08-28T02:15:50.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:50 smithi169 conmon[100500]: Updating smithi169:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.conf 2023-08-28T02:15:50.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:50 smithi169 conmon[100500]: cephadm 2023-08-28T02:15:48.409208+0000 mgr.y (mgr.34107) 15 : cephadm [INF] Updating smithi143:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.conf 2023-08-28T02:15:50.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:50 smithi169 conmon[100500]: cephadm 2023-08-28 2023-08-28T02:15:50.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:50 smithi169 conmon[100500]: T02:15:48.673891+0000 mgr.y (mgr.34107) 16 : cephadm [INF] Updating smithi143:/etc/ceph/ceph.client.admin.keyring 2023-08-28T02:15:50.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:50 smithi169 conmon[100500]: cephadm 2023- 2023-08-28T02:15:50.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:50 smithi169 conmon[100500]: 08-28T02:15:48.674028+0000 mgr.y (mgr.34107) 17 : cephadm [INF] 2023-08-28T02:15:50.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:50 smithi169 conmon[100500]: Updating smithi169:/etc/ceph/ceph.client.admin.keyring 2023-08-28T02:15:50.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:50 smithi169 conmon[100500]: cephadm 2023-08 2023-08-28T02:15:50.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:50 smithi169 conmon[100500]: -28T02:15:48.938020+0000 mgr.y (mgr.34107) 18 : cephadm [INF] 2023-08-28T02:15:50.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:50 smithi169 conmon[100500]: Updating smithi143:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.client.admin.keyring 2023-08-28T02:15:50.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:50 smithi169 conmon[100500]: cephadm 2023-08-28T02:15:48.938486+0000 2023-08-28T02:15:50.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:50 smithi169 conmon[100500]: mgr.y (mgr.34107) 19 : cephadm [INF] Updating smithi169:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.client.admin.keyring 2023-08-28T02:15:50.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:50 smithi169 conmon[100500]: cluster 2023-08-28T02:15 2023-08-28T02:15:50.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:50 smithi169 conmon[100500]: :48.948207+0000 mgr.y (mgr.34107) 20 : cluster 2023-08-28T02:15:50.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:50 smithi169 conmon[100500]: [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:50.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:50 smithi169 conmon[100500]: audit 2023-08-28T02:15: 2023-08-28T02:15:50.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:50 smithi169 conmon[100500]: 49.206891+0000 mon.a (mon.0) 34 : audit [INF] 2023-08-28T02:15:50.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:50 smithi169 conmon[100500]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:15:50.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:50 smithi169 conmon[100500]: audit 2023-08-28T02:15:50.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:50 smithi169 conmon[100500]: 2023-08-28T02:15:49.209517+0000 mon.a (mon. 2023-08-28T02:15:50.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:50 smithi169 conmon[100500]: 0) 35 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:15:50.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:50 smithi169 conmon[100500]: audit 2023-08-28T 2023-08-28T02:15:50.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:50 smithi169 conmon[100500]: 02:15:49.214706+0000 mon.a (mon.0 2023-08-28T02:15:50.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:50 smithi169 conmon[100500]: ) 36 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:15:50.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:50 smithi169 conmon[100500]: audit 2023-08-28T02 2023-08-28T02:15:50.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:50 smithi169 conmon[100500]: :15:49.218399+0000 mon.a (mon.0) 37 2023-08-28T02:15:50.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:50 smithi169 conmon[100500]: : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:15:50.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:50 smithi169 conmon[100500]: audit 2023-08-28T02 2023-08-28T02:15:50.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:50 smithi169 conmon[100500]: :15:49.223903+0000 mon.a (mon.0) 2023-08-28T02:15:50.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:50 smithi169 conmon[100500]: 38 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:15:50.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:50 smithi169 conmon[100500]: audit 2023-08-28T 2023-08-28T02:15:50.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:50 smithi169 conmon[100500]: 02:15:49.296001+0000 mon.c (mon.2) 2023-08-28T02:15:50.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:50 smithi169 conmon[100500]: 138 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:15:50.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:50 smithi169 conmon[100500]: audit 2023-08-28T02:15 2023-08-28T02:15:50.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:50 smithi169 conmon[100500]: :49.297465+0000 mon.c (mon.2) 139 : audit 2023-08-28T02:15:50.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:50 smithi169 conmon[100500]: [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:15:50.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:50 smithi169 conmon[100500]: audit 2023-08-28T02:15: 2023-08-28T02:15:50.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:50 smithi169 conmon[100500]: 49.298046+0000 mon.c (mon.2) 140 : audit 2023-08-28T02:15:50.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:50 smithi169 conmon[100500]: [INF] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:15:50.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:50 smithi169 conmon[100500]: audit 2023-08-28T02:15:50.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:50 smithi169 conmon[100500]: 2023-08-28T02:15:49.302863+0000 mon.a (mon.0) 39 : 2023-08-28T02:15:50.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:50 smithi169 conmon[100500]: audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:15:52.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:52 smithi143 conmon[109923]: cluster 2023-08-28T02:15:50.948674+0000 mgr.y (mgr.34107) 21 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:52.201 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:15:51 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:15:51] "GET /metrics HTTP/1.1" 200 31245 "" "Prometheus/2.33.4" 2023-08-28T02:15:52.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:52 smithi169 conmon[100500]: cluster 2023-08-28T02:15:50.948674+0000 mgr.y (mgr.34107) 21 : cluster [DBG] 2023-08-28T02:15:52.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:52 smithi169 conmon[100500]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:54.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:54 smithi143 conmon[109923]: cluster 2023-08-28T02:15:52.949364+0000 mgr.y (mgr.34107) 22 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:54.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:54 smithi169 conmon[100500]: cluster 2023-08-28T02:15:52.949364+0000 mgr.y (mgr.34107) 22 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:56.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:56 smithi143 conmon[109923]: cluster 2023-08-28T02:15:54.949774+0000 mgr.y (mgr.34107) 23 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:56.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:56 smithi143 conmon[109923]: audit 2023-08-28T02:15:56.010237+0000 mon.c (mon.2) 141 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:15:56.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:56 smithi169 conmon[100500]: cluster 2023-08-28T02:15:54.949774+0000 mgr.y (mgr.34107) 23 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:56.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:56 smithi169 conmon[100500]: audit 2023-08-28T02:15:56.010237+0000 mon.c (mon.2) 141 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:15:57.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:15:57 smithi143 conmon[109923]: cluster 2023-08-28T02:15:56.950243+0000 mgr.y (mgr.34107) 24 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:57.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:15:57 smithi169 conmon[100500]: cluster 2023-08-28T02:15:56.950243+0000 mgr.y (mgr.34107) 24 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:15:58.987 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:15:58 smithi169 conmon[150127]: ts=2023-08-28T02:15:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi169.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi143:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-08-28T02:16:00.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:00 smithi143 conmon[109923]: cluster 2023-08-28T02:15:58.950918+0000 mgr.y (mgr.34107) 25 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:00.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:00 smithi143 conmon[109923]: 2023-08-28T02:16:00.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:00 smithi169 conmon[100500]: cluster 2023-08-28T02:15:58.950918+0000 mgr.y (mgr.34107) 25 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:02.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:02 smithi143 conmon[109923]: cluster 2023-08-28T02:16:00.951441+0000 mgr.y (mgr.34107) 26 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:02.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:02 smithi143 conmon[109923]: 2023-08-28T02:16:02.201 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:16:01 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:16:01] "GET /metrics HTTP/1.1" 200 33942 "" "Prometheus/2.33.4" 2023-08-28T02:16:02.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:02 smithi169 conmon[100500]: cluster 2023-08-28T02:16:00.951441+0000 mgr.y (mgr.34107) 26 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:04.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:04 smithi143 conmon[109923]: cluster 2023-08-28T02:16:02.952046+0000 mgr.y (mgr.34107) 27 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:04.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:04 smithi169 conmon[100500]: cluster 2023-08-28T02:16:02.952046+0000 mgr.y (mgr.34107) 27 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:06.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:06 smithi143 conmon[109923]: cluster 2023-08-28T02:16:04.952608+0000 mgr.y (mgr.34107) 28 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:06.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:06 smithi143 conmon[109923]: 2023-08-28T02:16:06.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:06 smithi169 conmon[100500]: cluster 2023-08-28T02:16:04.952608+0000 mgr.y (mgr.34107) 28 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:08.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:08 smithi143 conmon[109923]: cluster 2023-08-28T02:16:06.953195+0000 mgr.y (mgr.34107) 29 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:08.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:08 smithi169 conmon[100500]: cluster 2023-08-28T02:16:06.953195+0000 mgr.y (mgr.34107) 29 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:08.987 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:16:08 smithi169 conmon[150127]: ts=2023-08-28T02:16:08.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi169.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi143:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-08-28T02:16:10.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:10 smithi143 conmon[109923]: cluster 2023-08-28T02:16:08.953603+0000 mgr.y (mgr.34107) 30 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:10.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:10 smithi169 conmon[100500]: cluster 2023-08-28T02:16:08.953603+0000 mgr.y (mgr.34107) 30 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:10.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:10 smithi169 conmon[100500]: 2023-08-28T02:16:11.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:11 smithi143 conmon[109923]: audit 2023-08-28T02:16:11.010464+0000 mon.c (mon.2) 142 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:16:11.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:11 smithi143 conmon[109923]: 2023-08-28T02:16:11.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:11 smithi169 conmon[100500]: audit 2023-08-28T02:16:11.010464+0000 mon.c (mon.2) 142 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:16:12.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:12 smithi143 conmon[109923]: cluster 2023-08-28T02:16:10.954130+0000 mgr.y (mgr.34107) 31 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:12.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:16:11 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:16:11] "GET /metrics HTTP/1.1" 200 33939 "" "Prometheus/2.33.4" 2023-08-28T02:16:12.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:12 smithi169 conmon[100500]: cluster 2023-08-28T02:16:10.954130+0000 mgr.y (mgr.34107) 31 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:14.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:14 smithi143 conmon[109923]: cluster 2023-08-28T02:16:12.954665+0000 mgr.y (mgr.34107) 32 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:14.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:14 smithi169 conmon[100500]: cluster 2023-08-28T02:16:12.954665+0000 mgr.y (mgr.34107) 32 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:16.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:16 smithi143 conmon[109923]: cluster 2023-08-28T02:16:14.955217+0000 mgr.y (mgr.34107) 33 : cluster [DBG] 2023-08-28T02:16:16.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:16 smithi143 conmon[109923]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:16.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:16 smithi169 conmon[100500]: cluster 2023-08-28T02:16:14.955217+0000 mgr.y (mgr.34107) 33 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:18.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:18 smithi143 conmon[109923]: cluster 2023-08-28T02:16:16.955785+0000 mgr.y (mgr.34107) 34 : cluster [DBG] 2023-08-28T02:16:18.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:18 smithi143 conmon[109923]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:18.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:18 smithi169 conmon[100500]: cluster 2023-08-28T02:16:16.955785+0000 mgr.y (mgr.34107) 34 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:18.987 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:16:18 smithi169 conmon[150127]: ts=2023-08-28T02:16:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi169.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi143:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-08-28T02:16:20.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:20 smithi143 conmon[109923]: cluster 2023-08-28T02:16:18.956261+0000 mgr.y (mgr.34107) 35 : cluster 2023-08-28T02:16:20.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:20 smithi143 conmon[109923]: [DBG] pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:20.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:20 smithi169 conmon[100500]: cluster 2023-08-28T02:16:18.956261+0000 mgr.y (mgr.34107) 35 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:22.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:22 smithi143 conmon[109923]: cluster 2023-08-28T02:16:20.956666+0000 mgr.y (mgr.34107) 36 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:22.201 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:16:21 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:16:21] "GET /metrics HTTP/1.1" 200 33939 "" "Prometheus/2.33.4" 2023-08-28T02:16:22.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:22 smithi169 conmon[100500]: cluster 2023-08-28T02:16:20.956666+0000 mgr.y (mgr.34107) 36 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:24.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:24 smithi143 conmon[109923]: cluster 2023-08-28T02:16:22.957270+0000 mgr.y (mgr.34107) 37 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:24.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:24 smithi169 conmon[100500]: cluster 2023-08-28T02:16:22.957270+0000 mgr.y (mgr.34107) 37 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:24.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:24 smithi169 conmon[100500]: 2023-08-28T02:16:26.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:26 smithi143 conmon[109923]: cluster 2023-08-28T02:16:24.957817+0000 mgr.y (mgr.34107) 38 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:26.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:26 smithi143 conmon[109923]: audit 2023-08-28T02:16:26.010901+0000 2023-08-28T02:16:26.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:26 smithi143 conmon[109923]: mon.c (mon.2) 143 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:16:26.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:26 smithi169 conmon[100500]: cluster 2023-08-28T02:16:24.957817+0000 mgr.y (mgr.34107) 38 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:26.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:26 smithi169 conmon[100500]: audit 2023-08-28T02:16:26.010901 2023-08-28T02:16:26.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:26 smithi169 conmon[100500]: +0000 mon.c (mon.2) 143 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:16:28.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:28 smithi143 conmon[109923]: cluster 2023-08-28T02:16:26.958403+0000 mgr.y (mgr.34107) 39 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:28.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:28 smithi169 conmon[100500]: cluster 2023-08-28T02:16:26.958403+0000 mgr.y (mgr.34107) 39 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:28.987 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:16:28 smithi169 conmon[150127]: ts=2023-08-28T02:16:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi169.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi143:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-08-28T02:16:30.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:30 smithi143 conmon[109923]: cluster 2023-08-28T02:16:28.958879+0000 mgr.y (mgr.34107) 40 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:30.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:30 smithi169 conmon[100500]: cluster 2023-08-28T02:16:28.958879+0000 mgr.y (mgr.34107) 40 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:32.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:32 smithi143 conmon[109923]: cluster 2023-08-28T02:16:30.959424+0000 mgr.y (mgr.34107) 41 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:32.201 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:16:31 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:16:31] "GET /metrics HTTP/1.1" 200 33937 "" "Prometheus/2.33.4" 2023-08-28T02:16:32.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:32 smithi169 conmon[100500]: cluster 2023-08-28T02:16:30.959424+0000 mgr.y (mgr.34107) 41 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:34.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:34 smithi143 conmon[109923]: cluster 2023-08-28T02:16:32.960019+0000 mgr.y (mgr.34107) 42 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:34.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:34 smithi169 conmon[100500]: cluster 2023-08-28T02:16:32.960019+0000 mgr.y (mgr.34107) 42 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:36.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:36 smithi143 conmon[109923]: cluster 2023-08-28T02:16:34.960465+0000 mgr.y (mgr.34107) 43 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:36.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:36 smithi169 conmon[100500]: cluster 2023-08-28T02:16:34.960465+0000 mgr.y (mgr.34107) 43 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:38.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:38 smithi143 conmon[109923]: cluster 2023-08-28T02:16:36.960969+0000 mgr.y (mgr.34107) 44 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:38.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:38 smithi169 conmon[100500]: cluster 2023-08-28T02:16:36.960969+0000 mgr.y (mgr.34107) 44 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:38.987 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:16:38 smithi169 conmon[150127]: ts=2023-08-28T02:16:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi169.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi143:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-08-28T02:16:40.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:40 smithi143 conmon[109923]: cluster 2023-08-28T02:16:38.961452+0000 mgr.y (mgr.34107) 45 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:40.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:40 smithi143 conmon[109923]: 2023-08-28T02:16:40.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:40 smithi169 conmon[100500]: cluster 2023-08-28T02:16:38.961452+0000 mgr.y (mgr.34107) 45 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:41.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:41 smithi143 conmon[109923]: audit 2023-08-28T02:16:41.011887+0000 mon.c (mon.2) 144 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:16:41.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:41 smithi143 conmon[109923]: 2023-08-28T02:16:41.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:41 smithi169 conmon[100500]: audit 2023-08-28T02:16:41.011887+0000 mon.c (mon.2) 144 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:16:42.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:16:41 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:16:41] "GET /metrics HTTP/1.1" 200 33941 "" "Prometheus/2.33.4" 2023-08-28T02:16:42.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:42 smithi143 conmon[109923]: cluster 2023-08-28T02:16:40.961834+0000 mgr.y (mgr.34107) 46 : cluster 2023-08-28T02:16:42.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:42 smithi143 conmon[109923]: [DBG] pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:42.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:42 smithi169 conmon[100500]: cluster 2023-08-28T02:16:40.961834+0000 mgr.y (mgr.34107) 46 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:44.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:44 smithi143 conmon[109923]: cluster 2023-08-28T02:16:42.962571+0000 mgr.y (mgr.34107) 47 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:44.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:44 smithi169 conmon[100500]: cluster 2023-08-28T02:16:42.962571+0000 mgr.y (mgr.34107) 47 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:46.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:46 smithi143 conmon[109923]: cluster 2023-08-28T02:16:44.963104+0000 mgr.y (mgr.34107) 48 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:46.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:46 smithi169 conmon[100500]: cluster 2023-08-28T02:16:44.963104+0000 mgr.y (mgr.34107) 48 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:47.836 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -e sha1=9cf208207dfe23ffccb33308b22e51570cb45e51 -- bash -c 'ceph orch upgrade resume' 2023-08-28T02:16:48.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:48 smithi143 conmon[109923]: cluster 2023-08-28T02:16:46.963864+0000 mgr.y (mgr.34107) 49 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:48.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:48 smithi169 conmon[100500]: cluster 2023-08-28T02:16:46.963864+0000 mgr.y (mgr.34107) 49 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:48.987 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:16:48 smithi169 conmon[150127]: ts=2023-08-28T02:16:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi169.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi143:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-08-28T02:16:49.679 INFO:teuthology.orchestra.run.smithi143.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8 2023-08-28T02:16:50.324 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -e sha1=9cf208207dfe23ffccb33308b22e51570cb45e51 -- bash -c 'echo wait for majority of mons to upgrade' 2023-08-28T02:16:50.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:50 smithi143 conmon[109923]: cluster 2023-08-28T02:16:48.964233+0000 mgr.y (mgr.34107) 50 : cluster [DBG] pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:50.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:50 smithi143 conmon[109923]: audit 2023-08-28T02:16:49.375035+0000 mon.c (mon.2) 145 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:16:50.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:50 smithi143 conmon[109923]: audit 2023-08-28T02:16:49.678899+0000 mon.a (mon.0) 40 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:16:50.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:50 smithi143 conmon[109923]: audit 2023-08-28T02:16:49.696002+0000 mon.a (mon.0) 41 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:16:50.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:50 smithi143 conmon[109923]: audit 2023-08-28T02:16:49.699366+0000 mon.a (mon.0) 42 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:16:50.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:50 smithi143 conmon[109923]: audit 2023-08-28T02:16:49.709537+0000 mon.a (mon.0) 43 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:16:50.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:50 smithi143 conmon[109923]: audit 2023-08-28T02:16:49.715195+0000 mon.a (mon.0) 44 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:16:50.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:50 smithi169 conmon[100500]: cluster 2023-08-28T02:16:48.964233+0000 mgr.y (mgr.34107) 50 : cluster [DBG] pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:50.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:50 smithi169 conmon[100500]: audit 2023-08-28T02:16:49.375035+0000 mon.c (mon.2) 145 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:16:50.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:50 smithi169 conmon[100500]: audit 2023-08-28T02:16:49.678899+0000 mon.a (mon.0) 40 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:16:50.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:50 smithi169 conmon[100500]: audit 2023-08-28T02:16:49.696002+0000 mon.a (mon.0) 41 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:16:50.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:50 smithi169 conmon[100500]: audit 2023-08-28T02:16:49.699366+0000 mon.a (mon.0) 42 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:16:50.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:50 smithi169 conmon[100500]: audit 2023-08-28T02:16:49.709537+0000 mon.a (mon.0) 43 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:16:50.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:50 smithi169 conmon[100500]: audit 2023-08-28T02 2023-08-28T02:16:50.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:50 smithi169 conmon[100500]: :16:49.715195+0000 mon.a (mon.0) 44 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:16:51.287 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:51 smithi143 conmon[109923]: audit 2023-08-28T02:16:49.674884+0000 mgr.y (mgr.34107) 51 : audit [DBG] from='client.34161 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:16:51.287 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:51 smithi143 conmon[109923]: cephadm 2023-08-28T02:16:49.675195+0000 mgr.y (mgr.34107) 52 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8 2023-08-28T02:16:51.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:51 smithi169 conmon[100500]: audit 2023-08-28T02:16:49.674884+0000 mgr.y ( 2023-08-28T02:16:51.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:51 smithi169 conmon[100500]: mgr.34107) 51 : audit [DBG] from='client.34161 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:16:51.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:51 smithi169 conmon[100500]: cephadm 2023-08-28T02:16:49.675195+0000 mgr.y (mgr.34107) 52 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8 2023-08-28T02:16:52.190 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:52 smithi143 conmon[109923]: cluster 2023-08-28T02:16:50.964560+0000 mgr.y (mgr.34107) 53 : cluster [DBG] 2023-08-28T02:16:52.190 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:52 smithi143 conmon[109923]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:52.190 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:16:51 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:16:51] "GET /metrics HTTP/1.1" 200 33941 "" "Prometheus/2.33.4" 2023-08-28T02:16:52.337 INFO:teuthology.orchestra.run.smithi143.stdout:wait for majority of mons to upgrade 2023-08-28T02:16:52.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:52 smithi169 conmon[100500]: cluster 2023-08-28T02:16:50.964560+0000 mgr.y 2023-08-28T02:16:52.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:52 smithi169 conmon[100500]: (mgr.34107) 53 : cluster [DBG] pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:52.917 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -e sha1=9cf208207dfe23ffccb33308b22e51570cb45e51 -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2023-08-28T02:16:54.164 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:53 smithi143 conmon[109923]: audit 2023-08-28T02:16:52.842629+0000 mon.a (mon.0) 45 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:16:54.164 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:53 smithi143 conmon[109923]: audit 2023-08-28T02:16:52.848714+0000 mon.a (mon.0) 46 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:16:54.164 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:53 smithi143 conmon[109923]: cluster 2023-08-28T02:16:52.965044+0000 mgr.y (mgr.34107) 54 : cluster [DBG] pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:54.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:53 smithi169 conmon[100500]: audit 2023-08-28T02:16:52.842629+0000 mon.a (mon.0) 45 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:16:54.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:53 smithi169 conmon[100500]: audit 2023-08-28T02:16:52.848714+0000 mon.a (mon.0) 46 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:16:54.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:53 smithi169 conmon[100500]: cluster 2023-08-28T02:16:52.965044+0000 mgr.y (mgr.34107) 54 : cluster [DBG] pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:55.297 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:54 smithi143 conmon[109923]: audit 2023-08-28T02:16:53.989411+0000 mon.a (mon.0) 47 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:16:55.297 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:54 smithi143 conmon[109923]: audit 2023-08-28T02:16:53.996198+0000 mon.a (mon.0) 48 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:16:55.297 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:54 smithi143 conmon[109923]: audit 2023-08-28T02:16:54.363092+0000 mon.c (mon.2) 146 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:16:55.297 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:54 smithi143 conmon[109923]: audit 2023-08-28T02:16:54.363795+0000 mon.c (mon.2) 147 : audit [INF] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:16:55.297 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:54 smithi143 conmon[109923]: audit 2023-08-28T02:16:54.368935+0000 mon.a (mon.0) 49 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:16:55.298 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:54 smithi143 conmon[109923]: audit 2023-08-28T02:16:54.476018+0000 mon.c (mon.2) 148 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:16:55.298 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:54 smithi143 conmon[109923]: audit 2023-08-28T02:16:54.478732+0000 mon.c (mon.2) 149 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:16:55.298 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:54 smithi143 conmon[109923]: audit 2023-08-28T02:16:54.479775+0000 mon.c (mon.2) 150 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-08-28T02:16:55.298 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:54 smithi143 conmon[109923]: audit 2023-08-28T02:16:54.480399+0000 mon.c (mon.2) 151 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-08-28T02:16:55.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:54 smithi169 conmon[100500]: audit 2023-08-28T02:16:53.989411+0000 mon.a (mon.0) 47 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:16:55.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:54 smithi169 conmon[100500]: audit 2023-08-28T02:16 2023-08-28T02:16:55.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:54 smithi169 conmon[100500]: :53.996198+0000 mon.a (mon.0) 48 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:16:55.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:54 smithi169 conmon[100500]: audit 2023-08-28T02:16:54.363092+0000 mon.c (mon.2) 146 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:16:55.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:54 smithi169 conmon[100500]: audit 2023-08-28T02:16 2023-08-28T02:16:55.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:54 smithi169 conmon[100500]: :54.363795+0000 mon.c (mon.2) 147 : audit [INF] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:16:55.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:54 smithi169 conmon[100500]: audit 2023-08-28T02:16:55.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:54 smithi169 conmon[100500]: 2023-08-28T02:16:54.368935+0000 mon.a (mon.0) 2023-08-28T02:16:55.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:54 smithi169 conmon[100500]: 49 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:16:55.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:54 smithi169 conmon[100500]: audit 2023-08-28T02 2023-08-28T02:16:55.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:54 smithi169 conmon[100500]: :16:54.476018+0000 mon.c (mon.2) 148 2023-08-28T02:16:55.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:54 smithi169 conmon[100500]: : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:16:55.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:54 smithi169 conmon[100500]: audit 2023-08-28T02:16 2023-08-28T02:16:55.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:54 smithi169 conmon[100500]: :54.478732+0000 mon.c (mon.2) 149 : audit 2023-08-28T02:16:55.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:54 smithi169 conmon[100500]: [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:16:55.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:54 smithi169 conmon[100500]: audit 2023-08-28T02:16:54. 2023-08-28T02:16:55.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:54 smithi169 conmon[100500]: 479775+0000 mon.c (mon.2) 150 : audit [DBG] 2023-08-28T02:16:55.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:54 smithi169 conmon[100500]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-08-28T02:16:55.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:54 smithi169 conmon[100500]: audit 2023-08-28T02:16:54. 2023-08-28T02:16:55.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:54 smithi169 conmon[100500]: 480399+0000 mon.c (mon.2) 151 : audit [DBG] 2023-08-28T02:16:55.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:54 smithi169 conmon[100500]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-08-28T02:16:56.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:55 smithi143 conmon[109923]: cephadm 2023-08-28T02:16:54.481339+0000 mgr.y (mgr.34107) 55 : 2023-08-28T02:16:56.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:55 smithi143 conmon[109923]: cephadm [INF] Upgrade: It appears safe to stop mon.c 2023-08-28T02:16:56.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:55 smithi143 conmon[109923]: cluster 2023-08-28T02:16:54.965338+0000 mgr.y (mgr.34107) 56 : cluster [DBG] pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:56.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:55 smithi143 conmon[109923]: audit 2023-08-28T02:16:55.170414+0000 mon.c (mon.2) 152 : audit [DBG] from='client.? 172.21.15.143:0/3782374877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:56.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:55 smithi143 conmon[109923]: audit 2023-08-28T02:16:55.951674+0000 mon.a (mon.0) 50 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:16:56.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:55 smithi143 conmon[109923]: audit 2023-08-28T02:16:55.952778+0000 mon.c (mon.2) 153 : audit [INF] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-28T02:16:56.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:55 smithi143 conmon[109923]: audit 2023-08-28T02:16:55.953561+0000 mon.a (mon.0) 51 : audit [DBG] from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-28T02:16:56.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:55 smithi143 conmon[109923]: audit 2023-08-28T02:16:55.954175+0000 mon.c (mon.2) 154 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:16:56.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:55 smithi169 conmon[100500]: cephadm 2023-08-28T02:16:54.481339+0000 mgr.y (mgr.34107) 55 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2023-08-28T02:16:56.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:55 smithi169 conmon[100500]: cluster 2023-08-28T02:16:54.965338+0000 mgr.y (mgr.34107) 56 : cluster [DBG] pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:16:56.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:55 smithi169 conmon[100500]: audit 2023-08-28T02:16:55.170414+0000 mon.c (mon.2) 152 : audit [DBG] from='client.? 172.21.15.143:0/3782374877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:56.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:55 smithi169 conmon[100500]: audit 2023-08-28T02:16:55.951674+0000 mon.a (mon.0) 50 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:16:56.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:55 smithi169 conmon[100500]: audit 2023-08-28T02:16:55.952778+0000 mon.c (mon.2) 153 : audit [INF] 2023-08-28T02:16:56.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:55 smithi169 conmon[100500]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-28T02:16:56.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:55 smithi169 conmon[100500]: audit 2023-08-28T02:16:56.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:55 smithi169 conmon[100500]: 2023-08-28T02:16:55.953561+0000 mon.a (mon 2023-08-28T02:16:56.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:55 smithi169 conmon[100500]: .0) 51 : audit [DBG] from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-28T02:16:56.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:55 smithi169 conmon[100500]: audit 2023-08-28T02 2023-08-28T02:16:56.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:55 smithi169 conmon[100500]: :16:55.954175+0000 mon.c (mon.2) 154 : audit [DBG] 2023-08-28T02:16:56.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:55 smithi169 conmon[100500]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:16:57.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:56 smithi143 conmon[109923]: cephadm 2023-08-28T02:16:55.947469+0000 mgr.y (mgr.34107) 57 : cephadm [INF] 2023-08-28T02:16:57.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:56 smithi143 conmon[109923]: Upgrade: Updating mon.c 2023-08-28T02:16:57.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:56 smithi143 conmon[109923]: cephadm 2023-08-28T02:16:55.955071+0000 mgr.y (mgr.34107) 58 : cephadm [INF] Deploying daemon mon.c on smithi143 2023-08-28T02:16:57.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:56 smithi143 conmon[109923]: audit 2023-08-28T02:16:56.011927+0000 mon.c (mon.2) 155 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:16:57.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:57 smithi143 systemd[1]: Stopping Ceph mon.c for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:16:57.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:56 smithi169 conmon[100500]: cephadm 2023-08-28T02:16:55.947469+0000 mgr.y (mgr.34107) 57 : cephadm [INF] Upgrade: Updating mon.c 2023-08-28T02:16:57.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:56 smithi169 conmon[100500]: cephadm 2023-08-28T02:16:55.955071+0000 mgr.y (mgr.34107) 58 : cephadm [INF] Deploying daemon mon.c on smithi143 2023-08-28T02:16:57.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:16:56 smithi169 conmon[100500]: audit 2023-08-28T02:16:56.011927+0000 mon.c (mon.2) 155 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:16:57.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:57 smithi143 conmon[109923]: debug 2023-08-28T02:16:57.507+0000 7f1b90c59700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-08-28T02:16:57.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:57 smithi143 conmon[109923]: debug 2023-08-28T02:16:57.507+0000 7f1b90c59700 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2023-08-28T02:16:57.950 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:16:57 smithi143 conmon[176053]: [28/Aug/2023:02:16:57] ENGINE Bus STOPPING 2023-08-28T02:16:57.951 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:16:57 smithi143 conmon[176053]: [28/Aug/2023:02:16:57] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-08-28T02:16:57.951 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:16:57 smithi143 conmon[176053]: [28/Aug/2023:02:16:57] ENGINE Bus STOPPED 2023-08-28T02:16:57.951 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:16:57 smithi143 conmon[176053]: [28/Aug/2023:02:16:57] ENGINE Bus STARTING 2023-08-28T02:16:57.951 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:16:57 smithi143 conmon[176053]: [28/Aug/2023:02:16:57] ENGINE Serving on http://:::9283 2023-08-28T02:16:57.951 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:16:57 smithi143 conmon[176053]: [28/Aug/2023:02:16:57] ENGINE Bus STARTED 2023-08-28T02:16:58.315 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 bash[189432]: 74b6ddae7a3fe1cab02b3d7211e0c901e58bbdc00cf4f2d40e2d06be40c2dd7b 2023-08-28T02:16:58.315 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 systemd[1]: ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@mon.c.service: Succeeded. 2023-08-28T02:16:58.315 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 systemd[1]: Stopped Ceph mon.c for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:16:58.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 systemd[1]: Starting Ceph mon.c for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:16:58.987 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:16:58 smithi169 conmon[150127]: ts=2023-08-28T02:16:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi169.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi143:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-08-28T02:16:59.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: set uid:gid to 167:167 (ceph:ceph) 2023-08-28T02:16:59.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: ceph version 18.2.0-187-g9cf20820 (9cf208207dfe23ffccb33308b22e51570cb45e51) reef (stable), process ceph-mon, pid 7 2023-08-28T02:16:59.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: pidfile_write: ignore empty --pid-file 2023-08-28T02:16:59.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: load: jerasure load: lrc 2023-08-28T02:16:59.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: RocksDB version: 7.9.2 2023-08-28T02:16:59.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Git sha 0 2023-08-28T02:16:59.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Compile date 2023-08-25 14:59:12 2023-08-28T02:16:59.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: DB SUMMARY 2023-08-28T02:16:59.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: DB Session ID: 03SOF0ORO3XASRKNQF2N 2023-08-28T02:16:59.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: CURRENT file: CURRENT 2023-08-28T02:16:59.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: IDENTITY file: IDENTITY 2023-08-28T02:16:59.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: MANIFEST file: MANIFEST-000005 size: 891 Bytes 2023-08-28T02:16:59.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000020.sst 2023-08-28T02:16:59.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000018.log size: 3305775 ; 2023-08-28T02:16:59.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.error_if_exists: 0 2023-08-28T02:16:59.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.create_if_missing: 0 2023-08-28T02:16:59.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.paranoid_checks: 1 2023-08-28T02:16:59.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.flush_verify_memtable_count: 1 2023-08-28T02:16:59.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-08-28T02:16:59.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-08-28T02:16:59.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.env: 0x560a9b687720 2023-08-28T02:16:59.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.fs: PosixFileSystem 2023-08-28T02:16:59.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.info_log: 0x560a9eb89420 2023-08-28T02:16:59.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.max_file_opening_threads: 16 2023-08-28T02:16:59.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.statistics: (nil) 2023-08-28T02:16:59.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.use_fsync: 0 2023-08-28T02:16:59.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.max_log_file_size: 0 2023-08-28T02:16:59.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-08-28T02:16:59.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.log_file_time_to_roll: 0 2023-08-28T02:16:59.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.keep_log_file_num: 1000 2023-08-28T02:16:59.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.recycle_log_file_num: 0 2023-08-28T02:16:59.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.allow_fallocate: 1 2023-08-28T02:16:59.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.allow_mmap_reads: 0 2023-08-28T02:16:59.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.allow_mmap_writes: 0 2023-08-28T02:16:59.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.use_direct_reads: 0 2023-08-28T02:16:59.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-08-28T02:16:59.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.create_missing_column_families: 0 2023-08-28T02:16:59.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.db_log_dir: 2023-08-28T02:16:59.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.wal_dir: 2023-08-28T02:16:59.205 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.table_cache_numshardbits: 6 2023-08-28T02:16:59.205 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.WAL_ttl_seconds: 0 2023-08-28T02:16:59.205 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.WAL_size_limit_MB: 0 2023-08-28T02:16:59.205 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-08-28T02:16:59.205 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-08-28T02:16:59.205 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.is_fd_close_on_exec: 1 2023-08-28T02:16:59.205 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.advise_random_on_open: 1 2023-08-28T02:16:59.206 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.db_write_buffer_size: 0 2023-08-28T02:16:59.206 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.write_buffer_manager: 0x560a9de18c80 2023-08-28T02:16:59.206 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-08-28T02:16:59.206 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-08-28T02:16:59.206 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.use_adaptive_mutex: 0 2023-08-28T02:16:59.206 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.rate_limiter: (nil) 2023-08-28T02:16:59.206 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-08-28T02:16:59.206 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.wal_recovery_mode: 2 2023-08-28T02:16:59.206 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.enable_thread_tracking: 0 2023-08-28T02:16:59.206 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.enable_pipelined_write: 0 2023-08-28T02:16:59.206 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.unordered_write: 0 2023-08-28T02:16:59.207 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-08-28T02:16:59.207 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-08-28T02:16:59.207 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-08-28T02:16:59.207 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-08-28T02:16:59.207 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.row_cache: None 2023-08-28T02:16:59.207 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.wal_filter: None 2023-08-28T02:16:59.207 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-08-28T02:16:59.207 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.allow_ingest_behind: 0 2023-08-28T02:16:59.207 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.two_write_queues: 0 2023-08-28T02:16:59.207 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.manual_wal_flush: 0 2023-08-28T02:16:59.207 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.wal_compression: 0 2023-08-28T02:16:59.208 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.atomic_flush: 0 2023-08-28T02:16:59.208 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-08-28T02:16:59.208 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.persist_stats_to_disk: 0 2023-08-28T02:16:59.208 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.write_dbid_to_manifest: 0 2023-08-28T02:16:59.208 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.log_readahead_size: 0 2023-08-28T02:16:59.208 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-08-28T02:16:59.208 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.best_efforts_recovery: 0 2023-08-28T02:16:59.208 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-08-28T02:16:59.208 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-08-28T02:16:59.208 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.allow_data_in_errors: 0 2023-08-28T02:16:59.208 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.db_host_id: __hostname__ 2023-08-28T02:16:59.208 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.enforce_single_del_contracts: true 2023-08-28T02:16:59.208 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.max_background_jobs: 2 2023-08-28T02:16:59.209 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.max_background_compactions: -1 2023-08-28T02:16:59.209 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.max_subcompactions: 1 2023-08-28T02:16:59.209 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-08-28T02:16:59.209 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-08-28T02:16:59.209 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.delayed_write_rate : 16777216 2023-08-28T02:16:59.209 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.max_total_wal_size: 0 2023-08-28T02:16:59.209 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-08-28T02:16:59.209 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.stats_dump_period_sec: 600 2023-08-28T02:16:59.209 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.stats_persist_period_sec: 600 2023-08-28T02:16:59.209 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-08-28T02:16:59.209 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.max_open_files: -1 2023-08-28T02:16:59.209 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.bytes_per_sync: 0 2023-08-28T02:16:59.209 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.wal_bytes_per_sync: 0 2023-08-28T02:16:59.210 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.strict_bytes_per_sync: 0 2023-08-28T02:16:59.210 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.compaction_readahead_size: 0 2023-08-28T02:16:59.210 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.max_background_flushes: -1 2023-08-28T02:16:59.210 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Compression algorithms supported: 2023-08-28T02:16:59.210 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-08-28T02:16:59.210 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: kZSTD supported: 0 2023-08-28T02:16:59.210 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: kXpressCompression supported: 0 2023-08-28T02:16:59.212 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: kLZ4HCCompression supported: 1 2023-08-28T02:16:59.212 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: kZlibCompression supported: 1 2023-08-28T02:16:59.212 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: kSnappyCompression supported: 1 2023-08-28T02:16:59.212 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: kLZ4Compression supported: 1 2023-08-28T02:16:59.212 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: kBZip2Compression supported: 0 2023-08-28T02:16:59.212 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Fast CRC32 supported: Supported on x86 2023-08-28T02:16:59.212 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: DMutex implementation: pthread_mutex_t 2023-08-28T02:16:59.212 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 2023-08-28T02:16:59.212 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-08-28T02:16:59.212 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-08-28T02:16:59.212 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.merge_operator: 2023-08-28T02:16:59.213 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.compaction_filter: None 2023-08-28T02:16:59.213 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.compaction_filter_factory: None 2023-08-28T02:16:59.213 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.sst_partitioner_factory: None 2023-08-28T02:16:59.213 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.memtable_factory: SkipListFactory 2023-08-28T02:16:59.213 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.table_factory: BlockBasedTable 2023-08-28T02:16:59.213 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x560a9eb89520) 2023-08-28T02:16:59.213 INFO:journalctl@ceph.mon.c.smithi143.stdout: cache_index_and_filter_blocks: 1 2023-08-28T02:16:59.213 INFO:journalctl@ceph.mon.c.smithi143.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-08-28T02:16:59.213 INFO:journalctl@ceph.mon.c.smithi143.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-08-28T02:16:59.213 INFO:journalctl@ceph.mon.c.smithi143.stdout: pin_top_level_index_and_filter: 1 2023-08-28T02:16:59.213 INFO:journalctl@ceph.mon.c.smithi143.stdout: index_type: 0 2023-08-28T02:16:59.213 INFO:journalctl@ceph.mon.c.smithi143.stdout: data_block_index_type: 0 2023-08-28T02:16:59.213 INFO:journalctl@ceph.mon.c.smithi143.stdout: index_shortening: 1 2023-08-28T02:16:59.214 INFO:journalctl@ceph.mon.c.smithi143.stdout: data_block_hash_table_util_ratio: 0.750000 2023-08-28T02:16:59.214 INFO:journalctl@ceph.mon.c.smithi143.stdout: checksum: 4 2023-08-28T02:16:59.214 INFO:journalctl@ceph.mon.c.smithi143.stdout: no_block_cache: 0 2023-08-28T02:16:59.214 INFO:journalctl@ceph.mon.c.smithi143.stdout: block_cache: 0x560a9de9b350 2023-08-28T02:16:59.214 INFO:journalctl@ceph.mon.c.smithi143.stdout: block_cache_name: BinnedLRUCache 2023-08-28T02:16:59.214 INFO:journalctl@ceph.mon.c.smithi143.stdout: block_cache_options: 2023-08-28T02:16:59.214 INFO:journalctl@ceph.mon.c.smithi143.stdout: capacity : 536870912 2023-08-28T02:16:59.214 INFO:journalctl@ceph.mon.c.smithi143.stdout: num_shard_bits : 4 2023-08-28T02:16:59.214 INFO:journalctl@ceph.mon.c.smithi143.stdout: strict_capacity_limit : 0 2023-08-28T02:16:59.214 INFO:journalctl@ceph.mon.c.smithi143.stdout: high_pri_pool_ratio: 0.000 2023-08-28T02:16:59.214 INFO:journalctl@ceph.mon.c.smithi143.stdout: block_cache_compressed: (nil) 2023-08-28T02:16:59.214 INFO:journalctl@ceph.mon.c.smithi143.stdout: persistent_cache: (nil) 2023-08-28T02:16:59.214 INFO:journalctl@ceph.mon.c.smithi143.stdout: block_size: 4096 2023-08-28T02:16:59.215 INFO:journalctl@ceph.mon.c.smithi143.stdout: block_size_deviation: 10 2023-08-28T02:16:59.215 INFO:journalctl@ceph.mon.c.smithi143.stdout: block_restart_interval: 16 2023-08-28T02:16:59.215 INFO:journalctl@ceph.mon.c.smithi143.stdout: index_block_restart_interval: 1 2023-08-28T02:16:59.215 INFO:journalctl@ceph.mon.c.smithi143.stdout: metadata_block_size: 4096 2023-08-28T02:16:59.215 INFO:journalctl@ceph.mon.c.smithi143.stdout: partition_filters: 0 2023-08-28T02:16:59.215 INFO:journalctl@ceph.mon.c.smithi143.stdout: use_delta_encoding: 1 2023-08-28T02:16:59.215 INFO:journalctl@ceph.mon.c.smithi143.stdout: filter_policy: bloomfilter 2023-08-28T02:16:59.215 INFO:journalctl@ceph.mon.c.smithi143.stdout: whole_key_filtering: 1 2023-08-28T02:16:59.215 INFO:journalctl@ceph.mon.c.smithi143.stdout: verify_compression: 0 2023-08-28T02:16:59.215 INFO:journalctl@ceph.mon.c.smithi143.stdout: read_amp_bytes_per_bit: 0 2023-08-28T02:16:59.215 INFO:journalctl@ceph.mon.c.smithi143.stdout: format_version: 5 2023-08-28T02:16:59.215 INFO:journalctl@ceph.mon.c.smithi143.stdout: enable_index_compression: 1 2023-08-28T02:16:59.215 INFO:journalctl@ceph.mon.c.smithi143.stdout: block_align: 0 2023-08-28T02:16:59.216 INFO:journalctl@ceph.mon.c.smithi143.stdout: max_auto_readahead_size: 262144 2023-08-28T02:16:59.216 INFO:journalctl@ceph.mon.c.smithi143.stdout: prepopulate_block_cache: 0 2023-08-28T02:16:59.216 INFO:journalctl@ceph.mon.c.smithi143.stdout: initial_auto_readahead_size: 8192 2023-08-28T02:16:59.216 INFO:journalctl@ceph.mon.c.smithi143.stdout: num_file_reads_for_auto_readahead: 2 2023-08-28T02:16:59.216 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.write_buffer_size: 33554432 2023-08-28T02:16:59.216 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.max_write_buffer_number: 2 2023-08-28T02:16:59.216 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.compression: NoCompression 2023-08-28T02:16:59.216 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.bottommost_compression: Disabled 2023-08-28T02:16:59.216 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.prefix_extractor: nullptr 2023-08-28T02:16:59.216 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-08-28T02:16:59.216 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.num_levels: 7 2023-08-28T02:16:59.216 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-08-28T02:16:59.216 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-08-28T02:16:59.217 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-08-28T02:16:59.217 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-08-28T02:16:59.217 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-08-28T02:16:59.217 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-08-28T02:16:59.217 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-08-28T02:16:59.217 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-08-28T02:16:59.217 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-08-28T02:16:59.217 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-08-28T02:16:59.217 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-08-28T02:16:59.217 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-08-28T02:16:59.217 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.compression_opts.window_bits: -14 2023-08-28T02:16:59.217 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.compression_opts.level: 32767 2023-08-28T02:16:59.217 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.compression_opts.strategy: 0 2023-08-28T02:16:59.218 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-08-28T02:16:59.218 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-08-28T02:16:59.218 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-08-28T02:16:59.218 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-08-28T02:16:59.218 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.compression_opts.enabled: false 2023-08-28T02:16:59.218 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-08-28T02:16:59.218 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-08-28T02:16:59.218 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-08-28T02:16:59.218 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-08-28T02:16:59.218 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.target_file_size_base: 67108864 2023-08-28T02:16:59.218 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.target_file_size_multiplier: 1 2023-08-28T02:16:59.218 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-08-28T02:16:59.218 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-08-28T02:16:59.219 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-08-28T02:16:59.219 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-08-28T02:16:59.219 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-08-28T02:16:59.219 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-08-28T02:16:59.219 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-08-28T02:16:59.219 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-08-28T02:16:59.219 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-08-28T02:16:59.219 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-08-28T02:16:59.219 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-08-28T02:16:59.219 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-08-28T02:16:59.219 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-08-28T02:16:59.219 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.arena_block_size: 1048576 2023-08-28T02:16:59.219 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-08-28T02:16:59.220 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-08-28T02:16:59.220 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.disable_auto_compactions: 0 2023-08-28T02:16:59.220 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-08-28T02:16:59.220 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-08-28T02:16:59.220 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-08-28T02:16:59.220 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-08-28T02:16:59.220 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-08-28T02:16:59.220 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-08-28T02:16:59.220 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-08-28T02:16:59.220 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-08-28T02:16:59.220 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-08-28T02:16:59.220 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-08-28T02:16:59.220 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.table_properties_collectors: 2023-08-28T02:16:59.221 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.inplace_update_support: 0 2023-08-28T02:16:59.221 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.inplace_update_num_locks: 10000 2023-08-28T02:16:59.221 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-08-28T02:16:59.221 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-08-28T02:16:59.221 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.memtable_huge_page_size: 0 2023-08-28T02:16:59.221 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.bloom_locality: 0 2023-08-28T02:16:59.221 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.max_successive_merges: 0 2023-08-28T02:16:59.221 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.optimize_filters_for_hits: 0 2023-08-28T02:16:59.221 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.paranoid_file_checks: 0 2023-08-28T02:16:59.221 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.force_consistency_checks: 1 2023-08-28T02:16:59.221 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.report_bg_io_stats: 0 2023-08-28T02:16:59.221 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.ttl: 2592000 2023-08-28T02:16:59.221 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.periodic_compaction_seconds: 0 2023-08-28T02:16:59.222 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-08-28T02:16:59.222 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-08-28T02:16:59.222 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.enable_blob_files: false 2023-08-28T02:16:59.222 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.min_blob_size: 0 2023-08-28T02:16:59.222 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.blob_file_size: 268435456 2023-08-28T02:16:59.222 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.blob_compression_type: NoCompression 2023-08-28T02:16:59.222 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.enable_blob_garbage_collection: false 2023-08-28T02:16:59.222 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-08-28T02:16:59.222 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-08-28T02:16:59.222 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-08-28T02:16:59.222 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.blob_file_starting_level: 0 2023-08-28T02:16:59.222 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-08-28T02:16:59.222 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 20.sst 2023-08-28T02:16:59.223 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-08-28T02:16:59.223 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 22, last_sequence is 12068, log_number is 18,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-08-28T02:16:59.223 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 18 2023-08-28T02:16:59.223 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: f216d8b2-2fb4-4d0c-bc50-eb21eaf48712 2023-08-28T02:16:59.223 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: EVENT_LOG_v1 {"time_micros": 1693189018830592, "job": 1, "event": "recovery_started", "wal_files": [18]} 2023-08-28T02:16:59.223 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #18 mode 2 2023-08-28T02:16:59.223 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 bash[189806]: f1328bdbe045a78328c14d489a5454e37f014ead8119f08718de2f18620a0449 2023-08-28T02:16:59.223 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: EVENT_LOG_v1 {"time_micros": 1693189018837768, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 23, "file_size": 2171446, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12073, "largest_seqno": 12721, "table_properties": {"data_size": 2167091, "index_size": 2825, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 517, "raw_key_size": 3864, "raw_average_key_size": 20, "raw_value_size": 2161336, "raw_average_value_size": 11315, "num_data_blocks": 125, "num_entries": 191, "num_filter_entries": 191, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1693189018, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "f216d8b2-2fb4-4d0c-bc50-eb21eaf48712", "db_session_id": "03SOF0ORO3XASRKNQF2N", "orig_file_number": 23, "seqno_to_time_mapping": "N/A"}} 2023-08-28T02:16:59.223 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: EVENT_LOG_v1 {"time_micros": 1693189018837857, "job": 1, "event": "recovery_finished"} 2023-08-28T02:16:59.223 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: [db/version_set.cc:5047] Creating manifest 25 2023-08-28T02:16:59.223 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-08-28T02:16:59.223 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 systemd[1]: Started Ceph mon.c for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:16:59.224 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000018.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-08-28T02:16:59.224 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x560a9df36000 2023-08-28T02:16:59.224 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: DB pointer 0x560a9df24000 2023-08-28T02:16:59.224 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-08-28T02:16:59.224 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-08-28T02:16:59.224 INFO:journalctl@ceph.mon.c.smithi143.stdout: ** DB Stats ** 2023-08-28T02:16:59.224 INFO:journalctl@ceph.mon.c.smithi143.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-08-28T02:16:59.224 INFO:journalctl@ceph.mon.c.smithi143.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-08-28T02:16:59.224 INFO:journalctl@ceph.mon.c.smithi143.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-08-28T02:16:59.224 INFO:journalctl@ceph.mon.c.smithi143.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-08-28T02:16:59.224 INFO:journalctl@ceph.mon.c.smithi143.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-08-28T02:16:59.224 INFO:journalctl@ceph.mon.c.smithi143.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-08-28T02:16:59.224 INFO:journalctl@ceph.mon.c.smithi143.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-08-28T02:16:59.225 INFO:journalctl@ceph.mon.c.smithi143.stdout: 2023-08-28T02:16:59.225 INFO:journalctl@ceph.mon.c.smithi143.stdout: ** Compaction Stats [default] ** 2023-08-28T02:16:59.225 INFO:journalctl@ceph.mon.c.smithi143.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-08-28T02:16:59.225 INFO:journalctl@ceph.mon.c.smithi143.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-08-28T02:16:59.225 INFO:journalctl@ceph.mon.c.smithi143.stdout: L0 1/0 2.07 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 626.8 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-08-28T02:16:59.225 INFO:journalctl@ceph.mon.c.smithi143.stdout: L6 1/0 24.08 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-08-28T02:16:59.225 INFO:journalctl@ceph.mon.c.smithi143.stdout: Sum 2/0 26.15 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 626.8 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-08-28T02:16:59.225 INFO:journalctl@ceph.mon.c.smithi143.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 626.8 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-08-28T02:16:59.225 INFO:journalctl@ceph.mon.c.smithi143.stdout: 2023-08-28T02:16:59.225 INFO:journalctl@ceph.mon.c.smithi143.stdout: ** Compaction Stats [default] ** 2023-08-28T02:16:59.225 INFO:journalctl@ceph.mon.c.smithi143.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-08-28T02:16:59.225 INFO:journalctl@ceph.mon.c.smithi143.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-08-28T02:16:59.225 INFO:journalctl@ceph.mon.c.smithi143.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 626.8 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-08-28T02:16:59.226 INFO:journalctl@ceph.mon.c.smithi143.stdout: 2023-08-28T02:16:59.226 INFO:journalctl@ceph.mon.c.smithi143.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-08-28T02:16:59.226 INFO:journalctl@ceph.mon.c.smithi143.stdout: 2023-08-28T02:16:59.226 INFO:journalctl@ceph.mon.c.smithi143.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-08-28T02:16:59.226 INFO:journalctl@ceph.mon.c.smithi143.stdout: Flush(GB): cumulative 0.002, interval 0.002 2023-08-28T02:16:59.226 INFO:journalctl@ceph.mon.c.smithi143.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-08-28T02:16:59.226 INFO:journalctl@ceph.mon.c.smithi143.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-08-28T02:16:59.226 INFO:journalctl@ceph.mon.c.smithi143.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-08-28T02:16:59.226 INFO:journalctl@ceph.mon.c.smithi143.stdout: AddFile(Keys): cumulative 0, interval 0 2023-08-28T02:16:59.226 INFO:journalctl@ceph.mon.c.smithi143.stdout: Cumulative compaction: 0.00 GB write, 173.90 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-08-28T02:16:59.226 INFO:journalctl@ceph.mon.c.smithi143.stdout: Interval compaction: 0.00 GB write, 173.90 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-08-28T02:16:59.226 INFO:journalctl@ceph.mon.c.smithi143.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-08-28T02:16:59.226 INFO:journalctl@ceph.mon.c.smithi143.stdout: Block cache BinnedLRUCache@0x560a9de9b350#7 capacity: 512.00 MB usage: 3.67 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2023-08-28T02:16:59.227 INFO:journalctl@ceph.mon.c.smithi143.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.59 KB,0.000113249%) IndexBlock(1,3.08 KB,0.000587106%) Misc(1,0.00 KB,0%) 2023-08-28T02:16:59.227 INFO:journalctl@ceph.mon.c.smithi143.stdout: 2023-08-28T02:16:59.227 INFO:journalctl@ceph.mon.c.smithi143.stdout: ** File Read Latency Histogram By Level [default] ** 2023-08-28T02:16:59.227 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: starting mon.c rank 2 at public addrs [v2:172.21.15.143:3301/0,v1:172.21.15.143:6790/0] at bind addrs [v2:172.21.15.143:3301/0,v1:172.21.15.143:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 2023-08-28T02:16:59.227 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: mon.c@-1(???) e3 preinit fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 2023-08-28T02:16:59.227 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: mon.c@-1(???).mds e7 new map 2023-08-28T02:16:59.227 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: mon.c@-1(???).mds e7 print_map 2023-08-28T02:16:59.227 INFO:journalctl@ceph.mon.c.smithi143.stdout: e7 2023-08-28T02:16:59.227 INFO:journalctl@ceph.mon.c.smithi143.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-08-28T02:16:59.227 INFO:journalctl@ceph.mon.c.smithi143.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-08-28T02:16:59.227 INFO:journalctl@ceph.mon.c.smithi143.stdout: legacy client fscid: 1 2023-08-28T02:16:59.227 INFO:journalctl@ceph.mon.c.smithi143.stdout: 2023-08-28T02:16:59.227 INFO:journalctl@ceph.mon.c.smithi143.stdout: Filesystem 'foo' (1) 2023-08-28T02:16:59.228 INFO:journalctl@ceph.mon.c.smithi143.stdout: fs_name foo 2023-08-28T02:16:59.228 INFO:journalctl@ceph.mon.c.smithi143.stdout: epoch 5 2023-08-28T02:16:59.228 INFO:journalctl@ceph.mon.c.smithi143.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-08-28T02:16:59.228 INFO:journalctl@ceph.mon.c.smithi143.stdout: created 2023-08-28T02:11:22.178022+0000 2023-08-28T02:16:59.228 INFO:journalctl@ceph.mon.c.smithi143.stdout: modified 2023-08-28T02:11:30.935128+0000 2023-08-28T02:16:59.228 INFO:journalctl@ceph.mon.c.smithi143.stdout: tableserver 0 2023-08-28T02:16:59.228 INFO:journalctl@ceph.mon.c.smithi143.stdout: root 0 2023-08-28T02:16:59.228 INFO:journalctl@ceph.mon.c.smithi143.stdout: session_timeout 60 2023-08-28T02:16:59.228 INFO:journalctl@ceph.mon.c.smithi143.stdout: session_autoclose 300 2023-08-28T02:16:59.228 INFO:journalctl@ceph.mon.c.smithi143.stdout: max_file_size 1099511627776 2023-08-28T02:16:59.228 INFO:journalctl@ceph.mon.c.smithi143.stdout: required_client_features {} 2023-08-28T02:16:59.228 INFO:journalctl@ceph.mon.c.smithi143.stdout: last_failure 0 2023-08-28T02:16:59.228 INFO:journalctl@ceph.mon.c.smithi143.stdout: last_failure_osd_epoch 0 2023-08-28T02:16:59.229 INFO:journalctl@ceph.mon.c.smithi143.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-08-28T02:16:59.229 INFO:journalctl@ceph.mon.c.smithi143.stdout: max_mds 1 2023-08-28T02:16:59.229 INFO:journalctl@ceph.mon.c.smithi143.stdout: in 0 2023-08-28T02:16:59.229 INFO:journalctl@ceph.mon.c.smithi143.stdout: up {0=14637} 2023-08-28T02:16:59.229 INFO:journalctl@ceph.mon.c.smithi143.stdout: failed 2023-08-28T02:16:59.229 INFO:journalctl@ceph.mon.c.smithi143.stdout: damaged 2023-08-28T02:16:59.229 INFO:journalctl@ceph.mon.c.smithi143.stdout: stopped 2023-08-28T02:16:59.229 INFO:journalctl@ceph.mon.c.smithi143.stdout: data_pools [4] 2023-08-28T02:16:59.229 INFO:journalctl@ceph.mon.c.smithi143.stdout: metadata_pool 3 2023-08-28T02:16:59.229 INFO:journalctl@ceph.mon.c.smithi143.stdout: inline_data disabled 2023-08-28T02:16:59.229 INFO:journalctl@ceph.mon.c.smithi143.stdout: balancer 2023-08-28T02:16:59.229 INFO:journalctl@ceph.mon.c.smithi143.stdout: bal_rank_mask -1 2023-08-28T02:16:59.229 INFO:journalctl@ceph.mon.c.smithi143.stdout: standby_count_wanted 1 2023-08-28T02:16:59.229 INFO:journalctl@ceph.mon.c.smithi143.stdout: [mds.foo.smithi143.spifrj{0:14637} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.143:6834/2211284212,v1:172.21.15.143:6835/2211284212] compat {c=[1],r=[1],i=[7ff]}] 2023-08-28T02:16:59.230 INFO:journalctl@ceph.mon.c.smithi143.stdout: 2023-08-28T02:16:59.230 INFO:journalctl@ceph.mon.c.smithi143.stdout: 2023-08-28T02:16:59.230 INFO:journalctl@ceph.mon.c.smithi143.stdout: Standby daemons: 2023-08-28T02:16:59.230 INFO:journalctl@ceph.mon.c.smithi143.stdout: 2023-08-28T02:16:59.230 INFO:journalctl@ceph.mon.c.smithi143.stdout: [mds.foo.smithi169.yeisur{-1:24520} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.169:6832/3278371000,v1:172.21.15.169:6833/3278371000] compat {c=[1],r=[1],i=[7ff]}] 2023-08-28T02:16:59.230 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: mon.c@-1(???).osd e86 crush map has features 3314933000854323200, adjusting msgr requires 2023-08-28T02:16:59.230 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: mon.c@-1(???).osd e86 crush map has features 432629239337189376, adjusting msgr requires 2023-08-28T02:16:59.230 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: mon.c@-1(???).osd e86 crush map has features 432629239337189376, adjusting msgr requires 2023-08-28T02:16:59.230 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: mon.c@-1(???).osd e86 crush map has features 432629239337189376, adjusting msgr requires 2023-08-28T02:16:59.230 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: mon.c@-1(???).paxosservice(auth 1..19) refresh upgraded, format 0 -> 3 2023-08-28T02:16:59.230 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: expand_channel_meta expand map: {default=false} 2023-08-28T02:16:59.230 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: expand_channel_meta from 'false' to 'false' 2023-08-28T02:16:59.230 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: expand_channel_meta expanded map: {default=false} 2023-08-28T02:16:59.231 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: expand_channel_meta expand map: {default=info} 2023-08-28T02:16:59.231 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: expand_channel_meta from 'info' to 'info' 2023-08-28T02:16:59.231 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: expand_channel_meta expanded map: {default=info} 2023-08-28T02:16:59.231 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: expand_channel_meta expand map: {default=daemon} 2023-08-28T02:16:59.231 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: expand_channel_meta from 'daemon' to 'daemon' 2023-08-28T02:16:59.231 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: expand_channel_meta expanded map: {default=daemon} 2023-08-28T02:16:59.231 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: expand_channel_meta expand map: {default=debug} 2023-08-28T02:16:59.231 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: expand_channel_meta from 'debug' to 'debug' 2023-08-28T02:16:59.231 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:16:58 smithi143 ceph-mon[189871]: expand_channel_meta expanded map: {default=debug} 2023-08-28T02:17:02.201 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:17:01 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:17:01] "GET /metrics HTTP/1.1" 200 33937 "" "Prometheus/2.33.4" 2023-08-28T02:17:04.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:04 smithi169 conmon[100500]: cluster 2023-08-28T02:16:56.965972+0000 mgr.y (mgr.34107) 59 : cluster [DBG] pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:04.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:04 smithi169 conmon[100500]: audit 2023-08-28T02:16:57. 2023-08-28T02:17:04.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:04 smithi169 conmon[100500]: 513639+0000 mon.b (mon.1) 112 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T02:17:04.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:04 smithi169 conmon[100500]: audit 2023-08-28T02:16:57.513853+0000 mon.b (mon.1) 113 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T02:17:04.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:04 smithi169 conmon[100500]: audit 2023-08-28T02:16:57.514040+0000 mon.b (mon.1) 114 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T02:17:04.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:04 smithi169 conmon[100500]: audit 2023-08-28T02:16:57.601316+0000 mon.a (mon.0) 52 : audit [DBG] from='client.? 172.21.15.143:0/2365514943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:05.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:05 smithi169 conmon[100500]: cluster 2023-08-28T02:16:59.052029+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-08-28T02:17:05.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:05 smithi169 conmon[100500]: cluster 2023-08-28T02:17:05.054349+0000 mon.b (mon.1) 117 : cluster [INF] mon.b calling monitor election 2023-08-28T02:17:05.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:05 smithi169 conmon[100500]: cluster 2023-08-28T02:17:05.054489+0000 mon.a (mon.0) 66 : cluster [INF] mon.a calling monitor election 2023-08-28T02:17:05.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:05 smithi169 conmon[100500]: cluster 2023-08-28T02:17:05.080310+0000 mon.a (mon.0) 67 : 2023-08-28T02:17:05.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:05 smithi169 conmon[100500]: cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-08-28T02:17:05.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:05 smithi169 conmon[100500]: cluster 2023-08-28T02:17:05 2023-08-28T02:17:05.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:05 smithi169 conmon[100500]: .088359+0000 mon.a (mon.0) 68 : cluster [DBG] 2023-08-28T02:17:05.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:05 smithi169 conmon[100500]: monmap e3: 3 mons at {a=[v2:172.21.15.143:3300/0,v1:172.21.15.143:6789/0],b=[v2:172.21.15.169:3300/0,v1:172.21.15.169:6789/0],c=[v2:172.21.15.143:3301/0,v1:172.21.15.143:6790/0]} removed_ranks: {} 2023-08-28T02:17:05.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:05 smithi169 conmon[100500]: cluster 2023-08-28T02:17:05.088813 2023-08-28T02:17:05.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:05 smithi169 conmon[100500]: +0000 mon.a (mon.0) 69 : cluster [DBG] fsmap foo:1 {0=foo.smithi143.spifrj=up:active} 1 up:standby 2023-08-28T02:17:05.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:05 smithi169 conmon[100500]: 2023-08-28T02:17:05.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:05 smithi169 conmon[100500]: cluster 2023-08-28T02:17:05.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:05 smithi169 conmon[100500]: 2023-08-28T02:17:05.088848+0000 mon.a (mon.0 2023-08-28T02:17:05.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:05 smithi169 conmon[100500]: ) 70 : cluster [DBG] osdmap e86: 8 total, 8 up, 8 in 2023-08-28T02:17:05.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:05 smithi169 conmon[100500]: cluster 2023-08-28T02: 2023-08-28T02:17:05.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:05 smithi169 conmon[100500]: 17:05.089168+0000 mon.a (mon.0) 71 : 2023-08-28T02:17:05.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:05 smithi169 conmon[100500]: cluster [DBG] mgrmap e34: y(active, since 84s), standbys: x 2023-08-28T02:17:05.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:05 smithi169 conmon[100500]: cluster 2023-08-28T02:17: 2023-08-28T02:17:05.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:05 smithi169 conmon[100500]: 05.089293+0000 mon.a (mon.0) 72 : cluster 2023-08-28T02:17:05.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:05 smithi169 conmon[100500]: [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-08-28T02:17:05.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:05 smithi169 conmon[100500]: cluster 2023-08-28T02:17: 2023-08-28T02:17:05.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:05 smithi169 conmon[100500]: 05.089307+0000 mon.a (mon.0) 73 : cluster 2023-08-28T02:17:05.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:05 smithi169 conmon[100500]: [INF] Cluster is now healthy 2023-08-28T02:17:05.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:05 smithi169 conmon[100500]: cluster 2023-08-28T02:17:05.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:05 smithi169 conmon[100500]: 2023-08-28T02:17:05.094403+0000 mon.a (mon. 2023-08-28T02:17:05.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:05 smithi169 conmon[100500]: 0) 74 : cluster [INF] overall HEALTH_OK 2023-08-28T02:17:06.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:06 smithi169 conmon[100500]: cluster 2023-08-28T02:17:04.968142+0000 mgr.y (mgr.34107) 63 : cluster [DBG] pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:06.837 INFO:teuthology.orchestra.run.smithi143.stdout: "ceph version 18.2.0-187-g9cf20820 (9cf208207dfe23ffccb33308b22e51570cb45e51) reef (stable)": 2 2023-08-28T02:17:07.485 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -e sha1=9cf208207dfe23ffccb33308b22e51570cb45e51 -- bash -c 'ceph orch ps' 2023-08-28T02:17:07.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:07 smithi169 conmon[100500]: audit 2023-08-28T02:17:06.720939+0000 mon.a (mon.0) 75 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:07.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:07 smithi169 conmon[100500]: audit 2023-08-28T02:17:06.727437+0000 mon.a (mon.0) 76 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:07.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:07 smithi169 conmon[100500]: audit 2023-08-28T02:17:06.825753+0000 mon.a (mon.0) 77 : audit [DBG] from='client.? 172.21.15.143:0/3847681820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:07.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:07 smithi169 conmon[100500]: cluster 2023-08-28T02 2023-08-28T02:17:07.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:07 smithi169 conmon[100500]: :17:06.968573+0000 mgr.y (mgr.34107) 64 : cluster [DBG] pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:07.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:07 smithi169 conmon[100500]: audit 2023-08-28T02:17:07 2023-08-28T02:17:07.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:07 smithi169 conmon[100500]: .662909+0000 mon.a (mon.0) 78 : 2023-08-28T02:17:07.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:07 smithi169 conmon[100500]: audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:07.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:07 smithi169 conmon[100500]: audit 2023-08-28T02:17: 2023-08-28T02:17:07.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:07 smithi169 conmon[100500]: 07.671754+0000 mon.a (mon.0) 79 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:07.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:07 smithi169 conmon[100500]: 2023-08-28T02:17:08.919 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:17:08 smithi143 conmon[176053]: 2023-08-28T02:17:08.850+0000 7f008786b700 -1 mgr.server handle_report got status from non-daemon mon.c 2023-08-28T02:17:08.987 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:17:08 smithi169 conmon[150127]: ts=2023-08-28T02:17:08.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi169.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi143:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-08-28T02:17:09.361 INFO:teuthology.orchestra.run.smithi143.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-08-28T02:17:09.361 INFO:teuthology.orchestra.run.smithi143.stdout:alertmanager.a smithi143 *:9093,9094 running (2m) 2s ago 6m 26.8M - 0.23.0 ba2b418f427c f216a6d30bb2 2023-08-28T02:17:09.361 INFO:teuthology.orchestra.run.smithi143.stdout:grafana.a smithi169 *:3000 running (2m) 16s ago 6m 51.7M - 8.3.5 dad864ee21e9 4c9eb6ad00e1 2023-08-28T02:17:09.361 INFO:teuthology.orchestra.run.smithi143.stdout:mds.foo.smithi143.spifrj smithi143 running (5m) 2s ago 5m 17.9M - 16.2.13-565-g595355f4 4099d2e00414 4eeedd3bc917 2023-08-28T02:17:09.361 INFO:teuthology.orchestra.run.smithi143.stdout:mds.foo.smithi169.yeisur smithi169 running (5m) 16s ago 5m 18.0M - 16.2.13-565-g595355f4 4099d2e00414 fed9013c1357 2023-08-28T02:17:09.361 INFO:teuthology.orchestra.run.smithi143.stdout:mgr.x smithi169 *:8443,9283,8765 running (110s) 16s ago 10m 446M - 18.2.0-187-g9cf20820 62b871dea10e 73a560edcde9 2023-08-28T02:17:09.361 INFO:teuthology.orchestra.run.smithi143.stdout:mgr.y smithi143 *:8443,9283,8765 running (2m) 2s ago 11m 501M - 18.2.0-187-g9cf20820 62b871dea10e 8de2036a51e3 2023-08-28T02:17:09.361 INFO:teuthology.orchestra.run.smithi143.stdout:mon.a smithi143 running (98s) 2s ago 11m 76.4M 2048M 18.2.0-187-g9cf20820 62b871dea10e 072272663d4f 2023-08-28T02:17:09.361 INFO:teuthology.orchestra.run.smithi143.stdout:mon.b smithi169 running (10m) 16s ago 10m 97.7M 2048M 16.2.13-565-g595355f4 4099d2e00414 f70db2856b37 2023-08-28T02:17:09.362 INFO:teuthology.orchestra.run.smithi143.stdout:mon.c smithi143 running (10s) 2s ago 10m 25.3M 2048M 18.2.0-187-g9cf20820 62b871dea10e f1328bdbe045 2023-08-28T02:17:09.362 INFO:teuthology.orchestra.run.smithi143.stdout:node-exporter.a smithi143 *:9100 running (3m) 2s ago 6m 28.1M - 1.3.1 1dbe0e931976 e389cc167c39 2023-08-28T02:17:09.362 INFO:teuthology.orchestra.run.smithi143.stdout:node-exporter.b smithi169 *:9100 running (2m) 16s ago 6m 21.1M - 1.3.1 1dbe0e931976 5e5215d11ac8 2023-08-28T02:17:09.362 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0 smithi143 running (9m) 2s ago 9m 73.4M 2500M 16.2.13-565-g595355f4 4099d2e00414 ce7c7eb91333 2023-08-28T02:17:09.362 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1 smithi143 running (9m) 2s ago 9m 90.4M 2500M 16.2.13-565-g595355f4 4099d2e00414 f27082b59650 2023-08-28T02:17:09.362 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2 smithi143 running (9m) 2s ago 9m 68.2M 2500M 16.2.13-565-g595355f4 4099d2e00414 e20c9920166c 2023-08-28T02:17:09.362 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3 smithi143 running (8m) 2s ago 8m 66.6M 2500M 16.2.13-565-g595355f4 4099d2e00414 ab95599724c4 2023-08-28T02:17:09.362 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4 smithi169 running (8m) 16s ago 8m 71.1M 2500M 16.2.13-565-g595355f4 4099d2e00414 e6ed776def27 2023-08-28T02:17:09.362 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5 smithi169 running (8m) 16s ago 8m 83.9M 2500M 16.2.13-565-g595355f4 4099d2e00414 6fe1195fca7e 2023-08-28T02:17:09.362 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6 smithi169 running (7m) 16s ago 7m 78.8M 2500M 16.2.13-565-g595355f4 4099d2e00414 12854ca48e76 2023-08-28T02:17:09.362 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7 smithi169 running (7m) 16s ago 7m 95.4M 2500M 16.2.13-565-g595355f4 4099d2e00414 64fa1422c85a 2023-08-28T02:17:09.362 INFO:teuthology.orchestra.run.smithi143.stdout:prometheus.a smithi169 *:9095 running (115s) 16s ago 6m 56.4M - 2.33.4 514e6a882f6e cd763ce7d282 2023-08-28T02:17:10.092 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -e sha1=9cf208207dfe23ffccb33308b22e51570cb45e51 -- bash -c 'ceph orch upgrade pause' 2023-08-28T02:17:10.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:09 smithi169 conmon[100500]: cluster 2023-08-28T02:17:08.968895+0000 mgr.y (mgr.34107) 65 : cluster [DBG] pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:11.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:10 smithi169 conmon[100500]: audit 2023-08-28T02:17:09.353548+0000 mgr.y (mgr.34107) 66 : audit [DBG] from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:17:12.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:17:11 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:17:11] "GET /metrics HTTP/1.1" 200 33933 "" "Prometheus/2.33.4" 2023-08-28T02:17:12.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:11 smithi169 conmon[100500]: cluster 2023-08-28T02:17:10.969404+0000 mgr.y (mgr.34107) 67 : cluster [DBG] pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:12.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:11 smithi169 conmon[100500]: audit 2023-08-28T02:17:11.012377+0000 mon.b (mon.1) 118 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:17:12.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:11 smithi169 conmon[100500]: audit 2023-08-28T02:17:11.749065+0000 mon.a (mon.0) 80 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:12.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:11 smithi169 conmon[100500]: audit 2023-08-28T02:17:12.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:11 smithi169 conmon[100500]: 2023-08-28T02:17:11.759611+0000 mon.a (mon.0) 81 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:12.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:11 smithi169 conmon[100500]: audit 2023-08-28T02:17:11.761140+0000 mon.b (mon.1 2023-08-28T02:17:12.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:11 smithi169 conmon[100500]: ) 119 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:12.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:11 smithi169 conmon[100500]: audit 2023-08-28T02:17:11.762030 2023-08-28T02:17:12.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:11 smithi169 conmon[100500]: +0000 mon.b (mon.1) 120 : audit [INF] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:17:12.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:11 smithi169 conmon[100500]: audit 2023-08-28 2023-08-28T02:17:12.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:11 smithi169 conmon[100500]: T02:17:11.767145+0000 mon.a (mon.0) 82 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:12.281 INFO:teuthology.orchestra.run.smithi143.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8 2023-08-28T02:17:12.934 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -e sha1=9cf208207dfe23ffccb33308b22e51570cb45e51 -- bash -c 'sleep 60' 2023-08-28T02:17:12.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:12 smithi169 conmon[100500]: audit 2023-08-28T02:17:11.904732+0000 mon.b (mon.1) 121 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:17:12.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:12 smithi169 conmon[100500]: audit 2023-08-28T02 2023-08-28T02:17:12.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:12 smithi169 conmon[100500]: :17:11.907616+0000 mon.b (mon.1) 122 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:17:12.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:12 smithi169 conmon[100500]: audit 2023-08-28T02:17:11.908832+0000 mon.b (mon.1) 123 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-08-28T02:17:12.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:12 smithi169 conmon[100500]: audit 2023-08-28T02:17:11.909606+0000 mon.b (mon.1) 124 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-08-28T02:17:12.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:12 smithi169 conmon[100500]: cephadm 2023-08-28T02:17:11.910059+0000 mgr.y (mgr.34107) 68 : cephadm [INF] Upgrade: It appears safe to stop mon.b 2023-08-28T02:17:12.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:12 smithi169 conmon[100500]: cluster 2023-08-28T02:17:12.076610+0000 mon.a (mon.0) 83 : cluster [DBG] mgrmap e35: y(active, since 91s), standbys: x 2023-08-28T02:17:12.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:12 smithi169 conmon[100500]: audit 2023-08-28T02:17:12.280092+0000 mon.a (mon. 2023-08-28T02:17:12.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:12 smithi169 conmon[100500]: 0) 84 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:14.112 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:13 smithi169 conmon[100500]: audit 2023-08-28T02:17:12.275922+0000 mgr.y (mgr.34107) 2023-08-28T02:17:14.113 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:13 smithi169 conmon[100500]: 69 : audit [DBG] from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:17:14.113 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:13 smithi169 conmon[100500]: cephadm 2023-08-28T02:17:12.276297+0000 mgr.y (mgr.34107) 70 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8 2023-08-28T02:17:14.113 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:13 smithi169 conmon[100500]: cluster 2023-08-28T02:17:12.970021+0000 mgr.y (mgr.34107) 71 : cluster [DBG] pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-08-28T02:17:14.113 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:13 smithi169 conmon[100500]: audit 2023-08-28T02:17:13.331551+0000 mon.a (mon.0) 85 : audit [INF] from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:14.113 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:13 smithi169 conmon[100500]: audit 2023-08-28T02:17:13.332444+0000 mon.b (mon.1) 125 : audit [INF] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-28T02:17:14.113 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:13 smithi169 conmon[100500]: audit 2023-08-28T02:17:13.333088+0000 mon.a (mon.0) 86 : audit [DBG] from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-28T02:17:14.113 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:13 smithi169 conmon[100500]: audit 2023-08-28T02:17:13.333746+0000 mon.b (mon.1) 126 : audit [DBG] from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:14.871 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:14 smithi169 systemd[1]: Stopping Ceph mon.b for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:17:15.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:17:14 smithi143 conmon[176053]: [28/Aug/2023:02:17:14] ENGINE Bus STOPPING 2023-08-28T02:17:15.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:17:14 smithi143 conmon[176053]: [28/Aug/2023:02:17:14] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-08-28T02:17:15.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:17:14 smithi143 conmon[176053]: [28/Aug/2023:02:17:14] ENGINE Bus STOPPED 2023-08-28T02:17:15.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:17:14 smithi143 conmon[176053]: [28/Aug/2023:02:17:14] ENGINE Bus STARTING 2023-08-28T02:17:15.201 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:17:15 smithi143 conmon[176053]: [28/Aug/2023:02:17:15] ENGINE Serving on http://:::9283 2023-08-28T02:17:15.201 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:17:15 smithi143 conmon[176053]: [28/Aug/2023:02:17:15] ENGINE Bus STARTED 2023-08-28T02:17:15.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:14 smithi169 conmon[100500]: cephadm 2023-08-28T02:17:13.326445+0000 2023-08-28T02:17:15.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:14 smithi169 conmon[100500]: mgr.y (mgr.34107) 72 : cephadm [INF] Upgrade: Updating mon.b 2023-08-28T02:17:15.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:14 smithi169 conmon[100500]: cephadm 2023-08-28T02:17:13.334389+0000 mgr.y (mgr.34107) 73 : cephadm [INF] Deploying daemon mon.b on smithi169 2023-08-28T02:17:15.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:14 smithi169 conmon[100500]: debug 2023-08-28T02:17:14.934+0000 7f2d4a9f3700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-08-28T02:17:15.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:14 smithi169 conmon[100500]: debug 2023-08-28T02:17:14.934+0000 7f2d4a9f3700 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2023-08-28T02:17:15.787 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:15 smithi169 bash[158984]: f70db2856b371aec7993513e6dcc7fead6faa126ca29a9ad317904181f61ff20 2023-08-28T02:17:15.787 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:15 smithi169 systemd[1]: ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@mon.b.service: Succeeded. 2023-08-28T02:17:15.787 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:15 smithi169 systemd[1]: Stopped Ceph mon.b for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:17:15.787 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:15 smithi169 systemd[1]: Starting Ceph mon.b for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:17:16.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: set uid:gid to 167:167 (ceph:ceph) 2023-08-28T02:17:16.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: ceph version 18.2.0-187-g9cf20820 (9cf208207dfe23ffccb33308b22e51570cb45e51) reef (stable), process ceph-mon, pid 7 2023-08-28T02:17:16.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: pidfile_write: ignore empty --pid-file 2023-08-28T02:17:16.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 bash[159349]: 21231a80906cd7cded78969a37642db5d6b043cbe05dc668c95b3ce47165dc4b 2023-08-28T02:17:16.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: load: jerasure load: lrc 2023-08-28T02:17:16.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: RocksDB version: 7.9.2 2023-08-28T02:17:16.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Git sha 0 2023-08-28T02:17:16.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Compile date 2023-08-25 14:59:12 2023-08-28T02:17:16.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: DB SUMMARY 2023-08-28T02:17:16.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: DB Session ID: TMMKJOKOJTZC3EW7ACPQ 2023-08-28T02:17:16.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: CURRENT file: CURRENT 2023-08-28T02:17:16.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: IDENTITY file: IDENTITY 2023-08-28T02:17:16.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: MANIFEST file: MANIFEST-000005 size: 891 Bytes 2023-08-28T02:17:16.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000020.sst 2023-08-28T02:17:16.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000018.log size: 4896678 ; 2023-08-28T02:17:16.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.error_if_exists: 0 2023-08-28T02:17:16.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.create_if_missing: 0 2023-08-28T02:17:16.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.paranoid_checks: 1 2023-08-28T02:17:16.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.flush_verify_memtable_count: 1 2023-08-28T02:17:16.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-08-28T02:17:16.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-08-28T02:17:16.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.env: 0x5587102e4720 2023-08-28T02:17:16.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.fs: PosixFileSystem 2023-08-28T02:17:16.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.info_log: 0x558712345420 2023-08-28T02:17:16.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.max_file_opening_threads: 16 2023-08-28T02:17:16.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.statistics: (nil) 2023-08-28T02:17:16.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.use_fsync: 0 2023-08-28T02:17:16.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.max_log_file_size: 0 2023-08-28T02:17:16.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-08-28T02:17:16.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.log_file_time_to_roll: 0 2023-08-28T02:17:16.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.keep_log_file_num: 1000 2023-08-28T02:17:16.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.recycle_log_file_num: 0 2023-08-28T02:17:16.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.allow_fallocate: 1 2023-08-28T02:17:16.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.allow_mmap_reads: 0 2023-08-28T02:17:16.741 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.allow_mmap_writes: 0 2023-08-28T02:17:16.741 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.use_direct_reads: 0 2023-08-28T02:17:16.741 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-08-28T02:17:16.741 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.create_missing_column_families: 0 2023-08-28T02:17:16.741 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.db_log_dir: 2023-08-28T02:17:16.741 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.wal_dir: 2023-08-28T02:17:16.741 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.table_cache_numshardbits: 6 2023-08-28T02:17:16.741 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.WAL_ttl_seconds: 0 2023-08-28T02:17:16.741 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.WAL_size_limit_MB: 0 2023-08-28T02:17:16.741 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-08-28T02:17:16.741 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-08-28T02:17:16.741 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.is_fd_close_on_exec: 1 2023-08-28T02:17:16.741 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.advise_random_on_open: 1 2023-08-28T02:17:16.741 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.db_write_buffer_size: 0 2023-08-28T02:17:16.741 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.write_buffer_manager: 0x5587115d4c80 2023-08-28T02:17:16.742 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-08-28T02:17:16.742 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-08-28T02:17:16.742 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.use_adaptive_mutex: 0 2023-08-28T02:17:16.742 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.rate_limiter: (nil) 2023-08-28T02:17:16.742 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-08-28T02:17:16.742 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.wal_recovery_mode: 2 2023-08-28T02:17:16.742 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.enable_thread_tracking: 0 2023-08-28T02:17:16.742 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.enable_pipelined_write: 0 2023-08-28T02:17:16.742 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.unordered_write: 0 2023-08-28T02:17:16.742 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-08-28T02:17:16.743 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-08-28T02:17:16.743 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-08-28T02:17:16.743 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-08-28T02:17:16.743 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.row_cache: None 2023-08-28T02:17:16.743 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.wal_filter: None 2023-08-28T02:17:16.744 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-08-28T02:17:16.744 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.allow_ingest_behind: 0 2023-08-28T02:17:16.744 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.two_write_queues: 0 2023-08-28T02:17:16.744 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.manual_wal_flush: 0 2023-08-28T02:17:16.744 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.wal_compression: 0 2023-08-28T02:17:16.744 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.atomic_flush: 0 2023-08-28T02:17:16.745 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-08-28T02:17:16.745 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.persist_stats_to_disk: 0 2023-08-28T02:17:16.745 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.write_dbid_to_manifest: 0 2023-08-28T02:17:16.745 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.log_readahead_size: 0 2023-08-28T02:17:16.745 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-08-28T02:17:16.745 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.best_efforts_recovery: 0 2023-08-28T02:17:16.746 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-08-28T02:17:16.746 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-08-28T02:17:16.746 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.allow_data_in_errors: 0 2023-08-28T02:17:16.747 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.db_host_id: __hostname__ 2023-08-28T02:17:16.747 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.enforce_single_del_contracts: true 2023-08-28T02:17:16.747 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.max_background_jobs: 2 2023-08-28T02:17:16.747 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.max_background_compactions: -1 2023-08-28T02:17:16.747 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.max_subcompactions: 1 2023-08-28T02:17:16.747 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-08-28T02:17:16.747 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-08-28T02:17:16.747 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.delayed_write_rate : 16777216 2023-08-28T02:17:16.748 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.max_total_wal_size: 0 2023-08-28T02:17:16.748 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-08-28T02:17:16.748 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.stats_dump_period_sec: 600 2023-08-28T02:17:16.748 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.stats_persist_period_sec: 600 2023-08-28T02:17:16.748 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-08-28T02:17:16.748 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.max_open_files: -1 2023-08-28T02:17:16.748 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.bytes_per_sync: 0 2023-08-28T02:17:16.748 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.wal_bytes_per_sync: 0 2023-08-28T02:17:16.748 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.strict_bytes_per_sync: 0 2023-08-28T02:17:16.748 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.compaction_readahead_size: 0 2023-08-28T02:17:16.748 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.max_background_flushes: -1 2023-08-28T02:17:16.748 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Compression algorithms supported: 2023-08-28T02:17:16.749 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-08-28T02:17:16.749 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: kZSTD supported: 0 2023-08-28T02:17:16.749 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: kXpressCompression supported: 0 2023-08-28T02:17:16.749 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: kLZ4HCCompression supported: 1 2023-08-28T02:17:16.749 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: kZlibCompression supported: 1 2023-08-28T02:17:16.749 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: kSnappyCompression supported: 1 2023-08-28T02:17:16.749 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: kLZ4Compression supported: 1 2023-08-28T02:17:16.749 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: kBZip2Compression supported: 0 2023-08-28T02:17:16.749 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Fast CRC32 supported: Supported on x86 2023-08-28T02:17:16.749 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: DMutex implementation: pthread_mutex_t 2023-08-28T02:17:16.749 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 2023-08-28T02:17:16.749 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-08-28T02:17:16.750 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-08-28T02:17:16.750 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.merge_operator: 2023-08-28T02:17:16.750 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.compaction_filter: None 2023-08-28T02:17:16.750 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.compaction_filter_factory: None 2023-08-28T02:17:16.750 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.sst_partitioner_factory: None 2023-08-28T02:17:16.750 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.memtable_factory: SkipListFactory 2023-08-28T02:17:16.750 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.table_factory: BlockBasedTable 2023-08-28T02:17:16.750 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x558712345520) 2023-08-28T02:17:16.750 INFO:journalctl@ceph.mon.b.smithi169.stdout: cache_index_and_filter_blocks: 1 2023-08-28T02:17:16.750 INFO:journalctl@ceph.mon.b.smithi169.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-08-28T02:17:16.750 INFO:journalctl@ceph.mon.b.smithi169.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-08-28T02:17:16.751 INFO:journalctl@ceph.mon.b.smithi169.stdout: pin_top_level_index_and_filter: 1 2023-08-28T02:17:16.751 INFO:journalctl@ceph.mon.b.smithi169.stdout: index_type: 0 2023-08-28T02:17:16.751 INFO:journalctl@ceph.mon.b.smithi169.stdout: data_block_index_type: 0 2023-08-28T02:17:16.751 INFO:journalctl@ceph.mon.b.smithi169.stdout: index_shortening: 1 2023-08-28T02:17:16.751 INFO:journalctl@ceph.mon.b.smithi169.stdout: data_block_hash_table_util_ratio: 0.750000 2023-08-28T02:17:16.751 INFO:journalctl@ceph.mon.b.smithi169.stdout: checksum: 4 2023-08-28T02:17:16.751 INFO:journalctl@ceph.mon.b.smithi169.stdout: no_block_cache: 0 2023-08-28T02:17:16.751 INFO:journalctl@ceph.mon.b.smithi169.stdout: block_cache: 0x558711657350 2023-08-28T02:17:16.751 INFO:journalctl@ceph.mon.b.smithi169.stdout: block_cache_name: BinnedLRUCache 2023-08-28T02:17:16.751 INFO:journalctl@ceph.mon.b.smithi169.stdout: block_cache_options: 2023-08-28T02:17:16.751 INFO:journalctl@ceph.mon.b.smithi169.stdout: capacity : 536870912 2023-08-28T02:17:16.752 INFO:journalctl@ceph.mon.b.smithi169.stdout: num_shard_bits : 4 2023-08-28T02:17:16.752 INFO:journalctl@ceph.mon.b.smithi169.stdout: strict_capacity_limit : 0 2023-08-28T02:17:16.752 INFO:journalctl@ceph.mon.b.smithi169.stdout: high_pri_pool_ratio: 0.000 2023-08-28T02:17:16.752 INFO:journalctl@ceph.mon.b.smithi169.stdout: block_cache_compressed: (nil) 2023-08-28T02:17:16.752 INFO:journalctl@ceph.mon.b.smithi169.stdout: persistent_cache: (nil) 2023-08-28T02:17:16.752 INFO:journalctl@ceph.mon.b.smithi169.stdout: block_size: 4096 2023-08-28T02:17:16.752 INFO:journalctl@ceph.mon.b.smithi169.stdout: block_size_deviation: 10 2023-08-28T02:17:16.752 INFO:journalctl@ceph.mon.b.smithi169.stdout: block_restart_interval: 16 2023-08-28T02:17:16.752 INFO:journalctl@ceph.mon.b.smithi169.stdout: index_block_restart_interval: 1 2023-08-28T02:17:16.752 INFO:journalctl@ceph.mon.b.smithi169.stdout: metadata_block_size: 4096 2023-08-28T02:17:16.752 INFO:journalctl@ceph.mon.b.smithi169.stdout: partition_filters: 0 2023-08-28T02:17:16.753 INFO:journalctl@ceph.mon.b.smithi169.stdout: use_delta_encoding: 1 2023-08-28T02:17:16.753 INFO:journalctl@ceph.mon.b.smithi169.stdout: filter_policy: bloomfilter 2023-08-28T02:17:16.753 INFO:journalctl@ceph.mon.b.smithi169.stdout: whole_key_filtering: 1 2023-08-28T02:17:16.753 INFO:journalctl@ceph.mon.b.smithi169.stdout: verify_compression: 0 2023-08-28T02:17:16.753 INFO:journalctl@ceph.mon.b.smithi169.stdout: read_amp_bytes_per_bit: 0 2023-08-28T02:17:16.753 INFO:journalctl@ceph.mon.b.smithi169.stdout: format_version: 5 2023-08-28T02:17:16.753 INFO:journalctl@ceph.mon.b.smithi169.stdout: enable_index_compression: 1 2023-08-28T02:17:16.753 INFO:journalctl@ceph.mon.b.smithi169.stdout: block_align: 0 2023-08-28T02:17:16.753 INFO:journalctl@ceph.mon.b.smithi169.stdout: max_auto_readahead_size: 262144 2023-08-28T02:17:16.753 INFO:journalctl@ceph.mon.b.smithi169.stdout: prepopulate_block_cache: 0 2023-08-28T02:17:16.753 INFO:journalctl@ceph.mon.b.smithi169.stdout: initial_auto_readahead_size: 8192 2023-08-28T02:17:16.753 INFO:journalctl@ceph.mon.b.smithi169.stdout: num_file_reads_for_auto_readahead: 2 2023-08-28T02:17:16.753 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.write_buffer_size: 33554432 2023-08-28T02:17:16.754 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.max_write_buffer_number: 2 2023-08-28T02:17:16.754 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.compression: NoCompression 2023-08-28T02:17:16.754 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.bottommost_compression: Disabled 2023-08-28T02:17:16.754 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.prefix_extractor: nullptr 2023-08-28T02:17:16.754 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-08-28T02:17:16.754 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.num_levels: 7 2023-08-28T02:17:16.754 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-08-28T02:17:16.754 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-08-28T02:17:16.754 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-08-28T02:17:16.754 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-08-28T02:17:16.754 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-08-28T02:17:16.754 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-08-28T02:17:16.755 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-08-28T02:17:16.755 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-08-28T02:17:16.755 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-08-28T02:17:16.755 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-08-28T02:17:16.755 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-08-28T02:17:16.755 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-08-28T02:17:16.755 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.compression_opts.window_bits: -14 2023-08-28T02:17:16.755 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.compression_opts.level: 32767 2023-08-28T02:17:16.755 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.compression_opts.strategy: 0 2023-08-28T02:17:16.755 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-08-28T02:17:16.755 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-08-28T02:17:16.755 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-08-28T02:17:16.755 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-08-28T02:17:16.756 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.compression_opts.enabled: false 2023-08-28T02:17:16.756 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-08-28T02:17:16.756 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-08-28T02:17:16.756 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-08-28T02:17:16.756 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-08-28T02:17:16.756 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.target_file_size_base: 67108864 2023-08-28T02:17:16.756 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.target_file_size_multiplier: 1 2023-08-28T02:17:16.756 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-08-28T02:17:16.756 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-08-28T02:17:16.756 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-08-28T02:17:16.756 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-08-28T02:17:16.756 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-08-28T02:17:16.756 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-08-28T02:17:16.757 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-08-28T02:17:16.757 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-08-28T02:17:16.757 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-08-28T02:17:16.757 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-08-28T02:17:16.757 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-08-28T02:17:16.757 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-08-28T02:17:16.757 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-08-28T02:17:16.757 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.arena_block_size: 1048576 2023-08-28T02:17:16.757 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-08-28T02:17:16.757 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-08-28T02:17:16.757 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.disable_auto_compactions: 0 2023-08-28T02:17:16.757 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-08-28T02:17:16.757 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-08-28T02:17:16.758 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-08-28T02:17:16.758 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-08-28T02:17:16.758 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-08-28T02:17:16.758 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-08-28T02:17:16.758 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-08-28T02:17:16.758 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-08-28T02:17:16.758 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-08-28T02:17:16.758 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-08-28T02:17:16.758 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.table_properties_collectors: 2023-08-28T02:17:16.758 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.inplace_update_support: 0 2023-08-28T02:17:16.758 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.inplace_update_num_locks: 10000 2023-08-28T02:17:16.758 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-08-28T02:17:16.758 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-08-28T02:17:16.759 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.memtable_huge_page_size: 0 2023-08-28T02:17:16.759 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.bloom_locality: 0 2023-08-28T02:17:16.759 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.max_successive_merges: 0 2023-08-28T02:17:16.759 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.optimize_filters_for_hits: 0 2023-08-28T02:17:16.759 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.paranoid_file_checks: 0 2023-08-28T02:17:16.759 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.force_consistency_checks: 1 2023-08-28T02:17:16.759 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.report_bg_io_stats: 0 2023-08-28T02:17:16.759 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.ttl: 2592000 2023-08-28T02:17:16.759 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.periodic_compaction_seconds: 0 2023-08-28T02:17:16.759 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-08-28T02:17:16.759 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-08-28T02:17:16.759 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.enable_blob_files: false 2023-08-28T02:17:16.759 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.min_blob_size: 0 2023-08-28T02:17:16.760 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.blob_file_size: 268435456 2023-08-28T02:17:16.760 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.blob_compression_type: NoCompression 2023-08-28T02:17:16.760 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.enable_blob_garbage_collection: false 2023-08-28T02:17:16.760 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-08-28T02:17:16.760 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-08-28T02:17:16.760 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-08-28T02:17:16.760 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.blob_file_starting_level: 0 2023-08-28T02:17:16.760 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-08-28T02:17:16.760 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 20.sst 2023-08-28T02:17:16.760 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-08-28T02:17:16.760 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 22, last_sequence is 12155, log_number is 18,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-08-28T02:17:16.760 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 18 2023-08-28T02:17:16.760 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: d450cb2e-fdf4-4e58-b366-0917c2a177b6 2023-08-28T02:17:16.761 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: EVENT_LOG_v1 {"time_micros": 1693189036365376, "job": 1, "event": "recovery_started", "wal_files": [18]} 2023-08-28T02:17:16.761 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #18 mode 2 2023-08-28T02:17:16.761 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 systemd[1]: Started Ceph mon.b for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:17:16.761 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: EVENT_LOG_v1 {"time_micros": 1693189036375891, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 23, "file_size": 3154970, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12160, "largest_seqno": 13102, "table_properties": {"data_size": 3149406, "index_size": 3841, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 709, "raw_key_size": 5444, "raw_average_key_size": 20, "raw_value_size": 3141456, "raw_average_value_size": 11592, "num_data_blocks": 170, "num_entries": 271, "num_filter_entries": 271, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1693189036, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "d450cb2e-fdf4-4e58-b366-0917c2a177b6", "db_session_id": "TMMKJOKOJTZC3EW7ACPQ", "orig_file_number": 23, "seqno_to_time_mapping": "N/A"}} 2023-08-28T02:17:16.761 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: EVENT_LOG_v1 {"time_micros": 1693189036375984, "job": 1, "event": "recovery_finished"} 2023-08-28T02:17:16.761 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: [db/version_set.cc:5047] Creating manifest 25 2023-08-28T02:17:16.761 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-08-28T02:17:16.761 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000018.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-08-28T02:17:16.761 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5587116f2000 2023-08-28T02:17:16.761 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: DB pointer 0x5587116e0000 2023-08-28T02:17:16.761 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-08-28T02:17:16.761 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-08-28T02:17:16.761 INFO:journalctl@ceph.mon.b.smithi169.stdout: ** DB Stats ** 2023-08-28T02:17:16.762 INFO:journalctl@ceph.mon.b.smithi169.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-08-28T02:17:16.762 INFO:journalctl@ceph.mon.b.smithi169.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-08-28T02:17:16.762 INFO:journalctl@ceph.mon.b.smithi169.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-08-28T02:17:16.762 INFO:journalctl@ceph.mon.b.smithi169.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-08-28T02:17:16.762 INFO:journalctl@ceph.mon.b.smithi169.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-08-28T02:17:16.762 INFO:journalctl@ceph.mon.b.smithi169.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-08-28T02:17:16.762 INFO:journalctl@ceph.mon.b.smithi169.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-08-28T02:17:16.762 INFO:journalctl@ceph.mon.b.smithi169.stdout: 2023-08-28T02:17:16.762 INFO:journalctl@ceph.mon.b.smithi169.stdout: ** Compaction Stats [default] ** 2023-08-28T02:17:16.762 INFO:journalctl@ceph.mon.b.smithi169.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-08-28T02:17:16.762 INFO:journalctl@ceph.mon.b.smithi169.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-08-28T02:17:16.762 INFO:journalctl@ceph.mon.b.smithi169.stdout: L0 1/0 3.01 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 699.1 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-08-28T02:17:16.762 INFO:journalctl@ceph.mon.b.smithi169.stdout: L6 1/0 24.08 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-08-28T02:17:16.763 INFO:journalctl@ceph.mon.b.smithi169.stdout: Sum 2/0 27.09 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 699.1 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-08-28T02:17:16.763 INFO:journalctl@ceph.mon.b.smithi169.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 699.1 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-08-28T02:17:16.763 INFO:journalctl@ceph.mon.b.smithi169.stdout: 2023-08-28T02:17:16.763 INFO:journalctl@ceph.mon.b.smithi169.stdout: ** Compaction Stats [default] ** 2023-08-28T02:17:16.763 INFO:journalctl@ceph.mon.b.smithi169.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-08-28T02:17:16.763 INFO:journalctl@ceph.mon.b.smithi169.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-08-28T02:17:16.763 INFO:journalctl@ceph.mon.b.smithi169.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 699.1 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-08-28T02:17:16.763 INFO:journalctl@ceph.mon.b.smithi169.stdout: 2023-08-28T02:17:16.763 INFO:journalctl@ceph.mon.b.smithi169.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-08-28T02:17:16.763 INFO:journalctl@ceph.mon.b.smithi169.stdout: 2023-08-28T02:17:16.763 INFO:journalctl@ceph.mon.b.smithi169.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-08-28T02:17:16.763 INFO:journalctl@ceph.mon.b.smithi169.stdout: Flush(GB): cumulative 0.003, interval 0.003 2023-08-28T02:17:16.763 INFO:journalctl@ceph.mon.b.smithi169.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-08-28T02:17:16.764 INFO:journalctl@ceph.mon.b.smithi169.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-08-28T02:17:16.764 INFO:journalctl@ceph.mon.b.smithi169.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-08-28T02:17:16.764 INFO:journalctl@ceph.mon.b.smithi169.stdout: AddFile(Keys): cumulative 0, interval 0 2023-08-28T02:17:16.764 INFO:journalctl@ceph.mon.b.smithi169.stdout: Cumulative compaction: 0.00 GB write, 225.53 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-08-28T02:17:16.764 INFO:journalctl@ceph.mon.b.smithi169.stdout: Interval compaction: 0.00 GB write, 225.53 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-08-28T02:17:16.764 INFO:journalctl@ceph.mon.b.smithi169.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-08-28T02:17:16.764 INFO:journalctl@ceph.mon.b.smithi169.stdout: Block cache BinnedLRUCache@0x558711657350#7 capacity: 512.00 MB usage: 4.86 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.2e-05 secs_since: 0 2023-08-28T02:17:16.764 INFO:journalctl@ceph.mon.b.smithi169.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.78 KB,0.000149012%) IndexBlock(1,4.08 KB,0.000777841%) Misc(1,0.00 KB,0%) 2023-08-28T02:17:16.764 INFO:journalctl@ceph.mon.b.smithi169.stdout: 2023-08-28T02:17:16.764 INFO:journalctl@ceph.mon.b.smithi169.stdout: ** File Read Latency Histogram By Level [default] ** 2023-08-28T02:17:16.764 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: starting mon.b rank 1 at public addrs [v2:172.21.15.169:3300/0,v1:172.21.15.169:6789/0] at bind addrs [v2:172.21.15.169:3300/0,v1:172.21.15.169:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 2023-08-28T02:17:16.764 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: mon.b@-1(???) e3 preinit fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 2023-08-28T02:17:16.764 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: mon.b@-1(???).mds e7 new map 2023-08-28T02:17:16.765 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: mon.b@-1(???).mds e7 print_map 2023-08-28T02:17:16.765 INFO:journalctl@ceph.mon.b.smithi169.stdout: e7 2023-08-28T02:17:16.765 INFO:journalctl@ceph.mon.b.smithi169.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-08-28T02:17:16.765 INFO:journalctl@ceph.mon.b.smithi169.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-08-28T02:17:16.765 INFO:journalctl@ceph.mon.b.smithi169.stdout: legacy client fscid: 1 2023-08-28T02:17:16.765 INFO:journalctl@ceph.mon.b.smithi169.stdout: 2023-08-28T02:17:16.765 INFO:journalctl@ceph.mon.b.smithi169.stdout: Filesystem 'foo' (1) 2023-08-28T02:17:16.765 INFO:journalctl@ceph.mon.b.smithi169.stdout: fs_name foo 2023-08-28T02:17:16.765 INFO:journalctl@ceph.mon.b.smithi169.stdout: epoch 5 2023-08-28T02:17:16.765 INFO:journalctl@ceph.mon.b.smithi169.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-08-28T02:17:16.765 INFO:journalctl@ceph.mon.b.smithi169.stdout: created 2023-08-28T02:11:22.178022+0000 2023-08-28T02:17:16.765 INFO:journalctl@ceph.mon.b.smithi169.stdout: modified 2023-08-28T02:11:30.935128+0000 2023-08-28T02:17:16.765 INFO:journalctl@ceph.mon.b.smithi169.stdout: tableserver 0 2023-08-28T02:17:16.765 INFO:journalctl@ceph.mon.b.smithi169.stdout: root 0 2023-08-28T02:17:16.766 INFO:journalctl@ceph.mon.b.smithi169.stdout: session_timeout 60 2023-08-28T02:17:16.766 INFO:journalctl@ceph.mon.b.smithi169.stdout: session_autoclose 300 2023-08-28T02:17:16.766 INFO:journalctl@ceph.mon.b.smithi169.stdout: max_file_size 1099511627776 2023-08-28T02:17:16.766 INFO:journalctl@ceph.mon.b.smithi169.stdout: required_client_features {} 2023-08-28T02:17:16.766 INFO:journalctl@ceph.mon.b.smithi169.stdout: last_failure 0 2023-08-28T02:17:16.766 INFO:journalctl@ceph.mon.b.smithi169.stdout: last_failure_osd_epoch 0 2023-08-28T02:17:16.766 INFO:journalctl@ceph.mon.b.smithi169.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-08-28T02:17:16.766 INFO:journalctl@ceph.mon.b.smithi169.stdout: max_mds 1 2023-08-28T02:17:16.766 INFO:journalctl@ceph.mon.b.smithi169.stdout: in 0 2023-08-28T02:17:16.766 INFO:journalctl@ceph.mon.b.smithi169.stdout: up {0=14637} 2023-08-28T02:17:16.766 INFO:journalctl@ceph.mon.b.smithi169.stdout: failed 2023-08-28T02:17:16.766 INFO:journalctl@ceph.mon.b.smithi169.stdout: damaged 2023-08-28T02:17:16.766 INFO:journalctl@ceph.mon.b.smithi169.stdout: stopped 2023-08-28T02:17:16.767 INFO:journalctl@ceph.mon.b.smithi169.stdout: data_pools [4] 2023-08-28T02:17:16.767 INFO:journalctl@ceph.mon.b.smithi169.stdout: metadata_pool 3 2023-08-28T02:17:16.767 INFO:journalctl@ceph.mon.b.smithi169.stdout: inline_data disabled 2023-08-28T02:17:16.767 INFO:journalctl@ceph.mon.b.smithi169.stdout: balancer 2023-08-28T02:17:16.767 INFO:journalctl@ceph.mon.b.smithi169.stdout: bal_rank_mask -1 2023-08-28T02:17:16.767 INFO:journalctl@ceph.mon.b.smithi169.stdout: standby_count_wanted 1 2023-08-28T02:17:16.767 INFO:journalctl@ceph.mon.b.smithi169.stdout: [mds.foo.smithi143.spifrj{0:14637} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.143:6834/2211284212,v1:172.21.15.143:6835/2211284212] compat {c=[1],r=[1],i=[7ff]}] 2023-08-28T02:17:16.767 INFO:journalctl@ceph.mon.b.smithi169.stdout: 2023-08-28T02:17:16.767 INFO:journalctl@ceph.mon.b.smithi169.stdout: 2023-08-28T02:17:16.767 INFO:journalctl@ceph.mon.b.smithi169.stdout: Standby daemons: 2023-08-28T02:17:16.767 INFO:journalctl@ceph.mon.b.smithi169.stdout: 2023-08-28T02:17:16.767 INFO:journalctl@ceph.mon.b.smithi169.stdout: [mds.foo.smithi169.yeisur{-1:24520} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.169:6832/3278371000,v1:172.21.15.169:6833/3278371000] compat {c=[1],r=[1],i=[7ff]}] 2023-08-28T02:17:16.767 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: mon.b@-1(???).osd e86 crush map has features 3314933000854323200, adjusting msgr requires 2023-08-28T02:17:16.768 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: mon.b@-1(???).osd e86 crush map has features 432629239337189376, adjusting msgr requires 2023-08-28T02:17:16.768 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: mon.b@-1(???).osd e86 crush map has features 432629239337189376, adjusting msgr requires 2023-08-28T02:17:16.768 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: mon.b@-1(???).osd e86 crush map has features 432629239337189376, adjusting msgr requires 2023-08-28T02:17:16.768 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: mon.b@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2023-08-28T02:17:16.768 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: expand_channel_meta expand map: {default=false} 2023-08-28T02:17:16.768 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: expand_channel_meta from 'false' to 'false' 2023-08-28T02:17:16.768 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: expand_channel_meta expanded map: {default=false} 2023-08-28T02:17:16.768 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: expand_channel_meta expand map: {default=info} 2023-08-28T02:17:16.768 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: expand_channel_meta from 'info' to 'info' 2023-08-28T02:17:16.768 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: expand_channel_meta expanded map: {default=info} 2023-08-28T02:17:16.768 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: expand_channel_meta expand map: {default=daemon} 2023-08-28T02:17:16.768 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: expand_channel_meta from 'daemon' to 'daemon' 2023-08-28T02:17:16.768 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: expand_channel_meta expanded map: {default=daemon} 2023-08-28T02:17:16.769 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: expand_channel_meta expand map: {default=debug} 2023-08-28T02:17:16.769 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: expand_channel_meta from 'debug' to 'debug' 2023-08-28T02:17:16.769 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:16 smithi169 ceph-mon[159415]: expand_channel_meta expanded map: {default=debug} 2023-08-28T02:17:18.987 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:17:18 smithi169 conmon[150127]: ts=2023-08-28T02:17:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi169.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi143:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-08-28T02:17:21.700 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:17:21 smithi143 conmon[176053]: 2023-08-28T02:17:21.386+0000 7f008786b700 -1 mgr.server handle_report got status from non-daemon mon.b 2023-08-28T02:17:21.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: mkfs 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 2023-08-28T02:17:21.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: Reconfiguring daemon prometheus.a on smithi169 2023-08-28T02:17:21.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: Reconfiguring node-exporter.b (dependencies changed)... 2023-08-28T02:17:21.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: Reconfiguring daemon node-exporter.b on smithi169 2023-08-28T02:17:21.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: Reconfiguring grafana.a (dependencies changed)... 2023-08-28T02:17:21.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: Reconfiguring daemon grafana.a on smithi169 2023-08-28T02:17:21.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: Upgrade: Need to upgrade myself (mgr.x) 2023-08-28T02:17:21.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: Upgrade: Need to upgrade myself (mgr.x) 2023-08-28T02:17:21.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: Upgrade: Updating mgr.y 2023-08-28T02:17:21.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: Deploying daemon mgr.y on smithi143 2023-08-28T02:17:21.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: [28/Aug/2023:02:15:03] ENGINE Bus STARTING 2023-08-28T02:17:21.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: [28/Aug/2023:02:15:03] ENGINE Serving on https://172.21.15.143:7150 2023-08-28T02:17:21.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: [28/Aug/2023:02:15:03] ENGINE Serving on http://172.21.15.143:8765 2023-08-28T02:17:21.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: [28/Aug/2023:02:15:03] ENGINE Bus STARTED 2023-08-28T02:17:21.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: Updating smithi143:/etc/ceph/ceph.conf 2023-08-28T02:17:21.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: Updating smithi169:/etc/ceph/ceph.conf 2023-08-28T02:17:21.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: Updating smithi169:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.conf 2023-08-28T02:17:21.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: Updating smithi143:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.conf 2023-08-28T02:17:21.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: Updating smithi169:/etc/ceph/ceph.client.admin.keyring 2023-08-28T02:17:21.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: Updating smithi143:/etc/ceph/ceph.client.admin.keyring 2023-08-28T02:17:21.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: Updating smithi169:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.client.admin.keyring 2023-08-28T02:17:21.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: Updating smithi143:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.client.admin.keyring 2023-08-28T02:17:21.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: Reconfiguring prometheus.a (dependencies changed)... 2023-08-28T02:17:21.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: Reconfiguring daemon prometheus.a on smithi169 2023-08-28T02:17:21.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: Upgrade: Updating mgr.x 2023-08-28T02:17:21.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: Deploying daemon mgr.x on smithi169 2023-08-28T02:17:21.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: Upgrade: Setting container_image for all mgr 2023-08-28T02:17:21.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: Upgrade: It appears safe to stop mon.a 2023-08-28T02:17:21.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: Upgrade: Updating mon.a 2023-08-28T02:17:21.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: Deploying daemon mon.a on smithi143 2023-08-28T02:17:21.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: [28/Aug/2023:02:15:41] ENGINE Bus STARTING 2023-08-28T02:17:21.704 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: [28/Aug/2023:02:15:41] ENGINE Serving on http://172.21.15.143:8765 2023-08-28T02:17:21.704 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: [28/Aug/2023:02:15:41] ENGINE Serving on https://172.21.15.143:7150 2023-08-28T02:17:21.704 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: [28/Aug/2023:02:15:41] ENGINE Bus STARTED 2023-08-28T02:17:21.704 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8 2023-08-28T02:17:21.704 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: Updating smithi143:/etc/ceph/ceph.conf 2023-08-28T02:17:21.704 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: Updating smithi169:/etc/ceph/ceph.conf 2023-08-28T02:17:21.704 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: Updating smithi169:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.conf 2023-08-28T02:17:21.704 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: Updating smithi143:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.conf 2023-08-28T02:17:21.704 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: Updating smithi143:/etc/ceph/ceph.client.admin.keyring 2023-08-28T02:17:21.704 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: Updating smithi169:/etc/ceph/ceph.client.admin.keyring 2023-08-28T02:17:21.704 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: Updating smithi143:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.client.admin.keyring 2023-08-28T02:17:21.704 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: Updating smithi169:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.client.admin.keyring 2023-08-28T02:17:21.704 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.705 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.705 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.705 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.705 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.705 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.705 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.705 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.705 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.705 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.705 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.705 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.705 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.705 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.706 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.706 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.706 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.706 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.706 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.706 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.706 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.706 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.706 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.706 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.706 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.706 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.706 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.707 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.707 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.707 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:17:21.707 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='client.34161 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:17:21.708 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8 2023-08-28T02:17:21.708 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:21.708 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:21.708 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:21.708 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:21.708 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:21.708 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.708 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:21.708 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:21.708 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.708 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:21.708 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:21.709 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:21.709 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:17:21.709 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:21.709 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:17:21.709 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:17:21.709 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-08-28T02:17:21.709 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-08-28T02:17:21.709 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: Upgrade: It appears safe to stop mon.c 2023-08-28T02:17:21.709 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.709 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3782374877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:21.709 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: Upgrade: Updating mon.c 2023-08-28T02:17:21.709 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:21.710 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-28T02:17:21.710 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-28T02:17:21.710 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:21.710 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: Deploying daemon mon.c on smithi143 2023-08-28T02:17:21.710 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:17:21.710 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.710 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T02:17:21.710 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T02:17:21.710 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T02:17:21.710 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2365514943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:21.710 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: mon.c calling monitor election 2023-08-28T02:17:21.710 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.710 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: mon.b calling monitor election 2023-08-28T02:17:21.711 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: mon.a calling monitor election 2023-08-28T02:17:21.711 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-08-28T02:17:21.711 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: monmap e3: 3 mons at {a=[v2:172.21.15.143:3300/0,v1:172.21.15.143:6789/0],b=[v2:172.21.15.169:3300/0,v1:172.21.15.169:6789/0],c=[v2:172.21.15.143:3301/0,v1:172.21.15.143:6790/0]} removed_ranks: {} 2023-08-28T02:17:21.711 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: fsmap foo:1 {0=foo.smithi143.spifrj=up:active} 1 up:standby 2023-08-28T02:17:21.711 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: osdmap e86: 8 total, 8 up, 8 in 2023-08-28T02:17:21.711 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: mgrmap e34: y(active, since 84s), standbys: x 2023-08-28T02:17:21.711 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-08-28T02:17:21.711 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: Cluster is now healthy 2023-08-28T02:17:21.711 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: overall HEALTH_OK 2023-08-28T02:17:21.711 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:21.711 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:21.711 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3847681820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:21.712 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.712 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:21.712 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:21.712 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.712 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:17:21.712 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.712 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:17:21.712 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:21.712 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:21.712 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:21.712 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:17:21.712 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:21.712 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:17:21.713 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:17:21.713 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-08-28T02:17:21.713 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-08-28T02:17:21.713 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: Upgrade: It appears safe to stop mon.b 2023-08-28T02:17:21.713 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: mgrmap e35: y(active, since 91s), standbys: x 2023-08-28T02:17:21.713 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:17:21.713 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8 2023-08-28T02:17:21.713 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:21.713 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-08-28T02:17:21.713 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: Upgrade: Updating mon.b 2023-08-28T02:17:21.713 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:21.713 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-28T02:17:21.713 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-28T02:17:21.714 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:21.714 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: Deploying daemon mon.b on smithi169 2023-08-28T02:17:21.714 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: overall HEALTH_OK 2023-08-28T02:17:21.714 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T02:17:21.714 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: mon.a calling monitor election 2023-08-28T02:17:21.714 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: mon.b calling monitor election 2023-08-28T02:17:21.714 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T02:17:21.714 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T02:17:21.714 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-08-28T02:17:21.714 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-08-28T02:17:21.714 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-08-28T02:17:21.714 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-08-28T02:17:21.714 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: monmap e4: 3 mons at {a=[v2:172.21.15.143:3300/0,v1:172.21.15.143:6789/0],b=[v2:172.21.15.169:3300/0,v1:172.21.15.169:6789/0],c=[v2:172.21.15.143:3301/0,v1:172.21.15.143:6790/0]} removed_ranks: {} 2023-08-28T02:17:21.715 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: fsmap foo:1 {0=foo.smithi143.spifrj=up:active} 1 up:standby 2023-08-28T02:17:21.715 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: osdmap e86: 8 total, 8 up, 8 in 2023-08-28T02:17:21.715 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: mgrmap e35: y(active, since 100s), standbys: x 2023-08-28T02:17:21.715 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-08-28T02:17:21.715 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-08-28T02:17:21.715 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-08-28T02:17:21.715 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: mon.c (rank 2) addr [v2:172.21.15.143:3301/0,v1:172.21.15.143:6790/0] is down (out of quorum) 2023-08-28T02:17:21.715 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:21.715 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:21.715 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:21 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:17:21.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: mkfs 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 2023-08-28T02:17:21.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: Reconfiguring daemon prometheus.a on smithi169 2023-08-28T02:17:21.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: Reconfiguring node-exporter.b (dependencies changed)... 2023-08-28T02:17:21.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: Reconfiguring daemon node-exporter.b on smithi169 2023-08-28T02:17:21.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: Reconfiguring grafana.a (dependencies changed)... 2023-08-28T02:17:21.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: Reconfiguring daemon grafana.a on smithi169 2023-08-28T02:17:21.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: Upgrade: Need to upgrade myself (mgr.x) 2023-08-28T02:17:21.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: Upgrade: Need to upgrade myself (mgr.x) 2023-08-28T02:17:21.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: Upgrade: Updating mgr.y 2023-08-28T02:17:21.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: Deploying daemon mgr.y on smithi143 2023-08-28T02:17:21.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: [28/Aug/2023:02:15:03] ENGINE Bus STARTING 2023-08-28T02:17:21.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: [28/Aug/2023:02:15:03] ENGINE Serving on https://172.21.15.143:7150 2023-08-28T02:17:21.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: [28/Aug/2023:02:15:03] ENGINE Serving on http://172.21.15.143:8765 2023-08-28T02:17:21.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: [28/Aug/2023:02:15:03] ENGINE Bus STARTED 2023-08-28T02:17:21.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: Updating smithi143:/etc/ceph/ceph.conf 2023-08-28T02:17:21.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: Updating smithi169:/etc/ceph/ceph.conf 2023-08-28T02:17:21.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: Updating smithi169:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.conf 2023-08-28T02:17:21.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: Updating smithi143:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.conf 2023-08-28T02:17:21.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: Updating smithi169:/etc/ceph/ceph.client.admin.keyring 2023-08-28T02:17:21.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: Updating smithi143:/etc/ceph/ceph.client.admin.keyring 2023-08-28T02:17:21.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: Updating smithi169:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.client.admin.keyring 2023-08-28T02:17:21.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: Updating smithi143:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.client.admin.keyring 2023-08-28T02:17:21.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: Reconfiguring prometheus.a (dependencies changed)... 2023-08-28T02:17:21.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: Reconfiguring daemon prometheus.a on smithi169 2023-08-28T02:17:21.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: Upgrade: Updating mgr.x 2023-08-28T02:17:21.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: Deploying daemon mgr.x on smithi169 2023-08-28T02:17:21.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: Upgrade: Setting container_image for all mgr 2023-08-28T02:17:21.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: Upgrade: It appears safe to stop mon.a 2023-08-28T02:17:21.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: Upgrade: Updating mon.a 2023-08-28T02:17:21.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: Deploying daemon mon.a on smithi143 2023-08-28T02:17:21.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: [28/Aug/2023:02:15:41] ENGINE Bus STARTING 2023-08-28T02:17:21.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: [28/Aug/2023:02:15:41] ENGINE Serving on http://172.21.15.143:8765 2023-08-28T02:17:21.991 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: [28/Aug/2023:02:15:41] ENGINE Serving on https://172.21.15.143:7150 2023-08-28T02:17:21.991 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: [28/Aug/2023:02:15:41] ENGINE Bus STARTED 2023-08-28T02:17:21.991 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8 2023-08-28T02:17:21.991 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: Updating smithi143:/etc/ceph/ceph.conf 2023-08-28T02:17:21.991 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: Updating smithi169:/etc/ceph/ceph.conf 2023-08-28T02:17:21.991 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: Updating smithi169:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.conf 2023-08-28T02:17:21.991 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: Updating smithi143:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.conf 2023-08-28T02:17:21.991 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: Updating smithi143:/etc/ceph/ceph.client.admin.keyring 2023-08-28T02:17:21.991 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: Updating smithi169:/etc/ceph/ceph.client.admin.keyring 2023-08-28T02:17:21.991 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: Updating smithi143:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.client.admin.keyring 2023-08-28T02:17:21.991 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: Updating smithi169:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.client.admin.keyring 2023-08-28T02:17:21.991 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.991 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.992 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.992 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.992 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.992 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.992 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.992 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.992 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.992 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.992 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.992 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.992 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.992 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.992 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.993 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.993 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.993 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.993 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.993 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.993 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.993 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.993 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.993 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.993 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.993 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.993 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.993 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.993 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.994 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:17:21.994 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='client.34161 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:17:21.994 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8 2023-08-28T02:17:21.994 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:21.994 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:21.995 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:21.995 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:21.995 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:21.995 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.995 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:21.995 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:21.995 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.995 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:21.995 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:21.995 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:21.995 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:17:21.995 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:21.995 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:17:21.996 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:17:21.996 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-08-28T02:17:21.996 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-08-28T02:17:21.996 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: Upgrade: It appears safe to stop mon.c 2023-08-28T02:17:21.996 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.996 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3782374877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:21.996 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: Upgrade: Updating mon.c 2023-08-28T02:17:21.996 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:21.996 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-28T02:17:21.996 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-28T02:17:21.996 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:21.996 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: Deploying daemon mon.c on smithi143 2023-08-28T02:17:21.996 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:17:21.997 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.997 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T02:17:21.997 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T02:17:21.997 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T02:17:21.997 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2365514943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:21.997 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: mon.c calling monitor election 2023-08-28T02:17:21.997 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.997 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: mon.b calling monitor election 2023-08-28T02:17:21.997 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: mon.a calling monitor election 2023-08-28T02:17:21.997 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-08-28T02:17:21.997 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: monmap e3: 3 mons at {a=[v2:172.21.15.143:3300/0,v1:172.21.15.143:6789/0],b=[v2:172.21.15.169:3300/0,v1:172.21.15.169:6789/0],c=[v2:172.21.15.143:3301/0,v1:172.21.15.143:6790/0]} removed_ranks: {} 2023-08-28T02:17:21.997 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: fsmap foo:1 {0=foo.smithi143.spifrj=up:active} 1 up:standby 2023-08-28T02:17:21.997 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: osdmap e86: 8 total, 8 up, 8 in 2023-08-28T02:17:21.997 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: mgrmap e34: y(active, since 84s), standbys: x 2023-08-28T02:17:21.998 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-08-28T02:17:21.998 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: Cluster is now healthy 2023-08-28T02:17:21.998 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: overall HEALTH_OK 2023-08-28T02:17:21.998 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:21.998 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:21.998 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3847681820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:21.998 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.998 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:21.998 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:21.998 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.998 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:17:21.998 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:21.998 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:17:21.999 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:21.999 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:21.999 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:21.999 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:17:21.999 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:21.999 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:17:21.999 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:17:21.999 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-08-28T02:17:21.999 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-08-28T02:17:21.999 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: Upgrade: It appears safe to stop mon.b 2023-08-28T02:17:21.999 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: mgrmap e35: y(active, since 91s), standbys: x 2023-08-28T02:17:21.999 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:17:22.000 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8 2023-08-28T02:17:22.000 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:22.000 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-08-28T02:17:22.000 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: Upgrade: Updating mon.b 2023-08-28T02:17:22.000 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:22.000 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-28T02:17:22.000 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-28T02:17:22.000 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:22.000 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: Deploying daemon mon.b on smithi169 2023-08-28T02:17:22.000 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: overall HEALTH_OK 2023-08-28T02:17:22.000 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T02:17:22.000 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: mon.a calling monitor election 2023-08-28T02:17:22.000 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: mon.b calling monitor election 2023-08-28T02:17:22.001 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T02:17:22.001 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T02:17:22.001 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-08-28T02:17:22.001 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-08-28T02:17:22.001 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-08-28T02:17:22.001 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-08-28T02:17:22.001 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: monmap e4: 3 mons at {a=[v2:172.21.15.143:3300/0,v1:172.21.15.143:6789/0],b=[v2:172.21.15.169:3300/0,v1:172.21.15.169:6789/0],c=[v2:172.21.15.143:3301/0,v1:172.21.15.143:6790/0]} removed_ranks: {} 2023-08-28T02:17:22.001 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: fsmap foo:1 {0=foo.smithi143.spifrj=up:active} 1 up:standby 2023-08-28T02:17:22.001 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: osdmap e86: 8 total, 8 up, 8 in 2023-08-28T02:17:22.001 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: mgrmap e35: y(active, since 100s), standbys: x 2023-08-28T02:17:22.001 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-08-28T02:17:22.001 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-08-28T02:17:22.002 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-08-28T02:17:22.002 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: mon.c (rank 2) addr [v2:172.21.15.143:3301/0,v1:172.21.15.143:6790/0] is down (out of quorum) 2023-08-28T02:17:22.002 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:22.002 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:22.002 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:21 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:17:22.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:17:21 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:17:21] "GET /metrics HTTP/1.1" 200 33933 "" "Prometheus/2.33.4" 2023-08-28T02:17:22.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[183059]: mon.c calling monitor election 2023-08-28T02:17:22.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[183059]: mon.b calling monitor election 2023-08-28T02:17:22.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[183059]: mon.a calling monitor election 2023-08-28T02:17:22.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[183059]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-08-28T02:17:22.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[183059]: monmap e4: 3 mons at {a=[v2:172.21.15.143:3300/0,v1:172.21.15.143:6789/0],b=[v2:172.21.15.169:3300/0,v1:172.21.15.169:6789/0],c=[v2:172.21.15.143:3301/0,v1:172.21.15.143:6790/0]} removed_ranks: {} 2023-08-28T02:17:22.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[183059]: fsmap foo:1 {0=foo.smithi143.spifrj=up:active} 1 up:standby 2023-08-28T02:17:22.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[183059]: osdmap e86: 8 total, 8 up, 8 in 2023-08-28T02:17:22.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[183059]: mgrmap e35: y(active, since 101s), standbys: x 2023-08-28T02:17:22.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[183059]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-08-28T02:17:22.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[183059]: Cluster is now healthy 2023-08-28T02:17:22.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[183059]: overall HEALTH_OK 2023-08-28T02:17:22.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: mkfs 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 2023-08-28T02:17:22.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Reconfiguring daemon prometheus.a on smithi169 2023-08-28T02:17:22.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Reconfiguring node-exporter.b (dependencies changed)... 2023-08-28T02:17:22.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Reconfiguring daemon node-exporter.b on smithi169 2023-08-28T02:17:22.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Reconfiguring grafana.a (dependencies changed)... 2023-08-28T02:17:22.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Reconfiguring daemon grafana.a on smithi169 2023-08-28T02:17:22.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Upgrade: Need to upgrade myself (mgr.x) 2023-08-28T02:17:22.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Upgrade: Need to upgrade myself (mgr.x) 2023-08-28T02:17:22.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Upgrade: Updating mgr.y 2023-08-28T02:17:22.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Deploying daemon mgr.y on smithi143 2023-08-28T02:17:22.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: [28/Aug/2023:02:15:03] ENGINE Bus STARTING 2023-08-28T02:17:22.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: [28/Aug/2023:02:15:03] ENGINE Serving on https://172.21.15.143:7150 2023-08-28T02:17:22.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: [28/Aug/2023:02:15:03] ENGINE Serving on http://172.21.15.143:8765 2023-08-28T02:17:22.956 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: [28/Aug/2023:02:15:03] ENGINE Bus STARTED 2023-08-28T02:17:22.956 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Updating smithi143:/etc/ceph/ceph.conf 2023-08-28T02:17:22.956 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Updating smithi169:/etc/ceph/ceph.conf 2023-08-28T02:17:22.956 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Updating smithi169:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.conf 2023-08-28T02:17:22.956 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Updating smithi143:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.conf 2023-08-28T02:17:22.956 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Updating smithi169:/etc/ceph/ceph.client.admin.keyring 2023-08-28T02:17:22.956 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Updating smithi143:/etc/ceph/ceph.client.admin.keyring 2023-08-28T02:17:22.956 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Updating smithi169:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.client.admin.keyring 2023-08-28T02:17:22.956 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Updating smithi143:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.client.admin.keyring 2023-08-28T02:17:22.956 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Reconfiguring prometheus.a (dependencies changed)... 2023-08-28T02:17:22.957 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Reconfiguring daemon prometheus.a on smithi169 2023-08-28T02:17:22.957 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Upgrade: Updating mgr.x 2023-08-28T02:17:22.957 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Deploying daemon mgr.x on smithi169 2023-08-28T02:17:22.957 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Upgrade: Setting container_image for all mgr 2023-08-28T02:17:22.957 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Upgrade: It appears safe to stop mon.a 2023-08-28T02:17:22.957 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Upgrade: Updating mon.a 2023-08-28T02:17:22.957 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Deploying daemon mon.a on smithi143 2023-08-28T02:17:22.957 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: [28/Aug/2023:02:15:41] ENGINE Bus STARTING 2023-08-28T02:17:22.957 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: [28/Aug/2023:02:15:41] ENGINE Serving on http://172.21.15.143:8765 2023-08-28T02:17:22.957 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: [28/Aug/2023:02:15:41] ENGINE Serving on https://172.21.15.143:7150 2023-08-28T02:17:22.958 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: [28/Aug/2023:02:15:41] ENGINE Bus STARTED 2023-08-28T02:17:22.958 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8 2023-08-28T02:17:22.958 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Updating smithi143:/etc/ceph/ceph.conf 2023-08-28T02:17:22.958 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Updating smithi169:/etc/ceph/ceph.conf 2023-08-28T02:17:22.958 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Updating smithi169:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.conf 2023-08-28T02:17:22.958 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Updating smithi143:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.conf 2023-08-28T02:17:22.958 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Updating smithi143:/etc/ceph/ceph.client.admin.keyring 2023-08-28T02:17:22.958 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Updating smithi169:/etc/ceph/ceph.client.admin.keyring 2023-08-28T02:17:22.958 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Updating smithi143:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.client.admin.keyring 2023-08-28T02:17:22.958 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Updating smithi169:/var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/config/ceph.client.admin.keyring 2023-08-28T02:17:22.959 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:22.959 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:22.959 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:22.959 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:22.959 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:22.959 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:22.959 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:22.959 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:22.959 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:22.959 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:22.959 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:22.959 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:22.959 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:22.960 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:22.960 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:22.960 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:22.960 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:22.960 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:22.960 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:22.960 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:22.960 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:22.960 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:22.960 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:22.960 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:22.960 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:22.960 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:22.961 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:22.961 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:22.961 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:22.961 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:17:22.961 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='client.34161 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:17:22.961 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8 2023-08-28T02:17:22.961 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:22.961 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:22.961 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:22.961 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:22.961 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:22.961 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:22.962 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:22.962 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:22.962 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:22.962 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:22.962 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:22.962 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:22.962 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:17:22.962 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:22.962 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:17:22.962 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:17:22.962 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-08-28T02:17:22.962 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-08-28T02:17:22.962 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Upgrade: It appears safe to stop mon.c 2023-08-28T02:17:22.963 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:22.963 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3782374877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:22.963 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Upgrade: Updating mon.c 2023-08-28T02:17:22.963 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:22.963 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-28T02:17:22.963 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-28T02:17:22.963 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:22.963 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Deploying daemon mon.c on smithi143 2023-08-28T02:17:22.963 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:17:22.963 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:22.963 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T02:17:22.963 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T02:17:22.963 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T02:17:22.964 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2365514943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:22.964 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: mon.c calling monitor election 2023-08-28T02:17:22.964 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:22.964 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: mon.b calling monitor election 2023-08-28T02:17:22.964 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: mon.a calling monitor election 2023-08-28T02:17:22.964 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-08-28T02:17:22.964 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: monmap e3: 3 mons at {a=[v2:172.21.15.143:3300/0,v1:172.21.15.143:6789/0],b=[v2:172.21.15.169:3300/0,v1:172.21.15.169:6789/0],c=[v2:172.21.15.143:3301/0,v1:172.21.15.143:6790/0]} removed_ranks: {} 2023-08-28T02:17:22.964 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: fsmap foo:1 {0=foo.smithi143.spifrj=up:active} 1 up:standby 2023-08-28T02:17:22.964 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: osdmap e86: 8 total, 8 up, 8 in 2023-08-28T02:17:22.964 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: mgrmap e34: y(active, since 84s), standbys: x 2023-08-28T02:17:22.964 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-08-28T02:17:22.965 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Cluster is now healthy 2023-08-28T02:17:22.965 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: overall HEALTH_OK 2023-08-28T02:17:22.965 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:22.965 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:22.965 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3847681820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:22.965 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:22.965 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:22.965 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:22.965 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:22.965 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:17:22.965 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:22.965 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:17:22.966 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:22.966 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:22.966 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:22.966 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:17:22.966 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:22.966 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:17:22.966 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:17:22.966 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-08-28T02:17:22.966 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-08-28T02:17:22.966 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Upgrade: It appears safe to stop mon.b 2023-08-28T02:17:22.966 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: mgrmap e35: y(active, since 91s), standbys: x 2023-08-28T02:17:22.966 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:17:22.967 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8 2023-08-28T02:17:22.967 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:22.967 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-08-28T02:17:22.967 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Upgrade: Updating mon.b 2023-08-28T02:17:22.967 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 ' entity='mgr.y' 2023-08-28T02:17:22.967 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-28T02:17:22.967 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-28T02:17:22.967 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:22.967 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Deploying daemon mon.b on smithi169 2023-08-28T02:17:22.967 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: overall HEALTH_OK 2023-08-28T02:17:22.967 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T02:17:22.967 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: mon.a calling monitor election 2023-08-28T02:17:22.968 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: mon.b calling monitor election 2023-08-28T02:17:22.968 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T02:17:22.968 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T02:17:22.968 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-08-28T02:17:22.968 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-08-28T02:17:22.968 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-08-28T02:17:22.968 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-08-28T02:17:22.968 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: monmap e4: 3 mons at {a=[v2:172.21.15.143:3300/0,v1:172.21.15.143:6789/0],b=[v2:172.21.15.169:3300/0,v1:172.21.15.169:6789/0],c=[v2:172.21.15.143:3301/0,v1:172.21.15.143:6790/0]} removed_ranks: {} 2023-08-28T02:17:22.968 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: fsmap foo:1 {0=foo.smithi143.spifrj=up:active} 1 up:standby 2023-08-28T02:17:22.968 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: osdmap e86: 8 total, 8 up, 8 in 2023-08-28T02:17:22.968 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: mgrmap e35: y(active, since 100s), standbys: x 2023-08-28T02:17:22.968 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-08-28T02:17:22.969 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-08-28T02:17:22.969 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-08-28T02:17:22.969 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: mon.c (rank 2) addr [v2:172.21.15.143:3301/0,v1:172.21.15.143:6790/0] is down (out of quorum) 2023-08-28T02:17:22.969 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:22.969 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:22.969 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:17:22.969 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: mon.c calling monitor election 2023-08-28T02:17:22.969 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: mon.b calling monitor election 2023-08-28T02:17:22.969 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: mon.a calling monitor election 2023-08-28T02:17:22.969 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-08-28T02:17:22.969 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: monmap e4: 3 mons at {a=[v2:172.21.15.143:3300/0,v1:172.21.15.143:6789/0],b=[v2:172.21.15.169:3300/0,v1:172.21.15.169:6789/0],c=[v2:172.21.15.143:3301/0,v1:172.21.15.143:6790/0]} removed_ranks: {} 2023-08-28T02:17:22.969 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: fsmap foo:1 {0=foo.smithi143.spifrj=up:active} 1 up:standby 2023-08-28T02:17:22.969 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: osdmap e86: 8 total, 8 up, 8 in 2023-08-28T02:17:22.969 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: mgrmap e35: y(active, since 101s), standbys: x 2023-08-28T02:17:22.970 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-08-28T02:17:22.970 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: Cluster is now healthy 2023-08-28T02:17:22.970 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:22 smithi143 ceph-mon[189871]: overall HEALTH_OK 2023-08-28T02:17:22.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:22 smithi169 ceph-mon[159415]: mon.c calling monitor election 2023-08-28T02:17:22.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:22 smithi169 ceph-mon[159415]: mon.b calling monitor election 2023-08-28T02:17:22.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:22 smithi169 ceph-mon[159415]: mon.a calling monitor election 2023-08-28T02:17:22.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:22 smithi169 ceph-mon[159415]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-08-28T02:17:22.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:22 smithi169 ceph-mon[159415]: monmap e4: 3 mons at {a=[v2:172.21.15.143:3300/0,v1:172.21.15.143:6789/0],b=[v2:172.21.15.169:3300/0,v1:172.21.15.169:6789/0],c=[v2:172.21.15.143:3301/0,v1:172.21.15.143:6790/0]} removed_ranks: {} 2023-08-28T02:17:22.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:22 smithi169 ceph-mon[159415]: fsmap foo:1 {0=foo.smithi143.spifrj=up:active} 1 up:standby 2023-08-28T02:17:22.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:22 smithi169 ceph-mon[159415]: osdmap e86: 8 total, 8 up, 8 in 2023-08-28T02:17:22.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:22 smithi169 ceph-mon[159415]: mgrmap e35: y(active, since 101s), standbys: x 2023-08-28T02:17:22.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:22 smithi169 ceph-mon[159415]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-08-28T02:17:22.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:22 smithi169 ceph-mon[159415]: Cluster is now healthy 2023-08-28T02:17:22.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:22 smithi169 ceph-mon[159415]: overall HEALTH_OK 2023-08-28T02:17:23.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:23 smithi143 ceph-mon[183059]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-08-28T02:17:23.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:23 smithi143 ceph-mon[189871]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-08-28T02:17:23.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:23 smithi169 ceph-mon[159415]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-08-28T02:17:26.003 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:25 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:26.003 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:25 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:26.003 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:25 smithi169 ceph-mon[159415]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:26.003 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:25 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:26.003 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:25 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:26.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:25 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:26.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:25 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:26.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:25 smithi143 ceph-mon[183059]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:26.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:25 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:26.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:25 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:26.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:25 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:26.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:25 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:26.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:25 smithi143 ceph-mon[189871]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:26.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:25 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:26.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:25 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:27.071 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:26 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:17:27.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:26 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:17:27.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:26 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:17:28.045 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:27 smithi169 ceph-mon[159415]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:28.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:27 smithi143 ceph-mon[183059]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:28.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:27 smithi143 ceph-mon[189871]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:28.987 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:17:28 smithi169 conmon[150127]: ts=2023-08-28T02:17:28.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi169.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi143:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-08-28T02:17:29.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:29 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:29.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:29 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:29.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:29 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:29.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:29 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:17:29.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:29 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:29.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:29 smithi143 ceph-mon[189871]: Reconfiguring mon.a (monmap changed)... 2023-08-28T02:17:29.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:29 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-28T02:17:29.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:29 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-28T02:17:29.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:29 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:29.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:29 smithi143 ceph-mon[189871]: Reconfiguring daemon mon.a on smithi143 2023-08-28T02:17:29.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:29 smithi143 ceph-mon[189871]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:29.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:29 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:29.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:29 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:29.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:29 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:29.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:29 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:17:29.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:29 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:29.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:29 smithi143 ceph-mon[183059]: Reconfiguring mon.a (monmap changed)... 2023-08-28T02:17:29.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:29 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-28T02:17:29.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:29 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-28T02:17:29.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:29 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:29.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:29 smithi143 ceph-mon[183059]: Reconfiguring daemon mon.a on smithi143 2023-08-28T02:17:29.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:29 smithi143 ceph-mon[183059]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:29.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:29 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:29.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:29 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:29.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:29 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:29.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:29 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:17:29.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:29 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:29.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:29 smithi169 ceph-mon[159415]: Reconfiguring mon.a (monmap changed)... 2023-08-28T02:17:29.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:29 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-28T02:17:29.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:29 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-28T02:17:29.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:29 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:29.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:29 smithi169 ceph-mon[159415]: Reconfiguring daemon mon.a on smithi143 2023-08-28T02:17:29.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:29 smithi169 ceph-mon[159415]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:31.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:31 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:31.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:31 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:31.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:31 smithi143 ceph-mon[183059]: Reconfiguring mgr.y (monmap changed)... 2023-08-28T02:17:31.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:31 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-28T02:17:31.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:31 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-08-28T02:17:31.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:31 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:31.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:31 smithi143 ceph-mon[183059]: Reconfiguring daemon mgr.y on smithi143 2023-08-28T02:17:31.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:31 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:31.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:31 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:31.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:31 smithi143 ceph-mon[189871]: Reconfiguring mgr.y (monmap changed)... 2023-08-28T02:17:31.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:31 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-28T02:17:31.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:31 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-08-28T02:17:31.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:31 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:31.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:31 smithi143 ceph-mon[189871]: Reconfiguring daemon mgr.y on smithi143 2023-08-28T02:17:31.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:31 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:31.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:31 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:31.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:31 smithi169 ceph-mon[159415]: Reconfiguring mgr.y (monmap changed)... 2023-08-28T02:17:31.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:31 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-28T02:17:31.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:31 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-08-28T02:17:31.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:31 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:31.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:31 smithi169 ceph-mon[159415]: Reconfiguring daemon mgr.y on smithi143 2023-08-28T02:17:32.055 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:17:31 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:17:31] "GET /metrics HTTP/1.1" 200 33930 "" "Prometheus/2.33.4" 2023-08-28T02:17:32.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:32 smithi143 ceph-mon[183059]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:32.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:32 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:32.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:32 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:32.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:32 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-28T02:17:32.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:32 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-28T02:17:32.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:32 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:32.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:32 smithi143 ceph-mon[189871]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:32.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:32 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:32.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:32 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:32.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:32 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-28T02:17:32.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:32 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-28T02:17:32.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:32 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:32.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:32 smithi169 ceph-mon[159415]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:32.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:32 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:32.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:32 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:32.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:32 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-28T02:17:32.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:32 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-28T02:17:32.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:32 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:33.272 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:33 smithi143 ceph-mon[189871]: Reconfiguring mon.c (monmap changed)... 2023-08-28T02:17:33.272 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:33 smithi143 ceph-mon[189871]: Reconfiguring daemon mon.c on smithi143 2023-08-28T02:17:33.272 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:33 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:33.272 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:33 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:33.272 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:33 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-08-28T02:17:33.272 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:33 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:33.273 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:33 smithi143 ceph-mon[183059]: Reconfiguring mon.c (monmap changed)... 2023-08-28T02:17:33.273 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:33 smithi143 ceph-mon[183059]: Reconfiguring daemon mon.c on smithi143 2023-08-28T02:17:33.273 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:33 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:33.273 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:33 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:33.273 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:33 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-08-28T02:17:33.273 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:33 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:33.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:33 smithi169 ceph-mon[159415]: Reconfiguring mon.c (monmap changed)... 2023-08-28T02:17:33.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:33 smithi169 ceph-mon[159415]: Reconfiguring daemon mon.c on smithi143 2023-08-28T02:17:33.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:33 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:33.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:33 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:33.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:33 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-08-28T02:17:33.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:33 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:34.428 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:34 smithi143 ceph-mon[183059]: Reconfiguring osd.0 (monmap changed)... 2023-08-28T02:17:34.429 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:34 smithi143 ceph-mon[183059]: Reconfiguring daemon osd.0 on smithi143 2023-08-28T02:17:34.429 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:34 smithi143 ceph-mon[183059]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:34.429 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:34 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:34.429 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:34 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:34.429 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:34 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-08-28T02:17:34.429 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:34 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:34.429 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:34 smithi143 ceph-mon[189871]: Reconfiguring osd.0 (monmap changed)... 2023-08-28T02:17:34.429 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:34 smithi143 ceph-mon[189871]: Reconfiguring daemon osd.0 on smithi143 2023-08-28T02:17:34.429 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:34 smithi143 ceph-mon[189871]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:34.430 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:34 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:34.430 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:34 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:34.430 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:34 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-08-28T02:17:34.430 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:34 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:34.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:34 smithi169 ceph-mon[159415]: Reconfiguring osd.0 (monmap changed)... 2023-08-28T02:17:34.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:34 smithi169 ceph-mon[159415]: Reconfiguring daemon osd.0 on smithi143 2023-08-28T02:17:34.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:34 smithi169 ceph-mon[159415]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:34.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:34 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:34.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:34 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:34.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:34 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-08-28T02:17:34.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:34 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:35.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:35 smithi143 ceph-mon[189871]: Reconfiguring osd.1 (monmap changed)... 2023-08-28T02:17:35.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:35 smithi143 ceph-mon[189871]: Reconfiguring daemon osd.1 on smithi143 2023-08-28T02:17:35.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:35 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:35.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:35 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:35.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:35 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-08-28T02:17:35.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:35 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:35.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:35 smithi143 ceph-mon[183059]: Reconfiguring osd.1 (monmap changed)... 2023-08-28T02:17:35.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:35 smithi143 ceph-mon[183059]: Reconfiguring daemon osd.1 on smithi143 2023-08-28T02:17:35.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:35 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:35.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:35 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:35.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:35 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-08-28T02:17:35.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:35 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:35.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:35 smithi169 ceph-mon[159415]: Reconfiguring osd.1 (monmap changed)... 2023-08-28T02:17:35.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:35 smithi169 ceph-mon[159415]: Reconfiguring daemon osd.1 on smithi143 2023-08-28T02:17:35.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:35 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:35.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:35 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:35.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:35 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-08-28T02:17:35.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:35 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:36.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:36 smithi143 ceph-mon[189871]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:36.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:36 smithi143 ceph-mon[189871]: Reconfiguring osd.2 (monmap changed)... 2023-08-28T02:17:36.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:36 smithi143 ceph-mon[189871]: Reconfiguring daemon osd.2 on smithi143 2023-08-28T02:17:36.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:36 smithi143 ceph-mon[183059]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:36.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:36 smithi143 ceph-mon[183059]: Reconfiguring osd.2 (monmap changed)... 2023-08-28T02:17:36.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:36 smithi143 ceph-mon[183059]: Reconfiguring daemon osd.2 on smithi143 2023-08-28T02:17:36.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:36 smithi169 ceph-mon[159415]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:36.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:36 smithi169 ceph-mon[159415]: Reconfiguring osd.2 (monmap changed)... 2023-08-28T02:17:36.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:36 smithi169 ceph-mon[159415]: Reconfiguring daemon osd.2 on smithi143 2023-08-28T02:17:37.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:37 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:37.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:37 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:37.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:37 smithi143 ceph-mon[183059]: Reconfiguring osd.3 (monmap changed)... 2023-08-28T02:17:37.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:37 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-08-28T02:17:37.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:37 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:37.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:37 smithi143 ceph-mon[183059]: Reconfiguring daemon osd.3 on smithi143 2023-08-28T02:17:37.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:37 smithi143 ceph-mon[183059]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:37.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:37 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:37.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:37 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:37.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:37 smithi143 ceph-mon[189871]: Reconfiguring osd.3 (monmap changed)... 2023-08-28T02:17:37.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:37 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-08-28T02:17:37.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:37 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:37.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:37 smithi143 ceph-mon[189871]: Reconfiguring daemon osd.3 on smithi143 2023-08-28T02:17:37.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:37 smithi143 ceph-mon[189871]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:37.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:37 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:37.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:37 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:37.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:37 smithi169 ceph-mon[159415]: Reconfiguring osd.3 (monmap changed)... 2023-08-28T02:17:37.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:37 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-08-28T02:17:37.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:37 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:37.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:37 smithi169 ceph-mon[159415]: Reconfiguring daemon osd.3 on smithi143 2023-08-28T02:17:37.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:37 smithi169 ceph-mon[159415]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:38.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:38 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:38.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:38 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:38.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:38 smithi143 ceph-mon[183059]: Reconfiguring mds.foo.smithi143.spifrj (monmap changed)... 2023-08-28T02:17:38.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:38 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi143.spifrj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-08-28T02:17:38.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:38 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:38.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:38 smithi143 ceph-mon[183059]: Reconfiguring daemon mds.foo.smithi143.spifrj on smithi143 2023-08-28T02:17:38.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:38 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:38.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:38 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:38.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:38 smithi143 ceph-mon[189871]: Reconfiguring mds.foo.smithi143.spifrj (monmap changed)... 2023-08-28T02:17:38.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:38 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi143.spifrj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-08-28T02:17:38.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:38 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:38.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:38 smithi143 ceph-mon[189871]: Reconfiguring daemon mds.foo.smithi143.spifrj on smithi143 2023-08-28T02:17:38.987 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:17:38 smithi169 conmon[150127]: ts=2023-08-28T02:17:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi169.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi143:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-08-28T02:17:38.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:38 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:38.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:38 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:38.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:38 smithi169 ceph-mon[159415]: Reconfiguring mds.foo.smithi143.spifrj (monmap changed)... 2023-08-28T02:17:38.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:38 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi143.spifrj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-08-28T02:17:38.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:38 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:38.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:38 smithi169 ceph-mon[159415]: Reconfiguring daemon mds.foo.smithi143.spifrj on smithi143 2023-08-28T02:17:39.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:39 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:39.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:39 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:39.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:39 smithi169 ceph-mon[159415]: Reconfiguring mon.b (monmap changed)... 2023-08-28T02:17:39.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:39 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-28T02:17:39.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:39 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-28T02:17:39.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:39 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:39.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:39 smithi169 ceph-mon[159415]: Reconfiguring daemon mon.b on smithi169 2023-08-28T02:17:39.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:39 smithi169 ceph-mon[159415]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:40.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:39 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:40.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:39 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:40.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:39 smithi143 ceph-mon[183059]: Reconfiguring mon.b (monmap changed)... 2023-08-28T02:17:40.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:39 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-28T02:17:40.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:39 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-28T02:17:40.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:39 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:40.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:39 smithi143 ceph-mon[183059]: Reconfiguring daemon mon.b on smithi169 2023-08-28T02:17:40.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:39 smithi143 ceph-mon[183059]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:40.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:39 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:40.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:39 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:40.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:39 smithi143 ceph-mon[189871]: Reconfiguring mon.b (monmap changed)... 2023-08-28T02:17:40.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:39 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-28T02:17:40.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:39 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-28T02:17:40.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:39 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:40.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:39 smithi143 ceph-mon[189871]: Reconfiguring daemon mon.b on smithi169 2023-08-28T02:17:40.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:39 smithi143 ceph-mon[189871]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:41.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:41 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:41.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:41 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:41.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:41 smithi169 ceph-mon[159415]: Reconfiguring mgr.x (monmap changed)... 2023-08-28T02:17:41.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:41 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-28T02:17:41.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:41 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-08-28T02:17:41.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:41 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:41.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:41 smithi169 ceph-mon[159415]: Reconfiguring daemon mgr.x on smithi169 2023-08-28T02:17:41.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:41 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:17:41.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:41 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:41.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:41 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:41.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:41 smithi143 ceph-mon[183059]: Reconfiguring mgr.x (monmap changed)... 2023-08-28T02:17:41.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:41 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-28T02:17:41.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:41 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-08-28T02:17:41.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:41 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:41.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:41 smithi143 ceph-mon[183059]: Reconfiguring daemon mgr.x on smithi169 2023-08-28T02:17:41.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:41 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:17:41.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:41 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:41.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:41 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:41.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:41 smithi143 ceph-mon[189871]: Reconfiguring mgr.x (monmap changed)... 2023-08-28T02:17:41.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:41 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-28T02:17:41.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:41 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-08-28T02:17:41.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:41 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:41.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:41 smithi143 ceph-mon[189871]: Reconfiguring daemon mgr.x on smithi169 2023-08-28T02:17:41.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:41 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:17:42.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:17:41 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:17:41] "GET /metrics HTTP/1.1" 200 33925 "" "Prometheus/2.33.4" 2023-08-28T02:17:42.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:42 smithi143 ceph-mon[183059]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:42.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:42 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:42.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:42 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:42.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:42 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-08-28T02:17:42.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:42 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:42.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:42 smithi143 ceph-mon[189871]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:42.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:42 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:42.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:42 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:42.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:42 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-08-28T02:17:42.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:42 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:42.225 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:42 smithi169 ceph-mon[159415]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:42.225 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:42 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:42.225 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:42 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:42.225 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:42 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-08-28T02:17:42.225 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:42 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:43.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:43 smithi143 ceph-mon[183059]: Reconfiguring osd.4 (monmap changed)... 2023-08-28T02:17:43.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:43 smithi143 ceph-mon[183059]: Reconfiguring daemon osd.4 on smithi169 2023-08-28T02:17:43.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:43 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:43.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:43 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:43.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:43 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-08-28T02:17:43.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:43 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:43.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:43 smithi143 ceph-mon[189871]: Reconfiguring osd.4 (monmap changed)... 2023-08-28T02:17:43.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:43 smithi143 ceph-mon[189871]: Reconfiguring daemon osd.4 on smithi169 2023-08-28T02:17:43.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:43 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:43.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:43 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:43.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:43 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-08-28T02:17:43.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:43 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:43.466 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:43 smithi169 ceph-mon[159415]: Reconfiguring osd.4 (monmap changed)... 2023-08-28T02:17:43.466 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:43 smithi169 ceph-mon[159415]: Reconfiguring daemon osd.4 on smithi169 2023-08-28T02:17:43.467 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:43 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:43.467 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:43 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:43.467 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:43 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-08-28T02:17:43.467 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:43 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:44.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:44 smithi143 ceph-mon[183059]: Reconfiguring osd.5 (monmap changed)... 2023-08-28T02:17:44.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:44 smithi143 ceph-mon[183059]: Reconfiguring daemon osd.5 on smithi169 2023-08-28T02:17:44.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:44 smithi143 ceph-mon[183059]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:44.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:44 smithi143 ceph-mon[189871]: Reconfiguring osd.5 (monmap changed)... 2023-08-28T02:17:44.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:44 smithi143 ceph-mon[189871]: Reconfiguring daemon osd.5 on smithi169 2023-08-28T02:17:44.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:44 smithi143 ceph-mon[189871]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:44.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:44 smithi169 ceph-mon[159415]: Reconfiguring osd.5 (monmap changed)... 2023-08-28T02:17:44.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:44 smithi169 ceph-mon[159415]: Reconfiguring daemon osd.5 on smithi169 2023-08-28T02:17:44.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:44 smithi169 ceph-mon[159415]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:45.441 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:45 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:45.441 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:45 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:45.441 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:45 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-08-28T02:17:45.441 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:45 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:45.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:45 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:45.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:45 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:45.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:45 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-08-28T02:17:45.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:45 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:45.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:45 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:45.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:45 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:45.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:45 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-08-28T02:17:45.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:45 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:46.432 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:46 smithi169 ceph-mon[159415]: Reconfiguring osd.6 (monmap changed)... 2023-08-28T02:17:46.432 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:46 smithi169 ceph-mon[159415]: Reconfiguring daemon osd.6 on smithi169 2023-08-28T02:17:46.432 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:46 smithi169 ceph-mon[159415]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:46.432 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:46 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:46.433 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:46 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:46.433 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:46 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-08-28T02:17:46.433 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:46 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:46.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:46 smithi143 ceph-mon[183059]: Reconfiguring osd.6 (monmap changed)... 2023-08-28T02:17:46.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:46 smithi143 ceph-mon[183059]: Reconfiguring daemon osd.6 on smithi169 2023-08-28T02:17:46.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:46 smithi143 ceph-mon[183059]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:46.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:46 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:46.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:46 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:46.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:46 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-08-28T02:17:46.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:46 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:46.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:46 smithi143 ceph-mon[189871]: Reconfiguring osd.6 (monmap changed)... 2023-08-28T02:17:46.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:46 smithi143 ceph-mon[189871]: Reconfiguring daemon osd.6 on smithi169 2023-08-28T02:17:46.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:46 smithi143 ceph-mon[189871]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:46.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:46 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:46.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:46 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:46.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:46 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-08-28T02:17:46.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:46 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:47.431 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:47 smithi169 ceph-mon[159415]: Reconfiguring osd.7 (monmap changed)... 2023-08-28T02:17:47.431 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:47 smithi169 ceph-mon[159415]: Reconfiguring daemon osd.7 on smithi169 2023-08-28T02:17:47.431 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:47 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:47.431 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:47 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:47.431 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:47 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi169.yeisur", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-08-28T02:17:47.431 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:47 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:47.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:47 smithi143 ceph-mon[183059]: Reconfiguring osd.7 (monmap changed)... 2023-08-28T02:17:47.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:47 smithi143 ceph-mon[183059]: Reconfiguring daemon osd.7 on smithi169 2023-08-28T02:17:47.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:47 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:47.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:47 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:47.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:47 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi169.yeisur", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-08-28T02:17:47.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:47 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:47.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:47 smithi143 ceph-mon[189871]: Reconfiguring osd.7 (monmap changed)... 2023-08-28T02:17:47.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:47 smithi143 ceph-mon[189871]: Reconfiguring daemon osd.7 on smithi169 2023-08-28T02:17:47.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:47 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:47.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:47 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:47.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:47 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi169.yeisur", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-08-28T02:17:47.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:47 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:48.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:48 smithi143 ceph-mon[183059]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:48.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:48 smithi143 ceph-mon[183059]: Reconfiguring mds.foo.smithi169.yeisur (monmap changed)... 2023-08-28T02:17:48.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:48 smithi143 ceph-mon[183059]: Reconfiguring daemon mds.foo.smithi169.yeisur on smithi169 2023-08-28T02:17:48.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:48 smithi143 ceph-mon[189871]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:48.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:48 smithi143 ceph-mon[189871]: Reconfiguring mds.foo.smithi169.yeisur (monmap changed)... 2023-08-28T02:17:48.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:48 smithi143 ceph-mon[189871]: Reconfiguring daemon mds.foo.smithi169.yeisur on smithi169 2023-08-28T02:17:48.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:48 smithi169 ceph-mon[159415]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:48.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:48 smithi169 ceph-mon[159415]: Reconfiguring mds.foo.smithi169.yeisur (monmap changed)... 2023-08-28T02:17:48.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:48 smithi169 ceph-mon[159415]: Reconfiguring daemon mds.foo.smithi169.yeisur on smithi169 2023-08-28T02:17:48.987 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:17:48 smithi169 conmon[150127]: ts=2023-08-28T02:17:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi169.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi143:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-08-28T02:17:49.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:49 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:49.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:49 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:49.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:49 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:17:49.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:49 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:49.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:49 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:17:49.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:49 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:49.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:49 smithi143 ceph-mon[183059]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:49.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:49 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:49.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:49 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:49.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:49 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:17:49.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:49 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:49.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:49 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:17:49.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:49 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:49.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:49 smithi143 ceph-mon[189871]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:49.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:49 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:49.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:49 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:49.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:49 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:17:49.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:49 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:49.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:49 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:17:49.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:49 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:17:49.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:49 smithi169 ceph-mon[159415]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:52.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:52 smithi143 ceph-mon[183059]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:52.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:52 smithi143 ceph-mon[189871]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:52.201 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:17:51 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:17:51] "GET /metrics HTTP/1.1" 200 33925 "" "Prometheus/2.33.4" 2023-08-28T02:17:52.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:52 smithi169 ceph-mon[159415]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:54.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:54 smithi143 ceph-mon[183059]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:54.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:54 smithi143 ceph-mon[189871]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:54.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:54 smithi169 ceph-mon[159415]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:56.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:56 smithi143 ceph-mon[183059]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:56.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:56 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:17:56.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:56 smithi143 ceph-mon[189871]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:56.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:56 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:17:56.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:56 smithi169 ceph-mon[159415]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:56.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:56 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:17:58.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:17:58 smithi143 ceph-mon[183059]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:58.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:17:58 smithi143 ceph-mon[189871]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:58.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:17:58 smithi169 ceph-mon[159415]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:17:58.987 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:17:58 smithi169 conmon[150127]: ts=2023-08-28T02:17:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi169.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi143:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-08-28T02:18:00.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:00 smithi143 ceph-mon[183059]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:00.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:00 smithi143 ceph-mon[189871]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:00.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:00 smithi169 ceph-mon[159415]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:02.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:02 smithi143 ceph-mon[183059]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:02.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:02 smithi143 ceph-mon[189871]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:02.201 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:18:01 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:18:01] "GET /metrics HTTP/1.1" 200 33942 "" "Prometheus/2.33.4" 2023-08-28T02:18:02.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:02 smithi169 ceph-mon[159415]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:04.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:04 smithi143 ceph-mon[183059]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:04.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:04 smithi143 ceph-mon[189871]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:04.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:04 smithi169 ceph-mon[159415]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:06.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:06 smithi143 ceph-mon[183059]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:06.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:06 smithi143 ceph-mon[189871]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:06.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:06 smithi169 ceph-mon[159415]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:08.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:08 smithi143 ceph-mon[183059]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:08.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:08 smithi143 ceph-mon[189871]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:08.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:08 smithi169 ceph-mon[159415]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:08.987 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:18:08 smithi169 conmon[150127]: ts=2023-08-28T02:18:08.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi169.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi143:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-08-28T02:18:10.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:10 smithi143 ceph-mon[183059]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:10.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:10 smithi143 ceph-mon[189871]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:10.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:10 smithi169 ceph-mon[159415]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:11.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:11 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:18:11.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:11 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:18:11.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:11 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:18:12.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:12 smithi143 ceph-mon[183059]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:12.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:12 smithi143 ceph-mon[189871]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:12.201 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:18:11 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:18:11] "GET /metrics HTTP/1.1" 200 33939 "" "Prometheus/2.33.4" 2023-08-28T02:18:12.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:12 smithi169 ceph-mon[159415]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:14.369 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:14 smithi143 ceph-mon[183059]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:14.370 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:14 smithi143 ceph-mon[189871]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:14.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:14 smithi169 ceph-mon[159415]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:14.851 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -e sha1=9cf208207dfe23ffccb33308b22e51570cb45e51 -- bash -c 'ceph orch upgrade resume' 2023-08-28T02:18:16.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:16 smithi143 ceph-mon[183059]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:16.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:16 smithi143 ceph-mon[189871]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:16.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:16 smithi169 ceph-mon[159415]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:16.654 INFO:teuthology.orchestra.run.smithi143.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8 2023-08-28T02:18:17.255 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -e sha1=9cf208207dfe23ffccb33308b22e51570cb45e51 -- bash -c 'echo wait for all mons to upgrade' 2023-08-28T02:18:17.659 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:17 smithi143 ceph-mon[183059]: from='client.34203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:18:17.660 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:17 smithi143 ceph-mon[183059]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8 2023-08-28T02:18:17.660 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:17 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:17.660 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:17 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:18:17.660 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:17 smithi143 ceph-mon[183059]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:17.661 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:17 smithi143 ceph-mon[189871]: from='client.34203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:18:17.661 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:17 smithi143 ceph-mon[189871]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8 2023-08-28T02:18:17.661 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:17 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:17.661 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:17 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:18:17.661 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:17 smithi143 ceph-mon[189871]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:17.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:17 smithi169 ceph-mon[159415]: from='client.34203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:18:17.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:17 smithi169 ceph-mon[159415]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8 2023-08-28T02:18:17.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:17 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:17.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:17 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:18:17.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:17 smithi169 ceph-mon[159415]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:18.987 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:18:18 smithi169 conmon[150127]: ts=2023-08-28T02:18:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi169.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi143:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-08-28T02:18:19.081 INFO:teuthology.orchestra.run.smithi143.stdout:wait for all mons to upgrade 2023-08-28T02:18:19.575 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -e sha1=9cf208207dfe23ffccb33308b22e51570cb45e51 -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2023-08-28T02:18:20.345 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:20 smithi143 ceph-mon[183059]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:20.345 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:20 smithi143 ceph-mon[189871]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:20.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:20 smithi169 ceph-mon[159415]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:21.743 INFO:teuthology.orchestra.run.smithi143.stdout: "ceph version 18.2.0-187-g9cf20820 (9cf208207dfe23ffccb33308b22e51570cb45e51) reef (stable)": 3 2023-08-28T02:18:22.080 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:18:21 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:18:21] "GET /metrics HTTP/1.1" 200 33939 "" "Prometheus/2.33.4" 2023-08-28T02:18:22.303 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -e sha1=9cf208207dfe23ffccb33308b22e51570cb45e51 -- bash -c 'ceph orch ps' 2023-08-28T02:18:22.331 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:22 smithi143 ceph-mon[183059]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:22.331 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:22 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:22.331 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:22 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:22.331 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:22 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:18:22.331 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:22 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:18:22.331 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:22 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:22.331 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:22 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:18:22.331 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:22 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:18:22.331 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:22 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:18:22.331 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:22 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:22.332 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:22 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-08-28T02:18:22.332 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:22 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-08-28T02:18:22.332 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:22 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-08-28T02:18:22.332 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:22 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-08-28T02:18:22.332 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:22 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-08-28T02:18:22.332 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:22 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-08-28T02:18:22.332 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:22 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:18:22.332 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:22 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:22.332 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:22 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1639231683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:22.332 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:22 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-08-28T02:18:22.333 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:22 smithi143 ceph-mon[189871]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:22.333 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:22.333 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:22.333 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:18:22.333 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:18:22.333 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:22.333 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:18:22.333 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:18:22.333 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:18:22.333 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:22.333 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-08-28T02:18:22.333 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-08-28T02:18:22.334 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-08-28T02:18:22.334 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-08-28T02:18:22.334 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-08-28T02:18:22.334 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-08-28T02:18:22.334 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:18:22.334 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:22.334 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:22 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1639231683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:22.334 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-08-28T02:18:22.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:22 smithi169 ceph-mon[159415]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:22.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:22 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:22.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:22 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:22.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:22 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:18:22.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:22 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:18:22.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:22 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:22.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:22 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:18:22.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:22 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:18:22.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:22 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:18:22.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:22 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:22.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:22 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-08-28T02:18:22.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:22 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-08-28T02:18:22.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:22 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-08-28T02:18:22.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:22 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-08-28T02:18:22.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:22 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-08-28T02:18:22.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:22 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-08-28T02:18:22.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:22 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:18:22.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:22 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:22.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:22 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1639231683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:22.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:22 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-08-28T02:18:23.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:23 smithi143 ceph-mon[189871]: Upgrade: Setting container_image for all mon 2023-08-28T02:18:23.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:23 smithi143 ceph-mon[189871]: Upgrade: Setting container_image for all crash 2023-08-28T02:18:23.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:23 smithi143 ceph-mon[189871]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-08-28T02:18:23.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:23 smithi143 ceph-mon[189871]: Upgrade: osd.0 is safe to restart 2023-08-28T02:18:23.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:23 smithi143 ceph-mon[183059]: Upgrade: Setting container_image for all mon 2023-08-28T02:18:23.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:23 smithi143 ceph-mon[183059]: Upgrade: Setting container_image for all crash 2023-08-28T02:18:23.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:23 smithi143 ceph-mon[183059]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-08-28T02:18:23.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:23 smithi143 ceph-mon[183059]: Upgrade: osd.0 is safe to restart 2023-08-28T02:18:23.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:23 smithi169 ceph-mon[159415]: Upgrade: Setting container_image for all mon 2023-08-28T02:18:23.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:23 smithi169 ceph-mon[159415]: Upgrade: Setting container_image for all crash 2023-08-28T02:18:23.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:23 smithi169 ceph-mon[159415]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-08-28T02:18:23.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:23 smithi169 ceph-mon[159415]: Upgrade: osd.0 is safe to restart 2023-08-28T02:18:24.170 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:24 smithi143 ceph-mon[183059]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:24.171 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:24 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:24.171 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:24 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-08-28T02:18:24.171 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:24 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:18:24.171 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:24 smithi143 ceph-mon[189871]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:24.171 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:24 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:24.171 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:24 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-08-28T02:18:24.171 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:24 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:18:24.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:24 smithi169 ceph-mon[159415]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:24.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:24 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:24.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:24 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-08-28T02:18:24.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:24 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:18:24.557 INFO:teuthology.orchestra.run.smithi143.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-08-28T02:18:24.557 INFO:teuthology.orchestra.run.smithi143.stdout:alertmanager.a smithi143 *:9093,9094 running (4m) 3s ago 7m 26.8M - 0.23.0 ba2b418f427c f216a6d30bb2 2023-08-28T02:18:24.557 INFO:teuthology.orchestra.run.smithi143.stdout:grafana.a smithi169 *:3000 running (3m) 59s ago 7m 56.0M - 8.3.5 dad864ee21e9 4c9eb6ad00e1 2023-08-28T02:18:24.558 INFO:teuthology.orchestra.run.smithi143.stdout:mds.foo.smithi143.spifrj smithi143 running (6m) 3s ago 6m 18.3M - 16.2.13-565-g595355f4 4099d2e00414 4eeedd3bc917 2023-08-28T02:18:24.558 INFO:teuthology.orchestra.run.smithi143.stdout:mds.foo.smithi169.yeisur smithi169 running (6m) 59s ago 6m 18.1M - 16.2.13-565-g595355f4 4099d2e00414 fed9013c1357 2023-08-28T02:18:24.558 INFO:teuthology.orchestra.run.smithi143.stdout:mgr.x smithi169 *:8443,9283,8765 running (3m) 59s ago 11m 447M - 18.2.0-187-g9cf20820 62b871dea10e 73a560edcde9 2023-08-28T02:18:24.558 INFO:teuthology.orchestra.run.smithi143.stdout:mgr.y smithi143 *:8443,9283,8765 running (3m) 3s ago 12m 504M - 18.2.0-187-g9cf20820 62b871dea10e 8de2036a51e3 2023-08-28T02:18:24.558 INFO:teuthology.orchestra.run.smithi143.stdout:mon.a smithi143 running (2m) 3s ago 13m 85.9M 2048M 18.2.0-187-g9cf20820 62b871dea10e 072272663d4f 2023-08-28T02:18:24.558 INFO:teuthology.orchestra.run.smithi143.stdout:mon.b smithi169 running (68s) 59s ago 11m 30.5M 2048M 18.2.0-187-g9cf20820 62b871dea10e 21231a80906c 2023-08-28T02:18:24.558 INFO:teuthology.orchestra.run.smithi143.stdout:mon.c smithi143 running (85s) 3s ago 11m 71.9M 2048M 18.2.0-187-g9cf20820 62b871dea10e f1328bdbe045 2023-08-28T02:18:24.558 INFO:teuthology.orchestra.run.smithi143.stdout:node-exporter.a smithi143 *:9100 running (4m) 3s ago 7m 26.1M - 1.3.1 1dbe0e931976 e389cc167c39 2023-08-28T02:18:24.558 INFO:teuthology.orchestra.run.smithi143.stdout:node-exporter.b smithi169 *:9100 running (3m) 59s ago 7m 21.1M - 1.3.1 1dbe0e931976 5e5215d11ac8 2023-08-28T02:18:24.558 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0 smithi143 running (11m) 3s ago 11m 76.0M 2500M 16.2.13-565-g595355f4 4099d2e00414 ce7c7eb91333 2023-08-28T02:18:24.558 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1 smithi143 running (10m) 3s ago 10m 92.8M 2500M 16.2.13-565-g595355f4 4099d2e00414 f27082b59650 2023-08-28T02:18:24.558 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2 smithi143 running (10m) 3s ago 10m 69.6M 2500M 16.2.13-565-g595355f4 4099d2e00414 e20c9920166c 2023-08-28T02:18:24.559 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3 smithi143 running (10m) 3s ago 10m 69.2M 2500M 16.2.13-565-g595355f4 4099d2e00414 ab95599724c4 2023-08-28T02:18:24.559 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4 smithi169 running (9m) 59s ago 9m 72.5M 2500M 16.2.13-565-g595355f4 4099d2e00414 e6ed776def27 2023-08-28T02:18:24.559 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5 smithi169 running (9m) 59s ago 9m 84.7M 2500M 16.2.13-565-g595355f4 4099d2e00414 6fe1195fca7e 2023-08-28T02:18:24.559 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6 smithi169 running (9m) 59s ago 9m 78.8M 2500M 16.2.13-565-g595355f4 4099d2e00414 12854ca48e76 2023-08-28T02:18:24.559 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7 smithi169 running (8m) 59s ago 8m 96.7M 2500M 16.2.13-565-g595355f4 4099d2e00414 64fa1422c85a 2023-08-28T02:18:24.559 INFO:teuthology.orchestra.run.smithi143.stdout:prometheus.a smithi169 *:9095 running (3m) 59s ago 7m 61.2M - 2.33.4 514e6a882f6e cd763ce7d282 2023-08-28T02:18:25.414 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:25 smithi143 ceph-mon[183059]: Upgrade: Updating osd.0 2023-08-28T02:18:25.415 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:25 smithi143 ceph-mon[183059]: Deploying daemon osd.0 on smithi143 2023-08-28T02:18:25.415 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:25 smithi143 ceph-mon[189871]: Upgrade: Updating osd.0 2023-08-28T02:18:25.415 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:25 smithi143 ceph-mon[189871]: Deploying daemon osd.0 on smithi143 2023-08-28T02:18:25.472 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -e sha1=9cf208207dfe23ffccb33308b22e51570cb45e51 -- bash -c 'ceph orch upgrade pause' 2023-08-28T02:18:25.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:25 smithi169 ceph-mon[159415]: Upgrade: Updating osd.0 2023-08-28T02:18:25.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:25 smithi169 ceph-mon[159415]: Deploying daemon osd.0 on smithi143 2023-08-28T02:18:26.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:26 smithi143 ceph-mon[189871]: from='client.44126 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:18:26.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:26 smithi143 ceph-mon[189871]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:26.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:26 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:18:26.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:26 smithi143 ceph-mon[183059]: from='client.44126 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:18:26.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:26 smithi143 ceph-mon[183059]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:26.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:26 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:18:26.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:26 smithi169 ceph-mon[159415]: from='client.44126 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:18:26.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:26 smithi169 ceph-mon[159415]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:26.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:26 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:18:27.400 INFO:journalctl@ceph.osd.0.smithi143.stdout:Aug 28 02:18:27 smithi143 systemd[1]: Stopping Ceph osd.0 for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:18:27.700 INFO:journalctl@ceph.osd.0.smithi143.stdout:Aug 28 02:18:27 smithi143 conmon[115063]: debug 2023-08-28T02:18:27.618+0000 7f21c9813700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-08-28T02:18:27.700 INFO:journalctl@ceph.osd.0.smithi143.stdout:Aug 28 02:18:27 smithi143 conmon[115063]: debug 2023-08-28T02:18:27.618+0000 7f21c9813700 -1 osd.0 86 *** Got signal Terminated *** 2023-08-28T02:18:27.700 INFO:journalctl@ceph.osd.0.smithi143.stdout:Aug 28 02:18:27 smithi143 conmon[115063]: debug 2023-08-28T02:18:27.618+0000 7f21c9813700 -1 osd.0 86 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-08-28T02:18:27.815 INFO:teuthology.orchestra.run.smithi143.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8 2023-08-28T02:18:28.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:28 smithi143 ceph-mon[183059]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:28.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:28 smithi143 ceph-mon[183059]: osd.0 marked itself down and dead 2023-08-28T02:18:28.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:28 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:28.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:28 smithi143 ceph-mon[189871]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:28.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:28 smithi143 ceph-mon[189871]: osd.0 marked itself down and dead 2023-08-28T02:18:28.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:28 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:28.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:28 smithi169 ceph-mon[159415]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:28.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:28 smithi169 ceph-mon[159415]: osd.0 marked itself down and dead 2023-08-28T02:18:28.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:28 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:28.821 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -e sha1=9cf208207dfe23ffccb33308b22e51570cb45e51 -- bash -c 'sleep 60' 2023-08-28T02:18:28.987 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:18:28 smithi169 conmon[150127]: ts=2023-08-28T02:18:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi169.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi143:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-08-28T02:18:29.109 INFO:journalctl@ceph.osd.0.smithi143.stdout:Aug 28 02:18:28 smithi143 bash[198628]: ce7c7eb91333a4297cb84582bd5d5db1392b86be9c55283e0b65596c19af2f06 2023-08-28T02:18:29.110 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:29 smithi143 ceph-mon[189871]: from='client.34221 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:18:29.110 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:29 smithi143 ceph-mon[189871]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8 2023-08-28T02:18:29.110 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:29 smithi143 ceph-mon[189871]: Health check failed: 1 osds down (OSD_DOWN) 2023-08-28T02:18:29.110 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:29 smithi143 ceph-mon[189871]: failed to encode map e87 with expected crc 2023-08-28T02:18:29.110 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:29 smithi143 ceph-mon[189871]: failed to encode map e87 with expected crc 2023-08-28T02:18:29.111 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:29 smithi143 ceph-mon[189871]: osdmap e87: 8 total, 7 up, 8 in 2023-08-28T02:18:29.362 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:29 smithi143 ceph-mon[183059]: from='client.34221 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:18:29.362 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:29 smithi143 ceph-mon[183059]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8 2023-08-28T02:18:29.362 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:29 smithi143 ceph-mon[183059]: Health check failed: 1 osds down (OSD_DOWN) 2023-08-28T02:18:29.362 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:29 smithi143 ceph-mon[183059]: failed to encode map e87 with expected crc 2023-08-28T02:18:29.362 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:29 smithi143 ceph-mon[183059]: failed to encode map e87 with expected crc 2023-08-28T02:18:29.362 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:29 smithi143 ceph-mon[183059]: osdmap e87: 8 total, 7 up, 8 in 2023-08-28T02:18:29.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:29 smithi169 ceph-mon[159415]: from='client.34221 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:18:29.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:29 smithi169 ceph-mon[159415]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8 2023-08-28T02:18:29.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:29 smithi169 ceph-mon[159415]: Health check failed: 1 osds down (OSD_DOWN) 2023-08-28T02:18:29.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:29 smithi169 ceph-mon[159415]: failed to encode map e87 with expected crc 2023-08-28T02:18:29.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:29 smithi169 ceph-mon[159415]: failed to encode map e87 with expected crc 2023-08-28T02:18:29.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:29 smithi169 ceph-mon[159415]: osdmap e87: 8 total, 7 up, 8 in 2023-08-28T02:18:30.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:30 smithi143 ceph-mon[189871]: pgmap v88: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:30.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:30 smithi143 ceph-mon[189871]: failed to encode map e88 with expected crc 2023-08-28T02:18:30.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:30 smithi143 ceph-mon[189871]: failed to encode map e88 with expected crc 2023-08-28T02:18:30.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:30 smithi143 ceph-mon[189871]: osdmap e88: 8 total, 7 up, 8 in 2023-08-28T02:18:30.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:30 smithi143 ceph-mon[189871]: failed to encode map e88 with expected crc 2023-08-28T02:18:30.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:30 smithi143 ceph-mon[189871]: failed to encode map e88 with expected crc 2023-08-28T02:18:30.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:30 smithi143 ceph-mon[189871]: failed to encode map e88 with expected crc 2023-08-28T02:18:30.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:30 smithi143 ceph-mon[189871]: failed to encode map e88 with expected crc 2023-08-28T02:18:30.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:30 smithi143 ceph-mon[189871]: failed to encode map e88 with expected crc 2023-08-28T02:18:30.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:30 smithi143 ceph-mon[189871]: failed to encode map e88 with expected crc 2023-08-28T02:18:30.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:30 smithi143 ceph-mon[183059]: pgmap v88: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:30.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:30 smithi143 ceph-mon[183059]: failed to encode map e88 with expected crc 2023-08-28T02:18:30.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:30 smithi143 ceph-mon[183059]: failed to encode map e88 with expected crc 2023-08-28T02:18:30.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:30 smithi143 ceph-mon[183059]: osdmap e88: 8 total, 7 up, 8 in 2023-08-28T02:18:30.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:30 smithi143 ceph-mon[183059]: failed to encode map e88 with expected crc 2023-08-28T02:18:30.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:30 smithi143 ceph-mon[183059]: failed to encode map e88 with expected crc 2023-08-28T02:18:30.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:30 smithi143 ceph-mon[183059]: failed to encode map e88 with expected crc 2023-08-28T02:18:30.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:30 smithi143 ceph-mon[183059]: failed to encode map e88 with expected crc 2023-08-28T02:18:30.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:30 smithi143 ceph-mon[183059]: failed to encode map e88 with expected crc 2023-08-28T02:18:30.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:30 smithi143 ceph-mon[183059]: failed to encode map e88 with expected crc 2023-08-28T02:18:30.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:30 smithi169 ceph-mon[159415]: pgmap v88: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:30.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:30 smithi169 ceph-mon[159415]: failed to encode map e88 with expected crc 2023-08-28T02:18:30.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:30 smithi169 ceph-mon[159415]: failed to encode map e88 with expected crc 2023-08-28T02:18:30.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:30 smithi169 ceph-mon[159415]: osdmap e88: 8 total, 7 up, 8 in 2023-08-28T02:18:30.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:30 smithi169 ceph-mon[159415]: failed to encode map e88 with expected crc 2023-08-28T02:18:30.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:30 smithi169 ceph-mon[159415]: failed to encode map e88 with expected crc 2023-08-28T02:18:30.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:30 smithi169 ceph-mon[159415]: failed to encode map e88 with expected crc 2023-08-28T02:18:30.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:30 smithi169 ceph-mon[159415]: failed to encode map e88 with expected crc 2023-08-28T02:18:30.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:30 smithi169 ceph-mon[159415]: failed to encode map e88 with expected crc 2023-08-28T02:18:30.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:30 smithi169 ceph-mon[159415]: failed to encode map e88 with expected crc 2023-08-28T02:18:31.450 INFO:journalctl@ceph.osd.0.smithi143.stdout:Aug 28 02:18:31 smithi143 systemd[1]: ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@osd.0.service: Succeeded. 2023-08-28T02:18:31.451 INFO:journalctl@ceph.osd.0.smithi143.stdout:Aug 28 02:18:31 smithi143 systemd[1]: Stopped Ceph osd.0 for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:18:31.451 INFO:journalctl@ceph.osd.0.smithi143.stdout:Aug 28 02:18:31 smithi143 systemd[1]: Starting Ceph osd.0 for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:18:31.951 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:18:31 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:18:31] "GET /metrics HTTP/1.1" 200 33938 "" "Prometheus/2.33.4" 2023-08-28T02:18:32.395 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:32 smithi143 ceph-mon[183059]: pgmap v90: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:32.396 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:32 smithi143 ceph-mon[189871]: pgmap v90: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:32.396 INFO:journalctl@ceph.osd.0.smithi143.stdout:Aug 28 02:18:32 smithi143 bash[199585]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-08-28T02:18:32.396 INFO:journalctl@ceph.osd.0.smithi143.stdout:Aug 28 02:18:32 smithi143 bash[199585]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2023-08-28T02:18:32.396 INFO:journalctl@ceph.osd.0.smithi143.stdout:Aug 28 02:18:32 smithi143 bash[199585]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2023-08-28T02:18:32.396 INFO:journalctl@ceph.osd.0.smithi143.stdout:Aug 28 02:18:32 smithi143 bash[199585]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-08-28T02:18:32.397 INFO:journalctl@ceph.osd.0.smithi143.stdout:Aug 28 02:18:32 smithi143 bash[199585]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2023-08-28T02:18:32.397 INFO:journalctl@ceph.osd.0.smithi143.stdout:Aug 28 02:18:32 smithi143 bash[199585]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-08-28T02:18:32.397 INFO:journalctl@ceph.osd.0.smithi143.stdout:Aug 28 02:18:32 smithi143 bash[199585]: --> ceph-volume raw activate successful for osd ID: 0 2023-08-28T02:18:32.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:32 smithi169 ceph-mon[159415]: pgmap v90: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:33.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:33 smithi143 ceph-mon[183059]: Health check failed: Degraded data redundancy: 14/84 objects degraded (16.667%), 9 pgs degraded (PG_DEGRADED) 2023-08-28T02:18:33.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:33 smithi143 ceph-mon[189871]: Health check failed: Degraded data redundancy: 14/84 objects degraded (16.667%), 9 pgs degraded (PG_DEGRADED) 2023-08-28T02:18:33.451 INFO:journalctl@ceph.osd.0.smithi143.stdout:Aug 28 02:18:33 smithi143 ceph-osd[200094]: -- 172.21.15.143:0/2758813480 <== mon.0 v2:172.21.15.143:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55d77a7864e0 con 0x55d77a759400 2023-08-28T02:18:33.451 INFO:journalctl@ceph.osd.0.smithi143.stdout:Aug 28 02:18:33 smithi143 bash[200029]: 241e17d94254922f328b2f9a77b934ad327995b56a487ead31443532bb1f29bf 2023-08-28T02:18:33.451 INFO:journalctl@ceph.osd.0.smithi143.stdout:Aug 28 02:18:33 smithi143 systemd[1]: Started Ceph osd.0 for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:18:33.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:33 smithi169 ceph-mon[159415]: Health check failed: Degraded data redundancy: 14/84 objects degraded (16.667%), 9 pgs degraded (PG_DEGRADED) 2023-08-28T02:18:34.168 INFO:journalctl@ceph.osd.0.smithi143.stdout:Aug 28 02:18:33 smithi143 conmon[200070]: 2023-08-28T02:18:33.862+0000 7feb12da6640 -1 Falling back to public interface 2023-08-28T02:18:34.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:34 smithi143 ceph-mon[183059]: pgmap v91: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-08-28T02:18:34.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:34 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:34.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:34 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:34.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:34 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:18:34.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:34 smithi143 ceph-mon[189871]: pgmap v91: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-08-28T02:18:34.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:34 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:34.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:34 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:34.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:34 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:18:34.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:34 smithi169 ceph-mon[159415]: pgmap v91: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-08-28T02:18:34.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:34 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:34.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:34 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:34.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:34 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:18:36.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:36 smithi143 ceph-mon[189871]: pgmap v92: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-08-28T02:18:36.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:36 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:36.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:36 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:36.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:36 smithi143 ceph-mon[183059]: pgmap v92: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-08-28T02:18:36.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:36 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:36.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:36 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:36.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:36 smithi169 ceph-mon[159415]: pgmap v92: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-08-28T02:18:36.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:36 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:36.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:36 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:37.200 INFO:journalctl@ceph.osd.0.smithi143.stdout:Aug 28 02:18:36 smithi143 conmon[200070]: 2023-08-28T02:18:36.868+0000 7feb12da6640 -1 osd.0 86 log_to_monitors true 2023-08-28T02:18:37.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:37 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:37.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:37 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:37.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:37 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:37.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:37 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:37.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:37 smithi143 ceph-mon[183059]: from='osd.0 [v2:172.21.15.143:6802/257144662,v1:172.21.15.143:6803/257144662]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-08-28T02:18:37.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:37 smithi143 ceph-mon[183059]: pgmap v93: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-08-28T02:18:37.951 INFO:journalctl@ceph.osd.0.smithi143.stdout:Aug 28 02:18:37 smithi143 conmon[200070]: 2023-08-28T02:18:37.784+0000 7feb0120c700 -1 osd.0 86 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-08-28T02:18:37.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:37 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:37.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:37 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:37.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:37 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:37.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:37 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:37.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:37 smithi143 ceph-mon[189871]: from='osd.0 [v2:172.21.15.143:6802/257144662,v1:172.21.15.143:6803/257144662]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-08-28T02:18:37.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:37 smithi143 ceph-mon[189871]: pgmap v93: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-08-28T02:18:37.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:37 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:37.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:37 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:37.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:37 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:37.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:37 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:37.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:37 smithi169 ceph-mon[159415]: from='osd.0 [v2:172.21.15.143:6802/257144662,v1:172.21.15.143:6803/257144662]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-08-28T02:18:37.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:37 smithi169 ceph-mon[159415]: pgmap v93: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-08-28T02:18:38.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:38 smithi169 ceph-mon[159415]: failed to encode map e89 with expected crc 2023-08-28T02:18:38.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:38 smithi169 ceph-mon[159415]: failed to encode map e89 with expected crc 2023-08-28T02:18:38.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:38 smithi169 ceph-mon[159415]: from='osd.0 [v2:172.21.15.143:6802/257144662,v1:172.21.15.143:6803/257144662]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-08-28T02:18:38.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:38 smithi169 ceph-mon[159415]: osdmap e89: 8 total, 7 up, 8 in 2023-08-28T02:18:38.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:38 smithi169 ceph-mon[159415]: from='osd.0 [v2:172.21.15.143:6802/257144662,v1:172.21.15.143:6803/257144662]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]: dispatch 2023-08-28T02:18:38.988 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:18:38 smithi169 conmon[150127]: ts=2023-08-28T02:18:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi169.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi143:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-08-28T02:18:39.103 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:38 smithi143 ceph-mon[183059]: failed to encode map e89 with expected crc 2023-08-28T02:18:39.103 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:38 smithi143 ceph-mon[183059]: failed to encode map e89 with expected crc 2023-08-28T02:18:39.103 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:38 smithi143 ceph-mon[183059]: from='osd.0 [v2:172.21.15.143:6802/257144662,v1:172.21.15.143:6803/257144662]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-08-28T02:18:39.103 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:38 smithi143 ceph-mon[183059]: osdmap e89: 8 total, 7 up, 8 in 2023-08-28T02:18:39.103 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:38 smithi143 ceph-mon[183059]: from='osd.0 [v2:172.21.15.143:6802/257144662,v1:172.21.15.143:6803/257144662]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]: dispatch 2023-08-28T02:18:39.103 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:38 smithi143 ceph-mon[189871]: failed to encode map e89 with expected crc 2023-08-28T02:18:39.104 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:38 smithi143 ceph-mon[189871]: failed to encode map e89 with expected crc 2023-08-28T02:18:39.104 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:38 smithi143 ceph-mon[189871]: from='osd.0 [v2:172.21.15.143:6802/257144662,v1:172.21.15.143:6803/257144662]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-08-28T02:18:39.104 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:38 smithi143 ceph-mon[189871]: osdmap e89: 8 total, 7 up, 8 in 2023-08-28T02:18:39.104 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:38 smithi143 ceph-mon[189871]: from='osd.0 [v2:172.21.15.143:6802/257144662,v1:172.21.15.143:6803/257144662]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]: dispatch 2023-08-28T02:18:39.786 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:39 smithi143 ceph-mon[183059]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-08-28T02:18:40.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:39 smithi143 ceph-mon[183059]: osd.0 [v2:172.21.15.143:6802/257144662,v1:172.21.15.143:6803/257144662] boot 2023-08-28T02:18:40.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:39 smithi143 ceph-mon[183059]: osdmap e90: 8 total, 8 up, 8 in 2023-08-28T02:18:40.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:39 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-28T02:18:40.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:39 smithi143 ceph-mon[183059]: failed to encode map e90 with expected crc 2023-08-28T02:18:40.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:39 smithi143 ceph-mon[183059]: pgmap v96: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-08-28T02:18:40.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:39 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:40.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:39 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:40.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:39 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:18:40.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:39 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:18:40.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:39 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:40.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:39 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:18:40.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:39 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:18:40.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:39 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:18:40.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:39 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:40.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:39 smithi143 ceph-mon[183059]: osdmap e91: 8 total, 8 up, 8 in 2023-08-28T02:18:40.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:39 smithi143 ceph-mon[189871]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-08-28T02:18:40.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:39 smithi143 ceph-mon[189871]: osd.0 [v2:172.21.15.143:6802/257144662,v1:172.21.15.143:6803/257144662] boot 2023-08-28T02:18:40.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:39 smithi143 ceph-mon[189871]: osdmap e90: 8 total, 8 up, 8 in 2023-08-28T02:18:40.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:39 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-28T02:18:40.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:39 smithi143 ceph-mon[189871]: failed to encode map e90 with expected crc 2023-08-28T02:18:40.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:39 smithi143 ceph-mon[189871]: pgmap v96: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-08-28T02:18:40.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:39 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:40.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:39 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:40.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:39 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:18:40.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:39 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:18:40.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:39 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:40.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:39 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:18:40.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:39 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:18:40.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:39 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:18:40.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:39 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:40.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:39 smithi143 ceph-mon[189871]: osdmap e91: 8 total, 8 up, 8 in 2023-08-28T02:18:40.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:39 smithi169 ceph-mon[159415]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-08-28T02:18:40.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:39 smithi169 ceph-mon[159415]: osd.0 [v2:172.21.15.143:6802/257144662,v1:172.21.15.143:6803/257144662] boot 2023-08-28T02:18:40.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:39 smithi169 ceph-mon[159415]: osdmap e90: 8 total, 8 up, 8 in 2023-08-28T02:18:40.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:39 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-28T02:18:40.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:39 smithi169 ceph-mon[159415]: failed to encode map e90 with expected crc 2023-08-28T02:18:40.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:39 smithi169 ceph-mon[159415]: pgmap v96: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-08-28T02:18:40.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:39 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:40.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:39 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:40.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:39 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:18:40.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:39 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:18:40.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:39 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:40.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:39 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:18:40.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:39 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:18:40.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:39 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:18:40.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:39 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:40.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:39 smithi169 ceph-mon[159415]: osdmap e91: 8 total, 8 up, 8 in 2023-08-28T02:18:41.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:40 smithi143 ceph-mon[183059]: failed to encode map e91 with expected crc 2023-08-28T02:18:41.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:40 smithi143 ceph-mon[183059]: failed to encode map e91 with expected crc 2023-08-28T02:18:41.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:40 smithi143 ceph-mon[183059]: failed to encode map e91 with expected crc 2023-08-28T02:18:41.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:40 smithi143 ceph-mon[183059]: failed to encode map e91 with expected crc 2023-08-28T02:18:41.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:40 smithi143 ceph-mon[183059]: failed to encode map e91 with expected crc 2023-08-28T02:18:41.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:40 smithi143 ceph-mon[183059]: failed to encode map e91 with expected crc 2023-08-28T02:18:41.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:40 smithi143 ceph-mon[183059]: failed to encode map e91 with expected crc 2023-08-28T02:18:41.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:40 smithi143 ceph-mon[183059]: failed to encode map e91 with expected crc 2023-08-28T02:18:41.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:40 smithi143 ceph-mon[183059]: failed to encode map e91 with expected crc 2023-08-28T02:18:41.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:40 smithi143 ceph-mon[189871]: failed to encode map e91 with expected crc 2023-08-28T02:18:41.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:40 smithi143 ceph-mon[189871]: failed to encode map e91 with expected crc 2023-08-28T02:18:41.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:40 smithi143 ceph-mon[189871]: failed to encode map e91 with expected crc 2023-08-28T02:18:41.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:40 smithi143 ceph-mon[189871]: failed to encode map e91 with expected crc 2023-08-28T02:18:41.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:40 smithi143 ceph-mon[189871]: failed to encode map e91 with expected crc 2023-08-28T02:18:41.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:40 smithi143 ceph-mon[189871]: failed to encode map e91 with expected crc 2023-08-28T02:18:41.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:40 smithi143 ceph-mon[189871]: failed to encode map e91 with expected crc 2023-08-28T02:18:41.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:40 smithi143 ceph-mon[189871]: failed to encode map e91 with expected crc 2023-08-28T02:18:41.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:40 smithi143 ceph-mon[189871]: failed to encode map e91 with expected crc 2023-08-28T02:18:41.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:40 smithi169 ceph-mon[159415]: failed to encode map e91 with expected crc 2023-08-28T02:18:41.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:40 smithi169 ceph-mon[159415]: failed to encode map e91 with expected crc 2023-08-28T02:18:41.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:40 smithi169 ceph-mon[159415]: failed to encode map e91 with expected crc 2023-08-28T02:18:41.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:40 smithi169 ceph-mon[159415]: failed to encode map e91 with expected crc 2023-08-28T02:18:41.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:40 smithi169 ceph-mon[159415]: failed to encode map e91 with expected crc 2023-08-28T02:18:41.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:40 smithi169 ceph-mon[159415]: failed to encode map e91 with expected crc 2023-08-28T02:18:41.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:40 smithi169 ceph-mon[159415]: failed to encode map e91 with expected crc 2023-08-28T02:18:41.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:40 smithi169 ceph-mon[159415]: failed to encode map e91 with expected crc 2023-08-28T02:18:41.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:40 smithi169 ceph-mon[159415]: failed to encode map e91 with expected crc 2023-08-28T02:18:42.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:42 smithi143 ceph-mon[183059]: pgmap v98: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-08-28T02:18:42.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:42 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:42.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:42 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:18:42.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:42 smithi143 ceph-mon[189871]: pgmap v98: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-08-28T02:18:42.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:42 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:42.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:42 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:18:42.201 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:18:41 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:18:41] "GET /metrics HTTP/1.1" 200 33850 "" "Prometheus/2.33.4" 2023-08-28T02:18:42.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:42 smithi169 ceph-mon[159415]: pgmap v98: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-08-28T02:18:42.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:42 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:42.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:42 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:18:43.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:43 smithi143 ceph-mon[189871]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 14/84 objects degraded (16.667%), 9 pgs degraded) 2023-08-28T02:18:43.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:43 smithi143 ceph-mon[189871]: Cluster is now healthy 2023-08-28T02:18:43.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:43 smithi143 ceph-mon[183059]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 14/84 objects degraded (16.667%), 9 pgs degraded) 2023-08-28T02:18:43.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:43 smithi143 ceph-mon[183059]: Cluster is now healthy 2023-08-28T02:18:43.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:43 smithi169 ceph-mon[159415]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 14/84 objects degraded (16.667%), 9 pgs degraded) 2023-08-28T02:18:43.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:43 smithi169 ceph-mon[159415]: Cluster is now healthy 2023-08-28T02:18:44.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:44 smithi143 ceph-mon[183059]: pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:44.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:44 smithi143 ceph-mon[189871]: pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:44.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:44 smithi169 ceph-mon[159415]: pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:46.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:46 smithi143 ceph-mon[183059]: pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:46.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:46 smithi143 ceph-mon[189871]: pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:46.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:46 smithi169 ceph-mon[159415]: pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:48.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:48 smithi143 ceph-mon[183059]: pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:48.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:48 smithi143 ceph-mon[189871]: pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:48.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:48 smithi169 ceph-mon[159415]: pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:48.987 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:18:48 smithi169 conmon[150127]: ts=2023-08-28T02:18:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.2.0-187-g9cf20820 (9cf208207dfe23ffccb33308b22e51570cb45e51) reef (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi143:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi169.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-08-28T02:18:50.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:50 smithi143 ceph-mon[183059]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:50.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:50 smithi143 ceph-mon[189871]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:50.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:50 smithi169 ceph-mon[159415]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:52.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:18:51 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:18:51] "GET /metrics HTTP/1.1" 200 33850 "" "Prometheus/2.33.4" 2023-08-28T02:18:52.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:52 smithi143 ceph-mon[183059]: pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:52.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:52 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:52.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:52 smithi143 ceph-mon[189871]: pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:52.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:52 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:52.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:52 smithi169 ceph-mon[159415]: pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:52.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:52 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:54.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:54 smithi143 ceph-mon[183059]: pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:54.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:54 smithi143 ceph-mon[189871]: pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:54.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:54 smithi169 ceph-mon[159415]: pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:56.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:56 smithi143 ceph-mon[183059]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:56.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:56 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:56.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:56 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:18:56.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:56 smithi143 ceph-mon[189871]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:56.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:56 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:56.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:56 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:18:56.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:56 smithi169 ceph-mon[159415]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:56.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:56 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:18:56.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:56 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:18:58.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:18:58 smithi143 ceph-mon[183059]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:58.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:18:58 smithi143 ceph-mon[189871]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:58.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:18:58 smithi169 ceph-mon[159415]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:18:58.987 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:18:58 smithi169 conmon[150127]: ts=2023-08-28T02:18:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.2.0-187-g9cf20820 (9cf208207dfe23ffccb33308b22e51570cb45e51) reef (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi143:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi169.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-08-28T02:19:00.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:00 smithi143 ceph-mon[183059]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:00.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:00 smithi143 ceph-mon[189871]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:00.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:00 smithi169 ceph-mon[159415]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:02.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:02 smithi143 ceph-mon[183059]: pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:02.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:02 smithi143 ceph-mon[189871]: pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:02.201 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:19:01 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:19:01] "GET /metrics HTTP/1.1" 200 33999 "" "Prometheus/2.33.4" 2023-08-28T02:19:02.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:02 smithi169 ceph-mon[159415]: pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:04.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:04 smithi143 ceph-mon[183059]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:04.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:04 smithi143 ceph-mon[189871]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:04.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:04 smithi169 ceph-mon[159415]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:06.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:06 smithi143 ceph-mon[183059]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:06.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:06 smithi143 ceph-mon[189871]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:06.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:06 smithi169 ceph-mon[159415]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:08.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:08 smithi143 ceph-mon[183059]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:08.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:08 smithi143 ceph-mon[189871]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:08.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:08 smithi169 ceph-mon[159415]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:08.987 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:19:08 smithi169 conmon[150127]: ts=2023-08-28T02:19:08.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.2.0-187-g9cf20820 (9cf208207dfe23ffccb33308b22e51570cb45e51) reef (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi143:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi169.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-08-28T02:19:10.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:10 smithi143 ceph-mon[183059]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:10.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:10 smithi143 ceph-mon[189871]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:10.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:10 smithi169 ceph-mon[159415]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:11.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:11 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:19:11.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:11 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:19:11.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:11 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:19:12.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:12 smithi143 ceph-mon[183059]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:12.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:12 smithi143 ceph-mon[189871]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:12.201 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:19:11 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:19:11] "GET /metrics HTTP/1.1" 200 34001 "" "Prometheus/2.33.4" 2023-08-28T02:19:12.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:12 smithi169 ceph-mon[159415]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:14.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:14 smithi143 ceph-mon[183059]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:14.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:14 smithi143 ceph-mon[189871]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:14.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:14 smithi169 ceph-mon[159415]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:16.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:16 smithi143 ceph-mon[183059]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:16.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:16 smithi143 ceph-mon[189871]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:16.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:16 smithi169 ceph-mon[159415]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:18.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:18 smithi143 ceph-mon[183059]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:18.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:18 smithi143 ceph-mon[189871]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:18.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:18 smithi169 ceph-mon[159415]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:18.987 INFO:journalctl@ceph.prometheus.a.smithi169.stdout:Aug 28 02:19:18 smithi169 conmon[150127]: ts=2023-08-28T02:19:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.2.0-187-g9cf20820 (9cf208207dfe23ffccb33308b22e51570cb45e51) reef (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi143:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)\", cluster_addr=\"172.21.15.143\", device_class=\"ssd\", hostname=\"smithi143\", instance=\"smithi169.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.143\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-08-28T02:19:20.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:20 smithi143 ceph-mon[183059]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:20.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:20 smithi143 ceph-mon[189871]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:20.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:20 smithi169 ceph-mon[159415]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:22.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:22 smithi143 ceph-mon[183059]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:22.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:22 smithi143 ceph-mon[189871]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:22.201 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:19:21 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:19:21] "GET /metrics HTTP/1.1" 200 34001 "" "Prometheus/2.33.4" 2023-08-28T02:19:22.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:22 smithi169 ceph-mon[159415]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:24.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:24 smithi143 ceph-mon[183059]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:24.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:24 smithi143 ceph-mon[189871]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:24.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:24 smithi169 ceph-mon[159415]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:26.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:26 smithi143 ceph-mon[183059]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:26.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:26 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:19:26.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:26 smithi143 ceph-mon[189871]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:26.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:26 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:19:26.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:26 smithi169 ceph-mon[159415]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:26.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:26 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:19:28.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:28 smithi143 ceph-mon[183059]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:28.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:28 smithi143 ceph-mon[189871]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:28.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:28 smithi169 ceph-mon[159415]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:30.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:30 smithi143 ceph-mon[183059]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:30.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:30 smithi143 ceph-mon[189871]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:30.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:30 smithi169 ceph-mon[159415]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:31.455 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -e sha1=9cf208207dfe23ffccb33308b22e51570cb45e51 -- bash -c 'ceph orch upgrade resume' 2023-08-28T02:19:32.150 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:19:31 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:19:31] "GET /metrics HTTP/1.1" 200 34000 "" "Prometheus/2.33.4" 2023-08-28T02:19:32.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:32 smithi143 ceph-mon[189871]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:32.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:32 smithi143 ceph-mon[183059]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:32.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:32 smithi169 ceph-mon[159415]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:33.299 INFO:teuthology.orchestra.run.smithi143.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8 2023-08-28T02:19:33.881 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -e sha1=9cf208207dfe23ffccb33308b22e51570cb45e51 -- bash -c 'echo wait for half of osds to upgrade' 2023-08-28T02:19:34.369 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:34 smithi143 ceph-mon[183059]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:34.369 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:34 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:34.369 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:34 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:19:34.370 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:34 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:19:34.370 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:34 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:19:34.370 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:34 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:34.370 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:34 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:19:34.370 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:34 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:19:34.370 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:34 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:19:34.370 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:34 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:19:34.370 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:34 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-08-28T02:19:34.370 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:34 smithi143 ceph-mon[189871]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:34.371 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:34 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:34.371 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:34 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:19:34.371 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:34 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:19:34.371 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:34 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:19:34.371 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:34 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:34.371 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:34 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:19:34.371 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:34 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:19:34.371 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:34 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:19:34.371 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:34 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:19:34.371 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:34 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-08-28T02:19:34.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:34 smithi169 ceph-mon[159415]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:34.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:34 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:34.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:34 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:19:34.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:34 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:19:34.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:34 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:19:34.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:34 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:34.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:34 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:19:34.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:34 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:19:34.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:34 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:19:34.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:34 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:19:34.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:34 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-08-28T02:19:35.416 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:35 smithi143 ceph-mon[183059]: from='client.44141 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:19:35.416 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:35 smithi143 ceph-mon[183059]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8 2023-08-28T02:19:35.416 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:35 smithi143 ceph-mon[183059]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-08-28T02:19:35.417 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:35 smithi143 ceph-mon[183059]: Upgrade: osd.1 is safe to restart 2023-08-28T02:19:35.417 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:35 smithi143 ceph-mon[189871]: from='client.44141 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:19:35.417 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:35 smithi143 ceph-mon[189871]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8 2023-08-28T02:19:35.417 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:35 smithi143 ceph-mon[189871]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-08-28T02:19:35.417 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:35 smithi143 ceph-mon[189871]: Upgrade: osd.1 is safe to restart 2023-08-28T02:19:35.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:35 smithi169 ceph-mon[159415]: from='client.44141 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:19:35.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:35 smithi169 ceph-mon[159415]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8 2023-08-28T02:19:35.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:35 smithi169 ceph-mon[159415]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-08-28T02:19:35.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:35 smithi169 ceph-mon[159415]: Upgrade: osd.1 is safe to restart 2023-08-28T02:19:35.792 INFO:teuthology.orchestra.run.smithi143.stdout:wait for half of osds to upgrade 2023-08-28T02:19:36.401 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:36 smithi143 ceph-mon[183059]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:36.401 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:36 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:36.401 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:36 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-08-28T02:19:36.401 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:36 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:19:36.402 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:36 smithi143 ceph-mon[189871]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:36.402 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:36 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:36.402 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:36 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-08-28T02:19:36.402 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:36 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:19:36.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:36 smithi169 ceph-mon[159415]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:36.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:36 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:36.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:36 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-08-28T02:19:36.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:36 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:19:36.499 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -e sha1=9cf208207dfe23ffccb33308b22e51570cb45e51 -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2023-08-28T02:19:37.428 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:37 smithi143 ceph-mon[183059]: Upgrade: Updating osd.1 2023-08-28T02:19:37.428 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:37 smithi143 ceph-mon[183059]: Deploying daemon osd.1 on smithi143 2023-08-28T02:19:37.429 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:37 smithi143 ceph-mon[189871]: Upgrade: Updating osd.1 2023-08-28T02:19:37.429 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:37 smithi143 ceph-mon[189871]: Deploying daemon osd.1 on smithi143 2023-08-28T02:19:37.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:37 smithi169 ceph-mon[159415]: Upgrade: Updating osd.1 2023-08-28T02:19:37.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:37 smithi169 ceph-mon[159415]: Deploying daemon osd.1 on smithi143 2023-08-28T02:19:38.423 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:38 smithi143 ceph-mon[183059]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:38.423 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:38 smithi143 ceph-mon[189871]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:38.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:38 smithi169 ceph-mon[159415]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:38.950 INFO:journalctl@ceph.osd.1.smithi143.stdout:Aug 28 02:19:38 smithi143 systemd[1]: Stopping Ceph osd.1 for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:19:39.394 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:39 smithi143 ceph-mon[183059]: osd.1 marked itself down and dead 2023-08-28T02:19:39.394 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:39 smithi143 ceph-mon[189871]: osd.1 marked itself down and dead 2023-08-28T02:19:39.394 INFO:journalctl@ceph.osd.1.smithi143.stdout:Aug 28 02:19:39 smithi143 conmon[120884]: debug 2023-08-28T02:19:39.123+0000 7f6413208700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-08-28T02:19:39.394 INFO:journalctl@ceph.osd.1.smithi143.stdout:Aug 28 02:19:39 smithi143 conmon[120884]: debug 2023-08-28T02:19:39.123+0000 7f6413208700 -1 osd.1 91 *** Got signal Terminated *** 2023-08-28T02:19:39.395 INFO:journalctl@ceph.osd.1.smithi143.stdout:Aug 28 02:19:39 smithi143 conmon[120884]: debug 2023-08-28T02:19:39.123+0000 7f6413208700 -1 osd.1 91 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-08-28T02:19:39.395 INFO:journalctl@ceph.osd.1.smithi143.stdout:Aug 28 02:19:39 smithi143 bash[204545]: f27082b596502e3259af7483879c630766acda5b8556203649d73a9db2703f36 2023-08-28T02:19:39.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:39 smithi169 ceph-mon[159415]: osd.1 marked itself down and dead 2023-08-28T02:19:40.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:40 smithi143 ceph-mon[183059]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:40.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:40 smithi143 ceph-mon[183059]: Health check failed: 1 osds down (OSD_DOWN) 2023-08-28T02:19:40.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:40 smithi143 ceph-mon[183059]: failed to encode map e92 with expected crc 2023-08-28T02:19:40.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:40 smithi143 ceph-mon[183059]: failed to encode map e92 with expected crc 2023-08-28T02:19:40.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:40 smithi143 ceph-mon[183059]: failed to encode map e92 with expected crc 2023-08-28T02:19:40.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:40 smithi143 ceph-mon[183059]: osdmap e92: 8 total, 7 up, 8 in 2023-08-28T02:19:40.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:40 smithi143 ceph-mon[183059]: failed to encode map e92 with expected crc 2023-08-28T02:19:40.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:40 smithi143 ceph-mon[183059]: failed to encode map e92 with expected crc 2023-08-28T02:19:40.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:40 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/689149759' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:19:40.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:40 smithi143 ceph-mon[189871]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:40.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:40 smithi143 ceph-mon[189871]: Health check failed: 1 osds down (OSD_DOWN) 2023-08-28T02:19:40.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:40 smithi143 ceph-mon[189871]: failed to encode map e92 with expected crc 2023-08-28T02:19:40.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:40 smithi143 ceph-mon[189871]: failed to encode map e92 with expected crc 2023-08-28T02:19:40.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:40 smithi143 ceph-mon[189871]: failed to encode map e92 with expected crc 2023-08-28T02:19:40.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:40 smithi143 ceph-mon[189871]: osdmap e92: 8 total, 7 up, 8 in 2023-08-28T02:19:40.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:40 smithi143 ceph-mon[189871]: failed to encode map e92 with expected crc 2023-08-28T02:19:40.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:40 smithi143 ceph-mon[189871]: failed to encode map e92 with expected crc 2023-08-28T02:19:40.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:40 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/689149759' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:19:40.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:40 smithi169 ceph-mon[159415]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:40.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:40 smithi169 ceph-mon[159415]: Health check failed: 1 osds down (OSD_DOWN) 2023-08-28T02:19:40.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:40 smithi169 ceph-mon[159415]: failed to encode map e92 with expected crc 2023-08-28T02:19:40.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:40 smithi169 ceph-mon[159415]: failed to encode map e92 with expected crc 2023-08-28T02:19:40.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:40 smithi169 ceph-mon[159415]: failed to encode map e92 with expected crc 2023-08-28T02:19:40.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:40 smithi169 ceph-mon[159415]: osdmap e92: 8 total, 7 up, 8 in 2023-08-28T02:19:40.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:40 smithi169 ceph-mon[159415]: failed to encode map e92 with expected crc 2023-08-28T02:19:40.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:40 smithi169 ceph-mon[159415]: failed to encode map e92 with expected crc 2023-08-28T02:19:40.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:40 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/689149759' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:19:41.451 INFO:journalctl@ceph.osd.1.smithi143.stdout:Aug 28 02:19:41 smithi143 systemd[1]: ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@osd.1.service: Succeeded. 2023-08-28T02:19:41.451 INFO:journalctl@ceph.osd.1.smithi143.stdout:Aug 28 02:19:41 smithi143 systemd[1]: Stopped Ceph osd.1 for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:19:41.451 INFO:journalctl@ceph.osd.1.smithi143.stdout:Aug 28 02:19:41 smithi143 systemd[1]: Starting Ceph osd.1 for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:19:41.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:41 smithi143 ceph-mon[183059]: failed to encode map e92 with expected crc 2023-08-28T02:19:41.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:41 smithi143 ceph-mon[183059]: failed to encode map e93 with expected crc 2023-08-28T02:19:41.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:41 smithi143 ceph-mon[183059]: osdmap e93: 8 total, 7 up, 8 in 2023-08-28T02:19:41.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:41 smithi143 ceph-mon[183059]: failed to encode map e93 with expected crc 2023-08-28T02:19:41.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:41 smithi143 ceph-mon[183059]: failed to encode map e93 with expected crc 2023-08-28T02:19:41.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:41 smithi143 ceph-mon[183059]: failed to encode map e93 with expected crc 2023-08-28T02:19:41.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:41 smithi143 ceph-mon[183059]: failed to encode map e93 with expected crc 2023-08-28T02:19:41.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:41 smithi143 ceph-mon[183059]: failed to encode map e93 with expected crc 2023-08-28T02:19:41.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:41 smithi143 ceph-mon[183059]: failed to encode map e93 with expected crc 2023-08-28T02:19:41.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:41 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [3, 5]}]: dispatch 2023-08-28T02:19:41.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:41 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [4, 5]}]: dispatch 2023-08-28T02:19:41.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:41 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [4, 0]}]: dispatch 2023-08-28T02:19:41.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:41 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [3, 0]}]: dispatch 2023-08-28T02:19:41.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:41 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 1]}]: dispatch 2023-08-28T02:19:41.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:41 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [3, 2]}]: dispatch 2023-08-28T02:19:41.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:41 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 2]}]: dispatch 2023-08-28T02:19:41.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:41 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.a"}]: dispatch 2023-08-28T02:19:41.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:41 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.13"}]: dispatch 2023-08-28T02:19:41.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:41 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.1c"}]: dispatch 2023-08-28T02:19:41.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:41 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:41.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:41 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:19:41.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:41 smithi143 ceph-mon[189871]: failed to encode map e92 with expected crc 2023-08-28T02:19:41.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:41 smithi143 ceph-mon[189871]: failed to encode map e93 with expected crc 2023-08-28T02:19:41.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:41 smithi143 ceph-mon[189871]: osdmap e93: 8 total, 7 up, 8 in 2023-08-28T02:19:41.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:41 smithi143 ceph-mon[189871]: failed to encode map e93 with expected crc 2023-08-28T02:19:41.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:41 smithi143 ceph-mon[189871]: failed to encode map e93 with expected crc 2023-08-28T02:19:41.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:41 smithi143 ceph-mon[189871]: failed to encode map e93 with expected crc 2023-08-28T02:19:41.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:41 smithi143 ceph-mon[189871]: failed to encode map e93 with expected crc 2023-08-28T02:19:41.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:41 smithi143 ceph-mon[189871]: failed to encode map e93 with expected crc 2023-08-28T02:19:41.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:41 smithi143 ceph-mon[189871]: failed to encode map e93 with expected crc 2023-08-28T02:19:41.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:41 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [3, 5]}]: dispatch 2023-08-28T02:19:41.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:41 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [4, 5]}]: dispatch 2023-08-28T02:19:41.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:41 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [4, 0]}]: dispatch 2023-08-28T02:19:41.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:41 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [3, 0]}]: dispatch 2023-08-28T02:19:41.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:41 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 1]}]: dispatch 2023-08-28T02:19:41.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:41 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [3, 2]}]: dispatch 2023-08-28T02:19:41.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:41 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 2]}]: dispatch 2023-08-28T02:19:41.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:41 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.a"}]: dispatch 2023-08-28T02:19:41.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:41 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.13"}]: dispatch 2023-08-28T02:19:41.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:41 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.1c"}]: dispatch 2023-08-28T02:19:41.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:41 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:41.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:41 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:19:41.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:41 smithi169 ceph-mon[159415]: failed to encode map e92 with expected crc 2023-08-28T02:19:41.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:41 smithi169 ceph-mon[159415]: failed to encode map e93 with expected crc 2023-08-28T02:19:41.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:41 smithi169 ceph-mon[159415]: osdmap e93: 8 total, 7 up, 8 in 2023-08-28T02:19:41.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:41 smithi169 ceph-mon[159415]: failed to encode map e93 with expected crc 2023-08-28T02:19:41.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:41 smithi169 ceph-mon[159415]: failed to encode map e93 with expected crc 2023-08-28T02:19:41.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:41 smithi169 ceph-mon[159415]: failed to encode map e93 with expected crc 2023-08-28T02:19:41.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:41 smithi169 ceph-mon[159415]: failed to encode map e93 with expected crc 2023-08-28T02:19:41.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:41 smithi169 ceph-mon[159415]: failed to encode map e93 with expected crc 2023-08-28T02:19:41.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:41 smithi169 ceph-mon[159415]: failed to encode map e93 with expected crc 2023-08-28T02:19:41.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:41 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [3, 5]}]: dispatch 2023-08-28T02:19:41.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:41 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [4, 5]}]: dispatch 2023-08-28T02:19:41.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:41 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [4, 0]}]: dispatch 2023-08-28T02:19:41.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:41 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [3, 0]}]: dispatch 2023-08-28T02:19:41.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:41 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 1]}]: dispatch 2023-08-28T02:19:41.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:41 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [3, 2]}]: dispatch 2023-08-28T02:19:41.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:41 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 2]}]: dispatch 2023-08-28T02:19:41.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:41 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.a"}]: dispatch 2023-08-28T02:19:41.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:41 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.13"}]: dispatch 2023-08-28T02:19:41.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:41 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.1c"}]: dispatch 2023-08-28T02:19:41.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:41 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:41.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:41 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:19:42.170 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:19:41 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:19:41] "GET /metrics HTTP/1.1" 200 34065 "" "Prometheus/2.33.4" 2023-08-28T02:19:42.450 INFO:journalctl@ceph.osd.1.smithi143.stdout:Aug 28 02:19:42 smithi143 bash[205204]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-08-28T02:19:42.451 INFO:journalctl@ceph.osd.1.smithi143.stdout:Aug 28 02:19:42 smithi143 bash[205204]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2023-08-28T02:19:42.451 INFO:journalctl@ceph.osd.1.smithi143.stdout:Aug 28 02:19:42 smithi143 bash[205204]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2023-08-28T02:19:42.451 INFO:journalctl@ceph.osd.1.smithi143.stdout:Aug 28 02:19:42 smithi143 bash[205204]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-08-28T02:19:42.451 INFO:journalctl@ceph.osd.1.smithi143.stdout:Aug 28 02:19:42 smithi143 bash[205204]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2023-08-28T02:19:42.451 INFO:journalctl@ceph.osd.1.smithi143.stdout:Aug 28 02:19:42 smithi143 bash[205204]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-08-28T02:19:42.451 INFO:journalctl@ceph.osd.1.smithi143.stdout:Aug 28 02:19:42 smithi143 bash[205204]: --> ceph-volume raw activate successful for osd ID: 1 2023-08-28T02:19:42.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:42 smithi143 ceph-mon[183059]: failed to encode map e93 with expected crc 2023-08-28T02:19:42.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:42 smithi143 ceph-mon[183059]: failed to encode map e93 with expected crc 2023-08-28T02:19:42.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:42 smithi143 ceph-mon[183059]: pgmap v130: 73 pgs: 10 stale+active+clean, 63 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:42.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:42 smithi143 ceph-mon[183059]: failed to encode map e94 with expected crc 2023-08-28T02:19:42.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:42 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [3, 5]}]': finished 2023-08-28T02:19:42.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:42 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [4, 5]}]': finished 2023-08-28T02:19:42.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:42 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [4, 0]}]': finished 2023-08-28T02:19:42.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:42 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [3, 0]}]': finished 2023-08-28T02:19:42.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:42 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 1]}]': finished 2023-08-28T02:19:42.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:42 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [3, 2]}]': finished 2023-08-28T02:19:42.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:42 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 2]}]': finished 2023-08-28T02:19:42.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:42 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.a"}]': finished 2023-08-28T02:19:42.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:42 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.13"}]': finished 2023-08-28T02:19:42.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:42 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.1c"}]': finished 2023-08-28T02:19:42.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:42 smithi143 ceph-mon[183059]: osdmap e94: 8 total, 7 up, 8 in 2023-08-28T02:19:42.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:42 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1154490847' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:19:42.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:42 smithi143 ceph-mon[189871]: failed to encode map e93 with expected crc 2023-08-28T02:19:42.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:42 smithi143 ceph-mon[189871]: failed to encode map e93 with expected crc 2023-08-28T02:19:42.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:42 smithi143 ceph-mon[189871]: pgmap v130: 73 pgs: 10 stale+active+clean, 63 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:42.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:42 smithi143 ceph-mon[189871]: failed to encode map e94 with expected crc 2023-08-28T02:19:42.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:42 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [3, 5]}]': finished 2023-08-28T02:19:42.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:42 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [4, 5]}]': finished 2023-08-28T02:19:42.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:42 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [4, 0]}]': finished 2023-08-28T02:19:42.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:42 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [3, 0]}]': finished 2023-08-28T02:19:42.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:42 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 1]}]': finished 2023-08-28T02:19:42.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:42 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [3, 2]}]': finished 2023-08-28T02:19:42.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:42 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 2]}]': finished 2023-08-28T02:19:42.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:42 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.a"}]': finished 2023-08-28T02:19:42.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:42 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.13"}]': finished 2023-08-28T02:19:42.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:42 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.1c"}]': finished 2023-08-28T02:19:42.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:42 smithi143 ceph-mon[189871]: osdmap e94: 8 total, 7 up, 8 in 2023-08-28T02:19:42.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:42 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1154490847' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:19:42.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:42 smithi169 ceph-mon[159415]: failed to encode map e93 with expected crc 2023-08-28T02:19:42.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:42 smithi169 ceph-mon[159415]: failed to encode map e93 with expected crc 2023-08-28T02:19:42.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:42 smithi169 ceph-mon[159415]: pgmap v130: 73 pgs: 10 stale+active+clean, 63 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:42.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:42 smithi169 ceph-mon[159415]: failed to encode map e94 with expected crc 2023-08-28T02:19:42.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:42 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [3, 5]}]': finished 2023-08-28T02:19:42.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:42 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [4, 5]}]': finished 2023-08-28T02:19:42.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:42 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [4, 0]}]': finished 2023-08-28T02:19:42.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:42 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [3, 0]}]': finished 2023-08-28T02:19:42.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:42 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 1]}]': finished 2023-08-28T02:19:42.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:42 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [3, 2]}]': finished 2023-08-28T02:19:42.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:42 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 2]}]': finished 2023-08-28T02:19:42.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:42 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.a"}]': finished 2023-08-28T02:19:42.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:42 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.13"}]': finished 2023-08-28T02:19:42.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:42 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.1c"}]': finished 2023-08-28T02:19:42.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:42 smithi169 ceph-mon[159415]: osdmap e94: 8 total, 7 up, 8 in 2023-08-28T02:19:42.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:42 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1154490847' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:19:43.450 INFO:journalctl@ceph.osd.1.smithi143.stdout:Aug 28 02:19:43 smithi143 bash[205650]: a6c1aefbbefc24e9a3e2149a6b0fc0c8d4e3cac37d416f907b60ceb1d1bec8e4 2023-08-28T02:19:43.450 INFO:journalctl@ceph.osd.1.smithi143.stdout:Aug 28 02:19:43 smithi143 ceph-osd[205715]: -- 172.21.15.143:0/2881285977 <== mon.2 v2:172.21.15.143:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x564291ac44e0 con 0x564291a97000 2023-08-28T02:19:43.450 INFO:journalctl@ceph.osd.1.smithi143.stdout:Aug 28 02:19:43 smithi143 systemd[1]: Started Ceph osd.1 for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:19:43.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:43 smithi143 ceph-mon[183059]: failed to encode map e94 with expected crc 2023-08-28T02:19:43.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:43 smithi143 ceph-mon[183059]: failed to encode map e95 with expected crc 2023-08-28T02:19:43.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:43 smithi143 ceph-mon[183059]: failed to encode map e95 with expected crc 2023-08-28T02:19:43.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:43 smithi143 ceph-mon[183059]: osdmap e95: 8 total, 7 up, 8 in 2023-08-28T02:19:43.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:43 smithi143 ceph-mon[183059]: failed to encode map e95 with expected crc 2023-08-28T02:19:43.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:43 smithi143 ceph-mon[183059]: failed to encode map e95 with expected crc 2023-08-28T02:19:43.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:43 smithi143 ceph-mon[183059]: failed to encode map e95 with expected crc 2023-08-28T02:19:43.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:43 smithi143 ceph-mon[183059]: failed to encode map e95 with expected crc 2023-08-28T02:19:43.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:43 smithi143 ceph-mon[189871]: failed to encode map e94 with expected crc 2023-08-28T02:19:43.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:43 smithi143 ceph-mon[189871]: failed to encode map e95 with expected crc 2023-08-28T02:19:43.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:43 smithi143 ceph-mon[189871]: failed to encode map e95 with expected crc 2023-08-28T02:19:43.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:43 smithi143 ceph-mon[189871]: osdmap e95: 8 total, 7 up, 8 in 2023-08-28T02:19:43.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:43 smithi143 ceph-mon[189871]: failed to encode map e95 with expected crc 2023-08-28T02:19:43.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:43 smithi143 ceph-mon[189871]: failed to encode map e95 with expected crc 2023-08-28T02:19:43.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:43 smithi143 ceph-mon[189871]: failed to encode map e95 with expected crc 2023-08-28T02:19:43.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:43 smithi143 ceph-mon[189871]: failed to encode map e95 with expected crc 2023-08-28T02:19:43.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:43 smithi169 ceph-mon[159415]: failed to encode map e94 with expected crc 2023-08-28T02:19:43.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:43 smithi169 ceph-mon[159415]: failed to encode map e95 with expected crc 2023-08-28T02:19:43.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:43 smithi169 ceph-mon[159415]: failed to encode map e95 with expected crc 2023-08-28T02:19:43.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:43 smithi169 ceph-mon[159415]: osdmap e95: 8 total, 7 up, 8 in 2023-08-28T02:19:43.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:43 smithi169 ceph-mon[159415]: failed to encode map e95 with expected crc 2023-08-28T02:19:43.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:43 smithi169 ceph-mon[159415]: failed to encode map e95 with expected crc 2023-08-28T02:19:43.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:43 smithi169 ceph-mon[159415]: failed to encode map e95 with expected crc 2023-08-28T02:19:43.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:43 smithi169 ceph-mon[159415]: failed to encode map e95 with expected crc 2023-08-28T02:19:44.172 INFO:journalctl@ceph.osd.1.smithi143.stdout:Aug 28 02:19:43 smithi143 conmon[205691]: 2023-08-28T02:19:43.899+0000 7f4b3839e640 -1 Falling back to public interface 2023-08-28T02:19:44.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:44 smithi143 ceph-mon[189871]: failed to encode map e95 with expected crc 2023-08-28T02:19:44.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:44 smithi143 ceph-mon[189871]: pgmap v133: 73 pgs: 3 remapped+peering, 5 peering, 6 active+undersized+degraded, 20 active+undersized, 39 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-08-28T02:19:44.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:44 smithi143 ceph-mon[189871]: Health check failed: Degraded data redundancy: 6/84 objects degraded (7.143%), 6 pgs degraded (PG_DEGRADED) 2023-08-28T02:19:44.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:44 smithi143 ceph-mon[189871]: failed to encode map e96 with expected crc 2023-08-28T02:19:44.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:44 smithi143 ceph-mon[189871]: osdmap e96: 8 total, 7 up, 8 in 2023-08-28T02:19:44.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:44 smithi143 ceph-mon[189871]: failed to encode map e96 with expected crc 2023-08-28T02:19:44.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:44 smithi143 ceph-mon[189871]: failed to encode map e96 with expected crc 2023-08-28T02:19:44.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:44 smithi143 ceph-mon[189871]: failed to encode map e96 with expected crc 2023-08-28T02:19:44.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:44 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:44.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:44 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:44.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:44 smithi143 ceph-mon[189871]: failed to encode map e96 with expected crc 2023-08-28T02:19:44.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:44 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:19:44.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:44 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/799092054' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:19:44.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:44 smithi143 ceph-mon[183059]: failed to encode map e95 with expected crc 2023-08-28T02:19:44.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:44 smithi143 ceph-mon[183059]: pgmap v133: 73 pgs: 3 remapped+peering, 5 peering, 6 active+undersized+degraded, 20 active+undersized, 39 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-08-28T02:19:44.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:44 smithi143 ceph-mon[183059]: Health check failed: Degraded data redundancy: 6/84 objects degraded (7.143%), 6 pgs degraded (PG_DEGRADED) 2023-08-28T02:19:44.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:44 smithi143 ceph-mon[183059]: failed to encode map e96 with expected crc 2023-08-28T02:19:44.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:44 smithi143 ceph-mon[183059]: osdmap e96: 8 total, 7 up, 8 in 2023-08-28T02:19:44.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:44 smithi143 ceph-mon[183059]: failed to encode map e96 with expected crc 2023-08-28T02:19:44.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:44 smithi143 ceph-mon[183059]: failed to encode map e96 with expected crc 2023-08-28T02:19:44.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:44 smithi143 ceph-mon[183059]: failed to encode map e96 with expected crc 2023-08-28T02:19:44.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:44 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:44.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:44 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:44.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:44 smithi143 ceph-mon[183059]: failed to encode map e96 with expected crc 2023-08-28T02:19:44.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:44 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:19:44.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:44 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/799092054' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:19:44.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:44 smithi169 ceph-mon[159415]: failed to encode map e95 with expected crc 2023-08-28T02:19:44.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:44 smithi169 ceph-mon[159415]: pgmap v133: 73 pgs: 3 remapped+peering, 5 peering, 6 active+undersized+degraded, 20 active+undersized, 39 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-08-28T02:19:44.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:44 smithi169 ceph-mon[159415]: Health check failed: Degraded data redundancy: 6/84 objects degraded (7.143%), 6 pgs degraded (PG_DEGRADED) 2023-08-28T02:19:44.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:44 smithi169 ceph-mon[159415]: failed to encode map e96 with expected crc 2023-08-28T02:19:44.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:44 smithi169 ceph-mon[159415]: osdmap e96: 8 total, 7 up, 8 in 2023-08-28T02:19:44.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:44 smithi169 ceph-mon[159415]: failed to encode map e96 with expected crc 2023-08-28T02:19:44.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:44 smithi169 ceph-mon[159415]: failed to encode map e96 with expected crc 2023-08-28T02:19:44.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:44 smithi169 ceph-mon[159415]: failed to encode map e96 with expected crc 2023-08-28T02:19:44.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:44 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:44.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:44 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:44.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:44 smithi169 ceph-mon[159415]: failed to encode map e96 with expected crc 2023-08-28T02:19:44.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:44 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:19:44.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:44 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/799092054' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:19:45.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:45 smithi143 ceph-mon[183059]: failed to encode map e96 with expected crc 2023-08-28T02:19:45.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:45 smithi143 ceph-mon[183059]: failed to encode map e96 with expected crc 2023-08-28T02:19:45.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:45 smithi143 ceph-mon[183059]: failed to encode map e96 with expected crc 2023-08-28T02:19:45.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:45 smithi143 ceph-mon[189871]: failed to encode map e96 with expected crc 2023-08-28T02:19:45.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:45 smithi143 ceph-mon[189871]: failed to encode map e96 with expected crc 2023-08-28T02:19:45.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:45 smithi143 ceph-mon[189871]: failed to encode map e96 with expected crc 2023-08-28T02:19:45.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:45 smithi169 ceph-mon[159415]: failed to encode map e96 with expected crc 2023-08-28T02:19:45.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:45 smithi169 ceph-mon[159415]: failed to encode map e96 with expected crc 2023-08-28T02:19:45.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:45 smithi169 ceph-mon[159415]: failed to encode map e96 with expected crc 2023-08-28T02:19:46.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:46 smithi143 ceph-mon[183059]: pgmap v135: 73 pgs: 3 remapped+peering, 5 peering, 6 active+undersized+degraded, 20 active+undersized, 39 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-08-28T02:19:46.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:46 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:46.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:46 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:46.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:46 smithi143 ceph-mon[189871]: pgmap v135: 73 pgs: 3 remapped+peering, 5 peering, 6 active+undersized+degraded, 20 active+undersized, 39 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-08-28T02:19:46.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:46 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:46.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:46 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:46.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:46 smithi169 ceph-mon[159415]: pgmap v135: 73 pgs: 3 remapped+peering, 5 peering, 6 active+undersized+degraded, 20 active+undersized, 39 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-08-28T02:19:46.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:46 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:46.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:46 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:47.450 INFO:journalctl@ceph.osd.1.smithi143.stdout:Aug 28 02:19:47 smithi143 conmon[205691]: 2023-08-28T02:19:47.189+0000 7f4b3839e640 -1 osd.1 91 log_to_monitors true 2023-08-28T02:19:47.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:47 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2773636530' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:19:47.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:47 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:47.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:47 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:47.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:47 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:47.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:47 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:47.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:47 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2773636530' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:19:47.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:47 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:47.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:47 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:47.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:47 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:47.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:47 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:47.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:47 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2773636530' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:19:47.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:47 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:47.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:47 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:47.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:47 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:47.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:47 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:48.285 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:48 smithi143 ceph-mon[183059]: pgmap v136: 73 pgs: 7 active+clean+remapped, 3 remapped+peering, 6 active+undersized+degraded, 20 active+undersized, 37 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-08-28T02:19:48.285 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:48 smithi143 ceph-mon[183059]: from='osd.1 [v2:172.21.15.143:6810/4171000628,v1:172.21.15.143:6811/4171000628]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-08-28T02:19:48.285 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:48 smithi143 ceph-mon[183059]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-08-28T02:19:48.285 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:48 smithi143 ceph-mon[189871]: pgmap v136: 73 pgs: 7 active+clean+remapped, 3 remapped+peering, 6 active+undersized+degraded, 20 active+undersized, 37 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-08-28T02:19:48.285 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:48 smithi143 ceph-mon[189871]: from='osd.1 [v2:172.21.15.143:6810/4171000628,v1:172.21.15.143:6811/4171000628]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-08-28T02:19:48.286 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:48 smithi143 ceph-mon[189871]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-08-28T02:19:48.286 INFO:journalctl@ceph.osd.1.smithi143.stdout:Aug 28 02:19:48 smithi143 conmon[205691]: 2023-08-28T02:19:48.215+0000 7f4b27806700 -1 osd.1 91 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-08-28T02:19:48.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:48 smithi169 ceph-mon[159415]: pgmap v136: 73 pgs: 7 active+clean+remapped, 3 remapped+peering, 6 active+undersized+degraded, 20 active+undersized, 37 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-08-28T02:19:48.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:48 smithi169 ceph-mon[159415]: from='osd.1 [v2:172.21.15.143:6810/4171000628,v1:172.21.15.143:6811/4171000628]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-08-28T02:19:48.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:48 smithi169 ceph-mon[159415]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-08-28T02:19:49.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:49 smithi143 ceph-mon[183059]: failed to encode map e97 with expected crc 2023-08-28T02:19:49.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:49 smithi143 ceph-mon[183059]: failed to encode map e97 with expected crc 2023-08-28T02:19:49.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:49 smithi143 ceph-mon[183059]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-08-28T02:19:49.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:49 smithi143 ceph-mon[183059]: osdmap e97: 8 total, 7 up, 8 in 2023-08-28T02:19:49.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:49 smithi143 ceph-mon[183059]: from='osd.1 [v2:172.21.15.143:6810/4171000628,v1:172.21.15.143:6811/4171000628]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]: dispatch 2023-08-28T02:19:49.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:49 smithi143 ceph-mon[183059]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]: dispatch 2023-08-28T02:19:49.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:49 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2402865818' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:19:49.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:49 smithi143 ceph-mon[189871]: failed to encode map e97 with expected crc 2023-08-28T02:19:49.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:49 smithi143 ceph-mon[189871]: failed to encode map e97 with expected crc 2023-08-28T02:19:49.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:49 smithi143 ceph-mon[189871]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-08-28T02:19:49.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:49 smithi143 ceph-mon[189871]: osdmap e97: 8 total, 7 up, 8 in 2023-08-28T02:19:49.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:49 smithi143 ceph-mon[189871]: from='osd.1 [v2:172.21.15.143:6810/4171000628,v1:172.21.15.143:6811/4171000628]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]: dispatch 2023-08-28T02:19:49.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:49 smithi143 ceph-mon[189871]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]: dispatch 2023-08-28T02:19:49.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:49 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2402865818' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:19:49.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:49 smithi169 ceph-mon[159415]: failed to encode map e97 with expected crc 2023-08-28T02:19:49.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:49 smithi169 ceph-mon[159415]: failed to encode map e97 with expected crc 2023-08-28T02:19:49.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:49 smithi169 ceph-mon[159415]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-08-28T02:19:49.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:49 smithi169 ceph-mon[159415]: osdmap e97: 8 total, 7 up, 8 in 2023-08-28T02:19:49.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:49 smithi169 ceph-mon[159415]: from='osd.1 [v2:172.21.15.143:6810/4171000628,v1:172.21.15.143:6811/4171000628]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]: dispatch 2023-08-28T02:19:49.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:49 smithi169 ceph-mon[159415]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]: dispatch 2023-08-28T02:19:49.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:49 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2402865818' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:19:49.950 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:19:49 smithi143 conmon[176053]: 2023-08-28T02:19:49.631+0000 7f008786b700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2023-08-28T02:19:50.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:50 smithi169 ceph-mon[159415]: failed to encode map e97 with expected crc 2023-08-28T02:19:50.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:50 smithi169 ceph-mon[159415]: pgmap v138: 73 pgs: 10 active+clean+remapped, 6 active+undersized+degraded, 20 active+undersized, 37 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-08-28T02:19:50.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:50 smithi169 ceph-mon[159415]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-08-28T02:19:50.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:50 smithi169 ceph-mon[159415]: failed to encode map e98 with expected crc 2023-08-28T02:19:50.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:50 smithi169 ceph-mon[159415]: failed to encode map e98 with expected crc 2023-08-28T02:19:50.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:50 smithi169 ceph-mon[159415]: osd.1 [v2:172.21.15.143:6810/4171000628,v1:172.21.15.143:6811/4171000628] boot 2023-08-28T02:19:50.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:50 smithi169 ceph-mon[159415]: osdmap e98: 8 total, 8 up, 8 in 2023-08-28T02:19:50.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:50 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-28T02:19:50.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:50 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:50.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:50 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:50.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:50 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:19:50.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:50 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:19:50.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:50 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:50.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:50 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:19:50.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:50 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:19:50.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:50 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:19:50.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:50 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:19:50.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:50 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-08-28T02:19:50.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:50 smithi143 ceph-mon[183059]: failed to encode map e97 with expected crc 2023-08-28T02:19:50.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:50 smithi143 ceph-mon[183059]: pgmap v138: 73 pgs: 10 active+clean+remapped, 6 active+undersized+degraded, 20 active+undersized, 37 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-08-28T02:19:50.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:50 smithi143 ceph-mon[183059]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-08-28T02:19:50.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:50 smithi143 ceph-mon[183059]: failed to encode map e98 with expected crc 2023-08-28T02:19:50.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:50 smithi143 ceph-mon[183059]: failed to encode map e98 with expected crc 2023-08-28T02:19:50.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:50 smithi143 ceph-mon[183059]: osd.1 [v2:172.21.15.143:6810/4171000628,v1:172.21.15.143:6811/4171000628] boot 2023-08-28T02:19:50.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:50 smithi143 ceph-mon[183059]: osdmap e98: 8 total, 8 up, 8 in 2023-08-28T02:19:50.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:50 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-28T02:19:50.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:50 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:50.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:50 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:50.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:50 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:19:50.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:50 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:19:50.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:50 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:50.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:50 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:19:50.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:50 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:19:50.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:50 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:19:50.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:50 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:19:50.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:50 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-08-28T02:19:50.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:50 smithi143 ceph-mon[189871]: failed to encode map e97 with expected crc 2023-08-28T02:19:50.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:50 smithi143 ceph-mon[189871]: pgmap v138: 73 pgs: 10 active+clean+remapped, 6 active+undersized+degraded, 20 active+undersized, 37 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-08-28T02:19:50.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:50 smithi143 ceph-mon[189871]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-08-28T02:19:50.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:50 smithi143 ceph-mon[189871]: failed to encode map e98 with expected crc 2023-08-28T02:19:50.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:50 smithi143 ceph-mon[189871]: failed to encode map e98 with expected crc 2023-08-28T02:19:50.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:50 smithi143 ceph-mon[189871]: osd.1 [v2:172.21.15.143:6810/4171000628,v1:172.21.15.143:6811/4171000628] boot 2023-08-28T02:19:50.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:50 smithi143 ceph-mon[189871]: osdmap e98: 8 total, 8 up, 8 in 2023-08-28T02:19:50.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:50 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-28T02:19:50.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:50 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:50.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:50 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:50.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:50 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:19:50.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:50 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:19:50.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:50 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:50.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:50 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:19:50.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:50 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:19:50.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:50 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:19:50.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:50 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:19:50.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:50 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-08-28T02:19:51.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:51 smithi169 ceph-mon[159415]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-08-28T02:19:51.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:51 smithi169 ceph-mon[159415]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2023-08-28T02:19:51.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:51 smithi169 ceph-mon[159415]: failed to encode map e99 with expected crc 2023-08-28T02:19:51.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:51 smithi169 ceph-mon[159415]: failed to encode map e99 with expected crc 2023-08-28T02:19:51.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:51 smithi169 ceph-mon[159415]: failed to encode map e99 with expected crc 2023-08-28T02:19:51.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:51 smithi169 ceph-mon[159415]: failed to encode map e99 with expected crc 2023-08-28T02:19:51.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:51 smithi169 ceph-mon[159415]: failed to encode map e99 with expected crc 2023-08-28T02:19:51.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:51 smithi169 ceph-mon[159415]: failed to encode map e99 with expected crc 2023-08-28T02:19:51.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:51 smithi169 ceph-mon[159415]: osdmap e99: 8 total, 8 up, 8 in 2023-08-28T02:19:51.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:51 smithi169 ceph-mon[159415]: failed to encode map e99 with expected crc 2023-08-28T02:19:51.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:51 smithi169 ceph-mon[159415]: failed to encode map e99 with expected crc 2023-08-28T02:19:51.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:51 smithi169 ceph-mon[159415]: failed to encode map e99 with expected crc 2023-08-28T02:19:51.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:51 smithi169 ceph-mon[159415]: failed to encode map e99 with expected crc 2023-08-28T02:19:51.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:51 smithi169 ceph-mon[159415]: failed to encode map e99 with expected crc 2023-08-28T02:19:51.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:51 smithi169 ceph-mon[159415]: failed to encode map e99 with expected crc 2023-08-28T02:19:51.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:51 smithi169 ceph-mon[159415]: failed to encode map e99 with expected crc 2023-08-28T02:19:51.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:51 smithi169 ceph-mon[159415]: failed to encode map e99 with expected crc 2023-08-28T02:19:51.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:51 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1198502734' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:19:51.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:51 smithi169 ceph-mon[159415]: pgmap v141: 73 pgs: 10 active+clean+remapped, 6 active+undersized+degraded, 20 active+undersized, 37 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-08-28T02:19:51.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:51 smithi169 ceph-mon[159415]: osdmap e100: 8 total, 8 up, 8 in 2023-08-28T02:19:51.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:51 smithi143 ceph-mon[183059]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-08-28T02:19:51.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:51 smithi143 ceph-mon[183059]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2023-08-28T02:19:51.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:51 smithi143 ceph-mon[183059]: failed to encode map e99 with expected crc 2023-08-28T02:19:51.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:51 smithi143 ceph-mon[183059]: failed to encode map e99 with expected crc 2023-08-28T02:19:51.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:51 smithi143 ceph-mon[183059]: failed to encode map e99 with expected crc 2023-08-28T02:19:51.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:51 smithi143 ceph-mon[183059]: failed to encode map e99 with expected crc 2023-08-28T02:19:51.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:51 smithi143 ceph-mon[183059]: failed to encode map e99 with expected crc 2023-08-28T02:19:51.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:51 smithi143 ceph-mon[183059]: failed to encode map e99 with expected crc 2023-08-28T02:19:51.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:51 smithi143 ceph-mon[183059]: osdmap e99: 8 total, 8 up, 8 in 2023-08-28T02:19:51.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:51 smithi143 ceph-mon[183059]: failed to encode map e99 with expected crc 2023-08-28T02:19:51.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:51 smithi143 ceph-mon[183059]: failed to encode map e99 with expected crc 2023-08-28T02:19:51.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:51 smithi143 ceph-mon[183059]: failed to encode map e99 with expected crc 2023-08-28T02:19:51.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:51 smithi143 ceph-mon[183059]: failed to encode map e99 with expected crc 2023-08-28T02:19:51.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:51 smithi143 ceph-mon[183059]: failed to encode map e99 with expected crc 2023-08-28T02:19:51.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:51 smithi143 ceph-mon[183059]: failed to encode map e99 with expected crc 2023-08-28T02:19:51.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:51 smithi143 ceph-mon[183059]: failed to encode map e99 with expected crc 2023-08-28T02:19:51.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:51 smithi143 ceph-mon[183059]: failed to encode map e99 with expected crc 2023-08-28T02:19:51.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:51 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1198502734' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:19:51.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:51 smithi143 ceph-mon[183059]: pgmap v141: 73 pgs: 10 active+clean+remapped, 6 active+undersized+degraded, 20 active+undersized, 37 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-08-28T02:19:51.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:51 smithi143 ceph-mon[183059]: osdmap e100: 8 total, 8 up, 8 in 2023-08-28T02:19:51.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:51 smithi143 ceph-mon[189871]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-08-28T02:19:51.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:51 smithi143 ceph-mon[189871]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2023-08-28T02:19:51.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:51 smithi143 ceph-mon[189871]: failed to encode map e99 with expected crc 2023-08-28T02:19:51.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:51 smithi143 ceph-mon[189871]: failed to encode map e99 with expected crc 2023-08-28T02:19:51.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:51 smithi143 ceph-mon[189871]: failed to encode map e99 with expected crc 2023-08-28T02:19:51.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:51 smithi143 ceph-mon[189871]: failed to encode map e99 with expected crc 2023-08-28T02:19:51.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:51 smithi143 ceph-mon[189871]: failed to encode map e99 with expected crc 2023-08-28T02:19:51.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:51 smithi143 ceph-mon[189871]: failed to encode map e99 with expected crc 2023-08-28T02:19:51.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:51 smithi143 ceph-mon[189871]: osdmap e99: 8 total, 8 up, 8 in 2023-08-28T02:19:51.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:51 smithi143 ceph-mon[189871]: failed to encode map e99 with expected crc 2023-08-28T02:19:51.706 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:51 smithi143 ceph-mon[189871]: failed to encode map e99 with expected crc 2023-08-28T02:19:51.706 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:51 smithi143 ceph-mon[189871]: failed to encode map e99 with expected crc 2023-08-28T02:19:51.706 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:51 smithi143 ceph-mon[189871]: failed to encode map e99 with expected crc 2023-08-28T02:19:51.706 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:51 smithi143 ceph-mon[189871]: failed to encode map e99 with expected crc 2023-08-28T02:19:51.706 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:51 smithi143 ceph-mon[189871]: failed to encode map e99 with expected crc 2023-08-28T02:19:51.706 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:51 smithi143 ceph-mon[189871]: failed to encode map e99 with expected crc 2023-08-28T02:19:51.706 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:51 smithi143 ceph-mon[189871]: failed to encode map e99 with expected crc 2023-08-28T02:19:51.706 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:51 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1198502734' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:19:51.706 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:51 smithi143 ceph-mon[189871]: pgmap v141: 73 pgs: 10 active+clean+remapped, 6 active+undersized+degraded, 20 active+undersized, 37 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-08-28T02:19:51.706 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:51 smithi143 ceph-mon[189871]: osdmap e100: 8 total, 8 up, 8 in 2023-08-28T02:19:52.201 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:19:51 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:19:51] "GET /metrics HTTP/1.1" 200 34065 "" "Prometheus/2.33.4" 2023-08-28T02:19:52.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:52 smithi169 ceph-mon[159415]: failed to encode map e100 with expected crc 2023-08-28T02:19:52.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:52 smithi169 ceph-mon[159415]: failed to encode map e100 with expected crc 2023-08-28T02:19:52.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:52 smithi169 ceph-mon[159415]: failed to encode map e100 with expected crc 2023-08-28T02:19:52.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:52 smithi169 ceph-mon[159415]: failed to encode map e100 with expected crc 2023-08-28T02:19:52.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:52 smithi169 ceph-mon[159415]: failed to encode map e100 with expected crc 2023-08-28T02:19:52.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:52 smithi169 ceph-mon[159415]: failed to encode map e100 with expected crc 2023-08-28T02:19:52.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:52 smithi169 ceph-mon[159415]: failed to encode map e100 with expected crc 2023-08-28T02:19:52.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:52 smithi143 ceph-mon[183059]: failed to encode map e100 with expected crc 2023-08-28T02:19:52.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:52 smithi143 ceph-mon[183059]: failed to encode map e100 with expected crc 2023-08-28T02:19:52.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:52 smithi143 ceph-mon[183059]: failed to encode map e100 with expected crc 2023-08-28T02:19:52.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:52 smithi143 ceph-mon[183059]: failed to encode map e100 with expected crc 2023-08-28T02:19:52.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:52 smithi143 ceph-mon[183059]: failed to encode map e100 with expected crc 2023-08-28T02:19:52.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:52 smithi143 ceph-mon[183059]: failed to encode map e100 with expected crc 2023-08-28T02:19:52.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:52 smithi143 ceph-mon[183059]: failed to encode map e100 with expected crc 2023-08-28T02:19:52.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:52 smithi143 ceph-mon[189871]: failed to encode map e100 with expected crc 2023-08-28T02:19:52.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:52 smithi143 ceph-mon[189871]: failed to encode map e100 with expected crc 2023-08-28T02:19:52.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:52 smithi143 ceph-mon[189871]: failed to encode map e100 with expected crc 2023-08-28T02:19:52.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:52 smithi143 ceph-mon[189871]: failed to encode map e100 with expected crc 2023-08-28T02:19:52.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:52 smithi143 ceph-mon[189871]: failed to encode map e100 with expected crc 2023-08-28T02:19:52.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:52 smithi143 ceph-mon[189871]: failed to encode map e100 with expected crc 2023-08-28T02:19:52.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:52 smithi143 ceph-mon[189871]: failed to encode map e100 with expected crc 2023-08-28T02:19:53.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:53 smithi169 ceph-mon[159415]: pgmap v143: 73 pgs: 1 active+clean+remapped, 1 active+undersized+degraded, 8 active+undersized, 63 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-08-28T02:19:53.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:53 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/259849324' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:19:53.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:53 smithi143 ceph-mon[183059]: pgmap v143: 73 pgs: 1 active+clean+remapped, 1 active+undersized+degraded, 8 active+undersized, 63 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-08-28T02:19:53.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:53 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/259849324' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:19:53.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:53 smithi143 ceph-mon[189871]: pgmap v143: 73 pgs: 1 active+clean+remapped, 1 active+undersized+degraded, 8 active+undersized, 63 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-08-28T02:19:53.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:53 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/259849324' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:19:54.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:54 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:19:54.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:54 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:19:54.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:54 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:19:55.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:55 smithi169 ceph-mon[159415]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:55.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:55 smithi143 ceph-mon[183059]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:55.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:55 smithi143 ceph-mon[189871]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:56.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:56 smithi169 ceph-mon[159415]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded) 2023-08-28T02:19:56.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:56 smithi169 ceph-mon[159415]: Cluster is now healthy 2023-08-28T02:19:56.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:56 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/346607562' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:19:56.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:56 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:56.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:56 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:19:56.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:56 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:56.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:56 smithi143 ceph-mon[183059]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded) 2023-08-28T02:19:56.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:56 smithi143 ceph-mon[183059]: Cluster is now healthy 2023-08-28T02:19:56.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:56 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/346607562' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:19:56.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:56 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:56.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:56 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:19:56.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:56 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:56.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:56 smithi143 ceph-mon[189871]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded) 2023-08-28T02:19:56.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:56 smithi143 ceph-mon[189871]: Cluster is now healthy 2023-08-28T02:19:56.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:56 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/346607562' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:19:56.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:56 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:56.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:56 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:19:56.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:56 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:19:57.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:57 smithi169 ceph-mon[159415]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:57.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:57 smithi143 ceph-mon[183059]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:57.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:57 smithi143 ceph-mon[189871]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:58.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:58 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3699805157' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:19:58.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:58 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3699805157' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:19:58.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:58 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3699805157' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:19:59.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:19:59 smithi143 ceph-mon[183059]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:59.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:19:59 smithi143 ceph-mon[189871]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:19:59.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:19:59 smithi169 ceph-mon[159415]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:20:00.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:00 smithi143 ceph-mon[183059]: overall HEALTH_OK 2023-08-28T02:20:00.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:00 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3050800678' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:00.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:00 smithi143 ceph-mon[189871]: overall HEALTH_OK 2023-08-28T02:20:00.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:00 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3050800678' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:00.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:00 smithi169 ceph-mon[159415]: overall HEALTH_OK 2023-08-28T02:20:00.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:00 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3050800678' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:02.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:02 smithi143 ceph-mon[183059]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:20:02.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:02 smithi143 ceph-mon[189871]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:20:02.201 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:20:01 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:20:01] "GET /metrics HTTP/1.1" 200 34058 "" "Prometheus/2.33.4" 2023-08-28T02:20:02.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:02 smithi169 ceph-mon[159415]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:20:03.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:03 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/4121486196' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:03.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:03 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/4121486196' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:03.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:03 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/4121486196' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:04.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:04 smithi143 ceph-mon[183059]: pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:20:04.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:04 smithi143 ceph-mon[189871]: pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:20:04.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:04 smithi169 ceph-mon[159415]: pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:20:05.383 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:05 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-08-28T02:20:05.383 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:05 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2465149536' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:05.383 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:05 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-08-28T02:20:05.383 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:05 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2465149536' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:05.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:05 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-08-28T02:20:05.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:05 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2465149536' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:06.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:06 smithi143 ceph-mon[183059]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-08-28T02:20:06.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:06 smithi143 ceph-mon[183059]: Upgrade: osd.2 is safe to restart 2023-08-28T02:20:06.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:06 smithi143 ceph-mon[183059]: pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:20:06.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:06 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:06.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:06 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-08-28T02:20:06.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:06 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:20:06.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:06 smithi143 ceph-mon[189871]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-08-28T02:20:06.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:06 smithi143 ceph-mon[189871]: Upgrade: osd.2 is safe to restart 2023-08-28T02:20:06.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:06 smithi143 ceph-mon[189871]: pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:20:06.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:06 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:06.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:06 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-08-28T02:20:06.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:06 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:20:06.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:06 smithi169 ceph-mon[159415]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-08-28T02:20:06.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:06 smithi169 ceph-mon[159415]: Upgrade: osd.2 is safe to restart 2023-08-28T02:20:06.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:06 smithi169 ceph-mon[159415]: pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:20:06.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:06 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:06.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:06 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-08-28T02:20:06.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:06 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:20:07.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:07 smithi143 ceph-mon[183059]: Upgrade: Updating osd.2 2023-08-28T02:20:07.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:07 smithi143 ceph-mon[183059]: Deploying daemon osd.2 on smithi143 2023-08-28T02:20:07.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:07 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/817744727' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:07.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:07 smithi143 ceph-mon[189871]: Upgrade: Updating osd.2 2023-08-28T02:20:07.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:07 smithi143 ceph-mon[189871]: Deploying daemon osd.2 on smithi143 2023-08-28T02:20:07.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:07 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/817744727' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:07.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:07 smithi169 ceph-mon[159415]: Upgrade: Updating osd.2 2023-08-28T02:20:07.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:07 smithi169 ceph-mon[159415]: Deploying daemon osd.2 on smithi143 2023-08-28T02:20:07.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:07 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/817744727' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:08.418 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:08 smithi143 ceph-mon[183059]: pgmap v150: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:20:08.419 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:08 smithi143 ceph-mon[189871]: pgmap v150: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:20:08.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:08 smithi169 ceph-mon[159415]: pgmap v150: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:20:08.700 INFO:journalctl@ceph.osd.2.smithi143.stdout:Aug 28 02:20:08 smithi143 systemd[1]: Stopping Ceph osd.2 for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:20:09.129 INFO:journalctl@ceph.osd.2.smithi143.stdout:Aug 28 02:20:08 smithi143 conmon[126784]: debug 2023-08-28T02:20:08.843+0000 7f1446267700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-08-28T02:20:09.130 INFO:journalctl@ceph.osd.2.smithi143.stdout:Aug 28 02:20:08 smithi143 conmon[126784]: debug 2023-08-28T02:20:08.843+0000 7f1446267700 -1 osd.2 100 *** Got signal Terminated *** 2023-08-28T02:20:09.130 INFO:journalctl@ceph.osd.2.smithi143.stdout:Aug 28 02:20:08 smithi143 conmon[126784]: debug 2023-08-28T02:20:08.843+0000 7f1446267700 -1 osd.2 100 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-08-28T02:20:09.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:09 smithi143 ceph-mon[189871]: osd.2 marked itself down and dead 2023-08-28T02:20:09.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:09 smithi143 ceph-mon[183059]: osd.2 marked itself down and dead 2023-08-28T02:20:09.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:09 smithi169 ceph-mon[159415]: osd.2 marked itself down and dead 2023-08-28T02:20:09.950 INFO:journalctl@ceph.osd.2.smithi143.stdout:Aug 28 02:20:09 smithi143 bash[209531]: e20c9920166cdda8447cde8cdd6f1aaca90a1b0c5580763282387365a9518066 2023-08-28T02:20:10.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:10 smithi143 ceph-mon[183059]: pgmap v151: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:20:10.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:10 smithi143 ceph-mon[183059]: Health check failed: 1 osds down (OSD_DOWN) 2023-08-28T02:20:10.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:10 smithi143 ceph-mon[183059]: failed to encode map e101 with expected crc 2023-08-28T02:20:10.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:10 smithi143 ceph-mon[183059]: failed to encode map e101 with expected crc 2023-08-28T02:20:10.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:10 smithi143 ceph-mon[183059]: failed to encode map e101 with expected crc 2023-08-28T02:20:10.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:10 smithi143 ceph-mon[183059]: failed to encode map e101 with expected crc 2023-08-28T02:20:10.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:10 smithi143 ceph-mon[183059]: osdmap e101: 8 total, 7 up, 8 in 2023-08-28T02:20:10.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:10 smithi143 ceph-mon[183059]: failed to encode map e101 with expected crc 2023-08-28T02:20:10.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:10 smithi143 ceph-mon[183059]: failed to encode map e101 with expected crc 2023-08-28T02:20:10.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:10 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/4157695464' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:10.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:10 smithi143 ceph-mon[189871]: pgmap v151: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:20:10.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:10 smithi143 ceph-mon[189871]: Health check failed: 1 osds down (OSD_DOWN) 2023-08-28T02:20:10.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:10 smithi143 ceph-mon[189871]: failed to encode map e101 with expected crc 2023-08-28T02:20:10.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:10 smithi143 ceph-mon[189871]: failed to encode map e101 with expected crc 2023-08-28T02:20:10.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:10 smithi143 ceph-mon[189871]: failed to encode map e101 with expected crc 2023-08-28T02:20:10.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:10 smithi143 ceph-mon[189871]: failed to encode map e101 with expected crc 2023-08-28T02:20:10.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:10 smithi143 ceph-mon[189871]: osdmap e101: 8 total, 7 up, 8 in 2023-08-28T02:20:10.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:10 smithi143 ceph-mon[189871]: failed to encode map e101 with expected crc 2023-08-28T02:20:10.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:10 smithi143 ceph-mon[189871]: failed to encode map e101 with expected crc 2023-08-28T02:20:10.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:10 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/4157695464' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:10.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:10 smithi169 ceph-mon[159415]: pgmap v151: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:20:10.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:10 smithi169 ceph-mon[159415]: Health check failed: 1 osds down (OSD_DOWN) 2023-08-28T02:20:10.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:10 smithi169 ceph-mon[159415]: failed to encode map e101 with expected crc 2023-08-28T02:20:10.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:10 smithi169 ceph-mon[159415]: failed to encode map e101 with expected crc 2023-08-28T02:20:10.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:10 smithi169 ceph-mon[159415]: failed to encode map e101 with expected crc 2023-08-28T02:20:10.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:10 smithi169 ceph-mon[159415]: failed to encode map e101 with expected crc 2023-08-28T02:20:10.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:10 smithi169 ceph-mon[159415]: osdmap e101: 8 total, 7 up, 8 in 2023-08-28T02:20:10.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:10 smithi169 ceph-mon[159415]: failed to encode map e101 with expected crc 2023-08-28T02:20:10.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:10 smithi169 ceph-mon[159415]: failed to encode map e101 with expected crc 2023-08-28T02:20:10.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:10 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/4157695464' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:11.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:11 smithi143 ceph-mon[183059]: failed to encode map e102 with expected crc 2023-08-28T02:20:11.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:11 smithi143 ceph-mon[183059]: failed to encode map e102 with expected crc 2023-08-28T02:20:11.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:11 smithi143 ceph-mon[183059]: osdmap e102: 8 total, 7 up, 8 in 2023-08-28T02:20:11.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:11 smithi143 ceph-mon[183059]: failed to encode map e102 with expected crc 2023-08-28T02:20:11.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:11 smithi143 ceph-mon[183059]: failed to encode map e102 with expected crc 2023-08-28T02:20:11.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:11 smithi143 ceph-mon[183059]: failed to encode map e102 with expected crc 2023-08-28T02:20:11.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:11 smithi143 ceph-mon[183059]: failed to encode map e102 with expected crc 2023-08-28T02:20:11.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:11 smithi143 ceph-mon[183059]: failed to encode map e102 with expected crc 2023-08-28T02:20:11.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:11 smithi143 ceph-mon[183059]: failed to encode map e102 with expected crc 2023-08-28T02:20:11.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:11 smithi143 ceph-mon[183059]: failed to encode map e102 with expected crc 2023-08-28T02:20:11.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:11 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:11.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:11 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:20:11.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:11 smithi143 ceph-mon[189871]: failed to encode map e102 with expected crc 2023-08-28T02:20:11.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:11 smithi143 ceph-mon[189871]: failed to encode map e102 with expected crc 2023-08-28T02:20:11.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:11 smithi143 ceph-mon[189871]: osdmap e102: 8 total, 7 up, 8 in 2023-08-28T02:20:11.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:11 smithi143 ceph-mon[189871]: failed to encode map e102 with expected crc 2023-08-28T02:20:11.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:11 smithi143 ceph-mon[189871]: failed to encode map e102 with expected crc 2023-08-28T02:20:11.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:11 smithi143 ceph-mon[189871]: failed to encode map e102 with expected crc 2023-08-28T02:20:11.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:11 smithi143 ceph-mon[189871]: failed to encode map e102 with expected crc 2023-08-28T02:20:11.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:11 smithi143 ceph-mon[189871]: failed to encode map e102 with expected crc 2023-08-28T02:20:11.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:11 smithi143 ceph-mon[189871]: failed to encode map e102 with expected crc 2023-08-28T02:20:11.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:11 smithi143 ceph-mon[189871]: failed to encode map e102 with expected crc 2023-08-28T02:20:11.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:11 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:11.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:11 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:20:11.454 INFO:journalctl@ceph.osd.2.smithi143.stdout:Aug 28 02:20:11 smithi143 systemd[1]: ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@osd.2.service: Succeeded. 2023-08-28T02:20:11.454 INFO:journalctl@ceph.osd.2.smithi143.stdout:Aug 28 02:20:11 smithi143 systemd[1]: Stopped Ceph osd.2 for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:20:11.454 INFO:journalctl@ceph.osd.2.smithi143.stdout:Aug 28 02:20:11 smithi143 systemd[1]: Starting Ceph osd.2 for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:20:11.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:11 smithi169 ceph-mon[159415]: failed to encode map e102 with expected crc 2023-08-28T02:20:11.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:11 smithi169 ceph-mon[159415]: failed to encode map e102 with expected crc 2023-08-28T02:20:11.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:11 smithi169 ceph-mon[159415]: osdmap e102: 8 total, 7 up, 8 in 2023-08-28T02:20:11.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:11 smithi169 ceph-mon[159415]: failed to encode map e102 with expected crc 2023-08-28T02:20:11.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:11 smithi169 ceph-mon[159415]: failed to encode map e102 with expected crc 2023-08-28T02:20:11.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:11 smithi169 ceph-mon[159415]: failed to encode map e102 with expected crc 2023-08-28T02:20:11.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:11 smithi169 ceph-mon[159415]: failed to encode map e102 with expected crc 2023-08-28T02:20:11.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:11 smithi169 ceph-mon[159415]: failed to encode map e102 with expected crc 2023-08-28T02:20:11.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:11 smithi169 ceph-mon[159415]: failed to encode map e102 with expected crc 2023-08-28T02:20:11.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:11 smithi169 ceph-mon[159415]: failed to encode map e102 with expected crc 2023-08-28T02:20:11.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:11 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:11.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:11 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:20:12.145 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:20:11 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:20:11] "GET /metrics HTTP/1.1" 200 34060 "" "Prometheus/2.33.4" 2023-08-28T02:20:12.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:12 smithi143 ceph-mon[183059]: pgmap v154: 73 pgs: 4 stale+active+clean, 69 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:20:12.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:12 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1532307881' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:12.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:12 smithi143 ceph-mon[189871]: pgmap v154: 73 pgs: 4 stale+active+clean, 69 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:20:12.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:12 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1532307881' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:12.451 INFO:journalctl@ceph.osd.2.smithi143.stdout:Aug 28 02:20:12 smithi143 bash[210207]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-08-28T02:20:12.451 INFO:journalctl@ceph.osd.2.smithi143.stdout:Aug 28 02:20:12 smithi143 bash[210207]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2023-08-28T02:20:12.451 INFO:journalctl@ceph.osd.2.smithi143.stdout:Aug 28 02:20:12 smithi143 bash[210207]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2023-08-28T02:20:12.451 INFO:journalctl@ceph.osd.2.smithi143.stdout:Aug 28 02:20:12 smithi143 bash[210207]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-08-28T02:20:12.451 INFO:journalctl@ceph.osd.2.smithi143.stdout:Aug 28 02:20:12 smithi143 bash[210207]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2023-08-28T02:20:12.451 INFO:journalctl@ceph.osd.2.smithi143.stdout:Aug 28 02:20:12 smithi143 bash[210207]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-08-28T02:20:12.451 INFO:journalctl@ceph.osd.2.smithi143.stdout:Aug 28 02:20:12 smithi143 bash[210207]: --> ceph-volume raw activate successful for osd ID: 2 2023-08-28T02:20:12.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:12 smithi169 ceph-mon[159415]: pgmap v154: 73 pgs: 4 stale+active+clean, 69 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:20:12.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:12 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1532307881' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:13.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:13 smithi143 ceph-mon[183059]: Health check failed: Degraded data redundancy: 5/84 objects degraded (5.952%), 3 pgs degraded (PG_DEGRADED) 2023-08-28T02:20:13.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:13 smithi143 ceph-mon[189871]: Health check failed: Degraded data redundancy: 5/84 objects degraded (5.952%), 3 pgs degraded (PG_DEGRADED) 2023-08-28T02:20:13.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:13 smithi169 ceph-mon[159415]: Health check failed: Degraded data redundancy: 5/84 objects degraded (5.952%), 3 pgs degraded (PG_DEGRADED) 2023-08-28T02:20:13.951 INFO:journalctl@ceph.osd.2.smithi143.stdout:Aug 28 02:20:13 smithi143 bash[210629]: edfc41dc7fbfbbf6fc7fb9476094f6f41c7d9855b0a12331f7891ac13efbdf0e 2023-08-28T02:20:13.951 INFO:journalctl@ceph.osd.2.smithi143.stdout:Aug 28 02:20:13 smithi143 ceph-osd[210694]: -- 172.21.15.143:0/587717916 <== mon.2 v2:172.21.15.143:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x560de60904e0 con 0x560de6063400 2023-08-28T02:20:13.951 INFO:journalctl@ceph.osd.2.smithi143.stdout:Aug 28 02:20:13 smithi143 systemd[1]: Started Ceph osd.2 for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:20:14.450 INFO:journalctl@ceph.osd.2.smithi143.stdout:Aug 28 02:20:13 smithi143 conmon[210670]: 2023-08-28T02:20:13.980+0000 7f7447d57640 -1 Falling back to public interface 2023-08-28T02:20:14.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:14 smithi143 ceph-mon[183059]: pgmap v155: 73 pgs: 3 active+undersized+degraded, 2 stale+active+clean, 10 active+undersized, 58 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-08-28T02:20:14.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:14 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:14.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:14 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:14.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:14 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:20:14.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:14 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3246573016' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:14.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:14 smithi143 ceph-mon[189871]: pgmap v155: 73 pgs: 3 active+undersized+degraded, 2 stale+active+clean, 10 active+undersized, 58 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-08-28T02:20:14.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:14 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:14.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:14 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:14.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:14 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:20:14.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:14 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3246573016' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:14.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:14 smithi169 ceph-mon[159415]: pgmap v155: 73 pgs: 3 active+undersized+degraded, 2 stale+active+clean, 10 active+undersized, 58 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-08-28T02:20:14.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:14 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:14.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:14 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:14.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:14 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:20:14.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:14 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3246573016' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:15.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:15 smithi169 ceph-mon[159415]: pgmap v156: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-08-28T02:20:15.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:15 smithi143 ceph-mon[183059]: pgmap v156: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-08-28T02:20:15.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:15 smithi143 ceph-mon[189871]: pgmap v156: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-08-28T02:20:17.200 INFO:journalctl@ceph.osd.2.smithi143.stdout:Aug 28 02:20:16 smithi143 conmon[210670]: 2023-08-28T02:20:16.940+0000 7f7447d57640 -1 osd.2 100 log_to_monitors true 2023-08-28T02:20:17.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:17 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:17.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:17 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:17.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:17 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/346878082' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:17.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:17 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:17.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:17 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:17.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:17 smithi143 ceph-mon[183059]: from='osd.2 [v2:172.21.15.143:6818/1552459422,v1:172.21.15.143:6819/1552459422]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-08-28T02:20:17.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:17 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:17.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:17 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:17.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:17 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/346878082' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:17.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:17 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:17.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:17 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:17.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:17 smithi143 ceph-mon[189871]: from='osd.2 [v2:172.21.15.143:6818/1552459422,v1:172.21.15.143:6819/1552459422]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-08-28T02:20:17.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:17 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:17.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:17 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:17.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:17 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/346878082' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:17.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:17 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:17.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:17 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:17.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:17 smithi169 ceph-mon[159415]: from='osd.2 [v2:172.21.15.143:6818/1552459422,v1:172.21.15.143:6819/1552459422]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-08-28T02:20:18.126 INFO:journalctl@ceph.osd.2.smithi143.stdout:Aug 28 02:20:17 smithi143 conmon[210670]: 2023-08-28T02:20:17.857+0000 7f74391c3700 -1 osd.2 100 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-08-28T02:20:18.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:18 smithi143 ceph-mon[183059]: pgmap v157: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-08-28T02:20:18.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:18 smithi143 ceph-mon[183059]: from='osd.2 [v2:172.21.15.143:6818/1552459422,v1:172.21.15.143:6819/1552459422]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-08-28T02:20:18.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:18 smithi143 ceph-mon[183059]: osdmap e103: 8 total, 7 up, 8 in 2023-08-28T02:20:18.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:18 smithi143 ceph-mon[183059]: from='osd.2 [v2:172.21.15.143:6818/1552459422,v1:172.21.15.143:6819/1552459422]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]: dispatch 2023-08-28T02:20:18.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:18 smithi143 ceph-mon[189871]: pgmap v157: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-08-28T02:20:18.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:18 smithi143 ceph-mon[189871]: from='osd.2 [v2:172.21.15.143:6818/1552459422,v1:172.21.15.143:6819/1552459422]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-08-28T02:20:18.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:18 smithi143 ceph-mon[189871]: osdmap e103: 8 total, 7 up, 8 in 2023-08-28T02:20:18.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:18 smithi143 ceph-mon[189871]: from='osd.2 [v2:172.21.15.143:6818/1552459422,v1:172.21.15.143:6819/1552459422]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]: dispatch 2023-08-28T02:20:18.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:18 smithi169 ceph-mon[159415]: pgmap v157: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-08-28T02:20:18.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:18 smithi169 ceph-mon[159415]: from='osd.2 [v2:172.21.15.143:6818/1552459422,v1:172.21.15.143:6819/1552459422]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-08-28T02:20:18.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:18 smithi169 ceph-mon[159415]: osdmap e103: 8 total, 7 up, 8 in 2023-08-28T02:20:18.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:18 smithi169 ceph-mon[159415]: from='osd.2 [v2:172.21.15.143:6818/1552459422,v1:172.21.15.143:6819/1552459422]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]: dispatch 2023-08-28T02:20:19.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:19 smithi143 ceph-mon[189871]: failed to encode map e103 with expected crc 2023-08-28T02:20:19.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:19 smithi143 ceph-mon[189871]: failed to encode map e103 with expected crc 2023-08-28T02:20:19.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:19 smithi143 ceph-mon[189871]: failed to encode map e103 with expected crc 2023-08-28T02:20:19.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:19 smithi143 ceph-mon[189871]: failed to encode map e103 with expected crc 2023-08-28T02:20:19.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:19 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1191738060' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:19.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:19 smithi143 ceph-mon[189871]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-08-28T02:20:19.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:19 smithi143 ceph-mon[189871]: osd.2 [v2:172.21.15.143:6818/1552459422,v1:172.21.15.143:6819/1552459422] boot 2023-08-28T02:20:19.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:19 smithi143 ceph-mon[189871]: osdmap e104: 8 total, 8 up, 8 in 2023-08-28T02:20:19.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:19 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-28T02:20:19.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:19 smithi143 ceph-mon[183059]: failed to encode map e103 with expected crc 2023-08-28T02:20:19.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:19 smithi143 ceph-mon[183059]: failed to encode map e103 with expected crc 2023-08-28T02:20:19.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:19 smithi143 ceph-mon[183059]: failed to encode map e103 with expected crc 2023-08-28T02:20:19.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:19 smithi143 ceph-mon[183059]: failed to encode map e103 with expected crc 2023-08-28T02:20:19.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:19 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1191738060' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:19.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:19 smithi143 ceph-mon[183059]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-08-28T02:20:19.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:19 smithi143 ceph-mon[183059]: osd.2 [v2:172.21.15.143:6818/1552459422,v1:172.21.15.143:6819/1552459422] boot 2023-08-28T02:20:19.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:19 smithi143 ceph-mon[183059]: osdmap e104: 8 total, 8 up, 8 in 2023-08-28T02:20:19.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:19 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-28T02:20:19.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:19 smithi169 ceph-mon[159415]: failed to encode map e103 with expected crc 2023-08-28T02:20:19.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:19 smithi169 ceph-mon[159415]: failed to encode map e103 with expected crc 2023-08-28T02:20:19.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:19 smithi169 ceph-mon[159415]: failed to encode map e103 with expected crc 2023-08-28T02:20:19.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:19 smithi169 ceph-mon[159415]: failed to encode map e103 with expected crc 2023-08-28T02:20:19.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:19 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1191738060' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:19.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:19 smithi169 ceph-mon[159415]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-08-28T02:20:19.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:19 smithi169 ceph-mon[159415]: osd.2 [v2:172.21.15.143:6818/1552459422,v1:172.21.15.143:6819/1552459422] boot 2023-08-28T02:20:19.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:19 smithi169 ceph-mon[159415]: osdmap e104: 8 total, 8 up, 8 in 2023-08-28T02:20:19.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:19 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-28T02:20:19.950 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:20:19 smithi143 conmon[176053]: 2023-08-28T02:20:19.901+0000 7f008786b700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2023-08-28T02:20:20.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:20 smithi143 ceph-mon[183059]: failed to encode map e104 with expected crc 2023-08-28T02:20:20.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:20 smithi143 ceph-mon[183059]: failed to encode map e104 with expected crc 2023-08-28T02:20:20.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:20 smithi143 ceph-mon[183059]: failed to encode map e104 with expected crc 2023-08-28T02:20:20.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:20 smithi143 ceph-mon[183059]: failed to encode map e104 with expected crc 2023-08-28T02:20:20.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:20 smithi143 ceph-mon[183059]: failed to encode map e104 with expected crc 2023-08-28T02:20:20.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:20 smithi143 ceph-mon[183059]: pgmap v160: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-08-28T02:20:20.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:20 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:20.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:20 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:20.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:20 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:20:20.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:20 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:20:20.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:20 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:20.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:20 smithi143 ceph-mon[183059]: osdmap e105: 8 total, 8 up, 8 in 2023-08-28T02:20:20.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:20 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:20:20.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:20 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:20:20.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:20 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:20:20.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:20 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:20:20.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:20 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-08-28T02:20:20.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:20 smithi143 ceph-mon[189871]: failed to encode map e104 with expected crc 2023-08-28T02:20:20.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:20 smithi143 ceph-mon[189871]: failed to encode map e104 with expected crc 2023-08-28T02:20:20.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:20 smithi143 ceph-mon[189871]: failed to encode map e104 with expected crc 2023-08-28T02:20:20.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:20 smithi143 ceph-mon[189871]: failed to encode map e104 with expected crc 2023-08-28T02:20:20.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:20 smithi143 ceph-mon[189871]: failed to encode map e104 with expected crc 2023-08-28T02:20:20.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:20 smithi143 ceph-mon[189871]: pgmap v160: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-08-28T02:20:20.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:20 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:20.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:20 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:20.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:20 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:20:20.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:20 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:20:20.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:20 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:20.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:20 smithi143 ceph-mon[189871]: osdmap e105: 8 total, 8 up, 8 in 2023-08-28T02:20:20.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:20 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:20:20.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:20 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:20:20.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:20 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:20:20.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:20 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:20:20.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:20 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-08-28T02:20:20.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:20 smithi169 ceph-mon[159415]: failed to encode map e104 with expected crc 2023-08-28T02:20:20.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:20 smithi169 ceph-mon[159415]: failed to encode map e104 with expected crc 2023-08-28T02:20:20.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:20 smithi169 ceph-mon[159415]: failed to encode map e104 with expected crc 2023-08-28T02:20:20.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:20 smithi169 ceph-mon[159415]: failed to encode map e104 with expected crc 2023-08-28T02:20:20.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:20 smithi169 ceph-mon[159415]: failed to encode map e104 with expected crc 2023-08-28T02:20:20.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:20 smithi169 ceph-mon[159415]: pgmap v160: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-08-28T02:20:20.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:20 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:20.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:20 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:20.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:20 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:20:20.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:20 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:20:20.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:20 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:20.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:20 smithi169 ceph-mon[159415]: osdmap e105: 8 total, 8 up, 8 in 2023-08-28T02:20:20.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:20 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:20:20.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:20 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:20:20.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:20 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:20:20.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:20 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:20:20.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:20 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-08-28T02:20:21.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:21 smithi143 ceph-mon[183059]: failed to encode map e105 with expected crc 2023-08-28T02:20:21.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:21 smithi143 ceph-mon[183059]: failed to encode map e105 with expected crc 2023-08-28T02:20:21.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:21 smithi143 ceph-mon[183059]: failed to encode map e105 with expected crc 2023-08-28T02:20:21.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:21 smithi143 ceph-mon[183059]: failed to encode map e105 with expected crc 2023-08-28T02:20:21.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:21 smithi143 ceph-mon[183059]: failed to encode map e105 with expected crc 2023-08-28T02:20:21.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:21 smithi143 ceph-mon[183059]: failed to encode map e105 with expected crc 2023-08-28T02:20:21.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:21 smithi143 ceph-mon[183059]: failed to encode map e105 with expected crc 2023-08-28T02:20:21.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:21 smithi143 ceph-mon[183059]: failed to encode map e105 with expected crc 2023-08-28T02:20:21.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:21 smithi143 ceph-mon[183059]: failed to encode map e105 with expected crc 2023-08-28T02:20:21.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:21 smithi143 ceph-mon[183059]: failed to encode map e105 with expected crc 2023-08-28T02:20:21.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:21 smithi143 ceph-mon[183059]: failed to encode map e105 with expected crc 2023-08-28T02:20:21.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:21 smithi143 ceph-mon[183059]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-08-28T02:20:21.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:21 smithi143 ceph-mon[183059]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2023-08-28T02:20:21.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:21 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/4212017274' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:21.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:21 smithi143 ceph-mon[189871]: failed to encode map e105 with expected crc 2023-08-28T02:20:21.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:21 smithi143 ceph-mon[189871]: failed to encode map e105 with expected crc 2023-08-28T02:20:21.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:21 smithi143 ceph-mon[189871]: failed to encode map e105 with expected crc 2023-08-28T02:20:21.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:21 smithi143 ceph-mon[189871]: failed to encode map e105 with expected crc 2023-08-28T02:20:21.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:21 smithi143 ceph-mon[189871]: failed to encode map e105 with expected crc 2023-08-28T02:20:21.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:21 smithi143 ceph-mon[189871]: failed to encode map e105 with expected crc 2023-08-28T02:20:21.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:21 smithi143 ceph-mon[189871]: failed to encode map e105 with expected crc 2023-08-28T02:20:21.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:21 smithi143 ceph-mon[189871]: failed to encode map e105 with expected crc 2023-08-28T02:20:21.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:21 smithi143 ceph-mon[189871]: failed to encode map e105 with expected crc 2023-08-28T02:20:21.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:21 smithi143 ceph-mon[189871]: failed to encode map e105 with expected crc 2023-08-28T02:20:21.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:21 smithi143 ceph-mon[189871]: failed to encode map e105 with expected crc 2023-08-28T02:20:21.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:21 smithi143 ceph-mon[189871]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-08-28T02:20:21.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:21 smithi143 ceph-mon[189871]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2023-08-28T02:20:21.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:21 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/4212017274' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:21.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:21 smithi169 ceph-mon[159415]: failed to encode map e105 with expected crc 2023-08-28T02:20:21.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:21 smithi169 ceph-mon[159415]: failed to encode map e105 with expected crc 2023-08-28T02:20:21.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:21 smithi169 ceph-mon[159415]: failed to encode map e105 with expected crc 2023-08-28T02:20:21.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:21 smithi169 ceph-mon[159415]: failed to encode map e105 with expected crc 2023-08-28T02:20:21.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:21 smithi169 ceph-mon[159415]: failed to encode map e105 with expected crc 2023-08-28T02:20:21.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:21 smithi169 ceph-mon[159415]: failed to encode map e105 with expected crc 2023-08-28T02:20:21.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:21 smithi169 ceph-mon[159415]: failed to encode map e105 with expected crc 2023-08-28T02:20:21.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:21 smithi169 ceph-mon[159415]: failed to encode map e105 with expected crc 2023-08-28T02:20:21.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:21 smithi169 ceph-mon[159415]: failed to encode map e105 with expected crc 2023-08-28T02:20:21.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:21 smithi169 ceph-mon[159415]: failed to encode map e105 with expected crc 2023-08-28T02:20:21.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:21 smithi169 ceph-mon[159415]: failed to encode map e105 with expected crc 2023-08-28T02:20:21.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:21 smithi169 ceph-mon[159415]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-08-28T02:20:21.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:21 smithi169 ceph-mon[159415]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2023-08-28T02:20:21.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:21 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/4212017274' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:22.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:22 smithi143 ceph-mon[183059]: pgmap v162: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-08-28T02:20:22.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:22 smithi143 ceph-mon[189871]: pgmap v162: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-08-28T02:20:22.201 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:20:21 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:20:21] "GET /metrics HTTP/1.1" 200 34060 "" "Prometheus/2.33.4" 2023-08-28T02:20:22.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:22 smithi169 ceph-mon[159415]: pgmap v162: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-08-28T02:20:23.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:23 smithi143 ceph-mon[183059]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/84 objects degraded (5.952%), 3 pgs degraded) 2023-08-28T02:20:23.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:23 smithi143 ceph-mon[183059]: Cluster is now healthy 2023-08-28T02:20:23.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:23 smithi143 ceph-mon[189871]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/84 objects degraded (5.952%), 3 pgs degraded) 2023-08-28T02:20:23.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:23 smithi143 ceph-mon[189871]: Cluster is now healthy 2023-08-28T02:20:23.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:23 smithi169 ceph-mon[159415]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/84 objects degraded (5.952%), 3 pgs degraded) 2023-08-28T02:20:23.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:23 smithi169 ceph-mon[159415]: Cluster is now healthy 2023-08-28T02:20:24.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:24 smithi143 ceph-mon[183059]: pgmap v163: 73 pgs: 4 active+undersized, 69 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:20:24.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:24 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3471702319' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:24.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:24 smithi143 ceph-mon[189871]: pgmap v163: 73 pgs: 4 active+undersized, 69 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:20:24.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:24 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3471702319' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:24.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:24 smithi169 ceph-mon[159415]: pgmap v163: 73 pgs: 4 active+undersized, 69 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:20:24.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:24 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3471702319' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:26.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:26 smithi143 ceph-mon[183059]: pgmap v164: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:20:26.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:26 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2054096070' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:26.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:26 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:26.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:26 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:20:26.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:26 smithi143 ceph-mon[189871]: pgmap v164: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:20:26.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:26 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2054096070' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:26.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:26 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:26.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:26 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:20:26.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:26 smithi169 ceph-mon[159415]: pgmap v164: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:20:26.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:26 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2054096070' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:26.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:26 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:26.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:26 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:20:28.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:28 smithi143 ceph-mon[183059]: pgmap v165: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:20:28.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:28 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1135355944' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:28.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:28 smithi143 ceph-mon[189871]: pgmap v165: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:20:28.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:28 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1135355944' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:28.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:28 smithi169 ceph-mon[159415]: pgmap v165: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:20:28.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:28 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1135355944' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:30.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:30 smithi143 ceph-mon[183059]: pgmap v166: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:20:30.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:30 smithi143 ceph-mon[189871]: pgmap v166: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:20:30.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:30 smithi169 ceph-mon[159415]: pgmap v166: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:20:31.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:31 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1889741377' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:31.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:31 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:31.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:31 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1889741377' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:31.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:31 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:31.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:31 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1889741377' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:31.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:31 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:32.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:20:31 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:20:31] "GET /metrics HTTP/1.1" 200 34053 "" "Prometheus/2.33.4" 2023-08-28T02:20:32.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:32 smithi169 ceph-mon[159415]: pgmap v167: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:20:32.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:32 smithi143 ceph-mon[183059]: pgmap v167: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:20:32.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:32 smithi143 ceph-mon[189871]: pgmap v167: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:20:33.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:33 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1983681630' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:33.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:33 smithi169 ceph-mon[159415]: pgmap v168: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:20:33.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:33 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1983681630' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:33.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:33 smithi143 ceph-mon[183059]: pgmap v168: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:20:33.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:33 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1983681630' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:33.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:33 smithi143 ceph-mon[189871]: pgmap v168: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:20:35.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:34 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/4151763437' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:35.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:34 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/4151763437' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:35.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:34 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/4151763437' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:35.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:35 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-08-28T02:20:35.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:35 smithi143 ceph-mon[183059]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-08-28T02:20:35.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:35 smithi143 ceph-mon[183059]: Upgrade: osd.3 is safe to restart 2023-08-28T02:20:35.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:35 smithi143 ceph-mon[183059]: pgmap v169: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:20:35.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:35 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-08-28T02:20:35.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:35 smithi143 ceph-mon[189871]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-08-28T02:20:35.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:35 smithi143 ceph-mon[189871]: Upgrade: osd.3 is safe to restart 2023-08-28T02:20:35.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:35 smithi143 ceph-mon[189871]: pgmap v169: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:20:36.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:35 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-08-28T02:20:36.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:35 smithi169 ceph-mon[159415]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-08-28T02:20:36.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:35 smithi169 ceph-mon[159415]: Upgrade: osd.3 is safe to restart 2023-08-28T02:20:36.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:35 smithi169 ceph-mon[159415]: pgmap v169: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:20:37.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:37 smithi143 ceph-mon[189871]: Upgrade: Updating osd.3 2023-08-28T02:20:37.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:37 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:37.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:37 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-08-28T02:20:37.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:37 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:20:37.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:37 smithi143 ceph-mon[189871]: Deploying daemon osd.3 on smithi143 2023-08-28T02:20:37.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:37 smithi143 ceph-mon[189871]: pgmap v170: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:20:37.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:37 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1789710537' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:37.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:37 smithi143 ceph-mon[183059]: Upgrade: Updating osd.3 2023-08-28T02:20:37.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:37 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:37.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:37 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-08-28T02:20:37.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:37 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:20:37.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:37 smithi143 ceph-mon[183059]: Deploying daemon osd.3 on smithi143 2023-08-28T02:20:37.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:37 smithi143 ceph-mon[183059]: pgmap v170: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:20:37.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:37 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1789710537' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:37.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:37 smithi169 ceph-mon[159415]: Upgrade: Updating osd.3 2023-08-28T02:20:37.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:37 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:37.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:37 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-08-28T02:20:37.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:37 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:20:37.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:37 smithi169 ceph-mon[159415]: Deploying daemon osd.3 on smithi143 2023-08-28T02:20:37.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:37 smithi169 ceph-mon[159415]: pgmap v170: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:20:37.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:37 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1789710537' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:39.132 INFO:journalctl@ceph.osd.3.smithi143.stdout:Aug 28 02:20:38 smithi143 systemd[1]: Stopping Ceph osd.3 for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:20:39.450 INFO:journalctl@ceph.osd.3.smithi143.stdout:Aug 28 02:20:39 smithi143 conmon[132739]: debug 2023-08-28T02:20:39.130+0000 7fa1d0825700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-08-28T02:20:39.450 INFO:journalctl@ceph.osd.3.smithi143.stdout:Aug 28 02:20:39 smithi143 conmon[132739]: debug 2023-08-28T02:20:39.130+0000 7fa1d0825700 -1 osd.3 105 *** Got signal Terminated *** 2023-08-28T02:20:39.450 INFO:journalctl@ceph.osd.3.smithi143.stdout:Aug 28 02:20:39 smithi143 conmon[132739]: debug 2023-08-28T02:20:39.130+0000 7fa1d0825700 -1 osd.3 105 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-08-28T02:20:40.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:40 smithi143 ceph-mon[183059]: pgmap v171: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:20:40.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:40 smithi143 ceph-mon[183059]: osd.3 marked itself down and dead 2023-08-28T02:20:40.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:40 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2113623820' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:40.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:40 smithi143 ceph-mon[189871]: pgmap v171: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:20:40.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:40 smithi143 ceph-mon[189871]: osd.3 marked itself down and dead 2023-08-28T02:20:40.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:40 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2113623820' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:40.452 INFO:journalctl@ceph.osd.3.smithi143.stdout:Aug 28 02:20:40 smithi143 bash[214494]: ab95599724c4c58b5887136fbb9a84a3f1eec2c3cc8894d08439ea1ec726eac1 2023-08-28T02:20:40.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:40 smithi169 ceph-mon[159415]: pgmap v171: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:20:40.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:40 smithi169 ceph-mon[159415]: osd.3 marked itself down and dead 2023-08-28T02:20:40.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:40 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2113623820' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:41.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:41 smithi143 ceph-mon[183059]: Health check failed: 1 osds down (OSD_DOWN) 2023-08-28T02:20:41.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:41 smithi143 ceph-mon[183059]: failed to encode map e106 with expected crc 2023-08-28T02:20:41.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:41 smithi143 ceph-mon[183059]: osdmap e106: 8 total, 7 up, 8 in 2023-08-28T02:20:41.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:41 smithi143 ceph-mon[183059]: failed to encode map e106 with expected crc 2023-08-28T02:20:41.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:41 smithi143 ceph-mon[183059]: failed to encode map e106 with expected crc 2023-08-28T02:20:41.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:41 smithi143 ceph-mon[183059]: failed to encode map e106 with expected crc 2023-08-28T02:20:41.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:41 smithi143 ceph-mon[183059]: failed to encode map e106 with expected crc 2023-08-28T02:20:41.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:41 smithi143 ceph-mon[183059]: failed to encode map e106 with expected crc 2023-08-28T02:20:41.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:41 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:41.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:41 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:20:41.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:41 smithi143 ceph-mon[189871]: Health check failed: 1 osds down (OSD_DOWN) 2023-08-28T02:20:41.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:41 smithi143 ceph-mon[189871]: failed to encode map e106 with expected crc 2023-08-28T02:20:41.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:41 smithi143 ceph-mon[189871]: osdmap e106: 8 total, 7 up, 8 in 2023-08-28T02:20:41.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:41 smithi143 ceph-mon[189871]: failed to encode map e106 with expected crc 2023-08-28T02:20:41.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:41 smithi143 ceph-mon[189871]: failed to encode map e106 with expected crc 2023-08-28T02:20:41.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:41 smithi143 ceph-mon[189871]: failed to encode map e106 with expected crc 2023-08-28T02:20:41.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:41 smithi143 ceph-mon[189871]: failed to encode map e106 with expected crc 2023-08-28T02:20:41.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:41 smithi143 ceph-mon[189871]: failed to encode map e106 with expected crc 2023-08-28T02:20:41.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:41 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:41.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:41 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:20:41.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:41 smithi169 ceph-mon[159415]: Health check failed: 1 osds down (OSD_DOWN) 2023-08-28T02:20:41.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:41 smithi169 ceph-mon[159415]: failed to encode map e106 with expected crc 2023-08-28T02:20:41.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:41 smithi169 ceph-mon[159415]: osdmap e106: 8 total, 7 up, 8 in 2023-08-28T02:20:41.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:41 smithi169 ceph-mon[159415]: failed to encode map e106 with expected crc 2023-08-28T02:20:41.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:41 smithi169 ceph-mon[159415]: failed to encode map e106 with expected crc 2023-08-28T02:20:41.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:41 smithi169 ceph-mon[159415]: failed to encode map e106 with expected crc 2023-08-28T02:20:41.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:41 smithi169 ceph-mon[159415]: failed to encode map e106 with expected crc 2023-08-28T02:20:41.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:41 smithi169 ceph-mon[159415]: failed to encode map e106 with expected crc 2023-08-28T02:20:41.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:41 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:41.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:41 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:20:42.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:20:41 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:20:41] "GET /metrics HTTP/1.1" 200 34052 "" "Prometheus/2.33.4" 2023-08-28T02:20:42.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:42 smithi143 ceph-mon[183059]: pgmap v173: 73 pgs: 10 stale+active+clean, 63 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:20:42.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:42 smithi143 ceph-mon[183059]: failed to encode map e107 with expected crc 2023-08-28T02:20:42.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:42 smithi143 ceph-mon[183059]: osdmap e107: 8 total, 7 up, 8 in 2023-08-28T02:20:42.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:42 smithi143 ceph-mon[183059]: failed to encode map e107 with expected crc 2023-08-28T02:20:42.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:42 smithi143 ceph-mon[183059]: failed to encode map e107 with expected crc 2023-08-28T02:20:42.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:42 smithi143 ceph-mon[183059]: failed to encode map e107 with expected crc 2023-08-28T02:20:42.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:42 smithi143 ceph-mon[183059]: failed to encode map e107 with expected crc 2023-08-28T02:20:42.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:42 smithi143 ceph-mon[183059]: failed to encode map e107 with expected crc 2023-08-28T02:20:42.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:42 smithi143 ceph-mon[183059]: failed to encode map e107 with expected crc 2023-08-28T02:20:42.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:42 smithi143 ceph-mon[183059]: failed to encode map e107 with expected crc 2023-08-28T02:20:42.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:42 smithi143 ceph-mon[183059]: failed to encode map e107 with expected crc 2023-08-28T02:20:42.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:42 smithi143 ceph-mon[183059]: failed to encode map e107 with expected crc 2023-08-28T02:20:42.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:42 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.9", "id": [4, 3]}]: dispatch 2023-08-28T02:20:42.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:42 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 3]}]: dispatch 2023-08-28T02:20:42.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:42 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [4, 2]}]: dispatch 2023-08-28T02:20:42.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:42 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 3]}]: dispatch 2023-08-28T02:20:42.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:42 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 3]}]: dispatch 2023-08-28T02:20:42.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:42 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 3]}]: dispatch 2023-08-28T02:20:42.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:42 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 3]}]: dispatch 2023-08-28T02:20:42.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:42 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 3]}]: dispatch 2023-08-28T02:20:42.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:42 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 3]}]: dispatch 2023-08-28T02:20:42.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:42 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.15"}]: dispatch 2023-08-28T02:20:42.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:42 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2575551417' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:42.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:42 smithi143 ceph-mon[189871]: pgmap v173: 73 pgs: 10 stale+active+clean, 63 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:20:42.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:42 smithi143 ceph-mon[189871]: failed to encode map e107 with expected crc 2023-08-28T02:20:42.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:42 smithi143 ceph-mon[189871]: osdmap e107: 8 total, 7 up, 8 in 2023-08-28T02:20:42.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:42 smithi143 ceph-mon[189871]: failed to encode map e107 with expected crc 2023-08-28T02:20:42.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:42 smithi143 ceph-mon[189871]: failed to encode map e107 with expected crc 2023-08-28T02:20:42.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:42 smithi143 ceph-mon[189871]: failed to encode map e107 with expected crc 2023-08-28T02:20:42.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:42 smithi143 ceph-mon[189871]: failed to encode map e107 with expected crc 2023-08-28T02:20:42.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:42 smithi143 ceph-mon[189871]: failed to encode map e107 with expected crc 2023-08-28T02:20:42.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:42 smithi143 ceph-mon[189871]: failed to encode map e107 with expected crc 2023-08-28T02:20:42.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:42 smithi143 ceph-mon[189871]: failed to encode map e107 with expected crc 2023-08-28T02:20:42.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:42 smithi143 ceph-mon[189871]: failed to encode map e107 with expected crc 2023-08-28T02:20:42.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:42 smithi143 ceph-mon[189871]: failed to encode map e107 with expected crc 2023-08-28T02:20:42.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:42 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.9", "id": [4, 3]}]: dispatch 2023-08-28T02:20:42.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:42 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 3]}]: dispatch 2023-08-28T02:20:42.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:42 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [4, 2]}]: dispatch 2023-08-28T02:20:42.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:42 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 3]}]: dispatch 2023-08-28T02:20:42.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:42 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 3]}]: dispatch 2023-08-28T02:20:42.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:42 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 3]}]: dispatch 2023-08-28T02:20:42.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:42 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 3]}]: dispatch 2023-08-28T02:20:42.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:42 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 3]}]: dispatch 2023-08-28T02:20:42.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:42 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 3]}]: dispatch 2023-08-28T02:20:42.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:42 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.15"}]: dispatch 2023-08-28T02:20:42.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:42 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2575551417' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:42.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:42 smithi169 ceph-mon[159415]: pgmap v173: 73 pgs: 10 stale+active+clean, 63 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:20:42.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:42 smithi169 ceph-mon[159415]: failed to encode map e107 with expected crc 2023-08-28T02:20:42.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:42 smithi169 ceph-mon[159415]: osdmap e107: 8 total, 7 up, 8 in 2023-08-28T02:20:42.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:42 smithi169 ceph-mon[159415]: failed to encode map e107 with expected crc 2023-08-28T02:20:42.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:42 smithi169 ceph-mon[159415]: failed to encode map e107 with expected crc 2023-08-28T02:20:42.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:42 smithi169 ceph-mon[159415]: failed to encode map e107 with expected crc 2023-08-28T02:20:42.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:42 smithi169 ceph-mon[159415]: failed to encode map e107 with expected crc 2023-08-28T02:20:42.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:42 smithi169 ceph-mon[159415]: failed to encode map e107 with expected crc 2023-08-28T02:20:42.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:42 smithi169 ceph-mon[159415]: failed to encode map e107 with expected crc 2023-08-28T02:20:42.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:42 smithi169 ceph-mon[159415]: failed to encode map e107 with expected crc 2023-08-28T02:20:42.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:42 smithi169 ceph-mon[159415]: failed to encode map e107 with expected crc 2023-08-28T02:20:42.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:42 smithi169 ceph-mon[159415]: failed to encode map e107 with expected crc 2023-08-28T02:20:42.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:42 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.9", "id": [4, 3]}]: dispatch 2023-08-28T02:20:42.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:42 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 3]}]: dispatch 2023-08-28T02:20:42.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:42 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [4, 2]}]: dispatch 2023-08-28T02:20:42.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:42 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 3]}]: dispatch 2023-08-28T02:20:42.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:42 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 3]}]: dispatch 2023-08-28T02:20:42.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:42 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 3]}]: dispatch 2023-08-28T02:20:42.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:42 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 3]}]: dispatch 2023-08-28T02:20:42.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:42 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 3]}]: dispatch 2023-08-28T02:20:42.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:42 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 3]}]: dispatch 2023-08-28T02:20:42.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:42 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.15"}]: dispatch 2023-08-28T02:20:42.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:42 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2575551417' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:42.700 INFO:journalctl@ceph.osd.3.smithi143.stdout:Aug 28 02:20:42 smithi143 systemd[1]: ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@osd.3.service: Succeeded. 2023-08-28T02:20:42.701 INFO:journalctl@ceph.osd.3.smithi143.stdout:Aug 28 02:20:42 smithi143 systemd[1]: Stopped Ceph osd.3 for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:20:42.701 INFO:journalctl@ceph.osd.3.smithi143.stdout:Aug 28 02:20:42 smithi143 systemd[1]: Starting Ceph osd.3 for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:20:43.451 INFO:journalctl@ceph.osd.3.smithi143.stdout:Aug 28 02:20:43 smithi143 bash[215184]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-08-28T02:20:43.451 INFO:journalctl@ceph.osd.3.smithi143.stdout:Aug 28 02:20:43 smithi143 bash[215184]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2023-08-28T02:20:43.451 INFO:journalctl@ceph.osd.3.smithi143.stdout:Aug 28 02:20:43 smithi143 bash[215184]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2023-08-28T02:20:43.451 INFO:journalctl@ceph.osd.3.smithi143.stdout:Aug 28 02:20:43 smithi143 bash[215184]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2023-08-28T02:20:43.451 INFO:journalctl@ceph.osd.3.smithi143.stdout:Aug 28 02:20:43 smithi143 bash[215184]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2023-08-28T02:20:43.451 INFO:journalctl@ceph.osd.3.smithi143.stdout:Aug 28 02:20:43 smithi143 bash[215184]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-08-28T02:20:43.451 INFO:journalctl@ceph.osd.3.smithi143.stdout:Aug 28 02:20:43 smithi143 bash[215184]: --> ceph-volume raw activate successful for osd ID: 3 2023-08-28T02:20:43.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:43 smithi143 ceph-mon[183059]: failed to encode map e108 with expected crc 2023-08-28T02:20:43.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:43 smithi143 ceph-mon[183059]: failed to encode map e108 with expected crc 2023-08-28T02:20:43.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:43 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.9", "id": [4, 3]}]': finished 2023-08-28T02:20:43.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:43 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 3]}]': finished 2023-08-28T02:20:43.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:43 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [4, 2]}]': finished 2023-08-28T02:20:43.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:43 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 3]}]': finished 2023-08-28T02:20:43.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:43 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 3]}]': finished 2023-08-28T02:20:43.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:43 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 3]}]': finished 2023-08-28T02:20:43.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:43 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 3]}]': finished 2023-08-28T02:20:43.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:43 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 3]}]': finished 2023-08-28T02:20:43.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:43 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 3]}]': finished 2023-08-28T02:20:43.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:43 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.15"}]': finished 2023-08-28T02:20:43.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:43 smithi143 ceph-mon[183059]: osdmap e108: 8 total, 7 up, 8 in 2023-08-28T02:20:43.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:43 smithi143 ceph-mon[183059]: failed to encode map e108 with expected crc 2023-08-28T02:20:43.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:43 smithi143 ceph-mon[183059]: failed to encode map e108 with expected crc 2023-08-28T02:20:43.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:43 smithi143 ceph-mon[183059]: failed to encode map e108 with expected crc 2023-08-28T02:20:43.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:43 smithi143 ceph-mon[189871]: failed to encode map e108 with expected crc 2023-08-28T02:20:43.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:43 smithi143 ceph-mon[189871]: failed to encode map e108 with expected crc 2023-08-28T02:20:43.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:43 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.9", "id": [4, 3]}]': finished 2023-08-28T02:20:43.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:43 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 3]}]': finished 2023-08-28T02:20:43.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:43 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [4, 2]}]': finished 2023-08-28T02:20:43.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:43 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 3]}]': finished 2023-08-28T02:20:43.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:43 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 3]}]': finished 2023-08-28T02:20:43.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:43 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 3]}]': finished 2023-08-28T02:20:43.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:43 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 3]}]': finished 2023-08-28T02:20:43.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:43 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 3]}]': finished 2023-08-28T02:20:43.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:43 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 3]}]': finished 2023-08-28T02:20:43.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:43 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.15"}]': finished 2023-08-28T02:20:43.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:43 smithi143 ceph-mon[189871]: osdmap e108: 8 total, 7 up, 8 in 2023-08-28T02:20:43.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:43 smithi143 ceph-mon[189871]: failed to encode map e108 with expected crc 2023-08-28T02:20:43.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:43 smithi143 ceph-mon[189871]: failed to encode map e108 with expected crc 2023-08-28T02:20:43.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:43 smithi143 ceph-mon[189871]: failed to encode map e108 with expected crc 2023-08-28T02:20:43.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:43 smithi169 ceph-mon[159415]: failed to encode map e108 with expected crc 2023-08-28T02:20:43.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:43 smithi169 ceph-mon[159415]: failed to encode map e108 with expected crc 2023-08-28T02:20:43.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:43 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.9", "id": [4, 3]}]': finished 2023-08-28T02:20:43.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:43 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 3]}]': finished 2023-08-28T02:20:43.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:43 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [4, 2]}]': finished 2023-08-28T02:20:43.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:43 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 3]}]': finished 2023-08-28T02:20:43.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:43 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 3]}]': finished 2023-08-28T02:20:43.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:43 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 3]}]': finished 2023-08-28T02:20:43.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:43 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 3]}]': finished 2023-08-28T02:20:43.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:43 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 3]}]': finished 2023-08-28T02:20:43.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:43 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 3]}]': finished 2023-08-28T02:20:43.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:43 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.15"}]': finished 2023-08-28T02:20:43.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:43 smithi169 ceph-mon[159415]: osdmap e108: 8 total, 7 up, 8 in 2023-08-28T02:20:43.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:43 smithi169 ceph-mon[159415]: failed to encode map e108 with expected crc 2023-08-28T02:20:43.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:43 smithi169 ceph-mon[159415]: failed to encode map e108 with expected crc 2023-08-28T02:20:43.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:43 smithi169 ceph-mon[159415]: failed to encode map e108 with expected crc 2023-08-28T02:20:44.413 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:44 smithi143 ceph-mon[183059]: pgmap v176: 73 pgs: 1 unknown, 1 peering, 3 active+undersized+degraded, 2 stale+active+clean, 19 active+undersized, 47 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-08-28T02:20:44.414 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:44 smithi143 ceph-mon[183059]: Health check failed: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2023-08-28T02:20:44.414 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:44 smithi143 ceph-mon[183059]: failed to encode map e109 with expected crc 2023-08-28T02:20:44.414 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:44 smithi143 ceph-mon[183059]: failed to encode map e109 with expected crc 2023-08-28T02:20:44.414 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:44 smithi143 ceph-mon[183059]: failed to encode map e109 with expected crc 2023-08-28T02:20:44.414 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:44 smithi143 ceph-mon[183059]: osdmap e109: 8 total, 7 up, 8 in 2023-08-28T02:20:44.414 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:44 smithi143 ceph-mon[183059]: failed to encode map e109 with expected crc 2023-08-28T02:20:44.414 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:44 smithi143 ceph-mon[183059]: failed to encode map e109 with expected crc 2023-08-28T02:20:44.414 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:44 smithi143 ceph-mon[183059]: failed to encode map e109 with expected crc 2023-08-28T02:20:44.414 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:44 smithi143 ceph-mon[183059]: failed to encode map e109 with expected crc 2023-08-28T02:20:44.414 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:44 smithi143 ceph-mon[183059]: failed to encode map e109 with expected crc 2023-08-28T02:20:44.414 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:44 smithi143 ceph-mon[183059]: failed to encode map e109 with expected crc 2023-08-28T02:20:44.415 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:44 smithi143 ceph-mon[183059]: failed to encode map e109 with expected crc 2023-08-28T02:20:44.415 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:44 smithi143 ceph-mon[183059]: failed to encode map e109 with expected crc 2023-08-28T02:20:44.415 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:44 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1758468992' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:44.415 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:44 smithi143 ceph-mon[189871]: pgmap v176: 73 pgs: 1 unknown, 1 peering, 3 active+undersized+degraded, 2 stale+active+clean, 19 active+undersized, 47 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-08-28T02:20:44.415 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:44 smithi143 ceph-mon[189871]: Health check failed: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2023-08-28T02:20:44.415 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:44 smithi143 ceph-mon[189871]: failed to encode map e109 with expected crc 2023-08-28T02:20:44.415 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:44 smithi143 ceph-mon[189871]: failed to encode map e109 with expected crc 2023-08-28T02:20:44.415 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:44 smithi143 ceph-mon[189871]: failed to encode map e109 with expected crc 2023-08-28T02:20:44.415 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:44 smithi143 ceph-mon[189871]: osdmap e109: 8 total, 7 up, 8 in 2023-08-28T02:20:44.415 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:44 smithi143 ceph-mon[189871]: failed to encode map e109 with expected crc 2023-08-28T02:20:44.416 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:44 smithi143 ceph-mon[189871]: failed to encode map e109 with expected crc 2023-08-28T02:20:44.416 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:44 smithi143 ceph-mon[189871]: failed to encode map e109 with expected crc 2023-08-28T02:20:44.416 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:44 smithi143 ceph-mon[189871]: failed to encode map e109 with expected crc 2023-08-28T02:20:44.416 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:44 smithi143 ceph-mon[189871]: failed to encode map e109 with expected crc 2023-08-28T02:20:44.416 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:44 smithi143 ceph-mon[189871]: failed to encode map e109 with expected crc 2023-08-28T02:20:44.416 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:44 smithi143 ceph-mon[189871]: failed to encode map e109 with expected crc 2023-08-28T02:20:44.416 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:44 smithi143 ceph-mon[189871]: failed to encode map e109 with expected crc 2023-08-28T02:20:44.416 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:44 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1758468992' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:44.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:44 smithi169 ceph-mon[159415]: pgmap v176: 73 pgs: 1 unknown, 1 peering, 3 active+undersized+degraded, 2 stale+active+clean, 19 active+undersized, 47 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-08-28T02:20:44.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:44 smithi169 ceph-mon[159415]: Health check failed: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2023-08-28T02:20:44.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:44 smithi169 ceph-mon[159415]: failed to encode map e109 with expected crc 2023-08-28T02:20:44.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:44 smithi169 ceph-mon[159415]: failed to encode map e109 with expected crc 2023-08-28T02:20:44.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:44 smithi169 ceph-mon[159415]: failed to encode map e109 with expected crc 2023-08-28T02:20:44.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:44 smithi169 ceph-mon[159415]: osdmap e109: 8 total, 7 up, 8 in 2023-08-28T02:20:44.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:44 smithi169 ceph-mon[159415]: failed to encode map e109 with expected crc 2023-08-28T02:20:44.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:44 smithi169 ceph-mon[159415]: failed to encode map e109 with expected crc 2023-08-28T02:20:44.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:44 smithi169 ceph-mon[159415]: failed to encode map e109 with expected crc 2023-08-28T02:20:44.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:44 smithi169 ceph-mon[159415]: failed to encode map e109 with expected crc 2023-08-28T02:20:44.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:44 smithi169 ceph-mon[159415]: failed to encode map e109 with expected crc 2023-08-28T02:20:44.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:44 smithi169 ceph-mon[159415]: failed to encode map e109 with expected crc 2023-08-28T02:20:44.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:44 smithi169 ceph-mon[159415]: failed to encode map e109 with expected crc 2023-08-28T02:20:44.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:44 smithi169 ceph-mon[159415]: failed to encode map e109 with expected crc 2023-08-28T02:20:44.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:44 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1758468992' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:44.950 INFO:journalctl@ceph.osd.3.smithi143.stdout:Aug 28 02:20:44 smithi143 ceph-osd[215689]: -- 172.21.15.143:0/2675442015 <== mon.0 v2:172.21.15.143:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55ff72af04e0 con 0x55ff72ac2c00 2023-08-28T02:20:44.950 INFO:journalctl@ceph.osd.3.smithi143.stdout:Aug 28 02:20:44 smithi143 bash[215623]: 842ce37f38d4efc64244d35b30c0257228c286cd40794ab56fe182ba1af8edd5 2023-08-28T02:20:44.950 INFO:journalctl@ceph.osd.3.smithi143.stdout:Aug 28 02:20:44 smithi143 systemd[1]: Started Ceph osd.3 for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:20:45.450 INFO:journalctl@ceph.osd.3.smithi143.stdout:Aug 28 02:20:45 smithi143 conmon[215665]: 2023-08-28T02:20:45.012+0000 7f58cc9ca640 -1 Falling back to public interface 2023-08-28T02:20:45.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:45 smithi143 ceph-mon[183059]: failed to encode map e110 with expected crc 2023-08-28T02:20:45.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:45 smithi143 ceph-mon[183059]: failed to encode map e110 with expected crc 2023-08-28T02:20:45.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:45 smithi143 ceph-mon[183059]: failed to encode map e110 with expected crc 2023-08-28T02:20:45.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:45 smithi143 ceph-mon[183059]: failed to encode map e110 with expected crc 2023-08-28T02:20:45.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:45 smithi143 ceph-mon[183059]: osdmap e110: 8 total, 7 up, 8 in 2023-08-28T02:20:45.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:45 smithi143 ceph-mon[183059]: failed to encode map e110 with expected crc 2023-08-28T02:20:45.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:45 smithi143 ceph-mon[183059]: failed to encode map e110 with expected crc 2023-08-28T02:20:45.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:45 smithi143 ceph-mon[183059]: failed to encode map e110 with expected crc 2023-08-28T02:20:45.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:45 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:45.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:45 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:45.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:45 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:20:45.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:45 smithi143 ceph-mon[189871]: failed to encode map e110 with expected crc 2023-08-28T02:20:45.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:45 smithi143 ceph-mon[189871]: failed to encode map e110 with expected crc 2023-08-28T02:20:45.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:45 smithi143 ceph-mon[189871]: failed to encode map e110 with expected crc 2023-08-28T02:20:45.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:45 smithi143 ceph-mon[189871]: failed to encode map e110 with expected crc 2023-08-28T02:20:45.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:45 smithi143 ceph-mon[189871]: osdmap e110: 8 total, 7 up, 8 in 2023-08-28T02:20:45.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:45 smithi143 ceph-mon[189871]: failed to encode map e110 with expected crc 2023-08-28T02:20:45.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:45 smithi143 ceph-mon[189871]: failed to encode map e110 with expected crc 2023-08-28T02:20:45.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:45 smithi143 ceph-mon[189871]: failed to encode map e110 with expected crc 2023-08-28T02:20:45.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:45 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:45.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:45 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:45.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:45 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:20:45.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:45 smithi169 ceph-mon[159415]: failed to encode map e110 with expected crc 2023-08-28T02:20:45.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:45 smithi169 ceph-mon[159415]: failed to encode map e110 with expected crc 2023-08-28T02:20:45.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:45 smithi169 ceph-mon[159415]: failed to encode map e110 with expected crc 2023-08-28T02:20:45.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:45 smithi169 ceph-mon[159415]: failed to encode map e110 with expected crc 2023-08-28T02:20:45.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:45 smithi169 ceph-mon[159415]: osdmap e110: 8 total, 7 up, 8 in 2023-08-28T02:20:45.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:45 smithi169 ceph-mon[159415]: failed to encode map e110 with expected crc 2023-08-28T02:20:45.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:45 smithi169 ceph-mon[159415]: failed to encode map e110 with expected crc 2023-08-28T02:20:45.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:45 smithi169 ceph-mon[159415]: failed to encode map e110 with expected crc 2023-08-28T02:20:45.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:45 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:45.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:45 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:45.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:45 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:20:46.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:46 smithi143 ceph-mon[183059]: pgmap v179: 73 pgs: 1 unknown, 1 peering, 3 active+undersized+degraded, 23 active+undersized, 45 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-08-28T02:20:46.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:46 smithi143 ceph-mon[189871]: pgmap v179: 73 pgs: 1 unknown, 1 peering, 3 active+undersized+degraded, 23 active+undersized, 45 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-08-28T02:20:46.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:46 smithi169 ceph-mon[159415]: pgmap v179: 73 pgs: 1 unknown, 1 peering, 3 active+undersized+degraded, 23 active+undersized, 45 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-08-28T02:20:47.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:47 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1986419716' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:47.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:47 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1986419716' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:47.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:47 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1986419716' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:48.200 INFO:journalctl@ceph.osd.3.smithi143.stdout:Aug 28 02:20:47 smithi143 conmon[215665]: 2023-08-28T02:20:47.919+0000 7f58cc9ca640 -1 osd.3 105 log_to_monitors true 2023-08-28T02:20:48.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:48 smithi143 ceph-mon[183059]: pgmap v180: 73 pgs: 7 active+clean+remapped, 1 unknown, 1 peering, 3 active+undersized+degraded, 23 active+undersized, 38 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%); 3 B/s, 0 objects/s recovering 2023-08-28T02:20:48.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:48 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:48.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:48 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:48.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:48 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:48.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:48 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:48.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:48 smithi143 ceph-mon[183059]: from='osd.3 [v2:172.21.15.143:6826/2677897674,v1:172.21.15.143:6827/2677897674]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-08-28T02:20:48.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:48 smithi143 ceph-mon[183059]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-08-28T02:20:48.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:48 smithi143 ceph-mon[189871]: pgmap v180: 73 pgs: 7 active+clean+remapped, 1 unknown, 1 peering, 3 active+undersized+degraded, 23 active+undersized, 38 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%); 3 B/s, 0 objects/s recovering 2023-08-28T02:20:48.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:48 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:48.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:48 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:48.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:48 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:48.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:48 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:48.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:48 smithi143 ceph-mon[189871]: from='osd.3 [v2:172.21.15.143:6826/2677897674,v1:172.21.15.143:6827/2677897674]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-08-28T02:20:48.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:48 smithi143 ceph-mon[189871]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-08-28T02:20:48.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:48 smithi169 ceph-mon[159415]: pgmap v180: 73 pgs: 7 active+clean+remapped, 1 unknown, 1 peering, 3 active+undersized+degraded, 23 active+undersized, 38 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%); 3 B/s, 0 objects/s recovering 2023-08-28T02:20:48.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:48 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:48.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:48 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:48.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:48 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:48.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:48 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:48.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:48 smithi169 ceph-mon[159415]: from='osd.3 [v2:172.21.15.143:6826/2677897674,v1:172.21.15.143:6827/2677897674]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-08-28T02:20:48.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:48 smithi169 ceph-mon[159415]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-08-28T02:20:48.951 INFO:journalctl@ceph.osd.3.smithi143.stdout:Aug 28 02:20:48 smithi143 conmon[215665]: 2023-08-28T02:20:48.822+0000 7f58bbe32700 -1 osd.3 105 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-08-28T02:20:49.406 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:49 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1349708776' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:49.406 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:49 smithi143 ceph-mon[183059]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-08-28T02:20:49.406 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:49 smithi143 ceph-mon[183059]: osdmap e111: 8 total, 7 up, 8 in 2023-08-28T02:20:49.406 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:49 smithi143 ceph-mon[183059]: from='osd.3 [v2:172.21.15.143:6826/2677897674,v1:172.21.15.143:6827/2677897674]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]: dispatch 2023-08-28T02:20:49.407 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:49 smithi143 ceph-mon[183059]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]: dispatch 2023-08-28T02:20:49.407 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:49 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1349708776' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:49.407 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:49 smithi143 ceph-mon[189871]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-08-28T02:20:49.407 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:49 smithi143 ceph-mon[189871]: osdmap e111: 8 total, 7 up, 8 in 2023-08-28T02:20:49.407 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:49 smithi143 ceph-mon[189871]: from='osd.3 [v2:172.21.15.143:6826/2677897674,v1:172.21.15.143:6827/2677897674]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]: dispatch 2023-08-28T02:20:49.407 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:49 smithi143 ceph-mon[189871]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]: dispatch 2023-08-28T02:20:49.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:49 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1349708776' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:49.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:49 smithi169 ceph-mon[159415]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-08-28T02:20:49.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:49 smithi169 ceph-mon[159415]: osdmap e111: 8 total, 7 up, 8 in 2023-08-28T02:20:49.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:49 smithi169 ceph-mon[159415]: from='osd.3 [v2:172.21.15.143:6826/2677897674,v1:172.21.15.143:6827/2677897674]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]: dispatch 2023-08-28T02:20:49.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:49 smithi169 ceph-mon[159415]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi143", "root=default"]}]: dispatch 2023-08-28T02:20:50.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:50 smithi143 ceph-mon[183059]: failed to encode map e111 with expected crc 2023-08-28T02:20:50.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:50 smithi143 ceph-mon[183059]: pgmap v182: 73 pgs: 9 active+clean+remapped, 3 active+undersized+degraded, 23 active+undersized, 38 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%); 3 B/s, 0 objects/s recovering 2023-08-28T02:20:50.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:50 smithi143 ceph-mon[183059]: failed to encode map e111 with expected crc 2023-08-28T02:20:50.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:50 smithi143 ceph-mon[183059]: failed to encode map e111 with expected crc 2023-08-28T02:20:50.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:50 smithi143 ceph-mon[183059]: failed to encode map e111 with expected crc 2023-08-28T02:20:50.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:50 smithi143 ceph-mon[183059]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-08-28T02:20:50.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:50 smithi143 ceph-mon[183059]: osd.3 [v2:172.21.15.143:6826/2677897674,v1:172.21.15.143:6827/2677897674] boot 2023-08-28T02:20:50.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:50 smithi143 ceph-mon[183059]: osdmap e112: 8 total, 8 up, 8 in 2023-08-28T02:20:50.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:50 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-28T02:20:50.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:50 smithi143 ceph-mon[189871]: failed to encode map e111 with expected crc 2023-08-28T02:20:50.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:50 smithi143 ceph-mon[189871]: pgmap v182: 73 pgs: 9 active+clean+remapped, 3 active+undersized+degraded, 23 active+undersized, 38 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%); 3 B/s, 0 objects/s recovering 2023-08-28T02:20:50.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:50 smithi143 ceph-mon[189871]: failed to encode map e111 with expected crc 2023-08-28T02:20:50.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:50 smithi143 ceph-mon[189871]: failed to encode map e111 with expected crc 2023-08-28T02:20:50.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:50 smithi143 ceph-mon[189871]: failed to encode map e111 with expected crc 2023-08-28T02:20:50.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:50 smithi143 ceph-mon[189871]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-08-28T02:20:50.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:50 smithi143 ceph-mon[189871]: osd.3 [v2:172.21.15.143:6826/2677897674,v1:172.21.15.143:6827/2677897674] boot 2023-08-28T02:20:50.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:50 smithi143 ceph-mon[189871]: osdmap e112: 8 total, 8 up, 8 in 2023-08-28T02:20:50.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:50 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-28T02:20:50.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:50 smithi169 ceph-mon[159415]: failed to encode map e111 with expected crc 2023-08-28T02:20:50.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:50 smithi169 ceph-mon[159415]: pgmap v182: 73 pgs: 9 active+clean+remapped, 3 active+undersized+degraded, 23 active+undersized, 38 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%); 3 B/s, 0 objects/s recovering 2023-08-28T02:20:50.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:50 smithi169 ceph-mon[159415]: failed to encode map e111 with expected crc 2023-08-28T02:20:50.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:50 smithi169 ceph-mon[159415]: failed to encode map e111 with expected crc 2023-08-28T02:20:50.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:50 smithi169 ceph-mon[159415]: failed to encode map e111 with expected crc 2023-08-28T02:20:50.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:50 smithi169 ceph-mon[159415]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-08-28T02:20:50.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:50 smithi169 ceph-mon[159415]: osd.3 [v2:172.21.15.143:6826/2677897674,v1:172.21.15.143:6827/2677897674] boot 2023-08-28T02:20:50.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:50 smithi169 ceph-mon[159415]: osdmap e112: 8 total, 8 up, 8 in 2023-08-28T02:20:50.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:50 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-28T02:20:50.950 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:20:50 smithi143 conmon[176053]: 2023-08-28T02:20:50.785+0000 7f008786b700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2023-08-28T02:20:51.048 INFO:teuthology.orchestra.run.smithi143.stdout: "ceph version 18.2.0-187-g9cf20820 (9cf208207dfe23ffccb33308b22e51570cb45e51) reef (stable)": 4 2023-08-28T02:20:51.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:51 smithi143 ceph-mon[183059]: failed to encode map e111 with expected crc 2023-08-28T02:20:51.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:51 smithi143 ceph-mon[183059]: failed to encode map e112 with expected crc 2023-08-28T02:20:51.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:51 smithi143 ceph-mon[183059]: failed to encode map e112 with expected crc 2023-08-28T02:20:51.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:51 smithi143 ceph-mon[183059]: failed to encode map e112 with expected crc 2023-08-28T02:20:51.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:51 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:51.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:51 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:51.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:51 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:20:51.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:51 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:20:51.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:51 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:51.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:51 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:20:51.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:51 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:20:51.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:51 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:20:51.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:51 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:20:51.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:51 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-08-28T02:20:51.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:51 smithi143 ceph-mon[183059]: osdmap e113: 8 total, 8 up, 8 in 2023-08-28T02:20:51.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:51 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/828160575' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:51.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:51 smithi143 ceph-mon[189871]: failed to encode map e111 with expected crc 2023-08-28T02:20:51.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:51 smithi143 ceph-mon[189871]: failed to encode map e112 with expected crc 2023-08-28T02:20:51.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:51 smithi143 ceph-mon[189871]: failed to encode map e112 with expected crc 2023-08-28T02:20:51.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:51 smithi143 ceph-mon[189871]: failed to encode map e112 with expected crc 2023-08-28T02:20:51.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:51 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:51.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:51 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:51.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:51 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:20:51.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:51 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:20:51.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:51 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:51.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:51 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:20:51.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:51 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:20:51.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:51 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:20:51.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:51 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:20:51.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:51 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-08-28T02:20:51.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:51 smithi143 ceph-mon[189871]: osdmap e113: 8 total, 8 up, 8 in 2023-08-28T02:20:51.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:51 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/828160575' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:51.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:51 smithi169 ceph-mon[159415]: failed to encode map e111 with expected crc 2023-08-28T02:20:51.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:51 smithi169 ceph-mon[159415]: failed to encode map e112 with expected crc 2023-08-28T02:20:51.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:51 smithi169 ceph-mon[159415]: failed to encode map e112 with expected crc 2023-08-28T02:20:51.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:51 smithi169 ceph-mon[159415]: failed to encode map e112 with expected crc 2023-08-28T02:20:51.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:51 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:51.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:51 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:51.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:51 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:20:51.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:51 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:20:51.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:51 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:51.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:51 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:20:51.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:51 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:20:51.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:51 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:20:51.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:51 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:20:51.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:51 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-08-28T02:20:51.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:51 smithi169 ceph-mon[159415]: osdmap e113: 8 total, 8 up, 8 in 2023-08-28T02:20:51.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:51 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/828160575' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-08-28T02:20:51.655 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -e sha1=9cf208207dfe23ffccb33308b22e51570cb45e51 -- bash -c 'ceph orch upgrade pause' 2023-08-28T02:20:52.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:20:51 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:20:51] "GET /metrics HTTP/1.1" 200 34052 "" "Prometheus/2.33.4" 2023-08-28T02:20:52.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:52 smithi143 ceph-mon[189871]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-08-28T02:20:52.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:52 smithi143 ceph-mon[189871]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2023-08-28T02:20:52.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:52 smithi143 ceph-mon[189871]: failed to encode map e113 with expected crc 2023-08-28T02:20:52.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:52 smithi143 ceph-mon[189871]: failed to encode map e113 with expected crc 2023-08-28T02:20:52.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:52 smithi143 ceph-mon[189871]: failed to encode map e113 with expected crc 2023-08-28T02:20:52.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:52 smithi143 ceph-mon[189871]: failed to encode map e113 with expected crc 2023-08-28T02:20:52.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:52 smithi143 ceph-mon[189871]: failed to encode map e113 with expected crc 2023-08-28T02:20:52.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:52 smithi143 ceph-mon[189871]: failed to encode map e113 with expected crc 2023-08-28T02:20:52.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:52 smithi143 ceph-mon[189871]: failed to encode map e113 with expected crc 2023-08-28T02:20:52.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:52 smithi143 ceph-mon[189871]: failed to encode map e113 with expected crc 2023-08-28T02:20:52.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:52 smithi143 ceph-mon[189871]: failed to encode map e113 with expected crc 2023-08-28T02:20:52.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:52 smithi143 ceph-mon[189871]: failed to encode map e113 with expected crc 2023-08-28T02:20:52.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:52 smithi143 ceph-mon[189871]: pgmap v185: 73 pgs: 9 active+clean+remapped, 3 active+undersized+degraded, 23 active+undersized, 38 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%); 3 B/s, 0 objects/s recovering 2023-08-28T02:20:52.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:52 smithi143 ceph-mon[189871]: osdmap e114: 8 total, 8 up, 8 in 2023-08-28T02:20:52.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:52 smithi143 ceph-mon[183059]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-08-28T02:20:52.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:52 smithi143 ceph-mon[183059]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2023-08-28T02:20:52.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:52 smithi143 ceph-mon[183059]: failed to encode map e113 with expected crc 2023-08-28T02:20:52.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:52 smithi143 ceph-mon[183059]: failed to encode map e113 with expected crc 2023-08-28T02:20:52.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:52 smithi143 ceph-mon[183059]: failed to encode map e113 with expected crc 2023-08-28T02:20:52.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:52 smithi143 ceph-mon[183059]: failed to encode map e113 with expected crc 2023-08-28T02:20:52.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:52 smithi143 ceph-mon[183059]: failed to encode map e113 with expected crc 2023-08-28T02:20:52.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:52 smithi143 ceph-mon[183059]: failed to encode map e113 with expected crc 2023-08-28T02:20:52.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:52 smithi143 ceph-mon[183059]: failed to encode map e113 with expected crc 2023-08-28T02:20:52.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:52 smithi143 ceph-mon[183059]: failed to encode map e113 with expected crc 2023-08-28T02:20:52.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:52 smithi143 ceph-mon[183059]: failed to encode map e113 with expected crc 2023-08-28T02:20:52.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:52 smithi143 ceph-mon[183059]: failed to encode map e113 with expected crc 2023-08-28T02:20:52.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:52 smithi143 ceph-mon[183059]: pgmap v185: 73 pgs: 9 active+clean+remapped, 3 active+undersized+degraded, 23 active+undersized, 38 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%); 3 B/s, 0 objects/s recovering 2023-08-28T02:20:52.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:52 smithi143 ceph-mon[183059]: osdmap e114: 8 total, 8 up, 8 in 2023-08-28T02:20:52.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:52 smithi169 ceph-mon[159415]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-08-28T02:20:52.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:52 smithi169 ceph-mon[159415]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2023-08-28T02:20:52.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:52 smithi169 ceph-mon[159415]: failed to encode map e113 with expected crc 2023-08-28T02:20:52.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:52 smithi169 ceph-mon[159415]: failed to encode map e113 with expected crc 2023-08-28T02:20:52.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:52 smithi169 ceph-mon[159415]: failed to encode map e113 with expected crc 2023-08-28T02:20:52.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:52 smithi169 ceph-mon[159415]: failed to encode map e113 with expected crc 2023-08-28T02:20:52.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:52 smithi169 ceph-mon[159415]: failed to encode map e113 with expected crc 2023-08-28T02:20:52.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:52 smithi169 ceph-mon[159415]: failed to encode map e113 with expected crc 2023-08-28T02:20:52.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:52 smithi169 ceph-mon[159415]: failed to encode map e113 with expected crc 2023-08-28T02:20:52.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:52 smithi169 ceph-mon[159415]: failed to encode map e113 with expected crc 2023-08-28T02:20:52.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:52 smithi169 ceph-mon[159415]: failed to encode map e113 with expected crc 2023-08-28T02:20:52.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:52 smithi169 ceph-mon[159415]: failed to encode map e113 with expected crc 2023-08-28T02:20:52.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:52 smithi169 ceph-mon[159415]: pgmap v185: 73 pgs: 9 active+clean+remapped, 3 active+undersized+degraded, 23 active+undersized, 38 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%); 3 B/s, 0 objects/s recovering 2023-08-28T02:20:52.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:52 smithi169 ceph-mon[159415]: osdmap e114: 8 total, 8 up, 8 in 2023-08-28T02:20:53.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:53 smithi143 ceph-mon[189871]: failed to encode map e114 with expected crc 2023-08-28T02:20:53.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:53 smithi143 ceph-mon[189871]: failed to encode map e114 with expected crc 2023-08-28T02:20:53.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:53 smithi143 ceph-mon[189871]: failed to encode map e114 with expected crc 2023-08-28T02:20:53.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:53 smithi143 ceph-mon[189871]: failed to encode map e114 with expected crc 2023-08-28T02:20:53.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:53 smithi143 ceph-mon[189871]: failed to encode map e114 with expected crc 2023-08-28T02:20:53.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:53 smithi143 ceph-mon[189871]: failed to encode map e114 with expected crc 2023-08-28T02:20:53.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:53 smithi143 ceph-mon[189871]: failed to encode map e114 with expected crc 2023-08-28T02:20:53.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:53 smithi143 ceph-mon[189871]: failed to encode map e114 with expected crc 2023-08-28T02:20:53.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:53 smithi143 ceph-mon[183059]: failed to encode map e114 with expected crc 2023-08-28T02:20:53.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:53 smithi143 ceph-mon[183059]: failed to encode map e114 with expected crc 2023-08-28T02:20:53.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:53 smithi143 ceph-mon[183059]: failed to encode map e114 with expected crc 2023-08-28T02:20:53.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:53 smithi143 ceph-mon[183059]: failed to encode map e114 with expected crc 2023-08-28T02:20:53.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:53 smithi143 ceph-mon[183059]: failed to encode map e114 with expected crc 2023-08-28T02:20:53.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:53 smithi143 ceph-mon[183059]: failed to encode map e114 with expected crc 2023-08-28T02:20:53.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:53 smithi143 ceph-mon[183059]: failed to encode map e114 with expected crc 2023-08-28T02:20:53.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:53 smithi143 ceph-mon[183059]: failed to encode map e114 with expected crc 2023-08-28T02:20:53.485 INFO:teuthology.orchestra.run.smithi143.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8 2023-08-28T02:20:53.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:53 smithi169 ceph-mon[159415]: failed to encode map e114 with expected crc 2023-08-28T02:20:53.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:53 smithi169 ceph-mon[159415]: failed to encode map e114 with expected crc 2023-08-28T02:20:53.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:53 smithi169 ceph-mon[159415]: failed to encode map e114 with expected crc 2023-08-28T02:20:53.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:53 smithi169 ceph-mon[159415]: failed to encode map e114 with expected crc 2023-08-28T02:20:53.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:53 smithi169 ceph-mon[159415]: failed to encode map e114 with expected crc 2023-08-28T02:20:53.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:53 smithi169 ceph-mon[159415]: failed to encode map e114 with expected crc 2023-08-28T02:20:53.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:53 smithi169 ceph-mon[159415]: failed to encode map e114 with expected crc 2023-08-28T02:20:53.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:53 smithi169 ceph-mon[159415]: failed to encode map e114 with expected crc 2023-08-28T02:20:54.071 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -e sha1=9cf208207dfe23ffccb33308b22e51570cb45e51 -- bash -c 'ceph orch ps' 2023-08-28T02:20:54.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:54 smithi143 ceph-mon[189871]: pgmap v187: 73 pgs: 2 active+clean+remapped, 2 active+undersized+degraded, 10 active+undersized, 59 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-08-28T02:20:54.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:54 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2023-08-28T02:20:54.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:54 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:54.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:54 smithi143 ceph-mon[183059]: pgmap v187: 73 pgs: 2 active+clean+remapped, 2 active+undersized+degraded, 10 active+undersized, 59 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-08-28T02:20:54.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:54 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2023-08-28T02:20:54.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:54 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:54.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:54 smithi169 ceph-mon[159415]: pgmap v187: 73 pgs: 2 active+clean+remapped, 2 active+undersized+degraded, 10 active+undersized, 59 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-08-28T02:20:54.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:54 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2023-08-28T02:20:54.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:54 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:55.429 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:55 smithi143 ceph-mon[183059]: from='client.44357 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:20:55.429 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:55 smithi143 ceph-mon[183059]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8 2023-08-28T02:20:55.429 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:55 smithi143 ceph-mon[189871]: from='client.44357 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:20:55.429 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:55 smithi143 ceph-mon[189871]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8 2023-08-28T02:20:55.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:55 smithi169 ceph-mon[159415]: from='client.44357 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:20:55.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:55 smithi169 ceph-mon[159415]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8 2023-08-28T02:20:55.769 INFO:teuthology.orchestra.run.smithi143.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-08-28T02:20:55.769 INFO:teuthology.orchestra.run.smithi143.stdout:alertmanager.a smithi143 *:9093,9094 running (6m) 8s ago 10m 27.3M - 0.23.0 ba2b418f427c f216a6d30bb2 2023-08-28T02:20:55.769 INFO:teuthology.orchestra.run.smithi143.stdout:grafana.a smithi169 *:3000 running (6m) 69s ago 9m 56.3M - 8.3.5 dad864ee21e9 4c9eb6ad00e1 2023-08-28T02:20:55.769 INFO:teuthology.orchestra.run.smithi143.stdout:mds.foo.smithi143.spifrj smithi143 running (9m) 8s ago 9m 19.1M - 16.2.13-565-g595355f4 4099d2e00414 4eeedd3bc917 2023-08-28T02:20:55.769 INFO:teuthology.orchestra.run.smithi143.stdout:mds.foo.smithi169.yeisur smithi169 running (9m) 69s ago 9m 18.7M - 16.2.13-565-g595355f4 4099d2e00414 fed9013c1357 2023-08-28T02:20:55.770 INFO:teuthology.orchestra.run.smithi143.stdout:mgr.x smithi169 *:8443,9283,8765 running (5m) 69s ago 14m 448M - 18.2.0-187-g9cf20820 62b871dea10e 73a560edcde9 2023-08-28T02:20:55.770 INFO:teuthology.orchestra.run.smithi143.stdout:mgr.y smithi143 *:8443,9283,8765 running (6m) 8s ago 15m 512M - 18.2.0-187-g9cf20820 62b871dea10e 8de2036a51e3 2023-08-28T02:20:55.770 INFO:teuthology.orchestra.run.smithi143.stdout:mon.a smithi143 running (5m) 8s ago 15m 97.1M 2048M 18.2.0-187-g9cf20820 62b871dea10e 072272663d4f 2023-08-28T02:20:55.770 INFO:teuthology.orchestra.run.smithi143.stdout:mon.b smithi169 running (3m) 69s ago 14m 75.6M 2048M 18.2.0-187-g9cf20820 62b871dea10e 21231a80906c 2023-08-28T02:20:55.770 INFO:teuthology.orchestra.run.smithi143.stdout:mon.c smithi143 running (3m) 8s ago 14m 76.7M 2048M 18.2.0-187-g9cf20820 62b871dea10e f1328bdbe045 2023-08-28T02:20:55.770 INFO:teuthology.orchestra.run.smithi143.stdout:node-exporter.a smithi143 *:9100 running (6m) 8s ago 10m 26.1M - 1.3.1 1dbe0e931976 e389cc167c39 2023-08-28T02:20:55.770 INFO:teuthology.orchestra.run.smithi143.stdout:node-exporter.b smithi169 *:9100 running (6m) 69s ago 10m 21.5M - 1.3.1 1dbe0e931976 5e5215d11ac8 2023-08-28T02:20:55.770 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0 smithi143 running (2m) 8s ago 13m 64.1M 2500M 18.2.0-187-g9cf20820 62b871dea10e 241e17d94254 2023-08-28T02:20:55.770 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1 smithi143 running (72s) 8s ago 13m 60.0M 2500M 18.2.0-187-g9cf20820 62b871dea10e a6c1aefbbefc 2023-08-28T02:20:55.770 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2 smithi143 running (42s) 8s ago 13m 49.0M 2500M 18.2.0-187-g9cf20820 62b871dea10e edfc41dc7fbf 2023-08-28T02:20:55.771 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3 smithi143 running (11s) 8s ago 12m 11.7M 2500M 18.2.0-187-g9cf20820 62b871dea10e 842ce37f38d4 2023-08-28T02:20:55.771 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4 smithi169 running (12m) 69s ago 12m 84.5M 2500M 16.2.13-565-g595355f4 4099d2e00414 e6ed776def27 2023-08-28T02:20:55.771 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5 smithi169 running (11m) 69s ago 11m 92.4M 2500M 16.2.13-565-g595355f4 4099d2e00414 6fe1195fca7e 2023-08-28T02:20:55.771 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6 smithi169 running (11m) 69s ago 11m 87.2M 2500M 16.2.13-565-g595355f4 4099d2e00414 12854ca48e76 2023-08-28T02:20:55.771 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7 smithi169 running (11m) 69s ago 11m 105M 2500M 16.2.13-565-g595355f4 4099d2e00414 64fa1422c85a 2023-08-28T02:20:55.771 INFO:teuthology.orchestra.run.smithi143.stdout:prometheus.a smithi169 *:9095 running (5m) 69s ago 10m 68.3M - 2.33.4 514e6a882f6e cd763ce7d282 2023-08-28T02:20:56.319 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -e sha1=9cf208207dfe23ffccb33308b22e51570cb45e51 -- bash -c 'ceph orch ps' 2023-08-28T02:20:56.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:56 smithi143 ceph-mon[189871]: pgmap v188: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:20:56.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:56 smithi143 ceph-mon[189871]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2023-08-28T02:20:56.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:56 smithi143 ceph-mon[189871]: Cluster is now healthy 2023-08-28T02:20:56.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:56 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:56.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:56 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:20:56.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:56 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:56.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:56 smithi143 ceph-mon[183059]: pgmap v188: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:20:56.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:56 smithi143 ceph-mon[183059]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2023-08-28T02:20:56.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:56 smithi143 ceph-mon[183059]: Cluster is now healthy 2023-08-28T02:20:56.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:56 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:56.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:56 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:20:56.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:56 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:56.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:56 smithi169 ceph-mon[159415]: pgmap v188: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:20:56.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:56 smithi169 ceph-mon[159415]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2023-08-28T02:20:56.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:56 smithi169 ceph-mon[159415]: Cluster is now healthy 2023-08-28T02:20:56.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:56 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:56.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:56 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:20:56.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:56 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:20:57.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:57 smithi143 ceph-mon[183059]: from='client.44363 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:20:57.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:57 smithi143 ceph-mon[189871]: from='client.44363 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:20:57.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:57 smithi169 ceph-mon[159415]: from='client.44363 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:20:58.066 INFO:teuthology.orchestra.run.smithi143.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-08-28T02:20:58.066 INFO:teuthology.orchestra.run.smithi143.stdout:alertmanager.a smithi143 *:9093,9094 running (6m) 10s ago 10m 27.3M - 0.23.0 ba2b418f427c f216a6d30bb2 2023-08-28T02:20:58.066 INFO:teuthology.orchestra.run.smithi143.stdout:grafana.a smithi169 *:3000 running (6m) 71s ago 9m 56.3M - 8.3.5 dad864ee21e9 4c9eb6ad00e1 2023-08-28T02:20:58.066 INFO:teuthology.orchestra.run.smithi143.stdout:mds.foo.smithi143.spifrj smithi143 running (9m) 10s ago 9m 19.1M - 16.2.13-565-g595355f4 4099d2e00414 4eeedd3bc917 2023-08-28T02:20:58.066 INFO:teuthology.orchestra.run.smithi143.stdout:mds.foo.smithi169.yeisur smithi169 running (9m) 71s ago 9m 18.7M - 16.2.13-565-g595355f4 4099d2e00414 fed9013c1357 2023-08-28T02:20:58.066 INFO:teuthology.orchestra.run.smithi143.stdout:mgr.x smithi169 *:8443,9283,8765 running (5m) 71s ago 14m 448M - 18.2.0-187-g9cf20820 62b871dea10e 73a560edcde9 2023-08-28T02:20:58.066 INFO:teuthology.orchestra.run.smithi143.stdout:mgr.y smithi143 *:8443,9283,8765 running (6m) 10s ago 15m 512M - 18.2.0-187-g9cf20820 62b871dea10e 8de2036a51e3 2023-08-28T02:20:58.066 INFO:teuthology.orchestra.run.smithi143.stdout:mon.a smithi143 running (5m) 10s ago 15m 97.1M 2048M 18.2.0-187-g9cf20820 62b871dea10e 072272663d4f 2023-08-28T02:20:58.067 INFO:teuthology.orchestra.run.smithi143.stdout:mon.b smithi169 running (3m) 71s ago 14m 75.6M 2048M 18.2.0-187-g9cf20820 62b871dea10e 21231a80906c 2023-08-28T02:20:58.067 INFO:teuthology.orchestra.run.smithi143.stdout:mon.c smithi143 running (3m) 10s ago 14m 76.7M 2048M 18.2.0-187-g9cf20820 62b871dea10e f1328bdbe045 2023-08-28T02:20:58.067 INFO:teuthology.orchestra.run.smithi143.stdout:node-exporter.a smithi143 *:9100 running (6m) 10s ago 10m 26.1M - 1.3.1 1dbe0e931976 e389cc167c39 2023-08-28T02:20:58.067 INFO:teuthology.orchestra.run.smithi143.stdout:node-exporter.b smithi169 *:9100 running (6m) 71s ago 10m 21.5M - 1.3.1 1dbe0e931976 5e5215d11ac8 2023-08-28T02:20:58.067 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0 smithi143 running (2m) 10s ago 13m 64.1M 2500M 18.2.0-187-g9cf20820 62b871dea10e 241e17d94254 2023-08-28T02:20:58.067 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1 smithi143 running (74s) 10s ago 13m 60.0M 2500M 18.2.0-187-g9cf20820 62b871dea10e a6c1aefbbefc 2023-08-28T02:20:58.067 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2 smithi143 running (44s) 10s ago 13m 49.0M 2500M 18.2.0-187-g9cf20820 62b871dea10e edfc41dc7fbf 2023-08-28T02:20:58.067 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3 smithi143 running (13s) 10s ago 12m 11.7M 2500M 18.2.0-187-g9cf20820 62b871dea10e 842ce37f38d4 2023-08-28T02:20:58.067 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4 smithi169 running (12m) 71s ago 12m 84.5M 2500M 16.2.13-565-g595355f4 4099d2e00414 e6ed776def27 2023-08-28T02:20:58.067 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5 smithi169 running (11m) 71s ago 11m 92.4M 2500M 16.2.13-565-g595355f4 4099d2e00414 6fe1195fca7e 2023-08-28T02:20:58.067 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6 smithi169 running (11m) 71s ago 11m 87.2M 2500M 16.2.13-565-g595355f4 4099d2e00414 12854ca48e76 2023-08-28T02:20:58.067 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7 smithi169 running (11m) 71s ago 11m 105M 2500M 16.2.13-565-g595355f4 4099d2e00414 64fa1422c85a 2023-08-28T02:20:58.067 INFO:teuthology.orchestra.run.smithi143.stdout:prometheus.a smithi169 *:9095 running (5m) 71s ago 10m 68.3M - 2.33.4 514e6a882f6e cd763ce7d282 2023-08-28T02:20:58.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:58 smithi143 ceph-mon[183059]: pgmap v189: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:20:58.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:58 smithi143 ceph-mon[189871]: pgmap v189: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:20:58.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:58 smithi169 ceph-mon[159415]: pgmap v189: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:20:58.687 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -e sha1=9cf208207dfe23ffccb33308b22e51570cb45e51 -- bash -c 'ceph versions' 2023-08-28T02:20:59.438 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:20:59 smithi143 ceph-mon[183059]: from='client.34461 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:20:59.438 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:20:59 smithi143 ceph-mon[189871]: from='client.34461 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:20:59.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:20:59 smithi169 ceph-mon[159415]: from='client.34461 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-08-28T02:21:00.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:00 smithi143 ceph-mon[183059]: pgmap v190: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:00.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:00 smithi143 ceph-mon[189871]: pgmap v190: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:00.459 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:21:00.459 INFO:teuthology.orchestra.run.smithi143.stdout: "mon": { 2023-08-28T02:21:00.459 INFO:teuthology.orchestra.run.smithi143.stdout: "ceph version 18.2.0-187-g9cf20820 (9cf208207dfe23ffccb33308b22e51570cb45e51) reef (stable)": 3 2023-08-28T02:21:00.459 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:21:00.459 INFO:teuthology.orchestra.run.smithi143.stdout: "mgr": { 2023-08-28T02:21:00.459 INFO:teuthology.orchestra.run.smithi143.stdout: "ceph version 18.2.0-187-g9cf20820 (9cf208207dfe23ffccb33308b22e51570cb45e51) reef (stable)": 2 2023-08-28T02:21:00.459 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:21:00.459 INFO:teuthology.orchestra.run.smithi143.stdout: "osd": { 2023-08-28T02:21:00.460 INFO:teuthology.orchestra.run.smithi143.stdout: "ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)": 4, 2023-08-28T02:21:00.460 INFO:teuthology.orchestra.run.smithi143.stdout: "ceph version 18.2.0-187-g9cf20820 (9cf208207dfe23ffccb33308b22e51570cb45e51) reef (stable)": 4 2023-08-28T02:21:00.460 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:21:00.460 INFO:teuthology.orchestra.run.smithi143.stdout: "mds": { 2023-08-28T02:21:00.460 INFO:teuthology.orchestra.run.smithi143.stdout: "ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)": 2 2023-08-28T02:21:00.460 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:21:00.460 INFO:teuthology.orchestra.run.smithi143.stdout: "overall": { 2023-08-28T02:21:00.460 INFO:teuthology.orchestra.run.smithi143.stdout: "ceph version 16.2.13-565-g595355f4 (595355f450898939abedfa92883f9a78b18c37aa) pacific (stable)": 6, 2023-08-28T02:21:00.460 INFO:teuthology.orchestra.run.smithi143.stdout: "ceph version 18.2.0-187-g9cf20820 (9cf208207dfe23ffccb33308b22e51570cb45e51) reef (stable)": 9 2023-08-28T02:21:00.460 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:21:00.460 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:21:00.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:00 smithi169 ceph-mon[159415]: pgmap v190: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:00.953 DEBUG:teuthology.parallel:result is None 2023-08-28T02:21:00.954 INFO:teuthology.run_tasks:Running task print... 2023-08-28T02:21:00.970 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2023-08-28T02:21:00.970 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-08-28T02:21:00.986 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-08-28T02:21:00.986 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-08-28T02:21:00.986 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph mgr dump --format=json 2023-08-28T02:21:01.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:01 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/644103502' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:21:01.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:01 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/644103502' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:21:01.335 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:21:01.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:01 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/644103502' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-08-28T02:21:02.186 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:21:01 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:21:01] "GET /metrics HTTP/1.1" 200 34042 "" "Prometheus/2.33.4" 2023-08-28T02:21:02.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:02 smithi143 ceph-mon[189871]: pgmap v191: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:02.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:02 smithi143 ceph-mon[183059]: pgmap v191: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:02.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:02 smithi169 ceph-mon[159415]: pgmap v191: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:02.898 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:21:03.411 INFO:teuthology.orchestra.run.smithi143.stdout:{"epoch":35,"active_gid":34107,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6800","nonce":3219810900},{"type":"v1","addr":"172.21.15.143:6801","nonce":3219810900}]},"active_addr":"172.21.15.143:6801/3219810900","active_change":"2023-08-28T02:15:40.929973+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":25699,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"alertmanager_web_password":{"name":"alertmanager_web_password","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"Alertmanager web password","long_desc":"","tags":[],"see_also":[]},"alertmanager_web_user":{"name":"alertmanager_web_user","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"Alertmanager web user","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"prometheus_web_password":{"name":"prometheus_web_password","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"Prometheus web password","long_desc":"","tags":[],"see_also":[]},"prometheus_web_user":{"name":"prometheus_web_user","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"Prometheus web user","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"alertmanager_web_password":{"name":"alertmanager_web_password","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"Alertmanager web password","long_desc":"","tags":[],"see_also":[]},"alertmanager_web_user":{"name":"alertmanager_web_user","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"Alertmanager web user","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"prometheus_web_password":{"name":"prometheus_web_password","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"Prometheus web password","long_desc":"","tags":[],"see_also":[]},"prometheus_web_user":{"name":"prometheus_web_user","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"Prometheus web user","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.143:8443/","prometheus":"http://172.21.15.143:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":86,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.143:0","nonce":2643911674}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.143:0","nonce":2975908186}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.143:0","nonce":4149006921}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.143:0","nonce":1130413972}]}]} 2023-08-28T02:21:03.416 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-08-28T02:21:03.417 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-08-28T02:21:03.417 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd dump --format=json 2023-08-28T02:21:03.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:03 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/390316054' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-08-28T02:21:03.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:03 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/390316054' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-08-28T02:21:03.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:03 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/390316054' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-08-28T02:21:03.765 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:21:04.245 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:04 smithi143 ceph-mon[183059]: pgmap v192: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:04.246 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:04 smithi143 ceph-mon[189871]: pgmap v192: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:04.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:04 smithi169 ceph-mon[159415]: pgmap v192: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:05.253 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:21:05.254 INFO:teuthology.orchestra.run.smithi143.stdout:{"epoch":114,"fsid":"2fcb0bfa-4547-11ee-817c-1b7a5f1ba315","created":"2023-08-28T02:05:20.056816+0000","modified":"2023-08-28T02:20:51.806521+0000","last_up_change":"2023-08-28T02:20:49.801135+0000","last_in_change":"2023-08-28T02:09:35.717712+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-08-28T02:08:01.318811+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"83","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-08-28T02:10:30.618550+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"77","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":77,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-08-28T02:11:20.682059+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-08-28T02:11:21.407431+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"718c8f27-245b-4e9d-9e2e-d1fc65ca7472","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":86,"up_from":90,"up_thru":113,"down_at":87,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6802","nonce":257144662},{"type":"v1","addr":"172.21.15.143:6803","nonce":257144662}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6804","nonce":257144662},{"type":"v1","addr":"172.21.15.143:6805","nonce":257144662}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6808","nonce":257144662},{"type":"v1","addr":"172.21.15.143:6809","nonce":257144662}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6806","nonce":257144662},{"type":"v1","addr":"172.21.15.143:6807","nonce":257144662}]},"public_addr":"172.21.15.143:6803/257144662","cluster_addr":"172.21.15.143:6805/257144662","heartbeat_back_addr":"172.21.15.143:6809/257144662","heartbeat_front_addr":"172.21.15.143:6807/257144662","state":["exists","up"]},{"osd":1,"uuid":"cf24771f-3b0c-4a00-8a73-13e1fee242dc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":91,"up_from":98,"up_thru":112,"down_at":92,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6810","nonce":4171000628},{"type":"v1","addr":"172.21.15.143:6811","nonce":4171000628}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6812","nonce":4171000628},{"type":"v1","addr":"172.21.15.143:6813","nonce":4171000628}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6816","nonce":4171000628},{"type":"v1","addr":"172.21.15.143:6817","nonce":4171000628}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6814","nonce":4171000628},{"type":"v1","addr":"172.21.15.143:6815","nonce":4171000628}]},"public_addr":"172.21.15.143:6811/4171000628","cluster_addr":"172.21.15.143:6813/4171000628","heartbeat_back_addr":"172.21.15.143:6817/4171000628","heartbeat_front_addr":"172.21.15.143:6815/4171000628","state":["exists","up"]},{"osd":2,"uuid":"4f8492d1-0e7a-49f8-b6a5-4f5e73842349","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":100,"up_from":104,"up_thru":112,"down_at":101,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6818","nonce":1552459422},{"type":"v1","addr":"172.21.15.143:6819","nonce":1552459422}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6820","nonce":1552459422},{"type":"v1","addr":"172.21.15.143:6821","nonce":1552459422}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6824","nonce":1552459422},{"type":"v1","addr":"172.21.15.143:6825","nonce":1552459422}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6822","nonce":1552459422},{"type":"v1","addr":"172.21.15.143:6823","nonce":1552459422}]},"public_addr":"172.21.15.143:6819/1552459422","cluster_addr":"172.21.15.143:6821/1552459422","heartbeat_back_addr":"172.21.15.143:6825/1552459422","heartbeat_front_addr":"172.21.15.143:6823/1552459422","state":["exists","up"]},{"osd":3,"uuid":"dd6abc8a-54a9-4e08-ba81-0b894f24d84a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":105,"up_from":112,"up_thru":113,"down_at":106,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6826","nonce":2677897674},{"type":"v1","addr":"172.21.15.143:6827","nonce":2677897674}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6828","nonce":2677897674},{"type":"v1","addr":"172.21.15.143:6829","nonce":2677897674}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6832","nonce":2677897674},{"type":"v1","addr":"172.21.15.143:6833","nonce":2677897674}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6830","nonce":2677897674},{"type":"v1","addr":"172.21.15.143:6831","nonce":2677897674}]},"public_addr":"172.21.15.143:6827/2677897674","cluster_addr":"172.21.15.143:6829/2677897674","heartbeat_back_addr":"172.21.15.143:6833/2677897674","heartbeat_front_addr":"172.21.15.143:6831/2677897674","state":["exists","up"]},{"osd":4,"uuid":"16122219-d985-46d2-bac1-f8ffcf48d8ad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":113,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6800","nonce":3960017153},{"type":"v1","addr":"172.21.15.169:6801","nonce":3960017153}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6802","nonce":3960017153},{"type":"v1","addr":"172.21.15.169:6803","nonce":3960017153}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6806","nonce":3960017153},{"type":"v1","addr":"172.21.15.169:6807","nonce":3960017153}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6804","nonce":3960017153},{"type":"v1","addr":"172.21.15.169:6805","nonce":3960017153}]},"public_addr":"172.21.15.169:6801/3960017153","cluster_addr":"172.21.15.169:6803/3960017153","heartbeat_back_addr":"172.21.15.169:6807/3960017153","heartbeat_front_addr":"172.21.15.169:6805/3960017153","state":["exists","up"]},{"osd":5,"uuid":"adbc3ea5-230f-419e-8f6f-99763378b296","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":113,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6808","nonce":4087672846},{"type":"v1","addr":"172.21.15.169:6809","nonce":4087672846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6810","nonce":4087672846},{"type":"v1","addr":"172.21.15.169:6811","nonce":4087672846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6814","nonce":4087672846},{"type":"v1","addr":"172.21.15.169:6815","nonce":4087672846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6812","nonce":4087672846},{"type":"v1","addr":"172.21.15.169:6813","nonce":4087672846}]},"public_addr":"172.21.15.169:6809/4087672846","cluster_addr":"172.21.15.169:6811/4087672846","heartbeat_back_addr":"172.21.15.169:6815/4087672846","heartbeat_front_addr":"172.21.15.169:6813/4087672846","state":["exists","up"]},{"osd":6,"uuid":"2abe4d77-c12c-477f-a4ec-eabb753ca4dd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":113,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6816","nonce":3801227982},{"type":"v1","addr":"172.21.15.169:6817","nonce":3801227982}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6818","nonce":3801227982},{"type":"v1","addr":"172.21.15.169:6819","nonce":3801227982}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6822","nonce":3801227982},{"type":"v1","addr":"172.21.15.169:6823","nonce":3801227982}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6820","nonce":3801227982},{"type":"v1","addr":"172.21.15.169:6821","nonce":3801227982}]},"public_addr":"172.21.15.169:6817/3801227982","cluster_addr":"172.21.15.169:6819/3801227982","heartbeat_back_addr":"172.21.15.169:6823/3801227982","heartbeat_front_addr":"172.21.15.169:6821/3801227982","state":["exists","up"]},{"osd":7,"uuid":"fc5ed8fc-8ad7-4b54-a8ea-bc6a9e846f63","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":112,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6824","nonce":3830851149},{"type":"v1","addr":"172.21.15.169:6825","nonce":3830851149}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6826","nonce":3830851149},{"type":"v1","addr":"172.21.15.169:6827","nonce":3830851149}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6830","nonce":3830851149},{"type":"v1","addr":"172.21.15.169:6831","nonce":3830851149}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6828","nonce":3830851149},{"type":"v1","addr":"172.21.15.169:6829","nonce":3830851149}]},"public_addr":"172.21.15.169:6825/3830851149","cluster_addr":"172.21.15.169:6827/3830851149","heartbeat_back_addr":"172.21.15.169:6831/3830851149","heartbeat_front_addr":"172.21.15.169:6829/3830851149","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:07:16.166767+0000","dead_epoch":86},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:07:37.634849+0000","dead_epoch":91},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:07:59.099590+0000","dead_epoch":100},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:08:21.737546+0000","dead_epoch":105},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:08:43.377647+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:09:06.116513+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:09:28.557330+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:09:51.159131+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.9","mappings":[{"from":4,"to":3}]},{"pgid":"3.a","mappings":[{"from":4,"to":3}]},{"pgid":"3.f","mappings":[{"from":4,"to":2}]},{"pgid":"4.2","mappings":[{"from":1,"to":3}]},{"pgid":"4.4","mappings":[{"from":3,"to":5}]},{"pgid":"4.6","mappings":[{"from":1,"to":3}]},{"pgid":"4.a","mappings":[{"from":1,"to":3}]},{"pgid":"4.b","mappings":[{"from":4,"to":5}]},{"pgid":"4.d","mappings":[{"from":1,"to":3}]},{"pgid":"4.e","mappings":[{"from":4,"to":0}]},{"pgid":"4.10","mappings":[{"from":3,"to":0}]},{"pgid":"4.17","mappings":[{"from":3,"to":2}]},{"pgid":"4.18","mappings":[{"from":1,"to":3}]},{"pgid":"4.1d","mappings":[{"from":3,"to":2}]},{"pgid":"4.1f","mappings":[{"from":1,"to":3}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.143:0/2467953564":"2023-08-29T02:15:40.929762+0000","172.21.15.143:6800/155264805":"2023-08-29T02:15:40.929762+0000","172.21.15.143:0/437060254":"2023-08-29T02:15:40.929762+0000","172.21.15.143:6801/155264805":"2023-08-29T02:15:40.929762+0000","172.21.15.143:0/1563442416":"2023-08-29T02:10:09.845824+0000","172.21.15.169:0/338571757":"2023-08-29T02:15:01.746015+0000","172.21.15.169:0/2988270055":"2023-08-29T02:15:01.746015+0000","172.21.15.143:0/1148418770":"2023-08-29T02:05:41.797747+0000","172.21.15.143:0/11141072":"2023-08-29T02:05:41.797747+0000","172.21.15.143:6800/907474889":"2023-08-29T02:06:02.911068+0000","172.21.15.143:0/2157302459":"2023-08-29T02:10:09.845824+0000","172.21.15.143:0/1035910192":"2023-08-29T02:13:52.442893+0000","172.21.15.143:0/2083962846":"2023-08-29T02:15:40.929762+0000","172.21.15.169:0/3392857751":"2023-08-29T02:15:01.746015+0000","172.21.15.143:6801/1828180492":"2023-08-29T02:10:09.845824+0000","172.21.15.143:0/1948284892":"2023-08-29T02:15:40.929762+0000","172.21.15.143:0/64297767":"2023-08-29T02:15:40.929762+0000","172.21.15.143:0/3251765285":"2023-08-29T02:13:52.442893+0000","172.21.15.169:0/1064560701":"2023-08-29T02:15:01.746015+0000","172.21.15.143:0/3606471716":"2023-08-29T02:06:02.911068+0000","172.21.15.143:6801/1019944263":"2023-08-29T02:05:41.797747+0000","172.21.15.169:0/2187902729":"2023-08-29T02:15:01.746015+0000","172.21.15.143:0/2323551826":"2023-08-29T02:06:02.911068+0000","172.21.15.143:6800/1828180492":"2023-08-29T02:10:09.845824+0000","172.21.15.143:0/550889863":"2023-08-29T02:15:40.929762+0000","172.21.15.143:6801/907474889":"2023-08-29T02:06:02.911068+0000","172.21.15.143:6800/1019944263":"2023-08-29T02:05:41.797747+0000","172.21.15.143:6800/1118800746":"2023-08-29T02:13:52.442893+0000","172.21.15.169:6834/3955030351":"2023-08-29T02:15:01.746015+0000","172.21.15.169:6835/3955030351":"2023-08-29T02:15:01.746015+0000","172.21.15.143:6801/1118800746":"2023-08-29T02:13:52.442893+0000","172.21.15.169:0/2161024377":"2023-08-29T02:15:01.746015+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-08-28T02:21:05.769 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-08-28T02:21:05.769 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd dump --format=json 2023-08-28T02:21:06.161 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:21:06.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:06 smithi169 ceph-mon[159415]: pgmap v193: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:06.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:06 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3461758983' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-28T02:21:06.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:06 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:21:06.544 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:06 smithi143 ceph-mon[183059]: pgmap v193: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:06.544 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:06 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3461758983' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-28T02:21:06.545 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:06 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:21:06.545 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:06 smithi143 ceph-mon[189871]: pgmap v193: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:06.545 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:06 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3461758983' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-28T02:21:06.545 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:06 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:21:07.772 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:21:07.772 INFO:teuthology.orchestra.run.smithi143.stdout:{"epoch":114,"fsid":"2fcb0bfa-4547-11ee-817c-1b7a5f1ba315","created":"2023-08-28T02:05:20.056816+0000","modified":"2023-08-28T02:20:51.806521+0000","last_up_change":"2023-08-28T02:20:49.801135+0000","last_in_change":"2023-08-28T02:09:35.717712+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-08-28T02:08:01.318811+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"83","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-08-28T02:10:30.618550+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"77","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":77,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-08-28T02:11:20.682059+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-08-28T02:11:21.407431+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"718c8f27-245b-4e9d-9e2e-d1fc65ca7472","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":86,"up_from":90,"up_thru":113,"down_at":87,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6802","nonce":257144662},{"type":"v1","addr":"172.21.15.143:6803","nonce":257144662}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6804","nonce":257144662},{"type":"v1","addr":"172.21.15.143:6805","nonce":257144662}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6808","nonce":257144662},{"type":"v1","addr":"172.21.15.143:6809","nonce":257144662}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6806","nonce":257144662},{"type":"v1","addr":"172.21.15.143:6807","nonce":257144662}]},"public_addr":"172.21.15.143:6803/257144662","cluster_addr":"172.21.15.143:6805/257144662","heartbeat_back_addr":"172.21.15.143:6809/257144662","heartbeat_front_addr":"172.21.15.143:6807/257144662","state":["exists","up"]},{"osd":1,"uuid":"cf24771f-3b0c-4a00-8a73-13e1fee242dc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":91,"up_from":98,"up_thru":112,"down_at":92,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6810","nonce":4171000628},{"type":"v1","addr":"172.21.15.143:6811","nonce":4171000628}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6812","nonce":4171000628},{"type":"v1","addr":"172.21.15.143:6813","nonce":4171000628}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6816","nonce":4171000628},{"type":"v1","addr":"172.21.15.143:6817","nonce":4171000628}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6814","nonce":4171000628},{"type":"v1","addr":"172.21.15.143:6815","nonce":4171000628}]},"public_addr":"172.21.15.143:6811/4171000628","cluster_addr":"172.21.15.143:6813/4171000628","heartbeat_back_addr":"172.21.15.143:6817/4171000628","heartbeat_front_addr":"172.21.15.143:6815/4171000628","state":["exists","up"]},{"osd":2,"uuid":"4f8492d1-0e7a-49f8-b6a5-4f5e73842349","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":100,"up_from":104,"up_thru":112,"down_at":101,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6818","nonce":1552459422},{"type":"v1","addr":"172.21.15.143:6819","nonce":1552459422}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6820","nonce":1552459422},{"type":"v1","addr":"172.21.15.143:6821","nonce":1552459422}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6824","nonce":1552459422},{"type":"v1","addr":"172.21.15.143:6825","nonce":1552459422}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6822","nonce":1552459422},{"type":"v1","addr":"172.21.15.143:6823","nonce":1552459422}]},"public_addr":"172.21.15.143:6819/1552459422","cluster_addr":"172.21.15.143:6821/1552459422","heartbeat_back_addr":"172.21.15.143:6825/1552459422","heartbeat_front_addr":"172.21.15.143:6823/1552459422","state":["exists","up"]},{"osd":3,"uuid":"dd6abc8a-54a9-4e08-ba81-0b894f24d84a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":105,"up_from":112,"up_thru":113,"down_at":106,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6826","nonce":2677897674},{"type":"v1","addr":"172.21.15.143:6827","nonce":2677897674}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6828","nonce":2677897674},{"type":"v1","addr":"172.21.15.143:6829","nonce":2677897674}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6832","nonce":2677897674},{"type":"v1","addr":"172.21.15.143:6833","nonce":2677897674}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.143:6830","nonce":2677897674},{"type":"v1","addr":"172.21.15.143:6831","nonce":2677897674}]},"public_addr":"172.21.15.143:6827/2677897674","cluster_addr":"172.21.15.143:6829/2677897674","heartbeat_back_addr":"172.21.15.143:6833/2677897674","heartbeat_front_addr":"172.21.15.143:6831/2677897674","state":["exists","up"]},{"osd":4,"uuid":"16122219-d985-46d2-bac1-f8ffcf48d8ad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":113,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6800","nonce":3960017153},{"type":"v1","addr":"172.21.15.169:6801","nonce":3960017153}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6802","nonce":3960017153},{"type":"v1","addr":"172.21.15.169:6803","nonce":3960017153}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6806","nonce":3960017153},{"type":"v1","addr":"172.21.15.169:6807","nonce":3960017153}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6804","nonce":3960017153},{"type":"v1","addr":"172.21.15.169:6805","nonce":3960017153}]},"public_addr":"172.21.15.169:6801/3960017153","cluster_addr":"172.21.15.169:6803/3960017153","heartbeat_back_addr":"172.21.15.169:6807/3960017153","heartbeat_front_addr":"172.21.15.169:6805/3960017153","state":["exists","up"]},{"osd":5,"uuid":"adbc3ea5-230f-419e-8f6f-99763378b296","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":113,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6808","nonce":4087672846},{"type":"v1","addr":"172.21.15.169:6809","nonce":4087672846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6810","nonce":4087672846},{"type":"v1","addr":"172.21.15.169:6811","nonce":4087672846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6814","nonce":4087672846},{"type":"v1","addr":"172.21.15.169:6815","nonce":4087672846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6812","nonce":4087672846},{"type":"v1","addr":"172.21.15.169:6813","nonce":4087672846}]},"public_addr":"172.21.15.169:6809/4087672846","cluster_addr":"172.21.15.169:6811/4087672846","heartbeat_back_addr":"172.21.15.169:6815/4087672846","heartbeat_front_addr":"172.21.15.169:6813/4087672846","state":["exists","up"]},{"osd":6,"uuid":"2abe4d77-c12c-477f-a4ec-eabb753ca4dd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":113,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6816","nonce":3801227982},{"type":"v1","addr":"172.21.15.169:6817","nonce":3801227982}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6818","nonce":3801227982},{"type":"v1","addr":"172.21.15.169:6819","nonce":3801227982}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6822","nonce":3801227982},{"type":"v1","addr":"172.21.15.169:6823","nonce":3801227982}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6820","nonce":3801227982},{"type":"v1","addr":"172.21.15.169:6821","nonce":3801227982}]},"public_addr":"172.21.15.169:6817/3801227982","cluster_addr":"172.21.15.169:6819/3801227982","heartbeat_back_addr":"172.21.15.169:6823/3801227982","heartbeat_front_addr":"172.21.15.169:6821/3801227982","state":["exists","up"]},{"osd":7,"uuid":"fc5ed8fc-8ad7-4b54-a8ea-bc6a9e846f63","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":112,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6824","nonce":3830851149},{"type":"v1","addr":"172.21.15.169:6825","nonce":3830851149}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6826","nonce":3830851149},{"type":"v1","addr":"172.21.15.169:6827","nonce":3830851149}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6830","nonce":3830851149},{"type":"v1","addr":"172.21.15.169:6831","nonce":3830851149}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.169:6828","nonce":3830851149},{"type":"v1","addr":"172.21.15.169:6829","nonce":3830851149}]},"public_addr":"172.21.15.169:6825/3830851149","cluster_addr":"172.21.15.169:6827/3830851149","heartbeat_back_addr":"172.21.15.169:6831/3830851149","heartbeat_front_addr":"172.21.15.169:6829/3830851149","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:07:16.166767+0000","dead_epoch":86},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:07:37.634849+0000","dead_epoch":91},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:07:59.099590+0000","dead_epoch":100},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:08:21.737546+0000","dead_epoch":105},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:08:43.377647+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:09:06.116513+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:09:28.557330+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-08-28T02:09:51.159131+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.9","mappings":[{"from":4,"to":3}]},{"pgid":"3.a","mappings":[{"from":4,"to":3}]},{"pgid":"3.f","mappings":[{"from":4,"to":2}]},{"pgid":"4.2","mappings":[{"from":1,"to":3}]},{"pgid":"4.4","mappings":[{"from":3,"to":5}]},{"pgid":"4.6","mappings":[{"from":1,"to":3}]},{"pgid":"4.a","mappings":[{"from":1,"to":3}]},{"pgid":"4.b","mappings":[{"from":4,"to":5}]},{"pgid":"4.d","mappings":[{"from":1,"to":3}]},{"pgid":"4.e","mappings":[{"from":4,"to":0}]},{"pgid":"4.10","mappings":[{"from":3,"to":0}]},{"pgid":"4.17","mappings":[{"from":3,"to":2}]},{"pgid":"4.18","mappings":[{"from":1,"to":3}]},{"pgid":"4.1d","mappings":[{"from":3,"to":2}]},{"pgid":"4.1f","mappings":[{"from":1,"to":3}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.169:0/338571757":"2023-08-29T02:15:01.746015+0000","172.21.15.169:0/2988270055":"2023-08-29T02:15:01.746015+0000","172.21.15.143:0/1148418770":"2023-08-29T02:05:41.797747+0000","172.21.15.143:0/11141072":"2023-08-29T02:05:41.797747+0000","172.21.15.143:6800/907474889":"2023-08-29T02:06:02.911068+0000","172.21.15.143:0/2157302459":"2023-08-29T02:10:09.845824+0000","172.21.15.143:0/2467953564":"2023-08-29T02:15:40.929762+0000","172.21.15.143:0/2083962846":"2023-08-29T02:15:40.929762+0000","172.21.15.169:0/3392857751":"2023-08-29T02:15:01.746015+0000","172.21.15.169:0/2161024377":"2023-08-29T02:15:01.746015+0000","172.21.15.143:6801/1118800746":"2023-08-29T02:13:52.442893+0000","172.21.15.143:6800/155264805":"2023-08-29T02:15:40.929762+0000","172.21.15.143:6801/1828180492":"2023-08-29T02:10:09.845824+0000","172.21.15.169:0/1064560701":"2023-08-29T02:15:01.746015+0000","172.21.15.143:0/3606471716":"2023-08-29T02:06:02.911068+0000","172.21.15.143:6801/1019944263":"2023-08-29T02:05:41.797747+0000","172.21.15.169:0/2187902729":"2023-08-29T02:15:01.746015+0000","172.21.15.143:0/2323551826":"2023-08-29T02:06:02.911068+0000","172.21.15.143:6800/1828180492":"2023-08-29T02:10:09.845824+0000","172.21.15.143:0/1563442416":"2023-08-29T02:10:09.845824+0000","172.21.15.143:0/437060254":"2023-08-29T02:15:40.929762+0000","172.21.15.143:6801/155264805":"2023-08-29T02:15:40.929762+0000","172.21.15.143:0/1948284892":"2023-08-29T02:15:40.929762+0000","172.21.15.143:0/3251765285":"2023-08-29T02:13:52.442893+0000","172.21.15.143:0/64297767":"2023-08-29T02:15:40.929762+0000","172.21.15.143:0/1035910192":"2023-08-29T02:13:52.442893+0000","172.21.15.143:6801/907474889":"2023-08-29T02:06:02.911068+0000","172.21.15.143:0/550889863":"2023-08-29T02:15:40.929762+0000","172.21.15.143:6800/1019944263":"2023-08-29T02:05:41.797747+0000","172.21.15.143:6800/1118800746":"2023-08-29T02:13:52.442893+0000","172.21.15.169:6834/3955030351":"2023-08-29T02:15:01.746015+0000","172.21.15.169:6835/3955030351":"2023-08-29T02:15:01.746015+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-08-28T02:21:08.327 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph tell osd.0 flush_pg_stats 2023-08-28T02:21:08.328 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph tell osd.1 flush_pg_stats 2023-08-28T02:21:08.328 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph tell osd.2 flush_pg_stats 2023-08-28T02:21:08.328 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph tell osd.3 flush_pg_stats 2023-08-28T02:21:08.329 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph tell osd.4 flush_pg_stats 2023-08-28T02:21:08.329 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph tell osd.5 flush_pg_stats 2023-08-28T02:21:08.330 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph tell osd.6 flush_pg_stats 2023-08-28T02:21:08.330 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph tell osd.7 flush_pg_stats 2023-08-28T02:21:08.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:08 smithi169 ceph-mon[159415]: pgmap v194: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:08.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:08 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/4113749414' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-28T02:21:08.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:08 smithi143 ceph-mon[183059]: pgmap v194: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:08.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:08 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/4113749414' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-28T02:21:08.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:08 smithi143 ceph-mon[189871]: pgmap v194: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:08.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:08 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/4113749414' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-28T02:21:09.920 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:21:09.971 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:21:10.035 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:21:10.111 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:09 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:21:10.111 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:09 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:21:10.111 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:09 smithi143 ceph-mon[183059]: pgmap v195: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:10.111 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:09 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:21:10.111 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:09 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:21:10.111 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:09 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:21:10.111 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:09 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:21:10.111 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:09 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:21:10.111 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:09 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:21:10.111 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:09 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:21:10.112 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:09 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:21:10.112 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:09 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:21:10.112 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:09 smithi143 ceph-mon[189871]: pgmap v195: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:10.112 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:09 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:21:10.112 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:09 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:21:10.112 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:09 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:21:10.112 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:09 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:21:10.112 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:09 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:21:10.112 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:09 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:21:10.112 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:09 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:21:10.125 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:21:10.203 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:21:10.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:09 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:21:10.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:09 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:21:10.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:09 smithi169 ceph-mon[159415]: pgmap v195: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:10.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:09 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:21:10.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:09 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:21:10.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:09 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:21:10.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:09 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:21:10.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:09 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:21:10.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:09 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:21:10.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:09 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:21:10.389 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:21:10.519 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:21:10.683 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:21:11.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:11 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:21:11.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:11 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:21:11.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:11 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:21:12.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:12 smithi143 ceph-mon[183059]: pgmap v196: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:12.201 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:21:11 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:21:11] "GET /metrics HTTP/1.1" 200 34046 "" "Prometheus/2.33.4" 2023-08-28T02:21:12.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:12 smithi143 ceph-mon[189871]: pgmap v196: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:12.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:12 smithi169 ceph-mon[159415]: pgmap v196: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:14.443 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:14 smithi143 ceph-mon[183059]: pgmap v197: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:14.443 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:14 smithi143 ceph-mon[189871]: pgmap v197: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:14.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:14 smithi169 ceph-mon[159415]: pgmap v197: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:16.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:16 smithi143 ceph-mon[183059]: pgmap v198: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:16.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:16 smithi143 ceph-mon[189871]: pgmap v198: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:16.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:16 smithi169 ceph-mon[159415]: pgmap v198: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:17.527 INFO:teuthology.orchestra.run.smithi143.stdout:180388626574 2023-08-28T02:21:17.527 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd last-stat-seq osd.6 2023-08-28T02:21:18.139 INFO:teuthology.orchestra.run.smithi143.stdout:206158430347 2023-08-28T02:21:18.139 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd last-stat-seq osd.7 2023-08-28T02:21:18.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:18 smithi143 ceph-mon[183059]: pgmap v199: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:18.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:18 smithi143 ceph-mon[189871]: pgmap v199: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:18.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:18 smithi169 ceph-mon[159415]: pgmap v199: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:19.025 INFO:teuthology.orchestra.run.smithi143.stdout:128849019032 2023-08-28T02:21:19.025 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd last-stat-seq osd.4 2023-08-28T02:21:19.467 INFO:teuthology.orchestra.run.smithi143.stdout:481036337159 2023-08-28T02:21:19.468 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd last-stat-seq osd.3 2023-08-28T02:21:20.170 INFO:teuthology.orchestra.run.smithi143.stdout:386547056674 2023-08-28T02:21:20.171 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd last-stat-seq osd.0 2023-08-28T02:21:20.435 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:20 smithi143 ceph-mon[183059]: pgmap v200: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:20.435 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:20 smithi143 ceph-mon[189871]: pgmap v200: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:20.436 INFO:teuthology.orchestra.run.smithi143.stdout:446676598797 2023-08-28T02:21:20.437 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd last-stat-seq osd.2 2023-08-28T02:21:20.466 INFO:teuthology.orchestra.run.smithi143.stdout:420906795027 2023-08-28T02:21:20.467 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd last-stat-seq osd.1 2023-08-28T02:21:20.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:20 smithi169 ceph-mon[159415]: pgmap v200: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:21.052 INFO:teuthology.orchestra.run.smithi143.stdout:154618822805 2023-08-28T02:21:21.053 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd last-stat-seq osd.5 2023-08-28T02:21:21.184 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:21:21.231 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:21:21.308 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:21:21.334 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:21:21.386 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:21:21.485 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:21:21.796 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:21:22.078 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:21:21 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:21:21] "GET /metrics HTTP/1.1" 200 34046 "" "Prometheus/2.33.4" 2023-08-28T02:21:22.362 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:22 smithi143 ceph-mon[183059]: pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:22.363 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:22 smithi143 ceph-mon[189871]: pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:22.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:22 smithi169 ceph-mon[159415]: pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:23.778 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:21:24.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:24 smithi143 ceph-mon[183059]: pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:24.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:24 smithi143 ceph-mon[189871]: pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:24.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:24 smithi169 ceph-mon[159415]: pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:25.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:25 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/742199130' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-08-28T02:21:25.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:25 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2478121174' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-08-28T02:21:25.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:25 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/742199130' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-08-28T02:21:25.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:25 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2478121174' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-08-28T02:21:25.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:25 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/742199130' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-08-28T02:21:25.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:25 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2478121174' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-08-28T02:21:25.890 INFO:teuthology.orchestra.run.smithi143.stdout:128849019034 2023-08-28T02:21:26.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:26 smithi143 ceph-mon[183059]: pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:26.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:26 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:21:26.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:26 smithi143 ceph-mon[189871]: pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:26.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:26 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:21:26.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:26 smithi169 ceph-mon[159415]: pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:26.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:26 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:21:26.949 INFO:teuthology.orchestra.run.smithi143.stdout:206158430349 2023-08-28T02:21:27.406 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849019032 got 128849019034 for osd.4 2023-08-28T02:21:27.406 DEBUG:teuthology.parallel:result is None 2023-08-28T02:21:27.972 INFO:teuthology.orchestra.run.smithi143.stdout:386547056675 2023-08-28T02:21:28.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:28 smithi143 ceph-mon[183059]: pgmap v204: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:28.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:28 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2205185902' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-08-28T02:21:28.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:28 smithi143 ceph-mon[189871]: pgmap v204: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:28.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:28 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2205185902' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-08-28T02:21:28.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:28 smithi169 ceph-mon[159415]: pgmap v204: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:28.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:28 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2205185902' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-08-28T02:21:28.818 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430347 got 206158430349 for osd.7 2023-08-28T02:21:28.818 DEBUG:teuthology.parallel:result is None 2023-08-28T02:21:29.139 INFO:teuthology.orchestra.run.smithi143.stdout:481036337161 2023-08-28T02:21:29.188 INFO:teuthology.orchestra.run.smithi143.stdout:180388626577 2023-08-28T02:21:29.404 INFO:teuthology.orchestra.run.smithi143.stdout:420906795029 2023-08-28T02:21:29.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:29 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3452944355' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-08-28T02:21:29.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:29 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/640727011' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-08-28T02:21:29.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:29 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3452944355' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-08-28T02:21:29.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:29 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/640727011' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-08-28T02:21:29.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:29 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3452944355' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-08-28T02:21:29.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:29 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/640727011' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-08-28T02:21:30.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:30 smithi143 ceph-mon[183059]: pgmap v205: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:30.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:30 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2577690694' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-08-28T02:21:30.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:30 smithi143 ceph-mon[189871]: pgmap v205: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:30.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:30 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2577690694' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-08-28T02:21:30.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:30 smithi169 ceph-mon[159415]: pgmap v205: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:30.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:30 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2577690694' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-08-28T02:21:30.516 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626574 got 180388626577 for osd.6 2023-08-28T02:21:30.516 DEBUG:teuthology.parallel:result is None 2023-08-28T02:21:30.722 INFO:tasks.cephadm.ceph_manager.ceph:need seq 481036337159 got 481036337161 for osd.3 2023-08-28T02:21:30.722 DEBUG:teuthology.parallel:result is None 2023-08-28T02:21:30.723 INFO:tasks.cephadm.ceph_manager.ceph:need seq 386547056674 got 386547056675 for osd.0 2023-08-28T02:21:30.723 DEBUG:teuthology.parallel:result is None 2023-08-28T02:21:31.006 INFO:tasks.cephadm.ceph_manager.ceph:need seq 420906795027 got 420906795029 for osd.1 2023-08-28T02:21:31.006 DEBUG:teuthology.parallel:result is None 2023-08-28T02:21:31.470 INFO:teuthology.orchestra.run.smithi143.stdout:154618822807 2023-08-28T02:21:31.548 INFO:teuthology.orchestra.run.smithi143.stdout:446676598800 2023-08-28T02:21:32.179 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:32 smithi143 ceph-mon[183059]: pgmap v206: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:32.180 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:32 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/4235560283' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-08-28T02:21:32.180 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:32 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2582841834' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-08-28T02:21:32.180 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:32 smithi143 ceph-mon[189871]: pgmap v206: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:32.180 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:32 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/4235560283' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-08-28T02:21:32.180 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:32 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2582841834' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-08-28T02:21:32.180 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:21:31 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:21:31] "GET /metrics HTTP/1.1" 200 34040 "" "Prometheus/2.33.4" 2023-08-28T02:21:32.198 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822805 got 154618822807 for osd.5 2023-08-28T02:21:32.198 DEBUG:teuthology.parallel:result is None 2023-08-28T02:21:32.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:32 smithi169 ceph-mon[159415]: pgmap v206: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:32.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:32 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/4235560283' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-08-28T02:21:32.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:32 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2582841834' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-08-28T02:21:32.591 INFO:tasks.cephadm.ceph_manager.ceph:need seq 446676598797 got 446676598800 for osd.2 2023-08-28T02:21:32.592 DEBUG:teuthology.parallel:result is None 2023-08-28T02:21:32.592 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-08-28T02:21:32.592 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph pg dump --format=json 2023-08-28T02:21:32.942 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:21:34.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:34 smithi143 ceph-mon[183059]: pgmap v207: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:34.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:34 smithi143 ceph-mon[189871]: pgmap v207: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:34.480 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:21:34.482 INFO:teuthology.orchestra.run.smithi143.stderr:dumped all 2023-08-28T02:21:34.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:34 smithi169 ceph-mon[159415]: pgmap v207: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:35.024 INFO:teuthology.orchestra.run.smithi143.stdout:{"pg_ready":true,"pg_map":{"version":207,"stamp":"2023-08-28T02:21:33.032391+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6288,"num_read_kb":13719,"num_write":1106,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1341828,"kb_used_data":23364,"kb_used_omap":29,"kb_used_meta":1318370,"kb_avail":748619388,"statfs":{"total":767960285184,"available":766586253312,"internally_reserved":0,"allocated":23924736,"data_stored":19705380,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30674,"internal_metadata":1350010926},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002732"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":100,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.824785+0000","last_change":"2023-08-28T02:20:51.824785+0000","last_active":"2023-08-28T02:20:51.824785+0000","last_peered":"2023-08-28T02:20:51.824785+0000","last_clean":"2023-08-28T02:20:51.824785+0000","last_became_active":"2023-08-28T02:20:51.824447+0000","last_became_peered":"2023-08-28T02:20:51.824447+0000","last_unstale":"2023-08-28T02:20:51.824785+0000","last_undegraded":"2023-08-28T02:20:51.824785+0000","last_fullsized":"2023-08-28T02:20:51.824785+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":65,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.813241+0000","last_change":"2023-08-28T02:20:19.850252+0000","last_active":"2023-08-28T02:20:50.813241+0000","last_peered":"2023-08-28T02:20:50.813241+0000","last_clean":"2023-08-28T02:20:50.813241+0000","last_became_active":"2023-08-28T02:20:19.849913+0000","last_became_peered":"2023-08-28T02:20:19.849913+0000","last_unstale":"2023-08-28T02:20:50.813241+0000","last_undegraded":"2023-08-28T02:20:50.813241+0000","last_fullsized":"2023-08-28T02:20:50.813241+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:48:47.713465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":98,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.826985+0000","last_change":"2023-08-28T02:20:50.823101+0000","last_active":"2023-08-28T02:20:50.826985+0000","last_peered":"2023-08-28T02:20:50.826985+0000","last_clean":"2023-08-28T02:20:50.826985+0000","last_became_active":"2023-08-28T02:20:50.822599+0000","last_became_peered":"2023-08-28T02:20:50.822599+0000","last_unstale":"2023-08-28T02:20:50.826985+0000","last_undegraded":"2023-08-28T02:20:50.826985+0000","last_fullsized":"2023-08-28T02:20:50.826985+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:28:57.827789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":85,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.826313+0000","last_change":"2023-08-28T02:20:50.825915+0000","last_active":"2023-08-28T02:20:50.826313+0000","last_peered":"2023-08-28T02:20:50.826313+0000","last_clean":"2023-08-28T02:20:50.826313+0000","last_became_active":"2023-08-28T02:20:50.825526+0000","last_became_peered":"2023-08-28T02:20:50.825526+0000","last_unstale":"2023-08-28T02:20:50.826313+0000","last_undegraded":"2023-08-28T02:20:50.826313+0000","last_fullsized":"2023-08-28T02:20:50.826313+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:25:20.142641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":95,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.820522+0000","last_change":"2023-08-28T02:20:50.819786+0000","last_active":"2023-08-28T02:20:50.820522+0000","last_peered":"2023-08-28T02:20:50.820522+0000","last_clean":"2023-08-28T02:20:50.820522+0000","last_became_active":"2023-08-28T02:20:50.819451+0000","last_became_peered":"2023-08-28T02:20:50.819451+0000","last_unstale":"2023-08-28T02:20:50.820522+0000","last_undegraded":"2023-08-28T02:20:50.820522+0000","last_fullsized":"2023-08-28T02:20:50.820522+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":68,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.824216+0000","last_change":"2023-08-28T02:11:22.186024+0000","last_active":"2023-08-28T02:20:50.824216+0000","last_peered":"2023-08-28T02:20:50.824216+0000","last_clean":"2023-08-28T02:20:50.824216+0000","last_became_active":"2023-08-28T02:11:22.185500+0000","last_became_peered":"2023-08-28T02:11:22.185500+0000","last_unstale":"2023-08-28T02:20:50.824216+0000","last_undegraded":"2023-08-28T02:20:50.824216+0000","last_fullsized":"2023-08-28T02:20:50.824216+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":95,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.820006+0000","last_change":"2023-08-28T02:20:50.819015+0000","last_active":"2023-08-28T02:20:50.820006+0000","last_peered":"2023-08-28T02:20:50.820006+0000","last_clean":"2023-08-28T02:20:50.820006+0000","last_became_active":"2023-08-28T02:20:50.818667+0000","last_became_peered":"2023-08-28T02:20:50.818667+0000","last_unstale":"2023-08-28T02:20:50.820006+0000","last_undegraded":"2023-08-28T02:20:50.820006+0000","last_fullsized":"2023-08-28T02:20:50.820006+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":83,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.822081+0000","last_change":"2023-08-28T02:19:50.218760+0000","last_active":"2023-08-28T02:20:50.822081+0000","last_peered":"2023-08-28T02:20:50.822081+0000","last_clean":"2023-08-28T02:20:50.822081+0000","last_became_active":"2023-08-28T02:19:50.218376+0000","last_became_peered":"2023-08-28T02:19:50.218376+0000","last_unstale":"2023-08-28T02:20:50.822081+0000","last_undegraded":"2023-08-28T02:20:50.822081+0000","last_fullsized":"2023-08-28T02:20:50.822081+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":116,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.817585+0000","last_change":"2023-08-28T02:20:50.817119+0000","last_active":"2023-08-28T02:20:50.817585+0000","last_peered":"2023-08-28T02:20:50.817585+0000","last_clean":"2023-08-28T02:20:50.817585+0000","last_became_active":"2023-08-28T02:20:50.816756+0000","last_became_peered":"2023-08-28T02:20:50.816756+0000","last_unstale":"2023-08-28T02:20:50.817585+0000","last_undegraded":"2023-08-28T02:20:50.817585+0000","last_fullsized":"2023-08-28T02:20:50.817585+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:12:52.257562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":50,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.817184+0000","last_change":"2023-08-28T02:18:39.781312+0000","last_active":"2023-08-28T02:20:51.817184+0000","last_peered":"2023-08-28T02:20:51.817184+0000","last_clean":"2023-08-28T02:20:51.817184+0000","last_became_active":"2023-08-28T02:18:39.780916+0000","last_became_peered":"2023-08-28T02:18:39.780916+0000","last_unstale":"2023-08-28T02:20:51.817184+0000","last_undegraded":"2023-08-28T02:20:51.817184+0000","last_fullsized":"2023-08-28T02:20:51.817184+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:37:11.571250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":65,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.814304+0000","last_change":"2023-08-28T02:11:22.437146+0000","last_active":"2023-08-28T02:20:50.814304+0000","last_peered":"2023-08-28T02:20:50.814304+0000","last_clean":"2023-08-28T02:20:50.814304+0000","last_became_active":"2023-08-28T02:11:22.436798+0000","last_became_peered":"2023-08-28T02:11:22.436798+0000","last_unstale":"2023-08-28T02:20:50.814304+0000","last_undegraded":"2023-08-28T02:20:50.814304+0000","last_fullsized":"2023-08-28T02:20:50.814304+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":65,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.813953+0000","last_change":"2023-08-28T02:19:50.221001+0000","last_active":"2023-08-28T02:20:50.813953+0000","last_peered":"2023-08-28T02:20:50.813953+0000","last_clean":"2023-08-28T02:20:50.813953+0000","last_became_active":"2023-08-28T02:19:50.220611+0000","last_became_peered":"2023-08-28T02:19:50.220611+0000","last_unstale":"2023-08-28T02:20:50.813953+0000","last_undegraded":"2023-08-28T02:20:50.813953+0000","last_fullsized":"2023-08-28T02:20:50.813953+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:23:10.637630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'122","reported_seq":570,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.816897+0000","last_change":"2023-08-28T02:20:19.851065+0000","last_active":"2023-08-28T02:20:50.816897+0000","last_peered":"2023-08-28T02:20:50.816897+0000","last_clean":"2023-08-28T02:20:50.816897+0000","last_became_active":"2023-08-28T02:20:19.850477+0000","last_became_peered":"2023-08-28T02:20:19.850477+0000","last_unstale":"2023-08-28T02:20:50.816897+0000","last_undegraded":"2023-08-28T02:20:50.816897+0000","last_fullsized":"2023-08-28T02:20:50.816897+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":857,"num_read_kb":2224,"num_write":99,"num_write_kb":2312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":70,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.823860+0000","last_change":"2023-08-28T02:20:50.822679+0000","last_active":"2023-08-28T02:20:50.823860+0000","last_peered":"2023-08-28T02:20:50.823860+0000","last_clean":"2023-08-28T02:20:50.823860+0000","last_became_active":"2023-08-28T02:20:50.822180+0000","last_became_peered":"2023-08-28T02:20:50.822180+0000","last_unstale":"2023-08-28T02:20:50.823860+0000","last_undegraded":"2023-08-28T02:20:50.823860+0000","last_fullsized":"2023-08-28T02:20:50.823860+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:02:40.005023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'112","reported_seq":356,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.809841+0000","last_change":"2023-08-28T02:19:50.218206+0000","last_active":"2023-08-28T02:20:50.809841+0000","last_peered":"2023-08-28T02:20:50.809841+0000","last_clean":"2023-08-28T02:20:50.809841+0000","last_became_active":"2023-08-28T02:19:50.217637+0000","last_became_peered":"2023-08-28T02:19:50.217637+0000","last_unstale":"2023-08-28T02:20:50.809841+0000","last_undegraded":"2023-08-28T02:20:50.809841+0000","last_fullsized":"2023-08-28T02:20:50.809841+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:12:06.224907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":444,"num_read_kb":473,"num_write":94,"num_write_kb":127,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":73,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.823867+0000","last_change":"2023-08-28T02:20:50.823477+0000","last_active":"2023-08-28T02:20:50.823867+0000","last_peered":"2023-08-28T02:20:50.823867+0000","last_clean":"2023-08-28T02:20:50.823867+0000","last_became_active":"2023-08-28T02:20:50.823158+0000","last_became_peered":"2023-08-28T02:20:50.823158+0000","last_unstale":"2023-08-28T02:20:50.823867+0000","last_undegraded":"2023-08-28T02:20:50.823867+0000","last_fullsized":"2023-08-28T02:20:50.823867+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:33:18.347341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":47,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.813160+0000","last_change":"2023-08-28T02:18:39.780069+0000","last_active":"2023-08-28T02:20:50.813160+0000","last_peered":"2023-08-28T02:20:50.813160+0000","last_clean":"2023-08-28T02:20:50.813160+0000","last_became_active":"2023-08-28T02:18:39.779829+0000","last_became_peered":"2023-08-28T02:18:39.779829+0000","last_unstale":"2023-08-28T02:20:50.813160+0000","last_undegraded":"2023-08-28T02:20:50.813160+0000","last_fullsized":"2023-08-28T02:20:50.813160+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:35:01.309659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":52,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.810821+0000","last_change":"2023-08-28T02:20:19.850916+0000","last_active":"2023-08-28T02:20:50.810821+0000","last_peered":"2023-08-28T02:20:50.810821+0000","last_clean":"2023-08-28T02:20:50.810821+0000","last_became_active":"2023-08-28T02:20:19.850627+0000","last_became_peered":"2023-08-28T02:20:19.850627+0000","last_unstale":"2023-08-28T02:20:50.810821+0000","last_undegraded":"2023-08-28T02:20:50.810821+0000","last_fullsized":"2023-08-28T02:20:50.810821+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:19:14.346189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'103","reported_seq":535,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.820309+0000","last_change":"2023-08-28T02:18:39.786876+0000","last_active":"2023-08-28T02:20:50.820309+0000","last_peered":"2023-08-28T02:20:50.820309+0000","last_clean":"2023-08-28T02:20:50.820309+0000","last_became_active":"2023-08-28T02:18:39.786414+0000","last_became_peered":"2023-08-28T02:18:39.786414+0000","last_unstale":"2023-08-28T02:20:50.820309+0000","last_undegraded":"2023-08-28T02:20:50.820309+0000","last_fullsized":"2023-08-28T02:20:50.820309+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":803,"num_read_kb":2080,"num_write":73,"num_write_kb":1244,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":97,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.886782+0000","last_change":"2023-08-28T02:20:51.886782+0000","last_active":"2023-08-28T02:20:51.886782+0000","last_peered":"2023-08-28T02:20:51.886782+0000","last_clean":"2023-08-28T02:20:51.886782+0000","last_became_active":"2023-08-28T02:20:51.886444+0000","last_became_peered":"2023-08-28T02:20:51.886444+0000","last_unstale":"2023-08-28T02:20:51.886782+0000","last_undegraded":"2023-08-28T02:20:51.886782+0000","last_fullsized":"2023-08-28T02:20:51.886782+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:57:03.704951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":61,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.823573+0000","last_change":"2023-08-28T02:20:50.823036+0000","last_active":"2023-08-28T02:20:50.823573+0000","last_peered":"2023-08-28T02:20:50.823573+0000","last_clean":"2023-08-28T02:20:50.823573+0000","last_became_active":"2023-08-28T02:20:50.822766+0000","last_became_peered":"2023-08-28T02:20:50.822766+0000","last_unstale":"2023-08-28T02:20:50.823573+0000","last_undegraded":"2023-08-28T02:20:50.823573+0000","last_fullsized":"2023-08-28T02:20:50.823573+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:09:26.786992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"78'148","reported_seq":659,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.818235+0000","last_change":"2023-08-28T02:19:50.221311+0000","last_active":"2023-08-28T02:20:50.818235+0000","last_peered":"2023-08-28T02:20:50.818235+0000","last_clean":"2023-08-28T02:20:50.818235+0000","last_became_active":"2023-08-28T02:19:50.220754+0000","last_became_peered":"2023-08-28T02:19:50.220754+0000","last_unstale":"2023-08-28T02:20:50.818235+0000","last_undegraded":"2023-08-28T02:20:50.818235+0000","last_fullsized":"2023-08-28T02:20:50.818235+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1092,"num_read_kb":1955,"num_write":127,"num_write_kb":5177,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":65,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.816396+0000","last_change":"2023-08-28T02:11:23.249640+0000","last_active":"2023-08-28T02:20:50.816396+0000","last_peered":"2023-08-28T02:20:50.816396+0000","last_clean":"2023-08-28T02:20:50.816396+0000","last_became_active":"2023-08-28T02:11:23.249355+0000","last_became_peered":"2023-08-28T02:11:23.249355+0000","last_unstale":"2023-08-28T02:20:50.816396+0000","last_undegraded":"2023-08-28T02:20:50.816396+0000","last_fullsized":"2023-08-28T02:20:50.816396+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":64,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.826633+0000","last_change":"2023-08-28T02:20:50.825728+0000","last_active":"2023-08-28T02:20:50.826633+0000","last_peered":"2023-08-28T02:20:50.826633+0000","last_clean":"2023-08-28T02:20:50.826633+0000","last_became_active":"2023-08-28T02:20:50.825418+0000","last_became_peered":"2023-08-28T02:20:50.825418+0000","last_unstale":"2023-08-28T02:20:50.826633+0000","last_undegraded":"2023-08-28T02:20:50.826633+0000","last_fullsized":"2023-08-28T02:20:50.826633+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:55:48.126269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"86'139","reported_seq":317,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.818120+0000","last_change":"2023-08-28T02:18:39.786823+0000","last_active":"2023-08-28T02:20:50.818120+0000","last_peered":"2023-08-28T02:20:50.818120+0000","last_clean":"2023-08-28T02:20:50.818120+0000","last_became_active":"2023-08-28T02:18:39.786357+0000","last_became_peered":"2023-08-28T02:18:39.786357+0000","last_unstale":"2023-08-28T02:20:50.818120+0000","last_undegraded":"2023-08-28T02:20:50.818120+0000","last_fullsized":"2023-08-28T02:20:50.818120+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_clean_scrub_stamp":"2023-08-28T02:08:02.043718+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'114","reported_seq":412,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.818867+0000","last_change":"2023-08-28T02:20:19.852152+0000","last_active":"2023-08-28T02:20:50.818867+0000","last_peered":"2023-08-28T02:20:50.818867+0000","last_clean":"2023-08-28T02:20:50.818867+0000","last_became_active":"2023-08-28T02:20:19.851615+0000","last_became_peered":"2023-08-28T02:20:19.851615+0000","last_unstale":"2023-08-28T02:20:50.818867+0000","last_undegraded":"2023-08-28T02:20:50.818867+0000","last_fullsized":"2023-08-28T02:20:50.818867+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":114,"log_dups_size":0,"ondisk_log_size":114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":358,"num_read_kb":1352,"num_write":102,"num_write_kb":1104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":84,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.808976+0000","last_change":"2023-08-28T02:19:50.211156+0000","last_active":"2023-08-28T02:20:50.808976+0000","last_peered":"2023-08-28T02:20:50.808976+0000","last_clean":"2023-08-28T02:20:50.808976+0000","last_became_active":"2023-08-28T02:19:50.210398+0000","last_became_peered":"2023-08-28T02:19:50.210398+0000","last_unstale":"2023-08-28T02:20:50.808976+0000","last_undegraded":"2023-08-28T02:20:50.808976+0000","last_fullsized":"2023-08-28T02:20:50.808976+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:14:41.748197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":52,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.808409+0000","last_change":"2023-08-28T02:20:19.851591+0000","last_active":"2023-08-28T02:20:50.808409+0000","last_peered":"2023-08-28T02:20:50.808409+0000","last_clean":"2023-08-28T02:20:50.808409+0000","last_became_active":"2023-08-28T02:20:19.851291+0000","last_became_peered":"2023-08-28T02:20:19.851291+0000","last_unstale":"2023-08-28T02:20:50.808409+0000","last_undegraded":"2023-08-28T02:20:50.808409+0000","last_fullsized":"2023-08-28T02:20:50.808409+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:33:51.259707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"78'47","reported_seq":266,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.819902+0000","last_change":"2023-08-28T02:20:50.819614+0000","last_active":"2023-08-28T02:20:50.819902+0000","last_peered":"2023-08-28T02:20:50.819902+0000","last_clean":"2023-08-28T02:20:50.819902+0000","last_became_active":"2023-08-28T02:20:50.819114+0000","last_became_peered":"2023-08-28T02:20:50.819114+0000","last_unstale":"2023-08-28T02:20:50.819902+0000","last_undegraded":"2023-08-28T02:20:50.819902+0000","last_fullsized":"2023-08-28T02:20:50.819902+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":47,"log_dups_size":0,"ondisk_log_size":47,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:29:07.212870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":162,"num_read_kb":1281,"num_write":48,"num_write_kb":1153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":51,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.810901+0000","last_change":"2023-08-28T02:20:19.851150+0000","last_active":"2023-08-28T02:20:50.810901+0000","last_peered":"2023-08-28T02:20:50.810901+0000","last_clean":"2023-08-28T02:20:50.810901+0000","last_became_active":"2023-08-28T02:20:19.850886+0000","last_became_peered":"2023-08-28T02:20:19.850886+0000","last_unstale":"2023-08-28T02:20:50.810901+0000","last_undegraded":"2023-08-28T02:20:50.810901+0000","last_fullsized":"2023-08-28T02:20:50.810901+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:50:28.287980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":81,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.813107+0000","last_change":"2023-08-28T02:18:39.779607+0000","last_active":"2023-08-28T02:20:50.813107+0000","last_peered":"2023-08-28T02:20:50.813107+0000","last_clean":"2023-08-28T02:20:50.813107+0000","last_became_active":"2023-08-28T02:18:39.779303+0000","last_became_peered":"2023-08-28T02:18:39.779303+0000","last_unstale":"2023-08-28T02:20:50.813107+0000","last_undegraded":"2023-08-28T02:20:50.813107+0000","last_fullsized":"2023-08-28T02:20:50.813107+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"78'204","reported_seq":1030,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.819453+0000","last_change":"2023-08-28T02:19:50.219949+0000","last_active":"2023-08-28T02:20:50.819453+0000","last_peered":"2023-08-28T02:20:50.819453+0000","last_clean":"2023-08-28T02:20:50.819453+0000","last_became_active":"2023-08-28T02:19:50.219356+0000","last_became_peered":"2023-08-28T02:19:50.219356+0000","last_unstale":"2023-08-28T02:20:50.819453+0000","last_undegraded":"2023-08-28T02:20:50.819453+0000","last_fullsized":"2023-08-28T02:20:50.819453+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":204,"log_dups_size":0,"ondisk_log_size":204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1521,"num_read_kb":2511,"num_write":178,"num_write_kb":1175,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":11,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.824031+0000","last_change":"2023-08-28T02:20:51.823558+0000","last_active":"2023-08-28T02:20:51.824031+0000","last_peered":"2023-08-28T02:20:51.824031+0000","last_clean":"2023-08-28T02:20:51.824031+0000","last_became_active":"2023-08-28T02:20:51.823259+0000","last_became_peered":"2023-08-28T02:20:51.823259+0000","last_unstale":"2023-08-28T02:20:51.824031+0000","last_undegraded":"2023-08-28T02:20:51.824031+0000","last_fullsized":"2023-08-28T02:20:51.824031+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:22:15.166170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":96,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.828624+0000","last_change":"2023-08-28T02:20:50.828341+0000","last_active":"2023-08-28T02:20:50.828624+0000","last_peered":"2023-08-28T02:20:50.828624+0000","last_clean":"2023-08-28T02:20:50.828624+0000","last_became_active":"2023-08-28T02:20:50.828019+0000","last_became_peered":"2023-08-28T02:20:50.828019+0000","last_unstale":"2023-08-28T02:20:50.828624+0000","last_undegraded":"2023-08-28T02:20:50.828624+0000","last_fullsized":"2023-08-28T02:20:50.828624+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'126","reported_seq":570,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.810726+0000","last_change":"2023-08-28T02:19:50.213556+0000","last_active":"2023-08-28T02:20:50.810726+0000","last_peered":"2023-08-28T02:20:50.810726+0000","last_clean":"2023-08-28T02:20:50.810726+0000","last_became_active":"2023-08-28T02:19:50.212996+0000","last_became_peered":"2023-08-28T02:19:50.212996+0000","last_unstale":"2023-08-28T02:20:50.810726+0000","last_undegraded":"2023-08-28T02:20:50.810726+0000","last_fullsized":"2023-08-28T02:20:50.810726+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:38:28.157352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":743,"num_read_kb":1331,"num_write":125,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":53,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.821699+0000","last_change":"2023-08-28T02:20:50.820676+0000","last_active":"2023-08-28T02:20:50.821699+0000","last_peered":"2023-08-28T02:20:50.821699+0000","last_clean":"2023-08-28T02:20:50.821699+0000","last_became_active":"2023-08-28T02:20:50.820389+0000","last_became_peered":"2023-08-28T02:20:50.820389+0000","last_unstale":"2023-08-28T02:20:50.821699+0000","last_undegraded":"2023-08-28T02:20:50.821699+0000","last_fullsized":"2023-08-28T02:20:50.821699+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:24:25.623757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":79,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.823878+0000","last_change":"2023-08-28T02:20:50.822792+0000","last_active":"2023-08-28T02:20:50.823878+0000","last_peered":"2023-08-28T02:20:50.823878+0000","last_clean":"2023-08-28T02:20:50.823878+0000","last_became_active":"2023-08-28T02:20:50.822157+0000","last_became_peered":"2023-08-28T02:20:50.822157+0000","last_unstale":"2023-08-28T02:20:50.823878+0000","last_undegraded":"2023-08-28T02:20:50.823878+0000","last_fullsized":"2023-08-28T02:20:50.823878+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:15:16.261246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":41,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.813102+0000","last_change":"2023-08-28T02:19:51.204017+0000","last_active":"2023-08-28T02:20:50.813102+0000","last_peered":"2023-08-28T02:20:50.813102+0000","last_clean":"2023-08-28T02:20:50.813102+0000","last_became_active":"2023-08-28T02:19:51.203627+0000","last_became_peered":"2023-08-28T02:19:51.203627+0000","last_unstale":"2023-08-28T02:20:50.813102+0000","last_undegraded":"2023-08-28T02:20:50.813102+0000","last_fullsized":"2023-08-28T02:20:50.813102+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:14:06.231701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":11,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.927939+0000","last_change":"2023-08-28T02:20:51.927612+0000","last_active":"2023-08-28T02:20:51.927939+0000","last_peered":"2023-08-28T02:20:51.927939+0000","last_clean":"2023-08-28T02:20:51.927939+0000","last_became_active":"2023-08-28T02:20:51.927256+0000","last_became_peered":"2023-08-28T02:20:51.927256+0000","last_unstale":"2023-08-28T02:20:51.927939+0000","last_undegraded":"2023-08-28T02:20:51.927939+0000","last_fullsized":"2023-08-28T02:20:51.927939+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:05:59.937193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":115,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.886978+0000","last_change":"2023-08-28T02:20:51.886978+0000","last_active":"2023-08-28T02:20:51.886978+0000","last_peered":"2023-08-28T02:20:51.886978+0000","last_clean":"2023-08-28T02:20:51.886978+0000","last_became_active":"2023-08-28T02:20:51.886625+0000","last_became_peered":"2023-08-28T02:20:51.886625+0000","last_unstale":"2023-08-28T02:20:51.886978+0000","last_undegraded":"2023-08-28T02:20:51.886978+0000","last_fullsized":"2023-08-28T02:20:51.886978+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":101,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.821045+0000","last_change":"2023-08-28T02:20:51.821045+0000","last_active":"2023-08-28T02:20:51.821045+0000","last_peered":"2023-08-28T02:20:51.821045+0000","last_clean":"2023-08-28T02:20:51.821045+0000","last_became_active":"2023-08-28T02:20:51.820588+0000","last_became_peered":"2023-08-28T02:20:51.820588+0000","last_unstale":"2023-08-28T02:20:51.821045+0000","last_undegraded":"2023-08-28T02:20:51.821045+0000","last_fullsized":"2023-08-28T02:20:51.821045+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":80,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.820658+0000","last_change":"2023-08-28T02:20:50.820134+0000","last_active":"2023-08-28T02:20:50.820658+0000","last_peered":"2023-08-28T02:20:50.820658+0000","last_clean":"2023-08-28T02:20:50.820658+0000","last_became_active":"2023-08-28T02:20:50.819765+0000","last_became_peered":"2023-08-28T02:20:50.819765+0000","last_unstale":"2023-08-28T02:20:50.820658+0000","last_undegraded":"2023-08-28T02:20:50.820658+0000","last_fullsized":"2023-08-28T02:20:50.820658+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":73,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.826281+0000","last_change":"2023-08-28T02:20:50.824864+0000","last_active":"2023-08-28T02:20:50.826281+0000","last_peered":"2023-08-28T02:20:50.826281+0000","last_clean":"2023-08-28T02:20:50.826281+0000","last_became_active":"2023-08-28T02:20:50.824495+0000","last_became_peered":"2023-08-28T02:20:50.824495+0000","last_unstale":"2023-08-28T02:20:50.826281+0000","last_undegraded":"2023-08-28T02:20:50.826281+0000","last_fullsized":"2023-08-28T02:20:50.826281+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:45:04.111154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":73,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.817140+0000","last_change":"2023-08-28T02:19:51.205245+0000","last_active":"2023-08-28T02:20:51.817140+0000","last_peered":"2023-08-28T02:20:51.817140+0000","last_clean":"2023-08-28T02:20:51.817140+0000","last_became_active":"2023-08-28T02:19:51.204896+0000","last_became_peered":"2023-08-28T02:19:51.204896+0000","last_unstale":"2023-08-28T02:20:51.817140+0000","last_undegraded":"2023-08-28T02:20:51.817140+0000","last_fullsized":"2023-08-28T02:20:51.817140+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:51:24.634702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":81,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.824924+0000","last_change":"2023-08-28T02:20:50.824679+0000","last_active":"2023-08-28T02:20:50.824924+0000","last_peered":"2023-08-28T02:20:50.824924+0000","last_clean":"2023-08-28T02:20:50.824924+0000","last_became_active":"2023-08-28T02:20:50.824339+0000","last_became_peered":"2023-08-28T02:20:50.824339+0000","last_unstale":"2023-08-28T02:20:50.824924+0000","last_undegraded":"2023-08-28T02:20:50.824924+0000","last_fullsized":"2023-08-28T02:20:50.824924+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":114,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.927844+0000","last_change":"2023-08-28T02:20:51.927844+0000","last_active":"2023-08-28T02:20:51.927844+0000","last_peered":"2023-08-28T02:20:51.927844+0000","last_clean":"2023-08-28T02:20:51.927844+0000","last_became_active":"2023-08-28T02:20:51.927433+0000","last_became_peered":"2023-08-28T02:20:51.927433+0000","last_unstale":"2023-08-28T02:20:51.927844+0000","last_undegraded":"2023-08-28T02:20:51.927844+0000","last_fullsized":"2023-08-28T02:20:51.927844+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":67,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.820415+0000","last_change":"2023-08-28T02:11:22.188982+0000","last_active":"2023-08-28T02:20:50.820415+0000","last_peered":"2023-08-28T02:20:50.820415+0000","last_clean":"2023-08-28T02:20:50.820415+0000","last_became_active":"2023-08-28T02:11:22.188663+0000","last_became_peered":"2023-08-28T02:11:22.188663+0000","last_unstale":"2023-08-28T02:20:50.820415+0000","last_undegraded":"2023-08-28T02:20:50.820415+0000","last_fullsized":"2023-08-28T02:20:50.820415+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":95,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.824117+0000","last_change":"2023-08-28T02:20:50.823610+0000","last_active":"2023-08-28T02:20:50.824117+0000","last_peered":"2023-08-28T02:20:50.824117+0000","last_clean":"2023-08-28T02:20:50.824117+0000","last_became_active":"2023-08-28T02:20:50.823066+0000","last_became_peered":"2023-08-28T02:20:50.823066+0000","last_unstale":"2023-08-28T02:20:50.824117+0000","last_undegraded":"2023-08-28T02:20:50.824117+0000","last_fullsized":"2023-08-28T02:20:50.824117+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":83,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.819174+0000","last_change":"2023-08-28T02:19:50.220079+0000","last_active":"2023-08-28T02:20:50.819174+0000","last_peered":"2023-08-28T02:20:50.819174+0000","last_clean":"2023-08-28T02:20:50.819174+0000","last_became_active":"2023-08-28T02:19:50.219671+0000","last_became_peered":"2023-08-28T02:19:50.219671+0000","last_unstale":"2023-08-28T02:20:50.819174+0000","last_undegraded":"2023-08-28T02:20:50.819174+0000","last_fullsized":"2023-08-28T02:20:50.819174+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":65,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.818304+0000","last_change":"2023-08-28T02:11:23.249061+0000","last_active":"2023-08-28T02:20:50.818304+0000","last_peered":"2023-08-28T02:20:50.818304+0000","last_clean":"2023-08-28T02:20:50.818304+0000","last_became_active":"2023-08-28T02:11:23.248739+0000","last_became_peered":"2023-08-28T02:11:23.248739+0000","last_unstale":"2023-08-28T02:20:50.818304+0000","last_undegraded":"2023-08-28T02:20:50.818304+0000","last_fullsized":"2023-08-28T02:20:50.818304+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":95,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.819235+0000","last_change":"2023-08-28T02:20:43.220383+0000","last_active":"2023-08-28T02:20:50.819235+0000","last_peered":"2023-08-28T02:20:50.819235+0000","last_clean":"2023-08-28T02:20:50.819235+0000","last_became_active":"2023-08-28T02:20:43.136191+0000","last_became_peered":"2023-08-28T02:20:43.136191+0000","last_unstale":"2023-08-28T02:20:50.819235+0000","last_undegraded":"2023-08-28T02:20:50.819235+0000","last_fullsized":"2023-08-28T02:20:50.819235+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":47,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.810990+0000","last_change":"2023-08-28T02:20:19.851128+0000","last_active":"2023-08-28T02:20:50.810990+0000","last_peered":"2023-08-28T02:20:50.810990+0000","last_clean":"2023-08-28T02:20:50.810990+0000","last_became_active":"2023-08-28T02:20:19.849358+0000","last_became_peered":"2023-08-28T02:20:19.849358+0000","last_unstale":"2023-08-28T02:20:50.810990+0000","last_undegraded":"2023-08-28T02:20:50.810990+0000","last_fullsized":"2023-08-28T02:20:50.810990+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:32:27.226152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":89,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.816878+0000","last_change":"2023-08-28T02:18:39.778728+0000","last_active":"2023-08-28T02:20:50.816878+0000","last_peered":"2023-08-28T02:20:50.816878+0000","last_clean":"2023-08-28T02:20:50.816878+0000","last_became_active":"2023-08-28T02:18:39.778283+0000","last_became_peered":"2023-08-28T02:18:39.778283+0000","last_unstale":"2023-08-28T02:20:50.816878+0000","last_undegraded":"2023-08-28T02:20:50.816878+0000","last_fullsized":"2023-08-28T02:20:50.816878+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":62,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.817071+0000","last_change":"2023-08-28T02:20:50.823670+0000","last_active":"2023-08-28T02:20:51.817071+0000","last_peered":"2023-08-28T02:20:51.817071+0000","last_clean":"2023-08-28T02:20:51.817071+0000","last_became_active":"2023-08-28T02:20:50.823229+0000","last_became_peered":"2023-08-28T02:20:50.823229+0000","last_unstale":"2023-08-28T02:20:51.817071+0000","last_undegraded":"2023-08-28T02:20:51.817071+0000","last_fullsized":"2023-08-28T02:20:51.817071+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:50:36.316012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":66,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.817149+0000","last_change":"2023-08-28T02:11:22.188327+0000","last_active":"2023-08-28T02:20:50.817149+0000","last_peered":"2023-08-28T02:20:50.817149+0000","last_clean":"2023-08-28T02:20:50.817149+0000","last_became_active":"2023-08-28T02:11:22.187870+0000","last_became_peered":"2023-08-28T02:11:22.187870+0000","last_unstale":"2023-08-28T02:20:50.817149+0000","last_undegraded":"2023-08-28T02:20:50.817149+0000","last_fullsized":"2023-08-28T02:20:50.817149+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":105,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.927775+0000","last_change":"2023-08-28T02:20:51.927775+0000","last_active":"2023-08-28T02:20:51.927775+0000","last_peered":"2023-08-28T02:20:51.927775+0000","last_clean":"2023-08-28T02:20:51.927775+0000","last_became_active":"2023-08-28T02:20:51.927401+0000","last_became_peered":"2023-08-28T02:20:51.927401+0000","last_unstale":"2023-08-28T02:20:51.927775+0000","last_undegraded":"2023-08-28T02:20:51.927775+0000","last_fullsized":"2023-08-28T02:20:51.927775+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":61,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.816807+0000","last_change":"2023-08-28T02:20:50.821233+0000","last_active":"2023-08-28T02:20:51.816807+0000","last_peered":"2023-08-28T02:20:51.816807+0000","last_clean":"2023-08-28T02:20:51.816807+0000","last_became_active":"2023-08-28T02:20:50.821062+0000","last_became_peered":"2023-08-28T02:20:50.821062+0000","last_unstale":"2023-08-28T02:20:51.816807+0000","last_undegraded":"2023-08-28T02:20:51.816807+0000","last_fullsized":"2023-08-28T02:20:51.816807+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:19:40.305189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":78,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.826178+0000","last_change":"2023-08-28T02:20:50.825633+0000","last_active":"2023-08-28T02:20:50.826178+0000","last_peered":"2023-08-28T02:20:50.826178+0000","last_clean":"2023-08-28T02:20:50.826178+0000","last_became_active":"2023-08-28T02:20:50.825328+0000","last_became_peered":"2023-08-28T02:20:50.825328+0000","last_unstale":"2023-08-28T02:20:50.826178+0000","last_undegraded":"2023-08-28T02:20:50.826178+0000","last_fullsized":"2023-08-28T02:20:50.826178+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:59:37.406892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":83,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.818201+0000","last_change":"2023-08-28T02:20:19.849490+0000","last_active":"2023-08-28T02:20:50.818201+0000","last_peered":"2023-08-28T02:20:50.818201+0000","last_clean":"2023-08-28T02:20:50.818201+0000","last_became_active":"2023-08-28T02:20:19.848962+0000","last_became_peered":"2023-08-28T02:20:19.848962+0000","last_unstale":"2023-08-28T02:20:50.818201+0000","last_undegraded":"2023-08-28T02:20:50.818201+0000","last_fullsized":"2023-08-28T02:20:50.818201+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":94,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.812125+0000","last_change":"2023-08-28T02:19:51.204118+0000","last_active":"2023-08-28T02:20:50.812125+0000","last_peered":"2023-08-28T02:20:50.812125+0000","last_clean":"2023-08-28T02:20:50.812125+0000","last_became_active":"2023-08-28T02:19:51.203745+0000","last_became_peered":"2023-08-28T02:19:51.203745+0000","last_unstale":"2023-08-28T02:20:50.812125+0000","last_undegraded":"2023-08-28T02:20:50.812125+0000","last_fullsized":"2023-08-28T02:20:50.812125+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"86'22","reported_seq":88,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.812982+0000","last_change":"2023-08-28T02:11:22.184497+0000","last_active":"2023-08-28T02:20:50.812982+0000","last_peered":"2023-08-28T02:20:50.812982+0000","last_clean":"2023-08-28T02:20:50.812982+0000","last_became_active":"2023-08-28T02:11:22.184173+0000","last_became_peered":"2023-08-28T02:11:22.184173+0000","last_unstale":"2023-08-28T02:20:50.812982+0000","last_undegraded":"2023-08-28T02:20:50.812982+0000","last_fullsized":"2023-08-28T02:20:50.812982+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":53,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.809557+0000","last_change":"2023-08-28T02:20:19.851616+0000","last_active":"2023-08-28T02:20:50.809557+0000","last_peered":"2023-08-28T02:20:50.809557+0000","last_clean":"2023-08-28T02:20:50.809557+0000","last_became_active":"2023-08-28T02:20:19.851322+0000","last_became_peered":"2023-08-28T02:20:19.851322+0000","last_unstale":"2023-08-28T02:20:50.809557+0000","last_undegraded":"2023-08-28T02:20:50.809557+0000","last_fullsized":"2023-08-28T02:20:50.809557+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:03:37.565646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":84,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.826581+0000","last_change":"2023-08-28T02:20:50.826189+0000","last_active":"2023-08-28T02:20:50.826581+0000","last_peered":"2023-08-28T02:20:50.826581+0000","last_clean":"2023-08-28T02:20:50.826581+0000","last_became_active":"2023-08-28T02:20:50.825824+0000","last_became_peered":"2023-08-28T02:20:50.825824+0000","last_unstale":"2023-08-28T02:20:50.826581+0000","last_undegraded":"2023-08-28T02:20:50.826581+0000","last_fullsized":"2023-08-28T02:20:50.826581+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":64,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.827001+0000","last_change":"2023-08-28T02:20:50.826216+0000","last_active":"2023-08-28T02:20:50.827001+0000","last_peered":"2023-08-28T02:20:50.827001+0000","last_clean":"2023-08-28T02:20:50.827001+0000","last_became_active":"2023-08-28T02:20:50.825949+0000","last_became_peered":"2023-08-28T02:20:50.825949+0000","last_unstale":"2023-08-28T02:20:50.827001+0000","last_undegraded":"2023-08-28T02:20:50.827001+0000","last_fullsized":"2023-08-28T02:20:50.827001+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:08:15.626853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":81,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.818941+0000","last_change":"2023-08-28T02:19:50.214698+0000","last_active":"2023-08-28T02:20:50.818941+0000","last_peered":"2023-08-28T02:20:50.818941+0000","last_clean":"2023-08-28T02:20:50.818941+0000","last_became_active":"2023-08-28T02:19:50.214323+0000","last_became_peered":"2023-08-28T02:19:50.214323+0000","last_unstale":"2023-08-28T02:20:50.818941+0000","last_undegraded":"2023-08-28T02:20:50.818941+0000","last_fullsized":"2023-08-28T02:20:50.818941+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":41,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.812255+0000","last_change":"2023-08-28T02:19:51.203925+0000","last_active":"2023-08-28T02:20:50.812255+0000","last_peered":"2023-08-28T02:20:50.812255+0000","last_clean":"2023-08-28T02:20:50.812255+0000","last_became_active":"2023-08-28T02:19:51.203594+0000","last_became_peered":"2023-08-28T02:19:51.203594+0000","last_unstale":"2023-08-28T02:20:50.812255+0000","last_undegraded":"2023-08-28T02:20:50.812255+0000","last_fullsized":"2023-08-28T02:20:50.812255+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:02:14.852254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,6],"acting":[0,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":61,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.817326+0000","last_change":"2023-08-28T02:20:50.828055+0000","last_active":"2023-08-28T02:20:51.817326+0000","last_peered":"2023-08-28T02:20:51.817326+0000","last_clean":"2023-08-28T02:20:51.817326+0000","last_became_active":"2023-08-28T02:20:50.827778+0000","last_became_peered":"2023-08-28T02:20:50.827778+0000","last_unstale":"2023-08-28T02:20:51.817326+0000","last_undegraded":"2023-08-28T02:20:51.817326+0000","last_fullsized":"2023-08-28T02:20:51.817326+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:50:35.464371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":100,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.823018+0000","last_change":"2023-08-28T02:20:51.823018+0000","last_active":"2023-08-28T02:20:51.823018+0000","last_peered":"2023-08-28T02:20:51.823018+0000","last_clean":"2023-08-28T02:20:51.823018+0000","last_became_active":"2023-08-28T02:20:51.822499+0000","last_became_peered":"2023-08-28T02:20:51.822499+0000","last_unstale":"2023-08-28T02:20:51.823018+0000","last_undegraded":"2023-08-28T02:20:51.823018+0000","last_fullsized":"2023-08-28T02:20:51.823018+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":94,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.821931+0000","last_change":"2023-08-28T02:20:50.819499+0000","last_active":"2023-08-28T02:20:50.821931+0000","last_peered":"2023-08-28T02:20:50.821931+0000","last_clean":"2023-08-28T02:20:50.821931+0000","last_became_active":"2023-08-28T02:20:50.819026+0000","last_became_peered":"2023-08-28T02:20:50.819026+0000","last_unstale":"2023-08-28T02:20:50.821931+0000","last_undegraded":"2023-08-28T02:20:50.821931+0000","last_fullsized":"2023-08-28T02:20:50.821931+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:59:22.177440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":62,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.827522+0000","last_change":"2023-08-28T02:20:50.823506+0000","last_active":"2023-08-28T02:20:50.827522+0000","last_peered":"2023-08-28T02:20:50.827522+0000","last_clean":"2023-08-28T02:20:50.827522+0000","last_became_active":"2023-08-28T02:20:50.822836+0000","last_became_peered":"2023-08-28T02:20:50.822836+0000","last_unstale":"2023-08-28T02:20:50.827522+0000","last_undegraded":"2023-08-28T02:20:50.827522+0000","last_fullsized":"2023-08-28T02:20:50.827522+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:43:06.363328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":52,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.821703+0000","last_change":"2023-08-28T02:20:50.820616+0000","last_active":"2023-08-28T02:20:50.821703+0000","last_peered":"2023-08-28T02:20:50.821703+0000","last_clean":"2023-08-28T02:20:50.821703+0000","last_became_active":"2023-08-28T02:20:50.820305+0000","last_became_peered":"2023-08-28T02:20:50.820305+0000","last_unstale":"2023-08-28T02:20:50.821703+0000","last_undegraded":"2023-08-28T02:20:50.821703+0000","last_fullsized":"2023-08-28T02:20:50.821703+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:41:30.684047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":47,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.811803+0000","last_change":"2023-08-28T02:20:19.850774+0000","last_active":"2023-08-28T02:20:50.811803+0000","last_peered":"2023-08-28T02:20:50.811803+0000","last_clean":"2023-08-28T02:20:50.811803+0000","last_became_active":"2023-08-28T02:20:19.850448+0000","last_became_peered":"2023-08-28T02:20:19.850448+0000","last_unstale":"2023-08-28T02:20:50.811803+0000","last_undegraded":"2023-08-28T02:20:50.811803+0000","last_fullsized":"2023-08-28T02:20:50.811803+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:40:37.288553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":97,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.812255+0000","last_change":"2023-08-28T02:20:19.849807+0000","last_active":"2023-08-28T02:20:50.812255+0000","last_peered":"2023-08-28T02:20:50.812255+0000","last_clean":"2023-08-28T02:20:50.812255+0000","last_became_active":"2023-08-28T02:20:19.849464+0000","last_became_peered":"2023-08-28T02:20:19.849464+0000","last_unstale":"2023-08-28T02:20:50.812255+0000","last_undegraded":"2023-08-28T02:20:50.812255+0000","last_fullsized":"2023-08-28T02:20:50.812255+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26810,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430351,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307016,"kb_used_data":5824,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93438136,"statfs":{"total":95995035648,"available":95680651264,"internally_reserved":0,"allocated":5963776,"data_stored":5448619,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:20:43 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.53400000000000003,"15min":0.53600000000000003},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.746,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.60299999999999998,"15min":0.60899999999999999},"min":{"1min":0.38200000000000001,"5min":0.374,"15min":0.374},"max":{"1min":0.97199999999999998,"5min":0.97199999999999998,"15min":0.97199999999999998},"last":0.47099999999999997}]},{"osd":1,"last update":"Mon Aug 28 02:20:50 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.44}]},{"osd":2,"last update":"Mon Aug 28 02:21:20 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.56100000000000005}]},{"osd":3,"last update":"Mon Aug 28 02:20:19 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.55700000000000005,"15min":0.54500000000000004},"min":{"1min":0.47599999999999998,"5min":0.35899999999999999,"15min":0.32600000000000001},"max":{"1min":0.877,"5min":0.88800000000000001,"15min":2.7509999999999999},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.56399999999999995,"15min":0.55900000000000005},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.35299999999999998},"max":{"1min":1.4770000000000001,"5min":1.4770000000000001,"15min":2.7080000000000002},"last":0.64500000000000002}]},{"osd":4,"last update":"Mon Aug 28 02:21:20 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.48399999999999999,"15min":0.47499999999999998},"min":{"1min":0.35299999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.71299999999999997,"5min":0.75700000000000001,"15min":0.77800000000000002},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.503,"5min":0.502,"15min":0.498},"min":{"1min":0.30199999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.67600000000000005,"5min":0.71599999999999997,"15min":0.79800000000000004},"last":0.53100000000000003}]},{"osd":5,"last update":"Mon Aug 28 02:21:20 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.504,"15min":0.48499999999999999},"min":{"1min":0.312,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.69799999999999995,"5min":0.71099999999999997,"15min":0.81100000000000005},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.57199999999999995,"15min":0.54400000000000004},"min":{"1min":0.32200000000000001,"5min":0.314,"15min":0.314},"max":{"1min":0.746,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.67800000000000005}]},{"osd":6,"last update":"Mon Aug 28 02:21:20 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.56200000000000006,"15min":0.53400000000000003},"min":{"1min":0.33900000000000002,"5min":0.32700000000000001,"15min":0.32200000000000001},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.61699999999999999,"15min":0.58299999999999996},"min":{"1min":0.41399999999999998,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.77200000000000002,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.71199999999999997}]}]},{"osd":6,"up_from":42,"seq":180388626578,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302908,"kb_used_data":1716,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93442244,"statfs":{"total":95995035648,"available":95684857856,"internally_reserved":0,"allocated":1757184,"data_stored":1253873,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:20:47 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.56499999999999995,"15min":0.57099999999999995},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.59099999999999997,"15min":0.59899999999999998},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.66900000000000004,"5min":0.746,"15min":0.746},"last":0.434}]},{"osd":1,"last update":"Mon Aug 28 02:20:51 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.57899999999999996}]},{"osd":2,"last update":"Mon Aug 28 02:21:23 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.575,"5min":1.575,"15min":1.575},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":1.6419999999999999,"5min":1.6419999999999999,"15min":1.6419999999999999},"last":0.47099999999999997}]},{"osd":3,"last update":"Mon Aug 28 02:19:54 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.55200000000000005,"15min":0.54900000000000004},"min":{"1min":0.41799999999999998,"5min":0.33400000000000002,"15min":0.30499999999999999},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.84399999999999997},"last":0.503},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.54400000000000004,"15min":0.52300000000000002},"min":{"1min":0.35599999999999998,"5min":0.34699999999999998,"15min":0.31900000000000001},"max":{"1min":0.69599999999999995,"5min":0.76500000000000001,"15min":1.02},"last":0.54000000000000004}]},{"osd":4,"last update":"Mon Aug 28 02:20:55 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.46200000000000002,"15min":0.44400000000000001},"min":{"1min":0.32400000000000001,"5min":0.27800000000000002,"15min":0.27200000000000002},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.877},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.51600000000000001,"15min":0.47599999999999998},"min":{"1min":0.38500000000000001,"5min":0.29699999999999999,"15min":0.28899999999999998},"max":{"1min":0.84099999999999997,"5min":0.84899999999999998,"15min":0.85599999999999998},"last":0.48199999999999998}]},{"osd":5,"last update":"Mon Aug 28 02:20:55 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.48199999999999998,"15min":0.45800000000000002},"min":{"1min":0.318,"5min":0.313,"15min":0.29299999999999998},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.79300000000000004},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.53900000000000003,"15min":0.51400000000000001},"min":{"1min":0.40300000000000002,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.82199999999999995,"5min":0.82499999999999996,"15min":2.0609999999999999},"last":0.61399999999999999}]},{"osd":7,"last update":"Mon Aug 28 02:21:16 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.57199999999999995,"15min":0.55000000000000004},"min":{"1min":0.41199999999999998,"5min":0.35799999999999998,"15min":0.32700000000000001},"max":{"1min":0.85699999999999998,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.65600000000000003,"5min":0.59899999999999998,"15min":0.57399999999999995},"min":{"1min":0.42799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.83499999999999996,"5min":0.88100000000000001,"15min":0.89400000000000002},"last":0.60399999999999998}]}]},{"osd":1,"up_from":98,"seq":420906795030,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36872,"kb_used_data":5312,"kb_used_omap":7,"kb_used_meta":31544,"kb_avail":93708280,"statfs":{"total":95995035648,"available":95957278720,"internally_reserved":0,"allocated":5439488,"data_stored":4873933,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":32301387},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:20:50 2023","interfaces":[{"interface":"back","average":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"last":0.26400000000000001},{"interface":"front","average":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.498,"5min":0.498,"15min":0.498},"last":0.25}]},{"osd":2,"last update":"Mon Aug 28 02:21:23 2023","interfaces":[{"interface":"back","average":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"last":0.22700000000000001},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.33800000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32700000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:20:50 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.90800000000000003,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.57399999999999995}]},{"osd":5,"last update":"Mon Aug 28 02:20:50 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.96599999999999997,"5min":0.96599999999999997,"15min":0.96599999999999997},"last":0.56399999999999995}]},{"osd":6,"last update":"Mon Aug 28 02:20:50 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.94199999999999995,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.58599999999999997}]},{"osd":7,"last update":"Mon Aug 28 02:20:50 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.499}]}]},{"osd":0,"up_from":90,"seq":386547056677,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":37464,"kb_used_data":5904,"kb_used_omap":8,"kb_used_meta":31543,"kb_avail":93707688,"statfs":{"total":95995035648,"available":95956672512,"internally_reserved":0,"allocated":6045696,"data_stored":5465837,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":32300393},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Aug 28 02:20:53 2023","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"min":{"1min":0.192,"5min":0.192,"15min":0.192},"max":{"1min":0.503,"5min":0.503,"15min":0.503},"last":0.378},{"interface":"front","average":{"1min":0.372,"5min":0.372,"15min":0.372},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.497,"5min":0.497,"15min":0.497},"last":0.27400000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:21:20 2023","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"last":3.4910000000000001},{"interface":"front","average":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"last":3.5619999999999998}]},{"osd":3,"last update":"Mon Aug 28 02:19:40 2023","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":0.435,"15min":0.435},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.48199999999999998}]},{"osd":4,"last update":"Mon Aug 28 02:20:43 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.65800000000000003,"15min":0.66600000000000004},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.82099999999999995,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.621,"5min":0.68899999999999995,"15min":0.70099999999999996},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.79800000000000004,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.67200000000000004}]},{"osd":5,"last update":"Mon Aug 28 02:20:43 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.64300000000000002,"15min":0.65000000000000002},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.82399999999999995,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.63800000000000001,"15min":0.64800000000000002},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.68300000000000005}]},{"osd":6,"last update":"Mon Aug 28 02:20:43 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.64200000000000002,"15min":0.65100000000000002},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.77200000000000002,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.622},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.69599999999999995,"15min":0.70399999999999996},"min":{"1min":0.42199999999999999,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.876,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.60799999999999998}]},{"osd":7,"last update":"Mon Aug 28 02:20:43 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.68400000000000005,"15min":0.69399999999999995},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.68999999999999995,"15min":0.69899999999999995},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.79500000000000004,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.66000000000000003}]}]},{"osd":2,"up_from":104,"seq":446676598801,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32680,"kb_used_data":1184,"kb_used_omap":10,"kb_used_meta":31477,"kb_avail":93712472,"statfs":{"total":95995035648,"available":95961571328,"internally_reserved":0,"allocated":1212416,"data_stored":672302,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10792,"internal_metadata":32232920},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:21:22 2023","interfaces":[{"interface":"back","average":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":0.29099999999999998},{"interface":"front","average":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.26300000000000001}]},{"osd":1,"last update":"Mon Aug 28 02:21:22 2023","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.317}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36099999999999999}]},{"osd":4,"last update":"Mon Aug 28 02:21:22 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.91100000000000003,"5min":0.91100000000000003,"15min":0.91100000000000003},"last":0.67900000000000005}]},{"osd":5,"last update":"Mon Aug 28 02:21:22 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.69799999999999995},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.499}]},{"osd":6,"last update":"Mon Aug 28 02:21:22 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.71599999999999997},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.72499999999999998}]},{"osd":7,"last update":"Mon Aug 28 02:21:22 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.745}]}]},{"osd":3,"up_from":112,"seq":481036337162,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28432,"kb_used_data":1160,"kb_used_omap":3,"kb_used_meta":27260,"kb_avail":93716720,"statfs":{"total":95995035648,"available":95965921280,"internally_reserved":0,"allocated":1187840,"data_stored":667951,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":27915170},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36599999999999999}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38600000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.433}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52800000000000002}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48999999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56399999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55600000000000005}]}]},{"osd":4,"up_from":30,"seq":128849019036,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298236,"kb_used_data":1140,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93446916,"statfs":{"total":95995035648,"available":95689641984,"internally_reserved":0,"allocated":1167360,"data_stored":663855,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:20:43 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.60399999999999998,"15min":0.61199999999999999},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.58599999999999997,"15min":0.59499999999999997},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.71399999999999997,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.33700000000000002}]},{"osd":1,"last update":"Mon Aug 28 02:20:49 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.28000000000000003},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.374}]},{"osd":2,"last update":"Mon Aug 28 02:21:19 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.29799999999999999}]},{"osd":3,"last update":"Mon Aug 28 02:20:01 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.58099999999999996,"15min":0.55300000000000005},"min":{"1min":0.45700000000000002,"5min":0.41799999999999998,"15min":0.307},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":1.1599999999999999},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.628,"5min":0.58099999999999996,"15min":0.55400000000000005},"min":{"1min":0.496,"5min":0.371,"15min":0.35899999999999999},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.871},"last":0.438}]},{"osd":5,"last update":"Mon Aug 28 02:20:32 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.49099999999999999,"15min":0.441},"min":{"1min":0.35399999999999998,"5min":0.30399999999999999,"15min":0.26600000000000001},"max":{"1min":0.72499999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.503,"5min":0.502,"15min":0.45500000000000002},"min":{"1min":0.36399999999999999,"5min":0.32500000000000001,"15min":0.27700000000000002},"max":{"1min":0.63300000000000001,"5min":0.78600000000000003,"15min":1.1839999999999999},"last":0.41199999999999998}]},{"osd":6,"last update":"Mon Aug 28 02:20:49 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.53000000000000003,"15min":0.48299999999999998},"min":{"1min":0.375,"5min":0.30499999999999999,"15min":0.27500000000000002},"max":{"1min":0.76000000000000001,"5min":0.77300000000000002,"15min":0.85099999999999998},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.58399999999999996,"15min":0.52100000000000002},"min":{"1min":0.45600000000000002,"5min":0.36799999999999999,"15min":0.318},"max":{"1min":0.77500000000000002,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.44900000000000001}]},{"osd":7,"last update":"Mon Aug 28 02:21:18 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.56899999999999995,"15min":0.53500000000000003},"min":{"1min":0.40400000000000003,"5min":0.36899999999999999,"15min":0.312},"max":{"1min":0.70999999999999996,"5min":0.82099999999999995,"15min":1.2110000000000001},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.59399999999999997,"15min":0.54500000000000004},"min":{"1min":0.41899999999999998,"5min":0.35199999999999998,"15min":0.32100000000000001},"max":{"1min":0.79700000000000004,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.51300000000000001}]}]},{"osd":5,"up_from":36,"seq":154618822808,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298220,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93446932,"statfs":{"total":95995035648,"available":95689658368,"internally_reserved":0,"allocated":1150976,"data_stored":659010,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:20:40 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.60999999999999999,"15min":0.62},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.61899999999999999,"15min":0.625},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.747,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.57599999999999996}]},{"osd":1,"last update":"Mon Aug 28 02:20:50 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.65000000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:21:22 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.62}]},{"osd":3,"last update":"Mon Aug 28 02:20:36 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57999999999999996,"15min":0.54900000000000004},"min":{"1min":0.39100000000000001,"5min":0.255,"15min":0.255},"max":{"1min":0.78400000000000003,"5min":1.1279999999999999,"15min":1.1779999999999999},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.59499999999999997,"15min":0.57299999999999995},"min":{"1min":0.40400000000000003,"5min":0.35299999999999998,"15min":0.307},"max":{"1min":0.80800000000000005,"5min":1.1679999999999999,"15min":1.1679999999999999},"last":0.51700000000000002}]},{"osd":4,"last update":"Mon Aug 28 02:20:36 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.47399999999999998,"15min":0.45300000000000001},"min":{"1min":0.33400000000000002,"5min":0.28000000000000003,"15min":0.27500000000000002},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.79200000000000004},"last":0.495},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.505,"15min":0.47799999999999998},"min":{"1min":0.31900000000000001,"5min":0.28799999999999998,"15min":0.28199999999999997},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":1.1499999999999999},"last":0.66700000000000004}]},{"osd":6,"last update":"Mon Aug 28 02:20:50 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.51300000000000001,"15min":0.51900000000000002},"min":{"1min":0.36699999999999999,"5min":0.32700000000000001,"15min":0.28399999999999997},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":5.2119999999999997},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.55400000000000005,"15min":0.51600000000000001},"min":{"1min":0.442,"5min":0.31900000000000001,"15min":0.30299999999999999},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":1.089},"last":0.71199999999999997}]},{"osd":7,"last update":"Mon Aug 28 02:21:13 2023","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.58299999999999996,"15min":0.55300000000000005},"min":{"1min":0.45000000000000001,"5min":0.35199999999999998,"15min":0.32900000000000001},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":1.119},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.63,"5min":0.57499999999999996,"15min":0.55300000000000005},"min":{"1min":0.40400000000000003,"5min":0.34999999999999998,"15min":0.29799999999999999},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.67700000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-08-28T02:21:35.030 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph pg dump --format=json 2023-08-28T02:21:35.378 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:21:36.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:36 smithi143 ceph-mon[183059]: from='client.34587 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:21:36.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:36 smithi143 ceph-mon[183059]: pgmap v208: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:36.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:36 smithi143 ceph-mon[189871]: from='client.34587 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:21:36.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:36 smithi143 ceph-mon[189871]: pgmap v208: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:36.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:36 smithi169 ceph-mon[159415]: from='client.34587 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:21:36.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:36 smithi169 ceph-mon[159415]: pgmap v208: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:36.910 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:21:36.912 INFO:teuthology.orchestra.run.smithi143.stderr:dumped all 2023-08-28T02:21:37.465 INFO:teuthology.orchestra.run.smithi143.stdout:{"pg_ready":true,"pg_map":{"version":208,"stamp":"2023-08-28T02:21:35.032809+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6288,"num_read_kb":13719,"num_write":1106,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1341828,"kb_used_data":23364,"kb_used_omap":29,"kb_used_meta":1318370,"kb_avail":748619388,"statfs":{"total":767960285184,"available":766586253312,"internally_reserved":0,"allocated":23924736,"data_stored":19705380,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30674,"internal_metadata":1350010926},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002743"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":100,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.824785+0000","last_change":"2023-08-28T02:20:51.824785+0000","last_active":"2023-08-28T02:20:51.824785+0000","last_peered":"2023-08-28T02:20:51.824785+0000","last_clean":"2023-08-28T02:20:51.824785+0000","last_became_active":"2023-08-28T02:20:51.824447+0000","last_became_peered":"2023-08-28T02:20:51.824447+0000","last_unstale":"2023-08-28T02:20:51.824785+0000","last_undegraded":"2023-08-28T02:20:51.824785+0000","last_fullsized":"2023-08-28T02:20:51.824785+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":65,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.813241+0000","last_change":"2023-08-28T02:20:19.850252+0000","last_active":"2023-08-28T02:20:50.813241+0000","last_peered":"2023-08-28T02:20:50.813241+0000","last_clean":"2023-08-28T02:20:50.813241+0000","last_became_active":"2023-08-28T02:20:19.849913+0000","last_became_peered":"2023-08-28T02:20:19.849913+0000","last_unstale":"2023-08-28T02:20:50.813241+0000","last_undegraded":"2023-08-28T02:20:50.813241+0000","last_fullsized":"2023-08-28T02:20:50.813241+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:48:47.713465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":98,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.826985+0000","last_change":"2023-08-28T02:20:50.823101+0000","last_active":"2023-08-28T02:20:50.826985+0000","last_peered":"2023-08-28T02:20:50.826985+0000","last_clean":"2023-08-28T02:20:50.826985+0000","last_became_active":"2023-08-28T02:20:50.822599+0000","last_became_peered":"2023-08-28T02:20:50.822599+0000","last_unstale":"2023-08-28T02:20:50.826985+0000","last_undegraded":"2023-08-28T02:20:50.826985+0000","last_fullsized":"2023-08-28T02:20:50.826985+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:28:57.827789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":85,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.826313+0000","last_change":"2023-08-28T02:20:50.825915+0000","last_active":"2023-08-28T02:20:50.826313+0000","last_peered":"2023-08-28T02:20:50.826313+0000","last_clean":"2023-08-28T02:20:50.826313+0000","last_became_active":"2023-08-28T02:20:50.825526+0000","last_became_peered":"2023-08-28T02:20:50.825526+0000","last_unstale":"2023-08-28T02:20:50.826313+0000","last_undegraded":"2023-08-28T02:20:50.826313+0000","last_fullsized":"2023-08-28T02:20:50.826313+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:25:20.142641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":95,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.820522+0000","last_change":"2023-08-28T02:20:50.819786+0000","last_active":"2023-08-28T02:20:50.820522+0000","last_peered":"2023-08-28T02:20:50.820522+0000","last_clean":"2023-08-28T02:20:50.820522+0000","last_became_active":"2023-08-28T02:20:50.819451+0000","last_became_peered":"2023-08-28T02:20:50.819451+0000","last_unstale":"2023-08-28T02:20:50.820522+0000","last_undegraded":"2023-08-28T02:20:50.820522+0000","last_fullsized":"2023-08-28T02:20:50.820522+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":68,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.824216+0000","last_change":"2023-08-28T02:11:22.186024+0000","last_active":"2023-08-28T02:20:50.824216+0000","last_peered":"2023-08-28T02:20:50.824216+0000","last_clean":"2023-08-28T02:20:50.824216+0000","last_became_active":"2023-08-28T02:11:22.185500+0000","last_became_peered":"2023-08-28T02:11:22.185500+0000","last_unstale":"2023-08-28T02:20:50.824216+0000","last_undegraded":"2023-08-28T02:20:50.824216+0000","last_fullsized":"2023-08-28T02:20:50.824216+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":95,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.820006+0000","last_change":"2023-08-28T02:20:50.819015+0000","last_active":"2023-08-28T02:20:50.820006+0000","last_peered":"2023-08-28T02:20:50.820006+0000","last_clean":"2023-08-28T02:20:50.820006+0000","last_became_active":"2023-08-28T02:20:50.818667+0000","last_became_peered":"2023-08-28T02:20:50.818667+0000","last_unstale":"2023-08-28T02:20:50.820006+0000","last_undegraded":"2023-08-28T02:20:50.820006+0000","last_fullsized":"2023-08-28T02:20:50.820006+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":83,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.822081+0000","last_change":"2023-08-28T02:19:50.218760+0000","last_active":"2023-08-28T02:20:50.822081+0000","last_peered":"2023-08-28T02:20:50.822081+0000","last_clean":"2023-08-28T02:20:50.822081+0000","last_became_active":"2023-08-28T02:19:50.218376+0000","last_became_peered":"2023-08-28T02:19:50.218376+0000","last_unstale":"2023-08-28T02:20:50.822081+0000","last_undegraded":"2023-08-28T02:20:50.822081+0000","last_fullsized":"2023-08-28T02:20:50.822081+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":116,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.817585+0000","last_change":"2023-08-28T02:20:50.817119+0000","last_active":"2023-08-28T02:20:50.817585+0000","last_peered":"2023-08-28T02:20:50.817585+0000","last_clean":"2023-08-28T02:20:50.817585+0000","last_became_active":"2023-08-28T02:20:50.816756+0000","last_became_peered":"2023-08-28T02:20:50.816756+0000","last_unstale":"2023-08-28T02:20:50.817585+0000","last_undegraded":"2023-08-28T02:20:50.817585+0000","last_fullsized":"2023-08-28T02:20:50.817585+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:12:52.257562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":50,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.817184+0000","last_change":"2023-08-28T02:18:39.781312+0000","last_active":"2023-08-28T02:20:51.817184+0000","last_peered":"2023-08-28T02:20:51.817184+0000","last_clean":"2023-08-28T02:20:51.817184+0000","last_became_active":"2023-08-28T02:18:39.780916+0000","last_became_peered":"2023-08-28T02:18:39.780916+0000","last_unstale":"2023-08-28T02:20:51.817184+0000","last_undegraded":"2023-08-28T02:20:51.817184+0000","last_fullsized":"2023-08-28T02:20:51.817184+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:37:11.571250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":65,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.814304+0000","last_change":"2023-08-28T02:11:22.437146+0000","last_active":"2023-08-28T02:20:50.814304+0000","last_peered":"2023-08-28T02:20:50.814304+0000","last_clean":"2023-08-28T02:20:50.814304+0000","last_became_active":"2023-08-28T02:11:22.436798+0000","last_became_peered":"2023-08-28T02:11:22.436798+0000","last_unstale":"2023-08-28T02:20:50.814304+0000","last_undegraded":"2023-08-28T02:20:50.814304+0000","last_fullsized":"2023-08-28T02:20:50.814304+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":65,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.813953+0000","last_change":"2023-08-28T02:19:50.221001+0000","last_active":"2023-08-28T02:20:50.813953+0000","last_peered":"2023-08-28T02:20:50.813953+0000","last_clean":"2023-08-28T02:20:50.813953+0000","last_became_active":"2023-08-28T02:19:50.220611+0000","last_became_peered":"2023-08-28T02:19:50.220611+0000","last_unstale":"2023-08-28T02:20:50.813953+0000","last_undegraded":"2023-08-28T02:20:50.813953+0000","last_fullsized":"2023-08-28T02:20:50.813953+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:23:10.637630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'122","reported_seq":570,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.816897+0000","last_change":"2023-08-28T02:20:19.851065+0000","last_active":"2023-08-28T02:20:50.816897+0000","last_peered":"2023-08-28T02:20:50.816897+0000","last_clean":"2023-08-28T02:20:50.816897+0000","last_became_active":"2023-08-28T02:20:19.850477+0000","last_became_peered":"2023-08-28T02:20:19.850477+0000","last_unstale":"2023-08-28T02:20:50.816897+0000","last_undegraded":"2023-08-28T02:20:50.816897+0000","last_fullsized":"2023-08-28T02:20:50.816897+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":857,"num_read_kb":2224,"num_write":99,"num_write_kb":2312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":70,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.823860+0000","last_change":"2023-08-28T02:20:50.822679+0000","last_active":"2023-08-28T02:20:50.823860+0000","last_peered":"2023-08-28T02:20:50.823860+0000","last_clean":"2023-08-28T02:20:50.823860+0000","last_became_active":"2023-08-28T02:20:50.822180+0000","last_became_peered":"2023-08-28T02:20:50.822180+0000","last_unstale":"2023-08-28T02:20:50.823860+0000","last_undegraded":"2023-08-28T02:20:50.823860+0000","last_fullsized":"2023-08-28T02:20:50.823860+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:02:40.005023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'112","reported_seq":356,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.809841+0000","last_change":"2023-08-28T02:19:50.218206+0000","last_active":"2023-08-28T02:20:50.809841+0000","last_peered":"2023-08-28T02:20:50.809841+0000","last_clean":"2023-08-28T02:20:50.809841+0000","last_became_active":"2023-08-28T02:19:50.217637+0000","last_became_peered":"2023-08-28T02:19:50.217637+0000","last_unstale":"2023-08-28T02:20:50.809841+0000","last_undegraded":"2023-08-28T02:20:50.809841+0000","last_fullsized":"2023-08-28T02:20:50.809841+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:12:06.224907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":444,"num_read_kb":473,"num_write":94,"num_write_kb":127,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":73,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.823867+0000","last_change":"2023-08-28T02:20:50.823477+0000","last_active":"2023-08-28T02:20:50.823867+0000","last_peered":"2023-08-28T02:20:50.823867+0000","last_clean":"2023-08-28T02:20:50.823867+0000","last_became_active":"2023-08-28T02:20:50.823158+0000","last_became_peered":"2023-08-28T02:20:50.823158+0000","last_unstale":"2023-08-28T02:20:50.823867+0000","last_undegraded":"2023-08-28T02:20:50.823867+0000","last_fullsized":"2023-08-28T02:20:50.823867+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:33:18.347341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":47,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.813160+0000","last_change":"2023-08-28T02:18:39.780069+0000","last_active":"2023-08-28T02:20:50.813160+0000","last_peered":"2023-08-28T02:20:50.813160+0000","last_clean":"2023-08-28T02:20:50.813160+0000","last_became_active":"2023-08-28T02:18:39.779829+0000","last_became_peered":"2023-08-28T02:18:39.779829+0000","last_unstale":"2023-08-28T02:20:50.813160+0000","last_undegraded":"2023-08-28T02:20:50.813160+0000","last_fullsized":"2023-08-28T02:20:50.813160+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:35:01.309659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":52,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.810821+0000","last_change":"2023-08-28T02:20:19.850916+0000","last_active":"2023-08-28T02:20:50.810821+0000","last_peered":"2023-08-28T02:20:50.810821+0000","last_clean":"2023-08-28T02:20:50.810821+0000","last_became_active":"2023-08-28T02:20:19.850627+0000","last_became_peered":"2023-08-28T02:20:19.850627+0000","last_unstale":"2023-08-28T02:20:50.810821+0000","last_undegraded":"2023-08-28T02:20:50.810821+0000","last_fullsized":"2023-08-28T02:20:50.810821+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:19:14.346189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'103","reported_seq":535,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.820309+0000","last_change":"2023-08-28T02:18:39.786876+0000","last_active":"2023-08-28T02:20:50.820309+0000","last_peered":"2023-08-28T02:20:50.820309+0000","last_clean":"2023-08-28T02:20:50.820309+0000","last_became_active":"2023-08-28T02:18:39.786414+0000","last_became_peered":"2023-08-28T02:18:39.786414+0000","last_unstale":"2023-08-28T02:20:50.820309+0000","last_undegraded":"2023-08-28T02:20:50.820309+0000","last_fullsized":"2023-08-28T02:20:50.820309+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":803,"num_read_kb":2080,"num_write":73,"num_write_kb":1244,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":97,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.886782+0000","last_change":"2023-08-28T02:20:51.886782+0000","last_active":"2023-08-28T02:20:51.886782+0000","last_peered":"2023-08-28T02:20:51.886782+0000","last_clean":"2023-08-28T02:20:51.886782+0000","last_became_active":"2023-08-28T02:20:51.886444+0000","last_became_peered":"2023-08-28T02:20:51.886444+0000","last_unstale":"2023-08-28T02:20:51.886782+0000","last_undegraded":"2023-08-28T02:20:51.886782+0000","last_fullsized":"2023-08-28T02:20:51.886782+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:57:03.704951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":61,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.823573+0000","last_change":"2023-08-28T02:20:50.823036+0000","last_active":"2023-08-28T02:20:50.823573+0000","last_peered":"2023-08-28T02:20:50.823573+0000","last_clean":"2023-08-28T02:20:50.823573+0000","last_became_active":"2023-08-28T02:20:50.822766+0000","last_became_peered":"2023-08-28T02:20:50.822766+0000","last_unstale":"2023-08-28T02:20:50.823573+0000","last_undegraded":"2023-08-28T02:20:50.823573+0000","last_fullsized":"2023-08-28T02:20:50.823573+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:09:26.786992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"78'148","reported_seq":659,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.818235+0000","last_change":"2023-08-28T02:19:50.221311+0000","last_active":"2023-08-28T02:20:50.818235+0000","last_peered":"2023-08-28T02:20:50.818235+0000","last_clean":"2023-08-28T02:20:50.818235+0000","last_became_active":"2023-08-28T02:19:50.220754+0000","last_became_peered":"2023-08-28T02:19:50.220754+0000","last_unstale":"2023-08-28T02:20:50.818235+0000","last_undegraded":"2023-08-28T02:20:50.818235+0000","last_fullsized":"2023-08-28T02:20:50.818235+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1092,"num_read_kb":1955,"num_write":127,"num_write_kb":5177,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":65,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.816396+0000","last_change":"2023-08-28T02:11:23.249640+0000","last_active":"2023-08-28T02:20:50.816396+0000","last_peered":"2023-08-28T02:20:50.816396+0000","last_clean":"2023-08-28T02:20:50.816396+0000","last_became_active":"2023-08-28T02:11:23.249355+0000","last_became_peered":"2023-08-28T02:11:23.249355+0000","last_unstale":"2023-08-28T02:20:50.816396+0000","last_undegraded":"2023-08-28T02:20:50.816396+0000","last_fullsized":"2023-08-28T02:20:50.816396+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":64,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.826633+0000","last_change":"2023-08-28T02:20:50.825728+0000","last_active":"2023-08-28T02:20:50.826633+0000","last_peered":"2023-08-28T02:20:50.826633+0000","last_clean":"2023-08-28T02:20:50.826633+0000","last_became_active":"2023-08-28T02:20:50.825418+0000","last_became_peered":"2023-08-28T02:20:50.825418+0000","last_unstale":"2023-08-28T02:20:50.826633+0000","last_undegraded":"2023-08-28T02:20:50.826633+0000","last_fullsized":"2023-08-28T02:20:50.826633+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:55:48.126269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"86'139","reported_seq":317,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.818120+0000","last_change":"2023-08-28T02:18:39.786823+0000","last_active":"2023-08-28T02:20:50.818120+0000","last_peered":"2023-08-28T02:20:50.818120+0000","last_clean":"2023-08-28T02:20:50.818120+0000","last_became_active":"2023-08-28T02:18:39.786357+0000","last_became_peered":"2023-08-28T02:18:39.786357+0000","last_unstale":"2023-08-28T02:20:50.818120+0000","last_undegraded":"2023-08-28T02:20:50.818120+0000","last_fullsized":"2023-08-28T02:20:50.818120+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_clean_scrub_stamp":"2023-08-28T02:08:02.043718+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'114","reported_seq":412,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.818867+0000","last_change":"2023-08-28T02:20:19.852152+0000","last_active":"2023-08-28T02:20:50.818867+0000","last_peered":"2023-08-28T02:20:50.818867+0000","last_clean":"2023-08-28T02:20:50.818867+0000","last_became_active":"2023-08-28T02:20:19.851615+0000","last_became_peered":"2023-08-28T02:20:19.851615+0000","last_unstale":"2023-08-28T02:20:50.818867+0000","last_undegraded":"2023-08-28T02:20:50.818867+0000","last_fullsized":"2023-08-28T02:20:50.818867+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":114,"log_dups_size":0,"ondisk_log_size":114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":358,"num_read_kb":1352,"num_write":102,"num_write_kb":1104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":84,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.808976+0000","last_change":"2023-08-28T02:19:50.211156+0000","last_active":"2023-08-28T02:20:50.808976+0000","last_peered":"2023-08-28T02:20:50.808976+0000","last_clean":"2023-08-28T02:20:50.808976+0000","last_became_active":"2023-08-28T02:19:50.210398+0000","last_became_peered":"2023-08-28T02:19:50.210398+0000","last_unstale":"2023-08-28T02:20:50.808976+0000","last_undegraded":"2023-08-28T02:20:50.808976+0000","last_fullsized":"2023-08-28T02:20:50.808976+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:14:41.748197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":52,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.808409+0000","last_change":"2023-08-28T02:20:19.851591+0000","last_active":"2023-08-28T02:20:50.808409+0000","last_peered":"2023-08-28T02:20:50.808409+0000","last_clean":"2023-08-28T02:20:50.808409+0000","last_became_active":"2023-08-28T02:20:19.851291+0000","last_became_peered":"2023-08-28T02:20:19.851291+0000","last_unstale":"2023-08-28T02:20:50.808409+0000","last_undegraded":"2023-08-28T02:20:50.808409+0000","last_fullsized":"2023-08-28T02:20:50.808409+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:33:51.259707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"78'47","reported_seq":266,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.819902+0000","last_change":"2023-08-28T02:20:50.819614+0000","last_active":"2023-08-28T02:20:50.819902+0000","last_peered":"2023-08-28T02:20:50.819902+0000","last_clean":"2023-08-28T02:20:50.819902+0000","last_became_active":"2023-08-28T02:20:50.819114+0000","last_became_peered":"2023-08-28T02:20:50.819114+0000","last_unstale":"2023-08-28T02:20:50.819902+0000","last_undegraded":"2023-08-28T02:20:50.819902+0000","last_fullsized":"2023-08-28T02:20:50.819902+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":47,"log_dups_size":0,"ondisk_log_size":47,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:29:07.212870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":162,"num_read_kb":1281,"num_write":48,"num_write_kb":1153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":51,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.810901+0000","last_change":"2023-08-28T02:20:19.851150+0000","last_active":"2023-08-28T02:20:50.810901+0000","last_peered":"2023-08-28T02:20:50.810901+0000","last_clean":"2023-08-28T02:20:50.810901+0000","last_became_active":"2023-08-28T02:20:19.850886+0000","last_became_peered":"2023-08-28T02:20:19.850886+0000","last_unstale":"2023-08-28T02:20:50.810901+0000","last_undegraded":"2023-08-28T02:20:50.810901+0000","last_fullsized":"2023-08-28T02:20:50.810901+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:50:28.287980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":81,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.813107+0000","last_change":"2023-08-28T02:18:39.779607+0000","last_active":"2023-08-28T02:20:50.813107+0000","last_peered":"2023-08-28T02:20:50.813107+0000","last_clean":"2023-08-28T02:20:50.813107+0000","last_became_active":"2023-08-28T02:18:39.779303+0000","last_became_peered":"2023-08-28T02:18:39.779303+0000","last_unstale":"2023-08-28T02:20:50.813107+0000","last_undegraded":"2023-08-28T02:20:50.813107+0000","last_fullsized":"2023-08-28T02:20:50.813107+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"78'204","reported_seq":1030,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.819453+0000","last_change":"2023-08-28T02:19:50.219949+0000","last_active":"2023-08-28T02:20:50.819453+0000","last_peered":"2023-08-28T02:20:50.819453+0000","last_clean":"2023-08-28T02:20:50.819453+0000","last_became_active":"2023-08-28T02:19:50.219356+0000","last_became_peered":"2023-08-28T02:19:50.219356+0000","last_unstale":"2023-08-28T02:20:50.819453+0000","last_undegraded":"2023-08-28T02:20:50.819453+0000","last_fullsized":"2023-08-28T02:20:50.819453+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":204,"log_dups_size":0,"ondisk_log_size":204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1521,"num_read_kb":2511,"num_write":178,"num_write_kb":1175,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":11,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.824031+0000","last_change":"2023-08-28T02:20:51.823558+0000","last_active":"2023-08-28T02:20:51.824031+0000","last_peered":"2023-08-28T02:20:51.824031+0000","last_clean":"2023-08-28T02:20:51.824031+0000","last_became_active":"2023-08-28T02:20:51.823259+0000","last_became_peered":"2023-08-28T02:20:51.823259+0000","last_unstale":"2023-08-28T02:20:51.824031+0000","last_undegraded":"2023-08-28T02:20:51.824031+0000","last_fullsized":"2023-08-28T02:20:51.824031+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:22:15.166170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":96,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.828624+0000","last_change":"2023-08-28T02:20:50.828341+0000","last_active":"2023-08-28T02:20:50.828624+0000","last_peered":"2023-08-28T02:20:50.828624+0000","last_clean":"2023-08-28T02:20:50.828624+0000","last_became_active":"2023-08-28T02:20:50.828019+0000","last_became_peered":"2023-08-28T02:20:50.828019+0000","last_unstale":"2023-08-28T02:20:50.828624+0000","last_undegraded":"2023-08-28T02:20:50.828624+0000","last_fullsized":"2023-08-28T02:20:50.828624+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'126","reported_seq":570,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.810726+0000","last_change":"2023-08-28T02:19:50.213556+0000","last_active":"2023-08-28T02:20:50.810726+0000","last_peered":"2023-08-28T02:20:50.810726+0000","last_clean":"2023-08-28T02:20:50.810726+0000","last_became_active":"2023-08-28T02:19:50.212996+0000","last_became_peered":"2023-08-28T02:19:50.212996+0000","last_unstale":"2023-08-28T02:20:50.810726+0000","last_undegraded":"2023-08-28T02:20:50.810726+0000","last_fullsized":"2023-08-28T02:20:50.810726+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:38:28.157352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":743,"num_read_kb":1331,"num_write":125,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":53,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.821699+0000","last_change":"2023-08-28T02:20:50.820676+0000","last_active":"2023-08-28T02:20:50.821699+0000","last_peered":"2023-08-28T02:20:50.821699+0000","last_clean":"2023-08-28T02:20:50.821699+0000","last_became_active":"2023-08-28T02:20:50.820389+0000","last_became_peered":"2023-08-28T02:20:50.820389+0000","last_unstale":"2023-08-28T02:20:50.821699+0000","last_undegraded":"2023-08-28T02:20:50.821699+0000","last_fullsized":"2023-08-28T02:20:50.821699+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:24:25.623757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":79,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.823878+0000","last_change":"2023-08-28T02:20:50.822792+0000","last_active":"2023-08-28T02:20:50.823878+0000","last_peered":"2023-08-28T02:20:50.823878+0000","last_clean":"2023-08-28T02:20:50.823878+0000","last_became_active":"2023-08-28T02:20:50.822157+0000","last_became_peered":"2023-08-28T02:20:50.822157+0000","last_unstale":"2023-08-28T02:20:50.823878+0000","last_undegraded":"2023-08-28T02:20:50.823878+0000","last_fullsized":"2023-08-28T02:20:50.823878+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:15:16.261246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":41,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.813102+0000","last_change":"2023-08-28T02:19:51.204017+0000","last_active":"2023-08-28T02:20:50.813102+0000","last_peered":"2023-08-28T02:20:50.813102+0000","last_clean":"2023-08-28T02:20:50.813102+0000","last_became_active":"2023-08-28T02:19:51.203627+0000","last_became_peered":"2023-08-28T02:19:51.203627+0000","last_unstale":"2023-08-28T02:20:50.813102+0000","last_undegraded":"2023-08-28T02:20:50.813102+0000","last_fullsized":"2023-08-28T02:20:50.813102+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:14:06.231701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":11,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.927939+0000","last_change":"2023-08-28T02:20:51.927612+0000","last_active":"2023-08-28T02:20:51.927939+0000","last_peered":"2023-08-28T02:20:51.927939+0000","last_clean":"2023-08-28T02:20:51.927939+0000","last_became_active":"2023-08-28T02:20:51.927256+0000","last_became_peered":"2023-08-28T02:20:51.927256+0000","last_unstale":"2023-08-28T02:20:51.927939+0000","last_undegraded":"2023-08-28T02:20:51.927939+0000","last_fullsized":"2023-08-28T02:20:51.927939+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:05:59.937193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":115,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.886978+0000","last_change":"2023-08-28T02:20:51.886978+0000","last_active":"2023-08-28T02:20:51.886978+0000","last_peered":"2023-08-28T02:20:51.886978+0000","last_clean":"2023-08-28T02:20:51.886978+0000","last_became_active":"2023-08-28T02:20:51.886625+0000","last_became_peered":"2023-08-28T02:20:51.886625+0000","last_unstale":"2023-08-28T02:20:51.886978+0000","last_undegraded":"2023-08-28T02:20:51.886978+0000","last_fullsized":"2023-08-28T02:20:51.886978+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":101,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.821045+0000","last_change":"2023-08-28T02:20:51.821045+0000","last_active":"2023-08-28T02:20:51.821045+0000","last_peered":"2023-08-28T02:20:51.821045+0000","last_clean":"2023-08-28T02:20:51.821045+0000","last_became_active":"2023-08-28T02:20:51.820588+0000","last_became_peered":"2023-08-28T02:20:51.820588+0000","last_unstale":"2023-08-28T02:20:51.821045+0000","last_undegraded":"2023-08-28T02:20:51.821045+0000","last_fullsized":"2023-08-28T02:20:51.821045+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":80,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.820658+0000","last_change":"2023-08-28T02:20:50.820134+0000","last_active":"2023-08-28T02:20:50.820658+0000","last_peered":"2023-08-28T02:20:50.820658+0000","last_clean":"2023-08-28T02:20:50.820658+0000","last_became_active":"2023-08-28T02:20:50.819765+0000","last_became_peered":"2023-08-28T02:20:50.819765+0000","last_unstale":"2023-08-28T02:20:50.820658+0000","last_undegraded":"2023-08-28T02:20:50.820658+0000","last_fullsized":"2023-08-28T02:20:50.820658+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":73,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.826281+0000","last_change":"2023-08-28T02:20:50.824864+0000","last_active":"2023-08-28T02:20:50.826281+0000","last_peered":"2023-08-28T02:20:50.826281+0000","last_clean":"2023-08-28T02:20:50.826281+0000","last_became_active":"2023-08-28T02:20:50.824495+0000","last_became_peered":"2023-08-28T02:20:50.824495+0000","last_unstale":"2023-08-28T02:20:50.826281+0000","last_undegraded":"2023-08-28T02:20:50.826281+0000","last_fullsized":"2023-08-28T02:20:50.826281+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:45:04.111154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":73,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.817140+0000","last_change":"2023-08-28T02:19:51.205245+0000","last_active":"2023-08-28T02:20:51.817140+0000","last_peered":"2023-08-28T02:20:51.817140+0000","last_clean":"2023-08-28T02:20:51.817140+0000","last_became_active":"2023-08-28T02:19:51.204896+0000","last_became_peered":"2023-08-28T02:19:51.204896+0000","last_unstale":"2023-08-28T02:20:51.817140+0000","last_undegraded":"2023-08-28T02:20:51.817140+0000","last_fullsized":"2023-08-28T02:20:51.817140+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:51:24.634702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":81,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.824924+0000","last_change":"2023-08-28T02:20:50.824679+0000","last_active":"2023-08-28T02:20:50.824924+0000","last_peered":"2023-08-28T02:20:50.824924+0000","last_clean":"2023-08-28T02:20:50.824924+0000","last_became_active":"2023-08-28T02:20:50.824339+0000","last_became_peered":"2023-08-28T02:20:50.824339+0000","last_unstale":"2023-08-28T02:20:50.824924+0000","last_undegraded":"2023-08-28T02:20:50.824924+0000","last_fullsized":"2023-08-28T02:20:50.824924+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":114,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.927844+0000","last_change":"2023-08-28T02:20:51.927844+0000","last_active":"2023-08-28T02:20:51.927844+0000","last_peered":"2023-08-28T02:20:51.927844+0000","last_clean":"2023-08-28T02:20:51.927844+0000","last_became_active":"2023-08-28T02:20:51.927433+0000","last_became_peered":"2023-08-28T02:20:51.927433+0000","last_unstale":"2023-08-28T02:20:51.927844+0000","last_undegraded":"2023-08-28T02:20:51.927844+0000","last_fullsized":"2023-08-28T02:20:51.927844+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":67,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.820415+0000","last_change":"2023-08-28T02:11:22.188982+0000","last_active":"2023-08-28T02:20:50.820415+0000","last_peered":"2023-08-28T02:20:50.820415+0000","last_clean":"2023-08-28T02:20:50.820415+0000","last_became_active":"2023-08-28T02:11:22.188663+0000","last_became_peered":"2023-08-28T02:11:22.188663+0000","last_unstale":"2023-08-28T02:20:50.820415+0000","last_undegraded":"2023-08-28T02:20:50.820415+0000","last_fullsized":"2023-08-28T02:20:50.820415+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":95,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.824117+0000","last_change":"2023-08-28T02:20:50.823610+0000","last_active":"2023-08-28T02:20:50.824117+0000","last_peered":"2023-08-28T02:20:50.824117+0000","last_clean":"2023-08-28T02:20:50.824117+0000","last_became_active":"2023-08-28T02:20:50.823066+0000","last_became_peered":"2023-08-28T02:20:50.823066+0000","last_unstale":"2023-08-28T02:20:50.824117+0000","last_undegraded":"2023-08-28T02:20:50.824117+0000","last_fullsized":"2023-08-28T02:20:50.824117+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":83,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.819174+0000","last_change":"2023-08-28T02:19:50.220079+0000","last_active":"2023-08-28T02:20:50.819174+0000","last_peered":"2023-08-28T02:20:50.819174+0000","last_clean":"2023-08-28T02:20:50.819174+0000","last_became_active":"2023-08-28T02:19:50.219671+0000","last_became_peered":"2023-08-28T02:19:50.219671+0000","last_unstale":"2023-08-28T02:20:50.819174+0000","last_undegraded":"2023-08-28T02:20:50.819174+0000","last_fullsized":"2023-08-28T02:20:50.819174+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":65,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.818304+0000","last_change":"2023-08-28T02:11:23.249061+0000","last_active":"2023-08-28T02:20:50.818304+0000","last_peered":"2023-08-28T02:20:50.818304+0000","last_clean":"2023-08-28T02:20:50.818304+0000","last_became_active":"2023-08-28T02:11:23.248739+0000","last_became_peered":"2023-08-28T02:11:23.248739+0000","last_unstale":"2023-08-28T02:20:50.818304+0000","last_undegraded":"2023-08-28T02:20:50.818304+0000","last_fullsized":"2023-08-28T02:20:50.818304+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":95,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.819235+0000","last_change":"2023-08-28T02:20:43.220383+0000","last_active":"2023-08-28T02:20:50.819235+0000","last_peered":"2023-08-28T02:20:50.819235+0000","last_clean":"2023-08-28T02:20:50.819235+0000","last_became_active":"2023-08-28T02:20:43.136191+0000","last_became_peered":"2023-08-28T02:20:43.136191+0000","last_unstale":"2023-08-28T02:20:50.819235+0000","last_undegraded":"2023-08-28T02:20:50.819235+0000","last_fullsized":"2023-08-28T02:20:50.819235+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":47,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.810990+0000","last_change":"2023-08-28T02:20:19.851128+0000","last_active":"2023-08-28T02:20:50.810990+0000","last_peered":"2023-08-28T02:20:50.810990+0000","last_clean":"2023-08-28T02:20:50.810990+0000","last_became_active":"2023-08-28T02:20:19.849358+0000","last_became_peered":"2023-08-28T02:20:19.849358+0000","last_unstale":"2023-08-28T02:20:50.810990+0000","last_undegraded":"2023-08-28T02:20:50.810990+0000","last_fullsized":"2023-08-28T02:20:50.810990+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:32:27.226152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":89,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.816878+0000","last_change":"2023-08-28T02:18:39.778728+0000","last_active":"2023-08-28T02:20:50.816878+0000","last_peered":"2023-08-28T02:20:50.816878+0000","last_clean":"2023-08-28T02:20:50.816878+0000","last_became_active":"2023-08-28T02:18:39.778283+0000","last_became_peered":"2023-08-28T02:18:39.778283+0000","last_unstale":"2023-08-28T02:20:50.816878+0000","last_undegraded":"2023-08-28T02:20:50.816878+0000","last_fullsized":"2023-08-28T02:20:50.816878+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":62,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.817071+0000","last_change":"2023-08-28T02:20:50.823670+0000","last_active":"2023-08-28T02:20:51.817071+0000","last_peered":"2023-08-28T02:20:51.817071+0000","last_clean":"2023-08-28T02:20:51.817071+0000","last_became_active":"2023-08-28T02:20:50.823229+0000","last_became_peered":"2023-08-28T02:20:50.823229+0000","last_unstale":"2023-08-28T02:20:51.817071+0000","last_undegraded":"2023-08-28T02:20:51.817071+0000","last_fullsized":"2023-08-28T02:20:51.817071+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:50:36.316012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":66,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.817149+0000","last_change":"2023-08-28T02:11:22.188327+0000","last_active":"2023-08-28T02:20:50.817149+0000","last_peered":"2023-08-28T02:20:50.817149+0000","last_clean":"2023-08-28T02:20:50.817149+0000","last_became_active":"2023-08-28T02:11:22.187870+0000","last_became_peered":"2023-08-28T02:11:22.187870+0000","last_unstale":"2023-08-28T02:20:50.817149+0000","last_undegraded":"2023-08-28T02:20:50.817149+0000","last_fullsized":"2023-08-28T02:20:50.817149+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":105,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.927775+0000","last_change":"2023-08-28T02:20:51.927775+0000","last_active":"2023-08-28T02:20:51.927775+0000","last_peered":"2023-08-28T02:20:51.927775+0000","last_clean":"2023-08-28T02:20:51.927775+0000","last_became_active":"2023-08-28T02:20:51.927401+0000","last_became_peered":"2023-08-28T02:20:51.927401+0000","last_unstale":"2023-08-28T02:20:51.927775+0000","last_undegraded":"2023-08-28T02:20:51.927775+0000","last_fullsized":"2023-08-28T02:20:51.927775+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":61,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.816807+0000","last_change":"2023-08-28T02:20:50.821233+0000","last_active":"2023-08-28T02:20:51.816807+0000","last_peered":"2023-08-28T02:20:51.816807+0000","last_clean":"2023-08-28T02:20:51.816807+0000","last_became_active":"2023-08-28T02:20:50.821062+0000","last_became_peered":"2023-08-28T02:20:50.821062+0000","last_unstale":"2023-08-28T02:20:51.816807+0000","last_undegraded":"2023-08-28T02:20:51.816807+0000","last_fullsized":"2023-08-28T02:20:51.816807+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:19:40.305189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":78,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.826178+0000","last_change":"2023-08-28T02:20:50.825633+0000","last_active":"2023-08-28T02:20:50.826178+0000","last_peered":"2023-08-28T02:20:50.826178+0000","last_clean":"2023-08-28T02:20:50.826178+0000","last_became_active":"2023-08-28T02:20:50.825328+0000","last_became_peered":"2023-08-28T02:20:50.825328+0000","last_unstale":"2023-08-28T02:20:50.826178+0000","last_undegraded":"2023-08-28T02:20:50.826178+0000","last_fullsized":"2023-08-28T02:20:50.826178+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:59:37.406892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":83,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.818201+0000","last_change":"2023-08-28T02:20:19.849490+0000","last_active":"2023-08-28T02:20:50.818201+0000","last_peered":"2023-08-28T02:20:50.818201+0000","last_clean":"2023-08-28T02:20:50.818201+0000","last_became_active":"2023-08-28T02:20:19.848962+0000","last_became_peered":"2023-08-28T02:20:19.848962+0000","last_unstale":"2023-08-28T02:20:50.818201+0000","last_undegraded":"2023-08-28T02:20:50.818201+0000","last_fullsized":"2023-08-28T02:20:50.818201+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":94,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.812125+0000","last_change":"2023-08-28T02:19:51.204118+0000","last_active":"2023-08-28T02:20:50.812125+0000","last_peered":"2023-08-28T02:20:50.812125+0000","last_clean":"2023-08-28T02:20:50.812125+0000","last_became_active":"2023-08-28T02:19:51.203745+0000","last_became_peered":"2023-08-28T02:19:51.203745+0000","last_unstale":"2023-08-28T02:20:50.812125+0000","last_undegraded":"2023-08-28T02:20:50.812125+0000","last_fullsized":"2023-08-28T02:20:50.812125+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"86'22","reported_seq":88,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.812982+0000","last_change":"2023-08-28T02:11:22.184497+0000","last_active":"2023-08-28T02:20:50.812982+0000","last_peered":"2023-08-28T02:20:50.812982+0000","last_clean":"2023-08-28T02:20:50.812982+0000","last_became_active":"2023-08-28T02:11:22.184173+0000","last_became_peered":"2023-08-28T02:11:22.184173+0000","last_unstale":"2023-08-28T02:20:50.812982+0000","last_undegraded":"2023-08-28T02:20:50.812982+0000","last_fullsized":"2023-08-28T02:20:50.812982+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":53,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.809557+0000","last_change":"2023-08-28T02:20:19.851616+0000","last_active":"2023-08-28T02:20:50.809557+0000","last_peered":"2023-08-28T02:20:50.809557+0000","last_clean":"2023-08-28T02:20:50.809557+0000","last_became_active":"2023-08-28T02:20:19.851322+0000","last_became_peered":"2023-08-28T02:20:19.851322+0000","last_unstale":"2023-08-28T02:20:50.809557+0000","last_undegraded":"2023-08-28T02:20:50.809557+0000","last_fullsized":"2023-08-28T02:20:50.809557+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:03:37.565646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":84,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.826581+0000","last_change":"2023-08-28T02:20:50.826189+0000","last_active":"2023-08-28T02:20:50.826581+0000","last_peered":"2023-08-28T02:20:50.826581+0000","last_clean":"2023-08-28T02:20:50.826581+0000","last_became_active":"2023-08-28T02:20:50.825824+0000","last_became_peered":"2023-08-28T02:20:50.825824+0000","last_unstale":"2023-08-28T02:20:50.826581+0000","last_undegraded":"2023-08-28T02:20:50.826581+0000","last_fullsized":"2023-08-28T02:20:50.826581+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":64,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.827001+0000","last_change":"2023-08-28T02:20:50.826216+0000","last_active":"2023-08-28T02:20:50.827001+0000","last_peered":"2023-08-28T02:20:50.827001+0000","last_clean":"2023-08-28T02:20:50.827001+0000","last_became_active":"2023-08-28T02:20:50.825949+0000","last_became_peered":"2023-08-28T02:20:50.825949+0000","last_unstale":"2023-08-28T02:20:50.827001+0000","last_undegraded":"2023-08-28T02:20:50.827001+0000","last_fullsized":"2023-08-28T02:20:50.827001+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:08:15.626853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":81,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.818941+0000","last_change":"2023-08-28T02:19:50.214698+0000","last_active":"2023-08-28T02:20:50.818941+0000","last_peered":"2023-08-28T02:20:50.818941+0000","last_clean":"2023-08-28T02:20:50.818941+0000","last_became_active":"2023-08-28T02:19:50.214323+0000","last_became_peered":"2023-08-28T02:19:50.214323+0000","last_unstale":"2023-08-28T02:20:50.818941+0000","last_undegraded":"2023-08-28T02:20:50.818941+0000","last_fullsized":"2023-08-28T02:20:50.818941+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":41,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.812255+0000","last_change":"2023-08-28T02:19:51.203925+0000","last_active":"2023-08-28T02:20:50.812255+0000","last_peered":"2023-08-28T02:20:50.812255+0000","last_clean":"2023-08-28T02:20:50.812255+0000","last_became_active":"2023-08-28T02:19:51.203594+0000","last_became_peered":"2023-08-28T02:19:51.203594+0000","last_unstale":"2023-08-28T02:20:50.812255+0000","last_undegraded":"2023-08-28T02:20:50.812255+0000","last_fullsized":"2023-08-28T02:20:50.812255+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:02:14.852254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,6],"acting":[0,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":61,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.817326+0000","last_change":"2023-08-28T02:20:50.828055+0000","last_active":"2023-08-28T02:20:51.817326+0000","last_peered":"2023-08-28T02:20:51.817326+0000","last_clean":"2023-08-28T02:20:51.817326+0000","last_became_active":"2023-08-28T02:20:50.827778+0000","last_became_peered":"2023-08-28T02:20:50.827778+0000","last_unstale":"2023-08-28T02:20:51.817326+0000","last_undegraded":"2023-08-28T02:20:51.817326+0000","last_fullsized":"2023-08-28T02:20:51.817326+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:50:35.464371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":100,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.823018+0000","last_change":"2023-08-28T02:20:51.823018+0000","last_active":"2023-08-28T02:20:51.823018+0000","last_peered":"2023-08-28T02:20:51.823018+0000","last_clean":"2023-08-28T02:20:51.823018+0000","last_became_active":"2023-08-28T02:20:51.822499+0000","last_became_peered":"2023-08-28T02:20:51.822499+0000","last_unstale":"2023-08-28T02:20:51.823018+0000","last_undegraded":"2023-08-28T02:20:51.823018+0000","last_fullsized":"2023-08-28T02:20:51.823018+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":94,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.821931+0000","last_change":"2023-08-28T02:20:50.819499+0000","last_active":"2023-08-28T02:20:50.821931+0000","last_peered":"2023-08-28T02:20:50.821931+0000","last_clean":"2023-08-28T02:20:50.821931+0000","last_became_active":"2023-08-28T02:20:50.819026+0000","last_became_peered":"2023-08-28T02:20:50.819026+0000","last_unstale":"2023-08-28T02:20:50.821931+0000","last_undegraded":"2023-08-28T02:20:50.821931+0000","last_fullsized":"2023-08-28T02:20:50.821931+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:59:22.177440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":62,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.827522+0000","last_change":"2023-08-28T02:20:50.823506+0000","last_active":"2023-08-28T02:20:50.827522+0000","last_peered":"2023-08-28T02:20:50.827522+0000","last_clean":"2023-08-28T02:20:50.827522+0000","last_became_active":"2023-08-28T02:20:50.822836+0000","last_became_peered":"2023-08-28T02:20:50.822836+0000","last_unstale":"2023-08-28T02:20:50.827522+0000","last_undegraded":"2023-08-28T02:20:50.827522+0000","last_fullsized":"2023-08-28T02:20:50.827522+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:43:06.363328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":52,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.821703+0000","last_change":"2023-08-28T02:20:50.820616+0000","last_active":"2023-08-28T02:20:50.821703+0000","last_peered":"2023-08-28T02:20:50.821703+0000","last_clean":"2023-08-28T02:20:50.821703+0000","last_became_active":"2023-08-28T02:20:50.820305+0000","last_became_peered":"2023-08-28T02:20:50.820305+0000","last_unstale":"2023-08-28T02:20:50.821703+0000","last_undegraded":"2023-08-28T02:20:50.821703+0000","last_fullsized":"2023-08-28T02:20:50.821703+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:41:30.684047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":47,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.811803+0000","last_change":"2023-08-28T02:20:19.850774+0000","last_active":"2023-08-28T02:20:50.811803+0000","last_peered":"2023-08-28T02:20:50.811803+0000","last_clean":"2023-08-28T02:20:50.811803+0000","last_became_active":"2023-08-28T02:20:19.850448+0000","last_became_peered":"2023-08-28T02:20:19.850448+0000","last_unstale":"2023-08-28T02:20:50.811803+0000","last_undegraded":"2023-08-28T02:20:50.811803+0000","last_fullsized":"2023-08-28T02:20:50.811803+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:40:37.288553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":97,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.812255+0000","last_change":"2023-08-28T02:20:19.849807+0000","last_active":"2023-08-28T02:20:50.812255+0000","last_peered":"2023-08-28T02:20:50.812255+0000","last_clean":"2023-08-28T02:20:50.812255+0000","last_became_active":"2023-08-28T02:20:19.849464+0000","last_became_peered":"2023-08-28T02:20:19.849464+0000","last_unstale":"2023-08-28T02:20:50.812255+0000","last_undegraded":"2023-08-28T02:20:50.812255+0000","last_fullsized":"2023-08-28T02:20:50.812255+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26810,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430351,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307016,"kb_used_data":5824,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93438136,"statfs":{"total":95995035648,"available":95680651264,"internally_reserved":0,"allocated":5963776,"data_stored":5448619,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:20:43 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.53400000000000003,"15min":0.53600000000000003},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.746,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.60299999999999998,"15min":0.60899999999999999},"min":{"1min":0.38200000000000001,"5min":0.374,"15min":0.374},"max":{"1min":0.97199999999999998,"5min":0.97199999999999998,"15min":0.97199999999999998},"last":0.47099999999999997}]},{"osd":1,"last update":"Mon Aug 28 02:20:50 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.44}]},{"osd":2,"last update":"Mon Aug 28 02:21:20 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.56100000000000005}]},{"osd":3,"last update":"Mon Aug 28 02:20:19 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.55700000000000005,"15min":0.54500000000000004},"min":{"1min":0.47599999999999998,"5min":0.35899999999999999,"15min":0.32600000000000001},"max":{"1min":0.877,"5min":0.88800000000000001,"15min":2.7509999999999999},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.56399999999999995,"15min":0.55900000000000005},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.35299999999999998},"max":{"1min":1.4770000000000001,"5min":1.4770000000000001,"15min":2.7080000000000002},"last":0.64500000000000002}]},{"osd":4,"last update":"Mon Aug 28 02:21:20 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.48399999999999999,"15min":0.47499999999999998},"min":{"1min":0.35299999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.71299999999999997,"5min":0.75700000000000001,"15min":0.77800000000000002},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.503,"5min":0.502,"15min":0.498},"min":{"1min":0.30199999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.67600000000000005,"5min":0.71599999999999997,"15min":0.79800000000000004},"last":0.53100000000000003}]},{"osd":5,"last update":"Mon Aug 28 02:21:20 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.504,"15min":0.48499999999999999},"min":{"1min":0.312,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.69799999999999995,"5min":0.71099999999999997,"15min":0.81100000000000005},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.57199999999999995,"15min":0.54400000000000004},"min":{"1min":0.32200000000000001,"5min":0.314,"15min":0.314},"max":{"1min":0.746,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.67800000000000005}]},{"osd":6,"last update":"Mon Aug 28 02:21:20 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.56200000000000006,"15min":0.53400000000000003},"min":{"1min":0.33900000000000002,"5min":0.32700000000000001,"15min":0.32200000000000001},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.61699999999999999,"15min":0.58299999999999996},"min":{"1min":0.41399999999999998,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.77200000000000002,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.71199999999999997}]}]},{"osd":6,"up_from":42,"seq":180388626578,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302908,"kb_used_data":1716,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93442244,"statfs":{"total":95995035648,"available":95684857856,"internally_reserved":0,"allocated":1757184,"data_stored":1253873,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:20:47 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.56499999999999995,"15min":0.57099999999999995},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.59099999999999997,"15min":0.59899999999999998},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.66900000000000004,"5min":0.746,"15min":0.746},"last":0.434}]},{"osd":1,"last update":"Mon Aug 28 02:20:51 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.57899999999999996}]},{"osd":2,"last update":"Mon Aug 28 02:21:23 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.575,"5min":1.575,"15min":1.575},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":1.6419999999999999,"5min":1.6419999999999999,"15min":1.6419999999999999},"last":0.47099999999999997}]},{"osd":3,"last update":"Mon Aug 28 02:19:54 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.55200000000000005,"15min":0.54900000000000004},"min":{"1min":0.41799999999999998,"5min":0.33400000000000002,"15min":0.30499999999999999},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.84399999999999997},"last":0.503},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.54400000000000004,"15min":0.52300000000000002},"min":{"1min":0.35599999999999998,"5min":0.34699999999999998,"15min":0.31900000000000001},"max":{"1min":0.69599999999999995,"5min":0.76500000000000001,"15min":1.02},"last":0.54000000000000004}]},{"osd":4,"last update":"Mon Aug 28 02:20:55 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.46200000000000002,"15min":0.44400000000000001},"min":{"1min":0.32400000000000001,"5min":0.27800000000000002,"15min":0.27200000000000002},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.877},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.51600000000000001,"15min":0.47599999999999998},"min":{"1min":0.38500000000000001,"5min":0.29699999999999999,"15min":0.28899999999999998},"max":{"1min":0.84099999999999997,"5min":0.84899999999999998,"15min":0.85599999999999998},"last":0.48199999999999998}]},{"osd":5,"last update":"Mon Aug 28 02:20:55 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.48199999999999998,"15min":0.45800000000000002},"min":{"1min":0.318,"5min":0.313,"15min":0.29299999999999998},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.79300000000000004},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.53900000000000003,"15min":0.51400000000000001},"min":{"1min":0.40300000000000002,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.82199999999999995,"5min":0.82499999999999996,"15min":2.0609999999999999},"last":0.61399999999999999}]},{"osd":7,"last update":"Mon Aug 28 02:21:16 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.57199999999999995,"15min":0.55000000000000004},"min":{"1min":0.41199999999999998,"5min":0.35799999999999998,"15min":0.32700000000000001},"max":{"1min":0.85699999999999998,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.65600000000000003,"5min":0.59899999999999998,"15min":0.57399999999999995},"min":{"1min":0.42799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.83499999999999996,"5min":0.88100000000000001,"15min":0.89400000000000002},"last":0.60399999999999998}]}]},{"osd":1,"up_from":98,"seq":420906795031,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36872,"kb_used_data":5312,"kb_used_omap":7,"kb_used_meta":31544,"kb_avail":93708280,"statfs":{"total":95995035648,"available":95957278720,"internally_reserved":0,"allocated":5439488,"data_stored":4873933,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":32301387},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:20:50 2023","interfaces":[{"interface":"back","average":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"last":0.252},{"interface":"front","average":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.498,"5min":0.498,"15min":0.498},"last":0.34499999999999997}]},{"osd":2,"last update":"Mon Aug 28 02:21:23 2023","interfaces":[{"interface":"back","average":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"last":0.29999999999999999},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.35599999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28699999999999998}]},{"osd":4,"last update":"Mon Aug 28 02:20:50 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.90800000000000003,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.57899999999999996}]},{"osd":5,"last update":"Mon Aug 28 02:20:50 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.96599999999999997,"5min":0.96599999999999997,"15min":0.96599999999999997},"last":0.501}]},{"osd":6,"last update":"Mon Aug 28 02:20:50 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.94199999999999995,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.55300000000000005}]},{"osd":7,"last update":"Mon Aug 28 02:20:50 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.437},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.49199999999999999}]}]},{"osd":0,"up_from":90,"seq":386547056677,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":37464,"kb_used_data":5904,"kb_used_omap":8,"kb_used_meta":31543,"kb_avail":93707688,"statfs":{"total":95995035648,"available":95956672512,"internally_reserved":0,"allocated":6045696,"data_stored":5465837,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":32300393},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Aug 28 02:20:53 2023","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"min":{"1min":0.192,"5min":0.192,"15min":0.192},"max":{"1min":0.503,"5min":0.503,"15min":0.503},"last":0.378},{"interface":"front","average":{"1min":0.372,"5min":0.372,"15min":0.372},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.497,"5min":0.497,"15min":0.497},"last":0.27400000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:21:20 2023","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"last":3.4910000000000001},{"interface":"front","average":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"last":3.5619999999999998}]},{"osd":3,"last update":"Mon Aug 28 02:19:40 2023","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":0.435,"15min":0.435},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.48199999999999998}]},{"osd":4,"last update":"Mon Aug 28 02:20:43 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.65800000000000003,"15min":0.66600000000000004},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.82099999999999995,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.621,"5min":0.68899999999999995,"15min":0.70099999999999996},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.79800000000000004,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.67200000000000004}]},{"osd":5,"last update":"Mon Aug 28 02:20:43 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.64300000000000002,"15min":0.65000000000000002},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.82399999999999995,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.63800000000000001,"15min":0.64800000000000002},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.68300000000000005}]},{"osd":6,"last update":"Mon Aug 28 02:20:43 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.64200000000000002,"15min":0.65100000000000002},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.77200000000000002,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.622},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.69599999999999995,"15min":0.70399999999999996},"min":{"1min":0.42199999999999999,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.876,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.60799999999999998}]},{"osd":7,"last update":"Mon Aug 28 02:20:43 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.68400000000000005,"15min":0.69399999999999995},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.68999999999999995,"15min":0.69899999999999995},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.79500000000000004,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.66000000000000003}]}]},{"osd":2,"up_from":104,"seq":446676598801,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32680,"kb_used_data":1184,"kb_used_omap":10,"kb_used_meta":31477,"kb_avail":93712472,"statfs":{"total":95995035648,"available":95961571328,"internally_reserved":0,"allocated":1212416,"data_stored":672302,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10792,"internal_metadata":32232920},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:21:22 2023","interfaces":[{"interface":"back","average":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":0.29099999999999998},{"interface":"front","average":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.26300000000000001}]},{"osd":1,"last update":"Mon Aug 28 02:21:22 2023","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.317}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36099999999999999}]},{"osd":4,"last update":"Mon Aug 28 02:21:22 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.91100000000000003,"5min":0.91100000000000003,"15min":0.91100000000000003},"last":0.67900000000000005}]},{"osd":5,"last update":"Mon Aug 28 02:21:22 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.69799999999999995},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.499}]},{"osd":6,"last update":"Mon Aug 28 02:21:22 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.71599999999999997},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.72499999999999998}]},{"osd":7,"last update":"Mon Aug 28 02:21:22 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.745}]}]},{"osd":3,"up_from":112,"seq":481036337163,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28432,"kb_used_data":1160,"kb_used_omap":3,"kb_used_meta":27260,"kb_avail":93716720,"statfs":{"total":95995035648,"available":95965921280,"internally_reserved":0,"allocated":1187840,"data_stored":667951,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":27915170},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48399999999999999}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.50700000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53000000000000003}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.73299999999999998}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.75800000000000001}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65200000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.79200000000000004}]}]},{"osd":4,"up_from":30,"seq":128849019036,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298236,"kb_used_data":1140,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93446916,"statfs":{"total":95995035648,"available":95689641984,"internally_reserved":0,"allocated":1167360,"data_stored":663855,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:20:43 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.60399999999999998,"15min":0.61199999999999999},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.58599999999999997,"15min":0.59499999999999997},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.71399999999999997,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.33700000000000002}]},{"osd":1,"last update":"Mon Aug 28 02:20:49 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.28000000000000003},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.374}]},{"osd":2,"last update":"Mon Aug 28 02:21:19 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.29799999999999999}]},{"osd":3,"last update":"Mon Aug 28 02:20:01 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.58099999999999996,"15min":0.55300000000000005},"min":{"1min":0.45700000000000002,"5min":0.41799999999999998,"15min":0.307},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":1.1599999999999999},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.628,"5min":0.58099999999999996,"15min":0.55400000000000005},"min":{"1min":0.496,"5min":0.371,"15min":0.35899999999999999},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.871},"last":0.438}]},{"osd":5,"last update":"Mon Aug 28 02:20:32 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.49099999999999999,"15min":0.441},"min":{"1min":0.35399999999999998,"5min":0.30399999999999999,"15min":0.26600000000000001},"max":{"1min":0.72499999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.503,"5min":0.502,"15min":0.45500000000000002},"min":{"1min":0.36399999999999999,"5min":0.32500000000000001,"15min":0.27700000000000002},"max":{"1min":0.63300000000000001,"5min":0.78600000000000003,"15min":1.1839999999999999},"last":0.41199999999999998}]},{"osd":6,"last update":"Mon Aug 28 02:20:49 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.53000000000000003,"15min":0.48299999999999998},"min":{"1min":0.375,"5min":0.30499999999999999,"15min":0.27500000000000002},"max":{"1min":0.76000000000000001,"5min":0.77300000000000002,"15min":0.85099999999999998},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.58399999999999996,"15min":0.52100000000000002},"min":{"1min":0.45600000000000002,"5min":0.36799999999999999,"15min":0.318},"max":{"1min":0.77500000000000002,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.44900000000000001}]},{"osd":7,"last update":"Mon Aug 28 02:21:18 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.56899999999999995,"15min":0.53500000000000003},"min":{"1min":0.40400000000000003,"5min":0.36899999999999999,"15min":0.312},"max":{"1min":0.70999999999999996,"5min":0.82099999999999995,"15min":1.2110000000000001},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.59399999999999997,"15min":0.54500000000000004},"min":{"1min":0.41899999999999998,"5min":0.35199999999999998,"15min":0.32100000000000001},"max":{"1min":0.79700000000000004,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.51300000000000001}]}]},{"osd":5,"up_from":36,"seq":154618822808,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298220,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93446932,"statfs":{"total":95995035648,"available":95689658368,"internally_reserved":0,"allocated":1150976,"data_stored":659010,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:20:40 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.60999999999999999,"15min":0.62},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.61899999999999999,"15min":0.625},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.747,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.57599999999999996}]},{"osd":1,"last update":"Mon Aug 28 02:20:50 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.65000000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:21:22 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.62}]},{"osd":3,"last update":"Mon Aug 28 02:20:36 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57999999999999996,"15min":0.54900000000000004},"min":{"1min":0.39100000000000001,"5min":0.255,"15min":0.255},"max":{"1min":0.78400000000000003,"5min":1.1279999999999999,"15min":1.1779999999999999},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.59499999999999997,"15min":0.57299999999999995},"min":{"1min":0.40400000000000003,"5min":0.35299999999999998,"15min":0.307},"max":{"1min":0.80800000000000005,"5min":1.1679999999999999,"15min":1.1679999999999999},"last":0.51700000000000002}]},{"osd":4,"last update":"Mon Aug 28 02:20:36 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.47399999999999998,"15min":0.45300000000000001},"min":{"1min":0.33400000000000002,"5min":0.28000000000000003,"15min":0.27500000000000002},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.79200000000000004},"last":0.495},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.505,"15min":0.47799999999999998},"min":{"1min":0.31900000000000001,"5min":0.28799999999999998,"15min":0.28199999999999997},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":1.1499999999999999},"last":0.66700000000000004}]},{"osd":6,"last update":"Mon Aug 28 02:20:50 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.51300000000000001,"15min":0.51900000000000002},"min":{"1min":0.36699999999999999,"5min":0.32700000000000001,"15min":0.28399999999999997},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":5.2119999999999997},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.55400000000000005,"15min":0.51600000000000001},"min":{"1min":0.442,"5min":0.31900000000000001,"15min":0.30299999999999999},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":1.089},"last":0.71199999999999997}]},{"osd":7,"last update":"Mon Aug 28 02:21:13 2023","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.58299999999999996,"15min":0.55300000000000005},"min":{"1min":0.45000000000000001,"5min":0.35199999999999998,"15min":0.32900000000000001},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":1.119},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.63,"5min":0.57499999999999996,"15min":0.55300000000000005},"min":{"1min":0.40400000000000003,"5min":0.34999999999999998,"15min":0.29799999999999999},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.67700000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-08-28T02:21:37.470 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-08-28T02:21:37.471 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-08-28T02:21:37.471 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-08-28T02:21:37.471 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph health --format=json 2023-08-28T02:21:37.820 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:21:38.340 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:38 smithi143 ceph-mon[183059]: from='client.44501 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:21:38.340 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:38 smithi143 ceph-mon[183059]: pgmap v209: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:38.340 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:38 smithi143 ceph-mon[189871]: from='client.44501 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:21:38.340 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:38 smithi143 ceph-mon[189871]: pgmap v209: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:38.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:38 smithi169 ceph-mon[159415]: from='client.44501 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:21:38.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:38 smithi169 ceph-mon[159415]: pgmap v209: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:39.349 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:21:39.350 INFO:teuthology.orchestra.run.smithi143.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-08-28T02:21:39.931 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-08-28T02:21:39.931 INFO:teuthology.run_tasks:Running task print... 2023-08-28T02:21:39.940 INFO:teuthology.task.print:**** applying stress + thrashing to mixed-version cluster **** 2023-08-28T02:21:39.940 INFO:teuthology.run_tasks:Running task parallel... 2023-08-28T02:21:39.948 INFO:teuthology.task.parallel:starting parallel... 2023-08-28T02:21:39.948 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-08-28T02:21:39.948 INFO:teuthology.task.sequential:In sequential, running task thrashosds... 2023-08-28T02:21:39.950 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-08-28T02:21:39.950 INFO:tasks.thrashosds:overrides is {'admin_socket': {'branch': 'reef'}, 'ceph': {'conf': {'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': '9cf208207dfe23ffccb33308b22e51570cb45e51'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': '9cf208207dfe23ffccb33308b22e51570cb45e51'}}, 'selinux': {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'workunit': {'branch': 'reef', 'sha1': '81fef490342f7b9544e89fe32a9e51929064494d'}} 2023-08-28T02:21:39.950 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-08-28T02:21:39.950 INFO:tasks.thrashosds:Beginning thrashosds... 2023-08-28T02:21:39.951 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-08-28T02:21:39.951 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph pg dump --format=json 2023-08-28T02:21:40.300 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:21:40.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:40 smithi143 ceph-mon[183059]: pgmap v210: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:40.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:40 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/4192520654' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-08-28T02:21:40.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:40 smithi143 ceph-mon[189871]: pgmap v210: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:40.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:40 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/4192520654' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-08-28T02:21:40.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:40 smithi169 ceph-mon[159415]: pgmap v210: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:40.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:40 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/4192520654' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-08-28T02:21:41.430 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:41 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:21:41.430 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:41 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:21:41.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:41 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:21:41.783 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:21:41.784 INFO:teuthology.orchestra.run.smithi143.stderr:dumped all 2023-08-28T02:21:42.179 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:21:41 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:21:41] "GET /metrics HTTP/1.1" 200 34037 "" "Prometheus/2.33.4" 2023-08-28T02:21:42.338 INFO:teuthology.orchestra.run.smithi143.stdout:{"pg_ready":true,"pg_map":{"version":211,"stamp":"2023-08-28T02:21:41.034227+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6288,"num_read_kb":13719,"num_write":1106,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1341828,"kb_used_data":23364,"kb_used_omap":29,"kb_used_meta":1318370,"kb_avail":748619388,"statfs":{"total":767960285184,"available":766586253312,"internally_reserved":0,"allocated":23924736,"data_stored":19705380,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30674,"internal_metadata":1350010926},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002801"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":100,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.824785+0000","last_change":"2023-08-28T02:20:51.824785+0000","last_active":"2023-08-28T02:20:51.824785+0000","last_peered":"2023-08-28T02:20:51.824785+0000","last_clean":"2023-08-28T02:20:51.824785+0000","last_became_active":"2023-08-28T02:20:51.824447+0000","last_became_peered":"2023-08-28T02:20:51.824447+0000","last_unstale":"2023-08-28T02:20:51.824785+0000","last_undegraded":"2023-08-28T02:20:51.824785+0000","last_fullsized":"2023-08-28T02:20:51.824785+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":65,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.813241+0000","last_change":"2023-08-28T02:20:19.850252+0000","last_active":"2023-08-28T02:20:50.813241+0000","last_peered":"2023-08-28T02:20:50.813241+0000","last_clean":"2023-08-28T02:20:50.813241+0000","last_became_active":"2023-08-28T02:20:19.849913+0000","last_became_peered":"2023-08-28T02:20:19.849913+0000","last_unstale":"2023-08-28T02:20:50.813241+0000","last_undegraded":"2023-08-28T02:20:50.813241+0000","last_fullsized":"2023-08-28T02:20:50.813241+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:48:47.713465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":98,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.826985+0000","last_change":"2023-08-28T02:20:50.823101+0000","last_active":"2023-08-28T02:20:50.826985+0000","last_peered":"2023-08-28T02:20:50.826985+0000","last_clean":"2023-08-28T02:20:50.826985+0000","last_became_active":"2023-08-28T02:20:50.822599+0000","last_became_peered":"2023-08-28T02:20:50.822599+0000","last_unstale":"2023-08-28T02:20:50.826985+0000","last_undegraded":"2023-08-28T02:20:50.826985+0000","last_fullsized":"2023-08-28T02:20:50.826985+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:28:57.827789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":85,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.826313+0000","last_change":"2023-08-28T02:20:50.825915+0000","last_active":"2023-08-28T02:20:50.826313+0000","last_peered":"2023-08-28T02:20:50.826313+0000","last_clean":"2023-08-28T02:20:50.826313+0000","last_became_active":"2023-08-28T02:20:50.825526+0000","last_became_peered":"2023-08-28T02:20:50.825526+0000","last_unstale":"2023-08-28T02:20:50.826313+0000","last_undegraded":"2023-08-28T02:20:50.826313+0000","last_fullsized":"2023-08-28T02:20:50.826313+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:25:20.142641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":95,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.820522+0000","last_change":"2023-08-28T02:20:50.819786+0000","last_active":"2023-08-28T02:20:50.820522+0000","last_peered":"2023-08-28T02:20:50.820522+0000","last_clean":"2023-08-28T02:20:50.820522+0000","last_became_active":"2023-08-28T02:20:50.819451+0000","last_became_peered":"2023-08-28T02:20:50.819451+0000","last_unstale":"2023-08-28T02:20:50.820522+0000","last_undegraded":"2023-08-28T02:20:50.820522+0000","last_fullsized":"2023-08-28T02:20:50.820522+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":68,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.824216+0000","last_change":"2023-08-28T02:11:22.186024+0000","last_active":"2023-08-28T02:20:50.824216+0000","last_peered":"2023-08-28T02:20:50.824216+0000","last_clean":"2023-08-28T02:20:50.824216+0000","last_became_active":"2023-08-28T02:11:22.185500+0000","last_became_peered":"2023-08-28T02:11:22.185500+0000","last_unstale":"2023-08-28T02:20:50.824216+0000","last_undegraded":"2023-08-28T02:20:50.824216+0000","last_fullsized":"2023-08-28T02:20:50.824216+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":95,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.820006+0000","last_change":"2023-08-28T02:20:50.819015+0000","last_active":"2023-08-28T02:20:50.820006+0000","last_peered":"2023-08-28T02:20:50.820006+0000","last_clean":"2023-08-28T02:20:50.820006+0000","last_became_active":"2023-08-28T02:20:50.818667+0000","last_became_peered":"2023-08-28T02:20:50.818667+0000","last_unstale":"2023-08-28T02:20:50.820006+0000","last_undegraded":"2023-08-28T02:20:50.820006+0000","last_fullsized":"2023-08-28T02:20:50.820006+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":83,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.822081+0000","last_change":"2023-08-28T02:19:50.218760+0000","last_active":"2023-08-28T02:20:50.822081+0000","last_peered":"2023-08-28T02:20:50.822081+0000","last_clean":"2023-08-28T02:20:50.822081+0000","last_became_active":"2023-08-28T02:19:50.218376+0000","last_became_peered":"2023-08-28T02:19:50.218376+0000","last_unstale":"2023-08-28T02:20:50.822081+0000","last_undegraded":"2023-08-28T02:20:50.822081+0000","last_fullsized":"2023-08-28T02:20:50.822081+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":116,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.817585+0000","last_change":"2023-08-28T02:20:50.817119+0000","last_active":"2023-08-28T02:20:50.817585+0000","last_peered":"2023-08-28T02:20:50.817585+0000","last_clean":"2023-08-28T02:20:50.817585+0000","last_became_active":"2023-08-28T02:20:50.816756+0000","last_became_peered":"2023-08-28T02:20:50.816756+0000","last_unstale":"2023-08-28T02:20:50.817585+0000","last_undegraded":"2023-08-28T02:20:50.817585+0000","last_fullsized":"2023-08-28T02:20:50.817585+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:12:52.257562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":50,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.817184+0000","last_change":"2023-08-28T02:18:39.781312+0000","last_active":"2023-08-28T02:20:51.817184+0000","last_peered":"2023-08-28T02:20:51.817184+0000","last_clean":"2023-08-28T02:20:51.817184+0000","last_became_active":"2023-08-28T02:18:39.780916+0000","last_became_peered":"2023-08-28T02:18:39.780916+0000","last_unstale":"2023-08-28T02:20:51.817184+0000","last_undegraded":"2023-08-28T02:20:51.817184+0000","last_fullsized":"2023-08-28T02:20:51.817184+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:37:11.571250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":65,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.814304+0000","last_change":"2023-08-28T02:11:22.437146+0000","last_active":"2023-08-28T02:20:50.814304+0000","last_peered":"2023-08-28T02:20:50.814304+0000","last_clean":"2023-08-28T02:20:50.814304+0000","last_became_active":"2023-08-28T02:11:22.436798+0000","last_became_peered":"2023-08-28T02:11:22.436798+0000","last_unstale":"2023-08-28T02:20:50.814304+0000","last_undegraded":"2023-08-28T02:20:50.814304+0000","last_fullsized":"2023-08-28T02:20:50.814304+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":65,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.813953+0000","last_change":"2023-08-28T02:19:50.221001+0000","last_active":"2023-08-28T02:20:50.813953+0000","last_peered":"2023-08-28T02:20:50.813953+0000","last_clean":"2023-08-28T02:20:50.813953+0000","last_became_active":"2023-08-28T02:19:50.220611+0000","last_became_peered":"2023-08-28T02:19:50.220611+0000","last_unstale":"2023-08-28T02:20:50.813953+0000","last_undegraded":"2023-08-28T02:20:50.813953+0000","last_fullsized":"2023-08-28T02:20:50.813953+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:23:10.637630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'122","reported_seq":570,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.816897+0000","last_change":"2023-08-28T02:20:19.851065+0000","last_active":"2023-08-28T02:20:50.816897+0000","last_peered":"2023-08-28T02:20:50.816897+0000","last_clean":"2023-08-28T02:20:50.816897+0000","last_became_active":"2023-08-28T02:20:19.850477+0000","last_became_peered":"2023-08-28T02:20:19.850477+0000","last_unstale":"2023-08-28T02:20:50.816897+0000","last_undegraded":"2023-08-28T02:20:50.816897+0000","last_fullsized":"2023-08-28T02:20:50.816897+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":857,"num_read_kb":2224,"num_write":99,"num_write_kb":2312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":70,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.823860+0000","last_change":"2023-08-28T02:20:50.822679+0000","last_active":"2023-08-28T02:20:50.823860+0000","last_peered":"2023-08-28T02:20:50.823860+0000","last_clean":"2023-08-28T02:20:50.823860+0000","last_became_active":"2023-08-28T02:20:50.822180+0000","last_became_peered":"2023-08-28T02:20:50.822180+0000","last_unstale":"2023-08-28T02:20:50.823860+0000","last_undegraded":"2023-08-28T02:20:50.823860+0000","last_fullsized":"2023-08-28T02:20:50.823860+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:02:40.005023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'112","reported_seq":356,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.809841+0000","last_change":"2023-08-28T02:19:50.218206+0000","last_active":"2023-08-28T02:20:50.809841+0000","last_peered":"2023-08-28T02:20:50.809841+0000","last_clean":"2023-08-28T02:20:50.809841+0000","last_became_active":"2023-08-28T02:19:50.217637+0000","last_became_peered":"2023-08-28T02:19:50.217637+0000","last_unstale":"2023-08-28T02:20:50.809841+0000","last_undegraded":"2023-08-28T02:20:50.809841+0000","last_fullsized":"2023-08-28T02:20:50.809841+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:12:06.224907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":444,"num_read_kb":473,"num_write":94,"num_write_kb":127,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":73,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.823867+0000","last_change":"2023-08-28T02:20:50.823477+0000","last_active":"2023-08-28T02:20:50.823867+0000","last_peered":"2023-08-28T02:20:50.823867+0000","last_clean":"2023-08-28T02:20:50.823867+0000","last_became_active":"2023-08-28T02:20:50.823158+0000","last_became_peered":"2023-08-28T02:20:50.823158+0000","last_unstale":"2023-08-28T02:20:50.823867+0000","last_undegraded":"2023-08-28T02:20:50.823867+0000","last_fullsized":"2023-08-28T02:20:50.823867+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:33:18.347341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":47,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.813160+0000","last_change":"2023-08-28T02:18:39.780069+0000","last_active":"2023-08-28T02:20:50.813160+0000","last_peered":"2023-08-28T02:20:50.813160+0000","last_clean":"2023-08-28T02:20:50.813160+0000","last_became_active":"2023-08-28T02:18:39.779829+0000","last_became_peered":"2023-08-28T02:18:39.779829+0000","last_unstale":"2023-08-28T02:20:50.813160+0000","last_undegraded":"2023-08-28T02:20:50.813160+0000","last_fullsized":"2023-08-28T02:20:50.813160+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:35:01.309659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":52,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.810821+0000","last_change":"2023-08-28T02:20:19.850916+0000","last_active":"2023-08-28T02:20:50.810821+0000","last_peered":"2023-08-28T02:20:50.810821+0000","last_clean":"2023-08-28T02:20:50.810821+0000","last_became_active":"2023-08-28T02:20:19.850627+0000","last_became_peered":"2023-08-28T02:20:19.850627+0000","last_unstale":"2023-08-28T02:20:50.810821+0000","last_undegraded":"2023-08-28T02:20:50.810821+0000","last_fullsized":"2023-08-28T02:20:50.810821+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:19:14.346189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'103","reported_seq":535,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.820309+0000","last_change":"2023-08-28T02:18:39.786876+0000","last_active":"2023-08-28T02:20:50.820309+0000","last_peered":"2023-08-28T02:20:50.820309+0000","last_clean":"2023-08-28T02:20:50.820309+0000","last_became_active":"2023-08-28T02:18:39.786414+0000","last_became_peered":"2023-08-28T02:18:39.786414+0000","last_unstale":"2023-08-28T02:20:50.820309+0000","last_undegraded":"2023-08-28T02:20:50.820309+0000","last_fullsized":"2023-08-28T02:20:50.820309+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":803,"num_read_kb":2080,"num_write":73,"num_write_kb":1244,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":97,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.886782+0000","last_change":"2023-08-28T02:20:51.886782+0000","last_active":"2023-08-28T02:20:51.886782+0000","last_peered":"2023-08-28T02:20:51.886782+0000","last_clean":"2023-08-28T02:20:51.886782+0000","last_became_active":"2023-08-28T02:20:51.886444+0000","last_became_peered":"2023-08-28T02:20:51.886444+0000","last_unstale":"2023-08-28T02:20:51.886782+0000","last_undegraded":"2023-08-28T02:20:51.886782+0000","last_fullsized":"2023-08-28T02:20:51.886782+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:57:03.704951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":61,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.823573+0000","last_change":"2023-08-28T02:20:50.823036+0000","last_active":"2023-08-28T02:20:50.823573+0000","last_peered":"2023-08-28T02:20:50.823573+0000","last_clean":"2023-08-28T02:20:50.823573+0000","last_became_active":"2023-08-28T02:20:50.822766+0000","last_became_peered":"2023-08-28T02:20:50.822766+0000","last_unstale":"2023-08-28T02:20:50.823573+0000","last_undegraded":"2023-08-28T02:20:50.823573+0000","last_fullsized":"2023-08-28T02:20:50.823573+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:09:26.786992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"78'148","reported_seq":659,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.818235+0000","last_change":"2023-08-28T02:19:50.221311+0000","last_active":"2023-08-28T02:20:50.818235+0000","last_peered":"2023-08-28T02:20:50.818235+0000","last_clean":"2023-08-28T02:20:50.818235+0000","last_became_active":"2023-08-28T02:19:50.220754+0000","last_became_peered":"2023-08-28T02:19:50.220754+0000","last_unstale":"2023-08-28T02:20:50.818235+0000","last_undegraded":"2023-08-28T02:20:50.818235+0000","last_fullsized":"2023-08-28T02:20:50.818235+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1092,"num_read_kb":1955,"num_write":127,"num_write_kb":5177,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":65,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.816396+0000","last_change":"2023-08-28T02:11:23.249640+0000","last_active":"2023-08-28T02:20:50.816396+0000","last_peered":"2023-08-28T02:20:50.816396+0000","last_clean":"2023-08-28T02:20:50.816396+0000","last_became_active":"2023-08-28T02:11:23.249355+0000","last_became_peered":"2023-08-28T02:11:23.249355+0000","last_unstale":"2023-08-28T02:20:50.816396+0000","last_undegraded":"2023-08-28T02:20:50.816396+0000","last_fullsized":"2023-08-28T02:20:50.816396+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":64,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.826633+0000","last_change":"2023-08-28T02:20:50.825728+0000","last_active":"2023-08-28T02:20:50.826633+0000","last_peered":"2023-08-28T02:20:50.826633+0000","last_clean":"2023-08-28T02:20:50.826633+0000","last_became_active":"2023-08-28T02:20:50.825418+0000","last_became_peered":"2023-08-28T02:20:50.825418+0000","last_unstale":"2023-08-28T02:20:50.826633+0000","last_undegraded":"2023-08-28T02:20:50.826633+0000","last_fullsized":"2023-08-28T02:20:50.826633+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:55:48.126269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"86'139","reported_seq":317,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.818120+0000","last_change":"2023-08-28T02:18:39.786823+0000","last_active":"2023-08-28T02:20:50.818120+0000","last_peered":"2023-08-28T02:20:50.818120+0000","last_clean":"2023-08-28T02:20:50.818120+0000","last_became_active":"2023-08-28T02:18:39.786357+0000","last_became_peered":"2023-08-28T02:18:39.786357+0000","last_unstale":"2023-08-28T02:20:50.818120+0000","last_undegraded":"2023-08-28T02:20:50.818120+0000","last_fullsized":"2023-08-28T02:20:50.818120+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_clean_scrub_stamp":"2023-08-28T02:08:02.043718+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'114","reported_seq":412,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.818867+0000","last_change":"2023-08-28T02:20:19.852152+0000","last_active":"2023-08-28T02:20:50.818867+0000","last_peered":"2023-08-28T02:20:50.818867+0000","last_clean":"2023-08-28T02:20:50.818867+0000","last_became_active":"2023-08-28T02:20:19.851615+0000","last_became_peered":"2023-08-28T02:20:19.851615+0000","last_unstale":"2023-08-28T02:20:50.818867+0000","last_undegraded":"2023-08-28T02:20:50.818867+0000","last_fullsized":"2023-08-28T02:20:50.818867+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":114,"log_dups_size":0,"ondisk_log_size":114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":358,"num_read_kb":1352,"num_write":102,"num_write_kb":1104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":84,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.808976+0000","last_change":"2023-08-28T02:19:50.211156+0000","last_active":"2023-08-28T02:20:50.808976+0000","last_peered":"2023-08-28T02:20:50.808976+0000","last_clean":"2023-08-28T02:20:50.808976+0000","last_became_active":"2023-08-28T02:19:50.210398+0000","last_became_peered":"2023-08-28T02:19:50.210398+0000","last_unstale":"2023-08-28T02:20:50.808976+0000","last_undegraded":"2023-08-28T02:20:50.808976+0000","last_fullsized":"2023-08-28T02:20:50.808976+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:14:41.748197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":52,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.808409+0000","last_change":"2023-08-28T02:20:19.851591+0000","last_active":"2023-08-28T02:20:50.808409+0000","last_peered":"2023-08-28T02:20:50.808409+0000","last_clean":"2023-08-28T02:20:50.808409+0000","last_became_active":"2023-08-28T02:20:19.851291+0000","last_became_peered":"2023-08-28T02:20:19.851291+0000","last_unstale":"2023-08-28T02:20:50.808409+0000","last_undegraded":"2023-08-28T02:20:50.808409+0000","last_fullsized":"2023-08-28T02:20:50.808409+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:33:51.259707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"78'47","reported_seq":266,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.819902+0000","last_change":"2023-08-28T02:20:50.819614+0000","last_active":"2023-08-28T02:20:50.819902+0000","last_peered":"2023-08-28T02:20:50.819902+0000","last_clean":"2023-08-28T02:20:50.819902+0000","last_became_active":"2023-08-28T02:20:50.819114+0000","last_became_peered":"2023-08-28T02:20:50.819114+0000","last_unstale":"2023-08-28T02:20:50.819902+0000","last_undegraded":"2023-08-28T02:20:50.819902+0000","last_fullsized":"2023-08-28T02:20:50.819902+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":47,"log_dups_size":0,"ondisk_log_size":47,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:29:07.212870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":162,"num_read_kb":1281,"num_write":48,"num_write_kb":1153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":51,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.810901+0000","last_change":"2023-08-28T02:20:19.851150+0000","last_active":"2023-08-28T02:20:50.810901+0000","last_peered":"2023-08-28T02:20:50.810901+0000","last_clean":"2023-08-28T02:20:50.810901+0000","last_became_active":"2023-08-28T02:20:19.850886+0000","last_became_peered":"2023-08-28T02:20:19.850886+0000","last_unstale":"2023-08-28T02:20:50.810901+0000","last_undegraded":"2023-08-28T02:20:50.810901+0000","last_fullsized":"2023-08-28T02:20:50.810901+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:50:28.287980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":81,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.813107+0000","last_change":"2023-08-28T02:18:39.779607+0000","last_active":"2023-08-28T02:20:50.813107+0000","last_peered":"2023-08-28T02:20:50.813107+0000","last_clean":"2023-08-28T02:20:50.813107+0000","last_became_active":"2023-08-28T02:18:39.779303+0000","last_became_peered":"2023-08-28T02:18:39.779303+0000","last_unstale":"2023-08-28T02:20:50.813107+0000","last_undegraded":"2023-08-28T02:20:50.813107+0000","last_fullsized":"2023-08-28T02:20:50.813107+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"78'204","reported_seq":1030,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.819453+0000","last_change":"2023-08-28T02:19:50.219949+0000","last_active":"2023-08-28T02:20:50.819453+0000","last_peered":"2023-08-28T02:20:50.819453+0000","last_clean":"2023-08-28T02:20:50.819453+0000","last_became_active":"2023-08-28T02:19:50.219356+0000","last_became_peered":"2023-08-28T02:19:50.219356+0000","last_unstale":"2023-08-28T02:20:50.819453+0000","last_undegraded":"2023-08-28T02:20:50.819453+0000","last_fullsized":"2023-08-28T02:20:50.819453+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":204,"log_dups_size":0,"ondisk_log_size":204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1521,"num_read_kb":2511,"num_write":178,"num_write_kb":1175,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":11,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.824031+0000","last_change":"2023-08-28T02:20:51.823558+0000","last_active":"2023-08-28T02:20:51.824031+0000","last_peered":"2023-08-28T02:20:51.824031+0000","last_clean":"2023-08-28T02:20:51.824031+0000","last_became_active":"2023-08-28T02:20:51.823259+0000","last_became_peered":"2023-08-28T02:20:51.823259+0000","last_unstale":"2023-08-28T02:20:51.824031+0000","last_undegraded":"2023-08-28T02:20:51.824031+0000","last_fullsized":"2023-08-28T02:20:51.824031+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:22:15.166170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":96,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.828624+0000","last_change":"2023-08-28T02:20:50.828341+0000","last_active":"2023-08-28T02:20:50.828624+0000","last_peered":"2023-08-28T02:20:50.828624+0000","last_clean":"2023-08-28T02:20:50.828624+0000","last_became_active":"2023-08-28T02:20:50.828019+0000","last_became_peered":"2023-08-28T02:20:50.828019+0000","last_unstale":"2023-08-28T02:20:50.828624+0000","last_undegraded":"2023-08-28T02:20:50.828624+0000","last_fullsized":"2023-08-28T02:20:50.828624+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'126","reported_seq":570,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.810726+0000","last_change":"2023-08-28T02:19:50.213556+0000","last_active":"2023-08-28T02:20:50.810726+0000","last_peered":"2023-08-28T02:20:50.810726+0000","last_clean":"2023-08-28T02:20:50.810726+0000","last_became_active":"2023-08-28T02:19:50.212996+0000","last_became_peered":"2023-08-28T02:19:50.212996+0000","last_unstale":"2023-08-28T02:20:50.810726+0000","last_undegraded":"2023-08-28T02:20:50.810726+0000","last_fullsized":"2023-08-28T02:20:50.810726+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:38:28.157352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":743,"num_read_kb":1331,"num_write":125,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":53,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.821699+0000","last_change":"2023-08-28T02:20:50.820676+0000","last_active":"2023-08-28T02:20:50.821699+0000","last_peered":"2023-08-28T02:20:50.821699+0000","last_clean":"2023-08-28T02:20:50.821699+0000","last_became_active":"2023-08-28T02:20:50.820389+0000","last_became_peered":"2023-08-28T02:20:50.820389+0000","last_unstale":"2023-08-28T02:20:50.821699+0000","last_undegraded":"2023-08-28T02:20:50.821699+0000","last_fullsized":"2023-08-28T02:20:50.821699+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:24:25.623757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":79,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.823878+0000","last_change":"2023-08-28T02:20:50.822792+0000","last_active":"2023-08-28T02:20:50.823878+0000","last_peered":"2023-08-28T02:20:50.823878+0000","last_clean":"2023-08-28T02:20:50.823878+0000","last_became_active":"2023-08-28T02:20:50.822157+0000","last_became_peered":"2023-08-28T02:20:50.822157+0000","last_unstale":"2023-08-28T02:20:50.823878+0000","last_undegraded":"2023-08-28T02:20:50.823878+0000","last_fullsized":"2023-08-28T02:20:50.823878+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:15:16.261246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":41,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.813102+0000","last_change":"2023-08-28T02:19:51.204017+0000","last_active":"2023-08-28T02:20:50.813102+0000","last_peered":"2023-08-28T02:20:50.813102+0000","last_clean":"2023-08-28T02:20:50.813102+0000","last_became_active":"2023-08-28T02:19:51.203627+0000","last_became_peered":"2023-08-28T02:19:51.203627+0000","last_unstale":"2023-08-28T02:20:50.813102+0000","last_undegraded":"2023-08-28T02:20:50.813102+0000","last_fullsized":"2023-08-28T02:20:50.813102+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:14:06.231701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":11,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.927939+0000","last_change":"2023-08-28T02:20:51.927612+0000","last_active":"2023-08-28T02:20:51.927939+0000","last_peered":"2023-08-28T02:20:51.927939+0000","last_clean":"2023-08-28T02:20:51.927939+0000","last_became_active":"2023-08-28T02:20:51.927256+0000","last_became_peered":"2023-08-28T02:20:51.927256+0000","last_unstale":"2023-08-28T02:20:51.927939+0000","last_undegraded":"2023-08-28T02:20:51.927939+0000","last_fullsized":"2023-08-28T02:20:51.927939+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:05:59.937193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":115,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.886978+0000","last_change":"2023-08-28T02:20:51.886978+0000","last_active":"2023-08-28T02:20:51.886978+0000","last_peered":"2023-08-28T02:20:51.886978+0000","last_clean":"2023-08-28T02:20:51.886978+0000","last_became_active":"2023-08-28T02:20:51.886625+0000","last_became_peered":"2023-08-28T02:20:51.886625+0000","last_unstale":"2023-08-28T02:20:51.886978+0000","last_undegraded":"2023-08-28T02:20:51.886978+0000","last_fullsized":"2023-08-28T02:20:51.886978+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":101,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.821045+0000","last_change":"2023-08-28T02:20:51.821045+0000","last_active":"2023-08-28T02:20:51.821045+0000","last_peered":"2023-08-28T02:20:51.821045+0000","last_clean":"2023-08-28T02:20:51.821045+0000","last_became_active":"2023-08-28T02:20:51.820588+0000","last_became_peered":"2023-08-28T02:20:51.820588+0000","last_unstale":"2023-08-28T02:20:51.821045+0000","last_undegraded":"2023-08-28T02:20:51.821045+0000","last_fullsized":"2023-08-28T02:20:51.821045+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":80,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.820658+0000","last_change":"2023-08-28T02:20:50.820134+0000","last_active":"2023-08-28T02:20:50.820658+0000","last_peered":"2023-08-28T02:20:50.820658+0000","last_clean":"2023-08-28T02:20:50.820658+0000","last_became_active":"2023-08-28T02:20:50.819765+0000","last_became_peered":"2023-08-28T02:20:50.819765+0000","last_unstale":"2023-08-28T02:20:50.820658+0000","last_undegraded":"2023-08-28T02:20:50.820658+0000","last_fullsized":"2023-08-28T02:20:50.820658+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":73,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.826281+0000","last_change":"2023-08-28T02:20:50.824864+0000","last_active":"2023-08-28T02:20:50.826281+0000","last_peered":"2023-08-28T02:20:50.826281+0000","last_clean":"2023-08-28T02:20:50.826281+0000","last_became_active":"2023-08-28T02:20:50.824495+0000","last_became_peered":"2023-08-28T02:20:50.824495+0000","last_unstale":"2023-08-28T02:20:50.826281+0000","last_undegraded":"2023-08-28T02:20:50.826281+0000","last_fullsized":"2023-08-28T02:20:50.826281+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:45:04.111154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":73,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.817140+0000","last_change":"2023-08-28T02:19:51.205245+0000","last_active":"2023-08-28T02:20:51.817140+0000","last_peered":"2023-08-28T02:20:51.817140+0000","last_clean":"2023-08-28T02:20:51.817140+0000","last_became_active":"2023-08-28T02:19:51.204896+0000","last_became_peered":"2023-08-28T02:19:51.204896+0000","last_unstale":"2023-08-28T02:20:51.817140+0000","last_undegraded":"2023-08-28T02:20:51.817140+0000","last_fullsized":"2023-08-28T02:20:51.817140+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:51:24.634702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":81,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.824924+0000","last_change":"2023-08-28T02:20:50.824679+0000","last_active":"2023-08-28T02:20:50.824924+0000","last_peered":"2023-08-28T02:20:50.824924+0000","last_clean":"2023-08-28T02:20:50.824924+0000","last_became_active":"2023-08-28T02:20:50.824339+0000","last_became_peered":"2023-08-28T02:20:50.824339+0000","last_unstale":"2023-08-28T02:20:50.824924+0000","last_undegraded":"2023-08-28T02:20:50.824924+0000","last_fullsized":"2023-08-28T02:20:50.824924+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":114,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.927844+0000","last_change":"2023-08-28T02:20:51.927844+0000","last_active":"2023-08-28T02:20:51.927844+0000","last_peered":"2023-08-28T02:20:51.927844+0000","last_clean":"2023-08-28T02:20:51.927844+0000","last_became_active":"2023-08-28T02:20:51.927433+0000","last_became_peered":"2023-08-28T02:20:51.927433+0000","last_unstale":"2023-08-28T02:20:51.927844+0000","last_undegraded":"2023-08-28T02:20:51.927844+0000","last_fullsized":"2023-08-28T02:20:51.927844+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":67,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.820415+0000","last_change":"2023-08-28T02:11:22.188982+0000","last_active":"2023-08-28T02:20:50.820415+0000","last_peered":"2023-08-28T02:20:50.820415+0000","last_clean":"2023-08-28T02:20:50.820415+0000","last_became_active":"2023-08-28T02:11:22.188663+0000","last_became_peered":"2023-08-28T02:11:22.188663+0000","last_unstale":"2023-08-28T02:20:50.820415+0000","last_undegraded":"2023-08-28T02:20:50.820415+0000","last_fullsized":"2023-08-28T02:20:50.820415+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":95,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.824117+0000","last_change":"2023-08-28T02:20:50.823610+0000","last_active":"2023-08-28T02:20:50.824117+0000","last_peered":"2023-08-28T02:20:50.824117+0000","last_clean":"2023-08-28T02:20:50.824117+0000","last_became_active":"2023-08-28T02:20:50.823066+0000","last_became_peered":"2023-08-28T02:20:50.823066+0000","last_unstale":"2023-08-28T02:20:50.824117+0000","last_undegraded":"2023-08-28T02:20:50.824117+0000","last_fullsized":"2023-08-28T02:20:50.824117+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":83,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.819174+0000","last_change":"2023-08-28T02:19:50.220079+0000","last_active":"2023-08-28T02:20:50.819174+0000","last_peered":"2023-08-28T02:20:50.819174+0000","last_clean":"2023-08-28T02:20:50.819174+0000","last_became_active":"2023-08-28T02:19:50.219671+0000","last_became_peered":"2023-08-28T02:19:50.219671+0000","last_unstale":"2023-08-28T02:20:50.819174+0000","last_undegraded":"2023-08-28T02:20:50.819174+0000","last_fullsized":"2023-08-28T02:20:50.819174+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":65,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.818304+0000","last_change":"2023-08-28T02:11:23.249061+0000","last_active":"2023-08-28T02:20:50.818304+0000","last_peered":"2023-08-28T02:20:50.818304+0000","last_clean":"2023-08-28T02:20:50.818304+0000","last_became_active":"2023-08-28T02:11:23.248739+0000","last_became_peered":"2023-08-28T02:11:23.248739+0000","last_unstale":"2023-08-28T02:20:50.818304+0000","last_undegraded":"2023-08-28T02:20:50.818304+0000","last_fullsized":"2023-08-28T02:20:50.818304+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":95,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.819235+0000","last_change":"2023-08-28T02:20:43.220383+0000","last_active":"2023-08-28T02:20:50.819235+0000","last_peered":"2023-08-28T02:20:50.819235+0000","last_clean":"2023-08-28T02:20:50.819235+0000","last_became_active":"2023-08-28T02:20:43.136191+0000","last_became_peered":"2023-08-28T02:20:43.136191+0000","last_unstale":"2023-08-28T02:20:50.819235+0000","last_undegraded":"2023-08-28T02:20:50.819235+0000","last_fullsized":"2023-08-28T02:20:50.819235+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":47,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.810990+0000","last_change":"2023-08-28T02:20:19.851128+0000","last_active":"2023-08-28T02:20:50.810990+0000","last_peered":"2023-08-28T02:20:50.810990+0000","last_clean":"2023-08-28T02:20:50.810990+0000","last_became_active":"2023-08-28T02:20:19.849358+0000","last_became_peered":"2023-08-28T02:20:19.849358+0000","last_unstale":"2023-08-28T02:20:50.810990+0000","last_undegraded":"2023-08-28T02:20:50.810990+0000","last_fullsized":"2023-08-28T02:20:50.810990+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:32:27.226152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":89,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.816878+0000","last_change":"2023-08-28T02:18:39.778728+0000","last_active":"2023-08-28T02:20:50.816878+0000","last_peered":"2023-08-28T02:20:50.816878+0000","last_clean":"2023-08-28T02:20:50.816878+0000","last_became_active":"2023-08-28T02:18:39.778283+0000","last_became_peered":"2023-08-28T02:18:39.778283+0000","last_unstale":"2023-08-28T02:20:50.816878+0000","last_undegraded":"2023-08-28T02:20:50.816878+0000","last_fullsized":"2023-08-28T02:20:50.816878+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":62,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.817071+0000","last_change":"2023-08-28T02:20:50.823670+0000","last_active":"2023-08-28T02:20:51.817071+0000","last_peered":"2023-08-28T02:20:51.817071+0000","last_clean":"2023-08-28T02:20:51.817071+0000","last_became_active":"2023-08-28T02:20:50.823229+0000","last_became_peered":"2023-08-28T02:20:50.823229+0000","last_unstale":"2023-08-28T02:20:51.817071+0000","last_undegraded":"2023-08-28T02:20:51.817071+0000","last_fullsized":"2023-08-28T02:20:51.817071+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:50:36.316012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":66,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.817149+0000","last_change":"2023-08-28T02:11:22.188327+0000","last_active":"2023-08-28T02:20:50.817149+0000","last_peered":"2023-08-28T02:20:50.817149+0000","last_clean":"2023-08-28T02:20:50.817149+0000","last_became_active":"2023-08-28T02:11:22.187870+0000","last_became_peered":"2023-08-28T02:11:22.187870+0000","last_unstale":"2023-08-28T02:20:50.817149+0000","last_undegraded":"2023-08-28T02:20:50.817149+0000","last_fullsized":"2023-08-28T02:20:50.817149+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":105,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.927775+0000","last_change":"2023-08-28T02:20:51.927775+0000","last_active":"2023-08-28T02:20:51.927775+0000","last_peered":"2023-08-28T02:20:51.927775+0000","last_clean":"2023-08-28T02:20:51.927775+0000","last_became_active":"2023-08-28T02:20:51.927401+0000","last_became_peered":"2023-08-28T02:20:51.927401+0000","last_unstale":"2023-08-28T02:20:51.927775+0000","last_undegraded":"2023-08-28T02:20:51.927775+0000","last_fullsized":"2023-08-28T02:20:51.927775+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":61,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.816807+0000","last_change":"2023-08-28T02:20:50.821233+0000","last_active":"2023-08-28T02:20:51.816807+0000","last_peered":"2023-08-28T02:20:51.816807+0000","last_clean":"2023-08-28T02:20:51.816807+0000","last_became_active":"2023-08-28T02:20:50.821062+0000","last_became_peered":"2023-08-28T02:20:50.821062+0000","last_unstale":"2023-08-28T02:20:51.816807+0000","last_undegraded":"2023-08-28T02:20:51.816807+0000","last_fullsized":"2023-08-28T02:20:51.816807+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:19:40.305189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":78,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.826178+0000","last_change":"2023-08-28T02:20:50.825633+0000","last_active":"2023-08-28T02:20:50.826178+0000","last_peered":"2023-08-28T02:20:50.826178+0000","last_clean":"2023-08-28T02:20:50.826178+0000","last_became_active":"2023-08-28T02:20:50.825328+0000","last_became_peered":"2023-08-28T02:20:50.825328+0000","last_unstale":"2023-08-28T02:20:50.826178+0000","last_undegraded":"2023-08-28T02:20:50.826178+0000","last_fullsized":"2023-08-28T02:20:50.826178+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:59:37.406892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":83,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.818201+0000","last_change":"2023-08-28T02:20:19.849490+0000","last_active":"2023-08-28T02:20:50.818201+0000","last_peered":"2023-08-28T02:20:50.818201+0000","last_clean":"2023-08-28T02:20:50.818201+0000","last_became_active":"2023-08-28T02:20:19.848962+0000","last_became_peered":"2023-08-28T02:20:19.848962+0000","last_unstale":"2023-08-28T02:20:50.818201+0000","last_undegraded":"2023-08-28T02:20:50.818201+0000","last_fullsized":"2023-08-28T02:20:50.818201+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":94,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.812125+0000","last_change":"2023-08-28T02:19:51.204118+0000","last_active":"2023-08-28T02:20:50.812125+0000","last_peered":"2023-08-28T02:20:50.812125+0000","last_clean":"2023-08-28T02:20:50.812125+0000","last_became_active":"2023-08-28T02:19:51.203745+0000","last_became_peered":"2023-08-28T02:19:51.203745+0000","last_unstale":"2023-08-28T02:20:50.812125+0000","last_undegraded":"2023-08-28T02:20:50.812125+0000","last_fullsized":"2023-08-28T02:20:50.812125+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"86'22","reported_seq":88,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.812982+0000","last_change":"2023-08-28T02:11:22.184497+0000","last_active":"2023-08-28T02:20:50.812982+0000","last_peered":"2023-08-28T02:20:50.812982+0000","last_clean":"2023-08-28T02:20:50.812982+0000","last_became_active":"2023-08-28T02:11:22.184173+0000","last_became_peered":"2023-08-28T02:11:22.184173+0000","last_unstale":"2023-08-28T02:20:50.812982+0000","last_undegraded":"2023-08-28T02:20:50.812982+0000","last_fullsized":"2023-08-28T02:20:50.812982+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":53,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.809557+0000","last_change":"2023-08-28T02:20:19.851616+0000","last_active":"2023-08-28T02:20:50.809557+0000","last_peered":"2023-08-28T02:20:50.809557+0000","last_clean":"2023-08-28T02:20:50.809557+0000","last_became_active":"2023-08-28T02:20:19.851322+0000","last_became_peered":"2023-08-28T02:20:19.851322+0000","last_unstale":"2023-08-28T02:20:50.809557+0000","last_undegraded":"2023-08-28T02:20:50.809557+0000","last_fullsized":"2023-08-28T02:20:50.809557+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:03:37.565646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":84,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.826581+0000","last_change":"2023-08-28T02:20:50.826189+0000","last_active":"2023-08-28T02:20:50.826581+0000","last_peered":"2023-08-28T02:20:50.826581+0000","last_clean":"2023-08-28T02:20:50.826581+0000","last_became_active":"2023-08-28T02:20:50.825824+0000","last_became_peered":"2023-08-28T02:20:50.825824+0000","last_unstale":"2023-08-28T02:20:50.826581+0000","last_undegraded":"2023-08-28T02:20:50.826581+0000","last_fullsized":"2023-08-28T02:20:50.826581+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":64,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.827001+0000","last_change":"2023-08-28T02:20:50.826216+0000","last_active":"2023-08-28T02:20:50.827001+0000","last_peered":"2023-08-28T02:20:50.827001+0000","last_clean":"2023-08-28T02:20:50.827001+0000","last_became_active":"2023-08-28T02:20:50.825949+0000","last_became_peered":"2023-08-28T02:20:50.825949+0000","last_unstale":"2023-08-28T02:20:50.827001+0000","last_undegraded":"2023-08-28T02:20:50.827001+0000","last_fullsized":"2023-08-28T02:20:50.827001+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:08:15.626853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":81,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.818941+0000","last_change":"2023-08-28T02:19:50.214698+0000","last_active":"2023-08-28T02:20:50.818941+0000","last_peered":"2023-08-28T02:20:50.818941+0000","last_clean":"2023-08-28T02:20:50.818941+0000","last_became_active":"2023-08-28T02:19:50.214323+0000","last_became_peered":"2023-08-28T02:19:50.214323+0000","last_unstale":"2023-08-28T02:20:50.818941+0000","last_undegraded":"2023-08-28T02:20:50.818941+0000","last_fullsized":"2023-08-28T02:20:50.818941+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":41,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.812255+0000","last_change":"2023-08-28T02:19:51.203925+0000","last_active":"2023-08-28T02:20:50.812255+0000","last_peered":"2023-08-28T02:20:50.812255+0000","last_clean":"2023-08-28T02:20:50.812255+0000","last_became_active":"2023-08-28T02:19:51.203594+0000","last_became_peered":"2023-08-28T02:19:51.203594+0000","last_unstale":"2023-08-28T02:20:50.812255+0000","last_undegraded":"2023-08-28T02:20:50.812255+0000","last_fullsized":"2023-08-28T02:20:50.812255+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:02:14.852254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,6],"acting":[0,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":61,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.817326+0000","last_change":"2023-08-28T02:20:50.828055+0000","last_active":"2023-08-28T02:20:51.817326+0000","last_peered":"2023-08-28T02:20:51.817326+0000","last_clean":"2023-08-28T02:20:51.817326+0000","last_became_active":"2023-08-28T02:20:50.827778+0000","last_became_peered":"2023-08-28T02:20:50.827778+0000","last_unstale":"2023-08-28T02:20:51.817326+0000","last_undegraded":"2023-08-28T02:20:51.817326+0000","last_fullsized":"2023-08-28T02:20:51.817326+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:50:35.464371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":100,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.823018+0000","last_change":"2023-08-28T02:20:51.823018+0000","last_active":"2023-08-28T02:20:51.823018+0000","last_peered":"2023-08-28T02:20:51.823018+0000","last_clean":"2023-08-28T02:20:51.823018+0000","last_became_active":"2023-08-28T02:20:51.822499+0000","last_became_peered":"2023-08-28T02:20:51.822499+0000","last_unstale":"2023-08-28T02:20:51.823018+0000","last_undegraded":"2023-08-28T02:20:51.823018+0000","last_fullsized":"2023-08-28T02:20:51.823018+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":94,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.821931+0000","last_change":"2023-08-28T02:20:50.819499+0000","last_active":"2023-08-28T02:20:50.821931+0000","last_peered":"2023-08-28T02:20:50.821931+0000","last_clean":"2023-08-28T02:20:50.821931+0000","last_became_active":"2023-08-28T02:20:50.819026+0000","last_became_peered":"2023-08-28T02:20:50.819026+0000","last_unstale":"2023-08-28T02:20:50.821931+0000","last_undegraded":"2023-08-28T02:20:50.821931+0000","last_fullsized":"2023-08-28T02:20:50.821931+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:59:22.177440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":62,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.827522+0000","last_change":"2023-08-28T02:20:50.823506+0000","last_active":"2023-08-28T02:20:50.827522+0000","last_peered":"2023-08-28T02:20:50.827522+0000","last_clean":"2023-08-28T02:20:50.827522+0000","last_became_active":"2023-08-28T02:20:50.822836+0000","last_became_peered":"2023-08-28T02:20:50.822836+0000","last_unstale":"2023-08-28T02:20:50.827522+0000","last_undegraded":"2023-08-28T02:20:50.827522+0000","last_fullsized":"2023-08-28T02:20:50.827522+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:43:06.363328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":52,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.821703+0000","last_change":"2023-08-28T02:20:50.820616+0000","last_active":"2023-08-28T02:20:50.821703+0000","last_peered":"2023-08-28T02:20:50.821703+0000","last_clean":"2023-08-28T02:20:50.821703+0000","last_became_active":"2023-08-28T02:20:50.820305+0000","last_became_peered":"2023-08-28T02:20:50.820305+0000","last_unstale":"2023-08-28T02:20:50.821703+0000","last_undegraded":"2023-08-28T02:20:50.821703+0000","last_fullsized":"2023-08-28T02:20:50.821703+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:41:30.684047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":47,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.811803+0000","last_change":"2023-08-28T02:20:19.850774+0000","last_active":"2023-08-28T02:20:50.811803+0000","last_peered":"2023-08-28T02:20:50.811803+0000","last_clean":"2023-08-28T02:20:50.811803+0000","last_became_active":"2023-08-28T02:20:19.850448+0000","last_became_peered":"2023-08-28T02:20:19.850448+0000","last_unstale":"2023-08-28T02:20:50.811803+0000","last_undegraded":"2023-08-28T02:20:50.811803+0000","last_fullsized":"2023-08-28T02:20:50.811803+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:40:37.288553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":97,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.812255+0000","last_change":"2023-08-28T02:20:19.849807+0000","last_active":"2023-08-28T02:20:50.812255+0000","last_peered":"2023-08-28T02:20:50.812255+0000","last_clean":"2023-08-28T02:20:50.812255+0000","last_became_active":"2023-08-28T02:20:19.849464+0000","last_became_peered":"2023-08-28T02:20:19.849464+0000","last_unstale":"2023-08-28T02:20:50.812255+0000","last_undegraded":"2023-08-28T02:20:50.812255+0000","last_fullsized":"2023-08-28T02:20:50.812255+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26810,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430352,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307016,"kb_used_data":5824,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93438136,"statfs":{"total":95995035648,"available":95680651264,"internally_reserved":0,"allocated":5963776,"data_stored":5448619,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:20:43 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.53400000000000003,"15min":0.53600000000000003},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.746,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.60299999999999998,"15min":0.60899999999999999},"min":{"1min":0.38200000000000001,"5min":0.374,"15min":0.374},"max":{"1min":0.97199999999999998,"5min":0.97199999999999998,"15min":0.97199999999999998},"last":0.56299999999999994}]},{"osd":1,"last update":"Mon Aug 28 02:20:50 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.63400000000000001}]},{"osd":2,"last update":"Mon Aug 28 02:21:20 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.67200000000000004}]},{"osd":3,"last update":"Mon Aug 28 02:20:19 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.55700000000000005,"15min":0.54500000000000004},"min":{"1min":0.47599999999999998,"5min":0.35899999999999999,"15min":0.32600000000000001},"max":{"1min":0.877,"5min":0.88800000000000001,"15min":2.7509999999999999},"last":0.62},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.56399999999999995,"15min":0.55900000000000005},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.35299999999999998},"max":{"1min":1.4770000000000001,"5min":1.4770000000000001,"15min":2.7080000000000002},"last":0.59599999999999997}]},{"osd":4,"last update":"Mon Aug 28 02:21:20 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.48399999999999999,"15min":0.47499999999999998},"min":{"1min":0.35299999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.71299999999999997,"5min":0.75700000000000001,"15min":0.77800000000000002},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.503,"5min":0.502,"15min":0.498},"min":{"1min":0.30199999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.67600000000000005,"5min":0.71599999999999997,"15min":0.79800000000000004},"last":0.501}]},{"osd":5,"last update":"Mon Aug 28 02:21:20 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.504,"15min":0.48499999999999999},"min":{"1min":0.312,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.69799999999999995,"5min":0.71099999999999997,"15min":0.81100000000000005},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.57199999999999995,"15min":0.54400000000000004},"min":{"1min":0.32200000000000001,"5min":0.314,"15min":0.314},"max":{"1min":0.746,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.52100000000000002}]},{"osd":6,"last update":"Mon Aug 28 02:21:20 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.56200000000000006,"15min":0.53400000000000003},"min":{"1min":0.33900000000000002,"5min":0.32700000000000001,"15min":0.32200000000000001},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.61699999999999999,"15min":0.58299999999999996},"min":{"1min":0.41399999999999998,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.77200000000000002,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.70199999999999996}]}]},{"osd":6,"up_from":42,"seq":180388626579,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302908,"kb_used_data":1716,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93442244,"statfs":{"total":95995035648,"available":95684857856,"internally_reserved":0,"allocated":1757184,"data_stored":1253873,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:20:47 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.56499999999999995,"15min":0.57099999999999995},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.59099999999999997,"15min":0.59899999999999998},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.66900000000000004,"5min":0.746,"15min":0.746},"last":0.63900000000000001}]},{"osd":1,"last update":"Mon Aug 28 02:20:51 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.59599999999999997}]},{"osd":2,"last update":"Mon Aug 28 02:21:23 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.575,"5min":1.575,"15min":1.575},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":1.6419999999999999,"5min":1.6419999999999999,"15min":1.6419999999999999},"last":0.65100000000000002}]},{"osd":3,"last update":"Mon Aug 28 02:19:54 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.55200000000000005,"15min":0.54900000000000004},"min":{"1min":0.41799999999999998,"5min":0.33400000000000002,"15min":0.30499999999999999},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.84399999999999997},"last":0.624},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.54400000000000004,"15min":0.52300000000000002},"min":{"1min":0.35599999999999998,"5min":0.34699999999999998,"15min":0.31900000000000001},"max":{"1min":0.69599999999999995,"5min":0.76500000000000001,"15min":1.02},"last":0.71399999999999997}]},{"osd":4,"last update":"Mon Aug 28 02:20:55 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.46200000000000002,"15min":0.44400000000000001},"min":{"1min":0.32400000000000001,"5min":0.27800000000000002,"15min":0.27200000000000002},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.877},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.51600000000000001,"15min":0.47599999999999998},"min":{"1min":0.38500000000000001,"5min":0.29699999999999999,"15min":0.28899999999999998},"max":{"1min":0.84099999999999997,"5min":0.84899999999999998,"15min":0.85599999999999998},"last":0.67500000000000004}]},{"osd":5,"last update":"Mon Aug 28 02:20:55 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.48199999999999998,"15min":0.45800000000000002},"min":{"1min":0.318,"5min":0.313,"15min":0.29299999999999998},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.79300000000000004},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.53900000000000003,"15min":0.51400000000000001},"min":{"1min":0.40300000000000002,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.82199999999999995,"5min":0.82499999999999996,"15min":2.0609999999999999},"last":0.72799999999999998}]},{"osd":7,"last update":"Mon Aug 28 02:21:16 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.57199999999999995,"15min":0.55000000000000004},"min":{"1min":0.41199999999999998,"5min":0.35799999999999998,"15min":0.32700000000000001},"max":{"1min":0.85699999999999998,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.76700000000000002},{"interface":"front","average":{"1min":0.65600000000000003,"5min":0.59899999999999998,"15min":0.57399999999999995},"min":{"1min":0.42799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.83499999999999996,"5min":0.88100000000000001,"15min":0.89400000000000002},"last":0.75600000000000001}]}]},{"osd":1,"up_from":98,"seq":420906795032,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36872,"kb_used_data":5312,"kb_used_omap":7,"kb_used_meta":31544,"kb_avail":93708280,"statfs":{"total":95995035648,"available":95957278720,"internally_reserved":0,"allocated":5439488,"data_stored":4873933,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":32301387},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:20:50 2023","interfaces":[{"interface":"back","average":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.498,"5min":0.498,"15min":0.498},"last":0.28000000000000003}]},{"osd":2,"last update":"Mon Aug 28 02:21:23 2023","interfaces":[{"interface":"back","average":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.311}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34599999999999997}]},{"osd":4,"last update":"Mon Aug 28 02:20:50 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.90800000000000003,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.54000000000000004}]},{"osd":5,"last update":"Mon Aug 28 02:20:50 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.96599999999999997,"5min":0.96599999999999997,"15min":0.96599999999999997},"last":0.53500000000000003}]},{"osd":6,"last update":"Mon Aug 28 02:20:50 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.94199999999999995,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.57499999999999996}]},{"osd":7,"last update":"Mon Aug 28 02:20:50 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.65200000000000002}]}]},{"osd":0,"up_from":90,"seq":386547056678,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":37464,"kb_used_data":5904,"kb_used_omap":8,"kb_used_meta":31543,"kb_avail":93707688,"statfs":{"total":95995035648,"available":95956672512,"internally_reserved":0,"allocated":6045696,"data_stored":5465837,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":32300393},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Aug 28 02:20:53 2023","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"min":{"1min":0.192,"5min":0.192,"15min":0.192},"max":{"1min":0.503,"5min":0.503,"15min":0.503},"last":0.23799999999999999},{"interface":"front","average":{"1min":0.372,"5min":0.372,"15min":0.372},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.497,"5min":0.497,"15min":0.497},"last":0.28499999999999998}]},{"osd":2,"last update":"Mon Aug 28 02:21:20 2023","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"last":0.253}]},{"osd":3,"last update":"Mon Aug 28 02:19:40 2023","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":0.435,"15min":0.435},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"last":0.29999999999999999},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.33300000000000002}]},{"osd":4,"last update":"Mon Aug 28 02:20:43 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.65800000000000003,"15min":0.66600000000000004},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.82099999999999995,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.621,"5min":0.68899999999999995,"15min":0.70099999999999996},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.79800000000000004,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.56399999999999995}]},{"osd":5,"last update":"Mon Aug 28 02:20:43 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.64300000000000002,"15min":0.65000000000000002},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.82399999999999995,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.501},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.63800000000000001,"15min":0.64800000000000002},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.59199999999999997}]},{"osd":6,"last update":"Mon Aug 28 02:20:43 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.64200000000000002,"15min":0.65100000000000002},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.77200000000000002,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.63},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.69599999999999995,"15min":0.70399999999999996},"min":{"1min":0.42199999999999999,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.876,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.51000000000000001}]},{"osd":7,"last update":"Mon Aug 28 02:20:43 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.68400000000000005,"15min":0.69399999999999995},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.68999999999999995,"15min":0.69899999999999995},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.79500000000000004,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.57699999999999996}]}]},{"osd":2,"up_from":104,"seq":446676598802,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32680,"kb_used_data":1184,"kb_used_omap":10,"kb_used_meta":31477,"kb_avail":93712472,"statfs":{"total":95995035648,"available":95961571328,"internally_reserved":0,"allocated":1212416,"data_stored":672302,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10792,"internal_metadata":32232920},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:21:22 2023","interfaces":[{"interface":"back","average":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.39500000000000002}]},{"osd":1,"last update":"Mon Aug 28 02:21:22 2023","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.433},{"interface":"front","average":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.317}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45900000000000002}]},{"osd":4,"last update":"Mon Aug 28 02:21:22 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.91100000000000003,"5min":0.91100000000000003,"15min":0.91100000000000003},"last":0.64400000000000002}]},{"osd":5,"last update":"Mon Aug 28 02:21:22 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.67100000000000004}]},{"osd":6,"last update":"Mon Aug 28 02:21:22 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.72499999999999998},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.68999999999999995}]},{"osd":7,"last update":"Mon Aug 28 02:21:22 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.55900000000000005}]}]},{"osd":3,"up_from":112,"seq":481036337164,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28432,"kb_used_data":1160,"kb_used_omap":3,"kb_used_meta":27260,"kb_avail":93716720,"statfs":{"total":95995035648,"available":95965921280,"internally_reserved":0,"allocated":1187840,"data_stored":667951,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":27915170},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.374}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41699999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28699999999999998}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69399999999999995}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67600000000000005}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64100000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.71299999999999997}]}]},{"osd":4,"up_from":30,"seq":128849019037,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298236,"kb_used_data":1140,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93446916,"statfs":{"total":95995035648,"available":95689641984,"internally_reserved":0,"allocated":1167360,"data_stored":663855,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:20:43 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.60399999999999998,"15min":0.61199999999999999},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.58599999999999997,"15min":0.59499999999999997},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.71399999999999997,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.59799999999999998}]},{"osd":1,"last update":"Mon Aug 28 02:20:49 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.44900000000000001}]},{"osd":2,"last update":"Mon Aug 28 02:21:19 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.434},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.53800000000000003}]},{"osd":3,"last update":"Mon Aug 28 02:20:01 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.58099999999999996,"15min":0.55300000000000005},"min":{"1min":0.45700000000000002,"5min":0.41799999999999998,"15min":0.307},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":1.1599999999999999},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.628,"5min":0.58099999999999996,"15min":0.55400000000000005},"min":{"1min":0.496,"5min":0.371,"15min":0.35899999999999999},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.871},"last":0.63200000000000001}]},{"osd":5,"last update":"Mon Aug 28 02:21:35 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.502,"15min":0.44900000000000001},"min":{"1min":0.36899999999999999,"5min":0.31900000000000001,"15min":0.26600000000000001},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.503,"15min":0.46100000000000002},"min":{"1min":0.34100000000000003,"5min":0.32500000000000001,"15min":0.27700000000000002},"max":{"1min":0.63800000000000001,"5min":0.78600000000000003,"15min":1.1839999999999999},"last":0.56499999999999995}]},{"osd":6,"last update":"Mon Aug 28 02:20:49 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.53000000000000003,"15min":0.48299999999999998},"min":{"1min":0.375,"5min":0.30499999999999999,"15min":0.27500000000000002},"max":{"1min":0.76000000000000001,"5min":0.77300000000000002,"15min":0.85099999999999998},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.58399999999999996,"15min":0.52100000000000002},"min":{"1min":0.45600000000000002,"5min":0.36799999999999999,"15min":0.318},"max":{"1min":0.77500000000000002,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.68600000000000005}]},{"osd":7,"last update":"Mon Aug 28 02:21:18 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.56899999999999995,"15min":0.53500000000000003},"min":{"1min":0.40400000000000003,"5min":0.36899999999999999,"15min":0.312},"max":{"1min":0.70999999999999996,"5min":0.82099999999999995,"15min":1.2110000000000001},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.59399999999999997,"15min":0.54500000000000004},"min":{"1min":0.41899999999999998,"5min":0.35199999999999998,"15min":0.32100000000000001},"max":{"1min":0.79700000000000004,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.67300000000000004}]}]},{"osd":5,"up_from":36,"seq":154618822809,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298220,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93446932,"statfs":{"total":95995035648,"available":95689658368,"internally_reserved":0,"allocated":1150976,"data_stored":659010,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:20:40 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.60999999999999999,"15min":0.62},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.61899999999999999,"15min":0.625},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.747,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.57599999999999996}]},{"osd":1,"last update":"Mon Aug 28 02:20:50 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.65000000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:21:22 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.62}]},{"osd":3,"last update":"Mon Aug 28 02:20:36 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57999999999999996,"15min":0.54900000000000004},"min":{"1min":0.39100000000000001,"5min":0.255,"15min":0.255},"max":{"1min":0.78400000000000003,"5min":1.1279999999999999,"15min":1.1779999999999999},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.59499999999999997,"15min":0.57299999999999995},"min":{"1min":0.40400000000000003,"5min":0.35299999999999998,"15min":0.307},"max":{"1min":0.80800000000000005,"5min":1.1679999999999999,"15min":1.1679999999999999},"last":0.51700000000000002}]},{"osd":4,"last update":"Mon Aug 28 02:20:36 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.47399999999999998,"15min":0.45300000000000001},"min":{"1min":0.33400000000000002,"5min":0.28000000000000003,"15min":0.27500000000000002},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.79200000000000004},"last":0.495},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.505,"15min":0.47799999999999998},"min":{"1min":0.31900000000000001,"5min":0.28799999999999998,"15min":0.28199999999999997},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":1.1499999999999999},"last":0.66700000000000004}]},{"osd":6,"last update":"Mon Aug 28 02:20:50 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.51300000000000001,"15min":0.51900000000000002},"min":{"1min":0.36699999999999999,"5min":0.32700000000000001,"15min":0.28399999999999997},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":5.2119999999999997},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.55400000000000005,"15min":0.51600000000000001},"min":{"1min":0.442,"5min":0.31900000000000001,"15min":0.30299999999999999},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":1.089},"last":0.71199999999999997}]},{"osd":7,"last update":"Mon Aug 28 02:21:13 2023","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.58299999999999996,"15min":0.55300000000000005},"min":{"1min":0.45000000000000001,"5min":0.35199999999999998,"15min":0.32900000000000001},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":1.119},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.63,"5min":0.57499999999999996,"15min":0.55300000000000005},"min":{"1min":0.40400000000000003,"5min":0.34999999999999998,"15min":0.29799999999999999},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.67700000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-08-28T02:21:42.344 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph pg dump --format=json 2023-08-28T02:21:42.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:42 smithi143 ceph-mon[189871]: pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:42.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:42 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.2"}]: dispatch 2023-08-28T02:21:42.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:42 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.a"}]: dispatch 2023-08-28T02:21:42.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:42 smithi143 ceph-mon[183059]: pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:42.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:42 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.2"}]: dispatch 2023-08-28T02:21:42.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:42 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.a"}]: dispatch 2023-08-28T02:21:42.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:42 smithi169 ceph-mon[159415]: pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:42.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:42 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.2"}]: dispatch 2023-08-28T02:21:42.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:42 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.a"}]: dispatch 2023-08-28T02:21:42.697 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:21:43.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:43 smithi143 ceph-mon[183059]: from='client.44513 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:21:43.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:43 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.2"}]': finished 2023-08-28T02:21:43.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:43 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.a"}]': finished 2023-08-28T02:21:43.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:43 smithi143 ceph-mon[183059]: osdmap e115: 8 total, 8 up, 8 in 2023-08-28T02:21:43.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:43 smithi143 ceph-mon[183059]: failed to encode map e115 with expected crc 2023-08-28T02:21:43.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:43 smithi143 ceph-mon[183059]: failed to encode map e115 with expected crc 2023-08-28T02:21:43.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:43 smithi143 ceph-mon[183059]: failed to encode map e115 with expected crc 2023-08-28T02:21:43.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:43 smithi143 ceph-mon[189871]: from='client.44513 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:21:43.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:43 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.2"}]': finished 2023-08-28T02:21:43.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:43 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.a"}]': finished 2023-08-28T02:21:43.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:43 smithi143 ceph-mon[189871]: osdmap e115: 8 total, 8 up, 8 in 2023-08-28T02:21:43.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:43 smithi143 ceph-mon[189871]: failed to encode map e115 with expected crc 2023-08-28T02:21:43.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:43 smithi143 ceph-mon[189871]: failed to encode map e115 with expected crc 2023-08-28T02:21:43.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:43 smithi143 ceph-mon[189871]: failed to encode map e115 with expected crc 2023-08-28T02:21:43.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:43 smithi169 ceph-mon[159415]: from='client.44513 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:21:43.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:43 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.2"}]': finished 2023-08-28T02:21:43.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:43 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.a"}]': finished 2023-08-28T02:21:43.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:43 smithi169 ceph-mon[159415]: osdmap e115: 8 total, 8 up, 8 in 2023-08-28T02:21:43.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:43 smithi169 ceph-mon[159415]: failed to encode map e115 with expected crc 2023-08-28T02:21:43.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:43 smithi169 ceph-mon[159415]: failed to encode map e115 with expected crc 2023-08-28T02:21:43.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:43 smithi169 ceph-mon[159415]: failed to encode map e115 with expected crc 2023-08-28T02:21:44.261 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:21:44.262 INFO:teuthology.orchestra.run.smithi143.stderr:dumped all 2023-08-28T02:21:44.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:44 smithi143 ceph-mon[189871]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:44.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:44 smithi143 ceph-mon[189871]: failed to encode map e116 with expected crc 2023-08-28T02:21:44.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:44 smithi143 ceph-mon[189871]: osdmap e116: 8 total, 8 up, 8 in 2023-08-28T02:21:44.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:44 smithi143 ceph-mon[189871]: failed to encode map e116 with expected crc 2023-08-28T02:21:44.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:44 smithi143 ceph-mon[189871]: failed to encode map e116 with expected crc 2023-08-28T02:21:44.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:44 smithi143 ceph-mon[183059]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:44.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:44 smithi143 ceph-mon[183059]: failed to encode map e116 with expected crc 2023-08-28T02:21:44.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:44 smithi143 ceph-mon[183059]: osdmap e116: 8 total, 8 up, 8 in 2023-08-28T02:21:44.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:44 smithi143 ceph-mon[183059]: failed to encode map e116 with expected crc 2023-08-28T02:21:44.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:44 smithi143 ceph-mon[183059]: failed to encode map e116 with expected crc 2023-08-28T02:21:44.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:44 smithi169 ceph-mon[159415]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:44.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:44 smithi169 ceph-mon[159415]: failed to encode map e116 with expected crc 2023-08-28T02:21:44.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:44 smithi169 ceph-mon[159415]: osdmap e116: 8 total, 8 up, 8 in 2023-08-28T02:21:44.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:44 smithi169 ceph-mon[159415]: failed to encode map e116 with expected crc 2023-08-28T02:21:44.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:44 smithi169 ceph-mon[159415]: failed to encode map e116 with expected crc 2023-08-28T02:21:44.880 INFO:teuthology.orchestra.run.smithi143.stdout:{"pg_ready":true,"pg_map":{"version":214,"stamp":"2023-08-28T02:21:43.195651+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6288,"num_read_kb":13719,"num_write":1106,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1341828,"kb_used_data":23364,"kb_used_omap":29,"kb_used_meta":1318370,"kb_avail":748619388,"statfs":{"total":767960285184,"available":766586253312,"internally_reserved":0,"allocated":23924736,"data_stored":19705380,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30674,"internal_metadata":1350010926},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.162841"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":100,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.824785+0000","last_change":"2023-08-28T02:20:51.824785+0000","last_active":"2023-08-28T02:20:51.824785+0000","last_peered":"2023-08-28T02:20:51.824785+0000","last_clean":"2023-08-28T02:20:51.824785+0000","last_became_active":"2023-08-28T02:20:51.824447+0000","last_became_peered":"2023-08-28T02:20:51.824447+0000","last_unstale":"2023-08-28T02:20:51.824785+0000","last_undegraded":"2023-08-28T02:20:51.824785+0000","last_fullsized":"2023-08-28T02:20:51.824785+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":66,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.817322+0000","last_change":"2023-08-28T02:20:19.850252+0000","last_active":"2023-08-28T02:20:51.817322+0000","last_peered":"2023-08-28T02:20:51.817322+0000","last_clean":"2023-08-28T02:20:51.817322+0000","last_became_active":"2023-08-28T02:20:19.849913+0000","last_became_peered":"2023-08-28T02:20:19.849913+0000","last_unstale":"2023-08-28T02:20:51.817322+0000","last_undegraded":"2023-08-28T02:20:51.817322+0000","last_fullsized":"2023-08-28T02:20:51.817322+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:48:47.713465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":98,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.826985+0000","last_change":"2023-08-28T02:20:50.823101+0000","last_active":"2023-08-28T02:20:50.826985+0000","last_peered":"2023-08-28T02:20:50.826985+0000","last_clean":"2023-08-28T02:20:50.826985+0000","last_became_active":"2023-08-28T02:20:50.822599+0000","last_became_peered":"2023-08-28T02:20:50.822599+0000","last_unstale":"2023-08-28T02:20:50.826985+0000","last_undegraded":"2023-08-28T02:20:50.826985+0000","last_fullsized":"2023-08-28T02:20:50.826985+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:28:57.827789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":85,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.826313+0000","last_change":"2023-08-28T02:20:50.825915+0000","last_active":"2023-08-28T02:20:50.826313+0000","last_peered":"2023-08-28T02:20:50.826313+0000","last_clean":"2023-08-28T02:20:50.826313+0000","last_became_active":"2023-08-28T02:20:50.825526+0000","last_became_peered":"2023-08-28T02:20:50.825526+0000","last_unstale":"2023-08-28T02:20:50.826313+0000","last_undegraded":"2023-08-28T02:20:50.826313+0000","last_fullsized":"2023-08-28T02:20:50.826313+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:25:20.142641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":95,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.820522+0000","last_change":"2023-08-28T02:20:50.819786+0000","last_active":"2023-08-28T02:20:50.820522+0000","last_peered":"2023-08-28T02:20:50.820522+0000","last_clean":"2023-08-28T02:20:50.820522+0000","last_became_active":"2023-08-28T02:20:50.819451+0000","last_became_peered":"2023-08-28T02:20:50.819451+0000","last_unstale":"2023-08-28T02:20:50.820522+0000","last_undegraded":"2023-08-28T02:20:50.820522+0000","last_fullsized":"2023-08-28T02:20:50.820522+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":68,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.824216+0000","last_change":"2023-08-28T02:11:22.186024+0000","last_active":"2023-08-28T02:20:50.824216+0000","last_peered":"2023-08-28T02:20:50.824216+0000","last_clean":"2023-08-28T02:20:50.824216+0000","last_became_active":"2023-08-28T02:11:22.185500+0000","last_became_peered":"2023-08-28T02:11:22.185500+0000","last_unstale":"2023-08-28T02:20:50.824216+0000","last_undegraded":"2023-08-28T02:20:50.824216+0000","last_fullsized":"2023-08-28T02:20:50.824216+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":95,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.820006+0000","last_change":"2023-08-28T02:20:50.819015+0000","last_active":"2023-08-28T02:20:50.820006+0000","last_peered":"2023-08-28T02:20:50.820006+0000","last_clean":"2023-08-28T02:20:50.820006+0000","last_became_active":"2023-08-28T02:20:50.818667+0000","last_became_peered":"2023-08-28T02:20:50.818667+0000","last_unstale":"2023-08-28T02:20:50.820006+0000","last_undegraded":"2023-08-28T02:20:50.820006+0000","last_fullsized":"2023-08-28T02:20:50.820006+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":83,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.822081+0000","last_change":"2023-08-28T02:19:50.218760+0000","last_active":"2023-08-28T02:20:50.822081+0000","last_peered":"2023-08-28T02:20:50.822081+0000","last_clean":"2023-08-28T02:20:50.822081+0000","last_became_active":"2023-08-28T02:19:50.218376+0000","last_became_peered":"2023-08-28T02:19:50.218376+0000","last_unstale":"2023-08-28T02:20:50.822081+0000","last_undegraded":"2023-08-28T02:20:50.822081+0000","last_fullsized":"2023-08-28T02:20:50.822081+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":117,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.885790+0000","last_change":"2023-08-28T02:20:50.817119+0000","last_active":"2023-08-28T02:20:51.885790+0000","last_peered":"2023-08-28T02:20:51.885790+0000","last_clean":"2023-08-28T02:20:51.885790+0000","last_became_active":"2023-08-28T02:20:50.816756+0000","last_became_peered":"2023-08-28T02:20:50.816756+0000","last_unstale":"2023-08-28T02:20:51.885790+0000","last_undegraded":"2023-08-28T02:20:51.885790+0000","last_fullsized":"2023-08-28T02:20:51.885790+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:12:52.257562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":51,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:21:07.794449+0000","last_change":"2023-08-28T02:18:39.781312+0000","last_active":"2023-08-28T02:21:07.794449+0000","last_peered":"2023-08-28T02:21:07.794449+0000","last_clean":"2023-08-28T02:21:07.794449+0000","last_became_active":"2023-08-28T02:18:39.780916+0000","last_became_peered":"2023-08-28T02:18:39.780916+0000","last_unstale":"2023-08-28T02:21:07.794449+0000","last_undegraded":"2023-08-28T02:21:07.794449+0000","last_fullsized":"2023-08-28T02:21:07.794449+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:37:11.571250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":65,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.814304+0000","last_change":"2023-08-28T02:11:22.437146+0000","last_active":"2023-08-28T02:20:50.814304+0000","last_peered":"2023-08-28T02:20:50.814304+0000","last_clean":"2023-08-28T02:20:50.814304+0000","last_became_active":"2023-08-28T02:11:22.436798+0000","last_became_peered":"2023-08-28T02:11:22.436798+0000","last_unstale":"2023-08-28T02:20:50.814304+0000","last_undegraded":"2023-08-28T02:20:50.814304+0000","last_fullsized":"2023-08-28T02:20:50.814304+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":66,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.817227+0000","last_change":"2023-08-28T02:19:50.221001+0000","last_active":"2023-08-28T02:20:51.817227+0000","last_peered":"2023-08-28T02:20:51.817227+0000","last_clean":"2023-08-28T02:20:51.817227+0000","last_became_active":"2023-08-28T02:19:50.220611+0000","last_became_peered":"2023-08-28T02:19:50.220611+0000","last_unstale":"2023-08-28T02:20:51.817227+0000","last_undegraded":"2023-08-28T02:20:51.817227+0000","last_fullsized":"2023-08-28T02:20:51.817227+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:23:10.637630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"78'122","reported_seq":570,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.816897+0000","last_change":"2023-08-28T02:20:19.851065+0000","last_active":"2023-08-28T02:20:50.816897+0000","last_peered":"2023-08-28T02:20:50.816897+0000","last_clean":"2023-08-28T02:20:50.816897+0000","last_became_active":"2023-08-28T02:20:19.850477+0000","last_became_peered":"2023-08-28T02:20:19.850477+0000","last_unstale":"2023-08-28T02:20:50.816897+0000","last_undegraded":"2023-08-28T02:20:50.816897+0000","last_fullsized":"2023-08-28T02:20:50.816897+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":857,"num_read_kb":2224,"num_write":99,"num_write_kb":2312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":70,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.823860+0000","last_change":"2023-08-28T02:20:50.822679+0000","last_active":"2023-08-28T02:20:50.823860+0000","last_peered":"2023-08-28T02:20:50.823860+0000","last_clean":"2023-08-28T02:20:50.823860+0000","last_became_active":"2023-08-28T02:20:50.822180+0000","last_became_peered":"2023-08-28T02:20:50.822180+0000","last_unstale":"2023-08-28T02:20:50.823860+0000","last_undegraded":"2023-08-28T02:20:50.823860+0000","last_fullsized":"2023-08-28T02:20:50.823860+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:02:40.005023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'112","reported_seq":356,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.809841+0000","last_change":"2023-08-28T02:19:50.218206+0000","last_active":"2023-08-28T02:20:50.809841+0000","last_peered":"2023-08-28T02:20:50.809841+0000","last_clean":"2023-08-28T02:20:50.809841+0000","last_became_active":"2023-08-28T02:19:50.217637+0000","last_became_peered":"2023-08-28T02:19:50.217637+0000","last_unstale":"2023-08-28T02:20:50.809841+0000","last_undegraded":"2023-08-28T02:20:50.809841+0000","last_fullsized":"2023-08-28T02:20:50.809841+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:12:06.224907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":444,"num_read_kb":473,"num_write":94,"num_write_kb":127,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":73,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.823867+0000","last_change":"2023-08-28T02:20:50.823477+0000","last_active":"2023-08-28T02:20:50.823867+0000","last_peered":"2023-08-28T02:20:50.823867+0000","last_clean":"2023-08-28T02:20:50.823867+0000","last_became_active":"2023-08-28T02:20:50.823158+0000","last_became_peered":"2023-08-28T02:20:50.823158+0000","last_unstale":"2023-08-28T02:20:50.823867+0000","last_undegraded":"2023-08-28T02:20:50.823867+0000","last_fullsized":"2023-08-28T02:20:50.823867+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:33:18.347341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":48,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.817069+0000","last_change":"2023-08-28T02:18:39.780069+0000","last_active":"2023-08-28T02:20:51.817069+0000","last_peered":"2023-08-28T02:20:51.817069+0000","last_clean":"2023-08-28T02:20:51.817069+0000","last_became_active":"2023-08-28T02:18:39.779829+0000","last_became_peered":"2023-08-28T02:18:39.779829+0000","last_unstale":"2023-08-28T02:20:51.817069+0000","last_undegraded":"2023-08-28T02:20:51.817069+0000","last_fullsized":"2023-08-28T02:20:51.817069+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:35:01.309659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":52,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.810821+0000","last_change":"2023-08-28T02:20:19.850916+0000","last_active":"2023-08-28T02:20:50.810821+0000","last_peered":"2023-08-28T02:20:50.810821+0000","last_clean":"2023-08-28T02:20:50.810821+0000","last_became_active":"2023-08-28T02:20:19.850627+0000","last_became_peered":"2023-08-28T02:20:19.850627+0000","last_unstale":"2023-08-28T02:20:50.810821+0000","last_undegraded":"2023-08-28T02:20:50.810821+0000","last_fullsized":"2023-08-28T02:20:50.810821+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:19:14.346189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'103","reported_seq":535,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.820309+0000","last_change":"2023-08-28T02:18:39.786876+0000","last_active":"2023-08-28T02:20:50.820309+0000","last_peered":"2023-08-28T02:20:50.820309+0000","last_clean":"2023-08-28T02:20:50.820309+0000","last_became_active":"2023-08-28T02:18:39.786414+0000","last_became_peered":"2023-08-28T02:18:39.786414+0000","last_unstale":"2023-08-28T02:20:50.820309+0000","last_undegraded":"2023-08-28T02:20:50.820309+0000","last_fullsized":"2023-08-28T02:20:50.820309+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":803,"num_read_kb":2080,"num_write":73,"num_write_kb":1244,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":98,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.887088+0000","last_change":"2023-08-28T02:20:51.886782+0000","last_active":"2023-08-28T02:20:51.887088+0000","last_peered":"2023-08-28T02:20:51.887088+0000","last_clean":"2023-08-28T02:20:51.887088+0000","last_became_active":"2023-08-28T02:20:51.886444+0000","last_became_peered":"2023-08-28T02:20:51.886444+0000","last_unstale":"2023-08-28T02:20:51.887088+0000","last_undegraded":"2023-08-28T02:20:51.887088+0000","last_fullsized":"2023-08-28T02:20:51.887088+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:57:03.704951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":62,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.816722+0000","last_change":"2023-08-28T02:20:50.823036+0000","last_active":"2023-08-28T02:20:51.816722+0000","last_peered":"2023-08-28T02:20:51.816722+0000","last_clean":"2023-08-28T02:20:51.816722+0000","last_became_active":"2023-08-28T02:20:50.822766+0000","last_became_peered":"2023-08-28T02:20:50.822766+0000","last_unstale":"2023-08-28T02:20:51.816722+0000","last_undegraded":"2023-08-28T02:20:51.816722+0000","last_fullsized":"2023-08-28T02:20:51.816722+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:09:26.786992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"78'148","reported_seq":659,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.818235+0000","last_change":"2023-08-28T02:19:50.221311+0000","last_active":"2023-08-28T02:20:50.818235+0000","last_peered":"2023-08-28T02:20:50.818235+0000","last_clean":"2023-08-28T02:20:50.818235+0000","last_became_active":"2023-08-28T02:19:50.220754+0000","last_became_peered":"2023-08-28T02:19:50.220754+0000","last_unstale":"2023-08-28T02:20:50.818235+0000","last_undegraded":"2023-08-28T02:20:50.818235+0000","last_fullsized":"2023-08-28T02:20:50.818235+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1092,"num_read_kb":1955,"num_write":127,"num_write_kb":5177,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":65,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.816396+0000","last_change":"2023-08-28T02:11:23.249640+0000","last_active":"2023-08-28T02:20:50.816396+0000","last_peered":"2023-08-28T02:20:50.816396+0000","last_clean":"2023-08-28T02:20:50.816396+0000","last_became_active":"2023-08-28T02:11:23.249355+0000","last_became_peered":"2023-08-28T02:11:23.249355+0000","last_unstale":"2023-08-28T02:20:50.816396+0000","last_undegraded":"2023-08-28T02:20:50.816396+0000","last_fullsized":"2023-08-28T02:20:50.816396+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":64,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.826633+0000","last_change":"2023-08-28T02:20:50.825728+0000","last_active":"2023-08-28T02:20:50.826633+0000","last_peered":"2023-08-28T02:20:50.826633+0000","last_clean":"2023-08-28T02:20:50.826633+0000","last_became_active":"2023-08-28T02:20:50.825418+0000","last_became_peered":"2023-08-28T02:20:50.825418+0000","last_unstale":"2023-08-28T02:20:50.826633+0000","last_undegraded":"2023-08-28T02:20:50.826633+0000","last_fullsized":"2023-08-28T02:20:50.826633+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:55:48.126269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"86'139","reported_seq":317,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.818120+0000","last_change":"2023-08-28T02:18:39.786823+0000","last_active":"2023-08-28T02:20:50.818120+0000","last_peered":"2023-08-28T02:20:50.818120+0000","last_clean":"2023-08-28T02:20:50.818120+0000","last_became_active":"2023-08-28T02:18:39.786357+0000","last_became_peered":"2023-08-28T02:18:39.786357+0000","last_unstale":"2023-08-28T02:20:50.818120+0000","last_undegraded":"2023-08-28T02:20:50.818120+0000","last_fullsized":"2023-08-28T02:20:50.818120+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_clean_scrub_stamp":"2023-08-28T02:08:02.043718+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"78'114","reported_seq":412,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.818867+0000","last_change":"2023-08-28T02:20:19.852152+0000","last_active":"2023-08-28T02:20:50.818867+0000","last_peered":"2023-08-28T02:20:50.818867+0000","last_clean":"2023-08-28T02:20:50.818867+0000","last_became_active":"2023-08-28T02:20:19.851615+0000","last_became_peered":"2023-08-28T02:20:19.851615+0000","last_unstale":"2023-08-28T02:20:50.818867+0000","last_undegraded":"2023-08-28T02:20:50.818867+0000","last_fullsized":"2023-08-28T02:20:50.818867+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":114,"log_dups_size":0,"ondisk_log_size":114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":358,"num_read_kb":1352,"num_write":102,"num_write_kb":1104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":84,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.808976+0000","last_change":"2023-08-28T02:19:50.211156+0000","last_active":"2023-08-28T02:20:50.808976+0000","last_peered":"2023-08-28T02:20:50.808976+0000","last_clean":"2023-08-28T02:20:50.808976+0000","last_became_active":"2023-08-28T02:19:50.210398+0000","last_became_peered":"2023-08-28T02:19:50.210398+0000","last_unstale":"2023-08-28T02:20:50.808976+0000","last_undegraded":"2023-08-28T02:20:50.808976+0000","last_fullsized":"2023-08-28T02:20:50.808976+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:14:41.748197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":52,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.808409+0000","last_change":"2023-08-28T02:20:19.851591+0000","last_active":"2023-08-28T02:20:50.808409+0000","last_peered":"2023-08-28T02:20:50.808409+0000","last_clean":"2023-08-28T02:20:50.808409+0000","last_became_active":"2023-08-28T02:20:19.851291+0000","last_became_peered":"2023-08-28T02:20:19.851291+0000","last_unstale":"2023-08-28T02:20:50.808409+0000","last_undegraded":"2023-08-28T02:20:50.808409+0000","last_fullsized":"2023-08-28T02:20:50.808409+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:33:51.259707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"78'47","reported_seq":267,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.885421+0000","last_change":"2023-08-28T02:20:50.819614+0000","last_active":"2023-08-28T02:20:51.885421+0000","last_peered":"2023-08-28T02:20:51.885421+0000","last_clean":"2023-08-28T02:20:51.885421+0000","last_became_active":"2023-08-28T02:20:50.819114+0000","last_became_peered":"2023-08-28T02:20:50.819114+0000","last_unstale":"2023-08-28T02:20:51.885421+0000","last_undegraded":"2023-08-28T02:20:51.885421+0000","last_fullsized":"2023-08-28T02:20:51.885421+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":47,"log_dups_size":0,"ondisk_log_size":47,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:29:07.212870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":162,"num_read_kb":1281,"num_write":48,"num_write_kb":1153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":51,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.810901+0000","last_change":"2023-08-28T02:20:19.851150+0000","last_active":"2023-08-28T02:20:50.810901+0000","last_peered":"2023-08-28T02:20:50.810901+0000","last_clean":"2023-08-28T02:20:50.810901+0000","last_became_active":"2023-08-28T02:20:19.850886+0000","last_became_peered":"2023-08-28T02:20:19.850886+0000","last_unstale":"2023-08-28T02:20:50.810901+0000","last_undegraded":"2023-08-28T02:20:50.810901+0000","last_fullsized":"2023-08-28T02:20:50.810901+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:50:28.287980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":81,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.813107+0000","last_change":"2023-08-28T02:18:39.779607+0000","last_active":"2023-08-28T02:20:50.813107+0000","last_peered":"2023-08-28T02:20:50.813107+0000","last_clean":"2023-08-28T02:20:50.813107+0000","last_became_active":"2023-08-28T02:18:39.779303+0000","last_became_peered":"2023-08-28T02:18:39.779303+0000","last_unstale":"2023-08-28T02:20:50.813107+0000","last_undegraded":"2023-08-28T02:20:50.813107+0000","last_fullsized":"2023-08-28T02:20:50.813107+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"78'204","reported_seq":1030,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.819453+0000","last_change":"2023-08-28T02:19:50.219949+0000","last_active":"2023-08-28T02:20:50.819453+0000","last_peered":"2023-08-28T02:20:50.819453+0000","last_clean":"2023-08-28T02:20:50.819453+0000","last_became_active":"2023-08-28T02:19:50.219356+0000","last_became_peered":"2023-08-28T02:19:50.219356+0000","last_unstale":"2023-08-28T02:20:50.819453+0000","last_undegraded":"2023-08-28T02:20:50.819453+0000","last_fullsized":"2023-08-28T02:20:50.819453+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":204,"log_dups_size":0,"ondisk_log_size":204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1521,"num_read_kb":2511,"num_write":178,"num_write_kb":1175,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":11,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.824031+0000","last_change":"2023-08-28T02:20:51.823558+0000","last_active":"2023-08-28T02:20:51.824031+0000","last_peered":"2023-08-28T02:20:51.824031+0000","last_clean":"2023-08-28T02:20:51.824031+0000","last_became_active":"2023-08-28T02:20:51.823259+0000","last_became_peered":"2023-08-28T02:20:51.823259+0000","last_unstale":"2023-08-28T02:20:51.824031+0000","last_undegraded":"2023-08-28T02:20:51.824031+0000","last_fullsized":"2023-08-28T02:20:51.824031+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:22:15.166170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":96,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.828624+0000","last_change":"2023-08-28T02:20:50.828341+0000","last_active":"2023-08-28T02:20:50.828624+0000","last_peered":"2023-08-28T02:20:50.828624+0000","last_clean":"2023-08-28T02:20:50.828624+0000","last_became_active":"2023-08-28T02:20:50.828019+0000","last_became_peered":"2023-08-28T02:20:50.828019+0000","last_unstale":"2023-08-28T02:20:50.828624+0000","last_undegraded":"2023-08-28T02:20:50.828624+0000","last_fullsized":"2023-08-28T02:20:50.828624+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"78'126","reported_seq":570,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.810726+0000","last_change":"2023-08-28T02:19:50.213556+0000","last_active":"2023-08-28T02:20:50.810726+0000","last_peered":"2023-08-28T02:20:50.810726+0000","last_clean":"2023-08-28T02:20:50.810726+0000","last_became_active":"2023-08-28T02:19:50.212996+0000","last_became_peered":"2023-08-28T02:19:50.212996+0000","last_unstale":"2023-08-28T02:20:50.810726+0000","last_undegraded":"2023-08-28T02:20:50.810726+0000","last_fullsized":"2023-08-28T02:20:50.810726+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:38:28.157352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":743,"num_read_kb":1331,"num_write":125,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":53,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.821699+0000","last_change":"2023-08-28T02:20:50.820676+0000","last_active":"2023-08-28T02:20:50.821699+0000","last_peered":"2023-08-28T02:20:50.821699+0000","last_clean":"2023-08-28T02:20:50.821699+0000","last_became_active":"2023-08-28T02:20:50.820389+0000","last_became_peered":"2023-08-28T02:20:50.820389+0000","last_unstale":"2023-08-28T02:20:50.821699+0000","last_undegraded":"2023-08-28T02:20:50.821699+0000","last_fullsized":"2023-08-28T02:20:50.821699+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:24:25.623757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":79,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.823878+0000","last_change":"2023-08-28T02:20:50.822792+0000","last_active":"2023-08-28T02:20:50.823878+0000","last_peered":"2023-08-28T02:20:50.823878+0000","last_clean":"2023-08-28T02:20:50.823878+0000","last_became_active":"2023-08-28T02:20:50.822157+0000","last_became_peered":"2023-08-28T02:20:50.822157+0000","last_unstale":"2023-08-28T02:20:50.823878+0000","last_undegraded":"2023-08-28T02:20:50.823878+0000","last_fullsized":"2023-08-28T02:20:50.823878+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:15:16.261246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":42,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.816972+0000","last_change":"2023-08-28T02:19:51.204017+0000","last_active":"2023-08-28T02:20:51.816972+0000","last_peered":"2023-08-28T02:20:51.816972+0000","last_clean":"2023-08-28T02:20:51.816972+0000","last_became_active":"2023-08-28T02:19:51.203627+0000","last_became_peered":"2023-08-28T02:19:51.203627+0000","last_unstale":"2023-08-28T02:20:51.816972+0000","last_undegraded":"2023-08-28T02:20:51.816972+0000","last_fullsized":"2023-08-28T02:20:51.816972+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:14:06.231701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":11,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.927939+0000","last_change":"2023-08-28T02:20:51.927612+0000","last_active":"2023-08-28T02:20:51.927939+0000","last_peered":"2023-08-28T02:20:51.927939+0000","last_clean":"2023-08-28T02:20:51.927939+0000","last_became_active":"2023-08-28T02:20:51.927256+0000","last_became_peered":"2023-08-28T02:20:51.927256+0000","last_unstale":"2023-08-28T02:20:51.927939+0000","last_undegraded":"2023-08-28T02:20:51.927939+0000","last_fullsized":"2023-08-28T02:20:51.927939+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:05:59.937193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":115,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.886978+0000","last_change":"2023-08-28T02:20:51.886978+0000","last_active":"2023-08-28T02:20:51.886978+0000","last_peered":"2023-08-28T02:20:51.886978+0000","last_clean":"2023-08-28T02:20:51.886978+0000","last_became_active":"2023-08-28T02:20:51.886625+0000","last_became_peered":"2023-08-28T02:20:51.886625+0000","last_unstale":"2023-08-28T02:20:51.886978+0000","last_undegraded":"2023-08-28T02:20:51.886978+0000","last_fullsized":"2023-08-28T02:20:51.886978+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":101,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.821045+0000","last_change":"2023-08-28T02:20:51.821045+0000","last_active":"2023-08-28T02:20:51.821045+0000","last_peered":"2023-08-28T02:20:51.821045+0000","last_clean":"2023-08-28T02:20:51.821045+0000","last_became_active":"2023-08-28T02:20:51.820588+0000","last_became_peered":"2023-08-28T02:20:51.820588+0000","last_unstale":"2023-08-28T02:20:51.821045+0000","last_undegraded":"2023-08-28T02:20:51.821045+0000","last_fullsized":"2023-08-28T02:20:51.821045+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":80,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.820658+0000","last_change":"2023-08-28T02:20:50.820134+0000","last_active":"2023-08-28T02:20:50.820658+0000","last_peered":"2023-08-28T02:20:50.820658+0000","last_clean":"2023-08-28T02:20:50.820658+0000","last_became_active":"2023-08-28T02:20:50.819765+0000","last_became_peered":"2023-08-28T02:20:50.819765+0000","last_unstale":"2023-08-28T02:20:50.820658+0000","last_undegraded":"2023-08-28T02:20:50.820658+0000","last_fullsized":"2023-08-28T02:20:50.820658+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":73,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.826281+0000","last_change":"2023-08-28T02:20:50.824864+0000","last_active":"2023-08-28T02:20:50.826281+0000","last_peered":"2023-08-28T02:20:50.826281+0000","last_clean":"2023-08-28T02:20:50.826281+0000","last_became_active":"2023-08-28T02:20:50.824495+0000","last_became_peered":"2023-08-28T02:20:50.824495+0000","last_unstale":"2023-08-28T02:20:50.826281+0000","last_undegraded":"2023-08-28T02:20:50.826281+0000","last_fullsized":"2023-08-28T02:20:50.826281+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:45:04.111154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":74,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:52.792709+0000","last_change":"2023-08-28T02:19:51.205245+0000","last_active":"2023-08-28T02:20:52.792709+0000","last_peered":"2023-08-28T02:20:52.792709+0000","last_clean":"2023-08-28T02:20:52.792709+0000","last_became_active":"2023-08-28T02:19:51.204896+0000","last_became_peered":"2023-08-28T02:19:51.204896+0000","last_unstale":"2023-08-28T02:20:52.792709+0000","last_undegraded":"2023-08-28T02:20:52.792709+0000","last_fullsized":"2023-08-28T02:20:52.792709+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:51:24.634702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":81,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.824924+0000","last_change":"2023-08-28T02:20:50.824679+0000","last_active":"2023-08-28T02:20:50.824924+0000","last_peered":"2023-08-28T02:20:50.824924+0000","last_clean":"2023-08-28T02:20:50.824924+0000","last_became_active":"2023-08-28T02:20:50.824339+0000","last_became_peered":"2023-08-28T02:20:50.824339+0000","last_unstale":"2023-08-28T02:20:50.824924+0000","last_undegraded":"2023-08-28T02:20:50.824924+0000","last_fullsized":"2023-08-28T02:20:50.824924+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":114,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.927844+0000","last_change":"2023-08-28T02:20:51.927844+0000","last_active":"2023-08-28T02:20:51.927844+0000","last_peered":"2023-08-28T02:20:51.927844+0000","last_clean":"2023-08-28T02:20:51.927844+0000","last_became_active":"2023-08-28T02:20:51.927433+0000","last_became_peered":"2023-08-28T02:20:51.927433+0000","last_unstale":"2023-08-28T02:20:51.927844+0000","last_undegraded":"2023-08-28T02:20:51.927844+0000","last_fullsized":"2023-08-28T02:20:51.927844+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":67,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.820415+0000","last_change":"2023-08-28T02:11:22.188982+0000","last_active":"2023-08-28T02:20:50.820415+0000","last_peered":"2023-08-28T02:20:50.820415+0000","last_clean":"2023-08-28T02:20:50.820415+0000","last_became_active":"2023-08-28T02:11:22.188663+0000","last_became_peered":"2023-08-28T02:11:22.188663+0000","last_unstale":"2023-08-28T02:20:50.820415+0000","last_undegraded":"2023-08-28T02:20:50.820415+0000","last_fullsized":"2023-08-28T02:20:50.820415+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":95,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.824117+0000","last_change":"2023-08-28T02:20:50.823610+0000","last_active":"2023-08-28T02:20:50.824117+0000","last_peered":"2023-08-28T02:20:50.824117+0000","last_clean":"2023-08-28T02:20:50.824117+0000","last_became_active":"2023-08-28T02:20:50.823066+0000","last_became_peered":"2023-08-28T02:20:50.823066+0000","last_unstale":"2023-08-28T02:20:50.824117+0000","last_undegraded":"2023-08-28T02:20:50.824117+0000","last_fullsized":"2023-08-28T02:20:50.824117+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":83,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.819174+0000","last_change":"2023-08-28T02:19:50.220079+0000","last_active":"2023-08-28T02:20:50.819174+0000","last_peered":"2023-08-28T02:20:50.819174+0000","last_clean":"2023-08-28T02:20:50.819174+0000","last_became_active":"2023-08-28T02:19:50.219671+0000","last_became_peered":"2023-08-28T02:19:50.219671+0000","last_unstale":"2023-08-28T02:20:50.819174+0000","last_undegraded":"2023-08-28T02:20:50.819174+0000","last_fullsized":"2023-08-28T02:20:50.819174+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":65,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.818304+0000","last_change":"2023-08-28T02:11:23.249061+0000","last_active":"2023-08-28T02:20:50.818304+0000","last_peered":"2023-08-28T02:20:50.818304+0000","last_clean":"2023-08-28T02:20:50.818304+0000","last_became_active":"2023-08-28T02:11:23.248739+0000","last_became_peered":"2023-08-28T02:11:23.248739+0000","last_unstale":"2023-08-28T02:20:50.818304+0000","last_undegraded":"2023-08-28T02:20:50.818304+0000","last_fullsized":"2023-08-28T02:20:50.818304+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":95,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.819235+0000","last_change":"2023-08-28T02:20:43.220383+0000","last_active":"2023-08-28T02:20:50.819235+0000","last_peered":"2023-08-28T02:20:50.819235+0000","last_clean":"2023-08-28T02:20:50.819235+0000","last_became_active":"2023-08-28T02:20:43.136191+0000","last_became_peered":"2023-08-28T02:20:43.136191+0000","last_unstale":"2023-08-28T02:20:50.819235+0000","last_undegraded":"2023-08-28T02:20:50.819235+0000","last_fullsized":"2023-08-28T02:20:50.819235+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":48,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.885989+0000","last_change":"2023-08-28T02:20:19.851128+0000","last_active":"2023-08-28T02:20:51.885989+0000","last_peered":"2023-08-28T02:20:51.885989+0000","last_clean":"2023-08-28T02:20:51.885989+0000","last_became_active":"2023-08-28T02:20:19.849358+0000","last_became_peered":"2023-08-28T02:20:19.849358+0000","last_unstale":"2023-08-28T02:20:51.885989+0000","last_undegraded":"2023-08-28T02:20:51.885989+0000","last_fullsized":"2023-08-28T02:20:51.885989+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:32:27.226152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":89,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.816878+0000","last_change":"2023-08-28T02:18:39.778728+0000","last_active":"2023-08-28T02:20:50.816878+0000","last_peered":"2023-08-28T02:20:50.816878+0000","last_clean":"2023-08-28T02:20:50.816878+0000","last_became_active":"2023-08-28T02:18:39.778283+0000","last_became_peered":"2023-08-28T02:18:39.778283+0000","last_unstale":"2023-08-28T02:20:50.816878+0000","last_undegraded":"2023-08-28T02:20:50.816878+0000","last_fullsized":"2023-08-28T02:20:50.816878+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":63,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:21:02.793865+0000","last_change":"2023-08-28T02:20:50.823670+0000","last_active":"2023-08-28T02:21:02.793865+0000","last_peered":"2023-08-28T02:21:02.793865+0000","last_clean":"2023-08-28T02:21:02.793865+0000","last_became_active":"2023-08-28T02:20:50.823229+0000","last_became_peered":"2023-08-28T02:20:50.823229+0000","last_unstale":"2023-08-28T02:21:02.793865+0000","last_undegraded":"2023-08-28T02:21:02.793865+0000","last_fullsized":"2023-08-28T02:21:02.793865+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:50:36.316012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":66,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.817149+0000","last_change":"2023-08-28T02:11:22.188327+0000","last_active":"2023-08-28T02:20:50.817149+0000","last_peered":"2023-08-28T02:20:50.817149+0000","last_clean":"2023-08-28T02:20:50.817149+0000","last_became_active":"2023-08-28T02:11:22.187870+0000","last_became_peered":"2023-08-28T02:11:22.187870+0000","last_unstale":"2023-08-28T02:20:50.817149+0000","last_undegraded":"2023-08-28T02:20:50.817149+0000","last_fullsized":"2023-08-28T02:20:50.817149+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":105,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.927775+0000","last_change":"2023-08-28T02:20:51.927775+0000","last_active":"2023-08-28T02:20:51.927775+0000","last_peered":"2023-08-28T02:20:51.927775+0000","last_clean":"2023-08-28T02:20:51.927775+0000","last_became_active":"2023-08-28T02:20:51.927401+0000","last_became_peered":"2023-08-28T02:20:51.927401+0000","last_unstale":"2023-08-28T02:20:51.927775+0000","last_undegraded":"2023-08-28T02:20:51.927775+0000","last_fullsized":"2023-08-28T02:20:51.927775+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":62,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:57.793198+0000","last_change":"2023-08-28T02:20:50.821233+0000","last_active":"2023-08-28T02:20:57.793198+0000","last_peered":"2023-08-28T02:20:57.793198+0000","last_clean":"2023-08-28T02:20:57.793198+0000","last_became_active":"2023-08-28T02:20:50.821062+0000","last_became_peered":"2023-08-28T02:20:50.821062+0000","last_unstale":"2023-08-28T02:20:57.793198+0000","last_undegraded":"2023-08-28T02:20:57.793198+0000","last_fullsized":"2023-08-28T02:20:57.793198+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:19:40.305189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":78,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.826178+0000","last_change":"2023-08-28T02:20:50.825633+0000","last_active":"2023-08-28T02:20:50.826178+0000","last_peered":"2023-08-28T02:20:50.826178+0000","last_clean":"2023-08-28T02:20:50.826178+0000","last_became_active":"2023-08-28T02:20:50.825328+0000","last_became_peered":"2023-08-28T02:20:50.825328+0000","last_unstale":"2023-08-28T02:20:50.826178+0000","last_undegraded":"2023-08-28T02:20:50.826178+0000","last_fullsized":"2023-08-28T02:20:50.826178+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:59:37.406892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":83,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.818201+0000","last_change":"2023-08-28T02:20:19.849490+0000","last_active":"2023-08-28T02:20:50.818201+0000","last_peered":"2023-08-28T02:20:50.818201+0000","last_clean":"2023-08-28T02:20:50.818201+0000","last_became_active":"2023-08-28T02:20:19.848962+0000","last_became_peered":"2023-08-28T02:20:19.848962+0000","last_unstale":"2023-08-28T02:20:50.818201+0000","last_undegraded":"2023-08-28T02:20:50.818201+0000","last_fullsized":"2023-08-28T02:20:50.818201+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":94,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.812125+0000","last_change":"2023-08-28T02:19:51.204118+0000","last_active":"2023-08-28T02:20:50.812125+0000","last_peered":"2023-08-28T02:20:50.812125+0000","last_clean":"2023-08-28T02:20:50.812125+0000","last_became_active":"2023-08-28T02:19:51.203745+0000","last_became_peered":"2023-08-28T02:19:51.203745+0000","last_unstale":"2023-08-28T02:20:50.812125+0000","last_undegraded":"2023-08-28T02:20:50.812125+0000","last_fullsized":"2023-08-28T02:20:50.812125+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"86'22","reported_seq":88,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.812982+0000","last_change":"2023-08-28T02:11:22.184497+0000","last_active":"2023-08-28T02:20:50.812982+0000","last_peered":"2023-08-28T02:20:50.812982+0000","last_clean":"2023-08-28T02:20:50.812982+0000","last_became_active":"2023-08-28T02:11:22.184173+0000","last_became_peered":"2023-08-28T02:11:22.184173+0000","last_unstale":"2023-08-28T02:20:50.812982+0000","last_undegraded":"2023-08-28T02:20:50.812982+0000","last_fullsized":"2023-08-28T02:20:50.812982+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":53,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.809557+0000","last_change":"2023-08-28T02:20:19.851616+0000","last_active":"2023-08-28T02:20:50.809557+0000","last_peered":"2023-08-28T02:20:50.809557+0000","last_clean":"2023-08-28T02:20:50.809557+0000","last_became_active":"2023-08-28T02:20:19.851322+0000","last_became_peered":"2023-08-28T02:20:19.851322+0000","last_unstale":"2023-08-28T02:20:50.809557+0000","last_undegraded":"2023-08-28T02:20:50.809557+0000","last_fullsized":"2023-08-28T02:20:50.809557+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:03:37.565646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":84,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.826581+0000","last_change":"2023-08-28T02:20:50.826189+0000","last_active":"2023-08-28T02:20:50.826581+0000","last_peered":"2023-08-28T02:20:50.826581+0000","last_clean":"2023-08-28T02:20:50.826581+0000","last_became_active":"2023-08-28T02:20:50.825824+0000","last_became_peered":"2023-08-28T02:20:50.825824+0000","last_unstale":"2023-08-28T02:20:50.826581+0000","last_undegraded":"2023-08-28T02:20:50.826581+0000","last_fullsized":"2023-08-28T02:20:50.826581+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":64,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.827001+0000","last_change":"2023-08-28T02:20:50.826216+0000","last_active":"2023-08-28T02:20:50.827001+0000","last_peered":"2023-08-28T02:20:50.827001+0000","last_clean":"2023-08-28T02:20:50.827001+0000","last_became_active":"2023-08-28T02:20:50.825949+0000","last_became_peered":"2023-08-28T02:20:50.825949+0000","last_unstale":"2023-08-28T02:20:50.827001+0000","last_undegraded":"2023-08-28T02:20:50.827001+0000","last_fullsized":"2023-08-28T02:20:50.827001+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:08:15.626853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":81,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.818941+0000","last_change":"2023-08-28T02:19:50.214698+0000","last_active":"2023-08-28T02:20:50.818941+0000","last_peered":"2023-08-28T02:20:50.818941+0000","last_clean":"2023-08-28T02:20:50.818941+0000","last_became_active":"2023-08-28T02:19:50.214323+0000","last_became_peered":"2023-08-28T02:19:50.214323+0000","last_unstale":"2023-08-28T02:20:50.818941+0000","last_undegraded":"2023-08-28T02:20:50.818941+0000","last_fullsized":"2023-08-28T02:20:50.818941+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":42,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.816629+0000","last_change":"2023-08-28T02:19:51.203925+0000","last_active":"2023-08-28T02:20:51.816629+0000","last_peered":"2023-08-28T02:20:51.816629+0000","last_clean":"2023-08-28T02:20:51.816629+0000","last_became_active":"2023-08-28T02:19:51.203594+0000","last_became_peered":"2023-08-28T02:19:51.203594+0000","last_unstale":"2023-08-28T02:20:51.816629+0000","last_undegraded":"2023-08-28T02:20:51.816629+0000","last_fullsized":"2023-08-28T02:20:51.816629+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:02:14.852254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,6],"acting":[0,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":62,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:21:02.793942+0000","last_change":"2023-08-28T02:20:50.828055+0000","last_active":"2023-08-28T02:21:02.793942+0000","last_peered":"2023-08-28T02:21:02.793942+0000","last_clean":"2023-08-28T02:21:02.793942+0000","last_became_active":"2023-08-28T02:20:50.827778+0000","last_became_peered":"2023-08-28T02:20:50.827778+0000","last_unstale":"2023-08-28T02:21:02.793942+0000","last_undegraded":"2023-08-28T02:21:02.793942+0000","last_fullsized":"2023-08-28T02:21:02.793942+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:50:35.464371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":100,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.823018+0000","last_change":"2023-08-28T02:20:51.823018+0000","last_active":"2023-08-28T02:20:51.823018+0000","last_peered":"2023-08-28T02:20:51.823018+0000","last_clean":"2023-08-28T02:20:51.823018+0000","last_became_active":"2023-08-28T02:20:51.822499+0000","last_became_peered":"2023-08-28T02:20:51.822499+0000","last_unstale":"2023-08-28T02:20:51.823018+0000","last_undegraded":"2023-08-28T02:20:51.823018+0000","last_fullsized":"2023-08-28T02:20:51.823018+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":94,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.821931+0000","last_change":"2023-08-28T02:20:50.819499+0000","last_active":"2023-08-28T02:20:50.821931+0000","last_peered":"2023-08-28T02:20:50.821931+0000","last_clean":"2023-08-28T02:20:50.821931+0000","last_became_active":"2023-08-28T02:20:50.819026+0000","last_became_peered":"2023-08-28T02:20:50.819026+0000","last_unstale":"2023-08-28T02:20:50.821931+0000","last_undegraded":"2023-08-28T02:20:50.821931+0000","last_fullsized":"2023-08-28T02:20:50.821931+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:59:22.177440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":63,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.817148+0000","last_change":"2023-08-28T02:20:50.823506+0000","last_active":"2023-08-28T02:20:51.817148+0000","last_peered":"2023-08-28T02:20:51.817148+0000","last_clean":"2023-08-28T02:20:51.817148+0000","last_became_active":"2023-08-28T02:20:50.822836+0000","last_became_peered":"2023-08-28T02:20:50.822836+0000","last_unstale":"2023-08-28T02:20:51.817148+0000","last_undegraded":"2023-08-28T02:20:51.817148+0000","last_fullsized":"2023-08-28T02:20:51.817148+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:43:06.363328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":52,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.821703+0000","last_change":"2023-08-28T02:20:50.820616+0000","last_active":"2023-08-28T02:20:50.821703+0000","last_peered":"2023-08-28T02:20:50.821703+0000","last_clean":"2023-08-28T02:20:50.821703+0000","last_became_active":"2023-08-28T02:20:50.820305+0000","last_became_peered":"2023-08-28T02:20:50.820305+0000","last_unstale":"2023-08-28T02:20:50.821703+0000","last_undegraded":"2023-08-28T02:20:50.821703+0000","last_fullsized":"2023-08-28T02:20:50.821703+0000","mapping_epoch":112,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":113,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:41:30.684047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":48,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-08-28T02:20:51.885245+0000","last_change":"2023-08-28T02:20:19.850774+0000","last_active":"2023-08-28T02:20:51.885245+0000","last_peered":"2023-08-28T02:20:51.885245+0000","last_clean":"2023-08-28T02:20:51.885245+0000","last_became_active":"2023-08-28T02:20:19.850448+0000","last_became_peered":"2023-08-28T02:20:19.850448+0000","last_unstale":"2023-08-28T02:20:51.885245+0000","last_undegraded":"2023-08-28T02:20:51.885245+0000","last_fullsized":"2023-08-28T02:20:51.885245+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:40:37.288553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":97,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-08-28T02:20:50.812255+0000","last_change":"2023-08-28T02:20:19.849807+0000","last_active":"2023-08-28T02:20:50.812255+0000","last_peered":"2023-08-28T02:20:50.812255+0000","last_clean":"2023-08-28T02:20:50.812255+0000","last_became_active":"2023-08-28T02:20:19.849464+0000","last_became_peered":"2023-08-28T02:20:19.849464+0000","last_unstale":"2023-08-28T02:20:50.812255+0000","last_undegraded":"2023-08-28T02:20:50.812255+0000","last_fullsized":"2023-08-28T02:20:50.812255+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26810,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430353,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307016,"kb_used_data":5824,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93438136,"statfs":{"total":95995035648,"available":95680651264,"internally_reserved":0,"allocated":5963776,"data_stored":5448619,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:20:43 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.53400000000000003,"15min":0.53600000000000003},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.746,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.60299999999999998,"15min":0.60899999999999999},"min":{"1min":0.38200000000000001,"5min":0.374,"15min":0.374},"max":{"1min":0.97199999999999998,"5min":0.97199999999999998,"15min":0.97199999999999998},"last":0.57099999999999995}]},{"osd":1,"last update":"Mon Aug 28 02:20:50 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.50800000000000001}]},{"osd":2,"last update":"Mon Aug 28 02:21:20 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.61399999999999999}]},{"osd":3,"last update":"Mon Aug 28 02:20:19 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.55700000000000005,"15min":0.54500000000000004},"min":{"1min":0.47599999999999998,"5min":0.35899999999999999,"15min":0.32600000000000001},"max":{"1min":0.877,"5min":0.88800000000000001,"15min":2.7509999999999999},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.56399999999999995,"15min":0.55900000000000005},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.35299999999999998},"max":{"1min":1.4770000000000001,"5min":1.4770000000000001,"15min":2.7080000000000002},"last":0.64000000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:21:20 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.48399999999999999,"15min":0.47499999999999998},"min":{"1min":0.35299999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.71299999999999997,"5min":0.75700000000000001,"15min":0.77800000000000002},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.503,"5min":0.502,"15min":0.498},"min":{"1min":0.30199999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.67600000000000005,"5min":0.71599999999999997,"15min":0.79800000000000004},"last":0.51700000000000002}]},{"osd":5,"last update":"Mon Aug 28 02:21:20 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.504,"15min":0.48499999999999999},"min":{"1min":0.312,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.69799999999999995,"5min":0.71099999999999997,"15min":0.81100000000000005},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.57199999999999995,"15min":0.54400000000000004},"min":{"1min":0.32200000000000001,"5min":0.314,"15min":0.314},"max":{"1min":0.746,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.66500000000000004}]},{"osd":6,"last update":"Mon Aug 28 02:21:20 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.56200000000000006,"15min":0.53400000000000003},"min":{"1min":0.33900000000000002,"5min":0.32700000000000001,"15min":0.32200000000000001},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.61699999999999999,"15min":0.58299999999999996},"min":{"1min":0.41399999999999998,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.77200000000000002,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.69399999999999995}]}]},{"osd":6,"up_from":42,"seq":180388626580,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302908,"kb_used_data":1716,"kb_used_omap":0,"kb_used_meta":301184,"kb_avail":93442244,"statfs":{"total":95995035648,"available":95684857856,"internally_reserved":0,"allocated":1757184,"data_stored":1253873,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308412416},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:20:47 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.56499999999999995,"15min":0.57099999999999995},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.432},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.59099999999999997,"15min":0.59899999999999998},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.66900000000000004,"5min":0.746,"15min":0.746},"last":0.40999999999999998}]},{"osd":1,"last update":"Mon Aug 28 02:20:51 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.51300000000000001}]},{"osd":2,"last update":"Mon Aug 28 02:21:23 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.575,"5min":1.575,"15min":1.575},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":1.6419999999999999,"5min":1.6419999999999999,"15min":1.6419999999999999},"last":0.47999999999999998}]},{"osd":3,"last update":"Mon Aug 28 02:19:54 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.55200000000000005,"15min":0.54900000000000004},"min":{"1min":0.41799999999999998,"5min":0.33400000000000002,"15min":0.30499999999999999},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.84399999999999997},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.54400000000000004,"15min":0.52300000000000002},"min":{"1min":0.35599999999999998,"5min":0.34699999999999998,"15min":0.31900000000000001},"max":{"1min":0.69599999999999995,"5min":0.76500000000000001,"15min":1.02},"last":0.58299999999999996}]},{"osd":4,"last update":"Mon Aug 28 02:20:55 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.46200000000000002,"15min":0.44400000000000001},"min":{"1min":0.32400000000000001,"5min":0.27800000000000002,"15min":0.27200000000000002},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.877},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.51600000000000001,"15min":0.47599999999999998},"min":{"1min":0.38500000000000001,"5min":0.29699999999999999,"15min":0.28899999999999998},"max":{"1min":0.84099999999999997,"5min":0.84899999999999998,"15min":0.85599999999999998},"last":0.61699999999999999}]},{"osd":5,"last update":"Mon Aug 28 02:20:55 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.48199999999999998,"15min":0.45800000000000002},"min":{"1min":0.318,"5min":0.313,"15min":0.29299999999999998},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.79300000000000004},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.53900000000000003,"15min":0.51400000000000001},"min":{"1min":0.40300000000000002,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.82199999999999995,"5min":0.82499999999999996,"15min":2.0609999999999999},"last":0.629}]},{"osd":7,"last update":"Mon Aug 28 02:21:16 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.57199999999999995,"15min":0.55000000000000004},"min":{"1min":0.41199999999999998,"5min":0.35799999999999998,"15min":0.32700000000000001},"max":{"1min":0.85699999999999998,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.65600000000000003,"5min":0.59899999999999998,"15min":0.57399999999999995},"min":{"1min":0.42799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.83499999999999996,"5min":0.88100000000000001,"15min":0.89400000000000002},"last":0.65400000000000003}]}]},{"osd":1,"up_from":98,"seq":420906795032,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36872,"kb_used_data":5312,"kb_used_omap":7,"kb_used_meta":31544,"kb_avail":93708280,"statfs":{"total":95995035648,"available":95957278720,"internally_reserved":0,"allocated":5439488,"data_stored":4873933,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":32301387},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:20:50 2023","interfaces":[{"interface":"back","average":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.498,"5min":0.498,"15min":0.498},"last":0.28000000000000003}]},{"osd":2,"last update":"Mon Aug 28 02:21:23 2023","interfaces":[{"interface":"back","average":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.311}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34599999999999997}]},{"osd":4,"last update":"Mon Aug 28 02:20:50 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.90800000000000003,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.54000000000000004}]},{"osd":5,"last update":"Mon Aug 28 02:20:50 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.96599999999999997,"5min":0.96599999999999997,"15min":0.96599999999999997},"last":0.53500000000000003}]},{"osd":6,"last update":"Mon Aug 28 02:20:50 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.94199999999999995,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.57499999999999996}]},{"osd":7,"last update":"Mon Aug 28 02:20:50 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.65200000000000002}]}]},{"osd":0,"up_from":90,"seq":386547056679,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":37464,"kb_used_data":5904,"kb_used_omap":8,"kb_used_meta":31543,"kb_avail":93707688,"statfs":{"total":95995035648,"available":95956672512,"internally_reserved":0,"allocated":6045696,"data_stored":5465837,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":32300393},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Aug 28 02:20:53 2023","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"min":{"1min":0.192,"5min":0.192,"15min":0.192},"max":{"1min":0.503,"5min":0.503,"15min":0.503},"last":0.505},{"interface":"front","average":{"1min":0.372,"5min":0.372,"15min":0.372},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.497,"5min":0.497,"15min":0.497},"last":0.39500000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:21:20 2023","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"last":0.432}]},{"osd":3,"last update":"Mon Aug 28 02:19:40 2023","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":0.435,"15min":0.435},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"last":0.377},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.44900000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:20:43 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.65800000000000003,"15min":0.66600000000000004},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.82099999999999995,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.621,"5min":0.68899999999999995,"15min":0.70099999999999996},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.79800000000000004,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.61599999999999999}]},{"osd":5,"last update":"Mon Aug 28 02:20:43 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.64300000000000002,"15min":0.65000000000000002},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.82399999999999995,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.63800000000000001,"15min":0.64800000000000002},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.79500000000000004}]},{"osd":6,"last update":"Mon Aug 28 02:20:43 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.64200000000000002,"15min":0.65100000000000002},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.77200000000000002,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.69599999999999995,"15min":0.70399999999999996},"min":{"1min":0.42199999999999999,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.876,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":1.538}]},{"osd":7,"last update":"Mon Aug 28 02:20:43 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.68400000000000005,"15min":0.69399999999999995},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.89200000000000002},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.68999999999999995,"15min":0.69899999999999995},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.79500000000000004,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.94899999999999995}]}]},{"osd":2,"up_from":104,"seq":446676598803,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32680,"kb_used_data":1184,"kb_used_omap":10,"kb_used_meta":31477,"kb_avail":93712472,"statfs":{"total":95995035648,"available":95961571328,"internally_reserved":0,"allocated":1212416,"data_stored":672302,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10792,"internal_metadata":32232920},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:21:22 2023","interfaces":[{"interface":"back","average":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":0.27900000000000003},{"interface":"front","average":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.29699999999999999}]},{"osd":1,"last update":"Mon Aug 28 02:21:22 2023","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.27100000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.35799999999999998}]},{"osd":4,"last update":"Mon Aug 28 02:21:22 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.91100000000000003,"5min":0.91100000000000003,"15min":0.91100000000000003},"last":0.67900000000000005}]},{"osd":5,"last update":"Mon Aug 28 02:21:22 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.48599999999999999}]},{"osd":6,"last update":"Mon Aug 28 02:21:22 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.52900000000000003}]},{"osd":7,"last update":"Mon Aug 28 02:21:22 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.55700000000000005}]}]},{"osd":3,"up_from":112,"seq":481036337164,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28432,"kb_used_data":1160,"kb_used_omap":3,"kb_used_meta":27260,"kb_avail":93716720,"statfs":{"total":95995035648,"available":95965921280,"internally_reserved":0,"allocated":1187840,"data_stored":667951,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":27915170},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.374}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41699999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28699999999999998}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69399999999999995}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67600000000000005}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64100000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.71299999999999997}]}]},{"osd":4,"up_from":30,"seq":128849019038,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298236,"kb_used_data":1140,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93446916,"statfs":{"total":95995035648,"available":95689641984,"internally_reserved":0,"allocated":1167360,"data_stored":663855,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:20:43 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.60399999999999998,"15min":0.61199999999999999},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.58599999999999997,"15min":0.59499999999999997},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.71399999999999997,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.56399999999999995}]},{"osd":1,"last update":"Mon Aug 28 02:20:49 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.59599999999999997}]},{"osd":2,"last update":"Mon Aug 28 02:21:19 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.53100000000000003}]},{"osd":3,"last update":"Mon Aug 28 02:20:01 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.58099999999999996,"15min":0.55300000000000005},"min":{"1min":0.45700000000000002,"5min":0.41799999999999998,"15min":0.307},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":1.1599999999999999},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.628,"5min":0.58099999999999996,"15min":0.55400000000000005},"min":{"1min":0.496,"5min":0.371,"15min":0.35899999999999999},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.871},"last":0.66600000000000004}]},{"osd":5,"last update":"Mon Aug 28 02:21:35 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.502,"15min":0.44900000000000001},"min":{"1min":0.36899999999999999,"5min":0.31900000000000001,"15min":0.26600000000000001},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.503,"15min":0.46100000000000002},"min":{"1min":0.34100000000000003,"5min":0.32500000000000001,"15min":0.27700000000000002},"max":{"1min":0.63800000000000001,"5min":0.78600000000000003,"15min":1.1839999999999999},"last":0.42899999999999999}]},{"osd":6,"last update":"Mon Aug 28 02:20:49 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.53000000000000003,"15min":0.48299999999999998},"min":{"1min":0.375,"5min":0.30499999999999999,"15min":0.27500000000000002},"max":{"1min":0.76000000000000001,"5min":0.77300000000000002,"15min":0.85099999999999998},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.58399999999999996,"15min":0.52100000000000002},"min":{"1min":0.45600000000000002,"5min":0.36799999999999999,"15min":0.318},"max":{"1min":0.77500000000000002,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.47299999999999998}]},{"osd":7,"last update":"Mon Aug 28 02:21:18 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.56899999999999995,"15min":0.53500000000000003},"min":{"1min":0.40400000000000003,"5min":0.36899999999999999,"15min":0.312},"max":{"1min":0.70999999999999996,"5min":0.82099999999999995,"15min":1.2110000000000001},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.59399999999999997,"15min":0.54500000000000004},"min":{"1min":0.41899999999999998,"5min":0.35199999999999998,"15min":0.32100000000000001},"max":{"1min":0.79700000000000004,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.61799999999999999}]}]},{"osd":5,"up_from":36,"seq":154618822810,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298220,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93446932,"statfs":{"total":95995035648,"available":95689658368,"internally_reserved":0,"allocated":1150976,"data_stored":659010,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:20:40 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.60999999999999999,"15min":0.62},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.61899999999999999,"15min":0.625},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.747,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.48499999999999999}]},{"osd":1,"last update":"Mon Aug 28 02:20:50 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.441}]},{"osd":2,"last update":"Mon Aug 28 02:21:22 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.505},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.55800000000000005}]},{"osd":3,"last update":"Mon Aug 28 02:20:36 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57999999999999996,"15min":0.54900000000000004},"min":{"1min":0.39100000000000001,"5min":0.255,"15min":0.255},"max":{"1min":0.78400000000000003,"5min":1.1279999999999999,"15min":1.1779999999999999},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.59499999999999997,"15min":0.57299999999999995},"min":{"1min":0.40400000000000003,"5min":0.35299999999999998,"15min":0.307},"max":{"1min":0.80800000000000005,"5min":1.1679999999999999,"15min":1.1679999999999999},"last":0.39400000000000002}]},{"osd":4,"last update":"Mon Aug 28 02:21:37 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.49199999999999999,"15min":0.46100000000000002},"min":{"1min":0.39400000000000002,"5min":0.28000000000000003,"15min":0.27500000000000002},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.79200000000000004},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.51300000000000001,"15min":0.48499999999999999},"min":{"1min":0.34499999999999997,"5min":0.28799999999999998,"15min":0.28199999999999997},"max":{"1min":0.75800000000000001,"5min":0.80000000000000004,"15min":1.1499999999999999},"last":0.59899999999999998}]},{"osd":6,"last update":"Mon Aug 28 02:20:50 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.51300000000000001,"15min":0.51900000000000002},"min":{"1min":0.36699999999999999,"5min":0.32700000000000001,"15min":0.28399999999999997},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":5.2119999999999997},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.55400000000000005,"15min":0.51600000000000001},"min":{"1min":0.442,"5min":0.31900000000000001,"15min":0.30299999999999999},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":1.089},"last":0.628}]},{"osd":7,"last update":"Mon Aug 28 02:21:13 2023","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.58299999999999996,"15min":0.55300000000000005},"min":{"1min":0.45000000000000001,"5min":0.35199999999999998,"15min":0.32900000000000001},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":1.119},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.63,"5min":0.57499999999999996,"15min":0.55300000000000005},"min":{"1min":0.40400000000000003,"5min":0.34999999999999998,"15min":0.29799999999999999},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.62}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-08-28T02:21:44.886 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-08-28T02:21:44.886 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd dump 2023-08-28T02:21:45.238 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:21:46.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:46 smithi143 ceph-mon[183059]: from='client.44519 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:21:46.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:46 smithi143 ceph-mon[183059]: pgmap v215: 73 pgs: 1 activating, 72 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:46.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:46 smithi143 ceph-mon[189871]: from='client.44519 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:21:46.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:46 smithi143 ceph-mon[189871]: pgmap v215: 73 pgs: 1 activating, 72 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:46.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:46 smithi169 ceph-mon[159415]: from='client.44519 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:21:46.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:46 smithi169 ceph-mon[159415]: pgmap v215: 73 pgs: 1 activating, 72 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:46.797 INFO:teuthology.orchestra.run.smithi143.stdout:epoch 116 2023-08-28T02:21:46.797 INFO:teuthology.orchestra.run.smithi143.stdout:fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 2023-08-28T02:21:46.797 INFO:teuthology.orchestra.run.smithi143.stdout:created 2023-08-28T02:05:20.056816+0000 2023-08-28T02:21:46.797 INFO:teuthology.orchestra.run.smithi143.stdout:modified 2023-08-28T02:21:43.189122+0000 2023-08-28T02:21:46.797 INFO:teuthology.orchestra.run.smithi143.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2023-08-28T02:21:46.797 INFO:teuthology.orchestra.run.smithi143.stdout:crush_version 22 2023-08-28T02:21:46.797 INFO:teuthology.orchestra.run.smithi143.stdout:full_ratio 0.95 2023-08-28T02:21:46.797 INFO:teuthology.orchestra.run.smithi143.stdout:backfillfull_ratio 0.9 2023-08-28T02:21:46.798 INFO:teuthology.orchestra.run.smithi143.stdout:nearfull_ratio 0.85 2023-08-28T02:21:46.798 INFO:teuthology.orchestra.run.smithi143.stdout:require_min_compat_client luminous 2023-08-28T02:21:46.798 INFO:teuthology.orchestra.run.smithi143.stdout:min_compat_client luminous 2023-08-28T02:21:46.798 INFO:teuthology.orchestra.run.smithi143.stdout:require_osd_release pacific 2023-08-28T02:21:46.798 INFO:teuthology.orchestra.run.smithi143.stdout:stretch_mode_enabled false 2023-08-28T02:21:46.798 INFO:teuthology.orchestra.run.smithi143.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 83 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr,mgr_devicehealth read_balance_score 7.89 2023-08-28T02:21:46.798 INFO:teuthology.orchestra.run.smithi143.stdout:pool 2 'rbd' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode off last_change 77 flags hashpspool,selfmanaged_snaps stripe_width 0 application rbd read_balance_score 2.00 2023-08-28T02:21:46.798 INFO:teuthology.orchestra.run.smithi143.stdout:pool 3 'cephfs.foo.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 59 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 1.75 2023-08-28T02:21:46.798 INFO:teuthology.orchestra.run.smithi143.stdout:pool 4 'cephfs.foo.data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 60 flags hashpspool,bulk stripe_width 0 application cephfs read_balance_score 2.00 2023-08-28T02:21:46.798 INFO:teuthology.orchestra.run.smithi143.stdout:max_osd 8 2023-08-28T02:21:46.798 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0 up in weight 1 up_from 90 up_thru 113 down_at 87 last_clean_interval [8,86) [v2:172.21.15.143:6802/257144662,v1:172.21.15.143:6803/257144662] [v2:172.21.15.143:6804/257144662,v1:172.21.15.143:6805/257144662] exists,up 718c8f27-245b-4e9d-9e2e-d1fc65ca7472 2023-08-28T02:21:46.798 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1 up in weight 1 up_from 98 up_thru 115 down_at 92 last_clean_interval [13,91) [v2:172.21.15.143:6810/4171000628,v1:172.21.15.143:6811/4171000628] [v2:172.21.15.143:6812/4171000628,v1:172.21.15.143:6813/4171000628] exists,up cf24771f-3b0c-4a00-8a73-13e1fee242dc 2023-08-28T02:21:46.799 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2 up in weight 1 up_from 104 up_thru 112 down_at 101 last_clean_interval [18,100) [v2:172.21.15.143:6818/1552459422,v1:172.21.15.143:6819/1552459422] [v2:172.21.15.143:6820/1552459422,v1:172.21.15.143:6821/1552459422] exists,up 4f8492d1-0e7a-49f8-b6a5-4f5e73842349 2023-08-28T02:21:46.799 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3 up in weight 1 up_from 112 up_thru 113 down_at 106 last_clean_interval [25,105) [v2:172.21.15.143:6826/2677897674,v1:172.21.15.143:6827/2677897674] [v2:172.21.15.143:6828/2677897674,v1:172.21.15.143:6829/2677897674] exists,up dd6abc8a-54a9-4e08-ba81-0b894f24d84a 2023-08-28T02:21:46.799 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4 up in weight 1 up_from 30 up_thru 113 down_at 0 last_clean_interval [0,0) [v2:172.21.15.169:6800/3960017153,v1:172.21.15.169:6801/3960017153] [v2:172.21.15.169:6802/3960017153,v1:172.21.15.169:6803/3960017153] exists,up 16122219-d985-46d2-bac1-f8ffcf48d8ad 2023-08-28T02:21:46.799 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5 up in weight 1 up_from 36 up_thru 113 down_at 0 last_clean_interval [0,0) [v2:172.21.15.169:6808/4087672846,v1:172.21.15.169:6809/4087672846] [v2:172.21.15.169:6810/4087672846,v1:172.21.15.169:6811/4087672846] exists,up adbc3ea5-230f-419e-8f6f-99763378b296 2023-08-28T02:21:46.799 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6 up in weight 1 up_from 42 up_thru 115 down_at 0 last_clean_interval [0,0) [v2:172.21.15.169:6816/3801227982,v1:172.21.15.169:6817/3801227982] [v2:172.21.15.169:6818/3801227982,v1:172.21.15.169:6819/3801227982] exists,up 2abe4d77-c12c-477f-a4ec-eabb753ca4dd 2023-08-28T02:21:46.799 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7 up in weight 1 up_from 48 up_thru 112 down_at 0 last_clean_interval [0,0) [v2:172.21.15.169:6824/3830851149,v1:172.21.15.169:6825/3830851149] [v2:172.21.15.169:6826/3830851149,v1:172.21.15.169:6827/3830851149] exists,up fc5ed8fc-8ad7-4b54-a8ea-bc6a9e846f63 2023-08-28T02:21:46.799 INFO:teuthology.orchestra.run.smithi143.stdout:pg_upmap_items 3.9 [4,3] 2023-08-28T02:21:46.799 INFO:teuthology.orchestra.run.smithi143.stdout:pg_upmap_items 3.a [4,3] 2023-08-28T02:21:46.799 INFO:teuthology.orchestra.run.smithi143.stdout:pg_upmap_items 3.f [4,2] 2023-08-28T02:21:46.799 INFO:teuthology.orchestra.run.smithi143.stdout:pg_upmap_items 4.4 [3,5] 2023-08-28T02:21:46.799 INFO:teuthology.orchestra.run.smithi143.stdout:pg_upmap_items 4.6 [1,3] 2023-08-28T02:21:46.799 INFO:teuthology.orchestra.run.smithi143.stdout:pg_upmap_items 4.b [4,5] 2023-08-28T02:21:46.800 INFO:teuthology.orchestra.run.smithi143.stdout:pg_upmap_items 4.d [1,3] 2023-08-28T02:21:46.800 INFO:teuthology.orchestra.run.smithi143.stdout:pg_upmap_items 4.e [4,0] 2023-08-28T02:21:46.800 INFO:teuthology.orchestra.run.smithi143.stdout:pg_upmap_items 4.10 [3,0] 2023-08-28T02:21:46.800 INFO:teuthology.orchestra.run.smithi143.stdout:pg_upmap_items 4.17 [3,2] 2023-08-28T02:21:46.800 INFO:teuthology.orchestra.run.smithi143.stdout:pg_upmap_items 4.18 [1,3] 2023-08-28T02:21:46.800 INFO:teuthology.orchestra.run.smithi143.stdout:pg_upmap_items 4.1d [3,2] 2023-08-28T02:21:46.800 INFO:teuthology.orchestra.run.smithi143.stdout:pg_upmap_items 4.1f [1,3] 2023-08-28T02:21:46.800 INFO:teuthology.orchestra.run.smithi143.stdout:blocklist 172.21.15.143:0/2467953564 expires 2023-08-29T02:15:40.929762+0000 2023-08-28T02:21:46.800 INFO:teuthology.orchestra.run.smithi143.stdout:blocklist 172.21.15.143:6800/155264805 expires 2023-08-29T02:15:40.929762+0000 2023-08-28T02:21:46.800 INFO:teuthology.orchestra.run.smithi143.stdout:blocklist 172.21.15.143:0/437060254 expires 2023-08-29T02:15:40.929762+0000 2023-08-28T02:21:46.800 INFO:teuthology.orchestra.run.smithi143.stdout:blocklist 172.21.15.143:6801/155264805 expires 2023-08-29T02:15:40.929762+0000 2023-08-28T02:21:46.800 INFO:teuthology.orchestra.run.smithi143.stdout:blocklist 172.21.15.143:0/1563442416 expires 2023-08-29T02:10:09.845824+0000 2023-08-28T02:21:46.800 INFO:teuthology.orchestra.run.smithi143.stdout:blocklist 172.21.15.169:0/338571757 expires 2023-08-29T02:15:01.746015+0000 2023-08-28T02:21:46.801 INFO:teuthology.orchestra.run.smithi143.stdout:blocklist 172.21.15.169:0/2988270055 expires 2023-08-29T02:15:01.746015+0000 2023-08-28T02:21:46.801 INFO:teuthology.orchestra.run.smithi143.stdout:blocklist 172.21.15.143:0/1148418770 expires 2023-08-29T02:05:41.797747+0000 2023-08-28T02:21:46.801 INFO:teuthology.orchestra.run.smithi143.stdout:blocklist 172.21.15.143:0/11141072 expires 2023-08-29T02:05:41.797747+0000 2023-08-28T02:21:46.801 INFO:teuthology.orchestra.run.smithi143.stdout:blocklist 172.21.15.143:6800/907474889 expires 2023-08-29T02:06:02.911068+0000 2023-08-28T02:21:46.801 INFO:teuthology.orchestra.run.smithi143.stdout:blocklist 172.21.15.143:0/2157302459 expires 2023-08-29T02:10:09.845824+0000 2023-08-28T02:21:46.801 INFO:teuthology.orchestra.run.smithi143.stdout:blocklist 172.21.15.143:0/1035910192 expires 2023-08-29T02:13:52.442893+0000 2023-08-28T02:21:46.801 INFO:teuthology.orchestra.run.smithi143.stdout:blocklist 172.21.15.143:0/2083962846 expires 2023-08-29T02:15:40.929762+0000 2023-08-28T02:21:46.801 INFO:teuthology.orchestra.run.smithi143.stdout:blocklist 172.21.15.169:0/3392857751 expires 2023-08-29T02:15:01.746015+0000 2023-08-28T02:21:46.801 INFO:teuthology.orchestra.run.smithi143.stdout:blocklist 172.21.15.143:6801/1828180492 expires 2023-08-29T02:10:09.845824+0000 2023-08-28T02:21:46.801 INFO:teuthology.orchestra.run.smithi143.stdout:blocklist 172.21.15.143:0/1948284892 expires 2023-08-29T02:15:40.929762+0000 2023-08-28T02:21:46.801 INFO:teuthology.orchestra.run.smithi143.stdout:blocklist 172.21.15.143:0/64297767 expires 2023-08-29T02:15:40.929762+0000 2023-08-28T02:21:46.801 INFO:teuthology.orchestra.run.smithi143.stdout:blocklist 172.21.15.143:0/3251765285 expires 2023-08-29T02:13:52.442893+0000 2023-08-28T02:21:46.802 INFO:teuthology.orchestra.run.smithi143.stdout:blocklist 172.21.15.169:0/1064560701 expires 2023-08-29T02:15:01.746015+0000 2023-08-28T02:21:46.802 INFO:teuthology.orchestra.run.smithi143.stdout:blocklist 172.21.15.143:0/3606471716 expires 2023-08-29T02:06:02.911068+0000 2023-08-28T02:21:46.802 INFO:teuthology.orchestra.run.smithi143.stdout:blocklist 172.21.15.143:6801/1019944263 expires 2023-08-29T02:05:41.797747+0000 2023-08-28T02:21:46.802 INFO:teuthology.orchestra.run.smithi143.stdout:blocklist 172.21.15.169:0/2187902729 expires 2023-08-29T02:15:01.746015+0000 2023-08-28T02:21:46.802 INFO:teuthology.orchestra.run.smithi143.stdout:blocklist 172.21.15.143:0/2323551826 expires 2023-08-29T02:06:02.911068+0000 2023-08-28T02:21:46.802 INFO:teuthology.orchestra.run.smithi143.stdout:blocklist 172.21.15.143:6800/1828180492 expires 2023-08-29T02:10:09.845824+0000 2023-08-28T02:21:46.802 INFO:teuthology.orchestra.run.smithi143.stdout:blocklist 172.21.15.143:0/550889863 expires 2023-08-29T02:15:40.929762+0000 2023-08-28T02:21:46.802 INFO:teuthology.orchestra.run.smithi143.stdout:blocklist 172.21.15.143:6801/907474889 expires 2023-08-29T02:06:02.911068+0000 2023-08-28T02:21:46.802 INFO:teuthology.orchestra.run.smithi143.stdout:blocklist 172.21.15.143:6800/1019944263 expires 2023-08-29T02:05:41.797747+0000 2023-08-28T02:21:46.802 INFO:teuthology.orchestra.run.smithi143.stdout:blocklist 172.21.15.143:6800/1118800746 expires 2023-08-29T02:13:52.442893+0000 2023-08-28T02:21:46.802 INFO:teuthology.orchestra.run.smithi143.stdout:blocklist 172.21.15.169:6834/3955030351 expires 2023-08-29T02:15:01.746015+0000 2023-08-28T02:21:46.802 INFO:teuthology.orchestra.run.smithi143.stdout:blocklist 172.21.15.169:6835/3955030351 expires 2023-08-29T02:15:01.746015+0000 2023-08-28T02:21:46.802 INFO:teuthology.orchestra.run.smithi143.stdout:blocklist 172.21.15.143:6801/1118800746 expires 2023-08-29T02:13:52.442893+0000 2023-08-28T02:21:46.802 INFO:teuthology.orchestra.run.smithi143.stdout:blocklist 172.21.15.169:0/2161024377 expires 2023-08-29T02:15:01.746015+0000 2023-08-28T02:21:47.388 INFO:tasks.cephadm.ceph_manager.ceph:['osd.0 up in weight 1 up_from 90 up_thru 113 down_at 87 last_clean_interval [8,86) [v2:172.21.15.143:6802/257144662,v1:172.21.15.143:6803/257144662] [v2:172.21.15.143:6804/257144662,v1:172.21.15.143:6805/257144662] exists,up 718c8f27-245b-4e9d-9e2e-d1fc65ca7472', 'osd.1 up in weight 1 up_from 98 up_thru 115 down_at 92 last_clean_interval [13,91) [v2:172.21.15.143:6810/4171000628,v1:172.21.15.143:6811/4171000628] [v2:172.21.15.143:6812/4171000628,v1:172.21.15.143:6813/4171000628] exists,up cf24771f-3b0c-4a00-8a73-13e1fee242dc', 'osd.2 up in weight 1 up_from 104 up_thru 112 down_at 101 last_clean_interval [18,100) [v2:172.21.15.143:6818/1552459422,v1:172.21.15.143:6819/1552459422] [v2:172.21.15.143:6820/1552459422,v1:172.21.15.143:6821/1552459422] exists,up 4f8492d1-0e7a-49f8-b6a5-4f5e73842349', 'osd.3 up in weight 1 up_from 112 up_thru 113 down_at 106 last_clean_interval [25,105) [v2:172.21.15.143:6826/2677897674,v1:172.21.15.143:6827/2677897674] [v2:172.21.15.143:6828/2677897674,v1:172.21.15.143:6829/2677897674] exists,up dd6abc8a-54a9-4e08-ba81-0b894f24d84a', 'osd.4 up in weight 1 up_from 30 up_thru 113 down_at 0 last_clean_interval [0,0) [v2:172.21.15.169:6800/3960017153,v1:172.21.15.169:6801/3960017153] [v2:172.21.15.169:6802/3960017153,v1:172.21.15.169:6803/3960017153] exists,up 16122219-d985-46d2-bac1-f8ffcf48d8ad', 'osd.5 up in weight 1 up_from 36 up_thru 113 down_at 0 last_clean_interval [0,0) [v2:172.21.15.169:6808/4087672846,v1:172.21.15.169:6809/4087672846] [v2:172.21.15.169:6810/4087672846,v1:172.21.15.169:6811/4087672846] exists,up adbc3ea5-230f-419e-8f6f-99763378b296', 'osd.6 up in weight 1 up_from 42 up_thru 115 down_at 0 last_clean_interval [0,0) [v2:172.21.15.169:6816/3801227982,v1:172.21.15.169:6817/3801227982] [v2:172.21.15.169:6818/3801227982,v1:172.21.15.169:6819/3801227982] exists,up 2abe4d77-c12c-477f-a4ec-eabb753ca4dd', 'osd.7 up in weight 1 up_from 48 up_thru 112 down_at 0 last_clean_interval [0,0) [v2:172.21.15.169:6824/3830851149,v1:172.21.15.169:6825/3830851149] [v2:172.21.15.169:6826/3830851149,v1:172.21.15.169:6827/3830851149] exists,up fc5ed8fc-8ad7-4b54-a8ea-bc6a9e846f63'] 2023-08-28T02:21:47.389 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon mon.a config show 2023-08-28T02:21:47.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:47 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/453543381' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-08-28T02:21:47.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:47 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/453543381' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-08-28T02:21:47.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:47 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/453543381' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-08-28T02:21:47.735 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:21:48.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:48 smithi169 ceph-mon[159415]: pgmap v216: 73 pgs: 1 activating, 72 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:48.682 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:48 smithi143 ceph-mon[183059]: pgmap v216: 73 pgs: 1 activating, 72 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:48.683 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:48 smithi143 ceph-mon[189871]: pgmap v216: 73 pgs: 1 activating, 72 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:49.092 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:21:49.092 INFO:teuthology.orchestra.run.smithi143.stdout: "name": "mon.a", 2023-08-28T02:21:49.092 INFO:teuthology.orchestra.run.smithi143.stdout: "cluster": "ceph", 2023-08-28T02:21:49.092 INFO:teuthology.orchestra.run.smithi143.stdout: "admin_socket": "/var/run/ceph/ceph-mon.a.asok", 2023-08-28T02:21:49.092 INFO:teuthology.orchestra.run.smithi143.stdout: "admin_socket_mode": "", 2023-08-28T02:21:49.092 INFO:teuthology.orchestra.run.smithi143.stdout: "auth_allow_insecure_global_id_reclaim": "false", 2023-08-28T02:21:49.092 INFO:teuthology.orchestra.run.smithi143.stdout: "auth_client_required": "cephx, none", 2023-08-28T02:21:49.092 INFO:teuthology.orchestra.run.smithi143.stdout: "auth_cluster_required": "cephx", 2023-08-28T02:21:49.093 INFO:teuthology.orchestra.run.smithi143.stdout: "auth_debug": "true", 2023-08-28T02:21:49.093 INFO:teuthology.orchestra.run.smithi143.stdout: "auth_expose_insecure_global_id_reclaim": "true", 2023-08-28T02:21:49.093 INFO:teuthology.orchestra.run.smithi143.stdout: "auth_mon_ticket_ttl": "660.000000", 2023-08-28T02:21:49.093 INFO:teuthology.orchestra.run.smithi143.stdout: "auth_service_required": "cephx", 2023-08-28T02:21:49.093 INFO:teuthology.orchestra.run.smithi143.stdout: "auth_service_ticket_ttl": "240.000000", 2023-08-28T02:21:49.093 INFO:teuthology.orchestra.run.smithi143.stdout: "auth_supported": "", 2023-08-28T02:21:49.093 INFO:teuthology.orchestra.run.smithi143.stdout: "bdev_aio": "true", 2023-08-28T02:21:49.093 INFO:teuthology.orchestra.run.smithi143.stdout: "bdev_aio_max_queue_depth": "1024", 2023-08-28T02:21:49.093 INFO:teuthology.orchestra.run.smithi143.stdout: "bdev_aio_poll_ms": "250", 2023-08-28T02:21:49.093 INFO:teuthology.orchestra.run.smithi143.stdout: "bdev_aio_reap_max": "16", 2023-08-28T02:21:49.093 INFO:teuthology.orchestra.run.smithi143.stdout: "bdev_async_discard": "false", 2023-08-28T02:21:49.093 INFO:teuthology.orchestra.run.smithi143.stdout: "bdev_block_size": "4096", 2023-08-28T02:21:49.094 INFO:teuthology.orchestra.run.smithi143.stdout: "bdev_debug_aio": "false", 2023-08-28T02:21:49.094 INFO:teuthology.orchestra.run.smithi143.stdout: "bdev_debug_aio_log_age": "5.000000", 2023-08-28T02:21:49.094 INFO:teuthology.orchestra.run.smithi143.stdout: "bdev_debug_aio_suicide_timeout": "60.000000", 2023-08-28T02:21:49.094 INFO:teuthology.orchestra.run.smithi143.stdout: "bdev_debug_inflight_ios": "false", 2023-08-28T02:21:49.094 INFO:teuthology.orchestra.run.smithi143.stdout: "bdev_enable_discard": "false", 2023-08-28T02:21:49.094 INFO:teuthology.orchestra.run.smithi143.stdout: "bdev_flock_retry": "3", 2023-08-28T02:21:49.094 INFO:teuthology.orchestra.run.smithi143.stdout: "bdev_flock_retry_interval": "0.100000", 2023-08-28T02:21:49.094 INFO:teuthology.orchestra.run.smithi143.stdout: "bdev_inject_crash": "0", 2023-08-28T02:21:49.094 INFO:teuthology.orchestra.run.smithi143.stdout: "bdev_inject_crash_flush_delay": "2", 2023-08-28T02:21:49.094 INFO:teuthology.orchestra.run.smithi143.stdout: "bdev_ioring": "false", 2023-08-28T02:21:49.094 INFO:teuthology.orchestra.run.smithi143.stdout: "bdev_ioring_hipri": "false", 2023-08-28T02:21:49.094 INFO:teuthology.orchestra.run.smithi143.stdout: "bdev_ioring_sqthread_poll": "false", 2023-08-28T02:21:49.095 INFO:teuthology.orchestra.run.smithi143.stdout: "bdev_nvme_unbind_from_kernel": "false", 2023-08-28T02:21:49.095 INFO:teuthology.orchestra.run.smithi143.stdout: "bdev_read_buffer_alignment": "4096", 2023-08-28T02:21:49.095 INFO:teuthology.orchestra.run.smithi143.stdout: "bdev_read_preallocated_huge_buffers": "", 2023-08-28T02:21:49.095 INFO:teuthology.orchestra.run.smithi143.stdout: "bdev_type": "", 2023-08-28T02:21:49.095 INFO:teuthology.orchestra.run.smithi143.stdout: "bluefs_alloc_size": "1048576", 2023-08-28T02:21:49.095 INFO:teuthology.orchestra.run.smithi143.stdout: "bluefs_allocator": "hybrid", 2023-08-28T02:21:49.095 INFO:teuthology.orchestra.run.smithi143.stdout: "bluefs_buffered_io": "true", 2023-08-28T02:21:49.095 INFO:teuthology.orchestra.run.smithi143.stdout: "bluefs_check_for_zeros": "false", 2023-08-28T02:21:49.095 INFO:teuthology.orchestra.run.smithi143.stdout: "bluefs_check_volume_selector_often": "false", 2023-08-28T02:21:49.095 INFO:teuthology.orchestra.run.smithi143.stdout: "bluefs_check_volume_selector_on_umount": "false", 2023-08-28T02:21:49.095 INFO:teuthology.orchestra.run.smithi143.stdout: "bluefs_compact_log_sync": "false", 2023-08-28T02:21:49.095 INFO:teuthology.orchestra.run.smithi143.stdout: "bluefs_failed_shared_alloc_cooldown": "600.000000", 2023-08-28T02:21:49.095 INFO:teuthology.orchestra.run.smithi143.stdout: "bluefs_log_compact_min_ratio": "5.000000", 2023-08-28T02:21:49.096 INFO:teuthology.orchestra.run.smithi143.stdout: "bluefs_log_compact_min_size": "16777216", 2023-08-28T02:21:49.096 INFO:teuthology.orchestra.run.smithi143.stdout: "bluefs_log_replay_check_allocations": "true", 2023-08-28T02:21:49.096 INFO:teuthology.orchestra.run.smithi143.stdout: "bluefs_max_log_runway": "4194304", 2023-08-28T02:21:49.096 INFO:teuthology.orchestra.run.smithi143.stdout: "bluefs_max_prefetch": "1048576", 2023-08-28T02:21:49.096 INFO:teuthology.orchestra.run.smithi143.stdout: "bluefs_min_flush_size": "524288", 2023-08-28T02:21:49.096 INFO:teuthology.orchestra.run.smithi143.stdout: "bluefs_min_log_runway": "1048576", 2023-08-28T02:21:49.096 INFO:teuthology.orchestra.run.smithi143.stdout: "bluefs_replay_recovery": "false", 2023-08-28T02:21:49.096 INFO:teuthology.orchestra.run.smithi143.stdout: "bluefs_replay_recovery_disable_compact": "false", 2023-08-28T02:21:49.096 INFO:teuthology.orchestra.run.smithi143.stdout: "bluefs_shared_alloc_size": "65536", 2023-08-28T02:21:49.096 INFO:teuthology.orchestra.run.smithi143.stdout: "bluefs_sync_write": "false", 2023-08-28T02:21:49.096 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_2q_cache_kin_ratio": "0.500000", 2023-08-28T02:21:49.096 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_2q_cache_kout_ratio": "0.500000", 2023-08-28T02:21:49.097 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_alloc_stats_dump_interval": "86400.000000", 2023-08-28T02:21:49.097 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_allocation_from_file": "true", 2023-08-28T02:21:49.097 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_allocator": "hybrid", 2023-08-28T02:21:49.097 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_avl_alloc_bf_free_pct": "4", 2023-08-28T02:21:49.097 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_avl_alloc_bf_threshold": "131072", 2023-08-28T02:21:49.097 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_avl_alloc_ff_max_search_bytes": "16777216", 2023-08-28T02:21:49.097 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_avl_alloc_ff_max_search_count": "100", 2023-08-28T02:21:49.097 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_bitmapallocator_blocks_per_zone": "1024", 2023-08-28T02:21:49.097 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_bitmapallocator_span_size": "1024", 2023-08-28T02:21:49.097 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_blobid_prealloc": "10240", 2023-08-28T02:21:49.097 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_block_create": "true", 2023-08-28T02:21:49.097 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_block_db_create": "false", 2023-08-28T02:21:49.098 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_block_db_path": "", 2023-08-28T02:21:49.098 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_block_db_size": "0", 2023-08-28T02:21:49.098 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_block_path": "", 2023-08-28T02:21:49.098 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_block_preallocate_file": "false", 2023-08-28T02:21:49.098 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_block_size": "107374182400", 2023-08-28T02:21:49.098 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_block_wal_create": "false", 2023-08-28T02:21:49.098 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_block_wal_path": "", 2023-08-28T02:21:49.098 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_block_wal_size": "100663296", 2023-08-28T02:21:49.098 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_bluefs": "true", 2023-08-28T02:21:49.098 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_bluefs_alloc_failure_dump_interval": "0.000000", 2023-08-28T02:21:49.098 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_bluefs_env_mirror": "false", 2023-08-28T02:21:49.098 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_bluefs_max_free": "10737418240", 2023-08-28T02:21:49.098 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_cache_age_bin_interval": "1.000000", 2023-08-28T02:21:49.099 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_cache_age_bins_data": "1 2 6 24 120 720 0 0 0 0", 2023-08-28T02:21:49.099 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_cache_age_bins_kv": "1 2 6 24 120 720 0 0 0 0", 2023-08-28T02:21:49.099 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_cache_age_bins_kv_onode": "0 0 0 0 0 0 0 0 0 720", 2023-08-28T02:21:49.099 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_cache_age_bins_meta": "1 2 6 24 120 720 0 0 0 0", 2023-08-28T02:21:49.099 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_cache_autotune": "true", 2023-08-28T02:21:49.099 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_cache_autotune_interval": "5.000000", 2023-08-28T02:21:49.099 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_cache_kv_onode_ratio": "0.040000", 2023-08-28T02:21:49.099 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_cache_kv_ratio": "0.450000", 2023-08-28T02:21:49.099 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_cache_meta_ratio": "0.450000", 2023-08-28T02:21:49.099 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_cache_size": "0", 2023-08-28T02:21:49.099 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_cache_size_hdd": "1073741824", 2023-08-28T02:21:49.099 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_cache_size_ssd": "3221225472", 2023-08-28T02:21:49.100 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_cache_trim_interval": "0.050000", 2023-08-28T02:21:49.100 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_cache_trim_max_skip_pinned": "1000", 2023-08-28T02:21:49.100 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_cache_type": "2q", 2023-08-28T02:21:49.100 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_cleaner_sleep_interval": "5.000000", 2023-08-28T02:21:49.100 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_clone_cow": "true", 2023-08-28T02:21:49.100 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_compression_algorithm": "snappy", 2023-08-28T02:21:49.100 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_compression_max_blob_size": "0", 2023-08-28T02:21:49.100 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_compression_max_blob_size_hdd": "65536", 2023-08-28T02:21:49.100 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_compression_max_blob_size_ssd": "65536", 2023-08-28T02:21:49.100 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_compression_min_blob_size": "0", 2023-08-28T02:21:49.100 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_compression_min_blob_size_hdd": "8192", 2023-08-28T02:21:49.100 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_compression_min_blob_size_ssd": "65536", 2023-08-28T02:21:49.100 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_compression_mode": "none", 2023-08-28T02:21:49.101 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_compression_required_ratio": "0.875000", 2023-08-28T02:21:49.101 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_csum_type": "crc32c", 2023-08-28T02:21:49.101 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_debug_enforce_settings": "default", 2023-08-28T02:21:49.101 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_debug_freelist": "false", 2023-08-28T02:21:49.101 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_debug_fsck_abort": "false", 2023-08-28T02:21:49.101 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_debug_inject_allocation_from_file_failure": "0.000000", 2023-08-28T02:21:49.101 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_debug_inject_bug21040": "false", 2023-08-28T02:21:49.101 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_debug_inject_csum_err_probability": "0.000000", 2023-08-28T02:21:49.101 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_debug_inject_read_err": "false", 2023-08-28T02:21:49.101 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_debug_legacy_omap": "false", 2023-08-28T02:21:49.101 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_debug_no_reuse_blocks": "false", 2023-08-28T02:21:49.101 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_debug_omit_block_device_write": "false", 2023-08-28T02:21:49.102 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_debug_omit_kv_commit": "false", 2023-08-28T02:21:49.102 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_debug_permit_any_bdev_label": "false", 2023-08-28T02:21:49.102 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_debug_prefill": "0.000000", 2023-08-28T02:21:49.102 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_debug_prefragment_max": "1048576", 2023-08-28T02:21:49.102 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_debug_random_read_err": "0.000000", 2023-08-28T02:21:49.102 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_debug_randomize_serial_transaction": "0", 2023-08-28T02:21:49.102 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_debug_small_allocations": "0", 2023-08-28T02:21:49.102 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_debug_too_many_blobs_threshold": "24576", 2023-08-28T02:21:49.102 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_default_buffered_read": "true", 2023-08-28T02:21:49.102 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_default_buffered_write": "false", 2023-08-28T02:21:49.103 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_deferred_batch_ops": "0", 2023-08-28T02:21:49.103 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_deferred_batch_ops_hdd": "64", 2023-08-28T02:21:49.103 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_deferred_batch_ops_ssd": "16", 2023-08-28T02:21:49.103 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_extent_map_inline_shard_prealloc_size": "256", 2023-08-28T02:21:49.103 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_extent_map_shard_max_size": "1200", 2023-08-28T02:21:49.103 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_extent_map_shard_min_size": "150", 2023-08-28T02:21:49.103 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_extent_map_shard_target_size": "500", 2023-08-28T02:21:49.103 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_extent_map_shard_target_size_slop": "0.200000", 2023-08-28T02:21:49.103 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_fail_eio": "false", 2023-08-28T02:21:49.103 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_freelist_blocks_per_key": "128", 2023-08-28T02:21:49.103 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_fsck_error_on_no_per_pg_omap": "false", 2023-08-28T02:21:49.103 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_fsck_error_on_no_per_pool_omap": "false", 2023-08-28T02:21:49.104 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_fsck_error_on_no_per_pool_stats": "false", 2023-08-28T02:21:49.104 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_fsck_on_mkfs": "true", 2023-08-28T02:21:49.104 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_fsck_on_mkfs_deep": "false", 2023-08-28T02:21:49.104 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_fsck_on_mount": "false", 2023-08-28T02:21:49.104 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_fsck_on_mount_deep": "false", 2023-08-28T02:21:49.104 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_fsck_on_umount": "false", 2023-08-28T02:21:49.104 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_fsck_on_umount_deep": "false", 2023-08-28T02:21:49.104 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_fsck_quick_fix_on_mount": "false", 2023-08-28T02:21:49.104 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_fsck_quick_fix_threads": "2", 2023-08-28T02:21:49.104 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_fsck_read_bytes_cap": "67108864", 2023-08-28T02:21:49.104 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_fsck_shared_blob_tracker_size": "0.031250", 2023-08-28T02:21:49.104 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_gc_enable_blob_threshold": "0", 2023-08-28T02:21:49.104 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_gc_enable_total_threshold": "0", 2023-08-28T02:21:49.105 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_hybrid_alloc_mem_cap": "67108864", 2023-08-28T02:21:49.105 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_ignore_data_csum": "false", 2023-08-28T02:21:49.105 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_kv_sync_util_logging_s": "10.000000", 2023-08-28T02:21:49.105 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_kvbackend": "rocksdb", 2023-08-28T02:21:49.105 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_log_collection_list_age": "60.000000", 2023-08-28T02:21:49.105 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_log_omap_iterator_age": "5.000000", 2023-08-28T02:21:49.105 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_log_op_age": "5.000000", 2023-08-28T02:21:49.105 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_max_alloc_size": "0", 2023-08-28T02:21:49.105 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_max_blob_size": "0", 2023-08-28T02:21:49.105 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_max_blob_size_hdd": "65536", 2023-08-28T02:21:49.105 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_max_blob_size_ssd": "65536", 2023-08-28T02:21:49.105 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_max_defer_interval": "3.000000", 2023-08-28T02:21:49.105 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_max_deferred_txc": "32", 2023-08-28T02:21:49.106 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_min_alloc_size": "0", 2023-08-28T02:21:49.106 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_min_alloc_size_hdd": "4096", 2023-08-28T02:21:49.106 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_min_alloc_size_ssd": "4096", 2023-08-28T02:21:49.106 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_nid_prealloc": "1024", 2023-08-28T02:21:49.106 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_prefer_deferred_size": "0", 2023-08-28T02:21:49.106 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_prefer_deferred_size_hdd": "65536", 2023-08-28T02:21:49.106 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_prefer_deferred_size_ssd": "0", 2023-08-28T02:21:49.106 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_qfsck_on_mount": "true", 2023-08-28T02:21:49.106 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_retry_disk_reads": "3", 2023-08-28T02:21:49.106 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_rocksdb_cf": "true", 2023-08-28T02:21:49.106 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_rocksdb_cfs": "m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L=min_write_buffer_number_to_merge=32 P=min_write_buffer_number_to_merge=32", 2023-08-28T02:21:49.106 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_rocksdb_options": "compression=kNoCompression,max_write_buffer_number=64,min_write_buffer_number_to_merge=6,compaction_style=kCompactionStyleLevel,write_buffer_size=16777216,max_background_jobs=4,level0_file_num_compaction_trigger=8,max_bytes_for_level_base=1073741824,max_bytes_for_level_multiplier=8,compaction_readahead_size=2MB,max_total_wal_size=1073741824,writable_file_max_buffer_size=0", 2023-08-28T02:21:49.107 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_rocksdb_options_annex": "", 2023-08-28T02:21:49.107 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_spdk_coremask": "0x1", 2023-08-28T02:21:49.107 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_spdk_io_sleep": "5", 2023-08-28T02:21:49.107 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_spdk_max_io_completion": "0", 2023-08-28T02:21:49.107 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_spdk_mem": "512", 2023-08-28T02:21:49.107 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_sync_submit_transaction": "false", 2023-08-28T02:21:49.107 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_throttle_bytes": "67108864", 2023-08-28T02:21:49.107 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_throttle_cost_per_io": "0", 2023-08-28T02:21:49.107 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_throttle_cost_per_io_hdd": "670000", 2023-08-28T02:21:49.107 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_throttle_cost_per_io_ssd": "4000", 2023-08-28T02:21:49.107 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_throttle_deferred_bytes": "134217728", 2023-08-28T02:21:49.107 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_throttle_trace_rate": "0.000000", 2023-08-28T02:21:49.108 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_tracing": "false", 2023-08-28T02:21:49.108 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_use_optimal_io_size_for_min_alloc_size": "false", 2023-08-28T02:21:49.108 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_volume_selection_policy": "use_some_extra", 2023-08-28T02:21:49.108 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_volume_selection_reserved": "0", 2023-08-28T02:21:49.108 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_volume_selection_reserved_factor": "2.000000", 2023-08-28T02:21:49.108 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_warn_on_bluefs_spillover": "true", 2023-08-28T02:21:49.108 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_warn_on_legacy_statfs": "true", 2023-08-28T02:21:49.108 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_warn_on_no_per_pg_omap": "false", 2023-08-28T02:21:49.108 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_warn_on_no_per_pool_omap": "true", 2023-08-28T02:21:49.108 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_warn_on_spurious_read_errors": "true", 2023-08-28T02:21:49.108 INFO:teuthology.orchestra.run.smithi143.stdout: "bluestore_zero_block_detection": "false", 2023-08-28T02:21:49.108 INFO:teuthology.orchestra.run.smithi143.stdout: "cephadm_path": "/usr/sbin/cephadm", 2023-08-28T02:21:49.109 INFO:teuthology.orchestra.run.smithi143.stdout: "cephfs_mirror_action_update_interval": "2", 2023-08-28T02:21:49.109 INFO:teuthology.orchestra.run.smithi143.stdout: "cephfs_mirror_directory_scan_interval": "10", 2023-08-28T02:21:49.109 INFO:teuthology.orchestra.run.smithi143.stdout: "cephfs_mirror_max_concurrent_directory_syncs": "3", 2023-08-28T02:21:49.109 INFO:teuthology.orchestra.run.smithi143.stdout: "cephfs_mirror_max_consecutive_failures_per_directory": "10", 2023-08-28T02:21:49.109 INFO:teuthology.orchestra.run.smithi143.stdout: "cephfs_mirror_max_snapshot_sync_per_cycle": "3", 2023-08-28T02:21:49.109 INFO:teuthology.orchestra.run.smithi143.stdout: "cephfs_mirror_mount_timeout": "10", 2023-08-28T02:21:49.109 INFO:teuthology.orchestra.run.smithi143.stdout: "cephfs_mirror_restart_mirror_on_blocklist_interval": "30", 2023-08-28T02:21:49.109 INFO:teuthology.orchestra.run.smithi143.stdout: "cephfs_mirror_restart_mirror_on_failure_interval": "20", 2023-08-28T02:21:49.109 INFO:teuthology.orchestra.run.smithi143.stdout: "cephfs_mirror_retry_failed_directories_interval": "60", 2023-08-28T02:21:49.109 INFO:teuthology.orchestra.run.smithi143.stdout: "cephsqlite_blocklist_dead_locker": "true", 2023-08-28T02:21:49.109 INFO:teuthology.orchestra.run.smithi143.stdout: "cephsqlite_lock_renewal_interval": "2000", 2023-08-28T02:21:49.109 INFO:teuthology.orchestra.run.smithi143.stdout: "cephsqlite_lock_renewal_timeout": "30000", 2023-08-28T02:21:49.109 INFO:teuthology.orchestra.run.smithi143.stdout: "cephx_cluster_require_signatures": "false", 2023-08-28T02:21:49.110 INFO:teuthology.orchestra.run.smithi143.stdout: "cephx_cluster_require_version": "2", 2023-08-28T02:21:49.110 INFO:teuthology.orchestra.run.smithi143.stdout: "cephx_require_signatures": "false", 2023-08-28T02:21:49.110 INFO:teuthology.orchestra.run.smithi143.stdout: "cephx_require_version": "2", 2023-08-28T02:21:49.110 INFO:teuthology.orchestra.run.smithi143.stdout: "cephx_service_require_signatures": "false", 2023-08-28T02:21:49.110 INFO:teuthology.orchestra.run.smithi143.stdout: "cephx_service_require_version": "2", 2023-08-28T02:21:49.110 INFO:teuthology.orchestra.run.smithi143.stdout: "cephx_sign_messages": "true", 2023-08-28T02:21:49.110 INFO:teuthology.orchestra.run.smithi143.stdout: "chdir": "", 2023-08-28T02:21:49.110 INFO:teuthology.orchestra.run.smithi143.stdout: "client_acl_type": "", 2023-08-28T02:21:49.110 INFO:teuthology.orchestra.run.smithi143.stdout: "client_asio_thread_count": "2", 2023-08-28T02:21:49.110 INFO:teuthology.orchestra.run.smithi143.stdout: "client_cache_mid": "0.750000", 2023-08-28T02:21:49.110 INFO:teuthology.orchestra.run.smithi143.stdout: "client_cache_size": "16384", 2023-08-28T02:21:49.110 INFO:teuthology.orchestra.run.smithi143.stdout: "client_caps_release_delay": "5", 2023-08-28T02:21:49.111 INFO:teuthology.orchestra.run.smithi143.stdout: "client_check_pool_perm": "true", 2023-08-28T02:21:49.111 INFO:teuthology.orchestra.run.smithi143.stdout: "client_collect_and_send_global_metrics": "false", 2023-08-28T02:21:49.111 INFO:teuthology.orchestra.run.smithi143.stdout: "client_debug_force_sync_read": "false", 2023-08-28T02:21:49.111 INFO:teuthology.orchestra.run.smithi143.stdout: "client_debug_getattr_caps": "false", 2023-08-28T02:21:49.111 INFO:teuthology.orchestra.run.smithi143.stdout: "client_debug_inject_tick_delay": "0", 2023-08-28T02:21:49.111 INFO:teuthology.orchestra.run.smithi143.stdout: "client_die_on_failed_dentry_invalidate": "true", 2023-08-28T02:21:49.111 INFO:teuthology.orchestra.run.smithi143.stdout: "client_die_on_failed_remount": "false", 2023-08-28T02:21:49.111 INFO:teuthology.orchestra.run.smithi143.stdout: "client_dirsize_rbytes": "true", 2023-08-28T02:21:49.111 INFO:teuthology.orchestra.run.smithi143.stdout: "client_force_lazyio": "false", 2023-08-28T02:21:49.111 INFO:teuthology.orchestra.run.smithi143.stdout: "client_fs": "", 2023-08-28T02:21:49.111 INFO:teuthology.orchestra.run.smithi143.stdout: "client_inject_fixed_oldest_tid": "false", 2023-08-28T02:21:49.111 INFO:teuthology.orchestra.run.smithi143.stdout: "client_inject_release_failure": "false", 2023-08-28T02:21:49.112 INFO:teuthology.orchestra.run.smithi143.stdout: "client_max_inline_size": "4096", 2023-08-28T02:21:49.112 INFO:teuthology.orchestra.run.smithi143.stdout: "client_max_retries_on_remount_failure": "5", 2023-08-28T02:21:49.112 INFO:teuthology.orchestra.run.smithi143.stdout: "client_mds_namespace": "", 2023-08-28T02:21:49.112 INFO:teuthology.orchestra.run.smithi143.stdout: "client_metadata": "", 2023-08-28T02:21:49.112 INFO:teuthology.orchestra.run.smithi143.stdout: "client_mount_gid": "-1", 2023-08-28T02:21:49.112 INFO:teuthology.orchestra.run.smithi143.stdout: "client_mount_timeout": "300", 2023-08-28T02:21:49.112 INFO:teuthology.orchestra.run.smithi143.stdout: "client_mount_uid": "-1", 2023-08-28T02:21:49.112 INFO:teuthology.orchestra.run.smithi143.stdout: "client_mountpoint": "/", 2023-08-28T02:21:49.112 INFO:teuthology.orchestra.run.smithi143.stdout: "client_notify_timeout": "10", 2023-08-28T02:21:49.112 INFO:teuthology.orchestra.run.smithi143.stdout: "client_oc": "true", 2023-08-28T02:21:49.112 INFO:teuthology.orchestra.run.smithi143.stdout: "client_oc_max_dirty": "104857600", 2023-08-28T02:21:49.112 INFO:teuthology.orchestra.run.smithi143.stdout: "client_oc_max_dirty_age": "5.000000", 2023-08-28T02:21:49.112 INFO:teuthology.orchestra.run.smithi143.stdout: "client_oc_max_objects": "1000", 2023-08-28T02:21:49.113 INFO:teuthology.orchestra.run.smithi143.stdout: "client_oc_size": "209715200", 2023-08-28T02:21:49.113 INFO:teuthology.orchestra.run.smithi143.stdout: "client_oc_target_dirty": "8388608", 2023-08-28T02:21:49.113 INFO:teuthology.orchestra.run.smithi143.stdout: "client_permissions": "true", 2023-08-28T02:21:49.113 INFO:teuthology.orchestra.run.smithi143.stdout: "client_quota": "true", 2023-08-28T02:21:49.113 INFO:teuthology.orchestra.run.smithi143.stdout: "client_quota_df": "true", 2023-08-28T02:21:49.113 INFO:teuthology.orchestra.run.smithi143.stdout: "client_readahead_max_bytes": "0", 2023-08-28T02:21:49.113 INFO:teuthology.orchestra.run.smithi143.stdout: "client_readahead_max_periods": "4", 2023-08-28T02:21:49.113 INFO:teuthology.orchestra.run.smithi143.stdout: "client_readahead_min": "131072", 2023-08-28T02:21:49.113 INFO:teuthology.orchestra.run.smithi143.stdout: "client_reconnect_stale": "false", 2023-08-28T02:21:49.113 INFO:teuthology.orchestra.run.smithi143.stdout: "client_shutdown_timeout": "30", 2023-08-28T02:21:49.113 INFO:teuthology.orchestra.run.smithi143.stdout: "client_snapdir": ".snap", 2023-08-28T02:21:49.113 INFO:teuthology.orchestra.run.smithi143.stdout: "client_tick_interval": "1", 2023-08-28T02:21:49.114 INFO:teuthology.orchestra.run.smithi143.stdout: "client_trace": "", 2023-08-28T02:21:49.114 INFO:teuthology.orchestra.run.smithi143.stdout: "client_try_dentry_invalidate": "false", 2023-08-28T02:21:49.114 INFO:teuthology.orchestra.run.smithi143.stdout: "client_use_faked_inos": "false", 2023-08-28T02:21:49.114 INFO:teuthology.orchestra.run.smithi143.stdout: "client_use_random_mds": "false", 2023-08-28T02:21:49.114 INFO:teuthology.orchestra.run.smithi143.stdout: "clog_to_graylog": "false", 2023-08-28T02:21:49.114 INFO:teuthology.orchestra.run.smithi143.stdout: "clog_to_graylog_host": "127.0.0.1", 2023-08-28T02:21:49.114 INFO:teuthology.orchestra.run.smithi143.stdout: "clog_to_graylog_port": "12201", 2023-08-28T02:21:49.114 INFO:teuthology.orchestra.run.smithi143.stdout: "clog_to_monitors": "default=true", 2023-08-28T02:21:49.114 INFO:teuthology.orchestra.run.smithi143.stdout: "clog_to_syslog": "false", 2023-08-28T02:21:49.114 INFO:teuthology.orchestra.run.smithi143.stdout: "clog_to_syslog_facility": "default=daemon audit=local0", 2023-08-28T02:21:49.114 INFO:teuthology.orchestra.run.smithi143.stdout: "clog_to_syslog_level": "info", 2023-08-28T02:21:49.114 INFO:teuthology.orchestra.run.smithi143.stdout: "cluster_addr": "-", 2023-08-28T02:21:49.115 INFO:teuthology.orchestra.run.smithi143.stdout: "cluster_network": "", 2023-08-28T02:21:49.115 INFO:teuthology.orchestra.run.smithi143.stdout: "cluster_network_interface": "", 2023-08-28T02:21:49.115 INFO:teuthology.orchestra.run.smithi143.stdout: "compressor_zlib_isal": "false", 2023-08-28T02:21:49.115 INFO:teuthology.orchestra.run.smithi143.stdout: "compressor_zlib_level": "5", 2023-08-28T02:21:49.115 INFO:teuthology.orchestra.run.smithi143.stdout: "compressor_zlib_winsize": "-15", 2023-08-28T02:21:49.115 INFO:teuthology.orchestra.run.smithi143.stdout: "compressor_zstd_level": "1", 2023-08-28T02:21:49.115 INFO:teuthology.orchestra.run.smithi143.stdout: "container_image": "quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8", 2023-08-28T02:21:49.115 INFO:teuthology.orchestra.run.smithi143.stdout: "crash_dir": "/var/lib/ceph/crash", 2023-08-28T02:21:49.115 INFO:teuthology.orchestra.run.smithi143.stdout: "crimson_alien_op_num_threads": "6", 2023-08-28T02:21:49.115 INFO:teuthology.orchestra.run.smithi143.stdout: "crimson_alien_thread_cpu_cores": "", 2023-08-28T02:21:49.115 INFO:teuthology.orchestra.run.smithi143.stdout: "crimson_osd_obc_lru_size": "10", 2023-08-28T02:21:49.115 INFO:teuthology.orchestra.run.smithi143.stdout: "crimson_osd_scheduler_concurrency": "0", 2023-08-28T02:21:49.116 INFO:teuthology.orchestra.run.smithi143.stdout: "crimson_seastar_smp": "1", 2023-08-28T02:21:49.116 INFO:teuthology.orchestra.run.smithi143.stdout: "crush_location": "", 2023-08-28T02:21:49.116 INFO:teuthology.orchestra.run.smithi143.stdout: "crush_location_hook": "", 2023-08-28T02:21:49.116 INFO:teuthology.orchestra.run.smithi143.stdout: "crush_location_hook_timeout": "10", 2023-08-28T02:21:49.116 INFO:teuthology.orchestra.run.smithi143.stdout: "daemonize": "false", 2023-08-28T02:21:49.116 INFO:teuthology.orchestra.run.smithi143.stdout: "daos_pool": "tank", 2023-08-28T02:21:49.116 INFO:teuthology.orchestra.run.smithi143.stdout: "dbstore_config_uri": "file:/var/lib/ceph/radosgw/dbstore-config.db", 2023-08-28T02:21:49.116 INFO:teuthology.orchestra.run.smithi143.stdout: "dbstore_db_dir": "/var/lib/ceph/radosgw", 2023-08-28T02:21:49.116 INFO:teuthology.orchestra.run.smithi143.stdout: "dbstore_db_name_prefix": "dbstore", 2023-08-28T02:21:49.116 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_alienstore": "0/5", 2023-08-28T02:21:49.117 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_allow_any_pool_priority": "false", 2023-08-28T02:21:49.117 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_asok": "1/5", 2023-08-28T02:21:49.117 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_asok_assert_abort": "false", 2023-08-28T02:21:49.117 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_asserts_on_shutdown": "true", 2023-08-28T02:21:49.117 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_auth": "1/5", 2023-08-28T02:21:49.117 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_bdev": "1/3", 2023-08-28T02:21:49.117 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_bluefs": "1/5", 2023-08-28T02:21:49.117 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_bluestore": "1/5", 2023-08-28T02:21:49.117 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_buffer": "0/1", 2023-08-28T02:21:49.117 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_ceph_exporter": "1/5", 2023-08-28T02:21:49.117 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_cephfs_mirror": "0/5", 2023-08-28T02:21:49.118 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_cephsqlite": "0/5", 2023-08-28T02:21:49.118 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_client": "0/5", 2023-08-28T02:21:49.118 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_compressor": "1/5", 2023-08-28T02:21:49.118 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_context": "0/1", 2023-08-28T02:21:49.118 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_crush": "1/1", 2023-08-28T02:21:49.118 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_crypto": "1/5", 2023-08-28T02:21:49.118 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_cyanstore": "0/5", 2023-08-28T02:21:49.118 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_deliberately_leak_memory": "false", 2023-08-28T02:21:49.118 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_disable_randomized_ping": "false", 2023-08-28T02:21:49.118 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_dpdk": "1/5", 2023-08-28T02:21:49.118 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_eventtrace": "1/5", 2023-08-28T02:21:49.118 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_filer": "0/1", 2023-08-28T02:21:49.118 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_filestore": "1/3", 2023-08-28T02:21:49.119 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_finisher": "1/1", 2023-08-28T02:21:49.119 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_fuse": "1/5", 2023-08-28T02:21:49.119 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_heartbeat_testing_span": "0", 2023-08-28T02:21:49.119 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_heartbeatmap": "1/5", 2023-08-28T02:21:49.119 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_immutable_obj_cache": "0/5", 2023-08-28T02:21:49.119 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_javaclient": "1/5", 2023-08-28T02:21:49.119 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_journal": "1/3", 2023-08-28T02:21:49.119 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_journaler": "0/5", 2023-08-28T02:21:49.119 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_kstore": "1/5", 2023-08-28T02:21:49.119 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_leveldb": "4/5", 2023-08-28T02:21:49.119 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_lockdep": "0/1", 2023-08-28T02:21:49.119 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_mclock": "1/5", 2023-08-28T02:21:49.120 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_mds": "1/5", 2023-08-28T02:21:49.120 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_mds_balancer": "1/5", 2023-08-28T02:21:49.120 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_mds_locker": "1/5", 2023-08-28T02:21:49.120 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_mds_log": "1/5", 2023-08-28T02:21:49.120 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_mds_log_expire": "1/5", 2023-08-28T02:21:49.120 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_mds_migrator": "1/5", 2023-08-28T02:21:49.120 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_memstore": "1/5", 2023-08-28T02:21:49.120 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_mgr": "2/5", 2023-08-28T02:21:49.120 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_mgrc": "1/5", 2023-08-28T02:21:49.120 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_mon": "20/20", 2023-08-28T02:21:49.120 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_monc": "0/10", 2023-08-28T02:21:49.120 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_ms": "1/1", 2023-08-28T02:21:49.121 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_none": "0/5", 2023-08-28T02:21:49.121 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_objclass": "0/5", 2023-08-28T02:21:49.121 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_objectcacher": "0/5", 2023-08-28T02:21:49.121 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_objecter": "0/1", 2023-08-28T02:21:49.121 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_optracker": "0/5", 2023-08-28T02:21:49.121 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_osd": "1/5", 2023-08-28T02:21:49.121 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_paxos": "20/20", 2023-08-28T02:21:49.121 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_perfcounter": "1/5", 2023-08-28T02:21:49.121 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_prioritycache": "1/5", 2023-08-28T02:21:49.121 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_rados": "0/5", 2023-08-28T02:21:49.121 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_rbd": "0/5", 2023-08-28T02:21:49.121 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_rbd_mirror": "0/5", 2023-08-28T02:21:49.122 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_rbd_pwl": "0/5", 2023-08-28T02:21:49.122 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_rbd_replay": "0/5", 2023-08-28T02:21:49.122 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_refs": "0/0", 2023-08-28T02:21:49.122 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_reserver": "1/1", 2023-08-28T02:21:49.122 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_rgw": "1/5", 2023-08-28T02:21:49.122 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_rgw_access": "1/5", 2023-08-28T02:21:49.122 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_rgw_datacache": "1/5", 2023-08-28T02:21:49.122 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_rgw_dbstore": "1/5", 2023-08-28T02:21:49.122 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_rgw_flight": "1/5", 2023-08-28T02:21:49.122 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_rgw_sync": "1/5", 2023-08-28T02:21:49.122 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_rocksdb": "4/5", 2023-08-28T02:21:49.122 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_seastore": "0/5", 2023-08-28T02:21:49.122 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_seastore_backref": "0/5", 2023-08-28T02:21:49.123 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_seastore_cache": "0/5", 2023-08-28T02:21:49.123 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_seastore_cleaner": "0/5", 2023-08-28T02:21:49.123 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_seastore_device": "0/5", 2023-08-28T02:21:49.123 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_seastore_epm": "0/5", 2023-08-28T02:21:49.123 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_seastore_fixedkv_tree": "0/5", 2023-08-28T02:21:49.123 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_seastore_journal": "0/5", 2023-08-28T02:21:49.123 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_seastore_lba": "0/5", 2023-08-28T02:21:49.123 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_seastore_odata": "0/5", 2023-08-28T02:21:49.123 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_seastore_omap": "0/5", 2023-08-28T02:21:49.123 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_seastore_onode": "0/5", 2023-08-28T02:21:49.123 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_seastore_t": "0/5", 2023-08-28T02:21:49.124 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_seastore_tm": "0/5", 2023-08-28T02:21:49.124 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_striper": "0/1", 2023-08-28T02:21:49.124 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_test": "0/5", 2023-08-28T02:21:49.124 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_throttle": "1/1", 2023-08-28T02:21:49.124 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_timer": "0/1", 2023-08-28T02:21:49.124 INFO:teuthology.orchestra.run.smithi143.stdout: "debug_tp": "0/5", 2023-08-28T02:21:49.124 INFO:teuthology.orchestra.run.smithi143.stdout: "device_failure_prediction_mode": "none", 2023-08-28T02:21:49.124 INFO:teuthology.orchestra.run.smithi143.stdout: "enable_experimental_unrecoverable_data_corrupting_features": "", 2023-08-28T02:21:49.124 INFO:teuthology.orchestra.run.smithi143.stdout: "erasure_code_dir": "/usr/lib64/ceph/erasure-code", 2023-08-28T02:21:49.124 INFO:teuthology.orchestra.run.smithi143.stdout: "err_to_graylog": "false", 2023-08-28T02:21:49.125 INFO:teuthology.orchestra.run.smithi143.stdout: "err_to_journald": "false", 2023-08-28T02:21:49.125 INFO:teuthology.orchestra.run.smithi143.stdout: "err_to_stderr": "true", 2023-08-28T02:21:49.125 INFO:teuthology.orchestra.run.smithi143.stdout: "err_to_syslog": "false", 2023-08-28T02:21:49.125 INFO:teuthology.orchestra.run.smithi143.stdout: "event_tracing": "false", 2023-08-28T02:21:49.125 INFO:teuthology.orchestra.run.smithi143.stdout: "exporter_addr": "0.0.0.0", 2023-08-28T02:21:49.125 INFO:teuthology.orchestra.run.smithi143.stdout: "exporter_http_port": "9926", 2023-08-28T02:21:49.125 INFO:teuthology.orchestra.run.smithi143.stdout: "exporter_prio_limit": "5", 2023-08-28T02:21:49.125 INFO:teuthology.orchestra.run.smithi143.stdout: "exporter_sock_dir": "/var/run/ceph/", 2023-08-28T02:21:49.125 INFO:teuthology.orchestra.run.smithi143.stdout: "exporter_sort_metrics": "true", 2023-08-28T02:21:49.125 INFO:teuthology.orchestra.run.smithi143.stdout: "exporter_stats_period": "5", 2023-08-28T02:21:49.125 INFO:teuthology.orchestra.run.smithi143.stdout: "fake_statfs_for_testing": "0", 2023-08-28T02:21:49.126 INFO:teuthology.orchestra.run.smithi143.stdout: "fatal_signal_handlers": "true", 2023-08-28T02:21:49.126 INFO:teuthology.orchestra.run.smithi143.stdout: "filer_max_purge_ops": "10", 2023-08-28T02:21:49.126 INFO:teuthology.orchestra.run.smithi143.stdout: "filer_max_truncate_ops": "128", 2023-08-28T02:21:49.126 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_apply_finisher_threads": "1", 2023-08-28T02:21:49.126 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_blackhole": "false", 2023-08-28T02:21:49.126 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_btrfs_clone_range": "true", 2023-08-28T02:21:49.126 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_btrfs_snap": "true", 2023-08-28T02:21:49.126 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_caller_concurrency": "10", 2023-08-28T02:21:49.126 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_collect_device_partition_information": "true", 2023-08-28T02:21:49.126 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_commit_timeout": "600.000000", 2023-08-28T02:21:49.126 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_debug_inject_read_err": "false", 2023-08-28T02:21:49.126 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_debug_omap_check": "false", 2023-08-28T02:21:49.127 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_debug_random_read_err": "0.000000", 2023-08-28T02:21:49.127 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_debug_verify_split": "false", 2023-08-28T02:21:49.127 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_dump_file": "", 2023-08-28T02:21:49.127 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_expected_throughput_bytes": "209715200.000000", 2023-08-28T02:21:49.127 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_expected_throughput_ops": "200.000000", 2023-08-28T02:21:49.127 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_fadvise": "true", 2023-08-28T02:21:49.127 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_fail_eio": "true", 2023-08-28T02:21:49.127 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_fd_cache_shards": "16", 2023-08-28T02:21:49.127 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_fd_cache_size": "128", 2023-08-28T02:21:49.127 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_fiemap": "false", 2023-08-28T02:21:49.127 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_fiemap_threshold": "4096", 2023-08-28T02:21:49.127 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_fsync_flushes_journal_data": "false", 2023-08-28T02:21:49.128 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_index_retry_probability": "0.000000", 2023-08-28T02:21:49.128 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_inject_stall": "0", 2023-08-28T02:21:49.128 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_journal_parallel": "false", 2023-08-28T02:21:49.128 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_journal_trailing": "false", 2023-08-28T02:21:49.128 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_journal_writeahead": "false", 2023-08-28T02:21:49.128 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_kill_at": "0", 2023-08-28T02:21:49.128 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_max_alloc_hint_size": "1048576", 2023-08-28T02:21:49.128 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_max_inline_xattr_size": "0", 2023-08-28T02:21:49.128 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_max_inline_xattr_size_btrfs": "2048", 2023-08-28T02:21:49.128 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_max_inline_xattr_size_other": "512", 2023-08-28T02:21:49.128 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_max_inline_xattr_size_xfs": "65536", 2023-08-28T02:21:49.128 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_max_inline_xattrs": "0", 2023-08-28T02:21:49.129 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_max_inline_xattrs_btrfs": "10", 2023-08-28T02:21:49.129 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_max_inline_xattrs_other": "2", 2023-08-28T02:21:49.129 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_max_inline_xattrs_xfs": "10", 2023-08-28T02:21:49.129 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_max_sync_interval": "5.000000", 2023-08-28T02:21:49.129 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_max_xattr_value_size": "0", 2023-08-28T02:21:49.129 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_max_xattr_value_size_btrfs": "65536", 2023-08-28T02:21:49.129 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_max_xattr_value_size_other": "1024", 2023-08-28T02:21:49.129 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_max_xattr_value_size_xfs": "65536", 2023-08-28T02:21:49.129 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_merge_threshold": "-10", 2023-08-28T02:21:49.129 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_min_sync_interval": "0.010000", 2023-08-28T02:21:49.129 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_odsync_write": "false", 2023-08-28T02:21:49.129 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_omap_backend": "rocksdb", 2023-08-28T02:21:49.130 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_omap_backend_path": "", 2023-08-28T02:21:49.130 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_omap_header_cache_size": "1024", 2023-08-28T02:21:49.130 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_ondisk_finisher_threads": "1", 2023-08-28T02:21:49.130 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_op_thread_suicide_timeout": "180", 2023-08-28T02:21:49.130 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_op_thread_timeout": "60", 2023-08-28T02:21:49.130 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_op_threads": "2", 2023-08-28T02:21:49.130 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_punch_hole": "false", 2023-08-28T02:21:49.130 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_queue_high_delay_multiple": "0.000000", 2023-08-28T02:21:49.130 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_queue_high_delay_multiple_bytes": "0.000000", 2023-08-28T02:21:49.130 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_queue_high_delay_multiple_ops": "0.000000", 2023-08-28T02:21:49.130 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_queue_high_threshhold": "0.900000", 2023-08-28T02:21:49.130 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_queue_low_threshhold": "0.300000", 2023-08-28T02:21:49.131 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_queue_max_bytes": "104857600", 2023-08-28T02:21:49.131 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_queue_max_delay_multiple": "0.000000", 2023-08-28T02:21:49.131 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_queue_max_delay_multiple_bytes": "0.000000", 2023-08-28T02:21:49.131 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_queue_max_delay_multiple_ops": "0.000000", 2023-08-28T02:21:49.131 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_queue_max_ops": "50", 2023-08-28T02:21:49.131 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_rocksdb_options": "max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression", 2023-08-28T02:21:49.131 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_seek_data_hole": "false", 2023-08-28T02:21:49.131 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_sloppy_crc": "false", 2023-08-28T02:21:49.131 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_sloppy_crc_block_size": "65536", 2023-08-28T02:21:49.131 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_splice": "false", 2023-08-28T02:21:49.131 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_split_multiple": "2", 2023-08-28T02:21:49.132 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_split_rand_factor": "20", 2023-08-28T02:21:49.132 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_update_to": "1000", 2023-08-28T02:21:49.132 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400", 2023-08-28T02:21:49.132 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040", 2023-08-28T02:21:49.132 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_wbthrottle_btrfs_inodes_hard_limit": "5000", 2023-08-28T02:21:49.132 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_wbthrottle_btrfs_inodes_start_flusher": "500", 2023-08-28T02:21:49.132 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_wbthrottle_btrfs_ios_hard_limit": "5000", 2023-08-28T02:21:49.132 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_wbthrottle_btrfs_ios_start_flusher": "500", 2023-08-28T02:21:49.132 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_wbthrottle_enable": "true", 2023-08-28T02:21:49.132 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_wbthrottle_xfs_bytes_hard_limit": "419430400", 2023-08-28T02:21:49.132 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_wbthrottle_xfs_bytes_start_flusher": "41943040", 2023-08-28T02:21:49.132 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_wbthrottle_xfs_inodes_hard_limit": "5000", 2023-08-28T02:21:49.133 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_wbthrottle_xfs_inodes_start_flusher": "500", 2023-08-28T02:21:49.133 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_wbthrottle_xfs_ios_hard_limit": "5000", 2023-08-28T02:21:49.133 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_wbthrottle_xfs_ios_start_flusher": "500", 2023-08-28T02:21:49.133 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_xfs_extsize": "false", 2023-08-28T02:21:49.133 INFO:teuthology.orchestra.run.smithi143.stdout: "filestore_zfs_snap": "false", 2023-08-28T02:21:49.133 INFO:teuthology.orchestra.run.smithi143.stdout: "fio_dir": "/tmp/fio", 2023-08-28T02:21:49.133 INFO:teuthology.orchestra.run.smithi143.stdout: "fsid": "2fcb0bfa-4547-11ee-817c-1b7a5f1ba315", 2023-08-28T02:21:49.133 INFO:teuthology.orchestra.run.smithi143.stdout: "fuse_allow_other": "true", 2023-08-28T02:21:49.133 INFO:teuthology.orchestra.run.smithi143.stdout: "fuse_atomic_o_trunc": "true", 2023-08-28T02:21:49.133 INFO:teuthology.orchestra.run.smithi143.stdout: "fuse_big_writes": "true", 2023-08-28T02:21:49.133 INFO:teuthology.orchestra.run.smithi143.stdout: "fuse_debug": "false", 2023-08-28T02:21:49.133 INFO:teuthology.orchestra.run.smithi143.stdout: "fuse_default_permissions": "false", 2023-08-28T02:21:49.133 INFO:teuthology.orchestra.run.smithi143.stdout: "fuse_disable_pagecache": "false", 2023-08-28T02:21:49.134 INFO:teuthology.orchestra.run.smithi143.stdout: "fuse_max_write": "0", 2023-08-28T02:21:49.134 INFO:teuthology.orchestra.run.smithi143.stdout: "fuse_multithreaded": "true", 2023-08-28T02:21:49.134 INFO:teuthology.orchestra.run.smithi143.stdout: "fuse_require_active_mds": "true", 2023-08-28T02:21:49.134 INFO:teuthology.orchestra.run.smithi143.stdout: "fuse_set_user_groups": "true", 2023-08-28T02:21:49.134 INFO:teuthology.orchestra.run.smithi143.stdout: "fuse_splice_move": "true", 2023-08-28T02:21:49.134 INFO:teuthology.orchestra.run.smithi143.stdout: "fuse_splice_read": "true", 2023-08-28T02:21:49.134 INFO:teuthology.orchestra.run.smithi143.stdout: "fuse_splice_write": "true", 2023-08-28T02:21:49.134 INFO:teuthology.orchestra.run.smithi143.stdout: "fuse_syncfs_on_mksnap": "true", 2023-08-28T02:21:49.134 INFO:teuthology.orchestra.run.smithi143.stdout: "fuse_use_invalidate_cb": "true", 2023-08-28T02:21:49.134 INFO:teuthology.orchestra.run.smithi143.stdout: "gss_ktab_client_file": "/var/lib/ceph/mon.a/gss_client_mon.a.ktab", 2023-08-28T02:21:49.134 INFO:teuthology.orchestra.run.smithi143.stdout: "gss_target_name": "ceph", 2023-08-28T02:21:49.135 INFO:teuthology.orchestra.run.smithi143.stdout: "heartbeat_file": "", 2023-08-28T02:21:49.135 INFO:teuthology.orchestra.run.smithi143.stdout: "heartbeat_inject_failure": "0", 2023-08-28T02:21:49.135 INFO:teuthology.orchestra.run.smithi143.stdout: "heartbeat_interval": "5", 2023-08-28T02:21:49.135 INFO:teuthology.orchestra.run.smithi143.stdout: "host": "smithi143", 2023-08-28T02:21:49.135 INFO:teuthology.orchestra.run.smithi143.stdout: "immutable_object_cache_client_dedicated_thread_num": "2", 2023-08-28T02:21:49.135 INFO:teuthology.orchestra.run.smithi143.stdout: "immutable_object_cache_max_inflight_ops": "128", 2023-08-28T02:21:49.135 INFO:teuthology.orchestra.run.smithi143.stdout: "immutable_object_cache_max_size": "1073741824", 2023-08-28T02:21:49.135 INFO:teuthology.orchestra.run.smithi143.stdout: "immutable_object_cache_path": "/tmp/ceph_immutable_object_cache", 2023-08-28T02:21:49.135 INFO:teuthology.orchestra.run.smithi143.stdout: "immutable_object_cache_qos_bps_burst": "0", 2023-08-28T02:21:49.135 INFO:teuthology.orchestra.run.smithi143.stdout: "immutable_object_cache_qos_bps_burst_seconds": "1", 2023-08-28T02:21:49.135 INFO:teuthology.orchestra.run.smithi143.stdout: "immutable_object_cache_qos_bps_limit": "0", 2023-08-28T02:21:49.136 INFO:teuthology.orchestra.run.smithi143.stdout: "immutable_object_cache_qos_iops_burst": "0", 2023-08-28T02:21:49.136 INFO:teuthology.orchestra.run.smithi143.stdout: "immutable_object_cache_qos_iops_burst_seconds": "1", 2023-08-28T02:21:49.136 INFO:teuthology.orchestra.run.smithi143.stdout: "immutable_object_cache_qos_iops_limit": "0", 2023-08-28T02:21:49.136 INFO:teuthology.orchestra.run.smithi143.stdout: "immutable_object_cache_qos_schedule_tick_min": "50", 2023-08-28T02:21:49.136 INFO:teuthology.orchestra.run.smithi143.stdout: "immutable_object_cache_sock": "/var/run/ceph/immutable_object_cache_sock", 2023-08-28T02:21:49.136 INFO:teuthology.orchestra.run.smithi143.stdout: "immutable_object_cache_watermark": "0.900000", 2023-08-28T02:21:49.136 INFO:teuthology.orchestra.run.smithi143.stdout: "inject_early_sigterm": "false", 2023-08-28T02:21:49.136 INFO:teuthology.orchestra.run.smithi143.stdout: "jaeger_agent_port": "6799", 2023-08-28T02:21:49.136 INFO:teuthology.orchestra.run.smithi143.stdout: "jaeger_tracing_enable": "false", 2023-08-28T02:21:49.136 INFO:teuthology.orchestra.run.smithi143.stdout: "journal_aio": "true", 2023-08-28T02:21:49.136 INFO:teuthology.orchestra.run.smithi143.stdout: "journal_align_min_size": "65536", 2023-08-28T02:21:49.137 INFO:teuthology.orchestra.run.smithi143.stdout: "journal_block_align": "true", 2023-08-28T02:21:49.137 INFO:teuthology.orchestra.run.smithi143.stdout: "journal_block_size": "4096", 2023-08-28T02:21:49.137 INFO:teuthology.orchestra.run.smithi143.stdout: "journal_dio": "true", 2023-08-28T02:21:49.137 INFO:teuthology.orchestra.run.smithi143.stdout: "journal_discard": "false", 2023-08-28T02:21:49.137 INFO:teuthology.orchestra.run.smithi143.stdout: "journal_force_aio": "false", 2023-08-28T02:21:49.137 INFO:teuthology.orchestra.run.smithi143.stdout: "journal_ignore_corruption": "false", 2023-08-28T02:21:49.137 INFO:teuthology.orchestra.run.smithi143.stdout: "journal_max_write_bytes": "10485760", 2023-08-28T02:21:49.137 INFO:teuthology.orchestra.run.smithi143.stdout: "journal_max_write_entries": "100", 2023-08-28T02:21:49.137 INFO:teuthology.orchestra.run.smithi143.stdout: "journal_replay_from": "0", 2023-08-28T02:21:49.137 INFO:teuthology.orchestra.run.smithi143.stdout: "journal_throttle_high_multiple": "0.000000", 2023-08-28T02:21:49.137 INFO:teuthology.orchestra.run.smithi143.stdout: "journal_throttle_high_threshhold": "0.900000", 2023-08-28T02:21:49.137 INFO:teuthology.orchestra.run.smithi143.stdout: "journal_throttle_low_threshhold": "0.600000", 2023-08-28T02:21:49.138 INFO:teuthology.orchestra.run.smithi143.stdout: "journal_throttle_max_multiple": "0.000000", 2023-08-28T02:21:49.138 INFO:teuthology.orchestra.run.smithi143.stdout: "journal_write_header_frequency": "0", 2023-08-28T02:21:49.138 INFO:teuthology.orchestra.run.smithi143.stdout: "journal_zero_on_create": "false", 2023-08-28T02:21:49.138 INFO:teuthology.orchestra.run.smithi143.stdout: "journaler_prefetch_periods": "10", 2023-08-28T02:21:49.138 INFO:teuthology.orchestra.run.smithi143.stdout: "journaler_prezero_periods": "5", 2023-08-28T02:21:49.138 INFO:teuthology.orchestra.run.smithi143.stdout: "journaler_write_head_interval": "15", 2023-08-28T02:21:49.138 INFO:teuthology.orchestra.run.smithi143.stdout: "key": "", 2023-08-28T02:21:49.138 INFO:teuthology.orchestra.run.smithi143.stdout: "keyfile": "", 2023-08-28T02:21:49.138 INFO:teuthology.orchestra.run.smithi143.stdout: "keyring": "/var/lib/ceph/mon/ceph-a/keyring", 2023-08-28T02:21:49.138 INFO:teuthology.orchestra.run.smithi143.stdout: "kstore_backend": "rocksdb", 2023-08-28T02:21:49.138 INFO:teuthology.orchestra.run.smithi143.stdout: "kstore_default_stripe_size": "65536", 2023-08-28T02:21:49.138 INFO:teuthology.orchestra.run.smithi143.stdout: "kstore_fsck_on_mount": "false", 2023-08-28T02:21:49.139 INFO:teuthology.orchestra.run.smithi143.stdout: "kstore_fsck_on_mount_deep": "true", 2023-08-28T02:21:49.139 INFO:teuthology.orchestra.run.smithi143.stdout: "kstore_max_bytes": "67108864", 2023-08-28T02:21:49.139 INFO:teuthology.orchestra.run.smithi143.stdout: "kstore_max_ops": "512", 2023-08-28T02:21:49.139 INFO:teuthology.orchestra.run.smithi143.stdout: "kstore_nid_prealloc": "1024", 2023-08-28T02:21:49.139 INFO:teuthology.orchestra.run.smithi143.stdout: "kstore_onode_map_size": "1024", 2023-08-28T02:21:49.139 INFO:teuthology.orchestra.run.smithi143.stdout: "kstore_rocksdb_options": "compression=kNoCompression", 2023-08-28T02:21:49.139 INFO:teuthology.orchestra.run.smithi143.stdout: "kstore_sync_submit_transaction": "false", 2023-08-28T02:21:49.139 INFO:teuthology.orchestra.run.smithi143.stdout: "kstore_sync_transaction": "false", 2023-08-28T02:21:49.139 INFO:teuthology.orchestra.run.smithi143.stdout: "leveldb_block_size": "65536", 2023-08-28T02:21:49.139 INFO:teuthology.orchestra.run.smithi143.stdout: "leveldb_bloom_size": "0", 2023-08-28T02:21:49.139 INFO:teuthology.orchestra.run.smithi143.stdout: "leveldb_cache_size": "536870912", 2023-08-28T02:21:49.139 INFO:teuthology.orchestra.run.smithi143.stdout: "leveldb_compact_on_mount": "false", 2023-08-28T02:21:49.140 INFO:teuthology.orchestra.run.smithi143.stdout: "leveldb_compression": "false", 2023-08-28T02:21:49.140 INFO:teuthology.orchestra.run.smithi143.stdout: "leveldb_log": "", 2023-08-28T02:21:49.140 INFO:teuthology.orchestra.run.smithi143.stdout: "leveldb_log_to_ceph_log": "true", 2023-08-28T02:21:49.140 INFO:teuthology.orchestra.run.smithi143.stdout: "leveldb_max_open_files": "0", 2023-08-28T02:21:49.140 INFO:teuthology.orchestra.run.smithi143.stdout: "leveldb_paranoid": "false", 2023-08-28T02:21:49.140 INFO:teuthology.orchestra.run.smithi143.stdout: "leveldb_write_buffer_size": "33554432", 2023-08-28T02:21:49.140 INFO:teuthology.orchestra.run.smithi143.stdout: "librados_thread_count": "2", 2023-08-28T02:21:49.140 INFO:teuthology.orchestra.run.smithi143.stdout: "lockdep": "false", 2023-08-28T02:21:49.140 INFO:teuthology.orchestra.run.smithi143.stdout: "lockdep_force_backtrace": "false", 2023-08-28T02:21:49.140 INFO:teuthology.orchestra.run.smithi143.stdout: "log_coarse_timestamps": "true", 2023-08-28T02:21:49.140 INFO:teuthology.orchestra.run.smithi143.stdout: "log_file": "/var/log/ceph/ceph-mon.a.log", 2023-08-28T02:21:49.140 INFO:teuthology.orchestra.run.smithi143.stdout: "log_flush_on_exit": "false", 2023-08-28T02:21:49.141 INFO:teuthology.orchestra.run.smithi143.stdout: "log_graylog_host": "127.0.0.1", 2023-08-28T02:21:49.141 INFO:teuthology.orchestra.run.smithi143.stdout: "log_graylog_port": "12201", 2023-08-28T02:21:49.141 INFO:teuthology.orchestra.run.smithi143.stdout: "log_max_new": "1000", 2023-08-28T02:21:49.141 INFO:teuthology.orchestra.run.smithi143.stdout: "log_max_recent": "10000", 2023-08-28T02:21:49.141 INFO:teuthology.orchestra.run.smithi143.stdout: "log_stderr_prefix": "", 2023-08-28T02:21:49.141 INFO:teuthology.orchestra.run.smithi143.stdout: "log_stop_at_utilization": "0.970000", 2023-08-28T02:21:49.141 INFO:teuthology.orchestra.run.smithi143.stdout: "log_to_file": "true", 2023-08-28T02:21:49.141 INFO:teuthology.orchestra.run.smithi143.stdout: "log_to_graylog": "false", 2023-08-28T02:21:49.141 INFO:teuthology.orchestra.run.smithi143.stdout: "log_to_journald": "false", 2023-08-28T02:21:49.141 INFO:teuthology.orchestra.run.smithi143.stdout: "log_to_stderr": "false", 2023-08-28T02:21:49.141 INFO:teuthology.orchestra.run.smithi143.stdout: "log_to_syslog": "false", 2023-08-28T02:21:49.141 INFO:teuthology.orchestra.run.smithi143.stdout: "max_rotating_auth_attempts": "10", 2023-08-28T02:21:49.142 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_abort_on_newly_corrupt_dentry": "true", 2023-08-28T02:21:49.142 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_action_on_write_error": "1", 2023-08-28T02:21:49.142 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_alternate_name_max": "8192", 2023-08-28T02:21:49.142 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_asio_thread_count": "2", 2023-08-28T02:21:49.142 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_bal_export_pin": "true", 2023-08-28T02:21:49.142 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_bal_fragment_dirs": "true", 2023-08-28T02:21:49.142 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_bal_fragment_fast_factor": "1.500000", 2023-08-28T02:21:49.142 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_bal_fragment_interval": "5", 2023-08-28T02:21:49.142 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_bal_fragment_size_max": "100000", 2023-08-28T02:21:49.142 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_bal_idle_threshold": "0.000000", 2023-08-28T02:21:49.142 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_bal_interval": "10", 2023-08-28T02:21:49.143 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_bal_max": "-1", 2023-08-28T02:21:49.143 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_bal_max_until": "-1", 2023-08-28T02:21:49.143 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_bal_merge_size": "50", 2023-08-28T02:21:49.143 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_bal_midchunk": "0.300000", 2023-08-28T02:21:49.143 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_bal_min_rebalance": "0.100000", 2023-08-28T02:21:49.143 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_bal_min_start": "0.200000", 2023-08-28T02:21:49.143 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_bal_minchunk": "0.001000", 2023-08-28T02:21:49.143 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_bal_mode": "0", 2023-08-28T02:21:49.143 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_bal_need_max": "1.200000", 2023-08-28T02:21:49.143 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_bal_need_min": "0.800000", 2023-08-28T02:21:49.143 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_bal_replicate_threshold": "8000.000000", 2023-08-28T02:21:49.143 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_bal_sample_interval": "3.000000", 2023-08-28T02:21:49.144 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_bal_split_bits": "3", 2023-08-28T02:21:49.144 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_bal_split_rd": "25000.000000", 2023-08-28T02:21:49.144 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_bal_split_size": "10000", 2023-08-28T02:21:49.144 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_bal_split_wr": "10000.000000", 2023-08-28T02:21:49.144 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_bal_target_decay": "10.000000", 2023-08-28T02:21:49.144 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_bal_unreplicate_threshold": "0.000000", 2023-08-28T02:21:49.144 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_beacon_grace": "15.000000", 2023-08-28T02:21:49.144 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_beacon_interval": "4.000000", 2023-08-28T02:21:49.144 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_beacon_mon_down_grace": "60", 2023-08-28T02:21:49.144 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_cache_memory_limit": "4294967296", 2023-08-28T02:21:49.144 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_cache_mid": "0.700000", 2023-08-28T02:21:49.144 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_cache_release_free_interval": "10", 2023-08-28T02:21:49.145 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_cache_reservation": "0.050000", 2023-08-28T02:21:49.145 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_cache_trim_decay_rate": "1.000000", 2023-08-28T02:21:49.145 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_cache_trim_interval": "1", 2023-08-28T02:21:49.145 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_cache_trim_threshold": "262144", 2023-08-28T02:21:49.145 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_cap_acquisition_throttle_retry_request_timeout": "0.500000", 2023-08-28T02:21:49.145 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_cap_revoke_eviction_timeout": "0.000000", 2023-08-28T02:21:49.145 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_client_delegate_inos_pct": "50", 2023-08-28T02:21:49.145 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_client_prealloc_inos": "1000", 2023-08-28T02:21:49.145 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_client_writeable_range_max_inc_objs": "1024", 2023-08-28T02:21:49.145 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_connect_bootstrapping": "false", 2023-08-28T02:21:49.145 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_damage_table_max_entries": "10000", 2023-08-28T02:21:49.145 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_data": "/var/lib/ceph/mds/ceph-a", 2023-08-28T02:21:49.145 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_debug_auth_pins": "false", 2023-08-28T02:21:49.145 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_debug_frag": "false", 2023-08-28T02:21:49.146 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_debug_scatterstat": "false", 2023-08-28T02:21:49.146 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_debug_subtrees": "false", 2023-08-28T02:21:49.146 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_decay_halflife": "5.000000", 2023-08-28T02:21:49.146 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_default_dir_hash": "2", 2023-08-28T02:21:49.146 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_defer_session_stale": "true", 2023-08-28T02:21:49.146 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_deny_all_reconnect": "false", 2023-08-28T02:21:49.146 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_dir_keys_per_op": "16384", 2023-08-28T02:21:49.146 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_dir_max_commit_size": "10", 2023-08-28T02:21:49.146 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_dir_max_entries": "0", 2023-08-28T02:21:49.146 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_dir_prefetch": "true", 2023-08-28T02:21:49.146 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_dirstat_min_interval": "1.000000", 2023-08-28T02:21:49.147 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_dump_cache_after_rejoin": "false", 2023-08-28T02:21:49.147 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_dump_cache_on_map": "false", 2023-08-28T02:21:49.147 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_dump_cache_threshold_file": "0", 2023-08-28T02:21:49.147 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_dump_cache_threshold_formatter": "1073741824", 2023-08-28T02:21:49.147 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_early_reply": "true", 2023-08-28T02:21:49.147 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_enable_op_tracker": "true", 2023-08-28T02:21:49.147 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_enforce_unique_name": "true", 2023-08-28T02:21:49.147 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_export_ephemeral_distributed": "true", 2023-08-28T02:21:49.147 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_export_ephemeral_distributed_factor": "2.000000", 2023-08-28T02:21:49.148 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_export_ephemeral_random": "true", 2023-08-28T02:21:49.148 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_export_ephemeral_random_max": "0.010000", 2023-08-28T02:21:49.149 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_extraordinary_events_dump_interval": "0", 2023-08-28T02:21:49.149 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_forward_all_requests_to_auth": "false", 2023-08-28T02:21:49.149 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_freeze_tree_timeout": "30.000000", 2023-08-28T02:21:49.149 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_fscrypt_last_block_max_size": "4096", 2023-08-28T02:21:49.149 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_go_bad_corrupt_dentry": "true", 2023-08-28T02:21:49.149 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_hack_allow_loading_invalid_metadata": "false", 2023-08-28T02:21:49.150 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_health_cache_threshold": "1.500000", 2023-08-28T02:21:49.150 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_health_summarize_threshold": "10", 2023-08-28T02:21:49.150 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_heartbeat_grace": "15.000000", 2023-08-28T02:21:49.150 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_heartbeat_reset_grace": "1000", 2023-08-28T02:21:49.150 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_inject_health_dummy": "false", 2023-08-28T02:21:49.150 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_inject_journal_corrupt_dentry_first": "0.000000", 2023-08-28T02:21:49.150 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_inject_migrator_session_race": "false", 2023-08-28T02:21:49.150 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_inject_rename_corrupt_dentry_first": "0.000000", 2023-08-28T02:21:49.151 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_inject_skip_replaying_inotable": "false", 2023-08-28T02:21:49.151 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_inject_traceless_reply_probability": "0.000000", 2023-08-28T02:21:49.151 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_join_fs": "", 2023-08-28T02:21:49.151 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_journal_format": "1", 2023-08-28T02:21:49.151 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_kill_create_at": "0", 2023-08-28T02:21:49.151 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_kill_export_at": "0", 2023-08-28T02:21:49.151 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_kill_import_at": "0", 2023-08-28T02:21:49.151 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_kill_journal_at": "0", 2023-08-28T02:21:49.151 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_kill_journal_expire_at": "0", 2023-08-28T02:21:49.151 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_kill_journal_replay_at": "0", 2023-08-28T02:21:49.151 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_kill_link_at": "0", 2023-08-28T02:21:49.151 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_kill_mdstable_at": "0", 2023-08-28T02:21:49.152 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_kill_openc_at": "0", 2023-08-28T02:21:49.152 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_kill_rename_at": "0", 2023-08-28T02:21:49.152 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_kill_skip_replaying_inotable": "false", 2023-08-28T02:21:49.152 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_log_events_per_segment": "1024", 2023-08-28T02:21:49.152 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_log_max_events": "-1", 2023-08-28T02:21:49.152 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_log_max_segments": "128", 2023-08-28T02:21:49.152 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_log_pause": "false", 2023-08-28T02:21:49.152 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_log_segment_size": "0", 2023-08-28T02:21:49.152 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_log_skip_corrupt_events": "false", 2023-08-28T02:21:49.152 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_log_warn_factor": "2.000000", 2023-08-28T02:21:49.152 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_max_caps_per_client": "1048576", 2023-08-28T02:21:49.153 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_max_completed_flushes": "100000", 2023-08-28T02:21:49.153 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_max_completed_requests": "100000", 2023-08-28T02:21:49.153 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_max_export_size": "20971520", 2023-08-28T02:21:49.153 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_max_file_recover": "32", 2023-08-28T02:21:49.153 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_max_purge_files": "64", 2023-08-28T02:21:49.153 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_max_purge_ops": "8192", 2023-08-28T02:21:49.153 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_max_purge_ops_per_pg": "0.500000", 2023-08-28T02:21:49.153 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_max_scrub_ops_in_progress": "5", 2023-08-28T02:21:49.153 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_max_snaps_per_dir": "100", 2023-08-28T02:21:49.153 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_max_xattr_pairs_size": "65536", 2023-08-28T02:21:49.153 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_metrics_update_interval": "2", 2023-08-28T02:21:49.153 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_min_caps_per_client": "100", 2023-08-28T02:21:49.154 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_min_caps_working_set": "10000", 2023-08-28T02:21:49.154 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_mon_shutdown_timeout": "5.000000", 2023-08-28T02:21:49.154 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_numa_node": "-1", 2023-08-28T02:21:49.154 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_oft_prefetch_dirfrags": "false", 2023-08-28T02:21:49.154 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_op_complaint_time": "30.000000", 2023-08-28T02:21:49.154 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_op_history_duration": "600", 2023-08-28T02:21:49.154 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_op_history_size": "20", 2023-08-28T02:21:49.154 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_op_log_threshold": "5", 2023-08-28T02:21:49.154 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_ping_grace": "15", 2023-08-28T02:21:49.154 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_ping_interval": "5", 2023-08-28T02:21:49.154 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_purge_queue_busy_flush_period": "1.000000", 2023-08-28T02:21:49.154 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_recall_global_max_decay_threshold": "131072", 2023-08-28T02:21:49.155 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_recall_max_caps": "30000", 2023-08-28T02:21:49.155 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_recall_max_decay_rate": "1.500000", 2023-08-28T02:21:49.155 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_recall_max_decay_threshold": "131072", 2023-08-28T02:21:49.155 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_recall_warning_decay_rate": "60.000000", 2023-08-28T02:21:49.155 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_recall_warning_threshold": "262144", 2023-08-28T02:21:49.155 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_reconnect_timeout": "45.000000", 2023-08-28T02:21:49.155 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_replay_interval": "1.000000", 2023-08-28T02:21:49.155 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_replay_unsafe_with_closed_session": "false", 2023-08-28T02:21:49.155 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_request_load_average_decay_rate": "60.000000", 2023-08-28T02:21:49.155 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_root_ino_gid": "0", 2023-08-28T02:21:49.155 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_root_ino_uid": "0", 2023-08-28T02:21:49.155 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_scatter_nudge_interval": "5.000000", 2023-08-28T02:21:49.156 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_session_blocklist_on_evict": "true", 2023-08-28T02:21:49.156 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_session_blocklist_on_timeout": "true", 2023-08-28T02:21:49.156 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_session_cache_liveness_decay_rate": "300.000000", 2023-08-28T02:21:49.156 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_session_cache_liveness_magnitude": "10", 2023-08-28T02:21:49.156 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_session_cap_acquisition_decay_rate": "30.000000", 2023-08-28T02:21:49.156 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_session_cap_acquisition_throttle": "100000", 2023-08-28T02:21:49.156 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_session_max_caps_throttle_ratio": "1.100000", 2023-08-28T02:21:49.156 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_sessionmap_keys_per_op": "1024", 2023-08-28T02:21:49.156 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_shutdown_check": "0", 2023-08-28T02:21:49.156 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_skip_ino": "0", 2023-08-28T02:21:49.156 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_sleep_rank_change": "0.000000", 2023-08-28T02:21:49.156 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_snap_max_uid": "4294967294", 2023-08-28T02:21:49.157 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_snap_min_uid": "0", 2023-08-28T02:21:49.157 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_snap_rstat": "false", 2023-08-28T02:21:49.157 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_standby_replay_damaged": "false", 2023-08-28T02:21:49.157 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_symlink_recovery": "true", 2023-08-28T02:21:49.157 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_task_status_update_interval": "2.000000", 2023-08-28T02:21:49.157 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_thrash_exports": "0", 2023-08-28T02:21:49.157 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_thrash_fragments": "0", 2023-08-28T02:21:49.157 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_tick_interval": "5.000000", 2023-08-28T02:21:49.157 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_valgrind_exit": "false", 2023-08-28T02:21:49.157 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_verify_backtrace": "1", 2023-08-28T02:21:49.157 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_verify_scatter": "false", 2023-08-28T02:21:49.157 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_wipe_ino_prealloc": "false", 2023-08-28T02:21:49.158 INFO:teuthology.orchestra.run.smithi143.stdout: "mds_wipe_sessions": "false", 2023-08-28T02:21:49.158 INFO:teuthology.orchestra.run.smithi143.stdout: "mempool_debug": "false", 2023-08-28T02:21:49.158 INFO:teuthology.orchestra.run.smithi143.stdout: "memstore_debug_omit_block_device_write": "false", 2023-08-28T02:21:49.158 INFO:teuthology.orchestra.run.smithi143.stdout: "memstore_device_bytes": "1073741824", 2023-08-28T02:21:49.158 INFO:teuthology.orchestra.run.smithi143.stdout: "memstore_page_set": "false", 2023-08-28T02:21:49.158 INFO:teuthology.orchestra.run.smithi143.stdout: "memstore_page_size": "65536", 2023-08-28T02:21:49.158 INFO:teuthology.orchestra.run.smithi143.stdout: "mgr_client_bytes": "134217728", 2023-08-28T02:21:49.158 INFO:teuthology.orchestra.run.smithi143.stdout: "mgr_client_messages": "512", 2023-08-28T02:21:49.158 INFO:teuthology.orchestra.run.smithi143.stdout: "mgr_client_service_daemon_unregister_timeout": "1.000000", 2023-08-28T02:21:49.158 INFO:teuthology.orchestra.run.smithi143.stdout: "mgr_connect_retry_interval": "1.000000", 2023-08-28T02:21:49.158 INFO:teuthology.orchestra.run.smithi143.stdout: "mgr_data": "/var/lib/ceph/mgr/ceph-a", 2023-08-28T02:21:49.158 INFO:teuthology.orchestra.run.smithi143.stdout: "mgr_debug_aggressive_pg_num_changes": "false", 2023-08-28T02:21:49.158 INFO:teuthology.orchestra.run.smithi143.stdout: "mgr_disabled_modules": "", 2023-08-28T02:21:49.159 INFO:teuthology.orchestra.run.smithi143.stdout: "mgr_initial_modules": "restful iostat nfs", 2023-08-28T02:21:49.159 INFO:teuthology.orchestra.run.smithi143.stdout: "mgr_max_pg_num_change": "128", 2023-08-28T02:21:49.159 INFO:teuthology.orchestra.run.smithi143.stdout: "mgr_mds_bytes": "134217728", 2023-08-28T02:21:49.159 INFO:teuthology.orchestra.run.smithi143.stdout: "mgr_mds_messages": "128", 2023-08-28T02:21:49.159 INFO:teuthology.orchestra.run.smithi143.stdout: "mgr_module_path": "/usr/share/ceph/mgr", 2023-08-28T02:21:49.159 INFO:teuthology.orchestra.run.smithi143.stdout: "mgr_mon_bytes": "134217728", 2023-08-28T02:21:49.159 INFO:teuthology.orchestra.run.smithi143.stdout: "mgr_mon_messages": "128", 2023-08-28T02:21:49.159 INFO:teuthology.orchestra.run.smithi143.stdout: "mgr_osd_bytes": "536870912", 2023-08-28T02:21:49.159 INFO:teuthology.orchestra.run.smithi143.stdout: "mgr_osd_messages": "8192", 2023-08-28T02:21:49.159 INFO:teuthology.orchestra.run.smithi143.stdout: "mgr_pool": "true", 2023-08-28T02:21:49.159 INFO:teuthology.orchestra.run.smithi143.stdout: "mgr_service_beacon_grace": "60.000000", 2023-08-28T02:21:49.159 INFO:teuthology.orchestra.run.smithi143.stdout: "mgr_standby_modules": "true", 2023-08-28T02:21:49.160 INFO:teuthology.orchestra.run.smithi143.stdout: "mgr_stats_period": "5", 2023-08-28T02:21:49.160 INFO:teuthology.orchestra.run.smithi143.stdout: "mgr_stats_threshold": "5", 2023-08-28T02:21:49.160 INFO:teuthology.orchestra.run.smithi143.stdout: "mgr_tick_period": "2", 2023-08-28T02:21:49.160 INFO:teuthology.orchestra.run.smithi143.stdout: "mgr_ttl_cache_expire_seconds": "0", 2023-08-28T02:21:49.160 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_accept_timeout_factor": "2.000000", 2023-08-28T02:21:49.160 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_allow_pool_delete": "true", 2023-08-28T02:21:49.160 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_allow_pool_size_one": "false", 2023-08-28T02:21:49.160 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_auth_validate_all_caps": "true", 2023-08-28T02:21:49.160 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_cache_target_full_warn_ratio": "0.660000", 2023-08-28T02:21:49.160 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_clean_pg_upmaps_per_chunk": "256", 2023-08-28T02:21:49.160 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_client_bytes": "104857600", 2023-08-28T02:21:49.160 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_client_directed_command_retry": "2", 2023-08-28T02:21:49.161 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_client_hunt_interval": "3.000000", 2023-08-28T02:21:49.161 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_client_hunt_interval_backoff": "1.500000", 2023-08-28T02:21:49.161 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_client_hunt_interval_max_multiple": "10.000000", 2023-08-28T02:21:49.161 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_client_hunt_interval_min_multiple": "1.000000", 2023-08-28T02:21:49.161 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_client_hunt_parallel": "3", 2023-08-28T02:21:49.161 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_client_log_interval": "1.000000", 2023-08-28T02:21:49.161 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_client_max_log_entries_per_message": "1000", 2023-08-28T02:21:49.161 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_client_ping_interval": "10.000000", 2023-08-28T02:21:49.161 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_client_ping_timeout": "30.000000", 2023-08-28T02:21:49.161 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_clock_drift_allowed": "1.000000", 2023-08-28T02:21:49.161 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_clock_drift_warn_backoff": "5.000000", 2023-08-28T02:21:49.161 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_cluster_log_file": "default=/var/log/ceph/ceph.$channel.log cluster=/var/log/ceph/ceph.log", 2023-08-28T02:21:49.162 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_cluster_log_file_level": "debug", 2023-08-28T02:21:49.162 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_cluster_log_to_file": "false", 2023-08-28T02:21:49.162 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_cluster_log_to_graylog": "false", 2023-08-28T02:21:49.162 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_cluster_log_to_graylog_host": "127.0.0.1", 2023-08-28T02:21:49.162 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_cluster_log_to_graylog_port": "12201", 2023-08-28T02:21:49.162 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_cluster_log_to_journald": "true", 2023-08-28T02:21:49.162 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_cluster_log_to_stderr": "false", 2023-08-28T02:21:49.162 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_cluster_log_to_syslog": "default=false", 2023-08-28T02:21:49.162 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_cluster_log_to_syslog_facility": "daemon", 2023-08-28T02:21:49.162 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_cluster_log_to_syslog_level": "info", 2023-08-28T02:21:49.162 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_compact_on_bootstrap": "false", 2023-08-28T02:21:49.162 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_compact_on_start": "false", 2023-08-28T02:21:49.162 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_compact_on_trim": "true", 2023-08-28T02:21:49.163 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_con_tracker_persist_interval": "10", 2023-08-28T02:21:49.163 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_con_tracker_score_halflife": "43200", 2023-08-28T02:21:49.163 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_config_key_max_entry_size": "65536", 2023-08-28T02:21:49.163 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_cpu_threads": "4", 2023-08-28T02:21:49.163 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_crush_min_required_version": "hammer", 2023-08-28T02:21:49.163 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_daemon_bytes": "419430400", 2023-08-28T02:21:49.163 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_data": "/var/lib/ceph/mon/ceph-a", 2023-08-28T02:21:49.163 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_data_avail_crit": "5", 2023-08-28T02:21:49.163 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_data_avail_warn": "5", 2023-08-28T02:21:49.163 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_data_size_warn": "16106127360", 2023-08-28T02:21:49.163 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_debug_block_osdmap_trim": "false", 2023-08-28T02:21:49.164 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_debug_deprecated_as_obsolete": "false", 2023-08-28T02:21:49.164 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_debug_dump_json": "false", 2023-08-28T02:21:49.164 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_debug_dump_location": "/var/log/ceph/ceph-mon.a.tdump", 2023-08-28T02:21:49.164 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_debug_dump_transactions": "false", 2023-08-28T02:21:49.164 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_debug_extra_checks": "false", 2023-08-28T02:21:49.164 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_debug_no_initial_persistent_features": "false", 2023-08-28T02:21:49.164 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_debug_no_require_bluestore_for_ec_overwrites": "false", 2023-08-28T02:21:49.164 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_debug_no_require_quincy": "false", 2023-08-28T02:21:49.164 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_debug_no_require_reef": "false", 2023-08-28T02:21:49.164 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false", 2023-08-28T02:21:49.164 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_delta_reset_interval": "10.000000", 2023-08-28T02:21:49.164 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_dns_srv_name": "ceph-mon", 2023-08-28T02:21:49.164 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_down_mkfs_grace": "60", 2023-08-28T02:21:49.165 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_election_default_strategy": "3", 2023-08-28T02:21:49.165 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_election_timeout": "5.000000", 2023-08-28T02:21:49.165 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_elector_ignore_propose_margin": "0.000500", 2023-08-28T02:21:49.165 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_elector_ping_divisor": "2", 2023-08-28T02:21:49.165 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_elector_ping_timeout": "2.000000", 2023-08-28T02:21:49.165 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_enable_op_tracker": "true", 2023-08-28T02:21:49.165 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_fake_pool_delete": "false", 2023-08-28T02:21:49.165 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_force_quorum_join": "false", 2023-08-28T02:21:49.165 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_globalid_prealloc": "10000", 2023-08-28T02:21:49.165 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_health_detail_to_clog": "true", 2023-08-28T02:21:49.165 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_health_log_update_period": "5", 2023-08-28T02:21:49.166 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_health_max_detail": "50", 2023-08-28T02:21:49.166 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_health_to_clog": "true", 2023-08-28T02:21:49.166 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_health_to_clog_interval": "600", 2023-08-28T02:21:49.166 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_health_to_clog_tick_interval": "60.000000", 2023-08-28T02:21:49.166 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_host": "[v2:172.21.15.143:3300/0,v1:172.21.15.143:6789/0] [v2:172.21.15.169:3300/0,v1:172.21.15.169:6789/0] [v2:172.21.15.143:3301/0,v1:172.21.15.143:6790/0]", 2023-08-28T02:21:49.166 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_host_override": "", 2023-08-28T02:21:49.166 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_initial_members": "", 2023-08-28T02:21:49.166 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_inject_pg_merge_bounce_probability": "0.000000", 2023-08-28T02:21:49.166 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_inject_sync_get_chunk_delay": "0.000000", 2023-08-28T02:21:49.166 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_inject_transaction_delay_max": "10.000000", 2023-08-28T02:21:49.166 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_inject_transaction_delay_probability": "0.000000", 2023-08-28T02:21:49.167 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_keyvaluedb": "rocksdb", 2023-08-28T02:21:49.167 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_lease": "5.000000", 2023-08-28T02:21:49.167 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_lease_ack_timeout_factor": "2.000000", 2023-08-28T02:21:49.167 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_lease_renew_interval_factor": "0.600000", 2023-08-28T02:21:49.167 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_log_full_interval": "50", 2023-08-28T02:21:49.167 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_log_max": "10000", 2023-08-28T02:21:49.167 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_log_max_summary": "50", 2023-08-28T02:21:49.167 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_max_log_entries_per_event": "4096", 2023-08-28T02:21:49.167 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_max_log_epochs": "500", 2023-08-28T02:21:49.167 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_max_mdsmap_epochs": "500", 2023-08-28T02:21:49.167 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_max_mgrmap_epochs": "500", 2023-08-28T02:21:49.167 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_max_osd": "10000", 2023-08-28T02:21:49.168 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_max_pg_per_osd": "10000", 2023-08-28T02:21:49.168 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_max_pool_pg_num": "65536", 2023-08-28T02:21:49.168 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_max_snap_prune_per_epoch": "100", 2023-08-28T02:21:49.168 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_mds_blocklist_interval": "86400.000000", 2023-08-28T02:21:49.168 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_mds_force_trim_to": "0", 2023-08-28T02:21:49.168 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_mds_skip_sanity": "true", 2023-08-28T02:21:49.168 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_memory_autotune": "true", 2023-08-28T02:21:49.168 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_memory_target": "2147483648", 2023-08-28T02:21:49.168 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_mgr_beacon_grace": "30", 2023-08-28T02:21:49.168 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_mgr_blocklist_interval": "86400.000000", 2023-08-28T02:21:49.168 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_mgr_digest_period": "5", 2023-08-28T02:21:49.168 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_mgr_inactive_grace": "60", 2023-08-28T02:21:49.169 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_mgr_mkfs_grace": "240", 2023-08-28T02:21:49.169 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_mgr_proxy_client_bytes_ratio": "0.300000", 2023-08-28T02:21:49.169 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_min_osdmap_epochs": "500", 2023-08-28T02:21:49.169 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_op_complaint_time": "30", 2023-08-28T02:21:49.169 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_op_history_duration": "600", 2023-08-28T02:21:49.169 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_op_history_size": "20", 2023-08-28T02:21:49.169 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_op_history_slow_op_size": "20", 2023-08-28T02:21:49.169 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_op_history_slow_op_threshold": "10", 2023-08-28T02:21:49.169 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_op_log_threshold": "5", 2023-08-28T02:21:49.169 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_osd_adjust_down_out_interval": "true", 2023-08-28T02:21:49.169 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_osd_adjust_heartbeat_grace": "true", 2023-08-28T02:21:49.169 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_osd_auto_mark_auto_out_in": "true", 2023-08-28T02:21:49.169 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_osd_auto_mark_in": "false", 2023-08-28T02:21:49.170 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_osd_auto_mark_new_in": "true", 2023-08-28T02:21:49.170 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_osd_backfillfull_ratio": "0.900000", 2023-08-28T02:21:49.170 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_osd_blocklist_default_expire": "3600.000000", 2023-08-28T02:21:49.170 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_osd_cache_size": "500", 2023-08-28T02:21:49.170 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_osd_cache_size_min": "134217728", 2023-08-28T02:21:49.170 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_osd_crush_smoke_test": "true", 2023-08-28T02:21:49.170 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_osd_destroyed_out_interval": "600", 2023-08-28T02:21:49.170 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_osd_down_out_interval": "600", 2023-08-28T02:21:49.170 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_osd_down_out_subtree_limit": "rack", 2023-08-28T02:21:49.170 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_osd_err_op_age_ratio": "128.000000", 2023-08-28T02:21:49.170 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_osd_force_trim_to": "0", 2023-08-28T02:21:49.170 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_osd_full_ratio": "0.950000", 2023-08-28T02:21:49.171 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_osd_initial_require_min_compat_client": "luminous", 2023-08-28T02:21:49.171 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_osd_laggy_halflife": "3600", 2023-08-28T02:21:49.171 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_osd_laggy_max_interval": "300", 2023-08-28T02:21:49.171 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_osd_laggy_weight": "0.300000", 2023-08-28T02:21:49.171 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_osd_mapping_pgs_per_chunk": "4096", 2023-08-28T02:21:49.171 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_osd_max_creating_pgs": "1024", 2023-08-28T02:21:49.171 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_osd_max_initial_pgs": "1024", 2023-08-28T02:21:49.171 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_osd_min_down_reporters": "2", 2023-08-28T02:21:49.171 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_osd_min_in_ratio": "0.750000", 2023-08-28T02:21:49.171 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_osd_min_up_ratio": "0.300000", 2023-08-28T02:21:49.171 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_osd_nearfull_ratio": "0.850000", 2023-08-28T02:21:49.171 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_osd_prime_pg_temp": "true", 2023-08-28T02:21:49.172 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_osd_prime_pg_temp_max_estimate": "0.250000", 2023-08-28T02:21:49.172 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_osd_prime_pg_temp_max_time": "0.500000", 2023-08-28T02:21:49.172 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_osd_report_timeout": "900", 2023-08-28T02:21:49.172 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_osd_reporter_subtree_level": "osd", 2023-08-28T02:21:49.172 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_osd_snap_trim_queue_warn_on": "32768", 2023-08-28T02:21:49.172 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_osd_warn_num_repaired": "10", 2023-08-28T02:21:49.172 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_osd_warn_op_age": "32.000000", 2023-08-28T02:21:49.172 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_osdmap_full_prune_enabled": "true", 2023-08-28T02:21:49.172 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_osdmap_full_prune_interval": "10", 2023-08-28T02:21:49.172 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_osdmap_full_prune_min": "10000", 2023-08-28T02:21:49.172 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_osdmap_full_prune_txsize": "100", 2023-08-28T02:21:49.172 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_pg_check_down_all_threshold": "0.500000", 2023-08-28T02:21:49.173 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_pg_stuck_threshold": "60", 2023-08-28T02:21:49.173 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_pg_warn_max_object_skew": "0.000000", 2023-08-28T02:21:49.173 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_pg_warn_min_objects": "10000", 2023-08-28T02:21:49.173 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_pg_warn_min_per_osd": "0", 2023-08-28T02:21:49.173 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_pg_warn_min_pool_objects": "1000", 2023-08-28T02:21:49.173 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_pool_quota_crit_threshold": "0", 2023-08-28T02:21:49.173 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_pool_quota_warn_threshold": "0", 2023-08-28T02:21:49.173 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_probe_timeout": "2.000000", 2023-08-28T02:21:49.173 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_reweight_max_change": "0.050000", 2023-08-28T02:21:49.173 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_reweight_max_osds": "4", 2023-08-28T02:21:49.173 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_reweight_min_bytes_per_osd": "10", 2023-08-28T02:21:49.173 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_reweight_min_pgs_per_osd": "4", 2023-08-28T02:21:49.174 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_rocksdb_options": "write_buffer_size=33554432,compression=kNoCompression,level_compaction_dynamic_level_bytes=true", 2023-08-28T02:21:49.174 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_scrub_inject_crc_mismatch": "0.000000", 2023-08-28T02:21:49.174 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_scrub_inject_missing_keys": "0.000000", 2023-08-28T02:21:49.174 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_scrub_interval": "86400", 2023-08-28T02:21:49.174 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_scrub_max_keys": "100", 2023-08-28T02:21:49.174 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_scrub_timeout": "300", 2023-08-28T02:21:49.174 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_session_timeout": "300", 2023-08-28T02:21:49.174 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_smart_report_timeout": "5", 2023-08-28T02:21:49.174 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_stat_smooth_intervals": "6", 2023-08-28T02:21:49.174 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_stretch_cluster_recovery_ratio": "0.600000", 2023-08-28T02:21:49.174 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_stretch_pool_min_size": "2", 2023-08-28T02:21:49.174 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_stretch_pool_size": "4", 2023-08-28T02:21:49.175 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_stretch_recovery_min_wait": "15.000000", 2023-08-28T02:21:49.175 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_subscribe_interval": "86400.000000", 2023-08-28T02:21:49.175 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_sync_debug": "false", 2023-08-28T02:21:49.175 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_sync_max_payload_keys": "2000", 2023-08-28T02:21:49.175 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_sync_max_payload_size": "1048576", 2023-08-28T02:21:49.175 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_sync_provider_kill_at": "0", 2023-08-28T02:21:49.175 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_sync_requester_kill_at": "0", 2023-08-28T02:21:49.175 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_sync_timeout": "60.000000", 2023-08-28T02:21:49.175 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_target_pg_per_osd": "100", 2023-08-28T02:21:49.175 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_tick_interval": "5", 2023-08-28T02:21:49.175 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_timecheck_interval": "300.000000", 2023-08-28T02:21:49.175 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_timecheck_skew_interval": "30.000000", 2023-08-28T02:21:49.176 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_use_min_delay_socket": "false", 2023-08-28T02:21:49.176 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_warn_older_version_delay": "604800", 2023-08-28T02:21:49.176 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_warn_on_cache_pools_without_hit_sets": "true", 2023-08-28T02:21:49.176 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_warn_on_crush_straw_calc_version_zero": "false", 2023-08-28T02:21:49.176 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_warn_on_degraded_stretch_mode": "true", 2023-08-28T02:21:49.176 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_warn_on_filestore_osds": "true", 2023-08-28T02:21:49.176 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_warn_on_insecure_global_id_reclaim": "false", 2023-08-28T02:21:49.176 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed": "false", 2023-08-28T02:21:49.176 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_warn_on_legacy_crush_tunables": "false", 2023-08-28T02:21:49.176 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_warn_on_misplaced": "false", 2023-08-28T02:21:49.177 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_warn_on_msgr2_not_enabled": "true", 2023-08-28T02:21:49.177 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_warn_on_older_version": "true", 2023-08-28T02:21:49.177 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_warn_on_osd_down_out_interval_zero": "false", 2023-08-28T02:21:49.177 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_warn_on_pool_no_app": "true", 2023-08-28T02:21:49.177 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_warn_on_pool_no_redundancy": "true", 2023-08-28T02:21:49.177 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_warn_on_pool_pg_num_not_power_of_two": "false", 2023-08-28T02:21:49.177 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_warn_on_slow_ping_ratio": "0.050000", 2023-08-28T02:21:49.177 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_warn_on_slow_ping_time": "0.000000", 2023-08-28T02:21:49.177 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_warn_on_too_few_osds": "false", 2023-08-28T02:21:49.177 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_warn_pg_not_deep_scrubbed_ratio": "0.750000", 2023-08-28T02:21:49.177 INFO:teuthology.orchestra.run.smithi143.stdout: "mon_warn_pg_not_scrubbed_ratio": "0.500000", 2023-08-28T02:21:49.177 INFO:teuthology.orchestra.run.smithi143.stdout: "monmap": "", 2023-08-28T02:21:49.178 INFO:teuthology.orchestra.run.smithi143.stdout: "motr_admin_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-08-28T02:21:49.178 INFO:teuthology.orchestra.run.smithi143.stdout: "motr_admin_fid": "0x7200000000000001:0x0", 2023-08-28T02:21:49.178 INFO:teuthology.orchestra.run.smithi143.stdout: "motr_ha_endpoint": "192.168.180.182@tcp:12345:1:1", 2023-08-28T02:21:49.178 INFO:teuthology.orchestra.run.smithi143.stdout: "motr_my_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-08-28T02:21:49.178 INFO:teuthology.orchestra.run.smithi143.stdout: "motr_my_fid": "0x7200000000000001:0x0", 2023-08-28T02:21:49.178 INFO:teuthology.orchestra.run.smithi143.stdout: "motr_profile_fid": "0x7000000000000001:0x0", 2023-08-28T02:21:49.178 INFO:teuthology.orchestra.run.smithi143.stdout: "motr_tracing_enabled": "false", 2023-08-28T02:21:49.178 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_async_op_threads": "3", 2023-08-28T02:21:49.179 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_async_rdma_buffer_size": "131072", 2023-08-28T02:21:49.179 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_async_rdma_cm": "false", 2023-08-28T02:21:49.179 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_async_rdma_device_name": "", 2023-08-28T02:21:49.179 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_async_rdma_dscp": "96", 2023-08-28T02:21:49.179 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_async_rdma_enable_hugepage": "false", 2023-08-28T02:21:49.179 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_async_rdma_gid_idx": "0", 2023-08-28T02:21:49.179 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_async_rdma_local_gid": "", 2023-08-28T02:21:49.179 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_async_rdma_polling_us": "1000", 2023-08-28T02:21:49.179 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_async_rdma_port_num": "1", 2023-08-28T02:21:49.179 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_async_rdma_receive_buffers": "32768", 2023-08-28T02:21:49.179 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_async_rdma_receive_queue_len": "4096", 2023-08-28T02:21:49.180 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_async_rdma_roce_ver": "1", 2023-08-28T02:21:49.180 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_async_rdma_send_buffers": "1024", 2023-08-28T02:21:49.180 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_async_rdma_sl": "3", 2023-08-28T02:21:49.180 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_async_rdma_support_srq": "true", 2023-08-28T02:21:49.180 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_async_rdma_type": "ib", 2023-08-28T02:21:49.180 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_async_reap_threshold": "5", 2023-08-28T02:21:49.180 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_bind_before_connect": "false", 2023-08-28T02:21:49.180 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_bind_ipv4": "true", 2023-08-28T02:21:49.180 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_bind_ipv6": "false", 2023-08-28T02:21:49.180 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_bind_msgr1": "true", 2023-08-28T02:21:49.180 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_bind_msgr2": "true", 2023-08-28T02:21:49.180 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_bind_port_max": "7568", 2023-08-28T02:21:49.180 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_bind_port_min": "6800", 2023-08-28T02:21:49.181 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_bind_prefer_ipv4": "false", 2023-08-28T02:21:49.181 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_bind_retry_count": "3", 2023-08-28T02:21:49.181 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_bind_retry_delay": "5", 2023-08-28T02:21:49.181 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_blackhole_client": "false", 2023-08-28T02:21:49.181 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_blackhole_mds": "false", 2023-08-28T02:21:49.181 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_blackhole_mgr": "false", 2023-08-28T02:21:49.181 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_blackhole_mon": "false", 2023-08-28T02:21:49.181 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_blackhole_osd": "false", 2023-08-28T02:21:49.181 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_client_mode": "crc secure", 2023-08-28T02:21:49.181 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_cluster_mode": "crc secure", 2023-08-28T02:21:49.181 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_cluster_type": "", 2023-08-28T02:21:49.181 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_compress_secure": "false", 2023-08-28T02:21:49.182 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_connection_idle_timeout": "900", 2023-08-28T02:21:49.182 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_connection_ready_timeout": "10", 2023-08-28T02:21:49.182 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_crc_data": "true", 2023-08-28T02:21:49.182 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_crc_header": "true", 2023-08-28T02:21:49.182 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_die_on_bad_msg": "false", 2023-08-28T02:21:49.182 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_die_on_bug": "true", 2023-08-28T02:21:49.182 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_die_on_old_message": "true", 2023-08-28T02:21:49.182 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_die_on_skipped_message": "false", 2023-08-28T02:21:49.182 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_die_on_unhandled_msg": "false", 2023-08-28T02:21:49.182 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_dispatch_throttle_bytes": "104857600", 2023-08-28T02:21:49.182 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_dpdk_coremask": "0xF", 2023-08-28T02:21:49.182 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_dpdk_debug_allow_loopback": "false", 2023-08-28T02:21:49.183 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_dpdk_devs_allowlist": "", 2023-08-28T02:21:49.183 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_dpdk_enable_tso": "true", 2023-08-28T02:21:49.183 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_dpdk_gateway_ipv4_addr": "", 2023-08-28T02:21:49.183 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_dpdk_host_ipv4_addr": "", 2023-08-28T02:21:49.183 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_dpdk_hugepages": "", 2023-08-28T02:21:49.183 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_dpdk_hw_flow_control": "true", 2023-08-28T02:21:49.183 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_dpdk_hw_queue_weight": "1.000000", 2023-08-28T02:21:49.183 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_dpdk_lro": "true", 2023-08-28T02:21:49.183 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_dpdk_memory_channel": "4", 2023-08-28T02:21:49.183 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_dpdk_netmask_ipv4_addr": "", 2023-08-28T02:21:49.183 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_dpdk_pmd": "", 2023-08-28T02:21:49.183 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_dpdk_port_id": "0", 2023-08-28T02:21:49.184 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_dpdk_rx_buffer_count_per_core": "8192", 2023-08-28T02:21:49.184 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_dump_corrupt_message_level": "1", 2023-08-28T02:21:49.184 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_dump_on_send": "false", 2023-08-28T02:21:49.184 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_initial_backoff": "0.200000", 2023-08-28T02:21:49.184 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_inject_delay_max": "1.000000", 2023-08-28T02:21:49.184 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_inject_delay_probability": "0.000000", 2023-08-28T02:21:49.184 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_inject_delay_type": "", 2023-08-28T02:21:49.184 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_inject_internal_delays": "0.000000", 2023-08-28T02:21:49.184 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_inject_network_congestion": "0", 2023-08-28T02:21:49.184 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_inject_socket_failures": "0", 2023-08-28T02:21:49.184 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_learn_addr_from_peer": "true", 2023-08-28T02:21:49.184 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_max_accept_failures": "4", 2023-08-28T02:21:49.185 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_max_backoff": "15.000000", 2023-08-28T02:21:49.185 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_mon_client_mode": "secure crc", 2023-08-28T02:21:49.185 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_mon_cluster_mode": "secure crc", 2023-08-28T02:21:49.185 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_mon_service_mode": "secure crc", 2023-08-28T02:21:49.185 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_osd_compress_min_size": "1024", 2023-08-28T02:21:49.185 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_osd_compress_mode": "none", 2023-08-28T02:21:49.185 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_osd_compression_algorithm": "snappy", 2023-08-28T02:21:49.185 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_pq_max_tokens_per_priority": "16777216", 2023-08-28T02:21:49.185 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_pq_min_cost": "65536", 2023-08-28T02:21:49.185 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_public_type": "", 2023-08-28T02:21:49.185 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_service_mode": "crc secure", 2023-08-28T02:21:49.185 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_tcp_listen_backlog": "512", 2023-08-28T02:21:49.186 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_tcp_nodelay": "true", 2023-08-28T02:21:49.186 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_tcp_prefetch_max_size": "4096", 2023-08-28T02:21:49.186 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_tcp_rcvbuf": "0", 2023-08-28T02:21:49.186 INFO:teuthology.orchestra.run.smithi143.stdout: "ms_type": "async+posix", 2023-08-28T02:21:49.186 INFO:teuthology.orchestra.run.smithi143.stdout: "no_config_file": "false", 2023-08-28T02:21:49.186 INFO:teuthology.orchestra.run.smithi143.stdout: "objecter_completion_locks_per_session": "32", 2023-08-28T02:21:49.186 INFO:teuthology.orchestra.run.smithi143.stdout: "objecter_debug_inject_relock_delay": "false", 2023-08-28T02:21:49.186 INFO:teuthology.orchestra.run.smithi143.stdout: "objecter_inflight_op_bytes": "104857600", 2023-08-28T02:21:49.186 INFO:teuthology.orchestra.run.smithi143.stdout: "objecter_inflight_ops": "1024", 2023-08-28T02:21:49.186 INFO:teuthology.orchestra.run.smithi143.stdout: "objecter_inject_no_watch_ping": "false", 2023-08-28T02:21:49.186 INFO:teuthology.orchestra.run.smithi143.stdout: "objecter_retry_writes_after_first_reply": "false", 2023-08-28T02:21:49.186 INFO:teuthology.orchestra.run.smithi143.stdout: "objecter_tick_interval": "5.000000", 2023-08-28T02:21:49.186 INFO:teuthology.orchestra.run.smithi143.stdout: "objecter_timeout": "10.000000", 2023-08-28T02:21:49.187 INFO:teuthology.orchestra.run.smithi143.stdout: "objectstore_blackhole": "false", 2023-08-28T02:21:49.187 INFO:teuthology.orchestra.run.smithi143.stdout: "openssl_engine_opts": "", 2023-08-28T02:21:49.187 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_agent_delay_time": "5.000000", 2023-08-28T02:21:49.187 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_agent_hist_halflife": "1000", 2023-08-28T02:21:49.187 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_agent_max_low_ops": "2", 2023-08-28T02:21:49.187 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_agent_max_ops": "4", 2023-08-28T02:21:49.187 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_agent_min_evict_effort": "0.100000", 2023-08-28T02:21:49.187 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_agent_quantize_effort": "0.100000", 2023-08-28T02:21:49.187 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_agent_slop": "0.020000", 2023-08-28T02:21:49.187 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_aggregated_slow_ops_logging": "true", 2023-08-28T02:21:49.187 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_allow_recovery_below_min_size": "true", 2023-08-28T02:21:49.187 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_asio_thread_count": "2", 2023-08-28T02:21:49.188 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_async_recovery_min_cost": "100", 2023-08-28T02:21:49.188 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_auto_mark_unfound_lost": "false", 2023-08-28T02:21:49.188 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_backfill_retry_interval": "30.000000", 2023-08-28T02:21:49.188 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_backfill_scan_max": "512", 2023-08-28T02:21:49.188 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_backfill_scan_min": "64", 2023-08-28T02:21:49.188 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_backoff_on_degraded": "false", 2023-08-28T02:21:49.188 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_backoff_on_peering": "false", 2023-08-28T02:21:49.188 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_backoff_on_unfound": "true", 2023-08-28T02:21:49.188 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_beacon_report_interval": "300", 2023-08-28T02:21:49.188 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_bench_duration": "30", 2023-08-28T02:21:49.188 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_bench_large_size_max_throughput": "104857600", 2023-08-28T02:21:49.188 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_bench_max_block_size": "67108864", 2023-08-28T02:21:49.188 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_bench_small_size_max_iops": "100", 2023-08-28T02:21:49.189 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_blkin_trace_all": "false", 2023-08-28T02:21:49.189 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_blocked_scrub_grace_period": "120", 2023-08-28T02:21:49.189 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_calc_pg_upmaps_aggressively": "true", 2023-08-28T02:21:49.189 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_calc_pg_upmaps_aggressively_fast": "true", 2023-08-28T02:21:49.189 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_calc_pg_upmaps_local_fallback_retries": "100", 2023-08-28T02:21:49.189 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_check_for_log_corruption": "false", 2023-08-28T02:21:49.189 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_check_max_object_name_len_on_startup": "true", 2023-08-28T02:21:49.189 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_class_default_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-08-28T02:21:49.189 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_class_dir": "/usr/lib64/rados-classes", 2023-08-28T02:21:49.189 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_class_load_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-08-28T02:21:49.189 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_class_update_on_start": "true", 2023-08-28T02:21:49.189 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_client_message_cap": "256", 2023-08-28T02:21:49.189 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_client_message_size_cap": "524288000", 2023-08-28T02:21:49.190 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_client_op_priority": "63", 2023-08-28T02:21:49.190 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_client_watch_timeout": "30", 2023-08-28T02:21:49.190 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_command_max_records": "256", 2023-08-28T02:21:49.190 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_compact_on_start": "false", 2023-08-28T02:21:49.190 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_copyfrom_max_chunk": "8388608", 2023-08-28T02:21:49.190 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_crush_chooseleaf_type": "1", 2023-08-28T02:21:49.190 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_crush_initial_weight": "-1.000000", 2023-08-28T02:21:49.190 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_crush_update_on_start": "true", 2023-08-28T02:21:49.190 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_crush_update_weight_set": "true", 2023-08-28T02:21:49.190 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_data": "/var/lib/ceph/osd/ceph-a", 2023-08-28T02:21:49.190 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_debug_crash_on_ignored_backoff": "false", 2023-08-28T02:21:49.190 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_debug_deep_scrub_sleep": "0.000000", 2023-08-28T02:21:49.191 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_debug_drop_ping_duration": "0", 2023-08-28T02:21:49.191 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_debug_drop_ping_probability": "0.000000", 2023-08-28T02:21:49.191 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_debug_feed_pullee": "-1", 2023-08-28T02:21:49.191 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_debug_inject_copyfrom_error": "false", 2023-08-28T02:21:49.191 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_debug_inject_dispatch_delay_duration": "0.100000", 2023-08-28T02:21:49.191 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_debug_inject_dispatch_delay_probability": "0.000000", 2023-08-28T02:21:49.191 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_debug_misdirected_ops": "false", 2023-08-28T02:21:49.191 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_debug_no_acting_change": "false", 2023-08-28T02:21:49.191 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_debug_no_purge_strays": "false", 2023-08-28T02:21:49.191 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_debug_op_order": "false", 2023-08-28T02:21:49.191 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_debug_pg_log_writeout": "false", 2023-08-28T02:21:49.191 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_debug_pretend_recovery_active": "false", 2023-08-28T02:21:49.192 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_debug_random_push_read_error": "0.000000", 2023-08-28T02:21:49.192 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_debug_reject_backfill_probability": "0.000000", 2023-08-28T02:21:49.192 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_debug_shutdown": "false", 2023-08-28T02:21:49.192 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_debug_skip_full_check_in_backfill_reservation": "false", 2023-08-28T02:21:49.192 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_debug_skip_full_check_in_recovery": "false", 2023-08-28T02:21:49.192 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_debug_verify_cached_snaps": "false", 2023-08-28T02:21:49.192 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_debug_verify_missing_on_start": "false", 2023-08-28T02:21:49.192 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_debug_verify_snaps": "false", 2023-08-28T02:21:49.192 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_debug_verify_stray_on_activate": "false", 2023-08-28T02:21:49.192 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_deep_scrub_interval": "604800.000000", 2023-08-28T02:21:49.192 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_deep_scrub_keys": "1024", 2023-08-28T02:21:49.192 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_deep_scrub_large_omap_object_key_threshold": "200000", 2023-08-28T02:21:49.192 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824", 2023-08-28T02:21:49.193 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_deep_scrub_randomize_ratio": "0.150000", 2023-08-28T02:21:49.193 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_deep_scrub_stride": "524288", 2023-08-28T02:21:49.193 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_deep_scrub_update_digest_min_age": "7200", 2023-08-28T02:21:49.193 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_default_data_pool_replay_window": "45", 2023-08-28T02:21:49.193 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_default_notify_timeout": "30", 2023-08-28T02:21:49.193 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_delete_sleep": "0.000000", 2023-08-28T02:21:49.193 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_delete_sleep_hdd": "5.000000", 2023-08-28T02:21:49.193 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_delete_sleep_hybrid": "1.000000", 2023-08-28T02:21:49.193 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_delete_sleep_ssd": "1.000000", 2023-08-28T02:21:49.193 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_discard_disconnected_ops": "true", 2023-08-28T02:21:49.193 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_enable_op_tracker": "true", 2023-08-28T02:21:49.193 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_erasure_code_plugins": "jerasure lrc", 2023-08-28T02:21:49.194 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_extblkdev_plugins": "vdo", 2023-08-28T02:21:49.194 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_failsafe_full_ratio": "0.970000", 2023-08-28T02:21:49.194 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_fast_fail_on_connection_refused": "true", 2023-08-28T02:21:49.194 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_fast_info": "true", 2023-08-28T02:21:49.194 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_fast_shutdown": "true", 2023-08-28T02:21:49.194 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_fast_shutdown_notify_mon": "true", 2023-08-28T02:21:49.194 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_fast_shutdown_timeout": "15", 2023-08-28T02:21:49.194 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_find_best_info_ignore_history_les": "false", 2023-08-28T02:21:49.194 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_force_auth_primary_missing_objects": "100", 2023-08-28T02:21:49.194 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_force_recovery_pg_log_entries_factor": "1.300000", 2023-08-28T02:21:49.194 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_function_tracing": "false", 2023-08-28T02:21:49.194 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_heartbeat_grace": "20", 2023-08-28T02:21:49.194 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_heartbeat_interval": "6", 2023-08-28T02:21:49.195 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_heartbeat_min_healthy_ratio": "0.330000", 2023-08-28T02:21:49.195 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_heartbeat_min_peers": "10", 2023-08-28T02:21:49.195 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_heartbeat_min_size": "2000", 2023-08-28T02:21:49.195 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_heartbeat_stale": "600", 2023-08-28T02:21:49.195 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_heartbeat_use_min_delay_socket": "false", 2023-08-28T02:21:49.195 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_hit_set_max_size": "100000", 2023-08-28T02:21:49.195 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_hit_set_min_size": "1000", 2023-08-28T02:21:49.195 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_hit_set_namespace": ".ceph-internal", 2023-08-28T02:21:49.195 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_ignore_stale_divergent_priors": "false", 2023-08-28T02:21:49.195 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_inject_bad_map_crc_probability": "0.000000", 2023-08-28T02:21:49.195 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_inject_failure_on_pg_removal": "false", 2023-08-28T02:21:49.196 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_journal": "/var/lib/ceph/osd/ceph-a/journal", 2023-08-28T02:21:49.196 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_journal_flush_on_shutdown": "true", 2023-08-28T02:21:49.196 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_journal_size": "5120", 2023-08-28T02:21:49.196 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_kill_backfill_at": "0", 2023-08-28T02:21:49.196 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_loop_before_reset_tphandle": "64", 2023-08-28T02:21:49.196 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_map_cache_size": "50", 2023-08-28T02:21:49.196 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_map_dedup": "true", 2023-08-28T02:21:49.196 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_map_message_max": "40", 2023-08-28T02:21:49.196 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_map_message_max_bytes": "10485760", 2023-08-28T02:21:49.196 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_map_share_max_epochs": "40", 2023-08-28T02:21:49.196 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_max_attr_name_len": "100", 2023-08-28T02:21:49.196 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_max_attr_size": "0", 2023-08-28T02:21:49.197 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_max_backfills": "1", 2023-08-28T02:21:49.197 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_max_markdown_count": "5", 2023-08-28T02:21:49.197 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_max_markdown_period": "600", 2023-08-28T02:21:49.197 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_max_object_name_len": "2048", 2023-08-28T02:21:49.197 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_max_object_namespace_len": "256", 2023-08-28T02:21:49.197 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_max_object_size": "134217728", 2023-08-28T02:21:49.197 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_max_omap_bytes_per_request": "1073741824", 2023-08-28T02:21:49.197 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_max_omap_entries_per_request": "1024", 2023-08-28T02:21:49.197 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_max_pg_blocked_by": "16", 2023-08-28T02:21:49.197 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_max_pg_log_entries": "10000", 2023-08-28T02:21:49.197 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_max_pg_per_osd_hard_ratio": "3.000000", 2023-08-28T02:21:49.197 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_max_pgls": "1024", 2023-08-28T02:21:49.197 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_max_push_cost": "8388608", 2023-08-28T02:21:49.198 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_max_push_objects": "10", 2023-08-28T02:21:49.198 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_max_scrubs": "1", 2023-08-28T02:21:49.198 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_max_snap_prune_intervals_per_epoch": "512", 2023-08-28T02:21:49.198 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_max_trimming_pgs": "2", 2023-08-28T02:21:49.198 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_max_write_op_reply_len": "64", 2023-08-28T02:21:49.198 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_max_write_size": "90", 2023-08-28T02:21:49.198 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_mclock_force_run_benchmark_on_init": "false", 2023-08-28T02:21:49.198 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_mclock_iops_capacity_threshold_hdd": "500.000000", 2023-08-28T02:21:49.198 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_mclock_iops_capacity_threshold_ssd": "80000.000000", 2023-08-28T02:21:49.198 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_mclock_max_capacity_iops_hdd": "315.000000", 2023-08-28T02:21:49.198 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_mclock_max_capacity_iops_ssd": "21500.000000", 2023-08-28T02:21:49.198 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_mclock_max_sequential_bandwidth_hdd": "157286400", 2023-08-28T02:21:49.199 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_mclock_max_sequential_bandwidth_ssd": "1258291200", 2023-08-28T02:21:49.199 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_mclock_override_recovery_settings": "false", 2023-08-28T02:21:49.199 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_mclock_profile": "balanced", 2023-08-28T02:21:49.199 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_mclock_scheduler_anticipation_timeout": "0.000000", 2023-08-28T02:21:49.199 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_mclock_scheduler_background_best_effort_lim": "0.000000", 2023-08-28T02:21:49.199 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_mclock_scheduler_background_best_effort_res": "0.000000", 2023-08-28T02:21:49.199 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_mclock_scheduler_background_best_effort_wgt": "1", 2023-08-28T02:21:49.199 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_mclock_scheduler_background_recovery_lim": "0.000000", 2023-08-28T02:21:49.199 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_mclock_scheduler_background_recovery_res": "0.000000", 2023-08-28T02:21:49.199 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_mclock_scheduler_background_recovery_wgt": "1", 2023-08-28T02:21:49.199 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_mclock_scheduler_client_lim": "0.000000", 2023-08-28T02:21:49.199 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_mclock_scheduler_client_res": "0.000000", 2023-08-28T02:21:49.200 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_mclock_scheduler_client_wgt": "1", 2023-08-28T02:21:49.200 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_mclock_skip_benchmark": "false", 2023-08-28T02:21:49.200 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_memory_base": "805306368", 2023-08-28T02:21:49.200 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_memory_cache_min": "134217728", 2023-08-28T02:21:49.200 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_memory_cache_resize_interval": "1.000000", 2023-08-28T02:21:49.200 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_memory_expected_fragmentation": "0.150000", 2023-08-28T02:21:49.200 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_memory_target": "4294967296", 2023-08-28T02:21:49.200 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_memory_target_autotune": "false", 2023-08-28T02:21:49.200 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_memory_target_cgroup_limit_ratio": "0.800000", 2023-08-28T02:21:49.200 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_min_pg_log_entries": "250", 2023-08-28T02:21:49.200 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_min_recovery_priority": "0", 2023-08-28T02:21:49.200 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_mon_heartbeat_interval": "30", 2023-08-28T02:21:49.200 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_mon_heartbeat_stat_stale": "3600", 2023-08-28T02:21:49.201 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_mon_report_interval": "5", 2023-08-28T02:21:49.201 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_mon_report_max_in_flight": "2", 2023-08-28T02:21:49.201 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_mon_shutdown_timeout": "5.000000", 2023-08-28T02:21:49.201 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_num_cache_shards": "32", 2023-08-28T02:21:49.201 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_num_op_tracker_shard": "32", 2023-08-28T02:21:49.201 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_numa_auto_affinity": "true", 2023-08-28T02:21:49.201 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_numa_node": "-1", 2023-08-28T02:21:49.201 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_numa_prefer_iface": "true", 2023-08-28T02:21:49.201 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_object_clean_region_max_num_intervals": "10", 2023-08-28T02:21:49.201 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_objecter_finishers": "1", 2023-08-28T02:21:49.201 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_objectstore": "bluestore", 2023-08-28T02:21:49.201 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_objectstore_fuse": "false", 2023-08-28T02:21:49.202 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_objectstore_tracing": "false", 2023-08-28T02:21:49.202 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_op_complaint_time": "30.000000", 2023-08-28T02:21:49.202 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_op_history_duration": "600", 2023-08-28T02:21:49.202 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_op_history_size": "20", 2023-08-28T02:21:49.202 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_op_history_slow_op_size": "20", 2023-08-28T02:21:49.202 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_op_history_slow_op_threshold": "10.000000", 2023-08-28T02:21:49.202 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_op_log_threshold": "5", 2023-08-28T02:21:49.202 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_op_num_shards": "0", 2023-08-28T02:21:49.202 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_op_num_shards_hdd": "5", 2023-08-28T02:21:49.202 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_op_num_shards_ssd": "8", 2023-08-28T02:21:49.202 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_op_num_threads_per_shard": "0", 2023-08-28T02:21:49.202 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_op_num_threads_per_shard_hdd": "1", 2023-08-28T02:21:49.203 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_op_num_threads_per_shard_ssd": "2", 2023-08-28T02:21:49.203 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_op_pq_max_tokens_per_priority": "4194304", 2023-08-28T02:21:49.203 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_op_pq_min_cost": "65536", 2023-08-28T02:21:49.203 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_op_queue": "mclock_scheduler", 2023-08-28T02:21:49.203 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_op_queue_cut_off": "high", 2023-08-28T02:21:49.203 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_op_thread_suicide_timeout": "150", 2023-08-28T02:21:49.203 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_op_thread_timeout": "15", 2023-08-28T02:21:49.203 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_open_classes_on_start": "true", 2023-08-28T02:21:49.203 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_os_flags": "0", 2023-08-28T02:21:49.203 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_peering_op_priority": "255", 2023-08-28T02:21:49.203 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_pg_delete_cost": "1048576", 2023-08-28T02:21:49.203 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_pg_delete_priority": "5", 2023-08-28T02:21:49.204 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_pg_epoch_max_lag_factor": "2.000000", 2023-08-28T02:21:49.204 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_pg_epoch_persisted_max_stale": "40", 2023-08-28T02:21:49.204 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_pg_log_dups_tracked": "3000", 2023-08-28T02:21:49.204 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_pg_log_trim_max": "10000", 2023-08-28T02:21:49.204 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_pg_log_trim_min": "100", 2023-08-28T02:21:49.204 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_pg_max_concurrent_snap_trims": "2", 2023-08-28T02:21:49.204 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_pg_object_context_cache_count": "64", 2023-08-28T02:21:49.204 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_pg_stat_report_interval_max": "500", 2023-08-28T02:21:49.204 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_pool_default_cache_max_evict_check_size": "10", 2023-08-28T02:21:49.204 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_pool_default_cache_min_evict_age": "0", 2023-08-28T02:21:49.204 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_pool_default_cache_min_flush_age": "0", 2023-08-28T02:21:49.204 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_pool_default_cache_target_dirty_high_ratio": "0.600000", 2023-08-28T02:21:49.204 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_pool_default_cache_target_dirty_ratio": "0.400000", 2023-08-28T02:21:49.205 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_pool_default_cache_target_full_ratio": "0.800000", 2023-08-28T02:21:49.205 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_pool_default_crimson": "false", 2023-08-28T02:21:49.205 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_pool_default_crush_rule": "-1", 2023-08-28T02:21:49.205 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_pool_default_ec_fast_read": "false", 2023-08-28T02:21:49.205 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_pool_default_erasure_code_profile": "plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd", 2023-08-28T02:21:49.205 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_pool_default_flag_bulk": "false", 2023-08-28T02:21:49.205 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_pool_default_flag_hashpspool": "true", 2023-08-28T02:21:49.205 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_pool_default_flag_nodelete": "false", 2023-08-28T02:21:49.205 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_pool_default_flag_nopgchange": "false", 2023-08-28T02:21:49.205 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_pool_default_flag_nosizechange": "false", 2023-08-28T02:21:49.205 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_pool_default_flags": "0", 2023-08-28T02:21:49.205 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_pool_default_hit_set_bloom_fpp": "0.050000", 2023-08-28T02:21:49.205 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_pool_default_min_size": "0", 2023-08-28T02:21:49.206 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_pool_default_pg_autoscale_mode": "off", 2023-08-28T02:21:49.206 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_pool_default_pg_num": "32", 2023-08-28T02:21:49.206 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_pool_default_pgp_num": "0", 2023-08-28T02:21:49.206 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_pool_default_read_lease_ratio": "0.800000", 2023-08-28T02:21:49.206 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_pool_default_size": "3", 2023-08-28T02:21:49.206 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_pool_default_type": "replicated", 2023-08-28T02:21:49.206 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_pool_erasure_code_stripe_unit": "4096", 2023-08-28T02:21:49.206 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_pool_use_gmt_hitset": "true", 2023-08-28T02:21:49.206 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_push_per_object_cost": "1000", 2023-08-28T02:21:49.206 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_read_ec_check_for_errors": "false", 2023-08-28T02:21:49.206 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_recover_clone_overlap": "true", 2023-08-28T02:21:49.206 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_recover_clone_overlap_limit": "10", 2023-08-28T02:21:49.207 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_recovery_cost": "20971520", 2023-08-28T02:21:49.207 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_recovery_delay_start": "0.000000", 2023-08-28T02:21:49.207 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_recovery_max_active": "0", 2023-08-28T02:21:49.207 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_recovery_max_active_hdd": "3", 2023-08-28T02:21:49.207 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_recovery_max_active_ssd": "10", 2023-08-28T02:21:49.207 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_recovery_max_chunk": "8388608", 2023-08-28T02:21:49.207 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_recovery_max_omap_entries_per_chunk": "8096", 2023-08-28T02:21:49.207 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_recovery_max_single_start": "1", 2023-08-28T02:21:49.207 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_recovery_op_priority": "3", 2023-08-28T02:21:49.207 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_recovery_op_warn_multiple": "16", 2023-08-28T02:21:49.207 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_recovery_priority": "5", 2023-08-28T02:21:49.207 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_recovery_retry_interval": "30.000000", 2023-08-28T02:21:49.208 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_recovery_sleep": "0.000000", 2023-08-28T02:21:49.208 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_recovery_sleep_hdd": "0.100000", 2023-08-28T02:21:49.208 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_recovery_sleep_hybrid": "0.025000", 2023-08-28T02:21:49.208 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_recovery_sleep_ssd": "0.000000", 2023-08-28T02:21:49.208 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_repair_during_recovery": "false", 2023-08-28T02:21:49.208 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_requested_scrub_priority": "120", 2023-08-28T02:21:49.208 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_rocksdb_iterator_bounds_enabled": "true", 2023-08-28T02:21:49.208 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_rollback_to_cluster_snap": "", 2023-08-28T02:21:49.208 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_scrub_auto_repair": "false", 2023-08-28T02:21:49.208 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_scrub_auto_repair_num_errors": "5", 2023-08-28T02:21:49.208 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_scrub_backoff_ratio": "0.660000", 2023-08-28T02:21:49.208 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_scrub_begin_hour": "0", 2023-08-28T02:21:49.208 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_scrub_begin_week_day": "0", 2023-08-28T02:21:49.209 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_scrub_chunk_max": "25", 2023-08-28T02:21:49.209 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_scrub_chunk_min": "5", 2023-08-28T02:21:49.209 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_scrub_cost": "52428800", 2023-08-28T02:21:49.209 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_scrub_during_recovery": "false", 2023-08-28T02:21:49.209 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_scrub_end_hour": "0", 2023-08-28T02:21:49.209 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_scrub_end_week_day": "0", 2023-08-28T02:21:49.209 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_scrub_event_cost": "4096", 2023-08-28T02:21:49.209 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_scrub_extended_sleep": "0.000000", 2023-08-28T02:21:49.209 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_scrub_interval_randomize_ratio": "0.500000", 2023-08-28T02:21:49.209 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_scrub_invalid_stats": "true", 2023-08-28T02:21:49.209 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_scrub_load_threshold": "0.500000", 2023-08-28T02:21:49.209 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_scrub_max_interval": "604800.000000", 2023-08-28T02:21:49.210 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_scrub_max_preemptions": "5", 2023-08-28T02:21:49.210 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_scrub_min_interval": "86400.000000", 2023-08-28T02:21:49.210 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_scrub_priority": "5", 2023-08-28T02:21:49.210 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_scrub_reservation_timeout": "5000", 2023-08-28T02:21:49.210 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_scrub_sleep": "0.000000", 2023-08-28T02:21:49.210 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_scrub_slow_reservation_response": "2200", 2023-08-28T02:21:49.210 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_shallow_scrub_chunk_max": "100", 2023-08-28T02:21:49.210 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_shallow_scrub_chunk_min": "50", 2023-08-28T02:21:49.210 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_shutdown_pgref_assert": "false", 2023-08-28T02:21:49.210 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_skip_data_digest": "false", 2023-08-28T02:21:49.210 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_smart_report_timeout": "5", 2023-08-28T02:21:49.210 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_snap_trim_cost": "1048576", 2023-08-28T02:21:49.211 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_snap_trim_priority": "5", 2023-08-28T02:21:49.211 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_snap_trim_sleep": "0.000000", 2023-08-28T02:21:49.211 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_snap_trim_sleep_hdd": "5.000000", 2023-08-28T02:21:49.211 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_snap_trim_sleep_hybrid": "2.000000", 2023-08-28T02:21:49.211 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_snap_trim_sleep_ssd": "0.000000", 2023-08-28T02:21:49.211 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_stats_update_period_not_scrubbing": "120", 2023-08-28T02:21:49.211 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_stats_update_period_scrubbing": "15", 2023-08-28T02:21:49.211 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_target_pg_log_entries_per_osd": "300000", 2023-08-28T02:21:49.211 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_target_transaction_size": "30", 2023-08-28T02:21:49.211 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_tier_default_cache_hit_set_count": "4", 2023-08-28T02:21:49.211 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_tier_default_cache_hit_set_grade_decay_rate": "20", 2023-08-28T02:21:49.212 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_tier_default_cache_hit_set_period": "1200", 2023-08-28T02:21:49.212 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_tier_default_cache_hit_set_search_last_n": "1", 2023-08-28T02:21:49.212 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_tier_default_cache_hit_set_type": "bloom", 2023-08-28T02:21:49.212 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_tier_default_cache_min_read_recency_for_promote": "1", 2023-08-28T02:21:49.212 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_tier_default_cache_min_write_recency_for_promote": "1", 2023-08-28T02:21:49.212 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_tier_default_cache_mode": "writeback", 2023-08-28T02:21:49.213 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_tier_promote_max_bytes_sec": "5242880", 2023-08-28T02:21:49.213 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_tier_promote_max_objects_sec": "25", 2023-08-28T02:21:49.213 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_tracing": "false", 2023-08-28T02:21:49.213 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_use_stale_snap": "false", 2023-08-28T02:21:49.213 INFO:teuthology.orchestra.run.smithi143.stdout: "osd_uuid": "00000000-0000-0000-0000-000000000000", 2023-08-28T02:21:49.213 INFO:teuthology.orchestra.run.smithi143.stdout: "osdc_blkin_trace_all": "false", 2023-08-28T02:21:49.213 INFO:teuthology.orchestra.run.smithi143.stdout: "paxos_kill_at": "0", 2023-08-28T02:21:49.213 INFO:teuthology.orchestra.run.smithi143.stdout: "paxos_max_join_drift": "10", 2023-08-28T02:21:49.213 INFO:teuthology.orchestra.run.smithi143.stdout: "paxos_min": "500", 2023-08-28T02:21:49.213 INFO:teuthology.orchestra.run.smithi143.stdout: "paxos_min_wait": "0.050000", 2023-08-28T02:21:49.213 INFO:teuthology.orchestra.run.smithi143.stdout: "paxos_propose_interval": "1.000000", 2023-08-28T02:21:49.213 INFO:teuthology.orchestra.run.smithi143.stdout: "paxos_service_trim_max": "500", 2023-08-28T02:21:49.214 INFO:teuthology.orchestra.run.smithi143.stdout: "paxos_service_trim_max_multiplier": "20", 2023-08-28T02:21:49.214 INFO:teuthology.orchestra.run.smithi143.stdout: "paxos_service_trim_min": "250", 2023-08-28T02:21:49.214 INFO:teuthology.orchestra.run.smithi143.stdout: "paxos_stash_full_interval": "25", 2023-08-28T02:21:49.214 INFO:teuthology.orchestra.run.smithi143.stdout: "paxos_trim_max": "500", 2023-08-28T02:21:49.214 INFO:teuthology.orchestra.run.smithi143.stdout: "paxos_trim_min": "250", 2023-08-28T02:21:49.214 INFO:teuthology.orchestra.run.smithi143.stdout: "perf": "true", 2023-08-28T02:21:49.214 INFO:teuthology.orchestra.run.smithi143.stdout: "pid_file": "", 2023-08-28T02:21:49.214 INFO:teuthology.orchestra.run.smithi143.stdout: "plugin_crypto_accelerator": "crypto_isal", 2023-08-28T02:21:49.214 INFO:teuthology.orchestra.run.smithi143.stdout: "plugin_dir": "/usr/lib64/ceph", 2023-08-28T02:21:49.214 INFO:teuthology.orchestra.run.smithi143.stdout: "public_addr": "-", 2023-08-28T02:21:49.214 INFO:teuthology.orchestra.run.smithi143.stdout: "public_addrv": "", 2023-08-28T02:21:49.214 INFO:teuthology.orchestra.run.smithi143.stdout: "public_bind_addr": "-", 2023-08-28T02:21:49.215 INFO:teuthology.orchestra.run.smithi143.stdout: "public_network": "172.21.0.0/20", 2023-08-28T02:21:49.215 INFO:teuthology.orchestra.run.smithi143.stdout: "public_network_interface": "", 2023-08-28T02:21:49.215 INFO:teuthology.orchestra.run.smithi143.stdout: "qat_compressor_enabled": "false", 2023-08-28T02:21:49.215 INFO:teuthology.orchestra.run.smithi143.stdout: "qat_compressor_session_max_number": "256", 2023-08-28T02:21:49.215 INFO:teuthology.orchestra.run.smithi143.stdout: "rados_mon_op_timeout": "0", 2023-08-28T02:21:49.215 INFO:teuthology.orchestra.run.smithi143.stdout: "rados_osd_op_timeout": "0", 2023-08-28T02:21:49.215 INFO:teuthology.orchestra.run.smithi143.stdout: "rados_tracing": "false", 2023-08-28T02:21:49.215 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_atime_update_interval": "60", 2023-08-28T02:21:49.215 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_auto_exclusive_lock_until_manual_request": "true", 2023-08-28T02:21:49.215 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_balance_parent_reads": "false", 2023-08-28T02:21:49.215 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_balance_snap_reads": "false", 2023-08-28T02:21:49.215 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_blkin_trace_all": "false", 2023-08-28T02:21:49.216 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_blocklist_expire_seconds": "0", 2023-08-28T02:21:49.216 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_blocklist_on_break_lock": "true", 2023-08-28T02:21:49.216 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_cache": "true", 2023-08-28T02:21:49.216 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_cache_block_writes_upfront": "false", 2023-08-28T02:21:49.216 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_cache_max_dirty": "25165824", 2023-08-28T02:21:49.216 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_cache_max_dirty_age": "1.000000", 2023-08-28T02:21:49.216 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_cache_max_dirty_object": "0", 2023-08-28T02:21:49.216 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_cache_policy": "writearound", 2023-08-28T02:21:49.216 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_cache_size": "33554432", 2023-08-28T02:21:49.216 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_cache_target_dirty": "16777216", 2023-08-28T02:21:49.216 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_cache_writethrough_until_flush": "true", 2023-08-28T02:21:49.216 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_clone_copy_on_read": "false", 2023-08-28T02:21:49.217 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_compression_hint": "none", 2023-08-28T02:21:49.217 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_concurrent_management_ops": "10", 2023-08-28T02:21:49.217 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_config_pool_override_update_timestamp": "0", 2023-08-28T02:21:49.217 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_default_clone_format": "auto", 2023-08-28T02:21:49.217 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_default_data_pool": "", 2023-08-28T02:21:49.217 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_default_features": "61", 2023-08-28T02:21:49.217 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_default_format": "2", 2023-08-28T02:21:49.217 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_default_map_options": "", 2023-08-28T02:21:49.217 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_default_order": "22", 2023-08-28T02:21:49.217 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_default_pool": "rbd", 2023-08-28T02:21:49.217 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_default_snapshot_quiesce_mode": "required", 2023-08-28T02:21:49.217 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_default_stripe_count": "0", 2023-08-28T02:21:49.218 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_default_stripe_unit": "0", 2023-08-28T02:21:49.218 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_disable_zero_copy_writes": "true", 2023-08-28T02:21:49.218 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_discard_granularity_bytes": "65536", 2023-08-28T02:21:49.218 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_discard_on_zeroed_write_same": "true", 2023-08-28T02:21:49.218 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_enable_alloc_hint": "true", 2023-08-28T02:21:49.218 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_invalidate_object_map_on_timeout": "true", 2023-08-28T02:21:49.218 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_io_scheduler": "simple", 2023-08-28T02:21:49.218 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_io_scheduler_simple_max_delay": "0", 2023-08-28T02:21:49.218 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_journal_commit_age": "5.000000", 2023-08-28T02:21:49.218 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_journal_max_concurrent_object_sets": "0", 2023-08-28T02:21:49.218 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_journal_max_payload_bytes": "16384", 2023-08-28T02:21:49.218 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_journal_object_flush_age": "0.000000", 2023-08-28T02:21:49.219 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_journal_object_flush_bytes": "1048576", 2023-08-28T02:21:49.219 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_journal_object_flush_interval": "0", 2023-08-28T02:21:49.219 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_journal_object_max_in_flight_appends": "0", 2023-08-28T02:21:49.219 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_journal_object_writethrough_until_flush": "true", 2023-08-28T02:21:49.219 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_journal_order": "24", 2023-08-28T02:21:49.219 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_journal_pool": "", 2023-08-28T02:21:49.219 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_journal_splay_width": "4", 2023-08-28T02:21:49.219 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_localize_parent_reads": "false", 2023-08-28T02:21:49.219 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_localize_snap_reads": "false", 2023-08-28T02:21:49.219 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_mirror_concurrent_image_deletions": "1", 2023-08-28T02:21:49.219 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_mirror_concurrent_image_syncs": "5", 2023-08-28T02:21:49.219 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_mirror_delete_retry_interval": "30.000000", 2023-08-28T02:21:49.220 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_mirror_image_perf_stats_prio": "5", 2023-08-28T02:21:49.220 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_mirror_image_policy_migration_throttle": "300", 2023-08-28T02:21:49.220 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_mirror_image_policy_rebalance_timeout": "0.000000", 2023-08-28T02:21:49.220 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_mirror_image_policy_type": "simple", 2023-08-28T02:21:49.220 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_mirror_image_policy_update_throttle_interval": "1.000000", 2023-08-28T02:21:49.220 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_mirror_image_state_check_interval": "30", 2023-08-28T02:21:49.220 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_mirror_journal_commit_age": "5.000000", 2023-08-28T02:21:49.220 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_mirror_journal_poll_age": "5.000000", 2023-08-28T02:21:49.220 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_mirror_leader_heartbeat_interval": "5", 2023-08-28T02:21:49.220 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_mirror_leader_max_acquire_attempts_before_break": "3", 2023-08-28T02:21:49.220 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_mirror_leader_max_missed_heartbeats": "2", 2023-08-28T02:21:49.220 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_mirror_memory_autotune": "true", 2023-08-28T02:21:49.220 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_mirror_memory_base": "805306368", 2023-08-28T02:21:49.221 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_mirror_memory_cache_autotune_interval": "30.000000", 2023-08-28T02:21:49.221 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_mirror_memory_cache_min": "134217728", 2023-08-28T02:21:49.221 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_mirror_memory_cache_resize_interval": "5.000000", 2023-08-28T02:21:49.221 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_mirror_memory_expected_fragmentation": "0.150000", 2023-08-28T02:21:49.221 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_mirror_memory_target": "4294967296", 2023-08-28T02:21:49.221 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_mirror_perf_stats_prio": "5", 2023-08-28T02:21:49.221 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_mirror_pool_replayers_refresh_interval": "30", 2023-08-28T02:21:49.221 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_mirror_sync_point_update_age": "30.000000", 2023-08-28T02:21:49.221 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_mirroring_delete_delay": "0", 2023-08-28T02:21:49.221 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_mirroring_max_mirroring_snapshots": "5", 2023-08-28T02:21:49.221 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_mirroring_replay_delay": "0", 2023-08-28T02:21:49.221 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_mirroring_resync_after_disconnect": "false", 2023-08-28T02:21:49.222 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_move_parent_to_trash_on_remove": "false", 2023-08-28T02:21:49.222 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_move_to_trash_on_remove": "false", 2023-08-28T02:21:49.222 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_move_to_trash_on_remove_expire_seconds": "0", 2023-08-28T02:21:49.222 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_mtime_update_interval": "60", 2023-08-28T02:21:49.222 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_non_blocking_aio": "true", 2023-08-28T02:21:49.222 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_op_thread_timeout": "60", 2023-08-28T02:21:49.222 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_op_threads": "1", 2023-08-28T02:21:49.222 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_parent_cache_enabled": "false", 2023-08-28T02:21:49.222 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_persistent_cache_mode": "disabled", 2023-08-28T02:21:49.222 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_persistent_cache_path": "/tmp", 2023-08-28T02:21:49.222 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_persistent_cache_size": "1073741824", 2023-08-28T02:21:49.222 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_plugins": "", 2023-08-28T02:21:49.223 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_qos_bps_burst": "0", 2023-08-28T02:21:49.223 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_qos_bps_burst_seconds": "1", 2023-08-28T02:21:49.223 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_qos_bps_limit": "0", 2023-08-28T02:21:49.223 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_qos_exclude_ops": "0", 2023-08-28T02:21:49.223 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_qos_iops_burst": "0", 2023-08-28T02:21:49.223 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_qos_iops_burst_seconds": "1", 2023-08-28T02:21:49.223 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_qos_iops_limit": "0", 2023-08-28T02:21:49.223 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_qos_read_bps_burst": "0", 2023-08-28T02:21:49.223 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_qos_read_bps_burst_seconds": "1", 2023-08-28T02:21:49.223 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_qos_read_bps_limit": "0", 2023-08-28T02:21:49.223 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_qos_read_iops_burst": "0", 2023-08-28T02:21:49.223 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_qos_read_iops_burst_seconds": "1", 2023-08-28T02:21:49.223 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_qos_read_iops_limit": "0", 2023-08-28T02:21:49.224 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_qos_schedule_tick_min": "50", 2023-08-28T02:21:49.224 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_qos_write_bps_burst": "0", 2023-08-28T02:21:49.224 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_qos_write_bps_burst_seconds": "1", 2023-08-28T02:21:49.224 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_qos_write_bps_limit": "0", 2023-08-28T02:21:49.224 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_qos_write_iops_burst": "0", 2023-08-28T02:21:49.224 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_qos_write_iops_burst_seconds": "1", 2023-08-28T02:21:49.224 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_qos_write_iops_limit": "0", 2023-08-28T02:21:49.224 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_quiesce_notification_attempts": "10", 2023-08-28T02:21:49.224 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_read_from_replica_policy": "default", 2023-08-28T02:21:49.224 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_readahead_disable_after_bytes": "52428800", 2023-08-28T02:21:49.224 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_readahead_max_bytes": "524288", 2023-08-28T02:21:49.224 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_readahead_trigger_requests": "10", 2023-08-28T02:21:49.225 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_request_timed_out_seconds": "30", 2023-08-28T02:21:49.225 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_skip_partial_discard": "true", 2023-08-28T02:21:49.225 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_sparse_read_threshold_bytes": "65536", 2023-08-28T02:21:49.225 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_tracing": "false", 2023-08-28T02:21:49.225 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_validate_names": "true", 2023-08-28T02:21:49.225 INFO:teuthology.orchestra.run.smithi143.stdout: "rbd_validate_pool": "true", 2023-08-28T02:21:49.225 INFO:teuthology.orchestra.run.smithi143.stdout: "restapi_base_url": "", 2023-08-28T02:21:49.225 INFO:teuthology.orchestra.run.smithi143.stdout: "restapi_log_level": "", 2023-08-28T02:21:49.225 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_acl_grants_max_num": "100", 2023-08-28T02:21:49.225 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_admin_entry": "admin", 2023-08-28T02:21:49.225 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_allow_notification_secrets_in_cleartext": "false", 2023-08-28T02:21:49.225 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_backend_store": "rados", 2023-08-28T02:21:49.226 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_barbican_url": "", 2023-08-28T02:21:49.226 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_beast_enable_async": "true", 2023-08-28T02:21:49.226 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_bucket_default_quota_max_objects": "-1", 2023-08-28T02:21:49.226 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_bucket_default_quota_max_size": "-1", 2023-08-28T02:21:49.226 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_bucket_index_max_aio": "128", 2023-08-28T02:21:49.226 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_bucket_index_transaction_instrumentation": "false", 2023-08-28T02:21:49.226 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_bucket_quota_cache_size": "10000", 2023-08-28T02:21:49.226 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_bucket_quota_ttl": "600", 2023-08-28T02:21:49.226 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_bucket_sync_spawn_window": "20", 2023-08-28T02:21:49.226 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_cache_enabled": "true", 2023-08-28T02:21:49.226 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_cache_expiry_interval": "900", 2023-08-28T02:21:49.227 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_cache_lru_size": "10000", 2023-08-28T02:21:49.227 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_config_store": "rados", 2023-08-28T02:21:49.227 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_content_length_compat": "false", 2023-08-28T02:21:49.227 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_copy_obj_progress": "true", 2023-08-28T02:21:49.227 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_copy_obj_progress_every_bytes": "1048576", 2023-08-28T02:21:49.227 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_cors_rules_max_num": "100", 2023-08-28T02:21:49.227 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_cross_domain_policy": "", 2023-08-28T02:21:49.227 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_crypt_default_encryption_key": "", 2023-08-28T02:21:49.227 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_crypt_kmip_addr": "", 2023-08-28T02:21:49.227 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_crypt_kmip_ca_path": "", 2023-08-28T02:21:49.227 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_crypt_kmip_client_cert": "", 2023-08-28T02:21:49.227 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_crypt_kmip_client_key": "", 2023-08-28T02:21:49.227 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_crypt_kmip_kms_key_template": "", 2023-08-28T02:21:49.228 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_crypt_kmip_password": "", 2023-08-28T02:21:49.228 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_crypt_kmip_s3_key_template": "$keyid", 2023-08-28T02:21:49.228 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_crypt_kmip_username": "", 2023-08-28T02:21:49.228 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_crypt_require_ssl": "true", 2023-08-28T02:21:49.228 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_crypt_s3_kms_backend": "barbican", 2023-08-28T02:21:49.228 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_crypt_s3_kms_encryption_keys": "", 2023-08-28T02:21:49.228 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_crypt_sse_s3_backend": "vault", 2023-08-28T02:21:49.228 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_crypt_sse_s3_key_template": "%bucket_id", 2023-08-28T02:21:49.228 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_crypt_sse_s3_vault_addr": "", 2023-08-28T02:21:49.228 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_crypt_sse_s3_vault_auth": "token", 2023-08-28T02:21:49.228 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_crypt_sse_s3_vault_namespace": "", 2023-08-28T02:21:49.228 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_crypt_sse_s3_vault_prefix": "", 2023-08-28T02:21:49.229 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_crypt_sse_s3_vault_secret_engine": "transit", 2023-08-28T02:21:49.229 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_crypt_sse_s3_vault_ssl_cacert": "", 2023-08-28T02:21:49.229 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_crypt_sse_s3_vault_ssl_clientcert": "", 2023-08-28T02:21:49.229 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_crypt_sse_s3_vault_ssl_clientkey": "", 2023-08-28T02:21:49.229 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_crypt_sse_s3_vault_token_file": "", 2023-08-28T02:21:49.229 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_crypt_sse_s3_vault_verify_ssl": "true", 2023-08-28T02:21:49.229 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_crypt_suppress_logs": "true", 2023-08-28T02:21:49.229 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_crypt_vault_addr": "", 2023-08-28T02:21:49.229 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_crypt_vault_auth": "token", 2023-08-28T02:21:49.229 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_crypt_vault_namespace": "", 2023-08-28T02:21:49.229 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_crypt_vault_prefix": "", 2023-08-28T02:21:49.229 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_crypt_vault_secret_engine": "transit", 2023-08-28T02:21:49.229 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_crypt_vault_ssl_cacert": "", 2023-08-28T02:21:49.230 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_crypt_vault_ssl_clientcert": "", 2023-08-28T02:21:49.230 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_crypt_vault_ssl_clientkey": "", 2023-08-28T02:21:49.230 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_crypt_vault_token_file": "", 2023-08-28T02:21:49.230 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_crypt_vault_verify_ssl": "true", 2023-08-28T02:21:49.230 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_curl_buffersize": "524288", 2023-08-28T02:21:49.230 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_curl_low_speed_limit": "1024", 2023-08-28T02:21:49.230 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_curl_low_speed_time": "300", 2023-08-28T02:21:49.230 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_curl_tcp_keepalive": "0", 2023-08-28T02:21:49.230 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_curl_wait_timeout_ms": "1000", 2023-08-28T02:21:49.230 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_d3n_l1_datacache_persistent_path": "/tmp/rgw_datacache/", 2023-08-28T02:21:49.230 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_d3n_l1_datacache_size": "1073741824", 2023-08-28T02:21:49.230 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_d3n_l1_evict_cache_on_start": "true", 2023-08-28T02:21:49.231 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_d3n_l1_eviction_policy": "lru", 2023-08-28T02:21:49.231 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_d3n_l1_fadvise": "4", 2023-08-28T02:21:49.231 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_d3n_l1_local_datacache_enabled": "false", 2023-08-28T02:21:49.231 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_d3n_libaio_aio_num": "64", 2023-08-28T02:21:49.231 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_d3n_libaio_aio_threads": "20", 2023-08-28T02:21:49.231 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_data": "/var/lib/ceph/radosgw/ceph-a", 2023-08-28T02:21:49.231 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_data_log_changes_size": "1000", 2023-08-28T02:21:49.231 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_data_log_num_shards": "128", 2023-08-28T02:21:49.231 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_data_log_obj_prefix": "data_log", 2023-08-28T02:21:49.231 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_data_log_window": "30", 2023-08-28T02:21:49.231 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_data_notify_interval_msec": "0", 2023-08-28T02:21:49.231 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_data_sync_poll_interval": "20", 2023-08-28T02:21:49.231 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_data_sync_spawn_window": "20", 2023-08-28T02:21:49.232 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_debug_inject_olh_cancel_modification_err": "false", 2023-08-28T02:21:49.232 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_debug_inject_set_olh_err": "0", 2023-08-28T02:21:49.232 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_default_data_log_backing": "fifo", 2023-08-28T02:21:49.232 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_default_realm_info_oid": "default.realm", 2023-08-28T02:21:49.232 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_default_region_info_oid": "default.region", 2023-08-28T02:21:49.232 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_default_zone_info_oid": "default.zone", 2023-08-28T02:21:49.232 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_default_zonegroup_info_oid": "default.zonegroup", 2023-08-28T02:21:49.232 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_defer_to_bucket_acls": "", 2023-08-28T02:21:49.232 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_delete_multi_obj_max_num": "1000", 2023-08-28T02:21:49.232 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_dmclock_admin_lim": "0.000000", 2023-08-28T02:21:49.232 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_dmclock_admin_res": "100.000000", 2023-08-28T02:21:49.232 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_dmclock_admin_wgt": "100.000000", 2023-08-28T02:21:49.232 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_dmclock_auth_lim": "0.000000", 2023-08-28T02:21:49.233 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_dmclock_auth_res": "200.000000", 2023-08-28T02:21:49.233 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_dmclock_auth_wgt": "100.000000", 2023-08-28T02:21:49.233 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_dmclock_data_lim": "0.000000", 2023-08-28T02:21:49.233 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_dmclock_data_res": "500.000000", 2023-08-28T02:21:49.233 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_dmclock_data_wgt": "500.000000", 2023-08-28T02:21:49.233 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_dmclock_metadata_lim": "0.000000", 2023-08-28T02:21:49.233 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_dmclock_metadata_res": "500.000000", 2023-08-28T02:21:49.233 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_dmclock_metadata_wgt": "500.000000", 2023-08-28T02:21:49.233 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_dns_name": "", 2023-08-28T02:21:49.233 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_dns_s3website_name": "", 2023-08-28T02:21:49.233 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_dynamic_resharding": "true", 2023-08-28T02:21:49.233 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_enable_apis": "s3, s3website, swift, swift_auth, admin, sts, iam, notifications", 2023-08-28T02:21:49.233 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_enable_gc_threads": "true", 2023-08-28T02:21:49.234 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_enable_lc_threads": "true", 2023-08-28T02:21:49.234 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_enable_ops_log": "false", 2023-08-28T02:21:49.234 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_enable_quota_threads": "true", 2023-08-28T02:21:49.234 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_enable_static_website": "false", 2023-08-28T02:21:49.234 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_enable_usage_log": "false", 2023-08-28T02:21:49.234 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_enforce_swift_acls": "true", 2023-08-28T02:21:49.234 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_exit_timeout_secs": "120", 2023-08-28T02:21:49.234 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_expose_bucket": "false", 2023-08-28T02:21:49.234 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_extended_http_attrs": "", 2023-08-28T02:21:49.234 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_filter": "none", 2023-08-28T02:21:49.234 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_frontend_defaults": "beast ssl_certificate=config://rgw/cert/$realm/$zone.crt ssl_private_key=config://rgw/cert/$realm/$zone.key", 2023-08-28T02:21:49.234 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_frontends": "beast port=7480", 2023-08-28T02:21:49.235 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_gc_max_concurrent_io": "10", 2023-08-28T02:21:49.235 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_gc_max_deferred": "50", 2023-08-28T02:21:49.235 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_gc_max_deferred_entries_size": "3072", 2023-08-28T02:21:49.235 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_gc_max_objs": "32", 2023-08-28T02:21:49.235 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_gc_max_queue_size": "134213632", 2023-08-28T02:21:49.235 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_gc_max_trim_chunk": "16", 2023-08-28T02:21:49.235 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_gc_obj_min_wait": "7200", 2023-08-28T02:21:49.235 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_gc_processor_max_time": "3600", 2023-08-28T02:21:49.235 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_gc_processor_period": "3600", 2023-08-28T02:21:49.235 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_get_obj_max_req_size": "4194304", 2023-08-28T02:21:49.235 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_get_obj_window_size": "16777216", 2023-08-28T02:21:49.235 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_healthcheck_disabling_path": "", 2023-08-28T02:21:49.235 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_ignore_get_invalid_range": "false", 2023-08-28T02:21:49.236 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_init_timeout": "300", 2023-08-28T02:21:49.236 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_inject_notify_timeout_probability": "0.000000", 2023-08-28T02:21:49.236 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_json_config": "/var/lib/ceph/radosgw/config.json", 2023-08-28T02:21:49.236 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_keystone_accepted_admin_roles": "", 2023-08-28T02:21:49.236 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_keystone_accepted_roles": "Member, admin", 2023-08-28T02:21:49.236 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_keystone_admin_domain": "", 2023-08-28T02:21:49.236 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_keystone_admin_password": "", 2023-08-28T02:21:49.236 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_keystone_admin_password_path": "", 2023-08-28T02:21:49.236 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_keystone_admin_project": "", 2023-08-28T02:21:49.236 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_keystone_admin_tenant": "", 2023-08-28T02:21:49.236 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_keystone_admin_token": "", 2023-08-28T02:21:49.236 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_keystone_admin_token_path": "", 2023-08-28T02:21:49.237 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_keystone_admin_user": "", 2023-08-28T02:21:49.237 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_keystone_api_version": "2", 2023-08-28T02:21:49.237 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_keystone_barbican_domain": "", 2023-08-28T02:21:49.237 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_keystone_barbican_password": "", 2023-08-28T02:21:49.237 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_keystone_barbican_project": "", 2023-08-28T02:21:49.237 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_keystone_barbican_tenant": "", 2023-08-28T02:21:49.237 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_keystone_barbican_user": "", 2023-08-28T02:21:49.237 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_keystone_expired_token_cache_expiration": "3600", 2023-08-28T02:21:49.237 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_keystone_implicit_tenants": "false", 2023-08-28T02:21:49.237 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_keystone_service_token_accepted_roles": "admin", 2023-08-28T02:21:49.237 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_keystone_service_token_enabled": "false", 2023-08-28T02:21:49.237 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_keystone_token_cache_size": "10000", 2023-08-28T02:21:49.238 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_keystone_url": "", 2023-08-28T02:21:49.238 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_keystone_verify_ssl": "true", 2023-08-28T02:21:49.238 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_lc_debug_interval": "-1", 2023-08-28T02:21:49.238 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_lc_lock_max_time": "90", 2023-08-28T02:21:49.238 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_lc_max_objs": "32", 2023-08-28T02:21:49.238 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_lc_max_rules": "1000", 2023-08-28T02:21:49.238 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_lc_max_worker": "3", 2023-08-28T02:21:49.238 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_lc_max_wp_worker": "3", 2023-08-28T02:21:49.238 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_lc_thread_delay": "0", 2023-08-28T02:21:49.238 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_ldap_binddn": "uid=admin,cn=users,dc=example,dc=com", 2023-08-28T02:21:49.238 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_ldap_dnattr": "uid", 2023-08-28T02:21:49.238 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_ldap_searchdn": "cn=users,cn=accounts,dc=example,dc=com", 2023-08-28T02:21:49.238 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_ldap_searchfilter": "", 2023-08-28T02:21:49.239 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_ldap_secret": "/etc/openldap/secret", 2023-08-28T02:21:49.239 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_ldap_uri": "ldaps://", 2023-08-28T02:21:49.239 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_lifecycle_work_time": "00:00-06:00", 2023-08-28T02:21:49.239 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_list_bucket_min_readahead": "1000", 2023-08-28T02:21:49.239 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_list_buckets_max_chunk": "1000", 2023-08-28T02:21:49.239 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_log_http_headers": "", 2023-08-28T02:21:49.239 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_log_nonexistent_bucket": "false", 2023-08-28T02:21:49.239 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_log_object_name": "%Y-%m-%d-%H-%i-%n", 2023-08-28T02:21:49.239 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_log_object_name_utc": "false", 2023-08-28T02:21:49.239 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_luarocks_location": "/tmp/luarocks", 2023-08-28T02:21:49.239 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_max_attr_name_len": "0", 2023-08-28T02:21:49.239 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_max_attr_size": "0", 2023-08-28T02:21:49.240 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_max_attrs_num_in_req": "0", 2023-08-28T02:21:49.240 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_max_chunk_size": "4194304", 2023-08-28T02:21:49.240 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_max_concurrent_requests": "1024", 2023-08-28T02:21:49.240 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_max_copy_obj_concurrent_io": "10", 2023-08-28T02:21:49.240 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_max_dynamic_shards": "1999", 2023-08-28T02:21:49.240 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_max_listing_results": "1000", 2023-08-28T02:21:49.240 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_max_notify_retries": "10", 2023-08-28T02:21:49.240 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_max_objs_per_shard": "100000", 2023-08-28T02:21:49.240 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_max_put_param_size": "1048576", 2023-08-28T02:21:49.240 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_max_put_size": "5368709120", 2023-08-28T02:21:49.240 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_max_slo_entries": "1000", 2023-08-28T02:21:49.240 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_md_log_max_shards": "64", 2023-08-28T02:21:49.240 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_md_notify_interval_msec": "200", 2023-08-28T02:21:49.241 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_meta_sync_poll_interval": "20", 2023-08-28T02:21:49.241 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_meta_sync_spawn_window": "20", 2023-08-28T02:21:49.241 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_mime_types_file": "/etc/mime.types", 2023-08-28T02:21:49.241 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_mp_lock_max_time": "600", 2023-08-28T02:21:49.241 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_multi_obj_del_max_aio": "16", 2023-08-28T02:21:49.241 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_multipart_min_part_size": "5242880", 2023-08-28T02:21:49.241 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_multipart_part_upload_limit": "10000", 2023-08-28T02:21:49.241 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_nfs_fhcache_partitions": "3", 2023-08-28T02:21:49.241 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_nfs_fhcache_size": "2017", 2023-08-28T02:21:49.241 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_nfs_frontends": "rgw-nfs", 2023-08-28T02:21:49.241 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_nfs_lru_lane_hiwat": "911", 2023-08-28T02:21:49.242 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_nfs_lru_lanes": "5", 2023-08-28T02:21:49.242 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_nfs_max_gc": "300", 2023-08-28T02:21:49.242 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_nfs_namespace_expire_secs": "300", 2023-08-28T02:21:49.242 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_nfs_run_gc_threads": "false", 2023-08-28T02:21:49.242 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_nfs_run_lc_threads": "false", 2023-08-28T02:21:49.242 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_nfs_run_quota_threads": "false", 2023-08-28T02:21:49.242 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_nfs_run_sync_thread": "false", 2023-08-28T02:21:49.242 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_nfs_s3_fast_attrs": "false", 2023-08-28T02:21:49.242 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_nfs_write_completion_interval_s": "10", 2023-08-28T02:21:49.242 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_num_async_rados_threads": "32", 2023-08-28T02:21:49.242 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_num_control_oids": "8", 2023-08-28T02:21:49.242 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_numa_node": "-1", 2023-08-28T02:21:49.243 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_obj_stripe_size": "4194304", 2023-08-28T02:21:49.243 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_obj_tombstone_cache_size": "1000", 2023-08-28T02:21:49.243 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_objexp_chunk_size": "100", 2023-08-28T02:21:49.243 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_objexp_gc_interval": "600", 2023-08-28T02:21:49.243 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_objexp_hints_num_shards": "127", 2023-08-28T02:21:49.243 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_olh_pending_timeout_sec": "3600", 2023-08-28T02:21:49.243 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_op_thread_suicide_timeout": "0", 2023-08-28T02:21:49.243 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_op_thread_timeout": "600", 2023-08-28T02:21:49.243 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_op_tracing": "false", 2023-08-28T02:21:49.243 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_opa_token": "", 2023-08-28T02:21:49.243 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_opa_url": "", 2023-08-28T02:21:49.243 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_opa_verify_ssl": "true", 2023-08-28T02:21:49.243 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_ops_log_data_backlog": "5242880", 2023-08-28T02:21:49.244 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_ops_log_file_path": "/var/log/ceph/ops-log-ceph-mon.a.log", 2023-08-28T02:21:49.244 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_ops_log_rados": "false", 2023-08-28T02:21:49.244 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_ops_log_socket_path": "", 2023-08-28T02:21:49.244 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_override_bucket_index_max_shards": "0", 2023-08-28T02:21:49.244 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_pending_bucket_index_op_expiration": "120", 2023-08-28T02:21:49.244 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_period_latest_epoch_info_oid": ".latest_epoch", 2023-08-28T02:21:49.244 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_period_push_interval": "2.000000", 2023-08-28T02:21:49.244 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_period_push_interval_max": "30.000000", 2023-08-28T02:21:49.244 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_period_root_pool": ".rgw.root", 2023-08-28T02:21:49.244 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_policy_reject_invalid_principals": "true", 2023-08-28T02:21:49.244 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_print_continue": "true", 2023-08-28T02:21:49.244 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_print_prohibited_content_length": "false", 2023-08-28T02:21:49.244 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_put_obj_max_window_size": "67108864", 2023-08-28T02:21:49.245 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_put_obj_min_window_size": "16777216", 2023-08-28T02:21:49.245 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_rados_pool_autoscale_bias": "4.000000", 2023-08-28T02:21:49.245 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_rados_pool_recovery_priority": "5", 2023-08-28T02:21:49.245 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_rados_tracing": "false", 2023-08-28T02:21:49.245 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_realm": "", 2023-08-28T02:21:49.245 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_realm_id": "", 2023-08-28T02:21:49.245 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_realm_root_pool": ".rgw.root", 2023-08-28T02:21:49.245 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_region": "", 2023-08-28T02:21:49.245 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_region_root_pool": ".rgw.root", 2023-08-28T02:21:49.245 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_relaxed_region_enforcement": "false", 2023-08-28T02:21:49.245 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_relaxed_s3_bucket_names": "false", 2023-08-28T02:21:49.245 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_remote_addr_param": "REMOTE_ADDR", 2023-08-28T02:21:49.246 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_request_uri": "", 2023-08-28T02:21:49.246 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_reshard_batch_size": "64", 2023-08-28T02:21:49.246 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_reshard_bucket_lock_duration": "360", 2023-08-28T02:21:49.246 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_reshard_max_aio": "128", 2023-08-28T02:21:49.246 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_reshard_num_logs": "16", 2023-08-28T02:21:49.246 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_reshard_thread_interval": "600", 2023-08-28T02:21:49.246 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_resolve_cname": "false", 2023-08-28T02:21:49.246 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_rest_getusage_op_compat": "false", 2023-08-28T02:21:49.246 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_run_sync_thread": "true", 2023-08-28T02:21:49.246 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_s3_auth_order": "sts, external, local", 2023-08-28T02:21:49.246 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_s3_auth_use_keystone": "false", 2023-08-28T02:21:49.246 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_s3_auth_use_ldap": "false", 2023-08-28T02:21:49.247 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_s3_auth_use_rados": "true", 2023-08-28T02:21:49.247 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_s3_auth_use_sts": "false", 2023-08-28T02:21:49.247 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_s3_client_max_sig_ver": "-1", 2023-08-28T02:21:49.247 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_s3_success_create_obj_status": "0", 2023-08-28T02:21:49.247 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_safe_max_objects_per_shard": "102400", 2023-08-28T02:21:49.247 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_scheduler_type": "throttler", 2023-08-28T02:21:49.247 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_script_uri": "", 2023-08-28T02:21:49.247 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_service_provider_name": "", 2023-08-28T02:21:49.247 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_shard_warning_threshold": "90.000000", 2023-08-28T02:21:49.247 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_sts_client_id": "", 2023-08-28T02:21:49.247 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_sts_client_secret": "", 2023-08-28T02:21:49.247 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_sts_entry": "sts", 2023-08-28T02:21:49.248 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_sts_key": "sts", 2023-08-28T02:21:49.248 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_sts_max_session_duration": "43200", 2023-08-28T02:21:49.248 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_sts_min_session_duration": "900", 2023-08-28T02:21:49.248 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_sts_token_introspection_url": "", 2023-08-28T02:21:49.248 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_swift_account_in_url": "false", 2023-08-28T02:21:49.248 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_swift_auth_entry": "auth", 2023-08-28T02:21:49.248 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_swift_auth_url": "", 2023-08-28T02:21:49.248 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_swift_custom_header": "", 2023-08-28T02:21:49.248 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_swift_enforce_content_length": "false", 2023-08-28T02:21:49.248 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_swift_need_stats": "true", 2023-08-28T02:21:49.248 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_swift_tenant_name": "", 2023-08-28T02:21:49.248 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_swift_token_expiration": "86400", 2023-08-28T02:21:49.248 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_swift_url": "", 2023-08-28T02:21:49.249 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_swift_url_prefix": "swift", 2023-08-28T02:21:49.249 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_swift_versioning_enabled": "false", 2023-08-28T02:21:49.249 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_sync_data_full_inject_err_probability": "0.000000", 2023-08-28T02:21:49.249 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_sync_data_inject_err_probability": "0.000000", 2023-08-28T02:21:49.249 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_sync_lease_period": "120", 2023-08-28T02:21:49.249 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_sync_log_trim_concurrent_buckets": "4", 2023-08-28T02:21:49.249 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_sync_log_trim_interval": "1200", 2023-08-28T02:21:49.249 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_sync_log_trim_max_buckets": "16", 2023-08-28T02:21:49.249 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_sync_log_trim_min_cold_buckets": "4", 2023-08-28T02:21:49.249 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_sync_meta_inject_err_probability": "0.000000", 2023-08-28T02:21:49.249 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_sync_obj_etag_verify": "false", 2023-08-28T02:21:49.249 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_sync_trace_history_size": "4096", 2023-08-28T02:21:49.250 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_sync_trace_per_node_log_size": "32", 2023-08-28T02:21:49.250 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_sync_trace_servicemap_update_interval": "10", 2023-08-28T02:21:49.250 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_thread_pool_size": "512", 2023-08-28T02:21:49.250 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_torrent_comment": "", 2023-08-28T02:21:49.250 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_torrent_createby": "", 2023-08-28T02:21:49.250 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_torrent_encoding": "", 2023-08-28T02:21:49.250 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_torrent_flag": "false", 2023-08-28T02:21:49.250 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_torrent_origin": "", 2023-08-28T02:21:49.250 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_torrent_sha_unit": "524288", 2023-08-28T02:21:49.250 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_torrent_tracker": "", 2023-08-28T02:21:49.250 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_trust_forwarded_https": "false", 2023-08-28T02:21:49.250 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_usage_log_flush_threshold": "1024", 2023-08-28T02:21:49.250 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_usage_log_tick_interval": "30", 2023-08-28T02:21:49.251 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_usage_max_shards": "32", 2023-08-28T02:21:49.251 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_usage_max_user_shards": "1", 2023-08-28T02:21:49.251 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_use_opa_authz": "false", 2023-08-28T02:21:49.251 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_user_default_quota_max_objects": "-1", 2023-08-28T02:21:49.251 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_user_default_quota_max_size": "-1", 2023-08-28T02:21:49.251 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_user_max_buckets": "1000", 2023-08-28T02:21:49.251 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_user_policies_max_num": "100", 2023-08-28T02:21:49.251 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_user_quota_bucket_sync_interval": "180", 2023-08-28T02:21:49.251 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_user_quota_sync_idle_users": "false", 2023-08-28T02:21:49.251 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_user_quota_sync_interval": "86400", 2023-08-28T02:21:49.251 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_user_quota_sync_wait_time": "86400", 2023-08-28T02:21:49.251 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_user_unique_email": "true", 2023-08-28T02:21:49.252 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_verify_ssl": "true", 2023-08-28T02:21:49.252 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_website_routing_rules_max_num": "50", 2023-08-28T02:21:49.252 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_zone": "", 2023-08-28T02:21:49.252 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_zone_id": "", 2023-08-28T02:21:49.252 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_zone_root_pool": ".rgw.root", 2023-08-28T02:21:49.252 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_zonegroup": "", 2023-08-28T02:21:49.252 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_zonegroup_id": "", 2023-08-28T02:21:49.252 INFO:teuthology.orchestra.run.smithi143.stdout: "rgw_zonegroup_root_pool": ".rgw.root", 2023-08-28T02:21:49.252 INFO:teuthology.orchestra.run.smithi143.stdout: "rgwlc_auto_session_clear": "true", 2023-08-28T02:21:49.252 INFO:teuthology.orchestra.run.smithi143.stdout: "rgwlc_skip_bucket_step": "false", 2023-08-28T02:21:49.252 INFO:teuthology.orchestra.run.smithi143.stdout: "rocksdb_block_size": "4096", 2023-08-28T02:21:49.252 INFO:teuthology.orchestra.run.smithi143.stdout: "rocksdb_bloom_bits_per_key": "20", 2023-08-28T02:21:49.252 INFO:teuthology.orchestra.run.smithi143.stdout: "rocksdb_cache_index_and_filter_blocks": "true", 2023-08-28T02:21:49.253 INFO:teuthology.orchestra.run.smithi143.stdout: "rocksdb_cache_index_and_filter_blocks_with_high_priority": "false", 2023-08-28T02:21:49.253 INFO:teuthology.orchestra.run.smithi143.stdout: "rocksdb_cache_row_ratio": "0.000000", 2023-08-28T02:21:49.253 INFO:teuthology.orchestra.run.smithi143.stdout: "rocksdb_cache_shard_bits": "4", 2023-08-28T02:21:49.253 INFO:teuthology.orchestra.run.smithi143.stdout: "rocksdb_cache_size": "536870912", 2023-08-28T02:21:49.253 INFO:teuthology.orchestra.run.smithi143.stdout: "rocksdb_cache_type": "binned_lru", 2023-08-28T02:21:49.253 INFO:teuthology.orchestra.run.smithi143.stdout: "rocksdb_cf_compact_on_deletion": "true", 2023-08-28T02:21:49.253 INFO:teuthology.orchestra.run.smithi143.stdout: "rocksdb_cf_compact_on_deletion_sliding_window": "32768", 2023-08-28T02:21:49.253 INFO:teuthology.orchestra.run.smithi143.stdout: "rocksdb_cf_compact_on_deletion_trigger": "16384", 2023-08-28T02:21:49.253 INFO:teuthology.orchestra.run.smithi143.stdout: "rocksdb_collect_compaction_stats": "false", 2023-08-28T02:21:49.253 INFO:teuthology.orchestra.run.smithi143.stdout: "rocksdb_collect_extended_stats": "false", 2023-08-28T02:21:49.253 INFO:teuthology.orchestra.run.smithi143.stdout: "rocksdb_collect_memory_stats": "false", 2023-08-28T02:21:49.253 INFO:teuthology.orchestra.run.smithi143.stdout: "rocksdb_delete_range_threshold": "1048576", 2023-08-28T02:21:49.254 INFO:teuthology.orchestra.run.smithi143.stdout: "rocksdb_index_type": "binary_search", 2023-08-28T02:21:49.254 INFO:teuthology.orchestra.run.smithi143.stdout: "rocksdb_log_to_ceph_log": "true", 2023-08-28T02:21:49.254 INFO:teuthology.orchestra.run.smithi143.stdout: "rocksdb_metadata_block_size": "4096", 2023-08-28T02:21:49.254 INFO:teuthology.orchestra.run.smithi143.stdout: "rocksdb_partition_filters": "false", 2023-08-28T02:21:49.254 INFO:teuthology.orchestra.run.smithi143.stdout: "rocksdb_perf": "false", 2023-08-28T02:21:49.254 INFO:teuthology.orchestra.run.smithi143.stdout: "rocksdb_pin_l0_filter_and_index_blocks_in_cache": "false", 2023-08-28T02:21:49.254 INFO:teuthology.orchestra.run.smithi143.stdout: "rotating_keys_bootstrap_timeout": "30", 2023-08-28T02:21:49.254 INFO:teuthology.orchestra.run.smithi143.stdout: "rotating_keys_renewal_timeout": "10", 2023-08-28T02:21:49.254 INFO:teuthology.orchestra.run.smithi143.stdout: "run_dir": "/var/run/ceph", 2023-08-28T02:21:49.254 INFO:teuthology.orchestra.run.smithi143.stdout: "seastore_block_create": "true", 2023-08-28T02:21:49.254 INFO:teuthology.orchestra.run.smithi143.stdout: "seastore_cache_lru_size": "67108864", 2023-08-28T02:21:49.254 INFO:teuthology.orchestra.run.smithi143.stdout: "seastore_cbjournal_size": "5368709120", 2023-08-28T02:21:49.254 INFO:teuthology.orchestra.run.smithi143.stdout: "seastore_default_max_object_size": "16777216", 2023-08-28T02:21:49.255 INFO:teuthology.orchestra.run.smithi143.stdout: "seastore_default_object_metadata_reservation": "16777216", 2023-08-28T02:21:49.255 INFO:teuthology.orchestra.run.smithi143.stdout: "seastore_device_size": "53687091200", 2023-08-28T02:21:49.255 INFO:teuthology.orchestra.run.smithi143.stdout: "seastore_journal_batch_capacity": "16", 2023-08-28T02:21:49.255 INFO:teuthology.orchestra.run.smithi143.stdout: "seastore_journal_batch_flush_size": "16777216", 2023-08-28T02:21:49.255 INFO:teuthology.orchestra.run.smithi143.stdout: "seastore_journal_batch_preferred_fullness": "0.950000", 2023-08-28T02:21:49.255 INFO:teuthology.orchestra.run.smithi143.stdout: "seastore_journal_iodepth_limit": "5", 2023-08-28T02:21:49.255 INFO:teuthology.orchestra.run.smithi143.stdout: "seastore_main_device_type": "SSD", 2023-08-28T02:21:49.255 INFO:teuthology.orchestra.run.smithi143.stdout: "seastore_max_concurrent_transactions": "8", 2023-08-28T02:21:49.255 INFO:teuthology.orchestra.run.smithi143.stdout: "seastore_multiple_tiers_default_evict_ratio": "0.600000", 2023-08-28T02:21:49.255 INFO:teuthology.orchestra.run.smithi143.stdout: "seastore_multiple_tiers_fast_evict_ratio": "0.700000", 2023-08-28T02:21:49.255 INFO:teuthology.orchestra.run.smithi143.stdout: "seastore_multiple_tiers_stop_evict_ratio": "0.500000", 2023-08-28T02:21:49.255 INFO:teuthology.orchestra.run.smithi143.stdout: "seastore_obj_data_write_amplification": "1.250000", 2023-08-28T02:21:49.256 INFO:teuthology.orchestra.run.smithi143.stdout: "seastore_segment_size": "67108864", 2023-08-28T02:21:49.256 INFO:teuthology.orchestra.run.smithi143.stdout: "set_keepcaps": "false", 2023-08-28T02:21:49.256 INFO:teuthology.orchestra.run.smithi143.stdout: "setgroup": "ceph", 2023-08-28T02:21:49.256 INFO:teuthology.orchestra.run.smithi143.stdout: "setuser": "ceph", 2023-08-28T02:21:49.256 INFO:teuthology.orchestra.run.smithi143.stdout: "setuser_match_path": "", 2023-08-28T02:21:49.256 INFO:teuthology.orchestra.run.smithi143.stdout: "target_max_misplaced_ratio": "0.050000", 2023-08-28T02:21:49.256 INFO:teuthology.orchestra.run.smithi143.stdout: "thp": "false", 2023-08-28T02:21:49.256 INFO:teuthology.orchestra.run.smithi143.stdout: "threadpool_default_timeout": "60", 2023-08-28T02:21:49.256 INFO:teuthology.orchestra.run.smithi143.stdout: "threadpool_empty_queue_max_wait": "2", 2023-08-28T02:21:49.256 INFO:teuthology.orchestra.run.smithi143.stdout: "throttler_perf_counter": "true" 2023-08-28T02:21:49.256 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:21:49.639 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=0 2023-08-28T02:21:49.989 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:21:50.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:50 smithi143 ceph-mon[183059]: pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:50.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:50 smithi143 ceph-mon[189871]: pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:50.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:50 smithi169 ceph-mon[159415]: pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:51.290 INFO:teuthology.orchestra.run.smithi143.stdout:mon.a: {} 2023-08-28T02:21:51.300 INFO:teuthology.orchestra.run.smithi143.stdout:mon.b: {} 2023-08-28T02:21:51.309 INFO:teuthology.orchestra.run.smithi143.stdout:mon.c: {} 2023-08-28T02:21:51.310 INFO:teuthology.orchestra.run.smithi143.stderr:mon.a: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-08-28T02:21:51.310 INFO:teuthology.orchestra.run.smithi143.stderr:mon.b: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-08-28T02:21:51.310 INFO:teuthology.orchestra.run.smithi143.stderr:mon.c: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-08-28T02:21:51.837 INFO:tasks.thrashosds.thrasher:Unable to test ceph-objectstore-tool, not available on all OSD nodes 2023-08-28T02:21:51.837 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2023-08-28T02:21:51.838 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-08-28T02:21:51.839 INFO:tasks.radosbench:Beginning radosbench... 2023-08-28T02:21:51.839 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2023-08-28T02:21:51.840 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd pool create unique_pool_0 16 2023-08-28T02:21:51.840 INFO:tasks.thrashosds.thrasher:starting do_thrash 2023-08-28T02:21:51.840 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-08-28T02:21:51.840 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-08-28T02:21:51.840 INFO:tasks.thrashosds.thrasher:Removing osd 3, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2023-08-28T02:21:51.841 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd out 3 2023-08-28T02:21:51.841 INFO:tasks.thrashosds.thrasher:starting do_sighup with a delay of 0.1 2023-08-28T02:21:51.841 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:21:51.841 INFO:tasks.thrashosds.thrasher:starting do_optrack_toggle with a delay of 2.0 2023-08-28T02:21:51.842 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:21:51.842 INFO:tasks.thrashosds.thrasher:starting do_dump_ops 2023-08-28T02:21:51.842 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.0 dump_ops_in_flight 2023-08-28T02:21:51.843 INFO:tasks.thrashosds.thrasher:starting do_noscrub_toggle with a delay of 2.0 2023-08-28T02:21:51.843 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set noscrub 2023-08-28T02:21:51.882 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:21:51.954 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:21:51.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:21:51.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:21:52.060 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:21:52.100 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:21:51 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:21:51] "GET /metrics HTTP/1.1" 200 34037 "" "Prometheus/2.33.4" 2023-08-28T02:21:52.103 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:21:52.190 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:21:52.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:21:52.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:21:52.294 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:21:52.317 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:21:52.334 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:21:52.347 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:21:52.393 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:21:52.397 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:52 smithi169 ceph-mon[159415]: pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:52.397 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:52 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3307895518' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-08-28T02:21:52.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:21:52.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:21:52.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:52 smithi143 ceph-mon[183059]: pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:52.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:52 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3307895518' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-08-28T02:21:52.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:52 smithi143 ceph-mon[189871]: pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:52.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:52 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3307895518' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-08-28T02:21:52.498 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:21:52.509 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:21:52.532 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:21:52.540 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:21:52.846 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:21:52.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:21:52.850 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:21:52.950 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:21:52.958 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:21:52.984 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:21:53.032 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:21:53.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:21:53.037 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:21:53.138 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:21:53.166 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:21:53.210 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:21:53.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:21:53.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:21:53.316 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:21:53.348 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:21:53.845 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:21:53.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:21:53.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:21:53.949 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:21:53.982 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:21:54.030 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:21:54.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:21:54.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:21:54.135 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:21:54.164 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:21:54.208 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:21:54.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:21:54.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:21:54.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:54 smithi169 ceph-mon[159415]: pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:54.316 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:21:54.342 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:21:54.385 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:21:54.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:21:54.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:21:54.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:54 smithi143 ceph-mon[183059]: pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:54.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:54 smithi143 ceph-mon[189871]: pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:54.491 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:21:54.521 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:21:55.147 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is set 2023-08-28T02:21:55.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:55 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/689479237' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:21:55.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:55 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:21:55.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:55 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/689479237' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:21:55.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:55 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:21:55.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:55 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/689479237' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:21:55.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:55 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:21:55.872 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:21:55.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:21:55.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:21:55.977 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:21:56.008 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:21:56.057 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:21:56.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:21:56.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:21:56.165 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:21:56.193 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:21:56.237 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:21:56.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:21:56.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:21:56.316 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:56 smithi143 ceph-mon[183059]: pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:56.316 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:56 smithi143 ceph-mon[183059]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:21:56.316 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:56 smithi143 ceph-mon[183059]: failed to encode map e117 with expected crc 2023-08-28T02:21:56.316 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:56 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:21:56.316 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:56 smithi143 ceph-mon[183059]: osdmap e117: 8 total, 8 up, 8 in 2023-08-28T02:21:56.316 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:56 smithi143 ceph-mon[183059]: failed to encode map e117 with expected crc 2023-08-28T02:21:56.316 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:56 smithi143 ceph-mon[183059]: failed to encode map e117 with expected crc 2023-08-28T02:21:56.317 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:56 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:21:56.317 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:56 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:21:56.317 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:56 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:21:56.317 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:56 smithi143 ceph-mon[189871]: pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:56.317 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:56 smithi143 ceph-mon[189871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:21:56.317 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:56 smithi143 ceph-mon[189871]: failed to encode map e117 with expected crc 2023-08-28T02:21:56.317 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:56 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:21:56.317 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:56 smithi143 ceph-mon[189871]: osdmap e117: 8 total, 8 up, 8 in 2023-08-28T02:21:56.317 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:56 smithi143 ceph-mon[189871]: failed to encode map e117 with expected crc 2023-08-28T02:21:56.318 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:56 smithi143 ceph-mon[189871]: failed to encode map e117 with expected crc 2023-08-28T02:21:56.318 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:56 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:21:56.318 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:56 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:21:56.318 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:56 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:21:56.344 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:21:56.368 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:56 smithi169 ceph-mon[159415]: pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:56.368 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:56 smithi169 ceph-mon[159415]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:21:56.368 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:56 smithi169 ceph-mon[159415]: failed to encode map e117 with expected crc 2023-08-28T02:21:56.368 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:56 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:21:56.368 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:56 smithi169 ceph-mon[159415]: osdmap e117: 8 total, 8 up, 8 in 2023-08-28T02:21:56.368 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:56 smithi169 ceph-mon[159415]: failed to encode map e117 with expected crc 2023-08-28T02:21:56.368 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:56 smithi169 ceph-mon[159415]: failed to encode map e117 with expected crc 2023-08-28T02:21:56.368 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:56 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:21:56.368 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:56 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:21:56.369 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:56 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:21:56.370 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:21:56.411 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:21:56.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:21:56.418 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:21:56.474 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:21:56.519 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:21:56.548 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:21:56.612 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:21:56.612 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:21:56.612 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:21:56.612 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:21:56.612 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:21:56.613 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:21:56.613 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:21:56.613 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:21:56.613 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:21:56.613 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:21:56.613 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:21:56.613 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:21:56.613 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:21:56.613 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:21:56.613 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:21:57.185 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:21:57.187 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:57 smithi143 ceph-mon[183059]: failed to encode map e117 with expected crc 2023-08-28T02:21:57.187 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:57 smithi143 ceph-mon[183059]: failed to encode map e117 with expected crc 2023-08-28T02:21:57.187 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:57 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2717717245' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:21:57.188 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:57 smithi143 ceph-mon[189871]: failed to encode map e117 with expected crc 2023-08-28T02:21:57.188 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:57 smithi143 ceph-mon[189871]: failed to encode map e117 with expected crc 2023-08-28T02:21:57.188 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:57 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2717717245' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:21:57.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:21:57.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:21:57.269 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:21:57.269 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:21:57.290 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:21:57.321 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:21:57.368 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:21:57.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:21:57.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:21:57.475 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:21:57.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:57 smithi169 ceph-mon[159415]: failed to encode map e117 with expected crc 2023-08-28T02:21:57.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:57 smithi169 ceph-mon[159415]: failed to encode map e117 with expected crc 2023-08-28T02:21:57.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:57 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2717717245' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:21:57.503 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:21:57.549 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:21:57.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:21:57.554 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:21:57.655 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:21:57.686 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:21:57.941 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:21:57.942 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:57 smithi143 conmon[183035]: 2023-08-28T02:21:57.477+0000 7fadc3887700 -1 mon.a@0(leader).osd e117 definitely_dead 0 2023-08-28T02:21:57.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:21:57.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:21:58.044 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:21:58.075 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:21:58.122 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:21:58.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:21:58.125 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:21:58.170 INFO:teuthology.orchestra.run.smithi143.stderr:marked out osd.3. 2023-08-28T02:21:58.192 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.0 dump_blocked_ops 2023-08-28T02:21:58.196 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:58 smithi143 ceph-mon[183059]: pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:58.197 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:58 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/717113930' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-08-28T02:21:58.197 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:58 smithi143 ceph-mon[183059]: Client client.admin marked osd.3 out, while it was still marked up 2023-08-28T02:21:58.197 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:58 smithi143 ceph-mon[189871]: pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:58.197 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:58 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/717113930' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-08-28T02:21:58.197 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:58 smithi143 ceph-mon[189871]: Client client.admin marked osd.3 out, while it was still marked up 2023-08-28T02:21:58.225 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:21:58.252 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:21:58.293 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:21:58.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:21:58.298 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:21:58.398 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:21:58.422 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:58 smithi169 ceph-mon[159415]: pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-08-28T02:21:58.422 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:58 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/717113930' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-08-28T02:21:58.422 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:58 smithi169 ceph-mon[159415]: Client client.admin marked osd.3 out, while it was still marked up 2023-08-28T02:21:58.424 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:21:58.438 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set nodeep-scrub 2023-08-28T02:21:58.466 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:21:58.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:21:58.470 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:21:58.571 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:21:58.596 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:21:58.637 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:21:58.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:21:58.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:21:58.745 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:21:58.771 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:21:58.812 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:21:58.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:21:58.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:21:58.912 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:21:58.918 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:21:58.948 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:21:58.964 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:21:59.035 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:21:59.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:21:59.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:21:59.140 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:21:59.169 INFO:teuthology.orchestra.run.smithi143.stderr:pool 'unique_pool_0' created 2023-08-28T02:21:59.184 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:21:59.226 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:21:59.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:21:59.230 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:21:59.331 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:21:59.356 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:59 smithi143 ceph-mon[189871]: failed to encode map e118 with expected crc 2023-08-28T02:21:59.356 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:59 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/717113930' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-08-28T02:21:59.356 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:59 smithi143 ceph-mon[189871]: osdmap e118: 8 total, 8 up, 7 in 2023-08-28T02:21:59.356 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:59 smithi143 ceph-mon[189871]: failed to encode map e118 with expected crc 2023-08-28T02:21:59.356 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:59 smithi143 ceph-mon[189871]: failed to encode map e118 with expected crc 2023-08-28T02:21:59.356 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:59 smithi143 ceph-mon[189871]: failed to encode map e118 with expected crc 2023-08-28T02:21:59.357 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:59 smithi143 ceph-mon[189871]: failed to encode map e118 with expected crc 2023-08-28T02:21:59.357 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:59 smithi143 ceph-mon[189871]: failed to encode map e118 with expected crc 2023-08-28T02:21:59.357 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:59 smithi143 ceph-mon[189871]: failed to encode map e118 with expected crc 2023-08-28T02:21:59.357 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:59 smithi143 ceph-mon[189871]: failed to encode map e118 with expected crc 2023-08-28T02:21:59.357 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:59 smithi143 ceph-mon[189871]: failed to encode map e118 with expected crc 2023-08-28T02:21:59.357 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:59 smithi143 ceph-mon[189871]: failed to encode map e118 with expected crc 2023-08-28T02:21:59.357 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:59 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/629448048' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-08-28T02:21:59.357 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:59 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-08-28T02:21:59.357 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:59 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-08-28T02:21:59.357 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:21:59 smithi143 ceph-mon[189871]: osdmap e119: 8 total, 8 up, 7 in 2023-08-28T02:21:59.358 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:59 smithi143 ceph-mon[183059]: failed to encode map e118 with expected crc 2023-08-28T02:21:59.358 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:59 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/717113930' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-08-28T02:21:59.358 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:59 smithi143 ceph-mon[183059]: osdmap e118: 8 total, 8 up, 7 in 2023-08-28T02:21:59.358 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:59 smithi143 ceph-mon[183059]: failed to encode map e118 with expected crc 2023-08-28T02:21:59.358 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:59 smithi143 ceph-mon[183059]: failed to encode map e118 with expected crc 2023-08-28T02:21:59.358 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:59 smithi143 ceph-mon[183059]: failed to encode map e118 with expected crc 2023-08-28T02:21:59.358 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:59 smithi143 ceph-mon[183059]: failed to encode map e118 with expected crc 2023-08-28T02:21:59.358 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:59 smithi143 ceph-mon[183059]: failed to encode map e118 with expected crc 2023-08-28T02:21:59.358 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:59 smithi143 ceph-mon[183059]: failed to encode map e118 with expected crc 2023-08-28T02:21:59.358 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:59 smithi143 ceph-mon[183059]: failed to encode map e118 with expected crc 2023-08-28T02:21:59.358 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:59 smithi143 ceph-mon[183059]: failed to encode map e118 with expected crc 2023-08-28T02:21:59.358 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:59 smithi143 ceph-mon[183059]: failed to encode map e118 with expected crc 2023-08-28T02:21:59.359 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:59 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/629448048' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-08-28T02:21:59.359 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:59 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-08-28T02:21:59.359 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:59 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-08-28T02:21:59.359 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:21:59 smithi143 ceph-mon[183059]: osdmap e119: 8 total, 8 up, 7 in 2023-08-28T02:21:59.359 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:21:59.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:59 smithi169 ceph-mon[159415]: failed to encode map e118 with expected crc 2023-08-28T02:21:59.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:59 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/717113930' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-08-28T02:21:59.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:59 smithi169 ceph-mon[159415]: osdmap e118: 8 total, 8 up, 7 in 2023-08-28T02:21:59.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:59 smithi169 ceph-mon[159415]: failed to encode map e118 with expected crc 2023-08-28T02:21:59.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:59 smithi169 ceph-mon[159415]: failed to encode map e118 with expected crc 2023-08-28T02:21:59.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:59 smithi169 ceph-mon[159415]: failed to encode map e118 with expected crc 2023-08-28T02:21:59.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:59 smithi169 ceph-mon[159415]: failed to encode map e118 with expected crc 2023-08-28T02:21:59.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:59 smithi169 ceph-mon[159415]: failed to encode map e118 with expected crc 2023-08-28T02:21:59.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:59 smithi169 ceph-mon[159415]: failed to encode map e118 with expected crc 2023-08-28T02:21:59.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:59 smithi169 ceph-mon[159415]: failed to encode map e118 with expected crc 2023-08-28T02:21:59.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:59 smithi169 ceph-mon[159415]: failed to encode map e118 with expected crc 2023-08-28T02:21:59.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:59 smithi169 ceph-mon[159415]: failed to encode map e118 with expected crc 2023-08-28T02:21:59.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:59 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/629448048' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-08-28T02:21:59.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:59 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-08-28T02:21:59.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:59 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-08-28T02:21:59.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:21:59 smithi169 ceph-mon[159415]: osdmap e119: 8 total, 8 up, 7 in 2023-08-28T02:21:59.992 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2023-08-28T02:22:00.122 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:22:00.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:00 smithi143 ceph-mon[183059]: pgmap v224: 73 pgs: 4 peering, 69 active+clean; 4.6 MiB data, 1.3 GiB used, 625 GiB / 626 GiB avail 2023-08-28T02:22:00.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:00 smithi143 ceph-mon[183059]: failed to encode map e119 with expected crc 2023-08-28T02:22:00.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:00 smithi143 ceph-mon[183059]: failed to encode map e119 with expected crc 2023-08-28T02:22:00.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:00 smithi143 ceph-mon[183059]: failed to encode map e119 with expected crc 2023-08-28T02:22:00.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:00 smithi143 ceph-mon[183059]: failed to encode map e119 with expected crc 2023-08-28T02:22:00.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:00 smithi143 ceph-mon[183059]: failed to encode map e119 with expected crc 2023-08-28T02:22:00.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:00 smithi143 ceph-mon[183059]: failed to encode map e119 with expected crc 2023-08-28T02:22:00.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:00 smithi143 ceph-mon[183059]: failed to encode map e119 with expected crc 2023-08-28T02:22:00.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:00 smithi143 ceph-mon[183059]: failed to encode map e119 with expected crc 2023-08-28T02:22:00.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:00 smithi143 ceph-mon[183059]: failed to encode map e119 with expected crc 2023-08-28T02:22:00.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:00 smithi143 ceph-mon[183059]: failed to encode map e119 with expected crc 2023-08-28T02:22:00.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:00 smithi143 ceph-mon[183059]: failed to encode map e119 with expected crc 2023-08-28T02:22:00.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:00 smithi143 ceph-mon[183059]: failed to encode map e119 with expected crc 2023-08-28T02:22:00.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:00 smithi143 ceph-mon[183059]: failed to encode map e119 with expected crc 2023-08-28T02:22:00.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:00 smithi143 ceph-mon[183059]: failed to encode map e119 with expected crc 2023-08-28T02:22:00.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:00 smithi143 ceph-mon[183059]: failed to encode map e119 with expected crc 2023-08-28T02:22:00.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:00 smithi143 ceph-mon[183059]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-08-28T02:22:00.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:00 smithi143 ceph-mon[183059]: osdmap e120: 8 total, 8 up, 7 in 2023-08-28T02:22:00.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:00 smithi143 ceph-mon[189871]: pgmap v224: 73 pgs: 4 peering, 69 active+clean; 4.6 MiB data, 1.3 GiB used, 625 GiB / 626 GiB avail 2023-08-28T02:22:00.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:00 smithi143 ceph-mon[189871]: failed to encode map e119 with expected crc 2023-08-28T02:22:00.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:00 smithi143 ceph-mon[189871]: failed to encode map e119 with expected crc 2023-08-28T02:22:00.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:00 smithi143 ceph-mon[189871]: failed to encode map e119 with expected crc 2023-08-28T02:22:00.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:00 smithi143 ceph-mon[189871]: failed to encode map e119 with expected crc 2023-08-28T02:22:00.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:00 smithi143 ceph-mon[189871]: failed to encode map e119 with expected crc 2023-08-28T02:22:00.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:00 smithi143 ceph-mon[189871]: failed to encode map e119 with expected crc 2023-08-28T02:22:00.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:00 smithi143 ceph-mon[189871]: failed to encode map e119 with expected crc 2023-08-28T02:22:00.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:00 smithi143 ceph-mon[189871]: failed to encode map e119 with expected crc 2023-08-28T02:22:00.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:00 smithi143 ceph-mon[189871]: failed to encode map e119 with expected crc 2023-08-28T02:22:00.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:00 smithi143 ceph-mon[189871]: failed to encode map e119 with expected crc 2023-08-28T02:22:00.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:00 smithi143 ceph-mon[189871]: failed to encode map e119 with expected crc 2023-08-28T02:22:00.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:00 smithi143 ceph-mon[189871]: failed to encode map e119 with expected crc 2023-08-28T02:22:00.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:00 smithi143 ceph-mon[189871]: failed to encode map e119 with expected crc 2023-08-28T02:22:00.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:00 smithi143 ceph-mon[189871]: failed to encode map e119 with expected crc 2023-08-28T02:22:00.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:00 smithi143 ceph-mon[189871]: failed to encode map e119 with expected crc 2023-08-28T02:22:00.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:00 smithi143 ceph-mon[189871]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-08-28T02:22:00.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:00 smithi143 ceph-mon[189871]: osdmap e120: 8 total, 8 up, 7 in 2023-08-28T02:22:00.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:00 smithi169 ceph-mon[159415]: pgmap v224: 73 pgs: 4 peering, 69 active+clean; 4.6 MiB data, 1.3 GiB used, 625 GiB / 626 GiB avail 2023-08-28T02:22:00.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:00 smithi169 ceph-mon[159415]: failed to encode map e119 with expected crc 2023-08-28T02:22:00.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:00 smithi169 ceph-mon[159415]: failed to encode map e119 with expected crc 2023-08-28T02:22:00.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:00 smithi169 ceph-mon[159415]: failed to encode map e119 with expected crc 2023-08-28T02:22:00.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:00 smithi169 ceph-mon[159415]: failed to encode map e119 with expected crc 2023-08-28T02:22:00.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:00 smithi169 ceph-mon[159415]: failed to encode map e119 with expected crc 2023-08-28T02:22:00.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:00 smithi169 ceph-mon[159415]: failed to encode map e119 with expected crc 2023-08-28T02:22:00.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:00 smithi169 ceph-mon[159415]: failed to encode map e119 with expected crc 2023-08-28T02:22:00.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:00 smithi169 ceph-mon[159415]: failed to encode map e119 with expected crc 2023-08-28T02:22:00.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:00 smithi169 ceph-mon[159415]: failed to encode map e119 with expected crc 2023-08-28T02:22:00.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:00 smithi169 ceph-mon[159415]: failed to encode map e119 with expected crc 2023-08-28T02:22:00.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:00 smithi169 ceph-mon[159415]: failed to encode map e119 with expected crc 2023-08-28T02:22:00.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:00 smithi169 ceph-mon[159415]: failed to encode map e119 with expected crc 2023-08-28T02:22:00.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:00 smithi169 ceph-mon[159415]: failed to encode map e119 with expected crc 2023-08-28T02:22:00.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:00 smithi169 ceph-mon[159415]: failed to encode map e119 with expected crc 2023-08-28T02:22:00.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:00 smithi169 ceph-mon[159415]: failed to encode map e119 with expected crc 2023-08-28T02:22:00.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:00 smithi169 ceph-mon[159415]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-08-28T02:22:00.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:00 smithi169 ceph-mon[159415]: osdmap e120: 8 total, 8 up, 7 in 2023-08-28T02:22:00.558 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:22:00.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:00.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:22:00.662 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:22:00.696 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:00.747 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:22:00.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:00.758 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:22:00.858 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:22:00.886 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:00.931 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:22:00.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:00.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:22:01.039 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:22:01.072 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:01.121 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:22:01.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:01.125 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:22:01.226 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:22:01.254 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:01.297 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:22:01.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:01.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:22:01.385 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:22:01.393 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:22:01.403 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:22:01.429 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:01.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:01 smithi143 ceph-mon[183059]: failed to encode map e120 with expected crc 2023-08-28T02:22:01.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:01 smithi143 ceph-mon[183059]: failed to encode map e120 with expected crc 2023-08-28T02:22:01.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:01 smithi143 ceph-mon[183059]: failed to encode map e120 with expected crc 2023-08-28T02:22:01.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:01 smithi143 ceph-mon[183059]: failed to encode map e120 with expected crc 2023-08-28T02:22:01.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:01 smithi143 ceph-mon[183059]: failed to encode map e120 with expected crc 2023-08-28T02:22:01.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:01 smithi143 ceph-mon[183059]: osdmap e121: 8 total, 8 up, 7 in 2023-08-28T02:22:01.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:01 smithi143 ceph-mon[189871]: failed to encode map e120 with expected crc 2023-08-28T02:22:01.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:01 smithi143 ceph-mon[189871]: failed to encode map e120 with expected crc 2023-08-28T02:22:01.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:01 smithi143 ceph-mon[189871]: failed to encode map e120 with expected crc 2023-08-28T02:22:01.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:01 smithi143 ceph-mon[189871]: failed to encode map e120 with expected crc 2023-08-28T02:22:01.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:01 smithi143 ceph-mon[189871]: failed to encode map e120 with expected crc 2023-08-28T02:22:01.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:01 smithi143 ceph-mon[189871]: osdmap e121: 8 total, 8 up, 7 in 2023-08-28T02:22:01.472 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:22:01.475 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:01 smithi169 ceph-mon[159415]: failed to encode map e120 with expected crc 2023-08-28T02:22:01.475 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:01 smithi169 ceph-mon[159415]: failed to encode map e120 with expected crc 2023-08-28T02:22:01.475 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:01 smithi169 ceph-mon[159415]: failed to encode map e120 with expected crc 2023-08-28T02:22:01.475 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:01 smithi169 ceph-mon[159415]: failed to encode map e120 with expected crc 2023-08-28T02:22:01.475 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:01 smithi169 ceph-mon[159415]: failed to encode map e120 with expected crc 2023-08-28T02:22:01.475 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:01 smithi169 ceph-mon[159415]: osdmap e121: 8 total, 8 up, 7 in 2023-08-28T02:22:01.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:01.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:22:01.577 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:22:01.603 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:01.644 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:22:01.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:01.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:22:01.715 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:22:01.715 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:22:01.749 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:22:01.775 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:01.817 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:22:01.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:01.821 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:22:01.921 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:22:01.951 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:02.193 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:22:01 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:22:01] "GET /metrics HTTP/1.1" 200 34115 "" "Prometheus/2.33.4" 2023-08-28T02:22:02.193 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is set 2023-08-28T02:22:02.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:02 smithi143 ceph-mon[183059]: pgmap v227: 89 pgs: 16 unknown, 4 peering, 69 active+clean; 4.6 MiB data, 1.3 GiB used, 625 GiB / 626 GiB avail 2023-08-28T02:22:02.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:02 smithi143 ceph-mon[183059]: failed to encode map e121 with expected crc 2023-08-28T02:22:02.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:02 smithi143 ceph-mon[183059]: failed to encode map e121 with expected crc 2023-08-28T02:22:02.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:02 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1180088105' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:22:02.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:02 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:22:02.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:02 smithi143 ceph-mon[189871]: pgmap v227: 89 pgs: 16 unknown, 4 peering, 69 active+clean; 4.6 MiB data, 1.3 GiB used, 625 GiB / 626 GiB avail 2023-08-28T02:22:02.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:02 smithi143 ceph-mon[189871]: failed to encode map e121 with expected crc 2023-08-28T02:22:02.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:02 smithi143 ceph-mon[189871]: failed to encode map e121 with expected crc 2023-08-28T02:22:02.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:02 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1180088105' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:22:02.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:02 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:22:02.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:02 smithi169 ceph-mon[159415]: pgmap v227: 89 pgs: 16 unknown, 4 peering, 69 active+clean; 4.6 MiB data, 1.3 GiB used, 625 GiB / 626 GiB avail 2023-08-28T02:22:02.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:02 smithi169 ceph-mon[159415]: failed to encode map e121 with expected crc 2023-08-28T02:22:02.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:02 smithi169 ceph-mon[159415]: failed to encode map e121 with expected crc 2023-08-28T02:22:02.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:02 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1180088105' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:22:02.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:02 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:22:02.749 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:22:02.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:02.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:22:02.852 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:22:02.885 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:02.934 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:22:02.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:02.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:22:03.041 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:22:03.069 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:03.112 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:22:03.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:03.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:22:03.219 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:22:03.249 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:03.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:03 smithi143 ceph-mon[183059]: failed to encode map e121 with expected crc 2023-08-28T02:22:03.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:03 smithi143 ceph-mon[183059]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:22:03.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:03 smithi143 ceph-mon[183059]: failed to encode map e122 with expected crc 2023-08-28T02:22:03.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:03 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:22:03.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:03 smithi143 ceph-mon[183059]: osdmap e122: 8 total, 8 up, 7 in 2023-08-28T02:22:03.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:03 smithi143 ceph-mon[183059]: failed to encode map e121 with expected crc 2023-08-28T02:22:03.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:03 smithi143 ceph-mon[183059]: failed to encode map e122 with expected crc 2023-08-28T02:22:03.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:03 smithi143 ceph-mon[183059]: failed to encode map e122 with expected crc 2023-08-28T02:22:03.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:03 smithi143 ceph-mon[189871]: failed to encode map e121 with expected crc 2023-08-28T02:22:03.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:03 smithi143 ceph-mon[189871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:22:03.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:03 smithi143 ceph-mon[189871]: failed to encode map e122 with expected crc 2023-08-28T02:22:03.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:03 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:22:03.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:03 smithi143 ceph-mon[189871]: osdmap e122: 8 total, 8 up, 7 in 2023-08-28T02:22:03.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:03 smithi143 ceph-mon[189871]: failed to encode map e121 with expected crc 2023-08-28T02:22:03.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:03 smithi143 ceph-mon[189871]: failed to encode map e122 with expected crc 2023-08-28T02:22:03.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:03 smithi143 ceph-mon[189871]: failed to encode map e122 with expected crc 2023-08-28T02:22:03.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:03 smithi169 ceph-mon[159415]: failed to encode map e121 with expected crc 2023-08-28T02:22:03.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:03 smithi169 ceph-mon[159415]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:22:03.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:03 smithi169 ceph-mon[159415]: failed to encode map e122 with expected crc 2023-08-28T02:22:03.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:03 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:22:03.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:03 smithi169 ceph-mon[159415]: osdmap e122: 8 total, 8 up, 7 in 2023-08-28T02:22:03.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:03 smithi169 ceph-mon[159415]: failed to encode map e121 with expected crc 2023-08-28T02:22:03.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:03 smithi169 ceph-mon[159415]: failed to encode map e122 with expected crc 2023-08-28T02:22:03.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:03 smithi169 ceph-mon[159415]: failed to encode map e122 with expected crc 2023-08-28T02:22:03.770 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:22:03.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:03.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:22:03.783 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7] out_osds: [3] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-08-28T02:22:03.783 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd reweight 0 1 2023-08-28T02:22:03.875 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:22:03.907 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:03.950 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.0 dump_historic_ops 2023-08-28T02:22:03.955 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:22:03.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:03.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:22:04.060 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:22:04.091 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:04.194 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:22:04.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:04.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:22:04.299 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:22:04.330 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:04.378 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:22:04.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:04.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:22:04.397 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:22:04.405 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:04 smithi143 ceph-mon[183059]: pgmap v230: 89 pgs: 2 unknown, 4 peering, 83 active+clean; 4.6 MiB data, 1.3 GiB used, 625 GiB / 626 GiB avail; 22 B/s, 3 objects/s recovering 2023-08-28T02:22:04.405 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:04 smithi143 ceph-mon[189871]: pgmap v230: 89 pgs: 2 unknown, 4 peering, 83 active+clean; 4.6 MiB data, 1.3 GiB used, 625 GiB / 626 GiB avail; 22 B/s, 3 objects/s recovering 2023-08-28T02:22:04.420 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:22:04.442 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:22:04.462 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:22:04.481 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:22:04.485 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:22:04.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:04 smithi169 ceph-mon[159415]: pgmap v230: 89 pgs: 2 unknown, 4 peering, 83 active+clean; 4.6 MiB data, 1.3 GiB used, 625 GiB / 626 GiB avail; 22 B/s, 3 objects/s recovering 2023-08-28T02:22:04.499 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:22:04.501 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:22:04.512 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:04.518 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:22:04.539 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:22:04.540 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:22:04.540 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:22:04.540 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:22:04.540 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:22:04.541 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:22:04.541 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:22:04.541 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:22:04.541 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:22:04.556 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:22:04.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:04.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:22:04.663 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:22:04.691 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:04.742 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:22:04.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:04.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:22:04.846 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:22:04.874 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:04.917 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:22:04.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:04.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:22:05.021 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:22:05.048 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:05.217 INFO:teuthology.orchestra.run.smithi143.stderr:enabled application 'rados' on pool 'unique_pool_0' 2023-08-28T02:22:05.411 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:05 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/894437178' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:22:05.411 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:05 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1307766708' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-28T02:22:05.411 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:05 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/894437178' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:22:05.411 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:05 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1307766708' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-28T02:22:05.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:05 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/894437178' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:22:05.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:05 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1307766708' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-28T02:22:05.516 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset noscrub 2023-08-28T02:22:05.996 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:22:05.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:05.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:22:06.000 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:22:06.100 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:22:06.132 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:06.180 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:22:06.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:06.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:22:06.288 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:22:06.316 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:06.337 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:06 smithi143 ceph-mon[183059]: pgmap v231: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 625 GiB / 626 GiB avail; 15 B/s, 2 objects/s recovering 2023-08-28T02:22:06.337 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:06 smithi143 ceph-mon[183059]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-08-28T02:22:06.337 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:06 smithi143 ceph-mon[183059]: failed to encode map e123 with expected crc 2023-08-28T02:22:06.338 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:06 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1307766708' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-08-28T02:22:06.338 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:06 smithi143 ceph-mon[183059]: osdmap e123: 8 total, 8 up, 7 in 2023-08-28T02:22:06.338 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:06 smithi143 ceph-mon[183059]: failed to encode map e123 with expected crc 2023-08-28T02:22:06.338 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:06 smithi143 ceph-mon[189871]: pgmap v231: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 625 GiB / 626 GiB avail; 15 B/s, 2 objects/s recovering 2023-08-28T02:22:06.338 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:06 smithi143 ceph-mon[189871]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-08-28T02:22:06.338 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:06 smithi143 ceph-mon[189871]: failed to encode map e123 with expected crc 2023-08-28T02:22:06.338 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:06 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1307766708' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-08-28T02:22:06.338 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:06 smithi143 ceph-mon[189871]: osdmap e123: 8 total, 8 up, 7 in 2023-08-28T02:22:06.338 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:06 smithi143 ceph-mon[189871]: failed to encode map e123 with expected crc 2023-08-28T02:22:06.360 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:22:06.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:06.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:22:06.467 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:22:06.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:06 smithi169 ceph-mon[159415]: pgmap v231: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 625 GiB / 626 GiB avail; 15 B/s, 2 objects/s recovering 2023-08-28T02:22:06.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:06 smithi169 ceph-mon[159415]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-08-28T02:22:06.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:06 smithi169 ceph-mon[159415]: failed to encode map e123 with expected crc 2023-08-28T02:22:06.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:06 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1307766708' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-08-28T02:22:06.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:06 smithi169 ceph-mon[159415]: osdmap e123: 8 total, 8 up, 7 in 2023-08-28T02:22:06.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:06 smithi169 ceph-mon[159415]: failed to encode map e123 with expected crc 2023-08-28T02:22:06.494 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:06.681 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:22:06.693 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:22:06.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:06.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:22:06.797 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:22:06.827 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:06.873 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:22:06.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:06.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:22:06.978 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:22:07.006 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:07.049 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:22:07.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:07.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:22:07.154 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:22:07.172 DEBUG:teuthology.orchestra.run.smithi143:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_0 bench 90 write' 2023-08-28T02:22:07.175 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-08-28T02:22:07.175 INFO:teuthology.orchestra.run:waiting for 3000 2023-08-28T02:22:07.183 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:07.222 INFO:teuthology.orchestra.run.smithi143.stderr:reweighted osd.0 to 1 (10000) 2023-08-28T02:22:07.227 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:hints = 1 2023-08-28T02:22:07.227 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-08-28T02:22:07.227 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Object prefix: benchmark_data_smithi143_236200 2023-08-28T02:22:07.227 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:22:07.228 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-08-28T02:22:07.228 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 0 0 0 0 0 0 - 0 2023-08-28T02:22:07.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:07.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:22:07.334 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:22:07.388 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:07.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:07 smithi169 ceph-mon[159415]: failed to encode map e123 with expected crc 2023-08-28T02:22:07.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:07 smithi169 ceph-mon[159415]: failed to encode map e123 with expected crc 2023-08-28T02:22:07.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:07 smithi169 ceph-mon[159415]: failed to encode map e123 with expected crc 2023-08-28T02:22:07.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:07 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2191431051' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-08-28T02:22:07.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:07 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-08-28T02:22:07.552 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:07 smithi143 ceph-mon[183059]: failed to encode map e123 with expected crc 2023-08-28T02:22:07.552 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:07 smithi143 ceph-mon[183059]: failed to encode map e123 with expected crc 2023-08-28T02:22:07.553 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:07 smithi143 ceph-mon[183059]: failed to encode map e123 with expected crc 2023-08-28T02:22:07.553 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:07 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2191431051' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-08-28T02:22:07.553 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:07 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-08-28T02:22:07.553 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:07 smithi143 ceph-mon[189871]: failed to encode map e123 with expected crc 2023-08-28T02:22:07.553 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:07 smithi143 ceph-mon[189871]: failed to encode map e123 with expected crc 2023-08-28T02:22:07.553 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:07 smithi143 ceph-mon[189871]: failed to encode map e123 with expected crc 2023-08-28T02:22:07.553 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:07 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2191431051' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-08-28T02:22:07.553 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:07 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-08-28T02:22:07.784 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:22:07.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:07.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:22:07.890 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:22:07.942 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:08.023 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:22:08.151 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:22:08.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:08.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:22:08.228 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 1 16 3863 3847 240.229 240.438 0.00786064 0.00414794 2023-08-28T02:22:08.256 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:22:08.304 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:08.397 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:22:08.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:08.404 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:22:08.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:08 smithi143 ceph-mon[183059]: pgmap v233: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 625 GiB / 626 GiB avail; 14 B/s, 2 objects/s recovering 2023-08-28T02:22:08.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:08 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-08-28T02:22:08.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:08 smithi143 ceph-mon[183059]: osdmap e124: 8 total, 8 up, 7 in 2023-08-28T02:22:08.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:08 smithi143 ceph-mon[183059]: failed to encode map e124 with expected crc 2023-08-28T02:22:08.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:08 smithi143 ceph-mon[183059]: failed to encode map e124 with expected crc 2023-08-28T02:22:08.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:08 smithi143 ceph-mon[183059]: failed to encode map e124 with expected crc 2023-08-28T02:22:08.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:08 smithi143 ceph-mon[183059]: failed to encode map e124 with expected crc 2023-08-28T02:22:08.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:08 smithi143 ceph-mon[189871]: pgmap v233: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 625 GiB / 626 GiB avail; 14 B/s, 2 objects/s recovering 2023-08-28T02:22:08.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:08 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-08-28T02:22:08.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:08 smithi143 ceph-mon[189871]: osdmap e124: 8 total, 8 up, 7 in 2023-08-28T02:22:08.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:08 smithi143 ceph-mon[189871]: failed to encode map e124 with expected crc 2023-08-28T02:22:08.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:08 smithi143 ceph-mon[189871]: failed to encode map e124 with expected crc 2023-08-28T02:22:08.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:08 smithi143 ceph-mon[189871]: failed to encode map e124 with expected crc 2023-08-28T02:22:08.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:08 smithi143 ceph-mon[189871]: failed to encode map e124 with expected crc 2023-08-28T02:22:08.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:08 smithi169 ceph-mon[159415]: pgmap v233: 89 pgs: 89 active+clean; 4.6 MiB data, 1.3 GiB used, 625 GiB / 626 GiB avail; 14 B/s, 2 objects/s recovering 2023-08-28T02:22:08.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:08 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-08-28T02:22:08.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:08 smithi169 ceph-mon[159415]: osdmap e124: 8 total, 8 up, 7 in 2023-08-28T02:22:08.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:08 smithi169 ceph-mon[159415]: failed to encode map e124 with expected crc 2023-08-28T02:22:08.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:08 smithi169 ceph-mon[159415]: failed to encode map e124 with expected crc 2023-08-28T02:22:08.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:08 smithi169 ceph-mon[159415]: failed to encode map e124 with expected crc 2023-08-28T02:22:08.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:08 smithi169 ceph-mon[159415]: failed to encode map e124 with expected crc 2023-08-28T02:22:08.505 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:22:08.556 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:08.647 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:22:08.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:08.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:22:08.759 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:22:08.785 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:08.874 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:22:08.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:08.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:22:08.981 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:22:09.011 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:09.228 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 2 16 6388 6372 199.021 157.812 0.0056987 0.00502117 2023-08-28T02:22:09.384 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:22:09.389 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:09 smithi143 ceph-mon[189871]: failed to encode map e124 with expected crc 2023-08-28T02:22:09.389 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:09 smithi143 ceph-mon[189871]: pgmap v235: 89 pgs: 89 active+clean; 101 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 14 MiB/s wr, 225 op/s; 0 B/s, 1 objects/s recovering 2023-08-28T02:22:09.389 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:09 smithi143 ceph-mon[183059]: failed to encode map e124 with expected crc 2023-08-28T02:22:09.389 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:09 smithi143 ceph-mon[183059]: pgmap v235: 89 pgs: 89 active+clean; 101 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 14 MiB/s wr, 225 op/s; 0 B/s, 1 objects/s recovering 2023-08-28T02:22:09.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:09.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:22:09.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:09 smithi169 ceph-mon[159415]: failed to encode map e124 with expected crc 2023-08-28T02:22:09.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:09 smithi169 ceph-mon[159415]: pgmap v235: 89 pgs: 89 active+clean; 101 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 14 MiB/s wr, 225 op/s; 0 B/s, 1 objects/s recovering 2023-08-28T02:22:09.491 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:22:09.504 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd reweight 1 1 2023-08-28T02:22:09.559 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:09.684 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:22:09.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:09.696 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:22:09.797 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:22:09.913 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:09.979 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:22:09.979 INFO:teuthology.orchestra.run.smithi143.stdout: "size": 20, 2023-08-28T02:22:09.979 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 600, 2023-08-28T02:22:09.980 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [ 2023-08-28T02:22:09.980 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:09.980 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:5477 5.5 5:ad533b78:::benchmark_data_smithi143_236200_object5476:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-08-28T02:22:09.980 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:22:08.665942+0000", 2023-08-28T02:22:09.980 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 1.3086397999999999, 2023-08-28T02:22:09.980 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.12534546699999999, 2023-08-28T02:22:09.980 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:22:09.980 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:22:09.980 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:22:09.980 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:22:09.980 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:22:09.981 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 5477 2023-08-28T02:22:09.981 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:09.981 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:22:09.981 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:09.981 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:22:09.981 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.665942+0000", 2023-08-28T02:22:09.981 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:22:09.981 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:09.981 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:09.981 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:22:09.981 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.665942+0000", 2023-08-28T02:22:09.981 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:22:09.982 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:09.982 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:09.982 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:22:09.982 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.665940+0000", 2023-08-28T02:22:09.982 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:22:09.982 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:09.982 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:09.982 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:22:09.982 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.665966+0000", 2023-08-28T02:22:09.982 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.5164000000000002e-05 2023-08-28T02:22:09.982 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:09.982 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:09.983 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:22:09.983 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.665982+0000", 2023-08-28T02:22:09.983 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.6495e-05 2023-08-28T02:22:09.983 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:09.983 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:09.983 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:22:09.983 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.665991+0000", 2023-08-28T02:22:09.983 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.0399999999999998e-06 2023-08-28T02:22:09.983 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:09.983 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:09.983 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:22:09.983 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.666061+0000", 2023-08-28T02:22:09.984 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.9907999999999995e-05 2023-08-28T02:22:09.984 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:09.984 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:09.984 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:22:09.984 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.666202+0000", 2023-08-28T02:22:09.984 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00014054000000000001 2023-08-28T02:22:09.984 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:09.984 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:09.984 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,4", 2023-08-28T02:22:09.984 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.666297+0000", 2023-08-28T02:22:09.984 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.5587999999999994e-05 2023-08-28T02:22:09.984 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:09.985 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:09.985 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:22:09.985 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.666784+0000", 2023-08-28T02:22:09.986 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000487054 2023-08-28T02:22:09.986 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:09.986 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:09.986 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:22:09.986 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.667501+0000", 2023-08-28T02:22:09.986 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00071644500000000002 2023-08-28T02:22:09.987 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:09.987 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:09.987 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:22:09.987 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.791195+0000", 2023-08-28T02:22:09.987 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.12369411800000001 2023-08-28T02:22:09.987 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:09.987 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:09.987 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:22:09.987 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.791247+0000", 2023-08-28T02:22:09.987 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.2036000000000001e-05 2023-08-28T02:22:09.987 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:09.987 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:09.988 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:22:09.988 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.791288+0000", 2023-08-28T02:22:09.988 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.0887999999999999e-05 2023-08-28T02:22:09.988 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:22:09.988 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:22:09.988 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:22:09.988 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:09.988 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:09.988 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.44594.0:6047 5.e e124/119 5:7f40ab5f:::benchmark_data_smithi143_236200_object6046:head v 124'345, mlcod=124'343)", 2023-08-28T02:22:09.988 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:22:08.943235+0000", 2023-08-28T02:22:09.988 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 1.031347287, 2023-08-28T02:22:09.988 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.045147299000000002, 2023-08-28T02:22:09.989 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:22:09.989 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:22:09.989 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:22:09.989 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:09.989 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:22:09.989 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.943235+0000", 2023-08-28T02:22:09.989 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:22:09.989 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:09.989 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:09.989 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:22:09.989 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.943235+0000", 2023-08-28T02:22:09.989 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:22:09.990 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:09.990 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:09.990 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:22:09.990 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.943234+0000", 2023-08-28T02:22:09.990 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:22:09.990 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:09.990 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:09.990 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:22:09.990 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.943265+0000", 2023-08-28T02:22:09.990 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.0985999999999998e-05 2023-08-28T02:22:09.990 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:09.990 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:09.991 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:22:09.991 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.943273+0000", 2023-08-28T02:22:09.991 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.9640000000000003e-06 2023-08-28T02:22:09.991 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:09.991 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:09.991 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:22:09.991 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.943279+0000", 2023-08-28T02:22:09.991 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.339e-06 2023-08-28T02:22:09.991 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:09.991 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:09.991 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:22:09.992 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.943339+0000", 2023-08-28T02:22:09.992 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.0304999999999999e-05 2023-08-28T02:22:09.992 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:09.992 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:09.992 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:22:09.992 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.943360+0000", 2023-08-28T02:22:09.992 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.0843999999999999e-05 2023-08-28T02:22:09.992 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:09.992 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:09.992 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:22:09.992 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.943919+0000", 2023-08-28T02:22:09.993 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00055935399999999997 2023-08-28T02:22:09.993 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:09.993 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:09.993 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:22:09.993 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.988382+0000", 2023-08-28T02:22:09.993 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.044462858000000001 2023-08-28T02:22:09.993 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:22:09.993 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:22:09.993 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:22:09.993 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:09.993 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:09.993 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.44594.0:6046 5.1 e124/119 5:88219906:::benchmark_data_smithi143_236200_object6045:head v 124'416, mlcod=124'413)", 2023-08-28T02:22:09.994 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:22:08.943346+0000", 2023-08-28T02:22:09.994 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 1.03123601, 2023-08-28T02:22:09.994 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.045071506999999997, 2023-08-28T02:22:09.994 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:22:09.994 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:22:09.994 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:22:09.994 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:09.994 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:22:09.994 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.943346+0000", 2023-08-28T02:22:09.994 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:22:09.994 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:09.994 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:09.995 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:22:09.995 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.943346+0000", 2023-08-28T02:22:09.995 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:22:09.995 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:09.995 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:09.995 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:22:09.995 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.943346+0000", 2023-08-28T02:22:09.995 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:22:09.995 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:09.995 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:09.995 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:22:09.995 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.943497+0000", 2023-08-28T02:22:09.996 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00015164200000000001 2023-08-28T02:22:09.996 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:09.996 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:09.996 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:22:09.996 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.943506+0000", 2023-08-28T02:22:09.996 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.2329999999999995e-06 2023-08-28T02:22:09.996 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:09.996 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:09.996 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:22:09.996 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.943514+0000", 2023-08-28T02:22:09.996 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.3980000000000002e-06 2023-08-28T02:22:09.996 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:09.996 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:09.997 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:22:09.997 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.943556+0000", 2023-08-28T02:22:09.997 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.2292000000000001e-05 2023-08-28T02:22:09.997 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:09.997 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:09.997 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:22:09.997 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.943575+0000", 2023-08-28T02:22:09.997 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.8868999999999998e-05 2023-08-28T02:22:09.997 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:09.997 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:09.997 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:22:09.997 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.944246+0000", 2023-08-28T02:22:09.998 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00067117399999999997 2023-08-28T02:22:09.998 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:09.998 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:09.998 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:22:09.998 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.988417+0000", 2023-08-28T02:22:09.998 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.044171357000000001 2023-08-28T02:22:09.998 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:22:09.998 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:22:09.998 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:22:09.999 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:09.999 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:09.999 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:6053 5.5 5:a8e431d0:::benchmark_data_smithi143_236200_object6052:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-08-28T02:22:09.999 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:22:08.943639+0000", 2023-08-28T02:22:09.999 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 1.030942808, 2023-08-28T02:22:09.999 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.048089663999999997, 2023-08-28T02:22:09.999 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:22:09.999 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:22:09.999 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:22:09.999 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:22:09.999 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:22:09.999 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 6053 2023-08-28T02:22:10.000 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.000 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:22:10.000 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.000 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:22:10.000 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.943639+0000", 2023-08-28T02:22:10.000 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:22:10.000 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.000 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.000 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:22:10.000 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.943639+0000", 2023-08-28T02:22:10.000 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:22:10.000 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.001 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.001 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:22:10.001 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.943638+0000", 2023-08-28T02:22:10.001 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:22:10.001 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.001 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.001 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:22:10.001 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.943663+0000", 2023-08-28T02:22:10.001 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.5150000000000001e-05 2023-08-28T02:22:10.001 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.001 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.001 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:22:10.001 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.943673+0000", 2023-08-28T02:22:10.002 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.9799999999999993e-06 2023-08-28T02:22:10.002 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.002 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.002 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:22:10.002 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.943680+0000", 2023-08-28T02:22:10.002 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.0620000000000003e-06 2023-08-28T02:22:10.002 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.002 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.002 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:22:10.002 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.943722+0000", 2023-08-28T02:22:10.002 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.1726e-05 2023-08-28T02:22:10.002 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.003 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.003 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:22:10.003 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.943856+0000", 2023-08-28T02:22:10.003 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00013388000000000001 2023-08-28T02:22:10.003 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.003 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.003 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,4", 2023-08-28T02:22:10.003 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.988475+0000", 2023-08-28T02:22:10.003 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.044619374000000003 2023-08-28T02:22:10.003 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.003 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.003 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:22:10.004 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.989790+0000", 2023-08-28T02:22:10.004 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.001315042 2023-08-28T02:22:10.004 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.004 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.004 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:22:10.004 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.990254+0000", 2023-08-28T02:22:10.004 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00046348800000000002 2023-08-28T02:22:10.004 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.004 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.004 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:22:10.004 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.991651+0000", 2023-08-28T02:22:10.004 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.001397276 2023-08-28T02:22:10.004 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.005 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.005 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:22:10.005 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.991694+0000", 2023-08-28T02:22:10.005 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.3068000000000002e-05 2023-08-28T02:22:10.005 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.005 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.005 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:22:10.005 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.991729+0000", 2023-08-28T02:22:10.005 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.4740999999999998e-05 2023-08-28T02:22:10.005 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:22:10.005 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:22:10.005 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:22:10.006 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.006 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.006 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.44594.0:6054 5.8 e124/119 5:1b0e35ba:::benchmark_data_smithi143_236200_object6053:head v 124'369, mlcod=124'366)", 2023-08-28T02:22:10.006 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:22:08.944192+0000", 2023-08-28T02:22:10.006 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 1.030390253, 2023-08-28T02:22:10.006 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.045977179999999999, 2023-08-28T02:22:10.006 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:22:10.006 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:22:10.006 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:22:10.006 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.006 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:22:10.006 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.944192+0000", 2023-08-28T02:22:10.007 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:22:10.007 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.007 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.007 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:22:10.007 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.944192+0000", 2023-08-28T02:22:10.007 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:22:10.007 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.007 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.007 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:22:10.007 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.944191+0000", 2023-08-28T02:22:10.007 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:22:10.007 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.008 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.008 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:22:10.008 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.944213+0000", 2023-08-28T02:22:10.008 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.1926999999999999e-05 2023-08-28T02:22:10.008 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.008 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.008 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:22:10.008 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.988367+0000", 2023-08-28T02:22:10.008 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.044154101000000001 2023-08-28T02:22:10.008 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.008 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.008 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:22:10.008 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.988386+0000", 2023-08-28T02:22:10.009 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.8635999999999999e-05 2023-08-28T02:22:10.009 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.009 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.009 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:22:10.009 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.988465+0000", 2023-08-28T02:22:10.009 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.9028000000000003e-05 2023-08-28T02:22:10.009 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.009 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.009 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:22:10.009 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.988489+0000", 2023-08-28T02:22:10.009 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.4545e-05 2023-08-28T02:22:10.009 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.010 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.010 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:22:10.010 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.990136+0000", 2023-08-28T02:22:10.010 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0016465519999999999 2023-08-28T02:22:10.010 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.010 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.010 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:22:10.010 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.990169+0000", 2023-08-28T02:22:10.010 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.2919000000000001e-05 2023-08-28T02:22:10.010 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:22:10.010 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:22:10.010 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:22:10.011 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.011 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.011 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.44594.0:6050 5.e e124/119 5:7f772344:::benchmark_data_smithi143_236200_object6049:head v 124'346, mlcod=124'343)", 2023-08-28T02:22:10.011 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:22:08.944249+0000", 2023-08-28T02:22:10.011 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 1.030333374, 2023-08-28T02:22:10.011 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.045858800999999998, 2023-08-28T02:22:10.011 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:22:10.011 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:22:10.011 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:22:10.011 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.011 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:22:10.011 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.944249+0000", 2023-08-28T02:22:10.012 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:22:10.012 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.012 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.012 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:22:10.012 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.944249+0000", 2023-08-28T02:22:10.012 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:22:10.012 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.012 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.012 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:22:10.012 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.944248+0000", 2023-08-28T02:22:10.012 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:22:10.012 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.012 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.013 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:22:10.013 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.944278+0000", 2023-08-28T02:22:10.013 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.0011000000000001e-05 2023-08-28T02:22:10.013 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.013 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.013 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:22:10.013 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.988372+0000", 2023-08-28T02:22:10.013 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.044093933000000002 2023-08-28T02:22:10.013 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.013 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.013 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:22:10.013 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.988385+0000", 2023-08-28T02:22:10.013 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3239e-05 2023-08-28T02:22:10.014 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.014 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.014 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:22:10.014 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.988587+0000", 2023-08-28T02:22:10.014 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00020147399999999999 2023-08-28T02:22:10.014 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.014 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.014 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:22:10.014 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.988607+0000", 2023-08-28T02:22:10.014 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.0136000000000002e-05 2023-08-28T02:22:10.014 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.014 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.014 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:22:10.015 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.990065+0000", 2023-08-28T02:22:10.015 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.001458476 2023-08-28T02:22:10.015 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.015 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.015 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:22:10.015 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.990107+0000", 2023-08-28T02:22:10.015 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.1983000000000001e-05 2023-08-28T02:22:10.015 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:22:10.015 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:22:10.015 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:22:10.015 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.015 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.016 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.44594.0:6055 5.2 e124/119 5:485099b3:::benchmark_data_smithi143_236200_object6054:head v 124'370, mlcod=124'367)", 2023-08-28T02:22:10.016 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:22:08.944352+0000", 2023-08-28T02:22:10.016 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 1.0302300179999999, 2023-08-28T02:22:10.016 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.045627266, 2023-08-28T02:22:10.016 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:22:10.016 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:22:10.016 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:22:10.016 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.016 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:22:10.016 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.944352+0000", 2023-08-28T02:22:10.016 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:22:10.016 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.016 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.017 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:22:10.017 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.944352+0000", 2023-08-28T02:22:10.017 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:22:10.017 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.017 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.017 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:22:10.017 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.944351+0000", 2023-08-28T02:22:10.017 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:22:10.017 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.017 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.017 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:22:10.017 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.944383+0000", 2023-08-28T02:22:10.018 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.2246999999999998e-05 2023-08-28T02:22:10.018 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.018 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.018 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:22:10.018 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.988363+0000", 2023-08-28T02:22:10.018 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.043979332000000003 2023-08-28T02:22:10.018 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.018 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.018 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:22:10.018 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.988385+0000", 2023-08-28T02:22:10.018 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.2541e-05 2023-08-28T02:22:10.018 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.018 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.019 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:22:10.019 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.988432+0000", 2023-08-28T02:22:10.019 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.6356000000000001e-05 2023-08-28T02:22:10.019 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.019 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.019 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:22:10.019 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.988460+0000", 2023-08-28T02:22:10.019 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.7820000000000001e-05 2023-08-28T02:22:10.019 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.019 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.019 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:22:10.019 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.989945+0000", 2023-08-28T02:22:10.020 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00148521 2023-08-28T02:22:10.020 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.020 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.020 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:22:10.020 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.989979+0000", 2023-08-28T02:22:10.020 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.4489000000000001e-05 2023-08-28T02:22:10.020 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:22:10.020 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:22:10.020 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:22:10.020 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.020 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.020 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:6077 5.3 5:ca48062c:::benchmark_data_smithi143_236200_object6076:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-08-28T02:22:10.020 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:22:08.990746+0000", 2023-08-28T02:22:10.021 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.98383589999999999, 2023-08-28T02:22:10.021 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.165224116, 2023-08-28T02:22:10.021 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:22:10.021 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:22:10.021 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:22:10.021 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:22:10.021 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:22:10.021 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 6077 2023-08-28T02:22:10.021 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.021 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:22:10.021 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.021 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:22:10.022 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.990746+0000", 2023-08-28T02:22:10.022 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:22:10.022 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.022 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.022 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:22:10.022 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.990746+0000", 2023-08-28T02:22:10.022 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:22:10.022 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.022 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.022 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:22:10.022 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.990744+0000", 2023-08-28T02:22:10.022 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:22:10.022 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.023 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.023 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:22:10.023 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.990773+0000", 2023-08-28T02:22:10.023 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.8649999999999998e-05 2023-08-28T02:22:10.023 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.023 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.023 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:22:10.023 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.990785+0000", 2023-08-28T02:22:10.023 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1929999999999999e-05 2023-08-28T02:22:10.023 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.023 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.023 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:22:10.024 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.990797+0000", 2023-08-28T02:22:10.024 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.198e-05 2023-08-28T02:22:10.024 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.024 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.024 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:22:10.024 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.990872+0000", 2023-08-28T02:22:10.024 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.5495999999999997e-05 2023-08-28T02:22:10.024 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.024 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.024 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:22:10.024 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.990984+0000", 2023-08-28T02:22:10.024 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000111888 2023-08-28T02:22:10.025 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.025 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.025 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 5,6", 2023-08-28T02:22:10.025 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.991583+0000", 2023-08-28T02:22:10.025 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00059899400000000005 2023-08-28T02:22:10.025 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.025 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.025 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:22:10.025 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.992551+0000", 2023-08-28T02:22:10.025 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00096798200000000004 2023-08-28T02:22:10.025 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.025 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.025 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:22:10.026 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.155853+0000", 2023-08-28T02:22:10.026 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.16330176199999999 2023-08-28T02:22:10.026 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.026 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.026 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:22:10.026 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.155884+0000", 2023-08-28T02:22:10.026 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.1349000000000001e-05 2023-08-28T02:22:10.026 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.026 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.026 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:22:10.026 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.155934+0000", 2023-08-28T02:22:10.027 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.9209999999999998e-05 2023-08-28T02:22:10.027 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.027 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.027 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:22:10.027 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.155970+0000", 2023-08-28T02:22:10.027 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.6562e-05 2023-08-28T02:22:10.027 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:22:10.027 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:22:10.027 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:22:10.027 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.027 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.027 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:6082 5.5 5:a72e6a2a:::benchmark_data_smithi143_236200_object6081:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-08-28T02:22:10.028 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:22:08.990871+0000", 2023-08-28T02:22:10.028 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.98371080499999997, 2023-08-28T02:22:10.028 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.165912857, 2023-08-28T02:22:10.028 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:22:10.028 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:22:10.028 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:22:10.028 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:22:10.028 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:22:10.028 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 6082 2023-08-28T02:22:10.028 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.028 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:22:10.028 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.029 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:22:10.029 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.990871+0000", 2023-08-28T02:22:10.029 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:22:10.029 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.029 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.029 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:22:10.029 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.990871+0000", 2023-08-28T02:22:10.029 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:22:10.029 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.029 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.029 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:22:10.029 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.990871+0000", 2023-08-28T02:22:10.030 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:22:10.030 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.030 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.030 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:22:10.030 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.990891+0000", 2023-08-28T02:22:10.030 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.0477000000000001e-05 2023-08-28T02:22:10.030 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.030 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.030 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:22:10.030 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.990898+0000", 2023-08-28T02:22:10.030 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.3660000000000003e-06 2023-08-28T02:22:10.030 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.031 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.031 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:22:10.031 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.990904+0000", 2023-08-28T02:22:10.031 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.609e-06 2023-08-28T02:22:10.031 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.031 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.031 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:22:10.031 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.991757+0000", 2023-08-28T02:22:10.031 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00085252600000000004 2023-08-28T02:22:10.031 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.031 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.031 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:22:10.032 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.991871+0000", 2023-08-28T02:22:10.032 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00011441499999999999 2023-08-28T02:22:10.032 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.032 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.032 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,4", 2023-08-28T02:22:10.032 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.992001+0000", 2023-08-28T02:22:10.032 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00013049399999999999 2023-08-28T02:22:10.032 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.032 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.032 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:22:10.032 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.993133+0000", 2023-08-28T02:22:10.032 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0011312029999999999 2023-08-28T02:22:10.033 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.033 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.033 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:22:10.033 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.993284+0000", 2023-08-28T02:22:10.033 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00015169900000000001 2023-08-28T02:22:10.033 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.033 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.033 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:22:10.033 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.156694+0000", 2023-08-28T02:22:10.033 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.163410051 2023-08-28T02:22:10.033 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.033 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.033 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:22:10.034 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.156747+0000", 2023-08-28T02:22:10.034 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.2911999999999998e-05 2023-08-28T02:22:10.034 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.034 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.034 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:22:10.034 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.156784+0000", 2023-08-28T02:22:10.034 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.6708999999999997e-05 2023-08-28T02:22:10.034 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:22:10.034 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:22:10.034 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:22:10.034 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.034 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.034 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:6085 5.5 5:a3a20b01:::benchmark_data_smithi143_236200_object6084:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-08-28T02:22:10.035 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:22:08.992912+0000", 2023-08-28T02:22:10.035 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.98166964599999995, 2023-08-28T02:22:10.035 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.16660082900000001, 2023-08-28T02:22:10.035 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:22:10.035 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:22:10.035 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:22:10.035 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:22:10.035 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:22:10.035 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 6085 2023-08-28T02:22:10.035 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.035 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:22:10.036 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.036 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:22:10.036 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.992912+0000", 2023-08-28T02:22:10.036 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:22:10.036 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.036 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.036 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:22:10.036 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.992912+0000", 2023-08-28T02:22:10.036 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:22:10.036 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.036 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.037 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:22:10.037 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.992911+0000", 2023-08-28T02:22:10.037 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:22:10.037 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.037 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.037 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:22:10.037 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.992936+0000", 2023-08-28T02:22:10.037 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.4807000000000002e-05 2023-08-28T02:22:10.037 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.037 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.037 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:22:10.037 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.992946+0000", 2023-08-28T02:22:10.038 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0593e-05 2023-08-28T02:22:10.038 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.038 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.038 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:22:10.038 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.992953+0000", 2023-08-28T02:22:10.038 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.917e-06 2023-08-28T02:22:10.038 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.038 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.038 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:22:10.038 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.155801+0000", 2023-08-28T02:22:10.038 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.16284750100000001 2023-08-28T02:22:10.038 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.039 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.039 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:22:10.039 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.155953+0000", 2023-08-28T02:22:10.039 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00015253399999999999 2023-08-28T02:22:10.039 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.039 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.039 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,4", 2023-08-28T02:22:10.039 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.156121+0000", 2023-08-28T02:22:10.039 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000167486 2023-08-28T02:22:10.039 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.039 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.040 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:22:10.040 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.157117+0000", 2023-08-28T02:22:10.040 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00099654600000000006 2023-08-28T02:22:10.040 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.040 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.040 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:22:10.040 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.157217+0000", 2023-08-28T02:22:10.040 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.9196000000000004e-05 2023-08-28T02:22:10.040 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.040 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.040 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:22:10.040 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.159436+0000", 2023-08-28T02:22:10.040 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0022190859999999999 2023-08-28T02:22:10.041 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.041 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.041 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:22:10.041 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.159479+0000", 2023-08-28T02:22:10.041 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.2796999999999997e-05 2023-08-28T02:22:10.041 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.041 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.041 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:22:10.041 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.159513+0000", 2023-08-28T02:22:10.041 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.4610999999999998e-05 2023-08-28T02:22:10.041 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:22:10.041 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:22:10.041 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:22:10.042 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.042 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.042 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.44594.0:6086 5.c e124/119 5:3cad0c94:::benchmark_data_smithi143_236200_object6085:head v 124'392, mlcod=124'389)", 2023-08-28T02:22:10.042 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:22:08.993114+0000", 2023-08-28T02:22:10.042 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.981468067, 2023-08-28T02:22:10.042 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.16274122299999999, 2023-08-28T02:22:10.042 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:22:10.042 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:22:10.042 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:22:10.042 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.042 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:22:10.042 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.993114+0000", 2023-08-28T02:22:10.042 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:22:10.043 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.043 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.043 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:22:10.043 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.993114+0000", 2023-08-28T02:22:10.043 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:22:10.043 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.043 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.043 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:22:10.043 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.993114+0000", 2023-08-28T02:22:10.043 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:22:10.043 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.043 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.044 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:22:10.044 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.993131+0000", 2023-08-28T02:22:10.044 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.6965000000000001e-05 2023-08-28T02:22:10.044 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.044 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.044 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:22:10.044 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.993135+0000", 2023-08-28T02:22:10.044 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.4750000000000004e-06 2023-08-28T02:22:10.044 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.044 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.044 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:22:10.044 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.993139+0000", 2023-08-28T02:22:10.045 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.9740000000000004e-06 2023-08-28T02:22:10.045 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.045 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.045 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:22:10.045 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.993191+0000", 2023-08-28T02:22:10.045 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.1885999999999998e-05 2023-08-28T02:22:10.045 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.045 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.045 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:22:10.045 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.993203+0000", 2023-08-28T02:22:10.045 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1938e-05 2023-08-28T02:22:10.045 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.045 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.046 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:22:10.046 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.155821+0000", 2023-08-28T02:22:10.046 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.16261857599999999 2023-08-28T02:22:10.046 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.046 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.046 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:22:10.046 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.155855+0000", 2023-08-28T02:22:10.046 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.3683999999999998e-05 2023-08-28T02:22:10.046 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:22:10.046 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:22:10.046 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:22:10.046 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.047 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.047 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop_reply(client.44594.0:6077 5.3 e124/119 ondisk, result = 0)", 2023-08-28T02:22:10.047 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:22:08.993278+0000", 2023-08-28T02:22:10.047 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.98130416200000004, 2023-08-28T02:22:10.047 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.16257867400000001, 2023-08-28T02:22:10.047 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:22:10.047 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "started", 2023-08-28T02:22:10.047 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:22:10.047 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.047 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:22:10.047 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.993278+0000", 2023-08-28T02:22:10.047 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:22:10.048 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.048 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.048 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:22:10.048 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.993278+0000", 2023-08-28T02:22:10.048 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:22:10.048 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.048 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.048 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:22:10.048 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.993277+0000", 2023-08-28T02:22:10.048 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:22:10.048 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.048 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.048 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:22:10.049 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.993283+0000", 2023-08-28T02:22:10.049 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.2029999999999997e-06 2023-08-28T02:22:10.049 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.049 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.049 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:22:10.049 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.993292+0000", 2023-08-28T02:22:10.049 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.9080000000000001e-06 2023-08-28T02:22:10.049 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.049 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.049 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:22:10.049 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.993301+0000", 2023-08-28T02:22:10.050 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.6340000000000007e-06 2023-08-28T02:22:10.050 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.050 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.050 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:22:10.050 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.155837+0000", 2023-08-28T02:22:10.050 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.16253591100000001 2023-08-28T02:22:10.050 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.050 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.050 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:22:10.050 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.155847+0000", 2023-08-28T02:22:10.050 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0044000000000001e-05 2023-08-28T02:22:10.050 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.051 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.051 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:22:10.051 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.155856+0000", 2023-08-28T02:22:10.051 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.5869999999999999e-06 2023-08-28T02:22:10.051 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:22:10.051 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:22:10.051 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:22:10.051 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.051 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.051 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop_reply(client.44594.0:6077 5.3 e124/119 ondisk, result = 0)", 2023-08-28T02:22:10.051 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:22:08.993367+0000", 2023-08-28T02:22:10.052 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.98121535299999996, 2023-08-28T02:22:10.052 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.16261359, 2023-08-28T02:22:10.052 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:22:10.052 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "started", 2023-08-28T02:22:10.052 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:22:10.052 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.052 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:22:10.052 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.993367+0000", 2023-08-28T02:22:10.052 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:22:10.052 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.052 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.052 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:22:10.053 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.993367+0000", 2023-08-28T02:22:10.053 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:22:10.053 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.053 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.053 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:22:10.053 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.993366+0000", 2023-08-28T02:22:10.053 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:22:10.053 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.053 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.053 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:22:10.053 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.993372+0000", 2023-08-28T02:22:10.053 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.6160000000000001e-06 2023-08-28T02:22:10.053 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.054 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.054 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:22:10.054 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.155811+0000", 2023-08-28T02:22:10.054 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.16243964899999999 2023-08-28T02:22:10.054 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.054 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.054 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:22:10.054 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.155821+0000", 2023-08-28T02:22:10.054 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.3449999999999995e-06 2023-08-28T02:22:10.054 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.054 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.054 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:22:10.055 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.155875+0000", 2023-08-28T02:22:10.055 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.3868e-05 2023-08-28T02:22:10.055 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.055 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.055 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:22:10.055 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.155880+0000", 2023-08-28T02:22:10.055 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.6289999999999998e-06 2023-08-28T02:22:10.055 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.055 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.055 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:22:10.055 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.155980+0000", 2023-08-28T02:22:10.055 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.9953999999999999e-05 2023-08-28T02:22:10.056 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:22:10.056 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:22:10.056 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:22:10.056 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.056 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.056 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.44594.0:6089 5.0 e124/119 5:0b48d546:::benchmark_data_smithi143_236200_object6088:head v 124'428, mlcod=124'426)", 2023-08-28T02:22:10.056 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:22:08.993518+0000", 2023-08-28T02:22:10.056 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.98106441099999997, 2023-08-28T02:22:10.056 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.163660257, 2023-08-28T02:22:10.056 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:22:10.056 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:22:10.056 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:22:10.056 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.057 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:22:10.057 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.993518+0000", 2023-08-28T02:22:10.057 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:22:10.057 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.057 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.057 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:22:10.057 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.993518+0000", 2023-08-28T02:22:10.057 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:22:10.057 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.057 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.057 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:22:10.057 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.993517+0000", 2023-08-28T02:22:10.058 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:22:10.058 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.058 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.058 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:22:10.058 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.993541+0000", 2023-08-28T02:22:10.058 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.4409e-05 2023-08-28T02:22:10.058 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.058 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.058 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:22:10.058 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.155801+0000", 2023-08-28T02:22:10.058 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.162259868 2023-08-28T02:22:10.058 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.059 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.059 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:22:10.059 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.155812+0000", 2023-08-28T02:22:10.059 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0681e-05 2023-08-28T02:22:10.059 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.059 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.059 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:22:10.059 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.155851+0000", 2023-08-28T02:22:10.059 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.9416999999999999e-05 2023-08-28T02:22:10.059 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.059 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.060 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:22:10.060 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.155872+0000", 2023-08-28T02:22:10.060 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.1009000000000002e-05 2023-08-28T02:22:10.060 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.060 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.060 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:22:10.060 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.157144+0000", 2023-08-28T02:22:10.060 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0012713690000000001 2023-08-28T02:22:10.060 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.060 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.060 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:22:10.060 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.157178+0000", 2023-08-28T02:22:10.060 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.4068e-05 2023-08-28T02:22:10.061 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:22:10.061 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:22:10.061 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:22:10.061 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.061 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.061 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:6099 5.3 5:c33791f2:::benchmark_data_smithi143_236200_object6098:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-08-28T02:22:10.061 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:22:08.996447+0000", 2023-08-28T02:22:10.061 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.97813533500000005, 2023-08-28T02:22:10.061 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.16158087600000001, 2023-08-28T02:22:10.061 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:22:10.061 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:22:10.062 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:22:10.062 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:22:10.062 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:22:10.062 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 6099 2023-08-28T02:22:10.062 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.062 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:22:10.062 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.062 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:22:10.062 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.996447+0000", 2023-08-28T02:22:10.062 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:22:10.062 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.062 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.063 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:22:10.063 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.996447+0000", 2023-08-28T02:22:10.063 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:22:10.063 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.063 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.063 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:22:10.063 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.996445+0000", 2023-08-28T02:22:10.063 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:22:10.063 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.063 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.063 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:22:10.063 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:08.996473+0000", 2023-08-28T02:22:10.064 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.8651000000000001e-05 2023-08-28T02:22:10.064 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.064 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.064 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:22:10.064 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.155797+0000", 2023-08-28T02:22:10.064 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.159323521 2023-08-28T02:22:10.064 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.064 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.064 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:22:10.064 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.155814+0000", 2023-08-28T02:22:10.064 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.7266e-05 2023-08-28T02:22:10.064 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.064 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.065 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:22:10.065 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.156476+0000", 2023-08-28T02:22:10.065 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000661722 2023-08-28T02:22:10.065 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.065 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.065 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:22:10.065 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.156600+0000", 2023-08-28T02:22:10.065 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00012435899999999999 2023-08-28T02:22:10.065 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.065 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.065 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 5,6", 2023-08-28T02:22:10.065 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.156728+0000", 2023-08-28T02:22:10.065 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00012832100000000001 2023-08-28T02:22:10.066 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.066 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.066 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:22:10.066 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.157525+0000", 2023-08-28T02:22:10.066 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00079685599999999997 2023-08-28T02:22:10.066 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.066 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.066 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:22:10.066 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.157907+0000", 2023-08-28T02:22:10.066 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00038123699999999998 2023-08-28T02:22:10.066 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.066 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.067 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:22:10.067 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.157939+0000", 2023-08-28T02:22:10.067 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.2125999999999997e-05 2023-08-28T02:22:10.067 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.067 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.067 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:22:10.067 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.157983+0000", 2023-08-28T02:22:10.067 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.4753999999999999e-05 2023-08-28T02:22:10.067 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.067 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.067 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:22:10.067 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.158028+0000", 2023-08-28T02:22:10.068 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.409e-05 2023-08-28T02:22:10.068 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:22:10.068 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:22:10.068 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:22:10.068 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.068 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.068 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:7926 5.5 5:a05a858e:::benchmark_data_smithi143_236200_object7925:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-08-28T02:22:10.068 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:22:09.770508+0000", 2023-08-28T02:22:10.068 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.20407356199999999, 2023-08-28T02:22:10.068 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.015620801, 2023-08-28T02:22:10.068 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:22:10.068 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:22:10.069 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:22:10.069 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:22:10.069 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:22:10.069 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 7926 2023-08-28T02:22:10.069 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.069 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:22:10.069 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.069 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:22:10.069 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.770508+0000", 2023-08-28T02:22:10.069 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:22:10.069 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.069 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.069 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:22:10.070 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.770508+0000", 2023-08-28T02:22:10.070 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:22:10.070 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.070 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.070 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:22:10.070 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.770508+0000", 2023-08-28T02:22:10.070 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:22:10.070 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.070 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.070 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:22:10.070 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.770534+0000", 2023-08-28T02:22:10.070 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.6140999999999999e-05 2023-08-28T02:22:10.070 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.071 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.071 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:22:10.071 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.770544+0000", 2023-08-28T02:22:10.071 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.8709999999999992e-06 2023-08-28T02:22:10.071 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.071 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.071 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:22:10.071 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.770550+0000", 2023-08-28T02:22:10.071 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.4649999999999999e-06 2023-08-28T02:22:10.071 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.071 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.072 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:22:10.072 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.771158+0000", 2023-08-28T02:22:10.072 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00060786 2023-08-28T02:22:10.072 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.072 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.072 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:22:10.072 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.771268+0000", 2023-08-28T02:22:10.072 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000110215 2023-08-28T02:22:10.072 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.072 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.072 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,4", 2023-08-28T02:22:10.073 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.771414+0000", 2023-08-28T02:22:10.073 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00014522800000000001 2023-08-28T02:22:10.073 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.073 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.073 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:22:10.073 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.773050+0000", 2023-08-28T02:22:10.073 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.001636298 2023-08-28T02:22:10.073 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.073 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.073 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:22:10.073 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.774617+0000", 2023-08-28T02:22:10.073 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0015668500000000001 2023-08-28T02:22:10.074 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.074 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.074 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:22:10.074 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.786040+0000", 2023-08-28T02:22:10.074 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.011423101 2023-08-28T02:22:10.074 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.074 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.074 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:22:10.074 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.786092+0000", 2023-08-28T02:22:10.074 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.1758000000000001e-05 2023-08-28T02:22:10.074 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.074 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.074 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:22:10.075 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.786129+0000", 2023-08-28T02:22:10.075 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.7524999999999997e-05 2023-08-28T02:22:10.075 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:22:10.075 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:22:10.075 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:22:10.075 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.075 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.075 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.44594.0:7966 5.e e124/119 5:775f6d72:::benchmark_data_smithi143_236200_object7965:head v 124'474, mlcod=124'471)", 2023-08-28T02:22:10.075 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:22:09.795024+0000", 2023-08-28T02:22:10.075 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.17955823700000001, 2023-08-28T02:22:10.075 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.014512148000000001, 2023-08-28T02:22:10.075 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:22:10.076 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:22:10.076 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:22:10.076 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.076 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:22:10.076 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.795024+0000", 2023-08-28T02:22:10.076 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:22:10.076 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.076 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.076 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:22:10.076 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.795024+0000", 2023-08-28T02:22:10.076 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:22:10.076 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.077 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.077 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:22:10.077 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.795023+0000", 2023-08-28T02:22:10.077 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:22:10.077 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.077 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.077 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:22:10.077 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.795070+0000", 2023-08-28T02:22:10.077 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.7271999999999997e-05 2023-08-28T02:22:10.077 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.077 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.077 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:22:10.077 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.795086+0000", 2023-08-28T02:22:10.078 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.5478999999999999e-05 2023-08-28T02:22:10.078 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.078 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.078 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:22:10.078 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.795094+0000", 2023-08-28T02:22:10.078 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.0770000000000008e-06 2023-08-28T02:22:10.078 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.078 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.078 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:22:10.078 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.795968+0000", 2023-08-28T02:22:10.078 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00087378100000000002 2023-08-28T02:22:10.078 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.079 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.079 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:22:10.079 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.795988+0000", 2023-08-28T02:22:10.079 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.0415999999999999e-05 2023-08-28T02:22:10.079 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.079 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.079 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:22:10.079 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.809500+0000", 2023-08-28T02:22:10.079 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.013512355 2023-08-28T02:22:10.079 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.079 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.079 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:22:10.079 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.809536+0000", 2023-08-28T02:22:10.080 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.5608000000000002e-05 2023-08-28T02:22:10.080 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:22:10.080 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:22:10.080 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:22:10.080 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.080 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.080 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.44594.0:7972 5.c e124/119 5:3194d6de:::benchmark_data_smithi143_236200_object7971:head v 124'522, mlcod=124'520)", 2023-08-28T02:22:10.080 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:22:09.802647+0000", 2023-08-28T02:22:10.080 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.17193539899999999, 2023-08-28T02:22:10.080 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.014222162999999999, 2023-08-28T02:22:10.080 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:22:10.081 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:22:10.081 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:22:10.081 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.081 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:22:10.081 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.802647+0000", 2023-08-28T02:22:10.081 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:22:10.081 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.081 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.081 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:22:10.081 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.802647+0000", 2023-08-28T02:22:10.081 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:22:10.081 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.082 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.082 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:22:10.082 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.802646+0000", 2023-08-28T02:22:10.082 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:22:10.082 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.082 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.082 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:22:10.082 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.802674+0000", 2023-08-28T02:22:10.082 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.853e-05 2023-08-28T02:22:10.082 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.082 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.082 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:22:10.083 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.802704+0000", 2023-08-28T02:22:10.083 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.9485e-05 2023-08-28T02:22:10.083 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.083 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.083 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:22:10.083 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.802713+0000", 2023-08-28T02:22:10.083 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.0799999999999995e-06 2023-08-28T02:22:10.083 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.083 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.083 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:22:10.083 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.802758+0000", 2023-08-28T02:22:10.083 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.5482999999999998e-05 2023-08-28T02:22:10.084 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.084 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.084 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:22:10.084 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.802783+0000", 2023-08-28T02:22:10.084 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.4201000000000001e-05 2023-08-28T02:22:10.084 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.084 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.084 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:22:10.084 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.816825+0000", 2023-08-28T02:22:10.084 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.014042057 2023-08-28T02:22:10.084 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.084 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.085 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:22:10.085 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.816869+0000", 2023-08-28T02:22:10.085 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.4186e-05 2023-08-28T02:22:10.085 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:22:10.085 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:22:10.085 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:22:10.085 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.085 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.085 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.44594.0:7998 5.e e124/119 5:7059b0e1:::benchmark_data_smithi143_236200_object7997:head v 124'475, mlcod=124'473)", 2023-08-28T02:22:10.085 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:22:09.824888+0000", 2023-08-28T02:22:10.085 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.14969368599999999, 2023-08-28T02:22:10.085 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.018203303000000001, 2023-08-28T02:22:10.085 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:22:10.086 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:22:10.086 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:22:10.086 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.086 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:22:10.086 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.824888+0000", 2023-08-28T02:22:10.086 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:22:10.086 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.086 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.086 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:22:10.086 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.824888+0000", 2023-08-28T02:22:10.086 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:22:10.086 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.087 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.087 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:22:10.087 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.824888+0000", 2023-08-28T02:22:10.087 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:22:10.087 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.087 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.087 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:22:10.087 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.824941+0000", 2023-08-28T02:22:10.087 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.3498999999999997e-05 2023-08-28T02:22:10.087 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.087 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.087 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:22:10.088 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.824953+0000", 2023-08-28T02:22:10.088 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1556000000000001e-05 2023-08-28T02:22:10.088 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.088 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.088 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:22:10.088 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.824960+0000", 2023-08-28T02:22:10.088 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.0859999999999996e-06 2023-08-28T02:22:10.088 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.088 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.088 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:22:10.088 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.825002+0000", 2023-08-28T02:22:10.088 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.2663999999999997e-05 2023-08-28T02:22:10.089 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.089 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.089 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:22:10.089 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.825048+0000", 2023-08-28T02:22:10.089 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.5438000000000001e-05 2023-08-28T02:22:10.089 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.089 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.089 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:22:10.089 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.843055+0000", 2023-08-28T02:22:10.089 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.018006742999999999 2023-08-28T02:22:10.089 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.089 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.090 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:22:10.090 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.843092+0000", 2023-08-28T02:22:10.090 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.7067000000000003e-05 2023-08-28T02:22:10.090 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:22:10.090 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:22:10.090 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:22:10.090 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.090 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.090 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:8141 5.5 5:ac0eccbc:::benchmark_data_smithi143_236200_object8140:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-08-28T02:22:10.090 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:22:09.899144+0000", 2023-08-28T02:22:10.090 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.075438064999999999, 2023-08-28T02:22:10.090 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.013604646, 2023-08-28T02:22:10.091 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:22:10.091 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:22:10.091 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:22:10.091 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:22:10.091 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:22:10.091 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 8141 2023-08-28T02:22:10.091 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.091 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:22:10.091 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.091 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:22:10.091 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.899144+0000", 2023-08-28T02:22:10.092 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:22:10.092 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.092 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.092 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:22:10.092 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.899144+0000", 2023-08-28T02:22:10.092 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:22:10.092 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.092 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.092 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:22:10.092 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.899142+0000", 2023-08-28T02:22:10.092 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:22:10.092 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.093 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.093 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:22:10.093 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.899166+0000", 2023-08-28T02:22:10.093 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.3764000000000002e-05 2023-08-28T02:22:10.093 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.093 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.093 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:22:10.093 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.899182+0000", 2023-08-28T02:22:10.093 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.5868000000000001e-05 2023-08-28T02:22:10.093 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.093 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.093 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:22:10.094 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.899192+0000", 2023-08-28T02:22:10.094 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.9660000000000008e-06 2023-08-28T02:22:10.094 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.094 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.094 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:22:10.094 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.899241+0000", 2023-08-28T02:22:10.094 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.8823999999999999e-05 2023-08-28T02:22:10.094 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.094 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.094 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:22:10.094 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.899399+0000", 2023-08-28T02:22:10.094 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00015867400000000001 2023-08-28T02:22:10.094 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.095 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.095 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,4", 2023-08-28T02:22:10.095 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.899554+0000", 2023-08-28T02:22:10.095 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000154537 2023-08-28T02:22:10.095 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.095 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.095 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:22:10.095 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.900363+0000", 2023-08-28T02:22:10.095 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00080872599999999995 2023-08-28T02:22:10.095 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.095 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.095 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:22:10.096 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.904175+0000", 2023-08-28T02:22:10.096 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0038128049999999998 2023-08-28T02:22:10.096 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.096 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.096 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:22:10.096 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.912668+0000", 2023-08-28T02:22:10.096 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0084923099999999994 2023-08-28T02:22:10.096 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.096 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.096 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:22:10.096 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.912712+0000", 2023-08-28T02:22:10.096 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.4518999999999998e-05 2023-08-28T02:22:10.096 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:22:10.097 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:22:10.097 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:22:10.097 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:09.912749+0000", 2023-08-28T02:22:10.097 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.6285999999999998e-05 2023-08-28T02:22:10.097 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:22:10.097 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:22:10.097 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:22:10.097 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:22:10.097 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:22:10.097 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:22:10.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:10.098 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:22:10.099 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:22:10.199 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:22:10.229 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 3 16 9000 8984 187.051 163.25 0.00282 0.00533446 2023-08-28T02:22:10.248 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:10.402 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:22:10.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:10.406 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:22:10.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:10 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:22:10.506 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:22:10.548 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:10 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:22:10.549 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:10 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:22:10.552 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:11.229 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 4 16 11314 11298 176.44 144.625 0.00432494 0.00565986 2023-08-28T02:22:11.268 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is unset 2023-08-28T02:22:11.540 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:22:11.546 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:11 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:22:11.547 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:11 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3827727408' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:22:11.547 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:11 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:22:11.547 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:11 smithi143 ceph-mon[183059]: pgmap v236: 89 pgs: 89 active+clean; 101 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 12 MiB/s wr, 192 op/s 2023-08-28T02:22:11.547 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:11 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:22:11.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:11.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:22:11.548 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:11 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:22:11.548 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:11 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3827727408' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:22:11.548 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:11 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:22:11.549 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:11 smithi143 ceph-mon[189871]: pgmap v236: 89 pgs: 89 active+clean; 101 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 12 MiB/s wr, 192 op/s 2023-08-28T02:22:11.549 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:11 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:22:11.648 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:22:11.698 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:11.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:11 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:22:11.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:11 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3827727408' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:22:11.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:11 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:22:11.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:11 smithi169 ceph-mon[159415]: pgmap v236: 89 pgs: 89 active+clean; 101 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 12 MiB/s wr, 192 op/s 2023-08-28T02:22:11.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:11 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:22:11.929 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:22:11 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:22:11] "GET /metrics HTTP/1.1" 200 36460 "" "Prometheus/2.33.4" 2023-08-28T02:22:12.187 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:22:12.230 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.1 dump_ops_in_flight 2023-08-28T02:22:12.230 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 5 16 14172 14156 176.869 178.625 0.00288494 0.00564602 2023-08-28T02:22:12.317 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:22:12.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:12.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:22:12.429 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:22:12.440 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:12 smithi143 ceph-mon[183059]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:22:12.440 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:12 smithi143 ceph-mon[183059]: failed to encode map e125 with expected crc 2023-08-28T02:22:12.440 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:12 smithi143 ceph-mon[183059]: failed to encode map e125 with expected crc 2023-08-28T02:22:12.440 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:12 smithi143 ceph-mon[183059]: failed to encode map e125 with expected crc 2023-08-28T02:22:12.440 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:12 smithi143 ceph-mon[183059]: failed to encode map e125 with expected crc 2023-08-28T02:22:12.440 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:12 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:22:12.440 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:12 smithi143 ceph-mon[183059]: osdmap e125: 8 total, 8 up, 7 in 2023-08-28T02:22:12.440 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:12 smithi143 ceph-mon[183059]: failed to encode map e125 with expected crc 2023-08-28T02:22:12.440 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:12 smithi143 ceph-mon[183059]: failed to encode map e125 with expected crc 2023-08-28T02:22:12.440 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:12 smithi143 ceph-mon[183059]: failed to encode map e125 with expected crc 2023-08-28T02:22:12.441 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:12 smithi143 ceph-mon[183059]: failed to encode map e125 with expected crc 2023-08-28T02:22:12.441 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:12 smithi143 ceph-mon[183059]: failed to encode map e125 with expected crc 2023-08-28T02:22:12.442 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:12 smithi143 ceph-mon[189871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:22:12.442 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:12 smithi143 ceph-mon[189871]: failed to encode map e125 with expected crc 2023-08-28T02:22:12.443 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:12 smithi143 ceph-mon[189871]: failed to encode map e125 with expected crc 2023-08-28T02:22:12.443 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:12 smithi143 ceph-mon[189871]: failed to encode map e125 with expected crc 2023-08-28T02:22:12.443 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:12 smithi143 ceph-mon[189871]: failed to encode map e125 with expected crc 2023-08-28T02:22:12.446 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:12 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:22:12.446 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:12 smithi143 ceph-mon[189871]: osdmap e125: 8 total, 8 up, 7 in 2023-08-28T02:22:12.446 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:12 smithi143 ceph-mon[189871]: failed to encode map e125 with expected crc 2023-08-28T02:22:12.446 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:12 smithi143 ceph-mon[189871]: failed to encode map e125 with expected crc 2023-08-28T02:22:12.446 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:12 smithi143 ceph-mon[189871]: failed to encode map e125 with expected crc 2023-08-28T02:22:12.447 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:12 smithi143 ceph-mon[189871]: failed to encode map e125 with expected crc 2023-08-28T02:22:12.447 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:12 smithi143 ceph-mon[189871]: failed to encode map e125 with expected crc 2023-08-28T02:22:12.535 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:12.702 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:22:12.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:12.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:22:12.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:12 smithi169 ceph-mon[159415]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:22:12.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:12 smithi169 ceph-mon[159415]: failed to encode map e125 with expected crc 2023-08-28T02:22:12.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:12 smithi169 ceph-mon[159415]: failed to encode map e125 with expected crc 2023-08-28T02:22:12.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:12 smithi169 ceph-mon[159415]: failed to encode map e125 with expected crc 2023-08-28T02:22:12.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:12 smithi169 ceph-mon[159415]: failed to encode map e125 with expected crc 2023-08-28T02:22:12.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:12 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:22:12.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:12 smithi169 ceph-mon[159415]: osdmap e125: 8 total, 8 up, 7 in 2023-08-28T02:22:12.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:12 smithi169 ceph-mon[159415]: failed to encode map e125 with expected crc 2023-08-28T02:22:12.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:12 smithi169 ceph-mon[159415]: failed to encode map e125 with expected crc 2023-08-28T02:22:12.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:12 smithi169 ceph-mon[159415]: failed to encode map e125 with expected crc 2023-08-28T02:22:12.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:12 smithi169 ceph-mon[159415]: failed to encode map e125 with expected crc 2023-08-28T02:22:12.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:12 smithi169 ceph-mon[159415]: failed to encode map e125 with expected crc 2023-08-28T02:22:12.807 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:22:12.831 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:22:12.857 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:13.004 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:22:13.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:13.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:22:13.114 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:22:13.164 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:13.230 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 6 16 16521 16505 171.859 146.812 0.00556103 0.00581176 2023-08-28T02:22:13.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:13 smithi143 ceph-mon[183059]: pgmap v238: 89 pgs: 89 active+clean; 824 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 105 MiB/s wr, 1.67k op/s 2023-08-28T02:22:13.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:13 smithi143 ceph-mon[189871]: pgmap v238: 89 pgs: 89 active+clean; 824 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 105 MiB/s wr, 1.67k op/s 2023-08-28T02:22:13.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:13 smithi169 ceph-mon[159415]: pgmap v238: 89 pgs: 89 active+clean; 824 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 105 MiB/s wr, 1.67k op/s 2023-08-28T02:22:14.184 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:22:14.230 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 7 16 18541 18525 165.344 126.25 0.00650975 0.00604153 2023-08-28T02:22:14.440 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset nodeep-scrub 2023-08-28T02:22:14.473 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:22:14.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:14.482 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:22:14.583 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:22:14.679 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:14.736 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:14 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:22:14.736 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:14 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:22:14.737 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:14 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:22:14.737 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:14 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:22:14.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:14 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:22:14.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:14 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:22:15.087 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:22:15.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:15.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:22:15.184 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:22:15.192 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:22:15.222 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:22:15.230 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 8 16 20758 20742 161.995 138.562 0.00763533 0.00616555 2023-08-28T02:22:15.239 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:15.260 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:22:15.293 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:22:15.330 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:22:15.331 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:22:15.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:15.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:22:15.364 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:22:15.402 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:22:15.435 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:22:15.437 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:22:15.437 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:22:15.437 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:22:15.437 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:22:15.437 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:22:15.437 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:22:15.438 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:22:15.438 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:22:15.438 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:22:15.489 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:15.555 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:22:15.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:15.560 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:22:15.661 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:22:15.745 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:15 smithi169 ceph-mon[159415]: pgmap v239: 89 pgs: 89 active+clean; 925 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-08-28T02:22:15.746 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:15 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1056918799' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:22:15.748 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:15.845 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:22:15.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:15.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:22:15.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:15 smithi143 ceph-mon[183059]: pgmap v239: 89 pgs: 89 active+clean; 925 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-08-28T02:22:15.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:15 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1056918799' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:22:15.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:15 smithi143 ceph-mon[189871]: pgmap v239: 89 pgs: 89 active+clean; 925 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-08-28T02:22:15.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:15 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1056918799' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:22:15.954 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:22:16.008 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:16.230 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 9 16 22854 22838 158.55 131 0.00148138 0.00620398 2023-08-28T02:22:16.640 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:22:16.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:16.643 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:22:16.744 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:22:16.770 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:17.230 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 10 16 24647 24631 153.901 112.062 0.0122578 0.00649153 2023-08-28T02:22:17.381 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:22:17.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:17.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:22:17.492 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:22:17.523 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:17.904 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:22:17.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:17.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:22:18.008 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:22:18.062 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:18.065 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:22:18.143 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:22:18.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:18.149 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:22:18.162 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:18 smithi143 ceph-mon[183059]: pgmap v240: 89 pgs: 89 active+clean; 1.2 GiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-08-28T02:22:18.163 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:18 smithi143 ceph-mon[189871]: pgmap v240: 89 pgs: 89 active+clean; 1.2 GiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-08-28T02:22:18.230 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 11 16 26376 26360 149.734 108.062 0.00471336 0.00667062 2023-08-28T02:22:18.250 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:22:18.295 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:18.382 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:22:18.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:18.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:22:18.388 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:18 smithi169 ceph-mon[159415]: pgmap v240: 89 pgs: 89 active+clean; 1.2 GiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-08-28T02:22:18.488 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:22:18.586 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:22:18.586 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:22:18.588 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:18.735 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:22:18.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:18.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:22:18.845 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:22:18.920 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:19.170 INFO:teuthology.orchestra.run.smithi143.stderr:reweighted osd.1 to 1 (10000) 2023-08-28T02:22:19.230 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 12 15 28337 28322 147.474 122.625 0.00393042 0.00677537 2023-08-28T02:22:19.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:19 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1838707072' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-08-28T02:22:19.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:19 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1838707072' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-08-28T02:22:19.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:19 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1838707072' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-08-28T02:22:20.094 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:22:20.231 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 13 16 30364 30348 145.859 126.625 0.0057188 0.006851 2023-08-28T02:22:20.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:20 smithi143 ceph-mon[183059]: pgmap v241: 89 pgs: 89 active+clean; 1.5 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 147 MiB/s wr, 2.36k op/s 2023-08-28T02:22:20.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:20 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1838707072' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-08-28T02:22:20.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:20 smithi143 ceph-mon[183059]: osdmap e126: 8 total, 8 up, 7 in 2023-08-28T02:22:20.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:20 smithi143 ceph-mon[183059]: failed to encode map e126 with expected crc 2023-08-28T02:22:20.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:20 smithi143 ceph-mon[183059]: failed to encode map e126 with expected crc 2023-08-28T02:22:20.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:20 smithi143 ceph-mon[183059]: failed to encode map e126 with expected crc 2023-08-28T02:22:20.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:20 smithi143 ceph-mon[183059]: failed to encode map e126 with expected crc 2023-08-28T02:22:20.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:20 smithi143 ceph-mon[183059]: failed to encode map e126 with expected crc 2023-08-28T02:22:20.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:20 smithi143 ceph-mon[183059]: failed to encode map e126 with expected crc 2023-08-28T02:22:20.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:20 smithi143 ceph-mon[189871]: pgmap v241: 89 pgs: 89 active+clean; 1.5 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 147 MiB/s wr, 2.36k op/s 2023-08-28T02:22:20.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:20 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1838707072' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-08-28T02:22:20.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:20 smithi143 ceph-mon[189871]: osdmap e126: 8 total, 8 up, 7 in 2023-08-28T02:22:20.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:20 smithi143 ceph-mon[189871]: failed to encode map e126 with expected crc 2023-08-28T02:22:20.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:20 smithi143 ceph-mon[189871]: failed to encode map e126 with expected crc 2023-08-28T02:22:20.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:20 smithi143 ceph-mon[189871]: failed to encode map e126 with expected crc 2023-08-28T02:22:20.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:20 smithi143 ceph-mon[189871]: failed to encode map e126 with expected crc 2023-08-28T02:22:20.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:20 smithi143 ceph-mon[189871]: failed to encode map e126 with expected crc 2023-08-28T02:22:20.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:20 smithi143 ceph-mon[189871]: failed to encode map e126 with expected crc 2023-08-28T02:22:20.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:20 smithi169 ceph-mon[159415]: pgmap v241: 89 pgs: 89 active+clean; 1.5 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 147 MiB/s wr, 2.36k op/s 2023-08-28T02:22:20.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:20 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1838707072' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-08-28T02:22:20.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:20 smithi169 ceph-mon[159415]: osdmap e126: 8 total, 8 up, 7 in 2023-08-28T02:22:20.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:20 smithi169 ceph-mon[159415]: failed to encode map e126 with expected crc 2023-08-28T02:22:20.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:20 smithi169 ceph-mon[159415]: failed to encode map e126 with expected crc 2023-08-28T02:22:20.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:20 smithi169 ceph-mon[159415]: failed to encode map e126 with expected crc 2023-08-28T02:22:20.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:20 smithi169 ceph-mon[159415]: failed to encode map e126 with expected crc 2023-08-28T02:22:20.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:20 smithi169 ceph-mon[159415]: failed to encode map e126 with expected crc 2023-08-28T02:22:20.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:20 smithi169 ceph-mon[159415]: failed to encode map e126 with expected crc 2023-08-28T02:22:21.233 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 14 16 31932 31916 142.429 98 0.00547946 0.00701633 2023-08-28T02:22:21.247 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:22:21.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:21.254 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:22:21.355 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:22:21.410 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:21.516 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:22:21.523 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd reweight 2 1 2023-08-28T02:22:21.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:21.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:22:21.573 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.1 dump_blocked_ops 2023-08-28T02:22:21.629 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:22:21.854 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:22.136 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:22:22.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:22.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:22:22.143 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:22:21 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:22:21] "GET /metrics HTTP/1.1" 200 36460 "" "Prometheus/2.33.4" 2023-08-28T02:22:22.233 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 15 16 33597 33581 139.865 104.062 0.00649024 0.00714575 2023-08-28T02:22:22.244 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:22:22.267 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:22:22.328 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:22.420 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:22:22.423 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:22 smithi143 ceph-mon[183059]: pgmap v243: 89 pgs: 89 active+clean; 1.5 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 150 MiB/s wr, 2.41k op/s 2023-08-28T02:22:22.424 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:22 smithi143 ceph-mon[189871]: pgmap v243: 89 pgs: 89 active+clean; 1.5 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 150 MiB/s wr, 2.41k op/s 2023-08-28T02:22:22.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:22.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:22:22.452 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:22:22.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:22 smithi169 ceph-mon[159415]: pgmap v243: 89 pgs: 89 active+clean; 1.5 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 150 MiB/s wr, 2.41k op/s 2023-08-28T02:22:22.526 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:22:22.577 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:22.582 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:22:22.710 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:22:22.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:22.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:22:22.821 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:22:22.863 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:22.967 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:22:22.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:22.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:22:23.075 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:22:23.138 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:23.200 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is unset 2023-08-28T02:22:23.222 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:22:23.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:23.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:22:23.234 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 16 16 35177 35161 137.295 98.75 0.00655318 0.00727786 2023-08-28T02:22:23.332 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:22:23.378 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:23 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3469977698' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:22:23.379 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:23 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:22:23.383 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:23.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:23 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3469977698' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:22:23.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:23 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:22:23.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:23 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3469977698' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:22:23.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:23 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:22:23.467 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:22:23.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:23.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:22:23.576 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:22:23.639 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:23.739 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:22:23.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:23.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:22:23.847 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:22:23.913 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:23.995 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:22:24.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:24.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:22:24.103 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:22:24.132 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:24.172 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:24 smithi169 ceph-mon[159415]: pgmap v244: 89 pgs: 89 active+clean; 2.0 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 124 MiB/s wr, 1.98k op/s 2023-08-28T02:22:24.191 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:22:24.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:24.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:22:24.234 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 17 16 36940 36924 135.7 110.188 0.00210593 0.00733372 2023-08-28T02:22:24.299 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:22:24.334 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:24.374 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:22:24.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:24.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:22:24.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:24 smithi143 ceph-mon[183059]: pgmap v244: 89 pgs: 89 active+clean; 2.0 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 124 MiB/s wr, 1.98k op/s 2023-08-28T02:22:24.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:24 smithi143 ceph-mon[183059]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:22:24.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:24 smithi143 ceph-mon[183059]: Cluster is now healthy 2023-08-28T02:22:24.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:24 smithi143 ceph-mon[183059]: failed to encode map e127 with expected crc 2023-08-28T02:22:24.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:24 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:22:24.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:24 smithi143 ceph-mon[183059]: osdmap e127: 8 total, 8 up, 7 in 2023-08-28T02:22:24.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:24 smithi143 ceph-mon[183059]: failed to encode map e127 with expected crc 2023-08-28T02:22:24.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:24 smithi143 ceph-mon[189871]: pgmap v244: 89 pgs: 89 active+clean; 2.0 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 124 MiB/s wr, 1.98k op/s 2023-08-28T02:22:24.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:24 smithi143 ceph-mon[189871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:22:24.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:24 smithi143 ceph-mon[189871]: Cluster is now healthy 2023-08-28T02:22:24.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:24 smithi143 ceph-mon[189871]: failed to encode map e127 with expected crc 2023-08-28T02:22:24.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:24 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:22:24.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:24 smithi143 ceph-mon[189871]: osdmap e127: 8 total, 8 up, 7 in 2023-08-28T02:22:24.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:24 smithi143 ceph-mon[189871]: failed to encode map e127 with expected crc 2023-08-28T02:22:24.481 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:22:24.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:24 smithi169 ceph-mon[159415]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:22:24.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:24 smithi169 ceph-mon[159415]: Cluster is now healthy 2023-08-28T02:22:24.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:24 smithi169 ceph-mon[159415]: failed to encode map e127 with expected crc 2023-08-28T02:22:24.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:24 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:22:24.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:24 smithi169 ceph-mon[159415]: osdmap e127: 8 total, 8 up, 7 in 2023-08-28T02:22:24.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:24 smithi169 ceph-mon[159415]: failed to encode map e127 with expected crc 2023-08-28T02:22:24.530 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:25.234 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 18 16 38372 38356 133.133 89.5 0.0072504 0.00750701 2023-08-28T02:22:25.492 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:22:25.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:25.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:22:25.596 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:22:25.632 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:26.171 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:22:26.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:26.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:22:26.229 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:26 smithi143 ceph-mon[183059]: pgmap v246: 89 pgs: 89 active+clean; 2.1 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-08-28T02:22:26.229 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:26 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:22:26.229 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:26 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:22:26.229 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:26 smithi143 ceph-mon[189871]: pgmap v246: 89 pgs: 89 active+clean; 2.1 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-08-28T02:22:26.229 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:26 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:22:26.230 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:26 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:22:26.234 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 19 16 39950 39934 131.317 98.625 0.00903619 0.00760957 2023-08-28T02:22:26.284 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:22:26.348 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:26.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:26 smithi169 ceph-mon[159415]: pgmap v246: 89 pgs: 89 active+clean; 2.1 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-08-28T02:22:26.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:26 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:22:26.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:26 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:22:26.746 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:22:26.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:26.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:22:26.853 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:22:26.919 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:27.235 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:2023-08-28T02:22:27.237200+0000 min lat: 0.00111735 max lat: 0.203269 avg lat: 0.00767721 2023-08-28T02:22:27.236 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-08-28T02:22:27.236 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 20 16 41686 41670 130.166 108.5 0.0078557 0.00767721 2023-08-28T02:22:27.625 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:22:27.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:27.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:22:27.738 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:22:27.775 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:27.852 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:22:27.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:27.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:22:27.965 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:22:28.015 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set noscrub 2023-08-28T02:22:28.050 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:28.106 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:22:28.148 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:22:28.197 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:22:28.244 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 21 16 43199 43183 128.453 94.5625 0.0286595 0.00777611 2023-08-28T02:22:28.244 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:22:28.245 INFO:teuthology.orchestra.run.smithi143.stderr:reweighted osd.2 to 1 (10000) 2023-08-28T02:22:28.283 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:22:28.287 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:28 smithi143 ceph-mon[183059]: pgmap v247: 89 pgs: 89 active+clean; 2.3 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 97 MiB/s wr, 1.56k op/s 2023-08-28T02:22:28.288 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:28 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2416179696' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:22:28.288 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:28 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/603024420' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-08-28T02:22:28.288 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:28 smithi143 ceph-mon[189871]: pgmap v247: 89 pgs: 89 active+clean; 2.3 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 97 MiB/s wr, 1.56k op/s 2023-08-28T02:22:28.288 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:28 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2416179696' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:22:28.288 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:28 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/603024420' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-08-28T02:22:28.339 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:22:28.369 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:22:28.410 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:22:28.413 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:22:28.414 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:22:28.414 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:22:28.414 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:22:28.414 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:22:28.414 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:22:28.414 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:22:28.414 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:22:28.426 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:22:28.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:28.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:22:28.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:28 smithi169 ceph-mon[159415]: pgmap v247: 89 pgs: 89 active+clean; 2.3 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 97 MiB/s wr, 1.56k op/s 2023-08-28T02:22:28.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:28 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2416179696' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:22:28.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:28 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/603024420' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-08-28T02:22:28.540 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:22:28.566 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:22:28.566 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [], 2023-08-28T02:22:28.566 INFO:teuthology.orchestra.run.smithi143.stdout: "complaint_time": 30, 2023-08-28T02:22:28.566 INFO:teuthology.orchestra.run.smithi143.stdout: "num_blocked_ops": 0 2023-08-28T02:22:28.566 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:22:28.648 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:29.241 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 22 16 44743 44727 126.985 96.5 0.00638885 0.00787001 2023-08-28T02:22:29.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:29 smithi169 ceph-mon[159415]: failed to encode map e128 with expected crc 2023-08-28T02:22:29.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:29 smithi169 ceph-mon[159415]: failed to encode map e128 with expected crc 2023-08-28T02:22:29.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:29 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/603024420' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-08-28T02:22:29.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:29 smithi169 ceph-mon[159415]: osdmap e128: 8 total, 8 up, 7 in 2023-08-28T02:22:29.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:29 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:22:29.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:29 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:22:29.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:29 smithi143 ceph-mon[183059]: failed to encode map e128 with expected crc 2023-08-28T02:22:29.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:29 smithi143 ceph-mon[183059]: failed to encode map e128 with expected crc 2023-08-28T02:22:29.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:29 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/603024420' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-08-28T02:22:29.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:29 smithi143 ceph-mon[183059]: osdmap e128: 8 total, 8 up, 7 in 2023-08-28T02:22:29.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:29 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:22:29.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:29 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:22:29.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:29 smithi143 ceph-mon[189871]: failed to encode map e128 with expected crc 2023-08-28T02:22:29.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:29 smithi143 ceph-mon[189871]: failed to encode map e128 with expected crc 2023-08-28T02:22:29.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:29 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/603024420' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-08-28T02:22:29.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:29 smithi143 ceph-mon[189871]: osdmap e128: 8 total, 8 up, 7 in 2023-08-28T02:22:29.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:29 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:22:29.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:29 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:22:30.241 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 23 16 46081 46065 125.1 83.625 0.0176481 0.00798826 2023-08-28T02:22:30.308 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd reweight 4 1 2023-08-28T02:22:30.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:30 smithi169 ceph-mon[159415]: pgmap v249: 89 pgs: 89 active+clean; 2.6 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 133 MiB/s wr, 2.13k op/s 2023-08-28T02:22:30.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:30 smithi143 ceph-mon[183059]: pgmap v249: 89 pgs: 89 active+clean; 2.6 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 133 MiB/s wr, 2.13k op/s 2023-08-28T02:22:30.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:30 smithi143 ceph-mon[189871]: pgmap v249: 89 pgs: 89 active+clean; 2.6 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 133 MiB/s wr, 2.13k op/s 2023-08-28T02:22:30.812 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.1 dump_historic_ops 2023-08-28T02:22:30.837 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:22:30.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:30.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:22:30.948 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:22:30.983 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:31.064 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:22:31.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:31.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:22:31.166 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:22:31.170 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:22:31.218 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:31.220 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:22:31.243 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 24 16 47413 47397 123.347 83.25 0.0140583 0.00810161 2023-08-28T02:22:31.297 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:22:31.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:31.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:22:31.406 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:22:31.460 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:31.745 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:22:31.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:31.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:22:31.850 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:22:31.906 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:32.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:22:31 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:22:31] "GET /metrics HTTP/1.1" 200 36467 "" "Prometheus/2.33.4" 2023-08-28T02:22:32.243 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 25 16 49009 48993 122.403 99.75 0.00267447 0.00816466 2023-08-28T02:22:32.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:32 smithi169 ceph-mon[159415]: pgmap v250: 89 pgs: 89 active+clean; 2.6 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-08-28T02:22:32.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:32 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:22:32.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:32 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:22:32.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:32 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:22:32.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:32 smithi143 ceph-mon[183059]: pgmap v250: 89 pgs: 89 active+clean; 2.6 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-08-28T02:22:32.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:32 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:22:32.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:32 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:22:32.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:32 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:22:32.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:32 smithi143 ceph-mon[189871]: pgmap v250: 89 pgs: 89 active+clean; 2.6 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-08-28T02:22:32.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:32 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:22:32.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:32 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:22:32.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:32 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:22:32.859 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:22:33.245 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 26 16 50217 50201 120.594 75.5 0.00601851 0.00828763 2023-08-28T02:22:33.615 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:22:33.637 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:22:33.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:33.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:22:33.750 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:22:33.817 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:33.910 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:22:33.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:33.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:22:34.018 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:22:34.080 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:22:34.086 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:34.245 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 27 16 51701 51685 119.563 92.75 0.00783545 0.00835885 2023-08-28T02:22:34.312 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:22:34.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:34.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:22:34.322 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:34 smithi143 ceph-mon[183059]: pgmap v251: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-08-28T02:22:34.323 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:34 smithi143 ceph-mon[189871]: pgmap v251: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-08-28T02:22:34.422 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:22:34.479 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:34.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:34 smithi169 ceph-mon[159415]: pgmap v251: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-08-28T02:22:35.248 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 28 16 53167 53151 118.555 91.625 0.0120183 0.00843093 2023-08-28T02:22:35.260 INFO:teuthology.orchestra.run.smithi143.stderr:reweighted osd.4 to 1 (10000) 2023-08-28T02:22:35.273 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is set 2023-08-28T02:22:35.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:35 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/994855737' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:22:35.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:35 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:22:35.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:35 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3393541275' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-08-28T02:22:35.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:35 smithi143 ceph-mon[183059]: pgmap v252: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-08-28T02:22:35.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:35 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/994855737' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:22:35.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:35 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:22:35.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:35 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3393541275' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-08-28T02:22:35.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:35 smithi143 ceph-mon[189871]: pgmap v252: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-08-28T02:22:35.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:35 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/994855737' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:22:35.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:35 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:22:35.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:35 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3393541275' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-08-28T02:22:35.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:35 smithi169 ceph-mon[159415]: pgmap v252: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-08-28T02:22:36.134 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:22:36.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:36.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:22:36.242 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:22:36.248 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 29 16 54775 54759 117.932 100.5 0.0152868 0.008475 2023-08-28T02:22:36.302 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:36.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:36 smithi143 ceph-mon[183059]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:22:36.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:36 smithi143 ceph-mon[183059]: failed to encode map e129 with expected crc 2023-08-28T02:22:36.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:36 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:22:36.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:36 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3393541275' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-08-28T02:22:36.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:36 smithi143 ceph-mon[183059]: osdmap e129: 8 total, 8 up, 7 in 2023-08-28T02:22:36.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:36 smithi143 ceph-mon[183059]: failed to encode map e129 with expected crc 2023-08-28T02:22:36.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:36 smithi143 ceph-mon[183059]: failed to encode map e129 with expected crc 2023-08-28T02:22:36.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:36 smithi143 ceph-mon[183059]: failed to encode map e129 with expected crc 2023-08-28T02:22:36.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:36 smithi143 ceph-mon[183059]: failed to encode map e129 with expected crc 2023-08-28T02:22:36.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:36 smithi143 ceph-mon[183059]: failed to encode map e129 with expected crc 2023-08-28T02:22:36.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:36 smithi143 ceph-mon[189871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:22:36.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:36 smithi143 ceph-mon[189871]: failed to encode map e129 with expected crc 2023-08-28T02:22:36.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:36 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:22:36.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:36 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3393541275' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-08-28T02:22:36.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:36 smithi143 ceph-mon[189871]: osdmap e129: 8 total, 8 up, 7 in 2023-08-28T02:22:36.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:36 smithi143 ceph-mon[189871]: failed to encode map e129 with expected crc 2023-08-28T02:22:36.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:36 smithi143 ceph-mon[189871]: failed to encode map e129 with expected crc 2023-08-28T02:22:36.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:36 smithi143 ceph-mon[189871]: failed to encode map e129 with expected crc 2023-08-28T02:22:36.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:36 smithi143 ceph-mon[189871]: failed to encode map e129 with expected crc 2023-08-28T02:22:36.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:36 smithi143 ceph-mon[189871]: failed to encode map e129 with expected crc 2023-08-28T02:22:36.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:36 smithi169 ceph-mon[159415]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:22:36.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:36 smithi169 ceph-mon[159415]: failed to encode map e129 with expected crc 2023-08-28T02:22:36.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:36 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:22:36.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:36 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3393541275' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-08-28T02:22:36.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:36 smithi169 ceph-mon[159415]: osdmap e129: 8 total, 8 up, 7 in 2023-08-28T02:22:36.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:36 smithi169 ceph-mon[159415]: failed to encode map e129 with expected crc 2023-08-28T02:22:36.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:36 smithi169 ceph-mon[159415]: failed to encode map e129 with expected crc 2023-08-28T02:22:36.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:36 smithi169 ceph-mon[159415]: failed to encode map e129 with expected crc 2023-08-28T02:22:36.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:36 smithi169 ceph-mon[159415]: failed to encode map e129 with expected crc 2023-08-28T02:22:36.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:36 smithi169 ceph-mon[159415]: failed to encode map e129 with expected crc 2023-08-28T02:22:37.148 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:22:37.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:37.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:22:37.248 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 30 16 55927 55911 116.402 72 0.011491 0.0085854 2023-08-28T02:22:37.254 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:22:37.314 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:37.399 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:22:37.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:37.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:22:37.409 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:37 smithi143 ceph-mon[183059]: pgmap v254: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-08-28T02:22:37.410 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd reweight 5 1 2023-08-28T02:22:37.411 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:37 smithi143 ceph-mon[189871]: pgmap v254: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-08-28T02:22:37.509 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:22:37.555 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:37 smithi169 ceph-mon[159415]: pgmap v254: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-08-28T02:22:37.560 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:37.657 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:22:37.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:37.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:22:37.765 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:22:37.812 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:37.899 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:22:37.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:37.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:22:38.010 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:22:38.071 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:38.152 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:22:38.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:38.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:22:38.248 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 31 16 57291 57275 115.397 85.25 0.00703643 0.0086614 2023-08-28T02:22:38.259 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:22:38.310 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:38.381 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:22:38.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:38.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:22:38.412 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:22:38.451 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:22:38.488 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:22:38.497 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:22:38.503 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:22:38.511 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:22:38.511 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:22:38.553 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:22:38.580 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:38.593 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:22:38.629 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:22:38.667 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:22:38.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:38 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2035166760' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:22:38.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:38 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2035166760' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:22:38.703 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:22:38.706 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:22:38.706 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:22:38.706 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:22:38.706 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:22:38.707 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:22:38.707 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:22:38.707 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:22:38.707 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:22:38.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:38 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2035166760' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:22:38.756 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:22:38.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:38.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:22:38.870 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:22:38.924 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:39.005 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:22:39.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:39.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:22:39.114 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:22:39.170 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:39.224 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:22:39.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:39.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:22:39.248 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 32 16 58565 58549 114.28 79.625 0.00845924 0.00873137 2023-08-28T02:22:39.330 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:22:39.399 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:39.434 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set nodeep-scrub 2023-08-28T02:22:39.443 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:22:39.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:39.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:22:39.549 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:22:39.575 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:39.618 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:22:39.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:39.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:22:39.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:39 smithi143 ceph-mon[183059]: pgmap v255: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 88 MiB/s wr, 1.42k op/s 2023-08-28T02:22:39.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:39 smithi143 ceph-mon[189871]: pgmap v255: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 88 MiB/s wr, 1.42k op/s 2023-08-28T02:22:39.731 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:22:39.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:39 smithi169 ceph-mon[159415]: pgmap v255: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 88 MiB/s wr, 1.42k op/s 2023-08-28T02:22:39.782 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:40.248 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 33 16 59481 59465 112.552 57.25 0.00594072 0.00888009 2023-08-28T02:22:40.502 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.2 dump_ops_in_flight 2023-08-28T02:22:40.643 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:22:40.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:40.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:22:40.755 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:22:40.810 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:40.873 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:22:40.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:40.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:22:40.979 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:22:41.013 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:41.057 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:22:41.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:41.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:22:41.165 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:22:41.191 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:41.231 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:22:41.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:41.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:22:41.248 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 34 16 60445 60429 111.015 60.25 0.00610778 0.00897589 2023-08-28T02:22:41.339 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:22:41.387 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:41.687 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:22:41.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:41.697 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:22:41.720 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:22:41.787 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:22:41.797 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:22:41.854 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:41.932 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:22:41 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:22:41] "GET /metrics HTTP/1.1" 200 36470 "" "Prometheus/2.33.4" 2023-08-28T02:22:41.949 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:22:41.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:41.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:22:42.060 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:22:42.110 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:42 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:22:42.110 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:42 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:22:42.110 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:42 smithi169 ceph-mon[159415]: pgmap v256: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 88 MiB/s wr, 1.42k op/s 2023-08-28T02:22:42.114 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:42.194 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:22:42.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:42 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:22:42.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:42 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:22:42.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:42 smithi143 ceph-mon[189871]: pgmap v256: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 88 MiB/s wr, 1.42k op/s 2023-08-28T02:22:42.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:42 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:22:42.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:42 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:22:42.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:42 smithi143 ceph-mon[183059]: pgmap v256: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 88 MiB/s wr, 1.42k op/s 2023-08-28T02:22:42.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:42.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:22:42.251 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 35 16 61793 61777 110.24 84.25 0.00360988 0.00906598 2023-08-28T02:22:42.311 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:22:42.373 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:42.654 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:22:43.251 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 36 16 62853 62837 109.019 66.25 0.0117429 0.00916824 2023-08-28T02:22:43.372 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:43 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 0]}]: dispatch 2023-08-28T02:22:43.372 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:43 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [1, 0]}]: dispatch 2023-08-28T02:22:43.373 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:43 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2023-08-28T02:22:43.373 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:43 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2023-08-28T02:22:43.373 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:43 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-08-28T02:22:43.373 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:43 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 0]}]: dispatch 2023-08-28T02:22:43.374 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:43 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [1, 0]}]: dispatch 2023-08-28T02:22:43.374 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:43 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2023-08-28T02:22:43.374 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:43 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2023-08-28T02:22:43.374 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:43 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-08-28T02:22:43.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:43 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 0]}]: dispatch 2023-08-28T02:22:43.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:43 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [1, 0]}]: dispatch 2023-08-28T02:22:43.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:43 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2023-08-28T02:22:43.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:43 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2023-08-28T02:22:43.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:43 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-08-28T02:22:43.555 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:22:43.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:43.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:22:43.663 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:22:43.708 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:43.775 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:22:43.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:43.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:22:43.878 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:22:43.904 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:43.985 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:22:43.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:43.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:22:44.083 INFO:teuthology.orchestra.run.smithi143.stderr:reweighted osd.5 to 1 (10000) 2023-08-28T02:22:44.088 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:22:44.143 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:44.252 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 37 16 63940 63924 107.909 67.9375 0.0110643 0.00926207 2023-08-28T02:22:44.255 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:22:44.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:44.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:22:44.366 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:22:44.412 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:44 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 0]}]': finished 2023-08-28T02:22:44.412 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:44 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [1, 0]}]': finished 2023-08-28T02:22:44.412 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:44 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2023-08-28T02:22:44.412 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:44 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]': finished 2023-08-28T02:22:44.412 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:44 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2023-08-28T02:22:44.412 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:44 smithi143 ceph-mon[183059]: osdmap e130: 8 total, 8 up, 7 in 2023-08-28T02:22:44.412 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:44 smithi143 ceph-mon[183059]: failed to encode map e130 with expected crc 2023-08-28T02:22:44.412 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:44 smithi143 ceph-mon[183059]: failed to encode map e130 with expected crc 2023-08-28T02:22:44.412 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:44 smithi143 ceph-mon[183059]: failed to encode map e130 with expected crc 2023-08-28T02:22:44.412 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:44 smithi143 ceph-mon[183059]: pgmap v258: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-08-28T02:22:44.413 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:44 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/820091940' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-08-28T02:22:44.413 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:44 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 0]}]': finished 2023-08-28T02:22:44.413 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:44 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [1, 0]}]': finished 2023-08-28T02:22:44.413 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:44 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2023-08-28T02:22:44.413 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:44 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]': finished 2023-08-28T02:22:44.413 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:44 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2023-08-28T02:22:44.413 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:44 smithi143 ceph-mon[189871]: osdmap e130: 8 total, 8 up, 7 in 2023-08-28T02:22:44.413 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:44 smithi143 ceph-mon[189871]: failed to encode map e130 with expected crc 2023-08-28T02:22:44.413 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:44 smithi143 ceph-mon[189871]: failed to encode map e130 with expected crc 2023-08-28T02:22:44.413 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:44 smithi143 ceph-mon[189871]: failed to encode map e130 with expected crc 2023-08-28T02:22:44.413 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:44 smithi143 ceph-mon[189871]: pgmap v258: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-08-28T02:22:44.414 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:44 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/820091940' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-08-28T02:22:44.414 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:44.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:44 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 0]}]': finished 2023-08-28T02:22:44.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:44 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [1, 0]}]': finished 2023-08-28T02:22:44.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:44 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2023-08-28T02:22:44.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:44 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]': finished 2023-08-28T02:22:44.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:44 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2023-08-28T02:22:44.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:44 smithi169 ceph-mon[159415]: osdmap e130: 8 total, 8 up, 7 in 2023-08-28T02:22:44.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:44 smithi169 ceph-mon[159415]: failed to encode map e130 with expected crc 2023-08-28T02:22:44.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:44 smithi169 ceph-mon[159415]: failed to encode map e130 with expected crc 2023-08-28T02:22:44.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:44 smithi169 ceph-mon[159415]: failed to encode map e130 with expected crc 2023-08-28T02:22:44.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:44 smithi169 ceph-mon[159415]: pgmap v258: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-08-28T02:22:44.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:44 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/820091940' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-08-28T02:22:45.180 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:22:45.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:45.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:22:45.258 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 38 15 65111 65096 106.996 73.25 0.0113859 0.00934048 2023-08-28T02:22:45.293 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:45 smithi143 ceph-mon[183059]: failed to encode map e131 with expected crc 2023-08-28T02:22:45.294 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:45 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/820091940' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-08-28T02:22:45.294 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:45 smithi143 ceph-mon[183059]: osdmap e131: 8 total, 8 up, 7 in 2023-08-28T02:22:45.294 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:45 smithi143 ceph-mon[183059]: failed to encode map e131 with expected crc 2023-08-28T02:22:45.294 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:45 smithi143 ceph-mon[183059]: failed to encode map e131 with expected crc 2023-08-28T02:22:45.294 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:45 smithi143 ceph-mon[183059]: failed to encode map e131 with expected crc 2023-08-28T02:22:45.294 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:45 smithi143 ceph-mon[183059]: failed to encode map e131 with expected crc 2023-08-28T02:22:45.294 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:45 smithi143 ceph-mon[183059]: failed to encode map e131 with expected crc 2023-08-28T02:22:45.294 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:45 smithi143 ceph-mon[189871]: failed to encode map e131 with expected crc 2023-08-28T02:22:45.295 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:45 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/820091940' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-08-28T02:22:45.295 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:45 smithi143 ceph-mon[189871]: osdmap e131: 8 total, 8 up, 7 in 2023-08-28T02:22:45.295 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:45 smithi143 ceph-mon[189871]: failed to encode map e131 with expected crc 2023-08-28T02:22:45.295 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:45 smithi143 ceph-mon[189871]: failed to encode map e131 with expected crc 2023-08-28T02:22:45.295 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:45 smithi143 ceph-mon[189871]: failed to encode map e131 with expected crc 2023-08-28T02:22:45.295 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:45 smithi143 ceph-mon[189871]: failed to encode map e131 with expected crc 2023-08-28T02:22:45.295 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:45 smithi143 ceph-mon[189871]: failed to encode map e131 with expected crc 2023-08-28T02:22:45.296 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd reweight 6 1 2023-08-28T02:22:45.296 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:22:45.374 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:45.410 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:22:45.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:45 smithi169 ceph-mon[159415]: failed to encode map e131 with expected crc 2023-08-28T02:22:45.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:45 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/820091940' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-08-28T02:22:45.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:45 smithi169 ceph-mon[159415]: osdmap e131: 8 total, 8 up, 7 in 2023-08-28T02:22:45.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:45 smithi169 ceph-mon[159415]: failed to encode map e131 with expected crc 2023-08-28T02:22:45.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:45 smithi169 ceph-mon[159415]: failed to encode map e131 with expected crc 2023-08-28T02:22:45.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:45 smithi169 ceph-mon[159415]: failed to encode map e131 with expected crc 2023-08-28T02:22:45.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:45 smithi169 ceph-mon[159415]: failed to encode map e131 with expected crc 2023-08-28T02:22:45.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:45 smithi169 ceph-mon[159415]: failed to encode map e131 with expected crc 2023-08-28T02:22:45.583 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:22:45.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:45.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:22:45.697 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:22:45.777 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:45.905 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:22:45.905 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:22:45.987 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:22:45.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:45.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:22:46.100 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:22:46.109 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is set 2023-08-28T02:22:46.136 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:46 smithi143 ceph-mon[183059]: pgmap v260: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 75 MiB/s wr, 1.21k op/s 2023-08-28T02:22:46.136 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:46 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1913004514' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:22:46.136 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:46 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:22:46.137 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:46 smithi143 ceph-mon[189871]: pgmap v260: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 75 MiB/s wr, 1.21k op/s 2023-08-28T02:22:46.137 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:46 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1913004514' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:22:46.137 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:46 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:22:46.160 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:46.241 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:22:46.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:46.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:22:46.253 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 39 16 66250 66234 106.076 71.125 0.00875823 0.00942184 2023-08-28T02:22:46.353 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:22:46.414 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:46.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:46 smithi169 ceph-mon[159415]: pgmap v260: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 75 MiB/s wr, 1.21k op/s 2023-08-28T02:22:46.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:46 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1913004514' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:22:46.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:46 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:22:47.015 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:22:47.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:47.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:22:47.120 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:22:47.187 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:47.252 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:2023-08-28T02:22:47.254106+0000 min lat: 0.00111735 max lat: 0.217218 avg lat: 0.00949448 2023-08-28T02:22:47.252 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-08-28T02:22:47.252 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 40 16 67420 67404 105.253 73.125 0.00271758 0.00949448 2023-08-28T02:22:47.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:47 smithi143 ceph-mon[189871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:22:47.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:47 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:22:47.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:47 smithi143 ceph-mon[189871]: osdmap e132: 8 total, 8 up, 7 in 2023-08-28T02:22:47.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:47 smithi143 ceph-mon[189871]: failed to encode map e132 with expected crc 2023-08-28T02:22:47.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:47 smithi143 ceph-mon[189871]: failed to encode map e132 with expected crc 2023-08-28T02:22:47.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:47 smithi143 ceph-mon[189871]: failed to encode map e132 with expected crc 2023-08-28T02:22:47.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:47 smithi143 ceph-mon[189871]: failed to encode map e132 with expected crc 2023-08-28T02:22:47.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:47 smithi143 ceph-mon[189871]: failed to encode map e132 with expected crc 2023-08-28T02:22:47.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:47 smithi143 ceph-mon[189871]: failed to encode map e132 with expected crc 2023-08-28T02:22:47.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:47 smithi143 ceph-mon[189871]: failed to encode map e132 with expected crc 2023-08-28T02:22:47.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:47 smithi143 ceph-mon[189871]: failed to encode map e132 with expected crc 2023-08-28T02:22:47.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:47 smithi143 ceph-mon[183059]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:22:47.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:47 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:22:47.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:47 smithi143 ceph-mon[183059]: osdmap e132: 8 total, 8 up, 7 in 2023-08-28T02:22:47.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:47 smithi143 ceph-mon[183059]: failed to encode map e132 with expected crc 2023-08-28T02:22:47.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:47 smithi143 ceph-mon[183059]: failed to encode map e132 with expected crc 2023-08-28T02:22:47.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:47 smithi143 ceph-mon[183059]: failed to encode map e132 with expected crc 2023-08-28T02:22:47.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:47 smithi143 ceph-mon[183059]: failed to encode map e132 with expected crc 2023-08-28T02:22:47.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:47 smithi143 ceph-mon[183059]: failed to encode map e132 with expected crc 2023-08-28T02:22:47.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:47 smithi143 ceph-mon[183059]: failed to encode map e132 with expected crc 2023-08-28T02:22:47.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:47 smithi143 ceph-mon[183059]: failed to encode map e132 with expected crc 2023-08-28T02:22:47.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:47 smithi143 ceph-mon[183059]: failed to encode map e132 with expected crc 2023-08-28T02:22:47.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:47 smithi169 ceph-mon[159415]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:22:47.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:47 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:22:47.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:47 smithi169 ceph-mon[159415]: osdmap e132: 8 total, 8 up, 7 in 2023-08-28T02:22:47.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:47 smithi169 ceph-mon[159415]: failed to encode map e132 with expected crc 2023-08-28T02:22:47.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:47 smithi169 ceph-mon[159415]: failed to encode map e132 with expected crc 2023-08-28T02:22:47.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:47 smithi169 ceph-mon[159415]: failed to encode map e132 with expected crc 2023-08-28T02:22:47.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:47 smithi169 ceph-mon[159415]: failed to encode map e132 with expected crc 2023-08-28T02:22:47.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:47 smithi169 ceph-mon[159415]: failed to encode map e132 with expected crc 2023-08-28T02:22:47.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:47 smithi169 ceph-mon[159415]: failed to encode map e132 with expected crc 2023-08-28T02:22:47.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:47 smithi169 ceph-mon[159415]: failed to encode map e132 with expected crc 2023-08-28T02:22:47.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:47 smithi169 ceph-mon[159415]: failed to encode map e132 with expected crc 2023-08-28T02:22:47.683 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.2 dump_blocked_ops 2023-08-28T02:22:48.254 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 41 16 68293 68277 104.017 54.5625 0.00931193 0.00960725 2023-08-28T02:22:48.254 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:22:48.275 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:22:48.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:48.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:22:48.384 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:22:48.439 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:48 smithi169 ceph-mon[159415]: pgmap v262: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-08-28T02:22:48.445 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:48.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:48 smithi143 ceph-mon[183059]: pgmap v262: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-08-28T02:22:48.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:48 smithi143 ceph-mon[189871]: pgmap v262: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-08-28T02:22:48.524 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:22:48.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:48.532 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:22:48.632 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:22:48.699 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:48.812 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:22:48.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:48.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:22:48.851 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:22:48.885 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:22:48.916 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:22:48.919 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:22:48.953 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:22:48.974 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:48.987 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:22:49.020 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:22:49.047 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:22:49.056 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:22:49.089 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:22:49.113 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:22:49.113 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:22:49.114 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:22:49.114 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:22:49.114 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:22:49.114 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:22:49.114 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:22:49.114 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:22:49.253 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 42 16 69412 69396 103.205 69.9375 0.00979778 0.00968411 2023-08-28T02:22:49.260 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:22:49.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:49.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:22:49.373 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:22:49.422 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:49 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3153147402' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:22:49.656 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:49 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3153147402' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:22:49.657 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:49.658 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:49 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3153147402' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:22:50.253 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 43 16 70589 70573 102.516 73.5625 0.0147717 0.00974933 2023-08-28T02:22:50.256 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset noscrub 2023-08-28T02:22:50.310 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:50 smithi143 ceph-mon[189871]: pgmap v263: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-08-28T02:22:50.312 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:50 smithi143 ceph-mon[183059]: pgmap v263: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-08-28T02:22:50.415 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:22:50.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:50.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:22:50.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:50 smithi169 ceph-mon[159415]: pgmap v263: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-08-28T02:22:50.521 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:22:50.556 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:50.638 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:22:50.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:50.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:22:50.746 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:22:50.821 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:50.985 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:22:50.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:50.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:22:51.092 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:22:51.163 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:51.250 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:22:51.254 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 44 16 71787 71771 101.888 74.875 0.00766013 0.00980968 2023-08-28T02:22:51.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:51.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:22:51.359 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:22:51.405 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:51.496 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:22:51.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:51.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:22:51.605 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:22:51.631 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:51.735 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:22:51.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:51.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:22:51.852 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:22:51.912 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:52.104 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:22:52.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:52.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:22:52.165 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:22:52.207 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:22:51 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:22:51] "GET /metrics HTTP/1.1" 200 36470 "" "Prometheus/2.33.4" 2023-08-28T02:22:52.207 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:52 smithi143 ceph-mon[189871]: pgmap v264: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 49 MiB/s wr, 779 op/s 2023-08-28T02:22:52.207 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:52 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/338145914' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-08-28T02:22:52.207 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:52 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-08-28T02:22:52.216 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:22:52.219 INFO:teuthology.orchestra.run.smithi143.stderr:reweighted osd.6 to 1 (10000) 2023-08-28T02:22:52.253 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 45 16 72833 72817 101.077 65.375 0.0123311 0.0098885 2023-08-28T02:22:52.264 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:52.338 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:22:52.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:52.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:22:52.444 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:22:52.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:52 smithi169 ceph-mon[159415]: pgmap v264: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 49 MiB/s wr, 779 op/s 2023-08-28T02:22:52.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:52 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/338145914' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-08-28T02:22:52.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:52 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-08-28T02:22:52.495 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:52 smithi143 ceph-mon[183059]: pgmap v264: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 49 MiB/s wr, 779 op/s 2023-08-28T02:22:52.495 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:52 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/338145914' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-08-28T02:22:52.496 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:52 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-08-28T02:22:52.505 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:52.685 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:22:52.997 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:22:52.997 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [], 2023-08-28T02:22:52.998 INFO:teuthology.orchestra.run.smithi143.stdout: "complaint_time": 30, 2023-08-28T02:22:52.998 INFO:teuthology.orchestra.run.smithi143.stdout: "num_blocked_ops": 0 2023-08-28T02:22:52.998 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:22:53.253 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 46 16 73741 73725 100.114 56.75 0.00989303 0.00998368 2023-08-28T02:22:53.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:53 smithi143 ceph-mon[183059]: failed to encode map e133 with expected crc 2023-08-28T02:22:53.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:53 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-08-28T02:22:53.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:53 smithi143 ceph-mon[183059]: osdmap e133: 8 total, 8 up, 7 in 2023-08-28T02:22:53.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:53 smithi143 ceph-mon[183059]: failed to encode map e133 with expected crc 2023-08-28T02:22:53.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:53 smithi143 ceph-mon[189871]: failed to encode map e133 with expected crc 2023-08-28T02:22:53.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:53 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-08-28T02:22:53.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:53 smithi143 ceph-mon[189871]: osdmap e133: 8 total, 8 up, 7 in 2023-08-28T02:22:53.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:53 smithi143 ceph-mon[189871]: failed to encode map e133 with expected crc 2023-08-28T02:22:53.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:53 smithi169 ceph-mon[159415]: failed to encode map e133 with expected crc 2023-08-28T02:22:53.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:53 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-08-28T02:22:53.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:53 smithi169 ceph-mon[159415]: osdmap e133: 8 total, 8 up, 7 in 2023-08-28T02:22:53.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:53 smithi169 ceph-mon[159415]: failed to encode map e133 with expected crc 2023-08-28T02:22:53.489 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd reweight 7 1 2023-08-28T02:22:53.595 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:22:53.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:53.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:22:53.703 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:22:53.766 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:53.862 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:22:53.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:53.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:22:53.971 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:22:54.038 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:54.215 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:54 smithi169 ceph-mon[159415]: pgmap v266: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 82 MiB/s wr, 1.30k op/s 2023-08-28T02:22:54.215 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:54 smithi143 ceph-mon[183059]: pgmap v266: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 82 MiB/s wr, 1.30k op/s 2023-08-28T02:22:54.255 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 47 16 74901 74885 99.5227 72.5 0.0107857 0.0100425 2023-08-28T02:22:54.324 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.2 dump_historic_ops 2023-08-28T02:22:54.487 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:54 smithi143 ceph-mon[189871]: pgmap v266: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 82 MiB/s wr, 1.30k op/s 2023-08-28T02:22:54.560 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:22:54.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:54.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:22:54.669 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:22:54.715 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:55.027 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:22:55.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:55.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:22:55.130 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:22:55.175 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:55.208 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:22:55.239 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:22:55.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:55.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:22:55.263 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 48 16 76051 76035 98.9297 71.875 0.0141423 0.0101022 2023-08-28T02:22:55.308 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:22:55.347 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:22:55.358 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:22:55.385 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:55.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:55 smithi143 ceph-mon[183059]: pgmap v267: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-08-28T02:22:55.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:55 smithi143 ceph-mon[189871]: pgmap v267: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-08-28T02:22:55.460 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:22:55.467 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:55 smithi169 ceph-mon[159415]: pgmap v267: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-08-28T02:22:55.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:55.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:22:55.569 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:22:55.638 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:56.264 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 49 16 77168 77152 98.3358 69.8125 0.0112942 0.0101651 2023-08-28T02:22:56.460 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:22:56.464 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:56 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:22:56.464 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:56 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:22:56.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:56.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:22:56.566 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:22:56.627 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:56.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:56 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:22:57.264 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 50 16 78285 78269 97.7652 69.8125 0.0224685 0.0102234 2023-08-28T02:22:57.271 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is unset 2023-08-28T02:22:57.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:57 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/96454005' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:22:57.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:57 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:22:57.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:57 smithi143 ceph-mon[183059]: pgmap v268: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-08-28T02:22:57.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:57 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/96454005' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:22:57.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:57 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:22:57.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:57 smithi143 ceph-mon[189871]: pgmap v268: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-08-28T02:22:57.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:57 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/96454005' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:22:57.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:57 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:22:57.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:57 smithi169 ceph-mon[159415]: pgmap v268: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-08-28T02:22:58.264 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 51 16 79375 79359 97.1843 68.125 0.0160126 0.0102849 2023-08-28T02:22:58.474 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:22:58.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:58.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:22:58.543 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:58 smithi143 ceph-mon[189871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:22:58.543 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:58 smithi143 ceph-mon[189871]: failed to encode map e134 with expected crc 2023-08-28T02:22:58.543 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:58 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:22:58.543 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:58 smithi143 ceph-mon[189871]: osdmap e134: 8 total, 8 up, 7 in 2023-08-28T02:22:58.543 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:58 smithi143 ceph-mon[189871]: failed to encode map e134 with expected crc 2023-08-28T02:22:58.546 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:58 smithi143 ceph-mon[183059]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:22:58.546 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:58 smithi143 ceph-mon[183059]: failed to encode map e134 with expected crc 2023-08-28T02:22:58.546 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:58 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:22:58.547 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:58 smithi143 ceph-mon[183059]: osdmap e134: 8 total, 8 up, 7 in 2023-08-28T02:22:58.547 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:58 smithi143 ceph-mon[183059]: failed to encode map e134 with expected crc 2023-08-28T02:22:58.583 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:22:58.622 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:58 smithi169 ceph-mon[159415]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:22:58.622 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:58 smithi169 ceph-mon[159415]: failed to encode map e134 with expected crc 2023-08-28T02:22:58.622 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:58 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:22:58.622 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:58 smithi169 ceph-mon[159415]: osdmap e134: 8 total, 8 up, 7 in 2023-08-28T02:22:58.622 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:58 smithi169 ceph-mon[159415]: failed to encode map e134 with expected crc 2023-08-28T02:22:58.627 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:58.716 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:22:58.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:58.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:22:58.842 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:22:58.900 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:59.264 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 52 16 80402 80386 96.55 64.1875 0.0149426 0.0103524 2023-08-28T02:22:59.485 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:22:59.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:59.490 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:22:59.541 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:22:59 smithi143 ceph-mon[183059]: pgmap v270: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 86 MiB/s wr, 1.37k op/s 2023-08-28T02:22:59.541 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:22:59 smithi143 ceph-mon[189871]: pgmap v270: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 86 MiB/s wr, 1.37k op/s 2023-08-28T02:22:59.591 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:22:59.639 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:22:59.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:22:59 smithi169 ceph-mon[159415]: pgmap v270: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 86 MiB/s wr, 1.37k op/s 2023-08-28T02:22:59.790 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:22:59.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:22:59.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:22:59.819 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:22:59.863 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:22:59.903 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:22:59.910 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:22:59.959 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:22:59.978 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:00.007 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:23:00.055 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:23:00.086 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:23:00.090 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:23:00.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:00.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:23:00.117 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:23:00.120 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:23:00.120 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:23:00.120 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:23:00.120 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:23:00.120 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:23:00.120 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:23:00.120 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:23:00.120 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:23:00.199 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:23:00.236 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:23:00.236 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:23:00.264 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 53 16 81454 81438 95.9691 65.75 0.0107561 0.0104142 2023-08-28T02:23:00.282 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:00.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:00 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3518514557' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:23:00.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:00 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3518514557' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:23:00.588 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset nodeep-scrub 2023-08-28T02:23:00.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:00 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3518514557' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:23:01.264 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 54 16 82355 82339 95.2351 56.3125 0.00291947 0.010481 2023-08-28T02:23:01.294 INFO:teuthology.orchestra.run.smithi143.stderr:reweighted osd.7 to 1 (10000) 2023-08-28T02:23:01.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:01 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3064681975' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-08-28T02:23:01.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:01 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-08-28T02:23:01.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:01 smithi143 ceph-mon[183059]: pgmap v271: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 61 MiB/s wr, 971 op/s 2023-08-28T02:23:01.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:01 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3064681975' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-08-28T02:23:01.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:01 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-08-28T02:23:01.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:01 smithi143 ceph-mon[189871]: pgmap v271: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 61 MiB/s wr, 971 op/s 2023-08-28T02:23:01.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:01 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3064681975' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-08-28T02:23:01.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:01 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-08-28T02:23:01.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:01 smithi169 ceph-mon[159415]: pgmap v271: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 61 MiB/s wr, 971 op/s 2023-08-28T02:23:01.780 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:23:01.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:01.784 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:23:01.784 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.3 dump_ops_in_flight 2023-08-28T02:23:01.884 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:23:01.987 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:02.204 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:23:01 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:23:01] "GET /metrics HTTP/1.1" 200 36469 "" "Prometheus/2.33.4" 2023-08-28T02:23:02.246 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:23:02.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:02.250 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:23:02.264 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 55 16 83376 83360 94.664 63.8125 0.0110938 0.0105585 2023-08-28T02:23:02.352 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:23:02.364 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-08-28T02:23:02.364 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph pg dump --format=json 2023-08-28T02:23:02.395 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:02.438 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:23:02.503 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:23:02.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:02.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:23:02.518 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:23:02.607 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:23:02.647 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:02 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-08-28T02:23:02.647 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:02 smithi169 ceph-mon[159415]: osdmap e135: 8 total, 8 up, 7 in 2023-08-28T02:23:02.647 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:02 smithi169 ceph-mon[159415]: failed to encode map e135 with expected crc 2023-08-28T02:23:02.647 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:02 smithi169 ceph-mon[159415]: failed to encode map e135 with expected crc 2023-08-28T02:23:02.647 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:02 smithi169 ceph-mon[159415]: failed to encode map e135 with expected crc 2023-08-28T02:23:02.647 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:02 smithi169 ceph-mon[159415]: failed to encode map e135 with expected crc 2023-08-28T02:23:02.649 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:02.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:02 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-08-28T02:23:02.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:02 smithi143 ceph-mon[183059]: osdmap e135: 8 total, 8 up, 7 in 2023-08-28T02:23:02.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:02 smithi143 ceph-mon[183059]: failed to encode map e135 with expected crc 2023-08-28T02:23:02.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:02 smithi143 ceph-mon[183059]: failed to encode map e135 with expected crc 2023-08-28T02:23:02.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:02 smithi143 ceph-mon[183059]: failed to encode map e135 with expected crc 2023-08-28T02:23:02.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:02 smithi143 ceph-mon[183059]: failed to encode map e135 with expected crc 2023-08-28T02:23:02.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:02 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-08-28T02:23:02.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:02 smithi143 ceph-mon[189871]: osdmap e135: 8 total, 8 up, 7 in 2023-08-28T02:23:02.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:02 smithi143 ceph-mon[189871]: failed to encode map e135 with expected crc 2023-08-28T02:23:02.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:02 smithi143 ceph-mon[189871]: failed to encode map e135 with expected crc 2023-08-28T02:23:02.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:02 smithi143 ceph-mon[189871]: failed to encode map e135 with expected crc 2023-08-28T02:23:02.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:02 smithi143 ceph-mon[189871]: failed to encode map e135 with expected crc 2023-08-28T02:23:02.743 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:23:02.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:02.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:23:02.851 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:23:02.897 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:02.982 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:23:02.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:02.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:23:03.089 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:23:03.198 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:03.264 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 56 16 84400 84384 94.1166 64 0.0161273 0.0106205 2023-08-28T02:23:03.412 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:23:03.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:03.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:23:03.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:03 smithi143 ceph-mon[183059]: pgmap v273: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-08-28T02:23:03.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:03 smithi143 ceph-mon[189871]: pgmap v273: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-08-28T02:23:03.507 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:23:03.520 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:23:03.586 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:03.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:03 smithi169 ceph-mon[159415]: pgmap v273: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-08-28T02:23:04.264 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 57 16 85336 85320 93.492 58.5 0.01322 0.0106916 2023-08-28T02:23:04.764 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:23:04.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:04.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:23:04.874 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:23:04.959 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:05.015 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:23:05.269 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 58 16 86328 86312 92.9419 62 0.0117962 0.0107539 2023-08-28T02:23:05.301 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:23:05.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:05.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:23:05.406 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:23:05.462 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:05.559 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:23:05.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:05.573 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:23:05.577 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:23:05.610 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:23:05.611 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:23:05.674 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:23:05.715 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:05.796 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:23:05.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:05.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:23:05.902 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:23:05.963 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:06.068 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:23:06.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:06.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:23:06.140 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is unset 2023-08-28T02:23:06.178 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:23:06.218 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:06.268 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:23:06.269 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 59 16 87299 87283 92.3955 60.6875 0.00824595 0.0108159 2023-08-28T02:23:06.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:06.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:23:06.383 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:23:06.406 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:06 smithi169 ceph-mon[159415]: pgmap v274: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-08-28T02:23:06.406 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:06 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2088044646' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:23:06.408 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:06.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:06 smithi143 ceph-mon[189871]: pgmap v274: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-08-28T02:23:06.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:06 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2088044646' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:23:06.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:06 smithi143 ceph-mon[183059]: pgmap v274: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-08-28T02:23:06.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:06 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2088044646' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:23:06.483 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:23:06.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:06.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:23:06.593 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:23:06.619 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:06.664 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:23:06.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:06.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:23:06.786 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:23:06.843 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:07.104 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:23:07.107 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.3 dump_blocked_ops 2023-08-28T02:23:07.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:07.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:23:07.216 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:23:07.263 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:07.269 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:2023-08-28T02:23:07.270876+0000 min lat: 0.00111735 max lat: 0.217218 avg lat: 0.0108836 2023-08-28T02:23:07.269 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-08-28T02:23:07.269 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 60 15 88241 88226 91.8381 58.9375 0.0179402 0.0108836 2023-08-28T02:23:07.356 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:23:07.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:07.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:23:07.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:07 smithi143 ceph-mon[183059]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:23:07.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:07 smithi143 ceph-mon[183059]: Cluster is now healthy 2023-08-28T02:23:07.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:07 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2088044646' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:23:07.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:07 smithi143 ceph-mon[183059]: osdmap e136: 8 total, 8 up, 7 in 2023-08-28T02:23:07.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:07 smithi143 ceph-mon[183059]: failed to encode map e136 with expected crc 2023-08-28T02:23:07.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:07 smithi143 ceph-mon[183059]: failed to encode map e136 with expected crc 2023-08-28T02:23:07.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:07 smithi143 ceph-mon[183059]: failed to encode map e136 with expected crc 2023-08-28T02:23:07.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:07 smithi143 ceph-mon[183059]: failed to encode map e136 with expected crc 2023-08-28T02:23:07.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:07 smithi143 ceph-mon[189871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:23:07.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:07 smithi143 ceph-mon[189871]: Cluster is now healthy 2023-08-28T02:23:07.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:07 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2088044646' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:23:07.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:07 smithi143 ceph-mon[189871]: osdmap e136: 8 total, 8 up, 7 in 2023-08-28T02:23:07.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:07 smithi143 ceph-mon[189871]: failed to encode map e136 with expected crc 2023-08-28T02:23:07.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:07 smithi143 ceph-mon[189871]: failed to encode map e136 with expected crc 2023-08-28T02:23:07.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:07 smithi143 ceph-mon[189871]: failed to encode map e136 with expected crc 2023-08-28T02:23:07.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:07 smithi143 ceph-mon[189871]: failed to encode map e136 with expected crc 2023-08-28T02:23:07.473 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:23:07.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:07 smithi169 ceph-mon[159415]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:23:07.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:07 smithi169 ceph-mon[159415]: Cluster is now healthy 2023-08-28T02:23:07.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:07 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2088044646' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:23:07.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:07 smithi169 ceph-mon[159415]: osdmap e136: 8 total, 8 up, 7 in 2023-08-28T02:23:07.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:07 smithi169 ceph-mon[159415]: failed to encode map e136 with expected crc 2023-08-28T02:23:07.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:07 smithi169 ceph-mon[159415]: failed to encode map e136 with expected crc 2023-08-28T02:23:07.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:07 smithi169 ceph-mon[159415]: failed to encode map e136 with expected crc 2023-08-28T02:23:07.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:07 smithi169 ceph-mon[159415]: failed to encode map e136 with expected crc 2023-08-28T02:23:07.518 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:08.269 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 61 16 89075 89059 91.1863 52.0625 0.0043669 0.0109616 2023-08-28T02:23:08.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:08 smithi143 ceph-mon[183059]: pgmap v276: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 56 MiB/s wr, 896 op/s 2023-08-28T02:23:08.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:08 smithi143 ceph-mon[189871]: pgmap v276: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 56 MiB/s wr, 896 op/s 2023-08-28T02:23:08.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:08 smithi169 ceph-mon[159415]: pgmap v276: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 56 MiB/s wr, 896 op/s 2023-08-28T02:23:08.705 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:23:08.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:08.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:23:08.828 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:23:08.911 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:09.032 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:23:09.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:09.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:23:09.143 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:23:09.196 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:09.269 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 62 16 90130 90114 90.7793 65.9375 0.00764325 0.0110105 2023-08-28T02:23:09.299 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:23:09.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:09.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:23:09.419 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:23:09.432 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:23:09.433 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:23:09.459 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:09.467 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:23:09.502 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:23:09.548 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:23:09.549 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:23:09.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:09.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:23:09.597 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:23:09.635 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:23:09.655 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:23:09.684 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:23:09.714 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:23:09.728 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:09.754 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:23:09.754 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:23:09.754 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:23:09.754 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:23:09.755 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:23:09.755 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:23:09.755 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:23:09.755 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:23:09.894 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:23:09.903 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:23:09.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:09.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:23:09.915 INFO:teuthology.orchestra.run.smithi143.stderr:dumped all 2023-08-28T02:23:10.012 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:23:10.086 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:10.175 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:23:10.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:10.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:23:10.269 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 63 16 91118 91102 90.3187 61.75 0.0195725 0.0110672 2023-08-28T02:23:10.286 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:23:10.323 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:10 smithi143 ceph-mon[183059]: pgmap v277: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-08-28T02:23:10.323 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:10 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1414459301' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:23:10.323 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:10 smithi143 ceph-mon[189871]: pgmap v277: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-08-28T02:23:10.323 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:10 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1414459301' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:23:10.336 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:10.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:10 smithi169 ceph-mon[159415]: pgmap v277: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-08-28T02:23:10.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:10 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1414459301' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:23:10.712 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set noscrub 2023-08-28T02:23:11.269 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 64 16 91972 91956 89.7417 53.375 0.00905955 0.0111381 2023-08-28T02:23:11.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:11 smithi143 ceph-mon[189871]: from='client.44681 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:23:11.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:11 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:23:11.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:11 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:23:11.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:11 smithi143 ceph-mon[183059]: from='client.44681 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:23:11.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:11 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:23:11.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:11 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:23:11.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:11 smithi169 ceph-mon[159415]: from='client.44681 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:23:11.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:11 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:23:11.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:11 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:23:11.599 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:23:11.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:11.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:23:11.709 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:23:11.779 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:11.888 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:23:11.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:11.896 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:23:11.929 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:23:11 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:23:11] "GET /metrics HTTP/1.1" 200 36474 "" "Prometheus/2.33.4" 2023-08-28T02:23:11.997 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:23:12.078 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:12.164 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:23:12.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:12.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:23:12.222 INFO:teuthology.orchestra.run.smithi143.stdout:{"pg_ready":true,"pg_map":{"version":277,"stamp":"2023-08-28T02:23:09.061287+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5803481011,"num_objects":88509,"num_object_clones":0,"num_object_copies":265527,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":88509,"num_whiteouts":0,"num_read":6288,"num_read_kb":13719,"num_write":89587,"num_write_kb":5677925,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":112,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":89651,"ondisk_log_size":89651,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":18328736,"kb_used_data":16883240,"kb_used_omap":26,"kb_used_meta":1445349,"kb_avail":637887328,"statfs":{"total":671965249536,"available":653196623872,"internally_reserved":0,"allocated":17288437760,"data_stored":17284475374,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27508,"internal_metadata":1480037516},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":649789440,"num_objects":9915,"num_object_clones":0,"num_object_copies":29745,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9915,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9915,"num_write_kb":634560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.003496"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":119,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.346696+0000","last_change":"2023-08-28T02:20:19.849807+0000","last_active":"2023-08-28T02:23:01.346696+0000","last_peered":"2023-08-28T02:23:01.346696+0000","last_clean":"2023-08-28T02:23:01.346696+0000","last_became_active":"2023-08-28T02:20:19.849464+0000","last_became_peered":"2023-08-28T02:20:19.849464+0000","last_unstale":"2023-08-28T02:23:01.346696+0000","last_undegraded":"2023-08-28T02:23:01.346696+0000","last_fullsized":"2023-08-28T02:23:01.346696+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":16,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.386067+0000","last_change":"2023-08-28T02:22:44.090579+0000","last_active":"2023-08-28T02:23:01.386067+0000","last_peered":"2023-08-28T02:23:01.386067+0000","last_clean":"2023-08-28T02:23:01.386067+0000","last_became_active":"2023-08-28T02:22:44.090300+0000","last_became_peered":"2023-08-28T02:22:44.090300+0000","last_unstale":"2023-08-28T02:23:01.386067+0000","last_undegraded":"2023-08-28T02:23:01.386067+0000","last_fullsized":"2023-08-28T02:23:01.386067+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:43:17.958063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":84,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.291118+0000","last_change":"2023-08-28T02:21:59.173280+0000","last_active":"2023-08-28T02:23:01.291118+0000","last_peered":"2023-08-28T02:23:01.291118+0000","last_clean":"2023-08-28T02:23:01.291118+0000","last_became_active":"2023-08-28T02:21:59.172795+0000","last_became_peered":"2023-08-28T02:21:59.172795+0000","last_unstale":"2023-08-28T02:23:01.291118+0000","last_undegraded":"2023-08-28T02:23:01.291118+0000","last_fullsized":"2023-08-28T02:23:01.291118+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:25:55.130302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":94,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.380889+0000","last_change":"2023-08-28T02:21:59.175898+0000","last_active":"2023-08-28T02:23:01.380889+0000","last_peered":"2023-08-28T02:23:01.380889+0000","last_clean":"2023-08-28T02:23:01.380889+0000","last_became_active":"2023-08-28T02:21:59.175580+0000","last_became_peered":"2023-08-28T02:21:59.175580+0000","last_unstale":"2023-08-28T02:23:01.380889+0000","last_undegraded":"2023-08-28T02:23:01.380889+0000","last_fullsized":"2023-08-28T02:23:01.380889+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:06:34.160608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":113,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.124430+0000","last_change":"2023-08-28T02:21:59.166971+0000","last_active":"2023-08-28T02:23:06.124430+0000","last_peered":"2023-08-28T02:23:06.124430+0000","last_clean":"2023-08-28T02:23:06.124430+0000","last_became_active":"2023-08-28T02:21:59.166677+0000","last_became_peered":"2023-08-28T02:21:59.166677+0000","last_unstale":"2023-08-28T02:23:06.124430+0000","last_undegraded":"2023-08-28T02:23:06.124430+0000","last_fullsized":"2023-08-28T02:23:06.124430+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:16:04.310997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":130,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.348253+0000","last_change":"2023-08-28T02:21:59.183009+0000","last_active":"2023-08-28T02:23:01.348253+0000","last_peered":"2023-08-28T02:23:01.348253+0000","last_clean":"2023-08-28T02:23:01.348253+0000","last_became_active":"2023-08-28T02:21:59.182621+0000","last_became_peered":"2023-08-28T02:21:59.182621+0000","last_unstale":"2023-08-28T02:23:01.348253+0000","last_undegraded":"2023-08-28T02:23:01.348253+0000","last_fullsized":"2023-08-28T02:23:01.348253+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":92,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.380783+0000","last_change":"2023-08-28T02:21:59.181634+0000","last_active":"2023-08-28T02:23:01.380783+0000","last_peered":"2023-08-28T02:23:01.380783+0000","last_clean":"2023-08-28T02:23:01.380783+0000","last_became_active":"2023-08-28T02:21:59.181300+0000","last_became_peered":"2023-08-28T02:21:59.181300+0000","last_unstale":"2023-08-28T02:23:01.380783+0000","last_undegraded":"2023-08-28T02:23:01.380783+0000","last_fullsized":"2023-08-28T02:23:01.380783+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:35:48.902315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":47,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.291261+0000","last_change":"2023-08-28T02:21:59.181238+0000","last_active":"2023-08-28T02:23:01.291261+0000","last_peered":"2023-08-28T02:23:01.291261+0000","last_clean":"2023-08-28T02:23:01.291261+0000","last_became_active":"2023-08-28T02:21:59.180913+0000","last_became_peered":"2023-08-28T02:21:59.180913+0000","last_unstale":"2023-08-28T02:23:01.291261+0000","last_undegraded":"2023-08-28T02:23:01.291261+0000","last_fullsized":"2023-08-28T02:23:01.291261+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:10:43.235404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":103,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.347476+0000","last_change":"2023-08-28T02:19:50.214698+0000","last_active":"2023-08-28T02:23:01.347476+0000","last_peered":"2023-08-28T02:23:01.347476+0000","last_clean":"2023-08-28T02:23:01.347476+0000","last_became_active":"2023-08-28T02:19:50.214323+0000","last_became_peered":"2023-08-28T02:19:50.214323+0000","last_unstale":"2023-08-28T02:23:01.347476+0000","last_undegraded":"2023-08-28T02:23:01.347476+0000","last_fullsized":"2023-08-28T02:23:01.347476+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":83,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.345629+0000","last_change":"2023-08-28T02:21:59.175974+0000","last_active":"2023-08-28T02:23:01.345629+0000","last_peered":"2023-08-28T02:23:01.345629+0000","last_clean":"2023-08-28T02:23:01.345629+0000","last_became_active":"2023-08-28T02:21:59.175624+0000","last_became_peered":"2023-08-28T02:21:59.175624+0000","last_unstale":"2023-08-28T02:23:01.345629+0000","last_undegraded":"2023-08-28T02:23:01.345629+0000","last_fullsized":"2023-08-28T02:23:01.345629+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,5],"acting":[7,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":121,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.346329+0000","last_change":"2023-08-28T02:21:59.278365+0000","last_active":"2023-08-28T02:23:01.346329+0000","last_peered":"2023-08-28T02:23:01.346329+0000","last_clean":"2023-08-28T02:23:01.346329+0000","last_became_active":"2023-08-28T02:21:59.171883+0000","last_became_peered":"2023-08-28T02:21:59.171883+0000","last_unstale":"2023-08-28T02:23:01.346329+0000","last_undegraded":"2023-08-28T02:23:01.346329+0000","last_fullsized":"2023-08-28T02:23:01.346329+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":76,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.291615+0000","last_change":"2023-08-28T02:20:19.851616+0000","last_active":"2023-08-28T02:23:01.291615+0000","last_peered":"2023-08-28T02:23:01.291615+0000","last_clean":"2023-08-28T02:23:01.291615+0000","last_became_active":"2023-08-28T02:20:19.851322+0000","last_became_peered":"2023-08-28T02:20:19.851322+0000","last_unstale":"2023-08-28T02:23:01.291615+0000","last_undegraded":"2023-08-28T02:23:01.291615+0000","last_fullsized":"2023-08-28T02:23:01.291615+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:03:37.565646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"86'22","reported_seq":110,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.346600+0000","last_change":"2023-08-28T02:11:22.184497+0000","last_active":"2023-08-28T02:23:01.346600+0000","last_peered":"2023-08-28T02:23:01.346600+0000","last_clean":"2023-08-28T02:23:01.346600+0000","last_became_active":"2023-08-28T02:11:22.184173+0000","last_became_peered":"2023-08-28T02:11:22.184173+0000","last_unstale":"2023-08-28T02:23:01.346600+0000","last_undegraded":"2023-08-28T02:23:01.346600+0000","last_fullsized":"2023-08-28T02:23:01.346600+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":116,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.345975+0000","last_change":"2023-08-28T02:19:51.204118+0000","last_active":"2023-08-28T02:23:01.345975+0000","last_peered":"2023-08-28T02:23:01.345975+0000","last_clean":"2023-08-28T02:23:01.345975+0000","last_became_active":"2023-08-28T02:19:51.203745+0000","last_became_peered":"2023-08-28T02:19:51.203745+0000","last_unstale":"2023-08-28T02:23:01.345975+0000","last_undegraded":"2023-08-28T02:23:01.345975+0000","last_fullsized":"2023-08-28T02:23:01.345975+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":105,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.345802+0000","last_change":"2023-08-28T02:20:19.849490+0000","last_active":"2023-08-28T02:23:01.345802+0000","last_peered":"2023-08-28T02:23:01.345802+0000","last_clean":"2023-08-28T02:23:01.345802+0000","last_became_active":"2023-08-28T02:20:19.848962+0000","last_became_peered":"2023-08-28T02:20:19.848962+0000","last_unstale":"2023-08-28T02:23:01.345802+0000","last_undegraded":"2023-08-28T02:23:01.345802+0000","last_fullsized":"2023-08-28T02:23:01.345802+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":97,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:03.285286+0000","last_change":"2023-08-28T02:21:59.175948+0000","last_active":"2023-08-28T02:23:03.285286+0000","last_peered":"2023-08-28T02:23:03.285286+0000","last_clean":"2023-08-28T02:23:03.285286+0000","last_became_active":"2023-08-28T02:21:59.175583+0000","last_became_peered":"2023-08-28T02:21:59.175583+0000","last_unstale":"2023-08-28T02:23:03.285286+0000","last_undegraded":"2023-08-28T02:23:03.285286+0000","last_fullsized":"2023-08-28T02:23:03.285286+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:06:13.068987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":92,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.380374+0000","last_change":"2023-08-28T02:21:59.167044+0000","last_active":"2023-08-28T02:23:01.380374+0000","last_peered":"2023-08-28T02:23:01.380374+0000","last_clean":"2023-08-28T02:23:01.380374+0000","last_became_active":"2023-08-28T02:21:59.166742+0000","last_became_peered":"2023-08-28T02:21:59.166742+0000","last_unstale":"2023-08-28T02:23:01.380374+0000","last_undegraded":"2023-08-28T02:23:01.380374+0000","last_fullsized":"2023-08-28T02:23:01.380374+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:35:59.546215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":136,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.347373+0000","last_change":"2023-08-28T02:21:59.180192+0000","last_active":"2023-08-28T02:23:01.347373+0000","last_peered":"2023-08-28T02:23:01.347373+0000","last_clean":"2023-08-28T02:23:01.347373+0000","last_became_active":"2023-08-28T02:21:59.179855+0000","last_became_peered":"2023-08-28T02:21:59.179855+0000","last_unstale":"2023-08-28T02:23:01.347373+0000","last_undegraded":"2023-08-28T02:23:01.347373+0000","last_fullsized":"2023-08-28T02:23:01.347373+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":88,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.345792+0000","last_change":"2023-08-28T02:11:22.188327+0000","last_active":"2023-08-28T02:23:01.345792+0000","last_peered":"2023-08-28T02:23:01.345792+0000","last_clean":"2023-08-28T02:23:01.345792+0000","last_became_active":"2023-08-28T02:11:22.187870+0000","last_became_peered":"2023-08-28T02:11:22.187870+0000","last_unstale":"2023-08-28T02:23:01.345792+0000","last_undegraded":"2023-08-28T02:23:01.345792+0000","last_fullsized":"2023-08-28T02:23:01.345792+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":93,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.380820+0000","last_change":"2023-08-28T02:21:59.176349+0000","last_active":"2023-08-28T02:23:01.380820+0000","last_peered":"2023-08-28T02:23:01.380820+0000","last_clean":"2023-08-28T02:23:01.380820+0000","last_became_active":"2023-08-28T02:21:59.175962+0000","last_became_peered":"2023-08-28T02:21:59.175962+0000","last_unstale":"2023-08-28T02:23:01.380820+0000","last_undegraded":"2023-08-28T02:23:01.380820+0000","last_fullsized":"2023-08-28T02:23:01.380820+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:43:54.669505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":110,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.347752+0000","last_change":"2023-08-28T02:18:39.778728+0000","last_active":"2023-08-28T02:23:01.347752+0000","last_peered":"2023-08-28T02:23:01.347752+0000","last_clean":"2023-08-28T02:23:01.347752+0000","last_became_active":"2023-08-28T02:18:39.778283+0000","last_became_peered":"2023-08-28T02:18:39.778283+0000","last_unstale":"2023-08-28T02:23:01.347752+0000","last_undegraded":"2023-08-28T02:23:01.347752+0000","last_fullsized":"2023-08-28T02:23:01.347752+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":90,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.346050+0000","last_change":"2023-08-28T02:22:44.096303+0000","last_active":"2023-08-28T02:23:01.346050+0000","last_peered":"2023-08-28T02:23:01.346050+0000","last_clean":"2023-08-28T02:23:01.346050+0000","last_became_active":"2023-08-28T02:22:44.095853+0000","last_became_peered":"2023-08-28T02:22:44.095853+0000","last_unstale":"2023-08-28T02:23:01.346050+0000","last_undegraded":"2023-08-28T02:23:01.346050+0000","last_fullsized":"2023-08-28T02:23:01.346050+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.9","version":"136'5464","reported_seq":5489,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.941340+0000","last_change":"2023-08-28T02:22:00.169476+0000","last_active":"2023-08-28T02:23:06.941340+0000","last_peered":"2023-08-28T02:23:06.941340+0000","last_clean":"2023-08-28T02:23:06.941340+0000","last_became_active":"2023-08-28T02:22:00.168754+0000","last_became_peered":"2023-08-28T02:22:00.168754+0000","last_unstale":"2023-08-28T02:23:06.941340+0000","last_undegraded":"2023-08-28T02:23:06.941340+0000","last_fullsized":"2023-08-28T02:23:06.941340+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":5464,"log_dups_size":0,"ondisk_log_size":5464,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358088704,"num_objects":5464,"num_object_clones":0,"num_object_copies":16392,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5464,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5464,"num_write_kb":349696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":117,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.346217+0000","last_change":"2023-08-28T02:20:43.220383+0000","last_active":"2023-08-28T02:23:01.346217+0000","last_peered":"2023-08-28T02:23:01.346217+0000","last_clean":"2023-08-28T02:23:01.346217+0000","last_became_active":"2023-08-28T02:20:43.136191+0000","last_became_peered":"2023-08-28T02:20:43.136191+0000","last_unstale":"2023-08-28T02:23:01.346217+0000","last_undegraded":"2023-08-28T02:23:01.346217+0000","last_fullsized":"2023-08-28T02:23:01.346217+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":87,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.346964+0000","last_change":"2023-08-28T02:11:23.249061+0000","last_active":"2023-08-28T02:23:01.346964+0000","last_peered":"2023-08-28T02:23:01.346964+0000","last_clean":"2023-08-28T02:23:01.346964+0000","last_became_active":"2023-08-28T02:11:23.248739+0000","last_became_peered":"2023-08-28T02:11:23.248739+0000","last_unstale":"2023-08-28T02:23:01.346964+0000","last_undegraded":"2023-08-28T02:23:01.346964+0000","last_fullsized":"2023-08-28T02:23:01.346964+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"136'5552","reported_seq":5577,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:07.868206+0000","last_change":"2023-08-28T02:22:00.165260+0000","last_active":"2023-08-28T02:23:07.868206+0000","last_peered":"2023-08-28T02:23:07.868206+0000","last_clean":"2023-08-28T02:23:07.868206+0000","last_became_active":"2023-08-28T02:22:00.164978+0000","last_became_peered":"2023-08-28T02:22:00.164978+0000","last_unstale":"2023-08-28T02:23:07.868206+0000","last_undegraded":"2023-08-28T02:23:07.868206+0000","last_fullsized":"2023-08-28T02:23:07.868206+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":5552,"log_dups_size":0,"ondisk_log_size":5552,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:18:41.711960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363855872,"num_objects":5552,"num_object_clones":0,"num_object_copies":16656,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5552,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5552,"num_write_kb":355328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":105,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.346133+0000","last_change":"2023-08-28T02:19:50.220079+0000","last_active":"2023-08-28T02:23:01.346133+0000","last_peered":"2023-08-28T02:23:01.346133+0000","last_clean":"2023-08-28T02:23:01.346133+0000","last_became_active":"2023-08-28T02:19:50.219671+0000","last_became_peered":"2023-08-28T02:19:50.219671+0000","last_unstale":"2023-08-28T02:23:01.346133+0000","last_undegraded":"2023-08-28T02:23:01.346133+0000","last_fullsized":"2023-08-28T02:23:01.346133+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":135,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.346153+0000","last_change":"2023-08-28T02:22:44.090586+0000","last_active":"2023-08-28T02:23:01.346153+0000","last_peered":"2023-08-28T02:23:01.346153+0000","last_clean":"2023-08-28T02:23:01.346153+0000","last_became_active":"2023-08-28T02:22:44.090339+0000","last_became_peered":"2023-08-28T02:22:44.090339+0000","last_unstale":"2023-08-28T02:23:01.346153+0000","last_undegraded":"2023-08-28T02:23:01.346153+0000","last_fullsized":"2023-08-28T02:23:01.346153+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"136'5553","reported_seq":5577,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:08.013097+0000","last_change":"2023-08-28T02:22:00.165317+0000","last_active":"2023-08-28T02:23:08.013097+0000","last_peered":"2023-08-28T02:23:08.013097+0000","last_clean":"2023-08-28T02:23:08.013097+0000","last_became_active":"2023-08-28T02:22:00.164989+0000","last_became_peered":"2023-08-28T02:22:00.164989+0000","last_unstale":"2023-08-28T02:23:08.013097+0000","last_undegraded":"2023-08-28T02:23:08.013097+0000","last_fullsized":"2023-08-28T02:23:08.013097+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":5553,"log_dups_size":0,"ondisk_log_size":5553,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:10:46.322659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363921408,"num_objects":5553,"num_object_clones":0,"num_object_copies":16659,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5553,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5553,"num_write_kb":355392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":89,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.346435+0000","last_change":"2023-08-28T02:11:22.188982+0000","last_active":"2023-08-28T02:23:01.346435+0000","last_peered":"2023-08-28T02:23:01.346435+0000","last_clean":"2023-08-28T02:23:01.346435+0000","last_became_active":"2023-08-28T02:11:22.188663+0000","last_became_peered":"2023-08-28T02:11:22.188663+0000","last_unstale":"2023-08-28T02:23:01.346435+0000","last_undegraded":"2023-08-28T02:23:01.346435+0000","last_fullsized":"2023-08-28T02:23:01.346435+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":144,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.347883+0000","last_change":"2023-08-28T02:21:43.205520+0000","last_active":"2023-08-28T02:23:01.347883+0000","last_peered":"2023-08-28T02:23:01.347883+0000","last_clean":"2023-08-28T02:23:01.347883+0000","last_became_active":"2023-08-28T02:21:43.205165+0000","last_became_peered":"2023-08-28T02:21:43.205165+0000","last_unstale":"2023-08-28T02:23:01.347883+0000","last_undegraded":"2023-08-28T02:23:01.347883+0000","last_fullsized":"2023-08-28T02:23:01.347883+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"136'5592","reported_seq":5615,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:08.013558+0000","last_change":"2023-08-28T02:22:00.166496+0000","last_active":"2023-08-28T02:23:08.013558+0000","last_peered":"2023-08-28T02:23:08.013558+0000","last_clean":"2023-08-28T02:23:08.013558+0000","last_became_active":"2023-08-28T02:22:00.166206+0000","last_became_peered":"2023-08-28T02:22:00.166206+0000","last_unstale":"2023-08-28T02:23:08.013558+0000","last_undegraded":"2023-08-28T02:23:08.013558+0000","last_fullsized":"2023-08-28T02:23:08.013558+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":5592,"log_dups_size":0,"ondisk_log_size":5592,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:16:10.265829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":366477312,"num_objects":5592,"num_object_clones":0,"num_object_copies":16776,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5592,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5592,"num_write_kb":357888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":112,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.347267+0000","last_change":"2023-08-28T02:21:59.176563+0000","last_active":"2023-08-28T02:23:01.347267+0000","last_peered":"2023-08-28T02:23:01.347267+0000","last_clean":"2023-08-28T02:23:01.347267+0000","last_became_active":"2023-08-28T02:21:59.175969+0000","last_became_peered":"2023-08-28T02:21:59.175969+0000","last_unstale":"2023-08-28T02:23:01.347267+0000","last_undegraded":"2023-08-28T02:23:01.347267+0000","last_fullsized":"2023-08-28T02:23:01.347267+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":96,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.124431+0000","last_change":"2023-08-28T02:19:51.205245+0000","last_active":"2023-08-28T02:23:06.124431+0000","last_peered":"2023-08-28T02:23:06.124431+0000","last_clean":"2023-08-28T02:23:06.124431+0000","last_became_active":"2023-08-28T02:19:51.204896+0000","last_became_peered":"2023-08-28T02:19:51.204896+0000","last_unstale":"2023-08-28T02:23:06.124431+0000","last_undegraded":"2023-08-28T02:23:06.124431+0000","last_fullsized":"2023-08-28T02:23:06.124431+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:51:24.634702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"136'5551","reported_seq":5574,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:07.893098+0000","last_change":"2023-08-28T02:22:00.167148+0000","last_active":"2023-08-28T02:23:07.893098+0000","last_peered":"2023-08-28T02:23:07.893098+0000","last_clean":"2023-08-28T02:23:07.893098+0000","last_became_active":"2023-08-28T02:22:00.166809+0000","last_became_peered":"2023-08-28T02:22:00.166809+0000","last_unstale":"2023-08-28T02:23:07.893098+0000","last_undegraded":"2023-08-28T02:23:07.893098+0000","last_fullsized":"2023-08-28T02:23:07.893098+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":5551,"log_dups_size":0,"ondisk_log_size":5551,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:03:13.526107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363790336,"num_objects":5551,"num_object_clones":0,"num_object_copies":16653,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5551,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5551,"num_write_kb":355264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":56,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.380945+0000","last_change":"2023-08-28T02:21:59.280044+0000","last_active":"2023-08-28T02:23:01.380945+0000","last_peered":"2023-08-28T02:23:01.380945+0000","last_clean":"2023-08-28T02:23:01.380945+0000","last_became_active":"2023-08-28T02:21:59.179617+0000","last_became_peered":"2023-08-28T02:21:59.179617+0000","last_unstale":"2023-08-28T02:23:01.380945+0000","last_undegraded":"2023-08-28T02:23:01.380945+0000","last_fullsized":"2023-08-28T02:23:01.380945+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:14:46.083577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":120,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.346487+0000","last_change":"2023-08-28T02:22:44.091351+0000","last_active":"2023-08-28T02:23:01.346487+0000","last_peered":"2023-08-28T02:23:01.346487+0000","last_clean":"2023-08-28T02:23:01.346487+0000","last_became_active":"2023-08-28T02:22:44.090971+0000","last_became_peered":"2023-08-28T02:22:44.090971+0000","last_unstale":"2023-08-28T02:23:01.346487+0000","last_undegraded":"2023-08-28T02:23:01.346487+0000","last_fullsized":"2023-08-28T02:23:01.346487+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"136'5671","reported_seq":5697,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:08.282470+0000","last_change":"2023-08-28T02:22:00.162375+0000","last_active":"2023-08-28T02:23:08.282470+0000","last_peered":"2023-08-28T02:23:08.282470+0000","last_clean":"2023-08-28T02:23:08.282470+0000","last_became_active":"2023-08-28T02:22:00.162190+0000","last_became_peered":"2023-08-28T02:22:00.162190+0000","last_unstale":"2023-08-28T02:23:08.282470+0000","last_undegraded":"2023-08-28T02:23:08.282470+0000","last_fullsized":"2023-08-28T02:23:08.282470+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":5671,"log_dups_size":0,"ondisk_log_size":5671,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:40:08.985631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":371654656,"num_objects":5671,"num_object_clones":0,"num_object_copies":17013,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5671,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5671,"num_write_kb":362944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":131,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.347985+0000","last_change":"2023-08-28T02:21:59.179699+0000","last_active":"2023-08-28T02:23:01.347985+0000","last_peered":"2023-08-28T02:23:01.347985+0000","last_clean":"2023-08-28T02:23:01.347985+0000","last_became_active":"2023-08-28T02:21:59.179345+0000","last_became_peered":"2023-08-28T02:21:59.179345+0000","last_unstale":"2023-08-28T02:23:01.347985+0000","last_undegraded":"2023-08-28T02:23:01.347985+0000","last_fullsized":"2023-08-28T02:23:01.347985+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":155,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.346557+0000","last_change":"2023-08-28T02:22:44.083121+0000","last_active":"2023-08-28T02:23:01.346557+0000","last_peered":"2023-08-28T02:23:01.346557+0000","last_clean":"2023-08-28T02:23:01.346557+0000","last_became_active":"2023-08-28T02:22:44.082776+0000","last_became_peered":"2023-08-28T02:22:44.082776+0000","last_unstale":"2023-08-28T02:23:01.346557+0000","last_undegraded":"2023-08-28T02:23:01.346557+0000","last_fullsized":"2023-08-28T02:23:01.346557+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"136'5501","reported_seq":5526,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.981931+0000","last_change":"2023-08-28T02:22:00.167630+0000","last_active":"2023-08-28T02:23:06.981931+0000","last_peered":"2023-08-28T02:23:06.981931+0000","last_clean":"2023-08-28T02:23:06.981931+0000","last_became_active":"2023-08-28T02:22:00.167305+0000","last_became_peered":"2023-08-28T02:22:00.167305+0000","last_unstale":"2023-08-28T02:23:06.981931+0000","last_undegraded":"2023-08-28T02:23:06.981931+0000","last_fullsized":"2023-08-28T02:23:06.981931+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":5501,"log_dups_size":0,"ondisk_log_size":5501,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":360513536,"num_objects":5501,"num_object_clones":0,"num_object_copies":16503,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5501,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5501,"num_write_kb":352064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":27,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.346245+0000","last_change":"2023-08-28T02:21:59.174435+0000","last_active":"2023-08-28T02:23:01.346245+0000","last_peered":"2023-08-28T02:23:01.346245+0000","last_clean":"2023-08-28T02:23:01.346245+0000","last_became_active":"2023-08-28T02:21:59.174079+0000","last_became_peered":"2023-08-28T02:21:59.174079+0000","last_unstale":"2023-08-28T02:23:01.346245+0000","last_undegraded":"2023-08-28T02:23:01.346245+0000","last_fullsized":"2023-08-28T02:23:01.346245+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":64,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.381395+0000","last_change":"2023-08-28T02:19:51.204017+0000","last_active":"2023-08-28T02:23:01.381395+0000","last_peered":"2023-08-28T02:23:01.381395+0000","last_clean":"2023-08-28T02:23:01.381395+0000","last_became_active":"2023-08-28T02:19:51.203627+0000","last_became_peered":"2023-08-28T02:19:51.203627+0000","last_unstale":"2023-08-28T02:23:01.381395+0000","last_undegraded":"2023-08-28T02:23:01.381395+0000","last_fullsized":"2023-08-28T02:23:01.381395+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:14:06.231701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.e","version":"136'5575","reported_seq":5600,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.951137+0000","last_change":"2023-08-28T02:22:00.166490+0000","last_active":"2023-08-28T02:23:06.951137+0000","last_peered":"2023-08-28T02:23:06.951137+0000","last_clean":"2023-08-28T02:23:06.951137+0000","last_became_active":"2023-08-28T02:22:00.166140+0000","last_became_peered":"2023-08-28T02:22:00.166140+0000","last_unstale":"2023-08-28T02:23:06.951137+0000","last_undegraded":"2023-08-28T02:23:06.951137+0000","last_fullsized":"2023-08-28T02:23:06.951137+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":5575,"log_dups_size":0,"ondisk_log_size":5575,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365363200,"num_objects":5575,"num_object_clones":0,"num_object_copies":16725,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5575,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5575,"num_write_kb":356800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":98,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.291168+0000","last_change":"2023-08-28T02:21:59.168708+0000","last_active":"2023-08-28T02:23:01.291168+0000","last_peered":"2023-08-28T02:23:01.291168+0000","last_clean":"2023-08-28T02:23:01.291168+0000","last_became_active":"2023-08-28T02:21:59.168481+0000","last_became_peered":"2023-08-28T02:21:59.168481+0000","last_unstale":"2023-08-28T02:23:01.291168+0000","last_undegraded":"2023-08-28T02:23:01.291168+0000","last_fullsized":"2023-08-28T02:23:01.291168+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:33:24.514653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":85,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.380628+0000","last_change":"2023-08-28T02:21:59.176160+0000","last_active":"2023-08-28T02:23:01.380628+0000","last_peered":"2023-08-28T02:23:01.380628+0000","last_clean":"2023-08-28T02:23:01.380628+0000","last_became_active":"2023-08-28T02:21:59.175793+0000","last_became_peered":"2023-08-28T02:21:59.175793+0000","last_unstale":"2023-08-28T02:23:01.380628+0000","last_undegraded":"2023-08-28T02:23:01.380628+0000","last_fullsized":"2023-08-28T02:23:01.380628+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:59:44.892535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"78'126","reported_seq":593,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.379989+0000","last_change":"2023-08-28T02:19:50.213556+0000","last_active":"2023-08-28T02:23:01.379989+0000","last_peered":"2023-08-28T02:23:01.379989+0000","last_clean":"2023-08-28T02:23:01.379989+0000","last_became_active":"2023-08-28T02:19:50.212996+0000","last_became_peered":"2023-08-28T02:19:50.212996+0000","last_unstale":"2023-08-28T02:23:01.379989+0000","last_undegraded":"2023-08-28T02:23:01.379989+0000","last_fullsized":"2023-08-28T02:23:01.379989+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:38:28.157352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":743,"num_read_kb":1331,"num_write":125,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":127,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.347442+0000","last_change":"2023-08-28T02:21:59.176292+0000","last_active":"2023-08-28T02:23:01.347442+0000","last_peered":"2023-08-28T02:23:01.347442+0000","last_clean":"2023-08-28T02:23:01.347442+0000","last_became_active":"2023-08-28T02:21:59.175956+0000","last_became_peered":"2023-08-28T02:21:59.175956+0000","last_unstale":"2023-08-28T02:23:01.347442+0000","last_undegraded":"2023-08-28T02:23:01.347442+0000","last_fullsized":"2023-08-28T02:23:01.347442+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":31,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.291565+0000","last_change":"2023-08-28T02:21:43.238044+0000","last_active":"2023-08-28T02:23:01.291565+0000","last_peered":"2023-08-28T02:23:01.291565+0000","last_clean":"2023-08-28T02:23:01.291565+0000","last_became_active":"2023-08-28T02:21:43.237792+0000","last_became_peered":"2023-08-28T02:21:43.237792+0000","last_unstale":"2023-08-28T02:23:01.291565+0000","last_undegraded":"2023-08-28T02:23:01.291565+0000","last_fullsized":"2023-08-28T02:23:01.291565+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:11:35.488503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"136'5500","reported_seq":5526,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:07.870172+0000","last_change":"2023-08-28T02:22:00.166578+0000","last_active":"2023-08-28T02:23:07.870172+0000","last_peered":"2023-08-28T02:23:07.870172+0000","last_clean":"2023-08-28T02:23:07.870172+0000","last_became_active":"2023-08-28T02:22:00.166311+0000","last_became_peered":"2023-08-28T02:22:00.166311+0000","last_unstale":"2023-08-28T02:23:07.870172+0000","last_undegraded":"2023-08-28T02:23:07.870172+0000","last_fullsized":"2023-08-28T02:23:07.870172+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":5500,"log_dups_size":0,"ondisk_log_size":5500,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:40:38.502180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":360448000,"num_objects":5500,"num_object_clones":0,"num_object_copies":16500,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5500,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5500,"num_write_kb":352000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"136'5541","reported_seq":5567,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:07.889501+0000","last_change":"2023-08-28T02:22:00.160991+0000","last_active":"2023-08-28T02:23:07.889501+0000","last_peered":"2023-08-28T02:23:07.889501+0000","last_clean":"2023-08-28T02:23:07.889501+0000","last_became_active":"2023-08-28T02:22:00.160755+0000","last_became_peered":"2023-08-28T02:22:00.160755+0000","last_unstale":"2023-08-28T02:23:07.889501+0000","last_undegraded":"2023-08-28T02:23:07.889501+0000","last_fullsized":"2023-08-28T02:23:07.889501+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":5541,"log_dups_size":0,"ondisk_log_size":5541,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:42:59.109288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363134976,"num_objects":5541,"num_object_clones":0,"num_object_copies":16623,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5541,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5541,"num_write_kb":354624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"78'204","reported_seq":1052,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.347142+0000","last_change":"2023-08-28T02:19:50.219949+0000","last_active":"2023-08-28T02:23:01.347142+0000","last_peered":"2023-08-28T02:23:01.347142+0000","last_clean":"2023-08-28T02:23:01.347142+0000","last_became_active":"2023-08-28T02:19:50.219356+0000","last_became_peered":"2023-08-28T02:19:50.219356+0000","last_unstale":"2023-08-28T02:23:01.347142+0000","last_undegraded":"2023-08-28T02:23:01.347142+0000","last_fullsized":"2023-08-28T02:23:01.347142+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":204,"log_dups_size":0,"ondisk_log_size":204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1521,"num_read_kb":2511,"num_write":178,"num_write_kb":1175,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":103,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.346134+0000","last_change":"2023-08-28T02:18:39.779607+0000","last_active":"2023-08-28T02:23:01.346134+0000","last_peered":"2023-08-28T02:23:01.346134+0000","last_clean":"2023-08-28T02:23:01.346134+0000","last_became_active":"2023-08-28T02:18:39.779303+0000","last_became_peered":"2023-08-28T02:18:39.779303+0000","last_unstale":"2023-08-28T02:23:01.346134+0000","last_undegraded":"2023-08-28T02:23:01.346134+0000","last_fullsized":"2023-08-28T02:23:01.346134+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":83,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.291826+0000","last_change":"2023-08-28T02:21:59.174327+0000","last_active":"2023-08-28T02:23:01.291826+0000","last_peered":"2023-08-28T02:23:01.291826+0000","last_clean":"2023-08-28T02:23:01.291826+0000","last_became_active":"2023-08-28T02:21:59.173986+0000","last_became_peered":"2023-08-28T02:21:59.173986+0000","last_unstale":"2023-08-28T02:23:01.291826+0000","last_undegraded":"2023-08-28T02:23:01.291826+0000","last_fullsized":"2023-08-28T02:23:01.291826+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:55:51.249985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.6","version":"136'5450","reported_seq":5475,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:08.007932+0000","last_change":"2023-08-28T02:22:00.166938+0000","last_active":"2023-08-28T02:23:08.007932+0000","last_peered":"2023-08-28T02:23:08.007932+0000","last_clean":"2023-08-28T02:23:08.007932+0000","last_became_active":"2023-08-28T02:22:00.166656+0000","last_became_peered":"2023-08-28T02:22:00.166656+0000","last_unstale":"2023-08-28T02:23:08.007932+0000","last_undegraded":"2023-08-28T02:23:08.007932+0000","last_fullsized":"2023-08-28T02:23:08.007932+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":5450,"log_dups_size":0,"ondisk_log_size":5450,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:57:00.014765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":357171200,"num_objects":5450,"num_object_clones":0,"num_object_copies":16350,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5450,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5450,"num_write_kb":348800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"78'47","reported_seq":320,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.387152+0000","last_change":"2023-08-28T02:21:59.287946+0000","last_active":"2023-08-28T02:23:01.387152+0000","last_peered":"2023-08-28T02:23:01.387152+0000","last_clean":"2023-08-28T02:23:01.387152+0000","last_became_active":"2023-08-28T02:21:59.173235+0000","last_became_peered":"2023-08-28T02:21:59.173235+0000","last_unstale":"2023-08-28T02:23:01.387152+0000","last_undegraded":"2023-08-28T02:23:01.387152+0000","last_fullsized":"2023-08-28T02:23:01.387152+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":47,"log_dups_size":0,"ondisk_log_size":47,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:02:39.919036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":162,"num_read_kb":1281,"num_write":48,"num_write_kb":1153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":75,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.291259+0000","last_change":"2023-08-28T02:20:19.851591+0000","last_active":"2023-08-28T02:23:01.291259+0000","last_peered":"2023-08-28T02:23:01.291259+0000","last_clean":"2023-08-28T02:23:01.291259+0000","last_became_active":"2023-08-28T02:20:19.851291+0000","last_became_peered":"2023-08-28T02:20:19.851291+0000","last_unstale":"2023-08-28T02:23:01.291259+0000","last_undegraded":"2023-08-28T02:23:01.291259+0000","last_fullsized":"2023-08-28T02:23:01.291259+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:33:51.259707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":107,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.380604+0000","last_change":"2023-08-28T02:19:50.211156+0000","last_active":"2023-08-28T02:23:01.380604+0000","last_peered":"2023-08-28T02:23:01.380604+0000","last_clean":"2023-08-28T02:23:01.380604+0000","last_became_active":"2023-08-28T02:19:50.210398+0000","last_became_peered":"2023-08-28T02:19:50.210398+0000","last_unstale":"2023-08-28T02:23:01.380604+0000","last_undegraded":"2023-08-28T02:23:01.380604+0000","last_fullsized":"2023-08-28T02:23:01.380604+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:14:41.748197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"78'114","reported_seq":434,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.347169+0000","last_change":"2023-08-28T02:20:19.852152+0000","last_active":"2023-08-28T02:23:01.347169+0000","last_peered":"2023-08-28T02:23:01.347169+0000","last_clean":"2023-08-28T02:23:01.347169+0000","last_became_active":"2023-08-28T02:20:19.851615+0000","last_became_peered":"2023-08-28T02:20:19.851615+0000","last_unstale":"2023-08-28T02:23:01.347169+0000","last_undegraded":"2023-08-28T02:23:01.347169+0000","last_fullsized":"2023-08-28T02:23:01.347169+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":114,"log_dups_size":0,"ondisk_log_size":114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":358,"num_read_kb":1352,"num_write":102,"num_write_kb":1104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"86'139","reported_seq":339,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.345771+0000","last_change":"2023-08-28T02:18:39.786823+0000","last_active":"2023-08-28T02:23:01.345771+0000","last_peered":"2023-08-28T02:23:01.345771+0000","last_clean":"2023-08-28T02:23:01.345771+0000","last_became_active":"2023-08-28T02:18:39.786357+0000","last_became_peered":"2023-08-28T02:18:39.786357+0000","last_unstale":"2023-08-28T02:23:01.345771+0000","last_undegraded":"2023-08-28T02:23:01.345771+0000","last_fullsized":"2023-08-28T02:23:01.345771+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_clean_scrub_stamp":"2023-08-28T02:08:02.043718+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":83,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.347171+0000","last_change":"2023-08-28T02:21:59.179583+0000","last_active":"2023-08-28T02:23:01.347171+0000","last_peered":"2023-08-28T02:23:01.347171+0000","last_clean":"2023-08-28T02:23:01.347171+0000","last_became_active":"2023-08-28T02:21:59.179239+0000","last_became_peered":"2023-08-28T02:21:59.179239+0000","last_unstale":"2023-08-28T02:23:01.347171+0000","last_undegraded":"2023-08-28T02:23:01.347171+0000","last_fullsized":"2023-08-28T02:23:01.347171+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":86,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.348119+0000","last_change":"2023-08-28T02:11:23.249640+0000","last_active":"2023-08-28T02:23:01.348119+0000","last_peered":"2023-08-28T02:23:01.348119+0000","last_clean":"2023-08-28T02:23:01.348119+0000","last_became_active":"2023-08-28T02:11:23.249355+0000","last_became_peered":"2023-08-28T02:11:23.249355+0000","last_unstale":"2023-08-28T02:23:01.348119+0000","last_undegraded":"2023-08-28T02:23:01.348119+0000","last_fullsized":"2023-08-28T02:23:01.348119+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"136'5460","reported_seq":5485,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.974616+0000","last_change":"2023-08-28T02:22:00.169046+0000","last_active":"2023-08-28T02:23:06.974616+0000","last_peered":"2023-08-28T02:23:06.974616+0000","last_clean":"2023-08-28T02:23:06.974616+0000","last_became_active":"2023-08-28T02:22:00.168696+0000","last_became_peered":"2023-08-28T02:22:00.168696+0000","last_unstale":"2023-08-28T02:23:06.974616+0000","last_undegraded":"2023-08-28T02:23:06.974616+0000","last_fullsized":"2023-08-28T02:23:06.974616+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":5460,"log_dups_size":0,"ondisk_log_size":5460,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":357826560,"num_objects":5460,"num_object_clones":0,"num_object_copies":16380,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5460,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5460,"num_write_kb":349440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.7","version":"136'5540","reported_seq":5565,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.948059+0000","last_change":"2023-08-28T02:22:00.165614+0000","last_active":"2023-08-28T02:23:06.948059+0000","last_peered":"2023-08-28T02:23:06.948059+0000","last_clean":"2023-08-28T02:23:06.948059+0000","last_became_active":"2023-08-28T02:22:00.165136+0000","last_became_peered":"2023-08-28T02:22:00.165136+0000","last_unstale":"2023-08-28T02:23:06.948059+0000","last_undegraded":"2023-08-28T02:23:06.948059+0000","last_fullsized":"2023-08-28T02:23:06.948059+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":5540,"log_dups_size":0,"ondisk_log_size":5540,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363069440,"num_objects":5540,"num_object_clones":0,"num_object_copies":16620,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5540,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5540,"num_write_kb":354560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"78'148","reported_seq":681,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.345808+0000","last_change":"2023-08-28T02:19:50.221311+0000","last_active":"2023-08-28T02:23:01.345808+0000","last_peered":"2023-08-28T02:23:01.345808+0000","last_clean":"2023-08-28T02:23:01.345808+0000","last_became_active":"2023-08-28T02:19:50.220754+0000","last_became_peered":"2023-08-28T02:19:50.220754+0000","last_unstale":"2023-08-28T02:23:01.345808+0000","last_undegraded":"2023-08-28T02:23:01.345808+0000","last_fullsized":"2023-08-28T02:23:01.345808+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1092,"num_read_kb":1955,"num_write":127,"num_write_kb":5177,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":93,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.289545+0000","last_change":"2023-08-28T02:21:59.180701+0000","last_active":"2023-08-28T02:23:01.289545+0000","last_peered":"2023-08-28T02:23:01.289545+0000","last_clean":"2023-08-28T02:23:01.289545+0000","last_became_active":"2023-08-28T02:21:59.180389+0000","last_became_peered":"2023-08-28T02:21:59.180389+0000","last_unstale":"2023-08-28T02:23:01.289545+0000","last_undegraded":"2023-08-28T02:23:01.289545+0000","last_fullsized":"2023-08-28T02:23:01.289545+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:36:31.599391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":129,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.381058+0000","last_change":"2023-08-28T02:21:59.167858+0000","last_active":"2023-08-28T02:23:01.381058+0000","last_peered":"2023-08-28T02:23:01.381058+0000","last_clean":"2023-08-28T02:23:01.381058+0000","last_became_active":"2023-08-28T02:21:59.167436+0000","last_became_peered":"2023-08-28T02:21:59.167436+0000","last_unstale":"2023-08-28T02:23:01.381058+0000","last_undegraded":"2023-08-28T02:23:01.381058+0000","last_fullsized":"2023-08-28T02:23:01.381058+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:17:50.717282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"78'103","reported_seq":557,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.346348+0000","last_change":"2023-08-28T02:18:39.786876+0000","last_active":"2023-08-28T02:23:01.346348+0000","last_peered":"2023-08-28T02:23:01.346348+0000","last_clean":"2023-08-28T02:23:01.346348+0000","last_became_active":"2023-08-28T02:18:39.786414+0000","last_became_peered":"2023-08-28T02:18:39.786414+0000","last_unstale":"2023-08-28T02:23:01.346348+0000","last_undegraded":"2023-08-28T02:23:01.346348+0000","last_fullsized":"2023-08-28T02:23:01.346348+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":803,"num_read_kb":2080,"num_write":73,"num_write_kb":1244,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":75,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.380185+0000","last_change":"2023-08-28T02:20:19.850916+0000","last_active":"2023-08-28T02:23:01.380185+0000","last_peered":"2023-08-28T02:23:01.380185+0000","last_clean":"2023-08-28T02:23:01.380185+0000","last_became_active":"2023-08-28T02:20:19.850627+0000","last_became_peered":"2023-08-28T02:20:19.850627+0000","last_unstale":"2023-08-28T02:23:01.380185+0000","last_undegraded":"2023-08-28T02:23:01.380185+0000","last_fullsized":"2023-08-28T02:23:01.380185+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:19:14.346189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":70,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.381464+0000","last_change":"2023-08-28T02:18:39.780069+0000","last_active":"2023-08-28T02:23:01.381464+0000","last_peered":"2023-08-28T02:23:01.381464+0000","last_clean":"2023-08-28T02:23:01.381464+0000","last_became_active":"2023-08-28T02:18:39.779829+0000","last_became_peered":"2023-08-28T02:18:39.779829+0000","last_unstale":"2023-08-28T02:23:01.381464+0000","last_undegraded":"2023-08-28T02:23:01.381464+0000","last_fullsized":"2023-08-28T02:23:01.381464+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:35:01.309659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"136'5387","reported_seq":5411,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.973809+0000","last_change":"2023-08-28T02:22:00.167814+0000","last_active":"2023-08-28T02:23:06.973809+0000","last_peered":"2023-08-28T02:23:06.973809+0000","last_clean":"2023-08-28T02:23:06.973809+0000","last_became_active":"2023-08-28T02:22:00.167513+0000","last_became_peered":"2023-08-28T02:22:00.167513+0000","last_unstale":"2023-08-28T02:23:06.973809+0000","last_undegraded":"2023-08-28T02:23:06.973809+0000","last_fullsized":"2023-08-28T02:23:06.973809+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":5387,"log_dups_size":0,"ondisk_log_size":5387,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":353042432,"num_objects":5387,"num_object_clones":0,"num_object_copies":16161,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5387,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5387,"num_write_kb":344768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":56,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.345960+0000","last_change":"2023-08-28T02:21:59.322116+0000","last_active":"2023-08-28T02:23:01.345960+0000","last_peered":"2023-08-28T02:23:01.345960+0000","last_clean":"2023-08-28T02:23:01.345960+0000","last_became_active":"2023-08-28T02:21:59.175477+0000","last_became_peered":"2023-08-28T02:21:59.175477+0000","last_unstale":"2023-08-28T02:23:01.345960+0000","last_undegraded":"2023-08-28T02:23:01.345960+0000","last_fullsized":"2023-08-28T02:23:01.345960+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"78'112","reported_seq":379,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.380353+0000","last_change":"2023-08-28T02:19:50.218206+0000","last_active":"2023-08-28T02:23:01.380353+0000","last_peered":"2023-08-28T02:23:01.380353+0000","last_clean":"2023-08-28T02:23:01.380353+0000","last_became_active":"2023-08-28T02:19:50.217637+0000","last_became_peered":"2023-08-28T02:19:50.217637+0000","last_unstale":"2023-08-28T02:23:01.380353+0000","last_undegraded":"2023-08-28T02:23:01.380353+0000","last_fullsized":"2023-08-28T02:23:01.380353+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:12:06.224907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":444,"num_read_kb":473,"num_write":94,"num_write_kb":127,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":89,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.345520+0000","last_change":"2023-08-28T02:21:59.176497+0000","last_active":"2023-08-28T02:23:01.345520+0000","last_peered":"2023-08-28T02:23:01.345520+0000","last_clean":"2023-08-28T02:23:01.345520+0000","last_became_active":"2023-08-28T02:21:59.176146+0000","last_became_peered":"2023-08-28T02:21:59.176146+0000","last_unstale":"2023-08-28T02:23:01.345520+0000","last_undegraded":"2023-08-28T02:23:01.345520+0000","last_fullsized":"2023-08-28T02:23:01.345520+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.1","version":"136'5508","reported_seq":5533,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.983261+0000","last_change":"2023-08-28T02:22:00.167365+0000","last_active":"2023-08-28T02:23:06.983261+0000","last_peered":"2023-08-28T02:23:06.983261+0000","last_clean":"2023-08-28T02:23:06.983261+0000","last_became_active":"2023-08-28T02:22:00.167035+0000","last_became_peered":"2023-08-28T02:22:00.167035+0000","last_unstale":"2023-08-28T02:23:06.983261+0000","last_undegraded":"2023-08-28T02:23:06.983261+0000","last_fullsized":"2023-08-28T02:23:06.983261+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":5508,"log_dups_size":0,"ondisk_log_size":5508,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":360972288,"num_objects":5508,"num_object_clones":0,"num_object_copies":16524,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5508,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5508,"num_write_kb":352512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"78'122","reported_seq":591,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.348218+0000","last_change":"2023-08-28T02:20:19.851065+0000","last_active":"2023-08-28T02:23:01.348218+0000","last_peered":"2023-08-28T02:23:01.348218+0000","last_clean":"2023-08-28T02:23:01.348218+0000","last_became_active":"2023-08-28T02:20:19.850477+0000","last_became_peered":"2023-08-28T02:20:19.850477+0000","last_unstale":"2023-08-28T02:23:01.348218+0000","last_undegraded":"2023-08-28T02:23:01.348218+0000","last_fullsized":"2023-08-28T02:23:01.348218+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":857,"num_read_kb":2224,"num_write":99,"num_write_kb":2312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":88,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.380976+0000","last_change":"2023-08-28T02:19:50.221001+0000","last_active":"2023-08-28T02:23:01.380976+0000","last_peered":"2023-08-28T02:23:01.380976+0000","last_clean":"2023-08-28T02:23:01.380976+0000","last_became_active":"2023-08-28T02:19:50.220611+0000","last_became_peered":"2023-08-28T02:19:50.220611+0000","last_unstale":"2023-08-28T02:23:01.380976+0000","last_undegraded":"2023-08-28T02:23:01.380976+0000","last_fullsized":"2023-08-28T02:23:01.380976+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:23:10.637630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":87,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.345675+0000","last_change":"2023-08-28T02:11:22.437146+0000","last_active":"2023-08-28T02:23:01.345675+0000","last_peered":"2023-08-28T02:23:01.345675+0000","last_clean":"2023-08-28T02:23:01.345675+0000","last_became_active":"2023-08-28T02:11:22.436798+0000","last_became_peered":"2023-08-28T02:11:22.436798+0000","last_unstale":"2023-08-28T02:23:01.345675+0000","last_undegraded":"2023-08-28T02:23:01.345675+0000","last_fullsized":"2023-08-28T02:23:01.345675+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"136'5636","reported_seq":5662,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:08.262791+0000","last_change":"2023-08-28T02:22:00.163491+0000","last_active":"2023-08-28T02:23:08.262791+0000","last_peered":"2023-08-28T02:23:08.262791+0000","last_clean":"2023-08-28T02:23:08.262791+0000","last_became_active":"2023-08-28T02:22:00.163135+0000","last_became_peered":"2023-08-28T02:22:00.163135+0000","last_unstale":"2023-08-28T02:23:08.262791+0000","last_undegraded":"2023-08-28T02:23:08.262791+0000","last_fullsized":"2023-08-28T02:23:08.262791+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":5636,"log_dups_size":0,"ondisk_log_size":5636,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:52:26.563181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":369360896,"num_objects":5636,"num_object_clones":0,"num_object_copies":16908,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5636,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5636,"num_write_kb":360704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":72,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.381044+0000","last_change":"2023-08-28T02:18:39.781312+0000","last_active":"2023-08-28T02:23:01.381044+0000","last_peered":"2023-08-28T02:23:01.381044+0000","last_clean":"2023-08-28T02:23:01.381044+0000","last_became_active":"2023-08-28T02:18:39.780916+0000","last_became_peered":"2023-08-28T02:18:39.780916+0000","last_unstale":"2023-08-28T02:23:01.381044+0000","last_undegraded":"2023-08-28T02:23:01.381044+0000","last_fullsized":"2023-08-28T02:23:01.381044+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:37:11.571250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":147,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.386674+0000","last_change":"2023-08-28T02:21:59.168665+0000","last_active":"2023-08-28T02:23:01.386674+0000","last_peered":"2023-08-28T02:23:01.386674+0000","last_clean":"2023-08-28T02:23:01.386674+0000","last_became_active":"2023-08-28T02:21:59.168430+0000","last_became_peered":"2023-08-28T02:21:59.168430+0000","last_unstale":"2023-08-28T02:23:01.386674+0000","last_undegraded":"2023-08-28T02:23:01.386674+0000","last_fullsized":"2023-08-28T02:23:01.386674+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:06:16.535160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":105,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.347336+0000","last_change":"2023-08-28T02:19:50.218760+0000","last_active":"2023-08-28T02:23:01.347336+0000","last_peered":"2023-08-28T02:23:01.347336+0000","last_clean":"2023-08-28T02:23:01.347336+0000","last_became_active":"2023-08-28T02:19:50.218376+0000","last_became_peered":"2023-08-28T02:19:50.218376+0000","last_unstale":"2023-08-28T02:23:01.347336+0000","last_undegraded":"2023-08-28T02:23:01.347336+0000","last_fullsized":"2023-08-28T02:23:01.347336+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":126,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.346302+0000","last_change":"2023-08-28T02:21:59.177791+0000","last_active":"2023-08-28T02:23:01.346302+0000","last_peered":"2023-08-28T02:23:01.346302+0000","last_clean":"2023-08-28T02:23:01.346302+0000","last_became_active":"2023-08-28T02:21:59.177436+0000","last_became_peered":"2023-08-28T02:21:59.177436+0000","last_unstale":"2023-08-28T02:23:01.346302+0000","last_undegraded":"2023-08-28T02:23:01.346302+0000","last_fullsized":"2023-08-28T02:23:01.346302+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":90,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.347524+0000","last_change":"2023-08-28T02:11:22.186024+0000","last_active":"2023-08-28T02:23:01.347524+0000","last_peered":"2023-08-28T02:23:01.347524+0000","last_clean":"2023-08-28T02:23:01.347524+0000","last_became_active":"2023-08-28T02:11:22.185500+0000","last_became_peered":"2023-08-28T02:11:22.185500+0000","last_unstale":"2023-08-28T02:23:01.347524+0000","last_undegraded":"2023-08-28T02:23:01.347524+0000","last_fullsized":"2023-08-28T02:23:01.347524+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":126,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.346715+0000","last_change":"2023-08-28T02:21:59.187663+0000","last_active":"2023-08-28T02:23:01.346715+0000","last_peered":"2023-08-28T02:23:01.346715+0000","last_clean":"2023-08-28T02:23:01.346715+0000","last_became_active":"2023-08-28T02:21:59.187366+0000","last_became_peered":"2023-08-28T02:21:59.187366+0000","last_unstale":"2023-08-28T02:23:01.346715+0000","last_undegraded":"2023-08-28T02:23:01.346715+0000","last_fullsized":"2023-08-28T02:23:01.346715+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":40,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.348224+0000","last_change":"2023-08-28T02:21:59.179758+0000","last_active":"2023-08-28T02:23:01.348224+0000","last_peered":"2023-08-28T02:23:01.348224+0000","last_clean":"2023-08-28T02:23:01.348224+0000","last_became_active":"2023-08-28T02:21:59.179436+0000","last_became_peered":"2023-08-28T02:21:59.179436+0000","last_unstale":"2023-08-28T02:23:01.348224+0000","last_undegraded":"2023-08-28T02:23:01.348224+0000","last_fullsized":"2023-08-28T02:23:01.348224+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":116,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.136785+0000","last_change":"2023-08-28T02:21:59.171738+0000","last_active":"2023-08-28T02:23:06.136785+0000","last_peered":"2023-08-28T02:23:06.136785+0000","last_clean":"2023-08-28T02:23:06.136785+0000","last_became_active":"2023-08-28T02:21:59.171422+0000","last_became_peered":"2023-08-28T02:21:59.171422+0000","last_unstale":"2023-08-28T02:23:06.136785+0000","last_undegraded":"2023-08-28T02:23:06.136785+0000","last_fullsized":"2023-08-28T02:23:06.136785+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:58:46.090765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":88,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.380702+0000","last_change":"2023-08-28T02:20:19.850252+0000","last_active":"2023-08-28T02:23:01.380702+0000","last_peered":"2023-08-28T02:23:01.380702+0000","last_clean":"2023-08-28T02:23:01.380702+0000","last_became_active":"2023-08-28T02:20:19.849913+0000","last_became_peered":"2023-08-28T02:20:19.849913+0000","last_unstale":"2023-08-28T02:23:01.380702+0000","last_undegraded":"2023-08-28T02:23:01.380702+0000","last_fullsized":"2023-08-28T02:23:01.380702+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:48:47.713465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":131,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-08-28T02:23:01.345640+0000","last_change":"2023-08-28T02:21:59.180201+0000","last_active":"2023-08-28T02:23:01.345640+0000","last_peered":"2023-08-28T02:23:01.345640+0000","last_clean":"2023-08-28T02:23:01.345640+0000","last_became_active":"2023-08-28T02:21:59.179863+0000","last_became_peered":"2023-08-28T02:21:59.179863+0000","last_unstale":"2023-08-28T02:23:01.345640+0000","last_undegraded":"2023-08-28T02:23:01.345640+0000","last_fullsized":"2023-08-28T02:23:01.345640+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":5798690816,"num_objects":88481,"num_object_clones":0,"num_object_copies":265443,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":88481,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":88481,"num_write_kb":5662784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17377853440,"data_stored":17377853440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":88481,"ondisk_log_size":88481,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":112,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16602,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26810,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430370,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2049072,"kb_used_data":1739496,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":91696080,"statfs":{"total":95995035648,"available":93896785920,"internally_reserved":0,"allocated":1781243904,"data_stored":1780721519,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:22:49 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.53900000000000003,"15min":0.53700000000000003},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":1.9370000000000001,"5min":1.9370000000000001,"15min":1.9370000000000001},"last":2.2749999999999999},{"interface":"front","average":{"1min":1.0009999999999999,"5min":0.65900000000000003,"15min":0.628},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":4.4710000000000001,"5min":4.4710000000000001,"15min":4.4710000000000001},"last":0.50600000000000001}]},{"osd":1,"last update":"Mon Aug 28 02:22:55 2023","interfaces":[{"interface":"back","average":{"1min":1.123,"5min":0.65200000000000002,"15min":0.56899999999999995},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":4.306,"5min":4.306,"15min":4.306},"last":0.26300000000000001},{"interface":"front","average":{"1min":1.0449999999999999,"5min":0.67500000000000004,"15min":0.61699999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":3.3460000000000001,"5min":3.3460000000000001,"15min":3.3460000000000001},"last":1.887}]},{"osd":2,"last update":"Mon Aug 28 02:22:22 2023","interfaces":[{"interface":"back","average":{"1min":0.72699999999999998,"5min":0.60499999999999998,"15min":0.58499999999999996},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":2.0489999999999999,"5min":2.0489999999999999,"15min":2.0489999999999999},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.79100000000000004,"5min":0.627,"15min":0.59899999999999998},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":3.1150000000000002,"5min":3.1150000000000002,"15min":3.1150000000000002},"last":0.56699999999999995}]},{"osd":3,"last update":"Mon Aug 28 02:22:55 2023","interfaces":[{"interface":"back","average":{"1min":1.1060000000000001,"5min":0.75600000000000001,"15min":0.69799999999999995},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.367,"5min":4.367,"15min":4.367},"last":0.41399999999999998},{"interface":"front","average":{"1min":1.0309999999999999,"5min":0.74199999999999999,"15min":0.69399999999999995},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":3.3159999999999998,"5min":3.3159999999999998,"15min":3.3159999999999998},"last":0.371}]},{"osd":4,"last update":"Mon Aug 28 02:22:22 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.51600000000000001,"15min":0.48599999999999999},"min":{"1min":0.28299999999999997,"5min":0.27700000000000002,"15min":0.25800000000000001},"max":{"1min":1.8779999999999999,"5min":1.8779999999999999,"15min":1.8779999999999999},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.53700000000000003,"15min":0.51000000000000001},"min":{"1min":0.316,"5min":0.29799999999999999,"15min":0.27100000000000002},"max":{"1min":1.8899999999999999,"5min":1.8899999999999999,"15min":1.8899999999999999},"last":1.466}]},{"osd":5,"last update":"Mon Aug 28 02:22:22 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.53200000000000003,"15min":0.495},"min":{"1min":0.33500000000000002,"5min":0.29899999999999999,"15min":0.28699999999999998},"max":{"1min":1.637,"5min":1.637,"15min":1.637},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.92300000000000004,"5min":0.65000000000000002,"15min":0.57099999999999995},"min":{"1min":0.35599999999999998,"5min":0.32200000000000001,"15min":0.314},"max":{"1min":3.1509999999999998,"5min":3.1509999999999998,"15min":3.1509999999999998},"last":0.53800000000000003}]},{"osd":6,"last update":"Mon Aug 28 02:22:22 2023","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.58399999999999996,"15min":0.54300000000000004},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.335,"5min":2.335,"15min":2.335},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.73099999999999998,"5min":0.64600000000000002,"15min":0.59499999999999997},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":1.8280000000000001,"5min":1.8280000000000001,"15min":1.8280000000000001},"last":0.46300000000000002}]}]},{"osd":6,"up_from":42,"seq":180388626597,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1702700,"kb_used_data":1393188,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92042452,"statfs":{"total":95995035648,"available":94251470848,"internally_reserved":0,"allocated":1426624512,"data_stored":1426113762,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:22:57 2023","interfaces":[{"interface":"back","average":{"1min":1.5589999999999999,"5min":0.75900000000000001,"15min":0.63600000000000001},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":7.4790000000000001,"5min":7.4790000000000001,"15min":7.4790000000000001},"last":0.27000000000000002},{"interface":"front","average":{"1min":0.83899999999999997,"5min":0.626,"15min":0.60999999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":3.1819999999999999,"5min":3.1819999999999999,"15min":3.1819999999999999},"last":0.42699999999999999}]},{"osd":1,"last update":"Mon Aug 28 02:22:57 2023","interfaces":[{"interface":"back","average":{"1min":3.8769999999999998,"5min":1.2230000000000001,"15min":0.76500000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":34.677999999999997,"5min":34.677999999999997,"15min":34.677999999999997},"last":0.30099999999999999},{"interface":"front","average":{"1min":3.9540000000000002,"5min":1.2609999999999999,"15min":0.80000000000000004},"min":{"1min":0.33300000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":34.348999999999997,"5min":34.348999999999997,"15min":34.348999999999997},"last":0.46400000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:22:26 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.59999999999999998,"15min":0.60599999999999998},"min":{"1min":0.39000000000000001,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.5349999999999999,"5min":1.575,"15min":1.575},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.56399999999999995,"15min":0.57199999999999995},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.89800000000000002,"5min":1.6419999999999999,"15min":1.6419999999999999},"last":1.1879999999999999}]},{"osd":3,"last update":"Mon Aug 28 02:22:57 2023","interfaces":[{"interface":"back","average":{"1min":0.997,"5min":0.73599999999999999,"15min":0.69199999999999995},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":4.2930000000000001,"5min":4.2930000000000001,"15min":4.2930000000000001},"last":0.52100000000000002},{"interface":"front","average":{"1min":1.546,"5min":0.85499999999999998,"15min":0.73999999999999999},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":7.9610000000000003,"5min":7.9610000000000003,"15min":7.9610000000000003},"last":0.51100000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:23:01 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":0.59499999999999997,"15min":0.496},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":4.3540000000000001,"5min":4.3540000000000001,"15min":4.3540000000000001},"last":0.78600000000000003},{"interface":"front","average":{"1min":1.093,"5min":0.65100000000000002,"15min":0.53100000000000003},"min":{"1min":0.32200000000000001,"5min":0.318,"15min":0.28899999999999998},"max":{"1min":6.1420000000000003,"5min":6.1420000000000003,"15min":6.1420000000000003},"last":0.54700000000000004}]},{"osd":5,"last update":"Mon Aug 28 02:23:01 2023","interfaces":[{"interface":"back","average":{"1min":1.865,"5min":0.77700000000000002,"15min":0.56000000000000005},"min":{"1min":0.30099999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":8.8330000000000002,"5min":8.8330000000000002,"15min":8.8330000000000002},"last":1.228},{"interface":"front","average":{"1min":1.2949999999999999,"5min":0.71299999999999997,"15min":0.57599999999999996},"min":{"1min":0.34300000000000003,"5min":0.32500000000000001,"15min":0.28799999999999998},"max":{"1min":7.3780000000000001,"5min":7.3780000000000001,"15min":7.3780000000000001},"last":0.58999999999999997}]},{"osd":7,"last update":"Mon Aug 28 02:22:19 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.59599999999999997,"15min":0.55800000000000005},"min":{"1min":0.38,"5min":0.36799999999999999,"15min":0.32700000000000001},"max":{"1min":1.272,"5min":1.272,"15min":1.272},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.94599999999999995,"5min":0.67700000000000005,"15min":0.59999999999999998},"min":{"1min":0.41399999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":4.9210000000000003,"5min":4.9210000000000003,"15min":4.9210000000000003},"last":0.59699999999999998}]}]},{"osd":1,"up_from":98,"seq":420906795050,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1835664,"kb_used_data":1790024,"kb_used_omap":7,"kb_used_meta":45624,"kb_avail":91909488,"statfs":{"total":95995035648,"available":94115315712,"internally_reserved":0,"allocated":1832984576,"data_stored":1832332903,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":46719307},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:22:55 2023","interfaces":[{"interface":"back","average":{"1min":1.4650000000000001,"5min":0.55700000000000005,"15min":0.40899999999999997},"min":{"1min":0.24199999999999999,"5min":0.222,"15min":0.222},"max":{"1min":6.5990000000000002,"5min":6.5990000000000002,"15min":6.5990000000000002},"last":4.5730000000000004},{"interface":"front","average":{"1min":2.2869999999999999,"5min":0.73699999999999999,"15min":0.48099999999999998},"min":{"1min":0.25700000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":12.574999999999999,"5min":12.574999999999999,"15min":12.574999999999999},"last":5.8579999999999997}]},{"osd":2,"last update":"Mon Aug 28 02:22:25 2023","interfaces":[{"interface":"back","average":{"1min":0.70799999999999996,"5min":0.41499999999999998,"15min":0.36599999999999999},"min":{"1min":0.26300000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":2.2709999999999999,"5min":2.2709999999999999,"15min":2.2709999999999999},"last":2.0760000000000001},{"interface":"front","average":{"1min":0.82199999999999995,"5min":0.46899999999999997,"15min":0.40999999999999998},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":9.2509999999999994,"5min":9.2509999999999994,"15min":9.2509999999999994},"last":4.548}]},{"osd":3,"last update":"Mon Aug 28 02:22:55 2023","interfaces":[{"interface":"back","average":{"1min":1.647,"5min":0.621,"15min":0.45000000000000001},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":12.544,"5min":12.544,"15min":12.544},"last":5.7949999999999999},{"interface":"front","average":{"1min":1.6160000000000001,"5min":0.69199999999999995,"15min":0.53800000000000003},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":8.9160000000000004,"5min":8.9160000000000004,"15min":8.9160000000000004},"last":2.1080000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:22:55 2023","interfaces":[{"interface":"back","average":{"1min":1.1259999999999999,"5min":0.73899999999999999,"15min":0.66400000000000003},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":5.8529999999999998,"5min":5.8529999999999998,"15min":5.8529999999999998},"last":2.1800000000000002},{"interface":"front","average":{"1min":0.90700000000000003,"5min":0.67300000000000004,"15min":0.63700000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":3.0499999999999998,"5min":3.0499999999999998,"15min":3.0499999999999998},"last":4.5049999999999999}]},{"osd":5,"last update":"Mon Aug 28 02:22:55 2023","interfaces":[{"interface":"back","average":{"1min":1.76,"5min":0.85299999999999998,"15min":0.70899999999999996},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":9.4469999999999992,"5min":9.4469999999999992,"15min":9.4469999999999992},"last":2.403},{"interface":"front","average":{"1min":1.2230000000000001,"5min":0.76400000000000001,"15min":0.68200000000000005},"min":{"1min":0.36699999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":4.6639999999999997,"5min":4.6639999999999997,"15min":4.6639999999999997},"last":2.234}]},{"osd":6,"last update":"Mon Aug 28 02:22:55 2023","interfaces":[{"interface":"back","average":{"1min":2.5870000000000002,"5min":1.0009999999999999,"15min":0.746},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":14.827,"5min":14.827,"15min":14.827},"last":2.4470000000000001},{"interface":"front","average":{"1min":1.1919999999999999,"5min":0.73799999999999999,"15min":0.65500000000000003},"min":{"1min":0.39000000000000001,"5min":0.371,"15min":0.371},"max":{"1min":4.468,"5min":4.468,"15min":4.468},"last":2.266}]},{"osd":7,"last update":"Mon Aug 28 02:22:55 2023","interfaces":[{"interface":"back","average":{"1min":0.94799999999999995,"5min":0.68700000000000006,"15min":0.65100000000000002},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":3.028,"5min":3.028,"15min":3.028},"last":4.5940000000000003},{"interface":"front","average":{"1min":1.748,"5min":0.85499999999999998,"15min":0.70999999999999996},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":9.5039999999999996,"5min":9.5039999999999996,"15min":9.5039999999999996},"last":2.4750000000000001}]}]},{"osd":0,"up_from":90,"seq":386547056696,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3986712,"kb_used_data":3888592,"kb_used_omap":8,"kb_used_meta":98103,"kb_avail":89758440,"statfs":{"total":95995035648,"available":91912642560,"internally_reserved":0,"allocated":3981918208,"data_stored":3981259077,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":100457833},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Aug 28 02:22:57 2023","interfaces":[{"interface":"back","average":{"1min":1.8200000000000001,"5min":0.64700000000000002,"15min":0.45100000000000001},"min":{"1min":0.186,"5min":0.186,"15min":0.186},"max":{"1min":11.115,"5min":11.115,"15min":11.115},"last":10.069000000000001},{"interface":"front","average":{"1min":0.97199999999999998,"5min":0.48299999999999998,"15min":0.40899999999999997},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":4.7999999999999998,"5min":4.7999999999999998,"15min":4.7999999999999998},"last":6.077}]},{"osd":2,"last update":"Mon Aug 28 02:22:21 2023","interfaces":[{"interface":"back","average":{"1min":0.80600000000000005,"5min":0.443,"15min":0.38300000000000001},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":3.7839999999999998,"5min":3.7839999999999998,"15min":3.7839999999999998},"last":6.5449999999999999},{"interface":"front","average":{"1min":1.069,"5min":0.52700000000000002,"15min":0.437},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":5.6849999999999996,"5min":5.6849999999999996,"15min":5.6849999999999996},"last":6.1029999999999998}]},{"osd":3,"last update":"Mon Aug 28 02:22:57 2023","interfaces":[{"interface":"back","average":{"1min":1.5,"5min":0.58999999999999997,"15min":0.438},"min":{"1min":0.26500000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":10.443,"5min":10.443,"15min":10.443},"last":1.1870000000000001},{"interface":"front","average":{"1min":1.508,"5min":0.61599999999999999,"15min":0.46800000000000003},"min":{"1min":0.29799999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":10.548999999999999,"5min":10.548999999999999,"15min":10.548999999999999},"last":1.2210000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:22:53 2023","interfaces":[{"interface":"back","average":{"1min":1.2150000000000001,"5min":0.75600000000000001,"15min":0.69799999999999995},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":6.1180000000000003,"5min":6.1180000000000003,"15min":6.1180000000000003},"last":6.657},{"interface":"front","average":{"1min":1.645,"5min":0.86099999999999999,"15min":0.75800000000000001},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":10.106999999999999,"5min":10.106999999999999,"15min":10.106999999999999},"last":1.248}]},{"osd":5,"last update":"Mon Aug 28 02:22:53 2023","interfaces":[{"interface":"back","average":{"1min":1.758,"5min":0.84099999999999997,"15min":0.71599999999999997},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":9.9209999999999994,"5min":9.9209999999999994,"15min":9.9209999999999994},"last":1.2749999999999999},{"interface":"front","average":{"1min":1.5109999999999999,"5min":0.80300000000000005,"15min":0.70299999999999996},"min":{"1min":0.373,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":5.0179999999999998,"5min":5.0179999999999998,"15min":5.0179999999999998},"last":6.593}]},{"osd":6,"last update":"Mon Aug 28 02:22:53 2023","interfaces":[{"interface":"back","average":{"1min":1.8580000000000001,"5min":0.87,"15min":0.72699999999999998},"min":{"1min":0.36299999999999999,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":10.073,"5min":10.073,"15min":10.073},"last":1.7529999999999999},{"interface":"front","average":{"1min":1.1599999999999999,"5min":0.77600000000000002,"15min":0.72999999999999998},"min":{"1min":0.39900000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":3.7160000000000002,"5min":3.7160000000000002,"15min":3.7160000000000002},"last":6.6269999999999998}]},{"osd":7,"last update":"Mon Aug 28 02:22:53 2023","interfaces":[{"interface":"back","average":{"1min":1.446,"5min":0.82099999999999995,"15min":0.73999999999999999},"min":{"1min":0.41399999999999998,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":6.0700000000000003,"5min":6.0700000000000003,"15min":6.0700000000000003},"last":7.5430000000000001},{"interface":"front","average":{"1min":0.88400000000000001,"5min":0.70699999999999996,"15min":0.70499999999999996},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":2.762,"5min":2.762,"15min":2.762},"last":6.0419999999999998}]}]},{"osd":2,"up_from":104,"seq":446676598820,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2181852,"kb_used_data":2110036,"kb_used_omap":10,"kb_used_meta":71797,"kb_avail":91563300,"statfs":{"total":95995035648,"available":93760819200,"internally_reserved":0,"allocated":2160676864,"data_stored":2160068555,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10792,"internal_metadata":73520600},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:22:25 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":0.39200000000000002,"15min":0.34399999999999997},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":4.149,"5min":4.149,"15min":4.149},"last":9.9390000000000001},{"interface":"front","average":{"1min":0.81599999999999995,"5min":0.45400000000000001,"15min":0.39400000000000002},"min":{"1min":0.246,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":5.1289999999999996,"5min":5.1289999999999996,"15min":5.1289999999999996},"last":0.48099999999999998}]},{"osd":1,"last update":"Mon Aug 28 02:22:25 2023","interfaces":[{"interface":"back","average":{"1min":2.3780000000000001,"5min":0.76200000000000001,"15min":0.49299999999999999},"min":{"1min":0.28199999999999997,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":30.888000000000002,"5min":30.888000000000002,"15min":30.888000000000002},"last":0.52400000000000002},{"interface":"front","average":{"1min":2.2149999999999999,"5min":0.70999999999999996,"15min":0.45900000000000002},"min":{"1min":0.27100000000000002,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":31.648,"5min":31.648,"15min":31.648},"last":0.623}]},{"osd":3,"last update":"Mon Aug 28 02:22:53 2023","interfaces":[{"interface":"back","average":{"1min":1.2270000000000001,"5min":0.57099999999999995,"15min":0.46200000000000002},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.9969999999999999,"5min":4.9969999999999999,"15min":4.9969999999999999},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.70399999999999996,"5min":0.45000000000000001,"15min":0.40799999999999997},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":2.3999999999999999,"5min":2.3999999999999999,"15min":2.3999999999999999},"last":0.69199999999999995}]},{"osd":4,"last update":"Mon Aug 28 02:22:25 2023","interfaces":[{"interface":"back","average":{"1min":0.79300000000000004,"5min":0.63300000000000001,"15min":0.60699999999999998},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":2.492,"5min":2.492,"15min":2.492},"last":0.72999999999999998},{"interface":"front","average":{"1min":1.069,"5min":0.73299999999999998,"15min":0.67700000000000005},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":4.1840000000000002,"5min":4.1840000000000002,"15min":4.1840000000000002},"last":0.83699999999999997}]},{"osd":5,"last update":"Mon Aug 28 02:22:25 2023","interfaces":[{"interface":"back","average":{"1min":0.93899999999999995,"5min":0.68100000000000005,"15min":0.63800000000000001},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":5.0880000000000001,"5min":5.0880000000000001,"15min":5.0880000000000001},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.70099999999999996,"5min":0.58699999999999997,"15min":0.56799999999999995},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":2.5430000000000001,"5min":2.5430000000000001,"15min":2.5430000000000001},"last":0.753}]},{"osd":6,"last update":"Mon Aug 28 02:22:25 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":0.67400000000000004,"15min":0.625},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":4.2279999999999998,"5min":4.2279999999999998,"15min":4.2279999999999998},"last":0.80500000000000005},{"interface":"front","average":{"1min":1.0529999999999999,"5min":0.70299999999999996,"15min":0.64500000000000002},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":5.1559999999999997,"5min":5.1559999999999997,"15min":5.1559999999999997},"last":0.65000000000000002}]},{"osd":7,"last update":"Mon Aug 28 02:22:25 2023","interfaces":[{"interface":"back","average":{"1min":1.0129999999999999,"5min":0.68200000000000005,"15min":0.627},"min":{"1min":0.41099999999999998,"5min":0.372,"15min":0.372},"max":{"1min":4.0670000000000002,"5min":4.0670000000000002,"15min":4.0670000000000002},"last":0.86199999999999999},{"interface":"front","average":{"1min":1.0940000000000001,"5min":0.70199999999999996,"15min":0.63700000000000001},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":5.181,"5min":5.181,"15min":5.181},"last":0.60599999999999998}]}]},{"osd":3,"up_from":0,"seq":481036337182,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019055,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3116516,"kb_used_data":2807004,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":90628636,"statfs":{"total":95995035648,"available":92803723264,"internally_reserved":0,"allocated":2874372096,"data_stored":2873861363,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:22:49 2023","interfaces":[{"interface":"back","average":{"1min":0.751,"5min":0.60899999999999999,"15min":0.61399999999999999},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":5.5039999999999996,"5min":5.5039999999999996,"15min":5.5039999999999996},"last":1.74},{"interface":"front","average":{"1min":1.0589999999999999,"5min":0.65200000000000002,"15min":0.61699999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":3.0299999999999998,"5min":3.0299999999999998,"15min":3.0299999999999998},"last":0.38100000000000001}]},{"osd":1,"last update":"Mon Aug 28 02:22:54 2023","interfaces":[{"interface":"back","average":{"1min":1.244,"5min":0.66300000000000003,"15min":0.56899999999999995},"min":{"1min":0.28100000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.5670000000000002,"5min":5.5670000000000002,"15min":5.5670000000000002},"last":1.806},{"interface":"front","average":{"1min":1.002,"5min":0.64200000000000002,"15min":0.58999999999999997},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":2.7850000000000001,"5min":2.7850000000000001,"15min":2.7850000000000001},"last":1.2909999999999999}]},{"osd":2,"last update":"Mon Aug 28 02:22:22 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.55700000000000005,"15min":0.55200000000000005},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.5629999999999999,"5min":1.5629999999999999,"15min":1.5629999999999999},"last":4.7279999999999998},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.56100000000000005,"15min":0.54700000000000004},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":1.478,"5min":1.478,"15min":1.478},"last":0.44700000000000001}]},{"osd":3,"last update":"Mon Aug 28 02:22:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":0.60799999999999998,"15min":0.53600000000000003},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":5.0970000000000004,"5min":5.0970000000000004,"15min":5.0970000000000004},"last":2.4489999999999998},{"interface":"front","average":{"1min":1.4330000000000001,"5min":0.753,"15min":0.64000000000000001},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":5.6529999999999996,"5min":5.6529999999999996,"15min":5.6529999999999996},"last":3.613}]},{"osd":5,"last update":"Mon Aug 28 02:22:36 2023","interfaces":[{"interface":"back","average":{"1min":0.86799999999999999,"5min":0.58099999999999996,"15min":0.48099999999999998},"min":{"1min":0.316,"5min":0.316,"15min":0.26600000000000001},"max":{"1min":4.5129999999999999,"5min":4.5129999999999999,"15min":4.5129999999999999},"last":0.40899999999999997},{"interface":"front","average":{"1min":1.0329999999999999,"5min":0.61399999999999999,"15min":0.504},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.27700000000000002},"max":{"1min":6.0899999999999999,"5min":6.0899999999999999,"15min":6.0899999999999999},"last":1.903}]},{"osd":6,"last update":"Mon Aug 28 02:22:54 2023","interfaces":[{"interface":"back","average":{"1min":1.262,"5min":0.69199999999999995,"15min":0.54700000000000004},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.27500000000000002},"max":{"1min":9.6920000000000002,"5min":9.6920000000000002,"15min":9.6920000000000002},"last":1.7829999999999999},{"interface":"front","average":{"1min":0.97999999999999998,"5min":0.67500000000000004,"15min":0.56399999999999995},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.318},"max":{"1min":2.4569999999999999,"5min":2.4569999999999999,"15min":2.4569999999999999},"last":0.42799999999999999}]},{"osd":7,"last update":"Mon Aug 28 02:22:19 2023","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":0.59999999999999998,"15min":0.54800000000000004},"min":{"1min":0.40799999999999997,"5min":0.36899999999999999,"15min":0.312},"max":{"1min":3.2320000000000002,"5min":3.2320000000000002,"15min":3.2320000000000002},"last":1.24},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.58999999999999997,"15min":0.54800000000000004},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32100000000000001},"max":{"1min":0.73999999999999999,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":6.9269999999999996}]}]},{"osd":5,"up_from":36,"seq":154618822827,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3456220,"kb_used_data":3154900,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":90288932,"statfs":{"total":95995035648,"available":92455866368,"internally_reserved":0,"allocated":3230617600,"data_stored":3230118195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:22:46 2023","interfaces":[{"interface":"back","average":{"1min":8.7870000000000008,"5min":2.2210000000000001,"15min":1.157},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":131.06100000000001,"5min":131.06100000000001,"15min":131.06100000000001},"last":3.1219999999999999},{"interface":"front","average":{"1min":8.9380000000000006,"5min":2.2650000000000001,"15min":1.173},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":131.262,"5min":131.262,"15min":131.262},"last":3.0830000000000002}]},{"osd":1,"last update":"Mon Aug 28 02:22:54 2023","interfaces":[{"interface":"back","average":{"1min":13.494,"5min":3.113,"15min":1.389},"min":{"1min":0.36899999999999999,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":183.97900000000001,"5min":183.97900000000001,"15min":183.97900000000001},"last":3.1389999999999998},{"interface":"front","average":{"1min":13.288,"5min":3.0830000000000002,"15min":1.3879999999999999},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":184.33699999999999,"5min":184.33699999999999,"15min":184.33699999999999},"last":3.4900000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:22:23 2023","interfaces":[{"interface":"back","average":{"1min":5.3550000000000004,"5min":1.5429999999999999,"15min":0.90700000000000003},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":72.944999999999993,"5min":72.944999999999993,"15min":72.944999999999993},"last":1.0449999999999999},{"interface":"front","average":{"1min":5.3710000000000004,"5min":1.544,"15min":0.90600000000000003},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":72.930999999999997,"5min":72.930999999999997,"15min":72.930999999999997},"last":1.411}]},{"osd":3,"last update":"Mon Aug 28 02:22:54 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":0.622,"15min":0.61499999999999999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":1.603,"5min":1.603,"15min":1.603},"last":3.637},{"interface":"front","average":{"1min":1.2210000000000001,"5min":0.745,"15min":0.66600000000000004},"min":{"1min":0.36199999999999999,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":3.3450000000000002,"5min":3.3450000000000002,"15min":3.3450000000000002},"last":1.5620000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:22:40 2023","interfaces":[{"interface":"back","average":{"1min":1.1140000000000001,"5min":0.623,"15min":0.50700000000000001},"min":{"1min":0.38400000000000001,"5min":0.32200000000000001,"15min":0.27500000000000002},"max":{"1min":4.2089999999999996,"5min":4.2089999999999996,"15min":4.2089999999999996},"last":1.5680000000000001},{"interface":"front","average":{"1min":0.96999999999999997,"5min":0.61299999999999999,"15min":0.52100000000000002},"min":{"1min":0.33200000000000002,"5min":0.31900000000000001,"15min":0.28199999999999997},"max":{"1min":4.0659999999999998,"5min":4.0659999999999998,"15min":4.0659999999999998},"last":0.99099999999999999}]},{"osd":6,"last update":"Mon Aug 28 02:22:54 2023","interfaces":[{"interface":"back","average":{"1min":1.7110000000000001,"5min":0.76600000000000001,"15min":0.60399999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.28399999999999997},"max":{"1min":4.8239999999999998,"5min":4.8239999999999998,"15min":5.2119999999999997},"last":1.54},{"interface":"front","average":{"1min":0.88500000000000001,"5min":0.64200000000000002,"15min":0.55200000000000005},"min":{"1min":0.31,"5min":0.31,"15min":0.30299999999999999},"max":{"1min":3.2999999999999998,"5min":3.2999999999999998,"15min":3.2999999999999998},"last":1.0640000000000001}]},{"osd":7,"last update":"Mon Aug 28 02:22:15 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.58999999999999997,"15min":0.55600000000000005},"min":{"1min":0.39600000000000002,"5min":0.35199999999999998,"15min":0.32900000000000001},"max":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":1.119},"last":1.022},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.57499999999999996,"15min":0.55300000000000005},"min":{"1min":0.39900000000000002,"5min":0.34999999999999998,"15min":0.29799999999999999},"max":{"1min":0.83799999999999997,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":1.361}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4013031424,"data_stored":4013031424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1838940160,"data_stored":1838940160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2176057344,"data_stored":2176057344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2897215488,"data_stored":2897215488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3232497664,"data_stored":3232497664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1428881408,"data_stored":1428881408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1791229952,"data_stored":1791229952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-08-28T02:23:12.228 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph pg dump --format=json 2023-08-28T02:23:12.269 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 65 16 93026 93010 89.3746 65.875 0.00858062 0.0111843 2023-08-28T02:23:12.281 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:23:12.413 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:12.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:12 smithi143 ceph-mon[183059]: pgmap v278: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-08-28T02:23:12.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:12 smithi143 ceph-mon[189871]: pgmap v278: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-08-28T02:23:12.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:12 smithi169 ceph-mon[159415]: pgmap v278: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-08-28T02:23:12.518 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:23:12.587 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:23:12.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:12.591 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:23:12.655 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:23:12.655 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [], 2023-08-28T02:23:12.655 INFO:teuthology.orchestra.run.smithi143.stdout: "complaint_time": 30, 2023-08-28T02:23:12.655 INFO:teuthology.orchestra.run.smithi143.stdout: "num_blocked_ops": 0 2023-08-28T02:23:12.655 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:23:12.692 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:23:12.739 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:12.835 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:23:12.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:12.842 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:23:12.943 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:23:12.978 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:13.072 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:23:13.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:13.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:23:13.179 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:23:13.207 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:13.258 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:23:13.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:13.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:23:13.269 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 66 16 93795 93779 88.749 48.0625 0.0112548 0.0112633 2023-08-28T02:23:13.363 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:23:13.411 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:13.441 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:23:13.659 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.3 dump_historic_ops 2023-08-28T02:23:13.866 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:23:13.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:13.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:23:13.972 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:23:14.032 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:14.081 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:23:14.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:14.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:23:14.186 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:23:14.246 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:14.270 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 67 16 94673 94657 88.2436 54.875 0.0171383 0.0113274 2023-08-28T02:23:14.377 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:23:14.387 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:14 smithi169 ceph-mon[159415]: pgmap v279: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 61 MiB/s wr, 971 op/s 2023-08-28T02:23:14.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:14.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:23:14.489 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:23:14.519 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:14 smithi143 ceph-mon[183059]: pgmap v279: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 61 MiB/s wr, 971 op/s 2023-08-28T02:23:14.519 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:14 smithi143 ceph-mon[189871]: pgmap v279: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 61 MiB/s wr, 971 op/s 2023-08-28T02:23:14.523 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:23:14.570 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:14.646 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:23:14.837 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:23:14.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:14.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:23:14.870 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:23:14.946 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:23:15.023 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:15.099 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:23:15.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:15.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:23:15.206 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:23:15.274 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 68 16 95623 95607 87.8132 59.375 0.0218514 0.0113825 2023-08-28T02:23:15.299 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:15.526 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:23:15.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:15.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:23:15.635 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:23:15.670 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:16.273 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is set 2023-08-28T02:23:16.274 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 69 16 96383 96367 87.2292 47.5 0.0323648 0.0114586 2023-08-28T02:23:16.550 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:16 smithi143 ceph-mon[183059]: pgmap v280: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 60 MiB/s wr, 965 op/s 2023-08-28T02:23:16.550 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:16 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/12537610' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:23:16.550 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:16 smithi143 ceph-mon[189871]: pgmap v280: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 60 MiB/s wr, 965 op/s 2023-08-28T02:23:16.550 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:16 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/12537610' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:23:16.562 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:23:16.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:16.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:23:16.667 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:23:16.695 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:16.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:16 smithi169 ceph-mon[159415]: pgmap v280: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 60 MiB/s wr, 965 op/s 2023-08-28T02:23:16.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:16 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/12537610' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:23:17.275 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 70 16 97291 97275 86.794 56.75 0.0160725 0.0115167 2023-08-28T02:23:17.506 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:17 smithi143 ceph-mon[183059]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:23:17.506 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:17 smithi143 ceph-mon[183059]: failed to encode map e137 with expected crc 2023-08-28T02:23:17.506 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:17 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/12537610' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:23:17.506 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:17 smithi143 ceph-mon[183059]: osdmap e137: 8 total, 8 up, 7 in 2023-08-28T02:23:17.506 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:17 smithi143 ceph-mon[189871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:23:17.506 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:17 smithi143 ceph-mon[189871]: failed to encode map e137 with expected crc 2023-08-28T02:23:17.507 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:17 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/12537610' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:23:17.507 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:17 smithi143 ceph-mon[189871]: osdmap e137: 8 total, 8 up, 7 in 2023-08-28T02:23:17.510 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:23:17.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:17.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:23:17.620 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:23:17.669 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:17 smithi169 ceph-mon[159415]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:23:17.669 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:17 smithi169 ceph-mon[159415]: failed to encode map e137 with expected crc 2023-08-28T02:23:17.669 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:17 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/12537610' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:23:17.669 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:17 smithi169 ceph-mon[159415]: osdmap e137: 8 total, 8 up, 7 in 2023-08-28T02:23:17.674 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:17.784 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:23:17.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:17.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:23:17.903 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:23:17.950 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:18.280 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 71 16 98309 98293 86.461 63.625 0.0127294 0.011561 2023-08-28T02:23:18.337 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:23:18.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:18.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:23:18.443 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:23:18.489 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:18.508 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:18 smithi143 ceph-mon[189871]: failed to encode map e137 with expected crc 2023-08-28T02:23:18.508 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:18 smithi143 ceph-mon[189871]: pgmap v282: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 59 MiB/s wr, 944 op/s 2023-08-28T02:23:18.510 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:18 smithi143 ceph-mon[183059]: failed to encode map e137 with expected crc 2023-08-28T02:23:18.510 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:18 smithi143 ceph-mon[183059]: pgmap v282: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 59 MiB/s wr, 944 op/s 2023-08-28T02:23:18.595 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:23:18.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:18.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:23:18.604 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:18 smithi169 ceph-mon[159415]: failed to encode map e137 with expected crc 2023-08-28T02:23:18.604 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:18 smithi169 ceph-mon[159415]: pgmap v282: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 59 MiB/s wr, 944 op/s 2023-08-28T02:23:18.705 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:23:18.772 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:18.903 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:23:18.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:18.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:23:19.011 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:23:19.033 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:23:19.064 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:23:19.078 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:19.118 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:23:19.119 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:23:19.141 INFO:teuthology.orchestra.run.smithi143.stderr:dumped all 2023-08-28T02:23:19.156 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:23:19.191 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:23:19.216 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:23:19.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:19.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:23:19.236 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:23:19.280 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 72 16 99184 99168 86.02 54.6875 0.0146728 0.0116202 2023-08-28T02:23:19.281 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:23:19.316 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:23:19.318 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:23:19.318 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:23:19.319 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:23:19.319 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:23:19.319 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:23:19.319 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:23:19.319 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:23:19.319 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:23:19.327 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:23:19.390 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:19.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:19 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1473273315' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:23:19.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:19 smithi143 ceph-mon[183059]: pgmap v283: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 57 MiB/s wr, 911 op/s 2023-08-28T02:23:19.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:19 smithi143 ceph-mon[183059]: from='client.44690 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:23:19.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:19 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1473273315' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:23:19.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:19 smithi143 ceph-mon[189871]: pgmap v283: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 57 MiB/s wr, 911 op/s 2023-08-28T02:23:19.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:19 smithi143 ceph-mon[189871]: from='client.44690 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:23:19.490 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:23:19.490 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:23:19.536 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set nodeep-scrub 2023-08-28T02:23:19.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:19 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1473273315' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:23:19.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:19 smithi169 ceph-mon[159415]: pgmap v283: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 57 MiB/s wr, 911 op/s 2023-08-28T02:23:19.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:19 smithi169 ceph-mon[159415]: from='client.44690 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:23:20.280 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 73 16 100091 100075 85.6183 56.6875 0.00768919 0.011675 2023-08-28T02:23:20.593 INFO:teuthology.orchestra.run.smithi143.stdout:{"pg_ready":true,"pg_map":{"version":283,"stamp":"2023-08-28T02:23:19.063499+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6401038259,"num_objects":97627,"num_object_clones":0,"num_object_copies":292881,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":97627,"num_whiteouts":0,"num_read":6288,"num_read_kb":13719,"num_write":98705,"num_write_kb":6261477,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":112,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":98769,"ondisk_log_size":98769,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":20106980,"kb_used_data":18645932,"kb_used_omap":26,"kb_used_meta":1460965,"kb_avail":636109084,"statfs":{"total":671965249536,"available":651375702016,"internally_reserved":0,"allocated":19093434368,"data_stored":19089466674,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27508,"internal_metadata":1496028300},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,4,7,9,7],"upper_bound":32},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":597557248,"num_objects":9118,"num_object_clones":0,"num_object_copies":27354,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9118,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9118,"num_write_kb":583552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002211"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":120,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.139259+0000","last_change":"2023-08-28T02:20:19.849807+0000","last_active":"2023-08-28T02:23:06.139259+0000","last_peered":"2023-08-28T02:23:06.139259+0000","last_clean":"2023-08-28T02:23:06.139259+0000","last_became_active":"2023-08-28T02:20:19.849464+0000","last_became_peered":"2023-08-28T02:20:19.849464+0000","last_unstale":"2023-08-28T02:23:06.139259+0000","last_undegraded":"2023-08-28T02:23:06.139259+0000","last_fullsized":"2023-08-28T02:23:06.139259+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":17,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.140392+0000","last_change":"2023-08-28T02:22:44.090579+0000","last_active":"2023-08-28T02:23:06.140392+0000","last_peered":"2023-08-28T02:23:06.140392+0000","last_clean":"2023-08-28T02:23:06.140392+0000","last_became_active":"2023-08-28T02:22:44.090300+0000","last_became_peered":"2023-08-28T02:22:44.090300+0000","last_unstale":"2023-08-28T02:23:06.140392+0000","last_undegraded":"2023-08-28T02:23:06.140392+0000","last_fullsized":"2023-08-28T02:23:06.140392+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:43:17.958063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":85,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.122971+0000","last_change":"2023-08-28T02:21:59.173280+0000","last_active":"2023-08-28T02:23:06.122971+0000","last_peered":"2023-08-28T02:23:06.122971+0000","last_clean":"2023-08-28T02:23:06.122971+0000","last_became_active":"2023-08-28T02:21:59.172795+0000","last_became_peered":"2023-08-28T02:21:59.172795+0000","last_unstale":"2023-08-28T02:23:06.122971+0000","last_undegraded":"2023-08-28T02:23:06.122971+0000","last_fullsized":"2023-08-28T02:23:06.122971+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:25:55.130302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":95,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.125400+0000","last_change":"2023-08-28T02:21:59.175898+0000","last_active":"2023-08-28T02:23:06.125400+0000","last_peered":"2023-08-28T02:23:06.125400+0000","last_clean":"2023-08-28T02:23:06.125400+0000","last_became_active":"2023-08-28T02:21:59.175580+0000","last_became_peered":"2023-08-28T02:21:59.175580+0000","last_unstale":"2023-08-28T02:23:06.125400+0000","last_undegraded":"2023-08-28T02:23:06.125400+0000","last_fullsized":"2023-08-28T02:23:06.125400+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:06:34.160608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":114,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:07.996448+0000","last_change":"2023-08-28T02:21:59.166971+0000","last_active":"2023-08-28T02:23:07.996448+0000","last_peered":"2023-08-28T02:23:07.996448+0000","last_clean":"2023-08-28T02:23:07.996448+0000","last_became_active":"2023-08-28T02:21:59.166677+0000","last_became_peered":"2023-08-28T02:21:59.166677+0000","last_unstale":"2023-08-28T02:23:07.996448+0000","last_undegraded":"2023-08-28T02:23:07.996448+0000","last_fullsized":"2023-08-28T02:23:07.996448+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:16:04.310997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":131,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.142206+0000","last_change":"2023-08-28T02:21:59.183009+0000","last_active":"2023-08-28T02:23:06.142206+0000","last_peered":"2023-08-28T02:23:06.142206+0000","last_clean":"2023-08-28T02:23:06.142206+0000","last_became_active":"2023-08-28T02:21:59.182621+0000","last_became_peered":"2023-08-28T02:21:59.182621+0000","last_unstale":"2023-08-28T02:23:06.142206+0000","last_undegraded":"2023-08-28T02:23:06.142206+0000","last_fullsized":"2023-08-28T02:23:06.142206+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":93,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.124991+0000","last_change":"2023-08-28T02:21:59.181634+0000","last_active":"2023-08-28T02:23:06.124991+0000","last_peered":"2023-08-28T02:23:06.124991+0000","last_clean":"2023-08-28T02:23:06.124991+0000","last_became_active":"2023-08-28T02:21:59.181300+0000","last_became_peered":"2023-08-28T02:21:59.181300+0000","last_unstale":"2023-08-28T02:23:06.124991+0000","last_undegraded":"2023-08-28T02:23:06.124991+0000","last_fullsized":"2023-08-28T02:23:06.124991+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:35:48.902315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":48,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.123572+0000","last_change":"2023-08-28T02:21:59.181238+0000","last_active":"2023-08-28T02:23:06.123572+0000","last_peered":"2023-08-28T02:23:06.123572+0000","last_clean":"2023-08-28T02:23:06.123572+0000","last_became_active":"2023-08-28T02:21:59.180913+0000","last_became_peered":"2023-08-28T02:21:59.180913+0000","last_unstale":"2023-08-28T02:23:06.123572+0000","last_undegraded":"2023-08-28T02:23:06.123572+0000","last_fullsized":"2023-08-28T02:23:06.123572+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:10:43.235404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":104,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.146304+0000","last_change":"2023-08-28T02:19:50.214698+0000","last_active":"2023-08-28T02:23:06.146304+0000","last_peered":"2023-08-28T02:23:06.146304+0000","last_clean":"2023-08-28T02:23:06.146304+0000","last_became_active":"2023-08-28T02:19:50.214323+0000","last_became_peered":"2023-08-28T02:19:50.214323+0000","last_unstale":"2023-08-28T02:23:06.146304+0000","last_undegraded":"2023-08-28T02:23:06.146304+0000","last_fullsized":"2023-08-28T02:23:06.146304+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":84,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.145592+0000","last_change":"2023-08-28T02:21:59.175974+0000","last_active":"2023-08-28T02:23:06.145592+0000","last_peered":"2023-08-28T02:23:06.145592+0000","last_clean":"2023-08-28T02:23:06.145592+0000","last_became_active":"2023-08-28T02:21:59.175624+0000","last_became_peered":"2023-08-28T02:21:59.175624+0000","last_unstale":"2023-08-28T02:23:06.145592+0000","last_undegraded":"2023-08-28T02:23:06.145592+0000","last_fullsized":"2023-08-28T02:23:06.145592+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,5],"acting":[7,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":122,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.144274+0000","last_change":"2023-08-28T02:21:59.278365+0000","last_active":"2023-08-28T02:23:06.144274+0000","last_peered":"2023-08-28T02:23:06.144274+0000","last_clean":"2023-08-28T02:23:06.144274+0000","last_became_active":"2023-08-28T02:21:59.171883+0000","last_became_peered":"2023-08-28T02:21:59.171883+0000","last_unstale":"2023-08-28T02:23:06.144274+0000","last_undegraded":"2023-08-28T02:23:06.144274+0000","last_fullsized":"2023-08-28T02:23:06.144274+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":77,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.122990+0000","last_change":"2023-08-28T02:20:19.851616+0000","last_active":"2023-08-28T02:23:06.122990+0000","last_peered":"2023-08-28T02:23:06.122990+0000","last_clean":"2023-08-28T02:23:06.122990+0000","last_became_active":"2023-08-28T02:20:19.851322+0000","last_became_peered":"2023-08-28T02:20:19.851322+0000","last_unstale":"2023-08-28T02:23:06.122990+0000","last_undegraded":"2023-08-28T02:23:06.122990+0000","last_fullsized":"2023-08-28T02:23:06.122990+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:03:37.565646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"86'22","reported_seq":111,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.139446+0000","last_change":"2023-08-28T02:11:22.184497+0000","last_active":"2023-08-28T02:23:06.139446+0000","last_peered":"2023-08-28T02:23:06.139446+0000","last_clean":"2023-08-28T02:23:06.139446+0000","last_became_active":"2023-08-28T02:11:22.184173+0000","last_became_peered":"2023-08-28T02:11:22.184173+0000","last_unstale":"2023-08-28T02:23:06.139446+0000","last_undegraded":"2023-08-28T02:23:06.139446+0000","last_fullsized":"2023-08-28T02:23:06.139446+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":117,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.136608+0000","last_change":"2023-08-28T02:19:51.204118+0000","last_active":"2023-08-28T02:23:06.136608+0000","last_peered":"2023-08-28T02:23:06.136608+0000","last_clean":"2023-08-28T02:23:06.136608+0000","last_became_active":"2023-08-28T02:19:51.203745+0000","last_became_peered":"2023-08-28T02:19:51.203745+0000","last_unstale":"2023-08-28T02:23:06.136608+0000","last_undegraded":"2023-08-28T02:23:06.136608+0000","last_fullsized":"2023-08-28T02:23:06.136608+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":106,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.143910+0000","last_change":"2023-08-28T02:20:19.849490+0000","last_active":"2023-08-28T02:23:06.143910+0000","last_peered":"2023-08-28T02:23:06.143910+0000","last_clean":"2023-08-28T02:23:06.143910+0000","last_became_active":"2023-08-28T02:20:19.848962+0000","last_became_peered":"2023-08-28T02:20:19.848962+0000","last_unstale":"2023-08-28T02:23:06.143910+0000","last_undegraded":"2023-08-28T02:23:06.143910+0000","last_fullsized":"2023-08-28T02:23:06.143910+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":98,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.123136+0000","last_change":"2023-08-28T02:21:59.175948+0000","last_active":"2023-08-28T02:23:06.123136+0000","last_peered":"2023-08-28T02:23:06.123136+0000","last_clean":"2023-08-28T02:23:06.123136+0000","last_became_active":"2023-08-28T02:21:59.175583+0000","last_became_peered":"2023-08-28T02:21:59.175583+0000","last_unstale":"2023-08-28T02:23:06.123136+0000","last_undegraded":"2023-08-28T02:23:06.123136+0000","last_fullsized":"2023-08-28T02:23:06.123136+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:06:13.068987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":94,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-08-28T02:23:16.265709+0000","last_change":"2023-08-28T02:21:59.167044+0000","last_active":"2023-08-28T02:23:16.265709+0000","last_peered":"2023-08-28T02:23:16.265709+0000","last_clean":"2023-08-28T02:23:16.265709+0000","last_became_active":"2023-08-28T02:21:59.166742+0000","last_became_peered":"2023-08-28T02:21:59.166742+0000","last_unstale":"2023-08-28T02:23:16.265709+0000","last_undegraded":"2023-08-28T02:23:16.265709+0000","last_fullsized":"2023-08-28T02:23:16.265709+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:35:59.546215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":137,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.146237+0000","last_change":"2023-08-28T02:21:59.180192+0000","last_active":"2023-08-28T02:23:06.146237+0000","last_peered":"2023-08-28T02:23:06.146237+0000","last_clean":"2023-08-28T02:23:06.146237+0000","last_became_active":"2023-08-28T02:21:59.179855+0000","last_became_peered":"2023-08-28T02:21:59.179855+0000","last_unstale":"2023-08-28T02:23:06.146237+0000","last_undegraded":"2023-08-28T02:23:06.146237+0000","last_fullsized":"2023-08-28T02:23:06.146237+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":89,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.145805+0000","last_change":"2023-08-28T02:11:22.188327+0000","last_active":"2023-08-28T02:23:06.145805+0000","last_peered":"2023-08-28T02:23:06.145805+0000","last_clean":"2023-08-28T02:23:06.145805+0000","last_became_active":"2023-08-28T02:11:22.187870+0000","last_became_peered":"2023-08-28T02:11:22.187870+0000","last_unstale":"2023-08-28T02:23:06.145805+0000","last_undegraded":"2023-08-28T02:23:06.145805+0000","last_fullsized":"2023-08-28T02:23:06.145805+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":94,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.125331+0000","last_change":"2023-08-28T02:21:59.176349+0000","last_active":"2023-08-28T02:23:06.125331+0000","last_peered":"2023-08-28T02:23:06.125331+0000","last_clean":"2023-08-28T02:23:06.125331+0000","last_became_active":"2023-08-28T02:21:59.175962+0000","last_became_peered":"2023-08-28T02:21:59.175962+0000","last_unstale":"2023-08-28T02:23:06.125331+0000","last_undegraded":"2023-08-28T02:23:06.125331+0000","last_fullsized":"2023-08-28T02:23:06.125331+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:43:54.669505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":111,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.141658+0000","last_change":"2023-08-28T02:18:39.778728+0000","last_active":"2023-08-28T02:23:06.141658+0000","last_peered":"2023-08-28T02:23:06.141658+0000","last_clean":"2023-08-28T02:23:06.141658+0000","last_became_active":"2023-08-28T02:18:39.778283+0000","last_became_peered":"2023-08-28T02:18:39.778283+0000","last_unstale":"2023-08-28T02:23:06.141658+0000","last_undegraded":"2023-08-28T02:23:06.141658+0000","last_fullsized":"2023-08-28T02:23:06.141658+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":91,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.144177+0000","last_change":"2023-08-28T02:22:44.096303+0000","last_active":"2023-08-28T02:23:06.144177+0000","last_peered":"2023-08-28T02:23:06.144177+0000","last_clean":"2023-08-28T02:23:06.144177+0000","last_became_active":"2023-08-28T02:22:44.095853+0000","last_became_peered":"2023-08-28T02:22:44.095853+0000","last_unstale":"2023-08-28T02:23:06.144177+0000","last_undegraded":"2023-08-28T02:23:06.144177+0000","last_fullsized":"2023-08-28T02:23:06.144177+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.9","version":"137'6035","reported_seq":6061,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-08-28T02:23:16.994350+0000","last_change":"2023-08-28T02:22:00.169476+0000","last_active":"2023-08-28T02:23:16.994350+0000","last_peered":"2023-08-28T02:23:16.994350+0000","last_clean":"2023-08-28T02:23:16.994350+0000","last_became_active":"2023-08-28T02:22:00.168754+0000","last_became_peered":"2023-08-28T02:22:00.168754+0000","last_unstale":"2023-08-28T02:23:16.994350+0000","last_undegraded":"2023-08-28T02:23:16.994350+0000","last_fullsized":"2023-08-28T02:23:16.994350+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":6035,"log_dups_size":0,"ondisk_log_size":6035,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":395509760,"num_objects":6035,"num_object_clones":0,"num_object_copies":18105,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6035,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6035,"num_write_kb":386240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":118,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.144284+0000","last_change":"2023-08-28T02:20:43.220383+0000","last_active":"2023-08-28T02:23:06.144284+0000","last_peered":"2023-08-28T02:23:06.144284+0000","last_clean":"2023-08-28T02:23:06.144284+0000","last_became_active":"2023-08-28T02:20:43.136191+0000","last_became_peered":"2023-08-28T02:20:43.136191+0000","last_unstale":"2023-08-28T02:23:06.144284+0000","last_undegraded":"2023-08-28T02:23:06.144284+0000","last_fullsized":"2023-08-28T02:23:06.144284+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":88,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.141119+0000","last_change":"2023-08-28T02:11:23.249061+0000","last_active":"2023-08-28T02:23:06.141119+0000","last_peered":"2023-08-28T02:23:06.141119+0000","last_clean":"2023-08-28T02:23:06.141119+0000","last_became_active":"2023-08-28T02:11:23.248739+0000","last_became_peered":"2023-08-28T02:11:23.248739+0000","last_unstale":"2023-08-28T02:23:06.141119+0000","last_undegraded":"2023-08-28T02:23:06.141119+0000","last_fullsized":"2023-08-28T02:23:06.141119+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"137'6141","reported_seq":6167,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-08-28T02:23:18.003989+0000","last_change":"2023-08-28T02:22:00.165260+0000","last_active":"2023-08-28T02:23:18.003989+0000","last_peered":"2023-08-28T02:23:18.003989+0000","last_clean":"2023-08-28T02:23:18.003989+0000","last_became_active":"2023-08-28T02:22:00.164978+0000","last_became_peered":"2023-08-28T02:22:00.164978+0000","last_unstale":"2023-08-28T02:23:18.003989+0000","last_undegraded":"2023-08-28T02:23:18.003989+0000","last_fullsized":"2023-08-28T02:23:18.003989+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":6141,"log_dups_size":0,"ondisk_log_size":6141,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:18:41.711960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":402456576,"num_objects":6141,"num_object_clones":0,"num_object_copies":18423,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6141,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6141,"num_write_kb":393024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":106,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.143977+0000","last_change":"2023-08-28T02:19:50.220079+0000","last_active":"2023-08-28T02:23:06.143977+0000","last_peered":"2023-08-28T02:23:06.143977+0000","last_clean":"2023-08-28T02:23:06.143977+0000","last_became_active":"2023-08-28T02:19:50.219671+0000","last_became_peered":"2023-08-28T02:19:50.219671+0000","last_unstale":"2023-08-28T02:23:06.143977+0000","last_undegraded":"2023-08-28T02:23:06.143977+0000","last_fullsized":"2023-08-28T02:23:06.143977+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":136,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.136865+0000","last_change":"2023-08-28T02:22:44.090586+0000","last_active":"2023-08-28T02:23:06.136865+0000","last_peered":"2023-08-28T02:23:06.136865+0000","last_clean":"2023-08-28T02:23:06.136865+0000","last_became_active":"2023-08-28T02:22:44.090339+0000","last_became_peered":"2023-08-28T02:22:44.090339+0000","last_unstale":"2023-08-28T02:23:06.136865+0000","last_undegraded":"2023-08-28T02:23:06.136865+0000","last_fullsized":"2023-08-28T02:23:06.136865+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"137'6101","reported_seq":6127,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-08-28T02:23:17.989890+0000","last_change":"2023-08-28T02:22:00.165317+0000","last_active":"2023-08-28T02:23:17.989890+0000","last_peered":"2023-08-28T02:23:17.989890+0000","last_clean":"2023-08-28T02:23:17.989890+0000","last_became_active":"2023-08-28T02:22:00.164989+0000","last_became_peered":"2023-08-28T02:22:00.164989+0000","last_unstale":"2023-08-28T02:23:17.989890+0000","last_undegraded":"2023-08-28T02:23:17.989890+0000","last_fullsized":"2023-08-28T02:23:17.989890+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":6101,"log_dups_size":0,"ondisk_log_size":6101,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:10:46.322659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":399835136,"num_objects":6101,"num_object_clones":0,"num_object_copies":18303,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6101,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6101,"num_write_kb":390464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":90,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.144439+0000","last_change":"2023-08-28T02:11:22.188982+0000","last_active":"2023-08-28T02:23:06.144439+0000","last_peered":"2023-08-28T02:23:06.144439+0000","last_clean":"2023-08-28T02:23:06.144439+0000","last_became_active":"2023-08-28T02:11:22.188663+0000","last_became_peered":"2023-08-28T02:11:22.188663+0000","last_unstale":"2023-08-28T02:23:06.144439+0000","last_undegraded":"2023-08-28T02:23:06.144439+0000","last_fullsized":"2023-08-28T02:23:06.144439+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":145,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.142035+0000","last_change":"2023-08-28T02:21:43.205520+0000","last_active":"2023-08-28T02:23:06.142035+0000","last_peered":"2023-08-28T02:23:06.142035+0000","last_clean":"2023-08-28T02:23:06.142035+0000","last_became_active":"2023-08-28T02:21:43.205165+0000","last_became_peered":"2023-08-28T02:21:43.205165+0000","last_unstale":"2023-08-28T02:23:06.142035+0000","last_undegraded":"2023-08-28T02:23:06.142035+0000","last_fullsized":"2023-08-28T02:23:06.142035+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"137'6143","reported_seq":6168,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-08-28T02:23:18.013415+0000","last_change":"2023-08-28T02:22:00.166496+0000","last_active":"2023-08-28T02:23:18.013415+0000","last_peered":"2023-08-28T02:23:18.013415+0000","last_clean":"2023-08-28T02:23:18.013415+0000","last_became_active":"2023-08-28T02:22:00.166206+0000","last_became_peered":"2023-08-28T02:22:00.166206+0000","last_unstale":"2023-08-28T02:23:18.013415+0000","last_undegraded":"2023-08-28T02:23:18.013415+0000","last_fullsized":"2023-08-28T02:23:18.013415+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":6143,"log_dups_size":0,"ondisk_log_size":6143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:16:10.265829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":402587648,"num_objects":6143,"num_object_clones":0,"num_object_copies":18429,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6143,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6143,"num_write_kb":393152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":113,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.142792+0000","last_change":"2023-08-28T02:21:59.176563+0000","last_active":"2023-08-28T02:23:06.142792+0000","last_peered":"2023-08-28T02:23:06.142792+0000","last_clean":"2023-08-28T02:23:06.142792+0000","last_became_active":"2023-08-28T02:21:59.175969+0000","last_became_peered":"2023-08-28T02:21:59.175969+0000","last_unstale":"2023-08-28T02:23:06.142792+0000","last_undegraded":"2023-08-28T02:23:06.142792+0000","last_fullsized":"2023-08-28T02:23:06.142792+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":97,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:07.996556+0000","last_change":"2023-08-28T02:19:51.205245+0000","last_active":"2023-08-28T02:23:07.996556+0000","last_peered":"2023-08-28T02:23:07.996556+0000","last_clean":"2023-08-28T02:23:07.996556+0000","last_became_active":"2023-08-28T02:19:51.204896+0000","last_became_peered":"2023-08-28T02:19:51.204896+0000","last_unstale":"2023-08-28T02:23:07.996556+0000","last_undegraded":"2023-08-28T02:23:07.996556+0000","last_fullsized":"2023-08-28T02:23:07.996556+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:51:24.634702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"137'6134","reported_seq":6160,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-08-28T02:23:18.013167+0000","last_change":"2023-08-28T02:22:00.167148+0000","last_active":"2023-08-28T02:23:18.013167+0000","last_peered":"2023-08-28T02:23:18.013167+0000","last_clean":"2023-08-28T02:23:18.013167+0000","last_became_active":"2023-08-28T02:22:00.166809+0000","last_became_peered":"2023-08-28T02:22:00.166809+0000","last_unstale":"2023-08-28T02:23:18.013167+0000","last_undegraded":"2023-08-28T02:23:18.013167+0000","last_fullsized":"2023-08-28T02:23:18.013167+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":6134,"log_dups_size":0,"ondisk_log_size":6134,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:03:13.526107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":401997824,"num_objects":6134,"num_object_clones":0,"num_object_copies":18402,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6134,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6134,"num_write_kb":392576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":57,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.124490+0000","last_change":"2023-08-28T02:21:59.280044+0000","last_active":"2023-08-28T02:23:06.124490+0000","last_peered":"2023-08-28T02:23:06.124490+0000","last_clean":"2023-08-28T02:23:06.124490+0000","last_became_active":"2023-08-28T02:21:59.179617+0000","last_became_peered":"2023-08-28T02:21:59.179617+0000","last_unstale":"2023-08-28T02:23:06.124490+0000","last_undegraded":"2023-08-28T02:23:06.124490+0000","last_fullsized":"2023-08-28T02:23:06.124490+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:14:46.083577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":121,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.137859+0000","last_change":"2023-08-28T02:22:44.091351+0000","last_active":"2023-08-28T02:23:06.137859+0000","last_peered":"2023-08-28T02:23:06.137859+0000","last_clean":"2023-08-28T02:23:06.137859+0000","last_became_active":"2023-08-28T02:22:44.090971+0000","last_became_peered":"2023-08-28T02:22:44.090971+0000","last_unstale":"2023-08-28T02:23:06.137859+0000","last_undegraded":"2023-08-28T02:23:06.137859+0000","last_fullsized":"2023-08-28T02:23:06.137859+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"137'6249","reported_seq":6276,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-08-28T02:23:18.294192+0000","last_change":"2023-08-28T02:22:00.162375+0000","last_active":"2023-08-28T02:23:18.294192+0000","last_peered":"2023-08-28T02:23:18.294192+0000","last_clean":"2023-08-28T02:23:18.294192+0000","last_became_active":"2023-08-28T02:22:00.162190+0000","last_became_peered":"2023-08-28T02:22:00.162190+0000","last_unstale":"2023-08-28T02:23:18.294192+0000","last_undegraded":"2023-08-28T02:23:18.294192+0000","last_fullsized":"2023-08-28T02:23:18.294192+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":6249,"log_dups_size":0,"ondisk_log_size":6249,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:40:08.985631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":409534464,"num_objects":6249,"num_object_clones":0,"num_object_copies":18747,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6249,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6249,"num_write_kb":399936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":132,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.142173+0000","last_change":"2023-08-28T02:21:59.179699+0000","last_active":"2023-08-28T02:23:06.142173+0000","last_peered":"2023-08-28T02:23:06.142173+0000","last_clean":"2023-08-28T02:23:06.142173+0000","last_became_active":"2023-08-28T02:21:59.179345+0000","last_became_peered":"2023-08-28T02:21:59.179345+0000","last_unstale":"2023-08-28T02:23:06.142173+0000","last_undegraded":"2023-08-28T02:23:06.142173+0000","last_fullsized":"2023-08-28T02:23:06.142173+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":156,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.139440+0000","last_change":"2023-08-28T02:22:44.083121+0000","last_active":"2023-08-28T02:23:06.139440+0000","last_peered":"2023-08-28T02:23:06.139440+0000","last_clean":"2023-08-28T02:23:06.139440+0000","last_became_active":"2023-08-28T02:22:44.082776+0000","last_became_peered":"2023-08-28T02:22:44.082776+0000","last_unstale":"2023-08-28T02:23:06.139440+0000","last_undegraded":"2023-08-28T02:23:06.139440+0000","last_fullsized":"2023-08-28T02:23:06.139440+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"137'6068","reported_seq":6094,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-08-28T02:23:16.986315+0000","last_change":"2023-08-28T02:22:00.167630+0000","last_active":"2023-08-28T02:23:16.986315+0000","last_peered":"2023-08-28T02:23:16.986315+0000","last_clean":"2023-08-28T02:23:16.986315+0000","last_became_active":"2023-08-28T02:22:00.167305+0000","last_became_peered":"2023-08-28T02:22:00.167305+0000","last_unstale":"2023-08-28T02:23:16.986315+0000","last_undegraded":"2023-08-28T02:23:16.986315+0000","last_fullsized":"2023-08-28T02:23:16.986315+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":6068,"log_dups_size":0,"ondisk_log_size":6068,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":397672448,"num_objects":6068,"num_object_clones":0,"num_object_copies":18204,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6068,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6068,"num_write_kb":388352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":28,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.136985+0000","last_change":"2023-08-28T02:21:59.174435+0000","last_active":"2023-08-28T02:23:06.136985+0000","last_peered":"2023-08-28T02:23:06.136985+0000","last_clean":"2023-08-28T02:23:06.136985+0000","last_became_active":"2023-08-28T02:21:59.174079+0000","last_became_peered":"2023-08-28T02:21:59.174079+0000","last_unstale":"2023-08-28T02:23:06.136985+0000","last_undegraded":"2023-08-28T02:23:06.136985+0000","last_fullsized":"2023-08-28T02:23:06.136985+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":65,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.125202+0000","last_change":"2023-08-28T02:19:51.204017+0000","last_active":"2023-08-28T02:23:06.125202+0000","last_peered":"2023-08-28T02:23:06.125202+0000","last_clean":"2023-08-28T02:23:06.125202+0000","last_became_active":"2023-08-28T02:19:51.203627+0000","last_became_peered":"2023-08-28T02:19:51.203627+0000","last_unstale":"2023-08-28T02:23:06.125202+0000","last_undegraded":"2023-08-28T02:23:06.125202+0000","last_fullsized":"2023-08-28T02:23:06.125202+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:14:06.231701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.e","version":"137'6140","reported_seq":6166,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-08-28T02:23:16.967320+0000","last_change":"2023-08-28T02:22:00.166490+0000","last_active":"2023-08-28T02:23:16.967320+0000","last_peered":"2023-08-28T02:23:16.967320+0000","last_clean":"2023-08-28T02:23:16.967320+0000","last_became_active":"2023-08-28T02:22:00.166140+0000","last_became_peered":"2023-08-28T02:22:00.166140+0000","last_unstale":"2023-08-28T02:23:16.967320+0000","last_undegraded":"2023-08-28T02:23:16.967320+0000","last_fullsized":"2023-08-28T02:23:16.967320+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":6140,"log_dups_size":0,"ondisk_log_size":6140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":402391040,"num_objects":6140,"num_object_clones":0,"num_object_copies":18420,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6140,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6140,"num_write_kb":392960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":99,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.123683+0000","last_change":"2023-08-28T02:21:59.168708+0000","last_active":"2023-08-28T02:23:06.123683+0000","last_peered":"2023-08-28T02:23:06.123683+0000","last_clean":"2023-08-28T02:23:06.123683+0000","last_became_active":"2023-08-28T02:21:59.168481+0000","last_became_peered":"2023-08-28T02:21:59.168481+0000","last_unstale":"2023-08-28T02:23:06.123683+0000","last_undegraded":"2023-08-28T02:23:06.123683+0000","last_fullsized":"2023-08-28T02:23:06.123683+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:33:24.514653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":86,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.123556+0000","last_change":"2023-08-28T02:21:59.176160+0000","last_active":"2023-08-28T02:23:06.123556+0000","last_peered":"2023-08-28T02:23:06.123556+0000","last_clean":"2023-08-28T02:23:06.123556+0000","last_became_active":"2023-08-28T02:21:59.175793+0000","last_became_peered":"2023-08-28T02:21:59.175793+0000","last_unstale":"2023-08-28T02:23:06.123556+0000","last_undegraded":"2023-08-28T02:23:06.123556+0000","last_fullsized":"2023-08-28T02:23:06.123556+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:59:44.892535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"78'126","reported_seq":594,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.123285+0000","last_change":"2023-08-28T02:19:50.213556+0000","last_active":"2023-08-28T02:23:06.123285+0000","last_peered":"2023-08-28T02:23:06.123285+0000","last_clean":"2023-08-28T02:23:06.123285+0000","last_became_active":"2023-08-28T02:19:50.212996+0000","last_became_peered":"2023-08-28T02:19:50.212996+0000","last_unstale":"2023-08-28T02:23:06.123285+0000","last_undegraded":"2023-08-28T02:23:06.123285+0000","last_fullsized":"2023-08-28T02:23:06.123285+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:38:28.157352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":743,"num_read_kb":1331,"num_write":125,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":128,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.146311+0000","last_change":"2023-08-28T02:21:59.176292+0000","last_active":"2023-08-28T02:23:06.146311+0000","last_peered":"2023-08-28T02:23:06.146311+0000","last_clean":"2023-08-28T02:23:06.146311+0000","last_became_active":"2023-08-28T02:21:59.175956+0000","last_became_peered":"2023-08-28T02:21:59.175956+0000","last_unstale":"2023-08-28T02:23:06.146311+0000","last_undegraded":"2023-08-28T02:23:06.146311+0000","last_fullsized":"2023-08-28T02:23:06.146311+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":32,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.123413+0000","last_change":"2023-08-28T02:21:43.238044+0000","last_active":"2023-08-28T02:23:06.123413+0000","last_peered":"2023-08-28T02:23:06.123413+0000","last_clean":"2023-08-28T02:23:06.123413+0000","last_became_active":"2023-08-28T02:21:43.237792+0000","last_became_peered":"2023-08-28T02:21:43.237792+0000","last_unstale":"2023-08-28T02:23:06.123413+0000","last_undegraded":"2023-08-28T02:23:06.123413+0000","last_fullsized":"2023-08-28T02:23:06.123413+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:11:35.488503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"137'6075","reported_seq":6100,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-08-28T02:23:17.989526+0000","last_change":"2023-08-28T02:22:00.166578+0000","last_active":"2023-08-28T02:23:17.989526+0000","last_peered":"2023-08-28T02:23:17.989526+0000","last_clean":"2023-08-28T02:23:17.989526+0000","last_became_active":"2023-08-28T02:22:00.166311+0000","last_became_peered":"2023-08-28T02:22:00.166311+0000","last_unstale":"2023-08-28T02:23:17.989526+0000","last_undegraded":"2023-08-28T02:23:17.989526+0000","last_fullsized":"2023-08-28T02:23:17.989526+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":6075,"log_dups_size":0,"ondisk_log_size":6075,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:40:38.502180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":398131200,"num_objects":6075,"num_object_clones":0,"num_object_copies":18225,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6075,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6075,"num_write_kb":388800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"137'6106","reported_seq":6133,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-08-28T02:23:17.961550+0000","last_change":"2023-08-28T02:22:00.160991+0000","last_active":"2023-08-28T02:23:17.961550+0000","last_peered":"2023-08-28T02:23:17.961550+0000","last_clean":"2023-08-28T02:23:17.961550+0000","last_became_active":"2023-08-28T02:22:00.160755+0000","last_became_peered":"2023-08-28T02:22:00.160755+0000","last_unstale":"2023-08-28T02:23:17.961550+0000","last_undegraded":"2023-08-28T02:23:17.961550+0000","last_fullsized":"2023-08-28T02:23:17.961550+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":6106,"log_dups_size":0,"ondisk_log_size":6106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:42:59.109288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":400162816,"num_objects":6106,"num_object_clones":0,"num_object_copies":18318,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6106,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6106,"num_write_kb":390784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"78'204","reported_seq":1053,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.143019+0000","last_change":"2023-08-28T02:19:50.219949+0000","last_active":"2023-08-28T02:23:06.143019+0000","last_peered":"2023-08-28T02:23:06.143019+0000","last_clean":"2023-08-28T02:23:06.143019+0000","last_became_active":"2023-08-28T02:19:50.219356+0000","last_became_peered":"2023-08-28T02:19:50.219356+0000","last_unstale":"2023-08-28T02:23:06.143019+0000","last_undegraded":"2023-08-28T02:23:06.143019+0000","last_fullsized":"2023-08-28T02:23:06.143019+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":204,"log_dups_size":0,"ondisk_log_size":204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1521,"num_read_kb":2511,"num_write":178,"num_write_kb":1175,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":104,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.137360+0000","last_change":"2023-08-28T02:18:39.779607+0000","last_active":"2023-08-28T02:23:06.137360+0000","last_peered":"2023-08-28T02:23:06.137360+0000","last_clean":"2023-08-28T02:23:06.137360+0000","last_became_active":"2023-08-28T02:18:39.779303+0000","last_became_peered":"2023-08-28T02:18:39.779303+0000","last_unstale":"2023-08-28T02:23:06.137360+0000","last_undegraded":"2023-08-28T02:23:06.137360+0000","last_fullsized":"2023-08-28T02:23:06.137360+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":84,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.123205+0000","last_change":"2023-08-28T02:21:59.174327+0000","last_active":"2023-08-28T02:23:06.123205+0000","last_peered":"2023-08-28T02:23:06.123205+0000","last_clean":"2023-08-28T02:23:06.123205+0000","last_became_active":"2023-08-28T02:21:59.173986+0000","last_became_peered":"2023-08-28T02:21:59.173986+0000","last_unstale":"2023-08-28T02:23:06.123205+0000","last_undegraded":"2023-08-28T02:23:06.123205+0000","last_fullsized":"2023-08-28T02:23:06.123205+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:55:51.249985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.6","version":"137'6041","reported_seq":6067,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-08-28T02:23:17.994746+0000","last_change":"2023-08-28T02:22:00.166938+0000","last_active":"2023-08-28T02:23:17.994746+0000","last_peered":"2023-08-28T02:23:17.994746+0000","last_clean":"2023-08-28T02:23:17.994746+0000","last_became_active":"2023-08-28T02:22:00.166656+0000","last_became_peered":"2023-08-28T02:22:00.166656+0000","last_unstale":"2023-08-28T02:23:17.994746+0000","last_undegraded":"2023-08-28T02:23:17.994746+0000","last_fullsized":"2023-08-28T02:23:17.994746+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":6041,"log_dups_size":0,"ondisk_log_size":6041,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:57:00.014765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":395902976,"num_objects":6041,"num_object_clones":0,"num_object_copies":18123,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6041,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6041,"num_write_kb":386624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"78'47","reported_seq":321,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.136889+0000","last_change":"2023-08-28T02:21:59.287946+0000","last_active":"2023-08-28T02:23:06.136889+0000","last_peered":"2023-08-28T02:23:06.136889+0000","last_clean":"2023-08-28T02:23:06.136889+0000","last_became_active":"2023-08-28T02:21:59.173235+0000","last_became_peered":"2023-08-28T02:21:59.173235+0000","last_unstale":"2023-08-28T02:23:06.136889+0000","last_undegraded":"2023-08-28T02:23:06.136889+0000","last_fullsized":"2023-08-28T02:23:06.136889+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":47,"log_dups_size":0,"ondisk_log_size":47,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:02:39.919036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":162,"num_read_kb":1281,"num_write":48,"num_write_kb":1153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":76,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.123497+0000","last_change":"2023-08-28T02:20:19.851591+0000","last_active":"2023-08-28T02:23:06.123497+0000","last_peered":"2023-08-28T02:23:06.123497+0000","last_clean":"2023-08-28T02:23:06.123497+0000","last_became_active":"2023-08-28T02:20:19.851291+0000","last_became_peered":"2023-08-28T02:20:19.851291+0000","last_unstale":"2023-08-28T02:23:06.123497+0000","last_undegraded":"2023-08-28T02:23:06.123497+0000","last_fullsized":"2023-08-28T02:23:06.123497+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:33:51.259707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":108,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.123482+0000","last_change":"2023-08-28T02:19:50.211156+0000","last_active":"2023-08-28T02:23:06.123482+0000","last_peered":"2023-08-28T02:23:06.123482+0000","last_clean":"2023-08-28T02:23:06.123482+0000","last_became_active":"2023-08-28T02:19:50.210398+0000","last_became_peered":"2023-08-28T02:19:50.210398+0000","last_unstale":"2023-08-28T02:23:06.123482+0000","last_undegraded":"2023-08-28T02:23:06.123482+0000","last_fullsized":"2023-08-28T02:23:06.123482+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:14:41.748197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"78'114","reported_seq":435,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.145519+0000","last_change":"2023-08-28T02:20:19.852152+0000","last_active":"2023-08-28T02:23:06.145519+0000","last_peered":"2023-08-28T02:23:06.145519+0000","last_clean":"2023-08-28T02:23:06.145519+0000","last_became_active":"2023-08-28T02:20:19.851615+0000","last_became_peered":"2023-08-28T02:20:19.851615+0000","last_unstale":"2023-08-28T02:23:06.145519+0000","last_undegraded":"2023-08-28T02:23:06.145519+0000","last_fullsized":"2023-08-28T02:23:06.145519+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":114,"log_dups_size":0,"ondisk_log_size":114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":358,"num_read_kb":1352,"num_write":102,"num_write_kb":1104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"86'139","reported_seq":340,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.143420+0000","last_change":"2023-08-28T02:18:39.786823+0000","last_active":"2023-08-28T02:23:06.143420+0000","last_peered":"2023-08-28T02:23:06.143420+0000","last_clean":"2023-08-28T02:23:06.143420+0000","last_became_active":"2023-08-28T02:18:39.786357+0000","last_became_peered":"2023-08-28T02:18:39.786357+0000","last_unstale":"2023-08-28T02:23:06.143420+0000","last_undegraded":"2023-08-28T02:23:06.143420+0000","last_fullsized":"2023-08-28T02:23:06.143420+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_clean_scrub_stamp":"2023-08-28T02:08:02.043718+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":84,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.143094+0000","last_change":"2023-08-28T02:21:59.179583+0000","last_active":"2023-08-28T02:23:06.143094+0000","last_peered":"2023-08-28T02:23:06.143094+0000","last_clean":"2023-08-28T02:23:06.143094+0000","last_became_active":"2023-08-28T02:21:59.179239+0000","last_became_peered":"2023-08-28T02:21:59.179239+0000","last_unstale":"2023-08-28T02:23:06.143094+0000","last_undegraded":"2023-08-28T02:23:06.143094+0000","last_fullsized":"2023-08-28T02:23:06.143094+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":87,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.142273+0000","last_change":"2023-08-28T02:11:23.249640+0000","last_active":"2023-08-28T02:23:06.142273+0000","last_peered":"2023-08-28T02:23:06.142273+0000","last_clean":"2023-08-28T02:23:06.142273+0000","last_became_active":"2023-08-28T02:11:23.249355+0000","last_became_peered":"2023-08-28T02:11:23.249355+0000","last_unstale":"2023-08-28T02:23:06.142273+0000","last_undegraded":"2023-08-28T02:23:06.142273+0000","last_fullsized":"2023-08-28T02:23:06.142273+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"137'6028","reported_seq":6054,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-08-28T02:23:16.960080+0000","last_change":"2023-08-28T02:22:00.169046+0000","last_active":"2023-08-28T02:23:16.960080+0000","last_peered":"2023-08-28T02:23:16.960080+0000","last_clean":"2023-08-28T02:23:16.960080+0000","last_became_active":"2023-08-28T02:22:00.168696+0000","last_became_peered":"2023-08-28T02:22:00.168696+0000","last_unstale":"2023-08-28T02:23:16.960080+0000","last_undegraded":"2023-08-28T02:23:16.960080+0000","last_fullsized":"2023-08-28T02:23:16.960080+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":6028,"log_dups_size":0,"ondisk_log_size":6028,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":395051008,"num_objects":6028,"num_object_clones":0,"num_object_copies":18084,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6028,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6028,"num_write_kb":385792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.7","version":"137'6126","reported_seq":6152,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-08-28T02:23:17.003212+0000","last_change":"2023-08-28T02:22:00.165614+0000","last_active":"2023-08-28T02:23:17.003212+0000","last_peered":"2023-08-28T02:23:17.003212+0000","last_clean":"2023-08-28T02:23:17.003212+0000","last_became_active":"2023-08-28T02:22:00.165136+0000","last_became_peered":"2023-08-28T02:22:00.165136+0000","last_unstale":"2023-08-28T02:23:17.003212+0000","last_undegraded":"2023-08-28T02:23:17.003212+0000","last_fullsized":"2023-08-28T02:23:17.003212+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":6126,"log_dups_size":0,"ondisk_log_size":6126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":401473536,"num_objects":6126,"num_object_clones":0,"num_object_copies":18378,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6126,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6126,"num_write_kb":392064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"78'148","reported_seq":682,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.143499+0000","last_change":"2023-08-28T02:19:50.221311+0000","last_active":"2023-08-28T02:23:06.143499+0000","last_peered":"2023-08-28T02:23:06.143499+0000","last_clean":"2023-08-28T02:23:06.143499+0000","last_became_active":"2023-08-28T02:19:50.220754+0000","last_became_peered":"2023-08-28T02:19:50.220754+0000","last_unstale":"2023-08-28T02:23:06.143499+0000","last_undegraded":"2023-08-28T02:23:06.143499+0000","last_fullsized":"2023-08-28T02:23:06.143499+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1092,"num_read_kb":1955,"num_write":127,"num_write_kb":5177,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":94,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.124305+0000","last_change":"2023-08-28T02:21:59.180701+0000","last_active":"2023-08-28T02:23:06.124305+0000","last_peered":"2023-08-28T02:23:06.124305+0000","last_clean":"2023-08-28T02:23:06.124305+0000","last_became_active":"2023-08-28T02:21:59.180389+0000","last_became_peered":"2023-08-28T02:21:59.180389+0000","last_unstale":"2023-08-28T02:23:06.124305+0000","last_undegraded":"2023-08-28T02:23:06.124305+0000","last_fullsized":"2023-08-28T02:23:06.124305+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:36:31.599391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":130,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.125558+0000","last_change":"2023-08-28T02:21:59.167858+0000","last_active":"2023-08-28T02:23:06.125558+0000","last_peered":"2023-08-28T02:23:06.125558+0000","last_clean":"2023-08-28T02:23:06.125558+0000","last_became_active":"2023-08-28T02:21:59.167436+0000","last_became_peered":"2023-08-28T02:21:59.167436+0000","last_unstale":"2023-08-28T02:23:06.125558+0000","last_undegraded":"2023-08-28T02:23:06.125558+0000","last_fullsized":"2023-08-28T02:23:06.125558+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:17:50.717282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"78'103","reported_seq":558,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.144357+0000","last_change":"2023-08-28T02:18:39.786876+0000","last_active":"2023-08-28T02:23:06.144357+0000","last_peered":"2023-08-28T02:23:06.144357+0000","last_clean":"2023-08-28T02:23:06.144357+0000","last_became_active":"2023-08-28T02:18:39.786414+0000","last_became_peered":"2023-08-28T02:18:39.786414+0000","last_unstale":"2023-08-28T02:23:06.144357+0000","last_undegraded":"2023-08-28T02:23:06.144357+0000","last_fullsized":"2023-08-28T02:23:06.144357+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":803,"num_read_kb":2080,"num_write":73,"num_write_kb":1244,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":76,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.123434+0000","last_change":"2023-08-28T02:20:19.850916+0000","last_active":"2023-08-28T02:23:06.123434+0000","last_peered":"2023-08-28T02:23:06.123434+0000","last_clean":"2023-08-28T02:23:06.123434+0000","last_became_active":"2023-08-28T02:20:19.850627+0000","last_became_peered":"2023-08-28T02:20:19.850627+0000","last_unstale":"2023-08-28T02:23:06.123434+0000","last_undegraded":"2023-08-28T02:23:06.123434+0000","last_fullsized":"2023-08-28T02:23:06.123434+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:19:14.346189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":71,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.124581+0000","last_change":"2023-08-28T02:18:39.780069+0000","last_active":"2023-08-28T02:23:06.124581+0000","last_peered":"2023-08-28T02:23:06.124581+0000","last_clean":"2023-08-28T02:23:06.124581+0000","last_became_active":"2023-08-28T02:18:39.779829+0000","last_became_peered":"2023-08-28T02:18:39.779829+0000","last_unstale":"2023-08-28T02:23:06.124581+0000","last_undegraded":"2023-08-28T02:23:06.124581+0000","last_fullsized":"2023-08-28T02:23:06.124581+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:35:01.309659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"137'5953","reported_seq":5977,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-08-28T02:23:16.970793+0000","last_change":"2023-08-28T02:22:00.167814+0000","last_active":"2023-08-28T02:23:16.970793+0000","last_peered":"2023-08-28T02:23:16.970793+0000","last_clean":"2023-08-28T02:23:16.970793+0000","last_became_active":"2023-08-28T02:22:00.167513+0000","last_became_peered":"2023-08-28T02:22:00.167513+0000","last_unstale":"2023-08-28T02:23:16.970793+0000","last_undegraded":"2023-08-28T02:23:16.970793+0000","last_fullsized":"2023-08-28T02:23:16.970793+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":5953,"log_dups_size":0,"ondisk_log_size":5953,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":390135808,"num_objects":5953,"num_object_clones":0,"num_object_copies":17859,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5953,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5953,"num_write_kb":380992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":57,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.144084+0000","last_change":"2023-08-28T02:21:59.322116+0000","last_active":"2023-08-28T02:23:06.144084+0000","last_peered":"2023-08-28T02:23:06.144084+0000","last_clean":"2023-08-28T02:23:06.144084+0000","last_became_active":"2023-08-28T02:21:59.175477+0000","last_became_peered":"2023-08-28T02:21:59.175477+0000","last_unstale":"2023-08-28T02:23:06.144084+0000","last_undegraded":"2023-08-28T02:23:06.144084+0000","last_fullsized":"2023-08-28T02:23:06.144084+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"78'112","reported_seq":380,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.123521+0000","last_change":"2023-08-28T02:19:50.218206+0000","last_active":"2023-08-28T02:23:06.123521+0000","last_peered":"2023-08-28T02:23:06.123521+0000","last_clean":"2023-08-28T02:23:06.123521+0000","last_became_active":"2023-08-28T02:19:50.217637+0000","last_became_peered":"2023-08-28T02:19:50.217637+0000","last_unstale":"2023-08-28T02:23:06.123521+0000","last_undegraded":"2023-08-28T02:23:06.123521+0000","last_fullsized":"2023-08-28T02:23:06.123521+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:12:06.224907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":444,"num_read_kb":473,"num_write":94,"num_write_kb":127,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":90,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.143272+0000","last_change":"2023-08-28T02:21:59.176497+0000","last_active":"2023-08-28T02:23:06.143272+0000","last_peered":"2023-08-28T02:23:06.143272+0000","last_clean":"2023-08-28T02:23:06.143272+0000","last_became_active":"2023-08-28T02:21:59.176146+0000","last_became_peered":"2023-08-28T02:21:59.176146+0000","last_unstale":"2023-08-28T02:23:06.143272+0000","last_undegraded":"2023-08-28T02:23:06.143272+0000","last_fullsized":"2023-08-28T02:23:06.143272+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.1","version":"137'6085","reported_seq":6109,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-08-28T02:23:16.998322+0000","last_change":"2023-08-28T02:22:00.167365+0000","last_active":"2023-08-28T02:23:16.998322+0000","last_peered":"2023-08-28T02:23:16.998322+0000","last_clean":"2023-08-28T02:23:16.998322+0000","last_became_active":"2023-08-28T02:22:00.167035+0000","last_became_peered":"2023-08-28T02:22:00.167035+0000","last_unstale":"2023-08-28T02:23:16.998322+0000","last_undegraded":"2023-08-28T02:23:16.998322+0000","last_fullsized":"2023-08-28T02:23:16.998322+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":6085,"log_dups_size":0,"ondisk_log_size":6085,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":398786560,"num_objects":6085,"num_object_clones":0,"num_object_copies":18255,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6085,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6085,"num_write_kb":389440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"78'122","reported_seq":592,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.142047+0000","last_change":"2023-08-28T02:20:19.851065+0000","last_active":"2023-08-28T02:23:06.142047+0000","last_peered":"2023-08-28T02:23:06.142047+0000","last_clean":"2023-08-28T02:23:06.142047+0000","last_became_active":"2023-08-28T02:20:19.850477+0000","last_became_peered":"2023-08-28T02:20:19.850477+0000","last_unstale":"2023-08-28T02:23:06.142047+0000","last_undegraded":"2023-08-28T02:23:06.142047+0000","last_fullsized":"2023-08-28T02:23:06.142047+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":857,"num_read_kb":2224,"num_write":99,"num_write_kb":2312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":89,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.125482+0000","last_change":"2023-08-28T02:19:50.221001+0000","last_active":"2023-08-28T02:23:06.125482+0000","last_peered":"2023-08-28T02:23:06.125482+0000","last_clean":"2023-08-28T02:23:06.125482+0000","last_became_active":"2023-08-28T02:19:50.220611+0000","last_became_peered":"2023-08-28T02:19:50.220611+0000","last_unstale":"2023-08-28T02:23:06.125482+0000","last_undegraded":"2023-08-28T02:23:06.125482+0000","last_fullsized":"2023-08-28T02:23:06.125482+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:23:10.637630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":88,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.136686+0000","last_change":"2023-08-28T02:11:22.437146+0000","last_active":"2023-08-28T02:23:06.136686+0000","last_peered":"2023-08-28T02:23:06.136686+0000","last_clean":"2023-08-28T02:23:06.136686+0000","last_became_active":"2023-08-28T02:11:22.436798+0000","last_became_peered":"2023-08-28T02:11:22.436798+0000","last_unstale":"2023-08-28T02:23:06.136686+0000","last_undegraded":"2023-08-28T02:23:06.136686+0000","last_fullsized":"2023-08-28T02:23:06.136686+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"137'6174","reported_seq":6201,"reported_epoch":137,"state":"active+clean","last_fresh":"2023-08-28T02:23:18.255367+0000","last_change":"2023-08-28T02:22:00.163491+0000","last_active":"2023-08-28T02:23:18.255367+0000","last_peered":"2023-08-28T02:23:18.255367+0000","last_clean":"2023-08-28T02:23:18.255367+0000","last_became_active":"2023-08-28T02:22:00.163135+0000","last_became_peered":"2023-08-28T02:22:00.163135+0000","last_unstale":"2023-08-28T02:23:18.255367+0000","last_undegraded":"2023-08-28T02:23:18.255367+0000","last_fullsized":"2023-08-28T02:23:18.255367+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":6174,"log_dups_size":0,"ondisk_log_size":6174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:52:26.563181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":404619264,"num_objects":6174,"num_object_clones":0,"num_object_copies":18522,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6174,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6174,"num_write_kb":395136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":73,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.124539+0000","last_change":"2023-08-28T02:18:39.781312+0000","last_active":"2023-08-28T02:23:06.124539+0000","last_peered":"2023-08-28T02:23:06.124539+0000","last_clean":"2023-08-28T02:23:06.124539+0000","last_became_active":"2023-08-28T02:18:39.780916+0000","last_became_peered":"2023-08-28T02:18:39.780916+0000","last_unstale":"2023-08-28T02:23:06.124539+0000","last_undegraded":"2023-08-28T02:23:06.124539+0000","last_fullsized":"2023-08-28T02:23:06.124539+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:37:11.571250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":148,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.138296+0000","last_change":"2023-08-28T02:21:59.168665+0000","last_active":"2023-08-28T02:23:06.138296+0000","last_peered":"2023-08-28T02:23:06.138296+0000","last_clean":"2023-08-28T02:23:06.138296+0000","last_became_active":"2023-08-28T02:21:59.168430+0000","last_became_peered":"2023-08-28T02:21:59.168430+0000","last_unstale":"2023-08-28T02:23:06.138296+0000","last_undegraded":"2023-08-28T02:23:06.138296+0000","last_fullsized":"2023-08-28T02:23:06.138296+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:06:16.535160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":106,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.142871+0000","last_change":"2023-08-28T02:19:50.218760+0000","last_active":"2023-08-28T02:23:06.142871+0000","last_peered":"2023-08-28T02:23:06.142871+0000","last_clean":"2023-08-28T02:23:06.142871+0000","last_became_active":"2023-08-28T02:19:50.218376+0000","last_became_peered":"2023-08-28T02:19:50.218376+0000","last_unstale":"2023-08-28T02:23:06.142871+0000","last_undegraded":"2023-08-28T02:23:06.142871+0000","last_fullsized":"2023-08-28T02:23:06.142871+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":127,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.137479+0000","last_change":"2023-08-28T02:21:59.177791+0000","last_active":"2023-08-28T02:23:06.137479+0000","last_peered":"2023-08-28T02:23:06.137479+0000","last_clean":"2023-08-28T02:23:06.137479+0000","last_became_active":"2023-08-28T02:21:59.177436+0000","last_became_peered":"2023-08-28T02:21:59.177436+0000","last_unstale":"2023-08-28T02:23:06.137479+0000","last_undegraded":"2023-08-28T02:23:06.137479+0000","last_fullsized":"2023-08-28T02:23:06.137479+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":91,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.146406+0000","last_change":"2023-08-28T02:11:22.186024+0000","last_active":"2023-08-28T02:23:06.146406+0000","last_peered":"2023-08-28T02:23:06.146406+0000","last_clean":"2023-08-28T02:23:06.146406+0000","last_became_active":"2023-08-28T02:11:22.185500+0000","last_became_peered":"2023-08-28T02:11:22.185500+0000","last_unstale":"2023-08-28T02:23:06.146406+0000","last_undegraded":"2023-08-28T02:23:06.146406+0000","last_fullsized":"2023-08-28T02:23:06.146406+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":127,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.139416+0000","last_change":"2023-08-28T02:21:59.187663+0000","last_active":"2023-08-28T02:23:06.139416+0000","last_peered":"2023-08-28T02:23:06.139416+0000","last_clean":"2023-08-28T02:23:06.139416+0000","last_became_active":"2023-08-28T02:21:59.187366+0000","last_became_peered":"2023-08-28T02:21:59.187366+0000","last_unstale":"2023-08-28T02:23:06.139416+0000","last_undegraded":"2023-08-28T02:23:06.139416+0000","last_fullsized":"2023-08-28T02:23:06.139416+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":41,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.142217+0000","last_change":"2023-08-28T02:21:59.179758+0000","last_active":"2023-08-28T02:23:06.142217+0000","last_peered":"2023-08-28T02:23:06.142217+0000","last_clean":"2023-08-28T02:23:06.142217+0000","last_became_active":"2023-08-28T02:21:59.179436+0000","last_became_peered":"2023-08-28T02:21:59.179436+0000","last_unstale":"2023-08-28T02:23:06.142217+0000","last_undegraded":"2023-08-28T02:23:06.142217+0000","last_fullsized":"2023-08-28T02:23:06.142217+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":117,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:08.019040+0000","last_change":"2023-08-28T02:21:59.171738+0000","last_active":"2023-08-28T02:23:08.019040+0000","last_peered":"2023-08-28T02:23:08.019040+0000","last_clean":"2023-08-28T02:23:08.019040+0000","last_became_active":"2023-08-28T02:21:59.171422+0000","last_became_peered":"2023-08-28T02:21:59.171422+0000","last_unstale":"2023-08-28T02:23:08.019040+0000","last_undegraded":"2023-08-28T02:23:08.019040+0000","last_fullsized":"2023-08-28T02:23:08.019040+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:58:46.090765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":89,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.124909+0000","last_change":"2023-08-28T02:20:19.850252+0000","last_active":"2023-08-28T02:23:06.124909+0000","last_peered":"2023-08-28T02:23:06.124909+0000","last_clean":"2023-08-28T02:23:06.124909+0000","last_became_active":"2023-08-28T02:20:19.849913+0000","last_became_peered":"2023-08-28T02:20:19.849913+0000","last_unstale":"2023-08-28T02:23:06.124909+0000","last_undegraded":"2023-08-28T02:23:06.124909+0000","last_fullsized":"2023-08-28T02:23:06.124909+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:48:47.713465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":132,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-08-28T02:23:06.136668+0000","last_change":"2023-08-28T02:21:59.180201+0000","last_active":"2023-08-28T02:23:06.136668+0000","last_peered":"2023-08-28T02:23:06.136668+0000","last_clean":"2023-08-28T02:23:06.136668+0000","last_became_active":"2023-08-28T02:21:59.179863+0000","last_became_peered":"2023-08-28T02:21:59.179863+0000","last_unstale":"2023-08-28T02:23:06.136668+0000","last_undegraded":"2023-08-28T02:23:06.136668+0000","last_fullsized":"2023-08-28T02:23:06.136668+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":6396248064,"num_objects":97599,"num_object_clones":0,"num_object_copies":292797,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":97599,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":97599,"num_write_kb":6246336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":19167117312,"data_stored":19167117312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97599,"ondisk_log_size":97599,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":112,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16602,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26810,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430372,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2226168,"kb_used_data":1920752,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":91518984,"statfs":{"total":95995035648,"available":93715439616,"internally_reserved":0,"allocated":1966850048,"data_stored":1966327452,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,1,0,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:22:49 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.53900000000000003,"15min":0.53700000000000003},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":1.9370000000000001,"5min":1.9370000000000001,"15min":1.9370000000000001},"last":0.38},{"interface":"front","average":{"1min":1.0009999999999999,"5min":0.65900000000000003,"15min":0.628},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":4.4710000000000001,"5min":4.4710000000000001,"15min":4.4710000000000001},"last":2.2679999999999998}]},{"osd":1,"last update":"Mon Aug 28 02:22:55 2023","interfaces":[{"interface":"back","average":{"1min":1.123,"5min":0.65200000000000002,"15min":0.56899999999999995},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":4.306,"5min":4.306,"15min":4.306},"last":2.8570000000000002},{"interface":"front","average":{"1min":1.0449999999999999,"5min":0.67500000000000004,"15min":0.61699999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":3.3460000000000001,"5min":3.3460000000000001,"15min":3.3460000000000001},"last":1.671}]},{"osd":2,"last update":"Mon Aug 28 02:22:22 2023","interfaces":[{"interface":"back","average":{"1min":0.72699999999999998,"5min":0.60499999999999998,"15min":0.58499999999999996},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":2.0489999999999999,"5min":2.0489999999999999,"15min":2.0489999999999999},"last":2.9300000000000002},{"interface":"front","average":{"1min":0.79100000000000004,"5min":0.627,"15min":0.59899999999999998},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":3.1150000000000002,"5min":3.1150000000000002,"15min":3.1150000000000002},"last":2.1480000000000001}]},{"osd":3,"last update":"Mon Aug 28 02:22:55 2023","interfaces":[{"interface":"back","average":{"1min":1.1060000000000001,"5min":0.75600000000000001,"15min":0.69799999999999995},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.367,"5min":4.367,"15min":4.367},"last":0.45600000000000002},{"interface":"front","average":{"1min":1.0309999999999999,"5min":0.74199999999999999,"15min":0.69399999999999995},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":3.3159999999999998,"5min":3.3159999999999998,"15min":3.3159999999999998},"last":1.7}]},{"osd":4,"last update":"Mon Aug 28 02:22:22 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.51600000000000001,"15min":0.48599999999999999},"min":{"1min":0.28299999999999997,"5min":0.27700000000000002,"15min":0.25800000000000001},"max":{"1min":1.8779999999999999,"5min":1.8779999999999999,"15min":1.8779999999999999},"last":3.4289999999999998},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.53700000000000003,"15min":0.51000000000000001},"min":{"1min":0.316,"5min":0.29799999999999999,"15min":0.27100000000000002},"max":{"1min":1.8899999999999999,"5min":1.8899999999999999,"15min":1.8899999999999999},"last":2.081}]},{"osd":5,"last update":"Mon Aug 28 02:22:22 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.53200000000000003,"15min":0.495},"min":{"1min":0.33500000000000002,"5min":0.29899999999999999,"15min":0.28699999999999998},"max":{"1min":1.637,"5min":1.637,"15min":1.637},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.92300000000000004,"5min":0.65000000000000002,"15min":0.57099999999999995},"min":{"1min":0.35599999999999998,"5min":0.32200000000000001,"15min":0.314},"max":{"1min":3.1509999999999998,"5min":3.1509999999999998,"15min":3.1509999999999998},"last":1.7350000000000001}]},{"osd":6,"last update":"Mon Aug 28 02:22:22 2023","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.58399999999999996,"15min":0.54300000000000004},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.335,"5min":2.335,"15min":2.335},"last":0.878},{"interface":"front","average":{"1min":0.73099999999999998,"5min":0.64600000000000002,"15min":0.59499999999999997},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":1.8280000000000001,"5min":1.8280000000000001,"15min":1.8280000000000001},"last":2.5219999999999998}]}]},{"osd":6,"up_from":42,"seq":180388626599,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1841780,"kb_used_data":1540460,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":91903372,"statfs":{"total":95995035648,"available":94109052928,"internally_reserved":0,"allocated":1577431040,"data_stored":1576920079,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:22:57 2023","interfaces":[{"interface":"back","average":{"1min":1.5589999999999999,"5min":0.75900000000000001,"15min":0.63600000000000001},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":7.4790000000000001,"5min":7.4790000000000001,"15min":7.4790000000000001},"last":1.0529999999999999},{"interface":"front","average":{"1min":0.83899999999999997,"5min":0.626,"15min":0.60999999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":3.1819999999999999,"5min":3.1819999999999999,"15min":3.1819999999999999},"last":0.34699999999999998}]},{"osd":1,"last update":"Mon Aug 28 02:22:57 2023","interfaces":[{"interface":"back","average":{"1min":3.8769999999999998,"5min":1.2230000000000001,"15min":0.76500000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":34.677999999999997,"5min":34.677999999999997,"15min":34.677999999999997},"last":0.41599999999999998},{"interface":"front","average":{"1min":3.9540000000000002,"5min":1.2609999999999999,"15min":0.80000000000000004},"min":{"1min":0.33300000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":34.348999999999997,"5min":34.348999999999997,"15min":34.348999999999997},"last":0.38100000000000001}]},{"osd":2,"last update":"Mon Aug 28 02:22:26 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.59999999999999998,"15min":0.60599999999999998},"min":{"1min":0.39000000000000001,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.5349999999999999,"5min":1.575,"15min":1.575},"last":3.8650000000000002},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.56399999999999995,"15min":0.57199999999999995},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.89800000000000002,"5min":1.6419999999999999,"15min":1.6419999999999999},"last":6.1890000000000001}]},{"osd":3,"last update":"Mon Aug 28 02:22:57 2023","interfaces":[{"interface":"back","average":{"1min":0.997,"5min":0.73599999999999999,"15min":0.69199999999999995},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":4.2930000000000001,"5min":4.2930000000000001,"15min":4.2930000000000001},"last":0.57699999999999996},{"interface":"front","average":{"1min":1.546,"5min":0.85499999999999998,"15min":0.73999999999999999},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":7.9610000000000003,"5min":7.9610000000000003,"15min":7.9610000000000003},"last":0.58699999999999997}]},{"osd":4,"last update":"Mon Aug 28 02:23:01 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":0.59499999999999997,"15min":0.496},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":4.3540000000000001,"5min":4.3540000000000001,"15min":4.3540000000000001},"last":6.4770000000000003},{"interface":"front","average":{"1min":1.093,"5min":0.65100000000000002,"15min":0.53100000000000003},"min":{"1min":0.32200000000000001,"5min":0.318,"15min":0.28899999999999998},"max":{"1min":6.1420000000000003,"5min":6.1420000000000003,"15min":6.1420000000000003},"last":2.6309999999999998}]},{"osd":5,"last update":"Mon Aug 28 02:23:01 2023","interfaces":[{"interface":"back","average":{"1min":1.865,"5min":0.77700000000000002,"15min":0.56000000000000005},"min":{"1min":0.30099999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":8.8330000000000002,"5min":8.8330000000000002,"15min":8.8330000000000002},"last":9.625},{"interface":"front","average":{"1min":1.2949999999999999,"5min":0.71299999999999997,"15min":0.57599999999999996},"min":{"1min":0.34300000000000003,"5min":0.32500000000000001,"15min":0.28799999999999998},"max":{"1min":7.3780000000000001,"5min":7.3780000000000001,"15min":7.3780000000000001},"last":0.44600000000000001}]},{"osd":7,"last update":"Mon Aug 28 02:22:19 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.59599999999999997,"15min":0.55800000000000005},"min":{"1min":0.38,"5min":0.36799999999999999,"15min":0.32700000000000001},"max":{"1min":1.272,"5min":1.272,"15min":1.272},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.94599999999999995,"5min":0.67700000000000005,"15min":0.59999999999999998},"min":{"1min":0.41399999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":4.9210000000000003,"5min":4.9210000000000003,"15min":4.9210000000000003},"last":3.9009999999999998}]}]},{"osd":1,"up_from":98,"seq":420906795052,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2046428,"kb_used_data":1970452,"kb_used_omap":7,"kb_used_meta":75960,"kb_avail":91698724,"statfs":{"total":95995035648,"available":93899493376,"internally_reserved":0,"allocated":2017742848,"data_stored":2017087344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":77783371},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:22:55 2023","interfaces":[{"interface":"back","average":{"1min":1.4650000000000001,"5min":0.55700000000000005,"15min":0.40899999999999997},"min":{"1min":0.24199999999999999,"5min":0.222,"15min":0.222},"max":{"1min":6.5990000000000002,"5min":6.5990000000000002,"15min":6.5990000000000002},"last":6.6710000000000003},{"interface":"front","average":{"1min":2.2869999999999999,"5min":0.73699999999999999,"15min":0.48099999999999998},"min":{"1min":0.25700000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":12.574999999999999,"5min":12.574999999999999,"15min":12.574999999999999},"last":5.4580000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:22:25 2023","interfaces":[{"interface":"back","average":{"1min":0.70799999999999996,"5min":0.41499999999999998,"15min":0.36599999999999999},"min":{"1min":0.26300000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":2.2709999999999999,"5min":2.2709999999999999,"15min":2.2709999999999999},"last":3.702},{"interface":"front","average":{"1min":0.82199999999999995,"5min":0.46899999999999997,"15min":0.40999999999999998},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":9.2509999999999994,"5min":9.2509999999999994,"15min":9.2509999999999994},"last":9.6669999999999998}]},{"osd":3,"last update":"Mon Aug 28 02:22:55 2023","interfaces":[{"interface":"back","average":{"1min":1.647,"5min":0.621,"15min":0.45000000000000001},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":12.544,"5min":12.544,"15min":12.544},"last":6.2359999999999998},{"interface":"front","average":{"1min":1.6160000000000001,"5min":0.69199999999999995,"15min":0.53800000000000003},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":8.9160000000000004,"5min":8.9160000000000004,"15min":8.9160000000000004},"last":0.32300000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:22:55 2023","interfaces":[{"interface":"back","average":{"1min":1.1259999999999999,"5min":0.73899999999999999,"15min":0.66400000000000003},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":5.8529999999999998,"5min":5.8529999999999998,"15min":5.8529999999999998},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.90700000000000003,"5min":0.67300000000000004,"15min":0.63700000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":3.0499999999999998,"5min":3.0499999999999998,"15min":3.0499999999999998},"last":0.42999999999999999}]},{"osd":5,"last update":"Mon Aug 28 02:22:55 2023","interfaces":[{"interface":"back","average":{"1min":1.76,"5min":0.85299999999999998,"15min":0.70899999999999996},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":9.4469999999999992,"5min":9.4469999999999992,"15min":9.4469999999999992},"last":3.1139999999999999},{"interface":"front","average":{"1min":1.2230000000000001,"5min":0.76400000000000001,"15min":0.68200000000000005},"min":{"1min":0.36699999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":4.6639999999999997,"5min":4.6639999999999997,"15min":4.6639999999999997},"last":0.35999999999999999}]},{"osd":6,"last update":"Mon Aug 28 02:22:55 2023","interfaces":[{"interface":"back","average":{"1min":2.5870000000000002,"5min":1.0009999999999999,"15min":0.746},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":14.827,"5min":14.827,"15min":14.827},"last":3.0819999999999999},{"interface":"front","average":{"1min":1.1919999999999999,"5min":0.73799999999999999,"15min":0.65500000000000003},"min":{"1min":0.39000000000000001,"5min":0.371,"15min":0.371},"max":{"1min":4.468,"5min":4.468,"15min":4.468},"last":3.665}]},{"osd":7,"last update":"Mon Aug 28 02:22:55 2023","interfaces":[{"interface":"back","average":{"1min":0.94799999999999995,"5min":0.68700000000000006,"15min":0.65100000000000002},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":3.028,"5min":3.028,"15min":3.028},"last":0.45100000000000001},{"interface":"front","average":{"1min":1.748,"5min":0.85499999999999998,"15min":0.70999999999999996},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":9.5039999999999996,"5min":9.5039999999999996,"15min":9.5039999999999996},"last":3.1459999999999999}]}]},{"osd":0,"up_from":90,"seq":386547056698,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4393184,"kb_used_data":4293400,"kb_used_omap":8,"kb_used_meta":99767,"kb_avail":89351968,"statfs":{"total":95995035648,"available":91496415232,"internally_reserved":0,"allocated":4396441600,"data_stored":4395782258,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":102161769},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,3,5,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Aug 28 02:22:57 2023","interfaces":[{"interface":"back","average":{"1min":1.8200000000000001,"5min":0.64700000000000002,"15min":0.45100000000000001},"min":{"1min":0.186,"5min":0.186,"15min":0.186},"max":{"1min":11.115,"5min":11.115,"15min":11.115},"last":40.313000000000002},{"interface":"front","average":{"1min":0.97199999999999998,"5min":0.48299999999999998,"15min":0.40899999999999997},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":4.7999999999999998,"5min":4.7999999999999998,"15min":4.7999999999999998},"last":0.247}]},{"osd":2,"last update":"Mon Aug 28 02:22:21 2023","interfaces":[{"interface":"back","average":{"1min":0.80600000000000005,"5min":0.443,"15min":0.38300000000000001},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":3.7839999999999998,"5min":3.7839999999999998,"15min":3.7839999999999998},"last":1.6699999999999999},{"interface":"front","average":{"1min":1.069,"5min":0.52700000000000002,"15min":0.437},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":5.6849999999999996,"5min":5.6849999999999996,"15min":5.6849999999999996},"last":0.30399999999999999}]},{"osd":3,"last update":"Mon Aug 28 02:22:57 2023","interfaces":[{"interface":"back","average":{"1min":1.5,"5min":0.58999999999999997,"15min":0.438},"min":{"1min":0.26500000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":10.443,"5min":10.443,"15min":10.443},"last":2.573},{"interface":"front","average":{"1min":1.508,"5min":0.61599999999999999,"15min":0.46800000000000003},"min":{"1min":0.29799999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":10.548999999999999,"5min":10.548999999999999,"15min":10.548999999999999},"last":2.484}]},{"osd":4,"last update":"Mon Aug 28 02:22:53 2023","interfaces":[{"interface":"back","average":{"1min":1.2150000000000001,"5min":0.75600000000000001,"15min":0.69799999999999995},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":6.1180000000000003,"5min":6.1180000000000003,"15min":6.1180000000000003},"last":0.46600000000000003},{"interface":"front","average":{"1min":1.645,"5min":0.86099999999999999,"15min":0.75800000000000001},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":10.106999999999999,"5min":10.106999999999999,"15min":10.106999999999999},"last":0.32500000000000001}]},{"osd":5,"last update":"Mon Aug 28 02:22:53 2023","interfaces":[{"interface":"back","average":{"1min":1.758,"5min":0.84099999999999997,"15min":0.71599999999999997},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":9.9209999999999994,"5min":9.9209999999999994,"15min":9.9209999999999994},"last":0.36199999999999999},{"interface":"front","average":{"1min":1.5109999999999999,"5min":0.80300000000000005,"15min":0.70299999999999996},"min":{"1min":0.373,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":5.0179999999999998,"5min":5.0179999999999998,"15min":5.0179999999999998},"last":0.54100000000000004}]},{"osd":6,"last update":"Mon Aug 28 02:22:53 2023","interfaces":[{"interface":"back","average":{"1min":1.8580000000000001,"5min":0.87,"15min":0.72699999999999998},"min":{"1min":0.36299999999999999,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":10.073,"5min":10.073,"15min":10.073},"last":2.444},{"interface":"front","average":{"1min":1.1599999999999999,"5min":0.77600000000000002,"15min":0.72999999999999998},"min":{"1min":0.39900000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":3.7160000000000002,"5min":3.7160000000000002,"15min":3.7160000000000002},"last":3.0640000000000001}]},{"osd":7,"last update":"Mon Aug 28 02:22:53 2023","interfaces":[{"interface":"back","average":{"1min":1.446,"5min":0.82099999999999995,"15min":0.73999999999999999},"min":{"1min":0.41399999999999998,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":6.0700000000000003,"5min":6.0700000000000003,"15min":6.0700000000000003},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.88400000000000001,"5min":0.70699999999999996,"15min":0.70499999999999996},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":2.762,"5min":2.762,"15min":2.762},"last":0.42799999999999999}]}]},{"osd":2,"up_from":104,"seq":446676598822,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2395492,"kb_used_data":2327772,"kb_used_omap":10,"kb_used_meta":67701,"kb_avail":91349660,"statfs":{"total":95995035648,"available":93542051840,"internally_reserved":0,"allocated":2383638528,"data_stored":2383030008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10792,"internal_metadata":69326296},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:22:25 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":0.39200000000000002,"15min":0.34399999999999997},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":4.149,"5min":4.149,"15min":4.149},"last":2.2450000000000001},{"interface":"front","average":{"1min":0.81599999999999995,"5min":0.45400000000000001,"15min":0.39400000000000002},"min":{"1min":0.246,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":5.1289999999999996,"5min":5.1289999999999996,"15min":5.1289999999999996},"last":0.45600000000000002}]},{"osd":1,"last update":"Mon Aug 28 02:22:25 2023","interfaces":[{"interface":"back","average":{"1min":2.3780000000000001,"5min":0.76200000000000001,"15min":0.49299999999999999},"min":{"1min":0.28199999999999997,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":30.888000000000002,"5min":30.888000000000002,"15min":30.888000000000002},"last":0.219},{"interface":"front","average":{"1min":2.2149999999999999,"5min":0.70999999999999996,"15min":0.45900000000000002},"min":{"1min":0.27100000000000002,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":31.648,"5min":31.648,"15min":31.648},"last":1.238}]},{"osd":3,"last update":"Mon Aug 28 02:22:53 2023","interfaces":[{"interface":"back","average":{"1min":1.2270000000000001,"5min":0.57099999999999995,"15min":0.46200000000000002},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.9969999999999999,"5min":4.9969999999999999,"15min":4.9969999999999999},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.70399999999999996,"5min":0.45000000000000001,"15min":0.40799999999999997},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":2.3999999999999999,"5min":2.3999999999999999,"15min":2.3999999999999999},"last":0.35999999999999999}]},{"osd":4,"last update":"Mon Aug 28 02:22:25 2023","interfaces":[{"interface":"back","average":{"1min":0.79300000000000004,"5min":0.63300000000000001,"15min":0.60699999999999998},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":2.492,"5min":2.492,"15min":2.492},"last":0.30599999999999999},{"interface":"front","average":{"1min":1.069,"5min":0.73299999999999998,"15min":0.67700000000000005},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":4.1840000000000002,"5min":4.1840000000000002,"15min":4.1840000000000002},"last":5.7080000000000002}]},{"osd":5,"last update":"Mon Aug 28 02:22:25 2023","interfaces":[{"interface":"back","average":{"1min":0.93899999999999995,"5min":0.68100000000000005,"15min":0.63800000000000001},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":5.0880000000000001,"5min":5.0880000000000001,"15min":5.0880000000000001},"last":1.2010000000000001},{"interface":"front","average":{"1min":0.70099999999999996,"5min":0.58699999999999997,"15min":0.56799999999999995},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":2.5430000000000001,"5min":2.5430000000000001,"15min":2.5430000000000001},"last":2.298}]},{"osd":6,"last update":"Mon Aug 28 02:22:25 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":0.67400000000000004,"15min":0.625},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":4.2279999999999998,"5min":4.2279999999999998,"15min":4.2279999999999998},"last":6.9139999999999997},{"interface":"front","average":{"1min":1.0529999999999999,"5min":0.70299999999999996,"15min":0.64500000000000002},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":5.1559999999999997,"5min":5.1559999999999997,"15min":5.1559999999999997},"last":0.42499999999999999}]},{"osd":7,"last update":"Mon Aug 28 02:22:25 2023","interfaces":[{"interface":"back","average":{"1min":1.0129999999999999,"5min":0.68200000000000005,"15min":0.627},"min":{"1min":0.41099999999999998,"5min":0.372,"15min":0.372},"max":{"1min":4.0670000000000002,"5min":4.0670000000000002,"15min":4.0670000000000002},"last":9.0909999999999993},{"interface":"front","average":{"1min":1.0940000000000001,"5min":0.70199999999999996,"15min":0.63700000000000001},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":5.181,"5min":5.181,"15min":5.181},"last":0.51400000000000001}]}]},{"osd":3,"up_from":0,"seq":481036337184,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019057,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3427444,"kb_used_data":3117932,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":90317708,"statfs":{"total":95995035648,"available":92485332992,"internally_reserved":0,"allocated":3192762368,"data_stored":3192251213,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:22:49 2023","interfaces":[{"interface":"back","average":{"1min":0.751,"5min":0.60899999999999999,"15min":0.61399999999999999},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":5.5039999999999996,"5min":5.5039999999999996,"15min":5.5039999999999996},"last":0.24199999999999999},{"interface":"front","average":{"1min":1.0589999999999999,"5min":0.65200000000000002,"15min":0.61699999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":3.0299999999999998,"5min":3.0299999999999998,"15min":3.0299999999999998},"last":0.39900000000000002}]},{"osd":1,"last update":"Mon Aug 28 02:22:54 2023","interfaces":[{"interface":"back","average":{"1min":1.244,"5min":0.66300000000000003,"15min":0.56899999999999995},"min":{"1min":0.28100000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.5670000000000002,"5min":5.5670000000000002,"15min":5.5670000000000002},"last":0.31},{"interface":"front","average":{"1min":1.002,"5min":0.64200000000000002,"15min":0.58999999999999997},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":2.7850000000000001,"5min":2.7850000000000001,"15min":2.7850000000000001},"last":3.4460000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:22:22 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.55700000000000005,"15min":0.55200000000000005},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.5629999999999999,"5min":1.5629999999999999,"15min":1.5629999999999999},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.56100000000000005,"15min":0.54700000000000004},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":1.478,"5min":1.478,"15min":1.478},"last":4.875}]},{"osd":3,"last update":"Mon Aug 28 02:22:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":0.60799999999999998,"15min":0.53600000000000003},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":5.0970000000000004,"5min":5.0970000000000004,"15min":5.0970000000000004},"last":1.46},{"interface":"front","average":{"1min":1.4330000000000001,"5min":0.753,"15min":0.64000000000000001},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":5.6529999999999996,"5min":5.6529999999999996,"15min":5.6529999999999996},"last":1.276}]},{"osd":5,"last update":"Mon Aug 28 02:22:36 2023","interfaces":[{"interface":"back","average":{"1min":0.86799999999999999,"5min":0.58099999999999996,"15min":0.48099999999999998},"min":{"1min":0.316,"5min":0.316,"15min":0.26600000000000001},"max":{"1min":4.5129999999999999,"5min":4.5129999999999999,"15min":4.5129999999999999},"last":0.44600000000000001},{"interface":"front","average":{"1min":1.0329999999999999,"5min":0.61399999999999999,"15min":0.504},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.27700000000000002},"max":{"1min":6.0899999999999999,"5min":6.0899999999999999,"15min":6.0899999999999999},"last":0.315}]},{"osd":6,"last update":"Mon Aug 28 02:22:54 2023","interfaces":[{"interface":"back","average":{"1min":1.262,"5min":0.69199999999999995,"15min":0.54700000000000004},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.27500000000000002},"max":{"1min":9.6920000000000002,"5min":9.6920000000000002,"15min":9.6920000000000002},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.97999999999999998,"5min":0.67500000000000004,"15min":0.56399999999999995},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.318},"max":{"1min":2.4569999999999999,"5min":2.4569999999999999,"15min":2.4569999999999999},"last":0.49299999999999999}]},{"osd":7,"last update":"Mon Aug 28 02:22:19 2023","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":0.59999999999999998,"15min":0.54800000000000004},"min":{"1min":0.40799999999999997,"5min":0.36899999999999999,"15min":0.312},"max":{"1min":3.2320000000000002,"5min":3.2320000000000002,"15min":3.2320000000000002},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.58999999999999997,"15min":0.54800000000000004},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32100000000000001},"max":{"1min":0.73999999999999999,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.53100000000000003}]}]},{"osd":5,"up_from":36,"seq":154618822829,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3776484,"kb_used_data":3475164,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":89968668,"statfs":{"total":95995035648,"available":92127916032,"internally_reserved":0,"allocated":3558567936,"data_stored":3558068320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:22:46 2023","interfaces":[{"interface":"back","average":{"1min":8.7870000000000008,"5min":2.2210000000000001,"15min":1.157},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":131.06100000000001,"5min":131.06100000000001,"15min":131.06100000000001},"last":1.6120000000000001},{"interface":"front","average":{"1min":8.9380000000000006,"5min":2.2650000000000001,"15min":1.173},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":131.262,"5min":131.262,"15min":131.262},"last":0.83099999999999996}]},{"osd":1,"last update":"Mon Aug 28 02:22:54 2023","interfaces":[{"interface":"back","average":{"1min":13.494,"5min":3.113,"15min":1.389},"min":{"1min":0.36899999999999999,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":183.97900000000001,"5min":183.97900000000001,"15min":183.97900000000001},"last":1.147},{"interface":"front","average":{"1min":13.288,"5min":3.0830000000000002,"15min":1.3879999999999999},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":184.33699999999999,"5min":184.33699999999999,"15min":184.33699999999999},"last":0.80600000000000005}]},{"osd":2,"last update":"Mon Aug 28 02:22:23 2023","interfaces":[{"interface":"back","average":{"1min":5.3550000000000004,"5min":1.5429999999999999,"15min":0.90700000000000003},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":72.944999999999993,"5min":72.944999999999993,"15min":72.944999999999993},"last":1.5840000000000001},{"interface":"front","average":{"1min":5.3710000000000004,"5min":1.544,"15min":0.90600000000000003},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":72.930999999999997,"5min":72.930999999999997,"15min":72.930999999999997},"last":4.7549999999999999}]},{"osd":3,"last update":"Mon Aug 28 02:22:54 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":0.622,"15min":0.61499999999999999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":1.603,"5min":1.603,"15min":1.603},"last":3.0979999999999999},{"interface":"front","average":{"1min":1.2210000000000001,"5min":0.745,"15min":0.66600000000000004},"min":{"1min":0.36199999999999999,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":3.3450000000000002,"5min":3.3450000000000002,"15min":3.3450000000000002},"last":0.91500000000000004}]},{"osd":4,"last update":"Mon Aug 28 02:22:40 2023","interfaces":[{"interface":"back","average":{"1min":1.1140000000000001,"5min":0.623,"15min":0.50700000000000001},"min":{"1min":0.38400000000000001,"5min":0.32200000000000001,"15min":0.27500000000000002},"max":{"1min":4.2089999999999996,"5min":4.2089999999999996,"15min":4.2089999999999996},"last":2.0840000000000001},{"interface":"front","average":{"1min":0.96999999999999997,"5min":0.61299999999999999,"15min":0.52100000000000002},"min":{"1min":0.33200000000000002,"5min":0.31900000000000001,"15min":0.28199999999999997},"max":{"1min":4.0659999999999998,"5min":4.0659999999999998,"15min":4.0659999999999998},"last":1.5109999999999999}]},{"osd":6,"last update":"Mon Aug 28 02:22:54 2023","interfaces":[{"interface":"back","average":{"1min":1.7110000000000001,"5min":0.76600000000000001,"15min":0.60399999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.28399999999999997},"max":{"1min":4.8239999999999998,"5min":4.8239999999999998,"15min":5.2119999999999997},"last":1.1870000000000001},{"interface":"front","average":{"1min":0.88500000000000001,"5min":0.64200000000000002,"15min":0.55200000000000005},"min":{"1min":0.31,"5min":0.31,"15min":0.30299999999999999},"max":{"1min":3.2999999999999998,"5min":3.2999999999999998,"15min":3.2999999999999998},"last":1.651}]},{"osd":7,"last update":"Mon Aug 28 02:23:15 2023","interfaces":[{"interface":"back","average":{"1min":1.4490000000000001,"5min":0.76800000000000002,"15min":0.61699999999999999},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":5.109,"5min":5.109,"15min":5.109},"last":1.5489999999999999},{"interface":"front","average":{"1min":1.6200000000000001,"5min":0.78900000000000003,"15min":0.624},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.29799999999999999},"max":{"1min":6.8710000000000004,"5min":6.8710000000000004,"15min":6.8710000000000004},"last":0.77600000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4424663040,"data_stored":4424663040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2026373120,"data_stored":2026373120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2401959936,"data_stored":2401959936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3192520704,"data_stored":3192520704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3565617152,"data_stored":3565617152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1577713664,"data_stored":1577713664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1978269696,"data_stored":1978269696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-08-28T02:23:20.600 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-08-28T02:23:20.600 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-08-28T02:23:20.600 INFO:tasks.thrashosds.thrasher:Removing osd 7, in_osds are: [0, 1, 2, 4, 5, 6, 7] 2023-08-28T02:23:20.600 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd out 7 2023-08-28T02:23:20.739 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:23:20.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:20.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:23:20.849 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:23:20.877 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:21.205 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:23:21.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:21.213 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:23:21.281 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 74 16 101042 101026 85.2647 59.4375 0.0157095 0.0117228 2023-08-28T02:23:21.314 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:23:21.319 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.4 dump_ops_in_flight 2023-08-28T02:23:21.400 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:23:21.404 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:21.553 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:23:21.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:21.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:23:21.615 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:23:21.660 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:23:21.725 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:21.862 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:23:21.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:21.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:23:21.950 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:23:21 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:23:21] "GET /metrics HTTP/1.1" 200 36474 "" "Prometheus/2.33.4" 2023-08-28T02:23:21.979 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:23:22.031 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:22.042 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:23:22.232 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:22 smithi143 ceph-mon[183059]: pgmap v284: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 57 MiB/s wr, 911 op/s 2023-08-28T02:23:22.236 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:22 smithi143 ceph-mon[189871]: pgmap v284: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 57 MiB/s wr, 911 op/s 2023-08-28T02:23:22.281 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 75 16 101939 101923 84.8755 56.0625 0.0043307 0.0117756 2023-08-28T02:23:22.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:22 smithi169 ceph-mon[159415]: pgmap v284: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 57 MiB/s wr, 911 op/s 2023-08-28T02:23:22.522 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:23:22.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:22.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:23:22.627 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:23:22.695 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:22.841 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:23:22.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:22.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:23:22.955 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:23:23.021 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:23.094 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:23:23.281 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 76 16 102808 102792 84.4735 54.3125 0.00717133 0.0118326 2023-08-28T02:23:23.447 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:23:23.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:23.451 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:23:23.552 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:23:23.602 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:23.810 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:23:23.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:23.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:23:23.933 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:23:24.009 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:24.024 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:23:24.102 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:23:24.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:24.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:23:24.208 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:23:24.262 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:24.284 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 77 16 103629 103613 84.0393 51.3125 0.0128479 0.0118935 2023-08-28T02:23:24.366 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:23:24.366 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:23:24.382 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:24 smithi143 ceph-mon[183059]: pgmap v285: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 56 MiB/s wr, 892 op/s 2023-08-28T02:23:24.383 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:24 smithi143 ceph-mon[189871]: pgmap v285: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 56 MiB/s wr, 892 op/s 2023-08-28T02:23:24.399 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:24 smithi169 ceph-mon[159415]: pgmap v285: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 56 MiB/s wr, 892 op/s 2023-08-28T02:23:24.402 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:23:24.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:24.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:23:24.513 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:23:24.575 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:24.676 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:23:24.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:24.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:23:24.780 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:23:24.832 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:24.875 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:23:24.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:24.877 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:23:24.978 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:23:25.028 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:25.097 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:23:25.101 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:24 smithi143 conmon[183035]: 2023-08-28T02:23:24.801+0000 7fadc3887700 -1 mon.a@0(leader).osd e137 definitely_dead 0 2023-08-28T02:23:25.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:25.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:23:25.200 INFO:teuthology.orchestra.run.smithi143.stderr:marked out osd.7. 2023-08-28T02:23:25.200 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is set 2023-08-28T02:23:25.204 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:23:25.235 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:25.284 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 78 16 104307 104291 83.5054 42.375 0.00382981 0.011963 2023-08-28T02:23:25.286 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.4 dump_blocked_ops 2023-08-28T02:23:25.298 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:23:25.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:25.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:23:25.402 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:23:25.425 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:25 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3472885967' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:23:25.425 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:25 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/897198728' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-08-28T02:23:25.425 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:25 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-08-28T02:23:25.426 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:25 smithi143 ceph-mon[183059]: Client client.admin marked osd.7 out, while it was still marked up 2023-08-28T02:23:25.426 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:25 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3472885967' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:23:25.426 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:25 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/897198728' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-08-28T02:23:25.426 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:25 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-08-28T02:23:25.426 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:25 smithi143 ceph-mon[189871]: Client client.admin marked osd.7 out, while it was still marked up 2023-08-28T02:23:25.427 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:25.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:25 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3472885967' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:23:25.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:25 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/897198728' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-08-28T02:23:25.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:25 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-08-28T02:23:25.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:25 smithi169 ceph-mon[159415]: Client client.admin marked osd.7 out, while it was still marked up 2023-08-28T02:23:25.626 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:23:26.073 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:23:26.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:26.077 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:23:26.178 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:23:26.229 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:26.273 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:23:26.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:26.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:23:26.284 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 79 16 104313 104297 82.4538 0.375 0.00354592 0.0119626 2023-08-28T02:23:26.376 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:23:26.405 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:26.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:26 smithi143 ceph-mon[183059]: pgmap v286: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 56 MiB/s wr, 893 op/s 2023-08-28T02:23:26.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:26 smithi143 ceph-mon[183059]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:23:26.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:26 smithi143 ceph-mon[183059]: failed to encode map e138 with expected crc 2023-08-28T02:23:26.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:26 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3472885967' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:23:26.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:26 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-08-28T02:23:26.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:26 smithi143 ceph-mon[183059]: osdmap e138: 8 total, 8 up, 6 in 2023-08-28T02:23:26.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:26 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:23:26.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:26 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:23:26.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:26 smithi143 ceph-mon[183059]: osdmap e139: 8 total, 8 up, 6 in 2023-08-28T02:23:26.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:26 smithi143 ceph-mon[189871]: pgmap v286: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 56 MiB/s wr, 893 op/s 2023-08-28T02:23:26.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:26 smithi143 ceph-mon[189871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:23:26.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:26 smithi143 ceph-mon[189871]: failed to encode map e138 with expected crc 2023-08-28T02:23:26.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:26 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3472885967' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:23:26.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:26 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-08-28T02:23:26.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:26 smithi143 ceph-mon[189871]: osdmap e138: 8 total, 8 up, 6 in 2023-08-28T02:23:26.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:26 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:23:26.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:26 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:23:26.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:26 smithi143 ceph-mon[189871]: osdmap e139: 8 total, 8 up, 6 in 2023-08-28T02:23:26.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:26 smithi169 ceph-mon[159415]: pgmap v286: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 56 MiB/s wr, 893 op/s 2023-08-28T02:23:26.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:26 smithi169 ceph-mon[159415]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:23:26.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:26 smithi169 ceph-mon[159415]: failed to encode map e138 with expected crc 2023-08-28T02:23:26.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:26 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3472885967' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:23:26.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:26 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-08-28T02:23:26.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:26 smithi169 ceph-mon[159415]: osdmap e138: 8 total, 8 up, 6 in 2023-08-28T02:23:26.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:26 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:23:26.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:26 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:23:26.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:26 smithi169 ceph-mon[159415]: osdmap e139: 8 total, 8 up, 6 in 2023-08-28T02:23:26.651 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:23:26.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:26.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:23:26.754 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:23:26.783 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:26.827 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:23:26.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:26.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:23:26.843 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:23:26.861 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:23:26.879 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:23:26.897 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:23:26.914 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:23:26.930 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:23:26.932 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:23:26.949 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:23:26.958 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:26.968 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:23:26.969 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:23:26.969 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:23:26.970 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:23:26.970 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:23:26.970 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:23:26.970 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:23:26.970 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:23:26.970 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:23:27.002 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:23:27.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:27.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:23:27.106 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:23:27.132 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:27.230 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:23:27.230 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:27 smithi143 ceph-mon[189871]: failed to encode map e139 with expected crc 2023-08-28T02:23:27.231 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:27 smithi143 ceph-mon[189871]: failed to encode map e139 with expected crc 2023-08-28T02:23:27.231 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:27 smithi143 ceph-mon[189871]: failed to encode map e139 with expected crc 2023-08-28T02:23:27.231 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:27 smithi143 ceph-mon[189871]: failed to encode map e139 with expected crc 2023-08-28T02:23:27.231 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:27 smithi143 ceph-mon[189871]: failed to encode map e139 with expected crc 2023-08-28T02:23:27.231 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:27 smithi143 ceph-mon[189871]: failed to encode map e139 with expected crc 2023-08-28T02:23:27.231 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:27 smithi143 ceph-mon[189871]: failed to encode map e139 with expected crc 2023-08-28T02:23:27.232 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:27 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3708175812' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:23:27.232 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:27 smithi143 ceph-mon[189871]: osdmap e140: 8 total, 8 up, 6 in 2023-08-28T02:23:27.232 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:27 smithi143 ceph-mon[183059]: failed to encode map e139 with expected crc 2023-08-28T02:23:27.232 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:27 smithi143 ceph-mon[183059]: failed to encode map e139 with expected crc 2023-08-28T02:23:27.233 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:27 smithi143 ceph-mon[183059]: failed to encode map e139 with expected crc 2023-08-28T02:23:27.233 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:27 smithi143 ceph-mon[183059]: failed to encode map e139 with expected crc 2023-08-28T02:23:27.233 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:27 smithi143 ceph-mon[183059]: failed to encode map e139 with expected crc 2023-08-28T02:23:27.233 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:27 smithi143 ceph-mon[183059]: failed to encode map e139 with expected crc 2023-08-28T02:23:27.233 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:27 smithi143 ceph-mon[183059]: failed to encode map e139 with expected crc 2023-08-28T02:23:27.233 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:27 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3708175812' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:23:27.234 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:27 smithi143 ceph-mon[183059]: osdmap e140: 8 total, 8 up, 6 in 2023-08-28T02:23:27.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:27.235 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:23:27.285 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:2023-08-28T02:23:27.286365+0000 min lat: 0.00111735 max lat: 0.221599 avg lat: 0.0119626 2023-08-28T02:23:27.285 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-08-28T02:23:27.285 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 80 16 104313 104297 81.4237 0 - 0.0119626 2023-08-28T02:23:27.335 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:23:27.361 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:27.376 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:23:27.377 INFO:teuthology.orchestra.run.smithi169.stdout: "ops": [], 2023-08-28T02:23:27.377 INFO:teuthology.orchestra.run.smithi169.stdout: "complaint_time": 30, 2023-08-28T02:23:27.377 INFO:teuthology.orchestra.run.smithi169.stdout: "num_blocked_ops": 0 2023-08-28T02:23:27.377 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:23:27.405 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:27 smithi169 ceph-mon[159415]: failed to encode map e139 with expected crc 2023-08-28T02:23:27.405 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:27 smithi169 ceph-mon[159415]: failed to encode map e139 with expected crc 2023-08-28T02:23:27.406 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:27 smithi169 ceph-mon[159415]: failed to encode map e139 with expected crc 2023-08-28T02:23:27.406 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:27 smithi169 ceph-mon[159415]: failed to encode map e139 with expected crc 2023-08-28T02:23:27.406 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:27 smithi169 ceph-mon[159415]: failed to encode map e139 with expected crc 2023-08-28T02:23:27.406 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:27 smithi169 ceph-mon[159415]: failed to encode map e139 with expected crc 2023-08-28T02:23:27.406 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:27 smithi169 ceph-mon[159415]: failed to encode map e139 with expected crc 2023-08-28T02:23:27.406 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:27 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3708175812' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:23:27.406 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:27 smithi169 ceph-mon[159415]: osdmap e140: 8 total, 8 up, 6 in 2023-08-28T02:23:27.413 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:23:27.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:27.426 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:23:27.527 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:23:27.575 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:27.754 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:23:27.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:27.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:23:27.861 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:23:27.887 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:27.929 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:23:27.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:27.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:23:28.036 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:23:28.098 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:28.285 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 81 16 104721 104705 80.7337 12.75 0.00973119 0.0123811 2023-08-28T02:23:28.331 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:23:28.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:28.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:23:28.371 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.4 dump_historic_ops 2023-08-28T02:23:28.373 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:28 smithi169 ceph-mon[159415]: pgmap v289: 89 pgs: 4 remapped+peering, 85 active+clean; 6.3 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 44 MiB/s wr, 702 op/s; 604 B/s, 1 keys/s, 0 objects/s recovering 2023-08-28T02:23:28.373 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:28 smithi169 ceph-mon[159415]: failed to encode map e140 with expected crc 2023-08-28T02:23:28.373 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:28 smithi169 ceph-mon[159415]: failed to encode map e140 with expected crc 2023-08-28T02:23:28.373 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:28 smithi169 ceph-mon[159415]: failed to encode map e140 with expected crc 2023-08-28T02:23:28.373 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:28 smithi169 ceph-mon[159415]: failed to encode map e140 with expected crc 2023-08-28T02:23:28.373 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:28 smithi169 ceph-mon[159415]: failed to encode map e140 with expected crc 2023-08-28T02:23:28.373 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:28 smithi169 ceph-mon[159415]: failed to encode map e140 with expected crc 2023-08-28T02:23:28.374 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:28 smithi169 ceph-mon[159415]: failed to encode map e140 with expected crc 2023-08-28T02:23:28.374 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:28 smithi169 ceph-mon[159415]: failed to encode map e140 with expected crc 2023-08-28T02:23:28.374 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:28 smithi169 ceph-mon[159415]: failed to encode map e140 with expected crc 2023-08-28T02:23:28.374 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:28 smithi169 ceph-mon[159415]: failed to encode map e140 with expected crc 2023-08-28T02:23:28.374 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:28 smithi169 ceph-mon[159415]: failed to encode map e140 with expected crc 2023-08-28T02:23:28.374 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:28 smithi169 ceph-mon[159415]: osdmap e141: 8 total, 8 up, 6 in 2023-08-28T02:23:28.439 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:23:28.506 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:28.555 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset noscrub 2023-08-28T02:23:28.599 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:23:28.609 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:28 smithi143 ceph-mon[189871]: pgmap v289: 89 pgs: 4 remapped+peering, 85 active+clean; 6.3 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 44 MiB/s wr, 702 op/s; 604 B/s, 1 keys/s, 0 objects/s recovering 2023-08-28T02:23:28.609 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:28 smithi143 ceph-mon[189871]: failed to encode map e140 with expected crc 2023-08-28T02:23:28.609 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:28 smithi143 ceph-mon[189871]: failed to encode map e140 with expected crc 2023-08-28T02:23:28.609 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:28 smithi143 ceph-mon[189871]: failed to encode map e140 with expected crc 2023-08-28T02:23:28.609 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:28 smithi143 ceph-mon[189871]: failed to encode map e140 with expected crc 2023-08-28T02:23:28.609 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:28 smithi143 ceph-mon[189871]: failed to encode map e140 with expected crc 2023-08-28T02:23:28.610 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:28 smithi143 ceph-mon[189871]: failed to encode map e140 with expected crc 2023-08-28T02:23:28.610 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:28 smithi143 ceph-mon[189871]: failed to encode map e140 with expected crc 2023-08-28T02:23:28.610 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:28 smithi143 ceph-mon[189871]: failed to encode map e140 with expected crc 2023-08-28T02:23:28.610 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:28 smithi143 ceph-mon[189871]: failed to encode map e140 with expected crc 2023-08-28T02:23:28.610 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:28 smithi143 ceph-mon[189871]: failed to encode map e140 with expected crc 2023-08-28T02:23:28.610 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:28 smithi143 ceph-mon[189871]: failed to encode map e140 with expected crc 2023-08-28T02:23:28.610 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:28 smithi143 ceph-mon[189871]: osdmap e141: 8 total, 8 up, 6 in 2023-08-28T02:23:28.612 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:28 smithi143 ceph-mon[183059]: pgmap v289: 89 pgs: 4 remapped+peering, 85 active+clean; 6.3 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 44 MiB/s wr, 702 op/s; 604 B/s, 1 keys/s, 0 objects/s recovering 2023-08-28T02:23:28.612 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:28 smithi143 ceph-mon[183059]: failed to encode map e140 with expected crc 2023-08-28T02:23:28.612 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:28 smithi143 ceph-mon[183059]: failed to encode map e140 with expected crc 2023-08-28T02:23:28.612 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:28 smithi143 ceph-mon[183059]: failed to encode map e140 with expected crc 2023-08-28T02:23:28.612 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:28 smithi143 ceph-mon[183059]: failed to encode map e140 with expected crc 2023-08-28T02:23:28.612 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:28 smithi143 ceph-mon[183059]: failed to encode map e140 with expected crc 2023-08-28T02:23:28.612 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:28 smithi143 ceph-mon[183059]: failed to encode map e140 with expected crc 2023-08-28T02:23:28.613 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:28 smithi143 ceph-mon[183059]: failed to encode map e140 with expected crc 2023-08-28T02:23:28.613 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:28 smithi143 ceph-mon[183059]: failed to encode map e140 with expected crc 2023-08-28T02:23:28.613 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:28 smithi143 ceph-mon[183059]: failed to encode map e140 with expected crc 2023-08-28T02:23:28.613 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:28 smithi143 ceph-mon[183059]: failed to encode map e140 with expected crc 2023-08-28T02:23:28.613 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:28 smithi143 ceph-mon[183059]: failed to encode map e140 with expected crc 2023-08-28T02:23:28.613 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:28 smithi143 ceph-mon[183059]: osdmap e141: 8 total, 8 up, 6 in 2023-08-28T02:23:28.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:28.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:23:28.715 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:23:28.793 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:28.889 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:23:28.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:28.896 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:23:28.967 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:23:28.997 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:23:29.074 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:29.188 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:23:29.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:29.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:23:29.206 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:23:29.285 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 82 16 105403 105387 80.2691 42.625 0.0309397 0.0124517 2023-08-28T02:23:29.296 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:23:29.316 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:29 smithi169 ceph-mon[159415]: failed to encode map e141 with expected crc 2023-08-28T02:23:29.316 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:29 smithi169 ceph-mon[159415]: failed to encode map e141 with expected crc 2023-08-28T02:23:29.316 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:29 smithi169 ceph-mon[159415]: failed to encode map e141 with expected crc 2023-08-28T02:23:29.316 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:29 smithi169 ceph-mon[159415]: osdmap e142: 8 total, 8 up, 6 in 2023-08-28T02:23:29.369 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:29.539 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:29 smithi143 ceph-mon[189871]: failed to encode map e141 with expected crc 2023-08-28T02:23:29.540 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:29 smithi143 ceph-mon[189871]: failed to encode map e141 with expected crc 2023-08-28T02:23:29.540 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:29 smithi143 ceph-mon[189871]: failed to encode map e141 with expected crc 2023-08-28T02:23:29.540 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:29 smithi143 ceph-mon[189871]: osdmap e142: 8 total, 8 up, 6 in 2023-08-28T02:23:29.540 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:29 smithi143 ceph-mon[183059]: failed to encode map e141 with expected crc 2023-08-28T02:23:29.540 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:29 smithi143 ceph-mon[183059]: failed to encode map e141 with expected crc 2023-08-28T02:23:29.540 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:29 smithi143 ceph-mon[183059]: failed to encode map e141 with expected crc 2023-08-28T02:23:29.540 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:29 smithi143 ceph-mon[183059]: osdmap e142: 8 total, 8 up, 6 in 2023-08-28T02:23:29.603 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:23:29.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:29.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:23:29.713 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:23:29.774 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:23:29.806 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:29.884 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:23:29.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:29.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:23:29.987 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:23:30.076 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:30.194 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:23:30.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:30.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:23:30.285 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 83 16 106115 106099 79.8384 44.5 0.0198962 0.0125199 2023-08-28T02:23:30.301 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:23:30.364 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:30.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:30 smithi143 ceph-mon[183059]: pgmap v292: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 4 remapped+peering, 78 active+clean; 6.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 35 MiB/s wr, 553 op/s; 19419/313467 objects degraded (6.195%); 1.2 MiB/s, 2 keys/s, 7 objects/s recovering 2023-08-28T02:23:30.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:30 smithi143 ceph-mon[183059]: failed to encode map e142 with expected crc 2023-08-28T02:23:30.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:30 smithi143 ceph-mon[183059]: failed to encode map e142 with expected crc 2023-08-28T02:23:30.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:30 smithi143 ceph-mon[183059]: failed to encode map e142 with expected crc 2023-08-28T02:23:30.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:30 smithi143 ceph-mon[183059]: Health check failed: Degraded data redundancy: 19419/313467 objects degraded (6.195%), 4 pgs degraded (PG_DEGRADED) 2023-08-28T02:23:30.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:30 smithi143 ceph-mon[189871]: pgmap v292: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 4 remapped+peering, 78 active+clean; 6.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 35 MiB/s wr, 553 op/s; 19419/313467 objects degraded (6.195%); 1.2 MiB/s, 2 keys/s, 7 objects/s recovering 2023-08-28T02:23:30.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:30 smithi143 ceph-mon[189871]: failed to encode map e142 with expected crc 2023-08-28T02:23:30.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:30 smithi143 ceph-mon[189871]: failed to encode map e142 with expected crc 2023-08-28T02:23:30.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:30 smithi143 ceph-mon[189871]: failed to encode map e142 with expected crc 2023-08-28T02:23:30.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:30 smithi143 ceph-mon[189871]: Health check failed: Degraded data redundancy: 19419/313467 objects degraded (6.195%), 4 pgs degraded (PG_DEGRADED) 2023-08-28T02:23:30.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:30 smithi169 ceph-mon[159415]: pgmap v292: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 4 remapped+peering, 78 active+clean; 6.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 35 MiB/s wr, 553 op/s; 19419/313467 objects degraded (6.195%); 1.2 MiB/s, 2 keys/s, 7 objects/s recovering 2023-08-28T02:23:30.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:30 smithi169 ceph-mon[159415]: failed to encode map e142 with expected crc 2023-08-28T02:23:30.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:30 smithi169 ceph-mon[159415]: failed to encode map e142 with expected crc 2023-08-28T02:23:30.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:30 smithi169 ceph-mon[159415]: failed to encode map e142 with expected crc 2023-08-28T02:23:30.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:30 smithi169 ceph-mon[159415]: Health check failed: Degraded data redundancy: 19419/313467 objects degraded (6.195%), 4 pgs degraded (PG_DEGRADED) 2023-08-28T02:23:30.978 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:23:31.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:31.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:23:31.106 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:23:31.125 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:23:31.210 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:31.285 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 84 16 106705 106689 79.3272 36.875 0.0108669 0.0125997 2023-08-28T02:23:31.348 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:23:31.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:31.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:23:31.461 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:23:31.494 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6] out_osds: [3, 7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-08-28T02:23:31.495 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-08-28T02:23:31.495 INFO:tasks.thrashosds.thrasher:Adding osd 7 2023-08-28T02:23:31.495 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd in 7 2023-08-28T02:23:31.504 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:23:31.504 INFO:teuthology.orchestra.run.smithi169.stdout: "size": 20, 2023-08-28T02:23:31.504 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 600, 2023-08-28T02:23:31.504 INFO:teuthology.orchestra.run.smithi169.stdout: "ops": [ 2023-08-28T02:23:31.504 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.504 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:6092 5.1 5:8f589190:::benchmark_data_smithi143_236200_object6091:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-08-28T02:23:31.504 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:22:08.993631+0000", 2023-08-28T02:23:31.504 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 82.504165685000004, 2023-08-28T02:23:31.505 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.164224859, 2023-08-28T02:23:31.505 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:31.505 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:31.505 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:31.505 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:31.505 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:31.505 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 6092 2023-08-28T02:23:31.505 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.505 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:31.505 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.505 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:31.506 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.993631+0000", 2023-08-28T02:23:31.506 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:31.506 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.506 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.506 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:31.506 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.993631+0000", 2023-08-28T02:23:31.506 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:23:31.506 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.506 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.506 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:31.506 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.993629+0000", 2023-08-28T02:23:31.507 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00016439699999999999 2023-08-28T02:23:31.507 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.507 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.507 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:31.507 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.993794+0000", 2023-08-28T02:23:31.507 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.5781e-05 2023-08-28T02:23:31.507 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.507 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.507 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:31.507 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.993809+0000", 2023-08-28T02:23:31.507 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.7460000000000003e-06 2023-08-28T02:23:31.508 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.508 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.508 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:31.508 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.993816+0000", 2023-08-28T02:23:31.508 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.3161000000000002e-05 2023-08-28T02:23:31.508 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.508 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.508 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:31.508 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.993859+0000", 2023-08-28T02:23:31.508 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012430699999999999 2023-08-28T02:23:31.509 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.509 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.509 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:31.509 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.993984+0000", 2023-08-28T02:23:31.509 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000146439 2023-08-28T02:23:31.509 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.509 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.509 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,7", 2023-08-28T02:23:31.509 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.994130+0000", 2023-08-28T02:23:31.509 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.001298963 2023-08-28T02:23:31.510 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.510 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.510 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:31.510 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.995429+0000", 2023-08-28T02:23:31.510 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000976394 2023-08-28T02:23:31.510 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.510 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.510 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:31.510 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.996405+0000", 2023-08-28T02:23:31.510 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.16134261599999999 2023-08-28T02:23:31.511 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.511 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.511 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:31.511 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:09.157748+0000", 2023-08-28T02:23:31.511 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.2463000000000002e-05 2023-08-28T02:23:31.511 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.511 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.511 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:31.511 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:09.157801+0000", 2023-08-28T02:23:31.511 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.5027e-05 2023-08-28T02:23:31.512 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.512 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.512 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:31.512 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:09.157856+0000", 2023-08-28T02:23:31.512 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.164224859 2023-08-28T02:23:31.512 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:31.512 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:31.512 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:31.512 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.512 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.513 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:6096 5.1 5:8eeb403e:::benchmark_data_smithi143_236200_object6095:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-08-28T02:23:31.513 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:22:08.995577+0000", 2023-08-28T02:23:31.513 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 82.502218908000003, 2023-08-28T02:23:31.513 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.16241417, 2023-08-28T02:23:31.513 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:31.513 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:31.513 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:31.513 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:31.513 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:31.513 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 6096 2023-08-28T02:23:31.514 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.514 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:31.514 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.514 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:31.514 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.995577+0000", 2023-08-28T02:23:31.514 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:31.514 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.514 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.514 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:31.515 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.995577+0000", 2023-08-28T02:23:31.515 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:23:31.515 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.515 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.515 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:31.515 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.995576+0000", 2023-08-28T02:23:31.515 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.2591e-05 2023-08-28T02:23:31.515 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.515 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.515 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:31.516 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.995609+0000", 2023-08-28T02:23:31.516 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.5361000000000001e-05 2023-08-28T02:23:31.516 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.516 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.516 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:31.516 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.995624+0000", 2023-08-28T02:23:31.516 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.3270000000000003e-06 2023-08-28T02:23:31.516 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.516 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.517 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:31.517 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.995631+0000", 2023-08-28T02:23:31.517 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.6556000000000004e-05 2023-08-28T02:23:31.517 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.517 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.517 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:31.517 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.995708+0000", 2023-08-28T02:23:31.517 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012302400000000001 2023-08-28T02:23:31.517 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.517 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.518 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:31.518 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.995831+0000", 2023-08-28T02:23:31.518 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000183124 2023-08-28T02:23:31.518 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.518 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.518 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,7", 2023-08-28T02:23:31.518 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.996014+0000", 2023-08-28T02:23:31.518 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.001072019 2023-08-28T02:23:31.518 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.518 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.519 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:31.519 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.997086+0000", 2023-08-28T02:23:31.519 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00092508399999999995 2023-08-28T02:23:31.519 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.519 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.519 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:31.519 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.998011+0000", 2023-08-28T02:23:31.519 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.15989225100000001 2023-08-28T02:23:31.519 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.519 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.519 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:31.520 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:09.157903+0000", 2023-08-28T02:23:31.520 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.9172999999999998e-05 2023-08-28T02:23:31.520 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.520 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.520 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:31.520 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:09.157952+0000", 2023-08-28T02:23:31.520 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.9140000000000001e-05 2023-08-28T02:23:31.520 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.520 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.521 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:31.521 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:09.157992+0000", 2023-08-28T02:23:31.521 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.16241417 2023-08-28T02:23:31.521 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:31.521 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:31.521 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:31.521 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.521 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.521 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:6098 5.1 5:86b0d884:::benchmark_data_smithi143_236200_object6097:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-08-28T02:23:31.522 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:22:08.996177+0000", 2023-08-28T02:23:31.522 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 82.501619551000005, 2023-08-28T02:23:31.522 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.16194524099999999, 2023-08-28T02:23:31.522 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:31.522 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:31.522 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:31.522 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:31.522 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:31.522 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 6098 2023-08-28T02:23:31.522 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.523 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:31.523 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.523 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:31.523 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.996177+0000", 2023-08-28T02:23:31.523 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:31.523 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.523 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.523 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:31.523 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.996177+0000", 2023-08-28T02:23:31.523 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:23:31.524 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.524 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.524 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:31.524 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.996176+0000", 2023-08-28T02:23:31.524 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00010009399999999999 2023-08-28T02:23:31.524 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.524 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.524 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:31.524 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.996276+0000", 2023-08-28T02:23:31.525 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1245e-05 2023-08-28T02:23:31.525 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.525 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.525 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:31.525 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.996287+0000", 2023-08-28T02:23:31.525 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.3400000000000003e-06 2023-08-28T02:23:31.525 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.525 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.525 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:31.525 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.996294+0000", 2023-08-28T02:23:31.526 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000141471 2023-08-28T02:23:31.526 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.526 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.526 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:31.526 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.996435+0000", 2023-08-28T02:23:31.526 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00011576199999999999 2023-08-28T02:23:31.526 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.526 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.526 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:31.527 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.996551+0000", 2023-08-28T02:23:31.527 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00017802000000000001 2023-08-28T02:23:31.527 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.527 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.527 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,7", 2023-08-28T02:23:31.527 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.996729+0000", 2023-08-28T02:23:31.527 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0011748259999999999 2023-08-28T02:23:31.527 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.527 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.527 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:31.528 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.997904+0000", 2023-08-28T02:23:31.528 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00081944699999999995 2023-08-28T02:23:31.528 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.528 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.528 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:31.528 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.998723+0000", 2023-08-28T02:23:31.528 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.15931441399999999 2023-08-28T02:23:31.528 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.528 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.528 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:31.528 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:09.158037+0000", 2023-08-28T02:23:31.529 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.7741999999999998e-05 2023-08-28T02:23:31.529 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.529 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.529 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:31.529 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:09.158085+0000", 2023-08-28T02:23:31.529 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.6860999999999998e-05 2023-08-28T02:23:31.529 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.529 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.529 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:31.530 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:09.158122+0000", 2023-08-28T02:23:31.530 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.16194524099999999 2023-08-28T02:23:31.530 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:31.530 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:31.530 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:31.530 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.530 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.530 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:6101 5.1 5:82184f45:::benchmark_data_smithi143_236200_object6100:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-08-28T02:23:31.530 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:22:08.997201+0000", 2023-08-28T02:23:31.530 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 82.500595136000001, 2023-08-28T02:23:31.531 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.16104807900000001, 2023-08-28T02:23:31.531 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:31.531 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:31.531 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:31.531 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:31.531 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:31.531 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 6101 2023-08-28T02:23:31.531 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.531 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:31.531 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.532 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:31.532 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.997201+0000", 2023-08-28T02:23:31.532 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:31.532 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.532 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.533 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:31.533 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.997201+0000", 2023-08-28T02:23:31.533 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:23:31.534 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.534 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.534 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:31.534 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.997200+0000", 2023-08-28T02:23:31.534 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.9884999999999999e-05 2023-08-28T02:23:31.534 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.534 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.534 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:31.534 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.997230+0000", 2023-08-28T02:23:31.534 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.6045e-05 2023-08-28T02:23:31.535 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.535 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.535 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:31.535 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.997246+0000", 2023-08-28T02:23:31.535 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.8950000000000001e-06 2023-08-28T02:23:31.535 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.535 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.535 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:31.535 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.997253+0000", 2023-08-28T02:23:31.535 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.7203999999999998e-05 2023-08-28T02:23:31.536 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.536 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.536 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:31.536 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.997290+0000", 2023-08-28T02:23:31.536 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000120172 2023-08-28T02:23:31.536 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.536 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.536 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:31.536 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.997411+0000", 2023-08-28T02:23:31.537 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00017984300000000001 2023-08-28T02:23:31.537 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.537 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.537 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,7", 2023-08-28T02:23:31.537 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.997590+0000", 2023-08-28T02:23:31.537 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00059586000000000003 2023-08-28T02:23:31.537 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.537 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.537 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:31.537 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.998186+0000", 2023-08-28T02:23:31.538 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00056894900000000002 2023-08-28T02:23:31.538 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.538 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.538 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:31.538 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.998755+0000", 2023-08-28T02:23:31.538 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.15941250200000001 2023-08-28T02:23:31.538 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.538 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.538 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:31.538 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:09.158168+0000", 2023-08-28T02:23:31.539 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.7942999999999998e-05 2023-08-28T02:23:31.539 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.539 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.539 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:31.539 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:09.158216+0000", 2023-08-28T02:23:31.539 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.3646999999999998e-05 2023-08-28T02:23:31.539 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.539 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.539 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:31.540 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:09.158249+0000", 2023-08-28T02:23:31.540 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.16104807900000001 2023-08-28T02:23:31.540 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:31.540 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:31.540 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:31.540 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.540 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.540 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:17316 5.e 5:73595770:::benchmark_data_smithi143_236200_object17315:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e125)", 2023-08-28T02:23:31.540 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:22:13.611417+0000", 2023-08-28T02:23:31.540 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 77.886379331000001, 2023-08-28T02:23:31.540 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.16042120200000001, 2023-08-28T02:23:31.541 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:31.541 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:31.541 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:31.541 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:31.541 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:31.541 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 17316 2023-08-28T02:23:31.541 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.541 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:31.541 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.541 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:31.542 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:13.611417+0000", 2023-08-28T02:23:31.542 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:31.542 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.542 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.542 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:31.542 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:13.611417+0000", 2023-08-28T02:23:31.542 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:23:31.542 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.542 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.542 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:31.543 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:13.611415+0000", 2023-08-28T02:23:31.543 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.1131999999999998e-05 2023-08-28T02:23:31.543 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.543 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.543 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:31.543 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:13.611466+0000", 2023-08-28T02:23:31.543 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.6076999999999999e-05 2023-08-28T02:23:31.543 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.543 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.543 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:31.544 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:13.611482+0000", 2023-08-28T02:23:31.544 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.6559999999999998e-06 2023-08-28T02:23:31.544 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.544 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.544 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:31.544 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:13.611491+0000", 2023-08-28T02:23:31.544 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.2317999999999997e-05 2023-08-28T02:23:31.544 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.544 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.544 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:31.544 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:13.611533+0000", 2023-08-28T02:23:31.545 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000124602 2023-08-28T02:23:31.545 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.545 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.545 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:31.545 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:13.611658+0000", 2023-08-28T02:23:31.545 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00016459100000000001 2023-08-28T02:23:31.545 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.545 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.545 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:23:31.545 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:13.611823+0000", 2023-08-28T02:23:31.546 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0015854809999999999 2023-08-28T02:23:31.546 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.546 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.546 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:31.546 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:13.613408+0000", 2023-08-28T02:23:31.546 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00094831600000000005 2023-08-28T02:23:31.546 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.546 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.546 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:31.546 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:13.614356+0000", 2023-08-28T02:23:31.546 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.15740479099999999 2023-08-28T02:23:31.547 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.547 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.547 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:31.547 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:13.771761+0000", 2023-08-28T02:23:31.547 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.3578999999999998e-05 2023-08-28T02:23:31.547 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.547 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.547 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:31.547 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:13.771795+0000", 2023-08-28T02:23:31.547 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.3531e-05 2023-08-28T02:23:31.547 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.548 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.548 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:31.548 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:13.771838+0000", 2023-08-28T02:23:31.548 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.16042120200000001 2023-08-28T02:23:31.548 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:31.548 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:31.548 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:31.548 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.548 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.548 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:17317 5.e 5:7684f0b7:::benchmark_data_smithi143_236200_object17316:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e125)", 2023-08-28T02:23:31.549 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:22:13.611748+0000", 2023-08-28T02:23:31.549 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 77.886048508000002, 2023-08-28T02:23:31.549 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.16018710899999999, 2023-08-28T02:23:31.549 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:31.549 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:31.549 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:31.549 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:31.549 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:31.549 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 17317 2023-08-28T02:23:31.550 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.550 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:31.550 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.550 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:31.550 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:13.611748+0000", 2023-08-28T02:23:31.550 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:31.550 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.550 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.550 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:31.550 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:13.611748+0000", 2023-08-28T02:23:31.550 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:23:31.551 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.551 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.551 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:31.551 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:13.611747+0000", 2023-08-28T02:23:31.551 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.0168999999999999e-05 2023-08-28T02:23:31.551 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.551 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.551 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:31.551 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:13.611777+0000", 2023-08-28T02:23:31.552 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0059e-05 2023-08-28T02:23:31.552 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.552 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.552 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:31.552 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:13.611787+0000", 2023-08-28T02:23:31.552 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.6459999999999998e-06 2023-08-28T02:23:31.552 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.552 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.552 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:31.552 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:13.611793+0000", 2023-08-28T02:23:31.552 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00073734500000000004 2023-08-28T02:23:31.553 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.553 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.553 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:31.553 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:13.612530+0000", 2023-08-28T02:23:31.553 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00011790000000000001 2023-08-28T02:23:31.553 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.553 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.553 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:31.553 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:13.612648+0000", 2023-08-28T02:23:31.554 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00019735799999999999 2023-08-28T02:23:31.554 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.554 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.554 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:23:31.554 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:13.612846+0000", 2023-08-28T02:23:31.554 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00074504599999999997 2023-08-28T02:23:31.554 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.554 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.554 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:31.554 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:13.613591+0000", 2023-08-28T02:23:31.554 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00078894999999999996 2023-08-28T02:23:31.555 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.555 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.555 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:31.555 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:13.614380+0000", 2023-08-28T02:23:31.555 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.15749091800000001 2023-08-28T02:23:31.555 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.555 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.555 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:31.555 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:13.771870+0000", 2023-08-28T02:23:31.556 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.7949e-05 2023-08-28T02:23:31.556 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.556 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.556 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:31.556 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:13.771898+0000", 2023-08-28T02:23:31.556 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.6573999999999999e-05 2023-08-28T02:23:31.556 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.556 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.556 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:31.557 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:13.771935+0000", 2023-08-28T02:23:31.557 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.16018710899999999 2023-08-28T02:23:31.557 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:31.557 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:31.557 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:31.557 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.557 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.557 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:72074 5.e 5:70511453:::benchmark_data_smithi143_236200_object72073:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e132)", 2023-08-28T02:23:31.557 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:22:51.486630+0000", 2023-08-28T02:23:31.557 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 40.011166832999997, 2023-08-28T02:23:31.558 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.16569899599999999, 2023-08-28T02:23:31.558 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:31.558 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:31.558 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:31.558 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:31.558 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:31.558 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 72074 2023-08-28T02:23:31.558 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.558 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:31.558 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.558 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:31.559 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.486630+0000", 2023-08-28T02:23:31.559 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:31.559 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.559 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.559 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:31.559 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.486630+0000", 2023-08-28T02:23:31.559 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:23:31.559 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.559 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.560 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:31.560 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.486627+0000", 2023-08-28T02:23:31.560 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.9319e-05 2023-08-28T02:23:31.560 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.560 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.560 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:31.560 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.486677+0000", 2023-08-28T02:23:31.560 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2948e-05 2023-08-28T02:23:31.560 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.560 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.561 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:31.561 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.486690+0000", 2023-08-28T02:23:31.561 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.7159999999999997e-06 2023-08-28T02:23:31.561 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.561 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.561 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:31.561 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.486696+0000", 2023-08-28T02:23:31.561 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00013991399999999999 2023-08-28T02:23:31.561 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.561 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.562 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:31.562 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.486836+0000", 2023-08-28T02:23:31.562 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000113594 2023-08-28T02:23:31.562 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.562 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.562 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:31.562 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.486950+0000", 2023-08-28T02:23:31.562 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00013785499999999999 2023-08-28T02:23:31.562 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.562 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.563 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:23:31.563 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.487088+0000", 2023-08-28T02:23:31.563 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0035394329999999998 2023-08-28T02:23:31.563 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.563 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.563 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:31.563 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.490627+0000", 2023-08-28T02:23:31.563 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0089625850000000003 2023-08-28T02:23:31.563 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.564 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.564 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:31.564 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.499590+0000", 2023-08-28T02:23:31.564 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.152635154 2023-08-28T02:23:31.564 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.564 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.564 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:31.564 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.652225+0000", 2023-08-28T02:23:31.564 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.6184000000000003e-05 2023-08-28T02:23:31.564 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.565 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.566 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:31.566 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.652281+0000", 2023-08-28T02:23:31.566 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.7391000000000001e-05 2023-08-28T02:23:31.566 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.566 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.566 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:31.566 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.652329+0000", 2023-08-28T02:23:31.567 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.16569899599999999 2023-08-28T02:23:31.567 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:31.567 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:31.567 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:31.567 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.567 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.567 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:72075 5.1 5:84d92808:::benchmark_data_smithi143_236200_object72074:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e132)", 2023-08-28T02:23:31.567 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:22:51.490259+0000", 2023-08-28T02:23:31.567 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 40.007537495999998, 2023-08-28T02:23:31.567 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.15438840500000001, 2023-08-28T02:23:31.568 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:31.568 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:31.568 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:31.568 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:31.568 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:31.568 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 72075 2023-08-28T02:23:31.568 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.568 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:31.568 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.568 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:31.569 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.490259+0000", 2023-08-28T02:23:31.569 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:31.569 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.569 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.569 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:31.569 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.490259+0000", 2023-08-28T02:23:31.569 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:23:31.569 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.569 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.569 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:31.569 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.490257+0000", 2023-08-28T02:23:31.570 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.0193e-05 2023-08-28T02:23:31.570 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.570 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.570 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:31.570 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.490298+0000", 2023-08-28T02:23:31.570 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.6427e-05 2023-08-28T02:23:31.570 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.570 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.570 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:31.571 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.490314+0000", 2023-08-28T02:23:31.571 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.0700000000000001e-06 2023-08-28T02:23:31.571 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.571 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.571 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:31.571 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.490321+0000", 2023-08-28T02:23:31.571 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.5729999999999997e-05 2023-08-28T02:23:31.571 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.571 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.572 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:31.572 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.490377+0000", 2023-08-28T02:23:31.572 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00014726600000000001 2023-08-28T02:23:31.572 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.572 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.572 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:31.572 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.490524+0000", 2023-08-28T02:23:31.572 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00017321700000000001 2023-08-28T02:23:31.572 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.572 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.572 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,7", 2023-08-28T02:23:31.573 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.490697+0000", 2023-08-28T02:23:31.573 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00318301 2023-08-28T02:23:31.573 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.573 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.573 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:31.573 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.493880+0000", 2023-08-28T02:23:31.573 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00023066699999999999 2023-08-28T02:23:31.573 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.573 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.573 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:31.573 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.494111+0000", 2023-08-28T02:23:31.573 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.15044105699999999 2023-08-28T02:23:31.574 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.574 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.574 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:31.574 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.644552+0000", 2023-08-28T02:23:31.574 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.2586999999999997e-05 2023-08-28T02:23:31.574 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.574 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.574 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:31.574 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.644595+0000", 2023-08-28T02:23:31.574 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.2661999999999998e-05 2023-08-28T02:23:31.574 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.575 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.575 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:31.575 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.644647+0000", 2023-08-28T02:23:31.575 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.15438840500000001 2023-08-28T02:23:31.575 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:31.575 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:31.575 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:31.575 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.575 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.575 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:72077 5.e 5:7ccf544d:::benchmark_data_smithi143_236200_object72076:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e132)", 2023-08-28T02:23:31.576 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:22:51.492538+0000", 2023-08-28T02:23:31.576 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 40.005258093000002, 2023-08-28T02:23:31.576 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.15993012500000001, 2023-08-28T02:23:31.576 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:31.576 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:31.576 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:31.576 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:31.576 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:31.576 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 72077 2023-08-28T02:23:31.576 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.576 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:31.577 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.577 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:31.577 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.492538+0000", 2023-08-28T02:23:31.577 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:31.577 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.577 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.577 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:31.577 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.492538+0000", 2023-08-28T02:23:31.577 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:23:31.577 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.578 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.578 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:31.578 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.492537+0000", 2023-08-28T02:23:31.578 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.7723000000000002e-05 2023-08-28T02:23:31.578 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.578 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.578 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:31.578 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.492604+0000", 2023-08-28T02:23:31.578 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.7759000000000001e-05 2023-08-28T02:23:31.578 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.579 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.579 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:31.579 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.492622+0000", 2023-08-28T02:23:31.579 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.9260000000000003e-06 2023-08-28T02:23:31.579 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.579 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.579 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:31.579 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.492629+0000", 2023-08-28T02:23:31.579 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.6986e-05 2023-08-28T02:23:31.579 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.580 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.580 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:31.580 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.492676+0000", 2023-08-28T02:23:31.580 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00015568900000000001 2023-08-28T02:23:31.580 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.580 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.580 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:31.580 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.492832+0000", 2023-08-28T02:23:31.580 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00018415400000000001 2023-08-28T02:23:31.580 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.581 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.581 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:23:31.581 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.493016+0000", 2023-08-28T02:23:31.581 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0049924679999999999 2023-08-28T02:23:31.581 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.581 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.581 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:31.581 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.498008+0000", 2023-08-28T02:23:31.581 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.001609164 2023-08-28T02:23:31.581 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.582 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.582 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:31.582 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.499617+0000", 2023-08-28T02:23:31.582 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.15276901100000001 2023-08-28T02:23:31.582 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.582 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.582 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:31.582 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.652386+0000", 2023-08-28T02:23:31.582 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.6070999999999999e-05 2023-08-28T02:23:31.582 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.583 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.583 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:31.583 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.652442+0000", 2023-08-28T02:23:31.583 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.5916999999999999e-05 2023-08-28T02:23:31.583 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.583 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.583 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:31.583 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.652468+0000", 2023-08-28T02:23:31.583 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.15993012500000001 2023-08-28T02:23:31.583 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:31.584 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:31.584 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:31.584 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.584 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.584 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:72081 5.e 5:702ec139:::benchmark_data_smithi143_236200_object72080:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e132)", 2023-08-28T02:23:31.584 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:22:51.494435+0000", 2023-08-28T02:23:31.584 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 40.003361710999997, 2023-08-28T02:23:31.584 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.15819324900000001, 2023-08-28T02:23:31.584 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:31.584 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:31.584 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:31.585 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:31.585 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:31.585 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 72081 2023-08-28T02:23:31.585 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.585 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:31.585 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.585 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:31.585 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.494435+0000", 2023-08-28T02:23:31.585 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:31.585 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.585 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.586 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:31.586 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.494435+0000", 2023-08-28T02:23:31.586 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:23:31.586 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.586 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.586 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:31.586 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.494433+0000", 2023-08-28T02:23:31.586 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.2079e-05 2023-08-28T02:23:31.586 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.586 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.586 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:31.586 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.494495+0000", 2023-08-28T02:23:31.587 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0601000000000001e-05 2023-08-28T02:23:31.587 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.587 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.587 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:31.587 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.494506+0000", 2023-08-28T02:23:31.587 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.7659999999999998e-06 2023-08-28T02:23:31.587 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.587 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.587 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:31.587 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.494511+0000", 2023-08-28T02:23:31.588 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00086822699999999995 2023-08-28T02:23:31.588 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.588 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.588 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:31.588 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.495379+0000", 2023-08-28T02:23:31.588 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000100559 2023-08-28T02:23:31.588 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.588 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.588 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:31.588 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.495479+0000", 2023-08-28T02:23:31.588 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012765300000000001 2023-08-28T02:23:31.588 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.589 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.589 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:23:31.589 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.495607+0000", 2023-08-28T02:23:31.589 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0025638269999999999 2023-08-28T02:23:31.589 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.589 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.589 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:31.589 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.498171+0000", 2023-08-28T02:23:31.589 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.001474043 2023-08-28T02:23:31.589 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.589 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.590 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:31.590 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.499645+0000", 2023-08-28T02:23:31.590 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.152895424 2023-08-28T02:23:31.590 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.590 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.590 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:31.590 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.652540+0000", 2023-08-28T02:23:31.590 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.9073999999999998e-05 2023-08-28T02:23:31.590 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.590 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.591 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:31.591 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.652589+0000", 2023-08-28T02:23:31.591 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.8531000000000001e-05 2023-08-28T02:23:31.591 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.591 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.591 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:31.591 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.652628+0000", 2023-08-28T02:23:31.591 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.15819324900000001 2023-08-28T02:23:31.591 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:31.591 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:31.592 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:31.592 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.592 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.592 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:72094 5.1 5:84c76282:::benchmark_data_smithi143_236200_object72093:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e132)", 2023-08-28T02:23:31.592 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:22:51.508951+0000", 2023-08-28T02:23:31.592 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 39.988845085000001, 2023-08-28T02:23:31.592 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.13893274799999999, 2023-08-28T02:23:31.592 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:31.592 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:31.592 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:31.593 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:31.593 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:31.593 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 72094 2023-08-28T02:23:31.593 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.593 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:31.593 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.593 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:31.593 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.508951+0000", 2023-08-28T02:23:31.593 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:31.593 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.593 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.594 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:31.594 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.508951+0000", 2023-08-28T02:23:31.594 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999914 2023-08-28T02:23:31.594 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.594 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.594 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:31.594 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.508943+0000", 2023-08-28T02:23:31.594 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.0431e-05 2023-08-28T02:23:31.594 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.594 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.595 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:31.595 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.508983+0000", 2023-08-28T02:23:31.595 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.7116999999999999e-05 2023-08-28T02:23:31.595 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.595 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.595 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:31.595 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.509000+0000", 2023-08-28T02:23:31.595 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.1489999999999997e-06 2023-08-28T02:23:31.595 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.595 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.596 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:31.596 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.509007+0000", 2023-08-28T02:23:31.596 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.6174000000000001e-05 2023-08-28T02:23:31.596 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.596 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.596 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:31.596 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.509064+0000", 2023-08-28T02:23:31.596 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000177133 2023-08-28T02:23:31.596 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.596 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.597 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:31.597 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.509241+0000", 2023-08-28T02:23:31.597 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000176983 2023-08-28T02:23:31.597 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.597 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.597 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,7", 2023-08-28T02:23:31.597 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.509418+0000", 2023-08-28T02:23:31.597 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0021259030000000002 2023-08-28T02:23:31.597 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.597 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.598 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:31.598 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.511544+0000", 2023-08-28T02:23:31.598 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00027018000000000002 2023-08-28T02:23:31.598 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.598 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.598 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:31.598 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.511814+0000", 2023-08-28T02:23:31.598 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.13596242 2023-08-28T02:23:31.598 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.598 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.599 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:31.599 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.647776+0000", 2023-08-28T02:23:31.599 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.7017e-05 2023-08-28T02:23:31.599 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.599 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.599 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:31.599 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.647833+0000", 2023-08-28T02:23:31.599 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.0781000000000001e-05 2023-08-28T02:23:31.599 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.599 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.599 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:31.600 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.647884+0000", 2023-08-28T02:23:31.600 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.13893274799999999 2023-08-28T02:23:31.600 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:31.600 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:31.600 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:31.600 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.600 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.600 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:72095 5.e 5:70250475:::benchmark_data_smithi143_236200_object72094:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e132)", 2023-08-28T02:23:31.601 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:22:51.509051+0000", 2023-08-28T02:23:31.601 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 39.988745068999997, 2023-08-28T02:23:31.601 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.148486697, 2023-08-28T02:23:31.601 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:31.601 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:31.601 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:31.601 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:31.601 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:31.601 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 72095 2023-08-28T02:23:31.601 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.602 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:31.602 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.602 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:31.602 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.509051+0000", 2023-08-28T02:23:31.602 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:31.602 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.602 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.602 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:31.602 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.509051+0000", 2023-08-28T02:23:31.602 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999909 2023-08-28T02:23:31.602 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.603 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.603 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:31.603 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.509042+0000", 2023-08-28T02:23:31.603 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.3120999999999994e-05 2023-08-28T02:23:31.603 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.603 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.603 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:31.603 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.509105+0000", 2023-08-28T02:23:31.603 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.7510000000000008e-06 2023-08-28T02:23:31.603 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.604 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.604 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:31.604 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.509113+0000", 2023-08-28T02:23:31.604 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.9470000000000002e-06 2023-08-28T02:23:31.604 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.604 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.604 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:31.604 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.509118+0000", 2023-08-28T02:23:31.604 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.0628999999999998e-05 2023-08-28T02:23:31.604 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.604 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.604 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:31.605 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.509179+0000", 2023-08-28T02:23:31.605 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000112297 2023-08-28T02:23:31.605 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.605 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.605 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:31.605 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.509291+0000", 2023-08-28T02:23:31.605 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000132507 2023-08-28T02:23:31.605 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.605 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.605 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:23:31.606 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.509423+0000", 2023-08-28T02:23:31.606 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0024930080000000001 2023-08-28T02:23:31.606 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.606 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.606 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:31.606 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.511916+0000", 2023-08-28T02:23:31.606 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00044497500000000001 2023-08-28T02:23:31.606 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.606 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.606 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:31.607 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.512361+0000", 2023-08-28T02:23:31.607 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.14509175499999999 2023-08-28T02:23:31.607 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.607 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.607 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:31.607 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.657453+0000", 2023-08-28T02:23:31.607 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.9057999999999997e-05 2023-08-28T02:23:31.607 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.607 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.607 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:31.608 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.657502+0000", 2023-08-28T02:23:31.608 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.5877000000000002e-05 2023-08-28T02:23:31.608 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.608 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.608 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:31.608 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.657538+0000", 2023-08-28T02:23:31.608 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.148486697 2023-08-28T02:23:31.608 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:31.608 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:31.608 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:31.608 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.609 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.609 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:73614 5.e 5:7235874c:::benchmark_data_smithi143_236200_object73613:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e133)", 2023-08-28T02:23:31.609 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:22:53.034952+0000", 2023-08-28T02:23:31.609 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 38.462844021999999, 2023-08-28T02:23:31.609 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.12804637599999999, 2023-08-28T02:23:31.609 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:31.609 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:31.609 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:31.609 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:31.609 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:31.610 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 73614 2023-08-28T02:23:31.610 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.610 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:31.610 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.610 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:31.610 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:53.034952+0000", 2023-08-28T02:23:31.610 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:31.610 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.610 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.610 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:31.611 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:53.034952+0000", 2023-08-28T02:23:31.611 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:23:31.611 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.611 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.611 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:31.611 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:53.034951+0000", 2023-08-28T02:23:31.611 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.9333999999999997e-05 2023-08-28T02:23:31.611 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.611 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.611 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:31.612 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:53.035020+0000", 2023-08-28T02:23:31.612 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2505e-05 2023-08-28T02:23:31.612 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.612 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.612 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:31.612 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:53.035032+0000", 2023-08-28T02:23:31.612 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.635e-06 2023-08-28T02:23:31.612 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.612 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.612 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:31.612 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:53.035039+0000", 2023-08-28T02:23:31.613 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.9327999999999998e-05 2023-08-28T02:23:31.613 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.613 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.613 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:31.613 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:53.035108+0000", 2023-08-28T02:23:31.613 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00015893999999999999 2023-08-28T02:23:31.613 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.613 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.613 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:31.613 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:53.035267+0000", 2023-08-28T02:23:31.614 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00015425700000000001 2023-08-28T02:23:31.614 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.614 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.614 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:23:31.614 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:53.035422+0000", 2023-08-28T02:23:31.614 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.002960889 2023-08-28T02:23:31.614 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.614 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.614 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:31.614 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:53.038383+0000", 2023-08-28T02:23:31.614 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0023600140000000001 2023-08-28T02:23:31.615 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.615 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.615 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:31.615 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:53.040743+0000", 2023-08-28T02:23:31.615 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.122153521 2023-08-28T02:23:31.615 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.615 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.615 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:31.615 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:53.162896+0000", 2023-08-28T02:23:31.615 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.8950000000000003e-05 2023-08-28T02:23:31.615 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.616 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.616 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:31.616 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:53.162955+0000", 2023-08-28T02:23:31.616 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.3696999999999999e-05 2023-08-28T02:23:31.616 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.616 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.616 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:31.616 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:53.162999+0000", 2023-08-28T02:23:31.616 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.12804637599999999 2023-08-28T02:23:31.616 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:31.617 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:31.617 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:31.617 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.617 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.617 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:77807 5.1 5:8dc7048a:::benchmark_data_smithi143_236200_object77806:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e133)", 2023-08-28T02:23:31.617 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:22:56.775511+0000", 2023-08-28T02:23:31.617 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 34.722285329999998, 2023-08-28T02:23:31.617 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.12895356299999999, 2023-08-28T02:23:31.617 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:31.617 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:31.617 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:31.618 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:31.618 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:31.618 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 77807 2023-08-28T02:23:31.618 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.618 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:31.618 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.618 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:31.618 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.775511+0000", 2023-08-28T02:23:31.618 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:31.618 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.619 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.619 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:31.619 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.775511+0000", 2023-08-28T02:23:31.619 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:23:31.619 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.619 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.619 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:31.619 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.775509+0000", 2023-08-28T02:23:31.619 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.9067999999999999e-05 2023-08-28T02:23:31.619 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.620 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.620 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:31.620 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.775548+0000", 2023-08-28T02:23:31.620 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.8471999999999999e-05 2023-08-28T02:23:31.620 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.620 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.620 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:31.620 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.775567+0000", 2023-08-28T02:23:31.620 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.6429999999999998e-06 2023-08-28T02:23:31.620 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.621 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.621 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:31.621 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.775573+0000", 2023-08-28T02:23:31.621 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.1519000000000005e-05 2023-08-28T02:23:31.621 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.621 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.621 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:31.621 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.775635+0000", 2023-08-28T02:23:31.621 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000114733 2023-08-28T02:23:31.622 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.622 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.622 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:31.622 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.775750+0000", 2023-08-28T02:23:31.622 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000110914 2023-08-28T02:23:31.622 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.622 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.622 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,7", 2023-08-28T02:23:31.622 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.775861+0000", 2023-08-28T02:23:31.622 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0057715400000000004 2023-08-28T02:23:31.623 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.623 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.623 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:31.623 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.781632+0000", 2023-08-28T02:23:31.623 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00021733899999999999 2023-08-28T02:23:31.623 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.623 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.623 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:31.623 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.781849+0000", 2023-08-28T02:23:31.623 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.122528811 2023-08-28T02:23:31.624 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.624 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.624 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:31.624 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.904378+0000", 2023-08-28T02:23:31.624 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.0847000000000003e-05 2023-08-28T02:23:31.624 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.624 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.624 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:31.624 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.904429+0000", 2023-08-28T02:23:31.624 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.5462e-05 2023-08-28T02:23:31.624 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.625 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.625 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:31.625 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.904465+0000", 2023-08-28T02:23:31.625 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.12895356299999999 2023-08-28T02:23:31.625 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:31.625 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:31.625 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:31.625 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.625 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.625 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:77809 5.1 5:87d586fa:::benchmark_data_smithi143_236200_object77808:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e133)", 2023-08-28T02:23:31.625 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:22:56.778815+0000", 2023-08-28T02:23:31.626 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 34.718981616000001, 2023-08-28T02:23:31.626 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.13123342900000001, 2023-08-28T02:23:31.626 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:31.626 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:31.626 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:31.626 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:31.626 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:31.626 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 77809 2023-08-28T02:23:31.626 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.626 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:31.627 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.627 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:31.627 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.778815+0000", 2023-08-28T02:23:31.627 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:31.627 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.627 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.627 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:31.627 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.778815+0000", 2023-08-28T02:23:31.627 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:23:31.627 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.628 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.628 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:31.628 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.778813+0000", 2023-08-28T02:23:31.628 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.8473999999999999e-05 2023-08-28T02:23:31.628 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.628 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.628 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:31.628 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.778842+0000", 2023-08-28T02:23:31.628 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.3738e-05 2023-08-28T02:23:31.628 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.628 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.629 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:31.629 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.778855+0000", 2023-08-28T02:23:31.629 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.827e-06 2023-08-28T02:23:31.629 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.629 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.629 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:31.629 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.778862+0000", 2023-08-28T02:23:31.629 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.7679000000000002e-05 2023-08-28T02:23:31.629 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.629 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.630 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:31.630 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.778910+0000", 2023-08-28T02:23:31.630 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00014033599999999999 2023-08-28T02:23:31.630 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.630 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.630 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:31.630 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.779050+0000", 2023-08-28T02:23:31.630 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00016777099999999999 2023-08-28T02:23:31.630 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.630 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.631 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,7", 2023-08-28T02:23:31.631 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.779218+0000", 2023-08-28T02:23:31.631 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0050502710000000003 2023-08-28T02:23:31.631 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.631 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.631 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:31.631 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.784268+0000", 2023-08-28T02:23:31.631 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0029595060000000002 2023-08-28T02:23:31.631 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.631 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.632 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:31.632 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.787228+0000", 2023-08-28T02:23:31.632 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.12272398 2023-08-28T02:23:31.632 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.632 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.632 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:31.632 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.909952+0000", 2023-08-28T02:23:31.632 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.4518000000000002e-05 2023-08-28T02:23:31.632 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.632 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.632 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:31.633 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.910006+0000", 2023-08-28T02:23:31.633 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.1943000000000001e-05 2023-08-28T02:23:31.633 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.633 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.633 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:31.633 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.910048+0000", 2023-08-28T02:23:31.633 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.13123342900000001 2023-08-28T02:23:31.633 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:31.633 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:31.633 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:31.634 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.634 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.634 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:77817 5.e 5:74618ea8:::benchmark_data_smithi143_236200_object77816:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e133)", 2023-08-28T02:23:31.634 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:22:56.784107+0000", 2023-08-28T02:23:31.634 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 34.713689610000003, 2023-08-28T02:23:31.634 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.12431911499999999, 2023-08-28T02:23:31.634 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:31.634 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:31.634 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:31.634 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:31.635 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:31.635 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 77817 2023-08-28T02:23:31.635 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.635 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:31.635 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.635 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:31.635 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.784107+0000", 2023-08-28T02:23:31.635 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:31.635 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.635 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.636 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:31.636 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.784107+0000", 2023-08-28T02:23:31.636 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:23:31.636 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.636 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.636 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:31.636 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.784105+0000", 2023-08-28T02:23:31.636 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.0307e-05 2023-08-28T02:23:31.636 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.636 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.637 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:31.637 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.784135+0000", 2023-08-28T02:23:31.637 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.4338999999999999e-05 2023-08-28T02:23:31.637 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.637 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.637 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:31.637 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.784150+0000", 2023-08-28T02:23:31.637 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.8173e-05 2023-08-28T02:23:31.637 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.637 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.637 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:31.638 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.784168+0000", 2023-08-28T02:23:31.638 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.6486000000000003e-05 2023-08-28T02:23:31.638 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.638 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.638 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:31.638 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.784204+0000", 2023-08-28T02:23:31.638 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000136074 2023-08-28T02:23:31.638 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.638 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.638 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:31.639 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.784341+0000", 2023-08-28T02:23:31.639 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00017445000000000001 2023-08-28T02:23:31.639 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.639 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.639 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:23:31.639 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.784515+0000", 2023-08-28T02:23:31.639 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0025544560000000001 2023-08-28T02:23:31.639 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.639 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.639 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:31.640 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.787069+0000", 2023-08-28T02:23:31.640 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00022998699999999999 2023-08-28T02:23:31.640 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.640 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.640 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:31.640 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.787299+0000", 2023-08-28T02:23:31.640 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.12102444800000001 2023-08-28T02:23:31.640 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.640 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.640 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:31.641 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.908324+0000", 2023-08-28T02:23:31.641 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.6583999999999999e-05 2023-08-28T02:23:31.641 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.641 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.641 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:31.641 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.908380+0000", 2023-08-28T02:23:31.641 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.5422e-05 2023-08-28T02:23:31.641 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.641 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.642 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:31.642 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.908426+0000", 2023-08-28T02:23:31.642 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.12431911499999999 2023-08-28T02:23:31.642 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:31.642 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:31.642 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:31.642 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.642 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.642 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:91763 5.e 5:7e8da77a:::benchmark_data_smithi143_236200_object91762:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-08-28T02:23:31.642 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:10.950140+0000", 2023-08-28T02:23:31.643 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 20.547656490000001, 2023-08-28T02:23:31.643 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.13649586699999999, 2023-08-28T02:23:31.643 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:31.643 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:31.643 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:31.643 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:31.643 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:31.643 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 91763 2023-08-28T02:23:31.643 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.643 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:31.644 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.644 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:31.644 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:10.950140+0000", 2023-08-28T02:23:31.644 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:31.644 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.644 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.644 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:31.644 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:10.950140+0000", 2023-08-28T02:23:31.644 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:23:31.644 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.644 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.645 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:31.645 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:10.950139+0000", 2023-08-28T02:23:31.645 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.6581000000000001e-05 2023-08-28T02:23:31.645 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.645 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.645 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:31.645 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:10.950176+0000", 2023-08-28T02:23:31.645 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.8530000000000004e-06 2023-08-28T02:23:31.645 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.645 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.645 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:31.646 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:10.950185+0000", 2023-08-28T02:23:31.646 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.6550000000000001e-06 2023-08-28T02:23:31.646 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.646 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.646 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:31.646 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:10.950191+0000", 2023-08-28T02:23:31.646 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0050284120000000003 2023-08-28T02:23:31.646 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.646 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.646 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:31.647 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:10.955219+0000", 2023-08-28T02:23:31.647 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00011729 2023-08-28T02:23:31.647 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.647 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.647 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:31.647 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:10.955337+0000", 2023-08-28T02:23:31.647 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00014178199999999999 2023-08-28T02:23:31.647 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.647 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.647 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:23:31.648 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:10.955479+0000", 2023-08-28T02:23:31.648 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0054157220000000004 2023-08-28T02:23:31.648 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.648 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.648 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:31.648 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:10.960894+0000", 2023-08-28T02:23:31.648 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.5230000000000001e-06 2023-08-28T02:23:31.648 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.648 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.649 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:31.649 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:10.960901+0000", 2023-08-28T02:23:31.649 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.12563465200000001 2023-08-28T02:23:31.649 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.649 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.649 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:31.649 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:11.086535+0000", 2023-08-28T02:23:31.649 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.0476999999999997e-05 2023-08-28T02:23:31.649 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.649 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.650 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:31.650 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:11.086596+0000", 2023-08-28T02:23:31.650 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.9854000000000002e-05 2023-08-28T02:23:31.650 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.650 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.650 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:31.650 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:11.086636+0000", 2023-08-28T02:23:31.650 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.13649586699999999 2023-08-28T02:23:31.650 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:31.650 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:31.651 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:31.651 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.651 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.651 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:91764 5.1 5:882848a8:::benchmark_data_smithi143_236200_object91763:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-08-28T02:23:31.651 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:10.952280+0000", 2023-08-28T02:23:31.651 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 20.545516427999999, 2023-08-28T02:23:31.651 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.13055265199999999, 2023-08-28T02:23:31.651 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:31.651 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:31.651 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:31.652 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:31.652 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:31.652 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 91764 2023-08-28T02:23:31.652 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.652 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:31.652 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.652 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:31.652 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:10.952280+0000", 2023-08-28T02:23:31.652 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:31.652 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.653 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.653 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:31.653 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:10.952280+0000", 2023-08-28T02:23:31.653 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:23:31.653 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.653 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.653 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:31.653 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:10.952279+0000", 2023-08-28T02:23:31.653 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.6607000000000002e-05 2023-08-28T02:23:31.653 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.653 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.654 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:31.654 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:10.952335+0000", 2023-08-28T02:23:31.654 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.4919999999999999e-05 2023-08-28T02:23:31.654 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.654 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.654 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:31.654 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:10.952350+0000", 2023-08-28T02:23:31.654 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.4210000000000001e-06 2023-08-28T02:23:31.654 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.654 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.654 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:31.655 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:10.952360+0000", 2023-08-28T02:23:31.655 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.8160000000000001e-05 2023-08-28T02:23:31.655 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.655 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.655 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:31.655 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:10.952398+0000", 2023-08-28T02:23:31.655 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00015322 2023-08-28T02:23:31.655 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.655 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.656 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:31.656 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:10.952551+0000", 2023-08-28T02:23:31.656 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00017462700000000001 2023-08-28T02:23:31.656 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.656 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.656 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,7", 2023-08-28T02:23:31.656 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:10.952726+0000", 2023-08-28T02:23:31.656 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.003684321 2023-08-28T02:23:31.656 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.656 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.656 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:31.657 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:10.956410+0000", 2023-08-28T02:23:31.657 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0091585780000000006 2023-08-28T02:23:31.657 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.657 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.657 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:31.657 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:10.965569+0000", 2023-08-28T02:23:31.657 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.117186753 2023-08-28T02:23:31.657 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.657 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.657 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:31.658 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:11.082755+0000", 2023-08-28T02:23:31.658 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.7401999999999999e-05 2023-08-28T02:23:31.658 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.658 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.658 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:31.658 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:11.082793+0000", 2023-08-28T02:23:31.658 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.9910000000000002e-05 2023-08-28T02:23:31.658 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.658 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.659 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:31.659 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:11.082833+0000", 2023-08-28T02:23:31.659 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.13055265199999999 2023-08-28T02:23:31.659 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:31.659 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:31.659 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:31.659 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.659 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.659 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:93768 5.e 5:7330778b:::benchmark_data_smithi143_236200_object93767:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-08-28T02:23:31.659 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:13.061910+0000", 2023-08-28T02:23:31.659 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 18.435886710999998, 2023-08-28T02:23:31.660 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.192889269, 2023-08-28T02:23:31.660 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:31.660 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:31.660 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:31.660 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:31.660 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:31.660 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 93768 2023-08-28T02:23:31.660 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.660 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:31.660 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.661 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:31.661 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:13.061910+0000", 2023-08-28T02:23:31.661 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:31.661 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.661 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.661 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:31.661 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:13.061910+0000", 2023-08-28T02:23:31.661 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:23:31.662 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.662 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.662 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:31.662 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:13.061908+0000", 2023-08-28T02:23:31.662 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.3115000000000003e-05 2023-08-28T02:23:31.662 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.662 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.662 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:31.662 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:13.061951+0000", 2023-08-28T02:23:31.663 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.4378e-05 2023-08-28T02:23:31.663 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.663 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.663 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:31.663 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:13.061965+0000", 2023-08-28T02:23:31.663 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.5059999999999998e-06 2023-08-28T02:23:31.663 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.663 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.663 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:31.664 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:13.061973+0000", 2023-08-28T02:23:31.664 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.1748e-05 2023-08-28T02:23:31.664 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.664 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.664 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:31.664 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:13.062024+0000", 2023-08-28T02:23:31.664 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012731400000000001 2023-08-28T02:23:31.664 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.664 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.664 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:31.665 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:13.062152+0000", 2023-08-28T02:23:31.665 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00015433100000000001 2023-08-28T02:23:31.665 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.665 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.665 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:23:31.665 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:13.062306+0000", 2023-08-28T02:23:31.665 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0055250619999999999 2023-08-28T02:23:31.665 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.665 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.665 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:31.666 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:13.067831+0000", 2023-08-28T02:23:31.666 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00017694999999999999 2023-08-28T02:23:31.666 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.666 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.666 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:31.666 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:13.068008+0000", 2023-08-28T02:23:31.666 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.18672372600000001 2023-08-28T02:23:31.666 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.666 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.666 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:31.667 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:13.254732+0000", 2023-08-28T02:23:31.667 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.8736000000000003e-05 2023-08-28T02:23:31.667 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.667 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.667 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:31.667 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:13.254770+0000", 2023-08-28T02:23:31.667 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.8422999999999999e-05 2023-08-28T02:23:31.667 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.667 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.668 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:31.668 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:13.254799+0000", 2023-08-28T02:23:31.668 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.192889269 2023-08-28T02:23:31.668 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:31.668 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:31.668 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:31.668 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.668 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.668 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:104463 5.e 5:7d9da7b1:::benchmark_data_smithi143_236200_object104462:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e140)", 2023-08-28T02:23:31.668 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:27.840236+0000", 2023-08-28T02:23:31.669 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 3.657560009, 2023-08-28T02:23:31.669 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.122362547, 2023-08-28T02:23:31.669 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:31.669 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:31.669 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:31.669 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:31.669 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:31.669 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 104463 2023-08-28T02:23:31.669 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.670 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:31.670 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.670 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:31.670 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.840236+0000", 2023-08-28T02:23:31.670 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:31.670 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.670 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.670 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:31.670 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.840236+0000", 2023-08-28T02:23:31.670 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:23:31.670 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.671 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.671 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:31.671 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.840235+0000", 2023-08-28T02:23:31.671 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.0787e-05 2023-08-28T02:23:31.671 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.671 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.671 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:31.671 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.840285+0000", 2023-08-28T02:23:31.671 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.6608000000000001e-05 2023-08-28T02:23:31.672 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.672 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.672 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:31.672 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.840302+0000", 2023-08-28T02:23:31.672 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.9099999999999999e-06 2023-08-28T02:23:31.672 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.672 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.672 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:31.672 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.840309+0000", 2023-08-28T02:23:31.672 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.5566999999999999e-05 2023-08-28T02:23:31.673 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.673 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.673 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:31.673 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.840354+0000", 2023-08-28T02:23:31.673 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000121481 2023-08-28T02:23:31.673 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.673 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.673 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:31.673 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.840476+0000", 2023-08-28T02:23:31.674 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012173299999999999 2023-08-28T02:23:31.674 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.674 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.674 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:23:31.674 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.840598+0000", 2023-08-28T02:23:31.674 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0053897770000000001 2023-08-28T02:23:31.674 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.674 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.674 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:31.675 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.845987+0000", 2023-08-28T02:23:31.675 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0059900609999999996 2023-08-28T02:23:31.675 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.675 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.675 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:31.675 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.851977+0000", 2023-08-28T02:23:31.675 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.11051549400000001 2023-08-28T02:23:31.675 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.675 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.676 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:31.676 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.962493+0000", 2023-08-28T02:23:31.676 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.9278999999999999e-05 2023-08-28T02:23:31.676 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.676 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.676 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:31.676 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.962552+0000", 2023-08-28T02:23:31.676 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.6638999999999998e-05 2023-08-28T02:23:31.676 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:31.676 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:31.677 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:31.677 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.962599+0000", 2023-08-28T02:23:31.677 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.122362547 2023-08-28T02:23:31.677 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:31.677 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:31.677 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:31.677 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:31.677 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:31.677 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:23:31.678 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:31.754 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:23:31.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:31.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:23:31.858 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:23:31.904 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:32.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:23:31 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:23:31] "GET /metrics HTTP/1.1" 200 36472 "" "Prometheus/2.33.4" 2023-08-28T02:23:32.255 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is unset 2023-08-28T02:23:32.285 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 85 16 107252 107236 78.7964 34.1875 0.0333496 0.0126793 2023-08-28T02:23:32.583 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:23:32.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:32.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:23:32.586 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:32 smithi169 ceph-mon[159415]: pgmap v294: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 4 remapped+peering, 78 active+clean; 6.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 28 MiB/s wr, 453 op/s; 19419/313467 objects degraded (6.195%); 969 KiB/s, 2 keys/s, 6 objects/s recovering 2023-08-28T02:23:32.586 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:32 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:23:32.586 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:32 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1171832898' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:23:32.586 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:32 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:23:32.684 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.5 dump_ops_in_flight 2023-08-28T02:23:32.687 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:23:32.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:32 smithi143 ceph-mon[183059]: pgmap v294: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 4 remapped+peering, 78 active+clean; 6.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 28 MiB/s wr, 453 op/s; 19419/313467 objects degraded (6.195%); 969 KiB/s, 2 keys/s, 6 objects/s recovering 2023-08-28T02:23:32.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:32 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:23:32.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:32 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1171832898' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:23:32.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:32 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:23:32.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:32 smithi143 ceph-mon[189871]: pgmap v294: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 4 remapped+peering, 78 active+clean; 6.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 28 MiB/s wr, 453 op/s; 19419/313467 objects degraded (6.195%); 969 KiB/s, 2 keys/s, 6 objects/s recovering 2023-08-28T02:23:32.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:32 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:23:32.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:32 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1171832898' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:23:32.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:32 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:23:32.731 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:32.906 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:23:32.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:32.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:23:33.010 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:23:33.091 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:33.285 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 86 16 107817 107801 78.291 35.3125 0.0245052 0.0127677 2023-08-28T02:23:33.342 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:23:33.346 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:33 smithi169 ceph-mon[159415]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:23:33.346 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:33 smithi169 ceph-mon[159415]: failed to encode map e143 with expected crc 2023-08-28T02:23:33.346 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:33 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:23:33.346 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:33 smithi169 ceph-mon[159415]: osdmap e143: 8 total, 8 up, 6 in 2023-08-28T02:23:33.346 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:33 smithi169 ceph-mon[159415]: pgmap v296: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.5 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 28 MiB/s wr, 452 op/s; 25912/317736 objects degraded (8.155%); 6451/317736 objects misplaced (2.030%); 2.0 MiB/s, 26 objects/s recovering 2023-08-28T02:23:33.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:33.347 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:23:33.448 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:23:33.510 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:33.542 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:33 smithi143 ceph-mon[183059]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:23:33.542 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:33 smithi143 ceph-mon[183059]: failed to encode map e143 with expected crc 2023-08-28T02:23:33.542 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:33 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:23:33.542 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:33 smithi143 ceph-mon[183059]: osdmap e143: 8 total, 8 up, 6 in 2023-08-28T02:23:33.542 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:33 smithi143 ceph-mon[183059]: pgmap v296: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.5 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 28 MiB/s wr, 452 op/s; 25912/317736 objects degraded (8.155%); 6451/317736 objects misplaced (2.030%); 2.0 MiB/s, 26 objects/s recovering 2023-08-28T02:23:33.543 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:33 smithi143 ceph-mon[189871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:23:33.543 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:33 smithi143 ceph-mon[189871]: failed to encode map e143 with expected crc 2023-08-28T02:23:33.543 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:33 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:23:33.544 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:33 smithi143 ceph-mon[189871]: osdmap e143: 8 total, 8 up, 6 in 2023-08-28T02:23:33.544 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:33 smithi143 ceph-mon[189871]: pgmap v296: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.5 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 28 MiB/s wr, 452 op/s; 25912/317736 objects degraded (8.155%); 6451/317736 objects misplaced (2.030%); 2.0 MiB/s, 26 objects/s recovering 2023-08-28T02:23:33.622 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:23:33.634 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:23:33.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:33.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:23:33.699 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:23:33.746 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:23:33.826 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:33.831 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:23:33.873 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:23:33.913 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:23:33.943 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:23:33.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:33.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:23:33.967 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:23:34.012 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:23:34.053 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:23:34.056 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:23:34.086 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:23:34.116 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:34.118 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:23:34.119 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:23:34.119 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:23:34.119 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:23:34.120 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:23:34.120 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:23:34.120 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:23:34.120 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:23:34.120 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:23:34.268 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:23:34.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:34.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:23:34.287 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 87 16 108423 108407 77.8249 37.875 0.0187678 0.0128428 2023-08-28T02:23:34.379 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:23:34.383 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:34 smithi143 ceph-mon[183059]: failed to encode map e143 with expected crc 2023-08-28T02:23:34.383 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:34 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2560609449' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:23:34.384 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:34 smithi143 ceph-mon[189871]: failed to encode map e143 with expected crc 2023-08-28T02:23:34.384 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:34 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2560609449' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:23:34.442 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:34.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:34 smithi169 ceph-mon[159415]: failed to encode map e143 with expected crc 2023-08-28T02:23:34.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:34 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2560609449' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:23:34.995 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:23:35.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:35.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:23:35.103 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:23:35.152 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:35.238 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:23:35.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:35.245 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:23:35.289 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 88 16 108997 108981 77.3472 35.875 0.0245859 0.0129225 2023-08-28T02:23:35.346 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:23:35.363 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset nodeep-scrub 2023-08-28T02:23:35.394 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:35.662 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:35 smithi143 ceph-mon[183059]: pgmap v297: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 29 MiB/s wr, 470 op/s; 25912/322482 objects degraded (8.035%); 6451/322482 objects misplaced (2.000%); 1.6 MiB/s, 20 objects/s recovering 2023-08-28T02:23:35.663 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:35 smithi143 ceph-mon[189871]: pgmap v297: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 29 MiB/s wr, 470 op/s; 25912/322482 objects degraded (8.035%); 6451/322482 objects misplaced (2.000%); 1.6 MiB/s, 20 objects/s recovering 2023-08-28T02:23:35.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:35 smithi169 ceph-mon[159415]: pgmap v297: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 29 MiB/s wr, 470 op/s; 25912/322482 objects degraded (8.035%); 6451/322482 objects misplaced (2.000%); 1.6 MiB/s, 20 objects/s recovering 2023-08-28T02:23:35.762 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:23:35.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:35.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:23:35.866 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:23:35.940 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:36.084 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:23:36.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:36.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:23:36.191 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:23:36.244 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:36.289 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 89 16 109592 109576 76.8962 37.1875 0.0158014 0.0129989 2023-08-28T02:23:36.335 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:23:36.384 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:23:36.387 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:23:36.387 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:23:36.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:36.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:23:36.421 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:36 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 25912/322482 objects degraded (8.035%), 5 pgs degraded (PG_DEGRADED) 2023-08-28T02:23:36.490 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:23:36.548 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:36.646 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:23:36.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:36.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:23:36.688 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:36 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 25912/322482 objects degraded (8.035%), 5 pgs degraded (PG_DEGRADED) 2023-08-28T02:23:36.689 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:36 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 25912/322482 objects degraded (8.035%), 5 pgs degraded (PG_DEGRADED) 2023-08-28T02:23:36.752 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:23:36.782 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:37.289 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Total time run: 90.023 2023-08-28T02:23:37.290 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Total writes made: 110140 2023-08-28T02:23:37.290 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Write size: 65536 2023-08-28T02:23:37.290 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Object size: 65536 2023-08-28T02:23:37.290 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Bandwidth (MB/sec): 76.4666 2023-08-28T02:23:37.290 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Stddev Bandwidth: 37.6258 2023-08-28T02:23:37.290 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Max bandwidth (MB/sec): 240.438 2023-08-28T02:23:37.290 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Min bandwidth (MB/sec): 0 2023-08-28T02:23:37.290 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Average IOPS: 1223 2023-08-28T02:23:37.290 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Stddev IOPS: 602.013 2023-08-28T02:23:37.290 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Max IOPS: 3847 2023-08-28T02:23:37.290 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Min IOPS: 0 2023-08-28T02:23:37.290 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Average Latency(s): 0.0130723 2023-08-28T02:23:37.291 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Stddev Latency(s): 0.0318855 2023-08-28T02:23:37.291 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Max latency(s): 2.62915 2023-08-28T02:23:37.291 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Min latency(s): 0.00111735 2023-08-28T02:23:37.295 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:37 smithi143 conmon[183035]: 2023-08-28T02:23:37.078+0000 7fadc3887700 -1 mon.a@0(leader).osd e143 definitely_dead 0 2023-08-28T02:23:37.302 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Cleaning up (deleting benchmark objects) 2023-08-28T02:23:37.335 INFO:teuthology.orchestra.run.smithi143.stderr:marked in osd.7. 2023-08-28T02:23:37.514 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:23:37.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:37.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:23:37.568 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:37 smithi169 ceph-mon[159415]: pgmap v298: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 32 MiB/s wr, 515 op/s; 25912/325857 objects degraded (7.952%); 6320/325857 objects misplaced (1.940%); 1.9 MiB/s, 32 objects/s recovering 2023-08-28T02:23:37.568 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:37 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2820638007' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-08-28T02:23:37.568 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:37 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-08-28T02:23:37.587 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.5 dump_blocked_ops 2023-08-28T02:23:37.619 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:23:37.645 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:37 smithi143 ceph-mon[183059]: pgmap v298: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 32 MiB/s wr, 515 op/s; 25912/325857 objects degraded (7.952%); 6320/325857 objects misplaced (1.940%); 1.9 MiB/s, 32 objects/s recovering 2023-08-28T02:23:37.645 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:37 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2820638007' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-08-28T02:23:37.645 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:37 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-08-28T02:23:37.645 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:37 smithi143 ceph-mon[189871]: pgmap v298: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 32 MiB/s wr, 515 op/s; 25912/325857 objects degraded (7.952%); 6320/325857 objects misplaced (1.940%); 1.9 MiB/s, 32 objects/s recovering 2023-08-28T02:23:37.645 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:37 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2820638007' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-08-28T02:23:37.645 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:37 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-08-28T02:23:37.648 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:37.694 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:23:38.052 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:23:38.183 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:23:38.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:38.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:23:38.287 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:23:38.319 INFO:tasks.thrashosds.thrasher:Added osd 7 2023-08-28T02:23:38.338 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:38.420 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:23:38.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:38.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:23:38.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:38 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-08-28T02:23:38.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:38 smithi143 ceph-mon[183059]: osdmap e144: 8 total, 8 up, 7 in 2023-08-28T02:23:38.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:38 smithi143 ceph-mon[183059]: failed to encode map e144 with expected crc 2023-08-28T02:23:38.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:38 smithi143 ceph-mon[183059]: failed to encode map e144 with expected crc 2023-08-28T02:23:38.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:38 smithi143 ceph-mon[183059]: failed to encode map e144 with expected crc 2023-08-28T02:23:38.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:38 smithi143 ceph-mon[183059]: failed to encode map e144 with expected crc 2023-08-28T02:23:38.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:38 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-08-28T02:23:38.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:38 smithi143 ceph-mon[189871]: osdmap e144: 8 total, 8 up, 7 in 2023-08-28T02:23:38.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:38 smithi143 ceph-mon[189871]: failed to encode map e144 with expected crc 2023-08-28T02:23:38.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:38 smithi143 ceph-mon[189871]: failed to encode map e144 with expected crc 2023-08-28T02:23:38.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:38 smithi143 ceph-mon[189871]: failed to encode map e144 with expected crc 2023-08-28T02:23:38.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:38 smithi143 ceph-mon[189871]: failed to encode map e144 with expected crc 2023-08-28T02:23:38.508 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:23:38.532 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:23:38.581 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:38 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-08-28T02:23:38.582 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:38 smithi169 ceph-mon[159415]: osdmap e144: 8 total, 8 up, 7 in 2023-08-28T02:23:38.582 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:38 smithi169 ceph-mon[159415]: failed to encode map e144 with expected crc 2023-08-28T02:23:38.582 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:38 smithi169 ceph-mon[159415]: failed to encode map e144 with expected crc 2023-08-28T02:23:38.582 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:38 smithi169 ceph-mon[159415]: failed to encode map e144 with expected crc 2023-08-28T02:23:38.582 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:38 smithi169 ceph-mon[159415]: failed to encode map e144 with expected crc 2023-08-28T02:23:38.609 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:38.810 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:23:38.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:38.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:23:38.922 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:23:38.980 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:39.120 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:23:39.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:39.124 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:23:39.225 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:23:39.289 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:39.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:39 smithi143 ceph-mon[189871]: failed to encode map e145 with expected crc 2023-08-28T02:23:39.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:39 smithi143 ceph-mon[189871]: osdmap e145: 8 total, 8 up, 7 in 2023-08-28T02:23:39.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:39 smithi143 ceph-mon[189871]: failed to encode map e145 with expected crc 2023-08-28T02:23:39.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:39 smithi143 ceph-mon[189871]: failed to encode map e145 with expected crc 2023-08-28T02:23:39.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:39 smithi143 ceph-mon[189871]: failed to encode map e145 with expected crc 2023-08-28T02:23:39.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:39 smithi143 ceph-mon[189871]: failed to encode map e145 with expected crc 2023-08-28T02:23:39.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:39 smithi143 ceph-mon[189871]: failed to encode map e145 with expected crc 2023-08-28T02:23:39.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:39 smithi143 ceph-mon[189871]: failed to encode map e145 with expected crc 2023-08-28T02:23:39.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:39 smithi143 ceph-mon[189871]: failed to encode map e145 with expected crc 2023-08-28T02:23:39.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:39 smithi143 ceph-mon[189871]: failed to encode map e145 with expected crc 2023-08-28T02:23:39.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:39 smithi143 ceph-mon[189871]: failed to encode map e145 with expected crc 2023-08-28T02:23:39.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:39 smithi143 ceph-mon[189871]: pgmap v301: 89 pgs: 3 remapped+peering, 1 activating, 6 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.7 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 42 MiB/s wr, 675 op/s; 6493/330285 objects degraded (1.966%); 6320/330285 objects misplaced (1.913%); 1.2 MiB/s, 19 objects/s recovering 2023-08-28T02:23:39.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:39 smithi143 ceph-mon[183059]: failed to encode map e145 with expected crc 2023-08-28T02:23:39.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:39 smithi143 ceph-mon[183059]: osdmap e145: 8 total, 8 up, 7 in 2023-08-28T02:23:39.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:39 smithi143 ceph-mon[183059]: failed to encode map e145 with expected crc 2023-08-28T02:23:39.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:39 smithi143 ceph-mon[183059]: failed to encode map e145 with expected crc 2023-08-28T02:23:39.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:39 smithi143 ceph-mon[183059]: failed to encode map e145 with expected crc 2023-08-28T02:23:39.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:39 smithi143 ceph-mon[183059]: failed to encode map e145 with expected crc 2023-08-28T02:23:39.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:39 smithi143 ceph-mon[183059]: failed to encode map e145 with expected crc 2023-08-28T02:23:39.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:39 smithi143 ceph-mon[183059]: failed to encode map e145 with expected crc 2023-08-28T02:23:39.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:39 smithi143 ceph-mon[183059]: failed to encode map e145 with expected crc 2023-08-28T02:23:39.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:39 smithi143 ceph-mon[183059]: failed to encode map e145 with expected crc 2023-08-28T02:23:39.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:39 smithi143 ceph-mon[183059]: failed to encode map e145 with expected crc 2023-08-28T02:23:39.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:39 smithi143 ceph-mon[183059]: pgmap v301: 89 pgs: 3 remapped+peering, 1 activating, 6 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.7 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 42 MiB/s wr, 675 op/s; 6493/330285 objects degraded (1.966%); 6320/330285 objects misplaced (1.913%); 1.2 MiB/s, 19 objects/s recovering 2023-08-28T02:23:39.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:39 smithi169 ceph-mon[159415]: failed to encode map e145 with expected crc 2023-08-28T02:23:39.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:39 smithi169 ceph-mon[159415]: osdmap e145: 8 total, 8 up, 7 in 2023-08-28T02:23:39.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:39 smithi169 ceph-mon[159415]: failed to encode map e145 with expected crc 2023-08-28T02:23:39.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:39 smithi169 ceph-mon[159415]: failed to encode map e145 with expected crc 2023-08-28T02:23:39.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:39 smithi169 ceph-mon[159415]: failed to encode map e145 with expected crc 2023-08-28T02:23:39.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:39 smithi169 ceph-mon[159415]: failed to encode map e145 with expected crc 2023-08-28T02:23:39.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:39 smithi169 ceph-mon[159415]: failed to encode map e145 with expected crc 2023-08-28T02:23:39.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:39 smithi169 ceph-mon[159415]: failed to encode map e145 with expected crc 2023-08-28T02:23:39.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:39 smithi169 ceph-mon[159415]: failed to encode map e145 with expected crc 2023-08-28T02:23:39.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:39 smithi169 ceph-mon[159415]: failed to encode map e145 with expected crc 2023-08-28T02:23:39.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:39 smithi169 ceph-mon[159415]: failed to encode map e145 with expected crc 2023-08-28T02:23:39.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:39 smithi169 ceph-mon[159415]: pgmap v301: 89 pgs: 3 remapped+peering, 1 activating, 6 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.7 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 42 MiB/s wr, 675 op/s; 6493/330285 objects degraded (1.966%); 6320/330285 objects misplaced (1.913%); 1.2 MiB/s, 19 objects/s recovering 2023-08-28T02:23:40.051 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:23:40.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:40.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:23:40.166 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:23:40.258 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:40.358 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is unset 2023-08-28T02:23:40.506 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:23:40.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:40.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:23:40.619 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:23:40.637 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:40 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3332519822' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:23:40.637 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:40 smithi143 ceph-mon[183059]: osdmap e146: 8 total, 8 up, 7 in 2023-08-28T02:23:40.637 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:40 smithi143 ceph-mon[183059]: failed to encode map e146 with expected crc 2023-08-28T02:23:40.637 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:40 smithi143 ceph-mon[183059]: failed to encode map e146 with expected crc 2023-08-28T02:23:40.638 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:40 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:23:40.638 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:40 smithi143 ceph-mon[183059]: failed to encode map e146 with expected crc 2023-08-28T02:23:40.638 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:40 smithi143 ceph-mon[183059]: failed to encode map e146 with expected crc 2023-08-28T02:23:40.638 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:40 smithi143 ceph-mon[183059]: failed to encode map e146 with expected crc 2023-08-28T02:23:40.638 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:40 smithi143 ceph-mon[183059]: failed to encode map e146 with expected crc 2023-08-28T02:23:40.638 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:40 smithi143 ceph-mon[183059]: failed to encode map e146 with expected crc 2023-08-28T02:23:40.638 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:40 smithi143 ceph-mon[183059]: failed to encode map e146 with expected crc 2023-08-28T02:23:40.638 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:40 smithi143 ceph-mon[183059]: failed to encode map e146 with expected crc 2023-08-28T02:23:40.638 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:40 smithi143 ceph-mon[183059]: failed to encode map e146 with expected crc 2023-08-28T02:23:40.638 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:40 smithi143 ceph-mon[183059]: failed to encode map e146 with expected crc 2023-08-28T02:23:40.638 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:40 smithi143 ceph-mon[183059]: failed to encode map e146 with expected crc 2023-08-28T02:23:40.638 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:40 smithi143 ceph-mon[183059]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:23:40.639 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:40 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:23:40.639 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:40 smithi143 ceph-mon[183059]: osdmap e147: 8 total, 8 up, 7 in 2023-08-28T02:23:40.639 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:40 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3332519822' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:23:40.639 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:40 smithi143 ceph-mon[189871]: osdmap e146: 8 total, 8 up, 7 in 2023-08-28T02:23:40.639 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:40 smithi143 ceph-mon[189871]: failed to encode map e146 with expected crc 2023-08-28T02:23:40.639 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:40 smithi143 ceph-mon[189871]: failed to encode map e146 with expected crc 2023-08-28T02:23:40.639 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:40 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:23:40.639 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:40 smithi143 ceph-mon[189871]: failed to encode map e146 with expected crc 2023-08-28T02:23:40.639 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:40 smithi143 ceph-mon[189871]: failed to encode map e146 with expected crc 2023-08-28T02:23:40.639 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:40 smithi143 ceph-mon[189871]: failed to encode map e146 with expected crc 2023-08-28T02:23:40.639 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:40 smithi143 ceph-mon[189871]: failed to encode map e146 with expected crc 2023-08-28T02:23:40.640 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:40 smithi143 ceph-mon[189871]: failed to encode map e146 with expected crc 2023-08-28T02:23:40.640 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:40 smithi143 ceph-mon[189871]: failed to encode map e146 with expected crc 2023-08-28T02:23:40.640 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:40 smithi143 ceph-mon[189871]: failed to encode map e146 with expected crc 2023-08-28T02:23:40.640 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:40 smithi143 ceph-mon[189871]: failed to encode map e146 with expected crc 2023-08-28T02:23:40.640 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:40 smithi143 ceph-mon[189871]: failed to encode map e146 with expected crc 2023-08-28T02:23:40.640 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:40 smithi143 ceph-mon[189871]: failed to encode map e146 with expected crc 2023-08-28T02:23:40.640 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:40 smithi143 ceph-mon[189871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:23:40.640 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:40 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:23:40.640 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:40 smithi143 ceph-mon[189871]: osdmap e147: 8 total, 8 up, 7 in 2023-08-28T02:23:40.662 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:40.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:40 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3332519822' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:23:40.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:40 smithi169 ceph-mon[159415]: osdmap e146: 8 total, 8 up, 7 in 2023-08-28T02:23:40.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:40 smithi169 ceph-mon[159415]: failed to encode map e146 with expected crc 2023-08-28T02:23:40.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:40 smithi169 ceph-mon[159415]: failed to encode map e146 with expected crc 2023-08-28T02:23:40.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:40 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:23:40.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:40 smithi169 ceph-mon[159415]: failed to encode map e146 with expected crc 2023-08-28T02:23:40.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:40 smithi169 ceph-mon[159415]: failed to encode map e146 with expected crc 2023-08-28T02:23:40.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:40 smithi169 ceph-mon[159415]: failed to encode map e146 with expected crc 2023-08-28T02:23:40.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:40 smithi169 ceph-mon[159415]: failed to encode map e146 with expected crc 2023-08-28T02:23:40.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:40 smithi169 ceph-mon[159415]: failed to encode map e146 with expected crc 2023-08-28T02:23:40.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:40 smithi169 ceph-mon[159415]: failed to encode map e146 with expected crc 2023-08-28T02:23:40.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:40 smithi169 ceph-mon[159415]: failed to encode map e146 with expected crc 2023-08-28T02:23:40.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:40 smithi169 ceph-mon[159415]: failed to encode map e146 with expected crc 2023-08-28T02:23:40.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:40 smithi169 ceph-mon[159415]: failed to encode map e146 with expected crc 2023-08-28T02:23:40.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:40 smithi169 ceph-mon[159415]: failed to encode map e146 with expected crc 2023-08-28T02:23:40.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:40 smithi169 ceph-mon[159415]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:23:40.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:40 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:23:40.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:40 smithi169 ceph-mon[159415]: osdmap e147: 8 total, 8 up, 7 in 2023-08-28T02:23:40.769 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:23:40.770 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:23:41.347 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:23:41.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:41.354 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:23:41.454 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:23:41.497 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:41.598 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:23:41.628 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:23:41.637 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:41 smithi143 ceph-mon[183059]: failed to encode map e147 with expected crc 2023-08-28T02:23:41.637 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:41 smithi143 ceph-mon[183059]: failed to encode map e147 with expected crc 2023-08-28T02:23:41.637 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:41 smithi143 ceph-mon[183059]: failed to encode map e147 with expected crc 2023-08-28T02:23:41.637 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:41 smithi143 ceph-mon[183059]: failed to encode map e147 with expected crc 2023-08-28T02:23:41.637 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:41 smithi143 ceph-mon[183059]: failed to encode map e147 with expected crc 2023-08-28T02:23:41.637 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:41 smithi143 ceph-mon[183059]: failed to encode map e147 with expected crc 2023-08-28T02:23:41.637 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:41 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:23:41.638 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:41 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:23:41.638 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:41 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:23:41.638 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:41 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:23:41.638 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:41 smithi143 ceph-mon[183059]: pgmap v304: 89 pgs: 3 remapped+peering, 1 activating, 6 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.7 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 24 MiB/s wr, 383 op/s; 6493/330285 objects degraded (1.966%); 6320/330285 objects misplaced (1.913%) 2023-08-28T02:23:41.638 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:41 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:23:41.638 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:41 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 6493/330285 objects degraded (1.966%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:23:41.638 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:41 smithi143 ceph-mon[189871]: failed to encode map e147 with expected crc 2023-08-28T02:23:41.638 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:41 smithi143 ceph-mon[189871]: failed to encode map e147 with expected crc 2023-08-28T02:23:41.639 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:41 smithi143 ceph-mon[189871]: failed to encode map e147 with expected crc 2023-08-28T02:23:41.639 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:41 smithi143 ceph-mon[189871]: failed to encode map e147 with expected crc 2023-08-28T02:23:41.639 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:41 smithi143 ceph-mon[189871]: failed to encode map e147 with expected crc 2023-08-28T02:23:41.639 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:41 smithi143 ceph-mon[189871]: failed to encode map e147 with expected crc 2023-08-28T02:23:41.639 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:41 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:23:41.639 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:41 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:23:41.639 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:41 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:23:41.639 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:41 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:23:41.639 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:41 smithi143 ceph-mon[189871]: pgmap v304: 89 pgs: 3 remapped+peering, 1 activating, 6 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.7 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 24 MiB/s wr, 383 op/s; 6493/330285 objects degraded (1.966%); 6320/330285 objects misplaced (1.913%) 2023-08-28T02:23:41.639 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:41 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:23:41.639 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:41 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 6493/330285 objects degraded (1.966%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:23:41.672 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:23:41.710 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:23:41.750 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:23:41.781 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:41 smithi169 ceph-mon[159415]: failed to encode map e147 with expected crc 2023-08-28T02:23:41.781 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:41 smithi169 ceph-mon[159415]: failed to encode map e147 with expected crc 2023-08-28T02:23:41.781 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:41 smithi169 ceph-mon[159415]: failed to encode map e147 with expected crc 2023-08-28T02:23:41.781 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:41 smithi169 ceph-mon[159415]: failed to encode map e147 with expected crc 2023-08-28T02:23:41.781 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:41 smithi169 ceph-mon[159415]: failed to encode map e147 with expected crc 2023-08-28T02:23:41.781 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:41 smithi169 ceph-mon[159415]: failed to encode map e147 with expected crc 2023-08-28T02:23:41.782 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:41 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:23:41.782 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:41 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:23:41.782 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:41 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:23:41.782 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:41 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:23:41.782 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:41 smithi169 ceph-mon[159415]: pgmap v304: 89 pgs: 3 remapped+peering, 1 activating, 6 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.7 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 24 MiB/s wr, 383 op/s; 6493/330285 objects degraded (1.966%); 6320/330285 objects misplaced (1.913%) 2023-08-28T02:23:41.782 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:41 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:23:41.782 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:41 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 6493/330285 objects degraded (1.966%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:23:41.788 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:23:41.816 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:23:41.817 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.5 dump_historic_ops 2023-08-28T02:23:41.818 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:23:41.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:41.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:23:41.855 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:23:41.874 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:23:41.874 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:23:41.874 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:23:41.874 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:23:41.875 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:23:41.875 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:23:41.875 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:23:41.875 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:23:41.925 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:23:41.973 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:42.114 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:23:42.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:42.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:23:42.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:23:41 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:23:41] "GET /metrics HTTP/1.1" 200 36492 "" "Prometheus/2.33.4" 2023-08-28T02:23:42.224 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:23:42.272 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:42.328 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:23:42.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:42.331 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:23:42.420 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:23:42.432 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:23:42.469 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:42.519 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:42 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2457419665' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:23:42.519 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:42 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:23:42.630 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:23:42.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:42.638 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:23:42.730 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:42 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2457419665' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:23:42.730 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:42 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:23:42.730 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:42 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:23:42.739 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:23:42.797 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:42.806 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:42 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:23:42.807 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:42 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2457419665' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:23:42.807 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:42 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:23:42.807 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:42 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:23:43.046 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:23:43.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:43.049 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:23:43.150 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:23:43.212 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:43.320 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7] out_osds: [3] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-08-28T02:23:43.320 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-08-28T02:23:43.320 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-08-28T02:23:43.320 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-08-28T02:23:43.320 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-08-28T02:23:43.320 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-08-28T02:23:43.329 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:23:43.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:43.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:23:43.347 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set noscrub 2023-08-28T02:23:43.437 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:23:43.476 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:43.576 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:23:43.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:43.601 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:23:43.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:43 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:23:43.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:43 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:23:43.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:43 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:23:43.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:43 smithi143 ceph-mon[189871]: pgmap v305: 89 pgs: 3 remapped+peering, 1 activating, 1 peering, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 17 MiB/s wr, 517 op/s; 399/326202 objects degraded (0.122%); 89/326202 objects misplaced (0.027%); 2.6 MiB/s, 51 objects/s recovering 2023-08-28T02:23:43.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:43 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:23:43.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:43 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:23:43.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:43 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:23:43.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:43 smithi143 ceph-mon[183059]: pgmap v305: 89 pgs: 3 remapped+peering, 1 activating, 1 peering, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 17 MiB/s wr, 517 op/s; 399/326202 objects degraded (0.122%); 89/326202 objects misplaced (0.027%); 2.6 MiB/s, 51 objects/s recovering 2023-08-28T02:23:43.702 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:23:43.731 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:43.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:43 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:23:43.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:43 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:23:43.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:43 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:23:43.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:43 smithi169 ceph-mon[159415]: pgmap v305: 89 pgs: 3 remapped+peering, 1 activating, 1 peering, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 6.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 17 MiB/s wr, 517 op/s; 399/326202 objects degraded (0.122%); 89/326202 objects misplaced (0.027%); 2.6 MiB/s, 51 objects/s recovering 2023-08-28T02:23:43.804 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:23:43.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:43.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:23:43.851 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:23:43.910 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:23:43.971 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:44.201 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:23:44.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:44.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:23:44.305 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:23:44.312 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:23:44.337 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:44.436 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:23:44.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:44.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:23:44.543 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:23:44.570 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:44.658 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:23:44.659 INFO:teuthology.orchestra.run.smithi169.stdout: "size": 20, 2023-08-28T02:23:44.659 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 600, 2023-08-28T02:23:44.659 INFO:teuthology.orchestra.run.smithi169.stdout: "ops": [ 2023-08-28T02:23:44.659 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.659 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:17315 5.7 5:e427df1e:::benchmark_data_smithi143_236200_object17314:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e125)", 2023-08-28T02:23:44.659 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:22:13.611088+0000", 2023-08-28T02:23:44.659 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 91.045945431999996, 2023-08-28T02:23:44.659 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.15911980000000001, 2023-08-28T02:23:44.659 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:44.659 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:44.660 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:44.660 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:44.660 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:44.660 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 17315 2023-08-28T02:23:44.660 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.660 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:44.660 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.660 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:44.660 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:13.611088+0000", 2023-08-28T02:23:44.660 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:44.660 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.660 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.661 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:44.661 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:13.611088+0000", 2023-08-28T02:23:44.661 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:23:44.661 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.661 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.661 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:44.661 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:13.611086+0000", 2023-08-28T02:23:44.661 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.6551000000000001e-05 2023-08-28T02:23:44.661 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.661 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.661 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:44.661 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:13.611143+0000", 2023-08-28T02:23:44.661 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.1339999999999999e-05 2023-08-28T02:23:44.662 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.662 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.662 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:44.662 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:13.611164+0000", 2023-08-28T02:23:44.662 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.9750000000000001e-06 2023-08-28T02:23:44.662 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.662 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.662 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:44.662 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:13.611171+0000", 2023-08-28T02:23:44.662 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00024026200000000001 2023-08-28T02:23:44.662 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.662 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.663 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:44.663 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:13.611411+0000", 2023-08-28T02:23:44.663 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000190409 2023-08-28T02:23:44.663 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.663 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.663 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:44.663 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:13.611602+0000", 2023-08-28T02:23:44.663 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00016581300000000001 2023-08-28T02:23:44.663 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.663 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.663 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,1", 2023-08-28T02:23:44.663 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:13.611768+0000", 2023-08-28T02:23:44.664 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0012937999999999999 2023-08-28T02:23:44.664 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.664 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.664 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:44.664 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:13.613061+0000", 2023-08-28T02:23:44.664 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00062938200000000005 2023-08-28T02:23:44.664 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.664 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.664 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:44.664 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:13.613691+0000", 2023-08-28T02:23:44.664 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.15633762900000001 2023-08-28T02:23:44.664 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.665 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.665 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:44.665 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:13.770028+0000", 2023-08-28T02:23:44.665 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00011249 2023-08-28T02:23:44.665 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.665 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.665 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:44.665 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:13.770141+0000", 2023-08-28T02:23:44.665 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.7028999999999998e-05 2023-08-28T02:23:44.665 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.665 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.665 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:44.666 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:13.770208+0000", 2023-08-28T02:23:44.666 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.15911980000000001 2023-08-28T02:23:44.666 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:44.666 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:44.666 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:44.666 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.666 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.666 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:45872 5.7 5:ec79c240:::benchmark_data_smithi143_236200_object45871:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-08-28T02:23:44.666 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:22:29.936812+0000", 2023-08-28T02:23:44.666 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 74.720221574999997, 2023-08-28T02:23:44.666 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.19466989300000001, 2023-08-28T02:23:44.667 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:44.667 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:44.667 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:44.667 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:44.667 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:44.667 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 45872 2023-08-28T02:23:44.667 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.667 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:44.667 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.667 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:44.667 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.936812+0000", 2023-08-28T02:23:44.667 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:44.668 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.668 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.668 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:44.668 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.936812+0000", 2023-08-28T02:23:44.668 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:23:44.668 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.668 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.668 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:44.668 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.936811+0000", 2023-08-28T02:23:44.668 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.6977000000000002e-05 2023-08-28T02:23:44.668 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.668 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.668 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:44.669 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.936848+0000", 2023-08-28T02:23:44.669 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.8570000000000001e-06 2023-08-28T02:23:44.669 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.669 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.669 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:44.669 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.936857+0000", 2023-08-28T02:23:44.669 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.7640000000000002e-06 2023-08-28T02:23:44.669 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.669 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.669 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:44.669 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.936863+0000", 2023-08-28T02:23:44.669 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.5926999999999997e-05 2023-08-28T02:23:44.670 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.670 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.670 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:44.670 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.936898+0000", 2023-08-28T02:23:44.670 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000126709 2023-08-28T02:23:44.670 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.670 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.670 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:44.670 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.937025+0000", 2023-08-28T02:23:44.670 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00015847600000000001 2023-08-28T02:23:44.670 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.670 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.671 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,1", 2023-08-28T02:23:44.671 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.937184+0000", 2023-08-28T02:23:44.671 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0018170529999999999 2023-08-28T02:23:44.671 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.671 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.671 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:44.671 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.939001+0000", 2023-08-28T02:23:44.671 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0030671990000000001 2023-08-28T02:23:44.671 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.671 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.671 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:44.671 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.942068+0000", 2023-08-28T02:23:44.672 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.189325665 2023-08-28T02:23:44.672 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.672 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.672 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:44.672 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:30.131394+0000", 2023-08-28T02:23:44.672 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.0983000000000003e-05 2023-08-28T02:23:44.672 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.672 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.672 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:44.672 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:30.131445+0000", 2023-08-28T02:23:44.672 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.7243000000000002e-05 2023-08-28T02:23:44.672 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.673 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.673 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:44.673 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:30.131482+0000", 2023-08-28T02:23:44.673 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.19466989300000001 2023-08-28T02:23:44.673 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:44.673 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:44.673 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:44.673 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.673 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.673 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:45876 5.7 5:eb222641:::benchmark_data_smithi143_236200_object45875:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-08-28T02:23:44.673 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:22:29.939130+0000", 2023-08-28T02:23:44.673 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 74.717903813000007, 2023-08-28T02:23:44.674 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.192484339, 2023-08-28T02:23:44.674 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:44.674 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:44.674 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:44.674 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:44.674 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:44.674 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 45876 2023-08-28T02:23:44.674 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.674 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:44.674 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.674 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:44.675 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.939130+0000", 2023-08-28T02:23:44.675 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:44.675 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.675 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.675 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:44.675 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.939130+0000", 2023-08-28T02:23:44.675 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:23:44.675 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.675 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.675 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:44.675 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.939128+0000", 2023-08-28T02:23:44.675 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.5680000000000003e-05 2023-08-28T02:23:44.676 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.676 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.676 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:44.676 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.939174+0000", 2023-08-28T02:23:44.676 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.4414e-05 2023-08-28T02:23:44.676 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.676 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.676 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:44.676 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.939188+0000", 2023-08-28T02:23:44.676 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.2540000000000003e-06 2023-08-28T02:23:44.676 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.676 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.677 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:44.677 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.939196+0000", 2023-08-28T02:23:44.677 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00022383700000000001 2023-08-28T02:23:44.677 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.677 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.677 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:44.677 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.939419+0000", 2023-08-28T02:23:44.677 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000139037 2023-08-28T02:23:44.677 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.677 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.677 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:44.677 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.939558+0000", 2023-08-28T02:23:44.678 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00033890800000000002 2023-08-28T02:23:44.678 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.678 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.678 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,1", 2023-08-28T02:23:44.678 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.939897+0000", 2023-08-28T02:23:44.678 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0024355760000000001 2023-08-28T02:23:44.678 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.678 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.678 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:44.678 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.942333+0000", 2023-08-28T02:23:44.678 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.001518625 2023-08-28T02:23:44.678 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.678 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.679 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:44.679 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.943852+0000", 2023-08-28T02:23:44.679 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.187662422 2023-08-28T02:23:44.679 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.679 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.679 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:44.679 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:30.131514+0000", 2023-08-28T02:23:44.679 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.5883000000000001e-05 2023-08-28T02:23:44.679 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.679 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.679 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:44.679 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:30.131560+0000", 2023-08-28T02:23:44.680 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.4103e-05 2023-08-28T02:23:44.680 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.680 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.680 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:44.680 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:30.131614+0000", 2023-08-28T02:23:44.680 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.192484339 2023-08-28T02:23:44.680 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:44.680 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:44.680 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:44.680 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.680 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.680 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:45886 5.7 5:e3b4df8e:::benchmark_data_smithi143_236200_object45885:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-08-28T02:23:44.681 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:22:29.943805+0000", 2023-08-28T02:23:44.681 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 74.713228814999994, 2023-08-28T02:23:44.681 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.18792346900000001, 2023-08-28T02:23:44.681 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:44.681 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:44.681 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:44.681 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:44.681 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:44.681 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 45886 2023-08-28T02:23:44.681 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.681 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:44.681 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.682 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:44.682 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.943805+0000", 2023-08-28T02:23:44.682 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:44.682 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.682 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.682 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:44.682 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.943805+0000", 2023-08-28T02:23:44.682 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:23:44.682 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.682 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.682 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:44.682 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.943803+0000", 2023-08-28T02:23:44.683 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.1723000000000005e-05 2023-08-28T02:23:44.683 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.683 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.683 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:44.683 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.943865+0000", 2023-08-28T02:23:44.683 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0234000000000001e-05 2023-08-28T02:23:44.683 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.683 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.683 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:44.683 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.943875+0000", 2023-08-28T02:23:44.683 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.6799999999999998e-06 2023-08-28T02:23:44.683 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.684 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.684 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:44.684 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.943881+0000", 2023-08-28T02:23:44.684 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.5068000000000002e-05 2023-08-28T02:23:44.684 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.684 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.684 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:44.684 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.943936+0000", 2023-08-28T02:23:44.684 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00013820999999999999 2023-08-28T02:23:44.684 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.684 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.684 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:44.684 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.944074+0000", 2023-08-28T02:23:44.685 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000171552 2023-08-28T02:23:44.685 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.685 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.685 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,1", 2023-08-28T02:23:44.685 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.944246+0000", 2023-08-28T02:23:44.685 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0046995479999999996 2023-08-28T02:23:44.685 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.685 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.685 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:44.685 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.948946+0000", 2023-08-28T02:23:44.685 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.5680000000000002e-05 2023-08-28T02:23:44.685 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.686 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.686 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:44.686 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.949001+0000", 2023-08-28T02:23:44.686 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.182646063 2023-08-28T02:23:44.686 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.686 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.686 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:44.686 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:30.131647+0000", 2023-08-28T02:23:44.686 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.6298000000000003e-05 2023-08-28T02:23:44.686 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.686 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.686 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:44.687 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:30.131694+0000", 2023-08-28T02:23:44.687 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.4573999999999998e-05 2023-08-28T02:23:44.687 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.687 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.687 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:44.687 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:30.131728+0000", 2023-08-28T02:23:44.687 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.18792346900000001 2023-08-28T02:23:44.687 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:44.687 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:44.687 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:44.687 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.687 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.688 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:45891 5.7 5:e4df42c7:::benchmark_data_smithi143_236200_object45890:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-08-28T02:23:44.688 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:22:29.946080+0000", 2023-08-28T02:23:44.688 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 74.710953095999997, 2023-08-28T02:23:44.688 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.18575723899999999, 2023-08-28T02:23:44.688 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:44.688 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:44.688 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:44.688 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:44.688 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:44.688 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 45891 2023-08-28T02:23:44.688 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.688 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:44.689 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.689 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:44.689 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.946080+0000", 2023-08-28T02:23:44.689 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:44.689 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.689 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.689 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:44.689 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.946080+0000", 2023-08-28T02:23:44.689 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:23:44.689 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.689 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.689 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:44.689 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.946079+0000", 2023-08-28T02:23:44.690 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.6539000000000002e-05 2023-08-28T02:23:44.690 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.690 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.690 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:44.690 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.946136+0000", 2023-08-28T02:23:44.690 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0481e-05 2023-08-28T02:23:44.690 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.690 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.690 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:44.690 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.946146+0000", 2023-08-28T02:23:44.690 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.7779e-05 2023-08-28T02:23:44.691 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.691 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.691 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:44.691 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.946164+0000", 2023-08-28T02:23:44.691 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0010492329999999999 2023-08-28T02:23:44.691 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.691 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.691 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:44.691 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.947213+0000", 2023-08-28T02:23:44.691 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00014276499999999999 2023-08-28T02:23:44.691 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.692 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.692 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:44.692 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.947356+0000", 2023-08-28T02:23:44.692 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00018582900000000001 2023-08-28T02:23:44.692 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.692 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.692 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,1", 2023-08-28T02:23:44.692 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.947542+0000", 2023-08-28T02:23:44.692 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0015901940000000001 2023-08-28T02:23:44.692 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.692 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.692 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:44.692 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.949132+0000", 2023-08-28T02:23:44.693 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00019757800000000001 2023-08-28T02:23:44.693 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.693 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.693 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:44.693 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.949329+0000", 2023-08-28T02:23:44.693 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.18243326500000001 2023-08-28T02:23:44.693 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.693 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.693 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:44.693 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:30.131763+0000", 2023-08-28T02:23:44.693 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.0436999999999999e-05 2023-08-28T02:23:44.693 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.694 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.694 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:44.694 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:30.131803+0000", 2023-08-28T02:23:44.694 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.4539000000000003e-05 2023-08-28T02:23:44.694 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.694 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.694 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:44.694 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:30.131838+0000", 2023-08-28T02:23:44.694 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.18575723899999999 2023-08-28T02:23:44.694 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:44.694 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:44.694 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:44.695 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.695 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.695 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:45895 5.7 5:e296ab7b:::benchmark_data_smithi143_236200_object45894:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-08-28T02:23:44.695 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:22:29.949232+0000", 2023-08-28T02:23:44.695 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 74.707801775999997, 2023-08-28T02:23:44.695 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.184808526, 2023-08-28T02:23:44.695 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:44.695 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:44.695 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:44.695 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:44.695 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:44.695 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 45895 2023-08-28T02:23:44.696 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.696 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:44.696 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.696 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:44.696 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.949232+0000", 2023-08-28T02:23:44.696 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:44.696 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.696 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.696 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:44.696 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.949232+0000", 2023-08-28T02:23:44.696 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:23:44.696 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.696 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.697 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:44.697 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.949230+0000", 2023-08-28T02:23:44.697 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000100463 2023-08-28T02:23:44.697 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.697 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.697 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:44.697 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.949331+0000", 2023-08-28T02:23:44.697 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.6290000000000005e-06 2023-08-28T02:23:44.697 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.697 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.697 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:44.697 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.949341+0000", 2023-08-28T02:23:44.698 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.1049999999999997e-06 2023-08-28T02:23:44.698 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.698 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.698 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:44.698 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.949345+0000", 2023-08-28T02:23:44.698 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.1885999999999999e-05 2023-08-28T02:23:44.698 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.698 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.698 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:44.698 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.949377+0000", 2023-08-28T02:23:44.698 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.1335000000000006e-05 2023-08-28T02:23:44.698 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.699 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.699 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:44.699 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.949458+0000", 2023-08-28T02:23:44.699 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000168498 2023-08-28T02:23:44.699 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.699 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.699 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,1", 2023-08-28T02:23:44.699 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.949626+0000", 2023-08-28T02:23:44.699 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0013563010000000001 2023-08-28T02:23:44.699 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.699 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.699 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:44.699 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.950983+0000", 2023-08-28T02:23:44.700 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00017011000000000001 2023-08-28T02:23:44.700 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.700 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.700 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:44.700 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.951153+0000", 2023-08-28T02:23:44.700 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.182796493 2023-08-28T02:23:44.700 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.700 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.700 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:44.700 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:30.133949+0000", 2023-08-28T02:23:44.700 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.9951000000000003e-05 2023-08-28T02:23:44.700 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.701 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.701 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:44.701 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:30.133999+0000", 2023-08-28T02:23:44.701 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.0957000000000001e-05 2023-08-28T02:23:44.701 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.701 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.701 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:44.701 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:30.134040+0000", 2023-08-28T02:23:44.701 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.184808526 2023-08-28T02:23:44.701 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:44.701 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:44.701 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:44.702 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.702 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.702 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:45898 5.7 5:e0c8937d:::benchmark_data_smithi143_236200_object45897:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-08-28T02:23:44.702 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:22:29.954192+0000", 2023-08-28T02:23:44.702 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 74.702841200999998, 2023-08-28T02:23:44.702 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.18140319799999999, 2023-08-28T02:23:44.702 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:44.702 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:44.702 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:44.702 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:44.702 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:44.702 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 45898 2023-08-28T02:23:44.703 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.703 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:44.703 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.703 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:44.703 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.954192+0000", 2023-08-28T02:23:44.703 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:44.703 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.703 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.703 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:44.703 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.954192+0000", 2023-08-28T02:23:44.703 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:23:44.704 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.704 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.704 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:44.704 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.954191+0000", 2023-08-28T02:23:44.704 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.7717999999999999e-05 2023-08-28T02:23:44.704 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.704 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.704 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:44.704 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.954249+0000", 2023-08-28T02:23:44.704 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2102e-05 2023-08-28T02:23:44.704 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.704 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.705 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:44.705 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.954261+0000", 2023-08-28T02:23:44.705 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.3629999999999999e-06 2023-08-28T02:23:44.705 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.705 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.705 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:44.705 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.954267+0000", 2023-08-28T02:23:44.705 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.3949000000000002e-05 2023-08-28T02:23:44.705 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.705 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.705 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:44.705 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.954311+0000", 2023-08-28T02:23:44.706 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.6087000000000004e-05 2023-08-28T02:23:44.706 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.706 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.706 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:44.706 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.954397+0000", 2023-08-28T02:23:44.706 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00010553 2023-08-28T02:23:44.706 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.706 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.706 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,1", 2023-08-28T02:23:44.706 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.954503+0000", 2023-08-28T02:23:44.706 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.001079322 2023-08-28T02:23:44.706 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.707 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.707 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:44.707 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.955582+0000", 2023-08-28T02:23:44.707 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000214515 2023-08-28T02:23:44.707 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.707 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.707 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:44.707 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:29.955797+0000", 2023-08-28T02:23:44.707 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.17970570399999999 2023-08-28T02:23:44.707 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.707 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.708 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:44.708 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:30.135502+0000", 2023-08-28T02:23:44.708 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.3857000000000003e-05 2023-08-28T02:23:44.708 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.708 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.708 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:44.708 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:30.135556+0000", 2023-08-28T02:23:44.708 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.9175999999999999e-05 2023-08-28T02:23:44.708 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.708 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.708 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:44.708 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:30.135595+0000", 2023-08-28T02:23:44.709 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.18140319799999999 2023-08-28T02:23:44.709 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:44.709 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:44.709 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:44.709 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.709 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.709 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:49467 5.7 5:ee46eff3:::benchmark_data_smithi143_236200_object49466:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-08-28T02:23:44.709 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:22:32.557920+0000", 2023-08-28T02:23:44.709 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 72.099113590000002, 2023-08-28T02:23:44.709 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.16165609, 2023-08-28T02:23:44.709 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:44.709 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:44.710 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:44.710 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:44.710 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:44.710 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 49467 2023-08-28T02:23:44.710 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.710 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:44.710 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.710 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:44.710 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:32.557920+0000", 2023-08-28T02:23:44.710 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:44.710 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.710 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.711 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:44.711 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:32.557920+0000", 2023-08-28T02:23:44.711 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:23:44.711 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.711 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.711 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:44.711 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:32.557919+0000", 2023-08-28T02:23:44.711 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000113566 2023-08-28T02:23:44.711 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.711 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.711 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:44.711 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:32.558032+0000", 2023-08-28T02:23:44.712 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2568999999999999e-05 2023-08-28T02:23:44.712 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.712 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.712 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:44.712 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:32.558045+0000", 2023-08-28T02:23:44.712 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.5599999999999999e-06 2023-08-28T02:23:44.712 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.712 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.712 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:44.712 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:32.558051+0000", 2023-08-28T02:23:44.712 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.5595e-05 2023-08-28T02:23:44.712 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.713 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.713 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:44.713 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:32.558097+0000", 2023-08-28T02:23:44.713 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000131376 2023-08-28T02:23:44.713 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.713 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.713 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:44.713 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:32.558228+0000", 2023-08-28T02:23:44.713 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000175193 2023-08-28T02:23:44.713 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.713 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.713 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,1", 2023-08-28T02:23:44.714 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:32.558404+0000", 2023-08-28T02:23:44.714 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.003335836 2023-08-28T02:23:44.714 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.714 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.714 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:44.714 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:32.561739+0000", 2023-08-28T02:23:44.714 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0012316969999999999 2023-08-28T02:23:44.714 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.714 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.714 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:44.714 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:32.562971+0000", 2023-08-28T02:23:44.714 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.15652497400000001 2023-08-28T02:23:44.714 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.715 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.715 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:44.715 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:32.719496+0000", 2023-08-28T02:23:44.715 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.0899000000000002e-05 2023-08-28T02:23:44.715 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.715 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.715 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:44.715 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:32.719547+0000", 2023-08-28T02:23:44.715 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.8966e-05 2023-08-28T02:23:44.715 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.715 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.715 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:44.716 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:32.719576+0000", 2023-08-28T02:23:44.716 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.16165609 2023-08-28T02:23:44.716 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:44.716 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:44.716 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:44.716 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.716 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.716 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:49469 5.7 5:e7c60c90:::benchmark_data_smithi143_236200_object49468:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-08-28T02:23:44.716 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:22:32.558062+0000", 2023-08-28T02:23:44.716 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 72.098971731000006, 2023-08-28T02:23:44.716 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.161673861, 2023-08-28T02:23:44.716 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:44.717 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:44.717 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:44.717 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:44.717 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:44.717 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 49469 2023-08-28T02:23:44.717 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.717 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:44.717 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.717 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:44.717 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:32.558062+0000", 2023-08-28T02:23:44.717 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:44.717 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.717 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.718 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:44.718 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:32.558062+0000", 2023-08-28T02:23:44.718 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:23:44.718 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.718 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.718 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:44.718 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:32.558061+0000", 2023-08-28T02:23:44.718 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.9838e-05 2023-08-28T02:23:44.718 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.718 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.718 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:44.718 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:32.558101+0000", 2023-08-28T02:23:44.719 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.2719999999999997e-06 2023-08-28T02:23:44.719 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.719 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.719 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:44.719 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:32.558106+0000", 2023-08-28T02:23:44.719 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.4089999999999999e-06 2023-08-28T02:23:44.719 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.719 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.719 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:44.719 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:32.558110+0000", 2023-08-28T02:23:44.719 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0017248210000000001 2023-08-28T02:23:44.719 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.720 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.720 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:44.720 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:32.559834+0000", 2023-08-28T02:23:44.720 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00013874100000000001 2023-08-28T02:23:44.720 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.720 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.720 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:44.720 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:32.559973+0000", 2023-08-28T02:23:44.720 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00016596199999999999 2023-08-28T02:23:44.720 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.720 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.720 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,1", 2023-08-28T02:23:44.720 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:32.560139+0000", 2023-08-28T02:23:44.721 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0028762779999999999 2023-08-28T02:23:44.721 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.721 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.721 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:44.721 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:32.563015+0000", 2023-08-28T02:23:44.721 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0024591909999999999 2023-08-28T02:23:44.721 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.721 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.721 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:44.721 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:32.565475+0000", 2023-08-28T02:23:44.721 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.15415958299999999 2023-08-28T02:23:44.721 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.722 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.722 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:44.722 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:32.719634+0000", 2023-08-28T02:23:44.722 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.7469000000000002e-05 2023-08-28T02:23:44.722 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.722 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.722 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:44.722 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:32.719692+0000", 2023-08-28T02:23:44.722 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.4026999999999997e-05 2023-08-28T02:23:44.722 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.722 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.722 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:44.723 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:32.719736+0000", 2023-08-28T02:23:44.723 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.161673861 2023-08-28T02:23:44.723 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:44.723 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:44.723 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:44.723 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.723 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.723 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:77808 5.7 5:e3d8141d:::benchmark_data_smithi143_236200_object77807:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e133)", 2023-08-28T02:23:44.723 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:22:56.777625+0000", 2023-08-28T02:23:44.723 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 47.879408458999997, 2023-08-28T02:23:44.723 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.13409547099999999, 2023-08-28T02:23:44.723 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:44.724 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:44.724 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:44.724 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:44.724 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:44.724 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 77808 2023-08-28T02:23:44.724 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.724 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:44.724 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.724 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:44.724 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.777625+0000", 2023-08-28T02:23:44.724 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:44.724 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.724 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.725 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:44.725 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.777625+0000", 2023-08-28T02:23:44.725 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999952 2023-08-28T02:23:44.725 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.725 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.725 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:44.725 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.777620+0000", 2023-08-28T02:23:44.725 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.8381000000000003e-05 2023-08-28T02:23:44.725 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.725 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.725 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:44.725 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.777668+0000", 2023-08-28T02:23:44.726 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.4019999999999996e-06 2023-08-28T02:23:44.726 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.726 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.726 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:44.726 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.777676+0000", 2023-08-28T02:23:44.726 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.5669999999999996e-06 2023-08-28T02:23:44.726 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.726 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.726 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:44.726 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.777680+0000", 2023-08-28T02:23:44.726 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.3219000000000002e-05 2023-08-28T02:23:44.726 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.727 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.727 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:44.727 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.777714+0000", 2023-08-28T02:23:44.727 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.9695000000000002e-05 2023-08-28T02:23:44.727 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.727 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.727 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:44.727 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.777793+0000", 2023-08-28T02:23:44.727 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.8913000000000007e-05 2023-08-28T02:23:44.727 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.727 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.727 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,1", 2023-08-28T02:23:44.728 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.777892+0000", 2023-08-28T02:23:44.728 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0057218190000000004 2023-08-28T02:23:44.728 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.728 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.728 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:44.728 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.783614+0000", 2023-08-28T02:23:44.728 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.097838284999999997 2023-08-28T02:23:44.728 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.728 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.728 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:44.728 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.881452+0000", 2023-08-28T02:23:44.728 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.030170354 2023-08-28T02:23:44.729 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.729 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.729 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:44.729 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.911623+0000", 2023-08-28T02:23:44.729 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.0538000000000002e-05 2023-08-28T02:23:44.729 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.729 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.729 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:44.729 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.911683+0000", 2023-08-28T02:23:44.729 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.7292999999999997e-05 2023-08-28T02:23:44.729 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.729 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.729 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:44.730 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.911720+0000", 2023-08-28T02:23:44.730 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.13409547099999999 2023-08-28T02:23:44.730 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:44.730 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:44.730 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:44.730 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.730 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.730 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:77810 5.7 5:e10a56eb:::benchmark_data_smithi143_236200_object77809:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e133)", 2023-08-28T02:23:44.730 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:22:56.778551+0000", 2023-08-28T02:23:44.730 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 47.878482450999996, 2023-08-28T02:23:44.730 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.13327017699999999, 2023-08-28T02:23:44.730 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:44.730 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:44.730 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:44.731 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:44.731 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:44.731 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 77810 2023-08-28T02:23:44.731 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.731 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:44.731 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.731 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:44.731 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.778551+0000", 2023-08-28T02:23:44.731 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:44.731 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.731 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.731 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:44.732 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.778551+0000", 2023-08-28T02:23:44.732 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:23:44.732 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.732 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.732 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:44.732 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.778550+0000", 2023-08-28T02:23:44.732 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012284399999999999 2023-08-28T02:23:44.732 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.732 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.732 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:44.732 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.778673+0000", 2023-08-28T02:23:44.732 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1168e-05 2023-08-28T02:23:44.733 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.733 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.733 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:44.733 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.778684+0000", 2023-08-28T02:23:44.733 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.7649999999999999e-06 2023-08-28T02:23:44.733 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.733 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.733 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:44.733 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.778690+0000", 2023-08-28T02:23:44.733 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0020734130000000001 2023-08-28T02:23:44.733 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.733 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.734 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:44.734 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.780763+0000", 2023-08-28T02:23:44.734 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.5895000000000003e-05 2023-08-28T02:23:44.734 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.734 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.734 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:44.734 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.780859+0000", 2023-08-28T02:23:44.734 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000116433 2023-08-28T02:23:44.734 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.734 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.734 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,1", 2023-08-28T02:23:44.734 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.780975+0000", 2023-08-28T02:23:44.735 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0028216980000000001 2023-08-28T02:23:44.735 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.735 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.735 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:44.735 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.783797+0000", 2023-08-28T02:23:44.735 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.097686496999999997 2023-08-28T02:23:44.735 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.735 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.735 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:44.735 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.881484+0000", 2023-08-28T02:23:44.735 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.030270308999999999 2023-08-28T02:23:44.735 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.736 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.736 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:44.736 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.911754+0000", 2023-08-28T02:23:44.736 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.6328000000000002e-05 2023-08-28T02:23:44.736 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.736 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.736 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:44.736 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.911790+0000", 2023-08-28T02:23:44.736 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.0945000000000003e-05 2023-08-28T02:23:44.736 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.736 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.736 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:44.737 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:56.911821+0000", 2023-08-28T02:23:44.737 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.13327017699999999 2023-08-28T02:23:44.737 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:44.737 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:44.737 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:44.737 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.737 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.737 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:104449 5.7 5:eb82dfb7:::benchmark_data_smithi143_236200_object104448:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e140)", 2023-08-28T02:23:44.737 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:27.818641+0000", 2023-08-28T02:23:44.737 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 16.838392097, 2023-08-28T02:23:44.737 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.14802080400000001, 2023-08-28T02:23:44.737 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:44.737 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:44.738 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:44.738 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:44.738 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:44.738 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 104449 2023-08-28T02:23:44.738 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.738 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:44.738 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.738 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:44.738 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.818641+0000", 2023-08-28T02:23:44.738 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:44.738 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.738 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.738 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:44.739 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.818641+0000", 2023-08-28T02:23:44.739 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:23:44.739 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.739 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.739 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:44.739 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.818641+0000", 2023-08-28T02:23:44.739 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.7041999999999999e-05 2023-08-28T02:23:44.739 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.739 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.739 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:44.739 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.818708+0000", 2023-08-28T02:23:44.739 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.1459999999999999e-06 2023-08-28T02:23:44.740 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.740 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.740 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:44.740 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.818715+0000", 2023-08-28T02:23:44.740 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.8790000000000002e-06 2023-08-28T02:23:44.740 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.740 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.740 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:44.740 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.818720+0000", 2023-08-28T02:23:44.740 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.017058558000000001 2023-08-28T02:23:44.740 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.740 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.740 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:44.741 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.835778+0000", 2023-08-28T02:23:44.741 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000145079 2023-08-28T02:23:44.741 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.741 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.741 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:44.741 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.835923+0000", 2023-08-28T02:23:44.741 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00015370800000000001 2023-08-28T02:23:44.741 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.741 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.741 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,1", 2023-08-28T02:23:44.741 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.836077+0000", 2023-08-28T02:23:44.741 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.003516117 2023-08-28T02:23:44.742 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.742 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.742 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:44.742 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.839593+0000", 2023-08-28T02:23:44.742 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.119163174 2023-08-28T02:23:44.742 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.742 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.742 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:44.742 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.958756+0000", 2023-08-28T02:23:44.742 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0078271680000000007 2023-08-28T02:23:44.742 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.742 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.742 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:44.743 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.966584+0000", 2023-08-28T02:23:44.743 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.3544999999999997e-05 2023-08-28T02:23:44.743 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.743 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.743 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:44.743 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.966627+0000", 2023-08-28T02:23:44.743 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.5108000000000003e-05 2023-08-28T02:23:44.743 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.743 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.743 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:44.743 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.966662+0000", 2023-08-28T02:23:44.743 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.14802080400000001 2023-08-28T02:23:44.744 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:44.744 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:44.744 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:44.744 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.744 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.744 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:104450 5.f 5:f6adaf78:::benchmark_data_smithi143_236200_object104449:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e140)", 2023-08-28T02:23:44.744 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:27.818784+0000", 2023-08-28T02:23:44.744 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 16.838249020999999, 2023-08-28T02:23:44.744 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.147784732, 2023-08-28T02:23:44.744 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:44.744 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:44.744 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:44.745 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:44.745 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:44.745 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 104450 2023-08-28T02:23:44.745 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.745 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:44.745 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.745 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:44.745 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.818784+0000", 2023-08-28T02:23:44.745 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:44.745 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.745 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.745 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:44.746 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.818784+0000", 2023-08-28T02:23:44.746 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:23:44.746 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.746 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.746 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:44.746 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.818784+0000", 2023-08-28T02:23:44.746 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.4469000000000003e-05 2023-08-28T02:23:44.746 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.746 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.746 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:44.746 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.818838+0000", 2023-08-28T02:23:44.746 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.0830000000000004e-06 2023-08-28T02:23:44.747 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.747 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.747 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:44.747 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.818846+0000", 2023-08-28T02:23:44.747 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.0810000000000001e-06 2023-08-28T02:23:44.747 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.747 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.747 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:44.747 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.818851+0000", 2023-08-28T02:23:44.747 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.01541789 2023-08-28T02:23:44.747 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.747 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.748 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:44.748 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.834269+0000", 2023-08-28T02:23:44.748 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000129064 2023-08-28T02:23:44.748 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.748 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.748 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:44.748 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.834398+0000", 2023-08-28T02:23:44.748 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000159364 2023-08-28T02:23:44.748 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.748 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.748 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 4,6", 2023-08-28T02:23:44.748 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.834558+0000", 2023-08-28T02:23:44.748 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0049398280000000003 2023-08-28T02:23:44.749 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.749 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.749 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:44.749 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.839497+0000", 2023-08-28T02:23:44.749 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.12297635 2023-08-28T02:23:44.749 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.749 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.749 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:44.749 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.962474+0000", 2023-08-28T02:23:44.749 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0039933950000000003 2023-08-28T02:23:44.749 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.749 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.750 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:44.750 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.966467+0000", 2023-08-28T02:23:44.750 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.7676999999999997e-05 2023-08-28T02:23:44.750 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.750 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.750 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:44.750 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.966525+0000", 2023-08-28T02:23:44.750 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.4385999999999998e-05 2023-08-28T02:23:44.750 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.750 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.750 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:44.750 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.966569+0000", 2023-08-28T02:23:44.750 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.147784732 2023-08-28T02:23:44.751 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:44.751 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:44.751 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:44.751 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.751 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.751 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:104452 5.f 5:f9ed2cfe:::benchmark_data_smithi143_236200_object104451:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e140)", 2023-08-28T02:23:44.751 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:27.832243+0000", 2023-08-28T02:23:44.751 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 16.824789980999999, 2023-08-28T02:23:44.751 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.13451121399999999, 2023-08-28T02:23:44.751 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:44.751 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:44.751 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:44.752 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:44.752 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:44.752 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 104452 2023-08-28T02:23:44.752 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.752 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:44.752 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.752 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:44.752 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.832243+0000", 2023-08-28T02:23:44.752 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:44.752 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.752 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.752 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:44.753 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.832243+0000", 2023-08-28T02:23:44.753 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:23:44.753 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.753 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.753 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:44.753 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.832242+0000", 2023-08-28T02:23:44.753 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.4122999999999998e-05 2023-08-28T02:23:44.753 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.753 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.753 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:44.753 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.832286+0000", 2023-08-28T02:23:44.753 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2656e-05 2023-08-28T02:23:44.754 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.754 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.754 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:44.754 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.832299+0000", 2023-08-28T02:23:44.754 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.4180000000000002e-06 2023-08-28T02:23:44.754 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.754 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.754 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:44.754 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.832305+0000", 2023-08-28T02:23:44.754 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.007240158 2023-08-28T02:23:44.754 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.754 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.754 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:44.755 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.839545+0000", 2023-08-28T02:23:44.755 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00013731299999999999 2023-08-28T02:23:44.755 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.755 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.755 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:44.755 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.839683+0000", 2023-08-28T02:23:44.755 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.115607949 2023-08-28T02:23:44.755 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.755 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.755 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 4,6", 2023-08-28T02:23:44.755 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.955291+0000", 2023-08-28T02:23:44.755 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.011344501999999999 2023-08-28T02:23:44.755 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.756 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.756 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:44.756 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.966635+0000", 2023-08-28T02:23:44.756 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.3052999999999997e-05 2023-08-28T02:23:44.756 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.756 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.756 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:44.756 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.966668+0000", 2023-08-28T02:23:44.756 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.9790000000000005e-06 2023-08-28T02:23:44.756 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.756 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.756 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:44.757 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.966678+0000", 2023-08-28T02:23:44.757 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.1248000000000002e-05 2023-08-28T02:23:44.757 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.757 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.757 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:44.757 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.966719+0000", 2023-08-28T02:23:44.757 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.5420999999999998e-05 2023-08-28T02:23:44.757 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.757 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.757 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:44.757 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.966755+0000", 2023-08-28T02:23:44.757 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.13451121399999999 2023-08-28T02:23:44.757 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:44.758 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:44.758 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:44.758 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.758 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.758 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:104453 5.f 5:f85e1207:::benchmark_data_smithi143_236200_object104452:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e140)", 2023-08-28T02:23:44.758 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:27.832312+0000", 2023-08-28T02:23:44.758 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 16.824721820000001, 2023-08-28T02:23:44.758 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.13592269000000001, 2023-08-28T02:23:44.758 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:44.758 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:44.758 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:44.758 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:44.758 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:44.759 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 104453 2023-08-28T02:23:44.759 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.759 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:44.759 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.759 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:44.759 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.832312+0000", 2023-08-28T02:23:44.759 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:44.759 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.759 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.759 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:44.759 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.832312+0000", 2023-08-28T02:23:44.759 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999995 2023-08-28T02:23:44.760 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.760 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.760 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:44.760 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.832311+0000", 2023-08-28T02:23:44.760 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.7583000000000003e-05 2023-08-28T02:23:44.760 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.760 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.760 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:44.760 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.832349+0000", 2023-08-28T02:23:44.760 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.5889999999999999e-06 2023-08-28T02:23:44.760 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.760 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.760 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:44.761 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.832355+0000", 2023-08-28T02:23:44.761 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.5549999999999996e-06 2023-08-28T02:23:44.761 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.761 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.761 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:44.761 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.832360+0000", 2023-08-28T02:23:44.761 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.12642231700000001 2023-08-28T02:23:44.761 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.761 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.761 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:44.761 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.958782+0000", 2023-08-28T02:23:44.761 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000150316 2023-08-28T02:23:44.761 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.762 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.762 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:44.762 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.958933+0000", 2023-08-28T02:23:44.762 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00016453099999999999 2023-08-28T02:23:44.762 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.762 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.762 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 4,6", 2023-08-28T02:23:44.762 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.959097+0000", 2023-08-28T02:23:44.762 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0077232810000000002 2023-08-28T02:23:44.762 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.762 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.762 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:44.762 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.966820+0000", 2023-08-28T02:23:44.763 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.2130999999999999e-05 2023-08-28T02:23:44.763 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.763 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.763 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:44.763 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.966842+0000", 2023-08-28T02:23:44.763 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.001290397 2023-08-28T02:23:44.763 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.763 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.763 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:44.763 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.968133+0000", 2023-08-28T02:23:44.763 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.9895999999999997e-05 2023-08-28T02:23:44.763 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.764 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.764 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:44.764 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.968193+0000", 2023-08-28T02:23:44.764 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.1594999999999997e-05 2023-08-28T02:23:44.764 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.764 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.764 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:44.764 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.968234+0000", 2023-08-28T02:23:44.764 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.13592269000000001 2023-08-28T02:23:44.764 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:44.764 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:44.764 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:44.764 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.765 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.765 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:104454 5.f 5:fb68fd32:::benchmark_data_smithi143_236200_object104453:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e140)", 2023-08-28T02:23:44.765 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:27.832364+0000", 2023-08-28T02:23:44.765 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 16.824669627999999, 2023-08-28T02:23:44.765 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.136021646, 2023-08-28T02:23:44.765 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:44.765 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:44.765 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:44.765 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:44.765 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:44.765 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 104454 2023-08-28T02:23:44.765 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.765 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:44.766 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.766 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:44.766 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.832364+0000", 2023-08-28T02:23:44.766 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:44.766 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.766 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.766 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:44.766 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.832364+0000", 2023-08-28T02:23:44.766 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999995 2023-08-28T02:23:44.766 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.766 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.766 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:44.766 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.832363+0000", 2023-08-28T02:23:44.767 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.1074000000000001e-05 2023-08-28T02:23:44.767 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.767 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.767 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:44.767 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.832394+0000", 2023-08-28T02:23:44.767 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.0249999999999999e-06 2023-08-28T02:23:44.767 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.767 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.767 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:44.767 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.832400+0000", 2023-08-28T02:23:44.767 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.6540000000000001e-06 2023-08-28T02:23:44.767 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.768 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.768 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:44.768 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.832404+0000", 2023-08-28T02:23:44.768 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.130170337 2023-08-28T02:23:44.768 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.768 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.768 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:44.768 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.962574+0000", 2023-08-28T02:23:44.768 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000111082 2023-08-28T02:23:44.768 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.768 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.768 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:44.768 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.962686+0000", 2023-08-28T02:23:44.769 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00013859199999999999 2023-08-28T02:23:44.769 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.769 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.769 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 4,6", 2023-08-28T02:23:44.769 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.962824+0000", 2023-08-28T02:23:44.769 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0043236749999999999 2023-08-28T02:23:44.769 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.769 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.769 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:44.769 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.967148+0000", 2023-08-28T02:23:44.769 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0011207190000000001 2023-08-28T02:23:44.769 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.770 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.770 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:44.770 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.968269+0000", 2023-08-28T02:23:44.770 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.4240000000000003e-05 2023-08-28T02:23:44.770 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.770 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.770 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:44.770 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.968313+0000", 2023-08-28T02:23:44.770 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.9814000000000002e-05 2023-08-28T02:23:44.770 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.770 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.770 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:44.770 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.968353+0000", 2023-08-28T02:23:44.771 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.2879000000000001e-05 2023-08-28T02:23:44.771 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.771 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.771 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:44.771 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.968385+0000", 2023-08-28T02:23:44.771 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.136021646 2023-08-28T02:23:44.771 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:44.771 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:44.771 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:44.771 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.771 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.771 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:106256 5.7 5:eacd22f9:::benchmark_data_smithi143_236200_object106255:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e142)", 2023-08-28T02:23:44.772 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:30.530796+0000", 2023-08-28T02:23:44.772 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 14.126237538, 2023-08-28T02:23:44.772 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.13318993500000001, 2023-08-28T02:23:44.772 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:44.772 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:44.772 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:44.772 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:44.772 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:44.772 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 106256 2023-08-28T02:23:44.772 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.772 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:44.772 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.772 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:44.773 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:30.530796+0000", 2023-08-28T02:23:44.773 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:44.773 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.773 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.773 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:44.773 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:30.530796+0000", 2023-08-28T02:23:44.773 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999995 2023-08-28T02:23:44.773 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.773 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.773 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:44.773 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:30.530795+0000", 2023-08-28T02:23:44.773 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.3376e-05 2023-08-28T02:23:44.773 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.774 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.774 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:44.774 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:30.530829+0000", 2023-08-28T02:23:44.774 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.2400000000000004e-06 2023-08-28T02:23:44.774 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.774 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.774 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:44.774 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:30.530835+0000", 2023-08-28T02:23:44.774 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.98e-06 2023-08-28T02:23:44.774 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.774 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.774 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:44.774 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:30.530839+0000", 2023-08-28T02:23:44.775 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0042489249999999998 2023-08-28T02:23:44.775 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.775 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.775 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:44.775 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:30.535088+0000", 2023-08-28T02:23:44.775 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000143931 2023-08-28T02:23:44.775 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.775 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.775 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:44.775 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:30.535232+0000", 2023-08-28T02:23:44.775 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00016260600000000001 2023-08-28T02:23:44.775 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.776 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.776 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,1", 2023-08-28T02:23:44.776 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:30.535394+0000", 2023-08-28T02:23:44.776 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0037419670000000001 2023-08-28T02:23:44.776 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.776 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.776 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:44.776 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:30.539136+0000", 2023-08-28T02:23:44.776 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0044253909999999999 2023-08-28T02:23:44.776 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.776 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.776 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:44.776 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:30.543562+0000", 2023-08-28T02:23:44.776 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.12035935 2023-08-28T02:23:44.777 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.777 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.777 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:44.777 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:30.663921+0000", 2023-08-28T02:23:44.777 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.6455999999999998e-05 2023-08-28T02:23:44.777 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.777 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.777 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:44.777 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:30.663958+0000", 2023-08-28T02:23:44.777 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.8238999999999998e-05 2023-08-28T02:23:44.777 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.777 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.778 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:44.778 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:30.663986+0000", 2023-08-28T02:23:44.778 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.13318993500000001 2023-08-28T02:23:44.778 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:44.778 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:44.778 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:44.778 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.778 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.778 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:114127 5.7 5:ec75b1ea:::benchmark_data_smithi143_236200_object3984:head [delete] snapc 0=[] ondisk+write+known_if_redirected e147)", 2023-08-28T02:23:44.778 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:43.602698+0000", 2023-08-28T02:23:44.778 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 1.054335096, 2023-08-28T02:23:44.778 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.13897567199999999, 2023-08-28T02:23:44.779 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:44.779 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:44.779 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:44.779 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:44.779 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:44.779 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 114127 2023-08-28T02:23:44.779 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.779 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:44.779 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.779 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:44.779 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.602698+0000", 2023-08-28T02:23:44.779 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:44.780 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.780 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.780 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:44.780 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.602698+0000", 2023-08-28T02:23:44.780 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:23:44.780 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.780 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.780 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:44.780 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.602697+0000", 2023-08-28T02:23:44.780 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.9719999999999992e-06 2023-08-28T02:23:44.780 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.780 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.780 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:44.781 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.602705+0000", 2023-08-28T02:23:44.781 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2561999999999999e-05 2023-08-28T02:23:44.781 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.781 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.781 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:44.781 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.602717+0000", 2023-08-28T02:23:44.781 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.9580000000000001e-06 2023-08-28T02:23:44.781 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.781 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.781 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:44.781 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.602724+0000", 2023-08-28T02:23:44.781 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.1270000000000003e-05 2023-08-28T02:23:44.782 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.782 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.782 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:44.782 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.602765+0000", 2023-08-28T02:23:44.782 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000109314 2023-08-28T02:23:44.782 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.782 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.782 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:44.782 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.602875+0000", 2023-08-28T02:23:44.782 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000115627 2023-08-28T02:23:44.782 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.782 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.783 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,1", 2023-08-28T02:23:44.783 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.602990+0000", 2023-08-28T02:23:44.783 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0083496209999999998 2023-08-28T02:23:44.783 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.783 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.783 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:44.783 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.611340+0000", 2023-08-28T02:23:44.783 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000222499 2023-08-28T02:23:44.783 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.783 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.783 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:44.783 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.611562+0000", 2023-08-28T02:23:44.783 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.13001283299999999 2023-08-28T02:23:44.784 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.784 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.784 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:44.784 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.741575+0000", 2023-08-28T02:23:44.784 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.3967e-05 2023-08-28T02:23:44.784 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.784 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.784 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:44.784 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.741629+0000", 2023-08-28T02:23:44.784 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.481e-05 2023-08-28T02:23:44.784 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.785 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.785 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:44.785 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.741674+0000", 2023-08-28T02:23:44.785 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.13897567199999999 2023-08-28T02:23:44.785 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:44.785 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:44.785 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:44.785 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.785 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.785 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:114141 5.f 5:f823ae24:::benchmark_data_smithi143_236200_object3998:head [delete] snapc 0=[] ondisk+write+known_if_redirected e147)", 2023-08-28T02:23:44.785 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:43.612321+0000", 2023-08-28T02:23:44.785 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 1.0447122719999999, 2023-08-28T02:23:44.785 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.15620606200000001, 2023-08-28T02:23:44.786 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:44.786 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:44.786 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:44.786 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:44.786 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:44.786 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 114141 2023-08-28T02:23:44.786 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.786 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:44.786 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.786 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:44.786 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.612321+0000", 2023-08-28T02:23:44.786 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:44.786 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.787 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.787 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:44.787 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.612321+0000", 2023-08-28T02:23:44.787 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:23:44.787 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.787 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.787 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:44.787 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.612320+0000", 2023-08-28T02:23:44.787 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.1970000000000004e-06 2023-08-28T02:23:44.787 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.787 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.787 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:44.787 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.612325+0000", 2023-08-28T02:23:44.788 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.4610000000000008e-06 2023-08-28T02:23:44.788 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.788 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.788 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:44.788 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.612334+0000", 2023-08-28T02:23:44.788 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.5379999999999996e-06 2023-08-28T02:23:44.788 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.788 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.788 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:44.788 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.612338+0000", 2023-08-28T02:23:44.788 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.8702999999999997e-05 2023-08-28T02:23:44.788 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.789 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.789 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:44.789 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.612397+0000", 2023-08-28T02:23:44.789 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000103094 2023-08-28T02:23:44.789 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.789 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.789 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:44.789 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.612500+0000", 2023-08-28T02:23:44.789 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000113575 2023-08-28T02:23:44.789 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.789 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.789 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 4,6", 2023-08-28T02:23:44.790 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.612614+0000", 2023-08-28T02:23:44.790 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.132457034 2023-08-28T02:23:44.790 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.790 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.790 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:44.790 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.745071+0000", 2023-08-28T02:23:44.790 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000235932 2023-08-28T02:23:44.790 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.790 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.790 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:44.790 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.745307+0000", 2023-08-28T02:23:44.790 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.023132363999999999 2023-08-28T02:23:44.790 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.791 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.791 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:44.791 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.768439+0000", 2023-08-28T02:23:44.791 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.0393999999999999e-05 2023-08-28T02:23:44.791 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.791 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.791 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:44.791 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.768489+0000", 2023-08-28T02:23:44.791 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.7926999999999998e-05 2023-08-28T02:23:44.791 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.791 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.791 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:44.792 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.768527+0000", 2023-08-28T02:23:44.792 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.15620606200000001 2023-08-28T02:23:44.792 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:44.792 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:44.792 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:44.792 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.792 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.792 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_repop(client.44594.0:114140 5.e e147/119 5:75aa45ca:::benchmark_data_smithi143_236200_object3997:head v 147'7209, mlcod=147'7207)", 2023-08-28T02:23:44.792 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:43.612682+0000", 2023-08-28T02:23:44.792 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 1.0443511249999999, 2023-08-28T02:23:44.792 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.137555439, 2023-08-28T02:23:44.792 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:44.792 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:44.793 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:44.793 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.793 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:44.793 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.612682+0000", 2023-08-28T02:23:44.793 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:44.793 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.793 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.793 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:44.793 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.612682+0000", 2023-08-28T02:23:44.793 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:23:44.793 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.793 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.794 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:44.794 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.612681+0000", 2023-08-28T02:23:44.794 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.3629999999999994e-06 2023-08-28T02:23:44.794 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.794 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.794 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:44.794 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.612690+0000", 2023-08-28T02:23:44.794 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.5550000000000008e-06 2023-08-28T02:23:44.794 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.794 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.794 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:44.794 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.612699+0000", 2023-08-28T02:23:44.794 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.5110000000000003e-06 2023-08-28T02:23:44.795 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.795 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.795 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:44.795 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.612705+0000", 2023-08-28T02:23:44.795 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.12882352799999999 2023-08-28T02:23:44.795 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.795 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.795 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:44.795 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.741528+0000", 2023-08-28T02:23:44.795 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.4595000000000001e-05 2023-08-28T02:23:44.795 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.795 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.796 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:44.796 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.741553+0000", 2023-08-28T02:23:44.796 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0086533019999999999 2023-08-28T02:23:44.796 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.796 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.796 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:44.796 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.750206+0000", 2023-08-28T02:23:44.796 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.1476000000000002e-05 2023-08-28T02:23:44.796 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:44.796 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:44.796 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:44.796 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.750238+0000", 2023-08-28T02:23:44.797 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.137555439 2023-08-28T02:23:44.797 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:44.797 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:44.797 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:44.797 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:44.797 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:44.797 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:23:44.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:44.798 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:23:44.798 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:23:44.827 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:23:44.899 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:23:44.926 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:44 smithi169 ceph-mon[159415]: osdmap e148: 8 total, 8 up, 7 in 2023-08-28T02:23:44.926 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:44 smithi169 ceph-mon[159415]: failed to encode map e148 with expected crc 2023-08-28T02:23:44.926 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:44 smithi169 ceph-mon[159415]: failed to encode map e148 with expected crc 2023-08-28T02:23:44.927 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:44 smithi169 ceph-mon[159415]: failed to encode map e148 with expected crc 2023-08-28T02:23:44.927 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:44 smithi169 ceph-mon[159415]: failed to encode map e148 with expected crc 2023-08-28T02:23:44.930 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:45.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:44 smithi143 ceph-mon[183059]: osdmap e148: 8 total, 8 up, 7 in 2023-08-28T02:23:45.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:44 smithi143 ceph-mon[183059]: failed to encode map e148 with expected crc 2023-08-28T02:23:45.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:44 smithi143 ceph-mon[183059]: failed to encode map e148 with expected crc 2023-08-28T02:23:45.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:44 smithi143 ceph-mon[183059]: failed to encode map e148 with expected crc 2023-08-28T02:23:45.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:44 smithi143 ceph-mon[183059]: failed to encode map e148 with expected crc 2023-08-28T02:23:45.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:44 smithi143 ceph-mon[189871]: osdmap e148: 8 total, 8 up, 7 in 2023-08-28T02:23:45.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:44 smithi143 ceph-mon[189871]: failed to encode map e148 with expected crc 2023-08-28T02:23:45.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:44 smithi143 ceph-mon[189871]: failed to encode map e148 with expected crc 2023-08-28T02:23:45.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:44 smithi143 ceph-mon[189871]: failed to encode map e148 with expected crc 2023-08-28T02:23:45.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:44 smithi143 ceph-mon[189871]: failed to encode map e148 with expected crc 2023-08-28T02:23:45.414 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:23:45.874 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:45 smithi169 ceph-mon[159415]: failed to encode map e149 with expected crc 2023-08-28T02:23:45.874 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:45 smithi169 ceph-mon[159415]: failed to encode map e149 with expected crc 2023-08-28T02:23:45.980 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:23:45.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:45.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:23:46.090 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:23:46.148 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:45 smithi169 ceph-mon[159415]: osdmap e149: 8 total, 8 up, 7 in 2023-08-28T02:23:46.149 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:45 smithi169 ceph-mon[159415]: failed to encode map e149 with expected crc 2023-08-28T02:23:46.149 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:45 smithi169 ceph-mon[159415]: failed to encode map e149 with expected crc 2023-08-28T02:23:46.149 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:45 smithi169 ceph-mon[159415]: pgmap v308: 89 pgs: 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 490 KiB/s wr, 571 op/s; 1138/320679 objects degraded (0.355%); 450/320679 objects misplaced (0.140%); 2.7 MiB/s, 54 objects/s recovering 2023-08-28T02:23:46.175 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.6 dump_ops_in_flight 2023-08-28T02:23:46.181 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:46.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:45 smithi143 ceph-mon[183059]: failed to encode map e149 with expected crc 2023-08-28T02:23:46.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:45 smithi143 ceph-mon[183059]: failed to encode map e149 with expected crc 2023-08-28T02:23:46.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:45 smithi143 ceph-mon[183059]: osdmap e149: 8 total, 8 up, 7 in 2023-08-28T02:23:46.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:45 smithi143 ceph-mon[183059]: failed to encode map e149 with expected crc 2023-08-28T02:23:46.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:45 smithi143 ceph-mon[183059]: failed to encode map e149 with expected crc 2023-08-28T02:23:46.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:45 smithi143 ceph-mon[183059]: pgmap v308: 89 pgs: 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 490 KiB/s wr, 571 op/s; 1138/320679 objects degraded (0.355%); 450/320679 objects misplaced (0.140%); 2.7 MiB/s, 54 objects/s recovering 2023-08-28T02:23:46.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:45 smithi143 ceph-mon[189871]: failed to encode map e149 with expected crc 2023-08-28T02:23:46.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:45 smithi143 ceph-mon[189871]: failed to encode map e149 with expected crc 2023-08-28T02:23:46.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:45 smithi143 ceph-mon[189871]: osdmap e149: 8 total, 8 up, 7 in 2023-08-28T02:23:46.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:45 smithi143 ceph-mon[189871]: failed to encode map e149 with expected crc 2023-08-28T02:23:46.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:45 smithi143 ceph-mon[189871]: failed to encode map e149 with expected crc 2023-08-28T02:23:46.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:45 smithi143 ceph-mon[189871]: pgmap v308: 89 pgs: 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 490 KiB/s wr, 571 op/s; 1138/320679 objects degraded (0.355%); 450/320679 objects misplaced (0.140%); 2.7 MiB/s, 54 objects/s recovering 2023-08-28T02:23:46.281 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:23:46.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:46.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:23:46.392 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:23:46.426 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:46.609 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:23:46.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:46.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:23:46.690 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:23:46.690 INFO:teuthology.orchestra.run.smithi169.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-08-28T02:23:46.690 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:23:46.713 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:23:46.743 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:46.837 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:23:46.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:46.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:23:46.941 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:23:46.967 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:46 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 1138/320679 objects degraded (0.355%), 8 pgs degraded (PG_DEGRADED) 2023-08-28T02:23:46.967 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:46 smithi169 ceph-mon[159415]: failed to encode map e150 with expected crc 2023-08-28T02:23:46.967 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:46 smithi169 ceph-mon[159415]: osdmap e150: 8 total, 8 up, 7 in 2023-08-28T02:23:46.967 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:46 smithi169 ceph-mon[159415]: failed to encode map e150 with expected crc 2023-08-28T02:23:46.967 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:46 smithi169 ceph-mon[159415]: failed to encode map e150 with expected crc 2023-08-28T02:23:46.967 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:46 smithi169 ceph-mon[159415]: failed to encode map e150 with expected crc 2023-08-28T02:23:46.967 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:46 smithi169 ceph-mon[159415]: failed to encode map e150 with expected crc 2023-08-28T02:23:46.967 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:46 smithi169 ceph-mon[159415]: failed to encode map e150 with expected crc 2023-08-28T02:23:46.967 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:46 smithi169 ceph-mon[159415]: failed to encode map e150 with expected crc 2023-08-28T02:23:46.967 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:46 smithi169 ceph-mon[159415]: failed to encode map e150 with expected crc 2023-08-28T02:23:46.968 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:46 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1741997990' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:23:46.968 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:46 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:23:46.969 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:47.014 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:23:47.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:47.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:23:47.093 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:46 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 1138/320679 objects degraded (0.355%), 8 pgs degraded (PG_DEGRADED) 2023-08-28T02:23:47.093 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:46 smithi143 ceph-mon[183059]: failed to encode map e150 with expected crc 2023-08-28T02:23:47.093 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:46 smithi143 ceph-mon[183059]: osdmap e150: 8 total, 8 up, 7 in 2023-08-28T02:23:47.093 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:46 smithi143 ceph-mon[183059]: failed to encode map e150 with expected crc 2023-08-28T02:23:47.093 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:46 smithi143 ceph-mon[183059]: failed to encode map e150 with expected crc 2023-08-28T02:23:47.093 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:46 smithi143 ceph-mon[183059]: failed to encode map e150 with expected crc 2023-08-28T02:23:47.093 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:46 smithi143 ceph-mon[183059]: failed to encode map e150 with expected crc 2023-08-28T02:23:47.094 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:46 smithi143 ceph-mon[183059]: failed to encode map e150 with expected crc 2023-08-28T02:23:47.094 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:46 smithi143 ceph-mon[183059]: failed to encode map e150 with expected crc 2023-08-28T02:23:47.094 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:46 smithi143 ceph-mon[183059]: failed to encode map e150 with expected crc 2023-08-28T02:23:47.094 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:46 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1741997990' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:23:47.094 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:46 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:23:47.094 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:46 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 1138/320679 objects degraded (0.355%), 8 pgs degraded (PG_DEGRADED) 2023-08-28T02:23:47.094 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:46 smithi143 ceph-mon[189871]: failed to encode map e150 with expected crc 2023-08-28T02:23:47.094 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:46 smithi143 ceph-mon[189871]: osdmap e150: 8 total, 8 up, 7 in 2023-08-28T02:23:47.094 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:46 smithi143 ceph-mon[189871]: failed to encode map e150 with expected crc 2023-08-28T02:23:47.094 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:46 smithi143 ceph-mon[189871]: failed to encode map e150 with expected crc 2023-08-28T02:23:47.095 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:46 smithi143 ceph-mon[189871]: failed to encode map e150 with expected crc 2023-08-28T02:23:47.095 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:46 smithi143 ceph-mon[189871]: failed to encode map e150 with expected crc 2023-08-28T02:23:47.095 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:46 smithi143 ceph-mon[189871]: failed to encode map e150 with expected crc 2023-08-28T02:23:47.095 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:46 smithi143 ceph-mon[189871]: failed to encode map e150 with expected crc 2023-08-28T02:23:47.095 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:46 smithi143 ceph-mon[189871]: failed to encode map e150 with expected crc 2023-08-28T02:23:47.095 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:46 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1741997990' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:23:47.095 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:46 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:23:47.118 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:23:47.149 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:47.194 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:23:47.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:47.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:23:47.205 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is set 2023-08-28T02:23:47.290 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:23:47.301 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:23:47.329 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:23:47.363 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:23:47.380 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:47.405 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:23:47.437 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:23:47.467 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:23:47.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:47.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:23:47.479 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:23:47.511 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:23:47.524 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:23:47.534 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:23:47.535 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:23:47.536 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:23:47.536 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:23:47.536 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:23:47.536 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:23:47.536 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:23:47.536 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:23:47.536 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:23:47.574 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:23:47.598 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:47.640 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:23:47.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:47.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:23:47.743 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:23:47.788 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:48.065 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:47 smithi143 ceph-mon[183059]: pgmap v310: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 469 KiB/s wr, 741 op/s; 776/317199 objects degraded (0.245%); 334/317199 objects misplaced (0.105%); 7.6 MiB/s, 131 objects/s recovering 2023-08-28T02:23:48.065 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:47 smithi143 ceph-mon[183059]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:23:48.065 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:47 smithi143 ceph-mon[183059]: failed to encode map e151 with expected crc 2023-08-28T02:23:48.065 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:47 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:23:48.065 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:47 smithi143 ceph-mon[183059]: osdmap e151: 8 total, 8 up, 7 in 2023-08-28T02:23:48.065 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:47 smithi143 ceph-mon[183059]: failed to encode map e151 with expected crc 2023-08-28T02:23:48.066 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:47 smithi143 ceph-mon[183059]: failed to encode map e151 with expected crc 2023-08-28T02:23:48.066 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:47 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/206046403' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:23:48.066 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:47 smithi143 ceph-mon[189871]: pgmap v310: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 469 KiB/s wr, 741 op/s; 776/317199 objects degraded (0.245%); 334/317199 objects misplaced (0.105%); 7.6 MiB/s, 131 objects/s recovering 2023-08-28T02:23:48.066 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:47 smithi143 ceph-mon[189871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:23:48.066 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:47 smithi143 ceph-mon[189871]: failed to encode map e151 with expected crc 2023-08-28T02:23:48.066 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:47 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:23:48.066 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:47 smithi143 ceph-mon[189871]: osdmap e151: 8 total, 8 up, 7 in 2023-08-28T02:23:48.066 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:47 smithi143 ceph-mon[189871]: failed to encode map e151 with expected crc 2023-08-28T02:23:48.066 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:47 smithi143 ceph-mon[189871]: failed to encode map e151 with expected crc 2023-08-28T02:23:48.066 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:47 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/206046403' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:23:48.232 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:47 smithi169 ceph-mon[159415]: pgmap v310: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 469 KiB/s wr, 741 op/s; 776/317199 objects degraded (0.245%); 334/317199 objects misplaced (0.105%); 7.6 MiB/s, 131 objects/s recovering 2023-08-28T02:23:48.232 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:47 smithi169 ceph-mon[159415]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:23:48.232 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:47 smithi169 ceph-mon[159415]: failed to encode map e151 with expected crc 2023-08-28T02:23:48.232 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:47 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:23:48.232 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:47 smithi169 ceph-mon[159415]: osdmap e151: 8 total, 8 up, 7 in 2023-08-28T02:23:48.232 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:47 smithi169 ceph-mon[159415]: failed to encode map e151 with expected crc 2023-08-28T02:23:48.232 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:47 smithi169 ceph-mon[159415]: failed to encode map e151 with expected crc 2023-08-28T02:23:48.232 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:47 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/206046403' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:23:48.602 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:23:48.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:48.608 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:23:48.709 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:23:48.782 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:48.939 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:23:48.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:48.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:23:49.059 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:23:49.106 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:49.181 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:23:49.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:49.188 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:23:49.289 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:23:49.381 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:49.529 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:23:49.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:49.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:23:49.620 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:23:49.621 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:23:49.640 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:23:49.672 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:49.747 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:23:49.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:49.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:23:49.853 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:23:49.902 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:50.017 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:23:50.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:50.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:23:50.126 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:23:50.153 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:50.195 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:23:50.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:50.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:23:50.300 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:23:50.382 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:50.409 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set nodeep-scrub 2023-08-28T02:23:50.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:50 smithi143 ceph-mon[183059]: pgmap v312: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 476 op/s; 776/313230 objects degraded (0.248%); 334/313230 objects misplaced (0.107%); 5.7 MiB/s, 91 objects/s recovering 2023-08-28T02:23:50.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:50 smithi143 ceph-mon[189871]: pgmap v312: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 476 op/s; 776/313230 objects degraded (0.248%); 334/313230 objects misplaced (0.107%); 5.7 MiB/s, 91 objects/s recovering 2023-08-28T02:23:50.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:50 smithi169 ceph-mon[159415]: pgmap v312: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 476 op/s; 776/313230 objects degraded (0.248%); 334/313230 objects misplaced (0.107%); 5.7 MiB/s, 91 objects/s recovering 2023-08-28T02:23:50.589 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:23:50.599 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.6 dump_blocked_ops 2023-08-28T02:23:50.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:50.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:23:50.644 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:23:50.708 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:23:50.806 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:50.918 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:23:50.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:50.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:23:51.001 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:23:51.023 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:23:51.095 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:51.247 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:23:51.248 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:23:51.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:51.260 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:23:51.361 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:23:51.408 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:51.434 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:23:51.630 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:23:51.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:51.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:23:51.750 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:23:51.825 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:51.987 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:23:51.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:51.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:23:52.096 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:23:52.148 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:52.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:52 smithi143 ceph-mon[183059]: pgmap v313: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 400 op/s; 776/313230 objects degraded (0.248%); 334/313230 objects misplaced (0.107%); 4.8 MiB/s, 76 objects/s recovering 2023-08-28T02:23:52.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:52 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:23:52.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:52 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 776/313230 objects degraded (0.248%), 7 pgs degraded (PG_DEGRADED) 2023-08-28T02:23:52.201 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:23:51 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:23:51] "GET /metrics HTTP/1.1" 200 36492 "" "Prometheus/2.33.4" 2023-08-28T02:23:52.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:52 smithi143 ceph-mon[189871]: pgmap v313: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 400 op/s; 776/313230 objects degraded (0.248%); 334/313230 objects misplaced (0.107%); 4.8 MiB/s, 76 objects/s recovering 2023-08-28T02:23:52.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:52 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:23:52.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:52 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 776/313230 objects degraded (0.248%), 7 pgs degraded (PG_DEGRADED) 2023-08-28T02:23:52.256 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:23:52.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:52.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:23:52.362 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:23:52.406 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:52.472 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:23:52.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:52.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:23:52.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:52 smithi169 ceph-mon[159415]: pgmap v313: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 400 op/s; 776/313230 objects degraded (0.248%); 334/313230 objects misplaced (0.107%); 4.8 MiB/s, 76 objects/s recovering 2023-08-28T02:23:52.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:52 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:23:52.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:52 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 776/313230 objects degraded (0.248%), 7 pgs degraded (PG_DEGRADED) 2023-08-28T02:23:52.534 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7] out_osds: [3] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-08-28T02:23:52.534 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-08-28T02:23:52.535 INFO:tasks.thrashosds.thrasher:Adding osd 3 2023-08-28T02:23:52.535 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd in 3 2023-08-28T02:23:52.576 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:23:52.603 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:52.645 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:23:52.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:52.648 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:23:52.748 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:23:52.801 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:53.066 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:23:53.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:53.070 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:23:53.171 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:23:53.206 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:53.584 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:23:53.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:53.588 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:23:53.689 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:23:53.808 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:53.828 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:23:53.829 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:23:53.910 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:23:53.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:53.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:23:54.021 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:23:54.112 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:23:54.113 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:54.290 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:23:54.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:54.293 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:23:54.368 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:23:54.387 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:23:54.394 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:23:54.406 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:23:54.436 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:54 smithi143 ceph-mon[189871]: pgmap v314: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 563 op/s; 776/307161 objects degraded (0.253%); 218/307161 objects misplaced (0.071%); 4.6 MiB/s, 74 objects/s recovering 2023-08-28T02:23:54.437 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:54 smithi143 ceph-mon[183059]: pgmap v314: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 563 op/s; 776/307161 objects degraded (0.253%); 218/307161 objects misplaced (0.071%); 4.6 MiB/s, 74 objects/s recovering 2023-08-28T02:23:54.439 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:23:54.439 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:54 smithi169 ceph-mon[159415]: pgmap v314: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 563 op/s; 776/307161 objects degraded (0.253%); 218/307161 objects misplaced (0.071%); 4.6 MiB/s, 74 objects/s recovering 2023-08-28T02:23:54.445 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:54.478 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:23:54.509 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:23:54.542 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:23:54.577 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:23:54.580 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:23:54.580 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:23:54.580 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:23:54.580 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:23:54.580 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:23:54.580 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:23:54.580 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:23:54.580 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:23:54.806 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:23:54.812 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.6 dump_historic_ops 2023-08-28T02:23:54.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:54.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:23:54.923 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:23:54.967 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:55.151 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:23:55.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:55.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:23:55.173 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:55 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1711076895' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:23:55.173 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:55 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2469915960' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:23:55.173 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:55 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:23:55.186 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is set 2023-08-28T02:23:55.255 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:23:55.280 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:55.355 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:23:55.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:55.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:23:55.425 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:23:55.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:55 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1711076895' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:23:55.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:55 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2469915960' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:23:55.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:55 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:23:55.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:55 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1711076895' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:23:55.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:55 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2469915960' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:23:55.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:55 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:23:55.463 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:23:55.494 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:55.577 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:23:55.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:55.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:23:55.680 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:23:55.773 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:55.818 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:23:55.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:55.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:23:55.921 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:23:55.992 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:56.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:56 smithi143 ceph-mon[183059]: pgmap v315: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 688 op/s; 776/302298 objects degraded (0.257%); 218/302298 objects misplaced (0.072%); 4.2 MiB/s, 66 objects/s recovering 2023-08-28T02:23:56.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:56 smithi143 ceph-mon[183059]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:23:56.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:56 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:23:56.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:56 smithi143 ceph-mon[183059]: osdmap e152: 8 total, 8 up, 7 in 2023-08-28T02:23:56.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:56 smithi143 ceph-mon[183059]: failed to encode map e152 with expected crc 2023-08-28T02:23:56.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:56 smithi143 ceph-mon[183059]: failed to encode map e152 with expected crc 2023-08-28T02:23:56.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:56 smithi143 ceph-mon[183059]: failed to encode map e152 with expected crc 2023-08-28T02:23:56.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:56 smithi143 ceph-mon[183059]: failed to encode map e152 with expected crc 2023-08-28T02:23:56.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:56 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:23:56.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:56 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:23:56.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:56 smithi143 ceph-mon[189871]: pgmap v315: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 688 op/s; 776/302298 objects degraded (0.257%); 218/302298 objects misplaced (0.072%); 4.2 MiB/s, 66 objects/s recovering 2023-08-28T02:23:56.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:56 smithi143 ceph-mon[189871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:23:56.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:56 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:23:56.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:56 smithi143 ceph-mon[189871]: osdmap e152: 8 total, 8 up, 7 in 2023-08-28T02:23:56.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:56 smithi143 ceph-mon[189871]: failed to encode map e152 with expected crc 2023-08-28T02:23:56.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:56 smithi143 ceph-mon[189871]: failed to encode map e152 with expected crc 2023-08-28T02:23:56.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:56 smithi143 ceph-mon[189871]: failed to encode map e152 with expected crc 2023-08-28T02:23:56.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:56 smithi143 ceph-mon[189871]: failed to encode map e152 with expected crc 2023-08-28T02:23:56.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:56 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:23:56.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:56 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:23:56.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:56 smithi169 ceph-mon[159415]: pgmap v315: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 688 op/s; 776/302298 objects degraded (0.257%); 218/302298 objects misplaced (0.072%); 4.2 MiB/s, 66 objects/s recovering 2023-08-28T02:23:56.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:56 smithi169 ceph-mon[159415]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:23:56.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:56 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:23:56.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:56 smithi169 ceph-mon[159415]: osdmap e152: 8 total, 8 up, 7 in 2023-08-28T02:23:56.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:56 smithi169 ceph-mon[159415]: failed to encode map e152 with expected crc 2023-08-28T02:23:56.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:56 smithi169 ceph-mon[159415]: failed to encode map e152 with expected crc 2023-08-28T02:23:56.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:56 smithi169 ceph-mon[159415]: failed to encode map e152 with expected crc 2023-08-28T02:23:56.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:56 smithi169 ceph-mon[159415]: failed to encode map e152 with expected crc 2023-08-28T02:23:56.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:56 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:23:56.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:56 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:23:56.638 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:23:56.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:56.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:23:56.743 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:23:56.816 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:56.971 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:23:56.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:56.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:23:57.083 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:23:57.159 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:57.178 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:57 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 776/302298 objects degraded (0.257%), 7 pgs degraded (PG_DEGRADED) 2023-08-28T02:23:57.349 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:23:57.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:57.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:23:57.419 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:57 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 776/302298 objects degraded (0.257%), 7 pgs degraded (PG_DEGRADED) 2023-08-28T02:23:57.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:57 smithi143 conmon[183035]: 2023-08-28T02:23:57.340+0000 7fadc3887700 -1 mon.a@0(leader).osd e152 definitely_dead 0 2023-08-28T02:23:57.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:57 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 776/302298 objects degraded (0.257%), 7 pgs degraded (PG_DEGRADED) 2023-08-28T02:23:57.462 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:23:57.502 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:57.564 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:23:57.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:57.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:23:57.655 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:23:57.656 INFO:teuthology.orchestra.run.smithi169.stdout: "size": 20, 2023-08-28T02:23:57.656 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 600, 2023-08-28T02:23:57.656 INFO:teuthology.orchestra.run.smithi169.stdout: "ops": [ 2023-08-28T02:23:57.656 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.656 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:6104 5.2 5:492807c4:::benchmark_data_smithi143_236200_object6103:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e124)", 2023-08-28T02:23:57.656 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:22:08.997857+0000", 2023-08-28T02:23:57.656 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 108.65438025100001, 2023-08-28T02:23:57.656 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.159556854, 2023-08-28T02:23:57.656 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:57.656 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:57.656 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:57.657 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:57.657 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:57.657 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 6104 2023-08-28T02:23:57.657 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.657 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:57.657 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.657 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:57.657 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.997857+0000", 2023-08-28T02:23:57.657 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:57.657 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.657 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.657 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:57.657 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.997857+0000", 2023-08-28T02:23:57.658 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:23:57.658 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.658 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.658 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:57.658 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.997855+0000", 2023-08-28T02:23:57.658 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.7426999999999998e-05 2023-08-28T02:23:57.658 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.658 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.658 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:57.658 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.997913+0000", 2023-08-28T02:23:57.658 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.9692999999999999e-05 2023-08-28T02:23:57.658 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.658 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.659 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:57.659 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.997932+0000", 2023-08-28T02:23:57.659 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.2269999999999993e-06 2023-08-28T02:23:57.659 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.659 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.659 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:57.659 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.997941+0000", 2023-08-28T02:23:57.659 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.6124e-05 2023-08-28T02:23:57.659 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.659 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.659 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:57.659 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.997997+0000", 2023-08-28T02:23:57.660 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00011772799999999999 2023-08-28T02:23:57.660 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.660 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.660 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:57.660 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.998114+0000", 2023-08-28T02:23:57.660 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00011285699999999999 2023-08-28T02:23:57.660 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.660 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.660 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:23:57.660 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.998227+0000", 2023-08-28T02:23:57.660 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000523728 2023-08-28T02:23:57.660 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.660 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.661 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:57.661 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.998751+0000", 2023-08-28T02:23:57.661 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00024091499999999999 2023-08-28T02:23:57.661 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.661 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.661 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:57.661 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:08.998992+0000", 2023-08-28T02:23:57.661 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.15832805799999999 2023-08-28T02:23:57.661 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.661 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.661 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:57.661 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:09.157320+0000", 2023-08-28T02:23:57.661 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.3788000000000001e-05 2023-08-28T02:23:57.662 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.662 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.662 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:57.662 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:09.157374+0000", 2023-08-28T02:23:57.662 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.9737999999999998e-05 2023-08-28T02:23:57.662 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.662 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.662 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:57.662 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:09.157414+0000", 2023-08-28T02:23:57.662 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.159556854 2023-08-28T02:23:57.662 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:57.662 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:57.662 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:57.663 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.663 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.663 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:72083 5.2 5:44459840:::benchmark_data_smithi143_236200_object72082:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e132)", 2023-08-28T02:23:57.663 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:22:51.496097+0000", 2023-08-28T02:23:57.663 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 66.156139608000004, 2023-08-28T02:23:57.663 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.15569572800000001, 2023-08-28T02:23:57.663 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:57.663 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:57.663 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:57.663 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:57.663 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:57.663 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 72083 2023-08-28T02:23:57.664 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.664 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:57.664 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.664 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:57.664 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.496097+0000", 2023-08-28T02:23:57.664 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:57.664 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.664 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.664 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:57.664 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.496097+0000", 2023-08-28T02:23:57.664 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999976 2023-08-28T02:23:57.664 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.664 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.664 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:57.665 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.496095+0000", 2023-08-28T02:23:57.665 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.5509000000000001e-05 2023-08-28T02:23:57.665 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.665 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.665 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:57.665 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.496140+0000", 2023-08-28T02:23:57.665 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.8080999999999997e-05 2023-08-28T02:23:57.665 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.665 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.665 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:57.665 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.496179+0000", 2023-08-28T02:23:57.665 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.3489999999999992e-06 2023-08-28T02:23:57.665 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.666 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.666 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:57.666 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.496187+0000", 2023-08-28T02:23:57.666 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.7048e-05 2023-08-28T02:23:57.666 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.666 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.666 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:57.666 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.496244+0000", 2023-08-28T02:23:57.666 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00014597699999999999 2023-08-28T02:23:57.666 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.666 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.666 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:57.666 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.496390+0000", 2023-08-28T02:23:57.667 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000160452 2023-08-28T02:23:57.667 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.667 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.667 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:23:57.667 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.496550+0000", 2023-08-28T02:23:57.667 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0047571219999999999 2023-08-28T02:23:57.667 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.667 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.667 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:57.667 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.501307+0000", 2023-08-28T02:23:57.667 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.9358999999999998e-05 2023-08-28T02:23:57.667 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.667 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.668 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:57.668 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.501377+0000", 2023-08-28T02:23:57.668 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.15030122600000001 2023-08-28T02:23:57.668 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.668 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.668 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:57.668 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.651678+0000", 2023-08-28T02:23:57.668 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.5188e-05 2023-08-28T02:23:57.668 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.668 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.668 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:57.668 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.651743+0000", 2023-08-28T02:23:57.668 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.9789000000000001e-05 2023-08-28T02:23:57.669 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.669 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.669 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:57.669 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.651793+0000", 2023-08-28T02:23:57.669 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.15569572800000001 2023-08-28T02:23:57.669 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:57.669 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:57.669 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:57.669 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.669 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.669 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:72084 5.2 5:486536da:::benchmark_data_smithi143_236200_object72083:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e132)", 2023-08-28T02:23:57.669 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:22:51.498843+0000", 2023-08-28T02:23:57.669 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 66.153393453000007, 2023-08-28T02:23:57.670 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.156005543, 2023-08-28T02:23:57.670 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:57.670 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:57.670 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:57.670 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:57.670 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:57.670 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 72084 2023-08-28T02:23:57.670 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.670 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:57.670 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.670 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:57.670 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.498843+0000", 2023-08-28T02:23:57.670 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:57.671 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.671 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.671 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:57.671 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.498843+0000", 2023-08-28T02:23:57.671 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:23:57.671 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.671 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.671 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:57.671 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.498842+0000", 2023-08-28T02:23:57.671 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.0946999999999999e-05 2023-08-28T02:23:57.671 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.671 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.671 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:57.672 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.498883+0000", 2023-08-28T02:23:57.672 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2283000000000001e-05 2023-08-28T02:23:57.672 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.672 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.672 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:57.672 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.498895+0000", 2023-08-28T02:23:57.672 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.438e-06 2023-08-28T02:23:57.672 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.672 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.672 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:57.672 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.498901+0000", 2023-08-28T02:23:57.672 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.8452999999999998e-05 2023-08-28T02:23:57.672 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.673 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.673 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:57.673 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.498950+0000", 2023-08-28T02:23:57.673 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000116932 2023-08-28T02:23:57.673 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.673 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.673 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:57.673 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.499067+0000", 2023-08-28T02:23:57.674 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00017901200000000001 2023-08-28T02:23:57.674 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.674 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.674 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:23:57.674 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.499246+0000", 2023-08-28T02:23:57.674 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0046196550000000003 2023-08-28T02:23:57.675 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.675 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.675 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:57.675 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.503865+0000", 2023-08-28T02:23:57.675 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0053357539999999998 2023-08-28T02:23:57.675 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.675 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.675 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:57.675 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.509201+0000", 2023-08-28T02:23:57.675 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.14553160400000001 2023-08-28T02:23:57.675 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.675 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.676 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:57.676 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.654733+0000", 2023-08-28T02:23:57.676 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.4144e-05 2023-08-28T02:23:57.676 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.676 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.676 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:57.676 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.654807+0000", 2023-08-28T02:23:57.676 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.1999999999999998e-05 2023-08-28T02:23:57.676 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.676 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.676 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:57.676 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.654849+0000", 2023-08-28T02:23:57.676 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.156005543 2023-08-28T02:23:57.677 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:57.677 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:57.677 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:57.677 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.677 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.677 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:72089 5.2 5:4988838b:::benchmark_data_smithi143_236200_object72088:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e132)", 2023-08-28T02:23:57.677 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:22:51.500287+0000", 2023-08-28T02:23:57.677 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 66.151949670999997, 2023-08-28T02:23:57.677 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.15686362400000001, 2023-08-28T02:23:57.677 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:57.677 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:57.677 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:57.678 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:57.678 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:57.678 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 72089 2023-08-28T02:23:57.678 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.678 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:57.678 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.678 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:57.678 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.500287+0000", 2023-08-28T02:23:57.678 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:57.678 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.678 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.678 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:57.678 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.500287+0000", 2023-08-28T02:23:57.679 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:23:57.679 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.679 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.679 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:57.679 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.500286+0000", 2023-08-28T02:23:57.679 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.0198000000000001e-05 2023-08-28T02:23:57.679 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.679 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.679 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:57.679 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.500316+0000", 2023-08-28T02:23:57.679 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.4369999999999996e-06 2023-08-28T02:23:57.679 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.679 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.680 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:57.680 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.500326+0000", 2023-08-28T02:23:57.680 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.4260000000000002e-06 2023-08-28T02:23:57.680 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.680 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.680 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:57.680 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.500331+0000", 2023-08-28T02:23:57.680 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.001075793 2023-08-28T02:23:57.680 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.680 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.680 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:57.680 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.501407+0000", 2023-08-28T02:23:57.680 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00013019299999999999 2023-08-28T02:23:57.681 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.681 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.681 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:57.681 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.501537+0000", 2023-08-28T02:23:57.681 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00015022700000000001 2023-08-28T02:23:57.681 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.681 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.681 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:23:57.681 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.501687+0000", 2023-08-28T02:23:57.681 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0025966650000000002 2023-08-28T02:23:57.681 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.681 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.681 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:57.682 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.504284+0000", 2023-08-28T02:23:57.682 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0050118309999999996 2023-08-28T02:23:57.682 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.682 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.682 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:57.682 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.509296+0000", 2023-08-28T02:23:57.682 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.14776678200000001 2023-08-28T02:23:57.682 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.682 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.682 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:57.682 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.657063+0000", 2023-08-28T02:23:57.682 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.8384000000000003e-05 2023-08-28T02:23:57.682 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.683 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.683 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:57.683 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.657111+0000", 2023-08-28T02:23:57.683 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.9937000000000001e-05 2023-08-28T02:23:57.683 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.683 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.683 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:57.683 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:51.657151+0000", 2023-08-28T02:23:57.683 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.15686362400000001 2023-08-28T02:23:57.683 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:57.683 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:57.683 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:57.683 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.684 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.684 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:73611 5.2 5:44947317:::benchmark_data_smithi143_236200_object73610:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e133)", 2023-08-28T02:23:57.684 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:22:53.034066+0000", 2023-08-28T02:23:57.684 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 64.618170508000006, 2023-08-28T02:23:57.684 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.12958734399999999, 2023-08-28T02:23:57.684 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:57.684 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:57.684 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:57.684 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:57.684 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:57.684 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 73611 2023-08-28T02:23:57.684 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.684 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:57.685 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.685 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:57.685 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:53.034066+0000", 2023-08-28T02:23:57.685 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:57.685 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.685 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.685 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:57.685 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:53.034066+0000", 2023-08-28T02:23:57.685 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:23:57.685 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.685 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.685 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:57.685 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:53.034064+0000", 2023-08-28T02:23:57.686 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00053028500000000002 2023-08-28T02:23:57.686 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.686 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.686 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:57.686 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:53.034595+0000", 2023-08-28T02:23:57.686 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.1980000000000004e-06 2023-08-28T02:23:57.686 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.686 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.686 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:57.686 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:53.034602+0000", 2023-08-28T02:23:57.686 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.4580000000000002e-06 2023-08-28T02:23:57.687 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.687 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.687 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:57.687 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:53.034605+0000", 2023-08-28T02:23:57.687 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00025787599999999998 2023-08-28T02:23:57.687 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.687 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.687 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:57.687 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:53.034863+0000", 2023-08-28T02:23:57.687 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000106669 2023-08-28T02:23:57.687 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.687 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.687 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:57.688 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:53.034970+0000", 2023-08-28T02:23:57.688 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000180006 2023-08-28T02:23:57.688 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.688 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.688 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:23:57.688 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:53.035150+0000", 2023-08-28T02:23:57.688 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0023956419999999999 2023-08-28T02:23:57.688 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.688 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.688 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:57.688 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:53.037545+0000", 2023-08-28T02:23:57.688 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00018312600000000001 2023-08-28T02:23:57.689 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.689 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.689 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:57.689 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:53.037729+0000", 2023-08-28T02:23:57.689 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.12582232800000001 2023-08-28T02:23:57.689 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.689 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.689 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:57.689 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:53.163551+0000", 2023-08-28T02:23:57.689 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.7358999999999997e-05 2023-08-28T02:23:57.689 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.689 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.689 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:57.689 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:53.163608+0000", 2023-08-28T02:23:57.690 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.5429999999999997e-05 2023-08-28T02:23:57.690 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.690 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.690 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:57.690 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:22:53.163654+0000", 2023-08-28T02:23:57.690 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.12958734399999999 2023-08-28T02:23:57.690 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:57.690 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:57.690 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:57.690 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.690 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.690 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:104320 5.9 5:9f16f9fc:::benchmark_data_smithi143_236200_object104319:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e140)", 2023-08-28T02:23:57.690 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:27.467854+0000", 2023-08-28T02:23:57.691 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 30.184383222000001, 2023-08-28T02:23:57.691 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.15896006300000001, 2023-08-28T02:23:57.691 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:57.691 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:57.691 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:57.691 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:57.691 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:57.691 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 104320 2023-08-28T02:23:57.691 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.691 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:57.691 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.691 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:57.691 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.467854+0000", 2023-08-28T02:23:57.692 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:57.692 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.692 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.692 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:57.692 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.467854+0000", 2023-08-28T02:23:57.692 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999971 2023-08-28T02:23:57.692 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.692 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.692 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:57.692 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.467851+0000", 2023-08-28T02:23:57.692 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.1936e-05 2023-08-28T02:23:57.692 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.692 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.693 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:57.693 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.467893+0000", 2023-08-28T02:23:57.693 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.7713e-05 2023-08-28T02:23:57.693 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.693 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.693 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:57.693 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.467920+0000", 2023-08-28T02:23:57.693 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1299e-05 2023-08-28T02:23:57.693 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.693 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.693 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:57.693 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.467932+0000", 2023-08-28T02:23:57.694 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.3537999999999999e-05 2023-08-28T02:23:57.694 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.694 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.694 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:57.694 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.468015+0000", 2023-08-28T02:23:57.694 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.9253999999999999e-05 2023-08-28T02:23:57.694 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.694 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.694 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:23:57.694 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.468034+0000", 2023-08-28T02:23:57.694 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.129889633 2023-08-28T02:23:57.694 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.694 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.695 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:57.695 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.597924+0000", 2023-08-28T02:23:57.695 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00018115300000000001 2023-08-28T02:23:57.695 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.695 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.695 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:57.695 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.598105+0000", 2023-08-28T02:23:57.695 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00023070499999999999 2023-08-28T02:23:57.695 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.695 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.695 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 1,4", 2023-08-28T02:23:57.695 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.598336+0000", 2023-08-28T02:23:57.695 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0083730100000000002 2023-08-28T02:23:57.696 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.696 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.696 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:57.696 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.606709+0000", 2023-08-28T02:23:57.696 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0052377090000000001 2023-08-28T02:23:57.696 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.696 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.696 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:57.696 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.611947+0000", 2023-08-28T02:23:57.696 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.012252231000000001 2023-08-28T02:23:57.696 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.696 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.697 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:57.697 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.624199+0000", 2023-08-28T02:23:57.697 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.9170000000000004e-05 2023-08-28T02:23:57.697 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.697 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.697 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:57.697 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.624268+0000", 2023-08-28T02:23:57.697 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0025457890000000001 2023-08-28T02:23:57.697 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.697 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.697 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:57.697 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.626814+0000", 2023-08-28T02:23:57.697 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.15896006300000001 2023-08-28T02:23:57.697 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:57.697 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:57.698 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:57.698 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.698 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.698 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:104323 5.9 5:9da8df7d:::benchmark_data_smithi143_236200_object104322:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e140)", 2023-08-28T02:23:57.698 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:27.504776+0000", 2023-08-28T02:23:57.698 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 30.147461139000001, 2023-08-28T02:23:57.698 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.124902922, 2023-08-28T02:23:57.698 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:57.698 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:57.698 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:57.698 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:57.698 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:57.699 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 104323 2023-08-28T02:23:57.699 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.699 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:57.699 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.699 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:57.699 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.504776+0000", 2023-08-28T02:23:57.699 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:57.699 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.699 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.699 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:57.699 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.504776+0000", 2023-08-28T02:23:57.699 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:23:57.699 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.700 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.700 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:57.700 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.504774+0000", 2023-08-28T02:23:57.700 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.8632e-05 2023-08-28T02:23:57.700 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.700 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.700 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:57.700 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.504803+0000", 2023-08-28T02:23:57.700 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.8464000000000001e-05 2023-08-28T02:23:57.700 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.700 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.700 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:57.700 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.504821+0000", 2023-08-28T02:23:57.701 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.5610000000000004e-06 2023-08-28T02:23:57.701 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.701 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.701 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:57.701 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.504828+0000", 2023-08-28T02:23:57.701 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.3081000000000001e-05 2023-08-28T02:23:57.701 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.701 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.701 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:57.701 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.504891+0000", 2023-08-28T02:23:57.701 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0696000000000001e-05 2023-08-28T02:23:57.701 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.701 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.701 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:23:57.702 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.504901+0000", 2023-08-28T02:23:57.702 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.096638496000000004 2023-08-28T02:23:57.702 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.702 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.702 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:57.702 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.601540+0000", 2023-08-28T02:23:57.702 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000121741 2023-08-28T02:23:57.702 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.702 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.702 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:57.702 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.601662+0000", 2023-08-28T02:23:57.702 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00014504399999999999 2023-08-28T02:23:57.703 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.703 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.703 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 1,4", 2023-08-28T02:23:57.703 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.601807+0000", 2023-08-28T02:23:57.703 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0049175540000000002 2023-08-28T02:23:57.703 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.703 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.703 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:57.703 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.606724+0000", 2023-08-28T02:23:57.703 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0052534900000000004 2023-08-28T02:23:57.703 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.703 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.704 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:57.704 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.611978+0000", 2023-08-28T02:23:57.704 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.014980464000000001 2023-08-28T02:23:57.704 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.704 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.704 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:57.704 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.626958+0000", 2023-08-28T02:23:57.704 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.0647999999999999e-05 2023-08-28T02:23:57.704 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.704 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.704 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:57.704 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.627019+0000", 2023-08-28T02:23:57.704 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.002659804 2023-08-28T02:23:57.704 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.705 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.705 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:57.705 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:27.629679+0000", 2023-08-28T02:23:57.705 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.124902922 2023-08-28T02:23:57.705 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:57.705 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:57.705 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:57.705 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.705 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.705 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:106262 5.2 5:4a60dcac:::benchmark_data_smithi143_236200_object106261:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e142)", 2023-08-28T02:23:57.705 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:30.539419+0000", 2023-08-28T02:23:57.705 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 27.112818145999999, 2023-08-28T02:23:57.705 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.12576027400000001, 2023-08-28T02:23:57.705 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:57.706 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:57.706 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:57.706 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:57.706 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:57.706 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 106262 2023-08-28T02:23:57.706 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.706 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:57.706 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.706 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:57.706 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:30.539419+0000", 2023-08-28T02:23:57.706 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:57.706 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.707 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.707 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:57.707 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:30.539419+0000", 2023-08-28T02:23:57.707 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:23:57.707 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.707 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.707 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:57.707 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:30.539417+0000", 2023-08-28T02:23:57.707 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.5351000000000001e-05 2023-08-28T02:23:57.707 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.707 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.707 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:57.708 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:30.539453+0000", 2023-08-28T02:23:57.708 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.3774000000000001e-05 2023-08-28T02:23:57.708 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.708 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.708 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:57.708 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:30.539466+0000", 2023-08-28T02:23:57.708 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.5529999999999998e-06 2023-08-28T02:23:57.708 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.708 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.708 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:57.708 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:30.539473+0000", 2023-08-28T02:23:57.708 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.3727999999999998e-05 2023-08-28T02:23:57.708 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.709 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.709 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:57.709 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:30.539527+0000", 2023-08-28T02:23:57.709 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00011975 2023-08-28T02:23:57.709 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.709 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.709 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:57.709 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:30.539646+0000", 2023-08-28T02:23:57.709 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.4643999999999997e-05 2023-08-28T02:23:57.709 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.709 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.709 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:23:57.709 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:30.539741+0000", 2023-08-28T02:23:57.709 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0039443509999999996 2023-08-28T02:23:57.710 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.710 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.710 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:57.710 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:30.543685+0000", 2023-08-28T02:23:57.710 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.013809833000000001 2023-08-28T02:23:57.710 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.710 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.710 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:57.710 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:30.557495+0000", 2023-08-28T02:23:57.710 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.10757058899999999 2023-08-28T02:23:57.710 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.710 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.710 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:57.711 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:30.665066+0000", 2023-08-28T02:23:57.711 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.6056999999999995e-05 2023-08-28T02:23:57.711 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.711 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.711 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:57.711 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:30.665132+0000", 2023-08-28T02:23:57.711 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.7219000000000003e-05 2023-08-28T02:23:57.711 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.711 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.711 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:57.711 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:30.665179+0000", 2023-08-28T02:23:57.711 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.12576027400000001 2023-08-28T02:23:57.712 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:57.712 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:57.712 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:57.712 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.712 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.712 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:107236 5.9 5:9b8f9284:::benchmark_data_smithi143_236200_object107235:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e142)", 2023-08-28T02:23:57.712 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:32.224596+0000", 2023-08-28T02:23:57.712 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 25.42764133, 2023-08-28T02:23:57.712 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.121841754, 2023-08-28T02:23:57.712 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:57.712 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:57.712 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:57.712 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:57.713 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:57.713 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 107236 2023-08-28T02:23:57.713 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.713 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:57.713 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.713 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:57.713 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:32.224596+0000", 2023-08-28T02:23:57.713 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:57.713 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.713 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.713 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:57.713 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:32.224596+0000", 2023-08-28T02:23:57.713 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:23:57.714 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.714 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.714 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:57.714 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:32.224594+0000", 2023-08-28T02:23:57.714 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00011548399999999999 2023-08-28T02:23:57.714 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.714 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.714 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:57.714 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:32.224709+0000", 2023-08-28T02:23:57.714 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1640999999999999e-05 2023-08-28T02:23:57.714 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.714 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.714 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:57.714 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:32.224721+0000", 2023-08-28T02:23:57.715 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.1430000000000003e-06 2023-08-28T02:23:57.715 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.715 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.715 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:57.715 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:32.224727+0000", 2023-08-28T02:23:57.715 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0081868789999999993 2023-08-28T02:23:57.715 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.715 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.715 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:57.715 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:32.232914+0000", 2023-08-28T02:23:57.715 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00017217299999999999 2023-08-28T02:23:57.715 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.716 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.716 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:57.716 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:32.233086+0000", 2023-08-28T02:23:57.716 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000212034 2023-08-28T02:23:57.716 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.716 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.716 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 1,4", 2023-08-28T02:23:57.716 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:32.233298+0000", 2023-08-28T02:23:57.716 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0035795000000000002 2023-08-28T02:23:57.716 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.716 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.716 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:57.717 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:32.236877+0000", 2023-08-28T02:23:57.717 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.090998493 2023-08-28T02:23:57.717 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.717 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.717 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:57.717 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:32.327876+0000", 2023-08-28T02:23:57.717 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.015870453999999999 2023-08-28T02:23:57.717 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.717 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.717 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:57.717 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:32.343746+0000", 2023-08-28T02:23:57.717 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.6372999999999996e-05 2023-08-28T02:23:57.717 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.718 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.718 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:57.718 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:32.343813+0000", 2023-08-28T02:23:57.718 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0026245299999999999 2023-08-28T02:23:57.718 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.718 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.718 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:57.718 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:32.346437+0000", 2023-08-28T02:23:57.718 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.121841754 2023-08-28T02:23:57.718 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:57.718 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:57.718 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:57.719 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.719 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.719 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:107250 5.9 5:97bc4668:::benchmark_data_smithi143_236200_object107249:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e142)", 2023-08-28T02:23:57.719 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:32.242738+0000", 2023-08-28T02:23:57.719 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 25.409498986999999, 2023-08-28T02:23:57.719 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.119855715, 2023-08-28T02:23:57.719 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:57.719 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:57.719 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:57.719 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:57.719 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:57.719 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 107250 2023-08-28T02:23:57.720 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.720 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:57.720 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.720 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:57.720 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:32.242738+0000", 2023-08-28T02:23:57.720 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:57.720 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.720 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.720 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:57.720 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:32.242738+0000", 2023-08-28T02:23:57.720 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999876 2023-08-28T02:23:57.720 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.720 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.720 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:57.721 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:32.242726+0000", 2023-08-28T02:23:57.721 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.3344000000000003e-05 2023-08-28T02:23:57.721 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.721 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.721 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:57.721 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:32.242779+0000", 2023-08-28T02:23:57.721 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.5065000000000001e-05 2023-08-28T02:23:57.721 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.721 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.721 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:57.721 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:32.242794+0000", 2023-08-28T02:23:57.721 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.2380000000000002e-06 2023-08-28T02:23:57.721 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.721 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.722 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:57.722 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:32.242800+0000", 2023-08-28T02:23:57.722 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.3639000000000005e-05 2023-08-28T02:23:57.722 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.722 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.722 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:57.722 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:32.242864+0000", 2023-08-28T02:23:57.722 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00014431399999999999 2023-08-28T02:23:57.722 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.722 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.722 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:57.722 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:32.243008+0000", 2023-08-28T02:23:57.722 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000206331 2023-08-28T02:23:57.723 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.723 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.723 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 1,4", 2023-08-28T02:23:57.723 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:32.243215+0000", 2023-08-28T02:23:57.723 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.084498259000000006 2023-08-28T02:23:57.723 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.723 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.723 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:57.723 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:32.327713+0000", 2023-08-28T02:23:57.723 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00022075600000000001 2023-08-28T02:23:57.723 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.723 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.723 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:57.724 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:32.327934+0000", 2023-08-28T02:23:57.724 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.032109026999999998 2023-08-28T02:23:57.724 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.724 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.724 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:57.724 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:32.360043+0000", 2023-08-28T02:23:57.724 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.4938000000000002e-05 2023-08-28T02:23:57.724 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.724 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.724 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:57.724 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:32.360088+0000", 2023-08-28T02:23:57.724 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0025060460000000001 2023-08-28T02:23:57.724 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.725 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.725 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:57.725 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:32.362594+0000", 2023-08-28T02:23:57.725 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.119855715 2023-08-28T02:23:57.725 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:57.725 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:57.725 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:57.725 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.725 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.725 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:107251 5.9 5:983a6dc9:::benchmark_data_smithi143_236200_object107250:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e142)", 2023-08-28T02:23:57.725 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:32.245472+0000", 2023-08-28T02:23:57.725 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 25.406764513999999, 2023-08-28T02:23:57.725 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.11954311200000001, 2023-08-28T02:23:57.725 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:57.726 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:57.726 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:57.726 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:57.726 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:57.726 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 107251 2023-08-28T02:23:57.726 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.726 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:57.726 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.726 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:57.726 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:32.245472+0000", 2023-08-28T02:23:57.726 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:57.726 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.726 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.726 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:57.727 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:32.245472+0000", 2023-08-28T02:23:57.727 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:23:57.727 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.727 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.727 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:57.727 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:32.245471+0000", 2023-08-28T02:23:57.727 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.1666000000000003e-05 2023-08-28T02:23:57.727 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.727 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.727 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:57.727 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:32.245523+0000", 2023-08-28T02:23:57.727 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2689e-05 2023-08-28T02:23:57.727 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.728 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.728 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:57.728 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:32.245535+0000", 2023-08-28T02:23:57.728 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.9530000000000002e-06 2023-08-28T02:23:57.728 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.728 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.728 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:57.728 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:32.245542+0000", 2023-08-28T02:23:57.728 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00067022000000000002 2023-08-28T02:23:57.728 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.728 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.728 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:57.728 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:32.246213+0000", 2023-08-28T02:23:57.728 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000148904 2023-08-28T02:23:57.729 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.729 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.729 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:57.729 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:32.246362+0000", 2023-08-28T02:23:57.729 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00018139299999999999 2023-08-28T02:23:57.729 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.729 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.729 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 1,4", 2023-08-28T02:23:57.729 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:32.246543+0000", 2023-08-28T02:23:57.729 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.081177062999999994 2023-08-28T02:23:57.729 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.729 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.729 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:57.730 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:32.327720+0000", 2023-08-28T02:23:57.730 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00027014399999999999 2023-08-28T02:23:57.730 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.730 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.730 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:57.730 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:32.327990+0000", 2023-08-28T02:23:57.730 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.034678759000000003 2023-08-28T02:23:57.730 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.730 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.730 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:57.730 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:32.362669+0000", 2023-08-28T02:23:57.730 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.2450000000000002e-05 2023-08-28T02:23:57.730 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.730 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.731 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:57.731 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:32.362711+0000", 2023-08-28T02:23:57.731 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.002304188 2023-08-28T02:23:57.731 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.731 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.731 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:57.731 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:32.365015+0000", 2023-08-28T02:23:57.731 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.11954311200000001 2023-08-28T02:23:57.731 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:57.731 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:57.731 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:57.731 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.731 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.732 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:113148 5.2 5:48f1e572:::benchmark_data_smithi143_236200_object3005:head [delete] snapc 0=[] ondisk+write+known_if_redirected e147)", 2023-08-28T02:23:57.732 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:42.298521+0000", 2023-08-28T02:23:57.732 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 15.353716392999999, 2023-08-28T02:23:57.732 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.162277536, 2023-08-28T02:23:57.732 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:57.732 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:57.732 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:57.732 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:57.732 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:57.732 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 113148 2023-08-28T02:23:57.732 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.732 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:57.732 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.733 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:57.733 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:42.298521+0000", 2023-08-28T02:23:57.733 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:57.733 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.733 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.733 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:57.733 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:42.298521+0000", 2023-08-28T02:23:57.733 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:23:57.733 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.733 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.733 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:57.733 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:42.298519+0000", 2023-08-28T02:23:57.733 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.75e-06 2023-08-28T02:23:57.734 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.734 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.734 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:57.734 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:42.298525+0000", 2023-08-28T02:23:57.734 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2043e-05 2023-08-28T02:23:57.734 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.734 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.734 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:57.734 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:42.298537+0000", 2023-08-28T02:23:57.734 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.6060000000000002e-06 2023-08-28T02:23:57.734 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.734 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.735 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:57.735 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:42.298543+0000", 2023-08-28T02:23:57.735 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.8819e-05 2023-08-28T02:23:57.735 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.735 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.735 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:57.735 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:42.298602+0000", 2023-08-28T02:23:57.735 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.8726000000000004e-05 2023-08-28T02:23:57.735 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.735 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.735 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:57.735 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:42.298690+0000", 2023-08-28T02:23:57.736 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.4852999999999995e-05 2023-08-28T02:23:57.736 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.736 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.736 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:23:57.736 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:42.298785+0000", 2023-08-28T02:23:57.736 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.012472555999999999 2023-08-28T02:23:57.736 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.736 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.736 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:57.736 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:42.311258+0000", 2023-08-28T02:23:57.736 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00081621299999999999 2023-08-28T02:23:57.736 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.737 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.737 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:57.737 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:42.312074+0000", 2023-08-28T02:23:57.737 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.148624114 2023-08-28T02:23:57.737 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.737 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.737 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:57.737 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:42.460698+0000", 2023-08-28T02:23:57.737 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.3180999999999998e-05 2023-08-28T02:23:57.737 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.737 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.737 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:57.737 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:42.460751+0000", 2023-08-28T02:23:57.738 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.6866000000000002e-05 2023-08-28T02:23:57.738 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.738 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.738 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:57.738 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:42.460798+0000", 2023-08-28T02:23:57.738 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.162277536 2023-08-28T02:23:57.738 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:57.738 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:57.738 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:57.738 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.738 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.738 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:113160 5.2 5:49dc7a23:::benchmark_data_smithi143_236200_object3017:head [delete] snapc 0=[] ondisk+write+known_if_redirected e147)", 2023-08-28T02:23:57.739 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:42.306709+0000", 2023-08-28T02:23:57.739 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 15.345528380999999, 2023-08-28T02:23:57.739 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.15419380499999999, 2023-08-28T02:23:57.739 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:57.739 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:57.739 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:57.739 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:57.739 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:57.739 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 113160 2023-08-28T02:23:57.739 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.739 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:57.739 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.740 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:57.740 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:42.306709+0000", 2023-08-28T02:23:57.740 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:57.740 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.740 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.740 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:57.740 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:42.306709+0000", 2023-08-28T02:23:57.740 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:23:57.740 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.740 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.740 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:57.740 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:42.306707+0000", 2023-08-28T02:23:57.740 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.8860000000000004e-06 2023-08-28T02:23:57.741 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.741 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.741 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:57.741 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:42.306714+0000", 2023-08-28T02:23:57.741 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.3835000000000001e-05 2023-08-28T02:23:57.741 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.741 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.741 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:57.741 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:42.306728+0000", 2023-08-28T02:23:57.741 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.6420000000000001e-06 2023-08-28T02:23:57.741 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.741 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.742 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:57.742 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:42.306735+0000", 2023-08-28T02:23:57.742 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.001492312 2023-08-28T02:23:57.742 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.742 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.742 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:57.742 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:42.308227+0000", 2023-08-28T02:23:57.742 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00014855599999999999 2023-08-28T02:23:57.742 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.742 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.742 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:57.742 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:42.308376+0000", 2023-08-28T02:23:57.743 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000149452 2023-08-28T02:23:57.743 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.743 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.743 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:23:57.743 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:42.308525+0000", 2023-08-28T02:23:57.743 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0028284959999999998 2023-08-28T02:23:57.743 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.746 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.746 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:57.746 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:42.311354+0000", 2023-08-28T02:23:57.746 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0039947330000000003 2023-08-28T02:23:57.746 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.746 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.746 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:57.746 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:42.315348+0000", 2023-08-28T02:23:57.746 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.14548657800000001 2023-08-28T02:23:57.746 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.746 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.747 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:57.747 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:42.460835+0000", 2023-08-28T02:23:57.747 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.5172999999999997e-05 2023-08-28T02:23:57.747 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.747 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.747 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:57.747 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:42.460870+0000", 2023-08-28T02:23:57.747 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.2152e-05 2023-08-28T02:23:57.747 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.747 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.747 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:57.747 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:42.460902+0000", 2023-08-28T02:23:57.748 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.15419380499999999 2023-08-28T02:23:57.748 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:57.748 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:57.748 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:57.748 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.748 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.748 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:114138 5.9 5:9eb8aa5f:::benchmark_data_smithi143_236200_object3995:head [delete] snapc 0=[] ondisk+write+known_if_redirected e147)", 2023-08-28T02:23:57.748 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:43.612310+0000", 2023-08-28T02:23:57.748 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 14.039926525, 2023-08-28T02:23:57.748 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.146063201, 2023-08-28T02:23:57.748 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:57.748 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:57.749 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:57.749 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:57.749 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:57.749 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 114138 2023-08-28T02:23:57.749 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.749 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:57.749 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.749 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:57.749 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.612310+0000", 2023-08-28T02:23:57.749 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:57.749 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.750 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.750 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:57.750 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.612310+0000", 2023-08-28T02:23:57.750 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:23:57.750 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.750 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.750 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:57.750 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.612309+0000", 2023-08-28T02:23:57.750 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.8139999999999997e-06 2023-08-28T02:23:57.750 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.750 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.750 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:57.751 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.612315+0000", 2023-08-28T02:23:57.751 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.4009999999999997e-06 2023-08-28T02:23:57.751 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.751 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.751 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:57.751 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.612323+0000", 2023-08-28T02:23:57.751 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.5720000000000002e-06 2023-08-28T02:23:57.751 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.751 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.751 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:57.751 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.612329+0000", 2023-08-28T02:23:57.751 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.9317999999999998e-05 2023-08-28T02:23:57.752 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.752 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.752 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:57.752 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.612378+0000", 2023-08-28T02:23:57.752 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.6669999999999994e-05 2023-08-28T02:23:57.752 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.752 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.752 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:57.752 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.612475+0000", 2023-08-28T02:23:57.752 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000109858 2023-08-28T02:23:57.752 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.752 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.753 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 4,7", 2023-08-28T02:23:57.753 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.612585+0000", 2023-08-28T02:23:57.753 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0072044759999999996 2023-08-28T02:23:57.753 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.753 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.753 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:57.753 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.619789+0000", 2023-08-28T02:23:57.753 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0042165859999999996 2023-08-28T02:23:57.753 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.753 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.753 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:57.753 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.624006+0000", 2023-08-28T02:23:57.753 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.133967007 2023-08-28T02:23:57.754 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.754 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.754 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:57.754 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.757973+0000", 2023-08-28T02:23:57.754 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.6412e-05 2023-08-28T02:23:57.754 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.754 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.754 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:57.754 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.758039+0000", 2023-08-28T02:23:57.754 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00033443600000000002 2023-08-28T02:23:57.754 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.754 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.754 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:57.755 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.758374+0000", 2023-08-28T02:23:57.755 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.146063201 2023-08-28T02:23:57.755 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:57.755 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:57.755 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:57.755 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.755 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.755 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:114139 5.9 5:9e268159:::benchmark_data_smithi143_236200_object3996:head [delete] snapc 0=[] ondisk+write+known_if_redirected e147)", 2023-08-28T02:23:57.755 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:43.612346+0000", 2023-08-28T02:23:57.755 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 14.039890917999999, 2023-08-28T02:23:57.755 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.14850207800000001, 2023-08-28T02:23:57.755 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:57.756 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:57.756 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:57.756 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:57.756 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:57.756 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 114139 2023-08-28T02:23:57.756 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.756 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:57.756 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.756 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:57.756 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.612346+0000", 2023-08-28T02:23:57.756 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:57.756 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.757 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.757 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:57.757 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.612346+0000", 2023-08-28T02:23:57.757 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999995 2023-08-28T02:23:57.757 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.757 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.757 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:57.757 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.612345+0000", 2023-08-28T02:23:57.757 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.2669999999999998e-06 2023-08-28T02:23:57.757 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.757 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.757 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:57.757 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.612349+0000", 2023-08-28T02:23:57.758 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.9210000000000003e-06 2023-08-28T02:23:57.758 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.758 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.758 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:57.758 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.612356+0000", 2023-08-28T02:23:57.758 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.3579999999999996e-06 2023-08-28T02:23:57.758 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.758 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.758 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:57.758 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.612360+0000", 2023-08-28T02:23:57.758 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00367564 2023-08-28T02:23:57.758 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.759 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.759 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:57.759 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.616036+0000", 2023-08-28T02:23:57.759 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00014236399999999999 2023-08-28T02:23:57.759 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.759 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.759 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:57.759 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.616178+0000", 2023-08-28T02:23:57.759 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000147508 2023-08-28T02:23:57.759 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.759 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.760 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 4,7", 2023-08-28T02:23:57.760 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.616326+0000", 2023-08-28T02:23:57.760 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0037459139999999999 2023-08-28T02:23:57.760 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.760 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.760 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:57.760 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.620071+0000", 2023-08-28T02:23:57.760 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0039837340000000001 2023-08-28T02:23:57.760 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.760 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.760 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:57.760 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.624055+0000", 2023-08-28T02:23:57.760 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.13642877 2023-08-28T02:23:57.761 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.761 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.761 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:57.761 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.760484+0000", 2023-08-28T02:23:57.761 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.6180000000000001e-05 2023-08-28T02:23:57.761 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.761 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.761 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:57.761 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.760540+0000", 2023-08-28T02:23:57.761 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00030796099999999999 2023-08-28T02:23:57.761 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.761 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.762 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:57.762 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.760848+0000", 2023-08-28T02:23:57.762 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.14850207800000001 2023-08-28T02:23:57.762 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:57.762 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:57.762 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:57.762 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.762 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.762 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:114614 5.9 5:91775d66:::benchmark_data_smithi143_236200_object4471:head [delete] snapc 0=[] ondisk+write+known_if_redirected e149)", 2023-08-28T02:23:57.762 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:45.197262+0000", 2023-08-28T02:23:57.762 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 12.454974649, 2023-08-28T02:23:57.762 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.145561732, 2023-08-28T02:23:57.762 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:57.763 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:57.763 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:57.763 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:57.763 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:57.763 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 114614 2023-08-28T02:23:57.763 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.763 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:57.763 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.763 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:57.763 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.197262+0000", 2023-08-28T02:23:57.763 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:57.763 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.764 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.764 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:57.764 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.197262+0000", 2023-08-28T02:23:57.764 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:23:57.764 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.764 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.764 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:57.764 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.197261+0000", 2023-08-28T02:23:57.764 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.8919999999999999e-06 2023-08-28T02:23:57.764 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.764 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.764 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:57.765 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.197266+0000", 2023-08-28T02:23:57.765 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.3299999999999999e-06 2023-08-28T02:23:57.765 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.765 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.765 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:57.765 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.197274+0000", 2023-08-28T02:23:57.765 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.3680000000000001e-06 2023-08-28T02:23:57.765 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.765 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.765 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:57.765 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.197279+0000", 2023-08-28T02:23:57.765 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0020190009999999999 2023-08-28T02:23:57.765 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.765 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.766 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:57.766 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.199298+0000", 2023-08-28T02:23:57.766 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00010425399999999999 2023-08-28T02:23:57.766 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.766 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.766 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:57.766 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.199403+0000", 2023-08-28T02:23:57.766 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000103341 2023-08-28T02:23:57.766 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.766 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.766 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 4,7", 2023-08-28T02:23:57.766 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.199506+0000", 2023-08-28T02:23:57.766 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0043564190000000003 2023-08-28T02:23:57.767 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.767 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.767 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:57.767 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.203862+0000", 2023-08-28T02:23:57.767 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.001277228 2023-08-28T02:23:57.767 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.767 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.767 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:57.767 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.205140+0000", 2023-08-28T02:23:57.767 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.137262144 2023-08-28T02:23:57.767 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.767 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.768 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:57.768 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.342402+0000", 2023-08-28T02:23:57.768 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.8430999999999997e-05 2023-08-28T02:23:57.768 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.768 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.768 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:57.768 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.342460+0000", 2023-08-28T02:23:57.768 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000363703 2023-08-28T02:23:57.768 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.768 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.768 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:57.768 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.342824+0000", 2023-08-28T02:23:57.768 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.145561732 2023-08-28T02:23:57.769 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:57.769 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:57.769 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:57.769 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.769 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.769 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:115417 5.2 5:4644143b:::benchmark_data_smithi143_236200_object5274:head [delete] snapc 0=[] ondisk+write+known_if_redirected e150)", 2023-08-28T02:23:57.769 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:46.300173+0000", 2023-08-28T02:23:57.769 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 11.352063555999999, 2023-08-28T02:23:57.769 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.13349918199999999, 2023-08-28T02:23:57.769 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:57.769 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:57.769 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:57.770 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:57.770 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:57.770 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 115417 2023-08-28T02:23:57.770 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.770 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:57.770 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.770 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:57.770 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.300173+0000", 2023-08-28T02:23:57.770 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:57.770 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.770 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.770 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:57.771 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.300173+0000", 2023-08-28T02:23:57.771 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:23:57.771 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.771 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.771 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:57.771 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.300172+0000", 2023-08-28T02:23:57.771 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.6389999999999992e-06 2023-08-28T02:23:57.771 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.771 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.771 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:57.771 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.300180+0000", 2023-08-28T02:23:57.771 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.0869999999999998e-05 2023-08-28T02:23:57.771 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.772 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.772 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:57.772 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.300200+0000", 2023-08-28T02:23:57.772 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.3860000000000007e-06 2023-08-28T02:23:57.772 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.772 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.772 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:57.772 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.300209+0000", 2023-08-28T02:23:57.772 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.6909e-05 2023-08-28T02:23:57.772 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.772 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.772 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:57.772 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.300266+0000", 2023-08-28T02:23:57.773 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00014285 2023-08-28T02:23:57.773 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.773 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.773 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:57.773 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.300409+0000", 2023-08-28T02:23:57.773 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00016642800000000001 2023-08-28T02:23:57.773 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.773 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.773 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:23:57.773 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.300575+0000", 2023-08-28T02:23:57.773 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0066883180000000004 2023-08-28T02:23:57.773 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.773 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.774 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:57.774 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.307263+0000", 2023-08-28T02:23:57.774 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.6626e-05 2023-08-28T02:23:57.774 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.774 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.774 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:57.774 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.307280+0000", 2023-08-28T02:23:57.774 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.126289395 2023-08-28T02:23:57.774 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.774 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.774 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:57.774 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.433569+0000", 2023-08-28T02:23:57.774 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.7382e-05 2023-08-28T02:23:57.775 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.775 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.775 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:57.775 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.433627+0000", 2023-08-28T02:23:57.775 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.5785999999999998e-05 2023-08-28T02:23:57.775 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.775 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.775 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:57.775 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.433673+0000", 2023-08-28T02:23:57.775 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.13349918199999999 2023-08-28T02:23:57.775 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:57.775 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:57.776 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:57.776 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.776 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.776 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:122090 5.9 5:9a6395dd:::benchmark_data_smithi143_236200_object11947:head [delete] snapc 0=[] ondisk+write+known_if_redirected e152)", 2023-08-28T02:23:57.776 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:56.359008+0000", 2023-08-28T02:23:57.776 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 1.2932287920000001, 2023-08-28T02:23:57.776 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.190802156, 2023-08-28T02:23:57.776 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:57.776 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:57.776 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:57.776 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:57.776 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:57.776 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 122090 2023-08-28T02:23:57.777 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.777 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:57.777 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.777 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:57.777 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.359008+0000", 2023-08-28T02:23:57.777 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:57.777 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.777 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.777 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:57.777 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.359008+0000", 2023-08-28T02:23:57.777 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:23:57.777 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.777 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.778 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:57.778 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.359006+0000", 2023-08-28T02:23:57.778 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.2130000000000008e-06 2023-08-28T02:23:57.778 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.778 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.778 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:57.778 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.359015+0000", 2023-08-28T02:23:57.778 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.3277e-05 2023-08-28T02:23:57.778 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.778 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.778 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:57.778 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.359028+0000", 2023-08-28T02:23:57.778 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.0299999999999996e-06 2023-08-28T02:23:57.779 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.779 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.779 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:57.779 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.359035+0000", 2023-08-28T02:23:57.779 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.6771000000000003e-05 2023-08-28T02:23:57.779 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.779 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.779 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:57.779 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.359082+0000", 2023-08-28T02:23:57.779 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000141734 2023-08-28T02:23:57.779 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.779 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.779 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:57.780 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.359223+0000", 2023-08-28T02:23:57.780 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000157331 2023-08-28T02:23:57.780 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.780 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.780 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 4,7", 2023-08-28T02:23:57.780 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.359381+0000", 2023-08-28T02:23:57.780 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0036132569999999999 2023-08-28T02:23:57.780 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.780 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.780 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:57.780 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.362994+0000", 2023-08-28T02:23:57.780 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.01001399 2023-08-28T02:23:57.781 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.781 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.781 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:57.781 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.373008+0000", 2023-08-28T02:23:57.781 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.17651315000000001 2023-08-28T02:23:57.781 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.781 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.781 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:57.781 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.549521+0000", 2023-08-28T02:23:57.781 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.1438000000000004e-05 2023-08-28T02:23:57.781 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.781 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.781 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:57.782 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.549583+0000", 2023-08-28T02:23:57.782 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00022760799999999999 2023-08-28T02:23:57.782 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.782 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.782 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:57.782 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.549810+0000", 2023-08-28T02:23:57.782 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.190802156 2023-08-28T02:23:57.782 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:57.782 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:57.782 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:57.782 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.782 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.782 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:122093 5.9 5:90145365:::benchmark_data_smithi143_236200_object11950:head [delete] snapc 0=[] ondisk+write+known_if_redirected e152)", 2023-08-28T02:23:57.783 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:56.359053+0000", 2023-08-28T02:23:57.783 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 1.293183682, 2023-08-28T02:23:57.783 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.19099266200000001, 2023-08-28T02:23:57.783 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:57.783 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:57.783 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:57.783 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:57.783 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:57.783 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 122093 2023-08-28T02:23:57.783 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.783 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:57.783 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.783 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:57.784 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.359053+0000", 2023-08-28T02:23:57.784 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:57.784 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.784 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.784 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:57.784 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.359053+0000", 2023-08-28T02:23:57.784 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999995 2023-08-28T02:23:57.784 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.784 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.784 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:57.784 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.359053+0000", 2023-08-28T02:23:57.784 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.146e-06 2023-08-28T02:23:57.784 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.785 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.785 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:57.785 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.359056+0000", 2023-08-28T02:23:57.785 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.5230000000000001e-06 2023-08-28T02:23:57.785 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.785 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.785 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:57.785 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.359062+0000", 2023-08-28T02:23:57.785 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.7600000000000002e-06 2023-08-28T02:23:57.785 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.785 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.785 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:57.785 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.359067+0000", 2023-08-28T02:23:57.786 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0067806539999999997 2023-08-28T02:23:57.786 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.786 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.786 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:57.786 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.365848+0000", 2023-08-28T02:23:57.786 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00013536200000000001 2023-08-28T02:23:57.786 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.786 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.786 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:57.786 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.365983+0000", 2023-08-28T02:23:57.786 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00015116000000000001 2023-08-28T02:23:57.786 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.786 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.787 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 4,7", 2023-08-28T02:23:57.787 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.366134+0000", 2023-08-28T02:23:57.787 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.002991437 2023-08-28T02:23:57.787 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.787 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.787 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:57.787 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.369126+0000", 2023-08-28T02:23:57.787 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0040113329999999997 2023-08-28T02:23:57.787 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.787 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.787 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:57.787 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.373137+0000", 2023-08-28T02:23:57.787 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.17671240499999999 2023-08-28T02:23:57.788 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.788 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.788 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:57.788 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.549850+0000", 2023-08-28T02:23:57.788 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.4039000000000003e-05 2023-08-28T02:23:57.788 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.788 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.788 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:57.788 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.549894+0000", 2023-08-28T02:23:57.788 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000152342 2023-08-28T02:23:57.788 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.788 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.788 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:57.789 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.550046+0000", 2023-08-28T02:23:57.789 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.19099266200000001 2023-08-28T02:23:57.789 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:57.789 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:57.789 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:57.789 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.789 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.789 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:122108 5.9 5:9c0cafde:::benchmark_data_smithi143_236200_object11965:head [delete] snapc 0=[] ondisk+write+known_if_redirected e152)", 2023-08-28T02:23:57.789 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:56.376442+0000", 2023-08-28T02:23:57.789 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 1.2757944110000001, 2023-08-28T02:23:57.789 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.17433146099999999, 2023-08-28T02:23:57.789 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:23:57.789 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:23:57.790 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:23:57.790 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:23:57.790 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:23:57.790 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 122108 2023-08-28T02:23:57.790 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.790 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:23:57.790 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.790 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:23:57.790 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.376442+0000", 2023-08-28T02:23:57.790 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:23:57.790 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.790 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.790 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:23:57.791 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.376442+0000", 2023-08-28T02:23:57.791 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:23:57.791 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.791 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.791 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:23:57.791 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.376441+0000", 2023-08-28T02:23:57.791 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.6969999999999998e-06 2023-08-28T02:23:57.791 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.791 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.791 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:23:57.791 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.376447+0000", 2023-08-28T02:23:57.791 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.4599999999999992e-06 2023-08-28T02:23:57.792 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.792 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.792 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:23:57.792 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.376456+0000", 2023-08-28T02:23:57.792 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.8930000000000004e-06 2023-08-28T02:23:57.792 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.792 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.792 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:23:57.792 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.376461+0000", 2023-08-28T02:23:57.792 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.3560999999999999e-05 2023-08-28T02:23:57.792 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.792 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.792 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:23:57.793 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.376495+0000", 2023-08-28T02:23:57.793 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.3605999999999998e-05 2023-08-28T02:23:57.793 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.793 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.793 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:23:57.793 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.376588+0000", 2023-08-28T02:23:57.793 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00010327600000000001 2023-08-28T02:23:57.793 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.793 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.793 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 4,7", 2023-08-28T02:23:57.793 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.376692+0000", 2023-08-28T02:23:57.793 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0025670110000000001 2023-08-28T02:23:57.793 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.794 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.794 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:23:57.794 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.379259+0000", 2023-08-28T02:23:57.794 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.170830232 2023-08-28T02:23:57.794 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.794 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.794 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:57.794 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.550089+0000", 2023-08-28T02:23:57.794 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000455723 2023-08-28T02:23:57.794 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.794 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.794 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:23:57.794 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.550544+0000", 2023-08-28T02:23:57.795 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.8425999999999999e-05 2023-08-28T02:23:57.795 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.795 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.795 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:23:57.795 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.550603+0000", 2023-08-28T02:23:57.795 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00017104699999999999 2023-08-28T02:23:57.795 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:23:57.795 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:23:57.795 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:23:57.795 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.550774+0000", 2023-08-28T02:23:57.795 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.17433146099999999 2023-08-28T02:23:57.795 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:57.795 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:57.796 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:57.796 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:23:57.796 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:23:57.796 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:23:57.796 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:23:57.902 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:58.041 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:23:58.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:58.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:23:58.154 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:23:58.187 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:58.208 INFO:teuthology.orchestra.run.smithi143.stderr:marked in osd.3. 2023-08-28T02:23:58.235 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:23:58.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:58.238 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:23:58.338 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:23:58.375 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:58.422 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:23:58.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:58.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:23:58.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:58 smithi169 ceph-mon[159415]: pgmap v317: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 653 op/s; 776/297831 objects degraded (0.261%); 105/297831 objects misplaced (0.035%); 1.4 MiB/s, 23 objects/s recovering 2023-08-28T02:23:58.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:58 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1676252340' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-08-28T02:23:58.503 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:23:58.525 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:23:58.527 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:58 smithi143 ceph-mon[183059]: pgmap v317: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 653 op/s; 776/297831 objects degraded (0.261%); 105/297831 objects misplaced (0.035%); 1.4 MiB/s, 23 objects/s recovering 2023-08-28T02:23:58.527 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:58 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1676252340' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-08-28T02:23:58.527 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:58 smithi143 ceph-mon[189871]: pgmap v317: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 6.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 653 op/s; 776/297831 objects degraded (0.261%); 105/297831 objects misplaced (0.035%); 1.4 MiB/s, 23 objects/s recovering 2023-08-28T02:23:58.527 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:58 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1676252340' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-08-28T02:23:58.552 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:58.646 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:23:58.647 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.7 dump_ops_in_flight 2023-08-28T02:23:58.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:58.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:23:58.752 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:23:58.779 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:58.826 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:23:58.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:58.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:23:58.862 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset noscrub 2023-08-28T02:23:58.933 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:23:58.961 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:59.012 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:23:59.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:59.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:23:59.059 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:23:59.114 INFO:tasks.thrashosds.thrasher:Added osd 3 2023-08-28T02:23:59.118 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:23:59.144 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:59.188 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:23:59.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:59.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:23:59.204 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:23:59.245 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:23:59.292 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:23:59.318 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:59.421 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:23:59.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:59.424 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:23:59.459 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:59 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1676252340' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-08-28T02:23:59.459 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:59 smithi169 ceph-mon[159415]: osdmap e153: 8 total, 8 up, 8 in 2023-08-28T02:23:59.459 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:59 smithi169 ceph-mon[159415]: failed to encode map e153 with expected crc 2023-08-28T02:23:59.459 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:59 smithi169 ceph-mon[159415]: failed to encode map e153 with expected crc 2023-08-28T02:23:59.459 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:23:59 smithi169 ceph-mon[159415]: osdmap e154: 8 total, 8 up, 8 in 2023-08-28T02:23:59.469 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:59 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1676252340' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-08-28T02:23:59.469 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:59 smithi143 ceph-mon[183059]: osdmap e153: 8 total, 8 up, 8 in 2023-08-28T02:23:59.469 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:59 smithi143 ceph-mon[183059]: failed to encode map e153 with expected crc 2023-08-28T02:23:59.469 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:59 smithi143 ceph-mon[183059]: failed to encode map e153 with expected crc 2023-08-28T02:23:59.469 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:23:59 smithi143 ceph-mon[183059]: osdmap e154: 8 total, 8 up, 8 in 2023-08-28T02:23:59.469 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:59 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1676252340' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-08-28T02:23:59.469 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:59 smithi143 ceph-mon[189871]: osdmap e153: 8 total, 8 up, 8 in 2023-08-28T02:23:59.469 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:59 smithi143 ceph-mon[189871]: failed to encode map e153 with expected crc 2023-08-28T02:23:59.469 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:59 smithi143 ceph-mon[189871]: failed to encode map e153 with expected crc 2023-08-28T02:23:59.470 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:23:59 smithi143 ceph-mon[189871]: osdmap e154: 8 total, 8 up, 8 in 2023-08-28T02:23:59.525 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:23:59.555 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:59.654 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:23:59.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:59.657 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:23:59.758 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:23:59.787 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:23:59.883 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:23:59.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:23:59.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:23:59.986 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:24:00.015 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:00.267 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:24:00.269 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:00 smithi169 ceph-mon[159415]: pgmap v319: 89 pgs: 1 unknown, 16 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 63 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 776/273885 objects degraded (0.283%); 105/273885 objects misplaced (0.038%); 1.8 MiB/s, 28 objects/s recovering 2023-08-28T02:24:00.269 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:00 smithi169 ceph-mon[159415]: failed to encode map e154 with expected crc 2023-08-28T02:24:00.269 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:00 smithi169 ceph-mon[159415]: failed to encode map e154 with expected crc 2023-08-28T02:24:00.269 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:00 smithi169 ceph-mon[159415]: failed to encode map e154 with expected crc 2023-08-28T02:24:00.269 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:00 smithi169 ceph-mon[159415]: failed to encode map e154 with expected crc 2023-08-28T02:24:00.269 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:00 smithi169 ceph-mon[159415]: failed to encode map e154 with expected crc 2023-08-28T02:24:00.269 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:00 smithi169 ceph-mon[159415]: failed to encode map e154 with expected crc 2023-08-28T02:24:00.269 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:00 smithi169 ceph-mon[159415]: failed to encode map e154 with expected crc 2023-08-28T02:24:00.269 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:00 smithi169 ceph-mon[159415]: failed to encode map e154 with expected crc 2023-08-28T02:24:00.269 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:00 smithi169 ceph-mon[159415]: Health check failed: Reduced data availability: 5 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-08-28T02:24:00.269 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:00 smithi169 ceph-mon[159415]: osdmap e155: 8 total, 8 up, 8 in 2023-08-28T02:24:00.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:00.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:24:00.371 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:24:00.442 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:00.598 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:00 smithi143 ceph-mon[183059]: pgmap v319: 89 pgs: 1 unknown, 16 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 63 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 776/273885 objects degraded (0.283%); 105/273885 objects misplaced (0.038%); 1.8 MiB/s, 28 objects/s recovering 2023-08-28T02:24:00.599 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:00 smithi143 ceph-mon[183059]: failed to encode map e154 with expected crc 2023-08-28T02:24:00.599 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:00 smithi143 ceph-mon[183059]: failed to encode map e154 with expected crc 2023-08-28T02:24:00.599 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:00 smithi143 ceph-mon[183059]: failed to encode map e154 with expected crc 2023-08-28T02:24:00.599 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:00 smithi143 ceph-mon[183059]: failed to encode map e154 with expected crc 2023-08-28T02:24:00.599 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:00 smithi143 ceph-mon[183059]: failed to encode map e154 with expected crc 2023-08-28T02:24:00.599 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:00 smithi143 ceph-mon[183059]: failed to encode map e154 with expected crc 2023-08-28T02:24:00.599 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:00 smithi143 ceph-mon[183059]: failed to encode map e154 with expected crc 2023-08-28T02:24:00.599 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:00 smithi143 ceph-mon[183059]: failed to encode map e154 with expected crc 2023-08-28T02:24:00.599 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:00 smithi143 ceph-mon[183059]: Health check failed: Reduced data availability: 5 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-08-28T02:24:00.599 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:00 smithi143 ceph-mon[183059]: osdmap e155: 8 total, 8 up, 8 in 2023-08-28T02:24:00.600 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:00 smithi143 ceph-mon[189871]: pgmap v319: 89 pgs: 1 unknown, 16 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 63 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 776/273885 objects degraded (0.283%); 105/273885 objects misplaced (0.038%); 1.8 MiB/s, 28 objects/s recovering 2023-08-28T02:24:00.600 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:00 smithi143 ceph-mon[189871]: failed to encode map e154 with expected crc 2023-08-28T02:24:00.600 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:00 smithi143 ceph-mon[189871]: failed to encode map e154 with expected crc 2023-08-28T02:24:00.600 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:00 smithi143 ceph-mon[189871]: failed to encode map e154 with expected crc 2023-08-28T02:24:00.600 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:00 smithi143 ceph-mon[189871]: failed to encode map e154 with expected crc 2023-08-28T02:24:00.600 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:00 smithi143 ceph-mon[189871]: failed to encode map e154 with expected crc 2023-08-28T02:24:00.600 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:00 smithi143 ceph-mon[189871]: failed to encode map e154 with expected crc 2023-08-28T02:24:00.600 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:00 smithi143 ceph-mon[189871]: failed to encode map e154 with expected crc 2023-08-28T02:24:00.600 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:00 smithi143 ceph-mon[189871]: failed to encode map e154 with expected crc 2023-08-28T02:24:00.600 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:00 smithi143 ceph-mon[189871]: Health check failed: Reduced data availability: 5 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-08-28T02:24:00.601 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:00 smithi143 ceph-mon[189871]: osdmap e155: 8 total, 8 up, 8 in 2023-08-28T02:24:00.901 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:24:00.901 INFO:teuthology.orchestra.run.smithi169.stdout: "ops": [], 2023-08-28T02:24:00.901 INFO:teuthology.orchestra.run.smithi169.stdout: "num_ops": 0 2023-08-28T02:24:00.902 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:24:01.356 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:24:01.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:01.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:24:01.460 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:24:01.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:01 smithi169 ceph-mon[159415]: failed to encode map e155 with expected crc 2023-08-28T02:24:01.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:01 smithi169 ceph-mon[159415]: failed to encode map e155 with expected crc 2023-08-28T02:24:01.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:01 smithi169 ceph-mon[159415]: 5.0 starting backfill to osd.3 from (0'0,0'0] MAX to 152'7773 2023-08-28T02:24:01.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:01 smithi169 ceph-mon[159415]: failed to encode map e155 with expected crc 2023-08-28T02:24:01.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:01 smithi169 ceph-mon[159415]: failed to encode map e155 with expected crc 2023-08-28T02:24:01.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:01 smithi169 ceph-mon[159415]: failed to encode map e155 with expected crc 2023-08-28T02:24:01.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:01 smithi169 ceph-mon[159415]: failed to encode map e155 with expected crc 2023-08-28T02:24:01.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:01 smithi169 ceph-mon[159415]: failed to encode map e155 with expected crc 2023-08-28T02:24:01.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:01 smithi169 ceph-mon[159415]: 5.1 starting backfill to osd.3 from (0'0,0'0] MAX to 152'7807 2023-08-28T02:24:01.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:01 smithi169 ceph-mon[159415]: failed to encode map e155 with expected crc 2023-08-28T02:24:01.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:01 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:24:01.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:01 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 776/273885 objects degraded (0.283%), 7 pgs degraded (PG_DEGRADED) 2023-08-28T02:24:01.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:01 smithi169 ceph-mon[159415]: osdmap e156: 8 total, 8 up, 8 in 2023-08-28T02:24:01.523 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:01 smithi143 ceph-mon[183059]: failed to encode map e155 with expected crc 2023-08-28T02:24:01.524 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:01 smithi143 ceph-mon[183059]: failed to encode map e155 with expected crc 2023-08-28T02:24:01.524 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:01 smithi143 ceph-mon[183059]: 5.0 starting backfill to osd.3 from (0'0,0'0] MAX to 152'7773 2023-08-28T02:24:01.524 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:01 smithi143 ceph-mon[183059]: failed to encode map e155 with expected crc 2023-08-28T02:24:01.524 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:01 smithi143 ceph-mon[183059]: failed to encode map e155 with expected crc 2023-08-28T02:24:01.524 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:01 smithi143 ceph-mon[183059]: failed to encode map e155 with expected crc 2023-08-28T02:24:01.524 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:01 smithi143 ceph-mon[183059]: failed to encode map e155 with expected crc 2023-08-28T02:24:01.524 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:01 smithi143 ceph-mon[183059]: failed to encode map e155 with expected crc 2023-08-28T02:24:01.524 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:01 smithi143 ceph-mon[183059]: 5.1 starting backfill to osd.3 from (0'0,0'0] MAX to 152'7807 2023-08-28T02:24:01.524 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:01 smithi143 ceph-mon[183059]: failed to encode map e155 with expected crc 2023-08-28T02:24:01.524 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:01 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:24:01.524 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:01 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 776/273885 objects degraded (0.283%), 7 pgs degraded (PG_DEGRADED) 2023-08-28T02:24:01.524 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:01 smithi143 ceph-mon[183059]: osdmap e156: 8 total, 8 up, 8 in 2023-08-28T02:24:01.526 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:01 smithi143 ceph-mon[189871]: failed to encode map e155 with expected crc 2023-08-28T02:24:01.526 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:01 smithi143 ceph-mon[189871]: failed to encode map e155 with expected crc 2023-08-28T02:24:01.527 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:01 smithi143 ceph-mon[189871]: 5.0 starting backfill to osd.3 from (0'0,0'0] MAX to 152'7773 2023-08-28T02:24:01.527 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:01 smithi143 ceph-mon[189871]: failed to encode map e155 with expected crc 2023-08-28T02:24:01.527 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:01 smithi143 ceph-mon[189871]: failed to encode map e155 with expected crc 2023-08-28T02:24:01.527 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:01 smithi143 ceph-mon[189871]: failed to encode map e155 with expected crc 2023-08-28T02:24:01.527 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:01 smithi143 ceph-mon[189871]: failed to encode map e155 with expected crc 2023-08-28T02:24:01.527 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:01 smithi143 ceph-mon[189871]: failed to encode map e155 with expected crc 2023-08-28T02:24:01.527 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:01 smithi143 ceph-mon[189871]: 5.1 starting backfill to osd.3 from (0'0,0'0] MAX to 152'7807 2023-08-28T02:24:01.527 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:01 smithi143 ceph-mon[189871]: failed to encode map e155 with expected crc 2023-08-28T02:24:01.527 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:01 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:24:01.527 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:01 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 776/273885 objects degraded (0.283%), 7 pgs degraded (PG_DEGRADED) 2023-08-28T02:24:01.527 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:01 smithi143 ceph-mon[189871]: osdmap e156: 8 total, 8 up, 8 in 2023-08-28T02:24:01.548 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:01.834 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.7 dump_blocked_ops 2023-08-28T02:24:01.883 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:24:01.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:01.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:24:01.978 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:24:01.991 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:24:02.013 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:24:02.051 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:24:02.070 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:24:01 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:24:01] "GET /metrics HTTP/1.1" 200 36508 "" "Prometheus/2.33.4" 2023-08-28T02:24:02.072 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:02.095 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:24:02.146 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:24:02.189 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:24:02.204 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:24:02.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:02.215 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:24:02.226 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:24:02.235 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:02 smithi169 ceph-mon[159415]: pgmap v322: 89 pgs: 1 unknown, 16 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 63 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 776/273885 objects degraded (0.283%); 105/273885 objects misplaced (0.038%); 1.2 MiB/s, 18 objects/s recovering 2023-08-28T02:24:02.235 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:02 smithi169 ceph-mon[159415]: failed to encode map e156 with expected crc 2023-08-28T02:24:02.235 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:02 smithi169 ceph-mon[159415]: failed to encode map e156 with expected crc 2023-08-28T02:24:02.235 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:02 smithi169 ceph-mon[159415]: failed to encode map e156 with expected crc 2023-08-28T02:24:02.235 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:02 smithi169 ceph-mon[159415]: failed to encode map e156 with expected crc 2023-08-28T02:24:02.235 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:02 smithi169 ceph-mon[159415]: failed to encode map e156 with expected crc 2023-08-28T02:24:02.235 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:02 smithi169 ceph-mon[159415]: failed to encode map e156 with expected crc 2023-08-28T02:24:02.235 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:02 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1715010688' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:24:02.262 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:24:02.283 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:24:02.283 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:24:02.283 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:24:02.284 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:24:02.284 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:24:02.284 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:24:02.284 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:24:02.284 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:24:02.315 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:24:02.395 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:02 smithi143 ceph-mon[183059]: pgmap v322: 89 pgs: 1 unknown, 16 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 63 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 776/273885 objects degraded (0.283%); 105/273885 objects misplaced (0.038%); 1.2 MiB/s, 18 objects/s recovering 2023-08-28T02:24:02.395 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:02 smithi143 ceph-mon[183059]: failed to encode map e156 with expected crc 2023-08-28T02:24:02.396 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:02 smithi143 ceph-mon[183059]: failed to encode map e156 with expected crc 2023-08-28T02:24:02.396 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:02 smithi143 ceph-mon[183059]: failed to encode map e156 with expected crc 2023-08-28T02:24:02.396 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:02 smithi143 ceph-mon[183059]: failed to encode map e156 with expected crc 2023-08-28T02:24:02.396 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:02 smithi143 ceph-mon[183059]: failed to encode map e156 with expected crc 2023-08-28T02:24:02.396 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:02 smithi143 ceph-mon[183059]: failed to encode map e156 with expected crc 2023-08-28T02:24:02.396 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:02 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1715010688' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:24:02.396 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:02 smithi143 ceph-mon[189871]: pgmap v322: 89 pgs: 1 unknown, 16 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 63 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 776/273885 objects degraded (0.283%); 105/273885 objects misplaced (0.038%); 1.2 MiB/s, 18 objects/s recovering 2023-08-28T02:24:02.396 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:02 smithi143 ceph-mon[189871]: failed to encode map e156 with expected crc 2023-08-28T02:24:02.396 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:02 smithi143 ceph-mon[189871]: failed to encode map e156 with expected crc 2023-08-28T02:24:02.396 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:02 smithi143 ceph-mon[189871]: failed to encode map e156 with expected crc 2023-08-28T02:24:02.397 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:02 smithi143 ceph-mon[189871]: failed to encode map e156 with expected crc 2023-08-28T02:24:02.397 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:02 smithi143 ceph-mon[189871]: failed to encode map e156 with expected crc 2023-08-28T02:24:02.397 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:02 smithi143 ceph-mon[189871]: failed to encode map e156 with expected crc 2023-08-28T02:24:02.397 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:02 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1715010688' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:24:02.407 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:02.426 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:24:02.576 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:24:02.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:02.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:24:02.694 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:24:02.743 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:02.786 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:24:02.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:02.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:24:02.890 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:24:02.951 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:03.276 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is unset 2023-08-28T02:24:03.323 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:24:03.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:03.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:24:03.427 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:24:03.457 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:03.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:03 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3514394086' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:24:03.501 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:24:03.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:03.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:24:03.604 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:24:03.637 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:03 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3514394086' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:24:03.637 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:03 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3514394086' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:24:03.639 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:03.694 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:24:03.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:03.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:24:03.796 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:24:03.823 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:04.065 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:24:04.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:04.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:24:04.115 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7, 3] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-08-28T02:24:04.115 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-08-28T02:24:04.116 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_0 2023-08-28T02:24:04.116 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph pg dump --format=json 2023-08-28T02:24:04.170 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:24:04.202 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:04.312 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:24:04.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:04.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:24:04.409 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:24:04.409 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:24:04.421 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:24:04.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:04 smithi143 ceph-mon[183059]: pgmap v324: 89 pgs: 1 active+remapped+backfilling, 65 active+clean, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 16 peering; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 332 op/s; 812/288192 objects degraded (0.282%); 11690/288192 objects misplaced (4.056%); 3.9 MiB/s, 52 objects/s recovering 2023-08-28T02:24:04.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:04 smithi143 ceph-mon[183059]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:24:04.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:04 smithi143 ceph-mon[183059]: failed to encode map e157 with expected crc 2023-08-28T02:24:04.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:04 smithi143 ceph-mon[183059]: failed to encode map e157 with expected crc 2023-08-28T02:24:04.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:04 smithi143 ceph-mon[183059]: failed to encode map e157 with expected crc 2023-08-28T02:24:04.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:04 smithi143 ceph-mon[183059]: failed to encode map e157 with expected crc 2023-08-28T02:24:04.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:04 smithi143 ceph-mon[183059]: failed to encode map e157 with expected crc 2023-08-28T02:24:04.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:04 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3514394086' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:24:04.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:04 smithi143 ceph-mon[183059]: osdmap e157: 8 total, 8 up, 8 in 2023-08-28T02:24:04.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:04 smithi143 ceph-mon[183059]: failed to encode map e157 with expected crc 2023-08-28T02:24:04.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:04 smithi143 ceph-mon[183059]: failed to encode map e157 with expected crc 2023-08-28T02:24:04.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:04 smithi143 ceph-mon[183059]: failed to encode map e157 with expected crc 2023-08-28T02:24:04.455 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:04 smithi143 ceph-mon[183059]: failed to encode map e157 with expected crc 2023-08-28T02:24:04.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:04 smithi143 ceph-mon[189871]: pgmap v324: 89 pgs: 1 active+remapped+backfilling, 65 active+clean, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 16 peering; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 332 op/s; 812/288192 objects degraded (0.282%); 11690/288192 objects misplaced (4.056%); 3.9 MiB/s, 52 objects/s recovering 2023-08-28T02:24:04.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:04 smithi143 ceph-mon[189871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:24:04.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:04 smithi143 ceph-mon[189871]: failed to encode map e157 with expected crc 2023-08-28T02:24:04.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:04 smithi143 ceph-mon[189871]: failed to encode map e157 with expected crc 2023-08-28T02:24:04.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:04 smithi143 ceph-mon[189871]: failed to encode map e157 with expected crc 2023-08-28T02:24:04.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:04 smithi143 ceph-mon[189871]: failed to encode map e157 with expected crc 2023-08-28T02:24:04.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:04 smithi143 ceph-mon[189871]: failed to encode map e157 with expected crc 2023-08-28T02:24:04.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:04 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3514394086' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:24:04.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:04 smithi143 ceph-mon[189871]: osdmap e157: 8 total, 8 up, 8 in 2023-08-28T02:24:04.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:04 smithi143 ceph-mon[189871]: failed to encode map e157 with expected crc 2023-08-28T02:24:04.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:04 smithi143 ceph-mon[189871]: failed to encode map e157 with expected crc 2023-08-28T02:24:04.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:04 smithi143 ceph-mon[189871]: failed to encode map e157 with expected crc 2023-08-28T02:24:04.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:04 smithi143 ceph-mon[189871]: failed to encode map e157 with expected crc 2023-08-28T02:24:04.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:04 smithi169 ceph-mon[159415]: pgmap v324: 89 pgs: 1 active+remapped+backfilling, 65 active+clean, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 16 peering; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 332 op/s; 812/288192 objects degraded (0.282%); 11690/288192 objects misplaced (4.056%); 3.9 MiB/s, 52 objects/s recovering 2023-08-28T02:24:04.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:04 smithi169 ceph-mon[159415]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:24:04.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:04 smithi169 ceph-mon[159415]: failed to encode map e157 with expected crc 2023-08-28T02:24:04.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:04 smithi169 ceph-mon[159415]: failed to encode map e157 with expected crc 2023-08-28T02:24:04.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:04 smithi169 ceph-mon[159415]: failed to encode map e157 with expected crc 2023-08-28T02:24:04.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:04 smithi169 ceph-mon[159415]: failed to encode map e157 with expected crc 2023-08-28T02:24:04.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:04 smithi169 ceph-mon[159415]: failed to encode map e157 with expected crc 2023-08-28T02:24:04.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:04 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3514394086' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:24:04.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:04 smithi169 ceph-mon[159415]: osdmap e157: 8 total, 8 up, 8 in 2023-08-28T02:24:04.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:04 smithi169 ceph-mon[159415]: failed to encode map e157 with expected crc 2023-08-28T02:24:04.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:04 smithi169 ceph-mon[159415]: failed to encode map e157 with expected crc 2023-08-28T02:24:04.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:04 smithi169 ceph-mon[159415]: failed to encode map e157 with expected crc 2023-08-28T02:24:04.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:04 smithi169 ceph-mon[159415]: failed to encode map e157 with expected crc 2023-08-28T02:24:04.496 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:04.641 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:24:04.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:04.650 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:24:04.725 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:24:04.750 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:24:04.839 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:04.936 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:24:04.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:04.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:24:05.043 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:24:05.084 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:05.292 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:05 smithi143 ceph-mon[189871]: failed to encode map e158 with expected crc 2023-08-28T02:24:05.293 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:05 smithi143 ceph-mon[189871]: failed to encode map e158 with expected crc 2023-08-28T02:24:05.293 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:05 smithi143 ceph-mon[189871]: failed to encode map e158 with expected crc 2023-08-28T02:24:05.293 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:05 smithi143 ceph-mon[189871]: osdmap e158: 8 total, 8 up, 8 in 2023-08-28T02:24:05.293 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:05 smithi143 ceph-mon[189871]: failed to encode map e158 with expected crc 2023-08-28T02:24:05.293 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:05 smithi143 ceph-mon[189871]: failed to encode map e158 with expected crc 2023-08-28T02:24:05.293 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:05 smithi143 ceph-mon[189871]: failed to encode map e158 with expected crc 2023-08-28T02:24:05.293 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:05 smithi143 ceph-mon[189871]: pgmap v327: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfilling, 79 active+clean, 1 active+remapped+backfill_wait, 2 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 88 MiB/s wr, 1.97k op/s; 6965/286188 objects degraded (2.434%); 12008/286188 objects misplaced (4.196%); 9.3 MiB/s, 139 objects/s recovering 2023-08-28T02:24:05.335 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:24:05.364 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:24:05.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:05.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:24:05.368 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:05 smithi169 ceph-mon[159415]: failed to encode map e158 with expected crc 2023-08-28T02:24:05.368 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:05 smithi169 ceph-mon[159415]: failed to encode map e158 with expected crc 2023-08-28T02:24:05.369 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:05 smithi169 ceph-mon[159415]: failed to encode map e158 with expected crc 2023-08-28T02:24:05.369 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:05 smithi169 ceph-mon[159415]: osdmap e158: 8 total, 8 up, 8 in 2023-08-28T02:24:05.369 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:05 smithi169 ceph-mon[159415]: failed to encode map e158 with expected crc 2023-08-28T02:24:05.369 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:05 smithi169 ceph-mon[159415]: failed to encode map e158 with expected crc 2023-08-28T02:24:05.369 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:05 smithi169 ceph-mon[159415]: failed to encode map e158 with expected crc 2023-08-28T02:24:05.369 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:05 smithi169 ceph-mon[159415]: pgmap v327: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfilling, 79 active+clean, 1 active+remapped+backfill_wait, 2 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 88 MiB/s wr, 1.97k op/s; 6965/286188 objects degraded (2.434%); 12008/286188 objects misplaced (4.196%); 9.3 MiB/s, 139 objects/s recovering 2023-08-28T02:24:05.444 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.7 dump_historic_ops 2023-08-28T02:24:05.468 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:24:05.542 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:05 smithi143 ceph-mon[183059]: failed to encode map e158 with expected crc 2023-08-28T02:24:05.543 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:05 smithi143 ceph-mon[183059]: failed to encode map e158 with expected crc 2023-08-28T02:24:05.543 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:05 smithi143 ceph-mon[183059]: failed to encode map e158 with expected crc 2023-08-28T02:24:05.543 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:05 smithi143 ceph-mon[183059]: osdmap e158: 8 total, 8 up, 8 in 2023-08-28T02:24:05.543 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:05 smithi143 ceph-mon[183059]: failed to encode map e158 with expected crc 2023-08-28T02:24:05.543 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:05 smithi143 ceph-mon[183059]: failed to encode map e158 with expected crc 2023-08-28T02:24:05.543 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:05 smithi143 ceph-mon[183059]: failed to encode map e158 with expected crc 2023-08-28T02:24:05.543 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:05 smithi143 ceph-mon[183059]: pgmap v327: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfilling, 79 active+clean, 1 active+remapped+backfill_wait, 2 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 88 MiB/s wr, 1.97k op/s; 6965/286188 objects degraded (2.434%); 12008/286188 objects misplaced (4.196%); 9.3 MiB/s, 139 objects/s recovering 2023-08-28T02:24:05.553 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:05.655 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:24:05.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:05.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:24:05.778 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:24:05.850 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:05.969 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:24:05.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:05.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:24:06.067 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset nodeep-scrub 2023-08-28T02:24:06.082 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:24:06.162 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:06.198 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:24:06.278 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:24:06.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:06.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:24:06.387 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:24:06.398 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:24:06.412 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:06 smithi169 ceph-mon[159415]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 5 pgs peering) 2023-08-28T02:24:06.412 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:06 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 6965/286188 objects degraded (2.434%), 5 pgs degraded (PG_DEGRADED) 2023-08-28T02:24:06.412 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:06 smithi169 ceph-mon[159415]: osdmap e159: 8 total, 8 up, 8 in 2023-08-28T02:24:06.413 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:06 smithi169 ceph-mon[159415]: failed to encode map e159 with expected crc 2023-08-28T02:24:06.413 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:06 smithi169 ceph-mon[159415]: failed to encode map e159 with expected crc 2023-08-28T02:24:06.415 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:06.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:06 smithi143 ceph-mon[183059]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 5 pgs peering) 2023-08-28T02:24:06.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:06 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 6965/286188 objects degraded (2.434%), 5 pgs degraded (PG_DEGRADED) 2023-08-28T02:24:06.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:06 smithi143 ceph-mon[183059]: osdmap e159: 8 total, 8 up, 8 in 2023-08-28T02:24:06.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:06 smithi143 ceph-mon[183059]: failed to encode map e159 with expected crc 2023-08-28T02:24:06.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:06 smithi143 ceph-mon[183059]: failed to encode map e159 with expected crc 2023-08-28T02:24:06.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:06 smithi143 ceph-mon[189871]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 5 pgs peering) 2023-08-28T02:24:06.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:06 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 6965/286188 objects degraded (2.434%), 5 pgs degraded (PG_DEGRADED) 2023-08-28T02:24:06.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:06 smithi143 ceph-mon[189871]: osdmap e159: 8 total, 8 up, 8 in 2023-08-28T02:24:06.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:06 smithi143 ceph-mon[189871]: failed to encode map e159 with expected crc 2023-08-28T02:24:06.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:06 smithi143 ceph-mon[189871]: failed to encode map e159 with expected crc 2023-08-28T02:24:06.561 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:24:06.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:06.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:24:06.664 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:24:06.693 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:06.745 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:24:06.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:06.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:24:06.849 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:24:06.877 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:06.924 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:24:06.931 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:24:06.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:06.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:24:06.935 INFO:teuthology.orchestra.run.smithi143.stderr:dumped all 2023-08-28T02:24:07.035 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:24:07.062 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:07.209 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:24:07.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:07.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:24:07.313 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:24:07.341 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:07.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:07 smithi143 ceph-mon[189871]: failed to encode map e159 with expected crc 2023-08-28T02:24:07.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:07 smithi143 ceph-mon[189871]: failed to encode map e159 with expected crc 2023-08-28T02:24:07.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:07 smithi143 ceph-mon[189871]: failed to encode map e159 with expected crc 2023-08-28T02:24:07.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:07 smithi143 ceph-mon[189871]: failed to encode map e159 with expected crc 2023-08-28T02:24:07.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:07 smithi143 ceph-mon[189871]: failed to encode map e159 with expected crc 2023-08-28T02:24:07.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:07 smithi143 ceph-mon[189871]: from='client.54655 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:24:07.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:07 smithi143 ceph-mon[189871]: pgmap v329: 89 pgs: 1 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 81 active+clean, 2 active+remapped+backfill_wait, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 73 MiB/s wr, 1.82k op/s; 6892/282948 objects degraded (2.436%); 11177/282948 objects misplaced (3.950%); 16 MiB/s, 241 objects/s recovering 2023-08-28T02:24:07.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:07 smithi143 ceph-mon[189871]: osdmap e160: 8 total, 8 up, 8 in 2023-08-28T02:24:07.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:07 smithi143 ceph-mon[189871]: failed to encode map e160 with expected crc 2023-08-28T02:24:07.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:07 smithi143 ceph-mon[189871]: failed to encode map e160 with expected crc 2023-08-28T02:24:07.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:07 smithi143 ceph-mon[189871]: failed to encode map e160 with expected crc 2023-08-28T02:24:07.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:07 smithi143 ceph-mon[183059]: failed to encode map e159 with expected crc 2023-08-28T02:24:07.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:07 smithi143 ceph-mon[183059]: failed to encode map e159 with expected crc 2023-08-28T02:24:07.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:07 smithi143 ceph-mon[183059]: failed to encode map e159 with expected crc 2023-08-28T02:24:07.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:07 smithi143 ceph-mon[183059]: failed to encode map e159 with expected crc 2023-08-28T02:24:07.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:07 smithi143 ceph-mon[183059]: failed to encode map e159 with expected crc 2023-08-28T02:24:07.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:07 smithi143 ceph-mon[183059]: from='client.54655 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:24:07.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:07 smithi143 ceph-mon[183059]: pgmap v329: 89 pgs: 1 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 81 active+clean, 2 active+remapped+backfill_wait, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 73 MiB/s wr, 1.82k op/s; 6892/282948 objects degraded (2.436%); 11177/282948 objects misplaced (3.950%); 16 MiB/s, 241 objects/s recovering 2023-08-28T02:24:07.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:07 smithi143 ceph-mon[183059]: osdmap e160: 8 total, 8 up, 8 in 2023-08-28T02:24:07.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:07 smithi143 ceph-mon[183059]: failed to encode map e160 with expected crc 2023-08-28T02:24:07.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:07 smithi143 ceph-mon[183059]: failed to encode map e160 with expected crc 2023-08-28T02:24:07.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:07 smithi143 ceph-mon[183059]: failed to encode map e160 with expected crc 2023-08-28T02:24:07.544 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:24:07.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:07.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:24:07.651 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:24:07.661 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:07 smithi169 ceph-mon[159415]: failed to encode map e159 with expected crc 2023-08-28T02:24:07.662 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:07 smithi169 ceph-mon[159415]: failed to encode map e159 with expected crc 2023-08-28T02:24:07.662 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:07 smithi169 ceph-mon[159415]: failed to encode map e159 with expected crc 2023-08-28T02:24:07.662 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:07 smithi169 ceph-mon[159415]: failed to encode map e159 with expected crc 2023-08-28T02:24:07.662 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:07 smithi169 ceph-mon[159415]: failed to encode map e159 with expected crc 2023-08-28T02:24:07.662 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:07 smithi169 ceph-mon[159415]: from='client.54655 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:24:07.662 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:07 smithi169 ceph-mon[159415]: pgmap v329: 89 pgs: 1 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 81 active+clean, 2 active+remapped+backfill_wait, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 73 MiB/s wr, 1.82k op/s; 6892/282948 objects degraded (2.436%); 11177/282948 objects misplaced (3.950%); 16 MiB/s, 241 objects/s recovering 2023-08-28T02:24:07.662 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:07 smithi169 ceph-mon[159415]: osdmap e160: 8 total, 8 up, 8 in 2023-08-28T02:24:07.663 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:07 smithi169 ceph-mon[159415]: failed to encode map e160 with expected crc 2023-08-28T02:24:07.663 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:07 smithi169 ceph-mon[159415]: failed to encode map e160 with expected crc 2023-08-28T02:24:07.663 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:07 smithi169 ceph-mon[159415]: failed to encode map e160 with expected crc 2023-08-28T02:24:07.731 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:07.838 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:24:07.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:07.842 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:24:07.943 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:24:08.006 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:08.197 INFO:teuthology.orchestra.run.smithi143.stdout:{"pg_ready":true,"pg_map":{"version":328,"stamp":"2023-08-28T02:24:06.195249+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6254761931,"num_objects":95396,"num_object_clones":0,"num_object_copies":286188,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6965,"num_objects_misplaced":12008,"num_objects_unfound":0,"num_objects_dirty":95396,"num_whiteouts":0,"num_read":6274,"num_read_kb":13707,"num_write":126020,"num_write_kb":7064102,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2006,"num_bytes_recovered":131476102,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":119629,"ondisk_log_size":119629,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":308,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":20067264,"kb_used_data":18534592,"kb_used_omap":29,"kb_used_meta":1532514,"kb_avail":729893952,"statfs":{"total":767960285184,"available":747411406848,"internally_reserved":0,"allocated":18979422208,"data_stored":18974181530,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30674,"internal_metadata":1569294382},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,4,7,5,4,4,0,0,2,0,0,6],"upper_bound":4096},"perf_stat":{"commit_latency_ms":12,"apply_latency_ms":12,"commit_latency_ns":12000000,"apply_latency_ns":12000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":268763136,"num_objects":4101,"num_object_clones":0,"num_object_copies":12303,"num_objects_missing_on_primary":0,"num_objects_missing":-2,"num_objects_degraded":6189,"num_objects_misplaced":11903,"num_objects_unfound":0,"num_objects_dirty":4101,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9651,"num_write_kb":440064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":681,"num_bytes_recovered":47776402,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.113592"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":139,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-08-28T02:24:00.201314+0000","last_change":"2023-08-28T02:20:19.849807+0000","last_active":"2023-08-28T02:24:00.201314+0000","last_peered":"2023-08-28T02:24:00.201314+0000","last_clean":"2023-08-28T02:24:00.201314+0000","last_became_active":"2023-08-28T02:20:19.849464+0000","last_became_peered":"2023-08-28T02:20:19.849464+0000","last_unstale":"2023-08-28T02:24:00.201314+0000","last_undegraded":"2023-08-28T02:24:00.201314+0000","last_fullsized":"2023-08-28T02:24:00.201314+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":14,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-08-28T02:24:01.206158+0000","last_change":"2023-08-28T02:23:59.204957+0000","last_active":"2023-08-28T02:24:01.206158+0000","last_peered":"2023-08-28T02:24:01.206158+0000","last_clean":"2023-08-28T02:24:01.206158+0000","last_became_active":"2023-08-28T02:23:59.204717+0000","last_became_peered":"2023-08-28T02:23:59.204717+0000","last_unstale":"2023-08-28T02:24:01.206158+0000","last_undegraded":"2023-08-28T02:24:01.206158+0000","last_fullsized":"2023-08-28T02:24:01.206158+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:34:58.977269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":114,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-08-28T02:24:01.203704+0000","last_change":"2023-08-28T02:23:59.212721+0000","last_active":"2023-08-28T02:24:01.203704+0000","last_peered":"2023-08-28T02:24:01.203704+0000","last_clean":"2023-08-28T02:24:01.203704+0000","last_became_active":"2023-08-28T02:23:59.212446+0000","last_became_peered":"2023-08-28T02:23:59.212446+0000","last_unstale":"2023-08-28T02:24:01.203704+0000","last_undegraded":"2023-08-28T02:24:01.203704+0000","last_fullsized":"2023-08-28T02:24:01.203704+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:03:54.701399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":141,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-08-28T02:24:00.198206+0000","last_change":"2023-08-28T02:23:59.223634+0000","last_active":"2023-08-28T02:24:00.198206+0000","last_peered":"2023-08-28T02:24:00.198206+0000","last_clean":"2023-08-28T02:24:00.198206+0000","last_became_active":"2023-08-28T02:23:59.222704+0000","last_became_peered":"2023-08-28T02:23:59.222704+0000","last_unstale":"2023-08-28T02:24:00.198206+0000","last_undegraded":"2023-08-28T02:24:00.198206+0000","last_fullsized":"2023-08-28T02:24:00.198206+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:51:52.752361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-08-28T02:24:01.205487+0000","last_change":"2023-08-28T02:23:59.220459+0000","last_active":"2023-08-28T02:24:01.205487+0000","last_peered":"2023-08-28T02:24:01.205487+0000","last_clean":"2023-08-28T02:24:01.205487+0000","last_became_active":"2023-08-28T02:23:59.219274+0000","last_became_peered":"2023-08-28T02:23:59.219274+0000","last_unstale":"2023-08-28T02:24:01.205487+0000","last_undegraded":"2023-08-28T02:24:01.205487+0000","last_fullsized":"2023-08-28T02:24:01.205487+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:31:53.889311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":150,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-08-28T02:24:00.203424+0000","last_change":"2023-08-28T02:21:59.183009+0000","last_active":"2023-08-28T02:24:00.203424+0000","last_peered":"2023-08-28T02:24:00.203424+0000","last_clean":"2023-08-28T02:24:00.203424+0000","last_became_active":"2023-08-28T02:21:59.182621+0000","last_became_peered":"2023-08-28T02:21:59.182621+0000","last_unstale":"2023-08-28T02:24:00.203424+0000","last_undegraded":"2023-08-28T02:24:00.203424+0000","last_fullsized":"2023-08-28T02:24:00.203424+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":122,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-08-28T02:24:00.197928+0000","last_change":"2023-08-28T02:23:59.227592+0000","last_active":"2023-08-28T02:24:00.197928+0000","last_peered":"2023-08-28T02:24:00.197928+0000","last_clean":"2023-08-28T02:24:00.197928+0000","last_became_active":"2023-08-28T02:23:59.227342+0000","last_became_peered":"2023-08-28T02:23:59.227342+0000","last_unstale":"2023-08-28T02:24:00.197928+0000","last_undegraded":"2023-08-28T02:24:00.197928+0000","last_fullsized":"2023-08-28T02:24:00.197928+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:08:19.874396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":14,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-08-28T02:24:01.205412+0000","last_change":"2023-08-28T02:23:59.220634+0000","last_active":"2023-08-28T02:24:01.205412+0000","last_peered":"2023-08-28T02:24:01.205412+0000","last_clean":"2023-08-28T02:24:01.205412+0000","last_became_active":"2023-08-28T02:23:59.220177+0000","last_became_peered":"2023-08-28T02:23:59.220177+0000","last_unstale":"2023-08-28T02:24:01.205412+0000","last_undegraded":"2023-08-28T02:24:01.205412+0000","last_fullsized":"2023-08-28T02:24:01.205412+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:16:26.906064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":141,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-08-28T02:24:00.205502+0000","last_change":"2023-08-28T02:23:38.339581+0000","last_active":"2023-08-28T02:24:00.205502+0000","last_peered":"2023-08-28T02:24:00.205502+0000","last_clean":"2023-08-28T02:24:00.205502+0000","last_became_active":"2023-08-28T02:23:38.339244+0000","last_became_peered":"2023-08-28T02:23:38.339244+0000","last_unstale":"2023-08-28T02:24:00.205502+0000","last_undegraded":"2023-08-28T02:24:00.205502+0000","last_fullsized":"2023-08-28T02:24:00.205502+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":14,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-08-28T02:24:01.205704+0000","last_change":"2023-08-28T02:23:59.220551+0000","last_active":"2023-08-28T02:24:01.205704+0000","last_peered":"2023-08-28T02:24:01.205704+0000","last_clean":"2023-08-28T02:24:01.205704+0000","last_became_active":"2023-08-28T02:23:59.219681+0000","last_became_peered":"2023-08-28T02:23:59.219681+0000","last_unstale":"2023-08-28T02:24:01.205704+0000","last_undegraded":"2023-08-28T02:24:01.205704+0000","last_fullsized":"2023-08-28T02:24:01.205704+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:39:58.811256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":42,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-08-28T02:24:00.203971+0000","last_change":"2023-08-28T02:23:59.316561+0000","last_active":"2023-08-28T02:24:00.203971+0000","last_peered":"2023-08-28T02:24:00.203971+0000","last_clean":"2023-08-28T02:24:00.203971+0000","last_became_active":"2023-08-28T02:23:59.227433+0000","last_became_peered":"2023-08-28T02:23:59.227433+0000","last_unstale":"2023-08-28T02:24:00.203971+0000","last_undegraded":"2023-08-28T02:24:00.203971+0000","last_fullsized":"2023-08-28T02:24:00.203971+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":97,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-08-28T02:24:01.203756+0000","last_change":"2023-08-28T02:20:19.851616+0000","last_active":"2023-08-28T02:24:01.203756+0000","last_peered":"2023-08-28T02:24:01.203756+0000","last_clean":"2023-08-28T02:24:01.203756+0000","last_became_active":"2023-08-28T02:20:19.851322+0000","last_became_peered":"2023-08-28T02:20:19.851322+0000","last_unstale":"2023-08-28T02:24:01.203756+0000","last_undegraded":"2023-08-28T02:24:01.203756+0000","last_fullsized":"2023-08-28T02:24:01.203756+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:03:37.565646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"86'22","reported_seq":168,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-08-28T02:24:00.201387+0000","last_change":"2023-08-28T02:23:38.368275+0000","last_active":"2023-08-28T02:24:00.201387+0000","last_peered":"2023-08-28T02:24:00.201387+0000","last_clean":"2023-08-28T02:24:00.201387+0000","last_became_active":"2023-08-28T02:23:38.342097+0000","last_became_peered":"2023-08-28T02:23:38.342097+0000","last_unstale":"2023-08-28T02:24:00.201387+0000","last_undegraded":"2023-08-28T02:24:00.201387+0000","last_fullsized":"2023-08-28T02:24:00.201387+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":136,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-08-28T02:24:00.201053+0000","last_change":"2023-08-28T02:19:51.204118+0000","last_active":"2023-08-28T02:24:00.201053+0000","last_peered":"2023-08-28T02:24:00.201053+0000","last_clean":"2023-08-28T02:24:00.201053+0000","last_became_active":"2023-08-28T02:19:51.203745+0000","last_became_peered":"2023-08-28T02:19:51.203745+0000","last_unstale":"2023-08-28T02:24:00.201053+0000","last_undegraded":"2023-08-28T02:24:00.201053+0000","last_fullsized":"2023-08-28T02:24:00.201053+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":27,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-08-28T02:24:00.203488+0000","last_change":"2023-08-28T02:23:38.983634+0000","last_active":"2023-08-28T02:24:00.203488+0000","last_peered":"2023-08-28T02:24:00.203488+0000","last_clean":"2023-08-28T02:24:00.203488+0000","last_became_active":"2023-08-28T02:23:38.344566+0000","last_became_peered":"2023-08-28T02:23:38.344566+0000","last_unstale":"2023-08-28T02:24:00.203488+0000","last_undegraded":"2023-08-28T02:24:00.203488+0000","last_fullsized":"2023-08-28T02:24:00.203488+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":14,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-08-28T02:24:01.205912+0000","last_change":"2023-08-28T02:23:59.216960+0000","last_active":"2023-08-28T02:24:01.205912+0000","last_peered":"2023-08-28T02:24:01.205912+0000","last_clean":"2023-08-28T02:24:01.205912+0000","last_became_active":"2023-08-28T02:23:59.215657+0000","last_became_peered":"2023-08-28T02:23:59.215657+0000","last_unstale":"2023-08-28T02:24:01.205912+0000","last_undegraded":"2023-08-28T02:24:01.205912+0000","last_fullsized":"2023-08-28T02:24:01.205912+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:31:14.407108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":140,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-08-28T02:24:00.198109+0000","last_change":"2023-08-28T02:23:59.223066+0000","last_active":"2023-08-28T02:24:00.198109+0000","last_peered":"2023-08-28T02:24:00.198109+0000","last_clean":"2023-08-28T02:24:00.198109+0000","last_became_active":"2023-08-28T02:23:59.222731+0000","last_became_peered":"2023-08-28T02:23:59.222731+0000","last_unstale":"2023-08-28T02:24:00.198109+0000","last_undegraded":"2023-08-28T02:24:00.198109+0000","last_fullsized":"2023-08-28T02:24:00.198109+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:48:37.558266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":183,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-08-28T02:24:00.205278+0000","last_change":"2023-08-28T02:23:59.210469+0000","last_active":"2023-08-28T02:24:00.205278+0000","last_peered":"2023-08-28T02:24:00.205278+0000","last_clean":"2023-08-28T02:24:00.205278+0000","last_became_active":"2023-08-28T02:23:59.210031+0000","last_became_peered":"2023-08-28T02:23:59.210031+0000","last_unstale":"2023-08-28T02:24:00.205278+0000","last_undegraded":"2023-08-28T02:24:00.205278+0000","last_fullsized":"2023-08-28T02:24:00.205278+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":21,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-08-28T02:24:00.203055+0000","last_change":"2023-08-28T02:23:38.370183+0000","last_active":"2023-08-28T02:24:00.203055+0000","last_peered":"2023-08-28T02:24:00.203055+0000","last_clean":"2023-08-28T02:24:00.203055+0000","last_became_active":"2023-08-28T02:23:38.368415+0000","last_became_peered":"2023-08-28T02:23:38.368415+0000","last_unstale":"2023-08-28T02:24:00.203055+0000","last_undegraded":"2023-08-28T02:24:00.203055+0000","last_fullsized":"2023-08-28T02:24:00.203055+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":141,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-08-28T02:24:00.198287+0000","last_change":"2023-08-28T02:23:59.220932+0000","last_active":"2023-08-28T02:24:00.198287+0000","last_peered":"2023-08-28T02:24:00.198287+0000","last_clean":"2023-08-28T02:24:00.198287+0000","last_became_active":"2023-08-28T02:23:59.220488+0000","last_became_peered":"2023-08-28T02:23:59.220488+0000","last_unstale":"2023-08-28T02:24:00.198287+0000","last_undegraded":"2023-08-28T02:24:00.198287+0000","last_fullsized":"2023-08-28T02:24:00.198287+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:53:09.216340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":130,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-08-28T02:24:00.202682+0000","last_change":"2023-08-28T02:18:39.778728+0000","last_active":"2023-08-28T02:24:00.202682+0000","last_peered":"2023-08-28T02:24:00.202682+0000","last_clean":"2023-08-28T02:24:00.202682+0000","last_became_active":"2023-08-28T02:18:39.778283+0000","last_became_peered":"2023-08-28T02:18:39.778283+0000","last_unstale":"2023-08-28T02:24:00.202682+0000","last_undegraded":"2023-08-28T02:24:00.202682+0000","last_fullsized":"2023-08-28T02:24:00.202682+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":14,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-08-28T02:24:01.206626+0000","last_change":"2023-08-28T02:23:59.216743+0000","last_active":"2023-08-28T02:24:01.206626+0000","last_peered":"2023-08-28T02:24:01.206626+0000","last_clean":"2023-08-28T02:24:01.206626+0000","last_became_active":"2023-08-28T02:23:59.216416+0000","last_became_peered":"2023-08-28T02:23:59.216416+0000","last_unstale":"2023-08-28T02:24:01.206626+0000","last_undegraded":"2023-08-28T02:24:01.206626+0000","last_fullsized":"2023-08-28T02:24:01.206626+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:25:13.056747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"156'7708","reported_seq":8392,"reported_epoch":156,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:24:01.994627+0000","last_change":"2023-08-28T02:23:59.231082+0000","last_active":"2023-08-28T02:24:01.994627+0000","last_peered":"2023-08-28T02:24:01.994627+0000","last_clean":"2023-08-28T02:23:25.010799+0000","last_became_active":"2023-08-28T02:23:38.380924+0000","last_became_peered":"2023-08-28T02:23:38.380924+0000","last_unstale":"2023-08-28T02:24:01.994627+0000","last_undegraded":"2023-08-28T02:23:59.230920+0000","last_fullsized":"2023-08-28T02:23:38.324894+0000","mapping_epoch":144,"log_start":"124'300","ondisk_log_start":"124'300","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":7408,"log_dups_size":0,"ondisk_log_size":7408,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393478144,"num_objects":6004,"num_object_clones":0,"num_object_copies":18012,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":56,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6004,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7708,"num_write_kb":438784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":311,"num_bytes_recovered":20250624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[6,4],"avail_no_missing":["6","4"],"object_location_counts":[{"shards":"1,4,6","objects":54},{"shards":"4,6","objects":5950}],"blocked_by":[],"up_primary":7,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":25,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-08-28T02:24:00.204599+0000","last_change":"2023-08-28T02:23:38.657185+0000","last_active":"2023-08-28T02:24:00.204599+0000","last_peered":"2023-08-28T02:24:00.204599+0000","last_clean":"2023-08-28T02:24:00.204599+0000","last_became_active":"2023-08-28T02:23:38.339016+0000","last_became_peered":"2023-08-28T02:23:38.339016+0000","last_unstale":"2023-08-28T02:24:00.204599+0000","last_undegraded":"2023-08-28T02:24:00.204599+0000","last_fullsized":"2023-08-28T02:24:00.204599+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":125,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-08-28T02:24:00.205099+0000","last_change":"2023-08-28T02:23:38.339886+0000","last_active":"2023-08-28T02:24:00.205099+0000","last_peered":"2023-08-28T02:24:00.205099+0000","last_clean":"2023-08-28T02:24:00.205099+0000","last_became_active":"2023-08-28T02:23:38.339514+0000","last_became_peered":"2023-08-28T02:23:38.339514+0000","last_unstale":"2023-08-28T02:24:00.205099+0000","last_undegraded":"2023-08-28T02:24:00.205099+0000","last_fullsized":"2023-08-28T02:24:00.205099+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"156'7819","reported_seq":7864,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-08-28T02:24:03.066734+0000","last_change":"2023-08-28T02:22:00.165260+0000","last_active":"2023-08-28T02:24:03.066734+0000","last_peered":"2023-08-28T02:24:03.066734+0000","last_clean":"2023-08-28T02:24:03.066734+0000","last_became_active":"2023-08-28T02:22:00.164978+0000","last_became_peered":"2023-08-28T02:22:00.164978+0000","last_unstale":"2023-08-28T02:24:03.066734+0000","last_undegraded":"2023-08-28T02:24:03.066734+0000","last_fullsized":"2023-08-28T02:24:03.066734+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":7819,"log_dups_size":0,"ondisk_log_size":7819,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:18:41.711960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393674752,"num_objects":6007,"num_object_clones":0,"num_object_copies":18021,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6007,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7819,"num_write_kb":442432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":25,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-08-28T02:24:00.203014+0000","last_change":"2023-08-28T02:23:38.555339+0000","last_active":"2023-08-28T02:24:00.203014+0000","last_peered":"2023-08-28T02:24:00.203014+0000","last_clean":"2023-08-28T02:24:00.203014+0000","last_became_active":"2023-08-28T02:23:38.337938+0000","last_became_peered":"2023-08-28T02:23:38.337938+0000","last_unstale":"2023-08-28T02:24:00.203014+0000","last_undegraded":"2023-08-28T02:24:00.203014+0000","last_fullsized":"2023-08-28T02:24:00.203014+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":164,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-08-28T02:24:00.218405+0000","last_change":"2023-08-28T02:23:59.208685+0000","last_active":"2023-08-28T02:24:00.218405+0000","last_peered":"2023-08-28T02:24:00.218405+0000","last_clean":"2023-08-28T02:24:00.218405+0000","last_became_active":"2023-08-28T02:23:59.208377+0000","last_became_peered":"2023-08-28T02:23:59.208377+0000","last_unstale":"2023-08-28T02:24:00.218405+0000","last_undegraded":"2023-08-28T02:24:00.218405+0000","last_fullsized":"2023-08-28T02:24:00.218405+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"156'7797","reported_seq":7842,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-08-28T02:24:03.059163+0000","last_change":"2023-08-28T02:22:00.165317+0000","last_active":"2023-08-28T02:24:03.059163+0000","last_peered":"2023-08-28T02:24:03.059163+0000","last_clean":"2023-08-28T02:24:03.059163+0000","last_became_active":"2023-08-28T02:22:00.164989+0000","last_became_peered":"2023-08-28T02:22:00.164989+0000","last_unstale":"2023-08-28T02:24:03.059163+0000","last_undegraded":"2023-08-28T02:24:03.059163+0000","last_fullsized":"2023-08-28T02:24:03.059163+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":7797,"log_dups_size":0,"ondisk_log_size":7797,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:10:46.322659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":388169728,"num_objects":5923,"num_object_clones":0,"num_object_copies":17769,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5923,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7797,"num_write_kb":439040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":25,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-08-28T02:24:00.203757+0000","last_change":"2023-08-28T02:23:38.749428+0000","last_active":"2023-08-28T02:24:00.203757+0000","last_peered":"2023-08-28T02:24:00.203757+0000","last_clean":"2023-08-28T02:24:00.203757+0000","last_became_active":"2023-08-28T02:23:38.339587+0000","last_became_peered":"2023-08-28T02:23:38.339587+0000","last_unstale":"2023-08-28T02:24:00.203757+0000","last_undegraded":"2023-08-28T02:24:00.203757+0000","last_fullsized":"2023-08-28T02:24:00.203757+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":182,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-08-28T02:24:00.202938+0000","last_change":"2023-08-28T02:23:38.340065+0000","last_active":"2023-08-28T02:24:00.202938+0000","last_peered":"2023-08-28T02:24:00.202938+0000","last_clean":"2023-08-28T02:24:00.202938+0000","last_became_active":"2023-08-28T02:23:38.328689+0000","last_became_peered":"2023-08-28T02:23:38.328689+0000","last_unstale":"2023-08-28T02:24:00.202938+0000","last_undegraded":"2023-08-28T02:24:00.202938+0000","last_fullsized":"2023-08-28T02:24:00.202938+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"156'7807","reported_seq":7860,"reported_epoch":156,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:24:03.086657+0000","last_change":"2023-08-28T02:24:00.446810+0000","last_active":"2023-08-28T02:24:03.086657+0000","last_peered":"2023-08-28T02:24:03.086657+0000","last_clean":"2023-08-28T02:23:58.182539+0000","last_became_active":"2023-08-28T02:24:00.420432+0000","last_became_peered":"2023-08-28T02:24:00.420432+0000","last_unstale":"2023-08-28T02:24:03.086657+0000","last_undegraded":"2023-08-28T02:24:00.242280+0000","last_fullsized":"2023-08-28T02:24:00.199333+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":7807,"log_dups_size":0,"ondisk_log_size":7807,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:57:53.818041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":390529024,"num_objects":5959,"num_object_clones":0,"num_object_copies":17877,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6883,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5959,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7807,"num_write_kb":440512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4],"avail_no_missing":["2","0","4"],"object_location_counts":[{"shards":"0,2,4","objects":5959},{"shards":"2,4","objects":-117}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":141,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-08-28T02:24:00.205357+0000","last_change":"2023-08-28T02:23:59.211466+0000","last_active":"2023-08-28T02:24:00.205357+0000","last_peered":"2023-08-28T02:24:00.205357+0000","last_clean":"2023-08-28T02:24:00.205357+0000","last_became_active":"2023-08-28T02:23:59.211175+0000","last_became_peered":"2023-08-28T02:23:59.211175+0000","last_unstale":"2023-08-28T02:24:00.205357+0000","last_undegraded":"2023-08-28T02:24:00.205357+0000","last_fullsized":"2023-08-28T02:24:00.205357+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":116,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-08-28T02:24:00.197609+0000","last_change":"2023-08-28T02:19:51.205245+0000","last_active":"2023-08-28T02:24:00.197609+0000","last_peered":"2023-08-28T02:24:00.197609+0000","last_clean":"2023-08-28T02:24:00.197609+0000","last_became_active":"2023-08-28T02:19:51.204896+0000","last_became_peered":"2023-08-28T02:19:51.204896+0000","last_unstale":"2023-08-28T02:24:00.197609+0000","last_undegraded":"2023-08-28T02:24:00.197609+0000","last_fullsized":"2023-08-28T02:24:00.197609+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:51:24.634702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"156'7862","reported_seq":8733,"reported_epoch":156,"state":"active+undersized+remapped","last_fresh":"2023-08-28T02:24:03.102609+0000","last_change":"2023-08-28T02:24:02.562775+0000","last_active":"2023-08-28T02:24:03.102609+0000","last_peered":"2023-08-28T02:24:03.102609+0000","last_clean":"2023-08-28T02:23:25.078528+0000","last_became_active":"2023-08-28T02:23:38.360745+0000","last_became_peered":"2023-08-28T02:23:38.360745+0000","last_unstale":"2023-08-28T02:24:03.102609+0000","last_undegraded":"2023-08-28T02:24:03.102609+0000","last_fullsized":"2023-08-28T02:23:38.317972+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":7862,"log_dups_size":0,"ondisk_log_size":7862,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:54:02.589214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393674776,"num_objects":6008,"num_object_clones":0,"num_object_copies":18024,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6008,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7862,"num_write_kb":443777,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":403,"num_bytes_recovered":26345496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,5],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"2,5","objects":6008}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":18,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-08-28T02:24:01.205618+0000","last_change":"2023-08-28T02:23:59.230059+0000","last_active":"2023-08-28T02:24:01.205618+0000","last_peered":"2023-08-28T02:24:01.205618+0000","last_clean":"2023-08-28T02:24:01.205618+0000","last_became_active":"2023-08-28T02:23:59.217783+0000","last_became_peered":"2023-08-28T02:23:59.217783+0000","last_unstale":"2023-08-28T02:24:01.205618+0000","last_undegraded":"2023-08-28T02:24:01.205618+0000","last_fullsized":"2023-08-28T02:24:01.205618+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:00:12.145395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":149,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-08-28T02:24:00.201052+0000","last_change":"2023-08-28T02:23:59.211286+0000","last_active":"2023-08-28T02:24:00.201052+0000","last_peered":"2023-08-28T02:24:00.201052+0000","last_clean":"2023-08-28T02:24:00.201052+0000","last_became_active":"2023-08-28T02:23:59.210942+0000","last_became_peered":"2023-08-28T02:23:59.210942+0000","last_unstale":"2023-08-28T02:24:00.201052+0000","last_undegraded":"2023-08-28T02:24:00.201052+0000","last_fullsized":"2023-08-28T02:24:00.201052+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"157'7965","reported_seq":8012,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-08-28T02:24:03.263806+0000","last_change":"2023-08-28T02:22:00.162375+0000","last_active":"2023-08-28T02:24:03.263806+0000","last_peered":"2023-08-28T02:24:03.263806+0000","last_clean":"2023-08-28T02:24:03.263806+0000","last_became_active":"2023-08-28T02:22:00.162190+0000","last_became_peered":"2023-08-28T02:22:00.162190+0000","last_unstale":"2023-08-28T02:24:03.263806+0000","last_undegraded":"2023-08-28T02:24:03.263806+0000","last_fullsized":"2023-08-28T02:24:03.263806+0000","mapping_epoch":119,"log_start":"125'1439","ondisk_log_start":"125'1439","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":6526,"log_dups_size":0,"ondisk_log_size":6526,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:40:08.985631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":394461184,"num_objects":6019,"num_object_clones":0,"num_object_copies":18057,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6019,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7965,"num_write_kb":447488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":151,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-08-28T02:24:00.203094+0000","last_change":"2023-08-28T02:21:59.179699+0000","last_active":"2023-08-28T02:24:00.203094+0000","last_peered":"2023-08-28T02:24:00.203094+0000","last_clean":"2023-08-28T02:24:00.203094+0000","last_became_active":"2023-08-28T02:21:59.179345+0000","last_became_peered":"2023-08-28T02:21:59.179345+0000","last_unstale":"2023-08-28T02:24:00.203094+0000","last_undegraded":"2023-08-28T02:24:00.203094+0000","last_fullsized":"2023-08-28T02:24:00.203094+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":175,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-08-28T02:24:00.201280+0000","last_change":"2023-08-28T02:22:44.083121+0000","last_active":"2023-08-28T02:24:00.201280+0000","last_peered":"2023-08-28T02:24:00.201280+0000","last_clean":"2023-08-28T02:24:00.201280+0000","last_became_active":"2023-08-28T02:22:44.082776+0000","last_became_peered":"2023-08-28T02:22:44.082776+0000","last_unstale":"2023-08-28T02:24:00.201280+0000","last_undegraded":"2023-08-28T02:24:00.201280+0000","last_fullsized":"2023-08-28T02:24:00.201280+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"156'7812","reported_seq":7857,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-08-28T02:24:02.003410+0000","last_change":"2023-08-28T02:22:00.167630+0000","last_active":"2023-08-28T02:24:02.003410+0000","last_peered":"2023-08-28T02:24:02.003410+0000","last_clean":"2023-08-28T02:24:02.003410+0000","last_became_active":"2023-08-28T02:22:00.167305+0000","last_became_peered":"2023-08-28T02:22:00.167305+0000","last_unstale":"2023-08-28T02:24:02.003410+0000","last_undegraded":"2023-08-28T02:24:02.003410+0000","last_fullsized":"2023-08-28T02:24:02.003410+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":7812,"log_dups_size":0,"ondisk_log_size":7812,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":392298496,"num_objects":5986,"num_object_clones":0,"num_object_copies":17958,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5986,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7812,"num_write_kb":441536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":65,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-08-28T02:24:00.218485+0000","last_change":"2023-08-28T02:23:38.347043+0000","last_active":"2023-08-28T02:24:00.218485+0000","last_peered":"2023-08-28T02:24:00.218485+0000","last_clean":"2023-08-28T02:24:00.218485+0000","last_became_active":"2023-08-28T02:23:38.346680+0000","last_became_peered":"2023-08-28T02:23:38.346680+0000","last_unstale":"2023-08-28T02:24:00.218485+0000","last_undegraded":"2023-08-28T02:24:00.218485+0000","last_fullsized":"2023-08-28T02:24:00.218485+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":84,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-08-28T02:24:00.198461+0000","last_change":"2023-08-28T02:19:51.204017+0000","last_active":"2023-08-28T02:24:00.198461+0000","last_peered":"2023-08-28T02:24:00.198461+0000","last_clean":"2023-08-28T02:24:00.198461+0000","last_became_active":"2023-08-28T02:19:51.203627+0000","last_became_peered":"2023-08-28T02:19:51.203627+0000","last_unstale":"2023-08-28T02:24:00.198461+0000","last_undegraded":"2023-08-28T02:24:00.198461+0000","last_fullsized":"2023-08-28T02:24:00.198461+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:14:06.231701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.e","version":"156'7854","reported_seq":7899,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-08-28T02:24:02.019005+0000","last_change":"2023-08-28T02:22:00.166490+0000","last_active":"2023-08-28T02:24:02.019005+0000","last_peered":"2023-08-28T02:24:02.019005+0000","last_clean":"2023-08-28T02:24:02.019005+0000","last_became_active":"2023-08-28T02:22:00.166140+0000","last_became_peered":"2023-08-28T02:22:00.166140+0000","last_unstale":"2023-08-28T02:24:02.019005+0000","last_undegraded":"2023-08-28T02:24:02.019005+0000","last_fullsized":"2023-08-28T02:24:02.019005+0000","mapping_epoch":119,"log_start":"124'600","ondisk_log_start":"124'600","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":7254,"log_dups_size":0,"ondisk_log_size":7254,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":401211392,"num_objects":6122,"num_object_clones":0,"num_object_copies":18366,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6122,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7854,"num_write_kb":447232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-08-28T02:24:01.205247+0000","last_change":"2023-08-28T02:23:59.216950+0000","last_active":"2023-08-28T02:24:01.205247+0000","last_peered":"2023-08-28T02:24:01.205247+0000","last_clean":"2023-08-28T02:24:01.205247+0000","last_became_active":"2023-08-28T02:23:59.216373+0000","last_became_peered":"2023-08-28T02:23:59.216373+0000","last_unstale":"2023-08-28T02:24:01.205247+0000","last_undegraded":"2023-08-28T02:24:01.205247+0000","last_fullsized":"2023-08-28T02:24:01.205247+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:13:14.736771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":133,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-08-28T02:24:01.204093+0000","last_change":"2023-08-28T02:23:59.220269+0000","last_active":"2023-08-28T02:24:01.204093+0000","last_peered":"2023-08-28T02:24:01.204093+0000","last_clean":"2023-08-28T02:24:01.204093+0000","last_became_active":"2023-08-28T02:23:59.219966+0000","last_became_peered":"2023-08-28T02:23:59.219966+0000","last_unstale":"2023-08-28T02:24:01.204093+0000","last_undegraded":"2023-08-28T02:24:01.204093+0000","last_fullsized":"2023-08-28T02:24:01.204093+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:16:08.119909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"78'126","reported_seq":633,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-08-28T02:24:01.204878+0000","last_change":"2023-08-28T02:23:39.351975+0000","last_active":"2023-08-28T02:24:01.204878+0000","last_peered":"2023-08-28T02:24:01.204878+0000","last_clean":"2023-08-28T02:24:01.204878+0000","last_became_active":"2023-08-28T02:23:39.351437+0000","last_became_peered":"2023-08-28T02:23:39.351437+0000","last_unstale":"2023-08-28T02:24:01.204878+0000","last_undegraded":"2023-08-28T02:24:01.204878+0000","last_fullsized":"2023-08-28T02:24:01.204878+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:25:22.203120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":743,"num_read_kb":1331,"num_write":125,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":156,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-08-28T02:24:00.205342+0000","last_change":"2023-08-28T02:23:59.207148+0000","last_active":"2023-08-28T02:24:00.205342+0000","last_peered":"2023-08-28T02:24:00.205342+0000","last_clean":"2023-08-28T02:24:00.205342+0000","last_became_active":"2023-08-28T02:23:59.206802+0000","last_became_peered":"2023-08-28T02:23:59.206802+0000","last_unstale":"2023-08-28T02:24:00.205342+0000","last_undegraded":"2023-08-28T02:24:00.205342+0000","last_fullsized":"2023-08-28T02:24:00.205342+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":53,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-08-28T02:24:01.204093+0000","last_change":"2023-08-28T02:21:43.238044+0000","last_active":"2023-08-28T02:24:01.204093+0000","last_peered":"2023-08-28T02:24:01.204093+0000","last_clean":"2023-08-28T02:24:01.204093+0000","last_became_active":"2023-08-28T02:21:43.237792+0000","last_became_peered":"2023-08-28T02:21:43.237792+0000","last_unstale":"2023-08-28T02:24:01.204093+0000","last_undegraded":"2023-08-28T02:24:01.204093+0000","last_fullsized":"2023-08-28T02:24:01.204093+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:11:35.488503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"156'7719","reported_seq":7765,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-08-28T02:24:03.034834+0000","last_change":"2023-08-28T02:22:00.166578+0000","last_active":"2023-08-28T02:24:03.034834+0000","last_peered":"2023-08-28T02:24:03.034834+0000","last_clean":"2023-08-28T02:24:03.034834+0000","last_became_active":"2023-08-28T02:22:00.166311+0000","last_became_peered":"2023-08-28T02:22:00.166311+0000","last_unstale":"2023-08-28T02:24:03.034834+0000","last_undegraded":"2023-08-28T02:24:03.034834+0000","last_fullsized":"2023-08-28T02:24:03.034834+0000","mapping_epoch":119,"log_start":"125'841","ondisk_log_start":"125'841","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":6878,"log_dups_size":0,"ondisk_log_size":6878,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:40:38.502180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":388694016,"num_objects":5931,"num_object_clones":0,"num_object_copies":17793,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5931,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7719,"num_write_kb":436800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"156'7776","reported_seq":7822,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-08-28T02:24:03.023984+0000","last_change":"2023-08-28T02:22:00.160991+0000","last_active":"2023-08-28T02:24:03.023984+0000","last_peered":"2023-08-28T02:24:03.023984+0000","last_clean":"2023-08-28T02:24:03.023984+0000","last_became_active":"2023-08-28T02:22:00.160755+0000","last_became_peered":"2023-08-28T02:22:00.160755+0000","last_unstale":"2023-08-28T02:24:03.023984+0000","last_undegraded":"2023-08-28T02:24:03.023984+0000","last_fullsized":"2023-08-28T02:24:03.023984+0000","mapping_epoch":119,"log_start":"125'898","ondisk_log_start":"125'898","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":6878,"log_dups_size":0,"ondisk_log_size":6878,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:42:59.109288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":391249920,"num_objects":5970,"num_object_clones":0,"num_object_copies":17910,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5970,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7776,"num_write_kb":439872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"78'204","reported_seq":1072,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-08-28T02:24:00.205189+0000","last_change":"2023-08-28T02:19:50.219949+0000","last_active":"2023-08-28T02:24:00.205189+0000","last_peered":"2023-08-28T02:24:00.205189+0000","last_clean":"2023-08-28T02:24:00.205189+0000","last_became_active":"2023-08-28T02:19:50.219356+0000","last_became_peered":"2023-08-28T02:19:50.219356+0000","last_unstale":"2023-08-28T02:24:00.205189+0000","last_undegraded":"2023-08-28T02:24:00.205189+0000","last_fullsized":"2023-08-28T02:24:00.205189+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":204,"log_dups_size":0,"ondisk_log_size":204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1521,"num_read_kb":2511,"num_write":178,"num_write_kb":1175,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":123,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-08-28T02:24:00.201258+0000","last_change":"2023-08-28T02:18:39.779607+0000","last_active":"2023-08-28T02:24:00.201258+0000","last_peered":"2023-08-28T02:24:00.201258+0000","last_clean":"2023-08-28T02:24:00.201258+0000","last_became_active":"2023-08-28T02:18:39.779303+0000","last_became_peered":"2023-08-28T02:18:39.779303+0000","last_unstale":"2023-08-28T02:24:00.201258+0000","last_undegraded":"2023-08-28T02:24:00.201258+0000","last_fullsized":"2023-08-28T02:24:00.201258+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":114,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-08-28T02:24:01.204800+0000","last_change":"2023-08-28T02:23:59.221979+0000","last_active":"2023-08-28T02:24:01.204800+0000","last_peered":"2023-08-28T02:24:01.204800+0000","last_clean":"2023-08-28T02:24:01.204800+0000","last_became_active":"2023-08-28T02:23:59.221619+0000","last_became_peered":"2023-08-28T02:23:59.221619+0000","last_unstale":"2023-08-28T02:24:01.204800+0000","last_undegraded":"2023-08-28T02:24:01.204800+0000","last_fullsized":"2023-08-28T02:24:01.204800+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:54:45.892284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.6","version":"156'7693","reported_seq":7805,"reported_epoch":156,"state":"active+recovering+undersized+remapped","last_fresh":"2023-08-28T02:24:03.102968+0000","last_change":"2023-08-28T02:24:02.645232+0000","last_active":"2023-08-28T02:24:03.102968+0000","last_peered":"2023-08-28T02:24:03.102968+0000","last_clean":"2023-08-28T02:23:25.096409+0000","last_became_active":"2023-08-28T02:23:38.375307+0000","last_became_peered":"2023-08-28T02:23:38.375307+0000","last_unstale":"2023-08-28T02:24:03.102968+0000","last_undegraded":"2023-08-28T02:24:03.102968+0000","last_fullsized":"2023-08-28T02:23:38.317719+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":7693,"log_dups_size":0,"ondisk_log_size":7693,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:50:23.010897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":381616128,"num_objects":5823,"num_object_clones":0,"num_object_copies":17469,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":318,"num_objects_unfound":0,"num_objects_dirty":5823,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7693,"num_write_kb":432512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":1572864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,4,5","objects":318},{"shards":"2,5","objects":5505}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"78'47","reported_seq":347,"reported_epoch":155,"state":"active+recovery_wait+degraded","last_fresh":"2023-08-28T02:24:00.199165+0000","last_change":"2023-08-28T02:23:59.210916+0000","last_active":"2023-08-28T02:24:00.199165+0000","last_peered":"2023-08-28T02:24:00.199165+0000","last_clean":"2023-08-28T02:23:55.313361+0000","last_became_active":"2023-08-28T02:23:59.210481+0000","last_became_peered":"2023-08-28T02:23:59.210481+0000","last_unstale":"2023-08-28T02:24:00.199165+0000","last_undegraded":"2023-08-28T02:23:59.198741+0000","last_fullsized":"2023-08-28T02:24:00.199165+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":47,"log_dups_size":0,"ondisk_log_size":47,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:40:18.542375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":162,"num_read_kb":1281,"num_write":48,"num_write_kb":1153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":["2","0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":97,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-08-28T02:24:01.203656+0000","last_change":"2023-08-28T02:20:19.851591+0000","last_active":"2023-08-28T02:24:01.203656+0000","last_peered":"2023-08-28T02:24:01.203656+0000","last_clean":"2023-08-28T02:24:01.203656+0000","last_became_active":"2023-08-28T02:20:19.851291+0000","last_became_peered":"2023-08-28T02:20:19.851291+0000","last_unstale":"2023-08-28T02:24:01.203656+0000","last_undegraded":"2023-08-28T02:24:01.203656+0000","last_fullsized":"2023-08-28T02:24:01.203656+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:33:51.259707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":128,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-08-28T02:24:01.204028+0000","last_change":"2023-08-28T02:19:50.211156+0000","last_active":"2023-08-28T02:24:01.204028+0000","last_peered":"2023-08-28T02:24:01.204028+0000","last_clean":"2023-08-28T02:24:01.204028+0000","last_became_active":"2023-08-28T02:19:50.210398+0000","last_became_peered":"2023-08-28T02:19:50.210398+0000","last_unstale":"2023-08-28T02:24:01.204028+0000","last_undegraded":"2023-08-28T02:24:01.204028+0000","last_fullsized":"2023-08-28T02:24:01.204028+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:14:41.748197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"78'114","reported_seq":520,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-08-28T02:24:01.210739+0000","last_change":"2023-08-28T02:24:01.210739+0000","last_active":"2023-08-28T02:24:01.210739+0000","last_peered":"2023-08-28T02:24:01.210739+0000","last_clean":"2023-08-28T02:24:01.210739+0000","last_became_active":"2023-08-28T02:24:01.210386+0000","last_became_peered":"2023-08-28T02:24:01.210386+0000","last_unstale":"2023-08-28T02:24:01.210739+0000","last_undegraded":"2023-08-28T02:24:01.210739+0000","last_fullsized":"2023-08-28T02:24:01.210739+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":114,"log_dups_size":0,"ondisk_log_size":114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":358,"num_read_kb":1352,"num_write":102,"num_write_kb":1104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"86'139","reported_seq":326,"reported_epoch":156,"state":"active+undersized+remapped","last_fresh":"2023-08-28T02:24:02.578354+0000","last_change":"2023-08-28T02:24:02.578616+0000","last_active":"2023-08-28T02:24:02.578354+0000","last_peered":"2023-08-28T02:24:02.578354+0000","last_clean":"2023-08-28T02:23:32.292225+0000","last_became_active":"2023-08-28T02:23:39.359394+0000","last_became_peered":"2023-08-28T02:23:39.359394+0000","last_unstale":"2023-08-28T02:24:02.578354+0000","last_undegraded":"2023-08-28T02:24:02.578354+0000","last_fullsized":"2023-08-28T02:23:39.335812+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":142,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_clean_scrub_stamp":"2023-08-28T02:08:02.043718+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:52:36.356494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[0,6],"avail_no_missing":["0","1","6","7"],"object_location_counts":[{"shards":"0,6","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-08-28T02:24:01.205751+0000","last_change":"2023-08-28T02:23:59.219981+0000","last_active":"2023-08-28T02:24:01.205751+0000","last_peered":"2023-08-28T02:24:01.205751+0000","last_clean":"2023-08-28T02:24:01.205751+0000","last_became_active":"2023-08-28T02:23:59.219633+0000","last_became_peered":"2023-08-28T02:23:59.219633+0000","last_unstale":"2023-08-28T02:24:01.205751+0000","last_undegraded":"2023-08-28T02:24:01.205751+0000","last_fullsized":"2023-08-28T02:24:01.205751+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:56:14.172518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":124,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-08-28T02:24:00.202738+0000","last_change":"2023-08-28T02:23:38.341351+0000","last_active":"2023-08-28T02:24:00.202738+0000","last_peered":"2023-08-28T02:24:00.202738+0000","last_clean":"2023-08-28T02:24:00.202738+0000","last_became_active":"2023-08-28T02:23:38.340997+0000","last_became_peered":"2023-08-28T02:23:38.340997+0000","last_unstale":"2023-08-28T02:24:00.202738+0000","last_undegraded":"2023-08-28T02:24:00.202738+0000","last_fullsized":"2023-08-28T02:24:00.202738+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"156'7746","reported_seq":8559,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-08-28T02:24:01.964110+0000","last_change":"2023-08-28T02:23:47.215362+0000","last_active":"2023-08-28T02:24:01.964110+0000","last_peered":"2023-08-28T02:24:01.964110+0000","last_clean":"2023-08-28T02:24:01.964110+0000","last_became_active":"2023-08-28T02:23:47.207109+0000","last_became_peered":"2023-08-28T02:23:47.207109+0000","last_unstale":"2023-08-28T02:24:01.964110+0000","last_undegraded":"2023-08-28T02:24:01.964110+0000","last_fullsized":"2023-08-28T02:24:01.964110+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":119,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":7746,"log_dups_size":0,"ondisk_log_size":7746,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":387317760,"num_objects":5910,"num_object_clones":0,"num_object_copies":17730,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5910,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7746,"num_write_kb":436992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":370,"num_bytes_recovered":24182784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.7","version":"156'7890","reported_seq":7935,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-08-28T02:24:01.984812+0000","last_change":"2023-08-28T02:22:00.165614+0000","last_active":"2023-08-28T02:24:01.984812+0000","last_peered":"2023-08-28T02:24:01.984812+0000","last_clean":"2023-08-28T02:24:01.984812+0000","last_became_active":"2023-08-28T02:22:00.165136+0000","last_became_peered":"2023-08-28T02:22:00.165136+0000","last_unstale":"2023-08-28T02:24:01.984812+0000","last_undegraded":"2023-08-28T02:24:01.984812+0000","last_fullsized":"2023-08-28T02:24:01.984812+0000","mapping_epoch":119,"log_start":"124'100","ondisk_log_start":"124'100","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":7790,"log_dups_size":0,"ondisk_log_size":7790,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":390856704,"num_objects":5964,"num_object_clones":0,"num_object_copies":17892,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5964,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7890,"num_write_kb":443328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"78'148","reported_seq":605,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-08-28T02:24:01.211494+0000","last_change":"2023-08-28T02:24:01.211494+0000","last_active":"2023-08-28T02:24:01.211494+0000","last_peered":"2023-08-28T02:24:01.211494+0000","last_clean":"2023-08-28T02:24:01.211494+0000","last_became_active":"2023-08-28T02:24:01.210957+0000","last_became_peered":"2023-08-28T02:24:01.210957+0000","last_unstale":"2023-08-28T02:24:01.211494+0000","last_undegraded":"2023-08-28T02:24:01.211494+0000","last_fullsized":"2023-08-28T02:24:01.211494+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1092,"num_read_kb":1955,"num_write":127,"num_write_kb":5177,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":122,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-08-28T02:24:00.197569+0000","last_change":"2023-08-28T02:23:59.226986+0000","last_active":"2023-08-28T02:24:00.197569+0000","last_peered":"2023-08-28T02:24:00.197569+0000","last_clean":"2023-08-28T02:24:00.197569+0000","last_became_active":"2023-08-28T02:23:59.226529+0000","last_became_peered":"2023-08-28T02:23:59.226529+0000","last_unstale":"2023-08-28T02:24:00.197569+0000","last_undegraded":"2023-08-28T02:24:00.197569+0000","last_fullsized":"2023-08-28T02:24:00.197569+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:22:09.193619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":149,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-08-28T02:24:00.197645+0000","last_change":"2023-08-28T02:21:59.167858+0000","last_active":"2023-08-28T02:24:00.197645+0000","last_peered":"2023-08-28T02:24:00.197645+0000","last_clean":"2023-08-28T02:24:00.197645+0000","last_became_active":"2023-08-28T02:21:59.167436+0000","last_became_peered":"2023-08-28T02:21:59.167436+0000","last_unstale":"2023-08-28T02:24:00.197645+0000","last_undegraded":"2023-08-28T02:24:00.197645+0000","last_fullsized":"2023-08-28T02:24:00.197645+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:17:50.717282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"78'103","reported_seq":540,"reported_epoch":155,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:24:00.198170+0000","last_change":"2023-08-28T02:23:39.369060+0000","last_active":"2023-08-28T02:24:00.198170+0000","last_peered":"2023-08-28T02:24:00.198170+0000","last_clean":"2023-08-28T02:23:32.292540+0000","last_became_active":"2023-08-28T02:23:39.368722+0000","last_became_peered":"2023-08-28T02:23:39.368722+0000","last_unstale":"2023-08-28T02:24:00.198170+0000","last_undegraded":"2023-08-28T02:23:39.341607+0000","last_fullsized":"2023-08-28T02:23:39.341178+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":142,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:57:53.284150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":803,"num_read_kb":2080,"num_write":73,"num_write_kb":1244,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[0,4],"avail_no_missing":["0","1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":97,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-08-28T02:24:01.205025+0000","last_change":"2023-08-28T02:20:19.850916+0000","last_active":"2023-08-28T02:24:01.205025+0000","last_peered":"2023-08-28T02:24:01.205025+0000","last_clean":"2023-08-28T02:24:01.205025+0000","last_became_active":"2023-08-28T02:20:19.850627+0000","last_became_peered":"2023-08-28T02:20:19.850627+0000","last_unstale":"2023-08-28T02:24:01.205025+0000","last_undegraded":"2023-08-28T02:24:01.205025+0000","last_fullsized":"2023-08-28T02:24:01.205025+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:19:14.346189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":108,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-08-28T02:24:00.197876+0000","last_change":"2023-08-28T02:23:38.329667+0000","last_active":"2023-08-28T02:24:00.197876+0000","last_peered":"2023-08-28T02:24:00.197876+0000","last_clean":"2023-08-28T02:24:00.197876+0000","last_became_active":"2023-08-28T02:23:38.329175+0000","last_became_peered":"2023-08-28T02:23:38.329175+0000","last_unstale":"2023-08-28T02:24:00.197876+0000","last_undegraded":"2023-08-28T02:24:00.197876+0000","last_fullsized":"2023-08-28T02:24:00.197876+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:21:34.815067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"156'7696","reported_seq":7740,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-08-28T02:24:01.979052+0000","last_change":"2023-08-28T02:22:00.167814+0000","last_active":"2023-08-28T02:24:01.979052+0000","last_peered":"2023-08-28T02:24:01.979052+0000","last_clean":"2023-08-28T02:24:01.979052+0000","last_became_active":"2023-08-28T02:22:00.167513+0000","last_became_peered":"2023-08-28T02:22:00.167513+0000","last_unstale":"2023-08-28T02:24:01.979052+0000","last_undegraded":"2023-08-28T02:24:01.979052+0000","last_fullsized":"2023-08-28T02:24:01.979052+0000","mapping_epoch":119,"log_start":"124'300","ondisk_log_start":"124'300","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":7396,"log_dups_size":0,"ondisk_log_size":7396,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":384696320,"num_objects":5870,"num_object_clones":0,"num_object_copies":17610,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5870,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7696,"num_write_kb":434112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":18,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-08-28T02:24:01.206556+0000","last_change":"2023-08-28T02:23:59.232099+0000","last_active":"2023-08-28T02:24:01.206556+0000","last_peered":"2023-08-28T02:24:01.206556+0000","last_clean":"2023-08-28T02:24:01.206556+0000","last_became_active":"2023-08-28T02:23:59.219318+0000","last_became_peered":"2023-08-28T02:23:59.219318+0000","last_unstale":"2023-08-28T02:24:01.206556+0000","last_undegraded":"2023-08-28T02:24:01.206556+0000","last_fullsized":"2023-08-28T02:24:01.206556+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:59:30.093289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'112","reported_seq":400,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-08-28T02:24:01.204287+0000","last_change":"2023-08-28T02:19:50.218206+0000","last_active":"2023-08-28T02:24:01.204287+0000","last_peered":"2023-08-28T02:24:01.204287+0000","last_clean":"2023-08-28T02:24:01.204287+0000","last_became_active":"2023-08-28T02:19:50.217637+0000","last_became_peered":"2023-08-28T02:19:50.217637+0000","last_unstale":"2023-08-28T02:24:01.204287+0000","last_undegraded":"2023-08-28T02:24:01.204287+0000","last_fullsized":"2023-08-28T02:24:01.204287+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:12:06.224907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":444,"num_read_kb":473,"num_write":94,"num_write_kb":127,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":14,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-08-28T02:24:01.205873+0000","last_change":"2023-08-28T02:23:59.221034+0000","last_active":"2023-08-28T02:24:01.205873+0000","last_peered":"2023-08-28T02:24:01.205873+0000","last_clean":"2023-08-28T02:24:01.205873+0000","last_became_active":"2023-08-28T02:23:59.220730+0000","last_became_peered":"2023-08-28T02:23:59.220730+0000","last_unstale":"2023-08-28T02:24:01.205873+0000","last_undegraded":"2023-08-28T02:24:01.205873+0000","last_fullsized":"2023-08-28T02:24:01.205873+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:25:52.364370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"156'7882","reported_seq":9607,"reported_epoch":156,"state":"active+remapped+backfilling","last_fresh":"2023-08-28T02:24:02.041381+0000","last_change":"2023-08-28T02:24:00.337896+0000","last_active":"2023-08-28T02:24:02.041381+0000","last_peered":"2023-08-28T02:24:02.041381+0000","last_clean":"2023-08-28T02:23:25.043301+0000","last_became_active":"2023-08-28T02:24:00.316639+0000","last_became_peered":"2023-08-28T02:24:00.316639+0000","last_unstale":"2023-08-28T02:24:02.041381+0000","last_undegraded":"2023-08-28T02:24:02.041381+0000","last_fullsized":"2023-08-28T02:24:02.041381+0000","mapping_epoch":154,"log_start":"124'600","ondisk_log_start":"124'600","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":7282,"log_dups_size":0,"ondisk_log_size":7282,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393740288,"num_objects":6008,"num_object_clones":0,"num_object_copies":18024,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5826,"num_objects_unfound":0,"num_objects_dirty":6008,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7882,"num_write_kb":444480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":822,"num_bytes_recovered":53739520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,7,0],"avail_no_missing":["4","0","5","7"],"object_location_counts":[{"shards":"0,4,7","objects":6008}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"78'122","reported_seq":656,"reported_epoch":155,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:24:00.202469+0000","last_change":"2023-08-28T02:23:39.366764+0000","last_active":"2023-08-28T02:24:00.202469+0000","last_peered":"2023-08-28T02:24:00.202469+0000","last_clean":"2023-08-28T02:23:32.323165+0000","last_became_active":"2023-08-28T02:23:39.366382+0000","last_became_peered":"2023-08-28T02:23:39.366382+0000","last_unstale":"2023-08-28T02:24:00.202469+0000","last_undegraded":"2023-08-28T02:23:39.349483+0000","last_fullsized":"2023-08-28T02:23:39.348937+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":142,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":857,"num_read_kb":2224,"num_write":99,"num_write_kb":2312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,2],"avail_no_missing":["6","1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":108,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-08-28T02:24:00.198012+0000","last_change":"2023-08-28T02:19:50.221001+0000","last_active":"2023-08-28T02:24:00.198012+0000","last_peered":"2023-08-28T02:24:00.198012+0000","last_clean":"2023-08-28T02:24:00.198012+0000","last_became_active":"2023-08-28T02:19:50.220611+0000","last_became_peered":"2023-08-28T02:19:50.220611+0000","last_unstale":"2023-08-28T02:24:00.198012+0000","last_undegraded":"2023-08-28T02:24:00.198012+0000","last_fullsized":"2023-08-28T02:24:00.198012+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:23:10.637630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":107,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-08-28T02:24:00.218332+0000","last_change":"2023-08-28T02:11:22.437146+0000","last_active":"2023-08-28T02:24:00.218332+0000","last_peered":"2023-08-28T02:24:00.218332+0000","last_clean":"2023-08-28T02:24:00.218332+0000","last_became_active":"2023-08-28T02:11:22.436798+0000","last_became_peered":"2023-08-28T02:11:22.436798+0000","last_unstale":"2023-08-28T02:24:00.218332+0000","last_undegraded":"2023-08-28T02:24:00.218332+0000","last_fullsized":"2023-08-28T02:24:00.218332+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"156'7888","reported_seq":7940,"reported_epoch":156,"state":"active+remapped+backfill_wait","last_fresh":"2023-08-28T02:24:03.053250+0000","last_change":"2023-08-28T02:24:00.310494+0000","last_active":"2023-08-28T02:24:03.053250+0000","last_peered":"2023-08-28T02:24:03.053250+0000","last_clean":"2023-08-28T02:23:58.174026+0000","last_became_active":"2023-08-28T02:24:00.290543+0000","last_became_peered":"2023-08-28T02:24:00.290543+0000","last_unstale":"2023-08-28T02:24:03.053250+0000","last_undegraded":"2023-08-28T02:24:03.053250+0000","last_fullsized":"2023-08-28T02:24:03.053250+0000","mapping_epoch":154,"log_start":"125'1377","ondisk_log_start":"125'1377","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":6511,"log_dups_size":0,"ondisk_log_size":6511,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:35:56.609960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":384303104,"num_objects":5864,"num_object_clones":0,"num_object_copies":17592,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5864,"num_objects_unfound":0,"num_objects_dirty":5864,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7888,"num_write_kb":440064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[0,1,4],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":5864}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":111,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-08-28T02:24:00.197788+0000","last_change":"2023-08-28T02:23:38.329474+0000","last_active":"2023-08-28T02:24:00.197788+0000","last_peered":"2023-08-28T02:24:00.197788+0000","last_clean":"2023-08-28T02:24:00.197788+0000","last_became_active":"2023-08-28T02:23:38.328505+0000","last_became_peered":"2023-08-28T02:23:38.328505+0000","last_unstale":"2023-08-28T02:24:00.197788+0000","last_undegraded":"2023-08-28T02:24:00.197788+0000","last_fullsized":"2023-08-28T02:24:00.197788+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:12:14.835809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":176,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-08-28T02:24:00.200137+0000","last_change":"2023-08-28T02:23:59.212036+0000","last_active":"2023-08-28T02:24:00.200137+0000","last_peered":"2023-08-28T02:24:00.200137+0000","last_clean":"2023-08-28T02:24:00.200137+0000","last_became_active":"2023-08-28T02:23:59.211709+0000","last_became_peered":"2023-08-28T02:23:59.211709+0000","last_unstale":"2023-08-28T02:24:00.200137+0000","last_undegraded":"2023-08-28T02:24:00.200137+0000","last_fullsized":"2023-08-28T02:24:00.200137+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T14:04:25.046626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":125,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-08-28T02:24:00.205372+0000","last_change":"2023-08-28T02:19:50.218760+0000","last_active":"2023-08-28T02:24:00.205372+0000","last_peered":"2023-08-28T02:24:00.205372+0000","last_clean":"2023-08-28T02:24:00.205372+0000","last_became_active":"2023-08-28T02:19:50.218376+0000","last_became_peered":"2023-08-28T02:19:50.218376+0000","last_unstale":"2023-08-28T02:24:00.205372+0000","last_undegraded":"2023-08-28T02:24:00.205372+0000","last_fullsized":"2023-08-28T02:24:00.205372+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":155,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-08-28T02:24:00.206554+0000","last_change":"2023-08-28T02:23:59.207822+0000","last_active":"2023-08-28T02:24:00.206554+0000","last_peered":"2023-08-28T02:24:00.206554+0000","last_clean":"2023-08-28T02:24:00.206554+0000","last_became_active":"2023-08-28T02:23:59.207475+0000","last_became_peered":"2023-08-28T02:23:59.207475+0000","last_unstale":"2023-08-28T02:24:00.206554+0000","last_undegraded":"2023-08-28T02:24:00.206554+0000","last_fullsized":"2023-08-28T02:24:00.206554+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":110,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-08-28T02:24:00.205443+0000","last_change":"2023-08-28T02:11:22.186024+0000","last_active":"2023-08-28T02:24:00.205443+0000","last_peered":"2023-08-28T02:24:00.205443+0000","last_clean":"2023-08-28T02:24:00.205443+0000","last_became_active":"2023-08-28T02:11:22.185500+0000","last_became_peered":"2023-08-28T02:11:22.185500+0000","last_unstale":"2023-08-28T02:24:00.205443+0000","last_undegraded":"2023-08-28T02:24:00.205443+0000","last_fullsized":"2023-08-28T02:24:00.205443+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":155,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-08-28T02:24:00.206636+0000","last_change":"2023-08-28T02:23:59.207929+0000","last_active":"2023-08-28T02:24:00.206636+0000","last_peered":"2023-08-28T02:24:00.206636+0000","last_clean":"2023-08-28T02:24:00.206636+0000","last_became_active":"2023-08-28T02:23:59.207589+0000","last_became_peered":"2023-08-28T02:23:59.207589+0000","last_unstale":"2023-08-28T02:24:00.206636+0000","last_undegraded":"2023-08-28T02:24:00.206636+0000","last_fullsized":"2023-08-28T02:24:00.206636+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":14,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-08-28T02:24:01.206353+0000","last_change":"2023-08-28T02:23:59.220437+0000","last_active":"2023-08-28T02:24:01.206353+0000","last_peered":"2023-08-28T02:24:01.206353+0000","last_clean":"2023-08-28T02:24:01.206353+0000","last_became_active":"2023-08-28T02:23:59.220091+0000","last_became_peered":"2023-08-28T02:23:59.220091+0000","last_unstale":"2023-08-28T02:24:01.206353+0000","last_undegraded":"2023-08-28T02:24:01.206353+0000","last_fullsized":"2023-08-28T02:24:01.206353+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:51:21.695566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":14,"reported_epoch":156,"state":"active+clean","last_fresh":"2023-08-28T02:24:01.205853+0000","last_change":"2023-08-28T02:23:59.217794+0000","last_active":"2023-08-28T02:24:01.205853+0000","last_peered":"2023-08-28T02:24:01.205853+0000","last_clean":"2023-08-28T02:24:01.205853+0000","last_became_active":"2023-08-28T02:23:59.217231+0000","last_became_peered":"2023-08-28T02:23:59.217231+0000","last_unstale":"2023-08-28T02:24:01.205853+0000","last_undegraded":"2023-08-28T02:24:01.205853+0000","last_fullsized":"2023-08-28T02:24:01.205853+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:36:26.297410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":108,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-08-28T02:24:00.197900+0000","last_change":"2023-08-28T02:20:19.850252+0000","last_active":"2023-08-28T02:24:00.197900+0000","last_peered":"2023-08-28T02:24:00.197900+0000","last_clean":"2023-08-28T02:24:00.197900+0000","last_became_active":"2023-08-28T02:20:19.849913+0000","last_became_peered":"2023-08-28T02:20:19.849913+0000","last_unstale":"2023-08-28T02:24:00.197900+0000","last_undegraded":"2023-08-28T02:24:00.197900+0000","last_fullsized":"2023-08-28T02:24:00.197900+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:48:47.713465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":151,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-08-28T02:24:00.200915+0000","last_change":"2023-08-28T02:21:59.180201+0000","last_active":"2023-08-28T02:24:00.200915+0000","last_peered":"2023-08-28T02:24:00.200915+0000","last_clean":"2023-08-28T02:24:00.200915+0000","last_became_active":"2023-08-28T02:21:59.179863+0000","last_became_peered":"2023-08-28T02:21:59.179863+0000","last_unstale":"2023-08-28T02:24:00.200915+0000","last_undegraded":"2023-08-28T02:24:00.200915+0000","last_fullsized":"2023-08-28T02:24:00.200915+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":6249971736,"num_objects":95368,"num_object_clones":0,"num_object_copies":286104,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6939,"num_objects_misplaced":12008,"num_objects_unfound":0,"num_objects_dirty":95368,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":124914,"num_write_kb":7048961,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1930,"num_bytes_recovered":126091288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18906427392,"data_stored":18906415176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":118459,"ondisk_log_size":118459,"up":48,"acting":44,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":9774,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":122880,"data_stored":21460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26810,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":2,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":26,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":51,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430381,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2191944,"kb_used_data":1890624,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":91553208,"statfs":{"total":95995035648,"available":93750484992,"internally_reserved":0,"allocated":1935998976,"data_stored":1935426867,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:23:54 2023","interfaces":[{"interface":"back","average":{"1min":7.2480000000000002,"5min":1.881,"15min":0.98499999999999999},"min":{"1min":0.25700000000000001,"5min":0.255,"15min":0.255},"max":{"1min":116.755,"5min":116.755,"15min":116.755},"last":0.30199999999999999},{"interface":"front","average":{"1min":8.5180000000000007,"5min":2.2400000000000002,"15min":1.155},"min":{"1min":0.33100000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":116.971,"5min":116.971,"15min":116.971},"last":0.57999999999999996}]},{"osd":1,"last update":"Mon Aug 28 02:23:58 2023","interfaces":[{"interface":"back","average":{"1min":1.0600000000000001,"5min":0.75900000000000001,"15min":0.60499999999999998},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":4.9489999999999998,"5min":4.9489999999999998,"15min":4.9489999999999998},"last":0.41899999999999998},{"interface":"front","average":{"1min":2.3500000000000001,"5min":1.0269999999999999,"15min":0.73399999999999999},"min":{"1min":0.36799999999999999,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":7.7210000000000001,"5min":7.7210000000000001,"15min":7.7210000000000001},"last":3.9420000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:23:25 2023","interfaces":[{"interface":"back","average":{"1min":1.46,"5min":0.78200000000000003,"15min":0.64400000000000002},"min":{"1min":0.37,"5min":0.309,"15min":0.309},"max":{"1min":4.3940000000000001,"5min":4.3940000000000001,"15min":4.3940000000000001},"last":4.4500000000000002},{"interface":"front","average":{"1min":2.0600000000000001,"5min":0.92100000000000004,"15min":0.69699999999999995},"min":{"1min":0.40200000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":6.548,"5min":6.548,"15min":6.548},"last":4.4619999999999997}]},{"osd":3,"last update":"Mon Aug 28 02:23:58 2023","interfaces":[{"interface":"back","average":{"1min":1,"5min":0.82199999999999995,"15min":0.71999999999999997},"min":{"1min":0.30599999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":5.1390000000000002,"5min":5.1390000000000002,"15min":5.1390000000000002},"last":0.52900000000000003},{"interface":"front","average":{"1min":2.4079999999999999,"5min":1.089,"15min":0.80900000000000005},"min":{"1min":0.371,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":7.734,"5min":7.734,"15min":7.734},"last":2.7349999999999999}]},{"osd":4,"last update":"Mon Aug 28 02:23:25 2023","interfaces":[{"interface":"back","average":{"1min":1.5980000000000001,"5min":0.747,"15min":0.56200000000000006},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.25800000000000001},"max":{"1min":6.5819999999999999,"5min":6.5819999999999999,"15min":6.5819999999999999},"last":0.34499999999999997},{"interface":"front","average":{"1min":1.7290000000000001,"5min":0.78700000000000003,"15min":0.59299999999999997},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.27100000000000002},"max":{"1min":8.6080000000000005,"5min":8.6080000000000005,"15min":8.6080000000000005},"last":0.316}]},{"osd":5,"last update":"Mon Aug 28 02:23:25 2023","interfaces":[{"interface":"back","average":{"1min":0.71899999999999997,"5min":0.58299999999999996,"15min":0.51200000000000001},"min":{"1min":0.33000000000000002,"5min":0.30299999999999999,"15min":0.28699999999999998},"max":{"1min":2.9529999999999998,"5min":2.9529999999999998,"15min":2.9529999999999998},"last":0.57099999999999995},{"interface":"front","average":{"1min":1.3859999999999999,"5min":0.82099999999999995,"15min":0.628},"min":{"1min":0.38700000000000001,"5min":0.32200000000000001,"15min":0.314},"max":{"1min":6.3899999999999997,"5min":6.3899999999999997,"15min":6.3899999999999997},"last":0.47899999999999998}]},{"osd":6,"last update":"Mon Aug 28 02:23:25 2023","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":0.65800000000000003,"15min":0.56599999999999995},"min":{"1min":0.35699999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.407,"5min":3.407,"15min":3.407},"last":0.38400000000000001},{"interface":"front","average":{"1min":1.49,"5min":0.82899999999999996,"15min":0.65700000000000003},"min":{"1min":0.36499999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":6.4749999999999996,"5min":6.4749999999999996,"15min":6.4749999999999996},"last":0.503}]}]},{"osd":6,"up_from":42,"seq":180388626608,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1841480,"kb_used_data":1531968,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":91903672,"statfs":{"total":95995035648,"available":94109360128,"internally_reserved":0,"allocated":1568735232,"data_stored":1568163349,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:23:59 2023","interfaces":[{"interface":"back","average":{"1min":4.2919999999999998,"5min":1.502,"15min":0.88400000000000001},"min":{"1min":0.23100000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":48.848999999999997,"5min":48.848999999999997,"15min":48.848999999999997},"last":0.36499999999999999},{"interface":"front","average":{"1min":5.0389999999999997,"5min":1.5129999999999999,"15min":0.90600000000000003},"min":{"1min":0.32800000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":48.771000000000001,"5min":48.771000000000001,"15min":48.771000000000001},"last":0.42699999999999999}]},{"osd":1,"last update":"Mon Aug 28 02:23:59 2023","interfaces":[{"interface":"back","average":{"1min":2.085,"5min":1.5329999999999999,"15min":0.86899999999999999},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":10.945,"5min":34.677999999999997,"15min":34.677999999999997},"last":0.29299999999999998},{"interface":"front","average":{"1min":2.657,"5min":1.679,"15min":0.93899999999999995},"min":{"1min":0.315,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":8.1240000000000006,"5min":34.348999999999997,"15min":34.348999999999997},"last":0.315}]},{"osd":2,"last update":"Mon Aug 28 02:23:29 2023","interfaces":[{"interface":"back","average":{"1min":1.5880000000000001,"5min":0.79600000000000004,"15min":0.67200000000000004},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":4.3650000000000002,"5min":4.3650000000000002,"15min":4.3650000000000002},"last":0.39200000000000002},{"interface":"front","average":{"1min":2.2080000000000002,"5min":0.89100000000000001,"15min":0.68100000000000005},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":7.2649999999999997,"5min":7.2649999999999997,"15min":7.2649999999999997},"last":0.46999999999999997}]},{"osd":3,"last update":"Mon Aug 28 02:23:59 2023","interfaces":[{"interface":"back","average":{"1min":2.169,"5min":1.0349999999999999,"15min":0.79200000000000004},"min":{"1min":0.41999999999999998,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":7.2089999999999996,"5min":7.2089999999999996,"15min":7.2089999999999996},"last":0.48099999999999998},{"interface":"front","average":{"1min":2.4980000000000002,"5min":1.218,"15min":0.86099999999999999},"min":{"1min":0.47599999999999998,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":7.1130000000000004,"5min":7.9610000000000003,"15min":7.9610000000000003},"last":0.53300000000000003}]},{"osd":4,"last update":"Mon Aug 28 02:23:01 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":0.59499999999999997,"15min":0.496},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":4.3540000000000001,"5min":4.3540000000000001,"15min":4.3540000000000001},"last":0.44600000000000001},{"interface":"front","average":{"1min":1.093,"5min":0.65100000000000002,"15min":0.53100000000000003},"min":{"1min":0.32200000000000001,"5min":0.318,"15min":0.28899999999999998},"max":{"1min":6.1420000000000003,"5min":6.1420000000000003,"15min":6.1420000000000003},"last":0.5}]},{"osd":5,"last update":"Mon Aug 28 02:23:01 2023","interfaces":[{"interface":"back","average":{"1min":1.865,"5min":0.77700000000000002,"15min":0.56000000000000005},"min":{"1min":0.30099999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":8.8330000000000002,"5min":8.8330000000000002,"15min":8.8330000000000002},"last":0.58999999999999997},{"interface":"front","average":{"1min":1.2949999999999999,"5min":0.71299999999999997,"15min":0.57599999999999996},"min":{"1min":0.34300000000000003,"5min":0.32500000000000001,"15min":0.28799999999999998},"max":{"1min":7.3780000000000001,"5min":7.3780000000000001,"15min":7.3780000000000001},"last":0.41199999999999998}]},{"osd":7,"last update":"Mon Aug 28 02:23:24 2023","interfaces":[{"interface":"back","average":{"1min":1.6220000000000001,"5min":0.81499999999999995,"15min":0.63},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.32700000000000001},"max":{"1min":7.1769999999999996,"5min":7.1769999999999996,"15min":7.1769999999999996},"last":0.50900000000000001},{"interface":"front","average":{"1min":1.494,"5min":0.86399999999999999,"15min":0.66300000000000003},"min":{"1min":0.40300000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.0860000000000003,"5min":6.0860000000000003,"15min":6.0860000000000003},"last":0.56699999999999995}]}]},{"osd":1,"up_from":98,"seq":420906795061,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2011896,"kb_used_data":1935920,"kb_used_omap":7,"kb_used_meta":75960,"kb_avail":91733256,"statfs":{"total":95995035648,"available":93934854144,"internally_reserved":0,"allocated":1982382080,"data_stored":1981604718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":77783371},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:23:57 2023","interfaces":[{"interface":"back","average":{"1min":2.4940000000000002,"5min":0.98899999999999999,"15min":0.55300000000000005},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":7.8959999999999999,"5min":7.8959999999999999,"15min":7.8959999999999999},"last":1.194},{"interface":"front","average":{"1min":1.9970000000000001,"5min":1.0649999999999999,"15min":0.59099999999999997},"min":{"1min":0.23499999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":5.8579999999999997,"5min":12.574999999999999,"15min":12.574999999999999},"last":0.22700000000000001}]},{"osd":2,"last update":"Mon Aug 28 02:23:25 2023","interfaces":[{"interface":"back","average":{"1min":2.4569999999999999,"5min":0.83799999999999997,"15min":0.50700000000000001},"min":{"1min":0.26600000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":12.635999999999999,"5min":12.635999999999999,"15min":12.635999999999999},"last":1.3580000000000001},{"interface":"front","average":{"1min":2.2789999999999999,"5min":0.84799999999999998,"15min":0.53600000000000003},"min":{"1min":0.23200000000000001,"5min":0.221,"15min":0.221},"max":{"1min":9.6669999999999998,"5min":9.6669999999999998,"15min":9.6669999999999998},"last":0.55000000000000004}]},{"osd":3,"last update":"Mon Aug 28 02:23:57 2023","interfaces":[{"interface":"back","average":{"1min":1.5660000000000001,"5min":0.86099999999999999,"15min":0.53000000000000003},"min":{"1min":0.29499999999999998,"5min":0.254,"15min":0.254},"max":{"1min":6.2359999999999998,"5min":12.544,"15min":12.544},"last":0.45300000000000001},{"interface":"front","average":{"1min":1.716,"5min":0.94299999999999995,"15min":0.621},"min":{"1min":0.314,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.8979999999999997,"5min":8.9160000000000004,"15min":8.9160000000000004},"last":1.5860000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:23:57 2023","interfaces":[{"interface":"back","average":{"1min":1.6339999999999999,"5min":0.93999999999999995,"15min":0.73099999999999998},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":5.883,"5min":5.883,"15min":5.883},"last":7.851},{"interface":"front","average":{"1min":1.571,"5min":0.86299999999999999,"15min":0.70099999999999996},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":6.3179999999999996,"5min":6.3179999999999996,"15min":6.3179999999999996},"last":0.39600000000000002}]},{"osd":5,"last update":"Mon Aug 28 02:23:57 2023","interfaces":[{"interface":"back","average":{"1min":1.8080000000000001,"5min":1.087,"15min":0.78700000000000003},"min":{"1min":0.40400000000000003,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":3.6030000000000002,"5min":9.4469999999999992,"15min":9.4469999999999992},"last":0.48299999999999998},{"interface":"front","average":{"1min":1.579,"5min":0.95099999999999996,"15min":0.74399999999999999},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":3.8690000000000002,"5min":4.6639999999999997,"15min":4.6639999999999997},"last":1.3939999999999999}]},{"osd":6,"last update":"Mon Aug 28 02:23:57 2023","interfaces":[{"interface":"back","average":{"1min":2.3090000000000002,"5min":1.339,"15min":0.85899999999999999},"min":{"1min":0.44500000000000001,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":10.618,"5min":14.827,"15min":14.827},"last":0.50700000000000001},{"interface":"front","average":{"1min":3.234,"5min":1.262,"15min":0.82999999999999996},"min":{"1min":0.42199999999999999,"5min":0.371,"15min":0.371},"max":{"1min":13.707000000000001,"5min":13.707000000000001,"15min":13.707000000000001},"last":1.4179999999999999}]},{"osd":7,"last update":"Mon Aug 28 02:23:57 2023","interfaces":[{"interface":"back","average":{"1min":2.0299999999999998,"5min":0.96599999999999997,"15min":0.74399999999999999},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":6.234,"5min":6.234,"15min":6.234},"last":4.5919999999999996},{"interface":"front","average":{"1min":1.948,"5min":1.117,"15min":0.79700000000000004},"min":{"1min":0.39900000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":6.0789999999999997,"5min":9.5039999999999996,"15min":9.5039999999999996},"last":0.53000000000000003}]}]},{"osd":0,"up_from":90,"seq":386547056707,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4394808,"kb_used_data":4258928,"kb_used_omap":8,"kb_used_meta":135863,"kb_avail":89350344,"statfs":{"total":95995035648,"available":91494752256,"internally_reserved":0,"allocated":4361142272,"data_stored":4360363358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":139124073},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,3,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Aug 28 02:24:01 2023","interfaces":[{"interface":"back","average":{"1min":6.7729999999999997,"5min":1.931,"15min":0.879},"min":{"1min":0.23400000000000001,"5min":0.186,"15min":0.186},"max":{"1min":40.313000000000002,"5min":40.313000000000002,"15min":40.313000000000002},"last":7.2009999999999996},{"interface":"front","average":{"1min":4.149,"5min":1.238,"15min":0.66000000000000003},"min":{"1min":0.17699999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":36.655000000000001,"5min":36.655000000000001,"15min":36.655000000000001},"last":6.359}]},{"osd":2,"last update":"Mon Aug 28 02:23:25 2023","interfaces":[{"interface":"back","average":{"1min":1.665,"5min":0.70599999999999996,"15min":0.46999999999999997},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":6.5449999999999999,"5min":6.5449999999999999,"15min":6.5449999999999999},"last":1.6890000000000001},{"interface":"front","average":{"1min":1.865,"5min":0.82199999999999995,"15min":0.53500000000000003},"min":{"1min":0.30399999999999999,"5min":0.253,"15min":0.253},"max":{"1min":6.1029999999999998,"5min":6.1029999999999998,"15min":6.1029999999999998},"last":6.327}]},{"osd":3,"last update":"Mon Aug 28 02:24:01 2023","interfaces":[{"interface":"back","average":{"1min":1.9890000000000001,"5min":0.91500000000000004,"15min":0.54700000000000004},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":4.5229999999999997,"5min":10.443,"15min":10.443},"last":3.0680000000000001},{"interface":"front","average":{"1min":2.169,"5min":0.97099999999999997,"15min":0.58599999999999997},"min":{"1min":0.29299999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":7.516,"5min":10.548999999999999,"15min":10.548999999999999},"last":3.0430000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:23:56 2023","interfaces":[{"interface":"back","average":{"1min":2.593,"5min":1.141,"15min":0.82699999999999996},"min":{"1min":0.35099999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":9.6649999999999991,"5min":9.6649999999999991,"15min":9.6649999999999991},"last":1.2130000000000001},{"interface":"front","average":{"1min":2.2330000000000001,"5min":1.167,"15min":0.85999999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":9.6470000000000002,"5min":10.106999999999999,"15min":10.106999999999999},"last":2.9369999999999998}]},{"osd":5,"last update":"Mon Aug 28 02:23:56 2023","interfaces":[{"interface":"back","average":{"1min":4.2030000000000003,"5min":1.55,"15min":0.95199999999999996},"min":{"1min":0.35799999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":44.000999999999998,"5min":44.000999999999998,"15min":44.000999999999998},"last":3.0129999999999999},{"interface":"front","average":{"1min":4.4089999999999998,"5min":1.5549999999999999,"15min":0.95299999999999996},"min":{"1min":0.33100000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":43.953000000000003,"5min":43.953000000000003,"15min":43.953000000000003},"last":1.1870000000000001}]},{"osd":6,"last update":"Mon Aug 28 02:23:56 2023","interfaces":[{"interface":"back","average":{"1min":3.6869999999999998,"5min":1.476,"15min":0.92900000000000005},"min":{"1min":0.374,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":32.417999999999999,"5min":32.417999999999999,"15min":32.417999999999999},"last":2.9809999999999999},{"interface":"front","average":{"1min":4.2389999999999999,"5min":1.482,"15min":0.96599999999999997},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":32.130000000000003,"5min":32.130000000000003,"15min":32.130000000000003},"last":1.2310000000000001}]},{"osd":7,"last update":"Mon Aug 28 02:23:56 2023","interfaces":[{"interface":"back","average":{"1min":2.6480000000000001,"5min":1.21,"15min":0.87},"min":{"1min":0.41499999999999998,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":9.6020000000000003,"5min":9.6020000000000003,"15min":9.6020000000000003},"last":1.143},{"interface":"front","average":{"1min":2.6110000000000002,"5min":1.0880000000000001,"15min":0.83199999999999996},"min":{"1min":0.42799999999999999,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":6.3079999999999998,"5min":6.3079999999999998,"15min":6.3079999999999998},"last":6.1980000000000004}]}]},{"osd":2,"up_from":104,"seq":446676598831,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2360636,"kb_used_data":2292916,"kb_used_omap":10,"kb_used_meta":67701,"kb_avail":91384516,"statfs":{"total":95995035648,"available":93577744384,"internally_reserved":0,"allocated":2347945984,"data_stored":2347221943,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10792,"internal_metadata":69326296},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,0,0,6],"upper_bound":4096},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:23:27 2023","interfaces":[{"interface":"back","average":{"1min":4.5839999999999996,"5min":1.2450000000000001,"15min":0.628},"min":{"1min":0.17599999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":36.890000000000001,"5min":36.890000000000001,"15min":36.890000000000001},"last":0.245},{"interface":"front","average":{"1min":3.79,"5min":1.139,"15min":0.622},"min":{"1min":0.28100000000000003,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":35.289000000000001,"5min":35.289000000000001,"15min":35.289000000000001},"last":0.22}]},{"osd":1,"last update":"Mon Aug 28 02:23:27 2023","interfaces":[{"interface":"back","average":{"1min":1.369,"5min":0.96399999999999997,"15min":0.56000000000000005},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":3.996,"5min":30.888000000000002,"15min":30.888000000000002},"last":0.28999999999999998},{"interface":"front","average":{"1min":1.5589999999999999,"5min":0.95499999999999996,"15min":0.54100000000000004},"min":{"1min":0.22700000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":5.3579999999999997,"5min":31.648,"15min":31.648},"last":0.379}]},{"osd":3,"last update":"Mon Aug 28 02:23:54 2023","interfaces":[{"interface":"back","average":{"1min":1.623,"5min":0.81399999999999995,"15min":0.54300000000000004},"min":{"1min":0.30299999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":6.0629999999999997,"5min":6.0629999999999997,"15min":6.0629999999999997},"last":0.51500000000000001},{"interface":"front","average":{"1min":1.889,"5min":0.75,"15min":0.50800000000000001},"min":{"1min":0.34100000000000003,"5min":0.246,"15min":0.246},"max":{"1min":7.9119999999999999,"5min":7.9119999999999999,"15min":7.9119999999999999},"last":0.52600000000000002}]},{"osd":4,"last update":"Mon Aug 28 02:23:27 2023","interfaces":[{"interface":"back","average":{"1min":1.427,"5min":0.80000000000000004,"15min":0.66200000000000003},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.3970000000000002,"5min":5.3970000000000002,"15min":5.3970000000000002},"last":0.42699999999999999},{"interface":"front","average":{"1min":1.6180000000000001,"5min":0.92700000000000005,"15min":0.74199999999999999},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":5.7080000000000002,"5min":5.7080000000000002,"15min":5.7080000000000002},"last":0.41499999999999998}]},{"osd":5,"last update":"Mon Aug 28 02:23:27 2023","interfaces":[{"interface":"back","average":{"1min":1.395,"5min":0.83699999999999997,"15min":0.68999999999999995},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":4.2300000000000004,"5min":5.0880000000000001,"15min":5.0880000000000001},"last":0.39800000000000002},{"interface":"front","average":{"1min":2.4489999999999998,"5min":0.96499999999999997,"15min":0.69399999999999995},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":6.6619999999999999,"5min":6.6619999999999999,"15min":6.6619999999999999},"last":0.46000000000000002}]},{"osd":6,"last update":"Mon Aug 28 02:23:27 2023","interfaces":[{"interface":"back","average":{"1min":2.1120000000000001,"5min":0.97599999999999998,"15min":0.72599999999999998},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":6.9139999999999997,"5min":6.9139999999999997,"15min":6.9139999999999997},"last":0.35999999999999999},{"interface":"front","average":{"1min":1.7569999999999999,"5min":0.93100000000000005,"15min":0.72099999999999997},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":5.0090000000000003,"5min":5.1559999999999997,"15min":5.1559999999999997},"last":0.47499999999999998}]},{"osd":7,"last update":"Mon Aug 28 02:23:27 2023","interfaces":[{"interface":"back","average":{"1min":2.0670000000000002,"5min":0.97599999999999998,"15min":0.72499999999999998},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":9.0909999999999993,"5min":9.0909999999999993,"15min":9.0909999999999993},"last":0.32100000000000001},{"interface":"front","average":{"1min":2.052,"5min":0.99199999999999999,"15min":0.73399999999999999},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":5.8120000000000003,"5min":5.8120000000000003,"15min":5.8120000000000003},"last":0.44}]}]},{"osd":3,"up_from":112,"seq":481036337193,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":50460,"kb_used_data":19092,"kb_used_omap":3,"kb_used_meta":31356,"kb_avail":93694692,"statfs":{"total":95995035648,"available":95943364608,"internally_reserved":0,"allocated":19550208,"data_stored":18850323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":32109474},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:23:56 2023","interfaces":[{"interface":"back","average":{"1min":6.8209999999999997,"5min":1.7909999999999999,"15min":0.82999999999999996},"min":{"1min":0.222,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":53.173000000000002,"5min":53.173000000000002,"15min":53.173000000000002},"last":0.28899999999999998},{"interface":"front","average":{"1min":7.4210000000000003,"5min":1.8959999999999999,"15min":0.85399999999999998},"min":{"1min":0.23100000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":52.753,"5min":52.753,"15min":52.753},"last":3.6240000000000001}]},{"osd":1,"last update":"Mon Aug 28 02:23:56 2023","interfaces":[{"interface":"back","average":{"1min":13.359999999999999,"5min":3.0390000000000001,"15min":1.2390000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":117.34699999999999,"5min":117.34699999999999,"15min":117.34699999999999},"last":0.39300000000000002},{"interface":"front","average":{"1min":13.702,"5min":3.1589999999999998,"15min":1.306},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":118.012,"5min":118.012,"15min":118.012},"last":0.34300000000000003}]},{"osd":2,"last update":"Mon Aug 28 02:23:56 2023","interfaces":[{"interface":"back","average":{"1min":12.25,"5min":2.8849999999999998,"15min":1.2110000000000001},"min":{"1min":0.255,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":102.563,"5min":102.563,"15min":102.563},"last":0.51800000000000002},{"interface":"front","average":{"1min":12.27,"5min":2.847,"15min":1.2},"min":{"1min":0.41299999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":102.57899999999999,"5min":102.57899999999999,"15min":102.57899999999999},"last":3.6619999999999999}]},{"osd":4,"last update":"Mon Aug 28 02:23:56 2023","interfaces":[{"interface":"back","average":{"1min":2.101,"5min":1.002,"15min":0.753},"min":{"1min":0.29499999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":7.423,"5min":7.423,"15min":7.423},"last":5.9740000000000002},{"interface":"front","average":{"1min":1.748,"5min":0.93600000000000005,"15min":0.73599999999999999},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":8.4039999999999999,"5min":8.4039999999999999,"15min":8.4039999999999999},"last":0.432}]},{"osd":5,"last update":"Mon Aug 28 02:23:56 2023","interfaces":[{"interface":"back","average":{"1min":1.4259999999999999,"5min":0.84399999999999997,"15min":0.70599999999999996},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":7.3369999999999997,"5min":7.3369999999999997,"15min":7.3369999999999997},"last":1.448},{"interface":"front","average":{"1min":1.9890000000000001,"5min":0.98499999999999999,"15min":0.748},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":9.3849999999999998,"5min":9.3849999999999998,"15min":9.3849999999999998},"last":1.4650000000000001}]},{"osd":6,"last update":"Mon Aug 28 02:23:56 2023","interfaces":[{"interface":"back","average":{"1min":5.4980000000000002,"5min":1.7130000000000001,"15min":0.995},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":72.116,"5min":72.116,"15min":72.116},"last":0.496},{"interface":"front","average":{"1min":5.4240000000000004,"5min":1.6319999999999999,"15min":0.94199999999999995},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":72.030000000000001,"5min":72.030000000000001,"15min":72.030000000000001},"last":3.6899999999999999}]},{"osd":7,"last update":"Mon Aug 28 02:23:56 2023","interfaces":[{"interface":"back","average":{"1min":1.722,"5min":0.98099999999999998,"15min":0.76500000000000001},"min":{"1min":0.39400000000000002,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":7.3129999999999997,"5min":7.3129999999999997,"15min":7.3129999999999997},"last":6.1669999999999998},{"interface":"front","average":{"1min":1.5720000000000001,"5min":0.95099999999999996,"15min":0.747},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":7.2960000000000003,"5min":7.2960000000000003,"15min":7.2960000000000003},"last":1.8799999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019066,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3427316,"kb_used_data":3121900,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":90317836,"statfs":{"total":95995035648,"available":92485464064,"internally_reserved":0,"allocated":3196825600,"data_stored":3196265658,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,0,0,0,0,2],"upper_bound":512},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:23:51 2023","interfaces":[{"interface":"back","average":{"1min":3.827,"5min":1.2509999999999999,"15min":0.82799999999999996},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":40.308,"5min":40.308,"15min":40.308},"last":0.24099999999999999},{"interface":"front","average":{"1min":2.9769999999999999,"5min":1.127,"15min":0.77500000000000002},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":40.332999999999998,"5min":40.332999999999998,"15min":40.332999999999998},"last":0.318}]},{"osd":1,"last update":"Mon Aug 28 02:23:56 2023","interfaces":[{"interface":"back","average":{"1min":4.0510000000000002,"5min":1.369,"15min":0.80500000000000005},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":53.622999999999998,"5min":53.622999999999998,"15min":53.622999999999998},"last":0.34200000000000003},{"interface":"front","average":{"1min":3.823,"5min":1.2929999999999999,"15min":0.80700000000000005},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":53.774000000000001,"5min":53.774000000000001,"15min":53.774000000000001},"last":0.29199999999999998}]},{"osd":2,"last update":"Mon Aug 28 02:23:24 2023","interfaces":[{"interface":"back","average":{"1min":5.524,"5min":1.5509999999999999,"15min":0.88300000000000001},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":74.099000000000004,"5min":74.099000000000004,"15min":74.099000000000004},"last":0.44600000000000001},{"interface":"front","average":{"1min":5.2480000000000002,"5min":1.502,"15min":0.85999999999999999},"min":{"1min":0.29499999999999998,"5min":0.25,"15min":0.25},"max":{"1min":74.081999999999994,"5min":74.081999999999994,"15min":74.081999999999994},"last":2.2679999999999998}]},{"osd":3,"last update":"Mon Aug 28 02:23:56 2023","interfaces":[{"interface":"back","average":{"1min":1.3140000000000001,"5min":0.77000000000000002,"15min":0.58999999999999997},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":6.6340000000000003,"5min":6.6340000000000003,"15min":6.6340000000000003},"last":0.309},{"interface":"front","average":{"1min":1.792,"5min":0.995,"15min":0.72099999999999997},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":6.8230000000000004,"5min":6.8230000000000004,"15min":6.8230000000000004},"last":0.38500000000000001}]},{"osd":5,"last update":"Mon Aug 28 02:23:37 2023","interfaces":[{"interface":"back","average":{"1min":1.478,"5min":0.78000000000000003,"15min":0.55300000000000005},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.26600000000000001},"max":{"1min":8.2560000000000002,"5min":8.2560000000000002,"15min":8.2560000000000002},"last":1.2410000000000001},{"interface":"front","average":{"1min":1.6919999999999999,"5min":0.84699999999999998,"15min":0.58999999999999997},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":6.5819999999999999,"5min":6.5819999999999999,"15min":6.5819999999999999},"last":0.33300000000000002}]},{"osd":6,"last update":"Mon Aug 28 02:23:56 2023","interfaces":[{"interface":"back","average":{"1min":1.9239999999999999,"5min":0.96699999999999997,"15min":0.64600000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.27500000000000002},"max":{"1min":8.3230000000000004,"5min":9.6920000000000002,"15min":9.6920000000000002},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.79900000000000004,"5min":0.71699999999999997,"15min":0.58599999999999997},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.318},"max":{"1min":4.2199999999999998,"5min":4.2199999999999998,"15min":4.2199999999999998},"last":1.2110000000000001}]},{"osd":7,"last update":"Mon Aug 28 02:23:20 2023","interfaces":[{"interface":"back","average":{"1min":1.6639999999999999,"5min":0.81699999999999995,"15min":0.625},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.312},"max":{"1min":8.6890000000000001,"5min":8.6890000000000001,"15min":8.6890000000000001},"last":0.50600000000000001},{"interface":"front","average":{"1min":2.2930000000000001,"5min":0.93500000000000005,"15min":0.66700000000000004},"min":{"1min":0.40100000000000002,"5min":0.32500000000000001,"15min":0.32100000000000001},"max":{"1min":8.7569999999999997,"5min":8.7569999999999997,"15min":8.7569999999999997},"last":7.1769999999999996}]}]},{"osd":5,"up_from":36,"seq":154618822838,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3788724,"kb_used_data":3483244,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":89956428,"statfs":{"total":95995035648,"available":92115382272,"internally_reserved":0,"allocated":3566841856,"data_stored":3566285314,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,5,0,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:23:49 2023","interfaces":[{"interface":"back","average":{"1min":3.669,"5min":2.8290000000000002,"15min":1.359},"min":{"1min":0.24099999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":44.680999999999997,"5min":131.06100000000001,"15min":131.06100000000001},"last":2.4769999999999999},{"interface":"front","average":{"1min":3.6949999999999998,"5min":2.879,"15min":1.3779999999999999},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":44.668999999999997,"5min":131.262,"15min":131.262},"last":4.1529999999999996}]},{"osd":1,"last update":"Mon Aug 28 02:23:58 2023","interfaces":[{"interface":"back","average":{"1min":1.97,"5min":3.4009999999999998,"15min":1.4850000000000001},"min":{"1min":0.34699999999999998,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":8.7799999999999994,"5min":183.97900000000001,"15min":183.97900000000001},"last":0.40799999999999997},{"interface":"front","average":{"1min":1.9950000000000001,"5min":3.3730000000000002,"15min":1.4850000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":8.1929999999999996,"5min":184.33699999999999,"15min":184.33699999999999},"last":4.2169999999999996}]},{"osd":2,"last update":"Mon Aug 28 02:23:24 2023","interfaces":[{"interface":"back","average":{"1min":4.6070000000000002,"5min":2.3460000000000001,"15min":1.175},"min":{"1min":0.28499999999999998,"5min":0.252,"15min":0.252},"max":{"1min":43.570999999999998,"5min":72.944999999999993,"15min":72.944999999999993},"last":2.4470000000000001},{"interface":"front","average":{"1min":5.2130000000000001,"5min":2.4689999999999999,"15min":1.2150000000000001},"min":{"1min":0.40000000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":43.582000000000001,"5min":72.930999999999997,"15min":72.930999999999997},"last":4.2409999999999997}]},{"osd":3,"last update":"Mon Aug 28 02:23:58 2023","interfaces":[{"interface":"back","average":{"1min":2.202,"5min":0.93999999999999995,"15min":0.72099999999999997},"min":{"1min":0.36199999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":6.8200000000000003,"5min":6.8200000000000003,"15min":6.8200000000000003},"last":2.5499999999999998},{"interface":"front","average":{"1min":2.2309999999999999,"5min":1.0660000000000001,"15min":0.77300000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":8.3539999999999992,"5min":8.3539999999999992,"15min":8.3539999999999992},"last":4.1870000000000003}]},{"osd":4,"last update":"Mon Aug 28 02:23:41 2023","interfaces":[{"interface":"back","average":{"1min":1.6479999999999999,"5min":0.86099999999999999,"15min":0.58799999999999997},"min":{"1min":0.312,"5min":0.312,"15min":0.27500000000000002},"max":{"1min":5.7789999999999999,"5min":5.7789999999999999,"15min":5.7789999999999999},"last":0.45600000000000002},{"interface":"front","average":{"1min":1.786,"5min":0.86899999999999999,"15min":0.61099999999999999},"min":{"1min":0.33600000000000002,"5min":0.31900000000000001,"15min":0.28199999999999997},"max":{"1min":6.6820000000000004,"5min":6.6820000000000004,"15min":6.6820000000000004},"last":2.5739999999999998}]},{"osd":6,"last update":"Mon Aug 28 02:23:58 2023","interfaces":[{"interface":"back","average":{"1min":2.8740000000000001,"5min":1.2450000000000001,"15min":0.76300000000000001},"min":{"1min":0.35799999999999998,"5min":0.30599999999999999,"15min":0.28399999999999997},"max":{"1min":14.803000000000001,"5min":14.803000000000001,"15min":14.803000000000001},"last":0.435},{"interface":"front","average":{"1min":1.9159999999999999,"5min":0.92400000000000004,"15min":0.64700000000000002},"min":{"1min":0.40899999999999997,"5min":0.31,"15min":0.30299999999999999},"max":{"1min":6.5359999999999996,"5min":6.5359999999999996,"15min":6.5359999999999996},"last":2.52}]},{"osd":7,"last update":"Mon Aug 28 02:23:15 2023","interfaces":[{"interface":"back","average":{"1min":1.4490000000000001,"5min":0.76800000000000002,"15min":0.61699999999999999},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":5.109,"5min":5.109,"15min":5.109},"last":2.4980000000000002},{"interface":"front","average":{"1min":1.6200000000000001,"5min":0.78900000000000003,"15min":0.624},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.29799999999999999},"max":{"1min":6.8710000000000004,"5min":6.8710000000000004,"15min":6.8710000000000004},"last":4.3109999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4348973056,"data_stored":4348968984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1971126272,"data_stored":1971126272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2332037120,"data_stored":2332033048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":32833536,"data_stored":32833536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3175284736,"data_stored":3175284736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3558739968,"data_stored":3558735896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1561657344,"data_stored":1561657344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1925775360,"data_stored":1925775360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-08-28T02:24:08.208 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd pool set unique_pool_0 pgp_num 16 2023-08-28T02:24:08.208 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:24:08.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:08.212 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:24:08.219 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:24:08.313 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:24:08.327 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:24:08.327 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:24:08.450 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:08.560 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:08 smithi143 ceph-mon[183059]: failed to encode map e160 with expected crc 2023-08-28T02:24:08.560 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:08 smithi143 ceph-mon[189871]: failed to encode map e160 with expected crc 2023-08-28T02:24:08.633 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:24:08.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:08.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:24:08.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:08 smithi169 ceph-mon[159415]: failed to encode map e160 with expected crc 2023-08-28T02:24:08.741 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:24:08.788 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:08.946 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:24:08.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:08.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:24:09.049 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:24:09.079 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:09.141 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:24:09.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:09.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:24:09.202 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.0 dump_ops_in_flight 2023-08-28T02:24:09.245 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:24:09.274 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:09.502 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:24:09.502 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:24:09.502 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:24:09.502 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:24:09.517 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:24:09.541 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:09 smithi143 ceph-mon[189871]: osdmap e161: 8 total, 8 up, 8 in 2023-08-28T02:24:09.541 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:09 smithi143 ceph-mon[189871]: failed to encode map e161 with expected crc 2023-08-28T02:24:09.542 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:09 smithi143 ceph-mon[189871]: failed to encode map e161 with expected crc 2023-08-28T02:24:09.542 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:09 smithi143 ceph-mon[189871]: failed to encode map e161 with expected crc 2023-08-28T02:24:09.542 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:09 smithi143 ceph-mon[189871]: failed to encode map e161 with expected crc 2023-08-28T02:24:09.542 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:09 smithi143 ceph-mon[189871]: failed to encode map e161 with expected crc 2023-08-28T02:24:09.542 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:09 smithi143 ceph-mon[189871]: pgmap v332: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 489 op/s; 6883/279126 objects degraded (2.466%); 10733/279126 objects misplaced (3.845%); 14 MiB/s, 222 objects/s recovering 2023-08-28T02:24:09.542 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:09 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/680984208' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:24:09.542 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:09 smithi143 ceph-mon[183059]: osdmap e161: 8 total, 8 up, 8 in 2023-08-28T02:24:09.543 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:09 smithi143 ceph-mon[183059]: failed to encode map e161 with expected crc 2023-08-28T02:24:09.543 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:09 smithi143 ceph-mon[183059]: failed to encode map e161 with expected crc 2023-08-28T02:24:09.543 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:09 smithi143 ceph-mon[183059]: failed to encode map e161 with expected crc 2023-08-28T02:24:09.543 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:09 smithi143 ceph-mon[183059]: failed to encode map e161 with expected crc 2023-08-28T02:24:09.543 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:09 smithi143 ceph-mon[183059]: failed to encode map e161 with expected crc 2023-08-28T02:24:09.544 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:09 smithi143 ceph-mon[183059]: pgmap v332: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 489 op/s; 6883/279126 objects degraded (2.466%); 10733/279126 objects misplaced (3.845%); 14 MiB/s, 222 objects/s recovering 2023-08-28T02:24:09.544 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:09 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/680984208' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:24:09.549 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:24:09.586 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:24:09.615 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:24:09.624 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:24:09.625 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:24:09.625 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:24:09.625 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:24:09.625 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:24:09.625 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:24:09.626 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:24:09.626 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:24:09.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:09 smithi169 ceph-mon[159415]: osdmap e161: 8 total, 8 up, 8 in 2023-08-28T02:24:09.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:09 smithi169 ceph-mon[159415]: failed to encode map e161 with expected crc 2023-08-28T02:24:09.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:09 smithi169 ceph-mon[159415]: failed to encode map e161 with expected crc 2023-08-28T02:24:09.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:09 smithi169 ceph-mon[159415]: failed to encode map e161 with expected crc 2023-08-28T02:24:09.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:09 smithi169 ceph-mon[159415]: failed to encode map e161 with expected crc 2023-08-28T02:24:09.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:09 smithi169 ceph-mon[159415]: failed to encode map e161 with expected crc 2023-08-28T02:24:09.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:09 smithi169 ceph-mon[159415]: pgmap v332: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 489 op/s; 6883/279126 objects degraded (2.466%); 10733/279126 objects misplaced (3.845%); 14 MiB/s, 222 objects/s recovering 2023-08-28T02:24:09.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:09 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/680984208' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:24:09.928 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:24:09.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:09.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:24:10.038 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:24:10.129 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:10.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:10 smithi143 ceph-mon[183059]: failed to encode map e162 with expected crc 2023-08-28T02:24:10.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:10 smithi143 ceph-mon[183059]: failed to encode map e162 with expected crc 2023-08-28T02:24:10.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:10 smithi143 ceph-mon[183059]: osdmap e162: 8 total, 8 up, 8 in 2023-08-28T02:24:10.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:10 smithi143 ceph-mon[183059]: failed to encode map e162 with expected crc 2023-08-28T02:24:10.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:10 smithi143 ceph-mon[183059]: failed to encode map e162 with expected crc 2023-08-28T02:24:10.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:10 smithi143 ceph-mon[189871]: failed to encode map e162 with expected crc 2023-08-28T02:24:10.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:10 smithi143 ceph-mon[189871]: failed to encode map e162 with expected crc 2023-08-28T02:24:10.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:10 smithi143 ceph-mon[189871]: osdmap e162: 8 total, 8 up, 8 in 2023-08-28T02:24:10.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:10 smithi143 ceph-mon[189871]: failed to encode map e162 with expected crc 2023-08-28T02:24:10.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:10 smithi143 ceph-mon[189871]: failed to encode map e162 with expected crc 2023-08-28T02:24:10.459 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:24:10.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:10.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:24:10.569 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:24:10.586 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:24:10.662 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:10.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:10 smithi169 ceph-mon[159415]: failed to encode map e162 with expected crc 2023-08-28T02:24:10.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:10 smithi169 ceph-mon[159415]: failed to encode map e162 with expected crc 2023-08-28T02:24:10.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:10 smithi169 ceph-mon[159415]: osdmap e162: 8 total, 8 up, 8 in 2023-08-28T02:24:10.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:10 smithi169 ceph-mon[159415]: failed to encode map e162 with expected crc 2023-08-28T02:24:10.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:10 smithi169 ceph-mon[159415]: failed to encode map e162 with expected crc 2023-08-28T02:24:10.801 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:24:10.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:10.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:24:10.911 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:24:11.004 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:11.095 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:24:11.153 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:24:11.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:11.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:24:11.270 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:24:11.367 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:11.638 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:11 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:24:11.639 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:11 smithi143 ceph-mon[183059]: pgmap v334: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 481 op/s; 6883/279126 objects degraded (2.466%); 10733/279126 objects misplaced (3.845%); 13 MiB/s, 218 objects/s recovering 2023-08-28T02:24:11.639 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:11 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 6883/279126 objects degraded (2.466%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:24:11.639 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:11 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:24:11.639 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:11 smithi143 ceph-mon[189871]: pgmap v334: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 481 op/s; 6883/279126 objects degraded (2.466%); 10733/279126 objects misplaced (3.845%); 13 MiB/s, 218 objects/s recovering 2023-08-28T02:24:11.639 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:11 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 6883/279126 objects degraded (2.466%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:24:11.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:11 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:24:11.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:11 smithi169 ceph-mon[159415]: pgmap v334: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 84 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 481 op/s; 6883/279126 objects degraded (2.466%); 10733/279126 objects misplaced (3.845%); 13 MiB/s, 218 objects/s recovering 2023-08-28T02:24:11.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:11 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 6883/279126 objects degraded (2.466%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:24:12.117 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:24:12.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:12.126 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:24:12.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:24:11 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:24:11] "GET /metrics HTTP/1.1" 200 36498 "" "Prometheus/2.33.4" 2023-08-28T02:24:12.227 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:24:12.254 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:12.348 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:24:12.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:12.356 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:24:12.363 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is unset 2023-08-28T02:24:12.457 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:24:12.514 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:12 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3804141053' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:24:12.514 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:12 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:24:12.519 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:12.614 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:24:12.615 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:24:12.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:12.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:24:12.689 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:12 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3804141053' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:24:12.690 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:12 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:24:12.690 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:12 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3804141053' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:24:12.690 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:12 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:24:12.720 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:24:12.760 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:12.841 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:24:12.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:12.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:24:12.954 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:24:13.026 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:13.367 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:13 smithi143 ceph-mon[183059]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:24:13.368 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:13 smithi143 ceph-mon[183059]: failed to encode map e163 with expected crc 2023-08-28T02:24:13.368 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:13 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:24:13.368 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:13 smithi143 ceph-mon[183059]: osdmap e163: 8 total, 8 up, 8 in 2023-08-28T02:24:13.368 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:13 smithi143 ceph-mon[183059]: pgmap v336: 89 pgs: 86 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 522 op/s; 6883/273720 objects degraded (2.515%); 10365/273720 objects misplaced (3.787%); 3.7 MiB/s, 61 objects/s recovering 2023-08-28T02:24:13.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:13 smithi143 ceph-mon[189871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:24:13.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:13 smithi143 ceph-mon[189871]: failed to encode map e163 with expected crc 2023-08-28T02:24:13.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:13 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:24:13.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:13 smithi143 ceph-mon[189871]: osdmap e163: 8 total, 8 up, 8 in 2023-08-28T02:24:13.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:13 smithi143 ceph-mon[189871]: pgmap v336: 89 pgs: 86 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 522 op/s; 6883/273720 objects degraded (2.515%); 10365/273720 objects misplaced (3.787%); 3.7 MiB/s, 61 objects/s recovering 2023-08-28T02:24:13.728 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:24:13.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:13 smithi169 ceph-mon[159415]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:24:13.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:13 smithi169 ceph-mon[159415]: failed to encode map e163 with expected crc 2023-08-28T02:24:13.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:13 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:24:13.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:13 smithi169 ceph-mon[159415]: osdmap e163: 8 total, 8 up, 8 in 2023-08-28T02:24:13.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:13 smithi169 ceph-mon[159415]: pgmap v336: 89 pgs: 86 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 522 op/s; 6883/273720 objects degraded (2.515%); 10365/273720 objects misplaced (3.787%); 3.7 MiB/s, 61 objects/s recovering 2023-08-28T02:24:13.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:13.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:24:13.840 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:24:13.886 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:13.958 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:24:13.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:13.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:24:14.068 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:24:14.110 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:14.198 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:24:14.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:14.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:24:14.305 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:24:14.345 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:14.418 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:24:14.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:14.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:24:14.525 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:24:14.578 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:14.680 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:24:14.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:14.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:24:14.790 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:24:14.811 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:24:14.908 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:15.111 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:24:15.112 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [ 2023-08-28T02:24:15.112 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:15.112 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.44594.0:131761 5.1 e163/154)", 2023-08-28T02:24:15.112 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:24:15.106090+0000", 2023-08-28T02:24:15.112 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.0043584180000000002, 2023-08-28T02:24:15.112 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0043995659999999997, 2023-08-28T02:24:15.112 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:24:15.112 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "queued for pg", 2023-08-28T02:24:15.112 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:24:15.112 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:15.112 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:24:15.113 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:15.106090+0000", 2023-08-28T02:24:15.113 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:15.113 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:15.113 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:15.113 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:24:15.113 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:15.106090+0000", 2023-08-28T02:24:15.113 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:15.113 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:15.113 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:15.113 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:24:15.113 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:15.106090+0000", 2023-08-28T02:24:15.113 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:24:15.113 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:15.114 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:15.114 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:24:15.114 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:15.106093+0000", 2023-08-28T02:24:15.114 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.4879999999999999e-06 2023-08-28T02:24:15.114 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:15.114 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:15.114 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:24:15.114 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:15.106099+0000", 2023-08-28T02:24:15.114 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.1619999999999999e-06 2023-08-28T02:24:15.114 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:15.114 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:15.114 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:24:15.114 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:15.106103+0000", 2023-08-28T02:24:15.115 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.1400000000000002e-06 2023-08-28T02:24:15.115 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:15.115 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:24:15.115 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:15.115 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:15.115 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:15.115 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.44594.0:131765 5.c e163/119 5:38b6e271:::benchmark_data_smithi143_236200_object21622:head v 163'8398, mlcod=163'8396)", 2023-08-28T02:24:15.115 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:24:15.108667+0000", 2023-08-28T02:24:15.115 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.0017812819999999999, 2023-08-28T02:24:15.115 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.001879646, 2023-08-28T02:24:15.115 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:24:15.115 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "started", 2023-08-28T02:24:15.116 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:24:15.116 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:15.116 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:24:15.116 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:15.108667+0000", 2023-08-28T02:24:15.116 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:15.116 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:15.116 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:15.116 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:24:15.116 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:15.108667+0000", 2023-08-28T02:24:15.116 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:15.116 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:15.116 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:15.117 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:24:15.117 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:15.108667+0000", 2023-08-28T02:24:15.117 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:24:15.117 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:15.117 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:15.117 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:24:15.117 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:15.108673+0000", 2023-08-28T02:24:15.117 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.5629999999999997e-06 2023-08-28T02:24:15.118 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:15.118 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:15.118 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:24:15.118 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:15.108694+0000", 2023-08-28T02:24:15.118 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.1177000000000001e-05 2023-08-28T02:24:15.118 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:15.118 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:15.118 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:24:15.118 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:15.108702+0000", 2023-08-28T02:24:15.118 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.3989999999999998e-06 2023-08-28T02:24:15.118 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:15.119 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:15.119 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:24:15.119 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:15.110048+0000", 2023-08-28T02:24:15.119 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.001346447 2023-08-28T02:24:15.119 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:15.119 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:15.119 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:24:15.119 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:15.110062+0000", 2023-08-28T02:24:15.119 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3865e-05 2023-08-28T02:24:15.119 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:15.119 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:24:15.119 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:15.120 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:15.120 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:15.120 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.44594.0:131756 5.1 e163/154 5:8d842ebe:::benchmark_data_smithi143_236200_object21613:head v 163'8363, mlcod=163'8360)", 2023-08-28T02:24:15.120 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:24:15.106045+0000", 2023-08-28T02:24:15.120 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.0044038489999999996, 2023-08-28T02:24:15.120 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0045654470000000003, 2023-08-28T02:24:15.120 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:24:15.120 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "started", 2023-08-28T02:24:15.120 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:24:15.120 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:15.120 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:24:15.120 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:15.106045+0000", 2023-08-28T02:24:15.120 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:15.121 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:15.121 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:15.121 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:24:15.121 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:15.106045+0000", 2023-08-28T02:24:15.121 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:15.121 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:15.121 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:15.121 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:24:15.121 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:15.106043+0000", 2023-08-28T02:24:15.121 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:24:15.121 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:15.121 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:15.121 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:24:15.122 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:15.106053+0000", 2023-08-28T02:24:15.122 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.482e-06 2023-08-28T02:24:15.122 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:15.122 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:15.122 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:24:15.122 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:15.106069+0000", 2023-08-28T02:24:15.122 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.6501e-05 2023-08-28T02:24:15.122 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:15.122 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:15.122 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:24:15.122 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:15.106078+0000", 2023-08-28T02:24:15.122 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.1990000000000006e-06 2023-08-28T02:24:15.122 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:15.123 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:15.123 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:24:15.123 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:15.109065+0000", 2023-08-28T02:24:15.123 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0029869599999999999 2023-08-28T02:24:15.123 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:15.123 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:15.123 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:24:15.123 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:15.109086+0000", 2023-08-28T02:24:15.123 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.1778999999999999e-05 2023-08-28T02:24:15.123 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:15.123 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:24:15.124 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:15.124 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:15.124 INFO:teuthology.orchestra.run.smithi143.stdout: ], 2023-08-28T02:24:15.124 INFO:teuthology.orchestra.run.smithi143.stdout: "num_ops": 3 2023-08-28T02:24:15.124 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:24:15.219 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:24:15.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:15.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:24:15.336 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:24:15.421 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:16.112 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set noscrub 2023-08-28T02:24:16.149 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:16 smithi143 ceph-mon[183059]: pgmap v337: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 450 op/s; 346/270438 objects degraded (0.128%); 15989/270438 objects misplaced (5.912%); 1.2 MiB/s, 21 objects/s recovering 2023-08-28T02:24:16.149 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:16 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2920061866' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]: dispatch 2023-08-28T02:24:16.210 INFO:teuthology.orchestra.run.smithi143.stderr:set pool 5 pgp_num to 16 2023-08-28T02:24:16.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:16 smithi143 ceph-mon[189871]: pgmap v337: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 450 op/s; 346/270438 objects degraded (0.128%); 15989/270438 objects misplaced (5.912%); 1.2 MiB/s, 21 objects/s recovering 2023-08-28T02:24:16.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:16 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2920061866' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]: dispatch 2023-08-28T02:24:16.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:16 smithi169 ceph-mon[159415]: pgmap v337: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 450 op/s; 346/270438 objects degraded (0.128%); 15989/270438 objects misplaced (5.912%); 1.2 MiB/s, 21 objects/s recovering 2023-08-28T02:24:16.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:16 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2920061866' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]: dispatch 2023-08-28T02:24:17.139 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.0 dump_blocked_ops 2023-08-28T02:24:17.398 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:24:17.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:17.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:24:17.405 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:17 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2920061866' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]': finished 2023-08-28T02:24:17.405 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:17 smithi143 ceph-mon[183059]: osdmap e164: 8 total, 8 up, 8 in 2023-08-28T02:24:17.405 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:17 smithi143 ceph-mon[183059]: failed to encode map e164 with expected crc 2023-08-28T02:24:17.405 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:17 smithi143 ceph-mon[183059]: failed to encode map e164 with expected crc 2023-08-28T02:24:17.405 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:17 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 346/270438 objects degraded (0.128%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:24:17.406 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:17 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2920061866' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]': finished 2023-08-28T02:24:17.406 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:17 smithi143 ceph-mon[189871]: osdmap e164: 8 total, 8 up, 8 in 2023-08-28T02:24:17.406 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:17 smithi143 ceph-mon[189871]: failed to encode map e164 with expected crc 2023-08-28T02:24:17.406 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:17 smithi143 ceph-mon[189871]: failed to encode map e164 with expected crc 2023-08-28T02:24:17.406 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:17 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 346/270438 objects degraded (0.128%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:24:17.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:17 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2920061866' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]': finished 2023-08-28T02:24:17.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:17 smithi169 ceph-mon[159415]: osdmap e164: 8 total, 8 up, 8 in 2023-08-28T02:24:17.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:17 smithi169 ceph-mon[159415]: failed to encode map e164 with expected crc 2023-08-28T02:24:17.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:17 smithi169 ceph-mon[159415]: failed to encode map e164 with expected crc 2023-08-28T02:24:17.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:17 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 346/270438 objects degraded (0.128%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:24:17.506 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:24:17.576 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:17.746 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:24:17.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:17.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:24:17.859 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:24:17.901 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:17.976 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:24:17.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:17.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:24:18.089 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:24:18.167 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:18.213 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:18 smithi169 ceph-mon[159415]: pgmap v339: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 552 op/s; 346/266259 objects degraded (0.130%); 15802/266259 objects misplaced (5.935%); 1.1 MiB/s, 18 objects/s recovering 2023-08-28T02:24:18.245 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:24:18.274 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:24:18.276 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:18 smithi143 ceph-mon[183059]: pgmap v339: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 552 op/s; 346/266259 objects degraded (0.130%); 15802/266259 objects misplaced (5.935%); 1.1 MiB/s, 18 objects/s recovering 2023-08-28T02:24:18.276 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:18 smithi143 ceph-mon[189871]: pgmap v339: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 552 op/s; 346/266259 objects degraded (0.130%); 15802/266259 objects misplaced (5.935%); 1.1 MiB/s, 18 objects/s recovering 2023-08-28T02:24:18.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:18.277 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:24:18.378 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:24:18.385 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:24:18.417 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:24:18.459 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:24:18.461 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:18.492 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:24:18.547 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:24:18.561 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:24:18.569 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:24:18.591 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:24:18.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:18.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:24:18.623 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:24:18.661 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:24:18.677 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:24:18.677 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:24:18.677 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:24:18.678 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:24:18.678 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:24:18.678 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:24:18.678 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:24:18.678 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:24:18.694 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:24:18.725 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:18.789 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:24:18.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:18.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:24:18.907 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:24:18.967 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:19.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:19 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1558958867' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:24:19.575 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:19 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1558958867' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:24:19.575 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:19 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1558958867' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:24:19.979 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:24:19.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:19.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:24:20.089 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:24:20.141 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:20.186 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:24:20.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:20.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:24:20.247 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:20 smithi143 ceph-mon[183059]: pgmap v340: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 765 op/s; 486/260733 objects degraded (0.186%); 15394/260733 objects misplaced (5.904%); 1.7 MiB/s, 28 objects/s recovering 2023-08-28T02:24:20.293 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:24:20.338 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:20.406 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:24:20.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:20.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:24:20.423 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:20 smithi169 ceph-mon[159415]: pgmap v340: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 765 op/s; 486/260733 objects degraded (0.186%); 15394/260733 objects misplaced (5.904%); 1.7 MiB/s, 28 objects/s recovering 2023-08-28T02:24:20.521 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:24:20.570 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:20.575 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:20 smithi143 ceph-mon[189871]: pgmap v340: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 765 op/s; 486/260733 objects degraded (0.186%); 15394/260733 objects misplaced (5.904%); 1.7 MiB/s, 28 objects/s recovering 2023-08-28T02:24:20.782 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:24:20.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:20.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:24:20.893 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:24:20.960 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:21.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:21 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 486/260733 objects degraded (0.186%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:24:21.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:21 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 486/260733 objects degraded (0.186%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:24:21.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:21 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 486/260733 objects degraded (0.186%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:24:21.940 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:24:21.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:21.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:24:22.050 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:24:22.104 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:22.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:24:21 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:24:21] "GET /metrics HTTP/1.1" 200 36498 "" "Prometheus/2.33.4" 2023-08-28T02:24:22.207 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:24:22.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:22.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:24:22.307 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is set 2023-08-28T02:24:22.317 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:24:22.351 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:22 smithi169 ceph-mon[159415]: pgmap v341: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 486/260733 objects degraded (0.186%); 15394/260733 objects misplaced (5.904%); 1.4 MiB/s, 21 objects/s recovering 2023-08-28T02:24:22.351 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:22 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/751597050' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:24:22.351 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:22 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:24:22.358 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:22.429 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:24:22.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:22.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:24:22.538 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:24:22.617 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:22 smithi143 ceph-mon[189871]: pgmap v341: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 486/260733 objects degraded (0.186%); 15394/260733 objects misplaced (5.904%); 1.4 MiB/s, 21 objects/s recovering 2023-08-28T02:24:22.618 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:22 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/751597050' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:24:22.618 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:22 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:24:22.619 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:22 smithi143 ceph-mon[183059]: pgmap v341: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 486/260733 objects degraded (0.186%); 15394/260733 objects misplaced (5.904%); 1.4 MiB/s, 21 objects/s recovering 2023-08-28T02:24:22.619 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:22 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/751597050' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:24:22.619 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:22 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:24:22.641 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:22.657 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:24:22.743 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7, 3] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-08-28T02:24:22.743 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-08-28T02:24:22.745 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-08-28T02:24:22.746 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-08-28T02:24:22.746 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-08-28T02:24:22.746 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-08-28T02:24:23.108 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:24:23.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:23.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:24:23.219 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:24:23.267 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:23.320 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:24:23.348 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:24:23.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:23.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:24:23.378 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:23 smithi143 ceph-mon[183059]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:24:23.378 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:23 smithi143 ceph-mon[183059]: failed to encode map e165 with expected crc 2023-08-28T02:24:23.378 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:23 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:24:23.378 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:23 smithi143 ceph-mon[183059]: osdmap e165: 8 total, 8 up, 8 in 2023-08-28T02:24:23.378 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:23 smithi143 ceph-mon[183059]: failed to encode map e165 with expected crc 2023-08-28T02:24:23.378 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:23 smithi143 ceph-mon[183059]: failed to encode map e165 with expected crc 2023-08-28T02:24:23.378 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:23 smithi143 ceph-mon[183059]: pgmap v343: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 486/254244 objects degraded (0.191%); 15007/254244 objects misplaced (5.903%); 671 KiB/s, 10 objects/s recovering 2023-08-28T02:24:23.379 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:23 smithi143 ceph-mon[189871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:24:23.379 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:23 smithi143 ceph-mon[189871]: failed to encode map e165 with expected crc 2023-08-28T02:24:23.379 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:23 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:24:23.379 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:23 smithi143 ceph-mon[189871]: osdmap e165: 8 total, 8 up, 8 in 2023-08-28T02:24:23.380 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:23 smithi143 ceph-mon[189871]: failed to encode map e165 with expected crc 2023-08-28T02:24:23.380 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:23 smithi143 ceph-mon[189871]: failed to encode map e165 with expected crc 2023-08-28T02:24:23.380 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:23 smithi143 ceph-mon[189871]: pgmap v343: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 486/254244 objects degraded (0.191%); 15007/254244 objects misplaced (5.903%); 671 KiB/s, 10 objects/s recovering 2023-08-28T02:24:23.467 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:24:23.556 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:23.632 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:23 smithi169 ceph-mon[159415]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:24:23.633 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:23 smithi169 ceph-mon[159415]: failed to encode map e165 with expected crc 2023-08-28T02:24:23.633 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:23 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:24:23.633 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:23 smithi169 ceph-mon[159415]: osdmap e165: 8 total, 8 up, 8 in 2023-08-28T02:24:23.633 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:23 smithi169 ceph-mon[159415]: failed to encode map e165 with expected crc 2023-08-28T02:24:23.633 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:23 smithi169 ceph-mon[159415]: failed to encode map e165 with expected crc 2023-08-28T02:24:23.633 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:23 smithi169 ceph-mon[159415]: pgmap v343: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 486/254244 objects degraded (0.191%); 15007/254244 objects misplaced (5.903%); 671 KiB/s, 10 objects/s recovering 2023-08-28T02:24:23.691 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:24:23.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:23.697 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:24:23.715 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:24:23.715 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:24:23.732 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:24:23.798 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:24:23.840 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:23.977 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:24:23.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:23.984 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:24:24.085 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:24:24.157 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:24.244 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:24:24.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:24.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:24:24.347 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:24:24.383 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:24.463 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:24:24.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:24.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:24:24.572 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:24:24.633 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:24.802 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:24:24.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:24.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:24:24.825 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.0 dump_historic_ops 2023-08-28T02:24:24.910 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:24:24.973 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:25.157 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:24:25.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:25.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:24:25.261 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set nodeep-scrub 2023-08-28T02:24:25.274 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:24:25.415 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:25.518 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:24:25.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:25.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:24:25.622 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:24:25.651 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:26.206 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:24:26.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:26.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:24:26.315 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:24:26.352 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:26 smithi169 ceph-mon[159415]: pgmap v344: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 587/250380 objects degraded (0.234%); 14809/250380 objects misplaced (5.915%); 1.3 MiB/s, 20 objects/s recovering 2023-08-28T02:24:26.352 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:26 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:24:26.364 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:26.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:26 smithi143 ceph-mon[183059]: pgmap v344: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 587/250380 objects degraded (0.234%); 14809/250380 objects misplaced (5.915%); 1.3 MiB/s, 20 objects/s recovering 2023-08-28T02:24:26.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:26 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:24:26.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:26 smithi143 ceph-mon[189871]: pgmap v344: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 587/250380 objects degraded (0.234%); 14809/250380 objects misplaced (5.915%); 1.3 MiB/s, 20 objects/s recovering 2023-08-28T02:24:26.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:26 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:24:26.505 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:24:26.566 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:24:26.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:26.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:24:26.668 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:24:26.668 INFO:teuthology.orchestra.run.smithi169.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-08-28T02:24:26.668 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:24:26.672 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:24:26.734 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:26.901 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:24:26.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:26.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:24:26.988 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:24:27.012 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:24:27.039 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:24:27.062 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:27.091 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:24:27.121 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:24:27.148 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:24:27.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:27.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:24:27.158 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:24:27.178 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:27 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 587/250380 objects degraded (0.234%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:24:27.178 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:27 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1051778233' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:24:27.193 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:24:27.246 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:24:27.254 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:24:27.290 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:24:27.308 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:27 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 587/250380 objects degraded (0.234%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:24:27.308 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:27 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1051778233' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:24:27.309 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:27.332 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:24:27.356 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:24:27.356 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:24:27.356 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:24:27.356 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:24:27.356 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:24:27.356 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:24:27.356 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:24:27.356 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:24:27.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:27 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 587/250380 objects degraded (0.234%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:24:27.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:27 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1051778233' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:24:27.527 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:24:27.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:27.543 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:24:27.643 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:24:27.706 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:27.789 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:24:27.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:27.794 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:24:27.895 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:24:27.952 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:28.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:28 smithi169 ceph-mon[159415]: pgmap v345: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 5.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 587/246234 objects degraded (0.238%); 14646/246234 objects misplaced (5.948%); 1.1 MiB/s, 18 objects/s recovering 2023-08-28T02:24:28.536 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:28 smithi143 ceph-mon[189871]: pgmap v345: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 5.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 587/246234 objects degraded (0.238%); 14646/246234 objects misplaced (5.948%); 1.1 MiB/s, 18 objects/s recovering 2023-08-28T02:24:28.539 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:28 smithi143 ceph-mon[183059]: pgmap v345: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 5.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 587/246234 objects degraded (0.238%); 14646/246234 objects misplaced (5.948%); 1.1 MiB/s, 18 objects/s recovering 2023-08-28T02:24:28.828 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:24:28.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:28.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:24:28.941 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:24:29.017 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:29.749 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:24:29.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:29.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:24:29.856 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:24:29.918 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:30.347 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:24:30.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:30.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:24:30.456 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:24:30.492 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:30.567 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:24:30.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:30.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:24:30.577 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:30 smithi169 ceph-mon[159415]: pgmap v346: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 668/240843 objects degraded (0.277%); 14250/240843 objects misplaced (5.917%); 1.3 MiB/s, 20 objects/s recovering 2023-08-28T02:24:30.677 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:24:30.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:30 smithi143 ceph-mon[183059]: pgmap v346: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 668/240843 objects degraded (0.277%); 14250/240843 objects misplaced (5.917%); 1.3 MiB/s, 20 objects/s recovering 2023-08-28T02:24:30.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:30 smithi143 ceph-mon[189871]: pgmap v346: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 668/240843 objects degraded (0.277%); 14250/240843 objects misplaced (5.917%); 1.3 MiB/s, 20 objects/s recovering 2023-08-28T02:24:30.771 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:30.960 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:24:30.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:30.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:24:31.071 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:24:31.128 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:31.185 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:24:31.247 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:24:31.247 INFO:teuthology.orchestra.run.smithi143.stdout: "size": 20, 2023-08-28T02:24:31.247 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 600, 2023-08-28T02:24:31.247 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [ 2023-08-28T02:24:31.248 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.248 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:45881 5.5 5:a849fe1c:::benchmark_data_smithi143_236200_object45880:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-08-28T02:24:31.248 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:22:29.939019+0000", 2023-08-28T02:24:31.248 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 121.30313213300001, 2023-08-28T02:24:31.248 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.189640275, 2023-08-28T02:24:31.248 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:24:31.248 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:24:31.248 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:24:31.248 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:24:31.248 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:24:31.248 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 45881 2023-08-28T02:24:31.248 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.249 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:24:31.249 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.249 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:24:31.249 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.939019+0000", 2023-08-28T02:24:31.249 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:31.249 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.249 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.249 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:24:31.249 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.939019+0000", 2023-08-28T02:24:31.249 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:31.249 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.249 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.250 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:24:31.250 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.939018+0000", 2023-08-28T02:24:31.250 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:24:31.250 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.250 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.250 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:24:31.250 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.939035+0000", 2023-08-28T02:24:31.250 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.7142e-05 2023-08-28T02:24:31.250 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.250 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.250 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:24:31.250 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.939041+0000", 2023-08-28T02:24:31.250 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.4479999999999999e-06 2023-08-28T02:24:31.251 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.251 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.251 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:24:31.251 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.939047+0000", 2023-08-28T02:24:31.251 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.361e-06 2023-08-28T02:24:31.251 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.251 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.251 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:24:31.251 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.939106+0000", 2023-08-28T02:24:31.251 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.9286999999999997e-05 2023-08-28T02:24:31.251 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.251 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.251 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:24:31.252 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.939188+0000", 2023-08-28T02:24:31.252 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.2118000000000002e-05 2023-08-28T02:24:31.252 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.252 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.252 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,4", 2023-08-28T02:24:31.252 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.939297+0000", 2023-08-28T02:24:31.252 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000109132 2023-08-28T02:24:31.252 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.252 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.252 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:24:31.252 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.941132+0000", 2023-08-28T02:24:31.252 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0018344489999999999 2023-08-28T02:24:31.252 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.253 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.253 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:31.253 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.942732+0000", 2023-08-28T02:24:31.253 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.001599788 2023-08-28T02:24:31.253 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.253 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.253 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:31.253 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.128574+0000", 2023-08-28T02:24:31.253 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.185842655 2023-08-28T02:24:31.253 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.253 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.253 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:24:31.253 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.128623+0000", 2023-08-28T02:24:31.254 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.9017999999999997e-05 2023-08-28T02:24:31.254 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.254 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.254 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:24:31.254 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.128659+0000", 2023-08-28T02:24:31.254 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.5568999999999998e-05 2023-08-28T02:24:31.254 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:31.254 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:24:31.254 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:31.254 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.254 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.254 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.44594.0:93768 5.e e136/119 5:7330778b:::benchmark_data_smithi143_236200_object93767:head v 136'5959, mlcod=136'5957)", 2023-08-28T02:24:31.254 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:13.064649+0000", 2023-08-28T02:24:31.255 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 78.177501562000003, 2023-08-28T02:24:31.255 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.18990683899999999, 2023-08-28T02:24:31.255 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:24:31.255 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:24:31.255 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:24:31.255 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.255 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:24:31.255 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:13.064649+0000", 2023-08-28T02:24:31.255 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:31.255 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.255 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.255 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:24:31.255 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:13.064649+0000", 2023-08-28T02:24:31.256 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:31.256 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.256 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.256 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:24:31.256 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:13.064649+0000", 2023-08-28T02:24:31.256 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967296 2023-08-28T02:24:31.256 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.256 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.256 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:24:31.256 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:13.064670+0000", 2023-08-28T02:24:31.256 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.0630999999999999e-05 2023-08-28T02:24:31.256 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.256 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.257 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:24:31.257 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:13.064674+0000", 2023-08-28T02:24:31.257 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.0300000000000004e-06 2023-08-28T02:24:31.257 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.257 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.257 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:24:31.257 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:13.064677+0000", 2023-08-28T02:24:31.257 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.9510000000000002e-06 2023-08-28T02:24:31.257 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.257 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.257 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:24:31.257 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:13.064714+0000", 2023-08-28T02:24:31.257 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.7169000000000003e-05 2023-08-28T02:24:31.258 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.258 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.258 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:24:31.258 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:13.064727+0000", 2023-08-28T02:24:31.258 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3128e-05 2023-08-28T02:24:31.258 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.258 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.258 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:24:31.258 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:13.254512+0000", 2023-08-28T02:24:31.258 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.18978546399999999 2023-08-28T02:24:31.258 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.258 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.258 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:24:31.259 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:13.254556+0000", 2023-08-28T02:24:31.259 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.3668000000000003e-05 2023-08-28T02:24:31.259 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:31.259 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:24:31.259 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:31.259 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.259 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.259 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:93769 5.3 5:c72a1d11:::benchmark_data_smithi143_236200_object93768:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-08-28T02:24:31.259 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:13.064842+0000", 2023-08-28T02:24:31.259 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 78.177309145999999, 2023-08-28T02:24:31.259 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.193393179, 2023-08-28T02:24:31.259 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:24:31.259 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:24:31.260 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:24:31.260 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:24:31.260 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:24:31.260 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 93769 2023-08-28T02:24:31.260 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.260 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:24:31.260 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.260 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:24:31.260 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:13.064842+0000", 2023-08-28T02:24:31.260 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:31.260 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.260 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.261 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:24:31.261 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:13.064842+0000", 2023-08-28T02:24:31.261 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:31.261 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.261 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.261 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:24:31.261 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:13.064841+0000", 2023-08-28T02:24:31.261 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:24:31.261 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.261 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.261 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:24:31.261 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:13.064860+0000", 2023-08-28T02:24:31.261 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.9431000000000001e-05 2023-08-28T02:24:31.262 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.262 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.262 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:24:31.262 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:13.248150+0000", 2023-08-28T02:24:31.262 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.18328965699999999 2023-08-28T02:24:31.262 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.262 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.262 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:24:31.262 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:13.248171+0000", 2023-08-28T02:24:31.262 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.0767e-05 2023-08-28T02:24:31.262 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.262 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.263 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:24:31.263 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:13.248232+0000", 2023-08-28T02:24:31.263 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.1722000000000003e-05 2023-08-28T02:24:31.263 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.263 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.263 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:24:31.263 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:13.248407+0000", 2023-08-28T02:24:31.263 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00017445500000000001 2023-08-28T02:24:31.263 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.263 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.263 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 5,6", 2023-08-28T02:24:31.263 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:13.248576+0000", 2023-08-28T02:24:31.263 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00016904499999999999 2023-08-28T02:24:31.264 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.264 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.264 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:31.264 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:13.258090+0000", 2023-08-28T02:24:31.264 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0095138459999999994 2023-08-28T02:24:31.264 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.264 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.264 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:24:31.264 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:13.258101+0000", 2023-08-28T02:24:31.264 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1589e-05 2023-08-28T02:24:31.264 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.264 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.265 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:31.265 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:13.258144+0000", 2023-08-28T02:24:31.265 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.2320000000000001e-05 2023-08-28T02:24:31.265 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.265 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.265 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:24:31.265 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:13.258197+0000", 2023-08-28T02:24:31.265 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.3393999999999997e-05 2023-08-28T02:24:31.265 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.265 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.265 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:24:31.265 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:13.258235+0000", 2023-08-28T02:24:31.265 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.7768999999999997e-05 2023-08-28T02:24:31.265 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:31.266 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:24:31.266 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:31.266 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.266 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.266 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:106246 5.3 5:cbc30a51:::benchmark_data_smithi143_236200_object106245:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e142)", 2023-08-28T02:24:31.266 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:30.467484+0000", 2023-08-28T02:24:31.266 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 60.774667211000001, 2023-08-28T02:24:31.266 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.18938422999999999, 2023-08-28T02:24:31.266 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:24:31.266 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:24:31.266 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:24:31.266 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:24:31.267 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:24:31.267 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 106246 2023-08-28T02:24:31.267 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.267 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:24:31.267 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.267 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:24:31.267 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:30.467484+0000", 2023-08-28T02:24:31.267 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:31.267 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.267 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.267 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:24:31.267 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:30.467484+0000", 2023-08-28T02:24:31.267 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:31.268 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.268 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.268 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:24:31.268 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:30.467482+0000", 2023-08-28T02:24:31.268 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:24:31.268 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.268 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.268 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:24:31.268 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:30.467509+0000", 2023-08-28T02:24:31.268 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.7223e-05 2023-08-28T02:24:31.268 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.268 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.269 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:24:31.269 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:30.467522+0000", 2023-08-28T02:24:31.269 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3495999999999999e-05 2023-08-28T02:24:31.269 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.269 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.269 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:24:31.269 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:30.467533+0000", 2023-08-28T02:24:31.269 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.042e-05 2023-08-28T02:24:31.269 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.269 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.269 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:24:31.269 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:30.467597+0000", 2023-08-28T02:24:31.269 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.4204999999999999e-05 2023-08-28T02:24:31.270 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.270 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.270 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:24:31.270 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:30.467721+0000", 2023-08-28T02:24:31.270 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00012433600000000001 2023-08-28T02:24:31.270 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.270 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.270 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 5,6", 2023-08-28T02:24:31.270 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:30.467858+0000", 2023-08-28T02:24:31.270 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00013655500000000001 2023-08-28T02:24:31.270 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.270 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.270 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:31.270 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:30.474122+0000", 2023-08-28T02:24:31.271 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0062645779999999998 2023-08-28T02:24:31.271 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.271 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.271 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:24:31.271 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:30.474576+0000", 2023-08-28T02:24:31.271 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00045395800000000002 2023-08-28T02:24:31.271 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.271 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.271 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:31.271 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:30.656776+0000", 2023-08-28T02:24:31.271 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.18219971500000001 2023-08-28T02:24:31.271 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.271 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.272 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:24:31.272 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:30.656827+0000", 2023-08-28T02:24:31.272 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.0649999999999998e-05 2023-08-28T02:24:31.272 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.272 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.272 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:24:31.272 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:30.656868+0000", 2023-08-28T02:24:31.272 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.1023999999999999e-05 2023-08-28T02:24:31.272 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:31.272 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:24:31.272 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:31.272 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.272 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.273 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:123521 5.0 5:0f371aae:::benchmark_data_smithi143_236200_object13378:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e154)", 2023-08-28T02:24:31.273 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:59.196550+0000", 2023-08-28T02:24:31.273 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 32.045601099000002, 2023-08-28T02:24:31.273 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.128653905, 2023-08-28T02:24:31.273 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:24:31.273 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:24:31.273 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:24:31.273 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:24:31.273 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:24:31.273 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 123521 2023-08-28T02:24:31.273 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.273 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:24:31.273 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.273 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:24:31.274 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196550+0000", 2023-08-28T02:24:31.274 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:31.274 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.274 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.274 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:24:31.274 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196550+0000", 2023-08-28T02:24:31.274 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:31.274 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.274 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.274 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:24:31.274 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196548+0000", 2023-08-28T02:24:31.274 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:24:31.274 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.275 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.275 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:24:31.275 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196553+0000", 2023-08-28T02:24:31.275 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.6439999999999999e-06 2023-08-28T02:24:31.275 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.275 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.275 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:24:31.275 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196564+0000", 2023-08-28T02:24:31.275 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0550999999999999e-05 2023-08-28T02:24:31.275 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.275 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.275 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:24:31.275 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196573+0000", 2023-08-28T02:24:31.276 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.8619999999999998e-06 2023-08-28T02:24:31.276 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.276 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.276 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:24:31.276 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196799+0000", 2023-08-28T02:24:31.276 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000225913 2023-08-28T02:24:31.276 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.276 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.276 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op must wait for map", 2023-08-28T02:24:31.276 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196806+0000", 2023-08-28T02:24:31.276 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.0720000000000002e-06 2023-08-28T02:24:31.276 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.276 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.277 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:24:31.277 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.208800+0000", 2023-08-28T02:24:31.277 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.011993996999999999 2023-08-28T02:24:31.277 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.277 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.277 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:24:31.277 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.208811+0000", 2023-08-28T02:24:31.277 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.048e-05 2023-08-28T02:24:31.277 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.277 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.277 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:24:31.277 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.305571+0000", 2023-08-28T02:24:31.277 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.096759813 2023-08-28T02:24:31.278 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.278 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.278 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:24:31.278 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.307358+0000", 2023-08-28T02:24:31.278 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0017876599999999999 2023-08-28T02:24:31.278 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.278 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.278 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,3,4", 2023-08-28T02:24:31.278 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.307473+0000", 2023-08-28T02:24:31.278 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000114498 2023-08-28T02:24:31.278 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.278 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.279 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:31.279 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.317779+0000", 2023-08-28T02:24:31.279 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.010306031 2023-08-28T02:24:31.279 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.279 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.279 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:31.279 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.317817+0000", 2023-08-28T02:24:31.279 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.7766999999999999e-05 2023-08-28T02:24:31.279 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.279 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.279 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:31.279 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.324897+0000", 2023-08-28T02:24:31.279 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0070804420000000002 2023-08-28T02:24:31.280 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.280 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.280 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:24:31.280 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325027+0000", 2023-08-28T02:24:31.280 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00012982799999999999 2023-08-28T02:24:31.280 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.280 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.280 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:24:31.280 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325088+0000", 2023-08-28T02:24:31.280 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.1532999999999995e-05 2023-08-28T02:24:31.280 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.280 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.280 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:24:31.281 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325204+0000", 2023-08-28T02:24:31.281 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00011503299999999999 2023-08-28T02:24:31.281 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:31.281 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:24:31.281 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:31.281 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.281 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.281 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:123522 5.0 5:0bc2203e:::benchmark_data_smithi143_236200_object13379:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e154)", 2023-08-28T02:24:31.281 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:59.196585+0000", 2023-08-28T02:24:31.281 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 32.045566047000001, 2023-08-28T02:24:31.281 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1287665410000001, 2023-08-28T02:24:31.281 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:24:31.281 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:24:31.282 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:24:31.282 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:24:31.282 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:24:31.282 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 123522 2023-08-28T02:24:31.282 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.282 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:24:31.282 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.282 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:24:31.282 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196585+0000", 2023-08-28T02:24:31.282 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:31.282 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.282 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.282 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:24:31.283 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196585+0000", 2023-08-28T02:24:31.283 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:31.283 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.283 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.283 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:24:31.283 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196584+0000", 2023-08-28T02:24:31.283 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:24:31.283 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.283 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.283 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:24:31.283 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196587+0000", 2023-08-28T02:24:31.283 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.238e-06 2023-08-28T02:24:31.283 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.284 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.284 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:24:31.284 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196593+0000", 2023-08-28T02:24:31.284 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.9710000000000001e-06 2023-08-28T02:24:31.284 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.284 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.284 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:24:31.284 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196598+0000", 2023-08-28T02:24:31.284 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.1120000000000004e-06 2023-08-28T02:24:31.284 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.284 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.284 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:24:31.284 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196858+0000", 2023-08-28T02:24:31.285 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00026082200000000002 2023-08-28T02:24:31.285 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.285 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.285 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting_for_map not empty", 2023-08-28T02:24:31.285 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196863+0000", 2023-08-28T02:24:31.285 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.8999999999999997e-06 2023-08-28T02:24:31.285 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.285 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.285 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:24:31.285 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.209793+0000", 2023-08-28T02:24:31.285 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.012929248000000001 2023-08-28T02:24:31.285 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.285 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.286 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:24:31.286 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.209798+0000", 2023-08-28T02:24:31.286 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.7250000000000002e-06 2023-08-28T02:24:31.286 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.286 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.286 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:24:31.286 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.310606+0000", 2023-08-28T02:24:31.286 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.10080757 2023-08-28T02:24:31.286 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.286 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.286 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:24:31.286 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.310743+0000", 2023-08-28T02:24:31.286 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00013726600000000001 2023-08-28T02:24:31.287 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.287 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.287 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,3,4", 2023-08-28T02:24:31.287 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.310871+0000", 2023-08-28T02:24:31.287 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00012777599999999999 2023-08-28T02:24:31.287 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.287 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.287 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:31.287 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.317856+0000", 2023-08-28T02:24:31.287 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0069848460000000003 2023-08-28T02:24:31.287 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.287 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.287 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:31.288 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.317896+0000", 2023-08-28T02:24:31.288 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.9743999999999998e-05 2023-08-28T02:24:31.288 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.288 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.288 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:31.288 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.324952+0000", 2023-08-28T02:24:31.288 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0070567060000000003 2023-08-28T02:24:31.288 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.288 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.288 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:24:31.288 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325214+0000", 2023-08-28T02:24:31.288 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00026183400000000001 2023-08-28T02:24:31.289 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.289 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.289 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:24:31.289 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325264+0000", 2023-08-28T02:24:31.289 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.9496999999999997e-05 2023-08-28T02:24:31.289 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.289 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.289 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:24:31.289 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325351+0000", 2023-08-28T02:24:31.289 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.763e-05 2023-08-28T02:24:31.289 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:31.289 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:24:31.289 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:31.290 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.290 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.290 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:123553 5.0 5:03eb21c5:::benchmark_data_smithi143_236200_object13410:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e154)", 2023-08-28T02:24:31.290 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:59.196602+0000", 2023-08-28T02:24:31.290 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 32.045548891999999, 2023-08-28T02:24:31.290 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.128974272, 2023-08-28T02:24:31.290 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:24:31.290 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:24:31.290 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:24:31.290 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:24:31.290 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:24:31.290 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 123553 2023-08-28T02:24:31.290 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.291 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:24:31.291 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.291 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:24:31.291 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196602+0000", 2023-08-28T02:24:31.291 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:31.291 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.291 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.291 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:24:31.291 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196602+0000", 2023-08-28T02:24:31.291 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:31.291 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.291 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.293 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:24:31.293 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196602+0000", 2023-08-28T02:24:31.293 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:24:31.293 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.293 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.293 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:24:31.293 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196604+0000", 2023-08-28T02:24:31.293 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.0990000000000001e-06 2023-08-28T02:24:31.293 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.293 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.293 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:24:31.294 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196609+0000", 2023-08-28T02:24:31.294 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.3920000000000002e-06 2023-08-28T02:24:31.294 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.294 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.294 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:24:31.294 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196613+0000", 2023-08-28T02:24:31.294 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.1679999999999997e-06 2023-08-28T02:24:31.294 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.294 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.294 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:24:31.294 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196883+0000", 2023-08-28T02:24:31.294 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00026973699999999998 2023-08-28T02:24:31.294 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.295 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.295 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting_for_map not empty", 2023-08-28T02:24:31.295 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196888+0000", 2023-08-28T02:24:31.295 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.3399999999999997e-06 2023-08-28T02:24:31.295 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.295 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.295 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:24:31.295 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.209815+0000", 2023-08-28T02:24:31.295 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.012926936 2023-08-28T02:24:31.295 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.295 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.295 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:24:31.295 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.209821+0000", 2023-08-28T02:24:31.296 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.049e-06 2023-08-28T02:24:31.296 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.296 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.296 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:24:31.296 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.314325+0000", 2023-08-28T02:24:31.296 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.10450329 2023-08-28T02:24:31.296 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.296 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.296 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:24:31.296 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.314416+0000", 2023-08-28T02:24:31.296 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.1209e-05 2023-08-28T02:24:31.296 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.296 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.297 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,3,4", 2023-08-28T02:24:31.297 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.314497+0000", 2023-08-28T02:24:31.297 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.0943000000000001e-05 2023-08-28T02:24:31.297 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.297 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.297 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:31.297 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.317937+0000", 2023-08-28T02:24:31.297 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.003440088 2023-08-28T02:24:31.297 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.297 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.297 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:24:31.297 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325361+0000", 2023-08-28T02:24:31.298 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0074246319999999996 2023-08-28T02:24:31.298 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.298 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.298 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:31.298 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325398+0000", 2023-08-28T02:24:31.298 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.6105000000000001e-05 2023-08-28T02:24:31.298 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.298 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.298 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:31.298 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325471+0000", 2023-08-28T02:24:31.298 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.3779e-05 2023-08-28T02:24:31.298 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.298 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.299 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:24:31.299 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325499+0000", 2023-08-28T02:24:31.299 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.7585000000000001e-05 2023-08-28T02:24:31.299 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.299 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.299 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:24:31.299 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325576+0000", 2023-08-28T02:24:31.299 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.7211000000000003e-05 2023-08-28T02:24:31.299 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:31.299 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:24:31.299 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:31.299 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.299 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.300 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:123563 5.0 5:0dabf8cb:::benchmark_data_smithi143_236200_object13420:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e154)", 2023-08-28T02:24:31.300 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:59.196617+0000", 2023-08-28T02:24:31.300 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 32.045534001, 2023-08-28T02:24:31.300 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1291040370000001, 2023-08-28T02:24:31.300 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:24:31.300 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:24:31.300 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:24:31.300 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:24:31.300 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:24:31.300 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 123563 2023-08-28T02:24:31.300 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.300 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:24:31.300 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.301 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:24:31.301 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196617+0000", 2023-08-28T02:24:31.301 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:31.301 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.301 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.301 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:24:31.301 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196617+0000", 2023-08-28T02:24:31.301 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:31.301 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.301 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.301 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:24:31.301 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196616+0000", 2023-08-28T02:24:31.301 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:24:31.302 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.302 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.302 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:24:31.302 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196618+0000", 2023-08-28T02:24:31.302 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.9970000000000001e-06 2023-08-28T02:24:31.302 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.302 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.302 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:24:31.302 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196623+0000", 2023-08-28T02:24:31.302 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.8509999999999998e-06 2023-08-28T02:24:31.302 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.302 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.302 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:24:31.302 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196628+0000", 2023-08-28T02:24:31.303 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.6149999999999999e-06 2023-08-28T02:24:31.303 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.303 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.303 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:24:31.303 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196908+0000", 2023-08-28T02:24:31.303 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000279788 2023-08-28T02:24:31.303 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.303 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.303 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting_for_map not empty", 2023-08-28T02:24:31.303 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196912+0000", 2023-08-28T02:24:31.303 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.544e-06 2023-08-28T02:24:31.303 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.303 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.304 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:24:31.304 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.209839+0000", 2023-08-28T02:24:31.304 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.012926524999999999 2023-08-28T02:24:31.304 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.304 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.304 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:24:31.304 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.209845+0000", 2023-08-28T02:24:31.304 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.579e-06 2023-08-28T02:24:31.304 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.304 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.304 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:24:31.304 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.317968+0000", 2023-08-28T02:24:31.305 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.108122914 2023-08-28T02:24:31.305 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.305 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.305 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:24:31.305 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.318146+0000", 2023-08-28T02:24:31.305 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00017759300000000001 2023-08-28T02:24:31.305 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.305 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.305 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,3,4", 2023-08-28T02:24:31.305 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.318278+0000", 2023-08-28T02:24:31.305 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00013184000000000001 2023-08-28T02:24:31.305 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.305 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.306 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:31.306 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325014+0000", 2023-08-28T02:24:31.306 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0067361549999999997 2023-08-28T02:24:31.306 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.306 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.306 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:24:31.306 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325368+0000", 2023-08-28T02:24:31.306 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00035389200000000002 2023-08-28T02:24:31.306 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.306 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.306 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:31.306 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325421+0000", 2023-08-28T02:24:31.307 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.2902000000000003e-05 2023-08-28T02:24:31.307 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.307 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.307 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:31.307 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325612+0000", 2023-08-28T02:24:31.307 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00019104500000000001 2023-08-28T02:24:31.307 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.307 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.307 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:24:31.307 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325640+0000", 2023-08-28T02:24:31.307 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.8425999999999999e-05 2023-08-28T02:24:31.307 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.308 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.308 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:24:31.308 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325721+0000", 2023-08-28T02:24:31.308 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.0642000000000005e-05 2023-08-28T02:24:31.308 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:31.308 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:24:31.308 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:31.308 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.308 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.308 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:123565 5.0 5:0ec9f19d:::benchmark_data_smithi143_236200_object13422:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e154)", 2023-08-28T02:24:31.308 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:59.196632+0000", 2023-08-28T02:24:31.308 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 32.045518946000001, 2023-08-28T02:24:31.309 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.129315396, 2023-08-28T02:24:31.309 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:24:31.309 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:24:31.309 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:24:31.309 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:24:31.309 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:24:31.309 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 123565 2023-08-28T02:24:31.309 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.309 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:24:31.309 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.309 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:24:31.309 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196632+0000", 2023-08-28T02:24:31.310 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:31.310 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.310 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.310 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:24:31.310 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196632+0000", 2023-08-28T02:24:31.310 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:31.310 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.310 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.310 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:24:31.310 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196631+0000", 2023-08-28T02:24:31.310 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:24:31.310 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.310 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.311 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:24:31.311 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196634+0000", 2023-08-28T02:24:31.311 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.0650000000000001e-06 2023-08-28T02:24:31.311 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.311 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.311 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:24:31.311 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196638+0000", 2023-08-28T02:24:31.311 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.6700000000000002e-06 2023-08-28T02:24:31.311 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.311 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.311 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:24:31.311 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196643+0000", 2023-08-28T02:24:31.312 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.7859999999999996e-06 2023-08-28T02:24:31.312 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.312 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.312 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:24:31.312 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196931+0000", 2023-08-28T02:24:31.312 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000288044 2023-08-28T02:24:31.312 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.312 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.312 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting_for_map not empty", 2023-08-28T02:24:31.312 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196935+0000", 2023-08-28T02:24:31.312 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.3930000000000001e-06 2023-08-28T02:24:31.312 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.313 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.313 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:24:31.313 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.209868+0000", 2023-08-28T02:24:31.313 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.012932073000000001 2023-08-28T02:24:31.313 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.313 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.313 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:24:31.313 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.209874+0000", 2023-08-28T02:24:31.313 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.6989999999999997e-06 2023-08-28T02:24:31.313 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.313 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.313 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:24:31.313 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.321422+0000", 2023-08-28T02:24:31.314 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.111547675 2023-08-28T02:24:31.314 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.314 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.314 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:24:31.314 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.321549+0000", 2023-08-28T02:24:31.314 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000127028 2023-08-28T02:24:31.314 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.314 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.314 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,3,4", 2023-08-28T02:24:31.314 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.321630+0000", 2023-08-28T02:24:31.314 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.0882000000000003e-05 2023-08-28T02:24:31.314 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.315 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.315 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:24:31.315 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325428+0000", 2023-08-28T02:24:31.315 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0037977200000000001 2023-08-28T02:24:31.315 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.315 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.315 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:31.315 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325447+0000", 2023-08-28T02:24:31.315 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.906e-05 2023-08-28T02:24:31.315 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.315 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.315 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:31.316 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325752+0000", 2023-08-28T02:24:31.316 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00030524300000000002 2023-08-28T02:24:31.316 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.316 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.316 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:31.316 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325801+0000", 2023-08-28T02:24:31.316 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.9429999999999999e-05 2023-08-28T02:24:31.316 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.316 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.316 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:24:31.316 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325845+0000", 2023-08-28T02:24:31.316 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.3900000000000003e-05 2023-08-28T02:24:31.316 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.316 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.317 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:24:31.317 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325947+0000", 2023-08-28T02:24:31.317 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000102007 2023-08-28T02:24:31.317 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:31.317 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:24:31.317 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:31.317 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.317 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.317 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:123578 5.0 5:0d4223f2:::benchmark_data_smithi143_236200_object13435:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e154)", 2023-08-28T02:24:31.317 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:59.196647+0000", 2023-08-28T02:24:31.317 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 32.045503850999999, 2023-08-28T02:24:31.317 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.14071996, 2023-08-28T02:24:31.318 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:24:31.318 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:24:31.318 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:24:31.318 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:24:31.318 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:24:31.318 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 123578 2023-08-28T02:24:31.318 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.318 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:24:31.318 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.318 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:24:31.318 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196647+0000", 2023-08-28T02:24:31.318 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:31.318 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.319 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.319 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:24:31.319 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196647+0000", 2023-08-28T02:24:31.319 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:31.319 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.319 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.319 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:24:31.319 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196647+0000", 2023-08-28T02:24:31.319 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:24:31.319 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.319 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.319 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:24:31.319 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196649+0000", 2023-08-28T02:24:31.320 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.4449999999999999e-06 2023-08-28T02:24:31.320 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.320 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.320 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:24:31.320 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196654+0000", 2023-08-28T02:24:31.320 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.1139999999999997e-06 2023-08-28T02:24:31.320 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.320 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.320 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:24:31.320 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196658+0000", 2023-08-28T02:24:31.320 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.2420000000000002e-06 2023-08-28T02:24:31.320 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.320 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.321 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:24:31.321 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196950+0000", 2023-08-28T02:24:31.321 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00029176700000000001 2023-08-28T02:24:31.321 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.321 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.321 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting_for_map not empty", 2023-08-28T02:24:31.321 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196953+0000", 2023-08-28T02:24:31.321 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.5359999999999999e-06 2023-08-28T02:24:31.321 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.321 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.321 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:24:31.321 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.209894+0000", 2023-08-28T02:24:31.321 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.012941413000000001 2023-08-28T02:24:31.322 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.322 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.322 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:24:31.322 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.209898+0000", 2023-08-28T02:24:31.322 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.484e-06 2023-08-28T02:24:31.322 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.322 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.322 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:24:31.322 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325997+0000", 2023-08-28T02:24:31.322 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.116099921 2023-08-28T02:24:31.322 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.322 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.323 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:24:31.323 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.326137+0000", 2023-08-28T02:24:31.323 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000139696 2023-08-28T02:24:31.323 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.323 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.323 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,3,4", 2023-08-28T02:24:31.323 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.326268+0000", 2023-08-28T02:24:31.323 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00013096399999999999 2023-08-28T02:24:31.323 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.323 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.323 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:31.323 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.335576+0000", 2023-08-28T02:24:31.324 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0093078999999999992 2023-08-28T02:24:31.324 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.324 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.324 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:31.324 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.335620+0000", 2023-08-28T02:24:31.324 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.3868000000000001e-05 2023-08-28T02:24:31.324 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.324 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.324 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:24:31.324 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.335661+0000", 2023-08-28T02:24:31.324 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.0908999999999997e-05 2023-08-28T02:24:31.324 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.325 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.325 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:31.325 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.337193+0000", 2023-08-28T02:24:31.325 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0015320399999999999 2023-08-28T02:24:31.325 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.325 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.325 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:24:31.325 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.337245+0000", 2023-08-28T02:24:31.325 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.2092999999999997e-05 2023-08-28T02:24:31.325 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.325 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.325 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:24:31.325 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.337367+0000", 2023-08-28T02:24:31.325 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000121902 2023-08-28T02:24:31.326 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:31.326 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:24:31.326 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:31.326 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.326 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.326 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:123581 5.0 5:06f44fcd:::benchmark_data_smithi143_236200_object13438:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e154)", 2023-08-28T02:24:31.326 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:59.196662+0000", 2023-08-28T02:24:31.326 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 32.045488274999997, 2023-08-28T02:24:31.326 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.14107384, 2023-08-28T02:24:31.326 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:24:31.326 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:24:31.326 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:24:31.327 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:24:31.327 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:24:31.327 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 123581 2023-08-28T02:24:31.327 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.327 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:24:31.327 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.327 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:24:31.327 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196662+0000", 2023-08-28T02:24:31.327 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:31.327 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.327 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.327 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:24:31.327 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196662+0000", 2023-08-28T02:24:31.327 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:31.328 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.328 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.328 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:24:31.328 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196662+0000", 2023-08-28T02:24:31.328 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:24:31.328 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.328 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.328 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:24:31.328 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196664+0000", 2023-08-28T02:24:31.328 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.6810000000000001e-06 2023-08-28T02:24:31.328 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.328 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.328 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:24:31.329 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196669+0000", 2023-08-28T02:24:31.329 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.0610000000000003e-06 2023-08-28T02:24:31.329 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.329 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.329 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:24:31.329 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196673+0000", 2023-08-28T02:24:31.329 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.2080000000000002e-06 2023-08-28T02:24:31.329 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.329 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.329 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:24:31.329 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196963+0000", 2023-08-28T02:24:31.329 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00029004399999999999 2023-08-28T02:24:31.329 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.330 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.330 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting_for_map not empty", 2023-08-28T02:24:31.330 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196966+0000", 2023-08-28T02:24:31.330 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.362e-06 2023-08-28T02:24:31.330 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.330 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.330 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:24:31.330 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.209987+0000", 2023-08-28T02:24:31.330 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.013021624000000001 2023-08-28T02:24:31.330 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.330 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.330 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:24:31.330 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.209994+0000", 2023-08-28T02:24:31.331 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.6440000000000003e-06 2023-08-28T02:24:31.331 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.331 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.331 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:24:31.331 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.332359+0000", 2023-08-28T02:24:31.331 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1223654620000001 2023-08-28T02:24:31.331 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.331 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.331 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:24:31.331 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.332502+0000", 2023-08-28T02:24:31.331 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00014240999999999999 2023-08-28T02:24:31.331 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.331 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.332 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,3,4", 2023-08-28T02:24:31.332 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.332652+0000", 2023-08-28T02:24:31.332 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000149853 2023-08-28T02:24:31.332 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.332 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.332 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:31.332 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.335652+0000", 2023-08-28T02:24:31.332 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0030004179999999999 2023-08-28T02:24:31.332 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.332 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.332 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:24:31.332 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.335671+0000", 2023-08-28T02:24:31.332 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.9049e-05 2023-08-28T02:24:31.333 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.333 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.333 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:31.333 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.336202+0000", 2023-08-28T02:24:31.333 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00053052899999999996 2023-08-28T02:24:31.333 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.333 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.333 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:31.333 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.337555+0000", 2023-08-28T02:24:31.333 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0013538479999999999 2023-08-28T02:24:31.333 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.333 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.333 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:24:31.334 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.337610+0000", 2023-08-28T02:24:31.334 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.4595000000000002e-05 2023-08-28T02:24:31.334 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.334 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.334 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:24:31.334 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.337736+0000", 2023-08-28T02:24:31.334 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00012634199999999999 2023-08-28T02:24:31.334 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:31.334 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:24:31.334 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:31.334 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.334 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.334 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:124089 5.0 5:06d6dacb:::benchmark_data_smithi143_236200_object13946:head [delete] snapc 0=[] ondisk+write+known_if_redirected e155)", 2023-08-28T02:24:31.335 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:24:01.180357+0000", 2023-08-28T02:24:31.335 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 30.061793629, 2023-08-28T02:24:31.335 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.22593939499999999, 2023-08-28T02:24:31.335 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:24:31.335 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:24:31.335 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:24:31.335 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:24:31.335 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:24:31.335 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 124089 2023-08-28T02:24:31.335 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.335 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:24:31.335 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.335 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:24:31.336 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.180357+0000", 2023-08-28T02:24:31.336 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:31.336 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.336 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.336 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:24:31.336 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.180357+0000", 2023-08-28T02:24:31.336 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:31.336 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.336 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.336 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:24:31.336 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.180356+0000", 2023-08-28T02:24:31.336 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:24:31.336 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.337 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.337 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:24:31.337 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.180363+0000", 2023-08-28T02:24:31.337 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.0899999999999999e-06 2023-08-28T02:24:31.337 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.337 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.337 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:24:31.337 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.180381+0000", 2023-08-28T02:24:31.337 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.8519e-05 2023-08-28T02:24:31.337 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.337 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.337 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:24:31.338 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.180391+0000", 2023-08-28T02:24:31.338 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0192e-05 2023-08-28T02:24:31.338 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.338 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.338 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:24:31.338 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.180456+0000", 2023-08-28T02:24:31.338 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.4084999999999994e-05 2023-08-28T02:24:31.338 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.338 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.338 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:24:31.338 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.180607+0000", 2023-08-28T02:24:31.338 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000151232 2023-08-28T02:24:31.338 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.339 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.339 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,3,4", 2023-08-28T02:24:31.339 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.181058+0000", 2023-08-28T02:24:31.339 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00045131499999999998 2023-08-28T02:24:31.339 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.339 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.339 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:24:31.339 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.187145+0000", 2023-08-28T02:24:31.339 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0060865600000000004 2023-08-28T02:24:31.339 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.339 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.339 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:31.339 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.187725+0000", 2023-08-28T02:24:31.340 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00058020299999999995 2023-08-28T02:24:31.340 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.340 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.340 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:31.340 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.189158+0000", 2023-08-28T02:24:31.340 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.001433134 2023-08-28T02:24:31.340 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.340 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.340 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:31.340 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.406130+0000", 2023-08-28T02:24:31.340 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.216971527 2023-08-28T02:24:31.340 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.340 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.341 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:24:31.341 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.406181+0000", 2023-08-28T02:24:31.341 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.1264999999999998e-05 2023-08-28T02:24:31.341 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.341 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.341 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:24:31.341 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.406296+0000", 2023-08-28T02:24:31.341 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000115686 2023-08-28T02:24:31.341 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:31.341 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:24:31.341 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:31.341 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.341 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.342 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:130455 5.3 5:c62e95e0:::benchmark_data_smithi143_236200_object20312:head [delete] snapc 0=[] ondisk+write+known_if_redirected e163)", 2023-08-28T02:24:31.342 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:24:13.094164+0000", 2023-08-28T02:24:31.342 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 18.147987183000001, 2023-08-28T02:24:31.342 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.23304668100000001, 2023-08-28T02:24:31.342 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:24:31.342 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:24:31.342 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:24:31.342 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:24:31.342 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:24:31.342 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 130455 2023-08-28T02:24:31.342 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.342 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:24:31.342 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.343 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:24:31.343 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.094164+0000", 2023-08-28T02:24:31.343 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:31.343 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.343 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.343 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:24:31.343 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.094164+0000", 2023-08-28T02:24:31.343 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:31.343 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.343 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.343 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:24:31.343 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.094162+0000", 2023-08-28T02:24:31.343 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:24:31.344 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.344 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.344 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:24:31.344 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.094168+0000", 2023-08-28T02:24:31.344 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.5480000000000001e-06 2023-08-28T02:24:31.344 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.344 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.344 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:24:31.344 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.200430+0000", 2023-08-28T02:24:31.344 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.106262407 2023-08-28T02:24:31.344 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.344 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.344 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:24:31.344 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.200446+0000", 2023-08-28T02:24:31.345 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.5396e-05 2023-08-28T02:24:31.345 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.345 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.345 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:24:31.345 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.208625+0000", 2023-08-28T02:24:31.345 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0081796609999999995 2023-08-28T02:24:31.345 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.345 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.345 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:24:31.345 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.208750+0000", 2023-08-28T02:24:31.345 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000124692 2023-08-28T02:24:31.345 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.345 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.345 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 5,6", 2023-08-28T02:24:31.346 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.208854+0000", 2023-08-28T02:24:31.346 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000103959 2023-08-28T02:24:31.346 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.346 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.346 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:24:31.346 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.212292+0000", 2023-08-28T02:24:31.346 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0034382129999999999 2023-08-28T02:24:31.346 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.346 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.346 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:31.346 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.327100+0000", 2023-08-28T02:24:31.346 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.114808038 2023-08-28T02:24:31.346 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.347 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.347 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:31.347 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.327129+0000", 2023-08-28T02:24:31.347 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.8972e-05 2023-08-28T02:24:31.347 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.347 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.347 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:24:31.347 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.327172+0000", 2023-08-28T02:24:31.347 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.3093000000000002e-05 2023-08-28T02:24:31.347 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.347 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.347 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:24:31.347 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.327210+0000", 2023-08-28T02:24:31.347 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.7783000000000001e-05 2023-08-28T02:24:31.349 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:31.350 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:24:31.350 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:31.350 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.350 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.350 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:140325 5.0 5:083c5755:::benchmark_data_smithi143_236200_object30182:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-08-28T02:24:31.350 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:24:28.047973+0000", 2023-08-28T02:24:31.350 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 3.1941775489999999, 2023-08-28T02:24:31.350 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.19925366999999999, 2023-08-28T02:24:31.350 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:24:31.350 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:24:31.350 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:24:31.350 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:24:31.350 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:24:31.351 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 140325 2023-08-28T02:24:31.351 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.351 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:24:31.351 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.351 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:24:31.351 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.047973+0000", 2023-08-28T02:24:31.351 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:31.351 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.351 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.351 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:24:31.351 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.047973+0000", 2023-08-28T02:24:31.351 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:31.351 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.352 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.352 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:24:31.352 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.047972+0000", 2023-08-28T02:24:31.352 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:24:31.352 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.352 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.352 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:24:31.352 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.047977+0000", 2023-08-28T02:24:31.352 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.1529999999999996e-06 2023-08-28T02:24:31.352 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.352 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.352 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:24:31.353 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.047985+0000", 2023-08-28T02:24:31.353 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.9980000000000003e-06 2023-08-28T02:24:31.353 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.353 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.353 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:24:31.353 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.047990+0000", 2023-08-28T02:24:31.353 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.0370000000000002e-06 2023-08-28T02:24:31.353 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.353 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.353 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:24:31.353 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.048044+0000", 2023-08-28T02:24:31.353 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.3211000000000003e-05 2023-08-28T02:24:31.353 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.353 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.354 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:24:31.354 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.048164+0000", 2023-08-28T02:24:31.354 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00012014 2023-08-28T02:24:31.354 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.354 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.354 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,3,4", 2023-08-28T02:24:31.354 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.048255+0000", 2023-08-28T02:24:31.354 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.1153e-05 2023-08-28T02:24:31.354 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.354 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.354 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:31.354 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.052016+0000", 2023-08-28T02:24:31.355 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.003761014 2023-08-28T02:24:31.355 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.355 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.355 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:31.355 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.240551+0000", 2023-08-28T02:24:31.355 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.18853502899999999 2023-08-28T02:24:31.355 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.355 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.355 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:24:31.355 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.244719+0000", 2023-08-28T02:24:31.355 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0041684679999999998 2023-08-28T02:24:31.355 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.356 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.356 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:31.356 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.247065+0000", 2023-08-28T02:24:31.356 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0023457529999999999 2023-08-28T02:24:31.356 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.356 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.356 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:24:31.356 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.247114+0000", 2023-08-28T02:24:31.356 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.8557000000000003e-05 2023-08-28T02:24:31.356 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.356 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.356 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:24:31.356 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.247227+0000", 2023-08-28T02:24:31.357 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00011322700000000001 2023-08-28T02:24:31.357 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:31.357 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:24:31.357 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:31.357 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.357 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.357 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.44594.0:140322 5.2 e165/119 5:42801a01:::benchmark_data_smithi143_236200_object30179:head v 165'8655, mlcod=165'8652)", 2023-08-28T02:24:31.357 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:24:28.048354+0000", 2023-08-28T02:24:31.357 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 3.1937967669999998, 2023-08-28T02:24:31.357 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.19222331000000001, 2023-08-28T02:24:31.357 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:24:31.357 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:24:31.357 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:24:31.358 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.358 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:24:31.358 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.048354+0000", 2023-08-28T02:24:31.358 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:31.358 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.358 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.358 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:24:31.358 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.048354+0000", 2023-08-28T02:24:31.358 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:31.358 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.358 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.358 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:24:31.358 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.048354+0000", 2023-08-28T02:24:31.359 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:24:31.359 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.359 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.359 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:24:31.359 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.048362+0000", 2023-08-28T02:24:31.359 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.1960000000000008e-06 2023-08-28T02:24:31.359 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.359 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.359 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:24:31.359 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.048369+0000", 2023-08-28T02:24:31.359 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.6290000000000001e-06 2023-08-28T02:24:31.359 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.359 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.360 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:24:31.360 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.048375+0000", 2023-08-28T02:24:31.360 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.609e-06 2023-08-28T02:24:31.360 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.360 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.360 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:24:31.360 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.048406+0000", 2023-08-28T02:24:31.360 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.0777e-05 2023-08-28T02:24:31.360 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.360 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.360 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:24:31.360 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.048425+0000", 2023-08-28T02:24:31.360 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.8938000000000001e-05 2023-08-28T02:24:31.360 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.361 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.361 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:24:31.361 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.240536+0000", 2023-08-28T02:24:31.361 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.19211083800000001 2023-08-28T02:24:31.361 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.361 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.361 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:24:31.361 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.240577+0000", 2023-08-28T02:24:31.361 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.1715999999999998e-05 2023-08-28T02:24:31.361 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:31.361 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:24:31.361 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:31.361 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.361 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.362 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.44594.0:140326 5.8 e165/119 5:1f9b0754:::benchmark_data_smithi143_236200_object30183:head v 165'8787, mlcod=165'8785)", 2023-08-28T02:24:31.362 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:24:28.051347+0000", 2023-08-28T02:24:31.362 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 3.1908033749999998, 2023-08-28T02:24:31.362 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.193406788, 2023-08-28T02:24:31.362 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:24:31.362 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:24:31.362 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:24:31.362 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.362 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:24:31.362 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.051347+0000", 2023-08-28T02:24:31.362 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:31.362 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.362 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.363 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:24:31.363 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.051347+0000", 2023-08-28T02:24:31.363 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:31.363 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.363 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.363 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:24:31.363 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.051346+0000", 2023-08-28T02:24:31.363 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:24:31.363 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.363 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.363 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:24:31.363 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.051355+0000", 2023-08-28T02:24:31.363 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.5640000000000002e-06 2023-08-28T02:24:31.364 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.364 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.364 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:24:31.364 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.051369+0000", 2023-08-28T02:24:31.364 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3141e-05 2023-08-28T02:24:31.364 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.364 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.364 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:24:31.364 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.051377+0000", 2023-08-28T02:24:31.364 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.1950000000000003e-06 2023-08-28T02:24:31.364 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.364 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.364 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:24:31.365 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.051418+0000", 2023-08-28T02:24:31.365 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.0759000000000001e-05 2023-08-28T02:24:31.365 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.365 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.365 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:24:31.365 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.051438+0000", 2023-08-28T02:24:31.365 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.0622999999999999e-05 2023-08-28T02:24:31.365 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.365 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.365 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:24:31.365 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.244721+0000", 2023-08-28T02:24:31.365 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.19328314699999999 2023-08-28T02:24:31.366 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.366 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.366 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:24:31.366 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.244754+0000", 2023-08-28T02:24:31.366 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.2818999999999999e-05 2023-08-28T02:24:31.366 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:31.366 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:24:31.366 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:31.366 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.366 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.366 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:140327 5.5 5:a6656200:::benchmark_data_smithi143_236200_object30184:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-08-28T02:24:31.366 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:24:28.051781+0000", 2023-08-28T02:24:31.367 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 3.1903699350000001, 2023-08-28T02:24:31.367 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.193112701, 2023-08-28T02:24:31.367 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:24:31.367 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:24:31.367 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:24:31.367 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:24:31.367 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:24:31.367 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 140327 2023-08-28T02:24:31.367 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.367 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:24:31.367 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.367 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:24:31.367 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.051781+0000", 2023-08-28T02:24:31.368 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:31.368 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.368 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.368 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:24:31.368 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.051781+0000", 2023-08-28T02:24:31.368 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:31.368 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.368 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.368 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:24:31.368 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.051779+0000", 2023-08-28T02:24:31.368 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:24:31.368 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.368 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.369 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:24:31.369 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.051786+0000", 2023-08-28T02:24:31.369 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.2659999999999998e-06 2023-08-28T02:24:31.369 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.369 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.369 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:24:31.369 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.051797+0000", 2023-08-28T02:24:31.369 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1583000000000001e-05 2023-08-28T02:24:31.369 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.369 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.369 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:24:31.369 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.051804+0000", 2023-08-28T02:24:31.370 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.2400000000000001e-06 2023-08-28T02:24:31.370 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.370 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.370 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:24:31.370 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.051855+0000", 2023-08-28T02:24:31.370 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.0390999999999999e-05 2023-08-28T02:24:31.370 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.370 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.370 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:24:31.370 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.051970+0000", 2023-08-28T02:24:31.370 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000115386 2023-08-28T02:24:31.370 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.371 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.371 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,4", 2023-08-28T02:24:31.371 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.052101+0000", 2023-08-28T02:24:31.371 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00013023900000000001 2023-08-28T02:24:31.371 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.371 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.371 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:24:31.371 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.240509+0000", 2023-08-28T02:24:31.371 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.18840891100000001 2023-08-28T02:24:31.371 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.371 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.371 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:31.371 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.240745+0000", 2023-08-28T02:24:31.372 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00023606899999999999 2023-08-28T02:24:31.372 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.372 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.372 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:31.372 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.244804+0000", 2023-08-28T02:24:31.372 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0040580939999999999 2023-08-28T02:24:31.372 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.372 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.372 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:24:31.372 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.244847+0000", 2023-08-28T02:24:31.372 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.2988999999999998e-05 2023-08-28T02:24:31.372 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.372 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.373 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:24:31.373 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.244893+0000", 2023-08-28T02:24:31.373 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.6924e-05 2023-08-28T02:24:31.373 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:31.373 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:24:31.373 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:31.373 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.373 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.373 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:140334 5.0 5:0466772d:::benchmark_data_smithi143_236200_object30191:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-08-28T02:24:31.373 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:24:28.055937+0000", 2023-08-28T02:24:31.373 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 3.1862138770000001, 2023-08-28T02:24:31.373 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.25254626600000002, 2023-08-28T02:24:31.374 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:24:31.374 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:24:31.374 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:24:31.374 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:24:31.374 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:24:31.374 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 140334 2023-08-28T02:24:31.374 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.374 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:24:31.374 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.374 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:24:31.374 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.055937+0000", 2023-08-28T02:24:31.374 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:31.374 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.375 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.375 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:24:31.375 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.055937+0000", 2023-08-28T02:24:31.375 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:31.375 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.375 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.375 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:24:31.375 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.055936+0000", 2023-08-28T02:24:31.375 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:24:31.375 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.375 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.375 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:24:31.375 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.055942+0000", 2023-08-28T02:24:31.376 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.9519999999999999e-06 2023-08-28T02:24:31.376 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.376 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.376 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:24:31.376 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.240512+0000", 2023-08-28T02:24:31.376 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.18457072699999999 2023-08-28T02:24:31.376 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.376 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.376 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:24:31.376 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.240525+0000", 2023-08-28T02:24:31.376 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.2515e-05 2023-08-28T02:24:31.376 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.376 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.377 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:24:31.377 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.240582+0000", 2023-08-28T02:24:31.377 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.6839000000000002e-05 2023-08-28T02:24:31.377 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.377 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.377 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:24:31.377 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.240726+0000", 2023-08-28T02:24:31.377 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0001448 2023-08-28T02:24:31.377 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.377 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.377 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,3,4", 2023-08-28T02:24:31.377 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.240865+0000", 2023-08-28T02:24:31.377 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000138601 2023-08-28T02:24:31.378 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.378 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.378 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:24:31.378 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.245256+0000", 2023-08-28T02:24:31.378 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0043909229999999997 2023-08-28T02:24:31.378 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.378 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.378 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:31.378 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.247591+0000", 2023-08-28T02:24:31.378 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0023347799999999998 2023-08-28T02:24:31.378 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.378 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.379 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:31.379 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.256080+0000", 2023-08-28T02:24:31.379 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0084893469999999995 2023-08-28T02:24:31.379 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.379 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.379 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:31.379 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.308312+0000", 2023-08-28T02:24:31.379 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.052232369000000001 2023-08-28T02:24:31.379 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.379 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.379 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:24:31.379 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.308369+0000", 2023-08-28T02:24:31.379 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.6746000000000002e-05 2023-08-28T02:24:31.380 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.380 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.380 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:24:31.380 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.308483+0000", 2023-08-28T02:24:31.380 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00011390700000000001 2023-08-28T02:24:31.380 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:31.380 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:24:31.380 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:31.380 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.380 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.380 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.44594.0:140332 5.c e165/119 5:38d68a55:::benchmark_data_smithi143_236200_object30189:head v 165'8978, mlcod=165'8976)", 2023-08-28T02:24:31.380 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:24:28.056257+0000", 2023-08-28T02:24:31.381 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 3.1858937749999998, 2023-08-28T02:24:31.381 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.195568192, 2023-08-28T02:24:31.381 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:24:31.381 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:24:31.381 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:24:31.381 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.381 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:24:31.381 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.056257+0000", 2023-08-28T02:24:31.381 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:31.381 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.381 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.381 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:24:31.381 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.056257+0000", 2023-08-28T02:24:31.382 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:31.382 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.382 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.382 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:24:31.382 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.056256+0000", 2023-08-28T02:24:31.382 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:24:31.382 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.382 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.382 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:24:31.382 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.056263+0000", 2023-08-28T02:24:31.382 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.365e-06 2023-08-28T02:24:31.382 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.383 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.383 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:24:31.383 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.240498+0000", 2023-08-28T02:24:31.383 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.18423572199999999 2023-08-28T02:24:31.383 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.383 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.383 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:24:31.383 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.240514+0000", 2023-08-28T02:24:31.383 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.5529999999999999e-05 2023-08-28T02:24:31.383 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.383 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.383 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:24:31.383 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.240576+0000", 2023-08-28T02:24:31.384 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.2249999999999995e-05 2023-08-28T02:24:31.384 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.384 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.384 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:24:31.384 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.240602+0000", 2023-08-28T02:24:31.384 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.5905999999999999e-05 2023-08-28T02:24:31.384 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.384 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.384 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:24:31.384 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.251794+0000", 2023-08-28T02:24:31.384 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.01119174 2023-08-28T02:24:31.384 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.384 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.385 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:24:31.385 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.251825+0000", 2023-08-28T02:24:31.385 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.1244000000000001e-05 2023-08-28T02:24:31.385 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:31.385 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:24:31.385 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:31.385 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.385 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.385 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.44594.0:140335 5.b e165/119 5:d5c4b0d2:::benchmark_data_smithi143_236200_object30192:head v 165'8688, mlcod=165'8685)", 2023-08-28T02:24:31.385 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:24:28.056719+0000", 2023-08-28T02:24:31.385 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 3.185431371, 2023-08-28T02:24:31.385 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.19169755399999999, 2023-08-28T02:24:31.385 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:24:31.385 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:24:31.386 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:24:31.386 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.386 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:24:31.386 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.056719+0000", 2023-08-28T02:24:31.386 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:31.386 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.386 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.386 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:24:31.386 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.056719+0000", 2023-08-28T02:24:31.386 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:31.386 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.386 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.386 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:24:31.387 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.056719+0000", 2023-08-28T02:24:31.387 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:24:31.387 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.387 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.387 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:24:31.387 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.056726+0000", 2023-08-28T02:24:31.387 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.7209999999999994e-06 2023-08-28T02:24:31.387 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.387 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.387 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:24:31.387 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.240498+0000", 2023-08-28T02:24:31.387 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.18377210899999999 2023-08-28T02:24:31.387 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.388 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.388 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:24:31.388 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.240514+0000", 2023-08-28T02:24:31.388 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.5531000000000001e-05 2023-08-28T02:24:31.388 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.388 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.388 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:24:31.388 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.240612+0000", 2023-08-28T02:24:31.388 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.8480999999999994e-05 2023-08-28T02:24:31.388 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.388 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.388 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:24:31.388 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.240633+0000", 2023-08-28T02:24:31.389 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.0602e-05 2023-08-28T02:24:31.389 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.389 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.389 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:24:31.389 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.248390+0000", 2023-08-28T02:24:31.389 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.007756944 2023-08-28T02:24:31.389 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:31.389 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:31.389 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:24:31.389 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.248417+0000", 2023-08-28T02:24:31.389 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.688e-05 2023-08-28T02:24:31.389 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:31.389 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:24:31.390 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:31.390 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:31.390 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:24:31.390 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:24:31.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:31.391 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:24:31.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:24:31.391 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is set 2023-08-28T02:24:31.460 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:31 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1703309834' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:24:31.460 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:31 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:24:31.461 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:31 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 668/240843 objects degraded (0.277%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:24:31.461 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:31 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1703309834' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:24:31.461 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:31 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:24:31.461 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:31 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 668/240843 objects degraded (0.277%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:24:31.492 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:24:31.535 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:31 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1703309834' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:24:31.535 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:31 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:24:31.535 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:31 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 668/240843 objects degraded (0.277%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:24:31.535 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:31.614 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:24:31.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:31.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:24:31.719 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:24:31.765 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:31.868 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:24:31.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:31.880 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:24:31.980 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:24:32.040 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:32.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:24:31 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:24:31] "GET /metrics HTTP/1.1" 200 36488 "" "Prometheus/2.33.4" 2023-08-28T02:24:32.542 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7, 3] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-08-28T02:24:32.542 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd reweight 0 1 2023-08-28T02:24:32.566 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:32 smithi143 ceph-mon[183059]: pgmap v347: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 668/240843 objects degraded (0.277%); 14250/240843 objects misplaced (5.917%); 1.3 MiB/s, 20 objects/s recovering 2023-08-28T02:24:32.566 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:32 smithi143 ceph-mon[183059]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:24:32.566 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:32 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:24:32.566 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:32 smithi143 ceph-mon[183059]: osdmap e166: 8 total, 8 up, 8 in 2023-08-28T02:24:32.566 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:32 smithi143 ceph-mon[183059]: failed to encode map e166 with expected crc 2023-08-28T02:24:32.566 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:32 smithi143 ceph-mon[189871]: pgmap v347: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 668/240843 objects degraded (0.277%); 14250/240843 objects misplaced (5.917%); 1.3 MiB/s, 20 objects/s recovering 2023-08-28T02:24:32.566 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:32 smithi143 ceph-mon[189871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:24:32.567 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:32 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:24:32.567 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:32 smithi143 ceph-mon[189871]: osdmap e166: 8 total, 8 up, 8 in 2023-08-28T02:24:32.567 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:32 smithi143 ceph-mon[189871]: failed to encode map e166 with expected crc 2023-08-28T02:24:32.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:32 smithi169 ceph-mon[159415]: pgmap v347: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 668/240843 objects degraded (0.277%); 14250/240843 objects misplaced (5.917%); 1.3 MiB/s, 20 objects/s recovering 2023-08-28T02:24:32.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:32 smithi169 ceph-mon[159415]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:24:32.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:32 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:24:32.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:32 smithi169 ceph-mon[159415]: osdmap e166: 8 total, 8 up, 8 in 2023-08-28T02:24:32.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:32 smithi169 ceph-mon[159415]: failed to encode map e166 with expected crc 2023-08-28T02:24:33.016 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:24:33.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:33.019 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:24:33.120 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:24:33.213 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:33.215 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.1 dump_ops_in_flight 2023-08-28T02:24:33.310 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:33 smithi143 ceph-mon[189871]: pgmap v349: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 668/236436 objects degraded (0.283%); 14056/236436 objects misplaced (5.945%); 1.3 MiB/s, 20 objects/s recovering 2023-08-28T02:24:33.314 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:33 smithi143 ceph-mon[183059]: pgmap v349: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 668/236436 objects degraded (0.283%); 14056/236436 objects misplaced (5.945%); 1.3 MiB/s, 20 objects/s recovering 2023-08-28T02:24:33.338 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:24:33.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:33.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:24:33.388 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:24:33.453 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:24:33.527 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:24:33.583 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:33.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:33 smithi169 ceph-mon[159415]: pgmap v349: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 668/236436 objects degraded (0.283%); 14056/236436 objects misplaced (5.945%); 1.3 MiB/s, 20 objects/s recovering 2023-08-28T02:24:33.848 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:24:33.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:33.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:24:33.960 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:24:34.031 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:34.710 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:24:34.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:34.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:24:34.816 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:24:34.842 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:34.922 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:24:34.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:34.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:24:35.029 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:24:35.057 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:35.139 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset noscrub 2023-08-28T02:24:35.398 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:24:35.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:35.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:24:35.502 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:24:35.532 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:35.573 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:24:35.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:35.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:24:35.680 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:24:35.741 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:36.032 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:24:36.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:36.037 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:24:36.038 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:24:36.138 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:24:36.260 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:36.315 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:36 smithi143 ceph-mon[183059]: pgmap v350: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 763/231015 objects degraded (0.330%); 13655/231015 objects misplaced (5.911%); 1.3 MiB/s, 20 objects/s recovering 2023-08-28T02:24:36.316 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:36 smithi143 ceph-mon[189871]: pgmap v350: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 763/231015 objects degraded (0.330%); 13655/231015 objects misplaced (5.911%); 1.3 MiB/s, 20 objects/s recovering 2023-08-28T02:24:36.409 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:24:36.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:36.422 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:24:36.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:36 smithi169 ceph-mon[159415]: pgmap v350: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 763/231015 objects degraded (0.330%); 13655/231015 objects misplaced (5.911%); 1.3 MiB/s, 20 objects/s recovering 2023-08-28T02:24:36.523 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:24:36.592 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:36.677 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:24:36.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:36.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:24:36.784 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:24:36.809 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:24:36.815 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:36.869 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:24:36.887 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:24:36.901 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:24:36.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:36.902 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:24:36.933 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:24:36.969 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:24:37.003 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:24:37.004 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:24:37.048 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:24:37.049 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:37.096 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:24:37.098 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:24:37.098 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:24:37.098 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:24:37.098 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:24:37.098 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:24:37.098 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:24:37.099 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:24:37.099 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:24:37.120 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:24:37.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:37.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:24:37.198 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:37 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 763/231015 objects degraded (0.330%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:24:37.198 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:37 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1754012671' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:24:37.233 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:24:37.286 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:37.368 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:24:37.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:37.376 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:24:37.455 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:37 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 763/231015 objects degraded (0.330%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:24:37.455 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:37 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1754012671' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:24:37.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:37 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 763/231015 objects degraded (0.330%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:24:37.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:37 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1754012671' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:24:37.477 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:24:37.594 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:38.043 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:24:38.252 INFO:teuthology.orchestra.run.smithi143.stderr:reweighted osd.0 to 1 (10000) 2023-08-28T02:24:38.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:38 smithi143 ceph-mon[183059]: pgmap v351: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 763/227295 objects degraded (0.336%); 13498/227295 objects misplaced (5.939%); 1.3 MiB/s, 20 objects/s recovering 2023-08-28T02:24:38.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:38 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3584973520' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-08-28T02:24:38.704 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:38 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-08-28T02:24:38.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:38 smithi143 ceph-mon[189871]: pgmap v351: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 763/227295 objects degraded (0.336%); 13498/227295 objects misplaced (5.939%); 1.3 MiB/s, 20 objects/s recovering 2023-08-28T02:24:38.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:38 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3584973520' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-08-28T02:24:38.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:38 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-08-28T02:24:38.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:38 smithi169 ceph-mon[159415]: pgmap v351: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 763/227295 objects degraded (0.336%); 13498/227295 objects misplaced (5.939%); 1.3 MiB/s, 20 objects/s recovering 2023-08-28T02:24:38.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:38 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3584973520' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-08-28T02:24:38.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:38 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-08-28T02:24:39.128 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:24:39.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:39.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:24:39.247 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:24:39.315 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:39.455 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:39 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-08-28T02:24:39.455 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:39 smithi143 ceph-mon[183059]: osdmap e167: 8 total, 8 up, 8 in 2023-08-28T02:24:39.455 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:39 smithi143 ceph-mon[183059]: failed to encode map e167 with expected crc 2023-08-28T02:24:39.455 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:39 smithi143 ceph-mon[183059]: failed to encode map e167 with expected crc 2023-08-28T02:24:39.455 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:39 smithi143 ceph-mon[183059]: failed to encode map e167 with expected crc 2023-08-28T02:24:39.455 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:39 smithi143 ceph-mon[183059]: failed to encode map e167 with expected crc 2023-08-28T02:24:39.455 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:39 smithi143 ceph-mon[183059]: failed to encode map e167 with expected crc 2023-08-28T02:24:39.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:39 smithi143 ceph-mon[183059]: failed to encode map e167 with expected crc 2023-08-28T02:24:39.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:39 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-08-28T02:24:39.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:39 smithi143 ceph-mon[189871]: osdmap e167: 8 total, 8 up, 8 in 2023-08-28T02:24:39.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:39 smithi143 ceph-mon[189871]: failed to encode map e167 with expected crc 2023-08-28T02:24:39.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:39 smithi143 ceph-mon[189871]: failed to encode map e167 with expected crc 2023-08-28T02:24:39.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:39 smithi143 ceph-mon[189871]: failed to encode map e167 with expected crc 2023-08-28T02:24:39.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:39 smithi143 ceph-mon[189871]: failed to encode map e167 with expected crc 2023-08-28T02:24:39.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:39 smithi143 ceph-mon[189871]: failed to encode map e167 with expected crc 2023-08-28T02:24:39.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:39 smithi143 ceph-mon[189871]: failed to encode map e167 with expected crc 2023-08-28T02:24:39.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:39 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-08-28T02:24:39.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:39 smithi169 ceph-mon[159415]: osdmap e167: 8 total, 8 up, 8 in 2023-08-28T02:24:39.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:39 smithi169 ceph-mon[159415]: failed to encode map e167 with expected crc 2023-08-28T02:24:39.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:39 smithi169 ceph-mon[159415]: failed to encode map e167 with expected crc 2023-08-28T02:24:39.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:39 smithi169 ceph-mon[159415]: failed to encode map e167 with expected crc 2023-08-28T02:24:39.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:39 smithi169 ceph-mon[159415]: failed to encode map e167 with expected crc 2023-08-28T02:24:39.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:39 smithi169 ceph-mon[159415]: failed to encode map e167 with expected crc 2023-08-28T02:24:39.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:39 smithi169 ceph-mon[159415]: failed to encode map e167 with expected crc 2023-08-28T02:24:39.879 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:24:39.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:39.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:24:39.944 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd reweight 1 1 2023-08-28T02:24:39.993 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:24:40.083 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:40.407 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:40 smithi143 ceph-mon[189871]: pgmap v353: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 872/221907 objects degraded (0.393%); 13105/221907 objects misplaced (5.906%); 1.5 MiB/s, 23 objects/s recovering 2023-08-28T02:24:40.408 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:40 smithi143 ceph-mon[183059]: pgmap v353: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 872/221907 objects degraded (0.393%); 13105/221907 objects misplaced (5.906%); 1.5 MiB/s, 23 objects/s recovering 2023-08-28T02:24:40.430 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:24:40.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:40.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:24:40.493 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:24:40.493 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:24:40.541 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:24:40.611 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:40.652 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:24:40.721 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:24:40.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:40.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:24:40.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:40 smithi169 ceph-mon[159415]: pgmap v353: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 872/221907 objects degraded (0.393%); 13105/221907 objects misplaced (5.906%); 1.5 MiB/s, 23 objects/s recovering 2023-08-28T02:24:40.832 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:24:40.859 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:41.039 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:24:41.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:41 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:24:41.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:41 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 872/221907 objects degraded (0.393%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:24:41.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:41 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:24:41.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:41 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 872/221907 objects degraded (0.393%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:24:41.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:41 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:24:41.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:41 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 872/221907 objects degraded (0.393%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:24:41.784 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:24:41.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:41.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:24:41.901 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:24:41.983 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:42.150 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:24:41 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:24:41] "GET /metrics HTTP/1.1" 200 36491 "" "Prometheus/2.33.4" 2023-08-28T02:24:42.239 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.1 dump_blocked_ops 2023-08-28T02:24:42.252 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:24:42.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:42.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:24:42.371 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:24:42.407 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:42.441 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:42 smithi143 ceph-mon[183059]: pgmap v354: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 872/221907 objects degraded (0.393%); 13105/221907 objects misplaced (5.906%); 1.2 MiB/s, 19 objects/s recovering 2023-08-28T02:24:42.441 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:42 smithi143 ceph-mon[189871]: pgmap v354: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 872/221907 objects degraded (0.393%); 13105/221907 objects misplaced (5.906%); 1.2 MiB/s, 19 objects/s recovering 2023-08-28T02:24:42.502 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:24:42.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:42.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:24:42.610 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:24:42.642 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:42 smithi169 ceph-mon[159415]: pgmap v354: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 872/221907 objects degraded (0.393%); 13105/221907 objects misplaced (5.906%); 1.2 MiB/s, 19 objects/s recovering 2023-08-28T02:24:42.650 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:42.744 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:24:42.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:42.757 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:24:42.858 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:24:42.902 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:42.997 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:24:43.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:43.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:24:43.116 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:24:43.154 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:43.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:43 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:24:43.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:43 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:24:43.613 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:24:43.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:43.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:24:43.680 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:24:43.727 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:24:43.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:43 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:24:43.808 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:44.124 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:24:44.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:44.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:24:44.233 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:24:44.246 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:24:44.312 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:44.363 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is unset 2023-08-28T02:24:44.572 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:44 smithi143 ceph-mon[183059]: pgmap v355: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 872/217509 objects degraded (0.401%); 12922/217509 objects misplaced (5.941%); 1.2 MiB/s, 18 objects/s recovering 2023-08-28T02:24:44.573 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:44 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1275411142' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:24:44.573 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:44 smithi143 ceph-mon[189871]: pgmap v355: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 872/217509 objects degraded (0.401%); 12922/217509 objects misplaced (5.941%); 1.2 MiB/s, 18 objects/s recovering 2023-08-28T02:24:44.573 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:44 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1275411142' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:24:44.583 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:24:44.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:44.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:24:44.697 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:24:44.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:44 smithi169 ceph-mon[159415]: pgmap v355: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 872/217509 objects degraded (0.401%); 12922/217509 objects misplaced (5.941%); 1.2 MiB/s, 18 objects/s recovering 2023-08-28T02:24:44.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:44 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1275411142' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:24:44.776 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:45.397 INFO:teuthology.orchestra.run.smithi143.stderr:reweighted osd.1 to 1 (10000) 2023-08-28T02:24:45.664 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:45 smithi143 ceph-mon[183059]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:24:45.665 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:45 smithi143 ceph-mon[183059]: failed to encode map e168 with expected crc 2023-08-28T02:24:45.665 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:45 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1275411142' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:24:45.665 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:45 smithi143 ceph-mon[183059]: osdmap e168: 8 total, 8 up, 8 in 2023-08-28T02:24:45.665 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:45 smithi143 ceph-mon[183059]: failed to encode map e168 with expected crc 2023-08-28T02:24:45.665 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:45 smithi143 ceph-mon[183059]: failed to encode map e168 with expected crc 2023-08-28T02:24:45.665 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:45 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2378841092' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-08-28T02:24:45.665 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:45 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-08-28T02:24:45.665 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:45 smithi143 ceph-mon[183059]: pgmap v357: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 961/212295 objects degraded (0.453%); 12521/212295 objects misplaced (5.898%); 1.6 MiB/s, 25 objects/s recovering 2023-08-28T02:24:45.673 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:45 smithi143 ceph-mon[189871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:24:45.673 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:45 smithi143 ceph-mon[189871]: failed to encode map e168 with expected crc 2023-08-28T02:24:45.673 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:45 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1275411142' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:24:45.673 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:45 smithi143 ceph-mon[189871]: osdmap e168: 8 total, 8 up, 8 in 2023-08-28T02:24:45.674 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:45 smithi143 ceph-mon[189871]: failed to encode map e168 with expected crc 2023-08-28T02:24:45.674 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:45 smithi143 ceph-mon[189871]: failed to encode map e168 with expected crc 2023-08-28T02:24:45.674 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:45 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2378841092' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-08-28T02:24:45.674 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:45 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-08-28T02:24:45.675 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:45 smithi143 ceph-mon[189871]: pgmap v357: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 961/212295 objects degraded (0.453%); 12521/212295 objects misplaced (5.898%); 1.6 MiB/s, 25 objects/s recovering 2023-08-28T02:24:45.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:45 smithi169 ceph-mon[159415]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:24:45.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:45 smithi169 ceph-mon[159415]: failed to encode map e168 with expected crc 2023-08-28T02:24:45.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:45 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1275411142' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:24:45.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:45 smithi169 ceph-mon[159415]: osdmap e168: 8 total, 8 up, 8 in 2023-08-28T02:24:45.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:45 smithi169 ceph-mon[159415]: failed to encode map e168 with expected crc 2023-08-28T02:24:45.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:45 smithi169 ceph-mon[159415]: failed to encode map e168 with expected crc 2023-08-28T02:24:45.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:45 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2378841092' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-08-28T02:24:45.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:45 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-08-28T02:24:45.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:45 smithi169 ceph-mon[159415]: pgmap v357: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 961/212295 objects degraded (0.453%); 12521/212295 objects misplaced (5.898%); 1.6 MiB/s, 25 objects/s recovering 2023-08-28T02:24:46.389 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:46 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-08-28T02:24:46.389 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:46 smithi143 ceph-mon[183059]: osdmap e169: 8 total, 8 up, 8 in 2023-08-28T02:24:46.389 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:46 smithi143 ceph-mon[183059]: failed to encode map e169 with expected crc 2023-08-28T02:24:46.389 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:46 smithi143 ceph-mon[183059]: failed to encode map e169 with expected crc 2023-08-28T02:24:46.389 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:46 smithi143 ceph-mon[183059]: failed to encode map e169 with expected crc 2023-08-28T02:24:46.389 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:46 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 961/212295 objects degraded (0.453%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:24:46.389 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:46 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-08-28T02:24:46.390 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:46 smithi143 ceph-mon[189871]: osdmap e169: 8 total, 8 up, 8 in 2023-08-28T02:24:46.390 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:46 smithi143 ceph-mon[189871]: failed to encode map e169 with expected crc 2023-08-28T02:24:46.390 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:46 smithi143 ceph-mon[189871]: failed to encode map e169 with expected crc 2023-08-28T02:24:46.390 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:46 smithi143 ceph-mon[189871]: failed to encode map e169 with expected crc 2023-08-28T02:24:46.390 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:46 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 961/212295 objects degraded (0.453%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:24:46.436 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:24:46.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:46.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:24:46.551 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:24:46.628 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:46 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-08-28T02:24:46.628 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:46 smithi169 ceph-mon[159415]: osdmap e169: 8 total, 8 up, 8 in 2023-08-28T02:24:46.628 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:46 smithi169 ceph-mon[159415]: failed to encode map e169 with expected crc 2023-08-28T02:24:46.628 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:46 smithi169 ceph-mon[159415]: failed to encode map e169 with expected crc 2023-08-28T02:24:46.628 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:46 smithi169 ceph-mon[159415]: failed to encode map e169 with expected crc 2023-08-28T02:24:46.628 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:46 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 961/212295 objects degraded (0.453%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:24:46.651 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:46.724 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:24:46.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:46.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:24:46.842 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:24:46.869 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:46.941 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:24:46.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:46.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:24:47.050 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:24:47.099 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:47.203 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:24:47.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:47.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:24:47.315 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:24:47.360 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:47.364 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd reweight 2 1 2023-08-28T02:24:47.455 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:24:47.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:47.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:24:47.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:47 smithi169 ceph-mon[159415]: pgmap v359: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 961/208359 objects degraded (0.461%); 12367/208359 objects misplaced (5.935%); 888 KiB/s, 13 objects/s recovering 2023-08-28T02:24:47.560 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:24:47.628 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:47.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:47 smithi143 ceph-mon[183059]: pgmap v359: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 961/208359 objects degraded (0.461%); 12367/208359 objects misplaced (5.935%); 888 KiB/s, 13 objects/s recovering 2023-08-28T02:24:47.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:47 smithi143 ceph-mon[189871]: pgmap v359: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 961/208359 objects degraded (0.461%); 12367/208359 objects misplaced (5.935%); 888 KiB/s, 13 objects/s recovering 2023-08-28T02:24:47.723 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:24:47.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:47.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:24:47.832 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:24:47.892 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:47.938 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:24:47.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:47.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:24:48.044 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:24:48.076 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:48.149 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:24:48.162 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset nodeep-scrub 2023-08-28T02:24:48.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:48.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:24:48.266 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:24:48.316 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:48.391 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:24:48.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:48.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:24:48.416 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:48 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:24:48.416 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:48 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:24:48.500 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:24:48.517 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:48 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:24:48.517 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:48 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:24:48.517 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:48 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:24:48.518 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:48 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:24:48.530 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:48.595 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:24:48.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:48.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:24:48.701 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:24:48.803 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:48.974 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:24:48.989 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:24:48.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:48.993 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:24:49.024 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:24:49.065 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:24:49.093 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:24:49.113 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:24:49.124 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:49.159 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:24:49.195 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:24:49.200 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:24:49.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:49.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:24:49.230 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:24:49.241 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:24:49.273 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:24:49.296 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:24:49.305 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:24:49.310 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:24:49.310 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:24:49.310 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:24:49.310 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:24:49.310 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:24:49.310 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:24:49.310 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:24:49.311 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:24:49.344 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:49.377 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:24:49.377 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [], 2023-08-28T02:24:49.377 INFO:teuthology.orchestra.run.smithi143.stdout: "complaint_time": 30, 2023-08-28T02:24:49.377 INFO:teuthology.orchestra.run.smithi143.stdout: "num_blocked_ops": 0 2023-08-28T02:24:49.377 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:24:49.416 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:24:49.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:49.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:24:49.521 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:24:49.627 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:49 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2123616047' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:24:49.627 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:49 smithi143 ceph-mon[189871]: pgmap v360: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 1065/203421 objects degraded (0.524%); 11993/203421 objects misplaced (5.896%); 1.7 MiB/s, 26 objects/s recovering 2023-08-28T02:24:49.633 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:49 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2123616047' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:24:49.633 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:49 smithi143 ceph-mon[183059]: pgmap v360: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 1065/203421 objects degraded (0.524%); 11993/203421 objects misplaced (5.896%); 1.7 MiB/s, 26 objects/s recovering 2023-08-28T02:24:49.655 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:49.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:49 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2123616047' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:24:49.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:49 smithi169 ceph-mon[159415]: pgmap v360: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 1065/203421 objects degraded (0.524%); 11993/203421 objects misplaced (5.896%); 1.7 MiB/s, 26 objects/s recovering 2023-08-28T02:24:50.272 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:24:50.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:50.277 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:24:50.378 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:24:50.405 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:51.478 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:24:51.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:51.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:24:51.584 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:24:51.626 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.1 dump_historic_ops 2023-08-28T02:24:51.631 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:51.716 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:24:51.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:51.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:24:51.822 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:24:51.867 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:51.930 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:24:51.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:51.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:24:51.952 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:24:51 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:24:51] "GET /metrics HTTP/1.1" 200 36491 "" "Prometheus/2.33.4" 2023-08-28T02:24:52.042 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:24:52.103 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:52.186 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:24:52.197 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:52 smithi169 ceph-mon[159415]: pgmap v361: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 1065/203421 objects degraded (0.524%); 11993/203421 objects misplaced (5.896%); 1.7 MiB/s, 26 objects/s recovering 2023-08-28T02:24:52.197 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:52 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 1065/203421 objects degraded (0.524%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:24:52.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:52.198 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:24:52.298 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:24:52.401 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:52.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:52 smithi143 ceph-mon[183059]: pgmap v361: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 1065/203421 objects degraded (0.524%); 11993/203421 objects misplaced (5.896%); 1.7 MiB/s, 26 objects/s recovering 2023-08-28T02:24:52.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:52 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 1065/203421 objects degraded (0.524%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:24:52.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:52 smithi143 ceph-mon[189871]: pgmap v361: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 1065/203421 objects degraded (0.524%); 11993/203421 objects misplaced (5.896%); 1.7 MiB/s, 26 objects/s recovering 2023-08-28T02:24:52.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:52 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 1065/203421 objects degraded (0.524%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:24:53.305 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:24:53.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:53.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:24:53.408 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:24:53.489 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:53.506 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:24:53.640 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:24:53.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:53.645 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:24:53.746 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:24:53.794 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:53.868 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:24:53.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:53.875 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:24:53.975 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:24:54.042 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:54.164 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:24:54.397 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:24:54.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:54.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:24:54.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:54 smithi143 ceph-mon[183059]: pgmap v362: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 489 op/s; 1065/199413 objects degraded (0.534%); 11827/199413 objects misplaced (5.931%); 766 KiB/s, 11 objects/s recovering 2023-08-28T02:24:54.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:54 smithi143 ceph-mon[189871]: pgmap v362: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 489 op/s; 1065/199413 objects degraded (0.534%); 11827/199413 objects misplaced (5.931%); 766 KiB/s, 11 objects/s recovering 2023-08-28T02:24:54.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:54 smithi169 ceph-mon[159415]: pgmap v362: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 489 op/s; 1065/199413 objects degraded (0.534%); 11827/199413 objects misplaced (5.931%); 766 KiB/s, 11 objects/s recovering 2023-08-28T02:24:54.556 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:24:54.667 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:54.833 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:24:54.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:54.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:24:54.898 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:24:54.965 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:24:54.999 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:55.079 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:24:55.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:55.084 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:24:55.185 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:24:55.247 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is unset 2023-08-28T02:24:55.269 INFO:teuthology.orchestra.run.smithi143.stderr:reweighted osd.2 to 1 (10000) 2023-08-28T02:24:55.274 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:55.495 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:55 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/461454679' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:24:55.495 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:55 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:24:55.495 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:55 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/672969476' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-08-28T02:24:55.495 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:55 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-08-28T02:24:55.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:55 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/461454679' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:24:55.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:55 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:24:55.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:55 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/672969476' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-08-28T02:24:55.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:55 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-08-28T02:24:55.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:55 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/461454679' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:24:55.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:55 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:24:55.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:55 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/672969476' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-08-28T02:24:55.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:55 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-08-28T02:24:56.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:56 smithi143 ceph-mon[183059]: pgmap v363: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 1138/194220 objects degraded (0.586%); 11448/194220 objects misplaced (5.894%); 1.4 MiB/s, 22 objects/s recovering 2023-08-28T02:24:56.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:56 smithi143 ceph-mon[183059]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:24:56.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:56 smithi143 ceph-mon[183059]: failed to encode map e170 with expected crc 2023-08-28T02:24:56.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:56 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:24:56.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:56 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-08-28T02:24:56.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:56 smithi143 ceph-mon[183059]: osdmap e170: 8 total, 8 up, 8 in 2023-08-28T02:24:56.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:56 smithi143 ceph-mon[183059]: failed to encode map e170 with expected crc 2023-08-28T02:24:56.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:56 smithi143 ceph-mon[183059]: failed to encode map e170 with expected crc 2023-08-28T02:24:56.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:56 smithi143 ceph-mon[183059]: failed to encode map e170 with expected crc 2023-08-28T02:24:56.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:56 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:24:56.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:56 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:24:56.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:56 smithi143 ceph-mon[189871]: pgmap v363: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 1138/194220 objects degraded (0.586%); 11448/194220 objects misplaced (5.894%); 1.4 MiB/s, 22 objects/s recovering 2023-08-28T02:24:56.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:56 smithi143 ceph-mon[189871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:24:56.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:56 smithi143 ceph-mon[189871]: failed to encode map e170 with expected crc 2023-08-28T02:24:56.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:56 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:24:56.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:56 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-08-28T02:24:56.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:56 smithi143 ceph-mon[189871]: osdmap e170: 8 total, 8 up, 8 in 2023-08-28T02:24:56.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:56 smithi143 ceph-mon[189871]: failed to encode map e170 with expected crc 2023-08-28T02:24:56.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:56 smithi143 ceph-mon[189871]: failed to encode map e170 with expected crc 2023-08-28T02:24:56.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:56 smithi143 ceph-mon[189871]: failed to encode map e170 with expected crc 2023-08-28T02:24:56.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:56 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:24:56.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:56 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:24:56.479 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:24:56.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:56 smithi169 ceph-mon[159415]: pgmap v363: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 1138/194220 objects degraded (0.586%); 11448/194220 objects misplaced (5.894%); 1.4 MiB/s, 22 objects/s recovering 2023-08-28T02:24:56.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:56 smithi169 ceph-mon[159415]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:24:56.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:56 smithi169 ceph-mon[159415]: failed to encode map e170 with expected crc 2023-08-28T02:24:56.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:56 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:24:56.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:56 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-08-28T02:24:56.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:56 smithi169 ceph-mon[159415]: osdmap e170: 8 total, 8 up, 8 in 2023-08-28T02:24:56.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:56 smithi169 ceph-mon[159415]: failed to encode map e170 with expected crc 2023-08-28T02:24:56.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:56 smithi169 ceph-mon[159415]: failed to encode map e170 with expected crc 2023-08-28T02:24:56.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:56 smithi169 ceph-mon[159415]: failed to encode map e170 with expected crc 2023-08-28T02:24:56.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:56 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:24:56.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:56 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:24:56.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:56.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:24:56.589 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:24:56.621 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:56.693 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:24:56.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:56.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:24:56.804 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:24:56.875 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:57.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:57 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 1138/194220 objects degraded (0.586%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:24:57.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:57 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 1138/194220 objects degraded (0.586%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:24:57.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:57 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 1138/194220 objects degraded (0.586%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:24:57.922 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd reweight 4 1 2023-08-28T02:24:57.943 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:24:57.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:57.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:24:58.047 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:24:58.097 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:58.174 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:24:58.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:58.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:24:58.279 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:24:58.322 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:58.326 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:24:58 smithi143 ceph-mon[183059]: pgmap v365: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 1138/190029 objects degraded (0.599%); 11290/190029 objects misplaced (5.941%); 1.4 MiB/s, 22 objects/s recovering 2023-08-28T02:24:58.326 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:24:58 smithi143 ceph-mon[189871]: pgmap v365: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 1138/190029 objects degraded (0.599%); 11290/190029 objects misplaced (5.941%); 1.4 MiB/s, 22 objects/s recovering 2023-08-28T02:24:58.436 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:24:58.440 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:24:58 smithi169 ceph-mon[159415]: pgmap v365: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 1138/190029 objects degraded (0.599%); 11290/190029 objects misplaced (5.941%); 1.4 MiB/s, 22 objects/s recovering 2023-08-28T02:24:58.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:58.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:24:58.542 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:24:58.611 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:58.655 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:24:58.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:58.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:24:58.760 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:24:58.786 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:58.828 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:24:58.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:58.833 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:24:58.934 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:24:59.008 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:59.588 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:24:59.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:59.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:24:59.698 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:24:59.739 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:24:59.808 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:24:59.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:24:59.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:24:59.918 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:24:59.931 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:24:59.931 INFO:teuthology.orchestra.run.smithi143.stdout: "size": 20, 2023-08-28T02:24:59.931 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 600, 2023-08-28T02:24:59.931 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [ 2023-08-28T02:24:59.931 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.931 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:45857 5.0 5:02266a25:::benchmark_data_smithi143_236200_object45856:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-08-28T02:24:59.932 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:22:29.926706+0000", 2023-08-28T02:24:59.932 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 149.99532127000001, 2023-08-28T02:24:59.932 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.20099894200000001, 2023-08-28T02:24:59.932 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:24:59.932 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:24:59.932 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:24:59.932 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:24:59.932 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:24:59.932 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 45857 2023-08-28T02:24:59.932 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.932 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:24:59.932 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.933 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:24:59.933 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.926706+0000", 2023-08-28T02:24:59.933 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:59.933 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.933 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.933 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:24:59.933 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.926706+0000", 2023-08-28T02:24:59.933 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:59.933 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.933 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.933 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:24:59.933 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.926706+0000", 2023-08-28T02:24:59.933 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:24:59.934 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.934 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.934 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:24:59.934 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.926725+0000", 2023-08-28T02:24:59.934 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.9286000000000001e-05 2023-08-28T02:24:59.934 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.934 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.934 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:24:59.934 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.926744+0000", 2023-08-28T02:24:59.934 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.9125000000000001e-05 2023-08-28T02:24:59.934 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.934 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.934 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:24:59.935 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.926751+0000", 2023-08-28T02:24:59.935 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.289e-06 2023-08-28T02:24:59.935 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.935 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.935 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:24:59.935 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.928156+0000", 2023-08-28T02:24:59.935 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.001404195 2023-08-28T02:24:59.935 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.935 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.935 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:24:59.935 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.928240+0000", 2023-08-28T02:24:59.935 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.4497000000000001e-05 2023-08-28T02:24:59.935 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.936 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.936 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,4", 2023-08-28T02:24:59.936 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.928342+0000", 2023-08-28T02:24:59.936 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000101699 2023-08-28T02:24:59.936 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.936 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.936 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:24:59.936 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.929633+0000", 2023-08-28T02:24:59.936 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0012916589999999999 2023-08-28T02:24:59.936 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.936 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.936 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:59.936 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.930440+0000", 2023-08-28T02:24:59.937 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00080684300000000001 2023-08-28T02:24:59.937 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.937 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.937 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:59.937 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.127618+0000", 2023-08-28T02:24:59.937 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.19717718100000001 2023-08-28T02:24:59.937 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.937 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.937 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:24:59.937 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.127662+0000", 2023-08-28T02:24:59.937 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.4737000000000002e-05 2023-08-28T02:24:59.937 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.937 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.938 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:24:59.938 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.127705+0000", 2023-08-28T02:24:59.938 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.3130000000000002e-05 2023-08-28T02:24:59.938 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:59.938 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:24:59.938 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:59.938 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.938 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.938 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:45862 5.c 5:3fd0b482:::benchmark_data_smithi143_236200_object45861:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-08-28T02:24:59.938 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:22:29.930174+0000", 2023-08-28T02:24:59.938 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 149.99185406399999, 2023-08-28T02:24:59.938 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.194363008, 2023-08-28T02:24:59.938 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:24:59.939 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:24:59.939 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:24:59.939 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:24:59.939 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:24:59.939 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 45862 2023-08-28T02:24:59.939 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.939 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:24:59.939 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.939 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:24:59.939 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.930174+0000", 2023-08-28T02:24:59.939 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:59.939 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.940 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.940 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:24:59.940 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.930174+0000", 2023-08-28T02:24:59.940 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:59.940 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.940 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.940 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:24:59.940 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.930173+0000", 2023-08-28T02:24:59.940 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:24:59.940 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.940 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.940 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:24:59.940 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.930192+0000", 2023-08-28T02:24:59.941 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.9391000000000001e-05 2023-08-28T02:24:59.941 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.941 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.941 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:24:59.941 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.930203+0000", 2023-08-28T02:24:59.941 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1134999999999999e-05 2023-08-28T02:24:59.941 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.941 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.941 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:24:59.941 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.930211+0000", 2023-08-28T02:24:59.941 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.6459999999999993e-06 2023-08-28T02:24:59.941 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.941 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.942 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:24:59.942 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.930254+0000", 2023-08-28T02:24:59.942 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.3362999999999998e-05 2023-08-28T02:24:59.942 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.942 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.942 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:24:59.942 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.930374+0000", 2023-08-28T02:24:59.942 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000120337 2023-08-28T02:24:59.942 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.942 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.942 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,4", 2023-08-28T02:24:59.942 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.930474+0000", 2023-08-28T02:24:59.943 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.9928999999999998e-05 2023-08-28T02:24:59.943 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.943 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.943 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:24:59.943 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.932207+0000", 2023-08-28T02:24:59.943 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.001733006 2023-08-28T02:24:59.943 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.943 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.943 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:59.943 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.932341+0000", 2023-08-28T02:24:59.943 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000133645 2023-08-28T02:24:59.943 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.944 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.944 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:59.944 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.124452+0000", 2023-08-28T02:24:59.944 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.19211102599999999 2023-08-28T02:24:59.944 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.944 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.944 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:24:59.944 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.124497+0000", 2023-08-28T02:24:59.944 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.5141e-05 2023-08-28T02:24:59.944 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.944 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.944 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:24:59.944 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.124537+0000", 2023-08-28T02:24:59.945 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.9440000000000002e-05 2023-08-28T02:24:59.945 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:59.945 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:24:59.945 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:59.945 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.945 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.945 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:45867 5.0 5:0bc76487:::benchmark_data_smithi143_236200_object45866:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-08-28T02:24:59.945 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:22:29.933114+0000", 2023-08-28T02:24:59.945 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 149.98891363000001, 2023-08-28T02:24:59.945 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.19481781000000001, 2023-08-28T02:24:59.945 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:24:59.945 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:24:59.946 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:24:59.946 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:24:59.946 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:24:59.946 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 45867 2023-08-28T02:24:59.946 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.946 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:24:59.946 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.946 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:24:59.946 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.933114+0000", 2023-08-28T02:24:59.946 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:59.946 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.946 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.947 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:24:59.947 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.933114+0000", 2023-08-28T02:24:59.947 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:59.947 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.947 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.947 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:24:59.947 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.933113+0000", 2023-08-28T02:24:59.947 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:24:59.947 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.947 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.947 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:24:59.947 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.933136+0000", 2023-08-28T02:24:59.948 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.3587e-05 2023-08-28T02:24:59.948 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.948 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.948 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:24:59.948 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.933148+0000", 2023-08-28T02:24:59.948 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1691999999999999e-05 2023-08-28T02:24:59.948 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.948 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.948 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:24:59.948 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.933155+0000", 2023-08-28T02:24:59.948 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.8920000000000003e-06 2023-08-28T02:24:59.949 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.949 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.949 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:24:59.949 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.933193+0000", 2023-08-28T02:24:59.949 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.8355999999999997e-05 2023-08-28T02:24:59.949 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.949 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.949 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:24:59.949 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.933297+0000", 2023-08-28T02:24:59.949 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000104149 2023-08-28T02:24:59.949 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.949 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.949 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,4", 2023-08-28T02:24:59.950 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.933434+0000", 2023-08-28T02:24:59.950 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000136491 2023-08-28T02:24:59.950 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.950 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.950 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:59.950 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.127755+0000", 2023-08-28T02:24:59.950 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.19432112100000001 2023-08-28T02:24:59.950 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.950 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.950 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:24:59.950 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.127763+0000", 2023-08-28T02:24:59.950 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.892e-06 2023-08-28T02:24:59.950 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.951 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.951 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:59.951 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.127851+0000", 2023-08-28T02:24:59.951 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.8060999999999996e-05 2023-08-28T02:24:59.951 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.951 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.951 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:24:59.951 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.127897+0000", 2023-08-28T02:24:59.951 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.6443000000000002e-05 2023-08-28T02:24:59.951 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.951 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.951 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:24:59.952 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.127932+0000", 2023-08-28T02:24:59.952 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.4514000000000002e-05 2023-08-28T02:24:59.952 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:59.952 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:24:59.952 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:59.952 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.952 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.952 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:45869 5.0 5:0a0258a1:::benchmark_data_smithi143_236200_object45868:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-08-28T02:24:59.952 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:22:29.933277+0000", 2023-08-28T02:24:59.952 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 149.98875062499999, 2023-08-28T02:24:59.952 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.19651866300000001, 2023-08-28T02:24:59.952 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:24:59.952 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:24:59.953 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:24:59.953 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:24:59.953 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:24:59.953 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 45869 2023-08-28T02:24:59.953 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.953 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:24:59.953 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.953 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:24:59.953 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.933277+0000", 2023-08-28T02:24:59.953 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:59.953 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.953 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.953 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:24:59.954 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.933277+0000", 2023-08-28T02:24:59.954 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:59.954 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.954 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.954 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:24:59.954 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.933276+0000", 2023-08-28T02:24:59.954 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:24:59.954 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.954 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.954 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:24:59.954 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.933299+0000", 2023-08-28T02:24:59.954 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.266e-05 2023-08-28T02:24:59.954 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.955 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.955 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:24:59.955 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.933311+0000", 2023-08-28T02:24:59.955 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.2713e-05 2023-08-28T02:24:59.955 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.955 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.955 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:24:59.955 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.933319+0000", 2023-08-28T02:24:59.955 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.9100000000000005e-06 2023-08-28T02:24:59.955 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.955 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.955 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:24:59.956 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.125884+0000", 2023-08-28T02:24:59.956 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.192564764 2023-08-28T02:24:59.956 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.956 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.956 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:24:59.956 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.126020+0000", 2023-08-28T02:24:59.956 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00013563200000000001 2023-08-28T02:24:59.956 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.956 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.956 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,4", 2023-08-28T02:24:59.956 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.126160+0000", 2023-08-28T02:24:59.956 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000140283 2023-08-28T02:24:59.956 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.957 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.957 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:59.957 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.129656+0000", 2023-08-28T02:24:59.957 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.003496337 2023-08-28T02:24:59.957 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.957 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.957 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:59.957 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.129697+0000", 2023-08-28T02:24:59.957 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.0803999999999998e-05 2023-08-28T02:24:59.957 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.957 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.957 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:24:59.957 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.129717+0000", 2023-08-28T02:24:59.958 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.0239000000000001e-05 2023-08-28T02:24:59.958 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.958 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.958 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:24:59.958 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.129761+0000", 2023-08-28T02:24:59.958 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.4002999999999999e-05 2023-08-28T02:24:59.958 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.958 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.958 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:24:59.958 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.129796+0000", 2023-08-28T02:24:59.958 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.4294000000000001e-05 2023-08-28T02:24:59.958 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:59.958 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:24:59.959 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:59.959 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.959 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.959 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop_reply(client.44594.0:45857 5.0 e128/119 ondisk, result = 0)", 2023-08-28T02:24:59.959 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:22:29.935059+0000", 2023-08-28T02:24:59.959 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 149.98696832900001, 2023-08-28T02:24:59.959 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.19265895199999999, 2023-08-28T02:24:59.959 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:24:59.959 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "started", 2023-08-28T02:24:59.959 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:24:59.959 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.959 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:24:59.959 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.935059+0000", 2023-08-28T02:24:59.960 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:59.960 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.960 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.960 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:24:59.960 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.935059+0000", 2023-08-28T02:24:59.960 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:59.960 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.960 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.960 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:24:59.960 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.935059+0000", 2023-08-28T02:24:59.960 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:24:59.960 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.961 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.961 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:24:59.961 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.935063+0000", 2023-08-28T02:24:59.961 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.6709999999999998e-06 2023-08-28T02:24:59.961 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.961 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.961 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:24:59.961 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.124274+0000", 2023-08-28T02:24:59.961 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.189210354 2023-08-28T02:24:59.961 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.961 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.961 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:24:59.961 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.124284+0000", 2023-08-28T02:24:59.962 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0553999999999999e-05 2023-08-28T02:24:59.962 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.962 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.962 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:24:59.962 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.127604+0000", 2023-08-28T02:24:59.962 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0033197869999999998 2023-08-28T02:24:59.962 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.962 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.962 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:24:59.962 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.127612+0000", 2023-08-28T02:24:59.962 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.6459999999999993e-06 2023-08-28T02:24:59.962 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.962 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.963 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:24:59.963 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.127718+0000", 2023-08-28T02:24:59.963 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00010658999999999999 2023-08-28T02:24:59.963 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:59.963 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:24:59.963 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:59.963 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.963 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.963 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.44594.0:45874 5.8 e128/119 5:1f0c8adb:::benchmark_data_smithi143_236200_object45873:head v 128'2824, mlcod=128'2822)", 2023-08-28T02:24:59.963 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:22:29.935903+0000", 2023-08-28T02:24:59.963 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 149.986125106, 2023-08-28T02:24:59.964 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.193952238, 2023-08-28T02:24:59.964 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:24:59.964 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:24:59.964 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:24:59.964 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.964 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:24:59.964 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.935903+0000", 2023-08-28T02:24:59.964 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:59.964 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.964 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.964 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:24:59.964 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.935903+0000", 2023-08-28T02:24:59.964 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:59.965 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.965 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.965 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:24:59.965 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.935902+0000", 2023-08-28T02:24:59.965 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:24:59.965 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.965 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.965 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:24:59.965 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.935925+0000", 2023-08-28T02:24:59.965 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.3197000000000002e-05 2023-08-28T02:24:59.965 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.965 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.965 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:24:59.966 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.124267+0000", 2023-08-28T02:24:59.966 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.188342173 2023-08-28T02:24:59.966 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.966 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.966 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:24:59.966 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.124279+0000", 2023-08-28T02:24:59.966 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.208e-05 2023-08-28T02:24:59.966 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.966 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.966 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:24:59.966 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.127786+0000", 2023-08-28T02:24:59.966 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0035070019999999999 2023-08-28T02:24:59.966 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.967 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.967 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:24:59.967 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.127809+0000", 2023-08-28T02:24:59.967 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.2206999999999999e-05 2023-08-28T02:24:59.967 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.967 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.967 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:24:59.967 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.129826+0000", 2023-08-28T02:24:59.967 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.002017146 2023-08-28T02:24:59.967 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.967 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.967 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:24:59.968 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.129855+0000", 2023-08-28T02:24:59.968 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.9145e-05 2023-08-28T02:24:59.968 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:59.968 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:24:59.968 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:59.968 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.968 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.968 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:96062 5.c 5:31f5f34f:::benchmark_data_smithi143_236200_object96061:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-08-28T02:24:59.968 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:15.770245+0000", 2023-08-28T02:24:59.968 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 104.151782756, 2023-08-28T02:24:59.968 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.17714152499999999, 2023-08-28T02:24:59.968 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:24:59.968 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:24:59.969 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:24:59.969 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:24:59.969 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:24:59.969 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 96062 2023-08-28T02:24:59.969 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.969 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:24:59.969 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.969 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:24:59.969 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:15.770245+0000", 2023-08-28T02:24:59.969 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:59.969 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.969 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.970 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:24:59.970 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:15.770245+0000", 2023-08-28T02:24:59.970 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:59.970 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.970 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.970 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:24:59.970 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:15.770243+0000", 2023-08-28T02:24:59.970 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:24:59.970 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.970 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.970 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:24:59.970 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:15.770266+0000", 2023-08-28T02:24:59.970 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.2215999999999999e-05 2023-08-28T02:24:59.971 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.971 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.971 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:24:59.971 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:15.770278+0000", 2023-08-28T02:24:59.971 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.2601e-05 2023-08-28T02:24:59.971 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.971 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.971 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:24:59.971 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:15.770287+0000", 2023-08-28T02:24:59.971 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.2199999999999998e-06 2023-08-28T02:24:59.971 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.971 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.972 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:24:59.972 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:15.770331+0000", 2023-08-28T02:24:59.972 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.4017999999999998e-05 2023-08-28T02:24:59.972 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.972 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.972 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:24:59.972 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:15.770442+0000", 2023-08-28T02:24:59.972 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00011047 2023-08-28T02:24:59.972 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.972 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.972 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,4", 2023-08-28T02:24:59.972 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:15.770543+0000", 2023-08-28T02:24:59.973 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000101121 2023-08-28T02:24:59.973 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.973 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.973 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:24:59.973 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:15.772947+0000", 2023-08-28T02:24:59.973 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0024040760000000002 2023-08-28T02:24:59.973 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.973 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.973 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:59.973 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:15.776698+0000", 2023-08-28T02:24:59.973 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0037513889999999999 2023-08-28T02:24:59.973 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.973 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.973 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:59.974 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:15.947292+0000", 2023-08-28T02:24:59.974 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.170593561 2023-08-28T02:24:59.974 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.974 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.974 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:24:59.974 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:15.947346+0000", 2023-08-28T02:24:59.974 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.3943999999999997e-05 2023-08-28T02:24:59.974 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.974 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.974 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:24:59.974 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:15.947386+0000", 2023-08-28T02:24:59.974 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.0493e-05 2023-08-28T02:24:59.974 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:59.975 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:24:59.975 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:59.975 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.975 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.975 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:96064 5.c 5:336d4f60:::benchmark_data_smithi143_236200_object96063:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-08-28T02:24:59.975 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:15.774099+0000", 2023-08-28T02:24:59.975 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 104.147928549, 2023-08-28T02:24:59.975 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.173390669, 2023-08-28T02:24:59.975 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:24:59.975 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:24:59.975 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:24:59.975 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:24:59.976 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:24:59.976 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 96064 2023-08-28T02:24:59.976 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.976 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:24:59.976 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.976 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:24:59.976 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:15.774099+0000", 2023-08-28T02:24:59.976 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:59.976 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.976 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.976 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:24:59.976 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:15.774099+0000", 2023-08-28T02:24:59.976 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:59.977 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.977 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.977 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:24:59.977 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:15.774098+0000", 2023-08-28T02:24:59.977 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:24:59.977 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.977 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.977 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:24:59.977 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:15.774122+0000", 2023-08-28T02:24:59.977 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.3587e-05 2023-08-28T02:24:59.977 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.977 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.977 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:24:59.978 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:15.774137+0000", 2023-08-28T02:24:59.978 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.4948e-05 2023-08-28T02:24:59.978 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.978 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.978 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:24:59.978 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:15.774145+0000", 2023-08-28T02:24:59.978 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.5709999999999998e-06 2023-08-28T02:24:59.978 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.978 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.978 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:24:59.978 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:15.774190+0000", 2023-08-28T02:24:59.978 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.5079e-05 2023-08-28T02:24:59.978 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.979 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.979 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:24:59.979 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:15.774328+0000", 2023-08-28T02:24:59.979 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00013824400000000001 2023-08-28T02:24:59.979 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.979 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.979 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,4", 2023-08-28T02:24:59.979 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:15.774477+0000", 2023-08-28T02:24:59.979 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00014811100000000001 2023-08-28T02:24:59.979 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.979 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.979 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:24:59.979 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:15.776799+0000", 2023-08-28T02:24:59.980 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0023222780000000001 2023-08-28T02:24:59.980 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.980 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.980 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:59.980 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:15.782534+0000", 2023-08-28T02:24:59.980 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.005734917 2023-08-28T02:24:59.980 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.980 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.980 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:59.980 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:15.947423+0000", 2023-08-28T02:24:59.980 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.16488945699999999 2023-08-28T02:24:59.980 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.981 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.981 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:24:59.981 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:15.947460+0000", 2023-08-28T02:24:59.981 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.65e-05 2023-08-28T02:24:59.981 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.981 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.981 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:24:59.981 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:15.947490+0000", 2023-08-28T02:24:59.981 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.0162e-05 2023-08-28T02:24:59.981 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:59.981 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:24:59.981 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:59.982 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.982 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.982 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:122091 5.0 5:0d3d7eef:::benchmark_data_smithi143_236200_object11948:head [delete] snapc 0=[] ondisk+write+known_if_redirected e152)", 2023-08-28T02:24:59.982 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:56.357099+0000", 2023-08-28T02:24:59.982 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 63.564929114999998, 2023-08-28T02:24:59.982 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.19015922599999999, 2023-08-28T02:24:59.982 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:24:59.982 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:24:59.982 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:24:59.982 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:24:59.982 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:24:59.982 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 122091 2023-08-28T02:24:59.982 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.983 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:24:59.983 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.983 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:24:59.983 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.357099+0000", 2023-08-28T02:24:59.983 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:59.983 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.983 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.983 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:24:59.983 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.357099+0000", 2023-08-28T02:24:59.983 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:59.983 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.983 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.983 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:24:59.984 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.357097+0000", 2023-08-28T02:24:59.984 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:24:59.984 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.984 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.984 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:24:59.984 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.357104+0000", 2023-08-28T02:24:59.984 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.759e-06 2023-08-28T02:24:59.984 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.984 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.984 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:24:59.984 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.357116+0000", 2023-08-28T02:24:59.984 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.2728000000000001e-05 2023-08-28T02:24:59.984 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.985 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.985 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:24:59.985 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.357126+0000", 2023-08-28T02:24:59.985 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.2080000000000006e-06 2023-08-28T02:24:59.985 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.985 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.985 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:24:59.985 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.357171+0000", 2023-08-28T02:24:59.985 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.5358000000000002e-05 2023-08-28T02:24:59.985 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.985 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.985 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:24:59.985 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.357301+0000", 2023-08-28T02:24:59.986 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00012978299999999999 2023-08-28T02:24:59.986 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.986 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.986 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,4", 2023-08-28T02:24:59.986 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.357427+0000", 2023-08-28T02:24:59.986 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000126487 2023-08-28T02:24:59.986 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.986 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.986 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:24:59.986 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.361063+0000", 2023-08-28T02:24:59.986 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0036359949999999999 2023-08-28T02:24:59.986 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.986 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.987 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:59.987 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.361171+0000", 2023-08-28T02:24:59.987 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000107351 2023-08-28T02:24:59.987 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.987 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.987 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:59.987 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.547178+0000", 2023-08-28T02:24:59.987 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.18600725600000001 2023-08-28T02:24:59.987 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.987 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.987 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:24:59.987 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.547222+0000", 2023-08-28T02:24:59.987 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.4178000000000003e-05 2023-08-28T02:24:59.988 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.988 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.988 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:24:59.988 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.547258+0000", 2023-08-28T02:24:59.988 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.5800999999999998e-05 2023-08-28T02:24:59.988 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:59.988 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:24:59.988 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:59.988 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.988 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.988 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:122103 5.0 5:0f2af708:::benchmark_data_smithi143_236200_object11960:head [delete] snapc 0=[] ondisk+write+known_if_redirected e152)", 2023-08-28T02:24:59.988 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:56.369607+0000", 2023-08-28T02:24:59.988 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 63.5524208, 2023-08-28T02:24:59.989 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.17778329600000001, 2023-08-28T02:24:59.989 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:24:59.989 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:24:59.989 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:24:59.989 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:24:59.989 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:24:59.989 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 122103 2023-08-28T02:24:59.989 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.989 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:24:59.989 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.989 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:24:59.989 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.369607+0000", 2023-08-28T02:24:59.989 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:59.990 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.990 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.990 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:24:59.990 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.369607+0000", 2023-08-28T02:24:59.990 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:59.990 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.990 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.990 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:24:59.990 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.369605+0000", 2023-08-28T02:24:59.990 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:24:59.990 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.990 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.991 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:24:59.991 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.369612+0000", 2023-08-28T02:24:59.991 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.4790000000000001e-06 2023-08-28T02:24:59.991 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.991 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.991 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:24:59.991 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.369627+0000", 2023-08-28T02:24:59.991 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.4997000000000001e-05 2023-08-28T02:24:59.991 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.991 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.991 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:24:59.991 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.369635+0000", 2023-08-28T02:24:59.991 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.1140000000000006e-06 2023-08-28T02:24:59.992 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.992 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.992 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:24:59.992 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.369677+0000", 2023-08-28T02:24:59.992 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.2459000000000001e-05 2023-08-28T02:24:59.992 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.992 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.992 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:24:59.992 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.369803+0000", 2023-08-28T02:24:59.992 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00012511299999999999 2023-08-28T02:24:59.992 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.992 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.993 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,4", 2023-08-28T02:24:59.993 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.369908+0000", 2023-08-28T02:24:59.993 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000105017 2023-08-28T02:24:59.993 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.993 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.993 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:24:59.993 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.372760+0000", 2023-08-28T02:24:59.993 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0028520709999999999 2023-08-28T02:24:59.993 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.993 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.993 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:59.993 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.377191+0000", 2023-08-28T02:24:59.993 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0044316770000000002 2023-08-28T02:24:59.994 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.994 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.994 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:24:59.994 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.547313+0000", 2023-08-28T02:24:59.994 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.17012127599999999 2023-08-28T02:24:59.994 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.994 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.994 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:24:59.994 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.547355+0000", 2023-08-28T02:24:59.994 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.2914000000000003e-05 2023-08-28T02:24:59.994 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.994 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.995 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:24:59.995 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.547390+0000", 2023-08-28T02:24:59.995 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.4745e-05 2023-08-28T02:24:59.995 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:59.995 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:24:59.995 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:24:59.995 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.995 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.995 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:122106 5.c 5:39a41a3e:::benchmark_data_smithi143_236200_object11963:head [delete] snapc 0=[] ondisk+write+known_if_redirected e152)", 2023-08-28T02:24:59.995 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:56.373118+0000", 2023-08-28T02:24:59.995 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 63.548910057999997, 2023-08-28T02:24:59.995 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.17411379799999999, 2023-08-28T02:24:59.995 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:24:59.996 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:24:59.996 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:24:59.996 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:24:59.996 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:24:59.996 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 122106 2023-08-28T02:24:59.996 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.996 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:24:59.996 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.996 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:24:59.996 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.373118+0000", 2023-08-28T02:24:59.996 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:59.996 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.996 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.997 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:24:59.997 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.373118+0000", 2023-08-28T02:24:59.997 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:24:59.997 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.997 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.997 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:24:59.997 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.373116+0000", 2023-08-28T02:24:59.997 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:24:59.997 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.997 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.997 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:24:59.997 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.373121+0000", 2023-08-28T02:24:59.998 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.057e-06 2023-08-28T02:24:59.998 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.998 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.998 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:24:59.998 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.373135+0000", 2023-08-28T02:24:59.998 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3234999999999999e-05 2023-08-28T02:24:59.998 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.998 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.998 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:24:59.998 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.373141+0000", 2023-08-28T02:24:59.998 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.0399999999999998e-06 2023-08-28T02:24:59.998 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.998 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.999 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:24:59.999 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.373196+0000", 2023-08-28T02:24:59.999 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.4923000000000003e-05 2023-08-28T02:24:59.999 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.999 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.999 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:24:59.999 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.373320+0000", 2023-08-28T02:24:59.999 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00012433999999999999 2023-08-28T02:24:59.999 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:24:59.999 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:24:59.999 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,4", 2023-08-28T02:24:59.999 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.373451+0000", 2023-08-28T02:25:00.000 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00013135600000000001 2023-08-28T02:25:00.000 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.000 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.000 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:25:00.000 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.378142+0000", 2023-08-28T02:25:00.000 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0046908740000000003 2023-08-28T02:25:00.000 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.000 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.000 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:25:00.000 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.379548+0000", 2023-08-28T02:25:00.000 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0014053850000000001 2023-08-28T02:25:00.000 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.000 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.001 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:25:00.001 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.547130+0000", 2023-08-28T02:25:00.001 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.16758196 2023-08-28T02:25:00.001 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.001 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.001 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:25:00.001 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.547180+0000", 2023-08-28T02:25:00.001 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.0025000000000003e-05 2023-08-28T02:25:00.001 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.001 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.001 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:25:00.001 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.547231+0000", 2023-08-28T02:25:00.001 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.1842999999999998e-05 2023-08-28T02:25:00.002 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:25:00.002 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:25:00.002 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:25:00.002 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.002 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.002 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:124097 5.c 5:363ea2bd:::benchmark_data_smithi143_236200_object13954:head [delete] snapc 0=[] ondisk+write+known_if_redirected e155)", 2023-08-28T02:25:00.002 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:24:01.195135+0000", 2023-08-28T02:25:00.002 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 58.726892534999998, 2023-08-28T02:25:00.002 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.21136323500000001, 2023-08-28T02:25:00.002 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:25:00.002 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:25:00.002 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:25:00.003 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:25:00.003 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:25:00.003 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 124097 2023-08-28T02:25:00.003 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.003 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:25:00.003 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.003 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:25:00.003 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.195135+0000", 2023-08-28T02:25:00.003 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:25:00.003 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.003 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.003 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:25:00.003 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.195135+0000", 2023-08-28T02:25:00.004 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:25:00.004 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.004 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.004 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:25:00.004 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.195133+0000", 2023-08-28T02:25:00.004 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:25:00.004 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.004 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.004 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:25:00.004 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.195144+0000", 2023-08-28T02:25:00.004 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0613e-05 2023-08-28T02:25:00.004 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.004 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.005 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:25:00.005 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.195160+0000", 2023-08-28T02:25:00.005 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.5760000000000002e-05 2023-08-28T02:25:00.005 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.005 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.005 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:25:00.005 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.195170+0000", 2023-08-28T02:25:00.005 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0282000000000001e-05 2023-08-28T02:25:00.005 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.005 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.005 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:25:00.005 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.195232+0000", 2023-08-28T02:25:00.005 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.2101999999999996e-05 2023-08-28T02:25:00.006 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.006 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.006 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:25:00.006 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.195382+0000", 2023-08-28T02:25:00.006 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00014973499999999999 2023-08-28T02:25:00.006 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.006 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.006 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,4", 2023-08-28T02:25:00.006 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.195531+0000", 2023-08-28T02:25:00.006 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00014867600000000001 2023-08-28T02:25:00.006 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.006 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.006 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:25:00.007 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.198673+0000", 2023-08-28T02:25:00.007 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0031423190000000002 2023-08-28T02:25:00.007 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.007 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.007 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:25:00.007 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.198720+0000", 2023-08-28T02:25:00.007 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.7618999999999999e-05 2023-08-28T02:25:00.007 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.007 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.007 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:25:00.007 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.406413+0000", 2023-08-28T02:25:00.007 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.20769302000000001 2023-08-28T02:25:00.008 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.008 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.008 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:25:00.008 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.406459+0000", 2023-08-28T02:25:00.008 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.5889000000000001e-05 2023-08-28T02:25:00.008 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.008 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.008 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:25:00.008 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.406498+0000", 2023-08-28T02:25:00.008 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.9017000000000003e-05 2023-08-28T02:25:00.008 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:25:00.008 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:25:00.008 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:25:00.009 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.009 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.009 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:130442 5.c 5:3d69323a:::benchmark_data_smithi143_236200_object20299:head [delete] snapc 0=[] ondisk+write+known_if_redirected e163)", 2023-08-28T02:25:00.009 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:24:13.079137+0000", 2023-08-28T02:25:00.009 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 46.842890421999996, 2023-08-28T02:25:00.009 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.244954062, 2023-08-28T02:25:00.009 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:25:00.009 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:25:00.009 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:25:00.009 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:25:00.009 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:25:00.009 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 130442 2023-08-28T02:25:00.010 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.010 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:25:00.010 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.010 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:25:00.010 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.079137+0000", 2023-08-28T02:25:00.010 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:25:00.010 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.010 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.010 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:25:00.010 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.079137+0000", 2023-08-28T02:25:00.010 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:25:00.010 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.010 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.011 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:25:00.011 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.079137+0000", 2023-08-28T02:25:00.011 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:25:00.011 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.011 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.011 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:25:00.011 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.079140+0000", 2023-08-28T02:25:00.011 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.621e-06 2023-08-28T02:25:00.011 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.011 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.011 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:25:00.011 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.079146+0000", 2023-08-28T02:25:00.012 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.9669999999999998e-06 2023-08-28T02:25:00.012 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.012 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.012 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:25:00.012 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.079150+0000", 2023-08-28T02:25:00.012 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.826e-06 2023-08-28T02:25:00.012 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.012 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.012 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:25:00.012 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.083017+0000", 2023-08-28T02:25:00.012 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0038665069999999999 2023-08-28T02:25:00.012 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.012 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.013 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:25:00.013 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.083142+0000", 2023-08-28T02:25:00.013 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00012473700000000001 2023-08-28T02:25:00.013 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.013 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.013 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,4", 2023-08-28T02:25:00.013 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.083252+0000", 2023-08-28T02:25:00.013 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000110258 2023-08-28T02:25:00.013 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.013 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.013 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:25:00.013 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.090235+0000", 2023-08-28T02:25:00.014 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0069833969999999997 2023-08-28T02:25:00.014 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.014 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.014 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:25:00.014 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.092255+0000", 2023-08-28T02:25:00.014 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0020200410000000002 2023-08-28T02:25:00.014 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.014 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.014 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:25:00.014 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.324000+0000", 2023-08-28T02:25:00.014 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.231744278 2023-08-28T02:25:00.014 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.014 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.015 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:25:00.015 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.324050+0000", 2023-08-28T02:25:00.015 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.0912000000000003e-05 2023-08-28T02:25:00.015 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.015 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.015 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:25:00.015 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.324091+0000", 2023-08-28T02:25:00.015 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.0877000000000002e-05 2023-08-28T02:25:00.015 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:25:00.015 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:25:00.015 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:25:00.015 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.015 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.016 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:130453 5.c 5:31ecd121:::benchmark_data_smithi143_236200_object20310:head [delete] snapc 0=[] ondisk+write+known_if_redirected e163)", 2023-08-28T02:25:00.016 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:24:13.092231+0000", 2023-08-28T02:25:00.016 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 46.829796297999998, 2023-08-28T02:25:00.016 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.231980675, 2023-08-28T02:25:00.016 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:25:00.016 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:25:00.016 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:25:00.016 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:25:00.016 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:25:00.016 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 130453 2023-08-28T02:25:00.016 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.016 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:25:00.017 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.017 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:25:00.017 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.092231+0000", 2023-08-28T02:25:00.017 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:25:00.017 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.017 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.017 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:25:00.017 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.092231+0000", 2023-08-28T02:25:00.017 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:25:00.017 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.017 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.017 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:25:00.017 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.092230+0000", 2023-08-28T02:25:00.018 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:25:00.018 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.018 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.018 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:25:00.018 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.092234+0000", 2023-08-28T02:25:00.018 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.5109999999999997e-06 2023-08-28T02:25:00.018 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.018 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.018 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:25:00.018 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.092243+0000", 2023-08-28T02:25:00.018 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.0650000000000005e-06 2023-08-28T02:25:00.018 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.018 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.019 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:25:00.019 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.092250+0000", 2023-08-28T02:25:00.019 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.0589999999999999e-06 2023-08-28T02:25:00.019 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.019 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.019 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:25:00.019 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.092281+0000", 2023-08-28T02:25:00.019 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.1158000000000002e-05 2023-08-28T02:25:00.019 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.019 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.019 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:25:00.019 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.092404+0000", 2023-08-28T02:25:00.019 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00012336899999999999 2023-08-28T02:25:00.020 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.020 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.020 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,4", 2023-08-28T02:25:00.020 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.092524+0000", 2023-08-28T02:25:00.020 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000119855 2023-08-28T02:25:00.020 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.020 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.020 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:25:00.020 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.096216+0000", 2023-08-28T02:25:00.020 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0036923099999999999 2023-08-28T02:25:00.020 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.020 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.021 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:25:00.021 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.099817+0000", 2023-08-28T02:25:00.021 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0036007690000000002 2023-08-28T02:25:00.021 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.021 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.021 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:25:00.021 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.324138+0000", 2023-08-28T02:25:00.021 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.224321459 2023-08-28T02:25:00.021 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.021 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.021 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:25:00.021 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.324180+0000", 2023-08-28T02:25:00.021 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.1684999999999998e-05 2023-08-28T02:25:00.022 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.022 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.022 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:25:00.022 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.324212+0000", 2023-08-28T02:25:00.022 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.1937999999999998e-05 2023-08-28T02:25:00.022 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:25:00.022 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:25:00.022 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:25:00.022 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.022 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.022 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:140332 5.c 5:38d68a55:::benchmark_data_smithi143_236200_object30189:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-08-28T02:25:00.022 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:24:28.055909+0000", 2023-08-28T02:25:00.023 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 31.86611916, 2023-08-28T02:25:00.023 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.198230936, 2023-08-28T02:25:00.023 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:25:00.023 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:25:00.023 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:25:00.023 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:25:00.023 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:25:00.023 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 140332 2023-08-28T02:25:00.023 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.023 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:25:00.023 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.023 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:25:00.023 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.055909+0000", 2023-08-28T02:25:00.024 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:25:00.024 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.024 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.024 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:25:00.024 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.055909+0000", 2023-08-28T02:25:00.024 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:25:00.024 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.024 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.024 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:25:00.024 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.055907+0000", 2023-08-28T02:25:00.024 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:25:00.024 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.024 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.025 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:25:00.025 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.055913+0000", 2023-08-28T02:25:00.025 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.3630000000000002e-06 2023-08-28T02:25:00.025 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.025 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.025 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:25:00.025 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.055922+0000", 2023-08-28T02:25:00.025 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.0999999999999993e-06 2023-08-28T02:25:00.025 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.025 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.025 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:25:00.025 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.055927+0000", 2023-08-28T02:25:00.026 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.2499999999999997e-06 2023-08-28T02:25:00.026 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.026 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.026 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:25:00.026 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.055968+0000", 2023-08-28T02:25:00.026 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.1242999999999998e-05 2023-08-28T02:25:00.026 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.026 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.026 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:25:00.026 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.056090+0000", 2023-08-28T02:25:00.026 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000121289 2023-08-28T02:25:00.026 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.026 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.027 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,4", 2023-08-28T02:25:00.027 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.056196+0000", 2023-08-28T02:25:00.027 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000106216 2023-08-28T02:25:00.027 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.027 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.027 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:25:00.027 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.059202+0000", 2023-08-28T02:25:00.027 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0030063199999999998 2023-08-28T02:25:00.027 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.027 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.027 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:25:00.027 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.064571+0000", 2023-08-28T02:25:00.028 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0053685110000000003 2023-08-28T02:25:00.028 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.028 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.028 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:25:00.028 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.254065+0000", 2023-08-28T02:25:00.028 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.18949487400000001 2023-08-28T02:25:00.029 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.029 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.029 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:25:00.030 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.254107+0000", 2023-08-28T02:25:00.030 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.1387000000000002e-05 2023-08-28T02:25:00.030 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.030 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.030 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:25:00.030 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.254139+0000", 2023-08-28T02:25:00.030 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.2654999999999998e-05 2023-08-28T02:25:00.030 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:25:00.030 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:25:00.030 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:25:00.030 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.030 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.031 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:140339 5.c 5:3390e4f0:::benchmark_data_smithi143_236200_object30196:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-08-28T02:25:00.031 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:24:28.065603+0000", 2023-08-28T02:25:00.031 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 31.856424240999999, 2023-08-28T02:25:00.031 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.188624818, 2023-08-28T02:25:00.031 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:25:00.031 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:25:00.031 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:25:00.031 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:25:00.031 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:25:00.031 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 140339 2023-08-28T02:25:00.031 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.031 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:25:00.031 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.032 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:25:00.032 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.065603+0000", 2023-08-28T02:25:00.032 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:25:00.032 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.032 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.032 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:25:00.032 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.065603+0000", 2023-08-28T02:25:00.032 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:25:00.032 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.032 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.032 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:25:00.032 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.065602+0000", 2023-08-28T02:25:00.032 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:25:00.033 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.033 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.033 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:25:00.033 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.065608+0000", 2023-08-28T02:25:00.033 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.9819999999999996e-06 2023-08-28T02:25:00.033 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.033 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.033 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:25:00.033 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.065618+0000", 2023-08-28T02:25:00.033 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.4779999999999997e-06 2023-08-28T02:25:00.033 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.033 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.034 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:25:00.034 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.065623+0000", 2023-08-28T02:25:00.034 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.383e-06 2023-08-28T02:25:00.034 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.034 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.034 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:25:00.034 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.065663+0000", 2023-08-28T02:25:00.034 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.0012000000000002e-05 2023-08-28T02:25:00.034 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.034 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.034 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:25:00.034 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.065752+0000", 2023-08-28T02:25:00.034 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.9171000000000004e-05 2023-08-28T02:25:00.035 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.035 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.035 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,4", 2023-08-28T02:25:00.035 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.065826+0000", 2023-08-28T02:25:00.035 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.3498999999999995e-05 2023-08-28T02:25:00.035 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.035 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.035 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:25:00.035 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.070482+0000", 2023-08-28T02:25:00.035 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0046563660000000003 2023-08-28T02:25:00.035 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.035 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.036 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:25:00.036 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.071688+0000", 2023-08-28T02:25:00.036 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.001206223 2023-08-28T02:25:00.036 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.036 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.036 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:25:00.036 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.254168+0000", 2023-08-28T02:25:00.036 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.182479893 2023-08-28T02:25:00.036 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.036 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.036 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:25:00.036 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.254202+0000", 2023-08-28T02:25:00.036 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.3467999999999999e-05 2023-08-28T02:25:00.037 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.037 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.037 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:25:00.037 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.254228+0000", 2023-08-28T02:25:00.037 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.6676999999999999e-05 2023-08-28T02:25:00.037 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:25:00.037 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:25:00.037 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:25:00.037 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.037 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.037 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:140340 5.c 5:3d53ef22:::benchmark_data_smithi143_236200_object30197:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-08-28T02:25:00.037 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:24:28.067273+0000", 2023-08-28T02:25:00.037 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 31.854754332999999, 2023-08-28T02:25:00.038 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.18703093900000001, 2023-08-28T02:25:00.038 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:25:00.038 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:25:00.038 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:25:00.038 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:25:00.038 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:25:00.038 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 140340 2023-08-28T02:25:00.038 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.038 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:25:00.038 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.038 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:25:00.038 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.067273+0000", 2023-08-28T02:25:00.038 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:25:00.039 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.039 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.039 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:25:00.039 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.067273+0000", 2023-08-28T02:25:00.039 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:25:00.039 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.039 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.039 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:25:00.039 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.067273+0000", 2023-08-28T02:25:00.039 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:25:00.039 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.039 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.040 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:25:00.040 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.067276+0000", 2023-08-28T02:25:00.040 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.6239999999999999e-06 2023-08-28T02:25:00.040 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.040 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.040 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:25:00.040 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.067284+0000", 2023-08-28T02:25:00.040 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.4479999999999997e-06 2023-08-28T02:25:00.040 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.040 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.040 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:25:00.040 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.067289+0000", 2023-08-28T02:25:00.040 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.763e-06 2023-08-28T02:25:00.041 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.041 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.041 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:25:00.041 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.068538+0000", 2023-08-28T02:25:00.041 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.001249018 2023-08-28T02:25:00.041 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.041 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.041 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:25:00.041 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.068628+0000", 2023-08-28T02:25:00.041 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.0068000000000006e-05 2023-08-28T02:25:00.041 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.041 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.042 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,4", 2023-08-28T02:25:00.042 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.068703+0000", 2023-08-28T02:25:00.042 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.4883999999999997e-05 2023-08-28T02:25:00.042 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.042 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.042 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:25:00.042 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.070608+0000", 2023-08-28T02:25:00.042 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0019058949999999999 2023-08-28T02:25:00.042 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.042 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.042 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:25:00.042 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.071841+0000", 2023-08-28T02:25:00.042 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.001232576 2023-08-28T02:25:00.043 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.043 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.043 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:25:00.043 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.254253+0000", 2023-08-28T02:25:00.043 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.18241223300000001 2023-08-28T02:25:00.043 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.043 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.043 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:25:00.043 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.254280+0000", 2023-08-28T02:25:00.043 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.7169e-05 2023-08-28T02:25:00.043 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.043 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.043 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:25:00.044 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.254304+0000", 2023-08-28T02:25:00.044 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.3887999999999999e-05 2023-08-28T02:25:00.044 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:25:00.044 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:25:00.044 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:25:00.044 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.044 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.044 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:154041 5.c 5:38b53cd8:::benchmark_data_smithi143_236200_object43898:head [delete] snapc 0=[] ondisk+write+known_if_redirected e169)", 2023-08-28T02:25:00.044 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:24:50.334265+0000", 2023-08-28T02:25:00.044 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 9.5877628129999994, 2023-08-28T02:25:00.044 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.21132811000000001, 2023-08-28T02:25:00.044 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:25:00.045 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:25:00.045 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:25:00.045 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:25:00.045 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:25:00.045 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 154041 2023-08-28T02:25:00.045 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.045 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:25:00.045 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.045 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:25:00.045 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.334265+0000", 2023-08-28T02:25:00.045 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:25:00.045 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.045 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.046 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:25:00.046 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.334265+0000", 2023-08-28T02:25:00.046 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:25:00.046 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.046 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.046 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:25:00.046 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.334263+0000", 2023-08-28T02:25:00.046 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:25:00.046 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.046 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.046 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:25:00.046 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.334271+0000", 2023-08-28T02:25:00.046 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.9780000000000005e-06 2023-08-28T02:25:00.047 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.047 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.047 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:25:00.047 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.334284+0000", 2023-08-28T02:25:00.047 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3158e-05 2023-08-28T02:25:00.047 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.047 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.047 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:25:00.047 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.334293+0000", 2023-08-28T02:25:00.047 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.0550000000000005e-06 2023-08-28T02:25:00.047 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.047 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.047 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:25:00.048 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.334349+0000", 2023-08-28T02:25:00.048 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.5129e-05 2023-08-28T02:25:00.048 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.048 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.048 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:25:00.048 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.334465+0000", 2023-08-28T02:25:00.048 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00011665199999999999 2023-08-28T02:25:00.048 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.048 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.048 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,4", 2023-08-28T02:25:00.048 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.334584+0000", 2023-08-28T02:25:00.048 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00011909 2023-08-28T02:25:00.049 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.049 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.049 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:25:00.049 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.340936+0000", 2023-08-28T02:25:00.049 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0063517970000000002 2023-08-28T02:25:00.049 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.049 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.049 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:25:00.049 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.352299+0000", 2023-08-28T02:25:00.049 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.011362921 2023-08-28T02:25:00.049 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.049 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.049 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:25:00.050 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.545477+0000", 2023-08-28T02:25:00.050 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.193178145 2023-08-28T02:25:00.050 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.050 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.050 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:25:00.050 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.545543+0000", 2023-08-28T02:25:00.050 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.5420999999999996e-05 2023-08-28T02:25:00.050 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.050 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.050 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:25:00.050 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.545593+0000", 2023-08-28T02:25:00.050 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.0386000000000002e-05 2023-08-28T02:25:00.050 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:25:00.051 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:25:00.051 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:25:00.051 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.051 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.051 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:154043 5.c 5:393c5f9d:::benchmark_data_smithi143_236200_object43900:head [delete] snapc 0=[] ondisk+write+known_if_redirected e169)", 2023-08-28T02:25:00.051 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:24:50.337310+0000", 2023-08-28T02:25:00.051 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 9.5847177299999995, 2023-08-28T02:25:00.051 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.210709331, 2023-08-28T02:25:00.051 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:25:00.051 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:25:00.051 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:25:00.051 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:25:00.052 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:25:00.052 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 154043 2023-08-28T02:25:00.052 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.052 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:25:00.052 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.052 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:25:00.052 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.337310+0000", 2023-08-28T02:25:00.052 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:25:00.052 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.052 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.052 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:25:00.053 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.337310+0000", 2023-08-28T02:25:00.053 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:25:00.053 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.053 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.053 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:25:00.053 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.337309+0000", 2023-08-28T02:25:00.053 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:25:00.053 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.053 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.053 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:25:00.053 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.337314+0000", 2023-08-28T02:25:00.053 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.626e-06 2023-08-28T02:25:00.053 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.054 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.054 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:25:00.054 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.337324+0000", 2023-08-28T02:25:00.054 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.5869999999999999e-06 2023-08-28T02:25:00.054 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.054 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.054 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:25:00.054 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.337331+0000", 2023-08-28T02:25:00.054 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.5599999999999999e-06 2023-08-28T02:25:00.054 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.054 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.054 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:25:00.055 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.337896+0000", 2023-08-28T02:25:00.055 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00056541399999999998 2023-08-28T02:25:00.055 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.055 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.055 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:25:00.055 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.337971+0000", 2023-08-28T02:25:00.055 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.5228999999999994e-05 2023-08-28T02:25:00.055 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.055 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.055 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,4", 2023-08-28T02:25:00.055 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.338049+0000", 2023-08-28T02:25:00.055 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.7358e-05 2023-08-28T02:25:00.056 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.056 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.056 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:25:00.056 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.341053+0000", 2023-08-28T02:25:00.056 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.003004039 2023-08-28T02:25:00.056 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.056 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.056 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:25:00.056 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.352329+0000", 2023-08-28T02:25:00.056 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.011276833999999999 2023-08-28T02:25:00.056 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.056 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.056 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:25:00.057 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.547929+0000", 2023-08-28T02:25:00.057 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.19559933900000001 2023-08-28T02:25:00.057 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.057 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.057 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:25:00.057 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.547975+0000", 2023-08-28T02:25:00.057 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.5739999999999999e-05 2023-08-28T02:25:00.057 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.057 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.057 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:25:00.057 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.548019+0000", 2023-08-28T02:25:00.057 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.4749999999999997e-05 2023-08-28T02:25:00.058 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:25:00.058 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:25:00.058 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:25:00.058 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.058 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.058 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:154052 5.c 5:30744441:::benchmark_data_smithi143_236200_object43909:head [delete] snapc 0=[] ondisk+write+known_if_redirected e169)", 2023-08-28T02:25:00.058 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:24:50.355115+0000", 2023-08-28T02:25:00.058 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 9.5669124760000006, 2023-08-28T02:25:00.058 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.193283705, 2023-08-28T02:25:00.058 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:25:00.058 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:25:00.058 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:25:00.059 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:25:00.059 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:25:00.059 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 154052 2023-08-28T02:25:00.059 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.059 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:25:00.059 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.059 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:25:00.059 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.355115+0000", 2023-08-28T02:25:00.059 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:25:00.059 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.059 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.059 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:25:00.059 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.355115+0000", 2023-08-28T02:25:00.060 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:25:00.060 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.060 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.060 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:25:00.060 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.355113+0000", 2023-08-28T02:25:00.060 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:25:00.060 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.060 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.060 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:25:00.060 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.355122+0000", 2023-08-28T02:25:00.060 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.3869999999999995e-06 2023-08-28T02:25:00.060 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.061 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.061 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:25:00.061 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.355134+0000", 2023-08-28T02:25:00.061 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1772e-05 2023-08-28T02:25:00.061 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.061 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.061 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:25:00.061 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.355142+0000", 2023-08-28T02:25:00.061 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.9619999999999993e-06 2023-08-28T02:25:00.061 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.061 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.061 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:25:00.062 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.355189+0000", 2023-08-28T02:25:00.062 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.7345999999999997e-05 2023-08-28T02:25:00.062 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.062 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.062 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:25:00.062 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.355323+0000", 2023-08-28T02:25:00.062 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00013421900000000001 2023-08-28T02:25:00.062 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.062 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.062 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,4", 2023-08-28T02:25:00.062 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.355444+0000", 2023-08-28T02:25:00.062 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00012092200000000001 2023-08-28T02:25:00.063 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.063 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.063 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:25:00.063 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.358770+0000", 2023-08-28T02:25:00.063 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.003326308 2023-08-28T02:25:00.063 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.063 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.063 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:25:00.063 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.359196+0000", 2023-08-28T02:25:00.063 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00042524199999999999 2023-08-28T02:25:00.063 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.063 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.063 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:25:00.064 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.548328+0000", 2023-08-28T02:25:00.064 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.189132252 2023-08-28T02:25:00.064 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.064 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.064 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:25:00.064 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.548367+0000", 2023-08-28T02:25:00.064 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.9487000000000003e-05 2023-08-28T02:25:00.064 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:25:00.064 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:25:00.064 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:25:00.064 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.548399+0000", 2023-08-28T02:25:00.064 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.163e-05 2023-08-28T02:25:00.064 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:25:00.065 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:25:00.065 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:25:00.065 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:25:00.065 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:25:00.065 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:25:00.066 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set noscrub 2023-08-28T02:25:00.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:00.067 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:00.067 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:25:00.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:25:00.168 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:25:00.310 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:00.313 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:25:00.365 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:25:00.403 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:25:00.434 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:25:00.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:00 smithi143 ceph-mon[183059]: pgmap v366: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 1229/184839 objects degraded (0.665%); 10885/184839 objects misplaced (5.889%); 1.4 MiB/s, 22 objects/s recovering 2023-08-28T02:25:00.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:00 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:25:00.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:00 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:25:00.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:00 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/79614827' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:25:00.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:00 smithi143 ceph-mon[189871]: pgmap v366: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 1229/184839 objects degraded (0.665%); 10885/184839 objects misplaced (5.889%); 1.4 MiB/s, 22 objects/s recovering 2023-08-28T02:25:00.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:00 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:25:00.457 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:00 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:25:00.457 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:00 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/79614827' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:25:00.467 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:25:00.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:00 smithi169 ceph-mon[159415]: pgmap v366: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 1229/184839 objects degraded (0.665%); 10885/184839 objects misplaced (5.889%); 1.4 MiB/s, 22 objects/s recovering 2023-08-28T02:25:00.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:00 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:25:00.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:00 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:25:00.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:00 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/79614827' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:25:00.502 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:25:00.552 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:25:00.763 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:25:00.763 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:25:00.763 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:25:00.763 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:25:00.763 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:25:00.764 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:25:00.764 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:25:00.764 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:25:00.764 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:25:01.632 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:25:01.636 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:01 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 1229/184839 objects degraded (0.665%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:25:01.636 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:01 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:25:01.636 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:01 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:25:01.636 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:01 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:25:01.636 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:01 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 1229/184839 objects degraded (0.665%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:25:01.636 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:01 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:25:01.636 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:01 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:25:01.636 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:01 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:25:01.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:01.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:25:01.738 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:25:01.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:01 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 1229/184839 objects degraded (0.665%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:25:01.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:01 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:25:01.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:01 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:25:01.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:01 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:25:01.794 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:01.853 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.2 dump_ops_in_flight 2023-08-28T02:25:01.864 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:25:01.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:01.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:25:01.975 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:25:02.078 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:02.123 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:25:02.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:25:01 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:25:01] "GET /metrics HTTP/1.1" 200 36495 "" "Prometheus/2.33.4" 2023-08-28T02:25:02.202 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:25:02.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:02.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:25:02.262 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:25:02.317 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:25:02.422 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:02.537 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:02 smithi143 ceph-mon[183059]: pgmap v367: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 1229/184839 objects degraded (0.665%); 10885/184839 objects misplaced (5.889%); 1.4 MiB/s, 22 objects/s recovering 2023-08-28T02:25:02.538 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:02 smithi143 ceph-mon[189871]: pgmap v367: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 1229/184839 objects degraded (0.665%); 10885/184839 objects misplaced (5.889%); 1.4 MiB/s, 22 objects/s recovering 2023-08-28T02:25:02.670 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:25:02.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:02.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:25:02.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:02 smithi169 ceph-mon[159415]: pgmap v367: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 1229/184839 objects degraded (0.665%); 10885/184839 objects misplaced (5.889%); 1.4 MiB/s, 22 objects/s recovering 2023-08-28T02:25:02.779 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:25:02.874 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:03.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:03 smithi143 ceph-mon[183059]: pgmap v368: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 1229/180921 objects degraded (0.679%); 10738/180921 objects misplaced (5.935%); 1.4 MiB/s, 22 objects/s recovering 2023-08-28T02:25:03.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:03 smithi143 ceph-mon[189871]: pgmap v368: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 1229/180921 objects degraded (0.679%); 10738/180921 objects misplaced (5.935%); 1.4 MiB/s, 22 objects/s recovering 2023-08-28T02:25:03.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:03 smithi169 ceph-mon[159415]: pgmap v368: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 1229/180921 objects degraded (0.679%); 10738/180921 objects misplaced (5.935%); 1.4 MiB/s, 22 objects/s recovering 2023-08-28T02:25:04.038 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:25:04.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:04.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:25:04.079 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:25:04.142 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:25:04.244 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:04.646 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:25:04.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:04.652 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:25:04.753 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:25:04.815 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:04.887 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:25:04.889 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:25:04.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:04.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:25:04.995 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:25:05.022 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:05.098 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:25:05.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:05.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:25:05.204 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:25:05.262 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:05.343 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:25:05.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:05.348 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:25:05.449 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:25:05.526 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:05.785 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:25:05.803 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:25:05.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:05.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:25:05.910 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:25:05.991 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:06.090 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:25:06.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:06.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:25:06.194 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:25:06.221 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is set 2023-08-28T02:25:06.232 INFO:teuthology.orchestra.run.smithi143.stderr:reweighted osd.4 to 1 (10000) 2023-08-28T02:25:06.240 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:06 smithi143 ceph-mon[183059]: pgmap v369: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 1284/175863 objects degraded (0.730%); 10368/175863 objects misplaced (5.895%); 1.4 MiB/s, 22 objects/s recovering 2023-08-28T02:25:06.240 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:06 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2742012466' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-08-28T02:25:06.241 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:06 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-08-28T02:25:06.241 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:06 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/4245623232' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:25:06.241 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:06 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:25:06.241 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:06 smithi143 ceph-mon[189871]: pgmap v369: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 1284/175863 objects degraded (0.730%); 10368/175863 objects misplaced (5.895%); 1.4 MiB/s, 22 objects/s recovering 2023-08-28T02:25:06.241 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:06 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2742012466' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-08-28T02:25:06.242 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:06 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-08-28T02:25:06.242 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:06 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/4245623232' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:25:06.242 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:06 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:25:06.296 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:06.406 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:25:06.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:06.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:25:06.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:06 smithi169 ceph-mon[159415]: pgmap v369: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 1284/175863 objects degraded (0.730%); 10368/175863 objects misplaced (5.895%); 1.4 MiB/s, 22 objects/s recovering 2023-08-28T02:25:06.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:06 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2742012466' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-08-28T02:25:06.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:06 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-08-28T02:25:06.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:06 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/4245623232' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:25:06.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:06 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:25:06.510 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:25:06.558 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:07.343 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:25:07.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:07.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:25:07.456 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:25:07.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:07 smithi169 ceph-mon[159415]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:25:07.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:07 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-08-28T02:25:07.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:07 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:25:07.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:07 smithi169 ceph-mon[159415]: osdmap e171: 8 total, 8 up, 8 in 2023-08-28T02:25:07.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:07 smithi169 ceph-mon[159415]: failed to encode map e171 with expected crc 2023-08-28T02:25:07.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:07 smithi169 ceph-mon[159415]: failed to encode map e171 with expected crc 2023-08-28T02:25:07.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:07 smithi169 ceph-mon[159415]: failed to encode map e171 with expected crc 2023-08-28T02:25:07.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:07 smithi169 ceph-mon[159415]: failed to encode map e171 with expected crc 2023-08-28T02:25:07.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:07 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 1284/175863 objects degraded (0.730%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:25:07.515 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:07.628 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:25:07.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:07.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:25:07.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:07 smithi143 ceph-mon[183059]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:25:07.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:07 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-08-28T02:25:07.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:07 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:25:07.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:07 smithi143 ceph-mon[183059]: osdmap e171: 8 total, 8 up, 8 in 2023-08-28T02:25:07.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:07 smithi143 ceph-mon[183059]: failed to encode map e171 with expected crc 2023-08-28T02:25:07.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:07 smithi143 ceph-mon[183059]: failed to encode map e171 with expected crc 2023-08-28T02:25:07.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:07 smithi143 ceph-mon[183059]: failed to encode map e171 with expected crc 2023-08-28T02:25:07.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:07 smithi143 ceph-mon[183059]: failed to encode map e171 with expected crc 2023-08-28T02:25:07.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:07 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 1284/175863 objects degraded (0.730%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:25:07.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:07 smithi143 ceph-mon[189871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:25:07.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:07 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-08-28T02:25:07.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:07 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:25:07.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:07 smithi143 ceph-mon[189871]: osdmap e171: 8 total, 8 up, 8 in 2023-08-28T02:25:07.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:07 smithi143 ceph-mon[189871]: failed to encode map e171 with expected crc 2023-08-28T02:25:07.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:07 smithi143 ceph-mon[189871]: failed to encode map e171 with expected crc 2023-08-28T02:25:07.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:07 smithi143 ceph-mon[189871]: failed to encode map e171 with expected crc 2023-08-28T02:25:07.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:07 smithi143 ceph-mon[189871]: failed to encode map e171 with expected crc 2023-08-28T02:25:07.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:07 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 1284/175863 objects degraded (0.730%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:25:07.740 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:25:07.766 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:07.840 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:25:07.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:07.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:25:07.950 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:25:07.991 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:08.076 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:25:08.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:08.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:25:08.186 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:25:08.213 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:08.308 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:25:08.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:08.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:25:08.427 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:25:08.454 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:08.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:08 smithi169 ceph-mon[159415]: pgmap v371: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 1284/171711 objects degraded (0.748%); 10195/171711 objects misplaced (5.937%); 1.4 MiB/s, 22 objects/s recovering 2023-08-28T02:25:08.676 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:08 smithi143 ceph-mon[183059]: pgmap v371: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 1284/171711 objects degraded (0.748%); 10195/171711 objects misplaced (5.937%); 1.4 MiB/s, 22 objects/s recovering 2023-08-28T02:25:08.676 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:08 smithi143 ceph-mon[189871]: pgmap v371: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 1284/171711 objects degraded (0.748%); 10195/171711 objects misplaced (5.937%); 1.4 MiB/s, 22 objects/s recovering 2023-08-28T02:25:08.736 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd reweight 5 1 2023-08-28T02:25:08.868 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:25:08.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:08.873 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:25:08.974 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:25:09.137 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:09.351 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:25:09.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:09.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:25:09.458 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:25:09.519 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:09.584 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:25:09.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:09.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:25:09.691 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:25:09.728 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:09.813 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:25:09.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:09.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:25:09.848 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:25:09.848 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:25:09.920 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:25:09.998 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:10.630 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:10 smithi143 ceph-mon[189871]: pgmap v372: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 1360/166590 objects degraded (0.816%); 9803/166590 objects misplaced (5.885%); 1.5 MiB/s, 24 objects/s recovering 2023-08-28T02:25:10.635 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:10 smithi143 ceph-mon[183059]: pgmap v372: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 1360/166590 objects degraded (0.816%); 9803/166590 objects misplaced (5.885%); 1.5 MiB/s, 24 objects/s recovering 2023-08-28T02:25:10.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:10 smithi169 ceph-mon[159415]: pgmap v372: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 1360/166590 objects degraded (0.816%); 9803/166590 objects misplaced (5.885%); 1.5 MiB/s, 24 objects/s recovering 2023-08-28T02:25:10.765 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:25:10.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:10.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:25:10.875 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:25:10.881 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.2 dump_blocked_ops 2023-08-28T02:25:10.936 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:11.052 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set nodeep-scrub 2023-08-28T02:25:11.117 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:25:11.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:11.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:25:11.231 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:25:11.319 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:25:11.366 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:11.505 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:25:11.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:11.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:25:11.551 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:25:11.590 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:25:11.615 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:25:11.639 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:25:11.679 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:25:11.715 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:11.717 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:25:11.747 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:25:11.790 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:25:11.830 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:25:11.845 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:25:11.846 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:25:11.846 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:25:11.846 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:25:11.846 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:25:11.846 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:25:11.846 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:25:11.846 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:25:11.851 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:25:11.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:11.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:25:11.966 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:25:12.013 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:12.062 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:25:11 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:25:11] "GET /metrics HTTP/1.1" 200 36489 "" "Prometheus/2.33.4" 2023-08-28T02:25:12.380 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:12 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:25:12.380 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:12 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:25:12.380 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:12 smithi143 ceph-mon[183059]: pgmap v373: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 1360/166590 objects degraded (0.816%); 9803/166590 objects misplaced (5.885%); 1.5 MiB/s, 24 objects/s recovering 2023-08-28T02:25:12.380 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:12 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 1360/166590 objects degraded (0.816%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:25:12.381 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:12 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2585370610' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:25:12.381 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:12 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:25:12.381 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:12 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:25:12.381 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:12 smithi143 ceph-mon[189871]: pgmap v373: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 1360/166590 objects degraded (0.816%); 9803/166590 objects misplaced (5.885%); 1.5 MiB/s, 24 objects/s recovering 2023-08-28T02:25:12.381 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:12 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 1360/166590 objects degraded (0.816%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:25:12.381 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:12 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2585370610' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:25:12.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:12 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:25:12.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:12 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:25:12.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:12 smithi169 ceph-mon[159415]: pgmap v373: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 1360/166590 objects degraded (0.816%); 9803/166590 objects misplaced (5.885%); 1.5 MiB/s, 24 objects/s recovering 2023-08-28T02:25:12.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:12 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 1360/166590 objects degraded (0.816%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:25:12.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:12 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2585370610' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:25:12.602 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:25:12.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:12.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:25:12.707 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:25:12.762 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:12.856 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:25:12.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:12.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:25:12.971 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:25:13.053 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:13.144 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:25:13.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:13.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:25:13.253 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:25:13.335 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:13.453 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:25:13.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:13.456 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:25:13.557 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:25:13.624 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:13.714 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:25:13.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:13.721 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:25:13.822 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:25:13.881 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:13.937 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:25:13.956 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:25:13.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:13.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:25:13.991 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:25:14.061 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:25:14.137 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:14.197 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:14 smithi143 ceph-mon[183059]: pgmap v374: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 1360/162747 objects degraded (0.836%); 9624/162747 objects misplaced (5.913%); 1.5 MiB/s, 24 objects/s recovering 2023-08-28T02:25:14.377 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:25:14.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:14.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:25:14.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:14 smithi143 ceph-mon[189871]: pgmap v374: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 1360/162747 objects degraded (0.836%); 9624/162747 objects misplaced (5.913%); 1.5 MiB/s, 24 objects/s recovering 2023-08-28T02:25:14.481 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:25:14.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:14 smithi169 ceph-mon[159415]: pgmap v374: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 1360/162747 objects degraded (0.836%); 9624/162747 objects misplaced (5.913%); 1.5 MiB/s, 24 objects/s recovering 2023-08-28T02:25:14.528 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:14.609 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:25:14.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:14.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:25:14.725 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:25:14.800 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:15.194 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:15 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3563760508' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-08-28T02:25:15.194 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:15 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-08-28T02:25:15.194 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:15 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3563760508' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-08-28T02:25:15.194 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:15 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-08-28T02:25:15.207 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:25:15.347 INFO:teuthology.orchestra.run.smithi143.stderr:reweighted osd.5 to 1 (10000) 2023-08-28T02:25:15.494 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:15 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3563760508' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-08-28T02:25:15.494 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:15 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-08-28T02:25:15.977 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:25:15.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:15.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:25:16.087 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:25:16.139 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:16.455 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:16 smithi143 ceph-mon[183059]: pgmap v375: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 1437/157503 objects degraded (0.912%); 9247/157503 objects misplaced (5.871%); 1.5 MiB/s, 24 objects/s recovering 2023-08-28T02:25:16.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:16 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-08-28T02:25:16.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:16 smithi143 ceph-mon[183059]: osdmap e172: 8 total, 8 up, 8 in 2023-08-28T02:25:16.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:16 smithi143 ceph-mon[183059]: failed to encode map e172 with expected crc 2023-08-28T02:25:16.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:16 smithi143 ceph-mon[183059]: failed to encode map e172 with expected crc 2023-08-28T02:25:16.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:16 smithi143 ceph-mon[183059]: failed to encode map e172 with expected crc 2023-08-28T02:25:16.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:16 smithi143 ceph-mon[183059]: failed to encode map e172 with expected crc 2023-08-28T02:25:16.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:16 smithi143 ceph-mon[189871]: pgmap v375: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 1437/157503 objects degraded (0.912%); 9247/157503 objects misplaced (5.871%); 1.5 MiB/s, 24 objects/s recovering 2023-08-28T02:25:16.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:16 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-08-28T02:25:16.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:16 smithi143 ceph-mon[189871]: osdmap e172: 8 total, 8 up, 8 in 2023-08-28T02:25:16.456 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:16 smithi143 ceph-mon[189871]: failed to encode map e172 with expected crc 2023-08-28T02:25:16.457 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:16 smithi143 ceph-mon[189871]: failed to encode map e172 with expected crc 2023-08-28T02:25:16.457 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:16 smithi143 ceph-mon[189871]: failed to encode map e172 with expected crc 2023-08-28T02:25:16.457 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:16 smithi143 ceph-mon[189871]: failed to encode map e172 with expected crc 2023-08-28T02:25:16.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:16 smithi169 ceph-mon[159415]: pgmap v375: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 1437/157503 objects degraded (0.912%); 9247/157503 objects misplaced (5.871%); 1.5 MiB/s, 24 objects/s recovering 2023-08-28T02:25:16.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:16 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-08-28T02:25:16.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:16 smithi169 ceph-mon[159415]: osdmap e172: 8 total, 8 up, 8 in 2023-08-28T02:25:16.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:16 smithi169 ceph-mon[159415]: failed to encode map e172 with expected crc 2023-08-28T02:25:16.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:16 smithi169 ceph-mon[159415]: failed to encode map e172 with expected crc 2023-08-28T02:25:16.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:16 smithi169 ceph-mon[159415]: failed to encode map e172 with expected crc 2023-08-28T02:25:16.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:16 smithi169 ceph-mon[159415]: failed to encode map e172 with expected crc 2023-08-28T02:25:16.675 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:25:16.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:16.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:25:16.785 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:25:16.814 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:17.367 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd reweight 6 1 2023-08-28T02:25:17.411 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:25:17.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:17.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:25:17.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:17 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 1437/157503 objects degraded (0.912%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:25:17.497 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:17 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 1437/157503 objects degraded (0.912%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:25:17.497 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:17 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 1437/157503 objects degraded (0.912%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:25:17.532 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:25:17.571 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:17.667 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:25:17.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:17.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:25:17.778 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:25:17.830 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:17.866 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:25:17.881 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:25:17.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:17.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:25:17.988 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:25:18.012 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:25:18.012 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [], 2023-08-28T02:25:18.012 INFO:teuthology.orchestra.run.smithi143.stdout: "complaint_time": 30, 2023-08-28T02:25:18.012 INFO:teuthology.orchestra.run.smithi143.stdout: "num_blocked_ops": 0 2023-08-28T02:25:18.013 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:25:18.023 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:18.088 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:25:18.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:18.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:25:18.194 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:25:18.295 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:18 smithi143 ceph-mon[183059]: pgmap v377: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 1437/153195 objects degraded (0.938%); 9069/153195 objects misplaced (5.920%); 1.5 MiB/s, 24 objects/s recovering 2023-08-28T02:25:18.295 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:18 smithi143 ceph-mon[189871]: pgmap v377: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 1437/153195 objects degraded (0.938%); 9069/153195 objects misplaced (5.920%); 1.5 MiB/s, 24 objects/s recovering 2023-08-28T02:25:18.343 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:18.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:18 smithi169 ceph-mon[159415]: pgmap v377: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 1437/153195 objects degraded (0.938%); 9069/153195 objects misplaced (5.920%); 1.5 MiB/s, 24 objects/s recovering 2023-08-28T02:25:18.922 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:25:18.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:18.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:25:19.028 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:25:19.082 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:19.182 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:25:19.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:19.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:25:19.292 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:25:19.349 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:19 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2589246926' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:25:19.350 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:19.359 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is set 2023-08-28T02:25:19.431 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:25:19.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:19.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:25:19.535 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:25:19.579 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.2 dump_historic_ops 2023-08-28T02:25:19.580 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:19 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2589246926' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:25:19.581 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:19 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2589246926' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:25:19.621 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:20.452 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:25:20.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:20.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:25:20.566 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:25:20.592 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:20 smithi169 ceph-mon[159415]: pgmap v378: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 1527/147987 objects degraded (1.032%); 8676/147987 objects misplaced (5.863%); 1.5 MiB/s, 23 objects/s recovering 2023-08-28T02:25:20.592 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:20 smithi169 ceph-mon[159415]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:25:20.592 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:20 smithi169 ceph-mon[159415]: failed to encode map e173 with expected crc 2023-08-28T02:25:20.592 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:20 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2589246926' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:25:20.592 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:20 smithi169 ceph-mon[159415]: osdmap e173: 8 total, 8 up, 8 in 2023-08-28T02:25:20.592 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:20 smithi169 ceph-mon[159415]: failed to encode map e173 with expected crc 2023-08-28T02:25:20.592 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:20 smithi169 ceph-mon[159415]: failed to encode map e173 with expected crc 2023-08-28T02:25:20.602 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:20.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:20 smithi143 ceph-mon[183059]: pgmap v378: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 1527/147987 objects degraded (1.032%); 8676/147987 objects misplaced (5.863%); 1.5 MiB/s, 23 objects/s recovering 2023-08-28T02:25:20.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:20 smithi143 ceph-mon[183059]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:25:20.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:20 smithi143 ceph-mon[183059]: failed to encode map e173 with expected crc 2023-08-28T02:25:20.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:20 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2589246926' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:25:20.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:20 smithi143 ceph-mon[183059]: osdmap e173: 8 total, 8 up, 8 in 2023-08-28T02:25:20.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:20 smithi143 ceph-mon[183059]: failed to encode map e173 with expected crc 2023-08-28T02:25:20.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:20 smithi143 ceph-mon[183059]: failed to encode map e173 with expected crc 2023-08-28T02:25:20.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:20 smithi143 ceph-mon[189871]: pgmap v378: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 1527/147987 objects degraded (1.032%); 8676/147987 objects misplaced (5.863%); 1.5 MiB/s, 23 objects/s recovering 2023-08-28T02:25:20.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:20 smithi143 ceph-mon[189871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:25:20.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:20 smithi143 ceph-mon[189871]: failed to encode map e173 with expected crc 2023-08-28T02:25:20.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:20 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2589246926' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:25:20.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:20 smithi143 ceph-mon[189871]: osdmap e173: 8 total, 8 up, 8 in 2023-08-28T02:25:20.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:20 smithi143 ceph-mon[189871]: failed to encode map e173 with expected crc 2023-08-28T02:25:20.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:20 smithi143 ceph-mon[189871]: failed to encode map e173 with expected crc 2023-08-28T02:25:20.709 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:25:20.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:20.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:25:20.823 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:25:20.888 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:20.987 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:25:21.027 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:25:21.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:21.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:25:21.132 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:25:21.205 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:25:21.217 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:21.414 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:25:21.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:21.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:25:21.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:21 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 1527/147987 objects degraded (1.032%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:25:21.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:21 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 1527/147987 objects degraded (1.032%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:25:21.520 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:25:21.594 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:21.713 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:25:21.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:21 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 1527/147987 objects degraded (1.032%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:25:21.780 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:25:21.820 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:25:21.858 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:25:21.891 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:25:21.935 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:25:21.971 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:25:22.009 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:25:22.032 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:25:22.032 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:25:22.032 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:25:22.033 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:25:22.033 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:25:22.033 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:25:22.033 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:25:22.033 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:25:22.051 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:25:21 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:25:21] "GET /metrics HTTP/1.1" 200 36489 "" "Prometheus/2.33.4" 2023-08-28T02:25:22.337 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:25:22.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:22.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:25:22.445 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:25:22.521 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:22.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:22 smithi143 ceph-mon[183059]: pgmap v380: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 1527/147987 objects degraded (1.032%); 8676/147987 objects misplaced (5.863%); 1.9 MiB/s, 29 objects/s recovering 2023-08-28T02:25:22.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:22 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/459944475' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:25:22.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:22 smithi143 ceph-mon[189871]: pgmap v380: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 1527/147987 objects degraded (1.032%); 8676/147987 objects misplaced (5.863%); 1.9 MiB/s, 29 objects/s recovering 2023-08-28T02:25:22.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:22 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/459944475' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:25:22.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:22 smithi169 ceph-mon[159415]: pgmap v380: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 1527/147987 objects degraded (1.032%); 8676/147987 objects misplaced (5.863%); 1.9 MiB/s, 29 objects/s recovering 2023-08-28T02:25:22.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:22 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/459944475' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:25:22.991 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset noscrub 2023-08-28T02:25:23.017 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:25:23.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:23.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:25:23.124 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:25:23.164 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:23.266 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:25:23.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:23.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:25:23.375 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:25:23.430 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:23.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:23 smithi169 ceph-mon[159415]: pgmap v381: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 2.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 1527/144048 objects degraded (1.060%); 8523/144048 objects misplaced (5.917%); 928 KiB/s, 14 objects/s recovering 2023-08-28T02:25:23.650 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:23 smithi143 ceph-mon[183059]: pgmap v381: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 2.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 1527/144048 objects degraded (1.060%); 8523/144048 objects misplaced (5.917%); 928 KiB/s, 14 objects/s recovering 2023-08-28T02:25:23.651 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:23 smithi143 ceph-mon[189871]: pgmap v381: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 2.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 1527/144048 objects degraded (1.060%); 8523/144048 objects misplaced (5.917%); 928 KiB/s, 14 objects/s recovering 2023-08-28T02:25:24.726 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:25:24.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:24.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:25:24.836 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:25:24.951 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:25.065 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:25:25.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:25.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:25:25.140 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:25:25.184 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:25:25.312 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:25.643 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:25:25.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:25.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:25:25.669 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:25:25.669 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:25:25.754 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:25:25.827 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:26.068 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:25:26.236 INFO:teuthology.orchestra.run.smithi143.stderr:reweighted osd.6 to 1 (10000) 2023-08-28T02:25:26.406 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:26 smithi143 ceph-mon[183059]: pgmap v382: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 1608/138873 objects degraded (1.158%); 8115/138873 objects misplaced (5.843%); 1.5 MiB/s, 23 objects/s recovering 2023-08-28T02:25:26.406 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:26 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/4128021014' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-08-28T02:25:26.407 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:26 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-08-28T02:25:26.407 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:26 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:25:26.407 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:26 smithi143 ceph-mon[189871]: pgmap v382: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 1608/138873 objects degraded (1.158%); 8115/138873 objects misplaced (5.843%); 1.5 MiB/s, 23 objects/s recovering 2023-08-28T02:25:26.407 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:26 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/4128021014' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-08-28T02:25:26.407 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:26 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-08-28T02:25:26.407 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:26 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:25:26.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:26 smithi169 ceph-mon[159415]: pgmap v382: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 1608/138873 objects degraded (1.158%); 8115/138873 objects misplaced (5.843%); 1.5 MiB/s, 23 objects/s recovering 2023-08-28T02:25:26.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:26 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/4128021014' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-08-28T02:25:26.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:26 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-08-28T02:25:26.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:26 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:25:27.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:27 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-08-28T02:25:27.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:27 smithi143 ceph-mon[183059]: osdmap e174: 8 total, 8 up, 8 in 2023-08-28T02:25:27.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:27 smithi143 ceph-mon[183059]: failed to encode map e174 with expected crc 2023-08-28T02:25:27.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:27 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 1608/138873 objects degraded (1.158%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:25:27.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:27 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-08-28T02:25:27.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:27 smithi143 ceph-mon[189871]: osdmap e174: 8 total, 8 up, 8 in 2023-08-28T02:25:27.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:27 smithi143 ceph-mon[189871]: failed to encode map e174 with expected crc 2023-08-28T02:25:27.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:27 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 1608/138873 objects degraded (1.158%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:25:27.539 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:25:27.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:27.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:25:27.649 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.3 dump_ops_in_flight 2023-08-28T02:25:27.653 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:25:27.694 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:27 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-08-28T02:25:27.694 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:27 smithi169 ceph-mon[159415]: osdmap e174: 8 total, 8 up, 8 in 2023-08-28T02:25:27.695 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:27 smithi169 ceph-mon[159415]: failed to encode map e174 with expected crc 2023-08-28T02:25:27.695 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:27 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 1608/138873 objects degraded (1.158%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:25:27.697 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:27.781 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:25:27.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:27.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:25:27.890 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:25:27.992 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:28.296 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd reweight 7 1 2023-08-28T02:25:28.353 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:25:28.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:28.379 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:25:28.481 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:25:28.555 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:28 smithi143 ceph-mon[189871]: pgmap v384: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 1608/134733 objects degraded (1.193%); 7954/134733 objects misplaced (5.904%); 935 KiB/s, 14 objects/s recovering 2023-08-28T02:25:28.557 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:28 smithi143 ceph-mon[183059]: pgmap v384: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 1608/134733 objects degraded (1.193%); 7954/134733 objects misplaced (5.904%); 935 KiB/s, 14 objects/s recovering 2023-08-28T02:25:28.559 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:28.631 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:25:28.734 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:25:28.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:28 smithi169 ceph-mon[159415]: pgmap v384: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 1608/134733 objects degraded (1.193%); 7954/134733 objects misplaced (5.904%); 935 KiB/s, 14 objects/s recovering 2023-08-28T02:25:28.770 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:25:28.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:28.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:25:28.879 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:25:28.982 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:29.238 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:25:29.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:29.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:25:29.355 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:25:29.388 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:29.452 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:25:29.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:29.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:25:29.558 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:25:29.612 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:29.685 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:25:29.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:29.700 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:25:29.801 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:25:29.881 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:30.350 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is unset 2023-08-28T02:25:30.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:30 smithi143 ceph-mon[183059]: pgmap v385: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 2.6 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 1665/129141 objects degraded (1.289%); 7551/129141 objects misplaced (5.847%); 1.6 MiB/s, 25 objects/s recovering 2023-08-28T02:25:30.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:30 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/4288720874' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:25:30.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:30 smithi143 ceph-mon[189871]: pgmap v385: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 2.6 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 1665/129141 objects degraded (1.289%); 7551/129141 objects misplaced (5.847%); 1.6 MiB/s, 25 objects/s recovering 2023-08-28T02:25:30.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:30 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/4288720874' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:25:30.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:30 smithi169 ceph-mon[159415]: pgmap v385: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 2.6 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 1665/129141 objects degraded (1.289%); 7551/129141 objects misplaced (5.847%); 1.6 MiB/s, 25 objects/s recovering 2023-08-28T02:25:30.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:30 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/4288720874' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:25:31.311 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:25:31.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:31.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:25:31.427 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:25:31.479 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:31.584 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:25:31.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:31.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:25:31.694 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:25:31.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:31 smithi143 ceph-mon[183059]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:25:31.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:31 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/4288720874' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:25:31.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:31 smithi143 ceph-mon[183059]: osdmap e175: 8 total, 8 up, 8 in 2023-08-28T02:25:31.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:31 smithi143 ceph-mon[183059]: failed to encode map e175 with expected crc 2023-08-28T02:25:31.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:31 smithi143 ceph-mon[183059]: failed to encode map e175 with expected crc 2023-08-28T02:25:31.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:31 smithi143 ceph-mon[183059]: failed to encode map e175 with expected crc 2023-08-28T02:25:31.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:31 smithi143 ceph-mon[183059]: failed to encode map e175 with expected crc 2023-08-28T02:25:31.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:31 smithi143 ceph-mon[183059]: failed to encode map e175 with expected crc 2023-08-28T02:25:31.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:31 smithi143 ceph-mon[183059]: failed to encode map e175 with expected crc 2023-08-28T02:25:31.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:31 smithi143 ceph-mon[183059]: pgmap v387: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 2.6 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 1665/129141 objects degraded (1.289%); 7551/129141 objects misplaced (5.847%); 2.0 MiB/s, 31 objects/s recovering 2023-08-28T02:25:31.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:31 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 1665/129141 objects degraded (1.289%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:25:31.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:31 smithi143 ceph-mon[189871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:25:31.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:31 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/4288720874' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:25:31.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:31 smithi143 ceph-mon[189871]: osdmap e175: 8 total, 8 up, 8 in 2023-08-28T02:25:31.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:31 smithi143 ceph-mon[189871]: failed to encode map e175 with expected crc 2023-08-28T02:25:31.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:31 smithi143 ceph-mon[189871]: failed to encode map e175 with expected crc 2023-08-28T02:25:31.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:31 smithi143 ceph-mon[189871]: failed to encode map e175 with expected crc 2023-08-28T02:25:31.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:31 smithi143 ceph-mon[189871]: failed to encode map e175 with expected crc 2023-08-28T02:25:31.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:31 smithi143 ceph-mon[189871]: failed to encode map e175 with expected crc 2023-08-28T02:25:31.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:31 smithi143 ceph-mon[189871]: failed to encode map e175 with expected crc 2023-08-28T02:25:31.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:31 smithi143 ceph-mon[189871]: pgmap v387: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 2.6 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 1665/129141 objects degraded (1.289%); 7551/129141 objects misplaced (5.847%); 2.0 MiB/s, 31 objects/s recovering 2023-08-28T02:25:31.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:31 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 1665/129141 objects degraded (1.289%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:25:31.717 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:31 smithi169 ceph-mon[159415]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:25:31.717 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:31 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/4288720874' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:25:31.718 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:31 smithi169 ceph-mon[159415]: osdmap e175: 8 total, 8 up, 8 in 2023-08-28T02:25:31.718 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:31 smithi169 ceph-mon[159415]: failed to encode map e175 with expected crc 2023-08-28T02:25:31.718 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:31 smithi169 ceph-mon[159415]: failed to encode map e175 with expected crc 2023-08-28T02:25:31.718 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:31 smithi169 ceph-mon[159415]: failed to encode map e175 with expected crc 2023-08-28T02:25:31.718 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:31 smithi169 ceph-mon[159415]: failed to encode map e175 with expected crc 2023-08-28T02:25:31.718 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:31 smithi169 ceph-mon[159415]: failed to encode map e175 with expected crc 2023-08-28T02:25:31.718 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:31 smithi169 ceph-mon[159415]: failed to encode map e175 with expected crc 2023-08-28T02:25:31.718 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:31 smithi169 ceph-mon[159415]: pgmap v387: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 2.6 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 1665/129141 objects degraded (1.289%); 7551/129141 objects misplaced (5.847%); 2.0 MiB/s, 31 objects/s recovering 2023-08-28T02:25:31.718 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:31 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 1665/129141 objects degraded (1.289%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:25:31.723 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:31.793 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:25:31.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:31.801 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:25:31.902 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:25:31.987 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:32.161 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:25:31 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:25:31] "GET /metrics HTTP/1.1" 200 36490 "" "Prometheus/2.33.4" 2023-08-28T02:25:32.180 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:25:32.181 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:25:32.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:32.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:25:32.295 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:25:32.391 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:32.523 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:25:32.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:32.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:25:32.635 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:25:32.674 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:32.746 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:25:32.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:32.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:25:32.855 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:25:32.856 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:25:32.886 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:25:32.942 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:25:32.951 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:32.967 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:25:32.967 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:25:32.967 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:25:32.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:32 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2441533961' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:25:33.024 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:25:33.062 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:25:33.126 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:25:33.163 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:25:33.181 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:25:33.181 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:25:33.181 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:25:33.181 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:25:33.181 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:25:33.182 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:25:33.182 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:25:33.182 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:25:33.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:32 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2441533961' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:25:33.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:32 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2441533961' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:25:34.115 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:25:34.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:34.125 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:25:34.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:33 smithi143 ceph-mon[183059]: pgmap v388: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 2.5 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 1665/124947 objects degraded (1.333%); 7398/124947 objects misplaced (5.921%); 1.1 MiB/s, 16 objects/s recovering 2023-08-28T02:25:34.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:33 smithi143 ceph-mon[189871]: pgmap v388: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 2.5 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 1665/124947 objects degraded (1.333%); 7398/124947 objects misplaced (5.921%); 1.1 MiB/s, 16 objects/s recovering 2023-08-28T02:25:34.205 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset nodeep-scrub 2023-08-28T02:25:34.226 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:25:34.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:33 smithi169 ceph-mon[159415]: pgmap v388: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 2.5 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 1665/124947 objects degraded (1.333%); 7398/124947 objects misplaced (5.921%); 1.1 MiB/s, 16 objects/s recovering 2023-08-28T02:25:34.282 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:34.342 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:25:34.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:34.349 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:25:34.450 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:25:34.528 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:34.637 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:25:34.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:34.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:25:34.749 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:25:34.775 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:34.855 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:25:34.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:34.877 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:25:34.978 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:25:35.048 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.3 dump_blocked_ops 2023-08-28T02:25:35.063 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:35.611 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:25:35.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:35.621 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:25:35.722 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:25:35.757 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:35.831 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:25:35.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:35.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:25:35.943 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:25:35.982 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:36.084 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:25:36.084 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:25:36.089 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:25:36.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:36.096 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:25:36.197 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:25:36.280 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:36.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:36 smithi169 ceph-mon[159415]: pgmap v389: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 2.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 1752/119541 objects degraded (1.466%); 6998/119541 objects misplaced (5.854%); 1.8 MiB/s, 28 objects/s recovering 2023-08-28T02:25:36.495 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:36 smithi143 ceph-mon[189871]: pgmap v389: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 2.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 1752/119541 objects degraded (1.466%); 6998/119541 objects misplaced (5.854%); 1.8 MiB/s, 28 objects/s recovering 2023-08-28T02:25:36.495 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:36 smithi143 ceph-mon[183059]: pgmap v389: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 2.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 1752/119541 objects degraded (1.466%); 6998/119541 objects misplaced (5.854%); 1.8 MiB/s, 28 objects/s recovering 2023-08-28T02:25:36.503 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:25:36.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:36.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:25:36.616 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:25:36.709 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:37.338 INFO:teuthology.orchestra.run.smithi143.stderr:reweighted osd.7 to 1 (10000) 2023-08-28T02:25:37.462 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:25:37.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:37.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:25:37.571 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:25:37.615 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:25:37.638 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:37 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 1752/119541 objects degraded (1.466%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:25:37.638 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:37 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3618919349' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-08-28T02:25:37.638 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:37 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-08-28T02:25:37.639 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:37 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 1752/119541 objects degraded (1.466%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:25:37.639 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:37 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3618919349' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-08-28T02:25:37.639 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:37 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-08-28T02:25:37.642 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:37.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:37 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 1752/119541 objects degraded (1.466%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:25:37.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:37 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3618919349' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-08-28T02:25:37.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:37 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-08-28T02:25:38.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:38 smithi143 ceph-mon[189871]: pgmap v390: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 2.4 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 1752/115716 objects degraded (1.514%); 6847/115716 objects misplaced (5.917%); 1.6 MiB/s, 25 objects/s recovering 2023-08-28T02:25:38.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:38 smithi143 ceph-mon[189871]: failed to encode map e176 with expected crc 2023-08-28T02:25:38.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:38 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-08-28T02:25:38.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:38 smithi143 ceph-mon[189871]: osdmap e176: 8 total, 8 up, 8 in 2023-08-28T02:25:38.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:38 smithi143 ceph-mon[189871]: failed to encode map e176 with expected crc 2023-08-28T02:25:38.704 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:38 smithi143 ceph-mon[183059]: pgmap v390: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 2.4 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 1752/115716 objects degraded (1.514%); 6847/115716 objects misplaced (5.917%); 1.6 MiB/s, 25 objects/s recovering 2023-08-28T02:25:38.704 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:38 smithi143 ceph-mon[183059]: failed to encode map e176 with expected crc 2023-08-28T02:25:38.704 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:38 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-08-28T02:25:38.704 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:38 smithi143 ceph-mon[183059]: osdmap e176: 8 total, 8 up, 8 in 2023-08-28T02:25:38.704 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:38 smithi143 ceph-mon[183059]: failed to encode map e176 with expected crc 2023-08-28T02:25:38.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:38 smithi169 ceph-mon[159415]: pgmap v390: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 2.4 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 1752/115716 objects degraded (1.514%); 6847/115716 objects misplaced (5.917%); 1.6 MiB/s, 25 objects/s recovering 2023-08-28T02:25:38.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:38 smithi169 ceph-mon[159415]: failed to encode map e176 with expected crc 2023-08-28T02:25:38.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:38 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-08-28T02:25:38.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:38 smithi169 ceph-mon[159415]: osdmap e176: 8 total, 8 up, 8 in 2023-08-28T02:25:38.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:38 smithi169 ceph-mon[159415]: failed to encode map e176 with expected crc 2023-08-28T02:25:39.033 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:25:39.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:39.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:25:39.143 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:25:39.193 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:39.757 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd reweight 3 1 2023-08-28T02:25:39.811 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:25:39.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:39.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:25:39.919 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:25:40.056 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:40.227 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:25:40.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:40.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:25:40.254 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:25:40.337 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:25:40.366 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:40.398 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:25:40.398 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [], 2023-08-28T02:25:40.398 INFO:teuthology.orchestra.run.smithi143.stdout: "complaint_time": 30, 2023-08-28T02:25:40.398 INFO:teuthology.orchestra.run.smithi143.stdout: "num_blocked_ops": 0 2023-08-28T02:25:40.398 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:25:40.419 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:25:40.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:40.430 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:25:40.531 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:25:40.563 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:40.564 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:40 smithi169 ceph-mon[159415]: pgmap v392: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 2.3 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 1802/110568 objects degraded (1.630%); 6460/110568 objects misplaced (5.843%); 1.9 MiB/s, 30 objects/s recovering 2023-08-28T02:25:40.631 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:25:40.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:40.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:25:40.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:40 smithi143 ceph-mon[183059]: pgmap v392: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 2.3 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 1802/110568 objects degraded (1.630%); 6460/110568 objects misplaced (5.843%); 1.9 MiB/s, 30 objects/s recovering 2023-08-28T02:25:40.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:40 smithi143 ceph-mon[189871]: pgmap v392: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 2.3 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 1802/110568 objects degraded (1.630%); 6460/110568 objects misplaced (5.843%); 1.9 MiB/s, 30 objects/s recovering 2023-08-28T02:25:40.740 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:25:40.792 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:40.912 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:25:40.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:40.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:25:41.028 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:25:41.095 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:41.141 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:25:41.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:41.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:25:41.247 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:25:41.272 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:41.344 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:25:41.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:41.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:25:41.355 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is unset 2023-08-28T02:25:41.451 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:25:41.530 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:41.684 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:41 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2299025982' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:25:41.684 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:41 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:25:41.684 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:41 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 1802/110568 objects degraded (1.630%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:25:41.684 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:41 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2299025982' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:25:41.684 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:41 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:25:41.685 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:41 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 1802/110568 objects degraded (1.630%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:25:41.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:41 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2299025982' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:25:41.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:41 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:25:41.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:41 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 1802/110568 objects degraded (1.630%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:25:41.848 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.3 dump_historic_ops 2023-08-28T02:25:41.887 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:25:41.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:41.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:25:41.954 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:25:41 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:25:41] "GET /metrics HTTP/1.1" 200 36485 "" "Prometheus/2.33.4" 2023-08-28T02:25:41.993 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:25:42.037 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:42.092 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:25:42.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:42.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:25:42.200 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:25:42.226 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:42.268 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:25:42.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:42.272 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:25:42.318 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:42 smithi169 ceph-mon[159415]: pgmap v393: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 2.3 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 1802/110568 objects degraded (1.630%); 6460/110568 objects misplaced (5.843%); 1.7 MiB/s, 27 objects/s recovering 2023-08-28T02:25:42.318 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:42 smithi169 ceph-mon[159415]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:25:42.319 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:42 smithi169 ceph-mon[159415]: failed to encode map e177 with expected crc 2023-08-28T02:25:42.319 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:42 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2299025982' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:25:42.319 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:42 smithi169 ceph-mon[159415]: osdmap e177: 8 total, 8 up, 8 in 2023-08-28T02:25:42.319 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:42 smithi169 ceph-mon[159415]: failed to encode map e177 with expected crc 2023-08-28T02:25:42.372 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:25:42.428 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:42.501 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:25:42.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:42.510 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:25:42.611 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:25:42.684 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:42 smithi143 ceph-mon[189871]: pgmap v393: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 2.3 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 1802/110568 objects degraded (1.630%); 6460/110568 objects misplaced (5.843%); 1.7 MiB/s, 27 objects/s recovering 2023-08-28T02:25:42.685 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:42 smithi143 ceph-mon[189871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:25:42.685 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:42 smithi143 ceph-mon[189871]: failed to encode map e177 with expected crc 2023-08-28T02:25:42.685 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:42 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2299025982' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:25:42.685 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:42 smithi143 ceph-mon[189871]: osdmap e177: 8 total, 8 up, 8 in 2023-08-28T02:25:42.685 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:42 smithi143 ceph-mon[189871]: failed to encode map e177 with expected crc 2023-08-28T02:25:42.686 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:42 smithi143 ceph-mon[183059]: pgmap v393: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 2.3 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 1802/110568 objects degraded (1.630%); 6460/110568 objects misplaced (5.843%); 1.7 MiB/s, 27 objects/s recovering 2023-08-28T02:25:42.686 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:42 smithi143 ceph-mon[183059]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:25:42.686 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:42 smithi143 ceph-mon[183059]: failed to encode map e177 with expected crc 2023-08-28T02:25:42.686 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:42 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2299025982' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:25:42.686 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:42 smithi143 ceph-mon[183059]: osdmap e177: 8 total, 8 up, 8 in 2023-08-28T02:25:42.686 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:42 smithi143 ceph-mon[183059]: failed to encode map e177 with expected crc 2023-08-28T02:25:42.693 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:43.373 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:25:43.550 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:25:43.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:43.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:25:43.670 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:25:43.767 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:43.776 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:25:43.954 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:25:43.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:43.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:25:44.066 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:25:44.141 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:44.355 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:25:44.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:44.365 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:25:44.433 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:25:44.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:44 smithi143 ceph-mon[183059]: pgmap v395: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 2.2 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 527 op/s; 1802/106872 objects degraded (1.686%); 6303/106872 objects misplaced (5.898%); 1.2 MiB/s, 18 objects/s recovering 2023-08-28T02:25:44.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:44 smithi143 ceph-mon[189871]: pgmap v395: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 2.2 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 527 op/s; 1802/106872 objects degraded (1.686%); 6303/106872 objects misplaced (5.898%); 1.2 MiB/s, 18 objects/s recovering 2023-08-28T02:25:44.466 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:25:44.472 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:25:44.512 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:44.515 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:25:44.549 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:25:44.586 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:25:44.587 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:25:44.592 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:44 smithi169 ceph-mon[159415]: pgmap v395: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 2.2 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 527 op/s; 1802/106872 objects degraded (1.686%); 6303/106872 objects misplaced (5.898%); 1.2 MiB/s, 18 objects/s recovering 2023-08-28T02:25:44.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:44.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:25:44.622 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:25:44.660 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:25:44.689 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:25:44.692 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:25:44.692 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:25:44.692 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:25:44.692 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:25:44.693 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:25:44.693 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:25:44.693 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:25:44.693 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:25:44.694 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:25:44.776 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:44.887 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set noscrub 2023-08-28T02:25:44.966 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:25:44.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:44.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:25:45.082 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:25:45.121 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:45.180 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:25:45.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:45.190 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:25:45.291 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:25:45.343 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:45.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:45 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3176942252' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:25:45.609 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:45 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3176942252' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:25:45.633 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:45 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3176942252' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:25:46.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:46 smithi143 ceph-mon[183059]: pgmap v396: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 2.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 1858/101817 objects degraded (1.825%); 5912/101817 objects misplaced (5.806%); 2.2 MiB/s, 35 objects/s recovering 2023-08-28T02:25:46.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:46 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 1858/101817 objects degraded (1.825%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:25:46.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:46 smithi143 ceph-mon[189871]: pgmap v396: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 2.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 1858/101817 objects degraded (1.825%); 5912/101817 objects misplaced (5.806%); 2.2 MiB/s, 35 objects/s recovering 2023-08-28T02:25:46.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:46 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 1858/101817 objects degraded (1.825%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:25:46.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:46 smithi169 ceph-mon[159415]: pgmap v396: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 2.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 1858/101817 objects degraded (1.825%); 5912/101817 objects misplaced (5.806%); 2.2 MiB/s, 35 objects/s recovering 2023-08-28T02:25:46.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:46 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 1858/101817 objects degraded (1.825%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:25:46.778 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:25:46.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:46.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:25:46.882 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:25:46.978 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:47.137 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:25:47.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:47.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:25:47.265 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:25:47.320 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:47.390 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:25:47.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:47.397 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:25:47.498 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:25:47.556 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:47.644 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:25:47.647 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:47 smithi169 ceph-mon[159415]: pgmap v397: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 2.0 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 1858/97722 objects degraded (1.901%); 5753/97722 objects misplaced (5.887%); 1.8 MiB/s, 28 objects/s recovering 2023-08-28T02:25:47.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:47.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:25:47.659 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:25:47.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:47 smithi143 ceph-mon[183059]: pgmap v397: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 2.0 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 1858/97722 objects degraded (1.901%); 5753/97722 objects misplaced (5.887%); 1.8 MiB/s, 28 objects/s recovering 2023-08-28T02:25:47.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:47 smithi143 ceph-mon[189871]: pgmap v397: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 2.0 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 1858/97722 objects degraded (1.901%); 5753/97722 objects misplaced (5.887%); 1.8 MiB/s, 28 objects/s recovering 2023-08-28T02:25:47.749 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:25:47.781 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:47.816 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:25:47.817 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:25:47.844 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:25:47.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:47.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:25:47.951 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:25:48.046 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:48.221 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:25:48.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:48.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:25:48.299 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:25:48.337 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:25:48.396 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:48.418 INFO:teuthology.orchestra.run.smithi143.stderr:reweighted osd.3 to 1 (10000) 2023-08-28T02:25:48.428 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:48 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1489484694' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-08-28T02:25:48.428 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:48 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-08-28T02:25:48.428 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:48 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1489484694' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-08-28T02:25:48.429 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:48 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-08-28T02:25:48.454 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:25:48.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:48.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:25:48.561 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:25:48.611 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:48.698 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:25:48.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:48.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:25:48.714 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:48 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1489484694' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-08-28T02:25:48.714 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:48 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-08-28T02:25:48.814 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:25:48.846 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:48.921 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:25:48.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:48.937 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:25:49.037 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:25:49.067 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:49.152 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:25:49.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:49.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:25:49.264 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:25:49.297 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:49.379 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:25:49.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:49.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:25:49.485 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:25:49.557 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:49 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-08-28T02:25:49.558 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:49 smithi143 ceph-mon[183059]: osdmap e178: 8 total, 8 up, 8 in 2023-08-28T02:25:49.558 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:49 smithi143 ceph-mon[183059]: failed to encode map e178 with expected crc 2023-08-28T02:25:49.558 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:49 smithi143 ceph-mon[183059]: failed to encode map e178 with expected crc 2023-08-28T02:25:49.558 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:49 smithi143 ceph-mon[183059]: failed to encode map e178 with expected crc 2023-08-28T02:25:49.558 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:49 smithi143 ceph-mon[183059]: pgmap v399: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.9 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 1925/92211 objects degraded (2.088%); 5348/92211 objects misplaced (5.800%); 2.2 MiB/s, 34 objects/s recovering 2023-08-28T02:25:49.560 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:49 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-08-28T02:25:49.560 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:49 smithi143 ceph-mon[189871]: osdmap e178: 8 total, 8 up, 8 in 2023-08-28T02:25:49.561 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:49 smithi143 ceph-mon[189871]: failed to encode map e178 with expected crc 2023-08-28T02:25:49.561 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:49 smithi143 ceph-mon[189871]: failed to encode map e178 with expected crc 2023-08-28T02:25:49.561 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:49 smithi143 ceph-mon[189871]: failed to encode map e178 with expected crc 2023-08-28T02:25:49.561 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:49 smithi143 ceph-mon[189871]: pgmap v399: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.9 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 1925/92211 objects degraded (2.088%); 5348/92211 objects misplaced (5.800%); 2.2 MiB/s, 34 objects/s recovering 2023-08-28T02:25:49.567 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:49.683 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.4 dump_ops_in_flight 2023-08-28T02:25:49.736 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:49 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-08-28T02:25:49.736 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:49 smithi169 ceph-mon[159415]: osdmap e178: 8 total, 8 up, 8 in 2023-08-28T02:25:49.736 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:49 smithi169 ceph-mon[159415]: failed to encode map e178 with expected crc 2023-08-28T02:25:49.736 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:49 smithi169 ceph-mon[159415]: failed to encode map e178 with expected crc 2023-08-28T02:25:49.736 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:49 smithi169 ceph-mon[159415]: failed to encode map e178 with expected crc 2023-08-28T02:25:49.736 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:49 smithi169 ceph-mon[159415]: pgmap v399: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.9 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 1925/92211 objects degraded (2.088%); 5348/92211 objects misplaced (5.800%); 2.2 MiB/s, 34 objects/s recovering 2023-08-28T02:25:49.890 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:25:49.899 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-08-28T02:25:49.899 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph pg dump --format=json 2023-08-28T02:25:49.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:49.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:25:50.002 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:25:50.068 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:50.146 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:25:50.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:50.170 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:25:50.266 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:25:50.272 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:25:50.342 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:50.819 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:25:50.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:50.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:25:50.920 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:25:50.925 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:25:50.974 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:51.063 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:25:51.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:51.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:25:51.167 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:25:51.250 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:51.270 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:25:51.638 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:25:51.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:51.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:25:51.745 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:25:51.779 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:51.869 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:25:51.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:51.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:25:51.936 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:25:51 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:25:51] "GET /metrics HTTP/1.1" 200 36485 "" "Prometheus/2.33.4" 2023-08-28T02:25:51.978 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:25:52.081 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:52.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:52 smithi169 ceph-mon[159415]: pgmap v400: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.9 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 1925/92211 objects degraded (2.088%); 5348/92211 objects misplaced (5.800%); 1.8 MiB/s, 28 objects/s recovering 2023-08-28T02:25:52.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:52 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 1925/92211 objects degraded (2.088%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:25:52.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:52 smithi143 ceph-mon[189871]: pgmap v400: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.9 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 1925/92211 objects degraded (2.088%); 5348/92211 objects misplaced (5.800%); 1.8 MiB/s, 28 objects/s recovering 2023-08-28T02:25:52.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:52 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 1925/92211 objects degraded (2.088%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:25:52.704 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:52 smithi143 ceph-mon[183059]: pgmap v400: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.9 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 1925/92211 objects degraded (2.088%); 5348/92211 objects misplaced (5.800%); 1.8 MiB/s, 28 objects/s recovering 2023-08-28T02:25:52.704 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:52 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 1925/92211 objects degraded (2.088%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:25:53.055 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:25:53.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:53.059 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:25:53.160 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:25:53.240 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:53.262 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is set 2023-08-28T02:25:53.291 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:53 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1404780920' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:25:53.291 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:53 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1404780920' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:25:53.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:53 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1404780920' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:25:53.805 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:25:53.805 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:25:54.290 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:25:54.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:54.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:25:54.393 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:25:54.442 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:54.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:54 smithi169 ceph-mon[159415]: pgmap v401: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.8 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 1925/88104 objects degraded (2.185%); 5181/88104 objects misplaced (5.881%); 1.7 MiB/s, 27 objects/s recovering 2023-08-28T02:25:54.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:54 smithi169 ceph-mon[159415]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:25:54.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:54 smithi169 ceph-mon[159415]: failed to encode map e179 with expected crc 2023-08-28T02:25:54.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:54 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1404780920' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:25:54.491 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:54 smithi169 ceph-mon[159415]: osdmap e179: 8 total, 8 up, 8 in 2023-08-28T02:25:54.502 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:54 smithi143 ceph-mon[183059]: pgmap v401: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.8 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 1925/88104 objects degraded (2.185%); 5181/88104 objects misplaced (5.881%); 1.7 MiB/s, 27 objects/s recovering 2023-08-28T02:25:54.502 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:54 smithi143 ceph-mon[183059]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:25:54.502 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:54 smithi143 ceph-mon[183059]: failed to encode map e179 with expected crc 2023-08-28T02:25:54.503 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:54 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1404780920' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:25:54.503 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:54 smithi143 ceph-mon[183059]: osdmap e179: 8 total, 8 up, 8 in 2023-08-28T02:25:54.503 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:54 smithi143 ceph-mon[189871]: pgmap v401: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.8 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 1925/88104 objects degraded (2.185%); 5181/88104 objects misplaced (5.881%); 1.7 MiB/s, 27 objects/s recovering 2023-08-28T02:25:54.503 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:54 smithi143 ceph-mon[189871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:25:54.503 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:54 smithi143 ceph-mon[189871]: failed to encode map e179 with expected crc 2023-08-28T02:25:54.503 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:54 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1404780920' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:25:54.503 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:54 smithi143 ceph-mon[189871]: osdmap e179: 8 total, 8 up, 8 in 2023-08-28T02:25:54.677 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:25:54.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:54.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:25:54.694 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.4 dump_blocked_ops 2023-08-28T02:25:54.786 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:25:54.813 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:55.117 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:25:55.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:55.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:25:55.224 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:25:55.261 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:55.295 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:25:55.372 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:25:55.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:55.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:25:55.407 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:25:55.447 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:25:55.478 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:25:55.492 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:25:55.532 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:55.544 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:25:55.587 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:25:55.617 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:25:55.626 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:25:55.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:55.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:25:55.659 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:25:55.709 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:25:55.728 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:25:55.728 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:25:55.728 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:25:55.729 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:25:55.729 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:25:55.729 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:25:55.729 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:25:55.729 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:25:55.729 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:25:55.816 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:56.032 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:25:56.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:56.054 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:25:56.155 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:25:56.205 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:25:56.208 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:56.218 INFO:teuthology.orchestra.run.smithi143.stderr:dumped all 2023-08-28T02:25:56.302 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:25:56.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:56.309 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:25:56.410 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:25:56.444 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:56.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:56 smithi143 ceph-mon[183059]: pgmap v403: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.7 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 1971/82623 objects degraded (2.386%); 4777/82623 objects misplaced (5.782%); 2.3 MiB/s, 36 objects/s recovering 2023-08-28T02:25:56.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:56 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2158537771' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:25:56.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:56 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:25:56.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:56 smithi143 ceph-mon[189871]: pgmap v403: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.7 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 1971/82623 objects degraded (2.386%); 4777/82623 objects misplaced (5.782%); 2.3 MiB/s, 36 objects/s recovering 2023-08-28T02:25:56.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:56 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2158537771' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:25:56.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:56 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:25:56.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:56 smithi169 ceph-mon[159415]: pgmap v403: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.7 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 1971/82623 objects degraded (2.386%); 4777/82623 objects misplaced (5.782%); 2.3 MiB/s, 36 objects/s recovering 2023-08-28T02:25:56.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:56 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2158537771' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:25:56.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:56 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:25:56.523 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:25:56.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:56.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:25:56.627 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:25:56.680 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:57.273 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set nodeep-scrub 2023-08-28T02:25:57.409 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:25:57.409 INFO:teuthology.orchestra.run.smithi169.stdout: "ops": [], 2023-08-28T02:25:57.410 INFO:teuthology.orchestra.run.smithi169.stdout: "complaint_time": 30, 2023-08-28T02:25:57.410 INFO:teuthology.orchestra.run.smithi169.stdout: "num_blocked_ops": 0 2023-08-28T02:25:57.410 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:25:57.433 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:57 smithi169 ceph-mon[159415]: from='client.54760 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:25:57.433 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:57 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 1971/82623 objects degraded (2.386%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:25:57.561 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:25:57.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:57.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:25:57.566 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:57 smithi143 ceph-mon[183059]: from='client.54760 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:25:57.566 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:57 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 1971/82623 objects degraded (2.386%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:25:57.567 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:57 smithi143 ceph-mon[189871]: from='client.54760 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:25:57.567 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:57 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 1971/82623 objects degraded (2.386%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:25:57.667 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:25:57.679 INFO:teuthology.orchestra.run.smithi143.stdout:{"pg_ready":true,"pg_map":{"version":403,"stamp":"2023-08-28T02:25:55.149267+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1807816651,"num_objects":27541,"num_object_clones":0,"num_object_copies":82623,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1971,"num_objects_misplaced":4777,"num_objects_unfound":0,"num_objects_dirty":27541,"num_whiteouts":0,"num_read":6274,"num_read_kb":13707,"num_write":193875,"num_write_kb":7064102,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5571,"num_bytes_recovered":363276934,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139484,"ondisk_log_size":139484,"up":267,"acting":266,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":279,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7404332,"kb_used_data":5753196,"kb_used_omap":29,"kb_used_meta":1650786,"kb_avail":742556884,"statfs":{"total":767960285184,"available":760378249216,"internally_reserved":0,"allocated":5891272704,"data_stored":5885758325,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30674,"internal_metadata":1690404910},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-329842688,"num_objects":-5033,"num_object_clones":0,"num_object_copies":-15099,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":113,"num_objects_misplaced":-976,"num_objects_unfound":0,"num_objects_dirty":-5033,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5033,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":294,"num_bytes_recovered":19267584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002127"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":161,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-08-28T02:25:41.334060+0000","last_change":"2023-08-28T02:20:19.849807+0000","last_active":"2023-08-28T02:25:41.334060+0000","last_peered":"2023-08-28T02:25:41.334060+0000","last_clean":"2023-08-28T02:25:41.334060+0000","last_became_active":"2023-08-28T02:20:19.849464+0000","last_became_peered":"2023-08-28T02:20:19.849464+0000","last_unstale":"2023-08-28T02:25:41.334060+0000","last_undegraded":"2023-08-28T02:25:41.334060+0000","last_fullsized":"2023-08-28T02:25:41.334060+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":36,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.427532+0000","last_change":"2023-08-28T02:23:59.204957+0000","last_active":"2023-08-28T02:25:48.427532+0000","last_peered":"2023-08-28T02:25:48.427532+0000","last_clean":"2023-08-28T02:25:48.427532+0000","last_became_active":"2023-08-28T02:23:59.204717+0000","last_became_peered":"2023-08-28T02:23:59.204717+0000","last_unstale":"2023-08-28T02:25:48.427532+0000","last_undegraded":"2023-08-28T02:25:48.427532+0000","last_fullsized":"2023-08-28T02:25:48.427532+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:34:58.977269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":137,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.408447+0000","last_change":"2023-08-28T02:23:59.212721+0000","last_active":"2023-08-28T02:25:48.408447+0000","last_peered":"2023-08-28T02:25:48.408447+0000","last_clean":"2023-08-28T02:25:48.408447+0000","last_became_active":"2023-08-28T02:23:59.212446+0000","last_became_peered":"2023-08-28T02:23:59.212446+0000","last_unstale":"2023-08-28T02:25:48.408447+0000","last_undegraded":"2023-08-28T02:25:48.408447+0000","last_fullsized":"2023-08-28T02:25:48.408447+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:03:54.701399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":165,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.508558+0000","last_change":"2023-08-28T02:23:59.223634+0000","last_active":"2023-08-28T02:25:48.508558+0000","last_peered":"2023-08-28T02:25:48.508558+0000","last_clean":"2023-08-28T02:25:48.508558+0000","last_became_active":"2023-08-28T02:23:59.222704+0000","last_became_peered":"2023-08-28T02:23:59.222704+0000","last_unstale":"2023-08-28T02:25:48.508558+0000","last_undegraded":"2023-08-28T02:25:48.508558+0000","last_fullsized":"2023-08-28T02:25:48.508558+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:51:52.752361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":36,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.428204+0000","last_change":"2023-08-28T02:23:59.220459+0000","last_active":"2023-08-28T02:25:48.428204+0000","last_peered":"2023-08-28T02:25:48.428204+0000","last_clean":"2023-08-28T02:25:48.428204+0000","last_became_active":"2023-08-28T02:23:59.219274+0000","last_became_peered":"2023-08-28T02:23:59.219274+0000","last_unstale":"2023-08-28T02:25:48.428204+0000","last_undegraded":"2023-08-28T02:25:48.428204+0000","last_fullsized":"2023-08-28T02:25:48.428204+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:31:53.889311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":172,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-08-28T02:25:41.336685+0000","last_change":"2023-08-28T02:21:59.183009+0000","last_active":"2023-08-28T02:25:41.336685+0000","last_peered":"2023-08-28T02:25:41.336685+0000","last_clean":"2023-08-28T02:25:41.336685+0000","last_became_active":"2023-08-28T02:21:59.182621+0000","last_became_peered":"2023-08-28T02:21:59.182621+0000","last_unstale":"2023-08-28T02:25:41.336685+0000","last_undegraded":"2023-08-28T02:25:41.336685+0000","last_fullsized":"2023-08-28T02:25:41.336685+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":146,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.260805+0000","last_change":"2023-08-28T02:23:59.227592+0000","last_active":"2023-08-28T02:25:53.260805+0000","last_peered":"2023-08-28T02:25:53.260805+0000","last_clean":"2023-08-28T02:25:53.260805+0000","last_became_active":"2023-08-28T02:23:59.227342+0000","last_became_peered":"2023-08-28T02:23:59.227342+0000","last_unstale":"2023-08-28T02:25:53.260805+0000","last_undegraded":"2023-08-28T02:25:53.260805+0000","last_fullsized":"2023-08-28T02:25:53.260805+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:08:19.874396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":36,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.428123+0000","last_change":"2023-08-28T02:23:59.220634+0000","last_active":"2023-08-28T02:25:48.428123+0000","last_peered":"2023-08-28T02:25:48.428123+0000","last_clean":"2023-08-28T02:25:48.428123+0000","last_became_active":"2023-08-28T02:23:59.220177+0000","last_became_peered":"2023-08-28T02:23:59.220177+0000","last_unstale":"2023-08-28T02:25:48.428123+0000","last_undegraded":"2023-08-28T02:25:48.428123+0000","last_fullsized":"2023-08-28T02:25:48.428123+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:16:26.906064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":163,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-08-28T02:25:41.331405+0000","last_change":"2023-08-28T02:23:38.339581+0000","last_active":"2023-08-28T02:25:41.331405+0000","last_peered":"2023-08-28T02:25:41.331405+0000","last_clean":"2023-08-28T02:25:41.331405+0000","last_became_active":"2023-08-28T02:23:38.339244+0000","last_became_peered":"2023-08-28T02:23:38.339244+0000","last_unstale":"2023-08-28T02:25:41.331405+0000","last_undegraded":"2023-08-28T02:25:41.331405+0000","last_fullsized":"2023-08-28T02:25:41.331405+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":36,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.428855+0000","last_change":"2023-08-28T02:23:59.220551+0000","last_active":"2023-08-28T02:25:48.428855+0000","last_peered":"2023-08-28T02:25:48.428855+0000","last_clean":"2023-08-28T02:25:48.428855+0000","last_became_active":"2023-08-28T02:23:59.219681+0000","last_became_peered":"2023-08-28T02:23:59.219681+0000","last_unstale":"2023-08-28T02:25:48.428855+0000","last_undegraded":"2023-08-28T02:25:48.428855+0000","last_fullsized":"2023-08-28T02:25:48.428855+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:39:58.811256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":64,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-08-28T02:25:41.337716+0000","last_change":"2023-08-28T02:23:59.316561+0000","last_active":"2023-08-28T02:25:41.337716+0000","last_peered":"2023-08-28T02:25:41.337716+0000","last_clean":"2023-08-28T02:25:41.337716+0000","last_became_active":"2023-08-28T02:23:59.227433+0000","last_became_peered":"2023-08-28T02:23:59.227433+0000","last_unstale":"2023-08-28T02:25:41.337716+0000","last_undegraded":"2023-08-28T02:25:41.337716+0000","last_fullsized":"2023-08-28T02:25:41.337716+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":120,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.408535+0000","last_change":"2023-08-28T02:20:19.851616+0000","last_active":"2023-08-28T02:25:48.408535+0000","last_peered":"2023-08-28T02:25:48.408535+0000","last_clean":"2023-08-28T02:25:48.408535+0000","last_became_active":"2023-08-28T02:20:19.851322+0000","last_became_peered":"2023-08-28T02:20:19.851322+0000","last_unstale":"2023-08-28T02:25:48.408535+0000","last_undegraded":"2023-08-28T02:25:48.408535+0000","last_fullsized":"2023-08-28T02:25:48.408535+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:03:37.565646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"86'22","reported_seq":190,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-08-28T02:25:41.334651+0000","last_change":"2023-08-28T02:23:38.368275+0000","last_active":"2023-08-28T02:25:41.334651+0000","last_peered":"2023-08-28T02:25:41.334651+0000","last_clean":"2023-08-28T02:25:41.334651+0000","last_became_active":"2023-08-28T02:23:38.342097+0000","last_became_peered":"2023-08-28T02:23:38.342097+0000","last_unstale":"2023-08-28T02:25:41.334651+0000","last_undegraded":"2023-08-28T02:25:41.334651+0000","last_fullsized":"2023-08-28T02:25:41.334651+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":158,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-08-28T02:25:41.333786+0000","last_change":"2023-08-28T02:19:51.204118+0000","last_active":"2023-08-28T02:25:41.333786+0000","last_peered":"2023-08-28T02:25:41.333786+0000","last_clean":"2023-08-28T02:25:41.333786+0000","last_became_active":"2023-08-28T02:19:51.203745+0000","last_became_peered":"2023-08-28T02:19:51.203745+0000","last_unstale":"2023-08-28T02:25:41.333786+0000","last_undegraded":"2023-08-28T02:25:41.333786+0000","last_fullsized":"2023-08-28T02:25:41.333786+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":49,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-08-28T02:25:41.337471+0000","last_change":"2023-08-28T02:23:38.983634+0000","last_active":"2023-08-28T02:25:41.337471+0000","last_peered":"2023-08-28T02:25:41.337471+0000","last_clean":"2023-08-28T02:25:41.337471+0000","last_became_active":"2023-08-28T02:23:38.344566+0000","last_became_peered":"2023-08-28T02:23:38.344566+0000","last_unstale":"2023-08-28T02:25:41.337471+0000","last_undegraded":"2023-08-28T02:25:41.337471+0000","last_fullsized":"2023-08-28T02:25:41.337471+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":36,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.427572+0000","last_change":"2023-08-28T02:23:59.216960+0000","last_active":"2023-08-28T02:25:48.427572+0000","last_peered":"2023-08-28T02:25:48.427572+0000","last_clean":"2023-08-28T02:25:48.427572+0000","last_became_active":"2023-08-28T02:23:59.215657+0000","last_became_peered":"2023-08-28T02:23:59.215657+0000","last_unstale":"2023-08-28T02:25:48.427572+0000","last_undegraded":"2023-08-28T02:25:48.427572+0000","last_fullsized":"2023-08-28T02:25:48.427572+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:31:14.407108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":164,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.508424+0000","last_change":"2023-08-28T02:23:59.223066+0000","last_active":"2023-08-28T02:25:48.508424+0000","last_peered":"2023-08-28T02:25:48.508424+0000","last_clean":"2023-08-28T02:25:48.508424+0000","last_became_active":"2023-08-28T02:23:59.222731+0000","last_became_peered":"2023-08-28T02:23:59.222731+0000","last_unstale":"2023-08-28T02:25:48.508424+0000","last_undegraded":"2023-08-28T02:25:48.508424+0000","last_fullsized":"2023-08-28T02:25:48.508424+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:48:37.558266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":205,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-08-28T02:25:41.330899+0000","last_change":"2023-08-28T02:23:59.210469+0000","last_active":"2023-08-28T02:25:41.330899+0000","last_peered":"2023-08-28T02:25:41.330899+0000","last_clean":"2023-08-28T02:25:41.330899+0000","last_became_active":"2023-08-28T02:23:59.210031+0000","last_became_peered":"2023-08-28T02:23:59.210031+0000","last_unstale":"2023-08-28T02:25:41.330899+0000","last_undegraded":"2023-08-28T02:25:41.330899+0000","last_fullsized":"2023-08-28T02:25:41.330899+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":43,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-08-28T02:25:41.337208+0000","last_change":"2023-08-28T02:23:38.370183+0000","last_active":"2023-08-28T02:25:41.337208+0000","last_peered":"2023-08-28T02:25:41.337208+0000","last_clean":"2023-08-28T02:25:41.337208+0000","last_became_active":"2023-08-28T02:23:38.368415+0000","last_became_peered":"2023-08-28T02:23:38.368415+0000","last_unstale":"2023-08-28T02:25:41.337208+0000","last_undegraded":"2023-08-28T02:25:41.337208+0000","last_fullsized":"2023-08-28T02:25:41.337208+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":165,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.260742+0000","last_change":"2023-08-28T02:23:59.220932+0000","last_active":"2023-08-28T02:25:53.260742+0000","last_peered":"2023-08-28T02:25:53.260742+0000","last_clean":"2023-08-28T02:25:53.260742+0000","last_became_active":"2023-08-28T02:23:59.220488+0000","last_became_peered":"2023-08-28T02:23:59.220488+0000","last_unstale":"2023-08-28T02:25:53.260742+0000","last_undegraded":"2023-08-28T02:25:53.260742+0000","last_fullsized":"2023-08-28T02:25:53.260742+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:53:09.216340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":152,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-08-28T02:25:41.335548+0000","last_change":"2023-08-28T02:18:39.778728+0000","last_active":"2023-08-28T02:25:41.335548+0000","last_peered":"2023-08-28T02:25:41.335548+0000","last_clean":"2023-08-28T02:25:41.335548+0000","last_became_active":"2023-08-28T02:18:39.778283+0000","last_became_peered":"2023-08-28T02:18:39.778283+0000","last_unstale":"2023-08-28T02:25:41.335548+0000","last_undegraded":"2023-08-28T02:25:41.335548+0000","last_fullsized":"2023-08-28T02:25:41.335548+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":36,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.427859+0000","last_change":"2023-08-28T02:23:59.216743+0000","last_active":"2023-08-28T02:25:48.427859+0000","last_peered":"2023-08-28T02:25:48.427859+0000","last_clean":"2023-08-28T02:25:48.427859+0000","last_became_active":"2023-08-28T02:23:59.216416+0000","last_became_peered":"2023-08-28T02:23:59.216416+0000","last_unstale":"2023-08-28T02:25:48.427859+0000","last_undegraded":"2023-08-28T02:25:48.427859+0000","last_fullsized":"2023-08-28T02:25:48.427859+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:25:13.056747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"178'11951","reported_seq":12783,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:52.064146+0000","last_change":"2023-08-28T02:24:09.355130+0000","last_active":"2023-08-28T02:25:52.064146+0000","last_peered":"2023-08-28T02:25:52.064146+0000","last_clean":"2023-08-28T02:25:52.064146+0000","last_became_active":"2023-08-28T02:24:09.341363+0000","last_became_peered":"2023-08-28T02:24:09.341363+0000","last_unstale":"2023-08-28T02:25:52.064146+0000","last_undegraded":"2023-08-28T02:25:52.064146+0000","last_fullsized":"2023-08-28T02:25:52.064146+0000","mapping_epoch":161,"log_start":"129'3300","ondisk_log_start":"129'3300","created":119,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":8651,"log_dups_size":0,"ondisk_log_size":8651,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115408896,"num_objects":1761,"num_object_clones":0,"num_object_copies":5283,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1761,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11951,"num_write_kb":438784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":367,"num_bytes_recovered":23789568,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":47,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-08-28T02:25:41.337589+0000","last_change":"2023-08-28T02:23:38.657185+0000","last_active":"2023-08-28T02:25:41.337589+0000","last_peered":"2023-08-28T02:25:41.337589+0000","last_clean":"2023-08-28T02:25:41.337589+0000","last_became_active":"2023-08-28T02:23:38.339016+0000","last_became_peered":"2023-08-28T02:23:38.339016+0000","last_unstale":"2023-08-28T02:25:41.337589+0000","last_undegraded":"2023-08-28T02:25:41.337589+0000","last_fullsized":"2023-08-28T02:25:41.337589+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":147,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-08-28T02:25:41.330456+0000","last_change":"2023-08-28T02:23:38.339886+0000","last_active":"2023-08-28T02:25:41.330456+0000","last_peered":"2023-08-28T02:25:41.330456+0000","last_clean":"2023-08-28T02:25:41.330456+0000","last_became_active":"2023-08-28T02:23:38.339514+0000","last_became_peered":"2023-08-28T02:23:38.339514+0000","last_unstale":"2023-08-28T02:25:41.330456+0000","last_undegraded":"2023-08-28T02:25:41.330456+0000","last_fullsized":"2023-08-28T02:25:41.330456+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"179'12116","reported_seq":12185,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.754107+0000","last_change":"2023-08-28T02:22:00.165260+0000","last_active":"2023-08-28T02:25:53.754107+0000","last_peered":"2023-08-28T02:25:53.754107+0000","last_clean":"2023-08-28T02:25:53.754107+0000","last_became_active":"2023-08-28T02:22:00.164978+0000","last_became_peered":"2023-08-28T02:22:00.164978+0000","last_unstale":"2023-08-28T02:25:53.754107+0000","last_undegraded":"2023-08-28T02:25:53.754107+0000","last_fullsized":"2023-08-28T02:25:53.754107+0000","mapping_epoch":119,"log_start":"128'3000","ondisk_log_start":"128'3000","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":9116,"log_dups_size":0,"ondisk_log_size":9116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:18:41.711960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112066560,"num_objects":1710,"num_object_clones":0,"num_object_copies":5130,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1710,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12116,"num_write_kb":442432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":47,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-08-28T02:25:41.337460+0000","last_change":"2023-08-28T02:23:38.555339+0000","last_active":"2023-08-28T02:25:41.337460+0000","last_peered":"2023-08-28T02:25:41.337460+0000","last_clean":"2023-08-28T02:25:41.337460+0000","last_became_active":"2023-08-28T02:23:38.337938+0000","last_became_peered":"2023-08-28T02:23:38.337938+0000","last_unstale":"2023-08-28T02:25:41.337460+0000","last_undegraded":"2023-08-28T02:25:41.337460+0000","last_fullsized":"2023-08-28T02:25:41.337460+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":186,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-08-28T02:25:41.334122+0000","last_change":"2023-08-28T02:23:59.208685+0000","last_active":"2023-08-28T02:25:41.334122+0000","last_peered":"2023-08-28T02:25:41.334122+0000","last_clean":"2023-08-28T02:25:41.334122+0000","last_became_active":"2023-08-28T02:23:59.208377+0000","last_became_peered":"2023-08-28T02:23:59.208377+0000","last_unstale":"2023-08-28T02:25:41.334122+0000","last_undegraded":"2023-08-28T02:25:41.334122+0000","last_fullsized":"2023-08-28T02:25:41.334122+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"179'12033","reported_seq":12102,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.813404+0000","last_change":"2023-08-28T02:22:00.165317+0000","last_active":"2023-08-28T02:25:53.813404+0000","last_peered":"2023-08-28T02:25:53.813404+0000","last_clean":"2023-08-28T02:25:53.813404+0000","last_became_active":"2023-08-28T02:22:00.164989+0000","last_became_peered":"2023-08-28T02:22:00.164989+0000","last_unstale":"2023-08-28T02:25:53.813404+0000","last_undegraded":"2023-08-28T02:25:53.813404+0000","last_fullsized":"2023-08-28T02:25:53.813404+0000","mapping_epoch":119,"log_start":"128'2900","ondisk_log_start":"128'2900","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":9133,"log_dups_size":0,"ondisk_log_size":9133,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:10:46.322659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110559232,"num_objects":1687,"num_object_clones":0,"num_object_copies":5061,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1687,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12033,"num_write_kb":439040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":47,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-08-28T02:25:41.337606+0000","last_change":"2023-08-28T02:23:38.749428+0000","last_active":"2023-08-28T02:25:41.337606+0000","last_peered":"2023-08-28T02:25:41.337606+0000","last_clean":"2023-08-28T02:25:41.337606+0000","last_became_active":"2023-08-28T02:23:38.339587+0000","last_became_peered":"2023-08-28T02:23:38.339587+0000","last_unstale":"2023-08-28T02:25:41.337606+0000","last_undegraded":"2023-08-28T02:25:41.337606+0000","last_fullsized":"2023-08-28T02:25:41.337606+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":204,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-08-28T02:25:41.334403+0000","last_change":"2023-08-28T02:23:38.340065+0000","last_active":"2023-08-28T02:25:41.334403+0000","last_peered":"2023-08-28T02:25:41.334403+0000","last_clean":"2023-08-28T02:25:41.334403+0000","last_became_active":"2023-08-28T02:23:38.328689+0000","last_became_peered":"2023-08-28T02:23:38.328689+0000","last_unstale":"2023-08-28T02:25:41.334403+0000","last_undegraded":"2023-08-28T02:25:41.334403+0000","last_fullsized":"2023-08-28T02:25:41.334403+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"179'12121","reported_seq":17119,"reported_epoch":179,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-08-28T02:25:53.819001+0000","last_change":"2023-08-28T02:24:08.250989+0000","last_active":"2023-08-28T02:25:53.819001+0000","last_peered":"2023-08-28T02:25:53.819001+0000","last_clean":"2023-08-28T02:23:58.182539+0000","last_became_active":"2023-08-28T02:24:00.420432+0000","last_became_peered":"2023-08-28T02:24:00.420432+0000","last_unstale":"2023-08-28T02:25:53.819001+0000","last_undegraded":"2023-08-28T02:24:00.242280+0000","last_fullsized":"2023-08-28T02:24:00.199333+0000","mapping_epoch":154,"log_start":"128'3000","ondisk_log_start":"128'3000","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":9121,"log_dups_size":0,"ondisk_log_size":9121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:57:53.818041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107806720,"num_objects":1645,"num_object_clones":0,"num_object_copies":4935,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1971,"num_objects_misplaced":1645,"num_objects_unfound":0,"num_objects_dirty":1645,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12121,"num_write_kb":440512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2460,"num_bytes_recovered":161218560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4],"avail_no_missing":["2","0","4"],"object_location_counts":[{"shards":"0,2,4","objects":1645},{"shards":"2,4","objects":-1971}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":163,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-08-28T02:25:41.335024+0000","last_change":"2023-08-28T02:23:59.211466+0000","last_active":"2023-08-28T02:25:41.335024+0000","last_peered":"2023-08-28T02:25:41.335024+0000","last_clean":"2023-08-28T02:25:41.335024+0000","last_became_active":"2023-08-28T02:23:59.211175+0000","last_became_peered":"2023-08-28T02:23:59.211175+0000","last_unstale":"2023-08-28T02:25:41.335024+0000","last_undegraded":"2023-08-28T02:25:41.335024+0000","last_fullsized":"2023-08-28T02:25:41.335024+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":140,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.414737+0000","last_change":"2023-08-28T02:19:51.205245+0000","last_active":"2023-08-28T02:25:48.414737+0000","last_peered":"2023-08-28T02:25:48.414737+0000","last_clean":"2023-08-28T02:25:48.414737+0000","last_became_active":"2023-08-28T02:19:51.204896+0000","last_became_peered":"2023-08-28T02:19:51.204896+0000","last_unstale":"2023-08-28T02:25:48.414737+0000","last_undegraded":"2023-08-28T02:25:48.414737+0000","last_fullsized":"2023-08-28T02:25:48.414737+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:51:24.634702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"179'12113","reported_seq":13015,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.744128+0000","last_change":"2023-08-28T02:24:04.294764+0000","last_active":"2023-08-28T02:25:53.744128+0000","last_peered":"2023-08-28T02:25:53.744128+0000","last_clean":"2023-08-28T02:25:53.744128+0000","last_became_active":"2023-08-28T02:24:04.280269+0000","last_became_peered":"2023-08-28T02:24:04.280269+0000","last_unstale":"2023-08-28T02:25:53.744128+0000","last_undegraded":"2023-08-28T02:25:53.744128+0000","last_fullsized":"2023-08-28T02:25:53.744128+0000","mapping_epoch":157,"log_start":"128'3000","ondisk_log_start":"128'3000","created":119,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":9113,"log_dups_size":0,"ondisk_log_size":9113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:18:55.996939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115081240,"num_objects":1757,"num_object_clones":0,"num_object_copies":5271,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1757,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12113,"num_write_kb":443777,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":403,"num_bytes_recovered":26345496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":40,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.427259+0000","last_change":"2023-08-28T02:23:59.230059+0000","last_active":"2023-08-28T02:25:48.427259+0000","last_peered":"2023-08-28T02:25:48.427259+0000","last_clean":"2023-08-28T02:25:48.427259+0000","last_became_active":"2023-08-28T02:23:59.217783+0000","last_became_peered":"2023-08-28T02:23:59.217783+0000","last_unstale":"2023-08-28T02:25:48.427259+0000","last_undegraded":"2023-08-28T02:25:48.427259+0000","last_fullsized":"2023-08-28T02:25:48.427259+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:00:12.145395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":171,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-08-28T02:25:41.334512+0000","last_change":"2023-08-28T02:23:59.211286+0000","last_active":"2023-08-28T02:25:41.334512+0000","last_peered":"2023-08-28T02:25:41.334512+0000","last_clean":"2023-08-28T02:25:41.334512+0000","last_became_active":"2023-08-28T02:23:59.210942+0000","last_became_peered":"2023-08-28T02:23:59.210942+0000","last_unstale":"2023-08-28T02:25:41.334512+0000","last_undegraded":"2023-08-28T02:25:41.334512+0000","last_fullsized":"2023-08-28T02:25:41.334512+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"179'12232","reported_seq":12302,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.411313+0000","last_change":"2023-08-28T02:22:00.162375+0000","last_active":"2023-08-28T02:25:53.411313+0000","last_peered":"2023-08-28T02:25:53.411313+0000","last_clean":"2023-08-28T02:25:53.411313+0000","last_became_active":"2023-08-28T02:22:00.162190+0000","last_became_peered":"2023-08-28T02:22:00.162190+0000","last_unstale":"2023-08-28T02:25:53.411313+0000","last_undegraded":"2023-08-28T02:25:53.411313+0000","last_fullsized":"2023-08-28T02:25:53.411313+0000","mapping_epoch":119,"log_start":"129'3839","ondisk_log_start":"129'3839","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":8393,"log_dups_size":0,"ondisk_log_size":8393,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:40:08.985631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114819072,"num_objects":1752,"num_object_clones":0,"num_object_copies":5256,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1752,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12232,"num_write_kb":447488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":173,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-08-28T02:25:41.334544+0000","last_change":"2023-08-28T02:21:59.179699+0000","last_active":"2023-08-28T02:25:41.334544+0000","last_peered":"2023-08-28T02:25:41.334544+0000","last_clean":"2023-08-28T02:25:41.334544+0000","last_became_active":"2023-08-28T02:21:59.179345+0000","last_became_peered":"2023-08-28T02:21:59.179345+0000","last_unstale":"2023-08-28T02:25:41.334544+0000","last_undegraded":"2023-08-28T02:25:41.334544+0000","last_fullsized":"2023-08-28T02:25:41.334544+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":197,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-08-28T02:25:41.334546+0000","last_change":"2023-08-28T02:22:44.083121+0000","last_active":"2023-08-28T02:25:41.334546+0000","last_peered":"2023-08-28T02:25:41.334546+0000","last_clean":"2023-08-28T02:25:41.334546+0000","last_became_active":"2023-08-28T02:22:44.082776+0000","last_became_peered":"2023-08-28T02:22:44.082776+0000","last_unstale":"2023-08-28T02:25:41.334546+0000","last_undegraded":"2023-08-28T02:25:41.334546+0000","last_fullsized":"2023-08-28T02:25:41.334546+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"178'12006","reported_seq":12073,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:52.097401+0000","last_change":"2023-08-28T02:22:00.167630+0000","last_active":"2023-08-28T02:25:52.097401+0000","last_peered":"2023-08-28T02:25:52.097401+0000","last_clean":"2023-08-28T02:25:52.097401+0000","last_became_active":"2023-08-28T02:22:00.167305+0000","last_became_peered":"2023-08-28T02:22:00.167305+0000","last_unstale":"2023-08-28T02:25:52.097401+0000","last_undegraded":"2023-08-28T02:25:52.097401+0000","last_fullsized":"2023-08-28T02:25:52.097401+0000","mapping_epoch":119,"log_start":"128'2900","ondisk_log_start":"128'2900","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":9106,"log_dups_size":0,"ondisk_log_size":9106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117440512,"num_objects":1792,"num_object_clones":0,"num_object_copies":5376,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1792,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12006,"num_write_kb":441536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":87,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-08-28T02:25:41.334269+0000","last_change":"2023-08-28T02:23:38.347043+0000","last_active":"2023-08-28T02:25:41.334269+0000","last_peered":"2023-08-28T02:25:41.334269+0000","last_clean":"2023-08-28T02:25:41.334269+0000","last_became_active":"2023-08-28T02:23:38.346680+0000","last_became_peered":"2023-08-28T02:23:38.346680+0000","last_unstale":"2023-08-28T02:25:41.334269+0000","last_undegraded":"2023-08-28T02:25:41.334269+0000","last_fullsized":"2023-08-28T02:25:41.334269+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":108,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.516414+0000","last_change":"2023-08-28T02:19:51.204017+0000","last_active":"2023-08-28T02:25:48.516414+0000","last_peered":"2023-08-28T02:25:48.516414+0000","last_clean":"2023-08-28T02:25:48.516414+0000","last_became_active":"2023-08-28T02:19:51.203627+0000","last_became_peered":"2023-08-28T02:19:51.203627+0000","last_unstale":"2023-08-28T02:25:48.516414+0000","last_undegraded":"2023-08-28T02:25:48.516414+0000","last_fullsized":"2023-08-28T02:25:48.516414+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:14:06.231701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.e","version":"178'12206","reported_seq":12273,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:52.083494+0000","last_change":"2023-08-28T02:22:00.166490+0000","last_active":"2023-08-28T02:25:52.083494+0000","last_peered":"2023-08-28T02:25:52.083494+0000","last_clean":"2023-08-28T02:25:52.083494+0000","last_became_active":"2023-08-28T02:22:00.166140+0000","last_became_peered":"2023-08-28T02:22:00.166140+0000","last_unstale":"2023-08-28T02:25:52.083494+0000","last_undegraded":"2023-08-28T02:25:52.083494+0000","last_fullsized":"2023-08-28T02:25:52.083494+0000","mapping_epoch":119,"log_start":"132'4300","ondisk_log_start":"132'4300","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":7906,"log_dups_size":0,"ondisk_log_size":7906,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115998720,"num_objects":1770,"num_object_clones":0,"num_object_copies":5310,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1770,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12206,"num_write_kb":447232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":36,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.427894+0000","last_change":"2023-08-28T02:23:59.216950+0000","last_active":"2023-08-28T02:25:48.427894+0000","last_peered":"2023-08-28T02:25:48.427894+0000","last_clean":"2023-08-28T02:25:48.427894+0000","last_became_active":"2023-08-28T02:23:59.216373+0000","last_became_peered":"2023-08-28T02:23:59.216373+0000","last_unstale":"2023-08-28T02:25:48.427894+0000","last_undegraded":"2023-08-28T02:25:48.427894+0000","last_fullsized":"2023-08-28T02:25:48.427894+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:13:14.736771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":156,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.411146+0000","last_change":"2023-08-28T02:23:59.220269+0000","last_active":"2023-08-28T02:25:48.411146+0000","last_peered":"2023-08-28T02:25:48.411146+0000","last_clean":"2023-08-28T02:25:48.411146+0000","last_became_active":"2023-08-28T02:23:59.219966+0000","last_became_peered":"2023-08-28T02:23:59.219966+0000","last_unstale":"2023-08-28T02:25:48.411146+0000","last_undegraded":"2023-08-28T02:25:48.411146+0000","last_fullsized":"2023-08-28T02:25:48.411146+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:16:08.119909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"78'126","reported_seq":655,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.408909+0000","last_change":"2023-08-28T02:23:39.351975+0000","last_active":"2023-08-28T02:25:48.408909+0000","last_peered":"2023-08-28T02:25:48.408909+0000","last_clean":"2023-08-28T02:25:48.408909+0000","last_became_active":"2023-08-28T02:23:39.351437+0000","last_became_peered":"2023-08-28T02:23:39.351437+0000","last_unstale":"2023-08-28T02:25:48.408909+0000","last_undegraded":"2023-08-28T02:25:48.408909+0000","last_fullsized":"2023-08-28T02:25:48.408909+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:25:22.203120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":743,"num_read_kb":1331,"num_write":125,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":178,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-08-28T02:25:41.330977+0000","last_change":"2023-08-28T02:23:59.207148+0000","last_active":"2023-08-28T02:25:41.330977+0000","last_peered":"2023-08-28T02:25:41.330977+0000","last_clean":"2023-08-28T02:25:41.330977+0000","last_became_active":"2023-08-28T02:23:59.206802+0000","last_became_peered":"2023-08-28T02:23:59.206802+0000","last_unstale":"2023-08-28T02:25:41.330977+0000","last_undegraded":"2023-08-28T02:25:41.330977+0000","last_fullsized":"2023-08-28T02:25:41.330977+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":76,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.241774+0000","last_change":"2023-08-28T02:21:43.238044+0000","last_active":"2023-08-28T02:25:53.241774+0000","last_peered":"2023-08-28T02:25:53.241774+0000","last_clean":"2023-08-28T02:25:53.241774+0000","last_became_active":"2023-08-28T02:21:43.237792+0000","last_became_peered":"2023-08-28T02:21:43.237792+0000","last_unstale":"2023-08-28T02:25:53.241774+0000","last_undegraded":"2023-08-28T02:25:53.241774+0000","last_fullsized":"2023-08-28T02:25:53.241774+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:11:35.488503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"179'11955","reported_seq":12023,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.471400+0000","last_change":"2023-08-28T02:22:00.166578+0000","last_active":"2023-08-28T02:25:53.471400+0000","last_peered":"2023-08-28T02:25:53.471400+0000","last_clean":"2023-08-28T02:25:53.471400+0000","last_became_active":"2023-08-28T02:22:00.166311+0000","last_became_peered":"2023-08-28T02:22:00.166311+0000","last_unstale":"2023-08-28T02:25:53.471400+0000","last_undegraded":"2023-08-28T02:25:53.471400+0000","last_fullsized":"2023-08-28T02:25:53.471400+0000","mapping_epoch":119,"log_start":"132'4241","ondisk_log_start":"132'4241","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":7714,"log_dups_size":0,"ondisk_log_size":7714,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:40:38.502180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111083520,"num_objects":1695,"num_object_clones":0,"num_object_copies":5085,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1695,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11955,"num_write_kb":436800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"179'12043","reported_seq":12113,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.545536+0000","last_change":"2023-08-28T02:22:00.160991+0000","last_active":"2023-08-28T02:25:53.545536+0000","last_peered":"2023-08-28T02:25:53.545536+0000","last_clean":"2023-08-28T02:25:53.545536+0000","last_became_active":"2023-08-28T02:22:00.160755+0000","last_became_peered":"2023-08-28T02:22:00.160755+0000","last_unstale":"2023-08-28T02:25:53.545536+0000","last_undegraded":"2023-08-28T02:25:53.545536+0000","last_fullsized":"2023-08-28T02:25:53.545536+0000","mapping_epoch":119,"log_start":"132'4298","ondisk_log_start":"132'4298","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":7745,"log_dups_size":0,"ondisk_log_size":7745,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:42:59.109288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111607808,"num_objects":1703,"num_object_clones":0,"num_object_copies":5109,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1703,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12043,"num_write_kb":439872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"78'204","reported_seq":1094,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-08-28T02:25:41.332793+0000","last_change":"2023-08-28T02:19:50.219949+0000","last_active":"2023-08-28T02:25:41.332793+0000","last_peered":"2023-08-28T02:25:41.332793+0000","last_clean":"2023-08-28T02:25:41.332793+0000","last_became_active":"2023-08-28T02:19:50.219356+0000","last_became_peered":"2023-08-28T02:19:50.219356+0000","last_unstale":"2023-08-28T02:25:41.332793+0000","last_undegraded":"2023-08-28T02:25:41.332793+0000","last_fullsized":"2023-08-28T02:25:41.332793+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":204,"log_dups_size":0,"ondisk_log_size":204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1521,"num_read_kb":2511,"num_write":178,"num_write_kb":1175,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":145,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-08-28T02:25:41.333924+0000","last_change":"2023-08-28T02:18:39.779607+0000","last_active":"2023-08-28T02:25:41.333924+0000","last_peered":"2023-08-28T02:25:41.333924+0000","last_clean":"2023-08-28T02:25:41.333924+0000","last_became_active":"2023-08-28T02:18:39.779303+0000","last_became_peered":"2023-08-28T02:18:39.779303+0000","last_unstale":"2023-08-28T02:25:41.333924+0000","last_undegraded":"2023-08-28T02:25:41.333924+0000","last_fullsized":"2023-08-28T02:25:41.333924+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":136,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.408837+0000","last_change":"2023-08-28T02:23:59.221979+0000","last_active":"2023-08-28T02:25:48.408837+0000","last_peered":"2023-08-28T02:25:48.408837+0000","last_clean":"2023-08-28T02:25:48.408837+0000","last_became_active":"2023-08-28T02:23:59.221619+0000","last_became_peered":"2023-08-28T02:23:59.221619+0000","last_unstale":"2023-08-28T02:25:48.408837+0000","last_undegraded":"2023-08-28T02:25:48.408837+0000","last_fullsized":"2023-08-28T02:25:48.408837+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:54:45.892284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.6","version":"179'11883","reported_seq":12666,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.810609+0000","last_change":"2023-08-28T02:24:07.218778+0000","last_active":"2023-08-28T02:25:53.810609+0000","last_peered":"2023-08-28T02:25:53.810609+0000","last_clean":"2023-08-28T02:25:53.810609+0000","last_became_active":"2023-08-28T02:24:07.205246+0000","last_became_peered":"2023-08-28T02:24:07.205246+0000","last_unstale":"2023-08-28T02:25:53.810609+0000","last_undegraded":"2023-08-28T02:25:53.810609+0000","last_fullsized":"2023-08-28T02:25:53.810609+0000","mapping_epoch":159,"log_start":"128'2700","ondisk_log_start":"128'2700","created":119,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":9183,"log_dups_size":0,"ondisk_log_size":9183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:30:08.510532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107020288,"num_objects":1633,"num_object_clones":0,"num_object_copies":4899,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1633,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11883,"num_write_kb":432512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":342,"num_bytes_recovered":22413312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"78'47","reported_seq":393,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.429189+0000","last_change":"2023-08-28T02:24:05.799647+0000","last_active":"2023-08-28T02:25:48.429189+0000","last_peered":"2023-08-28T02:25:48.429189+0000","last_clean":"2023-08-28T02:25:48.429189+0000","last_became_active":"2023-08-28T02:23:59.210481+0000","last_became_peered":"2023-08-28T02:23:59.210481+0000","last_unstale":"2023-08-28T02:25:48.429189+0000","last_undegraded":"2023-08-28T02:25:48.429189+0000","last_fullsized":"2023-08-28T02:25:48.429189+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":47,"log_dups_size":0,"ondisk_log_size":47,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:40:18.542375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":162,"num_read_kb":1281,"num_write":48,"num_write_kb":1153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":120,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.435421+0000","last_change":"2023-08-28T02:20:19.851591+0000","last_active":"2023-08-28T02:25:48.435421+0000","last_peered":"2023-08-28T02:25:48.435421+0000","last_clean":"2023-08-28T02:25:48.435421+0000","last_became_active":"2023-08-28T02:20:19.851291+0000","last_became_peered":"2023-08-28T02:20:19.851291+0000","last_unstale":"2023-08-28T02:25:48.435421+0000","last_undegraded":"2023-08-28T02:25:48.435421+0000","last_fullsized":"2023-08-28T02:25:48.435421+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:33:51.259707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":151,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.411064+0000","last_change":"2023-08-28T02:19:50.211156+0000","last_active":"2023-08-28T02:25:48.411064+0000","last_peered":"2023-08-28T02:25:48.411064+0000","last_clean":"2023-08-28T02:25:48.411064+0000","last_became_active":"2023-08-28T02:19:50.210398+0000","last_became_peered":"2023-08-28T02:19:50.210398+0000","last_unstale":"2023-08-28T02:25:48.411064+0000","last_undegraded":"2023-08-28T02:25:48.411064+0000","last_fullsized":"2023-08-28T02:25:48.411064+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:14:41.748197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"78'114","reported_seq":542,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-08-28T02:25:41.330753+0000","last_change":"2023-08-28T02:24:01.210739+0000","last_active":"2023-08-28T02:25:41.330753+0000","last_peered":"2023-08-28T02:25:41.330753+0000","last_clean":"2023-08-28T02:25:41.330753+0000","last_became_active":"2023-08-28T02:24:01.210386+0000","last_became_peered":"2023-08-28T02:24:01.210386+0000","last_unstale":"2023-08-28T02:25:41.330753+0000","last_undegraded":"2023-08-28T02:25:41.330753+0000","last_fullsized":"2023-08-28T02:25:41.330753+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":114,"log_dups_size":0,"ondisk_log_size":114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":358,"num_read_kb":1352,"num_write":102,"num_write_kb":1104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"86'139","reported_seq":336,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-08-28T02:25:41.337069+0000","last_change":"2023-08-28T02:24:04.276519+0000","last_active":"2023-08-28T02:25:41.337069+0000","last_peered":"2023-08-28T02:25:41.337069+0000","last_clean":"2023-08-28T02:25:41.337069+0000","last_became_active":"2023-08-28T02:24:04.276104+0000","last_became_peered":"2023-08-28T02:24:04.276104+0000","last_unstale":"2023-08-28T02:25:41.337069+0000","last_undegraded":"2023-08-28T02:25:41.337069+0000","last_fullsized":"2023-08-28T02:25:41.337069+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_clean_scrub_stamp":"2023-08-28T02:08:02.043718+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":36,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.428840+0000","last_change":"2023-08-28T02:23:59.219981+0000","last_active":"2023-08-28T02:25:48.428840+0000","last_peered":"2023-08-28T02:25:48.428840+0000","last_clean":"2023-08-28T02:25:48.428840+0000","last_became_active":"2023-08-28T02:23:59.219633+0000","last_became_peered":"2023-08-28T02:23:59.219633+0000","last_unstale":"2023-08-28T02:25:48.428840+0000","last_undegraded":"2023-08-28T02:25:48.428840+0000","last_fullsized":"2023-08-28T02:25:48.428840+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:56:14.172518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":146,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-08-28T02:25:41.335064+0000","last_change":"2023-08-28T02:23:38.341351+0000","last_active":"2023-08-28T02:25:41.335064+0000","last_peered":"2023-08-28T02:25:41.335064+0000","last_clean":"2023-08-28T02:25:41.335064+0000","last_became_active":"2023-08-28T02:23:38.340997+0000","last_became_peered":"2023-08-28T02:23:38.340997+0000","last_unstale":"2023-08-28T02:25:41.335064+0000","last_undegraded":"2023-08-28T02:25:41.335064+0000","last_fullsized":"2023-08-28T02:25:41.335064+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"178'11952","reported_seq":12787,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:52.027964+0000","last_change":"2023-08-28T02:23:47.215362+0000","last_active":"2023-08-28T02:25:52.027964+0000","last_peered":"2023-08-28T02:25:52.027964+0000","last_clean":"2023-08-28T02:25:52.027964+0000","last_became_active":"2023-08-28T02:23:47.207109+0000","last_became_peered":"2023-08-28T02:23:47.207109+0000","last_unstale":"2023-08-28T02:25:52.027964+0000","last_undegraded":"2023-08-28T02:25:52.027964+0000","last_fullsized":"2023-08-28T02:25:52.027964+0000","mapping_epoch":150,"log_start":"128'3300","ondisk_log_start":"128'3300","created":119,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":8652,"log_dups_size":0,"ondisk_log_size":8652,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111673344,"num_objects":1704,"num_object_clones":0,"num_object_copies":5112,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1704,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11952,"num_write_kb":436992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":370,"num_bytes_recovered":24182784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.7","version":"178'12097","reported_seq":12164,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:52.138275+0000","last_change":"2023-08-28T02:22:00.165614+0000","last_active":"2023-08-28T02:25:52.138275+0000","last_peered":"2023-08-28T02:25:52.138275+0000","last_clean":"2023-08-28T02:25:52.138275+0000","last_became_active":"2023-08-28T02:22:00.165136+0000","last_became_peered":"2023-08-28T02:22:00.165136+0000","last_unstale":"2023-08-28T02:25:52.138275+0000","last_undegraded":"2023-08-28T02:25:52.138275+0000","last_fullsized":"2023-08-28T02:25:52.138275+0000","mapping_epoch":119,"log_start":"128'3000","ondisk_log_start":"128'3000","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":9097,"log_dups_size":0,"ondisk_log_size":9097,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115146752,"num_objects":1757,"num_object_clones":0,"num_object_copies":5271,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1757,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12097,"num_write_kb":443328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"78'148","reported_seq":627,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-08-28T02:25:41.337018+0000","last_change":"2023-08-28T02:24:01.211494+0000","last_active":"2023-08-28T02:25:41.337018+0000","last_peered":"2023-08-28T02:25:41.337018+0000","last_clean":"2023-08-28T02:25:41.337018+0000","last_became_active":"2023-08-28T02:24:01.210957+0000","last_became_peered":"2023-08-28T02:24:01.210957+0000","last_unstale":"2023-08-28T02:25:41.337018+0000","last_undegraded":"2023-08-28T02:25:41.337018+0000","last_fullsized":"2023-08-28T02:25:41.337018+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1092,"num_read_kb":1955,"num_write":127,"num_write_kb":5177,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":146,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.414463+0000","last_change":"2023-08-28T02:23:59.226986+0000","last_active":"2023-08-28T02:25:48.414463+0000","last_peered":"2023-08-28T02:25:48.414463+0000","last_clean":"2023-08-28T02:25:48.414463+0000","last_became_active":"2023-08-28T02:23:59.226529+0000","last_became_peered":"2023-08-28T02:23:59.226529+0000","last_unstale":"2023-08-28T02:25:48.414463+0000","last_undegraded":"2023-08-28T02:25:48.414463+0000","last_fullsized":"2023-08-28T02:25:48.414463+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:22:09.193619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":173,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.508256+0000","last_change":"2023-08-28T02:21:59.167858+0000","last_active":"2023-08-28T02:25:48.508256+0000","last_peered":"2023-08-28T02:25:48.508256+0000","last_clean":"2023-08-28T02:25:48.508256+0000","last_became_active":"2023-08-28T02:21:59.167436+0000","last_became_peered":"2023-08-28T02:21:59.167436+0000","last_unstale":"2023-08-28T02:25:48.508256+0000","last_undegraded":"2023-08-28T02:25:48.508256+0000","last_fullsized":"2023-08-28T02:25:48.508256+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:17:50.717282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"78'103","reported_seq":554,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-08-28T02:25:41.337660+0000","last_change":"2023-08-28T02:24:07.212128+0000","last_active":"2023-08-28T02:25:41.337660+0000","last_peered":"2023-08-28T02:25:41.337660+0000","last_clean":"2023-08-28T02:25:41.337660+0000","last_became_active":"2023-08-28T02:24:07.211562+0000","last_became_peered":"2023-08-28T02:24:07.211562+0000","last_unstale":"2023-08-28T02:25:41.337660+0000","last_undegraded":"2023-08-28T02:25:41.337660+0000","last_fullsized":"2023-08-28T02:25:41.337660+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":803,"num_read_kb":2080,"num_write":73,"num_write_kb":1244,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":119,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.409054+0000","last_change":"2023-08-28T02:20:19.850916+0000","last_active":"2023-08-28T02:25:48.409054+0000","last_peered":"2023-08-28T02:25:48.409054+0000","last_clean":"2023-08-28T02:25:48.409054+0000","last_became_active":"2023-08-28T02:20:19.850627+0000","last_became_peered":"2023-08-28T02:20:19.850627+0000","last_unstale":"2023-08-28T02:25:48.409054+0000","last_undegraded":"2023-08-28T02:25:48.409054+0000","last_fullsized":"2023-08-28T02:25:48.409054+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:19:14.346189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":132,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.414947+0000","last_change":"2023-08-28T02:23:38.329667+0000","last_active":"2023-08-28T02:25:48.414947+0000","last_peered":"2023-08-28T02:25:48.414947+0000","last_clean":"2023-08-28T02:25:48.414947+0000","last_became_active":"2023-08-28T02:23:38.329175+0000","last_became_peered":"2023-08-28T02:23:38.329175+0000","last_unstale":"2023-08-28T02:25:48.414947+0000","last_undegraded":"2023-08-28T02:25:48.414947+0000","last_fullsized":"2023-08-28T02:25:48.414947+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:21:34.815067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"178'11810","reported_seq":11876,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:51.990925+0000","last_change":"2023-08-28T02:22:00.167814+0000","last_active":"2023-08-28T02:25:51.990925+0000","last_peered":"2023-08-28T02:25:51.990925+0000","last_clean":"2023-08-28T02:25:51.990925+0000","last_became_active":"2023-08-28T02:22:00.167513+0000","last_became_peered":"2023-08-28T02:22:00.167513+0000","last_unstale":"2023-08-28T02:25:51.990925+0000","last_undegraded":"2023-08-28T02:25:51.990925+0000","last_fullsized":"2023-08-28T02:25:51.990925+0000","mapping_epoch":119,"log_start":"126'2100","ondisk_log_start":"126'2100","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":9710,"log_dups_size":0,"ondisk_log_size":9710,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115081216,"num_objects":1756,"num_object_clones":0,"num_object_copies":5268,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1756,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11810,"num_write_kb":434112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":40,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.427805+0000","last_change":"2023-08-28T02:23:59.232099+0000","last_active":"2023-08-28T02:25:48.427805+0000","last_peered":"2023-08-28T02:25:48.427805+0000","last_clean":"2023-08-28T02:25:48.427805+0000","last_became_active":"2023-08-28T02:23:59.219318+0000","last_became_peered":"2023-08-28T02:23:59.219318+0000","last_unstale":"2023-08-28T02:25:48.427805+0000","last_undegraded":"2023-08-28T02:25:48.427805+0000","last_fullsized":"2023-08-28T02:25:48.427805+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:59:30.093289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'112","reported_seq":423,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.409169+0000","last_change":"2023-08-28T02:19:50.218206+0000","last_active":"2023-08-28T02:25:48.409169+0000","last_peered":"2023-08-28T02:25:48.409169+0000","last_clean":"2023-08-28T02:25:48.409169+0000","last_became_active":"2023-08-28T02:19:50.217637+0000","last_became_peered":"2023-08-28T02:19:50.217637+0000","last_unstale":"2023-08-28T02:25:48.409169+0000","last_undegraded":"2023-08-28T02:25:48.409169+0000","last_fullsized":"2023-08-28T02:25:48.409169+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:12:06.224907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":444,"num_read_kb":473,"num_write":94,"num_write_kb":127,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":36,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.427963+0000","last_change":"2023-08-28T02:23:59.221034+0000","last_active":"2023-08-28T02:25:48.427963+0000","last_peered":"2023-08-28T02:25:48.427963+0000","last_clean":"2023-08-28T02:25:48.427963+0000","last_became_active":"2023-08-28T02:23:59.220730+0000","last_became_peered":"2023-08-28T02:23:59.220730+0000","last_unstale":"2023-08-28T02:25:48.427963+0000","last_undegraded":"2023-08-28T02:25:48.427963+0000","last_fullsized":"2023-08-28T02:25:48.427963+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:25:52.364370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"178'12122","reported_seq":15284,"reported_epoch":178,"state":"active+remapped+backfill_wait","last_fresh":"2023-08-28T02:25:52.086305+0000","last_change":"2023-08-28T02:24:05.805308+0000","last_active":"2023-08-28T02:25:52.086305+0000","last_peered":"2023-08-28T02:25:52.086305+0000","last_clean":"2023-08-28T02:23:25.043301+0000","last_became_active":"2023-08-28T02:24:00.316639+0000","last_became_peered":"2023-08-28T02:24:00.316639+0000","last_unstale":"2023-08-28T02:25:52.086305+0000","last_undegraded":"2023-08-28T02:25:52.086305+0000","last_fullsized":"2023-08-28T02:25:52.086305+0000","mapping_epoch":154,"log_start":"131'4200","ondisk_log_start":"131'4200","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":7922,"log_dups_size":0,"ondisk_log_size":7922,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115867648,"num_objects":1768,"num_object_clones":0,"num_object_copies":5304,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1509,"num_objects_unfound":0,"num_objects_dirty":1768,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12122,"num_write_kb":444480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1527,"num_bytes_recovered":99942400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,7,0],"avail_no_missing":["4","0","5","7"],"object_location_counts":[{"shards":"0,4,7","objects":1768}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"78'122","reported_seq":707,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-08-28T02:25:41.336658+0000","last_change":"2023-08-28T02:24:04.276850+0000","last_active":"2023-08-28T02:25:41.336658+0000","last_peered":"2023-08-28T02:25:41.336658+0000","last_clean":"2023-08-28T02:25:41.336658+0000","last_became_active":"2023-08-28T02:24:04.276367+0000","last_became_peered":"2023-08-28T02:24:04.276367+0000","last_unstale":"2023-08-28T02:25:41.336658+0000","last_undegraded":"2023-08-28T02:25:41.336658+0000","last_fullsized":"2023-08-28T02:25:41.336658+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":857,"num_read_kb":2224,"num_write":99,"num_write_kb":2312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":132,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.508482+0000","last_change":"2023-08-28T02:19:50.221001+0000","last_active":"2023-08-28T02:25:48.508482+0000","last_peered":"2023-08-28T02:25:48.508482+0000","last_clean":"2023-08-28T02:25:48.508482+0000","last_became_active":"2023-08-28T02:19:50.220611+0000","last_became_peered":"2023-08-28T02:19:50.220611+0000","last_unstale":"2023-08-28T02:25:48.508482+0000","last_undegraded":"2023-08-28T02:25:48.508482+0000","last_fullsized":"2023-08-28T02:25:48.508482+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:23:10.637630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":129,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-08-28T02:25:41.333946+0000","last_change":"2023-08-28T02:11:22.437146+0000","last_active":"2023-08-28T02:25:41.333946+0000","last_peered":"2023-08-28T02:25:41.333946+0000","last_clean":"2023-08-28T02:25:41.333946+0000","last_became_active":"2023-08-28T02:11:22.436798+0000","last_became_peered":"2023-08-28T02:11:22.436798+0000","last_unstale":"2023-08-28T02:25:41.333946+0000","last_undegraded":"2023-08-28T02:25:41.333946+0000","last_fullsized":"2023-08-28T02:25:41.333946+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"179'12129","reported_seq":12205,"reported_epoch":179,"state":"active+remapped+backfill_wait","last_fresh":"2023-08-28T02:25:53.519240+0000","last_change":"2023-08-28T02:24:00.310494+0000","last_active":"2023-08-28T02:25:53.519240+0000","last_peered":"2023-08-28T02:25:53.519240+0000","last_clean":"2023-08-28T02:23:58.174026+0000","last_became_active":"2023-08-28T02:24:00.290543+0000","last_became_peered":"2023-08-28T02:24:00.290543+0000","last_unstale":"2023-08-28T02:25:53.519240+0000","last_undegraded":"2023-08-28T02:25:53.519240+0000","last_fullsized":"2023-08-28T02:25:53.519240+0000","mapping_epoch":154,"log_start":"132'4377","ondisk_log_start":"132'4377","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":7752,"log_dups_size":0,"ondisk_log_size":7752,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:35:56.609960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106364928,"num_objects":1623,"num_object_clones":0,"num_object_copies":4869,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1623,"num_objects_unfound":0,"num_objects_dirty":1623,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12129,"num_write_kb":440064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[0,1,4],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":1623}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":134,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.414877+0000","last_change":"2023-08-28T02:23:38.329474+0000","last_active":"2023-08-28T02:25:48.414877+0000","last_peered":"2023-08-28T02:25:48.414877+0000","last_clean":"2023-08-28T02:25:48.414877+0000","last_became_active":"2023-08-28T02:23:38.328505+0000","last_became_peered":"2023-08-28T02:23:38.328505+0000","last_unstale":"2023-08-28T02:25:48.414877+0000","last_undegraded":"2023-08-28T02:25:48.414877+0000","last_fullsized":"2023-08-28T02:25:48.414877+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:12:14.835809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":200,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.430448+0000","last_change":"2023-08-28T02:23:59.212036+0000","last_active":"2023-08-28T02:25:48.430448+0000","last_peered":"2023-08-28T02:25:48.430448+0000","last_clean":"2023-08-28T02:25:48.430448+0000","last_became_active":"2023-08-28T02:23:59.211709+0000","last_became_peered":"2023-08-28T02:23:59.211709+0000","last_unstale":"2023-08-28T02:25:48.430448+0000","last_undegraded":"2023-08-28T02:25:48.430448+0000","last_fullsized":"2023-08-28T02:25:48.430448+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T14:04:25.046626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":147,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-08-28T02:25:41.335114+0000","last_change":"2023-08-28T02:19:50.218760+0000","last_active":"2023-08-28T02:25:41.335114+0000","last_peered":"2023-08-28T02:25:41.335114+0000","last_clean":"2023-08-28T02:25:41.335114+0000","last_became_active":"2023-08-28T02:19:50.218376+0000","last_became_peered":"2023-08-28T02:19:50.218376+0000","last_unstale":"2023-08-28T02:25:41.335114+0000","last_undegraded":"2023-08-28T02:25:41.335114+0000","last_fullsized":"2023-08-28T02:25:41.335114+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":177,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-08-28T02:25:41.334065+0000","last_change":"2023-08-28T02:23:59.207822+0000","last_active":"2023-08-28T02:25:41.334065+0000","last_peered":"2023-08-28T02:25:41.334065+0000","last_clean":"2023-08-28T02:25:41.334065+0000","last_became_active":"2023-08-28T02:23:59.207475+0000","last_became_peered":"2023-08-28T02:23:59.207475+0000","last_unstale":"2023-08-28T02:25:41.334065+0000","last_undegraded":"2023-08-28T02:25:41.334065+0000","last_fullsized":"2023-08-28T02:25:41.334065+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":132,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-08-28T02:25:41.331112+0000","last_change":"2023-08-28T02:11:22.186024+0000","last_active":"2023-08-28T02:25:41.331112+0000","last_peered":"2023-08-28T02:25:41.331112+0000","last_clean":"2023-08-28T02:25:41.331112+0000","last_became_active":"2023-08-28T02:11:22.185500+0000","last_became_peered":"2023-08-28T02:11:22.185500+0000","last_unstale":"2023-08-28T02:25:41.331112+0000","last_undegraded":"2023-08-28T02:25:41.331112+0000","last_fullsized":"2023-08-28T02:25:41.331112+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":177,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-08-28T02:25:41.333978+0000","last_change":"2023-08-28T02:23:59.207929+0000","last_active":"2023-08-28T02:25:41.333978+0000","last_peered":"2023-08-28T02:25:41.333978+0000","last_clean":"2023-08-28T02:25:41.333978+0000","last_became_active":"2023-08-28T02:23:59.207589+0000","last_became_peered":"2023-08-28T02:23:59.207589+0000","last_unstale":"2023-08-28T02:25:41.333978+0000","last_undegraded":"2023-08-28T02:25:41.333978+0000","last_fullsized":"2023-08-28T02:25:41.333978+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":36,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.427767+0000","last_change":"2023-08-28T02:23:59.220437+0000","last_active":"2023-08-28T02:25:48.427767+0000","last_peered":"2023-08-28T02:25:48.427767+0000","last_clean":"2023-08-28T02:25:48.427767+0000","last_became_active":"2023-08-28T02:23:59.220091+0000","last_became_peered":"2023-08-28T02:23:59.220091+0000","last_unstale":"2023-08-28T02:25:48.427767+0000","last_undegraded":"2023-08-28T02:25:48.427767+0000","last_fullsized":"2023-08-28T02:25:48.427767+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:51:21.695566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":36,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.428919+0000","last_change":"2023-08-28T02:23:59.217794+0000","last_active":"2023-08-28T02:25:48.428919+0000","last_peered":"2023-08-28T02:25:48.428919+0000","last_clean":"2023-08-28T02:25:48.428919+0000","last_became_active":"2023-08-28T02:23:59.217231+0000","last_became_peered":"2023-08-28T02:23:59.217231+0000","last_unstale":"2023-08-28T02:25:48.428919+0000","last_undegraded":"2023-08-28T02:25:48.428919+0000","last_fullsized":"2023-08-28T02:25:48.428919+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:36:26.297410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":132,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.513284+0000","last_change":"2023-08-28T02:20:19.850252+0000","last_active":"2023-08-28T02:25:48.513284+0000","last_peered":"2023-08-28T02:25:48.513284+0000","last_clean":"2023-08-28T02:25:48.513284+0000","last_became_active":"2023-08-28T02:20:19.849913+0000","last_became_peered":"2023-08-28T02:20:19.849913+0000","last_unstale":"2023-08-28T02:25:48.513284+0000","last_undegraded":"2023-08-28T02:25:48.513284+0000","last_fullsized":"2023-08-28T02:25:48.513284+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:48:47.713465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":173,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-08-28T02:25:41.334848+0000","last_change":"2023-08-28T02:21:59.180201+0000","last_active":"2023-08-28T02:25:41.334848+0000","last_peered":"2023-08-28T02:25:41.334848+0000","last_clean":"2023-08-28T02:25:41.334848+0000","last_became_active":"2023-08-28T02:21:59.179863+0000","last_became_peered":"2023-08-28T02:21:59.179863+0000","last_unstale":"2023-08-28T02:25:41.334848+0000","last_undegraded":"2023-08-28T02:25:41.334848+0000","last_fullsized":"2023-08-28T02:25:41.334848+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":1803026456,"num_objects":27513,"num_object_clones":0,"num_object_copies":82539,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1971,"num_objects_misplaced":4777,"num_objects_unfound":0,"num_objects_dirty":27513,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":192769,"num_write_kb":7048961,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5469,"num_bytes_recovered":357892120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5784350720,"data_stored":5784338504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":138314,"ondisk_log_size":138314,"up":48,"acting":47,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":9774,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":21246,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26810,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430403,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":879296,"kb_used_data":577976,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92865856,"statfs":{"total":95995035648,"available":95094636544,"internally_reserved":0,"allocated":591847424,"data_stored":591264733,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:24:56 2023","interfaces":[{"interface":"back","average":{"1min":7.7380000000000004,"5min":3.3220000000000001,"15min":1.4650000000000001},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":53.595999999999997,"5min":116.755,"15min":116.755},"last":6.2149999999999999},{"interface":"front","average":{"1min":7.6120000000000001,"5min":3.6389999999999998,"15min":1.621},"min":{"1min":0.254,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":53.481000000000002,"5min":116.971,"15min":116.971},"last":1.4910000000000001}]},{"osd":1,"last update":"Mon Aug 28 02:25:02 2023","interfaces":[{"interface":"back","average":{"1min":1.395,"5min":0.93200000000000005,"15min":0.66200000000000003},"min":{"1min":0.28599999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":9.8710000000000004,"5min":9.8710000000000004,"15min":9.8710000000000004},"last":1.4450000000000001},{"interface":"front","average":{"1min":1.9670000000000001,"5min":1.3029999999999999,"15min":0.82599999999999996},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":11.696999999999999,"5min":11.696999999999999,"15min":11.696999999999999},"last":2.4089999999999998}]},{"osd":2,"last update":"Mon Aug 28 02:25:27 2023","interfaces":[{"interface":"back","average":{"1min":8.6959999999999997,"5min":3.8700000000000001,"15min":1.673},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":119.571,"5min":119.571,"15min":119.571},"last":6.2409999999999997},{"interface":"front","average":{"1min":9.7100000000000009,"5min":4.319,"15min":1.8300000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":119.554,"5min":119.554,"15min":119.554},"last":2.3769999999999998}]},{"osd":3,"last update":"Mon Aug 28 02:25:02 2023","interfaces":[{"interface":"back","average":{"1min":6.6299999999999999,"5min":2.0139999999999998,"15min":1.117},"min":{"1min":0.32500000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":107.916,"5min":107.916,"15min":107.916},"last":6.1369999999999996},{"interface":"front","average":{"1min":8.3409999999999993,"5min":2.6240000000000001,"15min":1.321},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":112.00700000000001,"5min":112.00700000000001,"15min":112.00700000000001},"last":2.4870000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:25:27 2023","interfaces":[{"interface":"back","average":{"1min":1.0980000000000001,"5min":2.0350000000000001,"15min":0.996},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":3.8359999999999999,"5min":63.048000000000002,"15min":63.048000000000002},"last":1.8340000000000001},{"interface":"front","average":{"1min":1.659,"5min":1.8759999999999999,"15min":0.96199999999999997},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":8.2720000000000002,"5min":62.912999999999997,"15min":62.912999999999997},"last":2.1179999999999999}]},{"osd":5,"last update":"Mon Aug 28 02:25:27 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":1.554,"15min":0.83999999999999997},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":5.7110000000000003,"5min":63.021000000000001,"15min":63.021000000000001},"last":1.4159999999999999},{"interface":"front","average":{"1min":1.4039999999999999,"5min":2.1779999999999999,"15min":1.089},"min":{"1min":0.35699999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":5.3470000000000004,"5min":63.082999999999998,"15min":63.082999999999998},"last":2.8359999999999999}]},{"osd":6,"last update":"Mon Aug 28 02:25:27 2023","interfaces":[{"interface":"back","average":{"1min":1.716,"5min":2.198,"15min":1.0860000000000001},"min":{"1min":0.317,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":6.0659999999999998,"5min":101.61,"15min":101.61},"last":6.1840000000000002},{"interface":"front","average":{"1min":1.3080000000000001,"5min":2.391,"15min":1.1890000000000001},"min":{"1min":0.34399999999999997,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":5.407,"5min":101.621,"15min":101.621},"last":1.675}]}]},{"osd":6,"up_from":42,"seq":180388626630,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":764848,"kb_used_data":455336,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92980304,"statfs":{"total":95995035648,"available":95211831296,"internally_reserved":0,"allocated":466264064,"data_stored":465697291,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:25:00 2023","interfaces":[{"interface":"back","average":{"1min":12.085000000000001,"5min":3.8039999999999998,"15min":1.651},"min":{"1min":0.23899999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":121.247,"5min":121.247,"15min":121.247},"last":6.3869999999999996},{"interface":"front","average":{"1min":12.305999999999999,"5min":3.8540000000000001,"15min":1.6859999999999999},"min":{"1min":0.28799999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":118.977,"5min":118.977,"15min":118.977},"last":3.419}]},{"osd":1,"last update":"Mon Aug 28 02:25:00 2023","interfaces":[{"interface":"back","average":{"1min":2.6080000000000001,"5min":1.9470000000000001,"15min":1.0069999999999999},"min":{"1min":0.26900000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":8.2420000000000009,"5min":34.677999999999997,"15min":34.677999999999997},"last":6.3639999999999999},{"interface":"front","average":{"1min":2.4660000000000002,"5min":2.0579999999999998,"15min":1.0660000000000001},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":14.33,"5min":34.348999999999997,"15min":34.348999999999997},"last":3.4649999999999999}]},{"osd":2,"last update":"Mon Aug 28 02:25:35 2023","interfaces":[{"interface":"back","average":{"1min":3.0590000000000002,"5min":2.2959999999999998,"15min":1.1719999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":15.762,"5min":49.555999999999997,"15min":49.555999999999997},"last":7.633},{"interface":"front","average":{"1min":2.9710000000000001,"5min":2.1160000000000001,"15min":1.089},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":11.503,"5min":49.470999999999997,"15min":49.470999999999997},"last":4.5060000000000002}]},{"osd":3,"last update":"Mon Aug 28 02:25:00 2023","interfaces":[{"interface":"back","average":{"1min":3.3010000000000002,"5min":1.5609999999999999,"15min":0.96699999999999997},"min":{"1min":0.35199999999999998,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":13.103,"5min":13.103,"15min":13.103},"last":6.3380000000000001},{"interface":"front","average":{"1min":3.093,"5min":1.7,"15min":1.022},"min":{"1min":0.42699999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":13.09,"5min":13.09,"15min":13.09},"last":3.3559999999999999}]},{"osd":4,"last update":"Mon Aug 28 02:25:04 2023","interfaces":[{"interface":"back","average":{"1min":2.2719999999999998,"5min":1.4570000000000001,"15min":0.78800000000000003},"min":{"1min":0.28299999999999997,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":5.9619999999999997,"5min":10.637,"15min":10.637},"last":0.34599999999999997},{"interface":"front","average":{"1min":1.629,"5min":1.514,"15min":0.83199999999999996},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":5.1399999999999997,"5min":10.055,"15min":10.055},"last":3.4380000000000002}]},{"osd":5,"last update":"Mon Aug 28 02:25:04 2023","interfaces":[{"interface":"back","average":{"1min":1.1379999999999999,"5min":1.655,"15min":0.86099999999999999},"min":{"1min":0.33200000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.4429999999999996,"5min":20.759,"15min":20.759},"last":0.49399999999999999},{"interface":"front","average":{"1min":1.909,"5min":1.246,"15min":0.76100000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.28799999999999998},"max":{"1min":6.2110000000000003,"5min":7.3780000000000001,"15min":7.3780000000000001},"last":0.45200000000000001}]},{"osd":7,"last update":"Mon Aug 28 02:25:33 2023","interfaces":[{"interface":"back","average":{"1min":1.7949999999999999,"5min":1.3320000000000001,"15min":0.80700000000000005},"min":{"1min":0.34799999999999998,"5min":0.314,"15min":0.314},"max":{"1min":6.4109999999999996,"5min":7.1769999999999996,"15min":7.1769999999999996},"last":6.3019999999999996},{"interface":"front","average":{"1min":1.9379999999999999,"5min":1.49,"15min":0.879},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.29799999999999999},"max":{"1min":6.8949999999999996,"5min":7.069,"15min":7.069},"last":3.3980000000000001}]}]},{"osd":1,"up_from":98,"seq":420906795083,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":659836,"kb_used_data":555892,"kb_used_omap":7,"kb_used_meta":103864,"kb_avail":93085316,"statfs":{"total":95995035648,"available":95319363584,"internally_reserved":0,"allocated":569233408,"data_stored":568376988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":106357067},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:24:59 2023","interfaces":[{"interface":"back","average":{"1min":3.0489999999999999,"5min":1.5309999999999999,"15min":0.73399999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":16.356000000000002,"5min":16.356000000000002,"15min":16.356000000000002},"last":0.216},{"interface":"front","average":{"1min":3.6000000000000001,"5min":1.714,"15min":0.80700000000000005},"min":{"1min":0.22700000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":17.175000000000001,"5min":17.175000000000001,"15min":17.175000000000001},"last":0.24199999999999999}]},{"osd":2,"last update":"Mon Aug 28 02:25:27 2023","interfaces":[{"interface":"back","average":{"1min":5.5549999999999997,"5min":2.2530000000000001,"15min":0.97899999999999998},"min":{"1min":0.26200000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":29.483000000000001,"5min":29.483000000000001,"15min":29.483000000000001},"last":125.889},{"interface":"front","average":{"1min":3.286,"5min":1.7470000000000001,"15min":0.83599999999999997},"min":{"1min":0.23200000000000001,"5min":0.221,"15min":0.221},"max":{"1min":19.045000000000002,"5min":19.045000000000002,"15min":19.045000000000002},"last":125.941}]},{"osd":3,"last update":"Mon Aug 28 02:24:59 2023","interfaces":[{"interface":"back","average":{"1min":3.3719999999999999,"5min":1.4630000000000001,"15min":0.73099999999999998},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":15.395,"5min":15.395,"15min":15.395},"last":0.29299999999999998},{"interface":"front","average":{"1min":3.2250000000000001,"5min":1.4950000000000001,"15min":0.80500000000000005},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":14.536,"5min":14.536,"15min":14.536},"last":0.26900000000000002}]},{"osd":4,"last update":"Mon Aug 28 02:24:59 2023","interfaces":[{"interface":"back","average":{"1min":2.8639999999999999,"5min":1.3879999999999999,"15min":0.88},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":10.323,"5min":10.323,"15min":10.323},"last":0.70599999999999996},{"interface":"front","average":{"1min":2.1579999999999999,"5min":1.1699999999999999,"15min":0.80300000000000005},"min":{"1min":0.32300000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":8.6679999999999993,"5min":8.6679999999999993,"15min":8.6679999999999993},"last":0.64000000000000001}]},{"osd":5,"last update":"Mon Aug 28 02:24:59 2023","interfaces":[{"interface":"back","average":{"1min":4.1289999999999996,"5min":1.786,"15min":1.02},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":15.544,"5min":15.544,"15min":15.544},"last":0.68400000000000005},{"interface":"front","average":{"1min":2.2669999999999999,"5min":1.276,"15min":0.85199999999999998},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":10.282,"5min":10.282,"15min":10.282},"last":0.66300000000000003}]},{"osd":6,"last update":"Mon Aug 28 02:24:59 2023","interfaces":[{"interface":"back","average":{"1min":4.2300000000000004,"5min":2.0619999999999998,"15min":1.1000000000000001},"min":{"1min":0.36699999999999999,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":15.512,"5min":15.512,"15min":15.512},"last":0.621},{"interface":"front","average":{"1min":2.98,"5min":1.7350000000000001,"15min":0.98699999999999999},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":10.35,"5min":13.707000000000001,"15min":13.707000000000001},"last":0.59499999999999997}]},{"osd":7,"last update":"Mon Aug 28 02:24:59 2023","interfaces":[{"interface":"back","average":{"1min":2.4630000000000001,"5min":1.3320000000000001,"15min":0.86599999999999999},"min":{"1min":0.34499999999999997,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":8.7240000000000002,"5min":8.7240000000000002,"15min":8.7240000000000002},"last":0.56999999999999995},{"interface":"front","average":{"1min":4.0149999999999997,"5min":1.792,"15min":1.022},"min":{"1min":0.46000000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":15.476000000000001,"5min":15.476000000000001,"15min":15.476000000000001},"last":0.72699999999999998}]}]},{"osd":0,"up_from":90,"seq":386547056729,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1681244,"kb_used_data":1504980,"kb_used_omap":8,"kb_used_meta":176055,"kb_avail":92063908,"statfs":{"total":95995035648,"available":94273441792,"internally_reserved":0,"allocated":1541099520,"data_stored":1540271232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":180280681},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Aug 28 02:25:05 2023","interfaces":[{"interface":"back","average":{"1min":2.306,"5min":2.3210000000000002,"15min":1.0089999999999999},"min":{"1min":0.185,"5min":0.185,"15min":0.185},"max":{"1min":9.2639999999999993,"5min":40.313000000000002,"15min":40.313000000000002},"last":4.1120000000000001},{"interface":"front","average":{"1min":3.6669999999999998,"5min":1.897,"15min":0.88},"min":{"1min":0.27500000000000002,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":19.564,"5min":36.655000000000001,"15min":36.655000000000001},"last":3.702}]},{"osd":2,"last update":"Mon Aug 28 02:25:27 2023","interfaces":[{"interface":"back","average":{"1min":4.4050000000000002,"5min":1.944,"15min":0.88300000000000001},"min":{"1min":0.40500000000000003,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":15.33,"5min":15.33,"15min":15.33},"last":4.532},{"interface":"front","average":{"1min":6.351,"5min":2.484,"15min":1.089},"min":{"1min":0.40799999999999997,"5min":0.224,"15min":0.224},"max":{"1min":18.870999999999999,"5min":18.870999999999999,"15min":18.870999999999999},"last":3.6709999999999998}]},{"osd":3,"last update":"Mon Aug 28 02:25:05 2023","interfaces":[{"interface":"back","average":{"1min":3.4239999999999999,"5min":1.5269999999999999,"15min":0.751},"min":{"1min":0.33600000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":18.545999999999999,"5min":18.545999999999999,"15min":18.545999999999999},"last":5.5609999999999999},{"interface":"front","average":{"1min":3.254,"5min":1.5429999999999999,"15min":0.77700000000000002},"min":{"1min":0.34599999999999997,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":14.534000000000001,"5min":14.534000000000001,"15min":14.534000000000001},"last":2.4900000000000002}]},{"osd":4,"last update":"Mon Aug 28 02:25:00 2023","interfaces":[{"interface":"back","average":{"1min":2.472,"5min":1.5009999999999999,"15min":0.94699999999999995},"min":{"1min":0.37,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":9.2330000000000005,"5min":9.6649999999999991,"15min":9.6649999999999991},"last":2.625},{"interface":"front","average":{"1min":2.8769999999999998,"5min":1.601,"15min":1.0049999999999999},"min":{"1min":0.34000000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":10.273999999999999,"5min":10.273999999999999,"15min":10.273999999999999},"last":1.702}]},{"osd":5,"last update":"Mon Aug 28 02:25:00 2023","interfaces":[{"interface":"back","average":{"1min":2.9929999999999999,"5min":2.0179999999999998,"15min":1.1080000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":11.318,"5min":44.000999999999998,"15min":44.000999999999998},"last":1.74},{"interface":"front","average":{"1min":2.3980000000000001,"5min":1.9039999999999999,"15min":1.0700000000000001},"min":{"1min":0.40899999999999997,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":9.2010000000000005,"5min":43.953000000000003,"15min":43.953000000000003},"last":2.5790000000000002}]},{"osd":6,"last update":"Mon Aug 28 02:25:00 2023","interfaces":[{"interface":"back","average":{"1min":3.3700000000000001,"5min":2.0190000000000001,"15min":1.1100000000000001},"min":{"1min":0.38900000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":11.379,"5min":32.417999999999999,"15min":32.417999999999999},"last":1.7609999999999999},{"interface":"front","average":{"1min":2.3260000000000001,"5min":1.806,"15min":1.0740000000000001},"min":{"1min":0.45500000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":6.3849999999999998,"5min":32.130000000000003,"15min":32.130000000000003},"last":4.0789999999999997}]},{"osd":7,"last update":"Mon Aug 28 02:25:00 2023","interfaces":[{"interface":"back","average":{"1min":2.5099999999999998,"5min":1.5720000000000001,"15min":0.98999999999999999},"min":{"1min":0.433,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":6.319,"5min":9.6020000000000003,"15min":9.6020000000000003},"last":2.6579999999999999},{"interface":"front","average":{"1min":4.0330000000000004,"5min":1.754,"15min":1.054},"min":{"1min":0.44600000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":12.228,"5min":12.228,"15min":12.228},"last":3.7269999999999999}]}]},{"osd":2,"up_from":104,"seq":446676598853,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":762496,"kb_used_data":650488,"kb_used_omap":10,"kb_used_meta":111989,"kb_avail":92982656,"statfs":{"total":95995035648,"available":95214239744,"internally_reserved":0,"allocated":666099712,"data_stored":665303486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10792,"internal_metadata":114677208},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:25:33 2023","interfaces":[{"interface":"back","average":{"1min":3.6179999999999999,"5min":2.2519999999999998,"15min":0.96399999999999997},"min":{"1min":0.188,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":18.146000000000001,"5min":36.890000000000001,"15min":36.890000000000001},"last":0.248},{"interface":"front","average":{"1min":4.625,"5min":2.4590000000000001,"15min":1.0620000000000001},"min":{"1min":0.214,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":25.414999999999999,"5min":35.289000000000001,"15min":35.289000000000001},"last":3.5409999999999999}]},{"osd":1,"last update":"Mon Aug 28 02:25:33 2023","interfaces":[{"interface":"back","average":{"1min":4.4900000000000002,"5min":2.4260000000000002,"15min":1.048},"min":{"1min":0.23999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":16.372,"5min":30.888000000000002,"15min":30.888000000000002},"last":8.2609999999999992},{"interface":"front","average":{"1min":3.8959999999999999,"5min":1.9299999999999999,"15min":0.86599999999999999},"min":{"1min":0.21299999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":14.855,"5min":31.648,"15min":31.648},"last":1.8360000000000001}]},{"osd":3,"last update":"Mon Aug 28 02:24:57 2023","interfaces":[{"interface":"back","average":{"1min":3.0430000000000001,"5min":1.341,"15min":0.71899999999999997},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":14.991,"5min":14.991,"15min":14.991},"last":3.5819999999999999},{"interface":"front","average":{"1min":4.5819999999999999,"5min":1.589,"15min":0.78700000000000003},"min":{"1min":0.29999999999999999,"5min":0.246,"15min":0.246},"max":{"1min":17.577000000000002,"5min":17.577000000000002,"15min":17.577000000000002},"last":1.8089999999999999}]},{"osd":4,"last update":"Mon Aug 28 02:25:33 2023","interfaces":[{"interface":"back","average":{"1min":3.5750000000000002,"5min":1.772,"15min":0.98599999999999999},"min":{"1min":0.34399999999999997,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":14.815,"5min":14.815,"15min":14.815},"last":1.879},{"interface":"front","average":{"1min":1.71,"5min":1.3300000000000001,"15min":0.876},"min":{"1min":0.33700000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.806,"5min":6.806,"15min":6.806},"last":0.42199999999999999}]},{"osd":5,"last update":"Mon Aug 28 02:25:33 2023","interfaces":[{"interface":"back","average":{"1min":1.9099999999999999,"5min":1.518,"15min":0.91700000000000004},"min":{"1min":0.36399999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":6.6900000000000004,"5min":9.1549999999999994,"15min":9.1549999999999994},"last":3.6080000000000001},{"interface":"front","average":{"1min":4.0650000000000004,"5min":2.0459999999999998,"15min":1.054},"min":{"1min":0.438,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":14.877000000000001,"5min":14.877000000000001,"15min":14.877000000000001},"last":1.859}]},{"osd":6,"last update":"Mon Aug 28 02:25:33 2023","interfaces":[{"interface":"back","average":{"1min":2.2280000000000002,"5min":1.659,"15min":0.95299999999999996},"min":{"1min":0.35699999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":8.7010000000000005,"5min":8.7010000000000005,"15min":8.7010000000000005},"last":0.45100000000000001},{"interface":"front","average":{"1min":2.1190000000000002,"5min":1.7350000000000001,"15min":0.98899999999999999},"min":{"1min":0.42999999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":8.7729999999999997,"5min":10.507,"15min":10.507},"last":4.8259999999999996}]},{"osd":7,"last update":"Mon Aug 28 02:25:33 2023","interfaces":[{"interface":"back","average":{"1min":2.0720000000000001,"5min":1.6100000000000001,"15min":0.93600000000000005},"min":{"1min":0.39800000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":8.7699999999999996,"5min":9.0909999999999993,"15min":9.0909999999999993},"last":0.51900000000000002},{"interface":"front","average":{"1min":2.278,"5min":1.673,"15min":0.96099999999999997},"min":{"1min":0.40400000000000003,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":7.1120000000000001,"5min":7.2569999999999997,"15min":7.2569999999999997},"last":7.173}]}]},{"osd":3,"up_from":112,"seq":481036337215,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":63376,"kb_used_data":17928,"kb_used_omap":3,"kb_used_meta":45436,"kb_avail":93681776,"statfs":{"total":95995035648,"available":95930138624,"internally_reserved":0,"allocated":18358272,"data_stored":17600758,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":46527394},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:24:56 2023","interfaces":[{"interface":"back","average":{"1min":18.995000000000001,"5min":5.5199999999999996,"15min":2.073},"min":{"1min":0.215,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":164.25299999999999,"5min":164.25299999999999,"15min":164.25299999999999},"last":2.0920000000000001},{"interface":"front","average":{"1min":19.428999999999998,"5min":5.7149999999999999,"15min":2.1269999999999998},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":164.55000000000001,"5min":164.55000000000001,"15min":164.55000000000001},"last":2.964}]},{"osd":1,"last update":"Mon Aug 28 02:24:56 2023","interfaces":[{"interface":"back","average":{"1min":1.9330000000000001,"5min":3.3570000000000002,"15min":1.345},"min":{"1min":0.247,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":5.5830000000000002,"5min":117.34699999999999,"15min":117.34699999999999},"last":2.1760000000000002},{"interface":"front","average":{"1min":1.4630000000000001,"5min":3.3759999999999999,"15min":1.3779999999999999},"min":{"1min":0.23400000000000001,"5min":0.224,"15min":0.224},"max":{"1min":4.4249999999999998,"5min":118.012,"15min":118.012},"last":2.1240000000000001}]},{"osd":2,"last update":"Mon Aug 28 02:24:56 2023","interfaces":[{"interface":"back","average":{"1min":10.507999999999999,"5min":4.9109999999999996,"15min":1.887},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":120.43300000000001,"5min":120.43300000000001,"15min":120.43300000000001},"last":11.380000000000001},{"interface":"front","average":{"1min":11.199,"5min":5.0119999999999996,"15min":1.9219999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":120.44499999999999,"5min":120.44499999999999,"15min":120.44499999999999},"last":3.0680000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:24:56 2023","interfaces":[{"interface":"back","average":{"1min":1.8959999999999999,"5min":1.256,"15min":0.83799999999999997},"min":{"1min":0.309,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":5.9740000000000002,"5min":7.423,"15min":7.423},"last":3.0030000000000001},{"interface":"front","average":{"1min":1.895,"5min":1.1879999999999999,"15min":0.81999999999999995},"min":{"1min":0.36799999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":5.4859999999999998,"5min":8.4039999999999999,"15min":8.4039999999999999},"last":2.2250000000000001}]},{"osd":5,"last update":"Mon Aug 28 02:24:56 2023","interfaces":[{"interface":"back","average":{"1min":2.048,"5min":1.1259999999999999,"15min":0.80000000000000004},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":5.5590000000000002,"5min":7.3369999999999997,"15min":7.3369999999999997},"last":2.2040000000000002},{"interface":"front","average":{"1min":1.228,"5min":1.1040000000000001,"15min":0.78800000000000003},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":4.8170000000000002,"5min":9.3849999999999998,"15min":9.3849999999999998},"last":2.145}]},{"osd":6,"last update":"Mon Aug 28 02:24:56 2023","interfaces":[{"interface":"back","average":{"1min":0.999,"5min":1.786,"15min":1.0189999999999999},"min":{"1min":0.35599999999999998,"5min":0.314,"15min":0.314},"max":{"1min":4.0999999999999996,"5min":72.116,"15min":72.116},"last":9.4529999999999994},{"interface":"front","average":{"1min":2.0990000000000002,"5min":1.9319999999999999,"15min":1.042},"min":{"1min":0.38800000000000001,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":4.9199999999999999,"5min":72.030000000000001,"15min":72.030000000000001},"last":3.0270000000000001}]},{"osd":7,"last update":"Mon Aug 28 02:24:56 2023","interfaces":[{"interface":"back","average":{"1min":2.1040000000000001,"5min":1.27,"15min":0.86199999999999999},"min":{"1min":0.41699999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":6.1669999999999998,"5min":7.3129999999999997,"15min":7.3129999999999997},"last":3.0470000000000002},{"interface":"front","average":{"1min":2.3700000000000001,"5min":1.2949999999999999,"15min":0.86199999999999999},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":6.5030000000000001,"5min":7.2960000000000003,"15min":7.2960000000000003},"last":2.3290000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019088,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1216176,"kb_used_data":918952,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":92528976,"statfs":{"total":95995035648,"available":94749671424,"internally_reserved":0,"allocated":941006848,"data_stored":940439881,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:24:55 2023","interfaces":[{"interface":"back","average":{"1min":17.632999999999999,"5min":4.6539999999999999,"15min":1.962},"min":{"1min":0.24099999999999999,"5min":0.223,"15min":0.223},"max":{"1min":138.67400000000001,"5min":138.67400000000001,"15min":138.67400000000001},"last":0.27000000000000002},{"interface":"front","average":{"1min":17.952000000000002,"5min":4.5979999999999999,"15min":1.9319999999999999},"min":{"1min":0.318,"5min":0.248,"15min":0.248},"max":{"1min":134.14599999999999,"5min":134.14599999999999,"15min":134.14599999999999},"last":0.28899999999999998}]},{"osd":1,"last update":"Mon Aug 28 02:24:58 2023","interfaces":[{"interface":"back","average":{"1min":1.125,"5min":1.4890000000000001,"15min":0.84499999999999997},"min":{"1min":0.27100000000000002,"5min":0.255,"15min":0.255},"max":{"1min":4.5209999999999999,"5min":53.622999999999998,"15min":53.622999999999998},"last":1.5840000000000001},{"interface":"front","average":{"1min":2.0299999999999998,"5min":1.5860000000000001,"15min":0.90500000000000003},"min":{"1min":0.29199999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":5.9649999999999999,"5min":53.774000000000001,"15min":53.774000000000001},"last":0.40899999999999997}]},{"osd":2,"last update":"Mon Aug 28 02:25:28 2023","interfaces":[{"interface":"back","average":{"1min":5.0780000000000003,"5min":3.6219999999999999,"15min":1.5740000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":56.295999999999999,"5min":74.099000000000004,"15min":74.099000000000004},"last":5.4009999999999998},{"interface":"front","average":{"1min":4.7229999999999999,"5min":3.4820000000000002,"15min":1.52},"min":{"1min":0.28599999999999998,"5min":0.25,"15min":0.25},"max":{"1min":56.314,"5min":74.081999999999994,"15min":74.081999999999994},"last":6.2569999999999997}]},{"osd":3,"last update":"Mon Aug 28 02:24:58 2023","interfaces":[{"interface":"back","average":{"1min":1.3899999999999999,"5min":0.94799999999999995,"15min":0.65000000000000002},"min":{"1min":0.309,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":5.8559999999999999,"5min":6.6340000000000003,"15min":6.6340000000000003},"last":0.32400000000000001},{"interface":"front","average":{"1min":1.6859999999999999,"5min":1.2150000000000001,"15min":0.79400000000000004},"min":{"1min":0.36199999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":5.7190000000000003,"5min":6.8230000000000004,"15min":6.8230000000000004},"last":0.39900000000000002}]},{"osd":5,"last update":"Mon Aug 28 02:25:41 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":0.97799999999999998,"15min":0.63700000000000001},"min":{"1min":0.26900000000000002,"5min":0.249,"15min":0.249},"max":{"1min":5.4080000000000004,"5min":8.2560000000000002,"15min":8.2560000000000002},"last":1.0900000000000001},{"interface":"front","average":{"1min":2.3010000000000002,"5min":1.3300000000000001,"15min":0.76700000000000002},"min":{"1min":0.27100000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":8.0619999999999994,"5min":8.0619999999999994,"15min":8.0619999999999994},"last":1.409}]},{"osd":6,"last update":"Mon Aug 28 02:24:58 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":1.0660000000000001,"15min":0.68500000000000005},"min":{"1min":0.32800000000000001,"5min":0.29199999999999998,"15min":0.27500000000000002},"max":{"1min":3.7989999999999999,"5min":9.6920000000000002,"15min":9.6920000000000002},"last":0.45900000000000002},{"interface":"front","average":{"1min":1.105,"5min":0.81499999999999995,"15min":0.629},"min":{"1min":0.38700000000000001,"5min":0.32200000000000001,"15min":0.318},"max":{"1min":4.3840000000000003,"5min":4.3840000000000003,"15min":4.3840000000000003},"last":0.46800000000000003}]},{"osd":7,"last update":"Mon Aug 28 02:25:22 2023","interfaces":[{"interface":"back","average":{"1min":1.825,"5min":1.0960000000000001,"15min":0.72499999999999998},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.312},"max":{"1min":7.9589999999999996,"5min":8.6890000000000001,"15min":8.6890000000000001},"last":6.3579999999999997},{"interface":"front","average":{"1min":1.8460000000000001,"5min":1.4450000000000001,"15min":0.84999999999999998},"min":{"1min":0.35499999999999998,"5min":0.32500000000000001,"15min":0.32100000000000001},"max":{"1min":5.8250000000000002,"5min":8.7569999999999997,"15min":8.7569999999999997},"last":0.371}]}]},{"osd":5,"up_from":36,"seq":154618822860,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1377060,"kb_used_data":1071644,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":92368092,"statfs":{"total":95995035648,"available":94584926208,"internally_reserved":0,"allocated":1097363456,"data_stored":1096803956,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:24:50 2023","interfaces":[{"interface":"back","average":{"1min":12.795999999999999,"5min":5.2640000000000002,"15min":2.1709999999999998},"min":{"1min":0.219,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":101.014,"5min":131.06100000000001,"15min":131.06100000000001},"last":4.133},{"interface":"front","average":{"1min":13.936999999999999,"5min":5.54,"15min":2.2650000000000001},"min":{"1min":0.36499999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":101.03,"5min":131.262,"15min":131.262},"last":4.0999999999999996}]},{"osd":1,"last update":"Mon Aug 28 02:25:00 2023","interfaces":[{"interface":"back","average":{"1min":7.7599999999999998,"5min":4.8479999999999999,"15min":1.968},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":104.748,"5min":183.97900000000001,"15min":183.97900000000001},"last":0.69999999999999996},{"interface":"front","average":{"1min":9.202,"5min":5.1059999999999999,"15min":2.0619999999999998},"min":{"1min":0.26300000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":105.666,"5min":184.33699999999999,"15min":184.33699999999999},"last":4.7480000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:25:30 2023","interfaces":[{"interface":"back","average":{"1min":3.5409999999999999,"5min":5.2169999999999996,"15min":2.1320000000000001},"min":{"1min":0.29999999999999999,"5min":0.252,"15min":0.252},"max":{"1min":12.584,"5min":110.051,"15min":110.051},"last":5.1740000000000004},{"interface":"front","average":{"1min":3.7709999999999999,"5min":5.4820000000000002,"15min":2.2189999999999999},"min":{"1min":0.29799999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":15.635,"5min":110.041,"15min":110.041},"last":5.1340000000000003}]},{"osd":3,"last update":"Mon Aug 28 02:25:00 2023","interfaces":[{"interface":"back","average":{"1min":2.7650000000000001,"5min":1.371,"15min":0.86499999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":10.576000000000001,"5min":10.576000000000001,"15min":10.576000000000001},"last":0.48099999999999998},{"interface":"front","average":{"1min":3.258,"5min":1.5920000000000001,"15min":0.94799999999999995},"min":{"1min":0.35799999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":10.459,"5min":10.459,"15min":10.459},"last":0.52500000000000002}]},{"osd":4,"last update":"Mon Aug 28 02:25:44 2023","interfaces":[{"interface":"back","average":{"1min":1.4239999999999999,"5min":1.355,"15min":0.76200000000000001},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":5.6890000000000001,"5min":10.759,"15min":10.759},"last":0.79300000000000004},{"interface":"front","average":{"1min":2.149,"5min":1.468,"15min":0.82199999999999995},"min":{"1min":0.31900000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.0170000000000003,"5min":8.6769999999999996,"15min":8.6769999999999996},"last":0.73799999999999999}]},{"osd":6,"last update":"Mon Aug 28 02:25:00 2023","interfaces":[{"interface":"back","average":{"1min":1.3540000000000001,"5min":1.4079999999999999,"15min":0.81999999999999995},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.28399999999999997},"max":{"1min":6.9100000000000001,"5min":14.803000000000001,"15min":14.803000000000001},"last":0.66700000000000004},{"interface":"front","average":{"1min":1.8580000000000001,"5min":1.1779999999999999,"15min":0.73799999999999999},"min":{"1min":0.315,"5min":0.31,"15min":0.30299999999999999},"max":{"1min":8.6479999999999997,"5min":8.6479999999999997,"15min":8.6479999999999997},"last":0.83699999999999997}]},{"osd":7,"last update":"Mon Aug 28 02:25:17 2023","interfaces":[{"interface":"back","average":{"1min":2.004,"5min":1.3180000000000001,"15min":0.80300000000000005},"min":{"1min":0.33900000000000002,"5min":0.31,"15min":0.31},"max":{"1min":9.3070000000000004,"5min":9.3070000000000004,"15min":9.3070000000000004},"last":0.81100000000000005},{"interface":"front","average":{"1min":1.871,"5min":1.341,"15min":0.80900000000000005},"min":{"1min":0.32200000000000001,"5min":0.32100000000000001,"15min":0.29799999999999999},"max":{"1min":12.787000000000001,"5min":12.787000000000001,"15min":12.787000000000001},"last":0.77700000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1514143744,"data_stored":1514143744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":558891008,"data_stored":558891008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":660279296,"data_stored":660275224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16449536,"data_stored":16449536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":920059904,"data_stored":920059904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1076826112,"data_stored":1076822040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":463077376,"data_stored":463077376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":574623744,"data_stored":574619672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-08-28T02:25:57.685 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph pg dump --format=json 2023-08-28T02:25:57.775 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:57.836 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:25:57.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:57.841 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:25:57.891 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:25:57.943 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:25:58.002 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:58.243 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:25:58.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:58.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:25:58.287 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.4 dump_historic_ops 2023-08-28T02:25:58.356 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:25:58.429 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:58.533 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:25:58.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:58.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:25:58.550 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:25:58 smithi169 ceph-mon[159415]: pgmap v404: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.6 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 1971/79800 objects degraded (2.470%); 4623/79800 objects misplaced (5.793%); 2.1 MiB/s, 33 objects/s recovering 2023-08-28T02:25:58.617 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:25:58 smithi143 ceph-mon[183059]: pgmap v404: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.6 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 1971/79800 objects degraded (2.470%); 4623/79800 objects misplaced (5.793%); 2.1 MiB/s, 33 objects/s recovering 2023-08-28T02:25:58.617 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:25:58 smithi143 ceph-mon[189871]: pgmap v404: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.6 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 1971/79800 objects degraded (2.470%); 4623/79800 objects misplaced (5.793%); 2.1 MiB/s, 33 objects/s recovering 2023-08-28T02:25:58.649 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:25:58.733 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:58.856 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:25:58.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:58.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:25:58.974 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:25:58.993 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:25:59.024 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:59.122 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:25:59.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:59.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:25:59.226 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:25:59.270 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:59.442 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:25:59.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:59.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:25:59.464 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:25:59.563 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:25:59.576 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:25:59.687 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:25:59.811 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:25:59.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:25:59.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:25:59.915 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:25:59.944 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:00.111 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:26:00.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:00.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:26:00.226 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:26:00.356 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:00.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:00 smithi143 ceph-mon[183059]: pgmap v405: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.5 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 1984/73251 objects degraded (2.708%); 4234/73251 objects misplaced (5.780%); 2.0 MiB/s, 32 objects/s recovering 2023-08-28T02:26:00.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:00 smithi143 ceph-mon[189871]: pgmap v405: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.5 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 1984/73251 objects degraded (2.708%); 4234/73251 objects misplaced (5.780%); 2.0 MiB/s, 32 objects/s recovering 2023-08-28T02:26:00.527 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:26:00.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:00.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:26:00.605 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:00 smithi169 ceph-mon[159415]: pgmap v405: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.5 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 1984/73251 objects degraded (2.708%); 4234/73251 objects misplaced (5.780%); 2.0 MiB/s, 32 objects/s recovering 2023-08-28T02:26:00.637 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:26:00.699 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:00.842 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:26:00.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:00.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:26:00.956 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:26:00.956 INFO:teuthology.orchestra.run.smithi169.stdout: "size": 20, 2023-08-28T02:26:00.956 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 600, 2023-08-28T02:26:00.956 INFO:teuthology.orchestra.run.smithi169.stdout: "ops": [ 2023-08-28T02:26:00.956 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.956 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:114254 5.1 5:8f7eaa26:::benchmark_data_smithi143_236200_object4111:head [delete] snapc 0=[] ondisk+write+known_if_redirected e147)", 2023-08-28T02:26:00.956 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:43.862931+0000", 2023-08-28T02:26:00.956 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 137.085670806, 2023-08-28T02:26:00.956 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0947160739999999, 2023-08-28T02:26:00.956 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:00.957 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "reached pg", 2023-08-28T02:26:00.957 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:26:00.957 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:26:00.957 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:00.957 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 114254 2023-08-28T02:26:00.957 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.957 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:00.957 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.957 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:00.957 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.862931+0000", 2023-08-28T02:26:00.957 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:00.957 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.957 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.958 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:00.958 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.862931+0000", 2023-08-28T02:26:00.958 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:26:00.958 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.958 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.958 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:00.958 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.862929+0000", 2023-08-28T02:26:00.958 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.157e-06 2023-08-28T02:26:00.958 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.958 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.958 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:00.958 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.862937+0000", 2023-08-28T02:26:00.958 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1687e-05 2023-08-28T02:26:00.959 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.959 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.959 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:00.959 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.862949+0000", 2023-08-28T02:26:00.959 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.3419999999999996e-06 2023-08-28T02:26:00.959 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.959 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.959 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:00.959 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.862955+0000", 2023-08-28T02:26:00.959 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0034338319999999999 2023-08-28T02:26:00.959 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.959 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.959 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:00.960 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.866389+0000", 2023-08-28T02:26:00.960 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00011907499999999999 2023-08-28T02:26:00.960 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.960 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.960 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:00.960 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.866508+0000", 2023-08-28T02:26:00.960 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00014797899999999999 2023-08-28T02:26:00.960 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.960 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.960 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,7", 2023-08-28T02:26:00.960 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.866656+0000", 2023-08-28T02:26:00.960 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0075255089999999997 2023-08-28T02:26:00.960 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.961 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.961 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:26:00.961 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.874182+0000", 2023-08-28T02:26:00.961 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0046194599999999997 2023-08-28T02:26:00.961 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.961 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.961 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:00.961 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.878801+0000", 2023-08-28T02:26:00.961 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1385e-05 2023-08-28T02:26:00.961 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.961 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.961 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:26:00.961 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.878813+0000", 2023-08-28T02:26:00.962 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.078743335 2023-08-28T02:26:00.962 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.962 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.962 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:00.962 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.957556+0000", 2023-08-28T02:26:00.962 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.1067000000000001e-05 2023-08-28T02:26:00.962 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.962 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.962 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:26:00.962 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.957647+0000", 2023-08-28T02:26:00.962 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0947160739999999 2023-08-28T02:26:00.962 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:00.962 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:00.962 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:00.963 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.963 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.963 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:114255 5.1 5:8ac446c3:::benchmark_data_smithi143_236200_object4112:head [delete] snapc 0=[] ondisk+write+known_if_redirected e147)", 2023-08-28T02:26:00.963 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:43.862970+0000", 2023-08-28T02:26:00.963 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 137.08563182699999, 2023-08-28T02:26:00.963 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.10295312, 2023-08-28T02:26:00.963 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:00.963 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "reached pg", 2023-08-28T02:26:00.963 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:26:00.963 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:26:00.963 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:00.963 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 114255 2023-08-28T02:26:00.963 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.964 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:00.964 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.964 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:00.964 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.862970+0000", 2023-08-28T02:26:00.964 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:00.964 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.964 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.964 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:00.964 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.862970+0000", 2023-08-28T02:26:00.964 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999995 2023-08-28T02:26:00.964 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.964 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.964 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:00.964 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.862969+0000", 2023-08-28T02:26:00.965 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.7919999999999999e-06 2023-08-28T02:26:00.965 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.965 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.965 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:00.965 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.862973+0000", 2023-08-28T02:26:00.965 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.6030000000000001e-06 2023-08-28T02:26:00.965 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.965 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.965 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:00.965 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.862980+0000", 2023-08-28T02:26:00.965 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.2400000000000001e-06 2023-08-28T02:26:00.965 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.965 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.966 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:00.966 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.862984+0000", 2023-08-28T02:26:00.966 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0072338599999999999 2023-08-28T02:26:00.966 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.966 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.966 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:00.966 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.870218+0000", 2023-08-28T02:26:00.966 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000128653 2023-08-28T02:26:00.966 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.966 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.966 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:00.966 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.870347+0000", 2023-08-28T02:26:00.966 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000162411 2023-08-28T02:26:00.967 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.967 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.967 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,7", 2023-08-28T02:26:00.967 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.870509+0000", 2023-08-28T02:26:00.967 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0083309839999999996 2023-08-28T02:26:00.967 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.967 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.967 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:00.967 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.878840+0000", 2023-08-28T02:26:00.967 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.272e-06 2023-08-28T02:26:00.967 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.967 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.967 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:26:00.968 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.878847+0000", 2023-08-28T02:26:00.968 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.079164749 2023-08-28T02:26:00.968 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.968 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.968 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:00.968 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.958012+0000", 2023-08-28T02:26:00.968 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00015897099999999999 2023-08-28T02:26:00.968 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.968 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.968 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for missing object", 2023-08-28T02:26:00.968 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.958171+0000", 2023-08-28T02:26:00.968 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0076846830000000003 2023-08-28T02:26:00.968 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.969 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.969 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:00.969 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.965856+0000", 2023-08-28T02:26:00.969 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.7471000000000005e-05 2023-08-28T02:26:00.969 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.969 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.969 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:26:00.969 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.965923+0000", 2023-08-28T02:26:00.969 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.10295312 2023-08-28T02:26:00.969 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:00.969 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:00.969 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:00.969 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.970 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.970 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:114271 5.1 5:8ca8421e:::benchmark_data_smithi143_236200_object4128:head [delete] snapc 0=[] ondisk+write+known_if_redirected e148)", 2023-08-28T02:26:00.970 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:43.884816+0000", 2023-08-28T02:26:00.970 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 137.06378536400001, 2023-08-28T02:26:00.970 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0813256689999999, 2023-08-28T02:26:00.970 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:00.970 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:00.970 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:26:00.970 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:26:00.970 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:00.970 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 114271 2023-08-28T02:26:00.970 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.970 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:00.971 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.971 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:00.971 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.884816+0000", 2023-08-28T02:26:00.971 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:00.971 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.971 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.971 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:00.971 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.884816+0000", 2023-08-28T02:26:00.971 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:26:00.971 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.971 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.971 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:00.971 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.884815+0000", 2023-08-28T02:26:00.972 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.8809999999999998e-06 2023-08-28T02:26:00.972 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.972 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.972 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:00.972 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.884822+0000", 2023-08-28T02:26:00.972 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1715999999999999e-05 2023-08-28T02:26:00.972 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.972 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.972 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:00.972 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.884833+0000", 2023-08-28T02:26:00.972 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.3779999999999998e-06 2023-08-28T02:26:00.972 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.973 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.973 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:00.973 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.884840+0000", 2023-08-28T02:26:00.973 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0016300780000000001 2023-08-28T02:26:00.973 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.973 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.973 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:00.973 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.886470+0000", 2023-08-28T02:26:00.973 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.6500000000000002e-06 2023-08-28T02:26:00.973 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.973 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.973 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:26:00.973 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.886478+0000", 2023-08-28T02:26:00.974 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0717474890000001 2023-08-28T02:26:00.974 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.974 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.974 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:00.974 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.958226+0000", 2023-08-28T02:26:00.974 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012793900000000001 2023-08-28T02:26:00.974 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.974 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.974 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:00.974 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.958354+0000", 2023-08-28T02:26:00.974 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000149421 2023-08-28T02:26:00.974 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.974 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.974 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,7", 2023-08-28T02:26:00.975 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.958503+0000", 2023-08-28T02:26:00.975 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0070383570000000003 2023-08-28T02:26:00.975 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.975 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.975 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:00.975 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.965542+0000", 2023-08-28T02:26:00.975 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.3148e-05 2023-08-28T02:26:00.975 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.975 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.975 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:26:00.975 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.965555+0000", 2023-08-28T02:26:00.975 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00044809299999999999 2023-08-28T02:26:00.975 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.976 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.976 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:00.976 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.966003+0000", 2023-08-28T02:26:00.976 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.1020999999999998e-05 2023-08-28T02:26:00.976 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.976 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.976 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:26:00.976 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.966054+0000", 2023-08-28T02:26:00.976 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.8335e-05 2023-08-28T02:26:00.976 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.976 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.976 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:26:00.976 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.966142+0000", 2023-08-28T02:26:00.977 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0813256689999999 2023-08-28T02:26:00.977 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:00.977 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:00.977 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:00.977 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.977 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.977 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:114280 5.1 5:85701856:::benchmark_data_smithi143_236200_object4137:head [delete] snapc 0=[] ondisk+write+known_if_redirected e148)", 2023-08-28T02:26:00.977 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:43.900301+0000", 2023-08-28T02:26:00.977 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 137.048300485, 2023-08-28T02:26:00.977 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0660135610000001, 2023-08-28T02:26:00.977 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:00.977 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:00.977 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:26:00.977 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:26:00.978 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:00.978 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 114280 2023-08-28T02:26:00.978 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.978 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:00.978 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.978 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:00.978 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.900301+0000", 2023-08-28T02:26:00.978 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:00.978 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.978 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.978 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:00.978 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.900301+0000", 2023-08-28T02:26:00.978 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:26:00.979 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.979 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.979 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:00.979 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.900299+0000", 2023-08-28T02:26:00.979 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.8669999999999999e-06 2023-08-28T02:26:00.979 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.979 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.979 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:00.979 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.900305+0000", 2023-08-28T02:26:00.979 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1285e-05 2023-08-28T02:26:00.979 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.979 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.979 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:00.980 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.900317+0000", 2023-08-28T02:26:00.980 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.5029999999999997e-06 2023-08-28T02:26:00.980 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.980 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.980 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:00.980 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.900322+0000", 2023-08-28T02:26:00.980 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.002364484 2023-08-28T02:26:00.980 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.980 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.980 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:00.980 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.902687+0000", 2023-08-28T02:26:00.980 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2643000000000001e-05 2023-08-28T02:26:00.981 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.981 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.981 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:26:00.981 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.902699+0000", 2023-08-28T02:26:00.981 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0592813210000001 2023-08-28T02:26:00.981 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.981 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.981 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:00.981 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.961981+0000", 2023-08-28T02:26:00.981 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000131493 2023-08-28T02:26:00.981 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.981 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.981 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:00.981 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.962112+0000", 2023-08-28T02:26:00.982 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00016603899999999999 2023-08-28T02:26:00.982 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.982 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.982 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,7", 2023-08-28T02:26:00.982 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.962278+0000", 2023-08-28T02:26:00.982 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0036423079999999999 2023-08-28T02:26:00.982 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.982 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.982 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:26:00.982 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.965920+0000", 2023-08-28T02:26:00.982 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00028787099999999998 2023-08-28T02:26:00.982 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.982 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.983 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:00.983 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.966208+0000", 2023-08-28T02:26:00.983 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.5313e-05 2023-08-28T02:26:00.983 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.983 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.983 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:00.983 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.966234+0000", 2023-08-28T02:26:00.983 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.6737000000000002e-05 2023-08-28T02:26:00.983 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.983 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.983 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:26:00.983 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.966260+0000", 2023-08-28T02:26:00.983 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.4552999999999998e-05 2023-08-28T02:26:00.984 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.984 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.984 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:26:00.984 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.966315+0000", 2023-08-28T02:26:00.984 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0660135610000001 2023-08-28T02:26:00.984 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:00.984 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:00.984 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:00.984 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.984 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.984 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:114309 5.1 5:87b49cb9:::benchmark_data_smithi143_236200_object4166:head [delete] snapc 0=[] ondisk+write+known_if_redirected e148)", 2023-08-28T02:26:00.984 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:43.938203+0000", 2023-08-28T02:26:00.984 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 137.01039933000001, 2023-08-28T02:26:00.984 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0461147120000001, 2023-08-28T02:26:00.985 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:00.985 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:00.985 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:26:00.985 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:26:00.985 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:00.985 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 114309 2023-08-28T02:26:00.985 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.985 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:00.985 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.985 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:00.985 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.938203+0000", 2023-08-28T02:26:00.985 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:00.986 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.986 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.986 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:00.986 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.938203+0000", 2023-08-28T02:26:00.986 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999976 2023-08-28T02:26:00.986 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.986 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.986 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:00.986 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.938200+0000", 2023-08-28T02:26:00.986 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.6900000000000004e-06 2023-08-28T02:26:00.986 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.986 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.986 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:00.987 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.938210+0000", 2023-08-28T02:26:00.987 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.3209e-05 2023-08-28T02:26:00.987 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.987 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.987 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:00.987 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.938223+0000", 2023-08-28T02:26:00.987 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.7580000000000003e-06 2023-08-28T02:26:00.987 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.987 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.987 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:00.987 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.938230+0000", 2023-08-28T02:26:00.987 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.0442999999999998e-05 2023-08-28T02:26:00.987 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.988 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.988 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:00.988 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.938280+0000", 2023-08-28T02:26:00.988 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0271e-05 2023-08-28T02:26:00.988 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.988 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.988 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:26:00.988 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.938291+0000", 2023-08-28T02:26:00.988 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0280542239999999 2023-08-28T02:26:00.988 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.988 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.988 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:00.988 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.966345+0000", 2023-08-28T02:26:00.989 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.8124e-05 2023-08-28T02:26:00.989 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.989 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.989 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:00.989 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.966413+0000", 2023-08-28T02:26:00.989 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000117788 2023-08-28T02:26:00.989 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.989 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.989 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,7", 2023-08-28T02:26:00.989 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.966531+0000", 2023-08-28T02:26:00.989 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.010700407 2023-08-28T02:26:00.989 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.989 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.990 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:00.990 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.977231+0000", 2023-08-28T02:26:00.990 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0069568429999999999 2023-08-28T02:26:00.990 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.990 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.990 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:00.990 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.984188+0000", 2023-08-28T02:26:00.990 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.8620000000000003e-06 2023-08-28T02:26:00.990 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.990 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.990 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:26:00.990 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.984196+0000", 2023-08-28T02:26:00.990 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.7914999999999999e-05 2023-08-28T02:26:00.991 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.991 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.991 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:26:00.991 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.984234+0000", 2023-08-28T02:26:00.991 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.3529999999999995e-05 2023-08-28T02:26:00.991 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.991 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.991 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:26:00.991 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.984317+0000", 2023-08-28T02:26:00.991 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0461147120000001 2023-08-28T02:26:00.991 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:00.991 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:00.991 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:00.992 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.992 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.992 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:114345 5.1 5:8edcacfd:::benchmark_data_smithi143_236200_object4202:head [delete] snapc 0=[] ondisk+write+known_if_redirected e148)", 2023-08-28T02:26:00.992 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:43.981100+0000", 2023-08-28T02:26:00.992 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 136.96750214299999, 2023-08-28T02:26:00.992 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0099165400000001, 2023-08-28T02:26:00.992 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:00.992 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:00.992 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:26:00.992 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:26:00.992 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:00.992 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 114345 2023-08-28T02:26:00.993 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.993 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:00.993 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.993 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:00.993 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.981100+0000", 2023-08-28T02:26:00.993 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:00.993 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.993 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.993 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:00.993 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.981100+0000", 2023-08-28T02:26:00.993 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:26:00.993 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.993 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.994 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:00.994 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.981098+0000", 2023-08-28T02:26:00.994 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.0620000000000007e-06 2023-08-28T02:26:00.994 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.994 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.994 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:00.994 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.981107+0000", 2023-08-28T02:26:00.994 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.3488e-05 2023-08-28T02:26:00.994 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.994 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.994 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:00.994 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.981120+0000", 2023-08-28T02:26:00.994 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.5180000000000001e-06 2023-08-28T02:26:00.995 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.995 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.995 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:00.995 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.981127+0000", 2023-08-28T02:26:00.995 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.5450000000000004e-05 2023-08-28T02:26:00.995 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.995 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.995 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:00.995 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.981202+0000", 2023-08-28T02:26:00.995 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1759999999999999e-05 2023-08-28T02:26:00.995 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.995 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.996 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:26:00.996 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.981214+0000", 2023-08-28T02:26:00.996 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.98907669099999995 2023-08-28T02:26:00.996 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.996 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.996 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:00.996 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.970291+0000", 2023-08-28T02:26:00.996 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.2393999999999998e-05 2023-08-28T02:26:00.996 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.996 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.996 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:00.996 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.970373+0000", 2023-08-28T02:26:00.996 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00024490299999999998 2023-08-28T02:26:00.996 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.997 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.997 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,7", 2023-08-28T02:26:00.997 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.970618+0000", 2023-08-28T02:26:00.997 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0066488270000000004 2023-08-28T02:26:00.997 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.997 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.997 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:00.997 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.977267+0000", 2023-08-28T02:26:00.997 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0070592379999999998 2023-08-28T02:26:00.997 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.997 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.997 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:26:00.997 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.984326+0000", 2023-08-28T02:26:00.998 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.006583144 2023-08-28T02:26:00.998 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.998 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.998 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:00.998 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.990909+0000", 2023-08-28T02:26:00.998 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.7954000000000003e-05 2023-08-28T02:26:00.998 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.998 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.998 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:26:00.998 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.990947+0000", 2023-08-28T02:26:00.998 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.9080999999999998e-05 2023-08-28T02:26:00.998 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.998 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.999 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:26:00.999 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.991016+0000", 2023-08-28T02:26:00.999 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0099165400000001 2023-08-28T02:26:00.999 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:00.999 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:00.999 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:00.999 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:00.999 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:00.999 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:114347 5.1 5:8f8588e4:::benchmark_data_smithi143_236200_object4204:head [delete] snapc 0=[] ondisk+write+known_if_redirected e148)", 2023-08-28T02:26:00.999 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:43.984527+0000", 2023-08-28T02:26:00.999 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 136.96407525999999, 2023-08-28T02:26:00.999 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.006606983, 2023-08-28T02:26:01.000 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:01.000 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:01.000 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:26:01.000 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:26:01.000 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:01.000 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 114347 2023-08-28T02:26:01.000 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.002 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:01.002 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.002 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:01.002 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.984527+0000", 2023-08-28T02:26:01.002 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:01.002 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.002 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.002 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:01.002 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.984527+0000", 2023-08-28T02:26:01.002 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:26:01.002 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.003 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.003 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:01.003 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.984525+0000", 2023-08-28T02:26:01.003 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.2899999999999997e-06 2023-08-28T02:26:01.003 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.003 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.003 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:01.003 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.984532+0000", 2023-08-28T02:26:01.003 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.3574000000000001e-05 2023-08-28T02:26:01.003 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.003 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.003 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:01.003 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.984545+0000", 2023-08-28T02:26:01.003 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.204e-06 2023-08-28T02:26:01.004 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.004 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.004 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:01.004 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.984553+0000", 2023-08-28T02:26:01.004 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0026792869999999998 2023-08-28T02:26:01.004 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.004 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.004 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:01.005 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.987232+0000", 2023-08-28T02:26:01.005 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2547e-05 2023-08-28T02:26:01.005 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.006 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.006 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:26:01.006 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.987244+0000", 2023-08-28T02:26:01.006 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.98630143800000003 2023-08-28T02:26:01.006 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.006 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.006 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:01.006 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.973546+0000", 2023-08-28T02:26:01.006 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.5401000000000004e-05 2023-08-28T02:26:01.006 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.006 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.006 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:01.007 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.973641+0000", 2023-08-28T02:26:01.007 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012306799999999999 2023-08-28T02:26:01.007 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.007 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.007 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,7", 2023-08-28T02:26:01.007 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.973764+0000", 2023-08-28T02:26:01.007 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.010567417000000001 2023-08-28T02:26:01.007 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.007 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.007 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:26:01.007 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.984332+0000", 2023-08-28T02:26:01.007 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.0612999999999999e-05 2023-08-28T02:26:01.007 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.008 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.008 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:01.008 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.984372+0000", 2023-08-28T02:26:01.008 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0066857319999999998 2023-08-28T02:26:01.008 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.008 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.008 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:01.008 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.991058+0000", 2023-08-28T02:26:01.008 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.2064000000000001e-05 2023-08-28T02:26:01.008 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.008 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.008 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:26:01.008 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.991080+0000", 2023-08-28T02:26:01.009 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.3366999999999999e-05 2023-08-28T02:26:01.009 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.009 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.009 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:26:01.009 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.991134+0000", 2023-08-28T02:26:01.009 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.006606983 2023-08-28T02:26:01.009 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:01.009 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:01.009 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:01.009 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.009 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.009 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:114352 5.1 5:89a4ee05:::benchmark_data_smithi143_236200_object4209:head [delete] snapc 0=[] ondisk+write+known_if_redirected e148)", 2023-08-28T02:26:01.009 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:43.989451+0000", 2023-08-28T02:26:01.010 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 136.95915038000001, 2023-08-28T02:26:01.010 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0017969799999999, 2023-08-28T02:26:01.010 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:01.010 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:01.010 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:26:01.010 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:26:01.010 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:01.010 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 114352 2023-08-28T02:26:01.010 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.010 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:01.010 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.010 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:01.010 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.989451+0000", 2023-08-28T02:26:01.011 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:01.011 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.011 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.011 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:01.011 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.989451+0000", 2023-08-28T02:26:01.011 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:26:01.011 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.011 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.011 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:01.011 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.989450+0000", 2023-08-28T02:26:01.011 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.8460000000000002e-06 2023-08-28T02:26:01.011 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.011 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.012 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:01.012 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.989457+0000", 2023-08-28T02:26:01.012 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.8840000000000006e-06 2023-08-28T02:26:01.012 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.012 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.012 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:01.012 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.989467+0000", 2023-08-28T02:26:01.012 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.9309999999999996e-06 2023-08-28T02:26:01.012 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.012 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.012 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:01.012 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.989473+0000", 2023-08-28T02:26:01.012 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.9379999999999998e-05 2023-08-28T02:26:01.013 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.013 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.013 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:01.013 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.989522+0000", 2023-08-28T02:26:01.013 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.7560000000000005e-06 2023-08-28T02:26:01.013 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.013 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.013 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:26:01.013 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.989531+0000", 2023-08-28T02:26:01.013 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.98783754999999995 2023-08-28T02:26:01.013 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.013 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.013 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:01.014 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.977368+0000", 2023-08-28T02:26:01.014 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000131094 2023-08-28T02:26:01.014 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.014 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.014 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:01.014 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.977499+0000", 2023-08-28T02:26:01.014 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00014025499999999999 2023-08-28T02:26:01.014 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.014 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.014 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,7", 2023-08-28T02:26:01.014 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.977640+0000", 2023-08-28T02:26:01.014 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0067410400000000002 2023-08-28T02:26:01.014 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.015 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.015 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:26:01.015 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.984381+0000", 2023-08-28T02:26:01.015 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0066601270000000001 2023-08-28T02:26:01.015 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.015 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.015 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:01.015 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.991041+0000", 2023-08-28T02:26:01.015 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00013248099999999999 2023-08-28T02:26:01.015 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.015 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.015 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:01.015 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.991173+0000", 2023-08-28T02:26:01.016 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.2561999999999998e-05 2023-08-28T02:26:01.016 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.016 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.016 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:26:01.016 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.991196+0000", 2023-08-28T02:26:01.016 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.2604e-05 2023-08-28T02:26:01.016 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.016 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.016 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:26:01.016 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.991248+0000", 2023-08-28T02:26:01.016 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0017969799999999 2023-08-28T02:26:01.016 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:01.016 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:01.017 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:01.017 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.017 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.017 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:114363 5.1 5:890fd397:::benchmark_data_smithi143_236200_object4220:head [delete] snapc 0=[] ondisk+write+known_if_redirected e148)", 2023-08-28T02:26:01.017 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:44.004622+0000", 2023-08-28T02:26:01.017 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 136.94397974699999, 2023-08-28T02:26:01.017 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.99759494800000004, 2023-08-28T02:26:01.017 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:01.017 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:01.017 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:26:01.017 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:26:01.017 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:01.018 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 114363 2023-08-28T02:26:01.018 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.018 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:01.018 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.018 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:01.018 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.004622+0000", 2023-08-28T02:26:01.018 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:01.018 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.018 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.018 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:01.018 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.004622+0000", 2023-08-28T02:26:01.018 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:26:01.018 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.019 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.019 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:01.019 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.004621+0000", 2023-08-28T02:26:01.019 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.4139999999999997e-06 2023-08-28T02:26:01.019 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.019 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.019 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:01.019 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.004628+0000", 2023-08-28T02:26:01.019 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.3670000000000003e-06 2023-08-28T02:26:01.019 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.019 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.019 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:01.019 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.004637+0000", 2023-08-28T02:26:01.020 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.3399999999999997e-06 2023-08-28T02:26:01.020 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.020 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.020 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:01.020 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.004643+0000", 2023-08-28T02:26:01.020 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.3288999999999998e-05 2023-08-28T02:26:01.020 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.020 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.020 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:01.020 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.004696+0000", 2023-08-28T02:26:01.020 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2778e-05 2023-08-28T02:26:01.020 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.020 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.021 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:26:01.021 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.004709+0000", 2023-08-28T02:26:01.021 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.97640097000000003 2023-08-28T02:26:01.021 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.021 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.021 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:01.021 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.981110+0000", 2023-08-28T02:26:01.021 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00014084300000000001 2023-08-28T02:26:01.021 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.021 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.021 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:01.021 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.981251+0000", 2023-08-28T02:26:01.021 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.3012999999999999e-05 2023-08-28T02:26:01.022 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.022 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.022 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,7", 2023-08-28T02:26:01.022 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.981344+0000", 2023-08-28T02:26:01.022 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.009928203 2023-08-28T02:26:01.022 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.022 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.022 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:01.022 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.991272+0000", 2023-08-28T02:26:01.022 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0073894390000000003 2023-08-28T02:26:01.022 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.022 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.022 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:01.023 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.998661+0000", 2023-08-28T02:26:01.023 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0033621649999999999 2023-08-28T02:26:01.023 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.023 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.023 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:26:01.023 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.002023+0000", 2023-08-28T02:26:01.023 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.8695e-05 2023-08-28T02:26:01.023 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.023 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.023 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:26:01.023 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.002082+0000", 2023-08-28T02:26:01.023 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00013499700000000001 2023-08-28T02:26:01.023 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.024 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.024 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:26:01.024 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.002217+0000", 2023-08-28T02:26:01.024 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.99759494800000004 2023-08-28T02:26:01.024 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:01.024 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:01.024 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:01.024 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.024 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.024 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:114367 5.1 5:8b417cad:::benchmark_data_smithi143_236200_object4224:head [delete] snapc 0=[] ondisk+write+known_if_redirected e148)", 2023-08-28T02:26:01.024 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:44.007552+0000", 2023-08-28T02:26:01.024 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 136.94104966399999, 2023-08-28T02:26:01.025 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.99493006100000003, 2023-08-28T02:26:01.025 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:01.025 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:01.025 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:26:01.025 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:26:01.025 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:01.025 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 114367 2023-08-28T02:26:01.025 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.025 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:01.025 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.025 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:01.025 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.007552+0000", 2023-08-28T02:26:01.025 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:01.026 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.026 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.026 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:01.026 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.007552+0000", 2023-08-28T02:26:01.026 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:26:01.026 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.026 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.026 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:01.026 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.007551+0000", 2023-08-28T02:26:01.026 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.8379999999999996e-06 2023-08-28T02:26:01.026 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.026 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.027 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:01.027 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.007557+0000", 2023-08-28T02:26:01.027 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.4807000000000001e-05 2023-08-28T02:26:01.027 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.027 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.027 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:01.027 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.007572+0000", 2023-08-28T02:26:01.027 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.2530000000000001e-06 2023-08-28T02:26:01.027 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.027 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.027 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:01.027 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.007578+0000", 2023-08-28T02:26:01.027 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.3174000000000002e-05 2023-08-28T02:26:01.028 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.028 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.028 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:01.028 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.007642+0000", 2023-08-28T02:26:01.028 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1406000000000001e-05 2023-08-28T02:26:01.028 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.028 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.028 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:26:01.028 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.007653+0000", 2023-08-28T02:26:01.028 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.97676323499999995 2023-08-28T02:26:01.028 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.029 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.029 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:01.029 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.984416+0000", 2023-08-28T02:26:01.029 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012243200000000001 2023-08-28T02:26:01.029 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.029 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.029 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:01.029 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.984539+0000", 2023-08-28T02:26:01.029 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012780699999999999 2023-08-28T02:26:01.029 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.029 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.029 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,7", 2023-08-28T02:26:01.029 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.984667+0000", 2023-08-28T02:26:01.030 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.014060197999999999 2023-08-28T02:26:01.030 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.030 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.030 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:01.030 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.998727+0000", 2023-08-28T02:26:01.030 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.003498091 2023-08-28T02:26:01.030 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.030 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.030 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:26:01.030 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.002225+0000", 2023-08-28T02:26:01.030 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.8292e-05 2023-08-28T02:26:01.030 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.030 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.031 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:01.031 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.002323+0000", 2023-08-28T02:26:01.031 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.0098e-05 2023-08-28T02:26:01.031 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.031 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.031 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:26:01.031 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.002373+0000", 2023-08-28T02:26:01.031 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000109016 2023-08-28T02:26:01.031 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.031 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.031 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:26:01.031 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.002482+0000", 2023-08-28T02:26:01.032 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.99493006100000003 2023-08-28T02:26:01.032 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:01.032 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:01.032 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:01.032 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.032 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.032 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:114373 5.1 5:8d61f17b:::benchmark_data_smithi143_236200_object4230:head [delete] snapc 0=[] ondisk+write+known_if_redirected e148)", 2023-08-28T02:26:01.032 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:44.020572+0000", 2023-08-28T02:26:01.032 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 136.928030221, 2023-08-28T02:26:01.032 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.98890386600000002, 2023-08-28T02:26:01.032 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:01.032 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:01.033 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:26:01.033 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:26:01.033 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:01.033 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 114373 2023-08-28T02:26:01.033 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.033 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:01.033 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.033 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:01.033 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.020572+0000", 2023-08-28T02:26:01.033 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:01.033 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.033 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.033 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:01.034 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.020572+0000", 2023-08-28T02:26:01.034 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:26:01.034 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.034 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.034 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:01.034 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.020570+0000", 2023-08-28T02:26:01.034 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0434999999999999e-05 2023-08-28T02:26:01.034 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.034 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.034 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:01.034 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.020580+0000", 2023-08-28T02:26:01.034 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.1787e-05 2023-08-28T02:26:01.034 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.035 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.035 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:01.035 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.020602+0000", 2023-08-28T02:26:01.035 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.2979999999999992e-06 2023-08-28T02:26:01.035 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.035 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.035 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:01.035 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.020610+0000", 2023-08-28T02:26:01.035 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.1524000000000002e-05 2023-08-28T02:26:01.035 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.035 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.035 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:01.036 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.020682+0000", 2023-08-28T02:26:01.036 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1493e-05 2023-08-28T02:26:01.036 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.036 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.036 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:26:01.036 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.020693+0000", 2023-08-28T02:26:01.036 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.96700611599999997 2023-08-28T02:26:01.036 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.036 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.036 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:01.036 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.987699+0000", 2023-08-28T02:26:01.036 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000114647 2023-08-28T02:26:01.036 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.037 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.037 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:01.037 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.987814+0000", 2023-08-28T02:26:01.037 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.221e-05 2023-08-28T02:26:01.037 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.037 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.037 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,7", 2023-08-28T02:26:01.037 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.987906+0000", 2023-08-28T02:26:01.037 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.014324356999999999 2023-08-28T02:26:01.037 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.037 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.037 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:26:01.038 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.002231+0000", 2023-08-28T02:26:01.038 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.9576000000000001e-05 2023-08-28T02:26:01.038 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.038 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.038 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:01.038 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.002280+0000", 2023-08-28T02:26:01.038 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0070311599999999998 2023-08-28T02:26:01.038 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.038 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.038 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:01.038 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.009311+0000", 2023-08-28T02:26:01.038 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.0016999999999999e-05 2023-08-28T02:26:01.038 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.039 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.039 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:26:01.039 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.009361+0000", 2023-08-28T02:26:01.039 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000114168 2023-08-28T02:26:01.039 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.039 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.039 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:26:01.039 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.009475+0000", 2023-08-28T02:26:01.039 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.98890386600000002 2023-08-28T02:26:01.039 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:01.039 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:01.039 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:01.039 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.040 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.040 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:114383 5.1 5:81a4d21f:::benchmark_data_smithi143_236200_object4240:head [delete] snapc 0=[] ondisk+write+known_if_redirected e148)", 2023-08-28T02:26:01.040 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:44.031002+0000", 2023-08-28T02:26:01.040 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 136.917599594, 2023-08-28T02:26:01.040 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.97863153999999997, 2023-08-28T02:26:01.040 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:01.040 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:01.040 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:26:01.040 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:26:01.040 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:01.040 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 114383 2023-08-28T02:26:01.040 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.041 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:01.041 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.041 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:01.041 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.031002+0000", 2023-08-28T02:26:01.041 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:01.041 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.041 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.041 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:01.041 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.031002+0000", 2023-08-28T02:26:01.041 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999995 2023-08-28T02:26:01.041 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.041 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.041 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:01.041 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.031002+0000", 2023-08-28T02:26:01.042 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.5300000000000001e-06 2023-08-28T02:26:01.042 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.042 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.042 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:01.042 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.031005+0000", 2023-08-28T02:26:01.042 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.1270000000000002e-06 2023-08-28T02:26:01.042 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.042 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.042 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:01.042 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.031010+0000", 2023-08-28T02:26:01.042 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.0199999999999999e-06 2023-08-28T02:26:01.042 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.043 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.043 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:01.043 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.031013+0000", 2023-08-28T02:26:01.043 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.9960000000000003e-05 2023-08-28T02:26:01.043 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.043 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.043 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:01.043 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.031063+0000", 2023-08-28T02:26:01.043 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.8900000000000002e-06 2023-08-28T02:26:01.043 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.043 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.043 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:26:01.044 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.031073+0000", 2023-08-28T02:26:01.044 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.96021787599999997 2023-08-28T02:26:01.044 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.044 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.044 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:01.044 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.991291+0000", 2023-08-28T02:26:01.044 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000106072 2023-08-28T02:26:01.044 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.044 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.044 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:01.044 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.991397+0000", 2023-08-28T02:26:01.044 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000111848 2023-08-28T02:26:01.044 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.045 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.045 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,7", 2023-08-28T02:26:01.045 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.991509+0000", 2023-08-28T02:26:01.045 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.010727258 2023-08-28T02:26:01.045 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.045 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.045 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:26:01.045 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.002236+0000", 2023-08-28T02:26:01.045 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00029985 2023-08-28T02:26:01.045 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.045 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.045 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:01.045 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.002536+0000", 2023-08-28T02:26:01.046 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.006984929 2023-08-28T02:26:01.046 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.046 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.046 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:01.046 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.009521+0000", 2023-08-28T02:26:01.046 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.3491000000000001e-05 2023-08-28T02:26:01.046 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.046 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.046 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:26:01.046 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.009564+0000", 2023-08-28T02:26:01.046 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.9277999999999996e-05 2023-08-28T02:26:01.046 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.046 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.047 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:26:01.047 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.009634+0000", 2023-08-28T02:26:01.047 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.97863153999999997 2023-08-28T02:26:01.047 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:01.047 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:01.047 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:01.047 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.047 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.047 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:114398 5.1 5:8f550161:::benchmark_data_smithi143_236200_object4255:head [delete] snapc 0=[] ondisk+write+known_if_redirected e148)", 2023-08-28T02:26:01.047 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:44.053922+0000", 2023-08-28T02:26:01.047 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 136.894679414, 2023-08-28T02:26:01.047 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.96977096500000004, 2023-08-28T02:26:01.048 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:01.048 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:01.048 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:26:01.048 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:26:01.048 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:01.048 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 114398 2023-08-28T02:26:01.048 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.048 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:01.048 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.048 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:01.048 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.053922+0000", 2023-08-28T02:26:01.048 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:01.049 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.049 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.049 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:01.049 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.053922+0000", 2023-08-28T02:26:01.049 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:26:01.049 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.049 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.049 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:01.049 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.053921+0000", 2023-08-28T02:26:01.049 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.3129999999999999e-06 2023-08-28T02:26:01.049 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.049 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.049 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:01.050 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.053929+0000", 2023-08-28T02:26:01.050 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.738e-05 2023-08-28T02:26:01.050 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.050 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.050 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:01.050 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.053947+0000", 2023-08-28T02:26:01.050 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.8860000000000004e-06 2023-08-28T02:26:01.050 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.050 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.050 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:01.050 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.053955+0000", 2023-08-28T02:26:01.050 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.8510999999999997e-05 2023-08-28T02:26:01.050 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.051 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.051 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:01.051 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.054003+0000", 2023-08-28T02:26:01.051 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.252e-05 2023-08-28T02:26:01.051 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.051 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.051 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:26:01.051 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.054016+0000", 2023-08-28T02:26:01.051 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.94065110100000005 2023-08-28T02:26:01.051 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.051 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.051 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:01.051 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.994667+0000", 2023-08-28T02:26:01.052 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000114616 2023-08-28T02:26:01.052 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.052 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.052 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:01.052 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.994781+0000", 2023-08-28T02:26:01.052 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00013167800000000001 2023-08-28T02:26:01.052 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.052 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.052 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,7", 2023-08-28T02:26:01.052 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.994913+0000", 2023-08-28T02:26:01.052 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0073757720000000001 2023-08-28T02:26:01.052 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.052 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.053 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:26:01.053 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.002289+0000", 2023-08-28T02:26:01.053 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0069236580000000001 2023-08-28T02:26:01.053 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.053 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.053 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:01.053 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.009213+0000", 2023-08-28T02:26:01.053 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.014330872999999999 2023-08-28T02:26:01.053 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.053 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.053 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:01.053 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.023543+0000", 2023-08-28T02:26:01.054 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.5627000000000002e-05 2023-08-28T02:26:01.054 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.054 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.054 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:26:01.054 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.023589+0000", 2023-08-28T02:26:01.054 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000104381 2023-08-28T02:26:01.054 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.054 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.054 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:26:01.054 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.023693+0000", 2023-08-28T02:26:01.054 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.96977096500000004 2023-08-28T02:26:01.054 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:01.054 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:01.055 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:01.055 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.055 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.055 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:114399 5.1 5:821d8672:::benchmark_data_smithi143_236200_object4256:head [delete] snapc 0=[] ondisk+write+known_if_redirected e148)", 2023-08-28T02:26:01.055 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:44.054511+0000", 2023-08-28T02:26:01.055 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 136.89409061699999, 2023-08-28T02:26:01.055 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.96934294700000001, 2023-08-28T02:26:01.055 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:01.055 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:01.055 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:26:01.055 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:26:01.055 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:01.055 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 114399 2023-08-28T02:26:01.056 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.056 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:01.056 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.056 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:01.056 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.054511+0000", 2023-08-28T02:26:01.056 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:01.056 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.056 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.056 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:01.056 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.054511+0000", 2023-08-28T02:26:01.056 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:26:01.056 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.056 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.057 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:01.057 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.054510+0000", 2023-08-28T02:26:01.057 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.6059999999999996e-06 2023-08-28T02:26:01.057 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.057 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.057 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:01.057 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.054515+0000", 2023-08-28T02:26:01.057 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.6590000000000002e-06 2023-08-28T02:26:01.057 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.057 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.057 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:01.057 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.054525+0000", 2023-08-28T02:26:01.057 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.8189999999999997e-06 2023-08-28T02:26:01.058 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.058 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.058 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:01.058 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.054531+0000", 2023-08-28T02:26:01.058 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.8594999999999999e-05 2023-08-28T02:26:01.058 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.058 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.058 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:01.058 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.054569+0000", 2023-08-28T02:26:01.058 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.7869999999999996e-06 2023-08-28T02:26:01.058 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.058 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.058 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:26:01.059 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.054579+0000", 2023-08-28T02:26:01.059 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.94418591900000004 2023-08-28T02:26:01.059 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.059 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.059 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:01.059 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.998765+0000", 2023-08-28T02:26:01.059 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000119363 2023-08-28T02:26:01.059 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.059 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.059 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:01.059 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.998884+0000", 2023-08-28T02:26:01.059 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000133683 2023-08-28T02:26:01.060 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.060 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.060 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,7", 2023-08-28T02:26:01.060 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.999018+0000", 2023-08-28T02:26:01.060 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.010199753000000001 2023-08-28T02:26:01.060 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.060 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.060 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:26:01.060 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.009218+0000", 2023-08-28T02:26:01.060 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00044989900000000002 2023-08-28T02:26:01.060 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.060 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.060 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:01.061 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.009668+0000", 2023-08-28T02:26:01.061 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.014060081 2023-08-28T02:26:01.061 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.061 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.061 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:01.061 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.023728+0000", 2023-08-28T02:26:01.061 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.6545000000000003e-05 2023-08-28T02:26:01.061 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.061 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.061 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:26:01.061 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.023764+0000", 2023-08-28T02:26:01.061 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.9993000000000004e-05 2023-08-28T02:26:01.061 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.062 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.062 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:26:01.062 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.023854+0000", 2023-08-28T02:26:01.062 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.96934294700000001 2023-08-28T02:26:01.062 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:01.062 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:01.062 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:01.062 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.062 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.062 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:114406 5.1 5:8f9a1570:::benchmark_data_smithi143_236200_object4263:head [delete] snapc 0=[] ondisk+write+known_if_redirected e148)", 2023-08-28T02:26:01.062 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:44.071870+0000", 2023-08-28T02:26:01.062 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 136.87673185099999, 2023-08-28T02:26:01.063 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.95214170300000001, 2023-08-28T02:26:01.063 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:01.063 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:01.063 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:26:01.063 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:26:01.063 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:01.063 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 114406 2023-08-28T02:26:01.063 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.063 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:01.063 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.063 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:01.063 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.071870+0000", 2023-08-28T02:26:01.063 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:01.064 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.064 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.064 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:01.064 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.071870+0000", 2023-08-28T02:26:01.064 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:26:01.064 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.064 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.064 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:01.064 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.071869+0000", 2023-08-28T02:26:01.064 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.541e-06 2023-08-28T02:26:01.064 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.064 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.064 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:01.065 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.071874+0000", 2023-08-28T02:26:01.065 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1824999999999999e-05 2023-08-28T02:26:01.065 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.065 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.065 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:01.065 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.071886+0000", 2023-08-28T02:26:01.065 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.9069999999999998e-06 2023-08-28T02:26:01.065 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.065 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.065 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:01.065 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.071891+0000", 2023-08-28T02:26:01.065 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.7457999999999998e-05 2023-08-28T02:26:01.065 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.066 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.066 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:01.066 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.071948+0000", 2023-08-28T02:26:01.066 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1953000000000001e-05 2023-08-28T02:26:01.066 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.066 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.066 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:26:01.066 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.071960+0000", 2023-08-28T02:26:01.066 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.93060808500000003 2023-08-28T02:26:01.066 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.066 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.066 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:01.066 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.002568+0000", 2023-08-28T02:26:01.067 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000125628 2023-08-28T02:26:01.067 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.067 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.067 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:01.067 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.002694+0000", 2023-08-28T02:26:01.067 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00013300900000000001 2023-08-28T02:26:01.067 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.067 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.067 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,7", 2023-08-28T02:26:01.067 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.002827+0000", 2023-08-28T02:26:01.067 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0064272879999999998 2023-08-28T02:26:01.067 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.068 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.068 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:26:01.068 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.009254+0000", 2023-08-28T02:26:01.068 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0036058919999999999 2023-08-28T02:26:01.068 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.068 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.068 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:01.068 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.012860+0000", 2023-08-28T02:26:01.068 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.011026055 2023-08-28T02:26:01.068 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.068 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.068 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:01.069 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.023886+0000", 2023-08-28T02:26:01.069 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.5678e-05 2023-08-28T02:26:01.069 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.069 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.069 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:26:01.069 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.023922+0000", 2023-08-28T02:26:01.069 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.9739000000000003e-05 2023-08-28T02:26:01.069 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.069 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.069 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:26:01.069 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.024012+0000", 2023-08-28T02:26:01.069 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.95214170300000001 2023-08-28T02:26:01.069 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:01.070 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:01.070 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:01.070 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.070 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.070 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:114414 5.1 5:82f807f5:::benchmark_data_smithi143_236200_object4271:head [delete] snapc 0=[] ondisk+write+known_if_redirected e148)", 2023-08-28T02:26:01.070 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:44.107189+0000", 2023-08-28T02:26:01.070 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 136.841413211, 2023-08-28T02:26:01.070 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.91698741100000003, 2023-08-28T02:26:01.070 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:01.070 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:01.070 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:26:01.070 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:26:01.071 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:01.071 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 114414 2023-08-28T02:26:01.071 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.071 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:01.071 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.071 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:01.071 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.107189+0000", 2023-08-28T02:26:01.071 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:01.071 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.071 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.071 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:01.071 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.107189+0000", 2023-08-28T02:26:01.071 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:26:01.072 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.072 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.072 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:01.072 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.107187+0000", 2023-08-28T02:26:01.072 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.9719999999999997e-06 2023-08-28T02:26:01.072 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.072 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.072 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:01.072 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.107193+0000", 2023-08-28T02:26:01.072 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2877000000000001e-05 2023-08-28T02:26:01.072 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.072 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.072 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:01.073 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.107206+0000", 2023-08-28T02:26:01.073 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.6389999999999997e-06 2023-08-28T02:26:01.073 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.073 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.073 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:01.073 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.107212+0000", 2023-08-28T02:26:01.073 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.7942000000000003e-05 2023-08-28T02:26:01.073 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.073 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.073 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:01.073 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.107260+0000", 2023-08-28T02:26:01.073 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.8300000000000008e-06 2023-08-28T02:26:01.073 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.074 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.074 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:26:01.074 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.107270+0000", 2023-08-28T02:26:01.074 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.89864702200000002 2023-08-28T02:26:01.074 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.074 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.074 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:01.074 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.005917+0000", 2023-08-28T02:26:01.074 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00014625200000000001 2023-08-28T02:26:01.074 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.074 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.074 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:01.074 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.006063+0000", 2023-08-28T02:26:01.075 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000161736 2023-08-28T02:26:01.075 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.075 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.075 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,7", 2023-08-28T02:26:01.075 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.006225+0000", 2023-08-28T02:26:01.075 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.003415029 2023-08-28T02:26:01.075 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.075 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.075 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:26:01.075 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.009640+0000", 2023-08-28T02:26:01.075 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.013842596 2023-08-28T02:26:01.075 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.075 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.076 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:01.076 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.023482+0000", 2023-08-28T02:26:01.076 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00056224300000000003 2023-08-28T02:26:01.076 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.076 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.076 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:01.076 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.024044+0000", 2023-08-28T02:26:01.076 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.5420999999999998e-05 2023-08-28T02:26:01.076 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.076 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.076 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:26:01.076 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.024080+0000", 2023-08-28T02:26:01.076 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.6132999999999996e-05 2023-08-28T02:26:01.077 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.077 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.077 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:26:01.077 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.024176+0000", 2023-08-28T02:26:01.077 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.91698741100000003 2023-08-28T02:26:01.077 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:01.077 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:01.077 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:01.077 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.077 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.077 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:123484 5.1 5:85bec03d:::benchmark_data_smithi143_236200_object13341:head [delete] snapc 0=[] ondisk+write+known_if_redirected e152)", 2023-08-28T02:26:01.077 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:58.166108+0000", 2023-08-28T02:26:01.078 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 122.78249413099999, 2023-08-28T02:26:01.078 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.1640762979999999, 2023-08-28T02:26:01.078 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:01.078 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "reached pg", 2023-08-28T02:26:01.078 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:26:01.078 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:26:01.078 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:01.078 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 123484 2023-08-28T02:26:01.078 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.078 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:01.078 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.078 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:01.079 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.166108+0000", 2023-08-28T02:26:01.079 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:01.079 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.079 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.079 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:01.079 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.166108+0000", 2023-08-28T02:26:01.079 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:26:01.079 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.079 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.079 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:01.079 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.166106+0000", 2023-08-28T02:26:01.079 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.661e-06 2023-08-28T02:26:01.080 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.080 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.080 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:01.080 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.166114+0000", 2023-08-28T02:26:01.080 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.537e-05 2023-08-28T02:26:01.080 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.080 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.080 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:01.080 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.166129+0000", 2023-08-28T02:26:01.080 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0088000000000001e-05 2023-08-28T02:26:01.080 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.080 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.080 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:01.081 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.166139+0000", 2023-08-28T02:26:01.081 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.003e-05 2023-08-28T02:26:01.081 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.081 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.081 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:01.081 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.166199+0000", 2023-08-28T02:26:01.081 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00014851600000000001 2023-08-28T02:26:01.081 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.081 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.081 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:01.081 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.166348+0000", 2023-08-28T02:26:01.081 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000186997 2023-08-28T02:26:01.081 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.082 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.082 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,7", 2023-08-28T02:26:01.082 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.166535+0000", 2023-08-28T02:26:01.082 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0066866920000000002 2023-08-28T02:26:01.082 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.082 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.082 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:26:01.082 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.173221+0000", 2023-08-28T02:26:01.082 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.1798000000000001e-05 2023-08-28T02:26:01.082 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.082 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.082 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:01.082 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.173273+0000", 2023-08-28T02:26:01.083 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.032931016 2023-08-28T02:26:01.083 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.083 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.083 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:01.083 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.206204+0000", 2023-08-28T02:26:01.083 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.2499999999999995e-06 2023-08-28T02:26:01.083 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.083 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.083 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:26:01.083 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.206214+0000", 2023-08-28T02:26:01.083 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.994048975 2023-08-28T02:26:01.083 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.084 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.084 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:01.084 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:59.200263+0000", 2023-08-28T02:26:01.084 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1748e-05 2023-08-28T02:26:01.084 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.084 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.084 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:26:01.084 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:59.200274+0000", 2023-08-28T02:26:01.084 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1298252950000001 2023-08-28T02:26:01.084 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.084 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.084 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:01.084 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:00.330100+0000", 2023-08-28T02:26:01.084 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.4437999999999994e-05 2023-08-28T02:26:01.085 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.085 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.085 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:26:01.085 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:00.330184+0000", 2023-08-28T02:26:01.085 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.1640762979999999 2023-08-28T02:26:01.085 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:01.085 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:01.085 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:01.085 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.085 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.085 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:123485 5.1 5:8d110a8f:::benchmark_data_smithi143_236200_object13342:head [delete] snapc 0=[] ondisk+write+known_if_redirected e152)", 2023-08-28T02:26:01.085 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:58.166177+0000", 2023-08-28T02:26:01.086 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 122.78242451, 2023-08-28T02:26:01.086 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.1643501280000002, 2023-08-28T02:26:01.086 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:01.086 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "reached pg", 2023-08-28T02:26:01.086 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:26:01.086 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:26:01.086 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:01.086 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 123485 2023-08-28T02:26:01.086 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.086 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:01.086 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.086 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:01.086 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.166177+0000", 2023-08-28T02:26:01.087 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:01.087 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.087 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.087 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:01.087 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.166177+0000", 2023-08-28T02:26:01.087 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:26:01.087 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.087 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.087 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:01.087 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.166177+0000", 2023-08-28T02:26:01.087 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.5520000000000005e-06 2023-08-28T02:26:01.087 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.087 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.088 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:01.088 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.166185+0000", 2023-08-28T02:26:01.088 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.7098000000000001e-05 2023-08-28T02:26:01.088 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.088 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.088 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:01.088 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.166202+0000", 2023-08-28T02:26:01.088 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.2490000000000001e-06 2023-08-28T02:26:01.088 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.088 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.088 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:01.088 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.166207+0000", 2023-08-28T02:26:01.088 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0040287659999999996 2023-08-28T02:26:01.089 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.089 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.089 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:01.089 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.170236+0000", 2023-08-28T02:26:01.089 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012299300000000001 2023-08-28T02:26:01.089 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.089 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.089 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:01.089 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.170359+0000", 2023-08-28T02:26:01.089 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000106042 2023-08-28T02:26:01.089 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.089 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.089 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,7", 2023-08-28T02:26:01.090 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.170465+0000", 2023-08-28T02:26:01.090 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0029179420000000002 2023-08-28T02:26:01.090 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.090 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.090 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:26:01.090 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.173383+0000", 2023-08-28T02:26:01.090 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00065289099999999995 2023-08-28T02:26:01.090 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.090 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.090 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:01.090 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.174036+0000", 2023-08-28T02:26:01.090 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.032203163999999999 2023-08-28T02:26:01.090 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.091 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.091 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:01.091 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.206239+0000", 2023-08-28T02:26:01.091 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.2450000000000001e-06 2023-08-28T02:26:01.091 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.091 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.091 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:26:01.091 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.206246+0000", 2023-08-28T02:26:01.091 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.99406602799999999 2023-08-28T02:26:01.091 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.091 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.091 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:01.092 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:59.200313+0000", 2023-08-28T02:26:01.092 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.7479999999999993e-06 2023-08-28T02:26:01.092 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.092 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.092 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:26:01.092 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:59.200320+0000", 2023-08-28T02:26:01.092 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.130151983 2023-08-28T02:26:01.092 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.092 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.092 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:01.092 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:00.330472+0000", 2023-08-28T02:26:01.092 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.5208999999999999e-05 2023-08-28T02:26:01.092 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.093 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.093 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:26:01.093 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:00.330527+0000", 2023-08-28T02:26:01.093 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.1643501280000002 2023-08-28T02:26:01.093 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:01.093 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:01.093 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:01.093 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.093 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.093 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:123528 5.1 5:826b0b9b:::benchmark_data_smithi143_236200_object13385:head [delete] snapc 0=[] ondisk+write+known_if_redirected e153)", 2023-08-28T02:26:01.093 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:58.314486+0000", 2023-08-28T02:26:01.093 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 122.63411605, 2023-08-28T02:26:01.094 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.0278054889999999, 2023-08-28T02:26:01.094 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:01.094 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:01.094 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:26:01.094 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:26:01.094 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:01.094 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 123528 2023-08-28T02:26:01.094 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.094 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:01.094 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.094 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:01.094 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.314486+0000", 2023-08-28T02:26:01.094 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:01.095 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.095 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.095 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:01.095 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.314486+0000", 2023-08-28T02:26:01.095 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:26:01.095 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.095 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.095 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:01.095 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.314484+0000", 2023-08-28T02:26:01.095 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.5689999999999999e-06 2023-08-28T02:26:01.095 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.095 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.095 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:01.095 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.314492+0000", 2023-08-28T02:26:01.096 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2577e-05 2023-08-28T02:26:01.096 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.096 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.096 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:01.096 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.314504+0000", 2023-08-28T02:26:01.096 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.9720000000000003e-06 2023-08-28T02:26:01.096 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.096 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.096 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:01.096 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.314511+0000", 2023-08-28T02:26:01.096 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.4861000000000001e-05 2023-08-28T02:26:01.096 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.097 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.097 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:01.097 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.314576+0000", 2023-08-28T02:26:01.097 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1195e-05 2023-08-28T02:26:01.097 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.097 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.097 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:26:01.097 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.314587+0000", 2023-08-28T02:26:01.097 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.88650818399999998 2023-08-28T02:26:01.097 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.097 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.097 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:01.097 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:59.201095+0000", 2023-08-28T02:26:01.098 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.064e-06 2023-08-28T02:26:01.098 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.098 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.098 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:26:01.098 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:59.201105+0000", 2023-08-28T02:26:01.098 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.129516108 2023-08-28T02:26:01.098 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.098 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.098 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:01.098 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:00.330621+0000", 2023-08-28T02:26:01.098 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000122156 2023-08-28T02:26:01.098 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.099 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.099 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:01.099 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:00.330743+0000", 2023-08-28T02:26:01.099 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00014396 2023-08-28T02:26:01.099 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.099 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.099 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,3,7", 2023-08-28T02:26:01.099 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:00.330887+0000", 2023-08-28T02:26:01.099 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0064385120000000004 2023-08-28T02:26:01.099 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.099 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.099 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:26:01.100 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:00.337325+0000", 2023-08-28T02:26:01.100 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00024257599999999999 2023-08-28T02:26:01.100 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.100 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.100 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:01.100 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:00.337568+0000", 2023-08-28T02:26:01.100 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00046788599999999999 2023-08-28T02:26:01.100 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.100 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.100 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:01.100 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:00.338036+0000", 2023-08-28T02:26:01.100 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0041302170000000003 2023-08-28T02:26:01.100 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.101 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.101 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:01.101 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:00.342166+0000", 2023-08-28T02:26:01.101 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.6479e-05 2023-08-28T02:26:01.101 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.101 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.101 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:26:01.101 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:00.342212+0000", 2023-08-28T02:26:01.101 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.8874000000000004e-05 2023-08-28T02:26:01.101 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.101 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.101 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:26:01.101 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:00.342291+0000", 2023-08-28T02:26:01.102 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.0278054889999999 2023-08-28T02:26:01.102 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:01.102 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:01.102 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:01.102 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.102 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.102 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_repop(client.44594.0:124092 5.9 e155/144 5:9a708815:::benchmark_data_smithi143_236200_object13949:head v 155'7684, mlcod=143'6802)", 2023-08-28T02:26:01.102 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:24:01.189691+0000", 2023-08-28T02:26:01.102 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 119.758911234, 2023-08-28T02:26:01.102 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.22055387700000001, 2023-08-28T02:26:01.102 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:01.102 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:01.102 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:01.103 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.103 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:01.103 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:01.189691+0000", 2023-08-28T02:26:01.103 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:01.103 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.103 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.103 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:01.103 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:01.189691+0000", 2023-08-28T02:26:01.103 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:26:01.103 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.103 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.103 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:01.103 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:01.189689+0000", 2023-08-28T02:26:01.104 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2235999999999999e-05 2023-08-28T02:26:01.104 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.104 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.104 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:01.104 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:01.189702+0000", 2023-08-28T02:26:01.104 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.057121234999999999 2023-08-28T02:26:01.104 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.104 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.104 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:01.104 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:01.246823+0000", 2023-08-28T02:26:01.104 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1090999999999999e-05 2023-08-28T02:26:01.104 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.104 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.105 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:01.105 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:01.246834+0000", 2023-08-28T02:26:01.105 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.9728999999999997e-05 2023-08-28T02:26:01.105 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.105 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.105 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:01.105 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:01.246894+0000", 2023-08-28T02:26:01.105 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.9166000000000002e-05 2023-08-28T02:26:01.105 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.105 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.105 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:01.105 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:01.246923+0000", 2023-08-28T02:26:01.105 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.16329518100000001 2023-08-28T02:26:01.106 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.106 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.106 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:26:01.106 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:01.410218+0000", 2023-08-28T02:26:01.106 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.6517999999999999e-05 2023-08-28T02:26:01.106 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:01.106 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:01.106 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:26:01.106 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:01.410244+0000", 2023-08-28T02:26:01.106 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.22055387700000001 2023-08-28T02:26:01.106 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:01.106 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:01.106 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:01.106 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:01.107 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:01.107 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:26:01.107 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:26:01.168 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:01.338 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:26:01.376 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:26:01.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:01.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:26:01.398 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is set 2023-08-28T02:26:01.422 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:01 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3723815664' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:26:01.422 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:01 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 1984/73251 objects degraded (2.708%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:26:01.422 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:01 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3723815664' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:26:01.422 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:01 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 1984/73251 objects degraded (2.708%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:26:01.486 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:26:01.584 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:01.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:01 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3723815664' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:26:01.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:01 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 1984/73251 objects degraded (2.708%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:26:01.822 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:26:01.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:01.830 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:26:01.868 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.5 dump_ops_in_flight 2023-08-28T02:26:01.931 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:26:02.002 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:02.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:26:01 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:26:01] "GET /metrics HTTP/1.1" 200 36489 "" "Prometheus/2.33.4" 2023-08-28T02:26:02.585 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:26:02.666 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:02 smithi143 ceph-mon[183059]: pgmap v406: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.5 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 1984/73251 objects degraded (2.708%); 4234/73251 objects misplaced (5.780%); 2.0 MiB/s, 32 objects/s recovering 2023-08-28T02:26:02.666 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:02 smithi143 ceph-mon[183059]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:26:02.667 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:02 smithi143 ceph-mon[183059]: failed to encode map e180 with expected crc 2023-08-28T02:26:02.667 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:02 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3723815664' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:26:02.667 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:02 smithi143 ceph-mon[183059]: osdmap e180: 8 total, 8 up, 8 in 2023-08-28T02:26:02.667 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:02 smithi143 ceph-mon[183059]: failed to encode map e180 with expected crc 2023-08-28T02:26:02.667 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:02 smithi143 ceph-mon[183059]: failed to encode map e180 with expected crc 2023-08-28T02:26:02.667 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:02 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:26:02.667 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:02 smithi143 ceph-mon[189871]: pgmap v406: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.5 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 1984/73251 objects degraded (2.708%); 4234/73251 objects misplaced (5.780%); 2.0 MiB/s, 32 objects/s recovering 2023-08-28T02:26:02.667 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:02 smithi143 ceph-mon[189871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:26:02.667 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:02 smithi143 ceph-mon[189871]: failed to encode map e180 with expected crc 2023-08-28T02:26:02.667 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:02 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3723815664' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:26:02.668 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:02 smithi143 ceph-mon[189871]: osdmap e180: 8 total, 8 up, 8 in 2023-08-28T02:26:02.668 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:02 smithi143 ceph-mon[189871]: failed to encode map e180 with expected crc 2023-08-28T02:26:02.668 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:02 smithi143 ceph-mon[189871]: failed to encode map e180 with expected crc 2023-08-28T02:26:02.668 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:02 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:26:02.729 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:26:02.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:02.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:26:02.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:02 smithi169 ceph-mon[159415]: pgmap v406: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.5 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 1984/73251 objects degraded (2.708%); 4234/73251 objects misplaced (5.780%); 2.0 MiB/s, 32 objects/s recovering 2023-08-28T02:26:02.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:02 smithi169 ceph-mon[159415]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:26:02.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:02 smithi169 ceph-mon[159415]: failed to encode map e180 with expected crc 2023-08-28T02:26:02.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:02 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3723815664' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:26:02.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:02 smithi169 ceph-mon[159415]: osdmap e180: 8 total, 8 up, 8 in 2023-08-28T02:26:02.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:02 smithi169 ceph-mon[159415]: failed to encode map e180 with expected crc 2023-08-28T02:26:02.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:02 smithi169 ceph-mon[159415]: failed to encode map e180 with expected crc 2023-08-28T02:26:02.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:02 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:26:02.835 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:26:02.932 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:03.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:03.263 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:26:03.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:26:03.364 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:26:03.373 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:26:03.402 INFO:teuthology.orchestra.run.smithi143.stderr:dumped all 2023-08-28T02:26:03.427 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:03.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:03 smithi143 ceph-mon[183059]: pgmap v408: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.4 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 1984/69531 objects degraded (2.853%); 4102/69531 objects misplaced (5.900%); 2.1 MiB/s, 32 objects/s recovering 2023-08-28T02:26:03.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:03 smithi143 ceph-mon[189871]: pgmap v408: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.4 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 1984/69531 objects degraded (2.853%); 4102/69531 objects misplaced (5.900%); 2.1 MiB/s, 32 objects/s recovering 2023-08-28T02:26:03.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:03 smithi169 ceph-mon[159415]: pgmap v408: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.4 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 1984/69531 objects degraded (2.853%); 4102/69531 objects misplaced (5.900%); 2.1 MiB/s, 32 objects/s recovering 2023-08-28T02:26:04.361 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:26:04.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:04.366 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:26:04.466 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:26:04.507 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:04.670 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset noscrub 2023-08-28T02:26:04.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:04 smithi143 ceph-mon[183059]: from='client.34959 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:26:04.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:04 smithi143 ceph-mon[189871]: from='client.34959 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:26:04.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:04 smithi169 ceph-mon[159415]: from='client.34959 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:26:04.863 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:26:04.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:04.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:26:04.974 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:26:05.038 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:05.067 INFO:teuthology.orchestra.run.smithi143.stdout:{"pg_ready":true,"pg_map":{"version":408,"stamp":"2023-08-28T02:26:03.155213+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1521817547,"num_objects":23177,"num_object_clones":0,"num_object_copies":69531,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1984,"num_objects_misplaced":4102,"num_objects_unfound":0,"num_objects_dirty":23177,"num_whiteouts":0,"num_read":6274,"num_read_kb":13707,"num_write":198239,"num_write_kb":7064102,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5746,"num_bytes_recovered":374745734,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139548,"ondisk_log_size":139548,"up":267,"acting":266,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":279,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6506864,"kb_used_data":4847792,"kb_used_omap":29,"kb_used_meta":1658978,"kb_avail":743454352,"statfs":{"total":767960285184,"available":761297256448,"internally_reserved":0,"allocated":4964139008,"data_stored":4958619820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30674,"internal_metadata":1698793518},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,4,3,6,7],"upper_bound":64},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-405733376,"num_objects":-6191,"num_object_clones":0,"num_object_copies":-18573,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":59,"num_objects_misplaced":-1079,"num_objects_unfound":0,"num_objects_dirty":-6191,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6191,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":326,"num_bytes_recovered":21364736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.921246"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":163,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.245963+0000","last_change":"2023-08-28T02:20:19.849807+0000","last_active":"2023-08-28T02:25:53.245963+0000","last_peered":"2023-08-28T02:25:53.245963+0000","last_clean":"2023-08-28T02:25:53.245963+0000","last_became_active":"2023-08-28T02:20:19.849464+0000","last_became_peered":"2023-08-28T02:20:19.849464+0000","last_unstale":"2023-08-28T02:25:53.245963+0000","last_undegraded":"2023-08-28T02:25:53.245963+0000","last_fullsized":"2023-08-28T02:25:53.245963+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":36,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.427532+0000","last_change":"2023-08-28T02:23:59.204957+0000","last_active":"2023-08-28T02:25:48.427532+0000","last_peered":"2023-08-28T02:25:48.427532+0000","last_clean":"2023-08-28T02:25:48.427532+0000","last_became_active":"2023-08-28T02:23:59.204717+0000","last_became_peered":"2023-08-28T02:23:59.204717+0000","last_unstale":"2023-08-28T02:25:48.427532+0000","last_undegraded":"2023-08-28T02:25:48.427532+0000","last_fullsized":"2023-08-28T02:25:48.427532+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:34:58.977269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":137,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.408447+0000","last_change":"2023-08-28T02:23:59.212721+0000","last_active":"2023-08-28T02:25:48.408447+0000","last_peered":"2023-08-28T02:25:48.408447+0000","last_clean":"2023-08-28T02:25:48.408447+0000","last_became_active":"2023-08-28T02:23:59.212446+0000","last_became_peered":"2023-08-28T02:23:59.212446+0000","last_unstale":"2023-08-28T02:25:48.408447+0000","last_undegraded":"2023-08-28T02:25:48.408447+0000","last_fullsized":"2023-08-28T02:25:48.408447+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:03:54.701399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":165,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.508558+0000","last_change":"2023-08-28T02:23:59.223634+0000","last_active":"2023-08-28T02:25:48.508558+0000","last_peered":"2023-08-28T02:25:48.508558+0000","last_clean":"2023-08-28T02:25:48.508558+0000","last_became_active":"2023-08-28T02:23:59.222704+0000","last_became_peered":"2023-08-28T02:23:59.222704+0000","last_unstale":"2023-08-28T02:25:48.508558+0000","last_undegraded":"2023-08-28T02:25:48.508558+0000","last_fullsized":"2023-08-28T02:25:48.508558+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:51:52.752361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":36,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.428204+0000","last_change":"2023-08-28T02:23:59.220459+0000","last_active":"2023-08-28T02:25:48.428204+0000","last_peered":"2023-08-28T02:25:48.428204+0000","last_clean":"2023-08-28T02:25:48.428204+0000","last_became_active":"2023-08-28T02:23:59.219274+0000","last_became_peered":"2023-08-28T02:23:59.219274+0000","last_unstale":"2023-08-28T02:25:48.428204+0000","last_undegraded":"2023-08-28T02:25:48.428204+0000","last_fullsized":"2023-08-28T02:25:48.428204+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:31:53.889311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":174,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.267216+0000","last_change":"2023-08-28T02:21:59.183009+0000","last_active":"2023-08-28T02:25:53.267216+0000","last_peered":"2023-08-28T02:25:53.267216+0000","last_clean":"2023-08-28T02:25:53.267216+0000","last_became_active":"2023-08-28T02:21:59.182621+0000","last_became_peered":"2023-08-28T02:21:59.182621+0000","last_unstale":"2023-08-28T02:25:53.267216+0000","last_undegraded":"2023-08-28T02:25:53.267216+0000","last_fullsized":"2023-08-28T02:25:53.267216+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":146,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.260805+0000","last_change":"2023-08-28T02:23:59.227592+0000","last_active":"2023-08-28T02:25:53.260805+0000","last_peered":"2023-08-28T02:25:53.260805+0000","last_clean":"2023-08-28T02:25:53.260805+0000","last_became_active":"2023-08-28T02:23:59.227342+0000","last_became_peered":"2023-08-28T02:23:59.227342+0000","last_unstale":"2023-08-28T02:25:53.260805+0000","last_undegraded":"2023-08-28T02:25:53.260805+0000","last_fullsized":"2023-08-28T02:25:53.260805+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:08:19.874396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":36,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.428123+0000","last_change":"2023-08-28T02:23:59.220634+0000","last_active":"2023-08-28T02:25:48.428123+0000","last_peered":"2023-08-28T02:25:48.428123+0000","last_clean":"2023-08-28T02:25:48.428123+0000","last_became_active":"2023-08-28T02:23:59.220177+0000","last_became_peered":"2023-08-28T02:23:59.220177+0000","last_unstale":"2023-08-28T02:25:48.428123+0000","last_undegraded":"2023-08-28T02:25:48.428123+0000","last_fullsized":"2023-08-28T02:25:48.428123+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:16:26.906064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":165,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.244676+0000","last_change":"2023-08-28T02:23:38.339581+0000","last_active":"2023-08-28T02:25:53.244676+0000","last_peered":"2023-08-28T02:25:53.244676+0000","last_clean":"2023-08-28T02:25:53.244676+0000","last_became_active":"2023-08-28T02:23:38.339244+0000","last_became_peered":"2023-08-28T02:23:38.339244+0000","last_unstale":"2023-08-28T02:25:53.244676+0000","last_undegraded":"2023-08-28T02:25:53.244676+0000","last_fullsized":"2023-08-28T02:25:53.244676+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":36,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.428855+0000","last_change":"2023-08-28T02:23:59.220551+0000","last_active":"2023-08-28T02:25:48.428855+0000","last_peered":"2023-08-28T02:25:48.428855+0000","last_clean":"2023-08-28T02:25:48.428855+0000","last_became_active":"2023-08-28T02:23:59.219681+0000","last_became_peered":"2023-08-28T02:23:59.219681+0000","last_unstale":"2023-08-28T02:25:48.428855+0000","last_undegraded":"2023-08-28T02:25:48.428855+0000","last_fullsized":"2023-08-28T02:25:48.428855+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:39:58.811256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":66,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.261879+0000","last_change":"2023-08-28T02:23:59.316561+0000","last_active":"2023-08-28T02:25:53.261879+0000","last_peered":"2023-08-28T02:25:53.261879+0000","last_clean":"2023-08-28T02:25:53.261879+0000","last_became_active":"2023-08-28T02:23:59.227433+0000","last_became_peered":"2023-08-28T02:23:59.227433+0000","last_unstale":"2023-08-28T02:25:53.261879+0000","last_undegraded":"2023-08-28T02:25:53.261879+0000","last_fullsized":"2023-08-28T02:25:53.261879+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":120,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.408535+0000","last_change":"2023-08-28T02:20:19.851616+0000","last_active":"2023-08-28T02:25:48.408535+0000","last_peered":"2023-08-28T02:25:48.408535+0000","last_clean":"2023-08-28T02:25:48.408535+0000","last_became_active":"2023-08-28T02:20:19.851322+0000","last_became_peered":"2023-08-28T02:20:19.851322+0000","last_unstale":"2023-08-28T02:25:48.408535+0000","last_undegraded":"2023-08-28T02:25:48.408535+0000","last_fullsized":"2023-08-28T02:25:48.408535+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:03:37.565646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"86'22","reported_seq":192,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.246110+0000","last_change":"2023-08-28T02:23:38.368275+0000","last_active":"2023-08-28T02:25:53.246110+0000","last_peered":"2023-08-28T02:25:53.246110+0000","last_clean":"2023-08-28T02:25:53.246110+0000","last_became_active":"2023-08-28T02:23:38.342097+0000","last_became_peered":"2023-08-28T02:23:38.342097+0000","last_unstale":"2023-08-28T02:25:53.246110+0000","last_undegraded":"2023-08-28T02:25:53.246110+0000","last_fullsized":"2023-08-28T02:25:53.246110+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":160,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.245818+0000","last_change":"2023-08-28T02:19:51.204118+0000","last_active":"2023-08-28T02:25:53.245818+0000","last_peered":"2023-08-28T02:25:53.245818+0000","last_clean":"2023-08-28T02:25:53.245818+0000","last_became_active":"2023-08-28T02:19:51.203745+0000","last_became_peered":"2023-08-28T02:19:51.203745+0000","last_unstale":"2023-08-28T02:25:53.245818+0000","last_undegraded":"2023-08-28T02:25:53.245818+0000","last_fullsized":"2023-08-28T02:25:53.245818+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":51,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.256468+0000","last_change":"2023-08-28T02:23:38.983634+0000","last_active":"2023-08-28T02:25:53.256468+0000","last_peered":"2023-08-28T02:25:53.256468+0000","last_clean":"2023-08-28T02:25:53.256468+0000","last_became_active":"2023-08-28T02:23:38.344566+0000","last_became_peered":"2023-08-28T02:23:38.344566+0000","last_unstale":"2023-08-28T02:25:53.256468+0000","last_undegraded":"2023-08-28T02:25:53.256468+0000","last_fullsized":"2023-08-28T02:25:53.256468+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":36,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.427572+0000","last_change":"2023-08-28T02:23:59.216960+0000","last_active":"2023-08-28T02:25:48.427572+0000","last_peered":"2023-08-28T02:25:48.427572+0000","last_clean":"2023-08-28T02:25:48.427572+0000","last_became_active":"2023-08-28T02:23:59.215657+0000","last_became_peered":"2023-08-28T02:23:59.215657+0000","last_unstale":"2023-08-28T02:25:48.427572+0000","last_undegraded":"2023-08-28T02:25:48.427572+0000","last_fullsized":"2023-08-28T02:25:48.427572+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:31:14.407108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":164,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.508424+0000","last_change":"2023-08-28T02:23:59.223066+0000","last_active":"2023-08-28T02:25:48.508424+0000","last_peered":"2023-08-28T02:25:48.508424+0000","last_clean":"2023-08-28T02:25:48.508424+0000","last_became_active":"2023-08-28T02:23:59.222731+0000","last_became_peered":"2023-08-28T02:23:59.222731+0000","last_unstale":"2023-08-28T02:25:48.508424+0000","last_undegraded":"2023-08-28T02:25:48.508424+0000","last_fullsized":"2023-08-28T02:25:48.508424+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:48:37.558266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":207,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.244867+0000","last_change":"2023-08-28T02:23:59.210469+0000","last_active":"2023-08-28T02:25:53.244867+0000","last_peered":"2023-08-28T02:25:53.244867+0000","last_clean":"2023-08-28T02:25:53.244867+0000","last_became_active":"2023-08-28T02:23:59.210031+0000","last_became_peered":"2023-08-28T02:23:59.210031+0000","last_unstale":"2023-08-28T02:25:53.244867+0000","last_undegraded":"2023-08-28T02:25:53.244867+0000","last_fullsized":"2023-08-28T02:25:53.244867+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":45,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.253780+0000","last_change":"2023-08-28T02:23:38.370183+0000","last_active":"2023-08-28T02:25:53.253780+0000","last_peered":"2023-08-28T02:25:53.253780+0000","last_clean":"2023-08-28T02:25:53.253780+0000","last_became_active":"2023-08-28T02:23:38.368415+0000","last_became_peered":"2023-08-28T02:23:38.368415+0000","last_unstale":"2023-08-28T02:25:53.253780+0000","last_undegraded":"2023-08-28T02:25:53.253780+0000","last_fullsized":"2023-08-28T02:25:53.253780+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":165,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.260742+0000","last_change":"2023-08-28T02:23:59.220932+0000","last_active":"2023-08-28T02:25:53.260742+0000","last_peered":"2023-08-28T02:25:53.260742+0000","last_clean":"2023-08-28T02:25:53.260742+0000","last_became_active":"2023-08-28T02:23:59.220488+0000","last_became_peered":"2023-08-28T02:23:59.220488+0000","last_unstale":"2023-08-28T02:25:53.260742+0000","last_undegraded":"2023-08-28T02:25:53.260742+0000","last_fullsized":"2023-08-28T02:25:53.260742+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:53:09.216340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":154,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.263838+0000","last_change":"2023-08-28T02:18:39.778728+0000","last_active":"2023-08-28T02:25:53.263838+0000","last_peered":"2023-08-28T02:25:53.263838+0000","last_clean":"2023-08-28T02:25:53.263838+0000","last_became_active":"2023-08-28T02:18:39.778283+0000","last_became_peered":"2023-08-28T02:18:39.778283+0000","last_unstale":"2023-08-28T02:25:53.263838+0000","last_undegraded":"2023-08-28T02:25:53.263838+0000","last_fullsized":"2023-08-28T02:25:53.263838+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":36,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.427859+0000","last_change":"2023-08-28T02:23:59.216743+0000","last_active":"2023-08-28T02:25:48.427859+0000","last_peered":"2023-08-28T02:25:48.427859+0000","last_clean":"2023-08-28T02:25:48.427859+0000","last_became_active":"2023-08-28T02:23:59.216416+0000","last_became_peered":"2023-08-28T02:23:59.216416+0000","last_unstale":"2023-08-28T02:25:48.427859+0000","last_undegraded":"2023-08-28T02:25:48.427859+0000","last_fullsized":"2023-08-28T02:25:48.427859+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:25:13.056747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"180'12325","reported_seq":13159,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:02.015742+0000","last_change":"2023-08-28T02:24:09.355130+0000","last_active":"2023-08-28T02:26:02.015742+0000","last_peered":"2023-08-28T02:26:02.015742+0000","last_clean":"2023-08-28T02:26:02.015742+0000","last_became_active":"2023-08-28T02:24:09.341363+0000","last_became_peered":"2023-08-28T02:24:09.341363+0000","last_unstale":"2023-08-28T02:26:02.015742+0000","last_undegraded":"2023-08-28T02:26:02.015742+0000","last_fullsized":"2023-08-28T02:26:02.015742+0000","mapping_epoch":161,"log_start":"129'3700","ondisk_log_start":"129'3700","created":119,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":8625,"log_dups_size":0,"ondisk_log_size":8625,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90898432,"num_objects":1387,"num_object_clones":0,"num_object_copies":4161,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1387,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12325,"num_write_kb":438784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":367,"num_bytes_recovered":23789568,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":49,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.253312+0000","last_change":"2023-08-28T02:23:38.657185+0000","last_active":"2023-08-28T02:25:53.253312+0000","last_peered":"2023-08-28T02:25:53.253312+0000","last_clean":"2023-08-28T02:25:53.253312+0000","last_became_active":"2023-08-28T02:23:38.339016+0000","last_became_peered":"2023-08-28T02:23:38.339016+0000","last_unstale":"2023-08-28T02:25:53.253312+0000","last_undegraded":"2023-08-28T02:25:53.253312+0000","last_fullsized":"2023-08-28T02:25:53.253312+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":149,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.244242+0000","last_change":"2023-08-28T02:23:38.339886+0000","last_active":"2023-08-28T02:25:53.244242+0000","last_peered":"2023-08-28T02:25:53.244242+0000","last_clean":"2023-08-28T02:25:53.244242+0000","last_became_active":"2023-08-28T02:23:38.339514+0000","last_became_peered":"2023-08-28T02:23:38.339514+0000","last_unstale":"2023-08-28T02:25:53.244242+0000","last_undegraded":"2023-08-28T02:25:53.244242+0000","last_fullsized":"2023-08-28T02:25:53.244242+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"179'12317","reported_seq":12386,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:58.812349+0000","last_change":"2023-08-28T02:22:00.165260+0000","last_active":"2023-08-28T02:25:58.812349+0000","last_peered":"2023-08-28T02:25:58.812349+0000","last_clean":"2023-08-28T02:25:58.812349+0000","last_became_active":"2023-08-28T02:22:00.164978+0000","last_became_peered":"2023-08-28T02:22:00.164978+0000","last_unstale":"2023-08-28T02:25:58.812349+0000","last_undegraded":"2023-08-28T02:25:58.812349+0000","last_fullsized":"2023-08-28T02:25:58.812349+0000","mapping_epoch":119,"log_start":"128'3200","ondisk_log_start":"128'3200","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":9117,"log_dups_size":0,"ondisk_log_size":9117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:18:41.711960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98893824,"num_objects":1509,"num_object_clones":0,"num_object_copies":4527,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1509,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12317,"num_write_kb":442432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":49,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.253478+0000","last_change":"2023-08-28T02:23:38.555339+0000","last_active":"2023-08-28T02:25:53.253478+0000","last_peered":"2023-08-28T02:25:53.253478+0000","last_clean":"2023-08-28T02:25:53.253478+0000","last_became_active":"2023-08-28T02:23:38.337938+0000","last_became_peered":"2023-08-28T02:23:38.337938+0000","last_unstale":"2023-08-28T02:25:53.253478+0000","last_undegraded":"2023-08-28T02:25:53.253478+0000","last_fullsized":"2023-08-28T02:25:53.253478+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":188,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.246012+0000","last_change":"2023-08-28T02:23:59.208685+0000","last_active":"2023-08-28T02:25:53.246012+0000","last_peered":"2023-08-28T02:25:53.246012+0000","last_clean":"2023-08-28T02:25:53.246012+0000","last_became_active":"2023-08-28T02:23:59.208377+0000","last_became_peered":"2023-08-28T02:23:59.208377+0000","last_unstale":"2023-08-28T02:25:53.246012+0000","last_undegraded":"2023-08-28T02:25:53.246012+0000","last_fullsized":"2023-08-28T02:25:53.246012+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"179'12252","reported_seq":12321,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:58.817775+0000","last_change":"2023-08-28T02:22:00.165317+0000","last_active":"2023-08-28T02:25:58.817775+0000","last_peered":"2023-08-28T02:25:58.817775+0000","last_clean":"2023-08-28T02:25:58.817775+0000","last_became_active":"2023-08-28T02:22:00.164989+0000","last_became_peered":"2023-08-28T02:22:00.164989+0000","last_unstale":"2023-08-28T02:25:58.817775+0000","last_undegraded":"2023-08-28T02:25:58.817775+0000","last_fullsized":"2023-08-28T02:25:58.817775+0000","mapping_epoch":119,"log_start":"128'3100","ondisk_log_start":"128'3100","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":9152,"log_dups_size":0,"ondisk_log_size":9152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:10:46.322659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96206848,"num_objects":1468,"num_object_clones":0,"num_object_copies":4404,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1468,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12252,"num_write_kb":439040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":49,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.261650+0000","last_change":"2023-08-28T02:23:38.749428+0000","last_active":"2023-08-28T02:25:53.261650+0000","last_peered":"2023-08-28T02:25:53.261650+0000","last_clean":"2023-08-28T02:25:53.261650+0000","last_became_active":"2023-08-28T02:23:38.339587+0000","last_became_peered":"2023-08-28T02:23:38.339587+0000","last_unstale":"2023-08-28T02:25:53.261650+0000","last_undegraded":"2023-08-28T02:25:53.261650+0000","last_fullsized":"2023-08-28T02:25:53.261650+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":206,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.267053+0000","last_change":"2023-08-28T02:23:38.340065+0000","last_active":"2023-08-28T02:25:53.267053+0000","last_peered":"2023-08-28T02:25:53.267053+0000","last_clean":"2023-08-28T02:25:53.267053+0000","last_became_active":"2023-08-28T02:23:38.328689+0000","last_became_peered":"2023-08-28T02:23:38.328689+0000","last_unstale":"2023-08-28T02:25:53.267053+0000","last_undegraded":"2023-08-28T02:25:53.267053+0000","last_fullsized":"2023-08-28T02:25:53.267053+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"179'12306","reported_seq":17654,"reported_epoch":179,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-08-28T02:25:58.827957+0000","last_change":"2023-08-28T02:24:08.250989+0000","last_active":"2023-08-28T02:25:58.827957+0000","last_peered":"2023-08-28T02:25:58.827957+0000","last_clean":"2023-08-28T02:23:58.182539+0000","last_became_active":"2023-08-28T02:24:00.420432+0000","last_became_peered":"2023-08-28T02:24:00.420432+0000","last_unstale":"2023-08-28T02:25:58.827957+0000","last_undegraded":"2023-08-28T02:24:00.242280+0000","last_fullsized":"2023-08-28T02:24:00.199333+0000","mapping_epoch":154,"log_start":"128'3200","ondisk_log_start":"128'3200","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":9106,"log_dups_size":0,"ondisk_log_size":9106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:57:53.818041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95682560,"num_objects":1460,"num_object_clones":0,"num_object_copies":4380,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1984,"num_objects_misplaced":1457,"num_objects_unfound":0,"num_objects_dirty":1460,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12306,"num_write_kb":440512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2635,"num_bytes_recovered":172687360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4],"avail_no_missing":["2","0","4"],"object_location_counts":[{"shards":"0,2,4","objects":1457},{"shards":"2,4","objects":-1981}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":165,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.244818+0000","last_change":"2023-08-28T02:23:59.211466+0000","last_active":"2023-08-28T02:25:53.244818+0000","last_peered":"2023-08-28T02:25:53.244818+0000","last_clean":"2023-08-28T02:25:53.244818+0000","last_became_active":"2023-08-28T02:23:59.211175+0000","last_became_peered":"2023-08-28T02:23:59.211175+0000","last_unstale":"2023-08-28T02:25:53.244818+0000","last_undegraded":"2023-08-28T02:25:53.244818+0000","last_fullsized":"2023-08-28T02:25:53.244818+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":140,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.414737+0000","last_change":"2023-08-28T02:19:51.205245+0000","last_active":"2023-08-28T02:25:48.414737+0000","last_peered":"2023-08-28T02:25:48.414737+0000","last_clean":"2023-08-28T02:25:48.414737+0000","last_became_active":"2023-08-28T02:19:51.204896+0000","last_became_peered":"2023-08-28T02:19:51.204896+0000","last_unstale":"2023-08-28T02:25:48.414737+0000","last_undegraded":"2023-08-28T02:25:48.414737+0000","last_fullsized":"2023-08-28T02:25:48.414737+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:51:24.634702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"179'12321","reported_seq":13222,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:58.817709+0000","last_change":"2023-08-28T02:24:04.294764+0000","last_active":"2023-08-28T02:25:58.817709+0000","last_peered":"2023-08-28T02:25:58.817709+0000","last_clean":"2023-08-28T02:25:58.817709+0000","last_became_active":"2023-08-28T02:24:04.280269+0000","last_became_peered":"2023-08-28T02:24:04.280269+0000","last_unstale":"2023-08-28T02:25:58.817709+0000","last_undegraded":"2023-08-28T02:25:58.817709+0000","last_fullsized":"2023-08-28T02:25:58.817709+0000","mapping_epoch":157,"log_start":"128'3200","ondisk_log_start":"128'3200","created":119,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":9121,"log_dups_size":0,"ondisk_log_size":9121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:18:55.996939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101449752,"num_objects":1549,"num_object_clones":0,"num_object_copies":4647,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1549,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12321,"num_write_kb":443777,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":403,"num_bytes_recovered":26345496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":40,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.427259+0000","last_change":"2023-08-28T02:23:59.230059+0000","last_active":"2023-08-28T02:25:48.427259+0000","last_peered":"2023-08-28T02:25:48.427259+0000","last_clean":"2023-08-28T02:25:48.427259+0000","last_became_active":"2023-08-28T02:23:59.217783+0000","last_became_peered":"2023-08-28T02:23:59.217783+0000","last_unstale":"2023-08-28T02:25:48.427259+0000","last_undegraded":"2023-08-28T02:25:48.427259+0000","last_fullsized":"2023-08-28T02:25:48.427259+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:00:12.145395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":173,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.245984+0000","last_change":"2023-08-28T02:23:59.211286+0000","last_active":"2023-08-28T02:25:53.245984+0000","last_peered":"2023-08-28T02:25:53.245984+0000","last_clean":"2023-08-28T02:25:53.245984+0000","last_became_active":"2023-08-28T02:23:59.210942+0000","last_became_peered":"2023-08-28T02:23:59.210942+0000","last_unstale":"2023-08-28T02:25:53.245984+0000","last_undegraded":"2023-08-28T02:25:53.245984+0000","last_fullsized":"2023-08-28T02:25:53.245984+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"179'12452","reported_seq":12522,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:58.406699+0000","last_change":"2023-08-28T02:22:00.162375+0000","last_active":"2023-08-28T02:25:58.406699+0000","last_peered":"2023-08-28T02:25:58.406699+0000","last_clean":"2023-08-28T02:25:58.406699+0000","last_became_active":"2023-08-28T02:22:00.162190+0000","last_became_peered":"2023-08-28T02:22:00.162190+0000","last_unstale":"2023-08-28T02:25:58.406699+0000","last_undegraded":"2023-08-28T02:25:58.406699+0000","last_fullsized":"2023-08-28T02:25:58.406699+0000","mapping_epoch":119,"log_start":"130'4039","ondisk_log_start":"130'4039","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":8413,"log_dups_size":0,"ondisk_log_size":8413,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:40:08.985631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100401152,"num_objects":1532,"num_object_clones":0,"num_object_copies":4596,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1532,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12452,"num_write_kb":447488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":175,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.267219+0000","last_change":"2023-08-28T02:21:59.179699+0000","last_active":"2023-08-28T02:25:53.267219+0000","last_peered":"2023-08-28T02:25:53.267219+0000","last_clean":"2023-08-28T02:25:53.267219+0000","last_became_active":"2023-08-28T02:21:59.179345+0000","last_became_peered":"2023-08-28T02:21:59.179345+0000","last_unstale":"2023-08-28T02:25:53.267219+0000","last_undegraded":"2023-08-28T02:25:53.267219+0000","last_fullsized":"2023-08-28T02:25:53.267219+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":199,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.246242+0000","last_change":"2023-08-28T02:22:44.083121+0000","last_active":"2023-08-28T02:25:53.246242+0000","last_peered":"2023-08-28T02:25:53.246242+0000","last_clean":"2023-08-28T02:25:53.246242+0000","last_became_active":"2023-08-28T02:22:44.082776+0000","last_became_peered":"2023-08-28T02:22:44.082776+0000","last_unstale":"2023-08-28T02:25:53.246242+0000","last_undegraded":"2023-08-28T02:25:53.246242+0000","last_fullsized":"2023-08-28T02:25:53.246242+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"180'12407","reported_seq":12476,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:02.085057+0000","last_change":"2023-08-28T02:22:00.167630+0000","last_active":"2023-08-28T02:26:02.085057+0000","last_peered":"2023-08-28T02:26:02.085057+0000","last_clean":"2023-08-28T02:26:02.085057+0000","last_became_active":"2023-08-28T02:22:00.167305+0000","last_became_peered":"2023-08-28T02:22:00.167305+0000","last_unstale":"2023-08-28T02:26:02.085057+0000","last_undegraded":"2023-08-28T02:26:02.085057+0000","last_fullsized":"2023-08-28T02:26:02.085057+0000","mapping_epoch":119,"log_start":"128'3300","ondisk_log_start":"128'3300","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":9107,"log_dups_size":0,"ondisk_log_size":9107,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91160576,"num_objects":1391,"num_object_clones":0,"num_object_copies":4173,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1391,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12407,"num_write_kb":441536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":89,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.246165+0000","last_change":"2023-08-28T02:23:38.347043+0000","last_active":"2023-08-28T02:25:53.246165+0000","last_peered":"2023-08-28T02:25:53.246165+0000","last_clean":"2023-08-28T02:25:53.246165+0000","last_became_active":"2023-08-28T02:23:38.346680+0000","last_became_peered":"2023-08-28T02:23:38.346680+0000","last_unstale":"2023-08-28T02:25:53.246165+0000","last_undegraded":"2023-08-28T02:25:53.246165+0000","last_fullsized":"2023-08-28T02:25:53.246165+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":108,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.516414+0000","last_change":"2023-08-28T02:19:51.204017+0000","last_active":"2023-08-28T02:25:48.516414+0000","last_peered":"2023-08-28T02:25:48.516414+0000","last_clean":"2023-08-28T02:25:48.516414+0000","last_became_active":"2023-08-28T02:19:51.203627+0000","last_became_peered":"2023-08-28T02:19:51.203627+0000","last_unstale":"2023-08-28T02:25:48.516414+0000","last_undegraded":"2023-08-28T02:25:48.516414+0000","last_fullsized":"2023-08-28T02:25:48.516414+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:14:06.231701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.e","version":"180'12573","reported_seq":12642,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:02.076738+0000","last_change":"2023-08-28T02:22:00.166490+0000","last_active":"2023-08-28T02:26:02.076738+0000","last_peered":"2023-08-28T02:26:02.076738+0000","last_clean":"2023-08-28T02:26:02.076738+0000","last_became_active":"2023-08-28T02:22:00.166140+0000","last_became_peered":"2023-08-28T02:22:00.166140+0000","last_unstale":"2023-08-28T02:26:02.076738+0000","last_undegraded":"2023-08-28T02:26:02.076738+0000","last_fullsized":"2023-08-28T02:26:02.076738+0000","mapping_epoch":119,"log_start":"132'4600","ondisk_log_start":"132'4600","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":7973,"log_dups_size":0,"ondisk_log_size":7973,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91947008,"num_objects":1403,"num_object_clones":0,"num_object_copies":4209,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1403,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12573,"num_write_kb":447232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":36,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.427894+0000","last_change":"2023-08-28T02:23:59.216950+0000","last_active":"2023-08-28T02:25:48.427894+0000","last_peered":"2023-08-28T02:25:48.427894+0000","last_clean":"2023-08-28T02:25:48.427894+0000","last_became_active":"2023-08-28T02:23:59.216373+0000","last_became_peered":"2023-08-28T02:23:59.216373+0000","last_unstale":"2023-08-28T02:25:48.427894+0000","last_undegraded":"2023-08-28T02:25:48.427894+0000","last_fullsized":"2023-08-28T02:25:48.427894+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:13:14.736771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":156,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.411146+0000","last_change":"2023-08-28T02:23:59.220269+0000","last_active":"2023-08-28T02:25:48.411146+0000","last_peered":"2023-08-28T02:25:48.411146+0000","last_clean":"2023-08-28T02:25:48.411146+0000","last_became_active":"2023-08-28T02:23:59.219966+0000","last_became_peered":"2023-08-28T02:23:59.219966+0000","last_unstale":"2023-08-28T02:25:48.411146+0000","last_undegraded":"2023-08-28T02:25:48.411146+0000","last_fullsized":"2023-08-28T02:25:48.411146+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:16:08.119909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"78'126","reported_seq":655,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.408909+0000","last_change":"2023-08-28T02:23:39.351975+0000","last_active":"2023-08-28T02:25:48.408909+0000","last_peered":"2023-08-28T02:25:48.408909+0000","last_clean":"2023-08-28T02:25:48.408909+0000","last_became_active":"2023-08-28T02:23:39.351437+0000","last_became_peered":"2023-08-28T02:23:39.351437+0000","last_unstale":"2023-08-28T02:25:48.408909+0000","last_undegraded":"2023-08-28T02:25:48.408909+0000","last_fullsized":"2023-08-28T02:25:48.408909+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:25:22.203120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":743,"num_read_kb":1331,"num_write":125,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":180,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.244923+0000","last_change":"2023-08-28T02:23:59.207148+0000","last_active":"2023-08-28T02:25:53.244923+0000","last_peered":"2023-08-28T02:25:53.244923+0000","last_clean":"2023-08-28T02:25:53.244923+0000","last_became_active":"2023-08-28T02:23:59.206802+0000","last_became_peered":"2023-08-28T02:23:59.206802+0000","last_unstale":"2023-08-28T02:25:53.244923+0000","last_undegraded":"2023-08-28T02:25:53.244923+0000","last_fullsized":"2023-08-28T02:25:53.244923+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":76,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.241774+0000","last_change":"2023-08-28T02:21:43.238044+0000","last_active":"2023-08-28T02:25:53.241774+0000","last_peered":"2023-08-28T02:25:53.241774+0000","last_clean":"2023-08-28T02:25:53.241774+0000","last_became_active":"2023-08-28T02:21:43.237792+0000","last_became_peered":"2023-08-28T02:21:43.237792+0000","last_unstale":"2023-08-28T02:25:53.241774+0000","last_undegraded":"2023-08-28T02:25:53.241774+0000","last_fullsized":"2023-08-28T02:25:53.241774+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:11:35.488503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"179'12141","reported_seq":12211,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:58.529040+0000","last_change":"2023-08-28T02:22:00.166578+0000","last_active":"2023-08-28T02:25:58.529040+0000","last_peered":"2023-08-28T02:25:58.529040+0000","last_clean":"2023-08-28T02:25:58.529040+0000","last_became_active":"2023-08-28T02:22:00.166311+0000","last_became_peered":"2023-08-28T02:22:00.166311+0000","last_unstale":"2023-08-28T02:25:58.529040+0000","last_undegraded":"2023-08-28T02:25:58.529040+0000","last_fullsized":"2023-08-28T02:25:58.529040+0000","mapping_epoch":119,"log_start":"132'4441","ondisk_log_start":"132'4441","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":7700,"log_dups_size":0,"ondisk_log_size":7700,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:40:38.502180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98893824,"num_objects":1509,"num_object_clones":0,"num_object_copies":4527,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1509,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12141,"num_write_kb":436800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"179'12236","reported_seq":12306,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:58.528533+0000","last_change":"2023-08-28T02:22:00.160991+0000","last_active":"2023-08-28T02:25:58.528533+0000","last_peered":"2023-08-28T02:25:58.528533+0000","last_clean":"2023-08-28T02:25:58.528533+0000","last_became_active":"2023-08-28T02:22:00.160755+0000","last_became_peered":"2023-08-28T02:22:00.160755+0000","last_unstale":"2023-08-28T02:25:58.528533+0000","last_undegraded":"2023-08-28T02:25:58.528533+0000","last_fullsized":"2023-08-28T02:25:58.528533+0000","mapping_epoch":119,"log_start":"133'4498","ondisk_log_start":"133'4498","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":7738,"log_dups_size":0,"ondisk_log_size":7738,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:42:59.109288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98959360,"num_objects":1510,"num_object_clones":0,"num_object_copies":4530,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1510,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12236,"num_write_kb":439872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"78'204","reported_seq":1096,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.244508+0000","last_change":"2023-08-28T02:19:50.219949+0000","last_active":"2023-08-28T02:25:53.244508+0000","last_peered":"2023-08-28T02:25:53.244508+0000","last_clean":"2023-08-28T02:25:53.244508+0000","last_became_active":"2023-08-28T02:19:50.219356+0000","last_became_peered":"2023-08-28T02:19:50.219356+0000","last_unstale":"2023-08-28T02:25:53.244508+0000","last_undegraded":"2023-08-28T02:25:53.244508+0000","last_fullsized":"2023-08-28T02:25:53.244508+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":204,"log_dups_size":0,"ondisk_log_size":204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1521,"num_read_kb":2511,"num_write":178,"num_write_kb":1175,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":147,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.245972+0000","last_change":"2023-08-28T02:18:39.779607+0000","last_active":"2023-08-28T02:25:53.245972+0000","last_peered":"2023-08-28T02:25:53.245972+0000","last_clean":"2023-08-28T02:25:53.245972+0000","last_became_active":"2023-08-28T02:18:39.779303+0000","last_became_peered":"2023-08-28T02:18:39.779303+0000","last_unstale":"2023-08-28T02:25:53.245972+0000","last_undegraded":"2023-08-28T02:25:53.245972+0000","last_fullsized":"2023-08-28T02:25:53.245972+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":136,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.408837+0000","last_change":"2023-08-28T02:23:59.221979+0000","last_active":"2023-08-28T02:25:48.408837+0000","last_peered":"2023-08-28T02:25:48.408837+0000","last_clean":"2023-08-28T02:25:48.408837+0000","last_became_active":"2023-08-28T02:23:59.221619+0000","last_became_peered":"2023-08-28T02:23:59.221619+0000","last_unstale":"2023-08-28T02:25:48.408837+0000","last_undegraded":"2023-08-28T02:25:48.408837+0000","last_fullsized":"2023-08-28T02:25:48.408837+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:54:45.892284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.6","version":"179'12048","reported_seq":12831,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:58.797953+0000","last_change":"2023-08-28T02:24:07.218778+0000","last_active":"2023-08-28T02:25:58.797953+0000","last_peered":"2023-08-28T02:25:58.797953+0000","last_clean":"2023-08-28T02:25:58.797953+0000","last_became_active":"2023-08-28T02:24:07.205246+0000","last_became_peered":"2023-08-28T02:24:07.205246+0000","last_unstale":"2023-08-28T02:25:58.797953+0000","last_undegraded":"2023-08-28T02:25:58.797953+0000","last_fullsized":"2023-08-28T02:25:58.797953+0000","mapping_epoch":159,"log_start":"128'2900","ondisk_log_start":"128'2900","created":119,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":9148,"log_dups_size":0,"ondisk_log_size":9148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:30:08.510532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96206848,"num_objects":1468,"num_object_clones":0,"num_object_copies":4404,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1468,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12048,"num_write_kb":432512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":342,"num_bytes_recovered":22413312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"78'47","reported_seq":393,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.429189+0000","last_change":"2023-08-28T02:24:05.799647+0000","last_active":"2023-08-28T02:25:48.429189+0000","last_peered":"2023-08-28T02:25:48.429189+0000","last_clean":"2023-08-28T02:25:48.429189+0000","last_became_active":"2023-08-28T02:23:59.210481+0000","last_became_peered":"2023-08-28T02:23:59.210481+0000","last_unstale":"2023-08-28T02:25:48.429189+0000","last_undegraded":"2023-08-28T02:25:48.429189+0000","last_fullsized":"2023-08-28T02:25:48.429189+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":47,"log_dups_size":0,"ondisk_log_size":47,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:40:18.542375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":162,"num_read_kb":1281,"num_write":48,"num_write_kb":1153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":120,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.435421+0000","last_change":"2023-08-28T02:20:19.851591+0000","last_active":"2023-08-28T02:25:48.435421+0000","last_peered":"2023-08-28T02:25:48.435421+0000","last_clean":"2023-08-28T02:25:48.435421+0000","last_became_active":"2023-08-28T02:20:19.851291+0000","last_became_peered":"2023-08-28T02:20:19.851291+0000","last_unstale":"2023-08-28T02:25:48.435421+0000","last_undegraded":"2023-08-28T02:25:48.435421+0000","last_fullsized":"2023-08-28T02:25:48.435421+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:33:51.259707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":151,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.411064+0000","last_change":"2023-08-28T02:19:50.211156+0000","last_active":"2023-08-28T02:25:48.411064+0000","last_peered":"2023-08-28T02:25:48.411064+0000","last_clean":"2023-08-28T02:25:48.411064+0000","last_became_active":"2023-08-28T02:19:50.210398+0000","last_became_peered":"2023-08-28T02:19:50.210398+0000","last_unstale":"2023-08-28T02:25:48.411064+0000","last_undegraded":"2023-08-28T02:25:48.411064+0000","last_fullsized":"2023-08-28T02:25:48.411064+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:14:41.748197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"78'114","reported_seq":544,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.244902+0000","last_change":"2023-08-28T02:24:01.210739+0000","last_active":"2023-08-28T02:25:53.244902+0000","last_peered":"2023-08-28T02:25:53.244902+0000","last_clean":"2023-08-28T02:25:53.244902+0000","last_became_active":"2023-08-28T02:24:01.210386+0000","last_became_peered":"2023-08-28T02:24:01.210386+0000","last_unstale":"2023-08-28T02:25:53.244902+0000","last_undegraded":"2023-08-28T02:25:53.244902+0000","last_fullsized":"2023-08-28T02:25:53.244902+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":114,"log_dups_size":0,"ondisk_log_size":114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":358,"num_read_kb":1352,"num_write":102,"num_write_kb":1104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"86'139","reported_seq":338,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.253107+0000","last_change":"2023-08-28T02:24:04.276519+0000","last_active":"2023-08-28T02:25:53.253107+0000","last_peered":"2023-08-28T02:25:53.253107+0000","last_clean":"2023-08-28T02:25:53.253107+0000","last_became_active":"2023-08-28T02:24:04.276104+0000","last_became_peered":"2023-08-28T02:24:04.276104+0000","last_unstale":"2023-08-28T02:25:53.253107+0000","last_undegraded":"2023-08-28T02:25:53.253107+0000","last_fullsized":"2023-08-28T02:25:53.253107+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_clean_scrub_stamp":"2023-08-28T02:08:02.043718+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":36,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.428840+0000","last_change":"2023-08-28T02:23:59.219981+0000","last_active":"2023-08-28T02:25:48.428840+0000","last_peered":"2023-08-28T02:25:48.428840+0000","last_clean":"2023-08-28T02:25:48.428840+0000","last_became_active":"2023-08-28T02:23:59.219633+0000","last_became_peered":"2023-08-28T02:23:59.219633+0000","last_unstale":"2023-08-28T02:25:48.428840+0000","last_undegraded":"2023-08-28T02:25:48.428840+0000","last_fullsized":"2023-08-28T02:25:48.428840+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:56:14.172518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":148,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.267069+0000","last_change":"2023-08-28T02:23:38.341351+0000","last_active":"2023-08-28T02:25:53.267069+0000","last_peered":"2023-08-28T02:25:53.267069+0000","last_clean":"2023-08-28T02:25:53.267069+0000","last_became_active":"2023-08-28T02:23:38.340997+0000","last_became_peered":"2023-08-28T02:23:38.340997+0000","last_unstale":"2023-08-28T02:25:53.267069+0000","last_undegraded":"2023-08-28T02:25:53.267069+0000","last_fullsized":"2023-08-28T02:25:53.267069+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"180'12297","reported_seq":13134,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:02.033258+0000","last_change":"2023-08-28T02:23:47.215362+0000","last_active":"2023-08-28T02:26:02.033258+0000","last_peered":"2023-08-28T02:26:02.033258+0000","last_clean":"2023-08-28T02:26:02.033258+0000","last_became_active":"2023-08-28T02:23:47.207109+0000","last_became_peered":"2023-08-28T02:23:47.207109+0000","last_unstale":"2023-08-28T02:26:02.033258+0000","last_undegraded":"2023-08-28T02:26:02.033258+0000","last_fullsized":"2023-08-28T02:26:02.033258+0000","mapping_epoch":150,"log_start":"129'3700","ondisk_log_start":"129'3700","created":119,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":8597,"log_dups_size":0,"ondisk_log_size":8597,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89063424,"num_objects":1359,"num_object_clones":0,"num_object_copies":4077,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1359,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12297,"num_write_kb":436992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":370,"num_bytes_recovered":24182784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.7","version":"180'12482","reported_seq":12551,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:02.051898+0000","last_change":"2023-08-28T02:22:00.165614+0000","last_active":"2023-08-28T02:26:02.051898+0000","last_peered":"2023-08-28T02:26:02.051898+0000","last_clean":"2023-08-28T02:26:02.051898+0000","last_became_active":"2023-08-28T02:22:00.165136+0000","last_became_peered":"2023-08-28T02:22:00.165136+0000","last_unstale":"2023-08-28T02:26:02.051898+0000","last_undegraded":"2023-08-28T02:26:02.051898+0000","last_fullsized":"2023-08-28T02:26:02.051898+0000","mapping_epoch":119,"log_start":"128'3300","ondisk_log_start":"128'3300","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":9182,"log_dups_size":0,"ondisk_log_size":9182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89915392,"num_objects":1372,"num_object_clones":0,"num_object_copies":4116,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1372,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12482,"num_write_kb":443328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"78'148","reported_seq":629,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.253023+0000","last_change":"2023-08-28T02:24:01.211494+0000","last_active":"2023-08-28T02:25:53.253023+0000","last_peered":"2023-08-28T02:25:53.253023+0000","last_clean":"2023-08-28T02:25:53.253023+0000","last_became_active":"2023-08-28T02:24:01.210957+0000","last_became_peered":"2023-08-28T02:24:01.210957+0000","last_unstale":"2023-08-28T02:25:53.253023+0000","last_undegraded":"2023-08-28T02:25:53.253023+0000","last_fullsized":"2023-08-28T02:25:53.253023+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1092,"num_read_kb":1955,"num_write":127,"num_write_kb":5177,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":146,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.414463+0000","last_change":"2023-08-28T02:23:59.226986+0000","last_active":"2023-08-28T02:25:48.414463+0000","last_peered":"2023-08-28T02:25:48.414463+0000","last_clean":"2023-08-28T02:25:48.414463+0000","last_became_active":"2023-08-28T02:23:59.226529+0000","last_became_peered":"2023-08-28T02:23:59.226529+0000","last_unstale":"2023-08-28T02:25:48.414463+0000","last_undegraded":"2023-08-28T02:25:48.414463+0000","last_fullsized":"2023-08-28T02:25:48.414463+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:22:09.193619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":173,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.508256+0000","last_change":"2023-08-28T02:21:59.167858+0000","last_active":"2023-08-28T02:25:48.508256+0000","last_peered":"2023-08-28T02:25:48.508256+0000","last_clean":"2023-08-28T02:25:48.508256+0000","last_became_active":"2023-08-28T02:21:59.167436+0000","last_became_peered":"2023-08-28T02:21:59.167436+0000","last_unstale":"2023-08-28T02:25:48.508256+0000","last_undegraded":"2023-08-28T02:25:48.508256+0000","last_fullsized":"2023-08-28T02:25:48.508256+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:17:50.717282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"78'103","reported_seq":556,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.261728+0000","last_change":"2023-08-28T02:24:07.212128+0000","last_active":"2023-08-28T02:25:53.261728+0000","last_peered":"2023-08-28T02:25:53.261728+0000","last_clean":"2023-08-28T02:25:53.261728+0000","last_became_active":"2023-08-28T02:24:07.211562+0000","last_became_peered":"2023-08-28T02:24:07.211562+0000","last_unstale":"2023-08-28T02:25:53.261728+0000","last_undegraded":"2023-08-28T02:25:53.261728+0000","last_fullsized":"2023-08-28T02:25:53.261728+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":803,"num_read_kb":2080,"num_write":73,"num_write_kb":1244,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":119,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.409054+0000","last_change":"2023-08-28T02:20:19.850916+0000","last_active":"2023-08-28T02:25:48.409054+0000","last_peered":"2023-08-28T02:25:48.409054+0000","last_clean":"2023-08-28T02:25:48.409054+0000","last_became_active":"2023-08-28T02:20:19.850627+0000","last_became_peered":"2023-08-28T02:20:19.850627+0000","last_unstale":"2023-08-28T02:25:48.409054+0000","last_undegraded":"2023-08-28T02:25:48.409054+0000","last_fullsized":"2023-08-28T02:25:48.409054+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:19:14.346189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":132,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.414947+0000","last_change":"2023-08-28T02:23:38.329667+0000","last_active":"2023-08-28T02:25:48.414947+0000","last_peered":"2023-08-28T02:25:48.414947+0000","last_clean":"2023-08-28T02:25:48.414947+0000","last_became_active":"2023-08-28T02:23:38.329175+0000","last_became_peered":"2023-08-28T02:23:38.329175+0000","last_unstale":"2023-08-28T02:25:48.414947+0000","last_undegraded":"2023-08-28T02:25:48.414947+0000","last_fullsized":"2023-08-28T02:25:48.414947+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:21:34.815067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"180'12181","reported_seq":12249,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:02.039326+0000","last_change":"2023-08-28T02:22:00.167814+0000","last_active":"2023-08-28T02:26:02.039326+0000","last_peered":"2023-08-28T02:26:02.039326+0000","last_clean":"2023-08-28T02:26:02.039326+0000","last_became_active":"2023-08-28T02:22:00.167513+0000","last_became_peered":"2023-08-28T02:22:00.167513+0000","last_unstale":"2023-08-28T02:26:02.039326+0000","last_undegraded":"2023-08-28T02:26:02.039326+0000","last_fullsized":"2023-08-28T02:26:02.039326+0000","mapping_epoch":119,"log_start":"127'2500","ondisk_log_start":"127'2500","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":9681,"log_dups_size":0,"ondisk_log_size":9681,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90767360,"num_objects":1385,"num_object_clones":0,"num_object_copies":4155,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1385,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12181,"num_write_kb":434112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":40,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.427805+0000","last_change":"2023-08-28T02:23:59.232099+0000","last_active":"2023-08-28T02:25:48.427805+0000","last_peered":"2023-08-28T02:25:48.427805+0000","last_clean":"2023-08-28T02:25:48.427805+0000","last_became_active":"2023-08-28T02:23:59.219318+0000","last_became_peered":"2023-08-28T02:23:59.219318+0000","last_unstale":"2023-08-28T02:25:48.427805+0000","last_undegraded":"2023-08-28T02:25:48.427805+0000","last_fullsized":"2023-08-28T02:25:48.427805+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:59:30.093289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'112","reported_seq":423,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.409169+0000","last_change":"2023-08-28T02:19:50.218206+0000","last_active":"2023-08-28T02:25:48.409169+0000","last_peered":"2023-08-28T02:25:48.409169+0000","last_clean":"2023-08-28T02:25:48.409169+0000","last_became_active":"2023-08-28T02:19:50.217637+0000","last_became_peered":"2023-08-28T02:19:50.217637+0000","last_unstale":"2023-08-28T02:25:48.409169+0000","last_undegraded":"2023-08-28T02:25:48.409169+0000","last_fullsized":"2023-08-28T02:25:48.409169+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:12:06.224907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":444,"num_read_kb":473,"num_write":94,"num_write_kb":127,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":37,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.265920+0000","last_change":"2023-08-28T02:23:59.221034+0000","last_active":"2023-08-28T02:25:53.265920+0000","last_peered":"2023-08-28T02:25:53.265920+0000","last_clean":"2023-08-28T02:25:53.265920+0000","last_became_active":"2023-08-28T02:23:59.220730+0000","last_became_peered":"2023-08-28T02:23:59.220730+0000","last_unstale":"2023-08-28T02:25:53.265920+0000","last_undegraded":"2023-08-28T02:25:53.265920+0000","last_fullsized":"2023-08-28T02:25:53.265920+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:25:52.364370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"180'12465","reported_seq":15629,"reported_epoch":180,"state":"active+remapped+backfill_wait","last_fresh":"2023-08-28T02:26:02.084795+0000","last_change":"2023-08-28T02:24:05.805308+0000","last_active":"2023-08-28T02:26:02.084795+0000","last_peered":"2023-08-28T02:26:02.084795+0000","last_clean":"2023-08-28T02:23:25.043301+0000","last_became_active":"2023-08-28T02:24:00.316639+0000","last_became_peered":"2023-08-28T02:24:00.316639+0000","last_unstale":"2023-08-28T02:26:02.084795+0000","last_undegraded":"2023-08-28T02:26:02.084795+0000","last_fullsized":"2023-08-28T02:26:02.084795+0000","mapping_epoch":154,"log_start":"132'4500","ondisk_log_start":"132'4500","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":7965,"log_dups_size":0,"ondisk_log_size":7965,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93388800,"num_objects":1425,"num_object_clones":0,"num_object_copies":4275,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1223,"num_objects_unfound":0,"num_objects_dirty":1425,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12465,"num_write_kb":444480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1527,"num_bytes_recovered":99942400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,7,0],"avail_no_missing":["4","0","5","7"],"object_location_counts":[{"shards":"0,4,7","objects":1425}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"78'122","reported_seq":709,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.267125+0000","last_change":"2023-08-28T02:24:04.276850+0000","last_active":"2023-08-28T02:25:53.267125+0000","last_peered":"2023-08-28T02:25:53.267125+0000","last_clean":"2023-08-28T02:25:53.267125+0000","last_became_active":"2023-08-28T02:24:04.276367+0000","last_became_peered":"2023-08-28T02:24:04.276367+0000","last_unstale":"2023-08-28T02:25:53.267125+0000","last_undegraded":"2023-08-28T02:25:53.267125+0000","last_fullsized":"2023-08-28T02:25:53.267125+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":857,"num_read_kb":2224,"num_write":99,"num_write_kb":2312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":132,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.508482+0000","last_change":"2023-08-28T02:19:50.221001+0000","last_active":"2023-08-28T02:25:48.508482+0000","last_peered":"2023-08-28T02:25:48.508482+0000","last_clean":"2023-08-28T02:25:48.508482+0000","last_became_active":"2023-08-28T02:19:50.220611+0000","last_became_peered":"2023-08-28T02:19:50.220611+0000","last_unstale":"2023-08-28T02:25:48.508482+0000","last_undegraded":"2023-08-28T02:25:48.508482+0000","last_fullsized":"2023-08-28T02:25:48.508482+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:23:10.637630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":131,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.245871+0000","last_change":"2023-08-28T02:11:22.437146+0000","last_active":"2023-08-28T02:25:53.245871+0000","last_peered":"2023-08-28T02:25:53.245871+0000","last_clean":"2023-08-28T02:25:53.245871+0000","last_became_active":"2023-08-28T02:11:22.436798+0000","last_became_peered":"2023-08-28T02:11:22.436798+0000","last_unstale":"2023-08-28T02:25:53.245871+0000","last_undegraded":"2023-08-28T02:25:53.245871+0000","last_fullsized":"2023-08-28T02:25:53.245871+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"179'12330","reported_seq":12406,"reported_epoch":179,"state":"active+remapped+backfill_wait","last_fresh":"2023-08-28T02:25:58.566074+0000","last_change":"2023-08-28T02:24:00.310494+0000","last_active":"2023-08-28T02:25:58.566074+0000","last_peered":"2023-08-28T02:25:58.566074+0000","last_clean":"2023-08-28T02:23:58.174026+0000","last_became_active":"2023-08-28T02:24:00.290543+0000","last_became_peered":"2023-08-28T02:24:00.290543+0000","last_unstale":"2023-08-28T02:25:58.566074+0000","last_undegraded":"2023-08-28T02:25:58.566074+0000","last_fullsized":"2023-08-28T02:25:58.566074+0000","mapping_epoch":154,"log_start":"132'4577","ondisk_log_start":"132'4577","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":7753,"log_dups_size":0,"ondisk_log_size":7753,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:35:56.609960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93192192,"num_objects":1422,"num_object_clones":0,"num_object_copies":4266,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1422,"num_objects_unfound":0,"num_objects_dirty":1422,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12330,"num_write_kb":440064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[0,1,4],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":1422}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":134,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.414877+0000","last_change":"2023-08-28T02:23:38.329474+0000","last_active":"2023-08-28T02:25:48.414877+0000","last_peered":"2023-08-28T02:25:48.414877+0000","last_clean":"2023-08-28T02:25:48.414877+0000","last_became_active":"2023-08-28T02:23:38.328505+0000","last_became_peered":"2023-08-28T02:23:38.328505+0000","last_unstale":"2023-08-28T02:25:48.414877+0000","last_undegraded":"2023-08-28T02:25:48.414877+0000","last_fullsized":"2023-08-28T02:25:48.414877+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:12:14.835809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":200,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.430448+0000","last_change":"2023-08-28T02:23:59.212036+0000","last_active":"2023-08-28T02:25:48.430448+0000","last_peered":"2023-08-28T02:25:48.430448+0000","last_clean":"2023-08-28T02:25:48.430448+0000","last_became_active":"2023-08-28T02:23:59.211709+0000","last_became_peered":"2023-08-28T02:23:59.211709+0000","last_unstale":"2023-08-28T02:25:48.430448+0000","last_undegraded":"2023-08-28T02:25:48.430448+0000","last_fullsized":"2023-08-28T02:25:48.430448+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T14:04:25.046626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":149,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.244697+0000","last_change":"2023-08-28T02:19:50.218760+0000","last_active":"2023-08-28T02:25:53.244697+0000","last_peered":"2023-08-28T02:25:53.244697+0000","last_clean":"2023-08-28T02:25:53.244697+0000","last_became_active":"2023-08-28T02:19:50.218376+0000","last_became_peered":"2023-08-28T02:19:50.218376+0000","last_unstale":"2023-08-28T02:25:53.244697+0000","last_undegraded":"2023-08-28T02:25:53.244697+0000","last_fullsized":"2023-08-28T02:25:53.244697+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":179,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.246132+0000","last_change":"2023-08-28T02:23:59.207822+0000","last_active":"2023-08-28T02:25:53.246132+0000","last_peered":"2023-08-28T02:25:53.246132+0000","last_clean":"2023-08-28T02:25:53.246132+0000","last_became_active":"2023-08-28T02:23:59.207475+0000","last_became_peered":"2023-08-28T02:23:59.207475+0000","last_unstale":"2023-08-28T02:25:53.246132+0000","last_undegraded":"2023-08-28T02:25:53.246132+0000","last_fullsized":"2023-08-28T02:25:53.246132+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":134,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.245048+0000","last_change":"2023-08-28T02:11:22.186024+0000","last_active":"2023-08-28T02:25:53.245048+0000","last_peered":"2023-08-28T02:25:53.245048+0000","last_clean":"2023-08-28T02:25:53.245048+0000","last_became_active":"2023-08-28T02:11:22.185500+0000","last_became_peered":"2023-08-28T02:11:22.185500+0000","last_unstale":"2023-08-28T02:25:53.245048+0000","last_undegraded":"2023-08-28T02:25:53.245048+0000","last_fullsized":"2023-08-28T02:25:53.245048+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":179,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.246191+0000","last_change":"2023-08-28T02:23:59.207929+0000","last_active":"2023-08-28T02:25:53.246191+0000","last_peered":"2023-08-28T02:25:53.246191+0000","last_clean":"2023-08-28T02:25:53.246191+0000","last_became_active":"2023-08-28T02:23:59.207589+0000","last_became_peered":"2023-08-28T02:23:59.207589+0000","last_unstale":"2023-08-28T02:25:53.246191+0000","last_undegraded":"2023-08-28T02:25:53.246191+0000","last_fullsized":"2023-08-28T02:25:53.246191+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":37,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.265902+0000","last_change":"2023-08-28T02:23:59.220437+0000","last_active":"2023-08-28T02:25:53.265902+0000","last_peered":"2023-08-28T02:25:53.265902+0000","last_clean":"2023-08-28T02:25:53.265902+0000","last_became_active":"2023-08-28T02:23:59.220091+0000","last_became_peered":"2023-08-28T02:23:59.220091+0000","last_unstale":"2023-08-28T02:25:53.265902+0000","last_undegraded":"2023-08-28T02:25:53.265902+0000","last_fullsized":"2023-08-28T02:25:53.265902+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:51:21.695566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":36,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.428919+0000","last_change":"2023-08-28T02:23:59.217794+0000","last_active":"2023-08-28T02:25:48.428919+0000","last_peered":"2023-08-28T02:25:48.428919+0000","last_clean":"2023-08-28T02:25:48.428919+0000","last_became_active":"2023-08-28T02:23:59.217231+0000","last_became_peered":"2023-08-28T02:23:59.217231+0000","last_unstale":"2023-08-28T02:25:48.428919+0000","last_undegraded":"2023-08-28T02:25:48.428919+0000","last_fullsized":"2023-08-28T02:25:48.428919+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:36:26.297410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":132,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-08-28T02:25:48.513284+0000","last_change":"2023-08-28T02:20:19.850252+0000","last_active":"2023-08-28T02:25:48.513284+0000","last_peered":"2023-08-28T02:25:48.513284+0000","last_clean":"2023-08-28T02:25:48.513284+0000","last_became_active":"2023-08-28T02:20:19.849913+0000","last_became_peered":"2023-08-28T02:20:19.849913+0000","last_unstale":"2023-08-28T02:25:48.513284+0000","last_undegraded":"2023-08-28T02:25:48.513284+0000","last_fullsized":"2023-08-28T02:25:48.513284+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:48:47.713465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":175,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-08-28T02:25:53.245740+0000","last_change":"2023-08-28T02:21:59.180201+0000","last_active":"2023-08-28T02:25:53.245740+0000","last_peered":"2023-08-28T02:25:53.245740+0000","last_clean":"2023-08-28T02:25:53.245740+0000","last_became_active":"2023-08-28T02:21:59.179863+0000","last_became_peered":"2023-08-28T02:21:59.179863+0000","last_unstale":"2023-08-28T02:25:53.245740+0000","last_undegraded":"2023-08-28T02:25:53.245740+0000","last_fullsized":"2023-08-28T02:25:53.245740+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":1517027352,"num_objects":23149,"num_object_clones":0,"num_object_copies":69447,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1984,"num_objects_misplaced":4102,"num_objects_unfound":0,"num_objects_dirty":23149,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":197133,"num_write_kb":7048961,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5644,"num_bytes_recovered":369360920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4870189056,"data_stored":4870176840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":138378,"ondisk_log_size":138378,"up":48,"acting":47,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":9774,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":21246,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26810,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430405,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":767176,"kb_used_data":461760,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":92977976,"statfs":{"total":95995035648,"available":95209447424,"internally_reserved":0,"allocated":472842240,"data_stored":472259404,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:25:56 2023","interfaces":[{"interface":"back","average":{"1min":2.536,"5min":3.7240000000000002,"15min":1.5980000000000001},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":9.5679999999999996,"5min":116.755,"15min":116.755},"last":4.6550000000000002},{"interface":"front","average":{"1min":3.1320000000000001,"5min":4.1520000000000001,"15min":1.7889999999999999},"min":{"1min":0.28000000000000003,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":12.206,"5min":116.971,"15min":116.971},"last":0.58899999999999997}]},{"osd":1,"last update":"Mon Aug 28 02:25:02 2023","interfaces":[{"interface":"back","average":{"1min":1.395,"5min":0.93200000000000005,"15min":0.66200000000000003},"min":{"1min":0.28599999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":9.8710000000000004,"5min":9.8710000000000004,"15min":9.8710000000000004},"last":0.33500000000000002},{"interface":"front","average":{"1min":1.9670000000000001,"5min":1.3029999999999999,"15min":0.82599999999999996},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":11.696999999999999,"5min":11.696999999999999,"15min":11.696999999999999},"last":0.42699999999999999}]},{"osd":2,"last update":"Mon Aug 28 02:25:27 2023","interfaces":[{"interface":"back","average":{"1min":8.6959999999999997,"5min":3.8700000000000001,"15min":1.673},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":119.571,"5min":119.571,"15min":119.571},"last":0.30299999999999999},{"interface":"front","average":{"1min":9.7100000000000009,"5min":4.319,"15min":1.8300000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":119.554,"5min":119.554,"15min":119.554},"last":0.38400000000000001}]},{"osd":3,"last update":"Mon Aug 28 02:25:02 2023","interfaces":[{"interface":"back","average":{"1min":6.6299999999999999,"5min":2.0139999999999998,"15min":1.117},"min":{"1min":0.32500000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":107.916,"5min":107.916,"15min":107.916},"last":2.9359999999999999},{"interface":"front","average":{"1min":8.3409999999999993,"5min":2.6240000000000001,"15min":1.321},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":112.00700000000001,"5min":112.00700000000001,"15min":112.00700000000001},"last":0.5}]},{"osd":4,"last update":"Mon Aug 28 02:25:27 2023","interfaces":[{"interface":"back","average":{"1min":1.0980000000000001,"5min":2.0350000000000001,"15min":0.996},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":3.8359999999999999,"5min":63.048000000000002,"15min":63.048000000000002},"last":0.40899999999999997},{"interface":"front","average":{"1min":1.659,"5min":1.8759999999999999,"15min":0.96199999999999997},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":8.2720000000000002,"5min":62.912999999999997,"15min":62.912999999999997},"last":0.35299999999999998}]},{"osd":5,"last update":"Mon Aug 28 02:25:27 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":1.554,"15min":0.83999999999999997},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":5.7110000000000003,"5min":63.021000000000001,"15min":63.021000000000001},"last":1.0289999999999999},{"interface":"front","average":{"1min":1.4039999999999999,"5min":2.1779999999999999,"15min":1.089},"min":{"1min":0.35699999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":5.3470000000000004,"5min":63.082999999999998,"15min":63.082999999999998},"last":0.36299999999999999}]},{"osd":6,"last update":"Mon Aug 28 02:25:27 2023","interfaces":[{"interface":"back","average":{"1min":1.716,"5min":2.198,"15min":1.0860000000000001},"min":{"1min":0.317,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":6.0659999999999998,"5min":101.61,"15min":101.61},"last":6.6189999999999998},{"interface":"front","average":{"1min":1.3080000000000001,"5min":2.391,"15min":1.1890000000000001},"min":{"1min":0.34399999999999997,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":5.407,"5min":101.621,"15min":101.621},"last":0.53600000000000003}]}]},{"osd":6,"up_from":42,"seq":180388626632,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":658048,"kb_used_data":356728,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93087104,"statfs":{"total":95995035648,"available":95321194496,"internally_reserved":0,"allocated":365289472,"data_stored":364722409,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:25:00 2023","interfaces":[{"interface":"back","average":{"1min":12.085000000000001,"5min":3.8039999999999998,"15min":1.651},"min":{"1min":0.23899999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":121.247,"5min":121.247,"15min":121.247},"last":0.36699999999999999},{"interface":"front","average":{"1min":12.305999999999999,"5min":3.8540000000000001,"15min":1.6859999999999999},"min":{"1min":0.28799999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":118.977,"5min":118.977,"15min":118.977},"last":8.4469999999999992}]},{"osd":1,"last update":"Mon Aug 28 02:26:02 2023","interfaces":[{"interface":"back","average":{"1min":2.0910000000000002,"5min":2.258,"15min":1.1100000000000001},"min":{"1min":0.307,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":6.7050000000000001,"5min":34.677999999999997,"15min":34.677999999999997},"last":0.437},{"interface":"front","average":{"1min":2.7850000000000001,"5min":2.5009999999999999,"15min":1.2130000000000001},"min":{"1min":0.317,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":9.6799999999999997,"5min":34.348999999999997,"15min":34.348999999999997},"last":0.54900000000000004}]},{"osd":2,"last update":"Mon Aug 28 02:25:35 2023","interfaces":[{"interface":"back","average":{"1min":3.0590000000000002,"5min":2.2959999999999998,"15min":1.1719999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":15.762,"5min":49.555999999999997,"15min":49.555999999999997},"last":0.41399999999999998},{"interface":"front","average":{"1min":2.9710000000000001,"5min":2.1160000000000001,"15min":1.089},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":11.503,"5min":49.470999999999997,"15min":49.470999999999997},"last":1.5600000000000001}]},{"osd":3,"last update":"Mon Aug 28 02:26:02 2023","interfaces":[{"interface":"back","average":{"1min":2.2869999999999999,"5min":1.885,"15min":1.075},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":6.4699999999999998,"5min":13.103,"15min":13.103},"last":0.68700000000000006},{"interface":"front","average":{"1min":2.508,"5min":2.0649999999999999,"15min":1.143},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":11.395,"5min":13.09,"15min":13.09},"last":0.58099999999999996}]},{"osd":4,"last update":"Mon Aug 28 02:25:04 2023","interfaces":[{"interface":"back","average":{"1min":2.2719999999999998,"5min":1.4570000000000001,"15min":0.78800000000000003},"min":{"1min":0.28299999999999997,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":5.9619999999999997,"5min":10.637,"15min":10.637},"last":0.64900000000000002},{"interface":"front","average":{"1min":1.629,"5min":1.514,"15min":0.83199999999999996},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":5.1399999999999997,"5min":10.055,"15min":10.055},"last":0.48799999999999999}]},{"osd":5,"last update":"Mon Aug 28 02:25:04 2023","interfaces":[{"interface":"back","average":{"1min":1.1379999999999999,"5min":1.655,"15min":0.86099999999999999},"min":{"1min":0.33200000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.4429999999999996,"5min":20.759,"15min":20.759},"last":0.42699999999999999},{"interface":"front","average":{"1min":1.909,"5min":1.246,"15min":0.76100000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.28799999999999998},"max":{"1min":6.2110000000000003,"5min":7.3780000000000001,"15min":7.3780000000000001},"last":2.4329999999999998}]},{"osd":7,"last update":"Mon Aug 28 02:25:33 2023","interfaces":[{"interface":"back","average":{"1min":1.7949999999999999,"5min":1.3320000000000001,"15min":0.80700000000000005},"min":{"1min":0.34799999999999998,"5min":0.314,"15min":0.314},"max":{"1min":6.4109999999999996,"5min":7.1769999999999996,"15min":7.1769999999999996},"last":0.71899999999999997},{"interface":"front","average":{"1min":1.9379999999999999,"5min":1.49,"15min":0.879},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.29799999999999999},"max":{"1min":6.8949999999999996,"5min":7.069,"15min":7.069},"last":0.60999999999999999}]}]},{"osd":1,"up_from":98,"seq":420906795084,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":599368,"kb_used_data":491392,"kb_used_omap":7,"kb_used_meta":107960,"kb_avail":93145784,"statfs":{"total":95995035648,"available":95381282816,"internally_reserved":0,"allocated":503185408,"data_stored":502325194,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":110551371},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:24:59 2023","interfaces":[{"interface":"back","average":{"1min":3.0489999999999999,"5min":1.5309999999999999,"15min":0.73399999999999999},"min":{"1min":0.17499999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":16.356000000000002,"5min":16.356000000000002,"15min":16.356000000000002},"last":77.537000000000006},{"interface":"front","average":{"1min":3.6000000000000001,"5min":1.714,"15min":0.80700000000000005},"min":{"1min":0.22700000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":17.175000000000001,"5min":17.175000000000001,"15min":17.175000000000001},"last":78.063000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:25:27 2023","interfaces":[{"interface":"back","average":{"1min":5.5549999999999997,"5min":2.2530000000000001,"15min":0.97899999999999998},"min":{"1min":0.26200000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":29.483000000000001,"5min":29.483000000000001,"15min":29.483000000000001},"last":0.65200000000000002},{"interface":"front","average":{"1min":3.286,"5min":1.7470000000000001,"15min":0.83599999999999997},"min":{"1min":0.23200000000000001,"5min":0.221,"15min":0.221},"max":{"1min":19.045000000000002,"5min":19.045000000000002,"15min":19.045000000000002},"last":0.70699999999999996}]},{"osd":3,"last update":"Mon Aug 28 02:24:59 2023","interfaces":[{"interface":"back","average":{"1min":3.3719999999999999,"5min":1.4630000000000001,"15min":0.73099999999999998},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":15.395,"5min":15.395,"15min":15.395},"last":1.0920000000000001},{"interface":"front","average":{"1min":3.2250000000000001,"5min":1.4950000000000001,"15min":0.80500000000000005},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":14.536,"5min":14.536,"15min":14.536},"last":0.37}]},{"osd":4,"last update":"Mon Aug 28 02:24:59 2023","interfaces":[{"interface":"back","average":{"1min":2.8639999999999999,"5min":1.3879999999999999,"15min":0.88},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":10.323,"5min":10.323,"15min":10.323},"last":0.32800000000000001},{"interface":"front","average":{"1min":2.1579999999999999,"5min":1.1699999999999999,"15min":0.80300000000000005},"min":{"1min":0.32300000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":8.6679999999999993,"5min":8.6679999999999993,"15min":8.6679999999999993},"last":0.66600000000000004}]},{"osd":5,"last update":"Mon Aug 28 02:24:59 2023","interfaces":[{"interface":"back","average":{"1min":4.1289999999999996,"5min":1.786,"15min":1.02},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":15.544,"5min":15.544,"15min":15.544},"last":1.048},{"interface":"front","average":{"1min":2.2669999999999999,"5min":1.276,"15min":0.85199999999999998},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":10.282,"5min":10.282,"15min":10.282},"last":0.41799999999999998}]},{"osd":6,"last update":"Mon Aug 28 02:24:59 2023","interfaces":[{"interface":"back","average":{"1min":4.2300000000000004,"5min":2.0619999999999998,"15min":1.1000000000000001},"min":{"1min":0.36699999999999999,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":15.512,"5min":15.512,"15min":15.512},"last":1.1200000000000001},{"interface":"front","average":{"1min":2.98,"5min":1.7350000000000001,"15min":0.98699999999999999},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":10.35,"5min":13.707000000000001,"15min":13.707000000000001},"last":0.39600000000000002}]},{"osd":7,"last update":"Mon Aug 28 02:24:59 2023","interfaces":[{"interface":"back","average":{"1min":2.4630000000000001,"5min":1.3320000000000001,"15min":0.86599999999999999},"min":{"1min":0.34499999999999997,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":8.7240000000000002,"5min":8.7240000000000002,"15min":8.7240000000000002},"last":0.74099999999999999},{"interface":"front","average":{"1min":4.0149999999999997,"5min":1.792,"15min":1.022},"min":{"1min":0.46000000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":15.476000000000001,"5min":15.476000000000001,"15min":15.476000000000001},"last":1.1459999999999999}]}]},{"osd":0,"up_from":90,"seq":386547056730,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1560420,"kb_used_data":1376156,"kb_used_omap":8,"kb_used_meta":184247,"kb_avail":92184732,"statfs":{"total":95995035648,"available":94397165568,"internally_reserved":0,"allocated":1409183744,"data_stored":1408355311,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":188669289},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Aug 28 02:25:05 2023","interfaces":[{"interface":"back","average":{"1min":2.306,"5min":2.3210000000000002,"15min":1.0089999999999999},"min":{"1min":0.185,"5min":0.185,"15min":0.185},"max":{"1min":9.2639999999999993,"5min":40.313000000000002,"15min":40.313000000000002},"last":4.1120000000000001},{"interface":"front","average":{"1min":3.6669999999999998,"5min":1.897,"15min":0.88},"min":{"1min":0.27500000000000002,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":19.564,"5min":36.655000000000001,"15min":36.655000000000001},"last":3.702}]},{"osd":2,"last update":"Mon Aug 28 02:25:27 2023","interfaces":[{"interface":"back","average":{"1min":4.4050000000000002,"5min":1.944,"15min":0.88300000000000001},"min":{"1min":0.40500000000000003,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":15.33,"5min":15.33,"15min":15.33},"last":4.532},{"interface":"front","average":{"1min":6.351,"5min":2.484,"15min":1.089},"min":{"1min":0.40799999999999997,"5min":0.224,"15min":0.224},"max":{"1min":18.870999999999999,"5min":18.870999999999999,"15min":18.870999999999999},"last":3.6709999999999998}]},{"osd":3,"last update":"Mon Aug 28 02:25:05 2023","interfaces":[{"interface":"back","average":{"1min":3.4239999999999999,"5min":1.5269999999999999,"15min":0.751},"min":{"1min":0.33600000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":18.545999999999999,"5min":18.545999999999999,"15min":18.545999999999999},"last":5.5609999999999999},{"interface":"front","average":{"1min":3.254,"5min":1.5429999999999999,"15min":0.77700000000000002},"min":{"1min":0.34599999999999997,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":14.534000000000001,"5min":14.534000000000001,"15min":14.534000000000001},"last":2.4900000000000002}]},{"osd":4,"last update":"Mon Aug 28 02:25:00 2023","interfaces":[{"interface":"back","average":{"1min":2.472,"5min":1.5009999999999999,"15min":0.94699999999999995},"min":{"1min":0.37,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":9.2330000000000005,"5min":9.6649999999999991,"15min":9.6649999999999991},"last":2.625},{"interface":"front","average":{"1min":2.8769999999999998,"5min":1.601,"15min":1.0049999999999999},"min":{"1min":0.34000000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":10.273999999999999,"5min":10.273999999999999,"15min":10.273999999999999},"last":1.702}]},{"osd":5,"last update":"Mon Aug 28 02:25:00 2023","interfaces":[{"interface":"back","average":{"1min":2.9929999999999999,"5min":2.0179999999999998,"15min":1.1080000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":11.318,"5min":44.000999999999998,"15min":44.000999999999998},"last":1.74},{"interface":"front","average":{"1min":2.3980000000000001,"5min":1.9039999999999999,"15min":1.0700000000000001},"min":{"1min":0.40899999999999997,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":9.2010000000000005,"5min":43.953000000000003,"15min":43.953000000000003},"last":2.5790000000000002}]},{"osd":6,"last update":"Mon Aug 28 02:25:00 2023","interfaces":[{"interface":"back","average":{"1min":3.3700000000000001,"5min":2.0190000000000001,"15min":1.1100000000000001},"min":{"1min":0.38900000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":11.379,"5min":32.417999999999999,"15min":32.417999999999999},"last":1.7609999999999999},{"interface":"front","average":{"1min":2.3260000000000001,"5min":1.806,"15min":1.0740000000000001},"min":{"1min":0.45500000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":6.3849999999999998,"5min":32.130000000000003,"15min":32.130000000000003},"last":4.0789999999999997}]},{"osd":7,"last update":"Mon Aug 28 02:25:00 2023","interfaces":[{"interface":"back","average":{"1min":2.5099999999999998,"5min":1.5720000000000001,"15min":0.98999999999999999},"min":{"1min":0.433,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":6.319,"5min":9.6020000000000003,"15min":9.6020000000000003},"last":2.6579999999999999},{"interface":"front","average":{"1min":4.0330000000000004,"5min":1.754,"15min":1.054},"min":{"1min":0.44600000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":12.228,"5min":12.228,"15min":12.228},"last":3.7269999999999999}]}]},{"osd":2,"up_from":104,"seq":446676598854,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":674816,"kb_used_data":575096,"kb_used_omap":10,"kb_used_meta":99701,"kb_avail":93070336,"statfs":{"total":95995035648,"available":95304024064,"internally_reserved":0,"allocated":588898304,"data_stored":588102078,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10792,"internal_metadata":102094296},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,2,7],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:25:33 2023","interfaces":[{"interface":"back","average":{"1min":3.6179999999999999,"5min":2.2519999999999998,"15min":0.96399999999999997},"min":{"1min":0.188,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":18.146000000000001,"5min":36.890000000000001,"15min":36.890000000000001},"last":27.972999999999999},{"interface":"front","average":{"1min":4.625,"5min":2.4590000000000001,"15min":1.0620000000000001},"min":{"1min":0.214,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":25.414999999999999,"5min":35.289000000000001,"15min":35.289000000000001},"last":20.844999999999999}]},{"osd":1,"last update":"Mon Aug 28 02:25:33 2023","interfaces":[{"interface":"back","average":{"1min":4.4900000000000002,"5min":2.4260000000000002,"15min":1.048},"min":{"1min":0.23999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":16.372,"5min":30.888000000000002,"15min":30.888000000000002},"last":0.27600000000000002},{"interface":"front","average":{"1min":3.8959999999999999,"5min":1.9299999999999999,"15min":0.86599999999999999},"min":{"1min":0.21299999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":14.855,"5min":31.648,"15min":31.648},"last":1.5149999999999999}]},{"osd":3,"last update":"Mon Aug 28 02:24:57 2023","interfaces":[{"interface":"back","average":{"1min":3.0430000000000001,"5min":1.341,"15min":0.71899999999999997},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":14.991,"5min":14.991,"15min":14.991},"last":0.309},{"interface":"front","average":{"1min":4.5819999999999999,"5min":1.589,"15min":0.78700000000000003},"min":{"1min":0.29999999999999999,"5min":0.246,"15min":0.246},"max":{"1min":17.577000000000002,"5min":17.577000000000002,"15min":17.577000000000002},"last":0.32200000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:25:33 2023","interfaces":[{"interface":"back","average":{"1min":3.5750000000000002,"5min":1.772,"15min":0.98599999999999999},"min":{"1min":0.34399999999999997,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":14.815,"5min":14.815,"15min":14.815},"last":0.371},{"interface":"front","average":{"1min":1.71,"5min":1.3300000000000001,"15min":0.876},"min":{"1min":0.33700000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.806,"5min":6.806,"15min":6.806},"last":0.39700000000000002}]},{"osd":5,"last update":"Mon Aug 28 02:25:33 2023","interfaces":[{"interface":"back","average":{"1min":1.9099999999999999,"5min":1.518,"15min":0.91700000000000004},"min":{"1min":0.36399999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":6.6900000000000004,"5min":9.1549999999999994,"15min":9.1549999999999994},"last":0.38400000000000001},{"interface":"front","average":{"1min":4.0650000000000004,"5min":2.0459999999999998,"15min":1.054},"min":{"1min":0.438,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":14.877000000000001,"5min":14.877000000000001,"15min":14.877000000000001},"last":0.43099999999999999}]},{"osd":6,"last update":"Mon Aug 28 02:25:33 2023","interfaces":[{"interface":"back","average":{"1min":2.2280000000000002,"5min":1.659,"15min":0.95299999999999996},"min":{"1min":0.35699999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":8.7010000000000005,"5min":8.7010000000000005,"15min":8.7010000000000005},"last":0.45800000000000002},{"interface":"front","average":{"1min":2.1190000000000002,"5min":1.7350000000000001,"15min":0.98899999999999999},"min":{"1min":0.42999999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":8.7729999999999997,"5min":10.507,"15min":10.507},"last":0.46800000000000003}]},{"osd":7,"last update":"Mon Aug 28 02:25:33 2023","interfaces":[{"interface":"back","average":{"1min":2.0720000000000001,"5min":1.6100000000000001,"15min":0.93600000000000005},"min":{"1min":0.39800000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":8.7699999999999996,"5min":9.0909999999999993,"15min":9.0909999999999993},"last":0.48999999999999999},{"interface":"front","average":{"1min":2.278,"5min":1.673,"15min":0.96099999999999997},"min":{"1min":0.40400000000000003,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":7.1120000000000001,"5min":7.2569999999999997,"15min":7.2569999999999997},"last":0.5}]}]},{"osd":3,"up_from":112,"seq":481036337216,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":66768,"kb_used_data":17224,"kb_used_omap":3,"kb_used_meta":49532,"kb_avail":93678384,"statfs":{"total":95995035648,"available":95926665216,"internally_reserved":0,"allocated":17637376,"data_stored":16879862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":50721698},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:24:56 2023","interfaces":[{"interface":"back","average":{"1min":18.995000000000001,"5min":5.5199999999999996,"15min":2.073},"min":{"1min":0.215,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":164.25299999999999,"5min":164.25299999999999,"15min":164.25299999999999},"last":4.3239999999999998},{"interface":"front","average":{"1min":19.428999999999998,"5min":5.7149999999999999,"15min":2.1269999999999998},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":164.55000000000001,"5min":164.55000000000001,"15min":164.55000000000001},"last":0.315}]},{"osd":1,"last update":"Mon Aug 28 02:24:56 2023","interfaces":[{"interface":"back","average":{"1min":1.9330000000000001,"5min":3.3570000000000002,"15min":1.345},"min":{"1min":0.247,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":5.5830000000000002,"5min":117.34699999999999,"15min":117.34699999999999},"last":1.4590000000000001},{"interface":"front","average":{"1min":1.4630000000000001,"5min":3.3759999999999999,"15min":1.3779999999999999},"min":{"1min":0.23400000000000001,"5min":0.224,"15min":0.224},"max":{"1min":4.4249999999999998,"5min":118.012,"15min":118.012},"last":4.3490000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:24:56 2023","interfaces":[{"interface":"back","average":{"1min":10.507999999999999,"5min":4.9109999999999996,"15min":1.887},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":120.43300000000001,"5min":120.43300000000001,"15min":120.43300000000001},"last":8.9369999999999994},{"interface":"front","average":{"1min":11.199,"5min":5.0119999999999996,"15min":1.9219999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":120.44499999999999,"5min":120.44499999999999,"15min":120.44499999999999},"last":8.984}]},{"osd":4,"last update":"Mon Aug 28 02:24:56 2023","interfaces":[{"interface":"back","average":{"1min":1.8959999999999999,"5min":1.256,"15min":0.83799999999999997},"min":{"1min":0.309,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":5.9740000000000002,"5min":7.423,"15min":7.423},"last":9.016},{"interface":"front","average":{"1min":1.895,"5min":1.1879999999999999,"15min":0.81999999999999995},"min":{"1min":0.36799999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":5.4859999999999998,"5min":8.4039999999999999,"15min":8.4039999999999999},"last":7.7089999999999996}]},{"osd":5,"last update":"Mon Aug 28 02:24:56 2023","interfaces":[{"interface":"back","average":{"1min":2.048,"5min":1.1259999999999999,"15min":0.80000000000000004},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":5.5590000000000002,"5min":7.3369999999999997,"15min":7.3369999999999997},"last":0.51600000000000001},{"interface":"front","average":{"1min":1.228,"5min":1.1040000000000001,"15min":0.78800000000000003},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":4.8170000000000002,"5min":9.3849999999999998,"15min":9.3849999999999998},"last":4.2889999999999997}]},{"osd":6,"last update":"Mon Aug 28 02:24:56 2023","interfaces":[{"interface":"back","average":{"1min":0.999,"5min":1.786,"15min":1.0189999999999999},"min":{"1min":0.35599999999999998,"5min":0.314,"15min":0.314},"max":{"1min":4.0999999999999996,"5min":72.116,"15min":72.116},"last":4.3710000000000004},{"interface":"front","average":{"1min":2.0990000000000002,"5min":1.9319999999999999,"15min":1.042},"min":{"1min":0.38800000000000001,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":4.9199999999999999,"5min":72.030000000000001,"15min":72.030000000000001},"last":0.85799999999999998}]},{"osd":7,"last update":"Mon Aug 28 02:24:56 2023","interfaces":[{"interface":"back","average":{"1min":2.1040000000000001,"5min":1.27,"15min":0.86199999999999999},"min":{"1min":0.41699999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":6.1669999999999998,"5min":7.3129999999999997,"15min":7.3129999999999997},"last":2.1360000000000001},{"interface":"front","average":{"1min":2.3700000000000001,"5min":1.2949999999999999,"15min":0.86199999999999999},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":6.5030000000000001,"5min":7.2960000000000003,"15min":7.2960000000000003},"last":3.8820000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019090,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1029432,"kb_used_data":719920,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92715720,"statfs":{"total":95995035648,"available":94940897280,"internally_reserved":0,"allocated":737198080,"data_stored":736630968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:25:59 2023","interfaces":[{"interface":"back","average":{"1min":6.2859999999999996,"5min":5.8010000000000002,"15min":2.3399999999999999},"min":{"1min":0.26500000000000001,"5min":0.223,"15min":0.223},"max":{"1min":43.548000000000002,"5min":138.67400000000001,"15min":138.67400000000001},"last":0.48299999999999998},{"interface":"front","average":{"1min":5.8789999999999996,"5min":5.6669999999999998,"15min":2.2839999999999998},"min":{"1min":0.25,"5min":0.248,"15min":0.248},"max":{"1min":43.515000000000001,"5min":134.14599999999999,"15min":134.14599999999999},"last":4.218}]},{"osd":1,"last update":"Mon Aug 28 02:25:59 2023","interfaces":[{"interface":"back","average":{"1min":2.6509999999999998,"5min":1.915,"15min":0.98699999999999999},"min":{"1min":0.32700000000000001,"5min":0.255,"15min":0.255},"max":{"1min":8.9570000000000007,"5min":53.622999999999998,"15min":53.622999999999998},"last":2.141},{"interface":"front","average":{"1min":2.1339999999999999,"5min":1.8999999999999999,"15min":1.01},"min":{"1min":0.28000000000000003,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":7.1580000000000004,"5min":53.774000000000001,"15min":53.774000000000001},"last":4.1459999999999999}]},{"osd":2,"last update":"Mon Aug 28 02:25:28 2023","interfaces":[{"interface":"back","average":{"1min":5.0780000000000003,"5min":3.6219999999999999,"15min":1.5740000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":56.295999999999999,"5min":74.099000000000004,"15min":74.099000000000004},"last":4.319},{"interface":"front","average":{"1min":4.7229999999999999,"5min":3.4820000000000002,"15min":1.52},"min":{"1min":0.28599999999999998,"5min":0.25,"15min":0.25},"max":{"1min":56.314,"5min":74.081999999999994,"15min":74.081999999999994},"last":4.1900000000000004}]},{"osd":3,"last update":"Mon Aug 28 02:25:59 2023","interfaces":[{"interface":"back","average":{"1min":3.3359999999999999,"5min":1.5149999999999999,"15min":0.83899999999999997},"min":{"1min":0.32400000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":8.6349999999999998,"5min":8.6349999999999998,"15min":8.6349999999999998},"last":1.4099999999999999},{"interface":"front","average":{"1min":2.9430000000000001,"5min":1.6870000000000001,"15min":0.95099999999999996},"min":{"1min":0.39900000000000002,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":10.956,"5min":10.956,"15min":10.956},"last":2.0699999999999998}]},{"osd":5,"last update":"Mon Aug 28 02:25:41 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":0.97799999999999998,"15min":0.63700000000000001},"min":{"1min":0.26900000000000002,"5min":0.249,"15min":0.249},"max":{"1min":5.4080000000000004,"5min":8.2560000000000002,"15min":8.2560000000000002},"last":4.1020000000000003},{"interface":"front","average":{"1min":2.3010000000000002,"5min":1.3300000000000001,"15min":0.76700000000000002},"min":{"1min":0.27100000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":8.0619999999999994,"5min":8.0619999999999994,"15min":8.0619999999999994},"last":1.3180000000000001}]},{"osd":6,"last update":"Mon Aug 28 02:25:59 2023","interfaces":[{"interface":"back","average":{"1min":2.1059999999999999,"5min":1.373,"15min":0.79600000000000004},"min":{"1min":0.32300000000000001,"5min":0.29199999999999998,"15min":0.27500000000000002},"max":{"1min":11.170999999999999,"5min":11.170999999999999,"15min":11.170999999999999},"last":0.51000000000000001},{"interface":"front","average":{"1min":1.403,"5min":0.97199999999999998,"15min":0.69199999999999995},"min":{"1min":0.33700000000000002,"5min":0.32200000000000001,"15min":0.318},"max":{"1min":7.3170000000000002,"5min":7.3170000000000002,"15min":7.3170000000000002},"last":4.2460000000000004}]},{"osd":7,"last update":"Mon Aug 28 02:25:22 2023","interfaces":[{"interface":"back","average":{"1min":1.825,"5min":1.0960000000000001,"15min":0.72499999999999998},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.312},"max":{"1min":7.9589999999999996,"5min":8.6890000000000001,"15min":8.6890000000000001},"last":2.544},{"interface":"front","average":{"1min":1.8460000000000001,"5min":1.4450000000000001,"15min":0.84999999999999998},"min":{"1min":0.35499999999999998,"5min":0.32500000000000001,"15min":0.32100000000000001},"max":{"1min":5.8250000000000002,"5min":8.7569999999999997,"15min":8.7569999999999997},"last":3.2709999999999999}]}]},{"osd":5,"up_from":36,"seq":154618822862,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1150836,"kb_used_data":849516,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92594316,"statfs":{"total":95995035648,"available":94816579584,"internally_reserved":0,"allocated":869904384,"data_stored":869344594,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:25:52 2023","interfaces":[{"interface":"back","average":{"1min":3.5249999999999999,"5min":5.8579999999999997,"15min":2.3639999999999999},"min":{"1min":0.24099999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":12.874000000000001,"5min":131.06100000000001,"15min":131.06100000000001},"last":1.2729999999999999},{"interface":"front","average":{"1min":3.758,"5min":6.1749999999999998,"15min":2.4740000000000002},"min":{"1min":0.24199999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":15.691000000000001,"5min":131.262,"15min":131.262},"last":1.2869999999999999}]},{"osd":1,"last update":"Mon Aug 28 02:25:00 2023","interfaces":[{"interface":"back","average":{"1min":7.7599999999999998,"5min":4.8479999999999999,"15min":1.968},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":104.748,"5min":183.97900000000001,"15min":183.97900000000001},"last":0.496},{"interface":"front","average":{"1min":9.202,"5min":5.1059999999999999,"15min":2.0619999999999998},"min":{"1min":0.26300000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":105.666,"5min":184.33699999999999,"15min":184.33699999999999},"last":1.177}]},{"osd":2,"last update":"Mon Aug 28 02:25:30 2023","interfaces":[{"interface":"back","average":{"1min":3.5409999999999999,"5min":5.2169999999999996,"15min":2.1320000000000001},"min":{"1min":0.29999999999999999,"5min":0.252,"15min":0.252},"max":{"1min":12.584,"5min":110.051,"15min":110.051},"last":1.4810000000000001},{"interface":"front","average":{"1min":3.7709999999999999,"5min":5.4820000000000002,"15min":2.2189999999999999},"min":{"1min":0.29799999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":15.635,"5min":110.041,"15min":110.041},"last":1.492}]},{"osd":3,"last update":"Mon Aug 28 02:25:00 2023","interfaces":[{"interface":"back","average":{"1min":2.7650000000000001,"5min":1.371,"15min":0.86499999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":10.576000000000001,"5min":10.576000000000001,"15min":10.576000000000001},"last":2.3969999999999998},{"interface":"front","average":{"1min":3.258,"5min":1.5920000000000001,"15min":0.94799999999999995},"min":{"1min":0.35799999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":10.459,"5min":10.459,"15min":10.459},"last":0.80700000000000005}]},{"osd":4,"last update":"Mon Aug 28 02:25:44 2023","interfaces":[{"interface":"back","average":{"1min":1.4239999999999999,"5min":1.355,"15min":0.76200000000000001},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":5.6890000000000001,"5min":10.759,"15min":10.759},"last":2.5840000000000001},{"interface":"front","average":{"1min":2.149,"5min":1.468,"15min":0.82199999999999995},"min":{"1min":0.31900000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.0170000000000003,"5min":8.6769999999999996,"15min":8.6769999999999996},"last":3.113}]},{"osd":6,"last update":"Mon Aug 28 02:25:00 2023","interfaces":[{"interface":"back","average":{"1min":1.3540000000000001,"5min":1.4079999999999999,"15min":0.81999999999999995},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.28399999999999997},"max":{"1min":6.9100000000000001,"5min":14.803000000000001,"15min":14.803000000000001},"last":4.0940000000000003},{"interface":"front","average":{"1min":1.8580000000000001,"5min":1.1779999999999999,"15min":0.73799999999999999},"min":{"1min":0.315,"5min":0.31,"15min":0.30299999999999999},"max":{"1min":8.6479999999999997,"5min":8.6479999999999997,"15min":8.6479999999999997},"last":0.45600000000000002}]},{"osd":7,"last update":"Mon Aug 28 02:25:17 2023","interfaces":[{"interface":"back","average":{"1min":2.004,"5min":1.3180000000000001,"15min":0.80300000000000005},"min":{"1min":0.33900000000000002,"5min":0.31,"15min":0.31},"max":{"1min":9.3070000000000004,"5min":9.3070000000000004,"15min":9.3070000000000004},"last":4.2480000000000002},{"interface":"front","average":{"1min":1.871,"5min":1.341,"15min":0.80900000000000005},"min":{"1min":0.32200000000000001,"5min":0.32100000000000001,"15min":0.29799999999999999},"max":{"1min":12.787000000000001,"5min":12.787000000000001,"15min":12.787000000000001},"last":0.37}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1385562112,"data_stored":1385562112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":492371968,"data_stored":492371968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":584519680,"data_stored":584515608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":14352384,"data_stored":14352384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":722141184,"data_stored":722141184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":853413888,"data_stored":853409816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":362348544,"data_stored":362348544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":455479296,"data_stored":455475224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-08-28T02:26:05.073 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph status --format=json 2023-08-28T02:26:05.135 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:26:05.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:05.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:26:05.142 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:26:05.142 INFO:teuthology.orchestra.run.smithi169.stdout: "ops": [ 2023-08-28T02:26:05.142 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.142 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_repop(client.44594.0:200073 5.3 e180/119 5:c5e7c95b:::benchmark_data_smithi143_236200_object89930:head v 180'12390, mlcod=180'12388)", 2023-08-28T02:26:05.142 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:26:05.127655+0000", 2023-08-28T02:26:05.143 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 0.014374636, 2023-08-28T02:26:05.143 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.014399966, 2023-08-28T02:26:05.143 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:05.143 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "started", 2023-08-28T02:26:05.143 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:05.143 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.143 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:05.143 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.127655+0000", 2023-08-28T02:26:05.143 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:05.143 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.143 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.143 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:05.143 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.127655+0000", 2023-08-28T02:26:05.144 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:26:05.144 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.144 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.144 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:05.144 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.127654+0000", 2023-08-28T02:26:05.144 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2411e-05 2023-08-28T02:26:05.144 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.144 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.144 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:05.144 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.127666+0000", 2023-08-28T02:26:05.144 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.5199e-05 2023-08-28T02:26:05.144 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.145 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.145 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:05.145 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.127682+0000", 2023-08-28T02:26:05.145 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.8769999999999993e-06 2023-08-28T02:26:05.145 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.145 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.145 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:05.145 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.127689+0000", 2023-08-28T02:26:05.145 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.1411e-05 2023-08-28T02:26:05.145 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.145 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.145 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:05.145 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.127741+0000", 2023-08-28T02:26:05.146 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.2682000000000001e-05 2023-08-28T02:26:05.146 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.146 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.146 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:05.146 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.127764+0000", 2023-08-28T02:26:05.146 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000108111 2023-08-28T02:26:05.146 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:05.146 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:05.146 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:05.146 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.146 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.146 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_repop(client.44594.0:200078 5.6 e180/159 5:6536c953:::benchmark_data_smithi143_236200_object89935:head v 180'12284, mlcod=180'12282)", 2023-08-28T02:26:05.147 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:26:05.128434+0000", 2023-08-28T02:26:05.147 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 0.013596288999999999, 2023-08-28T02:26:05.147 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.013672998, 2023-08-28T02:26:05.147 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:05.147 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "started", 2023-08-28T02:26:05.147 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:05.147 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.147 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:05.147 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.128434+0000", 2023-08-28T02:26:05.147 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:05.147 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.147 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.148 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:05.148 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.128434+0000", 2023-08-28T02:26:05.148 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:26:05.148 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.148 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.148 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:05.148 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.128433+0000", 2023-08-28T02:26:05.148 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0509999999999999e-05 2023-08-28T02:26:05.148 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.148 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.148 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:05.148 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.128443+0000", 2023-08-28T02:26:05.148 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2575000000000001e-05 2023-08-28T02:26:05.149 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.149 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.149 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:05.149 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.128456+0000", 2023-08-28T02:26:05.149 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.5279999999999998e-06 2023-08-28T02:26:05.149 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.149 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.149 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:05.149 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.128463+0000", 2023-08-28T02:26:05.149 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.3917e-05 2023-08-28T02:26:05.149 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.149 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.149 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:05.150 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.128507+0000", 2023-08-28T02:26:05.150 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.0098999999999998e-05 2023-08-28T02:26:05.150 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.150 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.150 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:05.150 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.128527+0000", 2023-08-28T02:26:05.150 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.3584000000000004e-05 2023-08-28T02:26:05.150 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:05.150 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:05.150 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:05.150 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.150 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.150 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_repop(client.44594.0:200072 5.e e180/119 5:777f63d0:::benchmark_data_smithi143_236200_object89929:head v 180'12692, mlcod=180'12690)", 2023-08-28T02:26:05.151 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:26:05.128939+0000", 2023-08-28T02:26:05.151 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 0.013090812, 2023-08-28T02:26:05.151 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.013241045999999999, 2023-08-28T02:26:05.151 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:05.151 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "started", 2023-08-28T02:26:05.151 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:05.151 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.151 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:05.151 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.128939+0000", 2023-08-28T02:26:05.151 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:05.151 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.151 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.152 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:05.152 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.128939+0000", 2023-08-28T02:26:05.152 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:26:05.152 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.152 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.152 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:05.152 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.128938+0000", 2023-08-28T02:26:05.152 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.3590000000000002e-06 2023-08-28T02:26:05.152 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.152 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.152 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:05.152 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.128946+0000", 2023-08-28T02:26:05.152 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0672999999999999e-05 2023-08-28T02:26:05.152 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.153 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.153 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:05.153 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.128956+0000", 2023-08-28T02:26:05.153 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.251e-06 2023-08-28T02:26:05.153 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.153 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.153 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:05.153 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.128963+0000", 2023-08-28T02:26:05.153 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.0837000000000002e-05 2023-08-28T02:26:05.153 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.153 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.153 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:05.153 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.129003+0000", 2023-08-28T02:26:05.153 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.9641e-05 2023-08-28T02:26:05.153 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.154 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.154 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:05.154 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.129023+0000", 2023-08-28T02:26:05.154 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.3832000000000006e-05 2023-08-28T02:26:05.154 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:05.154 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:05.154 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:05.154 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.154 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.154 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:200077 5.7 5:ed30bc52:::benchmark_data_smithi143_236200_object89934:head [delete] snapc 0=[] ondisk+write+known_if_redirected e180)", 2023-08-28T02:26:05.154 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:26:05.129408+0000", 2023-08-28T02:26:05.154 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 0.0126216, 2023-08-28T02:26:05.154 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.012820287, 2023-08-28T02:26:05.154 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:05.155 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "waiting for sub ops", 2023-08-28T02:26:05.155 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:26:05.155 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:26:05.155 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:05.155 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 200077 2023-08-28T02:26:05.155 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.155 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:05.155 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.155 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:05.155 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.129408+0000", 2023-08-28T02:26:05.155 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:05.155 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.155 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.155 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:05.156 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.129408+0000", 2023-08-28T02:26:05.156 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:26:05.156 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.156 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.156 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:05.156 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.129407+0000", 2023-08-28T02:26:05.156 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.3939999999999998e-06 2023-08-28T02:26:05.156 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.156 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.156 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:05.156 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.129414+0000", 2023-08-28T02:26:05.156 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.9992000000000001e-05 2023-08-28T02:26:05.156 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.156 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.157 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:05.157 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.129434+0000", 2023-08-28T02:26:05.157 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.9209999999999992e-06 2023-08-28T02:26:05.157 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.157 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.157 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:05.157 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.129442+0000", 2023-08-28T02:26:05.157 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0025215210000000001 2023-08-28T02:26:05.157 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.157 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.157 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:05.157 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.131964+0000", 2023-08-28T02:26:05.157 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000119518 2023-08-28T02:26:05.158 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.158 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.158 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:05.158 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.132083+0000", 2023-08-28T02:26:05.158 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00014067499999999999 2023-08-28T02:26:05.158 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.158 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.158 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,1", 2023-08-28T02:26:05.158 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.132224+0000", 2023-08-28T02:26:05.158 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0028154730000000002 2023-08-28T02:26:05.158 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:05.159 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:05.159 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:05.159 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.159 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.159 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_repop(client.44594.0:200076 5.4 e180/150 5:24952aee:::benchmark_data_smithi143_236200_object89933:head v 180'12401, mlcod=180'12399)", 2023-08-28T02:26:05.159 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:26:05.130207+0000", 2023-08-28T02:26:05.159 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 0.011823393, 2023-08-28T02:26:05.159 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.012105174999999999, 2023-08-28T02:26:05.159 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:05.159 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "started", 2023-08-28T02:26:05.159 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:05.159 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.159 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:05.160 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.130207+0000", 2023-08-28T02:26:05.160 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:05.160 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.160 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.160 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:05.160 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.130207+0000", 2023-08-28T02:26:05.160 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:26:05.160 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.160 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.160 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:05.160 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.130206+0000", 2023-08-28T02:26:05.161 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.3350000000000007e-06 2023-08-28T02:26:05.161 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.161 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.161 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:05.161 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.130214+0000", 2023-08-28T02:26:05.161 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.5438999999999999e-05 2023-08-28T02:26:05.161 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.161 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.161 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:05.161 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.130230+0000", 2023-08-28T02:26:05.162 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.516e-06 2023-08-28T02:26:05.162 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.162 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.162 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:05.162 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.130236+0000", 2023-08-28T02:26:05.162 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0041093839999999998 2023-08-28T02:26:05.162 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.162 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.162 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:05.162 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.134345+0000", 2023-08-28T02:26:05.163 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.7961e-05 2023-08-28T02:26:05.163 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.163 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.163 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:05.163 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.134363+0000", 2023-08-28T02:26:05.163 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0041567349999999999 2023-08-28T02:26:05.163 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:05.163 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:05.163 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:05.163 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.163 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.164 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_repop(client.44594.0:200082 5.d e180/157 5:b215547e:::benchmark_data_smithi143_236200_object89939:head v 180'12553, mlcod=180'12551)", 2023-08-28T02:26:05.164 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:26:05.130366+0000", 2023-08-28T02:26:05.164 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 0.011663651000000001, 2023-08-28T02:26:05.164 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.012004946000000001, 2023-08-28T02:26:05.164 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:05.164 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "started", 2023-08-28T02:26:05.164 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:05.164 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.164 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:05.164 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.130366+0000", 2023-08-28T02:26:05.164 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:05.164 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.165 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.165 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:05.165 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.130366+0000", 2023-08-28T02:26:05.165 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:26:05.165 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.165 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.165 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:05.165 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.130366+0000", 2023-08-28T02:26:05.165 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.7150000000000001e-06 2023-08-28T02:26:05.165 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.165 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.165 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:05.165 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.130372+0000", 2023-08-28T02:26:05.166 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.9499999999999996e-06 2023-08-28T02:26:05.166 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.166 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.166 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:05.166 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.130382+0000", 2023-08-28T02:26:05.166 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.1279999999999996e-06 2023-08-28T02:26:05.166 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.166 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.166 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:05.166 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.130388+0000", 2023-08-28T02:26:05.166 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.010051431 2023-08-28T02:26:05.166 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.167 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.167 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:05.167 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.140440+0000", 2023-08-28T02:26:05.167 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.0152e-05 2023-08-28T02:26:05.167 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.167 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.167 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:05.167 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.140460+0000", 2023-08-28T02:26:05.167 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.010093654000000001 2023-08-28T02:26:05.167 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:05.167 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:05.167 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:05.167 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.168 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.168 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_repop(client.44594.0:200074 5.3 e180/119 5:c80a1f7d:::benchmark_data_smithi143_236200_object89931:head v 180'12391, mlcod=180'12388)", 2023-08-28T02:26:05.168 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:26:05.136918+0000", 2023-08-28T02:26:05.168 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 0.0051116820000000002, 2023-08-28T02:26:05.168 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00551733, 2023-08-28T02:26:05.168 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:05.168 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "started", 2023-08-28T02:26:05.168 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:05.168 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.168 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:05.168 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.136918+0000", 2023-08-28T02:26:05.168 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:05.168 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.169 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.169 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:05.169 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.136918+0000", 2023-08-28T02:26:05.169 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:26:05.169 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.169 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.169 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:05.169 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.136917+0000", 2023-08-28T02:26:05.169 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.3580000000000003e-06 2023-08-28T02:26:05.169 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.169 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.169 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:05.169 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.136926+0000", 2023-08-28T02:26:05.170 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.7729999999999994e-06 2023-08-28T02:26:05.170 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.170 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.170 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:05.170 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.136936+0000", 2023-08-28T02:26:05.170 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.9960000000000001e-06 2023-08-28T02:26:05.170 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.170 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.170 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:05.170 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.136941+0000", 2023-08-28T02:26:05.170 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.7246999999999997e-05 2023-08-28T02:26:05.170 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.170 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.171 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:05.171 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.136988+0000", 2023-08-28T02:26:05.171 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.8813000000000001e-05 2023-08-28T02:26:05.171 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.171 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.171 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:05.171 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.137007+0000", 2023-08-28T02:26:05.171 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.8211e-05 2023-08-28T02:26:05.171 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:05.171 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:05.171 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:05.171 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.172 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.172 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_repop(client.44594.0:200080 5.2 e180/119 5:47cb6ae9:::benchmark_data_smithi143_236200_object89937:head v 180'12294, mlcod=180'12292)", 2023-08-28T02:26:05.172 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:26:05.137658+0000", 2023-08-28T02:26:05.172 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 0.0043721039999999999, 2023-08-28T02:26:05.172 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0048405000000000002, 2023-08-28T02:26:05.172 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:05.172 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "started", 2023-08-28T02:26:05.172 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:05.172 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.172 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:05.172 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.137658+0000", 2023-08-28T02:26:05.172 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:05.173 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.173 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.173 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:05.173 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.137658+0000", 2023-08-28T02:26:05.173 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999995 2023-08-28T02:26:05.173 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.173 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.173 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:05.173 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.137657+0000", 2023-08-28T02:26:05.173 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.0589999999999997e-06 2023-08-28T02:26:05.173 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.173 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.174 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:05.174 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.137664+0000", 2023-08-28T02:26:05.174 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.5339999999999999e-06 2023-08-28T02:26:05.174 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.174 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.174 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:05.174 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.137673+0000", 2023-08-28T02:26:05.174 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.3760000000000001e-06 2023-08-28T02:26:05.174 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.174 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.174 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:05.174 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.137677+0000", 2023-08-28T02:26:05.174 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.8198999999999998e-05 2023-08-28T02:26:05.175 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.175 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.175 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:05.175 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.137715+0000", 2023-08-28T02:26:05.175 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.0112999999999999e-05 2023-08-28T02:26:05.175 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.175 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.175 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:05.175 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.137736+0000", 2023-08-28T02:26:05.175 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.7569000000000002e-05 2023-08-28T02:26:05.175 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:05.175 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:05.176 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:05.176 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.176 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.176 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_repop(client.44594.0:200084 5.b e180/119 5:dd54e2be:::benchmark_data_smithi143_236200_object89941:head v 180'12476, mlcod=180'12474)", 2023-08-28T02:26:05.176 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:26:05.138674+0000", 2023-08-28T02:26:05.176 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 0.0033563930000000001, 2023-08-28T02:26:05.176 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0038928029999999998, 2023-08-28T02:26:05.176 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:05.176 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "started", 2023-08-28T02:26:05.176 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:05.176 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.176 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:05.176 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.138674+0000", 2023-08-28T02:26:05.176 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:05.177 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.177 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.177 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:05.177 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.138674+0000", 2023-08-28T02:26:05.177 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:26:05.177 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.177 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.177 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:05.177 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.138673+0000", 2023-08-28T02:26:05.177 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0491e-05 2023-08-28T02:26:05.177 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.177 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.177 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:05.177 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.138683+0000", 2023-08-28T02:26:05.177 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.3071000000000001e-05 2023-08-28T02:26:05.178 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.178 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.178 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:05.178 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.138696+0000", 2023-08-28T02:26:05.178 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.879e-06 2023-08-28T02:26:05.178 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.178 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.178 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:05.178 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.138702+0000", 2023-08-28T02:26:05.178 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.8541999999999998e-05 2023-08-28T02:26:05.178 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.178 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.178 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:05.179 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.138741+0000", 2023-08-28T02:26:05.179 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.8196999999999999e-05 2023-08-28T02:26:05.179 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.179 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.179 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:05.179 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.138759+0000", 2023-08-28T02:26:05.179 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.5376000000000004e-05 2023-08-28T02:26:05.179 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:05.179 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:05.179 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:05.179 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.179 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.179 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_repop(client.44594.0:200083 5.d e180/157)", 2023-08-28T02:26:05.180 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:26:05.138713+0000", 2023-08-28T02:26:05.180 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 0.003316678, 2023-08-28T02:26:05.180 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0039130119999999996, 2023-08-28T02:26:05.180 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:05.180 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "queued for pg", 2023-08-28T02:26:05.180 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:05.180 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.180 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:05.180 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.138713+0000", 2023-08-28T02:26:05.180 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:05.180 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.180 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.180 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:05.181 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.138713+0000", 2023-08-28T02:26:05.181 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999995 2023-08-28T02:26:05.181 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.181 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.181 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:05.181 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.138713+0000", 2023-08-28T02:26:05.181 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.822e-06 2023-08-28T02:26:05.181 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.181 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.181 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:05.181 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.138717+0000", 2023-08-28T02:26:05.181 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.5860000000000004e-06 2023-08-28T02:26:05.181 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.182 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.182 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:05.182 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.138722+0000", 2023-08-28T02:26:05.182 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.5880000000000002e-06 2023-08-28T02:26:05.182 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:05.182 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:05.182 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:05.182 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:05.138726+0000", 2023-08-28T02:26:05.182 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2659e-05 2023-08-28T02:26:05.182 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:05.182 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:05.182 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:05.182 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:05.182 INFO:teuthology.orchestra.run.smithi169.stdout: ], 2023-08-28T02:26:05.183 INFO:teuthology.orchestra.run.smithi169.stdout: "num_ops": 10 2023-08-28T02:26:05.183 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:26:05.241 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:26:05.390 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:05.404 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:05 smithi169 ceph-mon[159415]: pgmap v409: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.3 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 1990/64542 objects degraded (3.083%); 3732/64542 objects misplaced (5.782%); 2.3 MiB/s, 36 objects/s recovering 2023-08-28T02:26:05.557 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:26:05.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:05.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:26:05.568 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:05 smithi143 ceph-mon[183059]: pgmap v409: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.3 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 1990/64542 objects degraded (3.083%); 3732/64542 objects misplaced (5.782%); 2.3 MiB/s, 36 objects/s recovering 2023-08-28T02:26:05.570 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:05 smithi143 ceph-mon[189871]: pgmap v409: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.3 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 1990/64542 objects degraded (3.083%); 3732/64542 objects misplaced (5.782%); 2.3 MiB/s, 36 objects/s recovering 2023-08-28T02:26:05.631 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:26:05.657 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:26:05.663 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:26:05.681 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:26:05.708 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:05.714 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:26:05.765 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:26:05.794 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:26:05.813 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:26:05.868 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:26:05.904 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:26:05.931 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:26:05.931 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:26:05.931 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:26:05.931 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:26:05.932 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:26:05.932 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:26:05.932 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:26:05.932 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:26:05.981 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:26:05.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:05.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:26:06.096 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:26:06.153 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.5 dump_blocked_ops 2023-08-28T02:26:06.249 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:06.402 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:26:06.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:06.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:26:06.413 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:06 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1156635708' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:26:06.413 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:06 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 1990/64542 objects degraded (3.083%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:26:06.510 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:26:06.580 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:06.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:06 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1156635708' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:26:06.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:06 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 1990/64542 objects degraded (3.083%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:26:06.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:06 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1156635708' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:26:06.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:06 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 1990/64542 objects degraded (3.083%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:26:06.707 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:26:06.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:06.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:26:06.811 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:26:06.923 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:07.043 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:26:07.043 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:26:07.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:07.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:26:07.156 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:26:07.213 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:07.239 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:26:07.426 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:07 smithi169 ceph-mon[159415]: pgmap v410: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.3 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 1990/61854 objects degraded (3.217%); 3578/61854 objects misplaced (5.785%); 2.3 MiB/s, 36 objects/s recovering 2023-08-28T02:26:07.590 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:07 smithi143 ceph-mon[183059]: pgmap v410: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.3 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 1990/61854 objects degraded (3.217%); 3578/61854 objects misplaced (5.785%); 2.3 MiB/s, 36 objects/s recovering 2023-08-28T02:26:07.590 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:07 smithi143 ceph-mon[189871]: pgmap v410: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.3 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 1990/61854 objects degraded (3.217%); 3578/61854 objects misplaced (5.785%); 2.3 MiB/s, 36 objects/s recovering 2023-08-28T02:26:07.653 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:26:07.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:07.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:26:07.761 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:26:07.836 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:08.143 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:26:08.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:08.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:26:08.248 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:26:08.322 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:09.279 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:26:09.637 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:26:09.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:09.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:26:09.749 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:26:09.785 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:09.906 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:26:09.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:09.909 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:26:09.970 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:26:09.971 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:26:10.010 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:26:10.105 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:10.224 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:26:10.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:10.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:26:10.327 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:26:10.372 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:10.417 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:26:10.434 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:26:10.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:10.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:26:10.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:10 smithi169 ceph-mon[159415]: pgmap v411: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.1 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 584 op/s; 1990/55692 objects degraded (3.573%); 3193/55692 objects misplaced (5.733%); 2.5 MiB/s, 39 objects/s recovering 2023-08-28T02:26:10.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:10 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:26:10.490 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:10 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:26:10.538 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:26:10.648 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:10 smithi143 ceph-mon[183059]: pgmap v411: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.1 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 584 op/s; 1990/55692 objects degraded (3.573%); 3193/55692 objects misplaced (5.733%); 2.5 MiB/s, 39 objects/s recovering 2023-08-28T02:26:10.649 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:10 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:26:10.649 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:10 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:26:10.651 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:10 smithi143 ceph-mon[189871]: pgmap v411: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.1 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 584 op/s; 1990/55692 objects degraded (3.573%); 3193/55692 objects misplaced (5.733%); 2.5 MiB/s, 39 objects/s recovering 2023-08-28T02:26:10.651 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:10 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:26:10.652 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:10 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:26:10.661 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:10.858 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:26:10.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:10.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:26:10.903 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.5 dump_historic_ops 2023-08-28T02:26:10.972 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:26:11.029 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:26:11.030 INFO:teuthology.orchestra.run.smithi143.stdout:{"fsid":"2fcb0bfa-4547-11ee-817c-1b7a5f1ba315","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 1990/55692 objects degraded (3.573%), 1 pg degraded, 1 pg undersized","count":2},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":528,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":180,"num_osds":8,"num_up_osds":8,"osd_up_since":1693189249,"num_in_osds":8,"osd_in_since":1693189438,"num_remapped_pgs":3},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":86},{"state_name":"active+remapped+backfill_wait","count":2},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":18564,"data_bytes":1219499979,"bytes_used":5835898880,"bytes_avail":762124386304,"bytes_total":767960285184,"degraded_objects":1990,"degraded_total":55692,"degraded_ratio":0.035732241614594554,"misplaced_objects":3193,"misplaced_total":55692,"misplaced_ratio":0.05733318968613086,"recovering_objects_per_sec":39,"recovering_bytes_per_sec":2611974,"recovering_keys_per_sec":0,"num_objects_recovered":399,"num_bytes_recovered":26148864,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":584},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi143.spifrj","status":"up:active","gid":14637}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.143:8443/","prometheus":"http://172.21.15.143:9283/"}},"servicemap":{"epoch":22,"modified":"2023-08-28T02:21:49.036237+0000","services":{}},"progress_events":{"5de4a8b4-84fe-4f37-8817-bb52f7e413cf":{"message":"Global Recovery Event (2m)\n [===========================.] (remaining: 4s)","progress":0.96629214286804199,"add_to_ceph_s":true},"8e5f590e-829c-445f-9a5b-24676b2537ad":{"message":"Upgrade to 18.2.0-187-g9cf20820 (3m)\n [============................] (remaining: 4m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-08-28T02:26:11.066 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:11.156 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:26:11.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:11.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:26:11.263 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:26:11.272 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:11 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2745320663' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:26:11.274 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:11 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2745320663' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:26:11.274 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:11 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:26:11.274 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:11 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:26:11.275 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:11 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3318133891' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-08-28T02:26:11.311 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:11.313 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is unset 2023-08-28T02:26:11.407 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:26:11.423 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:11 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2745320663' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:26:11.423 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:11 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:26:11.423 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:11 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:26:11.423 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:11 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3318133891' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-08-28T02:26:11.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:11.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:26:11.525 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:26:11.603 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:26:11.611 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:11.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:11 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:26:11.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:11 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:26:11.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:11 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3318133891' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-08-28T02:26:11.727 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:26:11.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:11.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:26:11.839 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:26:11.897 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:12.202 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:26:11 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:26:11] "GET /metrics HTTP/1.1" 200 36487 "" "Prometheus/2.33.4" 2023-08-28T02:26:12.274 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:12 smithi169 ceph-mon[159415]: pgmap v412: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.1 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 584 op/s; 1990/55692 objects degraded (3.573%); 3193/55692 objects misplaced (5.733%); 2.5 MiB/s, 39 objects/s recovering 2023-08-28T02:26:12.274 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:12 smithi169 ceph-mon[159415]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:26:12.274 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:12 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 1990/55692 objects degraded (3.573%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:26:12.275 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:12 smithi169 ceph-mon[159415]: failed to encode map e181 with expected crc 2023-08-28T02:26:12.275 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:12 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:26:12.275 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:12 smithi169 ceph-mon[159415]: osdmap e181: 8 total, 8 up, 8 in 2023-08-28T02:26:12.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:12 smithi143 ceph-mon[183059]: pgmap v412: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.1 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 584 op/s; 1990/55692 objects degraded (3.573%); 3193/55692 objects misplaced (5.733%); 2.5 MiB/s, 39 objects/s recovering 2023-08-28T02:26:12.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:12 smithi143 ceph-mon[183059]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:26:12.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:12 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 1990/55692 objects degraded (3.573%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:26:12.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:12 smithi143 ceph-mon[183059]: failed to encode map e181 with expected crc 2023-08-28T02:26:12.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:12 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:26:12.704 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:12 smithi143 ceph-mon[183059]: osdmap e181: 8 total, 8 up, 8 in 2023-08-28T02:26:12.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:12 smithi143 ceph-mon[189871]: pgmap v412: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.1 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 584 op/s; 1990/55692 objects degraded (3.573%); 3193/55692 objects misplaced (5.733%); 2.5 MiB/s, 39 objects/s recovering 2023-08-28T02:26:12.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:12 smithi143 ceph-mon[189871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:26:12.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:12 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 1990/55692 objects degraded (3.573%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:26:12.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:12 smithi143 ceph-mon[189871]: failed to encode map e181 with expected crc 2023-08-28T02:26:12.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:12 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:26:12.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:12 smithi143 ceph-mon[189871]: osdmap e181: 8 total, 8 up, 8 in 2023-08-28T02:26:13.300 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-08-28T02:26:13.300 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph pg dump --format=json 2023-08-28T02:26:13.305 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:26:13.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:13.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:26:13.411 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:26:13.451 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:13.598 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:26:13.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:13.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:26:13.702 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:26:13.753 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:13.822 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:26:13.822 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:26:13.866 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:26:13.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:13.873 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:26:13.974 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:26:14.047 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:14.216 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:26:14.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:14.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:26:14.289 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:14 smithi143 ceph-mon[183059]: pgmap v414: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 1990/51576 objects degraded (3.858%); 3036/51576 objects misplaced (5.886%); 2.5 MiB/s, 39 objects/s recovering 2023-08-28T02:26:14.290 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:14 smithi143 ceph-mon[189871]: pgmap v414: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 1990/51576 objects degraded (3.858%); 3036/51576 objects misplaced (5.886%); 2.5 MiB/s, 39 objects/s recovering 2023-08-28T02:26:14.320 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:26:14.370 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:26:14.371 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:14.655 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:14 smithi169 ceph-mon[159415]: pgmap v414: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 1990/51576 objects degraded (3.858%); 3036/51576 objects misplaced (5.886%); 2.5 MiB/s, 39 objects/s recovering 2023-08-28T02:26:14.669 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:26:14.682 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.6 dump_ops_in_flight 2023-08-28T02:26:14.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:14.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:26:14.786 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:26:14.874 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:14.894 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:26:14.955 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:26:14.980 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:26:14.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:14.987 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:26:14.996 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:26:15.030 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:26:15.062 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:26:15.088 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:26:15.095 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:26:15.144 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:26:15.190 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:26:15.196 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:15.197 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:26:15.197 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:26:15.197 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:26:15.197 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:26:15.197 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:26:15.197 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:26:15.197 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:26:15.197 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:26:15.241 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:26:15.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:15 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2371783006' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:26:15.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:15 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2371783006' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:26:15.574 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:15 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2371783006' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:26:16.051 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:26:16.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:16.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:26:16.161 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:26:16.185 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset nodeep-scrub 2023-08-28T02:26:16.274 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:16.416 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:26:16.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:16.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:26:16.520 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:26:16.548 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:16.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:16 smithi143 ceph-mon[189871]: pgmap v415: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 973 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 1990/46554 objects degraded (4.275%); 2649/46554 objects misplaced (5.690%); 2.5 MiB/s, 40 objects/s recovering 2023-08-28T02:26:16.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:16 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:26:16.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:16 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:26:16.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:16 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 1990/46554 objects degraded (4.275%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:26:16.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:16 smithi143 ceph-mon[183059]: pgmap v415: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 973 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 1990/46554 objects degraded (4.275%); 2649/46554 objects misplaced (5.690%); 2.5 MiB/s, 40 objects/s recovering 2023-08-28T02:26:16.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:16 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:26:16.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:16 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:26:16.704 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:16 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 1990/46554 objects degraded (4.275%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:26:16.735 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:16 smithi169 ceph-mon[159415]: pgmap v415: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 973 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 1990/46554 objects degraded (4.275%); 2649/46554 objects misplaced (5.690%); 2.5 MiB/s, 40 objects/s recovering 2023-08-28T02:26:16.735 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:16 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:26:16.735 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:16 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:26:16.735 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:16 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 1990/46554 objects degraded (4.275%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:26:16.771 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:26:16.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:16.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:26:16.883 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:26:16.911 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:17.060 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:26:17.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:17.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:26:17.173 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:26:17.257 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:17.323 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:17 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:26:17.323 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:17 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:26:17.324 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:17 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:26:17.343 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:26:17.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:17.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:26:17.364 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:26:17.364 INFO:teuthology.orchestra.run.smithi169.stdout: "ops": [ 2023-08-28T02:26:17.364 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:17.364 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:207448 5.2 5:4886a3ef:::benchmark_data_smithi143_236200_object97305:head [delete] snapc 0=[] ondisk+write+known_if_redirected e181)", 2023-08-28T02:26:17.364 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:26:17.350009+0000", 2023-08-28T02:26:17.364 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 0.014992630999999999, 2023-08-28T02:26:17.364 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.015026457, 2023-08-28T02:26:17.365 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:17.365 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "waiting for sub ops", 2023-08-28T02:26:17.365 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:26:17.365 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:26:17.365 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:17.365 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 207448 2023-08-28T02:26:17.365 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:17.365 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:17.365 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:17.365 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:17.365 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:17.350009+0000", 2023-08-28T02:26:17.365 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:17.366 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:17.366 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:17.366 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:17.366 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:17.350009+0000", 2023-08-28T02:26:17.366 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:26:17.366 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:17.366 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:17.366 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:17.366 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:17.350007+0000", 2023-08-28T02:26:17.366 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.5859999999999994e-06 2023-08-28T02:26:17.366 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:17.366 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:17.366 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:17.367 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:17.350017+0000", 2023-08-28T02:26:17.367 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.7742000000000001e-05 2023-08-28T02:26:17.367 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:17.367 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:17.367 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:17.367 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:17.350035+0000", 2023-08-28T02:26:17.367 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.2190000000000004e-06 2023-08-28T02:26:17.367 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:17.367 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:17.367 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:17.367 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:17.350043+0000", 2023-08-28T02:26:17.367 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.6910999999999998e-05 2023-08-28T02:26:17.368 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:17.368 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:17.368 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:17.368 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:17.350100+0000", 2023-08-28T02:26:17.368 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00014533499999999999 2023-08-28T02:26:17.368 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:17.368 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:17.368 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:17.368 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:17.350245+0000", 2023-08-28T02:26:17.368 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00016074000000000001 2023-08-28T02:26:17.368 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:17.368 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:17.369 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:26:17.369 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:17.350406+0000", 2023-08-28T02:26:17.369 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0079810690000000004 2023-08-28T02:26:17.369 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:17.369 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:17.369 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:26:17.369 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:17.358387+0000", 2023-08-28T02:26:17.369 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0023833249999999999 2023-08-28T02:26:17.369 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:17.369 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:17.369 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:17.369 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:17.360770+0000", 2023-08-28T02:26:17.369 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.010761178999999999 2023-08-28T02:26:17.370 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:17.370 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:17.370 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:17.370 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:17.370 INFO:teuthology.orchestra.run.smithi169.stdout: ], 2023-08-28T02:26:17.370 INFO:teuthology.orchestra.run.smithi169.stdout: "num_ops": 1 2023-08-28T02:26:17.370 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:26:17.462 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:26:17.512 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:17.574 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:17 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:26:17.575 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:17 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:26:17.575 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:17 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:26:17.575 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:17 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:26:17.575 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:17 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:26:17.575 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:17 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:26:17.626 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:26:17.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:17.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:26:17.740 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:26:17.758 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:26:17.827 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:18.105 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:26:18.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:18.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:26:18.216 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:26:18.233 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.6 dump_blocked_ops 2023-08-28T02:26:18.334 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:18.384 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:18 smithi143 ceph-mon[183059]: pgmap v416: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 914 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 1990/43728 objects degraded (4.551%); 2470/43728 objects misplaced (5.649%); 2.5 MiB/s, 40 objects/s recovering 2023-08-28T02:26:18.384 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:18 smithi143 ceph-mon[189871]: pgmap v416: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 914 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 1990/43728 objects degraded (4.551%); 2470/43728 objects misplaced (5.649%); 2.5 MiB/s, 40 objects/s recovering 2023-08-28T02:26:18.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:18 smithi169 ceph-mon[159415]: pgmap v416: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 914 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 1990/43728 objects degraded (4.551%); 2470/43728 objects misplaced (5.649%); 2.5 MiB/s, 40 objects/s recovering 2023-08-28T02:26:18.571 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:26:18.581 INFO:teuthology.orchestra.run.smithi143.stderr:dumped all 2023-08-28T02:26:18.600 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:26:18.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:18.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:26:18.705 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:26:18.766 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:18.793 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:26:18.850 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:26:18.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:18.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:26:18.972 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:26:19.039 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:19.190 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:26:19.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:19.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:26:19.300 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:26:19.328 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:19.576 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:26:19.603 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:26:19.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:19.610 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:26:19.711 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:26:19.809 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:20.328 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:26:20.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:20.344 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:26:20.348 INFO:teuthology.orchestra.run.smithi143.stdout:{"pg_ready":true,"pg_map":{"version":416,"stamp":"2023-08-28T02:26:17.165424+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":958142411,"num_objects":14576,"num_object_clones":0,"num_object_copies":43728,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1990,"num_objects_misplaced":2470,"num_objects_unfound":0,"num_objects_dirty":14576,"num_whiteouts":0,"num_read":6274,"num_read_kb":13707,"num_write":206840,"num_write_kb":7064102,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6338,"num_bytes_recovered":413543046,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139749,"ondisk_log_size":139749,"up":267,"acting":266,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":279,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":4932920,"kb_used_data":3257336,"kb_used_omap":29,"kb_used_meta":1675362,"kb_avail":745028296,"statfs":{"total":767960285184,"available":762908975104,"internally_reserved":0,"allocated":3335512064,"data_stored":3329968952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30674,"internal_metadata":1715570734},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-395968512,"num_objects":-6042,"num_object_clones":0,"num_object_copies":-18126,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-1108,"num_objects_unfound":0,"num_objects_dirty":-6042,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6042,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":404,"num_bytes_recovered":26476544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004548"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":164,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.377405+0000","last_change":"2023-08-28T02:20:19.849807+0000","last_active":"2023-08-28T02:26:01.377405+0000","last_peered":"2023-08-28T02:26:01.377405+0000","last_clean":"2023-08-28T02:26:01.377405+0000","last_became_active":"2023-08-28T02:20:19.849464+0000","last_became_peered":"2023-08-28T02:20:19.849464+0000","last_unstale":"2023-08-28T02:26:01.377405+0000","last_undegraded":"2023-08-28T02:26:01.377405+0000","last_fullsized":"2023-08-28T02:26:01.377405+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":38,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.383093+0000","last_change":"2023-08-28T02:23:59.204957+0000","last_active":"2023-08-28T02:26:01.383093+0000","last_peered":"2023-08-28T02:26:01.383093+0000","last_clean":"2023-08-28T02:26:01.383093+0000","last_became_active":"2023-08-28T02:23:59.204717+0000","last_became_peered":"2023-08-28T02:23:59.204717+0000","last_unstale":"2023-08-28T02:26:01.383093+0000","last_undegraded":"2023-08-28T02:26:01.383093+0000","last_fullsized":"2023-08-28T02:26:01.383093+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:34:58.977269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":139,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.372535+0000","last_change":"2023-08-28T02:23:59.212721+0000","last_active":"2023-08-28T02:26:01.372535+0000","last_peered":"2023-08-28T02:26:01.372535+0000","last_clean":"2023-08-28T02:26:01.372535+0000","last_became_active":"2023-08-28T02:23:59.212446+0000","last_became_peered":"2023-08-28T02:23:59.212446+0000","last_unstale":"2023-08-28T02:26:01.372535+0000","last_undegraded":"2023-08-28T02:26:01.372535+0000","last_fullsized":"2023-08-28T02:26:01.372535+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:03:54.701399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":167,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.403894+0000","last_change":"2023-08-28T02:23:59.223634+0000","last_active":"2023-08-28T02:26:01.403894+0000","last_peered":"2023-08-28T02:26:01.403894+0000","last_clean":"2023-08-28T02:26:01.403894+0000","last_became_active":"2023-08-28T02:23:59.222704+0000","last_became_peered":"2023-08-28T02:23:59.222704+0000","last_unstale":"2023-08-28T02:26:01.403894+0000","last_undegraded":"2023-08-28T02:26:01.403894+0000","last_fullsized":"2023-08-28T02:26:01.403894+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:51:52.752361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":38,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.386878+0000","last_change":"2023-08-28T02:23:59.220459+0000","last_active":"2023-08-28T02:26:01.386878+0000","last_peered":"2023-08-28T02:26:01.386878+0000","last_clean":"2023-08-28T02:26:01.386878+0000","last_became_active":"2023-08-28T02:23:59.219274+0000","last_became_peered":"2023-08-28T02:23:59.219274+0000","last_unstale":"2023-08-28T02:26:01.386878+0000","last_undegraded":"2023-08-28T02:26:01.386878+0000","last_fullsized":"2023-08-28T02:26:01.386878+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:31:53.889311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":175,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.388954+0000","last_change":"2023-08-28T02:21:59.183009+0000","last_active":"2023-08-28T02:26:01.388954+0000","last_peered":"2023-08-28T02:26:01.388954+0000","last_clean":"2023-08-28T02:26:01.388954+0000","last_became_active":"2023-08-28T02:21:59.182621+0000","last_became_peered":"2023-08-28T02:21:59.182621+0000","last_unstale":"2023-08-28T02:26:01.388954+0000","last_undegraded":"2023-08-28T02:26:01.388954+0000","last_fullsized":"2023-08-28T02:26:01.388954+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":148,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.404479+0000","last_change":"2023-08-28T02:23:59.227592+0000","last_active":"2023-08-28T02:26:01.404479+0000","last_peered":"2023-08-28T02:26:01.404479+0000","last_clean":"2023-08-28T02:26:01.404479+0000","last_became_active":"2023-08-28T02:23:59.227342+0000","last_became_peered":"2023-08-28T02:23:59.227342+0000","last_unstale":"2023-08-28T02:26:01.404479+0000","last_undegraded":"2023-08-28T02:26:01.404479+0000","last_fullsized":"2023-08-28T02:26:01.404479+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:08:19.874396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":38,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.386833+0000","last_change":"2023-08-28T02:23:59.220634+0000","last_active":"2023-08-28T02:26:01.386833+0000","last_peered":"2023-08-28T02:26:01.386833+0000","last_clean":"2023-08-28T02:26:01.386833+0000","last_became_active":"2023-08-28T02:23:59.220177+0000","last_became_peered":"2023-08-28T02:23:59.220177+0000","last_unstale":"2023-08-28T02:26:01.386833+0000","last_undegraded":"2023-08-28T02:26:01.386833+0000","last_fullsized":"2023-08-28T02:26:01.386833+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:16:26.906064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":166,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.379078+0000","last_change":"2023-08-28T02:23:38.339581+0000","last_active":"2023-08-28T02:26:01.379078+0000","last_peered":"2023-08-28T02:26:01.379078+0000","last_clean":"2023-08-28T02:26:01.379078+0000","last_became_active":"2023-08-28T02:23:38.339244+0000","last_became_peered":"2023-08-28T02:23:38.339244+0000","last_unstale":"2023-08-28T02:26:01.379078+0000","last_undegraded":"2023-08-28T02:26:01.379078+0000","last_fullsized":"2023-08-28T02:26:01.379078+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":38,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.389601+0000","last_change":"2023-08-28T02:23:59.220551+0000","last_active":"2023-08-28T02:26:01.389601+0000","last_peered":"2023-08-28T02:26:01.389601+0000","last_clean":"2023-08-28T02:26:01.389601+0000","last_became_active":"2023-08-28T02:23:59.219681+0000","last_became_peered":"2023-08-28T02:23:59.219681+0000","last_unstale":"2023-08-28T02:26:01.389601+0000","last_undegraded":"2023-08-28T02:26:01.389601+0000","last_fullsized":"2023-08-28T02:26:01.389601+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:39:58.811256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":67,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.388014+0000","last_change":"2023-08-28T02:23:59.316561+0000","last_active":"2023-08-28T02:26:01.388014+0000","last_peered":"2023-08-28T02:26:01.388014+0000","last_clean":"2023-08-28T02:26:01.388014+0000","last_became_active":"2023-08-28T02:23:59.227433+0000","last_became_peered":"2023-08-28T02:23:59.227433+0000","last_unstale":"2023-08-28T02:26:01.388014+0000","last_undegraded":"2023-08-28T02:26:01.388014+0000","last_fullsized":"2023-08-28T02:26:01.388014+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":122,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.379997+0000","last_change":"2023-08-28T02:20:19.851616+0000","last_active":"2023-08-28T02:26:01.379997+0000","last_peered":"2023-08-28T02:26:01.379997+0000","last_clean":"2023-08-28T02:26:01.379997+0000","last_became_active":"2023-08-28T02:20:19.851322+0000","last_became_peered":"2023-08-28T02:20:19.851322+0000","last_unstale":"2023-08-28T02:26:01.379997+0000","last_undegraded":"2023-08-28T02:26:01.379997+0000","last_fullsized":"2023-08-28T02:26:01.379997+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:03:37.565646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"86'22","reported_seq":193,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.376330+0000","last_change":"2023-08-28T02:23:38.368275+0000","last_active":"2023-08-28T02:26:01.376330+0000","last_peered":"2023-08-28T02:26:01.376330+0000","last_clean":"2023-08-28T02:26:01.376330+0000","last_became_active":"2023-08-28T02:23:38.342097+0000","last_became_peered":"2023-08-28T02:23:38.342097+0000","last_unstale":"2023-08-28T02:26:01.376330+0000","last_undegraded":"2023-08-28T02:26:01.376330+0000","last_fullsized":"2023-08-28T02:26:01.376330+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":161,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.375627+0000","last_change":"2023-08-28T02:19:51.204118+0000","last_active":"2023-08-28T02:26:01.375627+0000","last_peered":"2023-08-28T02:26:01.375627+0000","last_clean":"2023-08-28T02:26:01.375627+0000","last_became_active":"2023-08-28T02:19:51.203745+0000","last_became_peered":"2023-08-28T02:19:51.203745+0000","last_unstale":"2023-08-28T02:26:01.375627+0000","last_undegraded":"2023-08-28T02:26:01.375627+0000","last_fullsized":"2023-08-28T02:26:01.375627+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":52,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.387870+0000","last_change":"2023-08-28T02:23:38.983634+0000","last_active":"2023-08-28T02:26:01.387870+0000","last_peered":"2023-08-28T02:26:01.387870+0000","last_clean":"2023-08-28T02:26:01.387870+0000","last_became_active":"2023-08-28T02:23:38.344566+0000","last_became_peered":"2023-08-28T02:23:38.344566+0000","last_unstale":"2023-08-28T02:26:01.387870+0000","last_undegraded":"2023-08-28T02:26:01.387870+0000","last_fullsized":"2023-08-28T02:26:01.387870+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":38,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.386488+0000","last_change":"2023-08-28T02:23:59.216960+0000","last_active":"2023-08-28T02:26:01.386488+0000","last_peered":"2023-08-28T02:26:01.386488+0000","last_clean":"2023-08-28T02:26:01.386488+0000","last_became_active":"2023-08-28T02:23:59.215657+0000","last_became_peered":"2023-08-28T02:23:59.215657+0000","last_unstale":"2023-08-28T02:26:01.386488+0000","last_undegraded":"2023-08-28T02:26:01.386488+0000","last_fullsized":"2023-08-28T02:26:01.386488+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:31:14.407108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":166,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.403653+0000","last_change":"2023-08-28T02:23:59.223066+0000","last_active":"2023-08-28T02:26:01.403653+0000","last_peered":"2023-08-28T02:26:01.403653+0000","last_clean":"2023-08-28T02:26:01.403653+0000","last_became_active":"2023-08-28T02:23:59.222731+0000","last_became_peered":"2023-08-28T02:23:59.222731+0000","last_unstale":"2023-08-28T02:26:01.403653+0000","last_undegraded":"2023-08-28T02:26:01.403653+0000","last_fullsized":"2023-08-28T02:26:01.403653+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:48:37.558266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":208,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.379065+0000","last_change":"2023-08-28T02:23:59.210469+0000","last_active":"2023-08-28T02:26:01.379065+0000","last_peered":"2023-08-28T02:26:01.379065+0000","last_clean":"2023-08-28T02:26:01.379065+0000","last_became_active":"2023-08-28T02:23:59.210031+0000","last_became_peered":"2023-08-28T02:23:59.210031+0000","last_unstale":"2023-08-28T02:26:01.379065+0000","last_undegraded":"2023-08-28T02:26:01.379065+0000","last_fullsized":"2023-08-28T02:26:01.379065+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":46,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.391240+0000","last_change":"2023-08-28T02:23:38.370183+0000","last_active":"2023-08-28T02:26:01.391240+0000","last_peered":"2023-08-28T02:26:01.391240+0000","last_clean":"2023-08-28T02:26:01.391240+0000","last_became_active":"2023-08-28T02:23:38.368415+0000","last_became_peered":"2023-08-28T02:23:38.368415+0000","last_unstale":"2023-08-28T02:26:01.391240+0000","last_undegraded":"2023-08-28T02:26:01.391240+0000","last_fullsized":"2023-08-28T02:26:01.391240+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":167,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.403963+0000","last_change":"2023-08-28T02:23:59.220932+0000","last_active":"2023-08-28T02:26:01.403963+0000","last_peered":"2023-08-28T02:26:01.403963+0000","last_clean":"2023-08-28T02:26:01.403963+0000","last_became_active":"2023-08-28T02:23:59.220488+0000","last_became_peered":"2023-08-28T02:23:59.220488+0000","last_unstale":"2023-08-28T02:26:01.403963+0000","last_undegraded":"2023-08-28T02:26:01.403963+0000","last_fullsized":"2023-08-28T02:26:01.403963+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:53:09.216340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":155,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.388805+0000","last_change":"2023-08-28T02:18:39.778728+0000","last_active":"2023-08-28T02:26:01.388805+0000","last_peered":"2023-08-28T02:26:01.388805+0000","last_clean":"2023-08-28T02:26:01.388805+0000","last_became_active":"2023-08-28T02:18:39.778283+0000","last_became_peered":"2023-08-28T02:18:39.778283+0000","last_unstale":"2023-08-28T02:26:01.388805+0000","last_undegraded":"2023-08-28T02:26:01.388805+0000","last_fullsized":"2023-08-28T02:26:01.388805+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":38,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.392372+0000","last_change":"2023-08-28T02:23:59.216743+0000","last_active":"2023-08-28T02:26:01.392372+0000","last_peered":"2023-08-28T02:26:01.392372+0000","last_clean":"2023-08-28T02:26:01.392372+0000","last_became_active":"2023-08-28T02:23:59.216416+0000","last_became_peered":"2023-08-28T02:23:59.216416+0000","last_unstale":"2023-08-28T02:26:01.392372+0000","last_undegraded":"2023-08-28T02:26:01.392372+0000","last_fullsized":"2023-08-28T02:26:01.392372+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:25:13.056747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"181'12899","reported_seq":13734,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:17.062565+0000","last_change":"2023-08-28T02:24:09.355130+0000","last_active":"2023-08-28T02:26:17.062565+0000","last_peered":"2023-08-28T02:26:17.062565+0000","last_clean":"2023-08-28T02:26:17.062565+0000","last_became_active":"2023-08-28T02:24:09.341363+0000","last_became_peered":"2023-08-28T02:24:09.341363+0000","last_unstale":"2023-08-28T02:26:17.062565+0000","last_undegraded":"2023-08-28T02:26:17.062565+0000","last_fullsized":"2023-08-28T02:26:17.062565+0000","mapping_epoch":161,"log_start":"132'4300","ondisk_log_start":"132'4300","created":119,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":8599,"log_dups_size":0,"ondisk_log_size":8599,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53280768,"num_objects":813,"num_object_clones":0,"num_object_copies":2439,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":813,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12899,"num_write_kb":438784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":367,"num_bytes_recovered":23789568,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":50,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.388272+0000","last_change":"2023-08-28T02:23:38.657185+0000","last_active":"2023-08-28T02:26:01.388272+0000","last_peered":"2023-08-28T02:26:01.388272+0000","last_clean":"2023-08-28T02:26:01.388272+0000","last_became_active":"2023-08-28T02:23:38.339016+0000","last_became_peered":"2023-08-28T02:23:38.339016+0000","last_unstale":"2023-08-28T02:26:01.388272+0000","last_undegraded":"2023-08-28T02:26:01.388272+0000","last_fullsized":"2023-08-28T02:26:01.388272+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":150,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.378672+0000","last_change":"2023-08-28T02:23:38.339886+0000","last_active":"2023-08-28T02:26:01.378672+0000","last_peered":"2023-08-28T02:26:01.378672+0000","last_clean":"2023-08-28T02:26:01.378672+0000","last_became_active":"2023-08-28T02:23:38.339514+0000","last_became_peered":"2023-08-28T02:23:38.339514+0000","last_unstale":"2023-08-28T02:26:01.378672+0000","last_undegraded":"2023-08-28T02:26:01.378672+0000","last_fullsized":"2023-08-28T02:26:01.378672+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"181'12877","reported_seq":12948,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:13.784647+0000","last_change":"2023-08-28T02:22:00.165260+0000","last_active":"2023-08-28T02:26:13.784647+0000","last_peered":"2023-08-28T02:26:13.784647+0000","last_clean":"2023-08-28T02:26:13.784647+0000","last_became_active":"2023-08-28T02:22:00.164978+0000","last_became_peered":"2023-08-28T02:22:00.164978+0000","last_unstale":"2023-08-28T02:26:13.784647+0000","last_undegraded":"2023-08-28T02:26:13.784647+0000","last_fullsized":"2023-08-28T02:26:13.784647+0000","mapping_epoch":119,"log_start":"129'3700","ondisk_log_start":"129'3700","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":9177,"log_dups_size":0,"ondisk_log_size":9177,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:18:41.711960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62193664,"num_objects":949,"num_object_clones":0,"num_object_copies":2847,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":949,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12877,"num_write_kb":442432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":50,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.388432+0000","last_change":"2023-08-28T02:23:38.555339+0000","last_active":"2023-08-28T02:26:01.388432+0000","last_peered":"2023-08-28T02:26:01.388432+0000","last_clean":"2023-08-28T02:26:01.388432+0000","last_became_active":"2023-08-28T02:23:38.337938+0000","last_became_peered":"2023-08-28T02:23:38.337938+0000","last_unstale":"2023-08-28T02:26:01.388432+0000","last_undegraded":"2023-08-28T02:26:01.388432+0000","last_fullsized":"2023-08-28T02:26:01.388432+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":189,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.375838+0000","last_change":"2023-08-28T02:23:59.208685+0000","last_active":"2023-08-28T02:26:01.375838+0000","last_peered":"2023-08-28T02:26:01.375838+0000","last_clean":"2023-08-28T02:26:01.375838+0000","last_became_active":"2023-08-28T02:23:59.208377+0000","last_became_peered":"2023-08-28T02:23:59.208377+0000","last_unstale":"2023-08-28T02:26:01.375838+0000","last_undegraded":"2023-08-28T02:26:01.375838+0000","last_fullsized":"2023-08-28T02:26:01.375838+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"181'12780","reported_seq":12851,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:13.851861+0000","last_change":"2023-08-28T02:22:00.165317+0000","last_active":"2023-08-28T02:26:13.851861+0000","last_peered":"2023-08-28T02:26:13.851861+0000","last_clean":"2023-08-28T02:26:13.851861+0000","last_became_active":"2023-08-28T02:22:00.164989+0000","last_became_peered":"2023-08-28T02:22:00.164989+0000","last_unstale":"2023-08-28T02:26:13.851861+0000","last_undegraded":"2023-08-28T02:26:13.851861+0000","last_fullsized":"2023-08-28T02:26:13.851861+0000","mapping_epoch":119,"log_start":"129'3600","ondisk_log_start":"129'3600","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":9180,"log_dups_size":0,"ondisk_log_size":9180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:10:46.322659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61603840,"num_objects":940,"num_object_clones":0,"num_object_copies":2820,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":940,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12780,"num_write_kb":439040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":50,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.387797+0000","last_change":"2023-08-28T02:23:38.749428+0000","last_active":"2023-08-28T02:26:01.387797+0000","last_peered":"2023-08-28T02:26:01.387797+0000","last_clean":"2023-08-28T02:26:01.387797+0000","last_became_active":"2023-08-28T02:23:38.339587+0000","last_became_peered":"2023-08-28T02:23:38.339587+0000","last_unstale":"2023-08-28T02:26:01.387797+0000","last_undegraded":"2023-08-28T02:26:01.387797+0000","last_fullsized":"2023-08-28T02:26:01.387797+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":207,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.391178+0000","last_change":"2023-08-28T02:23:38.340065+0000","last_active":"2023-08-28T02:26:01.391178+0000","last_peered":"2023-08-28T02:26:01.391178+0000","last_clean":"2023-08-28T02:26:01.391178+0000","last_became_active":"2023-08-28T02:23:38.328689+0000","last_became_peered":"2023-08-28T02:23:38.328689+0000","last_unstale":"2023-08-28T02:26:01.391178+0000","last_undegraded":"2023-08-28T02:26:01.391178+0000","last_fullsized":"2023-08-28T02:26:01.391178+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"181'12853","reported_seq":19387,"reported_epoch":181,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-08-28T02:26:13.866206+0000","last_change":"2023-08-28T02:24:08.250989+0000","last_active":"2023-08-28T02:26:13.866206+0000","last_peered":"2023-08-28T02:26:13.866206+0000","last_clean":"2023-08-28T02:23:58.182539+0000","last_became_active":"2023-08-28T02:24:00.420432+0000","last_became_peered":"2023-08-28T02:24:00.420432+0000","last_unstale":"2023-08-28T02:26:13.866206+0000","last_undegraded":"2023-08-28T02:24:00.242280+0000","last_fullsized":"2023-08-28T02:24:00.199333+0000","mapping_epoch":154,"log_start":"129'3700","ondisk_log_start":"129'3700","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":9153,"log_dups_size":0,"ondisk_log_size":9153,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:57:53.818041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59834368,"num_objects":913,"num_object_clones":0,"num_object_copies":2739,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1990,"num_objects_misplaced":859,"num_objects_unfound":0,"num_objects_dirty":913,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12853,"num_write_kb":440512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3227,"num_bytes_recovered":211484672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4],"avail_no_missing":["2","0","4"],"object_location_counts":[{"shards":"0,2,4","objects":859},{"shards":"2,4","objects":-1936}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":166,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.378973+0000","last_change":"2023-08-28T02:23:59.211466+0000","last_active":"2023-08-28T02:26:01.378973+0000","last_peered":"2023-08-28T02:26:01.378973+0000","last_clean":"2023-08-28T02:26:01.378973+0000","last_became_active":"2023-08-28T02:23:59.211175+0000","last_became_peered":"2023-08-28T02:23:59.211175+0000","last_unstale":"2023-08-28T02:26:01.378973+0000","last_undegraded":"2023-08-28T02:26:01.378973+0000","last_fullsized":"2023-08-28T02:26:01.378973+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":142,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.403482+0000","last_change":"2023-08-28T02:19:51.205245+0000","last_active":"2023-08-28T02:26:01.403482+0000","last_peered":"2023-08-28T02:26:01.403482+0000","last_clean":"2023-08-28T02:26:01.403482+0000","last_became_active":"2023-08-28T02:19:51.204896+0000","last_became_peered":"2023-08-28T02:19:51.204896+0000","last_unstale":"2023-08-28T02:26:01.403482+0000","last_undegraded":"2023-08-28T02:26:01.403482+0000","last_fullsized":"2023-08-28T02:26:01.403482+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:51:24.634702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"181'12879","reported_seq":13783,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:13.866887+0000","last_change":"2023-08-28T02:24:04.294764+0000","last_active":"2023-08-28T02:26:13.866887+0000","last_peered":"2023-08-28T02:26:13.866887+0000","last_clean":"2023-08-28T02:26:13.866887+0000","last_became_active":"2023-08-28T02:24:04.280269+0000","last_became_peered":"2023-08-28T02:24:04.280269+0000","last_unstale":"2023-08-28T02:26:13.866887+0000","last_undegraded":"2023-08-28T02:26:13.866887+0000","last_fullsized":"2023-08-28T02:26:13.866887+0000","mapping_epoch":157,"log_start":"129'3700","ondisk_log_start":"129'3700","created":119,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":9179,"log_dups_size":0,"ondisk_log_size":9179,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:18:55.996939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64880664,"num_objects":991,"num_object_clones":0,"num_object_copies":2973,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":991,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12879,"num_write_kb":443777,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":403,"num_bytes_recovered":26345496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":43,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.301933+0000","last_change":"2023-08-28T02:23:59.230059+0000","last_active":"2023-08-28T02:26:11.301933+0000","last_peered":"2023-08-28T02:26:11.301933+0000","last_clean":"2023-08-28T02:26:11.301933+0000","last_became_active":"2023-08-28T02:23:59.217783+0000","last_became_peered":"2023-08-28T02:23:59.217783+0000","last_unstale":"2023-08-28T02:26:11.301933+0000","last_undegraded":"2023-08-28T02:26:11.301933+0000","last_fullsized":"2023-08-28T02:26:11.301933+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:00:12.145395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":174,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.376173+0000","last_change":"2023-08-28T02:23:59.211286+0000","last_active":"2023-08-28T02:26:01.376173+0000","last_peered":"2023-08-28T02:26:01.376173+0000","last_clean":"2023-08-28T02:26:01.376173+0000","last_became_active":"2023-08-28T02:23:59.210942+0000","last_became_peered":"2023-08-28T02:23:59.210942+0000","last_unstale":"2023-08-28T02:26:01.376173+0000","last_undegraded":"2023-08-28T02:26:01.376173+0000","last_fullsized":"2023-08-28T02:26:01.376173+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"181'13011","reported_seq":13083,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:13.438606+0000","last_change":"2023-08-28T02:22:00.162375+0000","last_active":"2023-08-28T02:26:13.438606+0000","last_peered":"2023-08-28T02:26:13.438606+0000","last_clean":"2023-08-28T02:26:13.438606+0000","last_became_active":"2023-08-28T02:22:00.162190+0000","last_became_peered":"2023-08-28T02:22:00.162190+0000","last_unstale":"2023-08-28T02:26:13.438606+0000","last_undegraded":"2023-08-28T02:26:13.438606+0000","last_fullsized":"2023-08-28T02:26:13.438606+0000","mapping_epoch":119,"log_start":"133'4639","ondisk_log_start":"133'4639","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":8372,"log_dups_size":0,"ondisk_log_size":8372,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:40:08.985631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63766528,"num_objects":973,"num_object_clones":0,"num_object_copies":2919,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":973,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13011,"num_write_kb":447488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":176,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.391289+0000","last_change":"2023-08-28T02:21:59.179699+0000","last_active":"2023-08-28T02:26:01.391289+0000","last_peered":"2023-08-28T02:26:01.391289+0000","last_clean":"2023-08-28T02:26:01.391289+0000","last_became_active":"2023-08-28T02:21:59.179345+0000","last_became_peered":"2023-08-28T02:21:59.179345+0000","last_unstale":"2023-08-28T02:26:01.391289+0000","last_undegraded":"2023-08-28T02:26:01.391289+0000","last_fullsized":"2023-08-28T02:26:01.391289+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":200,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.376371+0000","last_change":"2023-08-28T02:22:44.083121+0000","last_active":"2023-08-28T02:26:01.376371+0000","last_peered":"2023-08-28T02:26:01.376371+0000","last_clean":"2023-08-28T02:26:01.376371+0000","last_became_active":"2023-08-28T02:22:44.082776+0000","last_became_peered":"2023-08-28T02:22:44.082776+0000","last_unstale":"2023-08-28T02:26:01.376371+0000","last_undegraded":"2023-08-28T02:26:01.376371+0000","last_fullsized":"2023-08-28T02:26:01.376371+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"181'12785","reported_seq":12855,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:12.001826+0000","last_change":"2023-08-28T02:22:00.167630+0000","last_active":"2023-08-28T02:26:12.001826+0000","last_peered":"2023-08-28T02:26:12.001826+0000","last_clean":"2023-08-28T02:26:12.001826+0000","last_became_active":"2023-08-28T02:22:00.167305+0000","last_became_peered":"2023-08-28T02:22:00.167305+0000","last_unstale":"2023-08-28T02:26:12.001826+0000","last_undegraded":"2023-08-28T02:26:12.001826+0000","last_fullsized":"2023-08-28T02:26:12.001826+0000","mapping_epoch":119,"log_start":"129'3600","ondisk_log_start":"129'3600","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":9185,"log_dups_size":0,"ondisk_log_size":9185,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66387968,"num_objects":1013,"num_object_clones":0,"num_object_copies":3039,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1013,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12785,"num_write_kb":441536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":90,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.375907+0000","last_change":"2023-08-28T02:23:38.347043+0000","last_active":"2023-08-28T02:26:01.375907+0000","last_peered":"2023-08-28T02:26:01.375907+0000","last_clean":"2023-08-28T02:26:01.375907+0000","last_became_active":"2023-08-28T02:23:38.346680+0000","last_became_peered":"2023-08-28T02:23:38.346680+0000","last_unstale":"2023-08-28T02:26:01.375907+0000","last_undegraded":"2023-08-28T02:26:01.375907+0000","last_fullsized":"2023-08-28T02:26:01.375907+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":110,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.404088+0000","last_change":"2023-08-28T02:19:51.204017+0000","last_active":"2023-08-28T02:26:01.404088+0000","last_peered":"2023-08-28T02:26:01.404088+0000","last_clean":"2023-08-28T02:26:01.404088+0000","last_became_active":"2023-08-28T02:19:51.203627+0000","last_became_peered":"2023-08-28T02:19:51.203627+0000","last_unstale":"2023-08-28T02:26:01.404088+0000","last_undegraded":"2023-08-28T02:26:01.404088+0000","last_fullsized":"2023-08-28T02:26:01.404088+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:14:06.231701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.e","version":"181'13144","reported_seq":13212,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:17.084090+0000","last_change":"2023-08-28T02:22:00.166490+0000","last_active":"2023-08-28T02:26:17.084090+0000","last_peered":"2023-08-28T02:26:17.084090+0000","last_clean":"2023-08-28T02:26:17.084090+0000","last_became_active":"2023-08-28T02:22:00.166140+0000","last_became_peered":"2023-08-28T02:22:00.166140+0000","last_unstale":"2023-08-28T02:26:17.084090+0000","last_undegraded":"2023-08-28T02:26:17.084090+0000","last_fullsized":"2023-08-28T02:26:17.084090+0000","mapping_epoch":119,"log_start":"134'5200","ondisk_log_start":"134'5200","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":7944,"log_dups_size":0,"ondisk_log_size":7944,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54525952,"num_objects":832,"num_object_clones":0,"num_object_copies":2496,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":832,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13144,"num_write_kb":447232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":39,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:08.930487+0000","last_change":"2023-08-28T02:23:59.216950+0000","last_active":"2023-08-28T02:26:08.930487+0000","last_peered":"2023-08-28T02:26:08.930487+0000","last_clean":"2023-08-28T02:26:08.930487+0000","last_became_active":"2023-08-28T02:23:59.216373+0000","last_became_peered":"2023-08-28T02:23:59.216373+0000","last_unstale":"2023-08-28T02:26:08.930487+0000","last_undegraded":"2023-08-28T02:26:08.930487+0000","last_fullsized":"2023-08-28T02:26:08.930487+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:13:14.736771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":158,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.375907+0000","last_change":"2023-08-28T02:23:59.220269+0000","last_active":"2023-08-28T02:26:01.375907+0000","last_peered":"2023-08-28T02:26:01.375907+0000","last_clean":"2023-08-28T02:26:01.375907+0000","last_became_active":"2023-08-28T02:23:59.219966+0000","last_became_peered":"2023-08-28T02:23:59.219966+0000","last_unstale":"2023-08-28T02:26:01.375907+0000","last_undegraded":"2023-08-28T02:26:01.375907+0000","last_fullsized":"2023-08-28T02:26:01.375907+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:16:08.119909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"78'126","reported_seq":658,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:03.441374+0000","last_change":"2023-08-28T02:23:39.351975+0000","last_active":"2023-08-28T02:26:03.441374+0000","last_peered":"2023-08-28T02:26:03.441374+0000","last_clean":"2023-08-28T02:26:03.441374+0000","last_became_active":"2023-08-28T02:23:39.351437+0000","last_became_peered":"2023-08-28T02:23:39.351437+0000","last_unstale":"2023-08-28T02:26:03.441374+0000","last_undegraded":"2023-08-28T02:26:03.441374+0000","last_fullsized":"2023-08-28T02:26:03.441374+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:25:22.203120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":743,"num_read_kb":1331,"num_write":125,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":181,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.379082+0000","last_change":"2023-08-28T02:23:59.207148+0000","last_active":"2023-08-28T02:26:01.379082+0000","last_peered":"2023-08-28T02:26:01.379082+0000","last_clean":"2023-08-28T02:26:01.379082+0000","last_became_active":"2023-08-28T02:23:59.206802+0000","last_became_peered":"2023-08-28T02:23:59.206802+0000","last_unstale":"2023-08-28T02:26:01.379082+0000","last_undegraded":"2023-08-28T02:26:01.379082+0000","last_fullsized":"2023-08-28T02:26:01.379082+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":78,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.380301+0000","last_change":"2023-08-28T02:21:43.238044+0000","last_active":"2023-08-28T02:26:01.380301+0000","last_peered":"2023-08-28T02:26:01.380301+0000","last_clean":"2023-08-28T02:26:01.380301+0000","last_became_active":"2023-08-28T02:21:43.237792+0000","last_became_peered":"2023-08-28T02:21:43.237792+0000","last_unstale":"2023-08-28T02:26:01.380301+0000","last_undegraded":"2023-08-28T02:26:01.380301+0000","last_fullsized":"2023-08-28T02:26:01.380301+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:11:35.488503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"181'12727","reported_seq":12800,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:13.631377+0000","last_change":"2023-08-28T02:22:00.166578+0000","last_active":"2023-08-28T02:26:13.631377+0000","last_peered":"2023-08-28T02:26:13.631377+0000","last_clean":"2023-08-28T02:26:13.631377+0000","last_became_active":"2023-08-28T02:22:00.166311+0000","last_became_peered":"2023-08-28T02:22:00.166311+0000","last_unstale":"2023-08-28T02:26:13.631377+0000","last_undegraded":"2023-08-28T02:26:13.631377+0000","last_fullsized":"2023-08-28T02:26:13.631377+0000","mapping_epoch":119,"log_start":"134'4941","ondisk_log_start":"134'4941","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":7786,"log_dups_size":0,"ondisk_log_size":7786,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:40:38.502180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60489728,"num_objects":923,"num_object_clones":0,"num_object_copies":2769,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":923,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12727,"num_write_kb":436800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"181'12795","reported_seq":12867,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:13.640343+0000","last_change":"2023-08-28T02:22:00.160991+0000","last_active":"2023-08-28T02:26:13.640343+0000","last_peered":"2023-08-28T02:26:13.640343+0000","last_clean":"2023-08-28T02:26:13.640343+0000","last_became_active":"2023-08-28T02:22:00.160755+0000","last_became_peered":"2023-08-28T02:22:00.160755+0000","last_unstale":"2023-08-28T02:26:13.640343+0000","last_undegraded":"2023-08-28T02:26:13.640343+0000","last_fullsized":"2023-08-28T02:26:13.640343+0000","mapping_epoch":119,"log_start":"134'5098","ondisk_log_start":"134'5098","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":7697,"log_dups_size":0,"ondisk_log_size":7697,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:42:59.109288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62324736,"num_objects":951,"num_object_clones":0,"num_object_copies":2853,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":951,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12795,"num_write_kb":439872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"78'204","reported_seq":1097,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.378841+0000","last_change":"2023-08-28T02:19:50.219949+0000","last_active":"2023-08-28T02:26:01.378841+0000","last_peered":"2023-08-28T02:26:01.378841+0000","last_clean":"2023-08-28T02:26:01.378841+0000","last_became_active":"2023-08-28T02:19:50.219356+0000","last_became_peered":"2023-08-28T02:19:50.219356+0000","last_unstale":"2023-08-28T02:26:01.378841+0000","last_undegraded":"2023-08-28T02:26:01.378841+0000","last_fullsized":"2023-08-28T02:26:01.378841+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":204,"log_dups_size":0,"ondisk_log_size":204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1521,"num_read_kb":2511,"num_write":178,"num_write_kb":1175,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":148,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.376080+0000","last_change":"2023-08-28T02:18:39.779607+0000","last_active":"2023-08-28T02:26:01.376080+0000","last_peered":"2023-08-28T02:26:01.376080+0000","last_clean":"2023-08-28T02:26:01.376080+0000","last_became_active":"2023-08-28T02:18:39.779303+0000","last_became_peered":"2023-08-28T02:18:39.779303+0000","last_unstale":"2023-08-28T02:26:01.376080+0000","last_undegraded":"2023-08-28T02:26:01.376080+0000","last_fullsized":"2023-08-28T02:26:01.376080+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":139,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:03.441353+0000","last_change":"2023-08-28T02:23:59.221979+0000","last_active":"2023-08-28T02:26:03.441353+0000","last_peered":"2023-08-28T02:26:03.441353+0000","last_clean":"2023-08-28T02:26:03.441353+0000","last_became_active":"2023-08-28T02:23:59.221619+0000","last_became_peered":"2023-08-28T02:23:59.221619+0000","last_unstale":"2023-08-28T02:26:03.441353+0000","last_undegraded":"2023-08-28T02:26:03.441353+0000","last_fullsized":"2023-08-28T02:26:03.441353+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:54:45.892284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.6","version":"181'12612","reported_seq":13397,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:13.857701+0000","last_change":"2023-08-28T02:24:07.218778+0000","last_active":"2023-08-28T02:26:13.857701+0000","last_peered":"2023-08-28T02:26:13.857701+0000","last_clean":"2023-08-28T02:26:13.857701+0000","last_became_active":"2023-08-28T02:24:07.205246+0000","last_became_peered":"2023-08-28T02:24:07.205246+0000","last_unstale":"2023-08-28T02:26:13.857701+0000","last_undegraded":"2023-08-28T02:26:13.857701+0000","last_fullsized":"2023-08-28T02:26:13.857701+0000","mapping_epoch":159,"log_start":"129'3500","ondisk_log_start":"129'3500","created":119,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":9112,"log_dups_size":0,"ondisk_log_size":9112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:30:08.510532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59244544,"num_objects":904,"num_object_clones":0,"num_object_copies":2712,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":904,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12612,"num_write_kb":432512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":342,"num_bytes_recovered":22413312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"78'47","reported_seq":395,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.376815+0000","last_change":"2023-08-28T02:24:05.799647+0000","last_active":"2023-08-28T02:26:01.376815+0000","last_peered":"2023-08-28T02:26:01.376815+0000","last_clean":"2023-08-28T02:26:01.376815+0000","last_became_active":"2023-08-28T02:23:59.210481+0000","last_became_peered":"2023-08-28T02:23:59.210481+0000","last_unstale":"2023-08-28T02:26:01.376815+0000","last_undegraded":"2023-08-28T02:26:01.376815+0000","last_fullsized":"2023-08-28T02:26:01.376815+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":47,"log_dups_size":0,"ondisk_log_size":47,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:40:18.542375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":162,"num_read_kb":1281,"num_write":48,"num_write_kb":1153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":122,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.374399+0000","last_change":"2023-08-28T02:20:19.851591+0000","last_active":"2023-08-28T02:26:01.374399+0000","last_peered":"2023-08-28T02:26:01.374399+0000","last_clean":"2023-08-28T02:26:01.374399+0000","last_became_active":"2023-08-28T02:20:19.851291+0000","last_became_peered":"2023-08-28T02:20:19.851291+0000","last_unstale":"2023-08-28T02:26:01.374399+0000","last_undegraded":"2023-08-28T02:26:01.374399+0000","last_fullsized":"2023-08-28T02:26:01.374399+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:33:51.259707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":153,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.375855+0000","last_change":"2023-08-28T02:19:50.211156+0000","last_active":"2023-08-28T02:26:01.375855+0000","last_peered":"2023-08-28T02:26:01.375855+0000","last_clean":"2023-08-28T02:26:01.375855+0000","last_became_active":"2023-08-28T02:19:50.210398+0000","last_became_peered":"2023-08-28T02:19:50.210398+0000","last_unstale":"2023-08-28T02:26:01.375855+0000","last_undegraded":"2023-08-28T02:26:01.375855+0000","last_fullsized":"2023-08-28T02:26:01.375855+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:14:41.748197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"78'114","reported_seq":545,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.378936+0000","last_change":"2023-08-28T02:24:01.210739+0000","last_active":"2023-08-28T02:26:01.378936+0000","last_peered":"2023-08-28T02:26:01.378936+0000","last_clean":"2023-08-28T02:26:01.378936+0000","last_became_active":"2023-08-28T02:24:01.210386+0000","last_became_peered":"2023-08-28T02:24:01.210386+0000","last_unstale":"2023-08-28T02:26:01.378936+0000","last_undegraded":"2023-08-28T02:26:01.378936+0000","last_fullsized":"2023-08-28T02:26:01.378936+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":114,"log_dups_size":0,"ondisk_log_size":114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":358,"num_read_kb":1352,"num_write":102,"num_write_kb":1104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"86'139","reported_seq":339,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.387556+0000","last_change":"2023-08-28T02:24:04.276519+0000","last_active":"2023-08-28T02:26:01.387556+0000","last_peered":"2023-08-28T02:26:01.387556+0000","last_clean":"2023-08-28T02:26:01.387556+0000","last_became_active":"2023-08-28T02:24:04.276104+0000","last_became_peered":"2023-08-28T02:24:04.276104+0000","last_unstale":"2023-08-28T02:26:01.387556+0000","last_undegraded":"2023-08-28T02:26:01.387556+0000","last_fullsized":"2023-08-28T02:26:01.387556+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_clean_scrub_stamp":"2023-08-28T02:08:02.043718+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":39,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:03.925622+0000","last_change":"2023-08-28T02:23:59.219981+0000","last_active":"2023-08-28T02:26:03.925622+0000","last_peered":"2023-08-28T02:26:03.925622+0000","last_clean":"2023-08-28T02:26:03.925622+0000","last_became_active":"2023-08-28T02:23:59.219633+0000","last_became_peered":"2023-08-28T02:23:59.219633+0000","last_unstale":"2023-08-28T02:26:03.925622+0000","last_undegraded":"2023-08-28T02:26:03.925622+0000","last_fullsized":"2023-08-28T02:26:03.925622+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:56:14.172518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":149,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.388994+0000","last_change":"2023-08-28T02:23:38.341351+0000","last_active":"2023-08-28T02:26:01.388994+0000","last_peered":"2023-08-28T02:26:01.388994+0000","last_clean":"2023-08-28T02:26:01.388994+0000","last_became_active":"2023-08-28T02:23:38.340997+0000","last_became_peered":"2023-08-28T02:23:38.340997+0000","last_unstale":"2023-08-28T02:26:01.388994+0000","last_undegraded":"2023-08-28T02:26:01.388994+0000","last_fullsized":"2023-08-28T02:26:01.388994+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"181'12865","reported_seq":13703,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:17.076090+0000","last_change":"2023-08-28T02:23:47.215362+0000","last_active":"2023-08-28T02:26:17.076090+0000","last_peered":"2023-08-28T02:26:17.076090+0000","last_clean":"2023-08-28T02:26:17.076090+0000","last_became_active":"2023-08-28T02:23:47.207109+0000","last_became_peered":"2023-08-28T02:23:47.207109+0000","last_unstale":"2023-08-28T02:26:17.076090+0000","last_undegraded":"2023-08-28T02:26:17.076090+0000","last_fullsized":"2023-08-28T02:26:17.076090+0000","mapping_epoch":150,"log_start":"132'4200","ondisk_log_start":"132'4200","created":119,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":8665,"log_dups_size":0,"ondisk_log_size":8665,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51838976,"num_objects":791,"num_object_clones":0,"num_object_copies":2373,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":791,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12865,"num_write_kb":436992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":370,"num_bytes_recovered":24182784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.7","version":"181'12850","reported_seq":12920,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:12.122302+0000","last_change":"2023-08-28T02:22:00.165614+0000","last_active":"2023-08-28T02:26:12.122302+0000","last_peered":"2023-08-28T02:26:12.122302+0000","last_clean":"2023-08-28T02:26:12.122302+0000","last_became_active":"2023-08-28T02:22:00.165136+0000","last_became_peered":"2023-08-28T02:22:00.165136+0000","last_unstale":"2023-08-28T02:26:12.122302+0000","last_undegraded":"2023-08-28T02:26:12.122302+0000","last_fullsized":"2023-08-28T02:26:12.122302+0000","mapping_epoch":119,"log_start":"129'3700","ondisk_log_start":"129'3700","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":9150,"log_dups_size":0,"ondisk_log_size":9150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65798144,"num_objects":1004,"num_object_clones":0,"num_object_copies":3012,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1004,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12850,"num_write_kb":443328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"78'148","reported_seq":630,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.387517+0000","last_change":"2023-08-28T02:24:01.211494+0000","last_active":"2023-08-28T02:26:01.387517+0000","last_peered":"2023-08-28T02:26:01.387517+0000","last_clean":"2023-08-28T02:26:01.387517+0000","last_became_active":"2023-08-28T02:24:01.210957+0000","last_became_peered":"2023-08-28T02:24:01.210957+0000","last_unstale":"2023-08-28T02:26:01.387517+0000","last_undegraded":"2023-08-28T02:26:01.387517+0000","last_fullsized":"2023-08-28T02:26:01.387517+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1092,"num_read_kb":1955,"num_write":127,"num_write_kb":5177,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":148,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.403190+0000","last_change":"2023-08-28T02:23:59.226986+0000","last_active":"2023-08-28T02:26:01.403190+0000","last_peered":"2023-08-28T02:26:01.403190+0000","last_clean":"2023-08-28T02:26:01.403190+0000","last_became_active":"2023-08-28T02:23:59.226529+0000","last_became_peered":"2023-08-28T02:23:59.226529+0000","last_unstale":"2023-08-28T02:26:01.403190+0000","last_undegraded":"2023-08-28T02:26:01.403190+0000","last_fullsized":"2023-08-28T02:26:01.403190+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:22:09.193619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":175,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.403271+0000","last_change":"2023-08-28T02:21:59.167858+0000","last_active":"2023-08-28T02:26:01.403271+0000","last_peered":"2023-08-28T02:26:01.403271+0000","last_clean":"2023-08-28T02:26:01.403271+0000","last_became_active":"2023-08-28T02:21:59.167436+0000","last_became_peered":"2023-08-28T02:21:59.167436+0000","last_unstale":"2023-08-28T02:26:01.403271+0000","last_undegraded":"2023-08-28T02:26:01.403271+0000","last_fullsized":"2023-08-28T02:26:01.403271+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:17:50.717282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"78'103","reported_seq":557,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.387880+0000","last_change":"2023-08-28T02:24:07.212128+0000","last_active":"2023-08-28T02:26:01.387880+0000","last_peered":"2023-08-28T02:26:01.387880+0000","last_clean":"2023-08-28T02:26:01.387880+0000","last_became_active":"2023-08-28T02:24:07.211562+0000","last_became_peered":"2023-08-28T02:24:07.211562+0000","last_unstale":"2023-08-28T02:26:01.387880+0000","last_undegraded":"2023-08-28T02:26:01.387880+0000","last_fullsized":"2023-08-28T02:26:01.387880+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":803,"num_read_kb":2080,"num_write":73,"num_write_kb":1244,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":122,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:03.441392+0000","last_change":"2023-08-28T02:20:19.850916+0000","last_active":"2023-08-28T02:26:03.441392+0000","last_peered":"2023-08-28T02:26:03.441392+0000","last_clean":"2023-08-28T02:26:03.441392+0000","last_became_active":"2023-08-28T02:20:19.850627+0000","last_became_peered":"2023-08-28T02:20:19.850627+0000","last_unstale":"2023-08-28T02:26:03.441392+0000","last_undegraded":"2023-08-28T02:26:03.441392+0000","last_fullsized":"2023-08-28T02:26:03.441392+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:19:14.346189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":134,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.403732+0000","last_change":"2023-08-28T02:23:38.329667+0000","last_active":"2023-08-28T02:26:01.403732+0000","last_peered":"2023-08-28T02:26:01.403732+0000","last_clean":"2023-08-28T02:26:01.403732+0000","last_became_active":"2023-08-28T02:23:38.329175+0000","last_became_peered":"2023-08-28T02:23:38.329175+0000","last_unstale":"2023-08-28T02:26:01.403732+0000","last_undegraded":"2023-08-28T02:26:01.403732+0000","last_fullsized":"2023-08-28T02:26:01.403732+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:21:34.815067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"181'12743","reported_seq":12812,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:17.030832+0000","last_change":"2023-08-28T02:22:00.167814+0000","last_active":"2023-08-28T02:26:17.030832+0000","last_peered":"2023-08-28T02:26:17.030832+0000","last_clean":"2023-08-28T02:26:17.030832+0000","last_became_active":"2023-08-28T02:22:00.167513+0000","last_became_peered":"2023-08-28T02:22:00.167513+0000","last_unstale":"2023-08-28T02:26:17.030832+0000","last_undegraded":"2023-08-28T02:26:17.030832+0000","last_fullsized":"2023-08-28T02:26:17.030832+0000","mapping_epoch":119,"log_start":"128'3000","ondisk_log_start":"128'3000","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":9743,"log_dups_size":0,"ondisk_log_size":9743,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53936128,"num_objects":823,"num_object_clones":0,"num_object_copies":2469,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":823,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12743,"num_write_kb":434112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":43,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:08.930634+0000","last_change":"2023-08-28T02:23:59.232099+0000","last_active":"2023-08-28T02:26:08.930634+0000","last_peered":"2023-08-28T02:26:08.930634+0000","last_clean":"2023-08-28T02:26:08.930634+0000","last_became_active":"2023-08-28T02:23:59.219318+0000","last_became_peered":"2023-08-28T02:23:59.219318+0000","last_unstale":"2023-08-28T02:26:08.930634+0000","last_undegraded":"2023-08-28T02:26:08.930634+0000","last_fullsized":"2023-08-28T02:26:08.930634+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:59:30.093289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'112","reported_seq":425,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.372904+0000","last_change":"2023-08-28T02:19:50.218206+0000","last_active":"2023-08-28T02:26:01.372904+0000","last_peered":"2023-08-28T02:26:01.372904+0000","last_clean":"2023-08-28T02:26:01.372904+0000","last_became_active":"2023-08-28T02:19:50.217637+0000","last_became_peered":"2023-08-28T02:19:50.217637+0000","last_unstale":"2023-08-28T02:26:01.372904+0000","last_undegraded":"2023-08-28T02:26:01.372904+0000","last_fullsized":"2023-08-28T02:26:01.372904+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:12:06.224907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":444,"num_read_kb":473,"num_write":94,"num_write_kb":127,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":39,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.386753+0000","last_change":"2023-08-28T02:23:59.221034+0000","last_active":"2023-08-28T02:26:01.386753+0000","last_peered":"2023-08-28T02:26:01.386753+0000","last_clean":"2023-08-28T02:26:01.386753+0000","last_became_active":"2023-08-28T02:23:59.220730+0000","last_became_peered":"2023-08-28T02:23:59.220730+0000","last_unstale":"2023-08-28T02:26:01.386753+0000","last_undegraded":"2023-08-28T02:26:01.386753+0000","last_fullsized":"2023-08-28T02:26:01.386753+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:25:52.364370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"181'13040","reported_seq":16205,"reported_epoch":181,"state":"active+remapped+backfill_wait","last_fresh":"2023-08-28T02:26:17.101713+0000","last_change":"2023-08-28T02:24:05.805308+0000","last_active":"2023-08-28T02:26:17.101713+0000","last_peered":"2023-08-28T02:26:17.101713+0000","last_clean":"2023-08-28T02:23:25.043301+0000","last_became_active":"2023-08-28T02:24:00.316639+0000","last_became_peered":"2023-08-28T02:24:00.316639+0000","last_unstale":"2023-08-28T02:26:17.101713+0000","last_undegraded":"2023-08-28T02:26:17.101713+0000","last_fullsized":"2023-08-28T02:26:17.101713+0000","mapping_epoch":154,"log_start":"134'5100","ondisk_log_start":"134'5100","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":7940,"log_dups_size":0,"ondisk_log_size":7940,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55705600,"num_objects":850,"num_object_clones":0,"num_object_copies":2550,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":733,"num_objects_unfound":0,"num_objects_dirty":850,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13040,"num_write_kb":444480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1527,"num_bytes_recovered":99942400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,7,0],"avail_no_missing":["4","0","5","7"],"object_location_counts":[{"shards":"0,4,7","objects":850}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"78'122","reported_seq":710,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.388760+0000","last_change":"2023-08-28T02:24:04.276850+0000","last_active":"2023-08-28T02:26:01.388760+0000","last_peered":"2023-08-28T02:26:01.388760+0000","last_clean":"2023-08-28T02:26:01.388760+0000","last_became_active":"2023-08-28T02:24:04.276367+0000","last_became_peered":"2023-08-28T02:24:04.276367+0000","last_unstale":"2023-08-28T02:26:01.388760+0000","last_undegraded":"2023-08-28T02:26:01.388760+0000","last_fullsized":"2023-08-28T02:26:01.388760+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":857,"num_read_kb":2224,"num_write":99,"num_write_kb":2312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":134,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.403821+0000","last_change":"2023-08-28T02:19:50.221001+0000","last_active":"2023-08-28T02:26:01.403821+0000","last_peered":"2023-08-28T02:26:01.403821+0000","last_clean":"2023-08-28T02:26:01.403821+0000","last_became_active":"2023-08-28T02:19:50.220611+0000","last_became_peered":"2023-08-28T02:19:50.220611+0000","last_unstale":"2023-08-28T02:26:01.403821+0000","last_undegraded":"2023-08-28T02:26:01.403821+0000","last_fullsized":"2023-08-28T02:26:01.403821+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:23:10.637630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":132,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.375732+0000","last_change":"2023-08-28T02:11:22.437146+0000","last_active":"2023-08-28T02:26:01.375732+0000","last_peered":"2023-08-28T02:26:01.375732+0000","last_clean":"2023-08-28T02:26:01.375732+0000","last_became_active":"2023-08-28T02:11:22.436798+0000","last_became_peered":"2023-08-28T02:11:22.436798+0000","last_unstale":"2023-08-28T02:26:01.375732+0000","last_undegraded":"2023-08-28T02:26:01.375732+0000","last_fullsized":"2023-08-28T02:26:01.375732+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"181'12874","reported_seq":12953,"reported_epoch":181,"state":"active+remapped+backfill_wait","last_fresh":"2023-08-28T02:26:13.547858+0000","last_change":"2023-08-28T02:24:00.310494+0000","last_active":"2023-08-28T02:26:13.547858+0000","last_peered":"2023-08-28T02:26:13.547858+0000","last_clean":"2023-08-28T02:23:58.174026+0000","last_became_active":"2023-08-28T02:24:00.290543+0000","last_became_peered":"2023-08-28T02:24:00.290543+0000","last_unstale":"2023-08-28T02:26:13.547858+0000","last_undegraded":"2023-08-28T02:26:13.547858+0000","last_fullsized":"2023-08-28T02:26:13.547858+0000","mapping_epoch":154,"log_start":"134'5177","ondisk_log_start":"134'5177","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":7697,"log_dups_size":0,"ondisk_log_size":7697,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:35:56.609960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57540608,"num_objects":878,"num_object_clones":0,"num_object_copies":2634,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":878,"num_objects_unfound":0,"num_objects_dirty":878,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12874,"num_write_kb":440064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[0,1,4],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":878}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":137,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:08.736401+0000","last_change":"2023-08-28T02:23:38.329474+0000","last_active":"2023-08-28T02:26:08.736401+0000","last_peered":"2023-08-28T02:26:08.736401+0000","last_clean":"2023-08-28T02:26:08.736401+0000","last_became_active":"2023-08-28T02:23:38.328505+0000","last_became_peered":"2023-08-28T02:23:38.328505+0000","last_unstale":"2023-08-28T02:26:08.736401+0000","last_undegraded":"2023-08-28T02:26:08.736401+0000","last_fullsized":"2023-08-28T02:26:08.736401+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:12:14.835809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":202,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.380860+0000","last_change":"2023-08-28T02:23:59.212036+0000","last_active":"2023-08-28T02:26:01.380860+0000","last_peered":"2023-08-28T02:26:01.380860+0000","last_clean":"2023-08-28T02:26:01.380860+0000","last_became_active":"2023-08-28T02:23:59.211709+0000","last_became_peered":"2023-08-28T02:23:59.211709+0000","last_unstale":"2023-08-28T02:26:01.380860+0000","last_undegraded":"2023-08-28T02:26:01.380860+0000","last_fullsized":"2023-08-28T02:26:01.380860+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T14:04:25.046626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":150,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.378982+0000","last_change":"2023-08-28T02:19:50.218760+0000","last_active":"2023-08-28T02:26:01.378982+0000","last_peered":"2023-08-28T02:26:01.378982+0000","last_clean":"2023-08-28T02:26:01.378982+0000","last_became_active":"2023-08-28T02:19:50.218376+0000","last_became_peered":"2023-08-28T02:19:50.218376+0000","last_unstale":"2023-08-28T02:26:01.378982+0000","last_undegraded":"2023-08-28T02:26:01.378982+0000","last_fullsized":"2023-08-28T02:26:01.378982+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":180,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.376164+0000","last_change":"2023-08-28T02:23:59.207822+0000","last_active":"2023-08-28T02:26:01.376164+0000","last_peered":"2023-08-28T02:26:01.376164+0000","last_clean":"2023-08-28T02:26:01.376164+0000","last_became_active":"2023-08-28T02:23:59.207475+0000","last_became_peered":"2023-08-28T02:23:59.207475+0000","last_unstale":"2023-08-28T02:26:01.376164+0000","last_undegraded":"2023-08-28T02:26:01.376164+0000","last_fullsized":"2023-08-28T02:26:01.376164+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":135,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.424241+0000","last_change":"2023-08-28T02:11:22.186024+0000","last_active":"2023-08-28T02:26:01.424241+0000","last_peered":"2023-08-28T02:26:01.424241+0000","last_clean":"2023-08-28T02:26:01.424241+0000","last_became_active":"2023-08-28T02:11:22.185500+0000","last_became_peered":"2023-08-28T02:11:22.185500+0000","last_unstale":"2023-08-28T02:26:01.424241+0000","last_undegraded":"2023-08-28T02:26:01.424241+0000","last_fullsized":"2023-08-28T02:26:01.424241+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":180,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.377430+0000","last_change":"2023-08-28T02:23:59.207929+0000","last_active":"2023-08-28T02:26:01.377430+0000","last_peered":"2023-08-28T02:26:01.377430+0000","last_clean":"2023-08-28T02:26:01.377430+0000","last_became_active":"2023-08-28T02:23:59.207589+0000","last_became_peered":"2023-08-28T02:23:59.207589+0000","last_unstale":"2023-08-28T02:26:01.377430+0000","last_undegraded":"2023-08-28T02:26:01.377430+0000","last_fullsized":"2023-08-28T02:26:01.377430+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":39,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.383280+0000","last_change":"2023-08-28T02:23:59.220437+0000","last_active":"2023-08-28T02:26:01.383280+0000","last_peered":"2023-08-28T02:26:01.383280+0000","last_clean":"2023-08-28T02:26:01.383280+0000","last_became_active":"2023-08-28T02:23:59.220091+0000","last_became_peered":"2023-08-28T02:23:59.220091+0000","last_unstale":"2023-08-28T02:26:01.383280+0000","last_undegraded":"2023-08-28T02:26:01.383280+0000","last_fullsized":"2023-08-28T02:26:01.383280+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:51:21.695566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":38,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.389668+0000","last_change":"2023-08-28T02:23:59.217794+0000","last_active":"2023-08-28T02:26:01.389668+0000","last_peered":"2023-08-28T02:26:01.389668+0000","last_clean":"2023-08-28T02:26:01.389668+0000","last_became_active":"2023-08-28T02:23:59.217231+0000","last_became_peered":"2023-08-28T02:23:59.217231+0000","last_unstale":"2023-08-28T02:26:01.389668+0000","last_undegraded":"2023-08-28T02:26:01.389668+0000","last_fullsized":"2023-08-28T02:26:01.389668+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:36:26.297410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":134,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.404402+0000","last_change":"2023-08-28T02:20:19.850252+0000","last_active":"2023-08-28T02:26:01.404402+0000","last_peered":"2023-08-28T02:26:01.404402+0000","last_clean":"2023-08-28T02:26:01.404402+0000","last_became_active":"2023-08-28T02:20:19.849913+0000","last_became_peered":"2023-08-28T02:20:19.849913+0000","last_unstale":"2023-08-28T02:26:01.404402+0000","last_undegraded":"2023-08-28T02:26:01.404402+0000","last_fullsized":"2023-08-28T02:26:01.404402+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:48:47.713465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":176,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.375546+0000","last_change":"2023-08-28T02:21:59.180201+0000","last_active":"2023-08-28T02:26:01.375546+0000","last_peered":"2023-08-28T02:26:01.375546+0000","last_clean":"2023-08-28T02:26:01.375546+0000","last_became_active":"2023-08-28T02:21:59.179863+0000","last_became_peered":"2023-08-28T02:21:59.179863+0000","last_unstale":"2023-08-28T02:26:01.375546+0000","last_undegraded":"2023-08-28T02:26:01.375546+0000","last_fullsized":"2023-08-28T02:26:01.375546+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":953352216,"num_objects":14548,"num_object_clones":0,"num_object_copies":43644,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1990,"num_objects_misplaced":2470,"num_objects_unfound":0,"num_objects_dirty":14548,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":205734,"num_write_kb":7048961,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6236,"num_bytes_recovered":408158232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3242930176,"data_stored":3242917960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":138579,"ondisk_log_size":138579,"up":48,"acting":47,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":9774,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":21246,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26810,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430408,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":576664,"kb_used_data":275344,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93168488,"statfs":{"total":95995035648,"available":95404531712,"internally_reserved":0,"allocated":281952256,"data_stored":281369130,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:25:56 2023","interfaces":[{"interface":"back","average":{"1min":2.536,"5min":3.7240000000000002,"15min":1.5980000000000001},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":9.5679999999999996,"5min":116.755,"15min":116.755},"last":0.49399999999999999},{"interface":"front","average":{"1min":3.1320000000000001,"5min":4.1520000000000001,"15min":1.7889999999999999},"min":{"1min":0.28000000000000003,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":12.206,"5min":116.971,"15min":116.971},"last":0.248}]},{"osd":1,"last update":"Mon Aug 28 02:26:03 2023","interfaces":[{"interface":"back","average":{"1min":1.5800000000000001,"5min":1.143,"15min":0.73299999999999998},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":9.0139999999999993,"5min":9.8710000000000004,"15min":9.8710000000000004},"last":0.26000000000000001},{"interface":"front","average":{"1min":3.0169999999999999,"5min":1.7889999999999999,"15min":0.98799999999999999},"min":{"1min":0.27700000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":11.140000000000001,"5min":11.696999999999999,"15min":11.696999999999999},"last":0.34599999999999997}]},{"osd":2,"last update":"Mon Aug 28 02:25:27 2023","interfaces":[{"interface":"back","average":{"1min":8.6959999999999997,"5min":3.8700000000000001,"15min":1.673},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":119.571,"5min":119.571,"15min":119.571},"last":4.8650000000000002},{"interface":"front","average":{"1min":9.7100000000000009,"5min":4.319,"15min":1.8300000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":119.554,"5min":119.554,"15min":119.554},"last":6.5}]},{"osd":3,"last update":"Mon Aug 28 02:26:03 2023","interfaces":[{"interface":"back","average":{"1min":2.2250000000000001,"5min":2.3260000000000001,"15min":1.2210000000000001},"min":{"1min":0.36799999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":6.5650000000000004,"5min":107.916,"15min":107.916},"last":0.40500000000000003},{"interface":"front","average":{"1min":2.548,"5min":2.9990000000000001,"15min":1.446},"min":{"1min":0.378,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":9.8460000000000001,"5min":112.00700000000001,"15min":112.00700000000001},"last":0.372}]},{"osd":4,"last update":"Mon Aug 28 02:25:27 2023","interfaces":[{"interface":"back","average":{"1min":1.0980000000000001,"5min":2.0350000000000001,"15min":0.996},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":3.8359999999999999,"5min":63.048000000000002,"15min":63.048000000000002},"last":0.32100000000000001},{"interface":"front","average":{"1min":1.659,"5min":1.8759999999999999,"15min":0.96199999999999997},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":8.2720000000000002,"5min":62.912999999999997,"15min":62.912999999999997},"last":0.39100000000000001}]},{"osd":5,"last update":"Mon Aug 28 02:25:27 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":1.554,"15min":0.83999999999999997},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":5.7110000000000003,"5min":63.021000000000001,"15min":63.021000000000001},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.4039999999999999,"5min":2.1779999999999999,"15min":1.089},"min":{"1min":0.35699999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":5.3470000000000004,"5min":63.082999999999998,"15min":63.082999999999998},"last":6.4550000000000001}]},{"osd":6,"last update":"Mon Aug 28 02:25:27 2023","interfaces":[{"interface":"back","average":{"1min":1.716,"5min":2.198,"15min":1.0860000000000001},"min":{"1min":0.317,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":6.0659999999999998,"5min":101.61,"15min":101.61},"last":0.47399999999999998},{"interface":"front","average":{"1min":1.3080000000000001,"5min":2.391,"15min":1.1890000000000001},"min":{"1min":0.34399999999999997,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":5.407,"5min":101.621,"15min":101.621},"last":0.48099999999999998}]}]},{"osd":6,"up_from":42,"seq":180388626635,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":520584,"kb_used_data":211008,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93224568,"statfs":{"total":95995035648,"available":95461957632,"internally_reserved":0,"allocated":216072192,"data_stored":215504984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:26:02 2023","interfaces":[{"interface":"back","average":{"1min":3.48,"5min":4.3949999999999996,"15min":1.845},"min":{"1min":0.245,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":13.260999999999999,"5min":121.247,"15min":121.247},"last":5.7320000000000002},{"interface":"front","average":{"1min":4.4560000000000004,"5min":4.6369999999999996,"15min":1.9430000000000001},"min":{"1min":0.371,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":13.298999999999999,"5min":118.977,"15min":118.977},"last":3.706}]},{"osd":1,"last update":"Mon Aug 28 02:26:02 2023","interfaces":[{"interface":"back","average":{"1min":2.0910000000000002,"5min":2.258,"15min":1.1100000000000001},"min":{"1min":0.307,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":6.7050000000000001,"5min":34.677999999999997,"15min":34.677999999999997},"last":0.27100000000000002},{"interface":"front","average":{"1min":2.7850000000000001,"5min":2.5009999999999999,"15min":1.2130000000000001},"min":{"1min":0.317,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":9.6799999999999997,"5min":34.348999999999997,"15min":34.348999999999997},"last":5.4980000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:25:35 2023","interfaces":[{"interface":"back","average":{"1min":3.0590000000000002,"5min":2.2959999999999998,"15min":1.1719999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":15.762,"5min":49.555999999999997,"15min":49.555999999999997},"last":3.7959999999999998},{"interface":"front","average":{"1min":2.9710000000000001,"5min":2.1160000000000001,"15min":1.089},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":11.503,"5min":49.470999999999997,"15min":49.470999999999997},"last":3.0179999999999998}]},{"osd":3,"last update":"Mon Aug 28 02:26:02 2023","interfaces":[{"interface":"back","average":{"1min":2.2869999999999999,"5min":1.885,"15min":1.075},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":6.4699999999999998,"5min":13.103,"15min":13.103},"last":1.0700000000000001},{"interface":"front","average":{"1min":2.508,"5min":2.0649999999999999,"15min":1.143},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":11.395,"5min":13.09,"15min":13.09},"last":3.766}]},{"osd":4,"last update":"Mon Aug 28 02:26:06 2023","interfaces":[{"interface":"back","average":{"1min":1.5880000000000001,"5min":1.669,"15min":0.86699999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":8.9580000000000002,"5min":10.637,"15min":10.637},"last":1.04},{"interface":"front","average":{"1min":2.0819999999999999,"5min":1.8100000000000001,"15min":0.94299999999999995},"min":{"1min":0.29899999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":9.7690000000000001,"5min":10.055,"15min":10.055},"last":0.51500000000000001}]},{"osd":5,"last update":"Mon Aug 28 02:26:06 2023","interfaces":[{"interface":"back","average":{"1min":1.982,"5min":1.9399999999999999,"15min":0.96499999999999997},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":9.9809999999999999,"5min":20.759,"15min":20.759},"last":2.3490000000000002},{"interface":"front","average":{"1min":1.8220000000000001,"5min":1.484,"15min":0.85099999999999998},"min":{"1min":0.32200000000000001,"5min":0.311,"15min":0.28799999999999998},"max":{"1min":5.5030000000000001,"5min":7.3780000000000001,"15min":7.3780000000000001},"last":0.36399999999999999}]},{"osd":7,"last update":"Mon Aug 28 02:25:33 2023","interfaces":[{"interface":"back","average":{"1min":1.7949999999999999,"5min":1.3320000000000001,"15min":0.80700000000000005},"min":{"1min":0.34799999999999998,"5min":0.314,"15min":0.314},"max":{"1min":6.4109999999999996,"5min":7.1769999999999996,"15min":7.1769999999999996},"last":0.34200000000000003},{"interface":"front","average":{"1min":1.9379999999999999,"5min":1.49,"15min":0.879},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.29799999999999999},"max":{"1min":6.8949999999999996,"5min":7.069,"15min":7.069},"last":0.58499999999999996}]}]},{"osd":1,"up_from":98,"seq":420906795087,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":413856,"kb_used_data":309976,"kb_used_omap":7,"kb_used_meta":103864,"kb_avail":93331296,"statfs":{"total":95995035648,"available":95571247104,"internally_reserved":0,"allocated":317415424,"data_stored":316547622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":106357067},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:26:01 2023","interfaces":[{"interface":"back","average":{"1min":14.807,"5min":4.4260000000000002,"15min":1.6990000000000001},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":128.60599999999999,"5min":128.60599999999999,"15min":128.60599999999999},"last":0.42899999999999999},{"interface":"front","average":{"1min":14.028,"5min":4.4489999999999998,"15min":1.7190000000000001},"min":{"1min":0.23699999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":128.685,"5min":128.685,"15min":128.685},"last":5.9459999999999997}]},{"osd":2,"last update":"Mon Aug 28 02:25:27 2023","interfaces":[{"interface":"back","average":{"1min":5.5549999999999997,"5min":2.2530000000000001,"15min":0.97899999999999998},"min":{"1min":0.26200000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":29.483000000000001,"5min":29.483000000000001,"15min":29.483000000000001},"last":1.3440000000000001},{"interface":"front","average":{"1min":3.286,"5min":1.7470000000000001,"15min":0.83599999999999997},"min":{"1min":0.23200000000000001,"5min":0.221,"15min":0.221},"max":{"1min":19.045000000000002,"5min":19.045000000000002,"15min":19.045000000000002},"last":10.718}]},{"osd":3,"last update":"Mon Aug 28 02:26:01 2023","interfaces":[{"interface":"back","average":{"1min":2.1200000000000001,"5min":1.8140000000000001,"15min":0.84799999999999998},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":12.321999999999999,"5min":15.395,"15min":15.395},"last":6.0979999999999999},{"interface":"front","average":{"1min":3.7250000000000001,"5min":2.1480000000000001,"15min":1.0229999999999999},"min":{"1min":0.26900000000000002,"5min":0.254,"15min":0.254},"max":{"1min":22.361999999999998,"5min":22.361999999999998,"15min":22.361999999999998},"last":0.30399999999999999}]},{"osd":4,"last update":"Mon Aug 28 02:26:01 2023","interfaces":[{"interface":"back","average":{"1min":3.3570000000000002,"5min":1.9339999999999999,"15min":1.0620000000000001},"min":{"1min":0.32800000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":17.600000000000001,"5min":17.600000000000001,"15min":17.600000000000001},"last":2.3919999999999999},{"interface":"front","average":{"1min":2.5499999999999998,"5min":1.556,"15min":0.93200000000000005},"min":{"1min":0.36299999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":19.010000000000002,"5min":19.010000000000002,"15min":19.010000000000002},"last":2.3809999999999998}]},{"osd":5,"last update":"Mon Aug 28 02:26:01 2023","interfaces":[{"interface":"back","average":{"1min":1.754,"5min":2.0089999999999999,"15min":1.0940000000000001},"min":{"1min":0.312,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.0579999999999998,"5min":15.544,"15min":15.544},"last":5.7389999999999999},{"interface":"front","average":{"1min":3.4529999999999998,"5min":1.839,"15min":1.04},"min":{"1min":0.38800000000000001,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":17.646999999999998,"5min":17.646999999999998,"15min":17.646999999999998},"last":0.36699999999999999}]},{"osd":6,"last update":"Mon Aug 28 02:26:01 2023","interfaces":[{"interface":"back","average":{"1min":1.992,"5min":2.3359999999999999,"15min":1.1910000000000001},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":6.0869999999999997,"5min":15.512,"15min":15.512},"last":5.8070000000000004},{"interface":"front","average":{"1min":3.3450000000000002,"5min":2.2810000000000001,"15min":1.169},"min":{"1min":0.39600000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":17.673999999999999,"5min":17.673999999999999,"15min":17.673999999999999},"last":0.34399999999999997}]},{"osd":7,"last update":"Mon Aug 28 02:26:01 2023","interfaces":[{"interface":"back","average":{"1min":3.1930000000000001,"5min":1.8440000000000001,"15min":1.0369999999999999},"min":{"1min":0.39900000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":19.061,"5min":19.061,"15min":19.061},"last":0.45800000000000002},{"interface":"front","average":{"1min":2.3500000000000001,"5min":2.1349999999999998,"15min":1.137},"min":{"1min":0.41899999999999998,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":6.1100000000000003,"5min":15.476000000000001,"15min":15.476000000000001},"last":5.8399999999999999}]}]},{"osd":0,"up_from":90,"seq":386547056733,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1205176,"kb_used_data":1020912,"kb_used_omap":8,"kb_used_meta":184247,"kb_avail":92539976,"statfs":{"total":95995035648,"available":94760935424,"internally_reserved":0,"allocated":1045413888,"data_stored":1044581516,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":188669289},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Aug 28 02:26:09 2023","interfaces":[{"interface":"back","average":{"1min":2.516,"5min":2.754,"15min":1.153},"min":{"1min":0.184,"5min":0.184,"15min":0.184},"max":{"1min":11.282,"5min":40.313000000000002,"15min":40.313000000000002},"last":4.8860000000000001},{"interface":"front","average":{"1min":3.8370000000000002,"5min":2.5899999999999999,"15min":1.111},"min":{"1min":0.23899999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":19.555,"5min":36.655000000000001,"15min":36.655000000000001},"last":0.46200000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:25:27 2023","interfaces":[{"interface":"back","average":{"1min":4.4050000000000002,"5min":1.944,"15min":0.88300000000000001},"min":{"1min":0.40500000000000003,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":15.33,"5min":15.33,"15min":15.33},"last":4.8410000000000002},{"interface":"front","average":{"1min":6.351,"5min":2.484,"15min":1.089},"min":{"1min":0.40799999999999997,"5min":0.224,"15min":0.224},"max":{"1min":18.870999999999999,"5min":18.870999999999999,"15min":18.870999999999999},"last":3.2130000000000001}]},{"osd":3,"last update":"Mon Aug 28 02:26:09 2023","interfaces":[{"interface":"back","average":{"1min":3.77,"5min":2.2090000000000001,"15min":0.97799999999999998},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":12.272,"5min":18.545999999999999,"15min":18.545999999999999},"last":0.23300000000000001},{"interface":"front","average":{"1min":3.3300000000000001,"5min":2.1309999999999998,"15min":0.97299999999999998},"min":{"1min":0.28999999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":12.474,"5min":14.534000000000001,"15min":14.534000000000001},"last":0.28999999999999998}]},{"osd":4,"last update":"Mon Aug 28 02:26:04 2023","interfaces":[{"interface":"back","average":{"1min":2.3700000000000001,"5min":1.8520000000000001,"15min":1.0600000000000001},"min":{"1min":0.29599999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":7.5979999999999999,"5min":9.6649999999999991,"15min":9.6649999999999991},"last":5.0300000000000002},{"interface":"front","average":{"1min":2.7679999999999998,"5min":2.0299999999999998,"15min":1.1419999999999999},"min":{"1min":0.45700000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":12.305,"5min":12.305,"15min":12.305},"last":2.5760000000000001}]},{"osd":5,"last update":"Mon Aug 28 02:26:04 2023","interfaces":[{"interface":"back","average":{"1min":3.0529999999999999,"5min":2.508,"15min":1.268},"min":{"1min":0.41299999999999998,"5min":0.313,"15min":0.313},"max":{"1min":12.337999999999999,"5min":44.000999999999998,"15min":44.000999999999998},"last":0.32600000000000001},{"interface":"front","average":{"1min":3.177,"5min":2.423,"15min":1.238},"min":{"1min":0.438,"5min":0.33100000000000002,"15min":0.30599999999999999},"max":{"1min":12.334,"5min":43.953000000000003,"15min":43.953000000000003},"last":4.9240000000000004}]},{"osd":6,"last update":"Mon Aug 28 02:26:04 2023","interfaces":[{"interface":"back","average":{"1min":3.1970000000000001,"5min":2.5409999999999999,"15min":1.2789999999999999},"min":{"1min":0.42799999999999999,"5min":0.36299999999999999,"15min":0.32000000000000001},"max":{"1min":12.361000000000001,"5min":32.417999999999999,"15min":32.417999999999999},"last":0.373},{"interface":"front","average":{"1min":3.0499999999999998,"5min":2.286,"15min":1.23},"min":{"1min":0.378,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":8.3829999999999991,"5min":32.130000000000003,"15min":32.130000000000003},"last":4.9859999999999998}]},{"osd":7,"last update":"Mon Aug 28 02:26:04 2023","interfaces":[{"interface":"back","average":{"1min":3.2690000000000001,"5min":2.1019999999999999,"15min":1.1619999999999999},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40000000000000002},"max":{"1min":8.9870000000000001,"5min":9.6020000000000003,"15min":9.6020000000000003},"last":4.9560000000000004},{"interface":"front","average":{"1min":3.649,"5min":2.3570000000000002,"15min":1.25},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":19.478999999999999,"5min":19.478999999999999,"15min":19.478999999999999},"last":0.52200000000000002}]}]},{"osd":2,"up_from":104,"seq":446676598857,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":474516,"kb_used_data":362508,"kb_used_omap":10,"kb_used_meta":111989,"kb_avail":93270636,"statfs":{"total":95995035648,"available":95509131264,"internally_reserved":0,"allocated":371208192,"data_stored":370408027,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10792,"internal_metadata":114677208},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:25:33 2023","interfaces":[{"interface":"back","average":{"1min":3.6179999999999999,"5min":2.2519999999999998,"15min":0.96399999999999997},"min":{"1min":0.188,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":18.146000000000001,"5min":36.890000000000001,"15min":36.890000000000001},"last":6.1870000000000003},{"interface":"front","average":{"1min":4.625,"5min":2.4590000000000001,"15min":1.0620000000000001},"min":{"1min":0.214,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":25.414999999999999,"5min":35.289000000000001,"15min":35.289000000000001},"last":10.25}]},{"osd":1,"last update":"Mon Aug 28 02:25:33 2023","interfaces":[{"interface":"back","average":{"1min":4.4900000000000002,"5min":2.4260000000000002,"15min":1.048},"min":{"1min":0.23999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":16.372,"5min":30.888000000000002,"15min":30.888000000000002},"last":6.1070000000000002},{"interface":"front","average":{"1min":3.8959999999999999,"5min":1.9299999999999999,"15min":0.86599999999999999},"min":{"1min":0.21299999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":14.855,"5min":31.648,"15min":31.648},"last":6.9610000000000003}]},{"osd":3,"last update":"Mon Aug 28 02:26:02 2023","interfaces":[{"interface":"back","average":{"1min":10.356,"5min":3.331,"15min":1.3819999999999999},"min":{"1min":0.309,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":128.63900000000001,"5min":128.63900000000001,"15min":128.63900000000001},"last":10.093},{"interface":"front","average":{"1min":10.528,"5min":3.6179999999999999,"15min":1.464},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":128.75700000000001,"5min":128.75700000000001,"15min":128.75700000000001},"last":10.103999999999999}]},{"osd":4,"last update":"Mon Aug 28 02:25:33 2023","interfaces":[{"interface":"back","average":{"1min":3.5750000000000002,"5min":1.772,"15min":0.98599999999999999},"min":{"1min":0.34399999999999997,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":14.815,"5min":14.815,"15min":14.815},"last":7.0129999999999999},{"interface":"front","average":{"1min":1.71,"5min":1.3300000000000001,"15min":0.876},"min":{"1min":0.33700000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.806,"5min":6.806,"15min":6.806},"last":6.1520000000000001}]},{"osd":5,"last update":"Mon Aug 28 02:25:33 2023","interfaces":[{"interface":"back","average":{"1min":1.9099999999999999,"5min":1.518,"15min":0.91700000000000004},"min":{"1min":0.36399999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":6.6900000000000004,"5min":9.1549999999999994,"15min":9.1549999999999994},"last":6.2380000000000004},{"interface":"front","average":{"1min":4.0650000000000004,"5min":2.0459999999999998,"15min":1.054},"min":{"1min":0.438,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":14.877000000000001,"5min":14.877000000000001,"15min":14.877000000000001},"last":7.0439999999999996}]},{"osd":6,"last update":"Mon Aug 28 02:25:33 2023","interfaces":[{"interface":"back","average":{"1min":2.2280000000000002,"5min":1.659,"15min":0.95299999999999996},"min":{"1min":0.35699999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":8.7010000000000005,"5min":8.7010000000000005,"15min":8.7010000000000005},"last":6.4370000000000003},{"interface":"front","average":{"1min":2.1190000000000002,"5min":1.7350000000000001,"15min":0.98899999999999999},"min":{"1min":0.42999999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":8.7729999999999997,"5min":10.507,"15min":10.507},"last":6.4240000000000004}]},{"osd":7,"last update":"Mon Aug 28 02:25:33 2023","interfaces":[{"interface":"back","average":{"1min":2.0720000000000001,"5min":1.6100000000000001,"15min":0.93600000000000005},"min":{"1min":0.39800000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":8.7699999999999996,"5min":9.0909999999999993,"15min":9.0909999999999993},"last":6.2119999999999997},{"interface":"front","average":{"1min":2.278,"5min":1.673,"15min":0.96099999999999997},"min":{"1min":0.40400000000000003,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":7.1120000000000001,"5min":7.2569999999999997,"15min":7.2569999999999997},"last":6.1699999999999999}]}]},{"osd":3,"up_from":112,"seq":481036337219,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":71464,"kb_used_data":13728,"kb_used_omap":3,"kb_used_meta":57724,"kb_avail":93673688,"statfs":{"total":95995035648,"available":95921856512,"internally_reserved":0,"allocated":14057472,"data_stored":13292370,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":59110306},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:26:00 2023","interfaces":[{"interface":"back","average":{"1min":1.96,"5min":5.8419999999999996,"15min":2.1800000000000002},"min":{"1min":0.25600000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":8.1150000000000002,"5min":164.25299999999999,"15min":164.25299999999999},"last":0.29699999999999999},{"interface":"front","average":{"1min":2.085,"5min":6.0650000000000004,"15min":2.2429999999999999},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":9.0530000000000008,"5min":164.55000000000001,"15min":164.55000000000001},"last":3.02}]},{"osd":1,"last update":"Mon Aug 28 02:26:00 2023","interfaces":[{"interface":"back","average":{"1min":3.25,"5min":3.9390000000000001,"15min":1.5389999999999999},"min":{"1min":0.251,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":13.936,"5min":117.34699999999999,"15min":117.34699999999999},"last":2.996},{"interface":"front","average":{"1min":1.8089999999999999,"5min":3.6619999999999999,"15min":1.474},"min":{"1min":0.28100000000000003,"5min":0.224,"15min":0.224},"max":{"1min":7.2679999999999998,"5min":118.012,"15min":118.012},"last":5.7629999999999999}]},{"osd":2,"last update":"Mon Aug 28 02:26:00 2023","interfaces":[{"interface":"back","average":{"1min":3.6829999999999998,"5min":5.5730000000000004,"15min":2.1070000000000002},"min":{"1min":0.314,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":18.718,"5min":120.43300000000001,"15min":120.43300000000001},"last":0.34200000000000003},{"interface":"front","average":{"1min":3.2949999999999999,"5min":5.5949999999999998,"15min":2.1160000000000001},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":18.774000000000001,"5min":120.44499999999999,"15min":120.44499999999999},"last":0.65700000000000003}]},{"osd":4,"last update":"Mon Aug 28 02:26:00 2023","interfaces":[{"interface":"back","average":{"1min":2.7559999999999998,"5min":1.681,"15min":0.97899999999999998},"min":{"1min":0.35199999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":9.016,"5min":9.016,"15min":9.016},"last":0.74399999999999999},{"interface":"front","average":{"1min":3.0409999999999999,"5min":1.669,"15min":0.97999999999999998},"min":{"1min":0.37,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":10.428000000000001,"5min":10.428000000000001,"15min":10.428000000000001},"last":1.6519999999999999}]},{"osd":5,"last update":"Mon Aug 28 02:26:00 2023","interfaces":[{"interface":"back","average":{"1min":2.4460000000000002,"5min":1.488,"15min":0.92000000000000004},"min":{"1min":0.41399999999999998,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":8.9789999999999992,"5min":8.9789999999999992,"15min":8.9789999999999992},"last":4.7460000000000004},{"interface":"front","average":{"1min":1.772,"5min":1.333,"15min":0.86399999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":13.331,"5min":13.331,"15min":13.331},"last":0.37}]},{"osd":6,"last update":"Mon Aug 28 02:26:00 2023","interfaces":[{"interface":"back","average":{"1min":2.427,"5min":2.1440000000000001,"15min":1.1379999999999999},"min":{"1min":0.36799999999999999,"5min":0.314,"15min":0.314},"max":{"1min":9.4529999999999994,"5min":72.116,"15min":72.116},"last":0.39700000000000002},{"interface":"front","average":{"1min":2.0459999999999998,"5min":2.2210000000000001,"15min":1.139},"min":{"1min":0.38,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":6.7110000000000003,"5min":72.030000000000001,"15min":72.030000000000001},"last":0.69699999999999995}]},{"osd":7,"last update":"Mon Aug 28 02:26:00 2023","interfaces":[{"interface":"back","average":{"1min":2.9329999999999998,"5min":1.7250000000000001,"15min":1.0129999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":11.949999999999999,"5min":11.949999999999999,"15min":11.949999999999999},"last":0.72299999999999998},{"interface":"front","average":{"1min":3.702,"5min":1.907,"15min":1.0660000000000001},"min":{"1min":0.438,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":13.890000000000001,"5min":13.890000000000001,"15min":13.890000000000001},"last":6.718}]}]},{"osd":4,"up_from":30,"seq":128849019093,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":744776,"kb_used_data":435264,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93000376,"statfs":{"total":95995035648,"available":95232385024,"internally_reserved":0,"allocated":445710336,"data_stored":445142934,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:25:59 2023","interfaces":[{"interface":"back","average":{"1min":6.2859999999999996,"5min":5.8010000000000002,"15min":2.3399999999999999},"min":{"1min":0.26500000000000001,"5min":0.223,"15min":0.223},"max":{"1min":43.548000000000002,"5min":138.67400000000001,"15min":138.67400000000001},"last":2.7360000000000002},{"interface":"front","average":{"1min":5.8789999999999996,"5min":5.6669999999999998,"15min":2.2839999999999998},"min":{"1min":0.25,"5min":0.248,"15min":0.248},"max":{"1min":43.515000000000001,"5min":134.14599999999999,"15min":134.14599999999999},"last":2.5920000000000001}]},{"osd":1,"last update":"Mon Aug 28 02:25:59 2023","interfaces":[{"interface":"back","average":{"1min":2.6509999999999998,"5min":1.915,"15min":0.98699999999999999},"min":{"1min":0.32700000000000001,"5min":0.255,"15min":0.255},"max":{"1min":8.9570000000000007,"5min":53.622999999999998,"15min":53.622999999999998},"last":3.9009999999999998},{"interface":"front","average":{"1min":2.1339999999999999,"5min":1.8999999999999999,"15min":1.01},"min":{"1min":0.28000000000000003,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":7.1580000000000004,"5min":53.774000000000001,"15min":53.774000000000001},"last":2.6230000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:25:28 2023","interfaces":[{"interface":"back","average":{"1min":5.0780000000000003,"5min":3.6219999999999999,"15min":1.5740000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":56.295999999999999,"5min":74.099000000000004,"15min":74.099000000000004},"last":2.7690000000000001},{"interface":"front","average":{"1min":4.7229999999999999,"5min":3.4820000000000002,"15min":1.52},"min":{"1min":0.28599999999999998,"5min":0.25,"15min":0.25},"max":{"1min":56.314,"5min":74.081999999999994,"15min":74.081999999999994},"last":2.6680000000000001}]},{"osd":3,"last update":"Mon Aug 28 02:25:59 2023","interfaces":[{"interface":"back","average":{"1min":3.3359999999999999,"5min":1.5149999999999999,"15min":0.83899999999999997},"min":{"1min":0.32400000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":8.6349999999999998,"5min":8.6349999999999998,"15min":8.6349999999999998},"last":8.5960000000000001},{"interface":"front","average":{"1min":2.9430000000000001,"5min":1.6870000000000001,"15min":0.95099999999999996},"min":{"1min":0.39900000000000002,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":10.956,"5min":10.956,"15min":10.956},"last":2.7029999999999998}]},{"osd":5,"last update":"Mon Aug 28 02:25:41 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":0.97799999999999998,"15min":0.63700000000000001},"min":{"1min":0.26900000000000002,"5min":0.249,"15min":0.249},"max":{"1min":5.4080000000000004,"5min":8.2560000000000002,"15min":8.2560000000000002},"last":2.645},{"interface":"front","average":{"1min":2.3010000000000002,"5min":1.3300000000000001,"15min":0.76700000000000002},"min":{"1min":0.27100000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":8.0619999999999994,"5min":8.0619999999999994,"15min":8.0619999999999994},"last":6.4560000000000004}]},{"osd":6,"last update":"Mon Aug 28 02:25:59 2023","interfaces":[{"interface":"back","average":{"1min":2.1059999999999999,"5min":1.373,"15min":0.79600000000000004},"min":{"1min":0.32300000000000001,"5min":0.29199999999999998,"15min":0.27500000000000002},"max":{"1min":11.170999999999999,"5min":11.170999999999999,"15min":11.170999999999999},"last":5.7160000000000002},{"interface":"front","average":{"1min":1.403,"5min":0.97199999999999998,"15min":0.69199999999999995},"min":{"1min":0.33700000000000002,"5min":0.32200000000000001,"15min":0.318},"max":{"1min":7.3170000000000002,"5min":7.3170000000000002,"15min":7.3170000000000002},"last":5.6900000000000004}]},{"osd":7,"last update":"Mon Aug 28 02:25:22 2023","interfaces":[{"interface":"back","average":{"1min":1.825,"5min":1.0960000000000001,"15min":0.72499999999999998},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.312},"max":{"1min":7.9589999999999996,"5min":8.6890000000000001,"15min":8.6890000000000001},"last":6.0259999999999998},{"interface":"front","average":{"1min":1.8460000000000001,"5min":1.4450000000000001,"15min":0.84999999999999998},"min":{"1min":0.35499999999999998,"5min":0.32500000000000001,"15min":0.32100000000000001},"max":{"1min":5.8250000000000002,"5min":8.7569999999999997,"15min":8.7569999999999997},"last":5.8499999999999996}]}]},{"osd":5,"up_from":36,"seq":154618822864,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":925884,"kb_used_data":628596,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":92819268,"statfs":{"total":95995035648,"available":95046930432,"internally_reserved":0,"allocated":643682304,"data_stored":643122369,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:25:52 2023","interfaces":[{"interface":"back","average":{"1min":3.5249999999999999,"5min":5.8579999999999997,"15min":2.3639999999999999},"min":{"1min":0.24099999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":12.874000000000001,"5min":131.06100000000001,"15min":131.06100000000001},"last":0.25800000000000001},{"interface":"front","average":{"1min":3.758,"5min":6.1749999999999998,"15min":2.4740000000000002},"min":{"1min":0.24199999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":15.691000000000001,"5min":131.262,"15min":131.262},"last":0.30199999999999999}]},{"osd":1,"last update":"Mon Aug 28 02:26:03 2023","interfaces":[{"interface":"back","average":{"1min":1.298,"5min":5.0019999999999998,"15min":2.0190000000000001},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":5.7240000000000002,"5min":183.97900000000001,"15min":183.97900000000001},"last":0.24399999999999999},{"interface":"front","average":{"1min":2.4929999999999999,"5min":5.4960000000000004,"15min":2.1920000000000002},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":8.5640000000000001,"5min":184.33699999999999,"15min":184.33699999999999},"last":0.33000000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:25:30 2023","interfaces":[{"interface":"back","average":{"1min":3.5409999999999999,"5min":5.2169999999999996,"15min":2.1320000000000001},"min":{"1min":0.29999999999999999,"5min":0.252,"15min":0.252},"max":{"1min":12.584,"5min":110.051,"15min":110.051},"last":2.746},{"interface":"front","average":{"1min":3.7709999999999999,"5min":5.4820000000000002,"15min":2.2189999999999999},"min":{"1min":0.29799999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":15.635,"5min":110.041,"15min":110.041},"last":2.7589999999999999}]},{"osd":3,"last update":"Mon Aug 28 02:26:03 2023","interfaces":[{"interface":"back","average":{"1min":2.004,"5min":1.649,"15min":0.95699999999999996},"min":{"1min":0.28899999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.0039999999999996,"5min":10.576000000000001,"15min":10.576000000000001},"last":2.423},{"interface":"front","average":{"1min":3.5499999999999998,"5min":2.177,"15min":1.143},"min":{"1min":0.34300000000000003,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":12.118,"5min":12.118,"15min":12.118},"last":7.1710000000000003}]},{"osd":4,"last update":"Mon Aug 28 02:25:44 2023","interfaces":[{"interface":"back","average":{"1min":1.4239999999999999,"5min":1.355,"15min":0.76200000000000001},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":5.6890000000000001,"5min":10.759,"15min":10.759},"last":0.41299999999999998},{"interface":"front","average":{"1min":2.149,"5min":1.468,"15min":0.82199999999999995},"min":{"1min":0.31900000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.0170000000000003,"5min":8.6769999999999996,"15min":8.6769999999999996},"last":0.34100000000000003}]},{"osd":6,"last update":"Mon Aug 28 02:26:03 2023","interfaces":[{"interface":"back","average":{"1min":1.968,"5min":1.694,"15min":0.91900000000000004},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.28399999999999997},"max":{"1min":5.9870000000000001,"5min":14.803000000000001,"15min":14.803000000000001},"last":0.45500000000000002},{"interface":"front","average":{"1min":1.6639999999999999,"5min":1.389,"15min":0.81699999999999995},"min":{"1min":0.33500000000000002,"5min":0.31,"15min":0.30299999999999999},"max":{"1min":8.4779999999999998,"5min":8.6479999999999997,"15min":8.6479999999999997},"last":0.39800000000000002}]},{"osd":7,"last update":"Mon Aug 28 02:25:17 2023","interfaces":[{"interface":"back","average":{"1min":2.004,"5min":1.3180000000000001,"15min":0.80300000000000005},"min":{"1min":0.33900000000000002,"5min":0.31,"15min":0.31},"max":{"1min":9.3070000000000004,"5min":9.3070000000000004,"15min":9.3070000000000004},"last":0.496},{"interface":"front","average":{"1min":1.871,"5min":1.341,"15min":0.80900000000000005},"min":{"1min":0.32200000000000001,"5min":0.32100000000000001,"15min":0.29799999999999999},"max":{"1min":12.787000000000001,"5min":12.787000000000001,"15min":12.787000000000001},"last":0.38400000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1015611392,"data_stored":1015611392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":309723136,"data_stored":309723136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":366546944,"data_stored":366542872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":11993088,"data_stored":11993088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":427819008,"data_stored":427819008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":629411840,"data_stored":629407768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":213647360,"data_stored":213647360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":268177408,"data_stored":268173336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-08-28T02:26:20.446 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:26:20.510 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:20.623 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:20 smithi169 ceph-mon[159415]: from='client.44930 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:26:20.624 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:20 smithi169 ceph-mon[159415]: pgmap v417: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 779 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 1990/37266 objects degraded (5.340%); 2112/37266 objects misplaced (5.667%); 2.4 MiB/s, 37 objects/s recovering 2023-08-28T02:26:20.645 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:20 smithi143 ceph-mon[189871]: from='client.44930 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:26:20.645 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:20 smithi143 ceph-mon[189871]: pgmap v417: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 779 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 1990/37266 objects degraded (5.340%); 2112/37266 objects misplaced (5.667%); 2.4 MiB/s, 37 objects/s recovering 2023-08-28T02:26:20.645 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:20 smithi143 ceph-mon[183059]: from='client.44930 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:26:20.645 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:20 smithi143 ceph-mon[183059]: pgmap v417: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 779 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 1990/37266 objects degraded (5.340%); 2112/37266 objects misplaced (5.667%); 2.4 MiB/s, 37 objects/s recovering 2023-08-28T02:26:20.672 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:26:20.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:20.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:26:20.776 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:26:20.828 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:20.838 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:26:20.924 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:26:20.930 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:26:20.930 INFO:teuthology.orchestra.run.smithi169.stdout: "ops": [], 2023-08-28T02:26:20.930 INFO:teuthology.orchestra.run.smithi169.stdout: "complaint_time": 30, 2023-08-28T02:26:20.930 INFO:teuthology.orchestra.run.smithi169.stdout: "num_blocked_ops": 0 2023-08-28T02:26:20.931 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:26:20.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:20.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:26:21.033 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:26:21.081 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:21.161 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:26:21.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:21.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:26:21.276 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:26:21.321 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:21.424 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:26:21.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:21.430 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:26:21.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:21 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 1990/37266 objects degraded (5.340%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:26:21.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:21 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/4057150813' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:26:21.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:21 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 1990/37266 objects degraded (5.340%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:26:21.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:21 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/4057150813' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:26:21.530 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:26:21.572 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:21 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 1990/37266 objects degraded (5.340%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:26:21.572 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:21 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/4057150813' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:26:21.579 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:21.806 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:26:21.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:21.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:26:21.890 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.6 dump_historic_ops 2023-08-28T02:26:21.915 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:26:21.929 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:26:21 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:26:21] "GET /metrics HTTP/1.1" 200 36487 "" "Prometheus/2.33.4" 2023-08-28T02:26:22.023 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:22.094 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:26:22.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:22.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:26:22.201 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:26:22.272 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:22.316 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is unset 2023-08-28T02:26:22.506 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:26:22.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:22 smithi143 ceph-mon[183059]: pgmap v418: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 779 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 1990/37266 objects degraded (5.340%); 2112/37266 objects misplaced (5.667%); 2.4 MiB/s, 37 objects/s recovering 2023-08-28T02:26:22.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:22 smithi143 ceph-mon[183059]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:26:22.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:22 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/4057150813' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:26:22.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:22 smithi143 ceph-mon[183059]: osdmap e182: 8 total, 8 up, 8 in 2023-08-28T02:26:22.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:22 smithi143 ceph-mon[189871]: pgmap v418: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 779 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 1990/37266 objects degraded (5.340%); 2112/37266 objects misplaced (5.667%); 2.4 MiB/s, 37 objects/s recovering 2023-08-28T02:26:22.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:22 smithi143 ceph-mon[189871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:26:22.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:22 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/4057150813' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:26:22.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:22 smithi143 ceph-mon[189871]: osdmap e182: 8 total, 8 up, 8 in 2023-08-28T02:26:22.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:22 smithi169 ceph-mon[159415]: pgmap v418: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 779 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 1990/37266 objects degraded (5.340%); 2112/37266 objects misplaced (5.667%); 2.4 MiB/s, 37 objects/s recovering 2023-08-28T02:26:22.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:22 smithi169 ceph-mon[159415]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:26:22.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:22 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/4057150813' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:26:22.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:22 smithi169 ceph-mon[159415]: osdmap e182: 8 total, 8 up, 8 in 2023-08-28T02:26:23.355 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph pg dump --format=json 2023-08-28T02:26:23.373 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:26:23.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:23.383 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:26:23.484 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:26:23.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:23 smithi169 ceph-mon[159415]: failed to encode map e182 with expected crc 2023-08-28T02:26:23.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:23 smithi169 ceph-mon[159415]: failed to encode map e182 with expected crc 2023-08-28T02:26:23.487 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:23 smithi143 ceph-mon[183059]: failed to encode map e182 with expected crc 2023-08-28T02:26:23.488 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:23 smithi143 ceph-mon[183059]: failed to encode map e182 with expected crc 2023-08-28T02:26:23.488 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:23 smithi143 ceph-mon[189871]: failed to encode map e182 with expected crc 2023-08-28T02:26:23.488 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:23 smithi143 ceph-mon[189871]: failed to encode map e182 with expected crc 2023-08-28T02:26:23.511 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:23.554 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:26:23.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:23.557 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:26:23.657 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:26:23.725 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:23.878 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:26:23.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:23.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:26:23.990 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:26:24.026 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:26:24.060 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:24.066 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:26:24.102 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:26:24.111 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:26:24.151 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:26:24.182 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:26:24.189 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:26:24.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:24.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:26:24.223 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:26:24.256 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:26:24.288 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:26:24.298 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:26:24.298 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:26:24.298 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:26:24.298 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:26:24.298 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:26:24.298 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:26:24.299 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:26:24.299 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:26:24.301 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:26:24.383 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:24.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:24 smithi143 ceph-mon[183059]: pgmap v420: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 689 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 1990/32919 objects degraded (6.045%); 1917/32919 objects misplaced (5.823%); 2.4 MiB/s, 37 objects/s recovering 2023-08-28T02:26:24.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:24 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1308757331' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:26:24.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:24 smithi143 ceph-mon[189871]: pgmap v420: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 689 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 1990/32919 objects degraded (6.045%); 1917/32919 objects misplaced (5.823%); 2.4 MiB/s, 37 objects/s recovering 2023-08-28T02:26:24.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:24 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1308757331' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:26:24.498 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:26:24.511 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:24 smithi169 ceph-mon[159415]: pgmap v420: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 689 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 1990/32919 objects degraded (6.045%); 1917/32919 objects misplaced (5.823%); 2.4 MiB/s, 37 objects/s recovering 2023-08-28T02:26:24.511 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:24 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1308757331' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:26:24.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:24.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:26:24.580 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:26:24.580 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:26:24.612 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:26:24.667 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:24.816 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:26:24.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:24.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:26:24.922 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:26:24.972 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:25.018 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:26:25.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:25.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:26:25.121 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:26:25.201 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:25.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:25 smithi143 ceph-mon[183059]: pgmap v421: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 580 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 1990/27690 objects degraded (7.187%); 1493/27690 objects misplaced (5.392%); 2.6 MiB/s, 40 objects/s recovering 2023-08-28T02:26:25.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:25 smithi143 ceph-mon[189871]: pgmap v421: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 580 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 1990/27690 objects degraded (7.187%); 1493/27690 objects misplaced (5.392%); 2.6 MiB/s, 40 objects/s recovering 2023-08-28T02:26:25.489 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set noscrub 2023-08-28T02:26:25.502 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.7 dump_ops_in_flight 2023-08-28T02:26:25.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:25 smithi169 ceph-mon[159415]: pgmap v421: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 580 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 1990/27690 objects degraded (7.187%); 1493/27690 objects misplaced (5.392%); 2.6 MiB/s, 40 objects/s recovering 2023-08-28T02:26:26.112 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:26:26.489 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:26:26.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:26.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:26:26.600 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:26:26.642 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:26.795 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:26:26.919 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:26:26.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:26.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:26:27.023 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:26:27.056 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:27.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:27 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:26:27.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:27 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:26:27.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:27 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 1990/27690 objects degraded (7.187%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:26:27.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:27 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:26:27.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:27 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:26:27.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:27 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 1990/27690 objects degraded (7.187%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:26:27.230 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:26:27.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:27 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:26:27.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:27 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:26:27.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:27 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 1990/27690 objects degraded (7.187%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:26:27.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:27.241 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:26:27.342 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:26:27.386 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:27.533 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:26:27.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:27.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:26:27.544 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:26:27.565 INFO:teuthology.orchestra.run.smithi143.stderr:dumped all 2023-08-28T02:26:27.642 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:26:27.708 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:28.126 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:26:28.126 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:26:28.150 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:28 smithi169 ceph-mon[159415]: pgmap v422: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 522 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 1990/24903 objects degraded (7.991%); 1341/24903 objects misplaced (5.385%); 2.6 MiB/s, 40 objects/s recovering 2023-08-28T02:26:28.362 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:26:28.386 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:28 smithi143 ceph-mon[183059]: pgmap v422: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 522 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 1990/24903 objects degraded (7.991%); 1341/24903 objects misplaced (5.385%); 2.6 MiB/s, 40 objects/s recovering 2023-08-28T02:26:28.387 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:28 smithi143 ceph-mon[189871]: pgmap v422: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 522 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 1990/24903 objects degraded (7.991%); 1341/24903 objects misplaced (5.385%); 2.6 MiB/s, 40 objects/s recovering 2023-08-28T02:26:28.959 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.7 dump_blocked_ops 2023-08-28T02:26:29.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:29 smithi169 ceph-mon[159415]: from='client.34989 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:26:29.302 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:26:29.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:29.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:26:29.370 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:29 smithi143 ceph-mon[183059]: from='client.34989 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:26:29.370 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:29 smithi143 ceph-mon[189871]: from='client.34989 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:26:29.409 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:26:29.470 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:29.474 INFO:teuthology.orchestra.run.smithi143.stdout:{"pg_ready":true,"pg_map":{"version":422,"stamp":"2023-08-28T02:26:27.174464+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":546904011,"num_objects":8301,"num_object_clones":0,"num_object_copies":24903,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1990,"num_objects_misplaced":1341,"num_objects_unfound":0,"num_objects_dirty":8301,"num_whiteouts":0,"num_read":6274,"num_read_kb":13707,"num_write":213115,"num_write_kb":7064102,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6747,"num_bytes_recovered":440347270,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139724,"ondisk_log_size":139724,"up":267,"acting":266,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":279,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":3735104,"kb_used_data":2053504,"kb_used_omap":29,"kb_used_meta":1681122,"kb_avail":746226112,"statfs":{"total":767960285184,"available":764135538688,"internally_reserved":0,"allocated":2102788096,"data_stored":2097229373,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30674,"internal_metadata":1721468974},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,3,1,12,3,5,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-411238400,"num_objects":-6275,"num_object_clones":0,"num_object_copies":-18825,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-1129,"num_objects_unfound":0,"num_objects_dirty":-6275,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6275,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":409,"num_bytes_recovered":26804224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.009039"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":165,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.297502+0000","last_change":"2023-08-28T02:20:19.849807+0000","last_active":"2023-08-28T02:26:11.297502+0000","last_peered":"2023-08-28T02:26:11.297502+0000","last_clean":"2023-08-28T02:26:11.297502+0000","last_became_active":"2023-08-28T02:20:19.849464+0000","last_became_peered":"2023-08-28T02:20:19.849464+0000","last_unstale":"2023-08-28T02:26:11.297502+0000","last_undegraded":"2023-08-28T02:26:11.297502+0000","last_fullsized":"2023-08-28T02:26:11.297502+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":39,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.302274+0000","last_change":"2023-08-28T02:23:59.204957+0000","last_active":"2023-08-28T02:26:11.302274+0000","last_peered":"2023-08-28T02:26:11.302274+0000","last_clean":"2023-08-28T02:26:11.302274+0000","last_became_active":"2023-08-28T02:23:59.204717+0000","last_became_peered":"2023-08-28T02:23:59.204717+0000","last_unstale":"2023-08-28T02:26:11.302274+0000","last_undegraded":"2023-08-28T02:26:11.302274+0000","last_fullsized":"2023-08-28T02:26:11.302274+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:34:58.977269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":140,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.297162+0000","last_change":"2023-08-28T02:23:59.212721+0000","last_active":"2023-08-28T02:26:11.297162+0000","last_peered":"2023-08-28T02:26:11.297162+0000","last_clean":"2023-08-28T02:26:11.297162+0000","last_became_active":"2023-08-28T02:23:59.212446+0000","last_became_peered":"2023-08-28T02:23:59.212446+0000","last_unstale":"2023-08-28T02:26:11.297162+0000","last_undegraded":"2023-08-28T02:26:11.297162+0000","last_fullsized":"2023-08-28T02:26:11.297162+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:03:54.701399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":168,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.301199+0000","last_change":"2023-08-28T02:23:59.223634+0000","last_active":"2023-08-28T02:26:11.301199+0000","last_peered":"2023-08-28T02:26:11.301199+0000","last_clean":"2023-08-28T02:26:11.301199+0000","last_became_active":"2023-08-28T02:23:59.222704+0000","last_became_peered":"2023-08-28T02:23:59.222704+0000","last_unstale":"2023-08-28T02:26:11.301199+0000","last_undegraded":"2023-08-28T02:26:11.301199+0000","last_fullsized":"2023-08-28T02:26:11.301199+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:51:52.752361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":40,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-08-28T02:26:22.312751+0000","last_change":"2023-08-28T02:23:59.220459+0000","last_active":"2023-08-28T02:26:22.312751+0000","last_peered":"2023-08-28T02:26:22.312751+0000","last_clean":"2023-08-28T02:26:22.312751+0000","last_became_active":"2023-08-28T02:23:59.219274+0000","last_became_peered":"2023-08-28T02:23:59.219274+0000","last_unstale":"2023-08-28T02:26:22.312751+0000","last_undegraded":"2023-08-28T02:26:22.312751+0000","last_fullsized":"2023-08-28T02:26:22.312751+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:31:53.889311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":176,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.310280+0000","last_change":"2023-08-28T02:21:59.183009+0000","last_active":"2023-08-28T02:26:11.310280+0000","last_peered":"2023-08-28T02:26:11.310280+0000","last_clean":"2023-08-28T02:26:11.310280+0000","last_became_active":"2023-08-28T02:21:59.182621+0000","last_became_peered":"2023-08-28T02:21:59.182621+0000","last_unstale":"2023-08-28T02:26:11.310280+0000","last_undegraded":"2023-08-28T02:26:11.310280+0000","last_fullsized":"2023-08-28T02:26:11.310280+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":149,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.301759+0000","last_change":"2023-08-28T02:23:59.227592+0000","last_active":"2023-08-28T02:26:11.301759+0000","last_peered":"2023-08-28T02:26:11.301759+0000","last_clean":"2023-08-28T02:26:11.301759+0000","last_became_active":"2023-08-28T02:23:59.227342+0000","last_became_peered":"2023-08-28T02:23:59.227342+0000","last_unstale":"2023-08-28T02:26:11.301759+0000","last_undegraded":"2023-08-28T02:26:11.301759+0000","last_fullsized":"2023-08-28T02:26:11.301759+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:08:19.874396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":40,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:18.946155+0000","last_change":"2023-08-28T02:23:59.220634+0000","last_active":"2023-08-28T02:26:18.946155+0000","last_peered":"2023-08-28T02:26:18.946155+0000","last_clean":"2023-08-28T02:26:18.946155+0000","last_became_active":"2023-08-28T02:23:59.220177+0000","last_became_peered":"2023-08-28T02:23:59.220177+0000","last_unstale":"2023-08-28T02:26:18.946155+0000","last_undegraded":"2023-08-28T02:26:18.946155+0000","last_fullsized":"2023-08-28T02:26:18.946155+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:16:26.906064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":166,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.379078+0000","last_change":"2023-08-28T02:23:38.339581+0000","last_active":"2023-08-28T02:26:01.379078+0000","last_peered":"2023-08-28T02:26:01.379078+0000","last_clean":"2023-08-28T02:26:01.379078+0000","last_became_active":"2023-08-28T02:23:38.339244+0000","last_became_peered":"2023-08-28T02:23:38.339244+0000","last_unstale":"2023-08-28T02:26:01.379078+0000","last_undegraded":"2023-08-28T02:26:01.379078+0000","last_fullsized":"2023-08-28T02:26:01.379078+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":40,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:18.946186+0000","last_change":"2023-08-28T02:23:59.220551+0000","last_active":"2023-08-28T02:26:18.946186+0000","last_peered":"2023-08-28T02:26:18.946186+0000","last_clean":"2023-08-28T02:26:18.946186+0000","last_became_active":"2023-08-28T02:23:59.219681+0000","last_became_peered":"2023-08-28T02:23:59.219681+0000","last_unstale":"2023-08-28T02:26:18.946186+0000","last_undegraded":"2023-08-28T02:26:18.946186+0000","last_fullsized":"2023-08-28T02:26:18.946186+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:39:58.811256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":68,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.323491+0000","last_change":"2023-08-28T02:23:59.316561+0000","last_active":"2023-08-28T02:26:11.323491+0000","last_peered":"2023-08-28T02:26:11.323491+0000","last_clean":"2023-08-28T02:26:11.323491+0000","last_became_active":"2023-08-28T02:23:59.227433+0000","last_became_peered":"2023-08-28T02:23:59.227433+0000","last_unstale":"2023-08-28T02:26:11.323491+0000","last_undegraded":"2023-08-28T02:26:11.323491+0000","last_fullsized":"2023-08-28T02:26:11.323491+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":123,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.297309+0000","last_change":"2023-08-28T02:20:19.851616+0000","last_active":"2023-08-28T02:26:11.297309+0000","last_peered":"2023-08-28T02:26:11.297309+0000","last_clean":"2023-08-28T02:26:11.297309+0000","last_became_active":"2023-08-28T02:20:19.851322+0000","last_became_peered":"2023-08-28T02:20:19.851322+0000","last_unstale":"2023-08-28T02:26:11.297309+0000","last_undegraded":"2023-08-28T02:26:11.297309+0000","last_fullsized":"2023-08-28T02:26:11.297309+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:03:37.565646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"86'22","reported_seq":194,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.297464+0000","last_change":"2023-08-28T02:23:38.368275+0000","last_active":"2023-08-28T02:26:11.297464+0000","last_peered":"2023-08-28T02:26:11.297464+0000","last_clean":"2023-08-28T02:26:11.297464+0000","last_became_active":"2023-08-28T02:23:38.342097+0000","last_became_peered":"2023-08-28T02:23:38.342097+0000","last_unstale":"2023-08-28T02:26:11.297464+0000","last_undegraded":"2023-08-28T02:26:11.297464+0000","last_fullsized":"2023-08-28T02:26:11.297464+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":162,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.297257+0000","last_change":"2023-08-28T02:19:51.204118+0000","last_active":"2023-08-28T02:26:11.297257+0000","last_peered":"2023-08-28T02:26:11.297257+0000","last_clean":"2023-08-28T02:26:11.297257+0000","last_became_active":"2023-08-28T02:19:51.203745+0000","last_became_peered":"2023-08-28T02:19:51.203745+0000","last_unstale":"2023-08-28T02:26:11.297257+0000","last_undegraded":"2023-08-28T02:26:11.297257+0000","last_fullsized":"2023-08-28T02:26:11.297257+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":53,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.322478+0000","last_change":"2023-08-28T02:23:38.983634+0000","last_active":"2023-08-28T02:26:11.322478+0000","last_peered":"2023-08-28T02:26:11.322478+0000","last_clean":"2023-08-28T02:26:11.322478+0000","last_became_active":"2023-08-28T02:23:38.344566+0000","last_became_peered":"2023-08-28T02:23:38.344566+0000","last_unstale":"2023-08-28T02:26:11.322478+0000","last_undegraded":"2023-08-28T02:26:11.322478+0000","last_fullsized":"2023-08-28T02:26:11.322478+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":40,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:18.946229+0000","last_change":"2023-08-28T02:23:59.216960+0000","last_active":"2023-08-28T02:26:18.946229+0000","last_peered":"2023-08-28T02:26:18.946229+0000","last_clean":"2023-08-28T02:26:18.946229+0000","last_became_active":"2023-08-28T02:23:59.215657+0000","last_became_peered":"2023-08-28T02:23:59.215657+0000","last_unstale":"2023-08-28T02:26:18.946229+0000","last_undegraded":"2023-08-28T02:26:18.946229+0000","last_fullsized":"2023-08-28T02:26:18.946229+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:31:14.407108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":167,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.304673+0000","last_change":"2023-08-28T02:23:59.223066+0000","last_active":"2023-08-28T02:26:11.304673+0000","last_peered":"2023-08-28T02:26:11.304673+0000","last_clean":"2023-08-28T02:26:11.304673+0000","last_became_active":"2023-08-28T02:23:59.222731+0000","last_became_peered":"2023-08-28T02:23:59.222731+0000","last_unstale":"2023-08-28T02:26:11.304673+0000","last_undegraded":"2023-08-28T02:26:11.304673+0000","last_fullsized":"2023-08-28T02:26:11.304673+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:48:37.558266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":208,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.379065+0000","last_change":"2023-08-28T02:23:59.210469+0000","last_active":"2023-08-28T02:26:01.379065+0000","last_peered":"2023-08-28T02:26:01.379065+0000","last_clean":"2023-08-28T02:26:01.379065+0000","last_became_active":"2023-08-28T02:23:59.210031+0000","last_became_peered":"2023-08-28T02:23:59.210031+0000","last_unstale":"2023-08-28T02:26:01.379065+0000","last_undegraded":"2023-08-28T02:26:01.379065+0000","last_fullsized":"2023-08-28T02:26:01.379065+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":47,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.322851+0000","last_change":"2023-08-28T02:23:38.370183+0000","last_active":"2023-08-28T02:26:11.322851+0000","last_peered":"2023-08-28T02:26:11.322851+0000","last_clean":"2023-08-28T02:26:11.322851+0000","last_became_active":"2023-08-28T02:23:38.368415+0000","last_became_peered":"2023-08-28T02:23:38.368415+0000","last_unstale":"2023-08-28T02:26:11.322851+0000","last_undegraded":"2023-08-28T02:26:11.322851+0000","last_fullsized":"2023-08-28T02:26:11.322851+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":168,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.301276+0000","last_change":"2023-08-28T02:23:59.220932+0000","last_active":"2023-08-28T02:26:11.301276+0000","last_peered":"2023-08-28T02:26:11.301276+0000","last_clean":"2023-08-28T02:26:11.301276+0000","last_became_active":"2023-08-28T02:23:59.220488+0000","last_became_peered":"2023-08-28T02:23:59.220488+0000","last_unstale":"2023-08-28T02:26:11.301276+0000","last_undegraded":"2023-08-28T02:26:11.301276+0000","last_fullsized":"2023-08-28T02:26:11.301276+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:53:09.216340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":156,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.310689+0000","last_change":"2023-08-28T02:18:39.778728+0000","last_active":"2023-08-28T02:26:11.310689+0000","last_peered":"2023-08-28T02:26:11.310689+0000","last_clean":"2023-08-28T02:26:11.310689+0000","last_became_active":"2023-08-28T02:18:39.778283+0000","last_became_peered":"2023-08-28T02:18:39.778283+0000","last_unstale":"2023-08-28T02:26:11.310689+0000","last_undegraded":"2023-08-28T02:26:11.310689+0000","last_fullsized":"2023-08-28T02:26:11.310689+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":40,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-08-28T02:26:22.313178+0000","last_change":"2023-08-28T02:23:59.216743+0000","last_active":"2023-08-28T02:26:22.313178+0000","last_peered":"2023-08-28T02:26:22.313178+0000","last_clean":"2023-08-28T02:26:22.313178+0000","last_became_active":"2023-08-28T02:23:59.216416+0000","last_became_peered":"2023-08-28T02:23:59.216416+0000","last_unstale":"2023-08-28T02:26:22.313178+0000","last_undegraded":"2023-08-28T02:26:22.313178+0000","last_fullsized":"2023-08-28T02:26:22.313178+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:25:13.056747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"182'13299","reported_seq":14134,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-08-28T02:26:26.975098+0000","last_change":"2023-08-28T02:24:09.355130+0000","last_active":"2023-08-28T02:26:26.975098+0000","last_peered":"2023-08-28T02:26:26.975098+0000","last_clean":"2023-08-28T02:26:26.975098+0000","last_became_active":"2023-08-28T02:24:09.341363+0000","last_became_peered":"2023-08-28T02:24:09.341363+0000","last_unstale":"2023-08-28T02:26:26.975098+0000","last_undegraded":"2023-08-28T02:26:26.975098+0000","last_fullsized":"2023-08-28T02:26:26.975098+0000","mapping_epoch":161,"log_start":"133'4700","ondisk_log_start":"133'4700","created":119,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":8599,"log_dups_size":0,"ondisk_log_size":8599,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27066368,"num_objects":413,"num_object_clones":0,"num_object_copies":1239,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":413,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13299,"num_write_kb":438784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":367,"num_bytes_recovered":23789568,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":51,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.323580+0000","last_change":"2023-08-28T02:23:38.657185+0000","last_active":"2023-08-28T02:26:11.323580+0000","last_peered":"2023-08-28T02:26:11.323580+0000","last_clean":"2023-08-28T02:26:11.323580+0000","last_became_active":"2023-08-28T02:23:38.339016+0000","last_became_peered":"2023-08-28T02:23:38.339016+0000","last_unstale":"2023-08-28T02:26:11.323580+0000","last_undegraded":"2023-08-28T02:26:11.323580+0000","last_fullsized":"2023-08-28T02:26:11.323580+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":150,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.378672+0000","last_change":"2023-08-28T02:23:38.339886+0000","last_active":"2023-08-28T02:26:01.378672+0000","last_peered":"2023-08-28T02:26:01.378672+0000","last_clean":"2023-08-28T02:26:01.378672+0000","last_became_active":"2023-08-28T02:23:38.339514+0000","last_became_peered":"2023-08-28T02:23:38.339514+0000","last_unstale":"2023-08-28T02:26:01.378672+0000","last_undegraded":"2023-08-28T02:26:01.378672+0000","last_fullsized":"2023-08-28T02:26:01.378672+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"182'13251","reported_seq":13324,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-08-28T02:26:23.866458+0000","last_change":"2023-08-28T02:22:00.165260+0000","last_active":"2023-08-28T02:26:23.866458+0000","last_peered":"2023-08-28T02:26:23.866458+0000","last_clean":"2023-08-28T02:26:23.866458+0000","last_became_active":"2023-08-28T02:22:00.164978+0000","last_became_peered":"2023-08-28T02:22:00.164978+0000","last_unstale":"2023-08-28T02:26:23.866458+0000","last_undegraded":"2023-08-28T02:26:23.866458+0000","last_fullsized":"2023-08-28T02:26:23.866458+0000","mapping_epoch":119,"log_start":"131'4100","ondisk_log_start":"131'4100","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":9151,"log_dups_size":0,"ondisk_log_size":9151,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:18:41.711960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37683200,"num_objects":575,"num_object_clones":0,"num_object_copies":1725,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":575,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13251,"num_write_kb":442432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":51,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.323305+0000","last_change":"2023-08-28T02:23:38.555339+0000","last_active":"2023-08-28T02:26:11.323305+0000","last_peered":"2023-08-28T02:26:11.323305+0000","last_clean":"2023-08-28T02:26:11.323305+0000","last_became_active":"2023-08-28T02:23:38.337938+0000","last_became_peered":"2023-08-28T02:23:38.337938+0000","last_unstale":"2023-08-28T02:26:11.323305+0000","last_undegraded":"2023-08-28T02:26:11.323305+0000","last_fullsized":"2023-08-28T02:26:11.323305+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":190,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.297404+0000","last_change":"2023-08-28T02:23:59.208685+0000","last_active":"2023-08-28T02:26:11.297404+0000","last_peered":"2023-08-28T02:26:11.297404+0000","last_clean":"2023-08-28T02:26:11.297404+0000","last_became_active":"2023-08-28T02:23:59.208377+0000","last_became_peered":"2023-08-28T02:23:59.208377+0000","last_unstale":"2023-08-28T02:26:11.297404+0000","last_undegraded":"2023-08-28T02:26:11.297404+0000","last_fullsized":"2023-08-28T02:26:11.297404+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"182'13152","reported_seq":13224,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-08-28T02:26:23.881884+0000","last_change":"2023-08-28T02:22:00.165317+0000","last_active":"2023-08-28T02:26:23.881884+0000","last_peered":"2023-08-28T02:26:23.881884+0000","last_clean":"2023-08-28T02:26:23.881884+0000","last_became_active":"2023-08-28T02:22:00.164989+0000","last_became_peered":"2023-08-28T02:22:00.164989+0000","last_unstale":"2023-08-28T02:26:23.881884+0000","last_undegraded":"2023-08-28T02:26:23.881884+0000","last_fullsized":"2023-08-28T02:26:23.881884+0000","mapping_epoch":119,"log_start":"131'4000","ondisk_log_start":"131'4000","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":9152,"log_dups_size":0,"ondisk_log_size":9152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:10:46.322659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37224448,"num_objects":568,"num_object_clones":0,"num_object_copies":1704,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":568,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13152,"num_write_kb":439040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":51,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.323266+0000","last_change":"2023-08-28T02:23:38.749428+0000","last_active":"2023-08-28T02:26:11.323266+0000","last_peered":"2023-08-28T02:26:11.323266+0000","last_clean":"2023-08-28T02:26:11.323266+0000","last_became_active":"2023-08-28T02:23:38.339587+0000","last_became_peered":"2023-08-28T02:23:38.339587+0000","last_unstale":"2023-08-28T02:26:11.323266+0000","last_undegraded":"2023-08-28T02:26:11.323266+0000","last_fullsized":"2023-08-28T02:26:11.323266+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":208,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.311042+0000","last_change":"2023-08-28T02:23:38.340065+0000","last_active":"2023-08-28T02:26:11.311042+0000","last_peered":"2023-08-28T02:26:11.311042+0000","last_clean":"2023-08-28T02:26:11.311042+0000","last_became_active":"2023-08-28T02:23:38.328689+0000","last_became_peered":"2023-08-28T02:23:38.328689+0000","last_unstale":"2023-08-28T02:26:11.311042+0000","last_undegraded":"2023-08-28T02:26:11.311042+0000","last_fullsized":"2023-08-28T02:26:11.311042+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"182'13228","reported_seq":20581,"reported_epoch":182,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-08-28T02:26:23.885144+0000","last_change":"2023-08-28T02:24:08.250989+0000","last_active":"2023-08-28T02:26:23.885144+0000","last_peered":"2023-08-28T02:26:23.885144+0000","last_clean":"2023-08-28T02:23:58.182539+0000","last_became_active":"2023-08-28T02:24:00.420432+0000","last_became_peered":"2023-08-28T02:24:00.420432+0000","last_unstale":"2023-08-28T02:26:23.885144+0000","last_undegraded":"2023-08-28T02:24:00.242280+0000","last_fullsized":"2023-08-28T02:24:00.199333+0000","mapping_epoch":154,"log_start":"131'4100","ondisk_log_start":"131'4100","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":9128,"log_dups_size":0,"ondisk_log_size":9128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:57:53.818041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35258368,"num_objects":538,"num_object_clones":0,"num_object_copies":1614,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1990,"num_objects_misplaced":450,"num_objects_unfound":0,"num_objects_dirty":538,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13228,"num_write_kb":440512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3636,"num_bytes_recovered":238288896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4],"avail_no_missing":["2","0","4"],"object_location_counts":[{"shards":"0,2,4","objects":450},{"shards":"2,4","objects":-1902}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":166,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.378973+0000","last_change":"2023-08-28T02:23:59.211466+0000","last_active":"2023-08-28T02:26:01.378973+0000","last_peered":"2023-08-28T02:26:01.378973+0000","last_clean":"2023-08-28T02:26:01.378973+0000","last_became_active":"2023-08-28T02:23:59.211175+0000","last_became_peered":"2023-08-28T02:23:59.211175+0000","last_unstale":"2023-08-28T02:26:01.378973+0000","last_undegraded":"2023-08-28T02:26:01.378973+0000","last_fullsized":"2023-08-28T02:26:01.378973+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":143,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.300846+0000","last_change":"2023-08-28T02:19:51.205245+0000","last_active":"2023-08-28T02:26:11.300846+0000","last_peered":"2023-08-28T02:26:11.300846+0000","last_clean":"2023-08-28T02:26:11.300846+0000","last_became_active":"2023-08-28T02:19:51.204896+0000","last_became_peered":"2023-08-28T02:19:51.204896+0000","last_unstale":"2023-08-28T02:26:11.300846+0000","last_undegraded":"2023-08-28T02:26:11.300846+0000","last_fullsized":"2023-08-28T02:26:11.300846+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:51:24.634702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"182'13276","reported_seq":14181,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-08-28T02:26:23.860659+0000","last_change":"2023-08-28T02:24:04.294764+0000","last_active":"2023-08-28T02:26:23.860659+0000","last_peered":"2023-08-28T02:26:23.860659+0000","last_clean":"2023-08-28T02:26:23.860659+0000","last_became_active":"2023-08-28T02:24:04.280269+0000","last_became_peered":"2023-08-28T02:24:04.280269+0000","last_unstale":"2023-08-28T02:26:23.860659+0000","last_undegraded":"2023-08-28T02:26:23.860659+0000","last_fullsized":"2023-08-28T02:26:23.860659+0000","mapping_epoch":157,"log_start":"132'4100","ondisk_log_start":"132'4100","created":119,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":9176,"log_dups_size":0,"ondisk_log_size":9176,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:18:55.996939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38862872,"num_objects":594,"num_object_clones":0,"num_object_copies":1782,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":594,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13276,"num_write_kb":443777,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":403,"num_bytes_recovered":26345496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":44,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:13.931087+0000","last_change":"2023-08-28T02:23:59.230059+0000","last_active":"2023-08-28T02:26:13.931087+0000","last_peered":"2023-08-28T02:26:13.931087+0000","last_clean":"2023-08-28T02:26:13.931087+0000","last_became_active":"2023-08-28T02:23:59.217783+0000","last_became_peered":"2023-08-28T02:23:59.217783+0000","last_unstale":"2023-08-28T02:26:13.931087+0000","last_undegraded":"2023-08-28T02:26:13.931087+0000","last_fullsized":"2023-08-28T02:26:13.931087+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:00:12.145395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":175,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.297385+0000","last_change":"2023-08-28T02:23:59.211286+0000","last_active":"2023-08-28T02:26:11.297385+0000","last_peered":"2023-08-28T02:26:11.297385+0000","last_clean":"2023-08-28T02:26:11.297385+0000","last_became_active":"2023-08-28T02:23:59.210942+0000","last_became_peered":"2023-08-28T02:23:59.210942+0000","last_unstale":"2023-08-28T02:26:11.297385+0000","last_undegraded":"2023-08-28T02:26:11.297385+0000","last_fullsized":"2023-08-28T02:26:11.297385+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"182'13436","reported_seq":13509,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-08-28T02:26:23.447431+0000","last_change":"2023-08-28T02:22:00.162375+0000","last_active":"2023-08-28T02:26:23.447431+0000","last_peered":"2023-08-28T02:26:23.447431+0000","last_clean":"2023-08-28T02:26:23.447431+0000","last_became_active":"2023-08-28T02:22:00.162190+0000","last_became_peered":"2023-08-28T02:22:00.162190+0000","last_unstale":"2023-08-28T02:26:23.447431+0000","last_undegraded":"2023-08-28T02:26:23.447431+0000","last_fullsized":"2023-08-28T02:26:23.447431+0000","mapping_epoch":119,"log_start":"134'5039","ondisk_log_start":"134'5039","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":8397,"log_dups_size":0,"ondisk_log_size":8397,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:40:08.985631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35913728,"num_objects":548,"num_object_clones":0,"num_object_copies":1644,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":548,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13436,"num_write_kb":447488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":177,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.310912+0000","last_change":"2023-08-28T02:21:59.179699+0000","last_active":"2023-08-28T02:26:11.310912+0000","last_peered":"2023-08-28T02:26:11.310912+0000","last_clean":"2023-08-28T02:26:11.310912+0000","last_became_active":"2023-08-28T02:21:59.179345+0000","last_became_peered":"2023-08-28T02:21:59.179345+0000","last_unstale":"2023-08-28T02:26:11.310912+0000","last_undegraded":"2023-08-28T02:26:11.310912+0000","last_fullsized":"2023-08-28T02:26:11.310912+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":201,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.297547+0000","last_change":"2023-08-28T02:22:44.083121+0000","last_active":"2023-08-28T02:26:11.297547+0000","last_peered":"2023-08-28T02:26:11.297547+0000","last_clean":"2023-08-28T02:26:11.297547+0000","last_became_active":"2023-08-28T02:22:44.082776+0000","last_became_peered":"2023-08-28T02:22:44.082776+0000","last_unstale":"2023-08-28T02:26:11.297547+0000","last_undegraded":"2023-08-28T02:26:11.297547+0000","last_fullsized":"2023-08-28T02:26:11.297547+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"181'13181","reported_seq":13251,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:22.038676+0000","last_change":"2023-08-28T02:22:00.167630+0000","last_active":"2023-08-28T02:26:22.038676+0000","last_peered":"2023-08-28T02:26:22.038676+0000","last_clean":"2023-08-28T02:26:22.038676+0000","last_became_active":"2023-08-28T02:22:00.167305+0000","last_became_peered":"2023-08-28T02:22:00.167305+0000","last_unstale":"2023-08-28T02:26:22.038676+0000","last_undegraded":"2023-08-28T02:26:22.038676+0000","last_fullsized":"2023-08-28T02:26:22.038676+0000","mapping_epoch":119,"log_start":"130'4000","ondisk_log_start":"130'4000","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":9181,"log_dups_size":0,"ondisk_log_size":9181,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40435712,"num_objects":617,"num_object_clones":0,"num_object_copies":1851,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":617,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13181,"num_write_kb":441536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":91,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.297494+0000","last_change":"2023-08-28T02:23:38.347043+0000","last_active":"2023-08-28T02:26:11.297494+0000","last_peered":"2023-08-28T02:26:11.297494+0000","last_clean":"2023-08-28T02:26:11.297494+0000","last_became_active":"2023-08-28T02:23:38.346680+0000","last_became_peered":"2023-08-28T02:23:38.346680+0000","last_unstale":"2023-08-28T02:26:11.297494+0000","last_undegraded":"2023-08-28T02:26:11.297494+0000","last_fullsized":"2023-08-28T02:26:11.297494+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":111,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.301533+0000","last_change":"2023-08-28T02:19:51.204017+0000","last_active":"2023-08-28T02:26:11.301533+0000","last_peered":"2023-08-28T02:26:11.301533+0000","last_clean":"2023-08-28T02:26:11.301533+0000","last_became_active":"2023-08-28T02:19:51.203627+0000","last_became_peered":"2023-08-28T02:19:51.203627+0000","last_unstale":"2023-08-28T02:26:11.301533+0000","last_undegraded":"2023-08-28T02:26:11.301533+0000","last_fullsized":"2023-08-28T02:26:11.301533+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:14:06.231701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.e","version":"182'13537","reported_seq":13608,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-08-28T02:26:26.986407+0000","last_change":"2023-08-28T02:22:00.166490+0000","last_active":"2023-08-28T02:26:26.986407+0000","last_peered":"2023-08-28T02:26:26.986407+0000","last_clean":"2023-08-28T02:26:26.986407+0000","last_became_active":"2023-08-28T02:22:00.166140+0000","last_became_peered":"2023-08-28T02:22:00.166140+0000","last_unstale":"2023-08-28T02:26:26.986407+0000","last_undegraded":"2023-08-28T02:26:26.986407+0000","last_fullsized":"2023-08-28T02:26:26.986407+0000","mapping_epoch":119,"log_start":"136'5600","ondisk_log_start":"136'5600","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":7937,"log_dups_size":0,"ondisk_log_size":7937,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28770304,"num_objects":439,"num_object_clones":0,"num_object_copies":1317,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":439,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13537,"num_write_kb":447232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":40,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.301771+0000","last_change":"2023-08-28T02:23:59.216950+0000","last_active":"2023-08-28T02:26:11.301771+0000","last_peered":"2023-08-28T02:26:11.301771+0000","last_clean":"2023-08-28T02:26:11.301771+0000","last_became_active":"2023-08-28T02:23:59.216373+0000","last_became_peered":"2023-08-28T02:23:59.216373+0000","last_unstale":"2023-08-28T02:26:11.301771+0000","last_undegraded":"2023-08-28T02:26:11.301771+0000","last_fullsized":"2023-08-28T02:26:11.301771+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:13:14.736771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":159,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.297609+0000","last_change":"2023-08-28T02:23:59.220269+0000","last_active":"2023-08-28T02:26:11.297609+0000","last_peered":"2023-08-28T02:26:11.297609+0000","last_clean":"2023-08-28T02:26:11.297609+0000","last_became_active":"2023-08-28T02:23:59.219966+0000","last_became_peered":"2023-08-28T02:23:59.219966+0000","last_unstale":"2023-08-28T02:26:11.297609+0000","last_undegraded":"2023-08-28T02:26:11.297609+0000","last_fullsized":"2023-08-28T02:26:11.297609+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:16:08.119909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"78'126","reported_seq":659,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.297477+0000","last_change":"2023-08-28T02:23:39.351975+0000","last_active":"2023-08-28T02:26:11.297477+0000","last_peered":"2023-08-28T02:26:11.297477+0000","last_clean":"2023-08-28T02:26:11.297477+0000","last_became_active":"2023-08-28T02:23:39.351437+0000","last_became_peered":"2023-08-28T02:23:39.351437+0000","last_unstale":"2023-08-28T02:26:11.297477+0000","last_undegraded":"2023-08-28T02:26:11.297477+0000","last_fullsized":"2023-08-28T02:26:11.297477+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:25:22.203120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":743,"num_read_kb":1331,"num_write":125,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":181,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.379082+0000","last_change":"2023-08-28T02:23:59.207148+0000","last_active":"2023-08-28T02:26:01.379082+0000","last_peered":"2023-08-28T02:26:01.379082+0000","last_clean":"2023-08-28T02:26:01.379082+0000","last_became_active":"2023-08-28T02:23:59.206802+0000","last_became_peered":"2023-08-28T02:23:59.206802+0000","last_unstale":"2023-08-28T02:26:01.379082+0000","last_undegraded":"2023-08-28T02:26:01.379082+0000","last_fullsized":"2023-08-28T02:26:01.379082+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":79,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.297654+0000","last_change":"2023-08-28T02:21:43.238044+0000","last_active":"2023-08-28T02:26:11.297654+0000","last_peered":"2023-08-28T02:26:11.297654+0000","last_clean":"2023-08-28T02:26:11.297654+0000","last_became_active":"2023-08-28T02:21:43.237792+0000","last_became_peered":"2023-08-28T02:21:43.237792+0000","last_unstale":"2023-08-28T02:26:11.297654+0000","last_undegraded":"2023-08-28T02:26:11.297654+0000","last_fullsized":"2023-08-28T02:26:11.297654+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:11:35.488503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"182'13101","reported_seq":13175,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-08-28T02:26:23.752122+0000","last_change":"2023-08-28T02:22:00.166578+0000","last_active":"2023-08-28T02:26:23.752122+0000","last_peered":"2023-08-28T02:26:23.752122+0000","last_clean":"2023-08-28T02:26:23.752122+0000","last_became_active":"2023-08-28T02:22:00.166311+0000","last_became_peered":"2023-08-28T02:22:00.166311+0000","last_unstale":"2023-08-28T02:26:23.752122+0000","last_undegraded":"2023-08-28T02:26:23.752122+0000","last_fullsized":"2023-08-28T02:26:23.752122+0000","mapping_epoch":119,"log_start":"135'5341","ondisk_log_start":"135'5341","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":7760,"log_dups_size":0,"ondisk_log_size":7760,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:40:38.502180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35979264,"num_objects":549,"num_object_clones":0,"num_object_copies":1647,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":549,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13101,"num_write_kb":436800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"182'13199","reported_seq":13273,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-08-28T02:26:23.726435+0000","last_change":"2023-08-28T02:22:00.160991+0000","last_active":"2023-08-28T02:26:23.726435+0000","last_peered":"2023-08-28T02:26:23.726435+0000","last_clean":"2023-08-28T02:26:23.726435+0000","last_became_active":"2023-08-28T02:22:00.160755+0000","last_became_peered":"2023-08-28T02:22:00.160755+0000","last_unstale":"2023-08-28T02:26:23.726435+0000","last_undegraded":"2023-08-28T02:26:23.726435+0000","last_fullsized":"2023-08-28T02:26:23.726435+0000","mapping_epoch":119,"log_start":"136'5498","ondisk_log_start":"136'5498","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":7701,"log_dups_size":0,"ondisk_log_size":7701,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:42:59.109288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35848192,"num_objects":547,"num_object_clones":0,"num_object_copies":1641,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":547,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13199,"num_write_kb":439872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"78'204","reported_seq":1097,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.378841+0000","last_change":"2023-08-28T02:19:50.219949+0000","last_active":"2023-08-28T02:26:01.378841+0000","last_peered":"2023-08-28T02:26:01.378841+0000","last_clean":"2023-08-28T02:26:01.378841+0000","last_became_active":"2023-08-28T02:19:50.219356+0000","last_became_peered":"2023-08-28T02:19:50.219356+0000","last_unstale":"2023-08-28T02:26:01.378841+0000","last_undegraded":"2023-08-28T02:26:01.378841+0000","last_fullsized":"2023-08-28T02:26:01.378841+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":204,"log_dups_size":0,"ondisk_log_size":204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1521,"num_read_kb":2511,"num_write":178,"num_write_kb":1175,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":149,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.297820+0000","last_change":"2023-08-28T02:18:39.779607+0000","last_active":"2023-08-28T02:26:11.297820+0000","last_peered":"2023-08-28T02:26:11.297820+0000","last_clean":"2023-08-28T02:26:11.297820+0000","last_became_active":"2023-08-28T02:18:39.779303+0000","last_became_peered":"2023-08-28T02:18:39.779303+0000","last_unstale":"2023-08-28T02:26:11.297820+0000","last_undegraded":"2023-08-28T02:26:11.297820+0000","last_fullsized":"2023-08-28T02:26:11.297820+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":140,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.297400+0000","last_change":"2023-08-28T02:23:59.221979+0000","last_active":"2023-08-28T02:26:11.297400+0000","last_peered":"2023-08-28T02:26:11.297400+0000","last_clean":"2023-08-28T02:26:11.297400+0000","last_became_active":"2023-08-28T02:23:59.221619+0000","last_became_peered":"2023-08-28T02:23:59.221619+0000","last_unstale":"2023-08-28T02:26:11.297400+0000","last_undegraded":"2023-08-28T02:26:11.297400+0000","last_fullsized":"2023-08-28T02:26:11.297400+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:54:45.892284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.6","version":"182'13008","reported_seq":13795,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-08-28T02:26:23.893183+0000","last_change":"2023-08-28T02:24:07.218778+0000","last_active":"2023-08-28T02:26:23.893183+0000","last_peered":"2023-08-28T02:26:23.893183+0000","last_clean":"2023-08-28T02:26:23.893183+0000","last_became_active":"2023-08-28T02:24:07.205246+0000","last_became_peered":"2023-08-28T02:24:07.205246+0000","last_unstale":"2023-08-28T02:26:23.893183+0000","last_undegraded":"2023-08-28T02:26:23.893183+0000","last_fullsized":"2023-08-28T02:26:23.893183+0000","mapping_epoch":159,"log_start":"131'3900","ondisk_log_start":"131'3900","created":119,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":9108,"log_dups_size":0,"ondisk_log_size":9108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:30:08.510532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33292288,"num_objects":508,"num_object_clones":0,"num_object_copies":1524,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":508,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13008,"num_write_kb":432512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":342,"num_bytes_recovered":22413312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"78'47","reported_seq":396,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.315056+0000","last_change":"2023-08-28T02:24:05.799647+0000","last_active":"2023-08-28T02:26:11.315056+0000","last_peered":"2023-08-28T02:26:11.315056+0000","last_clean":"2023-08-28T02:26:11.315056+0000","last_became_active":"2023-08-28T02:23:59.210481+0000","last_became_peered":"2023-08-28T02:23:59.210481+0000","last_unstale":"2023-08-28T02:26:11.315056+0000","last_undegraded":"2023-08-28T02:26:11.315056+0000","last_fullsized":"2023-08-28T02:26:11.315056+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":47,"log_dups_size":0,"ondisk_log_size":47,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:40:18.542375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":162,"num_read_kb":1281,"num_write":48,"num_write_kb":1153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":123,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.297964+0000","last_change":"2023-08-28T02:20:19.851591+0000","last_active":"2023-08-28T02:26:11.297964+0000","last_peered":"2023-08-28T02:26:11.297964+0000","last_clean":"2023-08-28T02:26:11.297964+0000","last_became_active":"2023-08-28T02:20:19.851291+0000","last_became_peered":"2023-08-28T02:20:19.851291+0000","last_unstale":"2023-08-28T02:26:11.297964+0000","last_undegraded":"2023-08-28T02:26:11.297964+0000","last_fullsized":"2023-08-28T02:26:11.297964+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:33:51.259707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":155,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:18.447649+0000","last_change":"2023-08-28T02:19:50.211156+0000","last_active":"2023-08-28T02:26:18.447649+0000","last_peered":"2023-08-28T02:26:18.447649+0000","last_clean":"2023-08-28T02:26:18.447649+0000","last_became_active":"2023-08-28T02:19:50.210398+0000","last_became_peered":"2023-08-28T02:19:50.210398+0000","last_unstale":"2023-08-28T02:26:18.447649+0000","last_undegraded":"2023-08-28T02:26:18.447649+0000","last_fullsized":"2023-08-28T02:26:18.447649+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:14:41.748197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"78'114","reported_seq":545,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.378936+0000","last_change":"2023-08-28T02:24:01.210739+0000","last_active":"2023-08-28T02:26:01.378936+0000","last_peered":"2023-08-28T02:26:01.378936+0000","last_clean":"2023-08-28T02:26:01.378936+0000","last_became_active":"2023-08-28T02:24:01.210386+0000","last_became_peered":"2023-08-28T02:24:01.210386+0000","last_unstale":"2023-08-28T02:26:01.378936+0000","last_undegraded":"2023-08-28T02:26:01.378936+0000","last_fullsized":"2023-08-28T02:26:01.378936+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":114,"log_dups_size":0,"ondisk_log_size":114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":358,"num_read_kb":1352,"num_write":102,"num_write_kb":1104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"86'139","reported_seq":340,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.322405+0000","last_change":"2023-08-28T02:24:04.276519+0000","last_active":"2023-08-28T02:26:11.322405+0000","last_peered":"2023-08-28T02:26:11.322405+0000","last_clean":"2023-08-28T02:26:11.322405+0000","last_became_active":"2023-08-28T02:24:04.276104+0000","last_became_peered":"2023-08-28T02:24:04.276104+0000","last_unstale":"2023-08-28T02:26:11.322405+0000","last_undegraded":"2023-08-28T02:26:11.322405+0000","last_fullsized":"2023-08-28T02:26:11.322405+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_clean_scrub_stamp":"2023-08-28T02:08:02.043718+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":40,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.302560+0000","last_change":"2023-08-28T02:23:59.219981+0000","last_active":"2023-08-28T02:26:11.302560+0000","last_peered":"2023-08-28T02:26:11.302560+0000","last_clean":"2023-08-28T02:26:11.302560+0000","last_became_active":"2023-08-28T02:23:59.219633+0000","last_became_peered":"2023-08-28T02:23:59.219633+0000","last_unstale":"2023-08-28T02:26:11.302560+0000","last_undegraded":"2023-08-28T02:26:11.302560+0000","last_fullsized":"2023-08-28T02:26:11.302560+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:56:14.172518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":150,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.310303+0000","last_change":"2023-08-28T02:23:38.341351+0000","last_active":"2023-08-28T02:26:11.310303+0000","last_peered":"2023-08-28T02:26:11.310303+0000","last_clean":"2023-08-28T02:26:11.310303+0000","last_became_active":"2023-08-28T02:23:38.340997+0000","last_became_peered":"2023-08-28T02:23:38.340997+0000","last_unstale":"2023-08-28T02:26:11.310303+0000","last_undegraded":"2023-08-28T02:26:11.310303+0000","last_fullsized":"2023-08-28T02:26:11.310303+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"182'13241","reported_seq":14080,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-08-28T02:26:27.000908+0000","last_change":"2023-08-28T02:23:47.215362+0000","last_active":"2023-08-28T02:26:27.000908+0000","last_peered":"2023-08-28T02:26:27.000908+0000","last_clean":"2023-08-28T02:26:27.000908+0000","last_became_active":"2023-08-28T02:23:47.207109+0000","last_became_peered":"2023-08-28T02:23:47.207109+0000","last_unstale":"2023-08-28T02:26:27.000908+0000","last_undegraded":"2023-08-28T02:26:27.000908+0000","last_fullsized":"2023-08-28T02:26:27.000908+0000","mapping_epoch":150,"log_start":"133'4600","ondisk_log_start":"133'4600","created":119,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":8641,"log_dups_size":0,"ondisk_log_size":8641,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27197440,"num_objects":415,"num_object_clones":0,"num_object_copies":1245,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":415,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13241,"num_write_kb":436992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":370,"num_bytes_recovered":24182784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.7","version":"181'13264","reported_seq":13334,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:22.179175+0000","last_change":"2023-08-28T02:22:00.165614+0000","last_active":"2023-08-28T02:26:22.179175+0000","last_peered":"2023-08-28T02:26:22.179175+0000","last_clean":"2023-08-28T02:26:22.179175+0000","last_became_active":"2023-08-28T02:22:00.165136+0000","last_became_peered":"2023-08-28T02:22:00.165136+0000","last_unstale":"2023-08-28T02:26:22.179175+0000","last_undegraded":"2023-08-28T02:26:22.179175+0000","last_fullsized":"2023-08-28T02:26:22.179175+0000","mapping_epoch":119,"log_start":"131'4100","ondisk_log_start":"131'4100","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":9164,"log_dups_size":0,"ondisk_log_size":9164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38666240,"num_objects":590,"num_object_clones":0,"num_object_copies":1770,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":590,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13264,"num_write_kb":443328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"78'148","reported_seq":631,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.322312+0000","last_change":"2023-08-28T02:24:01.211494+0000","last_active":"2023-08-28T02:26:11.322312+0000","last_peered":"2023-08-28T02:26:11.322312+0000","last_clean":"2023-08-28T02:26:11.322312+0000","last_became_active":"2023-08-28T02:24:01.210957+0000","last_became_peered":"2023-08-28T02:24:01.210957+0000","last_unstale":"2023-08-28T02:26:11.322312+0000","last_undegraded":"2023-08-28T02:26:11.322312+0000","last_fullsized":"2023-08-28T02:26:11.322312+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1092,"num_read_kb":1955,"num_write":127,"num_write_kb":5177,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":149,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.304261+0000","last_change":"2023-08-28T02:23:59.226986+0000","last_active":"2023-08-28T02:26:11.304261+0000","last_peered":"2023-08-28T02:26:11.304261+0000","last_clean":"2023-08-28T02:26:11.304261+0000","last_became_active":"2023-08-28T02:23:59.226529+0000","last_became_peered":"2023-08-28T02:23:59.226529+0000","last_unstale":"2023-08-28T02:26:11.304261+0000","last_undegraded":"2023-08-28T02:26:11.304261+0000","last_fullsized":"2023-08-28T02:26:11.304261+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:22:09.193619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":176,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.300982+0000","last_change":"2023-08-28T02:21:59.167858+0000","last_active":"2023-08-28T02:26:11.300982+0000","last_peered":"2023-08-28T02:26:11.300982+0000","last_clean":"2023-08-28T02:26:11.300982+0000","last_became_active":"2023-08-28T02:21:59.167436+0000","last_became_peered":"2023-08-28T02:21:59.167436+0000","last_unstale":"2023-08-28T02:26:11.300982+0000","last_undegraded":"2023-08-28T02:26:11.300982+0000","last_fullsized":"2023-08-28T02:26:11.300982+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:17:50.717282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"78'103","reported_seq":558,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.323352+0000","last_change":"2023-08-28T02:24:07.212128+0000","last_active":"2023-08-28T02:26:11.323352+0000","last_peered":"2023-08-28T02:26:11.323352+0000","last_clean":"2023-08-28T02:26:11.323352+0000","last_became_active":"2023-08-28T02:24:07.211562+0000","last_became_peered":"2023-08-28T02:24:07.211562+0000","last_unstale":"2023-08-28T02:26:11.323352+0000","last_undegraded":"2023-08-28T02:26:11.323352+0000","last_fullsized":"2023-08-28T02:26:11.323352+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":803,"num_read_kb":2080,"num_write":73,"num_write_kb":1244,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":123,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.297646+0000","last_change":"2023-08-28T02:20:19.850916+0000","last_active":"2023-08-28T02:26:11.297646+0000","last_peered":"2023-08-28T02:26:11.297646+0000","last_clean":"2023-08-28T02:26:11.297646+0000","last_became_active":"2023-08-28T02:20:19.850627+0000","last_became_peered":"2023-08-28T02:20:19.850627+0000","last_unstale":"2023-08-28T02:26:11.297646+0000","last_undegraded":"2023-08-28T02:26:11.297646+0000","last_fullsized":"2023-08-28T02:26:11.297646+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:19:14.346189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":135,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.301299+0000","last_change":"2023-08-28T02:23:38.329667+0000","last_active":"2023-08-28T02:26:11.301299+0000","last_peered":"2023-08-28T02:26:11.301299+0000","last_clean":"2023-08-28T02:26:11.301299+0000","last_became_active":"2023-08-28T02:23:38.329175+0000","last_became_peered":"2023-08-28T02:23:38.329175+0000","last_unstale":"2023-08-28T02:26:11.301299+0000","last_undegraded":"2023-08-28T02:26:11.301299+0000","last_fullsized":"2023-08-28T02:26:11.301299+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:21:34.815067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"182'13140","reported_seq":13210,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-08-28T02:26:27.009820+0000","last_change":"2023-08-28T02:22:00.167814+0000","last_active":"2023-08-28T02:26:27.009820+0000","last_peered":"2023-08-28T02:26:27.009820+0000","last_clean":"2023-08-28T02:26:27.009820+0000","last_became_active":"2023-08-28T02:22:00.167513+0000","last_became_peered":"2023-08-28T02:22:00.167513+0000","last_unstale":"2023-08-28T02:26:27.009820+0000","last_undegraded":"2023-08-28T02:26:27.009820+0000","last_fullsized":"2023-08-28T02:26:27.009820+0000","mapping_epoch":119,"log_start":"129'3400","ondisk_log_start":"129'3400","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":9740,"log_dups_size":0,"ondisk_log_size":9740,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27918336,"num_objects":426,"num_object_clones":0,"num_object_copies":1278,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":426,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13140,"num_write_kb":434112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":44,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.302541+0000","last_change":"2023-08-28T02:23:59.232099+0000","last_active":"2023-08-28T02:26:11.302541+0000","last_peered":"2023-08-28T02:26:11.302541+0000","last_clean":"2023-08-28T02:26:11.302541+0000","last_became_active":"2023-08-28T02:23:59.219318+0000","last_became_peered":"2023-08-28T02:23:59.219318+0000","last_unstale":"2023-08-28T02:26:11.302541+0000","last_undegraded":"2023-08-28T02:26:11.302541+0000","last_fullsized":"2023-08-28T02:26:11.302541+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:59:30.093289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'112","reported_seq":427,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:18.447624+0000","last_change":"2023-08-28T02:19:50.218206+0000","last_active":"2023-08-28T02:26:18.447624+0000","last_peered":"2023-08-28T02:26:18.447624+0000","last_clean":"2023-08-28T02:26:18.447624+0000","last_became_active":"2023-08-28T02:19:50.217637+0000","last_became_peered":"2023-08-28T02:19:50.217637+0000","last_unstale":"2023-08-28T02:26:18.447624+0000","last_undegraded":"2023-08-28T02:26:18.447624+0000","last_fullsized":"2023-08-28T02:26:18.447624+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:12:06.224907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":444,"num_read_kb":473,"num_write":94,"num_write_kb":127,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":40,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.301854+0000","last_change":"2023-08-28T02:23:59.221034+0000","last_active":"2023-08-28T02:26:11.301854+0000","last_peered":"2023-08-28T02:26:11.301854+0000","last_clean":"2023-08-28T02:26:11.301854+0000","last_became_active":"2023-08-28T02:23:59.220730+0000","last_became_peered":"2023-08-28T02:23:59.220730+0000","last_unstale":"2023-08-28T02:26:11.301854+0000","last_undegraded":"2023-08-28T02:26:11.301854+0000","last_fullsized":"2023-08-28T02:26:11.301854+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:25:52.364370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"182'13449","reported_seq":16615,"reported_epoch":182,"state":"active+remapped+backfill_wait","last_fresh":"2023-08-28T02:26:26.983104+0000","last_change":"2023-08-28T02:24:05.805308+0000","last_active":"2023-08-28T02:26:26.983104+0000","last_peered":"2023-08-28T02:26:26.983104+0000","last_clean":"2023-08-28T02:23:25.043301+0000","last_became_active":"2023-08-28T02:24:00.316639+0000","last_became_peered":"2023-08-28T02:24:00.316639+0000","last_unstale":"2023-08-28T02:26:26.983104+0000","last_undegraded":"2023-08-28T02:26:26.983104+0000","last_fullsized":"2023-08-28T02:26:26.983104+0000","mapping_epoch":154,"log_start":"136'5500","ondisk_log_start":"136'5500","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":7949,"log_dups_size":0,"ondisk_log_size":7949,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28901376,"num_objects":441,"num_object_clones":0,"num_object_copies":1323,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":386,"num_objects_unfound":0,"num_objects_dirty":441,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13449,"num_write_kb":444480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1527,"num_bytes_recovered":99942400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,7,0],"avail_no_missing":["4","0","5","7"],"object_location_counts":[{"shards":"0,4,7","objects":441}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"78'122","reported_seq":711,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.310201+0000","last_change":"2023-08-28T02:24:04.276850+0000","last_active":"2023-08-28T02:26:11.310201+0000","last_peered":"2023-08-28T02:26:11.310201+0000","last_clean":"2023-08-28T02:26:11.310201+0000","last_became_active":"2023-08-28T02:24:04.276367+0000","last_became_peered":"2023-08-28T02:24:04.276367+0000","last_unstale":"2023-08-28T02:26:11.310201+0000","last_undegraded":"2023-08-28T02:26:11.310201+0000","last_fullsized":"2023-08-28T02:26:11.310201+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":857,"num_read_kb":2224,"num_write":99,"num_write_kb":2312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":135,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.301071+0000","last_change":"2023-08-28T02:19:50.221001+0000","last_active":"2023-08-28T02:26:11.301071+0000","last_peered":"2023-08-28T02:26:11.301071+0000","last_clean":"2023-08-28T02:26:11.301071+0000","last_became_active":"2023-08-28T02:19:50.220611+0000","last_became_peered":"2023-08-28T02:19:50.220611+0000","last_unstale":"2023-08-28T02:26:11.301071+0000","last_undegraded":"2023-08-28T02:26:11.301071+0000","last_fullsized":"2023-08-28T02:26:11.301071+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:23:10.637630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":133,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.297200+0000","last_change":"2023-08-28T02:11:22.437146+0000","last_active":"2023-08-28T02:26:11.297200+0000","last_peered":"2023-08-28T02:26:11.297200+0000","last_clean":"2023-08-28T02:26:11.297200+0000","last_became_active":"2023-08-28T02:11:22.436798+0000","last_became_peered":"2023-08-28T02:11:22.436798+0000","last_unstale":"2023-08-28T02:26:11.297200+0000","last_undegraded":"2023-08-28T02:26:11.297200+0000","last_fullsized":"2023-08-28T02:26:11.297200+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"182'13247","reported_seq":13327,"reported_epoch":182,"state":"active+remapped+backfill_wait","last_fresh":"2023-08-28T02:26:23.720967+0000","last_change":"2023-08-28T02:24:00.310494+0000","last_active":"2023-08-28T02:26:23.720967+0000","last_peered":"2023-08-28T02:26:23.720967+0000","last_clean":"2023-08-28T02:23:58.174026+0000","last_became_active":"2023-08-28T02:24:00.290543+0000","last_became_peered":"2023-08-28T02:24:00.290543+0000","last_unstale":"2023-08-28T02:26:23.720967+0000","last_undegraded":"2023-08-28T02:26:23.720967+0000","last_fullsized":"2023-08-28T02:26:23.720967+0000","mapping_epoch":154,"log_start":"135'5477","ondisk_log_start":"135'5477","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":7770,"log_dups_size":0,"ondisk_log_size":7770,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:35:56.609960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33095680,"num_objects":505,"num_object_clones":0,"num_object_copies":1515,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":505,"num_objects_unfound":0,"num_objects_dirty":505,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13247,"num_write_kb":440064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[0,1,4],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":505}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":138,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.301215+0000","last_change":"2023-08-28T02:23:38.329474+0000","last_active":"2023-08-28T02:26:11.301215+0000","last_peered":"2023-08-28T02:26:11.301215+0000","last_clean":"2023-08-28T02:26:11.301215+0000","last_became_active":"2023-08-28T02:23:38.328505+0000","last_became_peered":"2023-08-28T02:23:38.328505+0000","last_unstale":"2023-08-28T02:26:11.301215+0000","last_undegraded":"2023-08-28T02:26:11.301215+0000","last_fullsized":"2023-08-28T02:26:11.301215+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:12:14.835809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":203,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.315659+0000","last_change":"2023-08-28T02:23:59.212036+0000","last_active":"2023-08-28T02:26:11.315659+0000","last_peered":"2023-08-28T02:26:11.315659+0000","last_clean":"2023-08-28T02:26:11.315659+0000","last_became_active":"2023-08-28T02:23:59.211709+0000","last_became_peered":"2023-08-28T02:23:59.211709+0000","last_unstale":"2023-08-28T02:26:11.315659+0000","last_undegraded":"2023-08-28T02:26:11.315659+0000","last_fullsized":"2023-08-28T02:26:11.315659+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T14:04:25.046626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":150,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.378982+0000","last_change":"2023-08-28T02:19:50.218760+0000","last_active":"2023-08-28T02:26:01.378982+0000","last_peered":"2023-08-28T02:26:01.378982+0000","last_clean":"2023-08-28T02:26:01.378982+0000","last_became_active":"2023-08-28T02:19:50.218376+0000","last_became_peered":"2023-08-28T02:19:50.218376+0000","last_unstale":"2023-08-28T02:26:01.378982+0000","last_undegraded":"2023-08-28T02:26:01.378982+0000","last_fullsized":"2023-08-28T02:26:01.378982+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":181,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.297838+0000","last_change":"2023-08-28T02:23:59.207822+0000","last_active":"2023-08-28T02:26:11.297838+0000","last_peered":"2023-08-28T02:26:11.297838+0000","last_clean":"2023-08-28T02:26:11.297838+0000","last_became_active":"2023-08-28T02:23:59.207475+0000","last_became_peered":"2023-08-28T02:23:59.207475+0000","last_unstale":"2023-08-28T02:26:11.297838+0000","last_undegraded":"2023-08-28T02:26:11.297838+0000","last_fullsized":"2023-08-28T02:26:11.297838+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":135,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-08-28T02:26:01.424241+0000","last_change":"2023-08-28T02:11:22.186024+0000","last_active":"2023-08-28T02:26:01.424241+0000","last_peered":"2023-08-28T02:26:01.424241+0000","last_clean":"2023-08-28T02:26:01.424241+0000","last_became_active":"2023-08-28T02:11:22.185500+0000","last_became_peered":"2023-08-28T02:11:22.185500+0000","last_unstale":"2023-08-28T02:26:01.424241+0000","last_undegraded":"2023-08-28T02:26:01.424241+0000","last_fullsized":"2023-08-28T02:26:01.424241+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":181,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.297629+0000","last_change":"2023-08-28T02:23:59.207929+0000","last_active":"2023-08-28T02:26:11.297629+0000","last_peered":"2023-08-28T02:26:11.297629+0000","last_clean":"2023-08-28T02:26:11.297629+0000","last_became_active":"2023-08-28T02:23:59.207589+0000","last_became_peered":"2023-08-28T02:23:59.207589+0000","last_unstale":"2023-08-28T02:26:11.297629+0000","last_undegraded":"2023-08-28T02:26:11.297629+0000","last_fullsized":"2023-08-28T02:26:11.297629+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":40,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.302332+0000","last_change":"2023-08-28T02:23:59.220437+0000","last_active":"2023-08-28T02:26:11.302332+0000","last_peered":"2023-08-28T02:26:11.302332+0000","last_clean":"2023-08-28T02:26:11.302332+0000","last_became_active":"2023-08-28T02:23:59.220091+0000","last_became_peered":"2023-08-28T02:23:59.220091+0000","last_unstale":"2023-08-28T02:26:11.302332+0000","last_undegraded":"2023-08-28T02:26:11.302332+0000","last_fullsized":"2023-08-28T02:26:11.302332+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:51:21.695566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":40,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-08-28T02:26:22.313417+0000","last_change":"2023-08-28T02:23:59.217794+0000","last_active":"2023-08-28T02:26:22.313417+0000","last_peered":"2023-08-28T02:26:22.313417+0000","last_clean":"2023-08-28T02:26:22.313417+0000","last_became_active":"2023-08-28T02:23:59.217231+0000","last_became_peered":"2023-08-28T02:23:59.217231+0000","last_unstale":"2023-08-28T02:26:22.313417+0000","last_undegraded":"2023-08-28T02:26:22.313417+0000","last_fullsized":"2023-08-28T02:26:22.313417+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:36:26.297410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":135,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.301690+0000","last_change":"2023-08-28T02:20:19.850252+0000","last_active":"2023-08-28T02:26:11.301690+0000","last_peered":"2023-08-28T02:26:11.301690+0000","last_clean":"2023-08-28T02:26:11.301690+0000","last_became_active":"2023-08-28T02:20:19.849913+0000","last_became_peered":"2023-08-28T02:20:19.849913+0000","last_unstale":"2023-08-28T02:26:11.301690+0000","last_undegraded":"2023-08-28T02:26:11.301690+0000","last_fullsized":"2023-08-28T02:26:11.301690+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:48:47.713465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":177,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-08-28T02:26:11.297188+0000","last_change":"2023-08-28T02:21:59.180201+0000","last_active":"2023-08-28T02:26:11.297188+0000","last_peered":"2023-08-28T02:26:11.297188+0000","last_clean":"2023-08-28T02:26:11.297188+0000","last_became_active":"2023-08-28T02:21:59.179863+0000","last_became_peered":"2023-08-28T02:21:59.179863+0000","last_unstale":"2023-08-28T02:26:11.297188+0000","last_undegraded":"2023-08-28T02:26:11.297188+0000","last_fullsized":"2023-08-28T02:26:11.297188+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":542113816,"num_objects":8273,"num_object_clones":0,"num_object_copies":24819,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1990,"num_objects_misplaced":1341,"num_objects_unfound":0,"num_objects_dirty":8273,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":212009,"num_write_kb":7048961,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6645,"num_bytes_recovered":434962456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2036281344,"data_stored":2036269128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":138554,"ondisk_log_size":138554,"up":48,"acting":47,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":9774,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":21246,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26810,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430410,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":456416,"kb_used_data":151000,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93288736,"statfs":{"total":95995035648,"available":95527665664,"internally_reserved":0,"allocated":154624000,"data_stored":154040729,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:25:56 2023","interfaces":[{"interface":"back","average":{"1min":2.536,"5min":3.7240000000000002,"15min":1.5980000000000001},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":9.5679999999999996,"5min":116.755,"15min":116.755},"last":0.36299999999999999},{"interface":"front","average":{"1min":3.1320000000000001,"5min":4.1520000000000001,"15min":1.7889999999999999},"min":{"1min":0.28000000000000003,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":12.206,"5min":116.971,"15min":116.971},"last":0.41699999999999998}]},{"osd":1,"last update":"Mon Aug 28 02:26:03 2023","interfaces":[{"interface":"back","average":{"1min":1.5800000000000001,"5min":1.143,"15min":0.73299999999999998},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":9.0139999999999993,"5min":9.8710000000000004,"15min":9.8710000000000004},"last":0.40799999999999997},{"interface":"front","average":{"1min":3.0169999999999999,"5min":1.7889999999999999,"15min":0.98799999999999999},"min":{"1min":0.27700000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":11.140000000000001,"5min":11.696999999999999,"15min":11.696999999999999},"last":0.34000000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:25:27 2023","interfaces":[{"interface":"back","average":{"1min":8.6959999999999997,"5min":3.8700000000000001,"15min":1.673},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":119.571,"5min":119.571,"15min":119.571},"last":0.378},{"interface":"front","average":{"1min":9.7100000000000009,"5min":4.319,"15min":1.8300000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":119.554,"5min":119.554,"15min":119.554},"last":0.35799999999999998}]},{"osd":3,"last update":"Mon Aug 28 02:26:03 2023","interfaces":[{"interface":"back","average":{"1min":2.2250000000000001,"5min":2.3260000000000001,"15min":1.2210000000000001},"min":{"1min":0.36799999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":6.5650000000000004,"5min":107.916,"15min":107.916},"last":0.34599999999999997},{"interface":"front","average":{"1min":2.548,"5min":2.9990000000000001,"15min":1.446},"min":{"1min":0.378,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":9.8460000000000001,"5min":112.00700000000001,"15min":112.00700000000001},"last":0.38700000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:25:27 2023","interfaces":[{"interface":"back","average":{"1min":1.0980000000000001,"5min":2.0350000000000001,"15min":0.996},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":3.8359999999999999,"5min":63.048000000000002,"15min":63.048000000000002},"last":11.404},{"interface":"front","average":{"1min":1.659,"5min":1.8759999999999999,"15min":0.96199999999999997},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":8.2720000000000002,"5min":62.912999999999997,"15min":62.912999999999997},"last":10.840999999999999}]},{"osd":5,"last update":"Mon Aug 28 02:25:27 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":1.554,"15min":0.83999999999999997},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":5.7110000000000003,"5min":63.021000000000001,"15min":63.021000000000001},"last":6.976},{"interface":"front","average":{"1min":1.4039999999999999,"5min":2.1779999999999999,"15min":1.089},"min":{"1min":0.35699999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":5.3470000000000004,"5min":63.082999999999998,"15min":63.082999999999998},"last":7.181}]},{"osd":6,"last update":"Mon Aug 28 02:25:27 2023","interfaces":[{"interface":"back","average":{"1min":1.716,"5min":2.198,"15min":1.0860000000000001},"min":{"1min":0.317,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":6.0659999999999998,"5min":101.61,"15min":101.61},"last":0.32100000000000001},{"interface":"front","average":{"1min":1.3080000000000001,"5min":2.391,"15min":1.1890000000000001},"min":{"1min":0.34399999999999997,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":5.407,"5min":101.621,"15min":101.621},"last":0.308}]}]},{"osd":6,"up_from":42,"seq":180388626637,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":408272,"kb_used_data":111048,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":93336880,"statfs":{"total":95995035648,"available":95576965120,"internally_reserved":0,"allocated":113713152,"data_stored":113145799,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:26:02 2023","interfaces":[{"interface":"back","average":{"1min":3.48,"5min":4.3949999999999996,"15min":1.845},"min":{"1min":0.245,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":13.260999999999999,"5min":121.247,"15min":121.247},"last":0.23400000000000001},{"interface":"front","average":{"1min":4.4560000000000004,"5min":4.6369999999999996,"15min":1.9430000000000001},"min":{"1min":0.371,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":13.298999999999999,"5min":118.977,"15min":118.977},"last":0.67800000000000005}]},{"osd":1,"last update":"Mon Aug 28 02:26:02 2023","interfaces":[{"interface":"back","average":{"1min":2.0910000000000002,"5min":2.258,"15min":1.1100000000000001},"min":{"1min":0.307,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":6.7050000000000001,"5min":34.677999999999997,"15min":34.677999999999997},"last":0.30599999999999999},{"interface":"front","average":{"1min":2.7850000000000001,"5min":2.5009999999999999,"15min":1.2130000000000001},"min":{"1min":0.317,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":9.6799999999999997,"5min":34.348999999999997,"15min":34.348999999999997},"last":0.72899999999999998}]},{"osd":2,"last update":"Mon Aug 28 02:25:35 2023","interfaces":[{"interface":"back","average":{"1min":3.0590000000000002,"5min":2.2959999999999998,"15min":1.1719999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":15.762,"5min":49.555999999999997,"15min":49.555999999999997},"last":0.753},{"interface":"front","average":{"1min":2.9710000000000001,"5min":2.1160000000000001,"15min":1.089},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":11.503,"5min":49.470999999999997,"15min":49.470999999999997},"last":3.9390000000000001}]},{"osd":3,"last update":"Mon Aug 28 02:26:02 2023","interfaces":[{"interface":"back","average":{"1min":2.2869999999999999,"5min":1.885,"15min":1.075},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":6.4699999999999998,"5min":13.103,"15min":13.103},"last":9.5269999999999992},{"interface":"front","average":{"1min":2.508,"5min":2.0649999999999999,"15min":1.143},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":11.395,"5min":13.09,"15min":13.09},"last":1.968}]},{"osd":4,"last update":"Mon Aug 28 02:26:06 2023","interfaces":[{"interface":"back","average":{"1min":1.5880000000000001,"5min":1.669,"15min":0.86699999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":8.9580000000000002,"5min":10.637,"15min":10.637},"last":9.4700000000000006},{"interface":"front","average":{"1min":2.0819999999999999,"5min":1.8100000000000001,"15min":0.94299999999999995},"min":{"1min":0.29899999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":9.7690000000000001,"5min":10.055,"15min":10.055},"last":1.9390000000000001}]},{"osd":5,"last update":"Mon Aug 28 02:26:06 2023","interfaces":[{"interface":"back","average":{"1min":1.982,"5min":1.9399999999999999,"15min":0.96499999999999997},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":9.9809999999999999,"5min":20.759,"15min":20.759},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.8220000000000001,"5min":1.484,"15min":0.85099999999999998},"min":{"1min":0.32200000000000001,"5min":0.311,"15min":0.28799999999999998},"max":{"1min":5.5030000000000001,"5min":7.3780000000000001,"15min":7.3780000000000001},"last":0.38900000000000001}]},{"osd":7,"last update":"Mon Aug 28 02:25:33 2023","interfaces":[{"interface":"back","average":{"1min":1.7949999999999999,"5min":1.3320000000000001,"15min":0.80700000000000005},"min":{"1min":0.34799999999999998,"5min":0.314,"15min":0.314},"max":{"1min":6.4109999999999996,"5min":7.1769999999999996,"15min":7.1769999999999996},"last":0.42299999999999999},{"interface":"front","average":{"1min":1.9379999999999999,"5min":1.49,"15min":0.879},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.29799999999999999},"max":{"1min":6.8949999999999996,"5min":7.069,"15min":7.069},"last":0.77500000000000002}]}]},{"osd":1,"up_from":98,"seq":420906795089,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":294828,"kb_used_data":182628,"kb_used_omap":7,"kb_used_meta":112056,"kb_avail":93450324,"statfs":{"total":95995035648,"available":95693131776,"internally_reserved":0,"allocated":187011072,"data_stored":186139476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":114745675},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:26:01 2023","interfaces":[{"interface":"back","average":{"1min":14.807,"5min":4.4260000000000002,"15min":1.6990000000000001},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":128.60599999999999,"5min":128.60599999999999,"15min":128.60599999999999},"last":7.0019999999999998},{"interface":"front","average":{"1min":14.028,"5min":4.4489999999999998,"15min":1.7190000000000001},"min":{"1min":0.23699999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":128.685,"5min":128.685,"15min":128.685},"last":9.5299999999999994}]},{"osd":2,"last update":"Mon Aug 28 02:25:27 2023","interfaces":[{"interface":"back","average":{"1min":5.5549999999999997,"5min":2.2530000000000001,"15min":0.97899999999999998},"min":{"1min":0.26200000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":29.483000000000001,"5min":29.483000000000001,"15min":29.483000000000001},"last":0.86799999999999999},{"interface":"front","average":{"1min":3.286,"5min":1.7470000000000001,"15min":0.83599999999999997},"min":{"1min":0.23200000000000001,"5min":0.221,"15min":0.221},"max":{"1min":19.045000000000002,"5min":19.045000000000002,"15min":19.045000000000002},"last":7.0339999999999998}]},{"osd":3,"last update":"Mon Aug 28 02:26:01 2023","interfaces":[{"interface":"back","average":{"1min":2.1200000000000001,"5min":1.8140000000000001,"15min":0.84799999999999998},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":12.321999999999999,"5min":15.395,"15min":15.395},"last":1.8160000000000001},{"interface":"front","average":{"1min":3.7250000000000001,"5min":2.1480000000000001,"15min":1.0229999999999999},"min":{"1min":0.26900000000000002,"5min":0.254,"15min":0.254},"max":{"1min":22.361999999999998,"5min":22.361999999999998,"15min":22.361999999999998},"last":0.754}]},{"osd":4,"last update":"Mon Aug 28 02:26:01 2023","interfaces":[{"interface":"back","average":{"1min":3.3570000000000002,"5min":1.9339999999999999,"15min":1.0620000000000001},"min":{"1min":0.32800000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":17.600000000000001,"5min":17.600000000000001,"15min":17.600000000000001},"last":0.82399999999999995},{"interface":"front","average":{"1min":2.5499999999999998,"5min":1.556,"15min":0.93200000000000005},"min":{"1min":0.36299999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":19.010000000000002,"5min":19.010000000000002,"15min":19.010000000000002},"last":0.79600000000000004}]},{"osd":5,"last update":"Mon Aug 28 02:26:01 2023","interfaces":[{"interface":"back","average":{"1min":1.754,"5min":2.0089999999999999,"15min":1.0940000000000001},"min":{"1min":0.312,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.0579999999999998,"5min":15.544,"15min":15.544},"last":1.927},{"interface":"front","average":{"1min":3.4529999999999998,"5min":1.839,"15min":1.04},"min":{"1min":0.38800000000000001,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":17.646999999999998,"5min":17.646999999999998,"15min":17.646999999999998},"last":0.90000000000000002}]},{"osd":6,"last update":"Mon Aug 28 02:26:01 2023","interfaces":[{"interface":"back","average":{"1min":1.992,"5min":2.3359999999999999,"15min":1.1910000000000001},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":6.0869999999999997,"5min":15.512,"15min":15.512},"last":1.899},{"interface":"front","average":{"1min":3.3450000000000002,"5min":2.2810000000000001,"15min":1.169},"min":{"1min":0.39600000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":17.673999999999999,"5min":17.673999999999999,"15min":17.673999999999999},"last":1.863}]},{"osd":7,"last update":"Mon Aug 28 02:26:01 2023","interfaces":[{"interface":"back","average":{"1min":3.1930000000000001,"5min":1.8440000000000001,"15min":1.0369999999999999},"min":{"1min":0.39900000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":19.061,"5min":19.061,"15min":19.061},"last":0.73099999999999998},{"interface":"front","average":{"1min":2.3500000000000001,"5min":2.1349999999999998,"15min":1.137},"min":{"1min":0.41899999999999998,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":6.1100000000000003,"5min":15.476000000000001,"15min":15.476000000000001},"last":2.504}]}]},{"osd":0,"up_from":90,"seq":386547056735,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":931588,"kb_used_data":751420,"kb_used_omap":8,"kb_used_meta":180151,"kb_avail":92813564,"statfs":{"total":95995035648,"available":95041089536,"internally_reserved":0,"allocated":769454080,"data_stored":768617914,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":184474985},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,4,2,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Aug 28 02:26:09 2023","interfaces":[{"interface":"back","average":{"1min":2.516,"5min":2.754,"15min":1.153},"min":{"1min":0.184,"5min":0.184,"15min":0.184},"max":{"1min":11.282,"5min":40.313000000000002,"15min":40.313000000000002},"last":2.7839999999999998},{"interface":"front","average":{"1min":3.8370000000000002,"5min":2.5899999999999999,"15min":1.111},"min":{"1min":0.23899999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":19.555,"5min":36.655000000000001,"15min":36.655000000000001},"last":2.097}]},{"osd":2,"last update":"Mon Aug 28 02:25:27 2023","interfaces":[{"interface":"back","average":{"1min":4.4050000000000002,"5min":1.944,"15min":0.88300000000000001},"min":{"1min":0.40500000000000003,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":15.33,"5min":15.33,"15min":15.33},"last":0.34100000000000003},{"interface":"front","average":{"1min":6.351,"5min":2.484,"15min":1.089},"min":{"1min":0.40799999999999997,"5min":0.224,"15min":0.224},"max":{"1min":18.870999999999999,"5min":18.870999999999999,"15min":18.870999999999999},"last":2.0649999999999999}]},{"osd":3,"last update":"Mon Aug 28 02:26:09 2023","interfaces":[{"interface":"back","average":{"1min":3.77,"5min":2.2090000000000001,"15min":0.97799999999999998},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":12.272,"5min":18.545999999999999,"15min":18.545999999999999},"last":3.613},{"interface":"front","average":{"1min":3.3300000000000001,"5min":2.1309999999999998,"15min":0.97299999999999998},"min":{"1min":0.28999999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":12.474,"5min":14.534000000000001,"15min":14.534000000000001},"last":3.9089999999999998}]},{"osd":4,"last update":"Mon Aug 28 02:26:04 2023","interfaces":[{"interface":"back","average":{"1min":2.3700000000000001,"5min":1.8520000000000001,"15min":1.0600000000000001},"min":{"1min":0.29599999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":7.5979999999999999,"5min":9.6649999999999991,"15min":9.6649999999999991},"last":0.379},{"interface":"front","average":{"1min":2.7679999999999998,"5min":2.0299999999999998,"15min":1.1419999999999999},"min":{"1min":0.45700000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":12.305,"5min":12.305,"15min":12.305},"last":3.6389999999999998}]},{"osd":5,"last update":"Mon Aug 28 02:26:04 2023","interfaces":[{"interface":"back","average":{"1min":3.0529999999999999,"5min":2.508,"15min":1.268},"min":{"1min":0.41299999999999998,"5min":0.313,"15min":0.313},"max":{"1min":12.337999999999999,"5min":44.000999999999998,"15min":44.000999999999998},"last":3.6970000000000001},{"interface":"front","average":{"1min":3.177,"5min":2.423,"15min":1.238},"min":{"1min":0.438,"5min":0.33100000000000002,"15min":0.30599999999999999},"max":{"1min":12.334,"5min":43.953000000000003,"15min":43.953000000000003},"last":0.40400000000000003}]},{"osd":6,"last update":"Mon Aug 28 02:26:04 2023","interfaces":[{"interface":"back","average":{"1min":3.1970000000000001,"5min":2.5409999999999999,"15min":1.2789999999999999},"min":{"1min":0.42799999999999999,"5min":0.36299999999999999,"15min":0.32000000000000001},"max":{"1min":12.361000000000001,"5min":32.417999999999999,"15min":32.417999999999999},"last":3.669},{"interface":"front","average":{"1min":3.0499999999999998,"5min":2.286,"15min":1.23},"min":{"1min":0.378,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":8.3829999999999991,"5min":32.130000000000003,"15min":32.130000000000003},"last":4.9880000000000004}]},{"osd":7,"last update":"Mon Aug 28 02:26:04 2023","interfaces":[{"interface":"back","average":{"1min":3.2690000000000001,"5min":2.1019999999999999,"15min":1.1619999999999999},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40000000000000002},"max":{"1min":8.9870000000000001,"5min":9.6020000000000003,"15min":9.6020000000000003},"last":3.254},{"interface":"front","average":{"1min":3.649,"5min":2.3570000000000002,"15min":1.25},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":19.478999999999999,"5min":19.478999999999999,"15min":19.478999999999999},"last":7.4699999999999998}]}]},{"osd":2,"up_from":104,"seq":446676598859,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":328544,"kb_used_data":224664,"kb_used_omap":10,"kb_used_meta":103797,"kb_avail":93416608,"statfs":{"total":95995035648,"available":95658606592,"internally_reserved":0,"allocated":230055936,"data_stored":229251977,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10792,"internal_metadata":106288600},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:25:33 2023","interfaces":[{"interface":"back","average":{"1min":3.6179999999999999,"5min":2.2519999999999998,"15min":0.96399999999999997},"min":{"1min":0.188,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":18.146000000000001,"5min":36.890000000000001,"15min":36.890000000000001},"last":51.920000000000002},{"interface":"front","average":{"1min":4.625,"5min":2.4590000000000001,"15min":1.0620000000000001},"min":{"1min":0.214,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":25.414999999999999,"5min":35.289000000000001,"15min":35.289000000000001},"last":51.682000000000002}]},{"osd":1,"last update":"Mon Aug 28 02:25:33 2023","interfaces":[{"interface":"back","average":{"1min":4.4900000000000002,"5min":2.4260000000000002,"15min":1.048},"min":{"1min":0.23999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":16.372,"5min":30.888000000000002,"15min":30.888000000000002},"last":0.23599999999999999},{"interface":"front","average":{"1min":3.8959999999999999,"5min":1.9299999999999999,"15min":0.86599999999999999},"min":{"1min":0.21299999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":14.855,"5min":31.648,"15min":31.648},"last":0.25800000000000001}]},{"osd":3,"last update":"Mon Aug 28 02:26:02 2023","interfaces":[{"interface":"back","average":{"1min":10.356,"5min":3.331,"15min":1.3819999999999999},"min":{"1min":0.309,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":128.63900000000001,"5min":128.63900000000001,"15min":128.63900000000001},"last":0.27400000000000002},{"interface":"front","average":{"1min":10.528,"5min":3.6179999999999999,"15min":1.464},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":128.75700000000001,"5min":128.75700000000001,"15min":128.75700000000001},"last":0.29499999999999998}]},{"osd":4,"last update":"Mon Aug 28 02:25:33 2023","interfaces":[{"interface":"back","average":{"1min":3.5750000000000002,"5min":1.772,"15min":0.98599999999999999},"min":{"1min":0.34399999999999997,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":14.815,"5min":14.815,"15min":14.815},"last":0.40799999999999997},{"interface":"front","average":{"1min":1.71,"5min":1.3300000000000001,"15min":0.876},"min":{"1min":0.33700000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.806,"5min":6.806,"15min":6.806},"last":0.33800000000000002}]},{"osd":5,"last update":"Mon Aug 28 02:25:33 2023","interfaces":[{"interface":"back","average":{"1min":1.9099999999999999,"5min":1.518,"15min":0.91700000000000004},"min":{"1min":0.36399999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":6.6900000000000004,"5min":9.1549999999999994,"15min":9.1549999999999994},"last":0.437},{"interface":"front","average":{"1min":4.0650000000000004,"5min":2.0459999999999998,"15min":1.054},"min":{"1min":0.438,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":14.877000000000001,"5min":14.877000000000001,"15min":14.877000000000001},"last":0.32100000000000001}]},{"osd":6,"last update":"Mon Aug 28 02:25:33 2023","interfaces":[{"interface":"back","average":{"1min":2.2280000000000002,"5min":1.659,"15min":0.95299999999999996},"min":{"1min":0.35699999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":8.7010000000000005,"5min":8.7010000000000005,"15min":8.7010000000000005},"last":0.307},{"interface":"front","average":{"1min":2.1190000000000002,"5min":1.7350000000000001,"15min":0.98899999999999999},"min":{"1min":0.42999999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":8.7729999999999997,"5min":10.507,"15min":10.507},"last":0.377}]},{"osd":7,"last update":"Mon Aug 28 02:25:33 2023","interfaces":[{"interface":"back","average":{"1min":2.0720000000000001,"5min":1.6100000000000001,"15min":0.93600000000000005},"min":{"1min":0.39800000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":8.7699999999999996,"5min":9.0909999999999993,"15min":9.0909999999999993},"last":0.42999999999999999},{"interface":"front","average":{"1min":2.278,"5min":1.673,"15min":0.96099999999999997},"min":{"1min":0.40400000000000003,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":7.1120000000000001,"5min":7.2569999999999997,"15min":7.2569999999999997},"last":0.34799999999999998}]}]},{"osd":3,"up_from":112,"seq":481036337221,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":80052,"kb_used_data":12460,"kb_used_omap":3,"kb_used_meta":67580,"kb_avail":93665100,"statfs":{"total":95995035648,"available":95913062400,"internally_reserved":0,"allocated":12759040,"data_stored":11990144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":69202850},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:26:00 2023","interfaces":[{"interface":"back","average":{"1min":1.96,"5min":5.8419999999999996,"15min":2.1800000000000002},"min":{"1min":0.25600000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":8.1150000000000002,"5min":164.25299999999999,"15min":164.25299999999999},"last":5.0389999999999997},{"interface":"front","average":{"1min":2.085,"5min":6.0650000000000004,"15min":2.2429999999999999},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":9.0530000000000008,"5min":164.55000000000001,"15min":164.55000000000001},"last":11.718}]},{"osd":1,"last update":"Mon Aug 28 02:26:00 2023","interfaces":[{"interface":"back","average":{"1min":3.25,"5min":3.9390000000000001,"15min":1.5389999999999999},"min":{"1min":0.251,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":13.936,"5min":117.34699999999999,"15min":117.34699999999999},"last":3.1640000000000001},{"interface":"front","average":{"1min":1.8089999999999999,"5min":3.6619999999999999,"15min":1.474},"min":{"1min":0.28100000000000003,"5min":0.224,"15min":0.224},"max":{"1min":7.2679999999999998,"5min":118.012,"15min":118.012},"last":5.0679999999999996}]},{"osd":2,"last update":"Mon Aug 28 02:26:00 2023","interfaces":[{"interface":"back","average":{"1min":3.6829999999999998,"5min":5.5730000000000004,"15min":2.1070000000000002},"min":{"1min":0.314,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":18.718,"5min":120.43300000000001,"15min":120.43300000000001},"last":0.48499999999999999},{"interface":"front","average":{"1min":3.2949999999999999,"5min":5.5949999999999998,"15min":2.1160000000000001},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":18.774000000000001,"5min":120.44499999999999,"15min":120.44499999999999},"last":0.45100000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:26:00 2023","interfaces":[{"interface":"back","average":{"1min":2.7559999999999998,"5min":1.681,"15min":0.97899999999999998},"min":{"1min":0.35199999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":9.016,"5min":9.016,"15min":9.016},"last":6.4550000000000001},{"interface":"front","average":{"1min":3.0409999999999999,"5min":1.669,"15min":0.97999999999999998},"min":{"1min":0.37,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":10.428000000000001,"5min":10.428000000000001,"15min":10.428000000000001},"last":11.768000000000001}]},{"osd":5,"last update":"Mon Aug 28 02:26:00 2023","interfaces":[{"interface":"back","average":{"1min":2.4460000000000002,"5min":1.488,"15min":0.92000000000000004},"min":{"1min":0.41399999999999998,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":8.9789999999999992,"5min":8.9789999999999992,"15min":8.9789999999999992},"last":6.556},{"interface":"front","average":{"1min":1.772,"5min":1.333,"15min":0.86399999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":13.331,"5min":13.331,"15min":13.331},"last":0.51800000000000002}]},{"osd":6,"last update":"Mon Aug 28 02:26:00 2023","interfaces":[{"interface":"back","average":{"1min":2.427,"5min":2.1440000000000001,"15min":1.1379999999999999},"min":{"1min":0.36799999999999999,"5min":0.314,"15min":0.314},"max":{"1min":9.4529999999999994,"5min":72.116,"15min":72.116},"last":0.54100000000000004},{"interface":"front","average":{"1min":2.0459999999999998,"5min":2.2210000000000001,"15min":1.139},"min":{"1min":0.38,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":6.7110000000000003,"5min":72.030000000000001,"15min":72.030000000000001},"last":0.38800000000000001}]},{"osd":7,"last update":"Mon Aug 28 02:26:00 2023","interfaces":[{"interface":"back","average":{"1min":2.9329999999999998,"5min":1.7250000000000001,"15min":1.0129999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":11.949999999999999,"5min":11.949999999999999,"15min":11.949999999999999},"last":0.42799999999999999},{"interface":"front","average":{"1min":3.702,"5min":1.907,"15min":1.0660000000000001},"min":{"1min":0.438,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":13.890000000000001,"5min":13.890000000000001,"15min":13.890000000000001},"last":7.2409999999999997}]}]},{"osd":4,"up_from":30,"seq":128849019095,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":522256,"kb_used_data":216840,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93222896,"statfs":{"total":95995035648,"available":95460245504,"internally_reserved":0,"allocated":222044160,"data_stored":221476613,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:25:59 2023","interfaces":[{"interface":"back","average":{"1min":6.2859999999999996,"5min":5.8010000000000002,"15min":2.3399999999999999},"min":{"1min":0.26500000000000001,"5min":0.223,"15min":0.223},"max":{"1min":43.548000000000002,"5min":138.67400000000001,"15min":138.67400000000001},"last":6.0350000000000001},{"interface":"front","average":{"1min":5.8789999999999996,"5min":5.6669999999999998,"15min":2.2839999999999998},"min":{"1min":0.25,"5min":0.248,"15min":0.248},"max":{"1min":43.515000000000001,"5min":134.14599999999999,"15min":134.14599999999999},"last":1.361}]},{"osd":1,"last update":"Mon Aug 28 02:25:59 2023","interfaces":[{"interface":"back","average":{"1min":2.6509999999999998,"5min":1.915,"15min":0.98699999999999999},"min":{"1min":0.32700000000000001,"5min":0.255,"15min":0.255},"max":{"1min":8.9570000000000007,"5min":53.622999999999998,"15min":53.622999999999998},"last":1.486},{"interface":"front","average":{"1min":2.1339999999999999,"5min":1.8999999999999999,"15min":1.01},"min":{"1min":0.28000000000000003,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":7.1580000000000004,"5min":53.774000000000001,"15min":53.774000000000001},"last":1.423}]},{"osd":2,"last update":"Mon Aug 28 02:25:28 2023","interfaces":[{"interface":"back","average":{"1min":5.0780000000000003,"5min":3.6219999999999999,"15min":1.5740000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":56.295999999999999,"5min":74.099000000000004,"15min":74.099000000000004},"last":6.5},{"interface":"front","average":{"1min":4.7229999999999999,"5min":3.4820000000000002,"15min":1.52},"min":{"1min":0.28599999999999998,"5min":0.25,"15min":0.25},"max":{"1min":56.314,"5min":74.081999999999994,"15min":74.081999999999994},"last":5.9180000000000001}]},{"osd":3,"last update":"Mon Aug 28 02:25:59 2023","interfaces":[{"interface":"back","average":{"1min":3.3359999999999999,"5min":1.5149999999999999,"15min":0.83899999999999997},"min":{"1min":0.32400000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":8.6349999999999998,"5min":8.6349999999999998,"15min":8.6349999999999998},"last":2.6560000000000001},{"interface":"front","average":{"1min":2.9430000000000001,"5min":1.6870000000000001,"15min":0.95099999999999996},"min":{"1min":0.39900000000000002,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":10.956,"5min":10.956,"15min":10.956},"last":1.51}]},{"osd":5,"last update":"Mon Aug 28 02:25:41 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":0.97799999999999998,"15min":0.63700000000000001},"min":{"1min":0.26900000000000002,"5min":0.249,"15min":0.249},"max":{"1min":5.4080000000000004,"5min":8.2560000000000002,"15min":8.2560000000000002},"last":5.883},{"interface":"front","average":{"1min":2.3010000000000002,"5min":1.3300000000000001,"15min":0.76700000000000002},"min":{"1min":0.27100000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":8.0619999999999994,"5min":8.0619999999999994,"15min":8.0619999999999994},"last":2.6930000000000001}]},{"osd":6,"last update":"Mon Aug 28 02:25:59 2023","interfaces":[{"interface":"back","average":{"1min":2.1059999999999999,"5min":1.373,"15min":0.79600000000000004},"min":{"1min":0.32300000000000001,"5min":0.29199999999999998,"15min":0.27500000000000002},"max":{"1min":11.170999999999999,"5min":11.170999999999999,"15min":11.170999999999999},"last":1.4530000000000001},{"interface":"front","average":{"1min":1.403,"5min":0.97199999999999998,"15min":0.69199999999999995},"min":{"1min":0.33700000000000002,"5min":0.32200000000000001,"15min":0.318},"max":{"1min":7.3170000000000002,"5min":7.3170000000000002,"15min":7.3170000000000002},"last":1.389}]},{"osd":7,"last update":"Mon Aug 28 02:26:24 2023","interfaces":[{"interface":"back","average":{"1min":4.2800000000000002,"5min":1.8400000000000001,"15min":0.97599999999999998},"min":{"1min":0.41399999999999998,"5min":0.33900000000000002,"15min":0.312},"max":{"1min":16.584,"5min":16.584,"15min":16.584},"last":6.21},{"interface":"front","average":{"1min":2.3399999999999999,"5min":1.79,"15min":0.97199999999999998},"min":{"1min":0.371,"5min":0.32500000000000001,"15min":0.32100000000000001},"max":{"1min":7.6360000000000001,"5min":8.7569999999999997,"15min":8.7569999999999997},"last":4.7599999999999998}]}]},{"osd":5,"up_from":36,"seq":154618822866,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":713148,"kb_used_data":403444,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93032004,"statfs":{"total":95995035648,"available":95264772096,"internally_reserved":0,"allocated":413126656,"data_stored":412566721,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,6,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:25:52 2023","interfaces":[{"interface":"back","average":{"1min":3.5249999999999999,"5min":5.8579999999999997,"15min":2.3639999999999999},"min":{"1min":0.24099999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":12.874000000000001,"5min":131.06100000000001,"15min":131.06100000000001},"last":0.67800000000000005},{"interface":"front","average":{"1min":3.758,"5min":6.1749999999999998,"15min":2.4740000000000002},"min":{"1min":0.24199999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":15.691000000000001,"5min":131.262,"15min":131.262},"last":2.3130000000000002}]},{"osd":1,"last update":"Mon Aug 28 02:26:03 2023","interfaces":[{"interface":"back","average":{"1min":1.298,"5min":5.0019999999999998,"15min":2.0190000000000001},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":5.7240000000000002,"5min":183.97900000000001,"15min":183.97900000000001},"last":7.6909999999999998},{"interface":"front","average":{"1min":2.4929999999999999,"5min":5.4960000000000004,"15min":2.1920000000000002},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":8.5640000000000001,"5min":184.33699999999999,"15min":184.33699999999999},"last":3.0760000000000001}]},{"osd":2,"last update":"Mon Aug 28 02:25:30 2023","interfaces":[{"interface":"back","average":{"1min":3.5409999999999999,"5min":5.2169999999999996,"15min":2.1320000000000001},"min":{"1min":0.29999999999999999,"5min":0.252,"15min":0.252},"max":{"1min":12.584,"5min":110.051,"15min":110.051},"last":7.3920000000000003},{"interface":"front","average":{"1min":3.7709999999999999,"5min":5.4820000000000002,"15min":2.2189999999999999},"min":{"1min":0.29799999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":15.635,"5min":110.041,"15min":110.041},"last":5.2190000000000003}]},{"osd":3,"last update":"Mon Aug 28 02:26:03 2023","interfaces":[{"interface":"back","average":{"1min":2.004,"5min":1.649,"15min":0.95699999999999996},"min":{"1min":0.28899999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.0039999999999996,"5min":10.576000000000001,"15min":10.576000000000001},"last":7.3449999999999998},{"interface":"front","average":{"1min":3.5499999999999998,"5min":2.177,"15min":1.143},"min":{"1min":0.34300000000000003,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":12.118,"5min":12.118,"15min":12.118},"last":7.5780000000000003}]},{"osd":4,"last update":"Mon Aug 28 02:25:44 2023","interfaces":[{"interface":"back","average":{"1min":1.4239999999999999,"5min":1.355,"15min":0.76200000000000001},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":5.6890000000000001,"5min":10.759,"15min":10.759},"last":3.4540000000000002},{"interface":"front","average":{"1min":2.149,"5min":1.468,"15min":0.82199999999999995},"min":{"1min":0.31900000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.0170000000000003,"5min":8.6769999999999996,"15min":8.6769999999999996},"last":0.64500000000000002}]},{"osd":6,"last update":"Mon Aug 28 02:26:03 2023","interfaces":[{"interface":"back","average":{"1min":1.968,"5min":1.694,"15min":0.91900000000000004},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.28399999999999997},"max":{"1min":5.9870000000000001,"5min":14.803000000000001,"15min":14.803000000000001},"last":3.488},{"interface":"front","average":{"1min":1.6639999999999999,"5min":1.389,"15min":0.81699999999999995},"min":{"1min":0.33500000000000002,"5min":0.31,"15min":0.30299999999999999},"max":{"1min":8.4779999999999998,"5min":8.6479999999999997,"15min":8.6479999999999997},"last":0.88100000000000001}]},{"osd":7,"last update":"Mon Aug 28 02:26:19 2023","interfaces":[{"interface":"back","average":{"1min":2.8159999999999998,"5min":1.75,"15min":0.95499999999999996},"min":{"1min":0.33500000000000002,"5min":0.31,"15min":0.31},"max":{"1min":9.9949999999999992,"5min":9.9949999999999992,"15min":9.9949999999999992},"last":0.70599999999999996},{"interface":"front","average":{"1min":1.54,"5min":1.5229999999999999,"15min":0.874},"min":{"1min":0.37,"5min":0.32100000000000001,"15min":0.29799999999999999},"max":{"1min":4.9889999999999999,"5min":12.787000000000001,"15min":12.787000000000001},"last":2.274}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":759496704,"data_stored":759496704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":178913280,"data_stored":178913280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":216338432,"data_stored":216334360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":10551296,"data_stored":10551296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":219217920,"data_stored":219217920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":401805312,"data_stored":401801240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":111017984,"data_stored":111017984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":138940416,"data_stored":138936344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-08-28T02:26:29.482 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph status --format=json 2023-08-28T02:26:29.571 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:26:29.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:29.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:26:29.606 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:26:29.680 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:26:29.724 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:29.814 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:26:29.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:29.817 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:26:29.918 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:26:29.945 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:26:30.013 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:30.276 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:30 smithi143 ceph-mon[183059]: pgmap v423: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 392 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 1990/18678 objects degraded (10.654%); 925/18678 objects misplaced (4.952%); 2.8 MiB/s, 44 objects/s recovering 2023-08-28T02:26:30.277 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:30 smithi143 ceph-mon[189871]: pgmap v423: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 392 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 1990/18678 objects degraded (10.654%); 925/18678 objects misplaced (4.952%); 2.8 MiB/s, 44 objects/s recovering 2023-08-28T02:26:30.281 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:26:30.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:30.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:26:30.364 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:30 smithi169 ceph-mon[159415]: pgmap v423: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 392 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 1990/18678 objects degraded (10.654%); 925/18678 objects misplaced (4.952%); 2.8 MiB/s, 44 objects/s recovering 2023-08-28T02:26:30.399 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:26:30.465 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:30.710 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:26:30.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:30.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:26:30.814 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:26:30.885 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:31.112 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:26:31.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:31.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:26:31.125 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:31 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/434833987' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:26:31.136 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is set 2023-08-28T02:26:31.221 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:26:31.248 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:31.292 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:26:31.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:31.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:26:31.395 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:26:31.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:31 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/434833987' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:26:31.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:31 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/434833987' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:26:31.461 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:31.633 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:26:31.634 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:26:32.123 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:26:31 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:26:31] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.33.4" 2023-08-28T02:26:32.293 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:26:32.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:32.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:26:32.311 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:26:32.398 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:26:32.435 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:32 smithi169 ceph-mon[159415]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:26:32.435 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:32 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/434833987' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:26:32.435 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:32 smithi169 ceph-mon[159415]: osdmap e183: 8 total, 8 up, 8 in 2023-08-28T02:26:32.436 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:32 smithi169 ceph-mon[159415]: failed to encode map e183 with expected crc 2023-08-28T02:26:32.436 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:32 smithi169 ceph-mon[159415]: pgmap v425: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 392 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 534 op/s; 1990/18678 objects degraded (10.654%); 925/18678 objects misplaced (4.952%); 3.2 MiB/s, 50 objects/s recovering 2023-08-28T02:26:32.436 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:32 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 1990/18678 objects degraded (10.654%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:26:32.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:32 smithi143 ceph-mon[183059]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:26:32.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:32 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/434833987' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:26:32.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:32 smithi143 ceph-mon[183059]: osdmap e183: 8 total, 8 up, 8 in 2023-08-28T02:26:32.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:32 smithi143 ceph-mon[183059]: failed to encode map e183 with expected crc 2023-08-28T02:26:32.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:32 smithi143 ceph-mon[183059]: pgmap v425: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 392 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 534 op/s; 1990/18678 objects degraded (10.654%); 925/18678 objects misplaced (4.952%); 3.2 MiB/s, 50 objects/s recovering 2023-08-28T02:26:32.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:32 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 1990/18678 objects degraded (10.654%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:26:32.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:32 smithi143 ceph-mon[189871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:26:32.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:32 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/434833987' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:26:32.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:32 smithi143 ceph-mon[189871]: osdmap e183: 8 total, 8 up, 8 in 2023-08-28T02:26:32.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:32 smithi143 ceph-mon[189871]: failed to encode map e183 with expected crc 2023-08-28T02:26:32.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:32 smithi143 ceph-mon[189871]: pgmap v425: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 392 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 534 op/s; 1990/18678 objects degraded (10.654%); 925/18678 objects misplaced (4.952%); 3.2 MiB/s, 50 objects/s recovering 2023-08-28T02:26:32.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:32 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 1990/18678 objects degraded (10.654%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:26:32.454 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.7 dump_historic_ops 2023-08-28T02:26:32.483 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:32.589 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:26:32.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:32.595 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:26:32.698 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:26:32.750 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:32.932 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:26:32.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:32.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:26:33.038 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:26:33.115 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:26:33.126 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:33.322 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:26:33.360 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:26:33.366 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:26:33.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:33.373 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:26:33.391 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:26:33.432 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:26:33.463 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:26:33.474 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:26:33.518 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:26:33.548 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:26:33.553 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:33.583 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:26:33.599 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:26:33.599 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:26:33.599 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:26:33.600 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:26:33.600 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:26:33.600 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:26:33.600 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:26:33.600 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:26:33.959 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:26:33.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:33.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:26:34.063 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:26:34.135 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:34.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:34 smithi143 ceph-mon[183059]: pgmap v426: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 314 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 1990/14931 objects degraded (13.328%); 767/14931 objects misplaced (5.137%); 2.8 MiB/s, 44 objects/s recovering 2023-08-28T02:26:34.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:34 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2203830596' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:26:34.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:34 smithi143 ceph-mon[189871]: pgmap v426: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 314 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 1990/14931 objects degraded (13.328%); 767/14931 objects misplaced (5.137%); 2.8 MiB/s, 44 objects/s recovering 2023-08-28T02:26:34.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:34 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2203830596' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:26:34.570 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set nodeep-scrub 2023-08-28T02:26:34.664 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:34 smithi169 ceph-mon[159415]: pgmap v426: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 314 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 1990/14931 objects degraded (13.328%); 767/14931 objects misplaced (5.137%); 2.8 MiB/s, 44 objects/s recovering 2023-08-28T02:26:34.665 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:34 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2203830596' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:26:34.884 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:26:34.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:34.908 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:26:35.009 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:26:35.078 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:35.210 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:26:35.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:35.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:26:35.309 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:26:35.309 INFO:teuthology.orchestra.run.smithi169.stdout: "size": 20, 2023-08-28T02:26:35.309 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 600, 2023-08-28T02:26:35.309 INFO:teuthology.orchestra.run.smithi169.stdout: "ops": [ 2023-08-28T02:26:35.310 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.310 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:93756 5.4 5:24326eee:::benchmark_data_smithi143_236200_object93755:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-08-28T02:26:35.310 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:13.054964+0000", 2023-08-28T02:26:35.310 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 202.24805226000001, 2023-08-28T02:26:35.310 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.21242322399999999, 2023-08-28T02:26:35.310 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:35.310 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:35.310 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:26:35.310 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:26:35.310 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:35.310 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 93756 2023-08-28T02:26:35.310 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.310 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:35.311 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.311 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:35.311 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:13.054964+0000", 2023-08-28T02:26:35.311 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:35.311 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.311 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.311 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:35.311 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:13.054964+0000", 2023-08-28T02:26:35.311 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:26:35.311 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.311 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.311 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:35.312 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:13.054962+0000", 2023-08-28T02:26:35.312 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.7784999999999998e-05 2023-08-28T02:26:35.312 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.312 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.312 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:35.312 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:13.055000+0000", 2023-08-28T02:26:35.312 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.573e-05 2023-08-28T02:26:35.312 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.312 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.312 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:35.312 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:13.055016+0000", 2023-08-28T02:26:35.312 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.7009999999999996e-06 2023-08-28T02:26:35.312 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.312 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.313 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:35.313 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:13.055023+0000", 2023-08-28T02:26:35.313 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.7620000000000006e-05 2023-08-28T02:26:35.313 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.313 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.313 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:35.313 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:13.055091+0000", 2023-08-28T02:26:35.313 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00016659500000000001 2023-08-28T02:26:35.313 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.313 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.313 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:35.313 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:13.055257+0000", 2023-08-28T02:26:35.313 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00017993999999999999 2023-08-28T02:26:35.314 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.314 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.314 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 2,5", 2023-08-28T02:26:35.314 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:13.055437+0000", 2023-08-28T02:26:35.314 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0062637309999999998 2023-08-28T02:26:35.314 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.314 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.314 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:26:35.314 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:13.061701+0000", 2023-08-28T02:26:35.314 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.0442000000000001e-05 2023-08-28T02:26:35.314 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.314 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.314 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:35.315 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:13.061732+0000", 2023-08-28T02:26:35.315 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.20556266200000001 2023-08-28T02:26:35.315 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.315 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.315 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:35.315 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:13.267294+0000", 2023-08-28T02:26:35.315 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.9750999999999999e-05 2023-08-28T02:26:35.315 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.315 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.315 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:26:35.315 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:13.267344+0000", 2023-08-28T02:26:35.315 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.3266999999999998e-05 2023-08-28T02:26:35.315 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.316 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.316 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:26:35.316 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:13.267387+0000", 2023-08-28T02:26:35.316 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.21242322399999999 2023-08-28T02:26:35.316 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:35.316 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:35.316 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:35.316 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.316 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.316 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:93760 5.4 5:24b7559b:::benchmark_data_smithi143_236200_object93759:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e136)", 2023-08-28T02:26:35.316 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:13.055568+0000", 2023-08-28T02:26:35.316 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 202.247448287, 2023-08-28T02:26:35.316 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.21196705900000001, 2023-08-28T02:26:35.317 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:35.317 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:35.317 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:26:35.317 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:26:35.317 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:35.317 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 93760 2023-08-28T02:26:35.317 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.317 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:35.317 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.317 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:35.317 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:13.055568+0000", 2023-08-28T02:26:35.317 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:35.317 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.318 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.318 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:35.318 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:13.055568+0000", 2023-08-28T02:26:35.318 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:26:35.318 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.318 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.318 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:35.318 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:13.055567+0000", 2023-08-28T02:26:35.318 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.1569000000000001e-05 2023-08-28T02:26:35.318 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.318 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.318 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:35.318 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:13.055609+0000", 2023-08-28T02:26:35.319 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.8240000000000006e-06 2023-08-28T02:26:35.319 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.319 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.319 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:35.319 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:13.055617+0000", 2023-08-28T02:26:35.319 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.6809999999999997e-06 2023-08-28T02:26:35.319 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.319 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.319 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:35.319 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:13.055622+0000", 2023-08-28T02:26:35.319 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0030379830000000002 2023-08-28T02:26:35.319 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.319 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.320 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:35.320 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:13.058660+0000", 2023-08-28T02:26:35.320 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00014730400000000001 2023-08-28T02:26:35.320 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.320 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.320 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:35.320 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:13.058807+0000", 2023-08-28T02:26:35.320 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00016239 2023-08-28T02:26:35.320 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.320 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.320 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 2,5", 2023-08-28T02:26:35.320 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:13.058970+0000", 2023-08-28T02:26:35.320 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.002766241 2023-08-28T02:26:35.320 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.321 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.321 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:26:35.321 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:13.061736+0000", 2023-08-28T02:26:35.321 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0037560639999999999 2023-08-28T02:26:35.321 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.321 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.321 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:35.321 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:13.065492+0000", 2023-08-28T02:26:35.321 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.201934896 2023-08-28T02:26:35.321 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.321 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.321 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:35.321 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:13.267427+0000", 2023-08-28T02:26:35.322 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.8265e-05 2023-08-28T02:26:35.322 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.322 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.322 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:26:35.322 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:13.267475+0000", 2023-08-28T02:26:35.322 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.9765999999999997e-05 2023-08-28T02:26:35.322 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.322 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.322 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:26:35.322 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:13.267535+0000", 2023-08-28T02:26:35.322 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.21196705900000001 2023-08-28T02:26:35.322 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:35.322 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:35.323 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:35.323 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.323 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.323 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:115340 5.4 5:2bae87bd:::benchmark_data_smithi143_236200_object5197:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e150)", 2023-08-28T02:26:35.323 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:46.202765+0000", 2023-08-28T02:26:35.323 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 169.10025159400001, 2023-08-28T02:26:35.323 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.01239904, 2023-08-28T02:26:35.323 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:35.323 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "reached pg", 2023-08-28T02:26:35.323 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:26:35.323 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:26:35.323 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:35.323 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 115340 2023-08-28T02:26:35.324 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.324 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:35.324 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.324 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:35.324 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.202765+0000", 2023-08-28T02:26:35.324 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:35.324 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.324 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.324 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:35.324 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.202765+0000", 2023-08-28T02:26:35.324 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:26:35.324 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.325 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.325 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:35.325 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.202763+0000", 2023-08-28T02:26:35.325 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.0499999999999997e-06 2023-08-28T02:26:35.325 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.325 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.325 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:35.325 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.202769+0000", 2023-08-28T02:26:35.325 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.764e-06 2023-08-28T02:26:35.325 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.325 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.325 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:35.325 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.202779+0000", 2023-08-28T02:26:35.326 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.5060000000000001e-06 2023-08-28T02:26:35.326 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.326 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.326 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:35.326 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.202785+0000", 2023-08-28T02:26:35.326 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.001139756 2023-08-28T02:26:35.326 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.326 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.326 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:35.326 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.203925+0000", 2023-08-28T02:26:35.326 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.6399999999999992e-06 2023-08-28T02:26:35.326 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.326 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.327 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:26:35.327 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.203935+0000", 2023-08-28T02:26:35.327 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0111562949999999 2023-08-28T02:26:35.327 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.327 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.327 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:35.327 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.215091+0000", 2023-08-28T02:26:35.327 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.2588000000000004e-05 2023-08-28T02:26:35.327 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.327 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.327 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:26:35.327 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.215164+0000", 2023-08-28T02:26:35.328 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.01239904 2023-08-28T02:26:35.328 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:35.328 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:35.328 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:35.328 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.328 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.328 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:115356 5.4 5:2fe6ff18:::benchmark_data_smithi143_236200_object5213:head [delete] snapc 0=[] ondisk+write+known_if_redirected e150)", 2023-08-28T02:26:35.328 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:46.208800+0000", 2023-08-28T02:26:35.328 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 169.09421621000001, 2023-08-28T02:26:35.328 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0175739580000001, 2023-08-28T02:26:35.328 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:35.328 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:35.328 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:26:35.329 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:26:35.329 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:35.329 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 115356 2023-08-28T02:26:35.329 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.329 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:35.329 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.329 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:35.329 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.208800+0000", 2023-08-28T02:26:35.329 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:35.329 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.329 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.329 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:35.329 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.208800+0000", 2023-08-28T02:26:35.330 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:26:35.330 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.330 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.330 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:35.330 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.208799+0000", 2023-08-28T02:26:35.330 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.4209999999999998e-06 2023-08-28T02:26:35.330 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.330 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.330 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:35.330 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.208806+0000", 2023-08-28T02:26:35.330 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0020000000000001e-05 2023-08-28T02:26:35.330 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.330 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.331 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:35.331 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.208816+0000", 2023-08-28T02:26:35.331 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.3469999999999998e-06 2023-08-28T02:26:35.331 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.331 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.331 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:35.331 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.208822+0000", 2023-08-28T02:26:35.331 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.7844999999999999e-05 2023-08-28T02:26:35.331 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.331 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.331 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:35.331 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.208879+0000", 2023-08-28T02:26:35.331 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.435e-06 2023-08-28T02:26:35.332 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.332 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.332 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:26:35.332 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.208887+0000", 2023-08-28T02:26:35.332 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.006549806 2023-08-28T02:26:35.332 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.332 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.332 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:35.332 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.215437+0000", 2023-08-28T02:26:35.332 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000105202 2023-08-28T02:26:35.332 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.332 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.332 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:35.333 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.215542+0000", 2023-08-28T02:26:35.333 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000114942 2023-08-28T02:26:35.333 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.333 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.333 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 2,5", 2023-08-28T02:26:35.333 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.215657+0000", 2023-08-28T02:26:35.333 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0061267980000000001 2023-08-28T02:26:35.333 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.333 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.333 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:35.333 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.221784+0000", 2023-08-28T02:26:35.333 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.7664000000000001e-05 2023-08-28T02:26:35.333 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.334 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.334 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:35.334 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.221811+0000", 2023-08-28T02:26:35.334 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0044988939999999998 2023-08-28T02:26:35.334 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.334 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.334 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:26:35.334 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.226310+0000", 2023-08-28T02:26:35.334 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.4276999999999997e-05 2023-08-28T02:26:35.334 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.334 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.334 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:26:35.334 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.226344+0000", 2023-08-28T02:26:35.335 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.9654000000000001e-05 2023-08-28T02:26:35.335 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.335 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.335 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:26:35.335 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.226374+0000", 2023-08-28T02:26:35.335 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0175739580000001 2023-08-28T02:26:35.335 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:35.335 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:35.335 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:35.335 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.335 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.335 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:115359 5.4 5:21dba740:::benchmark_data_smithi143_236200_object5216:head [delete] snapc 0=[] ondisk+write+known_if_redirected e150)", 2023-08-28T02:26:35.336 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:46.213222+0000", 2023-08-28T02:26:35.336 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 169.08979402700001, 2023-08-28T02:26:35.336 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.013340922, 2023-08-28T02:26:35.336 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:35.336 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:35.336 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:26:35.336 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:26:35.336 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:35.336 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 115359 2023-08-28T02:26:35.336 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.336 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:35.336 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.336 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:35.337 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.213222+0000", 2023-08-28T02:26:35.337 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:35.337 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.337 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.337 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:35.337 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.213222+0000", 2023-08-28T02:26:35.337 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999976 2023-08-28T02:26:35.337 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.337 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.337 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:35.338 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.213220+0000", 2023-08-28T02:26:35.338 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0625000000000001e-05 2023-08-28T02:26:35.338 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.338 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.338 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:35.338 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.213230+0000", 2023-08-28T02:26:35.338 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.6798999999999999e-05 2023-08-28T02:26:35.338 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.338 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.338 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:35.338 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.213257+0000", 2023-08-28T02:26:35.338 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0838e-05 2023-08-28T02:26:35.338 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.339 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.339 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:35.339 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.213268+0000", 2023-08-28T02:26:35.339 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000108974 2023-08-28T02:26:35.339 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.339 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.339 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:35.339 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.213377+0000", 2023-08-28T02:26:35.339 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.3584000000000001e-05 2023-08-28T02:26:35.339 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.339 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.339 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:26:35.340 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.213391+0000", 2023-08-28T02:26:35.340 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0057094150000001 2023-08-28T02:26:35.340 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.340 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.340 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:35.340 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.219100+0000", 2023-08-28T02:26:35.340 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000100673 2023-08-28T02:26:35.340 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.340 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.340 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:35.340 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.219201+0000", 2023-08-28T02:26:35.340 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.6006000000000002e-05 2023-08-28T02:26:35.340 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.341 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.341 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 2,5", 2023-08-28T02:26:35.341 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.219297+0000", 2023-08-28T02:26:35.341 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0070880350000000003 2023-08-28T02:26:35.341 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.341 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.341 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:26:35.341 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.226385+0000", 2023-08-28T02:26:35.341 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.0395000000000006e-05 2023-08-28T02:26:35.341 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.341 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.341 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:35.341 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.226455+0000", 2023-08-28T02:26:35.342 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.7385999999999997e-05 2023-08-28T02:26:35.342 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.342 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.342 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:35.342 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.226492+0000", 2023-08-28T02:26:35.342 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.6803e-05 2023-08-28T02:26:35.342 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.342 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.342 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:26:35.342 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.226529+0000", 2023-08-28T02:26:35.342 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.3908000000000001e-05 2023-08-28T02:26:35.342 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.342 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.343 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:26:35.343 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.226563+0000", 2023-08-28T02:26:35.343 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.013340922 2023-08-28T02:26:35.343 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:35.343 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:35.343 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:35.343 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.343 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.343 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:115388 5.4 5:21930be8:::benchmark_data_smithi143_236200_object5245:head [delete] snapc 0=[] ondisk+write+known_if_redirected e150)", 2023-08-28T02:26:35.343 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:46.267659+0000", 2023-08-28T02:26:35.343 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 169.03535701600001, 2023-08-28T02:26:35.343 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.96153673900000003, 2023-08-28T02:26:35.344 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:35.344 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:35.344 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:26:35.344 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:26:35.344 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:35.344 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 115388 2023-08-28T02:26:35.344 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.344 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:35.344 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.344 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:35.344 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.267659+0000", 2023-08-28T02:26:35.344 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:35.345 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.345 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.345 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:35.345 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.267659+0000", 2023-08-28T02:26:35.345 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:26:35.345 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.345 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.345 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:35.345 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.267657+0000", 2023-08-28T02:26:35.345 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0597e-05 2023-08-28T02:26:35.345 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.345 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.345 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:35.346 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.267668+0000", 2023-08-28T02:26:35.346 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.8224e-05 2023-08-28T02:26:35.346 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.346 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.346 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:35.346 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.267686+0000", 2023-08-28T02:26:35.346 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.2099999999999993e-06 2023-08-28T02:26:35.346 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.346 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.346 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:35.346 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.267694+0000", 2023-08-28T02:26:35.346 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.0723999999999996e-05 2023-08-28T02:26:35.346 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.347 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.347 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:35.347 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.267765+0000", 2023-08-28T02:26:35.347 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.221e-05 2023-08-28T02:26:35.347 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.347 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.347 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:26:35.347 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.267777+0000", 2023-08-28T02:26:35.347 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.95414679499999999 2023-08-28T02:26:35.347 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.347 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.347 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:35.347 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.221924+0000", 2023-08-28T02:26:35.348 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.5344000000000001e-05 2023-08-28T02:26:35.348 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.348 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.348 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:35.348 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.222009+0000", 2023-08-28T02:26:35.348 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.5743999999999998e-05 2023-08-28T02:26:35.348 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.348 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.348 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 2,5", 2023-08-28T02:26:35.348 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.222095+0000", 2023-08-28T02:26:35.348 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.006875651 2023-08-28T02:26:35.348 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.348 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.349 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:26:35.349 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.228971+0000", 2023-08-28T02:26:35.349 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.1415999999999998e-05 2023-08-28T02:26:35.349 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.349 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.349 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:35.349 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.229012+0000", 2023-08-28T02:26:35.349 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.5841000000000006e-05 2023-08-28T02:26:35.349 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.349 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.349 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:35.349 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.229108+0000", 2023-08-28T02:26:35.349 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.6836000000000003e-05 2023-08-28T02:26:35.350 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.350 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.350 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:26:35.350 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.229165+0000", 2023-08-28T02:26:35.350 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.1260999999999998e-05 2023-08-28T02:26:35.350 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.350 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.350 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:26:35.350 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.229196+0000", 2023-08-28T02:26:35.350 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.96153673900000003 2023-08-28T02:26:35.350 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:35.350 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:35.350 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:35.350 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.351 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.351 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:115391 5.4 5:2114e7b4:::benchmark_data_smithi143_236200_object5248:head [delete] snapc 0=[] ondisk+write+known_if_redirected e150)", 2023-08-28T02:26:35.351 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:46.270943+0000", 2023-08-28T02:26:35.351 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 169.03207343299999, 2023-08-28T02:26:35.351 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.95836357500000002, 2023-08-28T02:26:35.351 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:35.351 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:35.351 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:26:35.351 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:26:35.351 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:35.351 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 115391 2023-08-28T02:26:35.351 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.351 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:35.352 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.352 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:35.352 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.270943+0000", 2023-08-28T02:26:35.352 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:35.352 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.352 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.352 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:35.352 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.270943+0000", 2023-08-28T02:26:35.352 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:26:35.352 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.352 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.352 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:35.352 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.270941+0000", 2023-08-28T02:26:35.353 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.4160000000000004e-06 2023-08-28T02:26:35.353 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.353 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.353 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:35.353 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.270950+0000", 2023-08-28T02:26:35.353 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.3647e-05 2023-08-28T02:26:35.353 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.353 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.353 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:35.353 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.270963+0000", 2023-08-28T02:26:35.353 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.7989999999999996e-06 2023-08-28T02:26:35.353 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.354 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.354 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:35.354 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.270970+0000", 2023-08-28T02:26:35.354 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.1917999999999999e-05 2023-08-28T02:26:35.354 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.354 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.354 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:35.354 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.271032+0000", 2023-08-28T02:26:35.354 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1977000000000001e-05 2023-08-28T02:26:35.354 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.354 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.354 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:26:35.354 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.271044+0000", 2023-08-28T02:26:35.355 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.953200147 2023-08-28T02:26:35.355 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.355 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.355 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:35.355 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.224244+0000", 2023-08-28T02:26:35.355 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.6223000000000005e-05 2023-08-28T02:26:35.355 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.355 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.355 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:35.355 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.224320+0000", 2023-08-28T02:26:35.355 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.2144999999999994e-05 2023-08-28T02:26:35.355 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.356 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.356 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 2,5", 2023-08-28T02:26:35.356 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.224402+0000", 2023-08-28T02:26:35.356 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.004653318 2023-08-28T02:26:35.356 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.356 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.356 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:35.356 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.229056+0000", 2023-08-28T02:26:35.356 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0029e-05 2023-08-28T02:26:35.356 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.356 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.356 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:26:35.356 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.229066+0000", 2023-08-28T02:26:35.357 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000180416 2023-08-28T02:26:35.357 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.357 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.357 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:35.357 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.229246+0000", 2023-08-28T02:26:35.357 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.3538999999999999e-05 2023-08-28T02:26:35.357 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.357 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.357 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:26:35.357 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.229280+0000", 2023-08-28T02:26:35.357 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.6682999999999998e-05 2023-08-28T02:26:35.357 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.357 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.358 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:26:35.358 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.229306+0000", 2023-08-28T02:26:35.358 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.95836357500000002 2023-08-28T02:26:35.358 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:35.358 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:35.358 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:35.358 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.358 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.358 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:115403 5.4 5:2cb6de89:::benchmark_data_smithi143_236200_object5260:head [delete] snapc 0=[] ondisk+write+known_if_redirected e150)", 2023-08-28T02:26:35.358 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:46.282244+0000", 2023-08-28T02:26:35.358 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 169.02077180699999, 2023-08-28T02:26:35.358 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.95339742900000002, 2023-08-28T02:26:35.358 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:35.359 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:35.359 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:26:35.359 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:26:35.359 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:35.359 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 115403 2023-08-28T02:26:35.359 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.359 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:35.359 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.359 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:35.359 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.282244+0000", 2023-08-28T02:26:35.359 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:35.359 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.359 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.360 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:35.360 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.282244+0000", 2023-08-28T02:26:35.360 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:26:35.360 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.360 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.360 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:35.360 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.282243+0000", 2023-08-28T02:26:35.360 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.9199999999999993e-06 2023-08-28T02:26:35.360 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.360 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.360 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:35.360 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.282251+0000", 2023-08-28T02:26:35.360 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.7672999999999998e-05 2023-08-28T02:26:35.361 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.361 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.361 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:35.361 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.282269+0000", 2023-08-28T02:26:35.361 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.4270000000000008e-06 2023-08-28T02:26:35.361 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.361 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.361 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:35.361 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.282278+0000", 2023-08-28T02:26:35.361 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.0259999999999995e-05 2023-08-28T02:26:35.361 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.361 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.361 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:35.362 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.282348+0000", 2023-08-28T02:26:35.362 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.4467000000000001e-05 2023-08-28T02:26:35.362 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.362 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.362 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:26:35.362 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.282362+0000", 2023-08-28T02:26:35.362 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.94424789099999995 2023-08-28T02:26:35.362 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.362 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.362 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:35.362 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.226610+0000", 2023-08-28T02:26:35.362 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000113853 2023-08-28T02:26:35.362 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.363 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.363 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:35.363 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.226724+0000", 2023-08-28T02:26:35.363 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00013130300000000001 2023-08-28T02:26:35.363 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.363 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.363 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 2,5", 2023-08-28T02:26:35.363 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.226855+0000", 2023-08-28T02:26:35.363 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0025102879999999998 2023-08-28T02:26:35.363 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.363 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.363 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:35.363 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.229366+0000", 2023-08-28T02:26:35.364 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0539e-05 2023-08-28T02:26:35.364 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.364 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.364 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:26:35.364 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.229376+0000", 2023-08-28T02:26:35.364 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.006158864 2023-08-28T02:26:35.364 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.364 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.364 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:35.364 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.235535+0000", 2023-08-28T02:26:35.364 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.5773000000000003e-05 2023-08-28T02:26:35.364 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.364 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.364 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:26:35.365 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.235591+0000", 2023-08-28T02:26:35.365 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.1094999999999998e-05 2023-08-28T02:26:35.365 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.365 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.365 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:26:35.365 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.235642+0000", 2023-08-28T02:26:35.365 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.95339742900000002 2023-08-28T02:26:35.365 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:35.365 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:35.365 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:35.365 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.365 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.365 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:115415 5.4 5:22394c7b:::benchmark_data_smithi143_236200_object5272:head [delete] snapc 0=[] ondisk+write+known_if_redirected e150)", 2023-08-28T02:26:35.366 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:46.299189+0000", 2023-08-28T02:26:35.366 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 169.00382761500001, 2023-08-28T02:26:35.366 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.94624566499999996, 2023-08-28T02:26:35.366 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:35.366 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:35.366 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:26:35.366 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:26:35.366 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:35.366 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 115415 2023-08-28T02:26:35.366 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.366 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:35.366 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.366 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:35.367 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.299189+0000", 2023-08-28T02:26:35.367 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:35.367 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.367 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.367 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:35.367 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.299189+0000", 2023-08-28T02:26:35.367 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:26:35.367 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.367 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.367 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:35.367 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.299187+0000", 2023-08-28T02:26:35.367 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.2289999999999992e-06 2023-08-28T02:26:35.367 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.367 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.368 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:35.368 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.299196+0000", 2023-08-28T02:26:35.368 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.8862999999999999e-05 2023-08-28T02:26:35.368 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.368 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.368 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:35.368 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.299215+0000", 2023-08-28T02:26:35.368 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.4749999999999998e-06 2023-08-28T02:26:35.368 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.368 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.368 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:35.368 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.299222+0000", 2023-08-28T02:26:35.368 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.0647999999999999e-05 2023-08-28T02:26:35.369 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.369 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.369 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:35.369 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.299282+0000", 2023-08-28T02:26:35.369 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.4742999999999999e-05 2023-08-28T02:26:35.369 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.369 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.369 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:26:35.369 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.299297+0000", 2023-08-28T02:26:35.369 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.93010203300000005 2023-08-28T02:26:35.369 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.369 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.369 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:35.370 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.229399+0000", 2023-08-28T02:26:35.370 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.4791999999999998e-05 2023-08-28T02:26:35.370 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.370 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.370 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:35.370 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.229474+0000", 2023-08-28T02:26:35.370 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.3294000000000006e-05 2023-08-28T02:26:35.370 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.370 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.370 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 2,5", 2023-08-28T02:26:35.370 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.229557+0000", 2023-08-28T02:26:35.370 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0059212709999999997 2023-08-28T02:26:35.370 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.370 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.371 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:35.371 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.235478+0000", 2023-08-28T02:26:35.371 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00017909800000000001 2023-08-28T02:26:35.371 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.371 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.371 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:26:35.371 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.235657+0000", 2023-08-28T02:26:35.371 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0096717260000000003 2023-08-28T02:26:35.371 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.371 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.371 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:35.371 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.245329+0000", 2023-08-28T02:26:35.371 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.8638000000000003e-05 2023-08-28T02:26:35.372 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.372 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.372 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:26:35.372 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.245388+0000", 2023-08-28T02:26:35.372 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.6529000000000001e-05 2023-08-28T02:26:35.372 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.372 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.372 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:26:35.372 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.245434+0000", 2023-08-28T02:26:35.372 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.94624566499999996 2023-08-28T02:26:35.372 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:35.372 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:35.372 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:35.373 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.373 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.373 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:115430 5.4 5:20807434:::benchmark_data_smithi143_236200_object5287:head [delete] snapc 0=[] ondisk+write+known_if_redirected e150)", 2023-08-28T02:26:35.373 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:46.438144+0000", 2023-08-28T02:26:35.373 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 168.86487215299999, 2023-08-28T02:26:35.373 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.80751542300000001, 2023-08-28T02:26:35.373 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:35.373 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:35.373 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:26:35.373 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:26:35.373 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:35.373 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 115430 2023-08-28T02:26:35.373 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.373 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:35.374 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.374 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:35.374 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.438144+0000", 2023-08-28T02:26:35.374 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:35.374 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.374 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.374 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:35.374 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.438144+0000", 2023-08-28T02:26:35.374 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:26:35.374 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.374 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.374 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:35.374 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.438142+0000", 2023-08-28T02:26:35.375 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.507e-06 2023-08-28T02:26:35.375 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.375 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.375 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:35.375 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.438151+0000", 2023-08-28T02:26:35.375 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.4121000000000001e-05 2023-08-28T02:26:35.375 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.375 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.375 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:35.375 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.438185+0000", 2023-08-28T02:26:35.375 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.388e-06 2023-08-28T02:26:35.375 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.375 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.376 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:35.376 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.438193+0000", 2023-08-28T02:26:35.376 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.3328000000000002e-05 2023-08-28T02:26:35.376 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.376 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.376 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:35.376 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.438246+0000", 2023-08-28T02:26:35.376 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2381999999999999e-05 2023-08-28T02:26:35.376 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.376 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.376 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:26:35.376 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.438259+0000", 2023-08-28T02:26:35.376 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.79436576000000003 2023-08-28T02:26:35.377 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.377 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.377 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:35.377 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.232625+0000", 2023-08-28T02:26:35.377 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000106474 2023-08-28T02:26:35.377 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.377 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.377 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:35.377 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.232731+0000", 2023-08-28T02:26:35.377 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.1114999999999998e-05 2023-08-28T02:26:35.377 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.377 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.377 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 2,5", 2023-08-28T02:26:35.378 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.232812+0000", 2023-08-28T02:26:35.378 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.012393375 2023-08-28T02:26:35.378 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.378 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.378 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:26:35.378 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.245206+0000", 2023-08-28T02:26:35.378 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.6043000000000004e-05 2023-08-28T02:26:35.378 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.378 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.378 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:35.378 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.245272+0000", 2023-08-28T02:26:35.378 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00029820200000000002 2023-08-28T02:26:35.378 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.379 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.379 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:35.379 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.245570+0000", 2023-08-28T02:26:35.379 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.2108999999999999e-05 2023-08-28T02:26:35.379 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.379 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.379 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:26:35.379 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.245622+0000", 2023-08-28T02:26:35.379 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.7657000000000002e-05 2023-08-28T02:26:35.379 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.379 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.379 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:26:35.379 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.245660+0000", 2023-08-28T02:26:35.379 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.80751542300000001 2023-08-28T02:26:35.380 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:35.380 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:35.380 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:35.380 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.380 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.380 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:115433 5.4 5:22f6ceb8:::benchmark_data_smithi143_236200_object5290:head [delete] snapc 0=[] ondisk+write+known_if_redirected e150)", 2023-08-28T02:26:35.380 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:46.441952+0000", 2023-08-28T02:26:35.380 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 168.861064349, 2023-08-28T02:26:35.380 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.81153499200000001, 2023-08-28T02:26:35.380 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:35.380 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:35.380 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:26:35.380 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:26:35.381 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:35.381 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 115433 2023-08-28T02:26:35.381 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.381 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:35.381 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.381 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:35.381 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.441952+0000", 2023-08-28T02:26:35.381 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:35.381 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.381 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.381 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:35.381 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.441952+0000", 2023-08-28T02:26:35.381 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999971 2023-08-28T02:26:35.382 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.382 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.382 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:35.382 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.441949+0000", 2023-08-28T02:26:35.382 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2235999999999999e-05 2023-08-28T02:26:35.382 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.382 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.382 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:35.382 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.441962+0000", 2023-08-28T02:26:35.382 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.4902e-05 2023-08-28T02:26:35.382 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.382 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.382 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:35.383 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.441986+0000", 2023-08-28T02:26:35.383 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.2520000000000005e-06 2023-08-28T02:26:35.383 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.383 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.383 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:35.383 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.441996+0000", 2023-08-28T02:26:35.383 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.2984000000000005e-05 2023-08-28T02:26:35.383 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.383 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.383 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:35.383 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.442069+0000", 2023-08-28T02:26:35.383 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1915e-05 2023-08-28T02:26:35.383 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.384 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.384 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:26:35.384 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.442081+0000", 2023-08-28T02:26:35.384 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.79360673599999998 2023-08-28T02:26:35.384 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.384 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.384 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:35.384 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.235687+0000", 2023-08-28T02:26:35.384 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.9048999999999995e-05 2023-08-28T02:26:35.384 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.384 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.384 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:35.384 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.235766+0000", 2023-08-28T02:26:35.384 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000176825 2023-08-28T02:26:35.385 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.385 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.385 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 2,5", 2023-08-28T02:26:35.385 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.235943+0000", 2023-08-28T02:26:35.385 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0095067039999999995 2023-08-28T02:26:35.385 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.385 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.385 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:26:35.385 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.245450+0000", 2023-08-28T02:26:35.385 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.5024999999999995e-05 2023-08-28T02:26:35.385 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.385 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.385 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:35.386 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.245515+0000", 2023-08-28T02:26:35.386 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0078574400000000003 2023-08-28T02:26:35.386 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.386 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.386 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:35.386 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.253372+0000", 2023-08-28T02:26:35.386 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.1189e-05 2023-08-28T02:26:35.386 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.386 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.386 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:26:35.386 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.253434+0000", 2023-08-28T02:26:35.386 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.3395e-05 2023-08-28T02:26:35.386 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.387 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.387 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:26:35.387 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.253487+0000", 2023-08-28T02:26:35.387 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.81153499200000001 2023-08-28T02:26:35.387 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:35.387 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:35.387 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:35.387 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.387 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.387 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:115461 5.4 5:291789c6:::benchmark_data_smithi143_236200_object5318:head [delete] snapc 0=[] ondisk+write+known_if_redirected e150)", 2023-08-28T02:26:35.387 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:46.497880+0000", 2023-08-28T02:26:35.387 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 168.80513600899999, 2023-08-28T02:26:35.388 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.755782175, 2023-08-28T02:26:35.388 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:35.388 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:35.388 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:26:35.388 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:26:35.388 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:35.388 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 115461 2023-08-28T02:26:35.388 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.388 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:35.388 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.388 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:35.388 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.497880+0000", 2023-08-28T02:26:35.388 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:35.388 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.389 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.389 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:35.389 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.497880+0000", 2023-08-28T02:26:35.389 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999976 2023-08-28T02:26:35.389 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.389 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.389 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:35.389 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.497878+0000", 2023-08-28T02:26:35.389 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1885000000000001e-05 2023-08-28T02:26:35.389 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.389 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.389 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:35.389 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.497890+0000", 2023-08-28T02:26:35.390 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.4703999999999999e-05 2023-08-28T02:26:35.390 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.390 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.390 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:35.390 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.497915+0000", 2023-08-28T02:26:35.390 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.8650000000000007e-06 2023-08-28T02:26:35.390 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.390 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.390 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:35.390 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.497923+0000", 2023-08-28T02:26:35.390 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.0001999999999998e-05 2023-08-28T02:26:35.390 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.390 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.391 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:35.391 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.497993+0000", 2023-08-28T02:26:35.391 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1633e-05 2023-08-28T02:26:35.391 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.391 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.391 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:26:35.391 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.498005+0000", 2023-08-28T02:26:35.391 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.74034235599999998 2023-08-28T02:26:35.391 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.391 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.391 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:35.391 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.238347+0000", 2023-08-28T02:26:35.392 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000106808 2023-08-28T02:26:35.392 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.392 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.392 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:35.392 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.238454+0000", 2023-08-28T02:26:35.392 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000135096 2023-08-28T02:26:35.392 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.392 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.392 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 2,5", 2023-08-28T02:26:35.392 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.238589+0000", 2023-08-28T02:26:35.392 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0068695700000000002 2023-08-28T02:26:35.392 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.392 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.393 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:26:35.393 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.245459+0000", 2023-08-28T02:26:35.393 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0080750579999999995 2023-08-28T02:26:35.393 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.393 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.393 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:35.393 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.253534+0000", 2023-08-28T02:26:35.393 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.3621999999999998e-05 2023-08-28T02:26:35.393 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.393 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.393 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:35.393 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.253568+0000", 2023-08-28T02:26:35.393 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.0056999999999999e-05 2023-08-28T02:26:35.394 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.394 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.394 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:26:35.394 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.253618+0000", 2023-08-28T02:26:35.394 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.4873999999999998e-05 2023-08-28T02:26:35.394 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.394 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.394 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:26:35.394 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.253662+0000", 2023-08-28T02:26:35.394 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.755782175 2023-08-28T02:26:35.394 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:35.394 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:35.394 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:35.395 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.395 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.395 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:115466 5.4 5:22472d2c:::benchmark_data_smithi143_236200_object5323:head [delete] snapc 0=[] ondisk+write+known_if_redirected e150)", 2023-08-28T02:26:35.395 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:46.508560+0000", 2023-08-28T02:26:35.395 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 168.79445581900001, 2023-08-28T02:26:35.395 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.75962038399999998, 2023-08-28T02:26:35.395 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:35.395 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:35.395 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:26:35.395 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:26:35.395 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:35.395 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 115466 2023-08-28T02:26:35.395 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.396 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:35.396 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.396 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:35.396 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.508560+0000", 2023-08-28T02:26:35.396 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:35.396 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.396 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.396 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:35.396 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.508560+0000", 2023-08-28T02:26:35.396 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:26:35.396 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.396 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.396 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:35.397 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.508559+0000", 2023-08-28T02:26:35.397 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.1679999999999993e-06 2023-08-28T02:26:35.397 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.397 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.397 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:35.397 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.508568+0000", 2023-08-28T02:26:35.397 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.3961999999999999e-05 2023-08-28T02:26:35.397 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.397 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.397 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:35.397 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.508582+0000", 2023-08-28T02:26:35.397 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.3599999999999998e-06 2023-08-28T02:26:35.398 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.398 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.398 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:35.398 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.508589+0000", 2023-08-28T02:26:35.398 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.4589e-05 2023-08-28T02:26:35.398 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.398 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.398 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:35.398 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.508664+0000", 2023-08-28T02:26:35.398 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2138e-05 2023-08-28T02:26:35.398 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.398 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.398 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:26:35.399 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.508676+0000", 2023-08-28T02:26:35.399 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.73317837100000005 2023-08-28T02:26:35.399 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.399 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.399 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:35.399 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.241854+0000", 2023-08-28T02:26:35.399 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000117246 2023-08-28T02:26:35.399 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.399 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.399 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:35.399 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.241971+0000", 2023-08-28T02:26:35.399 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012837699999999999 2023-08-28T02:26:35.399 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.400 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.400 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 2,5", 2023-08-28T02:26:35.400 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.242100+0000", 2023-08-28T02:26:35.400 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0035736040000000002 2023-08-28T02:26:35.400 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.400 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.400 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:26:35.400 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.245673+0000", 2023-08-28T02:26:35.400 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0080461899999999999 2023-08-28T02:26:35.400 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.401 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.401 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:35.401 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.253720+0000", 2023-08-28T02:26:35.401 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.014366273000000001 2023-08-28T02:26:35.401 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.401 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.401 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:35.401 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.268086+0000", 2023-08-28T02:26:35.401 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.5237000000000001e-05 2023-08-28T02:26:35.401 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.401 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.401 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:26:35.402 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.268131+0000", 2023-08-28T02:26:35.402 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.9824999999999998e-05 2023-08-28T02:26:35.402 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.402 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.402 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:26:35.402 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.268181+0000", 2023-08-28T02:26:35.402 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.75962038399999998 2023-08-28T02:26:35.402 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:35.402 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:35.402 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:35.402 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.402 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.402 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:115469 5.4 5:274fde17:::benchmark_data_smithi143_236200_object5326:head [delete] snapc 0=[] ondisk+write+known_if_redirected e150)", 2023-08-28T02:26:35.403 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:46.512521+0000", 2023-08-28T02:26:35.403 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 168.79049518400001, 2023-08-28T02:26:35.403 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.75576967100000003, 2023-08-28T02:26:35.403 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:35.403 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:35.403 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:26:35.403 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:26:35.403 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:35.403 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 115469 2023-08-28T02:26:35.403 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.403 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:35.403 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.403 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:35.404 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.512521+0000", 2023-08-28T02:26:35.404 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:35.404 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.404 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.404 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:35.404 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.512521+0000", 2023-08-28T02:26:35.404 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:26:35.404 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.404 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.404 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:35.404 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.512519+0000", 2023-08-28T02:26:35.404 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.5517000000000001e-05 2023-08-28T02:26:35.404 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.405 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.405 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:35.405 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.512535+0000", 2023-08-28T02:26:35.405 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.5277e-05 2023-08-28T02:26:35.405 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.405 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.405 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:35.405 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.512550+0000", 2023-08-28T02:26:35.405 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.6669999999999996e-06 2023-08-28T02:26:35.405 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.405 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.405 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:35.405 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.512558+0000", 2023-08-28T02:26:35.406 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0020992430000000002 2023-08-28T02:26:35.406 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.406 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.406 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:35.406 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.514657+0000", 2023-08-28T02:26:35.406 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.6874000000000001e-05 2023-08-28T02:26:35.406 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.406 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.406 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:26:35.406 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.514684+0000", 2023-08-28T02:26:35.406 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.73102815600000004 2023-08-28T02:26:35.406 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.406 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.407 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:35.407 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.245712+0000", 2023-08-28T02:26:35.407 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00013840900000000001 2023-08-28T02:26:35.407 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.407 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.407 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:35.407 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.245851+0000", 2023-08-28T02:26:35.407 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000149841 2023-08-28T02:26:35.407 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.407 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.407 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 2,5", 2023-08-28T02:26:35.407 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.246000+0000", 2023-08-28T02:26:35.407 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.007757997 2023-08-28T02:26:35.407 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.408 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.408 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:35.408 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.253758+0000", 2023-08-28T02:26:35.408 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.981e-06 2023-08-28T02:26:35.408 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.408 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.408 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:26:35.408 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.253764+0000", 2023-08-28T02:26:35.408 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.014460132000000001 2023-08-28T02:26:35.408 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.408 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.408 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:35.408 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.268225+0000", 2023-08-28T02:26:35.409 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.4363000000000003e-05 2023-08-28T02:26:35.409 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.409 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.409 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:26:35.409 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.268259+0000", 2023-08-28T02:26:35.409 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.1850000000000002e-05 2023-08-28T02:26:35.409 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.409 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.409 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:26:35.409 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.268291+0000", 2023-08-28T02:26:35.409 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.75576967100000003 2023-08-28T02:26:35.409 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:35.409 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:35.409 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:35.410 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.410 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.410 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:115478 5.4 5:2893bc13:::benchmark_data_smithi143_236200_object5335:head [delete] snapc 0=[] ondisk+write+known_if_redirected e150)", 2023-08-28T02:26:35.410 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:46.525068+0000", 2023-08-28T02:26:35.410 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 168.777948307, 2023-08-28T02:26:35.410 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.74332375100000003, 2023-08-28T02:26:35.410 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:35.410 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:35.410 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:26:35.410 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:26:35.410 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:35.410 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 115478 2023-08-28T02:26:35.410 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.411 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:35.411 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.411 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:35.411 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.525068+0000", 2023-08-28T02:26:35.411 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:35.411 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.411 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.411 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:35.411 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.525068+0000", 2023-08-28T02:26:35.411 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:26:35.411 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.411 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.411 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:35.412 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.525067+0000", 2023-08-28T02:26:35.412 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.0609999999999998e-06 2023-08-28T02:26:35.412 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.412 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.412 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:35.412 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.525074+0000", 2023-08-28T02:26:35.412 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2438e-05 2023-08-28T02:26:35.412 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.412 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.412 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:35.412 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.525086+0000", 2023-08-28T02:26:35.412 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.4759999999999997e-06 2023-08-28T02:26:35.412 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.413 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.413 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:35.413 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.525092+0000", 2023-08-28T02:26:35.413 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.8484000000000003e-05 2023-08-28T02:26:35.413 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.413 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.413 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:35.413 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.525170+0000", 2023-08-28T02:26:35.413 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.4430000000000001e-05 2023-08-28T02:26:35.413 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.413 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.413 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:26:35.413 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.525184+0000", 2023-08-28T02:26:35.414 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.72438353700000002 2023-08-28T02:26:35.414 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.414 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.414 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:35.414 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.249568+0000", 2023-08-28T02:26:35.414 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00011211 2023-08-28T02:26:35.414 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.414 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.414 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:35.414 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.249680+0000", 2023-08-28T02:26:35.414 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00013465499999999999 2023-08-28T02:26:35.414 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.414 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.415 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 2,5", 2023-08-28T02:26:35.415 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.249815+0000", 2023-08-28T02:26:35.415 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0039542819999999999 2023-08-28T02:26:35.415 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.415 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.415 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:26:35.415 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.253769+0000", 2023-08-28T02:26:35.415 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.1517000000000003e-05 2023-08-28T02:26:35.415 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.415 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.415 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:35.415 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.253801+0000", 2023-08-28T02:26:35.416 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.014535591 2023-08-28T02:26:35.416 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.416 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.416 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:35.416 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.268336+0000", 2023-08-28T02:26:35.416 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.5709000000000001e-05 2023-08-28T02:26:35.416 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.416 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.416 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:26:35.416 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.268362+0000", 2023-08-28T02:26:35.416 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.9895000000000001e-05 2023-08-28T02:26:35.416 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.416 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.416 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:26:35.417 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.268392+0000", 2023-08-28T02:26:35.417 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.74332375100000003 2023-08-28T02:26:35.417 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:35.417 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:35.417 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:35.417 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.417 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.417 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:115494 5.4 5:2239329e:::benchmark_data_smithi143_236200_object5351:head [delete] snapc 0=[] ondisk+write+known_if_redirected e150)", 2023-08-28T02:26:35.417 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:46.551030+0000", 2023-08-28T02:26:35.417 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 168.751986012, 2023-08-28T02:26:35.417 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.71743900000000005, 2023-08-28T02:26:35.417 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:35.417 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:35.418 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:26:35.418 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:26:35.418 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:35.418 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 115494 2023-08-28T02:26:35.418 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.418 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:35.418 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.418 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:35.418 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.551030+0000", 2023-08-28T02:26:35.418 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:35.418 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.418 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.418 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:35.418 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.551030+0000", 2023-08-28T02:26:35.419 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:26:35.419 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.419 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.419 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:35.419 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.551029+0000", 2023-08-28T02:26:35.419 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.3619999999999995e-06 2023-08-28T02:26:35.419 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.419 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.419 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:35.419 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.551038+0000", 2023-08-28T02:26:35.419 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.5373999999999999e-05 2023-08-28T02:26:35.419 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.420 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.420 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:35.420 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.551053+0000", 2023-08-28T02:26:35.420 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.6860000000000006e-06 2023-08-28T02:26:35.420 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.420 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.420 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:35.420 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.551061+0000", 2023-08-28T02:26:35.420 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.5475e-05 2023-08-28T02:26:35.420 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.420 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.420 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:35.421 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.551116+0000", 2023-08-28T02:26:35.421 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.9089999999999995e-06 2023-08-28T02:26:35.421 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.421 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.421 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:26:35.421 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.551126+0000", 2023-08-28T02:26:35.421 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.70269407399999995 2023-08-28T02:26:35.421 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.421 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.421 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:35.421 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.253820+0000", 2023-08-28T02:26:35.421 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000117512 2023-08-28T02:26:35.421 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.421 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.422 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:35.422 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.253938+0000", 2023-08-28T02:26:35.422 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0013800170000000001 2023-08-28T02:26:35.422 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.422 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.422 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 2,5", 2023-08-28T02:26:35.422 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.255318+0000", 2023-08-28T02:26:35.422 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.009856343 2023-08-28T02:26:35.422 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.422 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.422 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:26:35.422 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.265174+0000", 2023-08-28T02:26:35.423 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.003139995 2023-08-28T02:26:35.423 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.423 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.423 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:35.423 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.268314+0000", 2023-08-28T02:26:35.423 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00010107600000000001 2023-08-28T02:26:35.423 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.423 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.423 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:35.423 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.268415+0000", 2023-08-28T02:26:35.423 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.4460999999999998e-05 2023-08-28T02:26:35.423 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.424 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.424 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:26:35.424 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.268440+0000", 2023-08-28T02:26:35.424 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.9451999999999999e-05 2023-08-28T02:26:35.424 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.424 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.424 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:26:35.424 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.268469+0000", 2023-08-28T02:26:35.424 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.71743900000000005 2023-08-28T02:26:35.424 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:35.424 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:35.424 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:35.424 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.424 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.425 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:115495 5.4 5:2ae0f4f8:::benchmark_data_smithi143_236200_object5352:head [delete] snapc 0=[] ondisk+write+known_if_redirected e150)", 2023-08-28T02:26:35.425 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:46.551129+0000", 2023-08-28T02:26:35.425 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 168.75188737100001, 2023-08-28T02:26:35.425 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.72474564100000005, 2023-08-28T02:26:35.425 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:35.425 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:35.425 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:26:35.425 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:26:35.425 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:35.425 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 115495 2023-08-28T02:26:35.425 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.425 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:35.425 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.426 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:35.426 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.551129+0000", 2023-08-28T02:26:35.426 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:35.426 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.426 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.426 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:35.426 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.551129+0000", 2023-08-28T02:26:35.426 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999995 2023-08-28T02:26:35.426 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.426 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.426 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:35.426 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.551128+0000", 2023-08-28T02:26:35.426 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.6580000000000001e-06 2023-08-28T02:26:35.427 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.427 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.427 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:35.427 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.551133+0000", 2023-08-28T02:26:35.427 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0040999999999999e-05 2023-08-28T02:26:35.427 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.427 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.427 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:35.427 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.551143+0000", 2023-08-28T02:26:35.427 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.0580000000000003e-06 2023-08-28T02:26:35.427 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.427 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.427 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:35.428 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.551149+0000", 2023-08-28T02:26:35.428 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.3900999999999997e-05 2023-08-28T02:26:35.428 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.428 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.428 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:35.428 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.551213+0000", 2023-08-28T02:26:35.428 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.9040000000000004e-06 2023-08-28T02:26:35.428 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.428 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.428 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:26:35.428 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.551223+0000", 2023-08-28T02:26:35.428 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.70743548999999994 2023-08-28T02:26:35.428 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.429 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.429 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:35.429 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.258658+0000", 2023-08-28T02:26:35.429 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.5086000000000004e-05 2023-08-28T02:26:35.429 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.429 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.429 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:35.429 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.258753+0000", 2023-08-28T02:26:35.429 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000113942 2023-08-28T02:26:35.429 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.429 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.429 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 2,5", 2023-08-28T02:26:35.429 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.258867+0000", 2023-08-28T02:26:35.430 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0091801000000000001 2023-08-28T02:26:35.430 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.430 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.430 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:26:35.430 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.268048+0000", 2023-08-28T02:26:35.430 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0044477529999999996 2023-08-28T02:26:35.430 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.430 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.430 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:35.430 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.272495+0000", 2023-08-28T02:26:35.430 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0032891740000000002 2023-08-28T02:26:35.430 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.430 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.431 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:35.431 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.275784+0000", 2023-08-28T02:26:35.431 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.876e-05 2023-08-28T02:26:35.431 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.431 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.431 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:26:35.431 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.275843+0000", 2023-08-28T02:26:35.431 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.1361e-05 2023-08-28T02:26:35.431 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.431 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.431 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:26:35.431 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.275875+0000", 2023-08-28T02:26:35.431 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.72474564100000005 2023-08-28T02:26:35.432 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:35.432 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:35.432 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:35.432 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.432 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.432 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:115525 5.4 5:2062ef58:::benchmark_data_smithi143_236200_object5382:head [delete] snapc 0=[] ondisk+write+known_if_redirected e150)", 2023-08-28T02:26:35.432 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:46.678047+0000", 2023-08-28T02:26:35.432 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 168.62496886400001, 2023-08-28T02:26:35.432 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.59805382799999995, 2023-08-28T02:26:35.432 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:35.432 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:35.432 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:26:35.432 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:26:35.433 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:35.433 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 115525 2023-08-28T02:26:35.433 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.433 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:35.433 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.433 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:35.433 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.678047+0000", 2023-08-28T02:26:35.433 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:35.433 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.433 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.433 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:35.433 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.678047+0000", 2023-08-28T02:26:35.433 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:26:35.433 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.434 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.434 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:35.434 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.678046+0000", 2023-08-28T02:26:35.434 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.9019999999999999e-06 2023-08-28T02:26:35.434 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.434 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.434 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:35.434 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.678054+0000", 2023-08-28T02:26:35.434 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.6011999999999998e-05 2023-08-28T02:26:35.434 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.434 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.434 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:35.434 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.678070+0000", 2023-08-28T02:26:35.435 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.9269999999999999e-06 2023-08-28T02:26:35.435 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.435 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.435 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:35.435 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.678077+0000", 2023-08-28T02:26:35.435 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.5090000000000002e-05 2023-08-28T02:26:35.435 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.435 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.435 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:35.435 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.678142+0000", 2023-08-28T02:26:35.435 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.7284000000000002e-05 2023-08-28T02:26:35.435 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.435 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.436 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:26:35.436 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:46.678169+0000", 2023-08-28T02:26:35.436 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.58350079300000002 2023-08-28T02:26:35.436 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.436 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.436 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:35.436 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.261670+0000", 2023-08-28T02:26:35.436 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000105834 2023-08-28T02:26:35.436 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.436 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.436 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:35.436 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.261776+0000", 2023-08-28T02:26:35.436 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012859699999999999 2023-08-28T02:26:35.437 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.437 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.437 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 2,5", 2023-08-28T02:26:35.437 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.261904+0000", 2023-08-28T02:26:35.437 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0062855519999999998 2023-08-28T02:26:35.437 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.437 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.437 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:26:35.437 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.268190+0000", 2023-08-28T02:26:35.437 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0077518539999999999 2023-08-28T02:26:35.437 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.437 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.437 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:35.437 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.275942+0000", 2023-08-28T02:26:35.438 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.7631000000000002e-05 2023-08-28T02:26:35.438 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.438 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.438 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:35.438 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.276029+0000", 2023-08-28T02:26:35.438 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.4066000000000002e-05 2023-08-28T02:26:35.438 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.438 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.438 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:26:35.438 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.276063+0000", 2023-08-28T02:26:35.438 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.7790999999999998e-05 2023-08-28T02:26:35.438 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.438 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.439 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:26:35.439 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:47.276101+0000", 2023-08-28T02:26:35.439 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.59805382799999995 2023-08-28T02:26:35.439 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:35.439 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:35.439 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:35.439 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.439 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.439 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:143200 5.4 5:238a36bb:::benchmark_data_smithi143_236200_object33057:head [delete] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-08-28T02:26:35.439 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:24:32.485479+0000", 2023-08-28T02:26:35.439 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 122.81753716, 2023-08-28T02:26:35.439 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.26978011899999998, 2023-08-28T02:26:35.439 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:35.440 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:35.440 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:26:35.440 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:26:35.440 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:35.440 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 143200 2023-08-28T02:26:35.440 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.440 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:35.440 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.440 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:35.440 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:32.485479+0000", 2023-08-28T02:26:35.440 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:35.440 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.440 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.440 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:35.441 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:32.485479+0000", 2023-08-28T02:26:35.441 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:26:35.441 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.441 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.441 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:35.441 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:32.485478+0000", 2023-08-28T02:26:35.441 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.8359999999999998e-06 2023-08-28T02:26:35.441 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.441 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.441 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:35.441 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:32.485487+0000", 2023-08-28T02:26:35.441 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.7711e-05 2023-08-28T02:26:35.441 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.442 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.442 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:35.442 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:32.485504+0000", 2023-08-28T02:26:35.442 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.6610000000000003e-06 2023-08-28T02:26:35.442 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.442 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.442 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:35.442 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:32.485511+0000", 2023-08-28T02:26:35.442 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.7921000000000003e-05 2023-08-28T02:26:35.442 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.442 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.442 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:35.442 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:32.485569+0000", 2023-08-28T02:26:35.443 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012735399999999999 2023-08-28T02:26:35.443 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.443 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.443 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:35.443 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:32.485696+0000", 2023-08-28T02:26:35.443 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00014385900000000001 2023-08-28T02:26:35.443 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.443 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.443 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 2,5", 2023-08-28T02:26:35.443 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:32.485840+0000", 2023-08-28T02:26:35.443 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0040326390000000002 2023-08-28T02:26:35.443 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.443 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.443 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:35.444 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:32.489873+0000", 2023-08-28T02:26:35.444 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.1428000000000006e-05 2023-08-28T02:26:35.444 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.444 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.444 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:26:35.444 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:32.489964+0000", 2023-08-28T02:26:35.444 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.265199511 2023-08-28T02:26:35.444 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.444 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.444 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:35.444 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:32.755164+0000", 2023-08-28T02:26:35.444 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.2101999999999997e-05 2023-08-28T02:26:35.444 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.445 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.445 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:26:35.445 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:32.755216+0000", 2023-08-28T02:26:35.445 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.3507000000000002e-05 2023-08-28T02:26:35.445 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.445 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.445 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:26:35.445 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:32.755259+0000", 2023-08-28T02:26:35.445 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.26978011899999998 2023-08-28T02:26:35.445 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:35.445 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:35.445 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:35.445 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.445 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.446 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:143203 5.4 5:23ec2e0e:::benchmark_data_smithi143_236200_object33060:head [delete] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-08-28T02:26:35.446 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:24:32.491570+0000", 2023-08-28T02:26:35.446 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 122.811446444, 2023-08-28T02:26:35.446 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.26380761200000002, 2023-08-28T02:26:35.446 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:26:35.446 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:35.446 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:26:35.446 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:26:35.446 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:35.446 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 143203 2023-08-28T02:26:35.446 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.446 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:26:35.446 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.447 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:26:35.447 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:32.491570+0000", 2023-08-28T02:26:35.447 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:26:35.447 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.447 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.447 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:26:35.447 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:32.491570+0000", 2023-08-28T02:26:35.447 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:26:35.447 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.447 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.447 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:26:35.447 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:32.491569+0000", 2023-08-28T02:26:35.447 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.6889999999999998e-06 2023-08-28T02:26:35.447 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.448 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.448 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:26:35.448 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:32.491575+0000", 2023-08-28T02:26:35.448 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2778e-05 2023-08-28T02:26:35.448 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.448 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.448 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:26:35.448 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:32.491588+0000", 2023-08-28T02:26:35.448 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.5300000000000004e-06 2023-08-28T02:26:35.448 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.448 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.448 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:26:35.448 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:32.491594+0000", 2023-08-28T02:26:35.449 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.7697000000000002e-05 2023-08-28T02:26:35.449 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.449 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.449 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:26:35.449 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:32.491631+0000", 2023-08-28T02:26:35.449 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.2082000000000004e-05 2023-08-28T02:26:35.449 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.449 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.449 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:26:35.449 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:32.491723+0000", 2023-08-28T02:26:35.449 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.8416000000000001e-05 2023-08-28T02:26:35.449 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.449 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.449 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 2,5", 2023-08-28T02:26:35.450 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:32.491822+0000", 2023-08-28T02:26:35.450 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0031649270000000001 2023-08-28T02:26:35.450 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.450 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.450 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:26:35.450 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:32.494987+0000", 2023-08-28T02:26:35.450 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00025955500000000002 2023-08-28T02:26:35.450 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.450 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.450 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:35.450 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:32.495246+0000", 2023-08-28T02:26:35.450 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.26004859299999999 2023-08-28T02:26:35.450 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.451 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.451 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:35.451 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:32.755295+0000", 2023-08-28T02:26:35.451 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.5889000000000001e-05 2023-08-28T02:26:35.451 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.451 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.451 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:26:35.451 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:32.755341+0000", 2023-08-28T02:26:35.451 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.6764999999999998e-05 2023-08-28T02:26:35.451 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:26:35.451 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:26:35.451 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:26:35.451 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:32.755377+0000", 2023-08-28T02:26:35.452 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.26380761200000002 2023-08-28T02:26:35.452 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:35.452 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:35.452 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:35.452 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:26:35.452 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:26:35.452 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:26:35.452 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:26:35.540 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:35.561 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:26:35.668 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:26:35.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:35.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:26:35.772 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:26:35.804 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:35.883 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:26:35.883 INFO:teuthology.orchestra.run.smithi143.stdout:{"fsid":"2fcb0bfa-4547-11ee-817c-1b7a5f1ba315","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 1990/9852 objects degraded (20.199%), 1 pg degraded, 1 pg undersized","count":2},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":553,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":183,"num_osds":8,"num_up_osds":8,"osd_up_since":1693189249,"num_in_osds":8,"osd_in_since":1693189438,"num_remapped_pgs":3},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":86},{"state_name":"active+remapped+backfill_wait","count":2},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":3284,"data_bytes":218109899,"bytes_used":2913689600,"bytes_avail":765046595584,"bytes_total":767960285184,"degraded_objects":1990,"degraded_total":9852,"degraded_ratio":0.2019894437677629,"misplaced_objects":379,"misplaced_total":9852,"misplaced_ratio":0.038469346325619165,"recovering_objects_per_sec":46,"recovering_bytes_per_sec":2948165,"recovering_keys_per_sec":0,"num_objects_recovered":470,"num_bytes_recovered":29491200,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":594},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi143.spifrj","status":"up:active","gid":14637}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.143:8443/","prometheus":"http://172.21.15.143:9283/"}},"servicemap":{"epoch":22,"modified":"2023-08-28T02:21:49.036237+0000","services":{}},"progress_events":{"5de4a8b4-84fe-4f37-8817-bb52f7e413cf":{"message":"Global Recovery Event (2m)\n [===========================.] (remaining: 5s)","progress":0.96629214286804199,"add_to_ceph_s":true},"8e5f590e-829c-445f-9a5b-24676b2537ad":{"message":"Upgrade to 18.2.0-187-g9cf20820 (3m)\n [============................] (remaining: 4m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-08-28T02:26:36.051 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:26:36.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:36.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:26:36.158 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:26:36.195 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.0 dump_ops_in_flight 2023-08-28T02:26:36.223 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:36.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:36 smithi143 ceph-mon[183059]: pgmap v427: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 208 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 1990/9852 objects degraded (20.199%); 379/9852 objects misplaced (3.847%); 2.8 MiB/s, 46 objects/s recovering 2023-08-28T02:26:36.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:36 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1870028117' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-08-28T02:26:36.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:36 smithi143 ceph-mon[189871]: pgmap v427: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 208 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 1990/9852 objects degraded (20.199%); 379/9852 objects misplaced (3.847%); 2.8 MiB/s, 46 objects/s recovering 2023-08-28T02:26:36.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:36 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1870028117' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-08-28T02:26:36.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:36 smithi169 ceph-mon[159415]: pgmap v427: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 208 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 1990/9852 objects degraded (20.199%); 379/9852 objects misplaced (3.847%); 2.8 MiB/s, 46 objects/s recovering 2023-08-28T02:26:36.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:36 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1870028117' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-08-28T02:26:36.925 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:26:37.275 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:37 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 1990/9852 objects degraded (20.199%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:26:37.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:37 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 1990/9852 objects degraded (20.199%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:26:37.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:37 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 1990/9852 objects degraded (20.199%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:26:37.975 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-08-28T02:26:37.976 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph pg dump --format=json 2023-08-28T02:26:37.991 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:26:38.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:38.009 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:26:38.110 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:26:38.125 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Removed 110140 objects 2023-08-28T02:26:38.125 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Clean up completed and total clean up time :180.823 2023-08-28T02:26:38.138 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:38.180 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:26:38.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:38.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:26:38.227 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:26:38.253 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:26:38.284 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:26:38.322 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:38.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:38 smithi169 ceph-mon[159415]: pgmap v428: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 170 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 562 op/s; 1990/8016 objects degraded (24.825%); 379/8016 objects misplaced (4.728%); 2.8 MiB/s, 46 objects/s recovering 2023-08-28T02:26:38.489 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:26:38.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:38.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:26:38.593 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:26:38.620 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:38.628 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:38 smithi143 ceph-mon[183059]: pgmap v428: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 170 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 562 op/s; 1990/8016 objects degraded (24.825%); 379/8016 objects misplaced (4.728%); 2.8 MiB/s, 46 objects/s recovering 2023-08-28T02:26:38.628 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:38 smithi143 ceph-mon[189871]: pgmap v428: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 170 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 562 op/s; 1990/8016 objects degraded (24.825%); 379/8016 objects misplaced (4.728%); 2.8 MiB/s, 46 objects/s recovering 2023-08-28T02:26:38.664 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:26:38.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:38.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:26:38.772 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:26:38.803 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:39.008 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:26:39.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:39.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:26:39.112 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:26:39.140 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:39.320 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is set 2023-08-28T02:26:39.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:39 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2473889801' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:26:39.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:39 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:26:39.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:39 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2473889801' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:26:39.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:39 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:26:39.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:39 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2473889801' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:26:39.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:39 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:26:40.641 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:26:40.643 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:40 smithi143 ceph-mon[189871]: pgmap v429: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 24 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 1990/999 objects degraded (199.199%); 43/999 objects misplaced (4.304%); 1.4 MiB/s, 54 objects/s recovering 2023-08-28T02:26:40.643 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:40 smithi143 ceph-mon[189871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:26:40.643 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:40 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:26:40.643 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:40 smithi143 ceph-mon[189871]: osdmap e184: 8 total, 8 up, 8 in 2023-08-28T02:26:40.643 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:40 smithi143 ceph-mon[189871]: failed to encode map e184 with expected crc 2023-08-28T02:26:40.643 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:40 smithi143 ceph-mon[189871]: failed to encode map e184 with expected crc 2023-08-28T02:26:40.643 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:40 smithi143 ceph-mon[189871]: failed to encode map e184 with expected crc 2023-08-28T02:26:40.643 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:40 smithi143 ceph-mon[189871]: failed to encode map e184 with expected crc 2023-08-28T02:26:40.644 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:40 smithi143 ceph-mon[183059]: pgmap v429: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 24 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 1990/999 objects degraded (199.199%); 43/999 objects misplaced (4.304%); 1.4 MiB/s, 54 objects/s recovering 2023-08-28T02:26:40.644 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:40 smithi143 ceph-mon[183059]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:26:40.644 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:40 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:26:40.644 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:40 smithi143 ceph-mon[183059]: osdmap e184: 8 total, 8 up, 8 in 2023-08-28T02:26:40.644 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:40 smithi143 ceph-mon[183059]: failed to encode map e184 with expected crc 2023-08-28T02:26:40.644 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:40 smithi143 ceph-mon[183059]: failed to encode map e184 with expected crc 2023-08-28T02:26:40.644 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:40 smithi143 ceph-mon[183059]: failed to encode map e184 with expected crc 2023-08-28T02:26:40.644 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:40 smithi143 ceph-mon[183059]: failed to encode map e184 with expected crc 2023-08-28T02:26:40.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:40.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:26:40.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:40 smithi169 ceph-mon[159415]: pgmap v429: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 24 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 1990/999 objects degraded (199.199%); 43/999 objects misplaced (4.304%); 1.4 MiB/s, 54 objects/s recovering 2023-08-28T02:26:40.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:40 smithi169 ceph-mon[159415]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:26:40.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:40 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:26:40.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:40 smithi169 ceph-mon[159415]: osdmap e184: 8 total, 8 up, 8 in 2023-08-28T02:26:40.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:40 smithi169 ceph-mon[159415]: failed to encode map e184 with expected crc 2023-08-28T02:26:40.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:40 smithi169 ceph-mon[159415]: failed to encode map e184 with expected crc 2023-08-28T02:26:40.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:40 smithi169 ceph-mon[159415]: failed to encode map e184 with expected crc 2023-08-28T02:26:40.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:40 smithi169 ceph-mon[159415]: failed to encode map e184 with expected crc 2023-08-28T02:26:40.746 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:26:40.778 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:40.825 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:26:40.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:40.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:26:40.930 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:26:40.957 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:41.002 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:26:41.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:41.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:26:41.110 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:26:41.142 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:41.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:41 smithi143 ceph-mon[183059]: failed to encode map e184 with expected crc 2023-08-28T02:26:41.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:41 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:26:41.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:41 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:26:41.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:41 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 1990/999 objects degraded (199.199%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:26:41.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:41 smithi143 ceph-mon[189871]: failed to encode map e184 with expected crc 2023-08-28T02:26:41.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:41 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:26:41.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:41 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:26:41.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:41 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 1990/999 objects degraded (199.199%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:26:41.735 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2023-08-28T02:26:41.735 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2023-08-28T02:26:41.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:41 smithi169 ceph-mon[159415]: failed to encode map e184 with expected crc 2023-08-28T02:26:41.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:41 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:26:41.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:41 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:26:41.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:41 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 1990/999 objects degraded (199.199%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:26:41.909 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:26:41.969 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:26:41.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:41.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:26:42.040 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:26:41 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:26:41] "GET /metrics HTTP/1.1" 200 36499 "" "Prometheus/2.33.4" 2023-08-28T02:26:42.075 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:26:42.107 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:42.156 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:26:42.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:42.163 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:26:42.264 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:26:42.298 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:42.483 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:26:42.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:42 smithi169 ceph-mon[159415]: pgmap v431: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 24 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 1990/999 objects degraded (199.199%); 43/999 objects misplaced (4.304%); 1.4 MiB/s, 54 objects/s recovering 2023-08-28T02:26:42.504 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:26:42.532 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:26:42.553 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:42 smithi143 ceph-mon[183059]: pgmap v431: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 24 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 1990/999 objects degraded (199.199%); 43/999 objects misplaced (4.304%); 1.4 MiB/s, 54 objects/s recovering 2023-08-28T02:26:42.554 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:42 smithi143 ceph-mon[189871]: pgmap v431: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 24 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 1990/999 objects degraded (199.199%); 43/999 objects misplaced (4.304%); 1.4 MiB/s, 54 objects/s recovering 2023-08-28T02:26:42.561 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:26:42.577 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:26:42.577 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:26:42.586 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:26:42.605 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:26:42.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:42.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:26:42.612 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:26:42.636 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:26:42.658 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:26:42.660 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:26:42.660 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:26:42.661 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:26:42.661 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:26:42.661 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:26:42.661 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:26:42.661 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:26:42.661 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:26:42.710 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:26:42.739 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:43.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:43 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3999240001' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:26:43.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:43 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3999240001' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:26:43.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:43 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3999240001' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:26:43.899 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:26:43.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:43.902 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:26:43.948 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset noscrub 2023-08-28T02:26:44.003 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:26:44.034 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:44.084 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:26:44.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:44.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:26:44.189 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:26:44.216 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:44.576 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:44 smithi143 ceph-mon[183059]: pgmap v432: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 494 op/s; 1990/84 objects degraded (2369.048%); 1.4 MiB/s, 54 objects/s recovering 2023-08-28T02:26:44.576 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:44 smithi143 ceph-mon[189871]: pgmap v432: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 494 op/s; 1990/84 objects degraded (2369.048%); 1.4 MiB/s, 54 objects/s recovering 2023-08-28T02:26:44.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:44 smithi169 ceph-mon[159415]: pgmap v432: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 494 op/s; 1990/84 objects degraded (2369.048%); 1.4 MiB/s, 54 objects/s recovering 2023-08-28T02:26:45.241 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:26:45.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:45.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:26:45.344 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:26:45.375 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:45.408 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:26:45.429 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:26:45.433 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.0 dump_blocked_ops 2023-08-28T02:26:45.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:45.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:26:45.536 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:26:45.566 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:45.713 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:26:45.728 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:26:45.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:45.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:26:45.833 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:26:45.866 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:45.913 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:26:45.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:45.918 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:26:46.019 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:26:46.046 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:46.091 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:26:46.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:46.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:26:46.109 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:26:46.110 INFO:teuthology.orchestra.run.smithi143.stderr:dumped all 2023-08-28T02:26:46.199 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:26:46.235 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:46.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:46 smithi143 ceph-mon[183059]: pgmap v433: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 325 op/s; 1665/84 objects degraded (1982.143%); 0 B/s, 111 objects/s recovering 2023-08-28T02:26:46.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:46 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 1665/84 objects degraded (1982.143%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:26:46.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:46 smithi143 ceph-mon[189871]: pgmap v433: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 325 op/s; 1665/84 objects degraded (1982.143%); 0 B/s, 111 objects/s recovering 2023-08-28T02:26:46.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:46 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 1665/84 objects degraded (1982.143%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:26:46.535 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:26:46.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:46.538 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:26:46.639 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:26:46.668 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:46 smithi169 ceph-mon[159415]: pgmap v433: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 325 op/s; 1665/84 objects degraded (1982.143%); 0 B/s, 111 objects/s recovering 2023-08-28T02:26:46.668 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:46 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 1665/84 objects degraded (1982.143%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:26:46.671 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:46.718 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:26:46.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:46.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:26:46.824 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:26:46.850 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:47.347 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:26:47.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:47.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:26:47.353 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:26:47.451 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:26:47.478 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:47.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:47 smithi143 ceph-mon[183059]: from='client.44960 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:26:47.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:47 smithi143 ceph-mon[189871]: from='client.44960 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:26:47.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:47 smithi169 ceph-mon[159415]: from='client.44960 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:26:48.418 INFO:teuthology.orchestra.run.smithi143.stdout:{"pg_ready":true,"pg_map":{"version":433,"stamp":"2023-08-28T02:26:45.179383+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1665,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6274,"num_read_kb":13707,"num_write":221388,"num_write_kb":7064102,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8329,"num_bytes_recovered":469838470,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139697,"ondisk_log_size":139697,"up":267,"acting":266,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":279,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2190364,"kb_used_data":458652,"kb_used_omap":29,"kb_used_meta":1731618,"kb_avail":747770852,"statfs":{"total":767960285184,"available":765717352448,"internally_reserved":0,"allocated":469659648,"data_stored":464081484,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30674,"internal_metadata":1773176878},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-213319704,"num_objects":-3256,"num_object_clones":0,"num_object_copies":-9768,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-325,"num_objects_misplaced":-379,"num_objects_unfound":0,"num_objects_dirty":-3256,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3256,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1112,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002622"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":167,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.141931+0000","last_change":"2023-08-28T02:20:19.849807+0000","last_active":"2023-08-28T02:26:31.141931+0000","last_peered":"2023-08-28T02:26:31.141931+0000","last_clean":"2023-08-28T02:26:31.141931+0000","last_became_active":"2023-08-28T02:20:19.849464+0000","last_became_peered":"2023-08-28T02:20:19.849464+0000","last_unstale":"2023-08-28T02:26:31.141931+0000","last_undegraded":"2023-08-28T02:26:31.141931+0000","last_fullsized":"2023-08-28T02:26:31.141931+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":42,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.296225+0000","last_change":"2023-08-28T02:23:59.204957+0000","last_active":"2023-08-28T02:26:31.296225+0000","last_peered":"2023-08-28T02:26:31.296225+0000","last_clean":"2023-08-28T02:26:31.296225+0000","last_became_active":"2023-08-28T02:23:59.204717+0000","last_became_peered":"2023-08-28T02:23:59.204717+0000","last_unstale":"2023-08-28T02:26:31.296225+0000","last_undegraded":"2023-08-28T02:26:31.296225+0000","last_fullsized":"2023-08-28T02:26:31.296225+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:34:58.977269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":142,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.138425+0000","last_change":"2023-08-28T02:23:59.212721+0000","last_active":"2023-08-28T02:26:31.138425+0000","last_peered":"2023-08-28T02:26:31.138425+0000","last_clean":"2023-08-28T02:26:31.138425+0000","last_became_active":"2023-08-28T02:23:59.212446+0000","last_became_peered":"2023-08-28T02:23:59.212446+0000","last_unstale":"2023-08-28T02:26:31.138425+0000","last_undegraded":"2023-08-28T02:26:31.138425+0000","last_fullsized":"2023-08-28T02:26:31.138425+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:03:54.701399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":171,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:38.789505+0000","last_change":"2023-08-28T02:23:59.223634+0000","last_active":"2023-08-28T02:26:38.789505+0000","last_peered":"2023-08-28T02:26:38.789505+0000","last_clean":"2023-08-28T02:26:38.789505+0000","last_became_active":"2023-08-28T02:23:59.222704+0000","last_became_peered":"2023-08-28T02:23:59.222704+0000","last_unstale":"2023-08-28T02:26:38.789505+0000","last_undegraded":"2023-08-28T02:26:38.789505+0000","last_fullsized":"2023-08-28T02:26:38.789505+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:51:52.752361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":42,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.132325+0000","last_change":"2023-08-28T02:23:59.220459+0000","last_active":"2023-08-28T02:26:31.132325+0000","last_peered":"2023-08-28T02:26:31.132325+0000","last_clean":"2023-08-28T02:26:31.132325+0000","last_became_active":"2023-08-28T02:23:59.219274+0000","last_became_peered":"2023-08-28T02:23:59.219274+0000","last_unstale":"2023-08-28T02:26:31.132325+0000","last_undegraded":"2023-08-28T02:26:31.132325+0000","last_fullsized":"2023-08-28T02:26:31.132325+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:31:53.889311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":178,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.264825+0000","last_change":"2023-08-28T02:21:59.183009+0000","last_active":"2023-08-28T02:26:31.264825+0000","last_peered":"2023-08-28T02:26:31.264825+0000","last_clean":"2023-08-28T02:26:31.264825+0000","last_became_active":"2023-08-28T02:21:59.182621+0000","last_became_peered":"2023-08-28T02:21:59.182621+0000","last_unstale":"2023-08-28T02:26:31.264825+0000","last_undegraded":"2023-08-28T02:26:31.264825+0000","last_fullsized":"2023-08-28T02:26:31.264825+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":151,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.300307+0000","last_change":"2023-08-28T02:23:59.227592+0000","last_active":"2023-08-28T02:26:31.300307+0000","last_peered":"2023-08-28T02:26:31.300307+0000","last_clean":"2023-08-28T02:26:31.300307+0000","last_became_active":"2023-08-28T02:23:59.227342+0000","last_became_peered":"2023-08-28T02:23:59.227342+0000","last_unstale":"2023-08-28T02:26:31.300307+0000","last_undegraded":"2023-08-28T02:26:31.300307+0000","last_fullsized":"2023-08-28T02:26:31.300307+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:08:19.874396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":42,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.132273+0000","last_change":"2023-08-28T02:23:59.220634+0000","last_active":"2023-08-28T02:26:31.132273+0000","last_peered":"2023-08-28T02:26:31.132273+0000","last_clean":"2023-08-28T02:26:31.132273+0000","last_became_active":"2023-08-28T02:23:59.220177+0000","last_became_peered":"2023-08-28T02:23:59.220177+0000","last_unstale":"2023-08-28T02:26:31.132273+0000","last_undegraded":"2023-08-28T02:26:31.132273+0000","last_fullsized":"2023-08-28T02:26:31.132273+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:16:26.906064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":169,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.305052+0000","last_change":"2023-08-28T02:23:38.339581+0000","last_active":"2023-08-28T02:26:31.305052+0000","last_peered":"2023-08-28T02:26:31.305052+0000","last_clean":"2023-08-28T02:26:31.305052+0000","last_became_active":"2023-08-28T02:23:38.339244+0000","last_became_peered":"2023-08-28T02:23:38.339244+0000","last_unstale":"2023-08-28T02:26:31.305052+0000","last_undegraded":"2023-08-28T02:26:31.305052+0000","last_fullsized":"2023-08-28T02:26:31.305052+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":42,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.132401+0000","last_change":"2023-08-28T02:23:59.220551+0000","last_active":"2023-08-28T02:26:31.132401+0000","last_peered":"2023-08-28T02:26:31.132401+0000","last_clean":"2023-08-28T02:26:31.132401+0000","last_became_active":"2023-08-28T02:23:59.219681+0000","last_became_peered":"2023-08-28T02:23:59.219681+0000","last_unstale":"2023-08-28T02:26:31.132401+0000","last_undegraded":"2023-08-28T02:26:31.132401+0000","last_fullsized":"2023-08-28T02:26:31.132401+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:39:58.811256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":70,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.307370+0000","last_change":"2023-08-28T02:23:59.316561+0000","last_active":"2023-08-28T02:26:31.307370+0000","last_peered":"2023-08-28T02:26:31.307370+0000","last_clean":"2023-08-28T02:26:31.307370+0000","last_became_active":"2023-08-28T02:23:59.227433+0000","last_became_peered":"2023-08-28T02:23:59.227433+0000","last_unstale":"2023-08-28T02:26:31.307370+0000","last_undegraded":"2023-08-28T02:26:31.307370+0000","last_fullsized":"2023-08-28T02:26:31.307370+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":125,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.138422+0000","last_change":"2023-08-28T02:20:19.851616+0000","last_active":"2023-08-28T02:26:31.138422+0000","last_peered":"2023-08-28T02:26:31.138422+0000","last_clean":"2023-08-28T02:26:31.138422+0000","last_became_active":"2023-08-28T02:20:19.851322+0000","last_became_peered":"2023-08-28T02:20:19.851322+0000","last_unstale":"2023-08-28T02:26:31.138422+0000","last_undegraded":"2023-08-28T02:26:31.138422+0000","last_fullsized":"2023-08-28T02:26:31.138422+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:03:37.565646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"86'22","reported_seq":196,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.142659+0000","last_change":"2023-08-28T02:23:38.368275+0000","last_active":"2023-08-28T02:26:31.142659+0000","last_peered":"2023-08-28T02:26:31.142659+0000","last_clean":"2023-08-28T02:26:31.142659+0000","last_became_active":"2023-08-28T02:23:38.342097+0000","last_became_peered":"2023-08-28T02:23:38.342097+0000","last_unstale":"2023-08-28T02:26:31.142659+0000","last_undegraded":"2023-08-28T02:26:31.142659+0000","last_fullsized":"2023-08-28T02:26:31.142659+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":164,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.141933+0000","last_change":"2023-08-28T02:19:51.204118+0000","last_active":"2023-08-28T02:26:31.141933+0000","last_peered":"2023-08-28T02:26:31.141933+0000","last_clean":"2023-08-28T02:26:31.141933+0000","last_became_active":"2023-08-28T02:19:51.203745+0000","last_became_peered":"2023-08-28T02:19:51.203745+0000","last_unstale":"2023-08-28T02:26:31.141933+0000","last_undegraded":"2023-08-28T02:26:31.141933+0000","last_fullsized":"2023-08-28T02:26:31.141933+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":55,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.308426+0000","last_change":"2023-08-28T02:23:38.983634+0000","last_active":"2023-08-28T02:26:31.308426+0000","last_peered":"2023-08-28T02:26:31.308426+0000","last_clean":"2023-08-28T02:26:31.308426+0000","last_became_active":"2023-08-28T02:23:38.344566+0000","last_became_peered":"2023-08-28T02:23:38.344566+0000","last_unstale":"2023-08-28T02:26:31.308426+0000","last_undegraded":"2023-08-28T02:26:31.308426+0000","last_fullsized":"2023-08-28T02:26:31.308426+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":42,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.296057+0000","last_change":"2023-08-28T02:23:59.216960+0000","last_active":"2023-08-28T02:26:31.296057+0000","last_peered":"2023-08-28T02:26:31.296057+0000","last_clean":"2023-08-28T02:26:31.296057+0000","last_became_active":"2023-08-28T02:23:59.215657+0000","last_became_peered":"2023-08-28T02:23:59.215657+0000","last_unstale":"2023-08-28T02:26:31.296057+0000","last_undegraded":"2023-08-28T02:26:31.296057+0000","last_fullsized":"2023-08-28T02:26:31.296057+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:31:14.407108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":169,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.307052+0000","last_change":"2023-08-28T02:23:59.223066+0000","last_active":"2023-08-28T02:26:31.307052+0000","last_peered":"2023-08-28T02:26:31.307052+0000","last_clean":"2023-08-28T02:26:31.307052+0000","last_became_active":"2023-08-28T02:23:59.222731+0000","last_became_peered":"2023-08-28T02:23:59.222731+0000","last_unstale":"2023-08-28T02:26:31.307052+0000","last_undegraded":"2023-08-28T02:26:31.307052+0000","last_fullsized":"2023-08-28T02:26:31.307052+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:48:37.558266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":211,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.302440+0000","last_change":"2023-08-28T02:23:59.210469+0000","last_active":"2023-08-28T02:26:31.302440+0000","last_peered":"2023-08-28T02:26:31.302440+0000","last_clean":"2023-08-28T02:26:31.302440+0000","last_became_active":"2023-08-28T02:23:59.210031+0000","last_became_peered":"2023-08-28T02:23:59.210031+0000","last_unstale":"2023-08-28T02:26:31.302440+0000","last_undegraded":"2023-08-28T02:26:31.302440+0000","last_fullsized":"2023-08-28T02:26:31.302440+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":49,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.309888+0000","last_change":"2023-08-28T02:23:38.370183+0000","last_active":"2023-08-28T02:26:31.309888+0000","last_peered":"2023-08-28T02:26:31.309888+0000","last_clean":"2023-08-28T02:26:31.309888+0000","last_became_active":"2023-08-28T02:23:38.368415+0000","last_became_peered":"2023-08-28T02:23:38.368415+0000","last_unstale":"2023-08-28T02:26:31.309888+0000","last_undegraded":"2023-08-28T02:26:31.309888+0000","last_fullsized":"2023-08-28T02:26:31.309888+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":170,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.300596+0000","last_change":"2023-08-28T02:23:59.220932+0000","last_active":"2023-08-28T02:26:31.300596+0000","last_peered":"2023-08-28T02:26:31.300596+0000","last_clean":"2023-08-28T02:26:31.300596+0000","last_became_active":"2023-08-28T02:23:59.220488+0000","last_became_peered":"2023-08-28T02:23:59.220488+0000","last_unstale":"2023-08-28T02:26:31.300596+0000","last_undegraded":"2023-08-28T02:26:31.300596+0000","last_fullsized":"2023-08-28T02:26:31.300596+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:53:09.216340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":158,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.264386+0000","last_change":"2023-08-28T02:18:39.778728+0000","last_active":"2023-08-28T02:26:31.264386+0000","last_peered":"2023-08-28T02:26:31.264386+0000","last_clean":"2023-08-28T02:26:31.264386+0000","last_became_active":"2023-08-28T02:18:39.778283+0000","last_became_peered":"2023-08-28T02:18:39.778283+0000","last_unstale":"2023-08-28T02:26:31.264386+0000","last_undegraded":"2023-08-28T02:26:31.264386+0000","last_fullsized":"2023-08-28T02:26:31.264386+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":42,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.296434+0000","last_change":"2023-08-28T02:23:59.216743+0000","last_active":"2023-08-28T02:26:31.296434+0000","last_peered":"2023-08-28T02:26:31.296434+0000","last_clean":"2023-08-28T02:26:31.296434+0000","last_became_active":"2023-08-28T02:23:59.216416+0000","last_became_peered":"2023-08-28T02:23:59.216416+0000","last_unstale":"2023-08-28T02:26:31.296434+0000","last_undegraded":"2023-08-28T02:26:31.296434+0000","last_fullsized":"2023-08-28T02:26:31.296434+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:25:13.056747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"183'13712","reported_seq":14550,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:38.105853+0000","last_change":"2023-08-28T02:24:09.355130+0000","last_active":"2023-08-28T02:26:38.105853+0000","last_peered":"2023-08-28T02:26:38.105853+0000","last_clean":"2023-08-28T02:26:38.105853+0000","last_became_active":"2023-08-28T02:24:09.341363+0000","last_became_peered":"2023-08-28T02:24:09.341363+0000","last_unstale":"2023-08-28T02:26:38.105853+0000","last_undegraded":"2023-08-28T02:26:38.105853+0000","last_fullsized":"2023-08-28T02:26:38.105853+0000","mapping_epoch":161,"log_start":"134'5100","ondisk_log_start":"134'5100","created":119,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":8612,"log_dups_size":0,"ondisk_log_size":8612,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13712,"num_write_kb":438784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":367,"num_bytes_recovered":23789568,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":53,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.310004+0000","last_change":"2023-08-28T02:23:38.657185+0000","last_active":"2023-08-28T02:26:31.310004+0000","last_peered":"2023-08-28T02:26:31.310004+0000","last_clean":"2023-08-28T02:26:31.310004+0000","last_became_active":"2023-08-28T02:23:38.339016+0000","last_became_peered":"2023-08-28T02:23:38.339016+0000","last_unstale":"2023-08-28T02:26:31.310004+0000","last_undegraded":"2023-08-28T02:26:31.310004+0000","last_fullsized":"2023-08-28T02:26:31.310004+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":153,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.302434+0000","last_change":"2023-08-28T02:23:38.339886+0000","last_active":"2023-08-28T02:26:31.302434+0000","last_peered":"2023-08-28T02:26:31.302434+0000","last_clean":"2023-08-28T02:26:31.302434+0000","last_became_active":"2023-08-28T02:23:38.339514+0000","last_became_peered":"2023-08-28T02:23:38.339514+0000","last_unstale":"2023-08-28T02:26:31.302434+0000","last_undegraded":"2023-08-28T02:26:31.302434+0000","last_fullsized":"2023-08-28T02:26:31.302434+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"183'13826","reported_seq":13901,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:38.121649+0000","last_change":"2023-08-28T02:22:00.165260+0000","last_active":"2023-08-28T02:26:38.121649+0000","last_peered":"2023-08-28T02:26:38.121649+0000","last_clean":"2023-08-28T02:26:38.121649+0000","last_became_active":"2023-08-28T02:22:00.164978+0000","last_became_peered":"2023-08-28T02:22:00.164978+0000","last_unstale":"2023-08-28T02:26:38.121649+0000","last_undegraded":"2023-08-28T02:26:38.121649+0000","last_fullsized":"2023-08-28T02:26:38.121649+0000","mapping_epoch":119,"log_start":"133'4700","ondisk_log_start":"133'4700","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":9126,"log_dups_size":0,"ondisk_log_size":9126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:18:41.711960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13826,"num_write_kb":442432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":53,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.309483+0000","last_change":"2023-08-28T02:23:38.555339+0000","last_active":"2023-08-28T02:26:31.309483+0000","last_peered":"2023-08-28T02:26:31.309483+0000","last_clean":"2023-08-28T02:26:31.309483+0000","last_became_active":"2023-08-28T02:23:38.337938+0000","last_became_peered":"2023-08-28T02:23:38.337938+0000","last_unstale":"2023-08-28T02:26:31.309483+0000","last_undegraded":"2023-08-28T02:26:31.309483+0000","last_fullsized":"2023-08-28T02:26:31.309483+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":192,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.141850+0000","last_change":"2023-08-28T02:23:59.208685+0000","last_active":"2023-08-28T02:26:31.141850+0000","last_peered":"2023-08-28T02:26:31.141850+0000","last_clean":"2023-08-28T02:26:31.141850+0000","last_became_active":"2023-08-28T02:23:59.208377+0000","last_became_peered":"2023-08-28T02:23:59.208377+0000","last_unstale":"2023-08-28T02:26:31.141850+0000","last_undegraded":"2023-08-28T02:26:31.141850+0000","last_fullsized":"2023-08-28T02:26:31.141850+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"183'13720","reported_seq":13795,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:38.088470+0000","last_change":"2023-08-28T02:22:00.165317+0000","last_active":"2023-08-28T02:26:38.088470+0000","last_peered":"2023-08-28T02:26:38.088470+0000","last_clean":"2023-08-28T02:26:38.088470+0000","last_became_active":"2023-08-28T02:22:00.164989+0000","last_became_peered":"2023-08-28T02:22:00.164989+0000","last_unstale":"2023-08-28T02:26:38.088470+0000","last_undegraded":"2023-08-28T02:26:38.088470+0000","last_fullsized":"2023-08-28T02:26:38.088470+0000","mapping_epoch":119,"log_start":"133'4600","ondisk_log_start":"133'4600","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":9120,"log_dups_size":0,"ondisk_log_size":9120,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:10:46.322659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13720,"num_write_kb":439040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":53,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.309692+0000","last_change":"2023-08-28T02:23:38.749428+0000","last_active":"2023-08-28T02:26:31.309692+0000","last_peered":"2023-08-28T02:26:31.309692+0000","last_clean":"2023-08-28T02:26:31.309692+0000","last_became_active":"2023-08-28T02:23:38.339587+0000","last_became_peered":"2023-08-28T02:23:38.339587+0000","last_unstale":"2023-08-28T02:26:31.309692+0000","last_undegraded":"2023-08-28T02:26:31.309692+0000","last_fullsized":"2023-08-28T02:26:31.309692+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":210,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.264509+0000","last_change":"2023-08-28T02:23:38.340065+0000","last_active":"2023-08-28T02:26:31.264509+0000","last_peered":"2023-08-28T02:26:31.264509+0000","last_clean":"2023-08-28T02:26:31.264509+0000","last_became_active":"2023-08-28T02:23:38.328689+0000","last_became_peered":"2023-08-28T02:23:38.328689+0000","last_unstale":"2023-08-28T02:26:31.264509+0000","last_undegraded":"2023-08-28T02:26:31.264509+0000","last_fullsized":"2023-08-28T02:26:31.264509+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"183'13766","reported_seq":24286,"reported_epoch":184,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-08-28T02:26:43.898653+0000","last_change":"2023-08-28T02:24:08.250989+0000","last_active":"2023-08-28T02:26:43.898653+0000","last_peered":"2023-08-28T02:26:43.898653+0000","last_clean":"2023-08-28T02:23:58.182539+0000","last_became_active":"2023-08-28T02:24:00.420432+0000","last_became_peered":"2023-08-28T02:24:00.420432+0000","last_unstale":"2023-08-28T02:26:43.898653+0000","last_undegraded":"2023-08-28T02:24:00.242280+0000","last_fullsized":"2023-08-28T02:24:00.199333+0000","mapping_epoch":154,"log_start":"133'4600","ondisk_log_start":"133'4600","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":9166,"log_dups_size":0,"ondisk_log_size":9166,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:57:53.818041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1665,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13766,"num_write_kb":440512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5218,"num_bytes_recovered":267780096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4],"avail_no_missing":["2","0","4"],"object_location_counts":[{"shards":"2,4","objects":-1665}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":169,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.302308+0000","last_change":"2023-08-28T02:23:59.211466+0000","last_active":"2023-08-28T02:26:31.302308+0000","last_peered":"2023-08-28T02:26:31.302308+0000","last_clean":"2023-08-28T02:26:31.302308+0000","last_became_active":"2023-08-28T02:23:59.211175+0000","last_became_peered":"2023-08-28T02:23:59.211175+0000","last_unstale":"2023-08-28T02:26:31.302308+0000","last_undegraded":"2023-08-28T02:26:31.302308+0000","last_fullsized":"2023-08-28T02:26:31.302308+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":145,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.299352+0000","last_change":"2023-08-28T02:19:51.205245+0000","last_active":"2023-08-28T02:26:31.299352+0000","last_peered":"2023-08-28T02:26:31.299352+0000","last_clean":"2023-08-28T02:26:31.299352+0000","last_became_active":"2023-08-28T02:19:51.204896+0000","last_became_peered":"2023-08-28T02:19:51.204896+0000","last_unstale":"2023-08-28T02:26:31.299352+0000","last_undegraded":"2023-08-28T02:26:31.299352+0000","last_fullsized":"2023-08-28T02:26:31.299352+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:51:24.634702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"183'13870","reported_seq":14778,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:38.132506+0000","last_change":"2023-08-28T02:24:04.294764+0000","last_active":"2023-08-28T02:26:38.132506+0000","last_peered":"2023-08-28T02:26:38.132506+0000","last_clean":"2023-08-28T02:26:38.132506+0000","last_became_active":"2023-08-28T02:24:04.280269+0000","last_became_peered":"2023-08-28T02:24:04.280269+0000","last_unstale":"2023-08-28T02:26:38.132506+0000","last_undegraded":"2023-08-28T02:26:38.132506+0000","last_fullsized":"2023-08-28T02:26:38.132506+0000","mapping_epoch":157,"log_start":"133'4700","ondisk_log_start":"133'4700","created":119,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":9170,"log_dups_size":0,"ondisk_log_size":9170,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:18:55.996939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13870,"num_write_kb":443777,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":403,"num_bytes_recovered":26345496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":46,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.132487+0000","last_change":"2023-08-28T02:23:59.230059+0000","last_active":"2023-08-28T02:26:31.132487+0000","last_peered":"2023-08-28T02:26:31.132487+0000","last_clean":"2023-08-28T02:26:31.132487+0000","last_became_active":"2023-08-28T02:23:59.217783+0000","last_became_peered":"2023-08-28T02:23:59.217783+0000","last_unstale":"2023-08-28T02:26:31.132487+0000","last_undegraded":"2023-08-28T02:26:31.132487+0000","last_fullsized":"2023-08-28T02:26:31.132487+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:00:12.145395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":177,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.142004+0000","last_change":"2023-08-28T02:23:59.211286+0000","last_active":"2023-08-28T02:26:31.142004+0000","last_peered":"2023-08-28T02:26:31.142004+0000","last_clean":"2023-08-28T02:26:31.142004+0000","last_became_active":"2023-08-28T02:23:59.210942+0000","last_became_peered":"2023-08-28T02:23:59.210942+0000","last_unstale":"2023-08-28T02:26:31.142004+0000","last_undegraded":"2023-08-28T02:26:31.142004+0000","last_fullsized":"2023-08-28T02:26:31.142004+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"183'13984","reported_seq":14060,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:38.102829+0000","last_change":"2023-08-28T02:22:00.162375+0000","last_active":"2023-08-28T02:26:38.102829+0000","last_peered":"2023-08-28T02:26:38.102829+0000","last_clean":"2023-08-28T02:26:38.102829+0000","last_became_active":"2023-08-28T02:22:00.162190+0000","last_became_peered":"2023-08-28T02:22:00.162190+0000","last_unstale":"2023-08-28T02:26:38.102829+0000","last_undegraded":"2023-08-28T02:26:38.102829+0000","last_fullsized":"2023-08-28T02:26:38.102829+0000","mapping_epoch":119,"log_start":"136'5639","ondisk_log_start":"136'5639","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":8345,"log_dups_size":0,"ondisk_log_size":8345,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:40:08.985631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13984,"num_write_kb":447488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":179,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.264140+0000","last_change":"2023-08-28T02:21:59.179699+0000","last_active":"2023-08-28T02:26:31.264140+0000","last_peered":"2023-08-28T02:26:31.264140+0000","last_clean":"2023-08-28T02:26:31.264140+0000","last_became_active":"2023-08-28T02:21:59.179345+0000","last_became_peered":"2023-08-28T02:21:59.179345+0000","last_unstale":"2023-08-28T02:26:31.264140+0000","last_undegraded":"2023-08-28T02:26:31.264140+0000","last_fullsized":"2023-08-28T02:26:31.264140+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":203,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.142293+0000","last_change":"2023-08-28T02:22:44.083121+0000","last_active":"2023-08-28T02:26:31.142293+0000","last_peered":"2023-08-28T02:26:31.142293+0000","last_clean":"2023-08-28T02:26:31.142293+0000","last_became_active":"2023-08-28T02:22:44.082776+0000","last_became_peered":"2023-08-28T02:22:44.082776+0000","last_unstale":"2023-08-28T02:26:31.142293+0000","last_undegraded":"2023-08-28T02:26:31.142293+0000","last_fullsized":"2023-08-28T02:26:31.142293+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"183'13798","reported_seq":13871,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:38.109798+0000","last_change":"2023-08-28T02:22:00.167630+0000","last_active":"2023-08-28T02:26:38.109798+0000","last_peered":"2023-08-28T02:26:38.109798+0000","last_clean":"2023-08-28T02:26:38.109798+0000","last_became_active":"2023-08-28T02:22:00.167305+0000","last_became_peered":"2023-08-28T02:22:00.167305+0000","last_unstale":"2023-08-28T02:26:38.109798+0000","last_undegraded":"2023-08-28T02:26:38.109798+0000","last_fullsized":"2023-08-28T02:26:38.109798+0000","mapping_epoch":119,"log_start":"133'4700","ondisk_log_start":"133'4700","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":9098,"log_dups_size":0,"ondisk_log_size":9098,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13798,"num_write_kb":441536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":93,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.141938+0000","last_change":"2023-08-28T02:23:38.347043+0000","last_active":"2023-08-28T02:26:31.141938+0000","last_peered":"2023-08-28T02:26:31.141938+0000","last_clean":"2023-08-28T02:26:31.141938+0000","last_became_active":"2023-08-28T02:23:38.346680+0000","last_became_peered":"2023-08-28T02:23:38.346680+0000","last_unstale":"2023-08-28T02:26:31.141938+0000","last_undegraded":"2023-08-28T02:26:31.141938+0000","last_fullsized":"2023-08-28T02:26:31.141938+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":114,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.300923+0000","last_change":"2023-08-28T02:19:51.204017+0000","last_active":"2023-08-28T02:26:31.300923+0000","last_peered":"2023-08-28T02:26:31.300923+0000","last_clean":"2023-08-28T02:26:31.300923+0000","last_became_active":"2023-08-28T02:19:51.203627+0000","last_became_peered":"2023-08-28T02:19:51.203627+0000","last_unstale":"2023-08-28T02:26:31.300923+0000","last_undegraded":"2023-08-28T02:26:31.300923+0000","last_fullsized":"2023-08-28T02:26:31.300923+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:14:06.231701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.e","version":"183'13976","reported_seq":14049,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:38.097861+0000","last_change":"2023-08-28T02:22:00.166490+0000","last_active":"2023-08-28T02:26:38.097861+0000","last_peered":"2023-08-28T02:26:38.097861+0000","last_clean":"2023-08-28T02:26:38.097861+0000","last_became_active":"2023-08-28T02:22:00.166140+0000","last_became_peered":"2023-08-28T02:22:00.166140+0000","last_unstale":"2023-08-28T02:26:38.097861+0000","last_undegraded":"2023-08-28T02:26:38.097861+0000","last_fullsized":"2023-08-28T02:26:38.097861+0000","mapping_epoch":119,"log_start":"136'6000","ondisk_log_start":"136'6000","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":7976,"log_dups_size":0,"ondisk_log_size":7976,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13976,"num_write_kb":447232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":42,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.132168+0000","last_change":"2023-08-28T02:23:59.216950+0000","last_active":"2023-08-28T02:26:31.132168+0000","last_peered":"2023-08-28T02:26:31.132168+0000","last_clean":"2023-08-28T02:26:31.132168+0000","last_became_active":"2023-08-28T02:23:59.216373+0000","last_became_peered":"2023-08-28T02:23:59.216373+0000","last_unstale":"2023-08-28T02:26:31.132168+0000","last_undegraded":"2023-08-28T02:26:31.132168+0000","last_fullsized":"2023-08-28T02:26:31.132168+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:13:14.736771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":162,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:33.467552+0000","last_change":"2023-08-28T02:23:59.220269+0000","last_active":"2023-08-28T02:26:33.467552+0000","last_peered":"2023-08-28T02:26:33.467552+0000","last_clean":"2023-08-28T02:26:33.467552+0000","last_became_active":"2023-08-28T02:23:59.219966+0000","last_became_peered":"2023-08-28T02:23:59.219966+0000","last_unstale":"2023-08-28T02:26:33.467552+0000","last_undegraded":"2023-08-28T02:26:33.467552+0000","last_fullsized":"2023-08-28T02:26:33.467552+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:16:08.119909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"78'126","reported_seq":661,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.362112+0000","last_change":"2023-08-28T02:23:39.351975+0000","last_active":"2023-08-28T02:26:31.362112+0000","last_peered":"2023-08-28T02:26:31.362112+0000","last_clean":"2023-08-28T02:26:31.362112+0000","last_became_active":"2023-08-28T02:23:39.351437+0000","last_became_peered":"2023-08-28T02:23:39.351437+0000","last_unstale":"2023-08-28T02:26:31.362112+0000","last_undegraded":"2023-08-28T02:26:31.362112+0000","last_fullsized":"2023-08-28T02:26:31.362112+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:25:22.203120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":743,"num_read_kb":1331,"num_write":125,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":184,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.302329+0000","last_change":"2023-08-28T02:23:59.207148+0000","last_active":"2023-08-28T02:26:31.302329+0000","last_peered":"2023-08-28T02:26:31.302329+0000","last_clean":"2023-08-28T02:26:31.302329+0000","last_became_active":"2023-08-28T02:23:59.206802+0000","last_became_peered":"2023-08-28T02:23:59.206802+0000","last_unstale":"2023-08-28T02:26:31.302329+0000","last_undegraded":"2023-08-28T02:26:31.302329+0000","last_fullsized":"2023-08-28T02:26:31.302329+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":81,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.358074+0000","last_change":"2023-08-28T02:21:43.238044+0000","last_active":"2023-08-28T02:26:31.358074+0000","last_peered":"2023-08-28T02:26:31.358074+0000","last_clean":"2023-08-28T02:26:31.358074+0000","last_became_active":"2023-08-28T02:21:43.237792+0000","last_became_peered":"2023-08-28T02:21:43.237792+0000","last_unstale":"2023-08-28T02:26:31.358074+0000","last_undegraded":"2023-08-28T02:26:31.358074+0000","last_fullsized":"2023-08-28T02:26:31.358074+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:11:35.488503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"183'13650","reported_seq":13726,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:38.082145+0000","last_change":"2023-08-28T02:22:00.166578+0000","last_active":"2023-08-28T02:26:38.082145+0000","last_peered":"2023-08-28T02:26:38.082145+0000","last_clean":"2023-08-28T02:26:38.082145+0000","last_became_active":"2023-08-28T02:22:00.166311+0000","last_became_peered":"2023-08-28T02:22:00.166311+0000","last_unstale":"2023-08-28T02:26:38.082145+0000","last_undegraded":"2023-08-28T02:26:38.082145+0000","last_fullsized":"2023-08-28T02:26:38.082145+0000","mapping_epoch":119,"log_start":"136'5941","ondisk_log_start":"136'5941","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":7709,"log_dups_size":0,"ondisk_log_size":7709,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:40:38.502180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13650,"num_write_kb":436800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"183'13746","reported_seq":13822,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:38.116544+0000","last_change":"2023-08-28T02:22:00.160991+0000","last_active":"2023-08-28T02:26:38.116544+0000","last_peered":"2023-08-28T02:26:38.116544+0000","last_clean":"2023-08-28T02:26:38.116544+0000","last_became_active":"2023-08-28T02:22:00.160755+0000","last_became_peered":"2023-08-28T02:22:00.160755+0000","last_unstale":"2023-08-28T02:26:38.116544+0000","last_undegraded":"2023-08-28T02:26:38.116544+0000","last_fullsized":"2023-08-28T02:26:38.116544+0000","mapping_epoch":119,"log_start":"136'5998","ondisk_log_start":"136'5998","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":7748,"log_dups_size":0,"ondisk_log_size":7748,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:42:59.109288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13746,"num_write_kb":439872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"78'204","reported_seq":1100,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.302523+0000","last_change":"2023-08-28T02:19:50.219949+0000","last_active":"2023-08-28T02:26:31.302523+0000","last_peered":"2023-08-28T02:26:31.302523+0000","last_clean":"2023-08-28T02:26:31.302523+0000","last_became_active":"2023-08-28T02:19:50.219356+0000","last_became_peered":"2023-08-28T02:19:50.219356+0000","last_unstale":"2023-08-28T02:26:31.302523+0000","last_undegraded":"2023-08-28T02:26:31.302523+0000","last_fullsized":"2023-08-28T02:26:31.302523+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":204,"log_dups_size":0,"ondisk_log_size":204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1521,"num_read_kb":2511,"num_write":178,"num_write_kb":1175,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":151,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.142055+0000","last_change":"2023-08-28T02:18:39.779607+0000","last_active":"2023-08-28T02:26:31.142055+0000","last_peered":"2023-08-28T02:26:31.142055+0000","last_clean":"2023-08-28T02:26:31.142055+0000","last_became_active":"2023-08-28T02:18:39.779303+0000","last_became_peered":"2023-08-28T02:18:39.779303+0000","last_unstale":"2023-08-28T02:26:31.142055+0000","last_undegraded":"2023-08-28T02:26:31.142055+0000","last_fullsized":"2023-08-28T02:26:31.142055+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":142,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.362064+0000","last_change":"2023-08-28T02:23:59.221979+0000","last_active":"2023-08-28T02:26:31.362064+0000","last_peered":"2023-08-28T02:26:31.362064+0000","last_clean":"2023-08-28T02:26:31.362064+0000","last_became_active":"2023-08-28T02:23:59.221619+0000","last_became_peered":"2023-08-28T02:23:59.221619+0000","last_unstale":"2023-08-28T02:26:31.362064+0000","last_undegraded":"2023-08-28T02:26:31.362064+0000","last_fullsized":"2023-08-28T02:26:31.362064+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:54:45.892284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.6","version":"183'13516","reported_seq":14305,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:38.118002+0000","last_change":"2023-08-28T02:24:07.218778+0000","last_active":"2023-08-28T02:26:38.118002+0000","last_peered":"2023-08-28T02:26:38.118002+0000","last_clean":"2023-08-28T02:26:38.118002+0000","last_became_active":"2023-08-28T02:24:07.205246+0000","last_became_peered":"2023-08-28T02:24:07.205246+0000","last_unstale":"2023-08-28T02:26:38.118002+0000","last_undegraded":"2023-08-28T02:26:38.118002+0000","last_fullsized":"2023-08-28T02:26:38.118002+0000","mapping_epoch":159,"log_start":"132'4400","ondisk_log_start":"132'4400","created":119,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":9116,"log_dups_size":0,"ondisk_log_size":9116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:30:08.510532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13516,"num_write_kb":432512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":342,"num_bytes_recovered":22413312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"78'47","reported_seq":399,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:33.902846+0000","last_change":"2023-08-28T02:24:05.799647+0000","last_active":"2023-08-28T02:26:33.902846+0000","last_peered":"2023-08-28T02:26:33.902846+0000","last_clean":"2023-08-28T02:26:33.902846+0000","last_became_active":"2023-08-28T02:23:59.210481+0000","last_became_peered":"2023-08-28T02:23:59.210481+0000","last_unstale":"2023-08-28T02:26:33.902846+0000","last_undegraded":"2023-08-28T02:26:33.902846+0000","last_fullsized":"2023-08-28T02:26:33.902846+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":47,"log_dups_size":0,"ondisk_log_size":47,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:40:18.542375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":162,"num_read_kb":1281,"num_write":48,"num_write_kb":1153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":125,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.138528+0000","last_change":"2023-08-28T02:20:19.851591+0000","last_active":"2023-08-28T02:26:31.138528+0000","last_peered":"2023-08-28T02:26:31.138528+0000","last_clean":"2023-08-28T02:26:31.138528+0000","last_became_active":"2023-08-28T02:20:19.851291+0000","last_became_peered":"2023-08-28T02:20:19.851291+0000","last_unstale":"2023-08-28T02:26:31.138528+0000","last_undegraded":"2023-08-28T02:26:31.138528+0000","last_fullsized":"2023-08-28T02:26:31.138528+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:33:51.259707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":157,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.358704+0000","last_change":"2023-08-28T02:19:50.211156+0000","last_active":"2023-08-28T02:26:31.358704+0000","last_peered":"2023-08-28T02:26:31.358704+0000","last_clean":"2023-08-28T02:26:31.358704+0000","last_became_active":"2023-08-28T02:19:50.210398+0000","last_became_peered":"2023-08-28T02:19:50.210398+0000","last_unstale":"2023-08-28T02:26:31.358704+0000","last_undegraded":"2023-08-28T02:26:31.358704+0000","last_fullsized":"2023-08-28T02:26:31.358704+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:14:41.748197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"78'114","reported_seq":548,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.302692+0000","last_change":"2023-08-28T02:24:01.210739+0000","last_active":"2023-08-28T02:26:31.302692+0000","last_peered":"2023-08-28T02:26:31.302692+0000","last_clean":"2023-08-28T02:26:31.302692+0000","last_became_active":"2023-08-28T02:24:01.210386+0000","last_became_peered":"2023-08-28T02:24:01.210386+0000","last_unstale":"2023-08-28T02:26:31.302692+0000","last_undegraded":"2023-08-28T02:26:31.302692+0000","last_fullsized":"2023-08-28T02:26:31.302692+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":114,"log_dups_size":0,"ondisk_log_size":114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":358,"num_read_kb":1352,"num_write":102,"num_write_kb":1104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"86'139","reported_seq":342,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.308757+0000","last_change":"2023-08-28T02:24:04.276519+0000","last_active":"2023-08-28T02:26:31.308757+0000","last_peered":"2023-08-28T02:26:31.308757+0000","last_clean":"2023-08-28T02:26:31.308757+0000","last_became_active":"2023-08-28T02:24:04.276104+0000","last_became_peered":"2023-08-28T02:24:04.276104+0000","last_unstale":"2023-08-28T02:26:31.308757+0000","last_undegraded":"2023-08-28T02:26:31.308757+0000","last_fullsized":"2023-08-28T02:26:31.308757+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_clean_scrub_stamp":"2023-08-28T02:08:02.043718+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":42,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.296912+0000","last_change":"2023-08-28T02:23:59.219981+0000","last_active":"2023-08-28T02:26:31.296912+0000","last_peered":"2023-08-28T02:26:31.296912+0000","last_clean":"2023-08-28T02:26:31.296912+0000","last_became_active":"2023-08-28T02:23:59.219633+0000","last_became_peered":"2023-08-28T02:23:59.219633+0000","last_unstale":"2023-08-28T02:26:31.296912+0000","last_undegraded":"2023-08-28T02:26:31.296912+0000","last_fullsized":"2023-08-28T02:26:31.296912+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:56:14.172518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":152,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.264687+0000","last_change":"2023-08-28T02:23:38.341351+0000","last_active":"2023-08-28T02:26:31.264687+0000","last_peered":"2023-08-28T02:26:31.264687+0000","last_clean":"2023-08-28T02:26:31.264687+0000","last_became_active":"2023-08-28T02:23:38.340997+0000","last_became_peered":"2023-08-28T02:23:38.340997+0000","last_unstale":"2023-08-28T02:26:31.264687+0000","last_undegraded":"2023-08-28T02:26:31.264687+0000","last_fullsized":"2023-08-28T02:26:31.264687+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"183'13656","reported_seq":14497,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:38.113472+0000","last_change":"2023-08-28T02:23:47.215362+0000","last_active":"2023-08-28T02:26:38.113472+0000","last_peered":"2023-08-28T02:26:38.113472+0000","last_clean":"2023-08-28T02:26:38.113472+0000","last_became_active":"2023-08-28T02:23:47.207109+0000","last_became_peered":"2023-08-28T02:23:47.207109+0000","last_unstale":"2023-08-28T02:26:38.113472+0000","last_undegraded":"2023-08-28T02:26:38.113472+0000","last_fullsized":"2023-08-28T02:26:38.113472+0000","mapping_epoch":150,"log_start":"134'5000","ondisk_log_start":"134'5000","created":119,"last_epoch_clean":151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":8656,"log_dups_size":0,"ondisk_log_size":8656,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13656,"num_write_kb":436992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":370,"num_bytes_recovered":24182784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.7","version":"183'13854","reported_seq":13927,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:38.090717+0000","last_change":"2023-08-28T02:22:00.165614+0000","last_active":"2023-08-28T02:26:38.090717+0000","last_peered":"2023-08-28T02:26:38.090717+0000","last_clean":"2023-08-28T02:26:38.090717+0000","last_became_active":"2023-08-28T02:22:00.165136+0000","last_became_peered":"2023-08-28T02:22:00.165136+0000","last_unstale":"2023-08-28T02:26:38.090717+0000","last_undegraded":"2023-08-28T02:26:38.090717+0000","last_fullsized":"2023-08-28T02:26:38.090717+0000","mapping_epoch":119,"log_start":"133'4700","ondisk_log_start":"133'4700","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":9154,"log_dups_size":0,"ondisk_log_size":9154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13854,"num_write_kb":443328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"78'148","reported_seq":633,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.308674+0000","last_change":"2023-08-28T02:24:01.211494+0000","last_active":"2023-08-28T02:26:31.308674+0000","last_peered":"2023-08-28T02:26:31.308674+0000","last_clean":"2023-08-28T02:26:31.308674+0000","last_became_active":"2023-08-28T02:24:01.210957+0000","last_became_peered":"2023-08-28T02:24:01.210957+0000","last_unstale":"2023-08-28T02:26:31.308674+0000","last_undegraded":"2023-08-28T02:26:31.308674+0000","last_fullsized":"2023-08-28T02:26:31.308674+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1092,"num_read_kb":1955,"num_write":127,"num_write_kb":5177,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":151,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.298132+0000","last_change":"2023-08-28T02:23:59.226986+0000","last_active":"2023-08-28T02:26:31.298132+0000","last_peered":"2023-08-28T02:26:31.298132+0000","last_clean":"2023-08-28T02:26:31.298132+0000","last_became_active":"2023-08-28T02:23:59.226529+0000","last_became_peered":"2023-08-28T02:23:59.226529+0000","last_unstale":"2023-08-28T02:26:31.298132+0000","last_undegraded":"2023-08-28T02:26:31.298132+0000","last_fullsized":"2023-08-28T02:26:31.298132+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:22:09.193619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":178,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.300684+0000","last_change":"2023-08-28T02:21:59.167858+0000","last_active":"2023-08-28T02:26:31.300684+0000","last_peered":"2023-08-28T02:26:31.300684+0000","last_clean":"2023-08-28T02:26:31.300684+0000","last_became_active":"2023-08-28T02:21:59.167436+0000","last_became_peered":"2023-08-28T02:21:59.167436+0000","last_unstale":"2023-08-28T02:26:31.300684+0000","last_undegraded":"2023-08-28T02:26:31.300684+0000","last_fullsized":"2023-08-28T02:26:31.300684+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:17:50.717282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"78'103","reported_seq":560,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.307506+0000","last_change":"2023-08-28T02:24:07.212128+0000","last_active":"2023-08-28T02:26:31.307506+0000","last_peered":"2023-08-28T02:26:31.307506+0000","last_clean":"2023-08-28T02:26:31.307506+0000","last_became_active":"2023-08-28T02:24:07.211562+0000","last_became_peered":"2023-08-28T02:24:07.211562+0000","last_unstale":"2023-08-28T02:26:31.307506+0000","last_undegraded":"2023-08-28T02:26:31.307506+0000","last_fullsized":"2023-08-28T02:26:31.307506+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":803,"num_read_kb":2080,"num_write":73,"num_write_kb":1244,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":125,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.362204+0000","last_change":"2023-08-28T02:20:19.850916+0000","last_active":"2023-08-28T02:26:31.362204+0000","last_peered":"2023-08-28T02:26:31.362204+0000","last_clean":"2023-08-28T02:26:31.362204+0000","last_became_active":"2023-08-28T02:20:19.850627+0000","last_became_peered":"2023-08-28T02:20:19.850627+0000","last_unstale":"2023-08-28T02:26:31.362204+0000","last_undegraded":"2023-08-28T02:26:31.362204+0000","last_fullsized":"2023-08-28T02:26:31.362204+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:19:14.346189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":137,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.299585+0000","last_change":"2023-08-28T02:23:38.329667+0000","last_active":"2023-08-28T02:26:31.299585+0000","last_peered":"2023-08-28T02:26:31.299585+0000","last_clean":"2023-08-28T02:26:31.299585+0000","last_became_active":"2023-08-28T02:23:38.329175+0000","last_became_peered":"2023-08-28T02:23:38.329175+0000","last_unstale":"2023-08-28T02:26:31.299585+0000","last_undegraded":"2023-08-28T02:26:31.299585+0000","last_fullsized":"2023-08-28T02:26:31.299585+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:21:34.815067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"183'13566","reported_seq":13638,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:38.022776+0000","last_change":"2023-08-28T02:22:00.167814+0000","last_active":"2023-08-28T02:26:38.022776+0000","last_peered":"2023-08-28T02:26:38.022776+0000","last_clean":"2023-08-28T02:26:38.022776+0000","last_became_active":"2023-08-28T02:22:00.167513+0000","last_became_peered":"2023-08-28T02:22:00.167513+0000","last_unstale":"2023-08-28T02:26:38.022776+0000","last_undegraded":"2023-08-28T02:26:38.022776+0000","last_fullsized":"2023-08-28T02:26:38.022776+0000","mapping_epoch":119,"log_start":"129'3800","ondisk_log_start":"129'3800","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":9766,"log_dups_size":0,"ondisk_log_size":9766,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13566,"num_write_kb":434112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":46,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.296565+0000","last_change":"2023-08-28T02:23:59.232099+0000","last_active":"2023-08-28T02:26:31.296565+0000","last_peered":"2023-08-28T02:26:31.296565+0000","last_clean":"2023-08-28T02:26:31.296565+0000","last_became_active":"2023-08-28T02:23:59.219318+0000","last_became_peered":"2023-08-28T02:23:59.219318+0000","last_unstale":"2023-08-28T02:26:31.296565+0000","last_undegraded":"2023-08-28T02:26:31.296565+0000","last_fullsized":"2023-08-28T02:26:31.296565+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:59:30.093289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'112","reported_seq":429,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.358376+0000","last_change":"2023-08-28T02:19:50.218206+0000","last_active":"2023-08-28T02:26:31.358376+0000","last_peered":"2023-08-28T02:26:31.358376+0000","last_clean":"2023-08-28T02:26:31.358376+0000","last_became_active":"2023-08-28T02:19:50.217637+0000","last_became_peered":"2023-08-28T02:19:50.217637+0000","last_unstale":"2023-08-28T02:26:31.358376+0000","last_undegraded":"2023-08-28T02:26:31.358376+0000","last_fullsized":"2023-08-28T02:26:31.358376+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:12:06.224907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":444,"num_read_kb":473,"num_write":94,"num_write_kb":127,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":42,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.132195+0000","last_change":"2023-08-28T02:23:59.221034+0000","last_active":"2023-08-28T02:26:31.132195+0000","last_peered":"2023-08-28T02:26:31.132195+0000","last_clean":"2023-08-28T02:26:31.132195+0000","last_became_active":"2023-08-28T02:23:59.220730+0000","last_became_peered":"2023-08-28T02:23:59.220730+0000","last_unstale":"2023-08-28T02:26:31.132195+0000","last_undegraded":"2023-08-28T02:26:31.132195+0000","last_fullsized":"2023-08-28T02:26:31.132195+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:25:52.364370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"183'13890","reported_seq":17058,"reported_epoch":183,"state":"active+remapped+backfill_wait","last_fresh":"2023-08-28T02:26:38.059623+0000","last_change":"2023-08-28T02:24:05.805308+0000","last_active":"2023-08-28T02:26:38.059623+0000","last_peered":"2023-08-28T02:26:38.059623+0000","last_clean":"2023-08-28T02:23:25.043301+0000","last_became_active":"2023-08-28T02:24:00.316639+0000","last_became_peered":"2023-08-28T02:24:00.316639+0000","last_unstale":"2023-08-28T02:26:38.059623+0000","last_undegraded":"2023-08-28T02:26:38.059623+0000","last_fullsized":"2023-08-28T02:26:38.059623+0000","mapping_epoch":154,"log_start":"136'5900","ondisk_log_start":"136'5900","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":7990,"log_dups_size":0,"ondisk_log_size":7990,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13890,"num_write_kb":444480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1527,"num_bytes_recovered":99942400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,7,0],"avail_no_missing":["4","0","3","5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"78'122","reported_seq":713,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.264781+0000","last_change":"2023-08-28T02:24:04.276850+0000","last_active":"2023-08-28T02:26:31.264781+0000","last_peered":"2023-08-28T02:26:31.264781+0000","last_clean":"2023-08-28T02:26:31.264781+0000","last_became_active":"2023-08-28T02:24:04.276367+0000","last_became_peered":"2023-08-28T02:24:04.276367+0000","last_unstale":"2023-08-28T02:26:31.264781+0000","last_undegraded":"2023-08-28T02:26:31.264781+0000","last_fullsized":"2023-08-28T02:26:31.264781+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":857,"num_read_kb":2224,"num_write":99,"num_write_kb":2312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":137,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.300768+0000","last_change":"2023-08-28T02:19:50.221001+0000","last_active":"2023-08-28T02:26:31.300768+0000","last_peered":"2023-08-28T02:26:31.300768+0000","last_clean":"2023-08-28T02:26:31.300768+0000","last_became_active":"2023-08-28T02:19:50.220611+0000","last_became_peered":"2023-08-28T02:19:50.220611+0000","last_unstale":"2023-08-28T02:26:31.300768+0000","last_undegraded":"2023-08-28T02:26:31.300768+0000","last_fullsized":"2023-08-28T02:26:31.300768+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:23:10.637630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":135,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.141767+0000","last_change":"2023-08-28T02:11:22.437146+0000","last_active":"2023-08-28T02:26:31.141767+0000","last_peered":"2023-08-28T02:26:31.141767+0000","last_clean":"2023-08-28T02:26:31.141767+0000","last_became_active":"2023-08-28T02:11:22.436798+0000","last_became_peered":"2023-08-28T02:11:22.436798+0000","last_unstale":"2023-08-28T02:26:31.141767+0000","last_undegraded":"2023-08-28T02:26:31.141767+0000","last_fullsized":"2023-08-28T02:26:31.141767+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"183'13752","reported_seq":13834,"reported_epoch":183,"state":"active+remapped+backfill_wait","last_fresh":"2023-08-28T02:26:38.126970+0000","last_change":"2023-08-28T02:24:00.310494+0000","last_active":"2023-08-28T02:26:38.126970+0000","last_peered":"2023-08-28T02:26:38.126970+0000","last_clean":"2023-08-28T02:23:58.174026+0000","last_became_active":"2023-08-28T02:24:00.290543+0000","last_became_peered":"2023-08-28T02:24:00.290543+0000","last_unstale":"2023-08-28T02:26:38.126970+0000","last_undegraded":"2023-08-28T02:26:38.126970+0000","last_fullsized":"2023-08-28T02:26:38.126970+0000","mapping_epoch":154,"log_start":"136'5977","ondisk_log_start":"136'5977","created":119,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:21:59.147188+0000","last_clean_scrub_stamp":"2023-08-28T02:21:59.147188+0000","objects_scrubbed":0,"log_size":7775,"log_dups_size":0,"ondisk_log_size":7775,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:35:56.609960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13752,"num_write_kb":440064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[0,1,4],"avail_no_missing":["0","1","3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":140,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.299511+0000","last_change":"2023-08-28T02:23:38.329474+0000","last_active":"2023-08-28T02:26:31.299511+0000","last_peered":"2023-08-28T02:26:31.299511+0000","last_clean":"2023-08-28T02:26:31.299511+0000","last_became_active":"2023-08-28T02:23:38.328505+0000","last_became_peered":"2023-08-28T02:23:38.328505+0000","last_unstale":"2023-08-28T02:26:31.299511+0000","last_undegraded":"2023-08-28T02:26:31.299511+0000","last_fullsized":"2023-08-28T02:26:31.299511+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:12:14.835809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":205,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.345566+0000","last_change":"2023-08-28T02:23:59.212036+0000","last_active":"2023-08-28T02:26:31.345566+0000","last_peered":"2023-08-28T02:26:31.345566+0000","last_clean":"2023-08-28T02:26:31.345566+0000","last_became_active":"2023-08-28T02:23:59.211709+0000","last_became_peered":"2023-08-28T02:23:59.211709+0000","last_unstale":"2023-08-28T02:26:31.345566+0000","last_undegraded":"2023-08-28T02:26:31.345566+0000","last_fullsized":"2023-08-28T02:26:31.345566+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T14:04:25.046626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":153,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.302845+0000","last_change":"2023-08-28T02:19:50.218760+0000","last_active":"2023-08-28T02:26:31.302845+0000","last_peered":"2023-08-28T02:26:31.302845+0000","last_clean":"2023-08-28T02:26:31.302845+0000","last_became_active":"2023-08-28T02:19:50.218376+0000","last_became_peered":"2023-08-28T02:19:50.218376+0000","last_unstale":"2023-08-28T02:26:31.302845+0000","last_undegraded":"2023-08-28T02:26:31.302845+0000","last_fullsized":"2023-08-28T02:26:31.302845+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":183,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.142144+0000","last_change":"2023-08-28T02:23:59.207822+0000","last_active":"2023-08-28T02:26:31.142144+0000","last_peered":"2023-08-28T02:26:31.142144+0000","last_clean":"2023-08-28T02:26:31.142144+0000","last_became_active":"2023-08-28T02:23:59.207475+0000","last_became_peered":"2023-08-28T02:23:59.207475+0000","last_unstale":"2023-08-28T02:26:31.142144+0000","last_undegraded":"2023-08-28T02:26:31.142144+0000","last_fullsized":"2023-08-28T02:26:31.142144+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":138,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.303056+0000","last_change":"2023-08-28T02:11:22.186024+0000","last_active":"2023-08-28T02:26:31.303056+0000","last_peered":"2023-08-28T02:26:31.303056+0000","last_clean":"2023-08-28T02:26:31.303056+0000","last_became_active":"2023-08-28T02:11:22.185500+0000","last_became_peered":"2023-08-28T02:11:22.185500+0000","last_unstale":"2023-08-28T02:26:31.303056+0000","last_undegraded":"2023-08-28T02:26:31.303056+0000","last_fullsized":"2023-08-28T02:26:31.303056+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":183,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.142062+0000","last_change":"2023-08-28T02:23:59.207929+0000","last_active":"2023-08-28T02:26:31.142062+0000","last_peered":"2023-08-28T02:26:31.142062+0000","last_clean":"2023-08-28T02:26:31.142062+0000","last_became_active":"2023-08-28T02:23:59.207589+0000","last_became_peered":"2023-08-28T02:23:59.207589+0000","last_unstale":"2023-08-28T02:26:31.142062+0000","last_undegraded":"2023-08-28T02:26:31.142062+0000","last_fullsized":"2023-08-28T02:26:31.142062+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":42,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.296193+0000","last_change":"2023-08-28T02:23:59.220437+0000","last_active":"2023-08-28T02:26:31.296193+0000","last_peered":"2023-08-28T02:26:31.296193+0000","last_clean":"2023-08-28T02:26:31.296193+0000","last_became_active":"2023-08-28T02:23:59.220091+0000","last_became_peered":"2023-08-28T02:23:59.220091+0000","last_unstale":"2023-08-28T02:26:31.296193+0000","last_undegraded":"2023-08-28T02:26:31.296193+0000","last_fullsized":"2023-08-28T02:26:31.296193+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:51:21.695566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":42,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.132427+0000","last_change":"2023-08-28T02:23:59.217794+0000","last_active":"2023-08-28T02:26:31.132427+0000","last_peered":"2023-08-28T02:26:31.132427+0000","last_clean":"2023-08-28T02:26:31.132427+0000","last_became_active":"2023-08-28T02:23:59.217231+0000","last_became_peered":"2023-08-28T02:23:59.217231+0000","last_unstale":"2023-08-28T02:26:31.132427+0000","last_undegraded":"2023-08-28T02:26:31.132427+0000","last_fullsized":"2023-08-28T02:26:31.132427+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:36:26.297410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":137,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.300829+0000","last_change":"2023-08-28T02:20:19.850252+0000","last_active":"2023-08-28T02:26:31.300829+0000","last_peered":"2023-08-28T02:26:31.300829+0000","last_clean":"2023-08-28T02:26:31.300829+0000","last_became_active":"2023-08-28T02:20:19.849913+0000","last_became_peered":"2023-08-28T02:20:19.849913+0000","last_unstale":"2023-08-28T02:26:31.300829+0000","last_undegraded":"2023-08-28T02:26:31.300829+0000","last_fullsized":"2023-08-28T02:26:31.300829+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:48:47.713465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":179,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-08-28T02:26:31.141500+0000","last_change":"2023-08-28T02:21:59.180201+0000","last_active":"2023-08-28T02:26:31.141500+0000","last_peered":"2023-08-28T02:26:31.141500+0000","last_clean":"2023-08-28T02:26:31.141500+0000","last_became_active":"2023-08-28T02:21:59.179863+0000","last_became_peered":"2023-08-28T02:21:59.179863+0000","last_unstale":"2023-08-28T02:26:31.141500+0000","last_undegraded":"2023-08-28T02:26:31.141500+0000","last_fullsized":"2023-08-28T02:26:31.141500+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1665,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":220282,"num_write_kb":7048961,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8227,"num_bytes_recovered":464453656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":439746560,"data_stored":439746560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":138527,"ondisk_log_size":138527,"up":48,"acting":47,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":9774,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":21246,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26810,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430413,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":303660,"kb_used_data":6436,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":93441492,"statfs":{"total":95995035648,"available":95684087808,"internally_reserved":0,"allocated":6590464,"data_stored":6010975,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:25:56 2023","interfaces":[{"interface":"back","average":{"1min":2.536,"5min":3.7240000000000002,"15min":1.5980000000000001},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":9.5679999999999996,"5min":116.755,"15min":116.755},"last":2.806},{"interface":"front","average":{"1min":3.1320000000000001,"5min":4.1520000000000001,"15min":1.7889999999999999},"min":{"1min":0.28000000000000003,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":12.206,"5min":116.971,"15min":116.971},"last":1.153}]},{"osd":1,"last update":"Mon Aug 28 02:26:03 2023","interfaces":[{"interface":"back","average":{"1min":1.5800000000000001,"5min":1.143,"15min":0.73299999999999998},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":9.0139999999999993,"5min":9.8710000000000004,"15min":9.8710000000000004},"last":3.9550000000000001},{"interface":"front","average":{"1min":3.0169999999999999,"5min":1.7889999999999999,"15min":0.98799999999999999},"min":{"1min":0.27700000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":11.140000000000001,"5min":11.696999999999999,"15min":11.696999999999999},"last":4.556}]},{"osd":2,"last update":"Mon Aug 28 02:26:28 2023","interfaces":[{"interface":"back","average":{"1min":2.5600000000000001,"5min":4.2670000000000003,"15min":1.8049999999999999},"min":{"1min":0.29499999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":8.4540000000000006,"5min":119.571,"15min":119.571},"last":12.558999999999999},{"interface":"front","average":{"1min":5.4619999999999997,"5min":5.2939999999999996,"15min":2.1549999999999998},"min":{"1min":0.35799999999999998,"5min":0.312,"15min":0.312},"max":{"1min":54.191000000000003,"5min":119.554,"15min":119.554},"last":15.951000000000001}]},{"osd":3,"last update":"Mon Aug 28 02:26:03 2023","interfaces":[{"interface":"back","average":{"1min":2.2250000000000001,"5min":2.3260000000000001,"15min":1.2210000000000001},"min":{"1min":0.36799999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":6.5650000000000004,"5min":107.916,"15min":107.916},"last":1.7350000000000001},{"interface":"front","average":{"1min":2.548,"5min":2.9990000000000001,"15min":1.446},"min":{"1min":0.378,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":9.8460000000000001,"5min":112.00700000000001,"15min":112.00700000000001},"last":4.1299999999999999}]},{"osd":4,"last update":"Mon Aug 28 02:26:28 2023","interfaces":[{"interface":"back","average":{"1min":2.9569999999999999,"5min":2.5219999999999998,"15min":1.163},"min":{"1min":0.32100000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":11.696,"5min":63.048000000000002,"15min":63.048000000000002},"last":0.32900000000000001},{"interface":"front","average":{"1min":2.4009999999999998,"5min":2.2559999999999998,"15min":1.0900000000000001},"min":{"1min":0.28899999999999998,"5min":0.247,"15min":0.247},"max":{"1min":10.840999999999999,"5min":62.912999999999997,"15min":62.912999999999997},"last":2.024}]},{"osd":5,"last update":"Mon Aug 28 02:26:28 2023","interfaces":[{"interface":"back","average":{"1min":2.3809999999999998,"5min":1.919,"15min":0.96699999999999997},"min":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":13.788,"5min":63.021000000000001,"15min":63.021000000000001},"last":0.39200000000000002},{"interface":"front","average":{"1min":2.855,"5min":2.6259999999999999,"15min":1.244},"min":{"1min":0.36299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":7.4690000000000003,"5min":63.082999999999998,"15min":63.082999999999998},"last":0.36599999999999999}]},{"osd":6,"last update":"Mon Aug 28 02:26:28 2023","interfaces":[{"interface":"back","average":{"1min":2.5219999999999998,"5min":2.5779999999999998,"15min":1.2190000000000001},"min":{"1min":0.32100000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":9.7070000000000007,"5min":101.61,"15min":101.61},"last":0.41799999999999998},{"interface":"front","average":{"1min":2.4609999999999999,"5min":2.7519999999999998,"15min":1.3160000000000001},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":7.6239999999999997,"5min":101.621,"15min":101.621},"last":0.52000000000000002}]}]},{"osd":6,"up_from":42,"seq":180388626640,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307744,"kb_used_data":2328,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93437408,"statfs":{"total":95995035648,"available":95679905792,"internally_reserved":0,"allocated":2383872,"data_stored":1816229,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:26:02 2023","interfaces":[{"interface":"back","average":{"1min":3.48,"5min":4.3949999999999996,"15min":1.845},"min":{"1min":0.245,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":13.260999999999999,"5min":121.247,"15min":121.247},"last":0.35799999999999998},{"interface":"front","average":{"1min":4.4560000000000004,"5min":4.6369999999999996,"15min":1.9430000000000001},"min":{"1min":0.371,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":13.298999999999999,"5min":118.977,"15min":118.977},"last":0.56699999999999995}]},{"osd":1,"last update":"Mon Aug 28 02:26:02 2023","interfaces":[{"interface":"back","average":{"1min":2.0910000000000002,"5min":2.258,"15min":1.1100000000000001},"min":{"1min":0.307,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":6.7050000000000001,"5min":34.677999999999997,"15min":34.677999999999997},"last":0.46000000000000002},{"interface":"front","average":{"1min":2.7850000000000001,"5min":2.5009999999999999,"15min":1.2130000000000001},"min":{"1min":0.317,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":9.6799999999999997,"5min":34.348999999999997,"15min":34.348999999999997},"last":0.50900000000000001}]},{"osd":2,"last update":"Mon Aug 28 02:26:39 2023","interfaces":[{"interface":"back","average":{"1min":9.5429999999999993,"5min":4.0819999999999999,"15min":1.7669999999999999},"min":{"1min":0.34000000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":79.787000000000006,"5min":79.787000000000006,"15min":79.787000000000006},"last":0.40400000000000003},{"interface":"front","average":{"1min":10.275,"5min":4.056,"15min":1.736},"min":{"1min":0.35799999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":79.935000000000002,"5min":79.935000000000002,"15min":79.935000000000002},"last":0.38300000000000001}]},{"osd":3,"last update":"Mon Aug 28 02:26:02 2023","interfaces":[{"interface":"back","average":{"1min":2.2869999999999999,"5min":1.885,"15min":1.075},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":6.4699999999999998,"5min":13.103,"15min":13.103},"last":0.52700000000000002},{"interface":"front","average":{"1min":2.508,"5min":2.0649999999999999,"15min":1.143},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":11.395,"5min":13.09,"15min":13.09},"last":0.61099999999999999}]},{"osd":4,"last update":"Mon Aug 28 02:26:06 2023","interfaces":[{"interface":"back","average":{"1min":1.5880000000000001,"5min":1.669,"15min":0.86699999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":8.9580000000000002,"5min":10.637,"15min":10.637},"last":0.58799999999999997},{"interface":"front","average":{"1min":2.0819999999999999,"5min":1.8100000000000001,"15min":0.94299999999999995},"min":{"1min":0.29899999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":9.7690000000000001,"5min":10.055,"15min":10.055},"last":0.63300000000000001}]},{"osd":5,"last update":"Mon Aug 28 02:26:06 2023","interfaces":[{"interface":"back","average":{"1min":1.982,"5min":1.9399999999999999,"15min":0.96499999999999997},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":9.9809999999999999,"5min":20.759,"15min":20.759},"last":0.442},{"interface":"front","average":{"1min":1.8220000000000001,"5min":1.484,"15min":0.85099999999999998},"min":{"1min":0.32200000000000001,"5min":0.311,"15min":0.28799999999999998},"max":{"1min":5.5030000000000001,"5min":7.3780000000000001,"15min":7.3780000000000001},"last":0.61899999999999999}]},{"osd":7,"last update":"Mon Aug 28 02:26:35 2023","interfaces":[{"interface":"back","average":{"1min":1.7490000000000001,"5min":1.5529999999999999,"15min":0.88700000000000001},"min":{"1min":0.34200000000000003,"5min":0.314,"15min":0.314},"max":{"1min":9.0830000000000002,"5min":9.0830000000000002,"15min":9.0830000000000002},"last":0.65500000000000003},{"interface":"front","average":{"1min":1.915,"5min":1.742,"15min":0.96999999999999997},"min":{"1min":0.38700000000000001,"5min":0.33500000000000002,"15min":0.29799999999999999},"max":{"1min":9.7110000000000003,"5min":9.7110000000000003,"15min":9.7110000000000003},"last":0.66400000000000003}]}]},{"osd":1,"up_from":98,"seq":420906795093,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":110084,"kb_used_data":6204,"kb_used_omap":7,"kb_used_meta":103864,"kb_avail":93635068,"statfs":{"total":95995035648,"available":95882309632,"internally_reserved":0,"allocated":6352896,"data_stored":5473712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":106357067},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:26:01 2023","interfaces":[{"interface":"back","average":{"1min":14.807,"5min":4.4260000000000002,"15min":1.6990000000000001},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":128.60599999999999,"5min":128.60599999999999,"15min":128.60599999999999},"last":0.28999999999999998},{"interface":"front","average":{"1min":14.028,"5min":4.4489999999999998,"15min":1.7190000000000001},"min":{"1min":0.23699999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":128.685,"5min":128.685,"15min":128.685},"last":0.34899999999999998}]},{"osd":2,"last update":"Mon Aug 28 02:26:31 2023","interfaces":[{"interface":"back","average":{"1min":16.405999999999999,"5min":5.4660000000000002,"15min":2.0499999999999998},"min":{"1min":0.27600000000000002,"5min":0.26200000000000001,"15min":0.22700000000000001},"max":{"1min":135.25999999999999,"5min":135.25999999999999,"15min":135.25999999999999},"last":0.222},{"interface":"front","average":{"1min":16.736000000000001,"5min":5.0179999999999998,"15min":1.9259999999999999},"min":{"1min":0.34300000000000003,"5min":0.221,"15min":0.221},"max":{"1min":135.10300000000001,"5min":135.10300000000001,"15min":135.10300000000001},"last":0.32600000000000001}]},{"osd":3,"last update":"Mon Aug 28 02:26:01 2023","interfaces":[{"interface":"back","average":{"1min":2.1200000000000001,"5min":1.8140000000000001,"15min":0.84799999999999998},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":12.321999999999999,"5min":15.395,"15min":15.395},"last":0.25700000000000001},{"interface":"front","average":{"1min":3.7250000000000001,"5min":2.1480000000000001,"15min":1.0229999999999999},"min":{"1min":0.26900000000000002,"5min":0.254,"15min":0.254},"max":{"1min":22.361999999999998,"5min":22.361999999999998,"15min":22.361999999999998},"last":0.38800000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:26:01 2023","interfaces":[{"interface":"back","average":{"1min":3.3570000000000002,"5min":1.9339999999999999,"15min":1.0620000000000001},"min":{"1min":0.32800000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":17.600000000000001,"5min":17.600000000000001,"15min":17.600000000000001},"last":0.54100000000000004},{"interface":"front","average":{"1min":2.5499999999999998,"5min":1.556,"15min":0.93200000000000005},"min":{"1min":0.36299999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":19.010000000000002,"5min":19.010000000000002,"15min":19.010000000000002},"last":0.58699999999999997}]},{"osd":5,"last update":"Mon Aug 28 02:26:01 2023","interfaces":[{"interface":"back","average":{"1min":1.754,"5min":2.0089999999999999,"15min":1.0940000000000001},"min":{"1min":0.312,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.0579999999999998,"5min":15.544,"15min":15.544},"last":0.64900000000000002},{"interface":"front","average":{"1min":3.4529999999999998,"5min":1.839,"15min":1.04},"min":{"1min":0.38800000000000001,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":17.646999999999998,"5min":17.646999999999998,"15min":17.646999999999998},"last":0.56599999999999995}]},{"osd":6,"last update":"Mon Aug 28 02:26:01 2023","interfaces":[{"interface":"back","average":{"1min":1.992,"5min":2.3359999999999999,"15min":1.1910000000000001},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":6.0869999999999997,"5min":15.512,"15min":15.512},"last":0.57299999999999995},{"interface":"front","average":{"1min":3.3450000000000002,"5min":2.2810000000000001,"15min":1.169},"min":{"1min":0.39600000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":17.673999999999999,"5min":17.673999999999999,"15min":17.673999999999999},"last":0.49099999999999999}]},{"osd":7,"last update":"Mon Aug 28 02:26:01 2023","interfaces":[{"interface":"back","average":{"1min":3.1930000000000001,"5min":1.8440000000000001,"15min":1.0369999999999999},"min":{"1min":0.39900000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":19.061,"5min":19.061,"15min":19.061},"last":0.629},{"interface":"front","average":{"1min":2.3500000000000001,"5min":2.1349999999999998,"15min":1.137},"min":{"1min":0.41899999999999998,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":6.1100000000000003,"5min":15.476000000000001,"15min":15.476000000000001},"last":0.61899999999999999}]}]},{"osd":0,"up_from":90,"seq":386547056739,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":624604,"kb_used_data":395604,"kb_used_omap":8,"kb_used_meta":228983,"kb_avail":93120548,"statfs":{"total":95995035648,"available":95355441152,"internally_reserved":0,"allocated":405098496,"data_stored":404254742,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":234478953},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Aug 28 02:26:09 2023","interfaces":[{"interface":"back","average":{"1min":2.516,"5min":2.754,"15min":1.153},"min":{"1min":0.184,"5min":0.184,"15min":0.184},"max":{"1min":11.282,"5min":40.313000000000002,"15min":40.313000000000002},"last":1.6060000000000001},{"interface":"front","average":{"1min":3.8370000000000002,"5min":2.5899999999999999,"15min":1.111},"min":{"1min":0.23899999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":19.555,"5min":36.655000000000001,"15min":36.655000000000001},"last":1.115}]},{"osd":2,"last update":"Mon Aug 28 02:26:28 2023","interfaces":[{"interface":"back","average":{"1min":3.492,"5min":2.5720000000000001,"15min":1.0920000000000001},"min":{"1min":0.32100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":8.4169999999999998,"5min":15.33,"15min":15.33},"last":1.5680000000000001},{"interface":"front","average":{"1min":6.0800000000000001,"5min":3.621,"15min":1.468},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":39.877000000000002,"5min":39.877000000000002,"15min":39.877000000000002},"last":1.2849999999999999}]},{"osd":3,"last update":"Mon Aug 28 02:26:09 2023","interfaces":[{"interface":"back","average":{"1min":3.77,"5min":2.2090000000000001,"15min":0.97799999999999998},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":12.272,"5min":18.545999999999999,"15min":18.545999999999999},"last":0.67000000000000004},{"interface":"front","average":{"1min":3.3300000000000001,"5min":2.1309999999999998,"15min":0.97299999999999998},"min":{"1min":0.28999999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":12.474,"5min":14.534000000000001,"15min":14.534000000000001},"last":0.78800000000000003}]},{"osd":4,"last update":"Mon Aug 28 02:26:04 2023","interfaces":[{"interface":"back","average":{"1min":2.3700000000000001,"5min":1.8520000000000001,"15min":1.0600000000000001},"min":{"1min":0.29599999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":7.5979999999999999,"5min":9.6649999999999991,"15min":9.6649999999999991},"last":1.724},{"interface":"front","average":{"1min":2.7679999999999998,"5min":2.0299999999999998,"15min":1.1419999999999999},"min":{"1min":0.45700000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":12.305,"5min":12.305,"15min":12.305},"last":0.76400000000000001}]},{"osd":5,"last update":"Mon Aug 28 02:26:04 2023","interfaces":[{"interface":"back","average":{"1min":3.0529999999999999,"5min":2.508,"15min":1.268},"min":{"1min":0.41299999999999998,"5min":0.313,"15min":0.313},"max":{"1min":12.337999999999999,"5min":44.000999999999998,"15min":44.000999999999998},"last":0.73499999999999999},{"interface":"front","average":{"1min":3.177,"5min":2.423,"15min":1.238},"min":{"1min":0.438,"5min":0.33100000000000002,"15min":0.30599999999999999},"max":{"1min":12.334,"5min":43.953000000000003,"15min":43.953000000000003},"last":1.6719999999999999}]},{"osd":6,"last update":"Mon Aug 28 02:26:04 2023","interfaces":[{"interface":"back","average":{"1min":3.1970000000000001,"5min":2.5409999999999999,"15min":1.2789999999999999},"min":{"1min":0.42799999999999999,"5min":0.36299999999999999,"15min":0.32000000000000001},"max":{"1min":12.361000000000001,"5min":32.417999999999999,"15min":32.417999999999999},"last":0.70799999999999996},{"interface":"front","average":{"1min":3.0499999999999998,"5min":2.286,"15min":1.23},"min":{"1min":0.378,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":8.3829999999999991,"5min":32.130000000000003,"15min":32.130000000000003},"last":1.6990000000000001}]},{"osd":7,"last update":"Mon Aug 28 02:26:04 2023","interfaces":[{"interface":"back","average":{"1min":3.2690000000000001,"5min":2.1019999999999999,"15min":1.1619999999999999},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40000000000000002},"max":{"1min":8.9870000000000001,"5min":9.6020000000000003,"15min":9.6020000000000003},"last":1.6419999999999999},{"interface":"front","average":{"1min":3.649,"5min":2.3570000000000002,"15min":1.25},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":19.478999999999999,"5min":19.478999999999999,"15min":19.478999999999999},"last":0.90100000000000002}]}]},{"osd":2,"up_from":104,"seq":446676598863,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":132084,"kb_used_data":2028,"kb_used_omap":10,"kb_used_meta":130037,"kb_avail":93613068,"statfs":{"total":95995035648,"available":95859781632,"internally_reserved":0,"allocated":2076672,"data_stored":1269197,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10792,"internal_metadata":133158360},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:26:33 2023","interfaces":[{"interface":"back","average":{"1min":8.6509999999999998,"5min":3.9180000000000001,"15min":1.5189999999999999},"min":{"1min":0.17699999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":51.920000000000002,"5min":51.920000000000002,"15min":51.920000000000002},"last":0.246},{"interface":"front","average":{"1min":22.567,"5min":6.899,"15min":2.5419999999999998},"min":{"1min":0.45000000000000001,"5min":0.214,"15min":0.19900000000000001},"max":{"1min":147.286,"5min":147.286,"15min":147.286},"last":0.41399999999999998}]},{"osd":1,"last update":"Mon Aug 28 02:26:33 2023","interfaces":[{"interface":"back","average":{"1min":4.452,"5min":3.2450000000000001,"15min":1.321},"min":{"1min":0.23599999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":11.031000000000001,"5min":30.888000000000002,"15min":30.888000000000002},"last":0.29899999999999999},{"interface":"front","average":{"1min":3.9380000000000002,"5min":2.6509999999999998,"15min":1.1060000000000001},"min":{"1min":0.25800000000000001,"5min":0.21299999999999999,"15min":0.20999999999999999},"max":{"1min":20.259,"5min":31.648,"15min":31.648},"last":0.309}]},{"osd":3,"last update":"Mon Aug 28 02:26:02 2023","interfaces":[{"interface":"back","average":{"1min":10.356,"5min":3.331,"15min":1.3819999999999999},"min":{"1min":0.309,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":128.63900000000001,"5min":128.63900000000001,"15min":128.63900000000001},"last":0.34999999999999998},{"interface":"front","average":{"1min":10.528,"5min":3.6179999999999999,"15min":1.464},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":128.75700000000001,"5min":128.75700000000001,"15min":128.75700000000001},"last":0.26500000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:26:33 2023","interfaces":[{"interface":"back","average":{"1min":3.6869999999999998,"5min":2.391,"15min":1.1930000000000001},"min":{"1min":0.32200000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":17.024999999999999,"5min":17.024999999999999,"15min":17.024999999999999},"last":0.61699999999999999},{"interface":"front","average":{"1min":3.351,"5min":1.8700000000000001,"15min":1.056},"min":{"1min":0.28999999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":16.936,"5min":16.936,"15min":16.936},"last":0.63400000000000001}]},{"osd":5,"last update":"Mon Aug 28 02:26:33 2023","interfaces":[{"interface":"back","average":{"1min":10.657,"5min":3.5259999999999998,"15min":1.5860000000000001},"min":{"1min":0.38400000000000001,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":128.696,"5min":128.696,"15min":128.696},"last":0.65100000000000002},{"interface":"front","average":{"1min":9.4239999999999995,"5min":3.819,"15min":1.645},"min":{"1min":0.32100000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":128.84100000000001,"5min":128.84100000000001,"15min":128.84100000000001},"last":0.56499999999999995}]},{"osd":6,"last update":"Mon Aug 28 02:26:33 2023","interfaces":[{"interface":"back","average":{"1min":2.7410000000000001,"5min":2.0870000000000002,"15min":1.0960000000000001},"min":{"1min":0.307,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":11.285,"5min":11.285,"15min":11.285},"last":0.55900000000000005},{"interface":"front","average":{"1min":3.7509999999999999,"5min":2.3620000000000001,"15min":1.198},"min":{"1min":0.377,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":11.055999999999999,"5min":11.055999999999999,"15min":11.055999999999999},"last":0.625}]},{"osd":7,"last update":"Mon Aug 28 02:26:33 2023","interfaces":[{"interface":"back","average":{"1min":9.8529999999999998,"5min":3.4609999999999999,"15min":1.5529999999999999},"min":{"1min":0.34499999999999997,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":128.911,"5min":128.911,"15min":128.911},"last":0.57899999999999996},{"interface":"front","average":{"1min":11.334,"5min":3.819,"15min":1.6759999999999999},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":128.72399999999999,"5min":128.72399999999999,"15min":128.72399999999999},"last":0.68300000000000005}]}]},{"osd":3,"up_from":112,"seq":481036337225,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":69580,"kb_used_data":1988,"kb_used_omap":3,"kb_used_meta":67580,"kb_avail":93675572,"statfs":{"total":95995035648,"available":95923785728,"internally_reserved":0,"allocated":2035712,"data_stored":1259228,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":69202850},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:26:00 2023","interfaces":[{"interface":"back","average":{"1min":1.96,"5min":5.8419999999999996,"15min":2.1800000000000002},"min":{"1min":0.25600000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":8.1150000000000002,"5min":164.25299999999999,"15min":164.25299999999999},"last":0.26300000000000001},{"interface":"front","average":{"1min":2.085,"5min":6.0650000000000004,"15min":2.2429999999999999},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":9.0530000000000008,"5min":164.55000000000001,"15min":164.55000000000001},"last":0.23200000000000001}]},{"osd":1,"last update":"Mon Aug 28 02:26:00 2023","interfaces":[{"interface":"back","average":{"1min":3.25,"5min":3.9390000000000001,"15min":1.5389999999999999},"min":{"1min":0.251,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":13.936,"5min":117.34699999999999,"15min":117.34699999999999},"last":0.249},{"interface":"front","average":{"1min":1.8089999999999999,"5min":3.6619999999999999,"15min":1.474},"min":{"1min":0.28100000000000003,"5min":0.224,"15min":0.224},"max":{"1min":7.2679999999999998,"5min":118.012,"15min":118.012},"last":0.30299999999999999}]},{"osd":2,"last update":"Mon Aug 28 02:26:00 2023","interfaces":[{"interface":"back","average":{"1min":3.6829999999999998,"5min":5.5730000000000004,"15min":2.1070000000000002},"min":{"1min":0.314,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":18.718,"5min":120.43300000000001,"15min":120.43300000000001},"last":0.35599999999999998},{"interface":"front","average":{"1min":3.2949999999999999,"5min":5.5949999999999998,"15min":2.1160000000000001},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":18.774000000000001,"5min":120.44499999999999,"15min":120.44499999999999},"last":0.36799999999999999}]},{"osd":4,"last update":"Mon Aug 28 02:26:00 2023","interfaces":[{"interface":"back","average":{"1min":2.7559999999999998,"5min":1.681,"15min":0.97899999999999998},"min":{"1min":0.35199999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":9.016,"5min":9.016,"15min":9.016},"last":0.68000000000000005},{"interface":"front","average":{"1min":3.0409999999999999,"5min":1.669,"15min":0.97999999999999998},"min":{"1min":0.37,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":10.428000000000001,"5min":10.428000000000001,"15min":10.428000000000001},"last":0.66400000000000003}]},{"osd":5,"last update":"Mon Aug 28 02:26:00 2023","interfaces":[{"interface":"back","average":{"1min":2.4460000000000002,"5min":1.488,"15min":0.92000000000000004},"min":{"1min":0.41399999999999998,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":8.9789999999999992,"5min":8.9789999999999992,"15min":8.9789999999999992},"last":0.626},{"interface":"front","average":{"1min":1.772,"5min":1.333,"15min":0.86399999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":13.331,"5min":13.331,"15min":13.331},"last":0.64100000000000001}]},{"osd":6,"last update":"Mon Aug 28 02:26:00 2023","interfaces":[{"interface":"back","average":{"1min":2.427,"5min":2.1440000000000001,"15min":1.1379999999999999},"min":{"1min":0.36799999999999999,"5min":0.314,"15min":0.314},"max":{"1min":9.4529999999999994,"5min":72.116,"15min":72.116},"last":0.52900000000000003},{"interface":"front","average":{"1min":2.0459999999999998,"5min":2.2210000000000001,"15min":1.139},"min":{"1min":0.38,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":6.7110000000000003,"5min":72.030000000000001,"15min":72.030000000000001},"last":0.58799999999999997}]},{"osd":7,"last update":"Mon Aug 28 02:26:00 2023","interfaces":[{"interface":"back","average":{"1min":2.9329999999999998,"5min":1.7250000000000001,"15min":1.0129999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":11.949999999999999,"5min":11.949999999999999,"15min":11.949999999999999},"last":0.65300000000000002},{"interface":"front","average":{"1min":3.702,"5min":1.907,"15min":1.0660000000000001},"min":{"1min":0.438,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":13.890000000000001,"5min":13.890000000000001,"15min":13.890000000000001},"last":0.57499999999999996}]}]},{"osd":4,"up_from":30,"seq":128849019098,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":303072,"kb_used_data":1752,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93442080,"statfs":{"total":95995035648,"available":95684689920,"internally_reserved":0,"allocated":1794048,"data_stored":1226211,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:25:59 2023","interfaces":[{"interface":"back","average":{"1min":6.2859999999999996,"5min":5.8010000000000002,"15min":2.3399999999999999},"min":{"1min":0.26500000000000001,"5min":0.223,"15min":0.223},"max":{"1min":43.548000000000002,"5min":138.67400000000001,"15min":138.67400000000001},"last":0.502},{"interface":"front","average":{"1min":5.8789999999999996,"5min":5.6669999999999998,"15min":2.2839999999999998},"min":{"1min":0.25,"5min":0.248,"15min":0.248},"max":{"1min":43.515000000000001,"5min":134.14599999999999,"15min":134.14599999999999},"last":0.45400000000000001}]},{"osd":1,"last update":"Mon Aug 28 02:25:59 2023","interfaces":[{"interface":"back","average":{"1min":2.6509999999999998,"5min":1.915,"15min":0.98699999999999999},"min":{"1min":0.32700000000000001,"5min":0.255,"15min":0.255},"max":{"1min":8.9570000000000007,"5min":53.622999999999998,"15min":53.622999999999998},"last":0.439},{"interface":"front","average":{"1min":2.1339999999999999,"5min":1.8999999999999999,"15min":1.01},"min":{"1min":0.28000000000000003,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":7.1580000000000004,"5min":53.774000000000001,"15min":53.774000000000001},"last":0.40300000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:26:30 2023","interfaces":[{"interface":"back","average":{"1min":6.0960000000000001,"5min":4.7309999999999999,"15min":1.9430000000000001},"min":{"1min":0.30499999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":38.488999999999997,"5min":74.099000000000004,"15min":74.099000000000004},"last":0.38},{"interface":"front","average":{"1min":4.6980000000000004,"5min":4.3129999999999997,"15min":1.7969999999999999},"min":{"1min":0.45500000000000002,"5min":0.25,"15min":0.25},"max":{"1min":35.134999999999998,"5min":74.081999999999994,"15min":74.081999999999994},"last":0.34399999999999997}]},{"osd":3,"last update":"Mon Aug 28 02:25:59 2023","interfaces":[{"interface":"back","average":{"1min":3.3359999999999999,"5min":1.5149999999999999,"15min":0.83899999999999997},"min":{"1min":0.32400000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":8.6349999999999998,"5min":8.6349999999999998,"15min":8.6349999999999998},"last":0.42099999999999999},{"interface":"front","average":{"1min":2.9430000000000001,"5min":1.6870000000000001,"15min":0.95099999999999996},"min":{"1min":0.39900000000000002,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":10.956,"5min":10.956,"15min":10.956},"last":0.47499999999999998}]},{"osd":5,"last update":"Mon Aug 28 02:26:41 2023","interfaces":[{"interface":"back","average":{"1min":2.5350000000000001,"5min":1.383,"15min":0.77900000000000003},"min":{"1min":0.34899999999999998,"5min":0.249,"15min":0.249},"max":{"1min":7.7350000000000003,"5min":8.2560000000000002,"15min":8.2560000000000002},"last":0.48799999999999999},{"interface":"front","average":{"1min":3.2109999999999999,"5min":1.875,"15min":0.94999999999999996},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":16.497,"5min":16.497,"15min":16.497},"last":0.46700000000000003}]},{"osd":6,"last update":"Mon Aug 28 02:25:59 2023","interfaces":[{"interface":"back","average":{"1min":2.1059999999999999,"5min":1.373,"15min":0.79600000000000004},"min":{"1min":0.32300000000000001,"5min":0.29199999999999998,"15min":0.27500000000000002},"max":{"1min":11.170999999999999,"5min":11.170999999999999,"15min":11.170999999999999},"last":0.56599999999999995},{"interface":"front","average":{"1min":1.403,"5min":0.97199999999999998,"15min":0.69199999999999995},"min":{"1min":0.33700000000000002,"5min":0.32200000000000001,"15min":0.318},"max":{"1min":7.3170000000000002,"5min":7.3170000000000002,"15min":7.3170000000000002},"last":0.52800000000000002}]},{"osd":7,"last update":"Mon Aug 28 02:26:24 2023","interfaces":[{"interface":"back","average":{"1min":4.2800000000000002,"5min":1.8400000000000001,"15min":0.97599999999999998},"min":{"1min":0.41399999999999998,"5min":0.33900000000000002,"15min":0.312},"max":{"1min":16.584,"5min":16.584,"15min":16.584},"last":0.51400000000000001},{"interface":"front","average":{"1min":2.3399999999999999,"5min":1.79,"15min":0.97199999999999998},"min":{"1min":0.371,"5min":0.32500000000000001,"15min":0.32100000000000001},"max":{"1min":7.6360000000000001,"5min":8.7569999999999997,"15min":8.7569999999999997},"last":0.53900000000000003}]}]},{"osd":5,"up_from":36,"seq":154618822870,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":339536,"kb_used_data":42312,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":93405616,"statfs":{"total":95995035648,"available":95647350784,"internally_reserved":0,"allocated":43327488,"data_stored":42771190,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:25:52 2023","interfaces":[{"interface":"back","average":{"1min":3.5249999999999999,"5min":5.8579999999999997,"15min":2.3639999999999999},"min":{"1min":0.24099999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":12.874000000000001,"5min":131.06100000000001,"15min":131.06100000000001},"last":0.53400000000000003},{"interface":"front","average":{"1min":3.758,"5min":6.1749999999999998,"15min":2.4740000000000002},"min":{"1min":0.24199999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":15.691000000000001,"5min":131.262,"15min":131.262},"last":0.56399999999999995}]},{"osd":1,"last update":"Mon Aug 28 02:26:03 2023","interfaces":[{"interface":"back","average":{"1min":1.298,"5min":5.0019999999999998,"15min":2.0190000000000001},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":5.7240000000000002,"5min":183.97900000000001,"15min":183.97900000000001},"last":0.44600000000000001},{"interface":"front","average":{"1min":2.4929999999999999,"5min":5.4960000000000004,"15min":2.1920000000000002},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":8.5640000000000001,"5min":184.33699999999999,"15min":184.33699999999999},"last":0.51000000000000001}]},{"osd":2,"last update":"Mon Aug 28 02:26:33 2023","interfaces":[{"interface":"back","average":{"1min":15.742000000000001,"5min":8.2479999999999993,"15min":3.1419999999999999},"min":{"1min":0.315,"5min":0.252,"15min":0.252},"max":{"1min":176.69499999999999,"5min":176.69499999999999,"15min":176.69499999999999},"last":0.60999999999999999},{"interface":"front","average":{"1min":16.274999999999999,"5min":8.6199999999999992,"15min":3.2650000000000001},"min":{"1min":1.492,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":176.71199999999999,"5min":176.71199999999999,"15min":176.71199999999999},"last":0.46100000000000002}]},{"osd":3,"last update":"Mon Aug 28 02:26:03 2023","interfaces":[{"interface":"back","average":{"1min":2.004,"5min":1.649,"15min":0.95699999999999996},"min":{"1min":0.28899999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.0039999999999996,"5min":10.576000000000001,"15min":10.576000000000001},"last":0.57799999999999996},{"interface":"front","average":{"1min":3.5499999999999998,"5min":2.177,"15min":1.143},"min":{"1min":0.34300000000000003,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":12.118,"5min":12.118,"15min":12.118},"last":0.59899999999999998}]},{"osd":4,"last update":"Mon Aug 28 02:25:44 2023","interfaces":[{"interface":"back","average":{"1min":1.4239999999999999,"5min":1.355,"15min":0.76200000000000001},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":5.6890000000000001,"5min":10.759,"15min":10.759},"last":0.58699999999999997},{"interface":"front","average":{"1min":2.149,"5min":1.468,"15min":0.82199999999999995},"min":{"1min":0.31900000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.0170000000000003,"5min":8.6769999999999996,"15min":8.6769999999999996},"last":0.65300000000000002}]},{"osd":6,"last update":"Mon Aug 28 02:26:03 2023","interfaces":[{"interface":"back","average":{"1min":1.968,"5min":1.694,"15min":0.91900000000000004},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.28399999999999997},"max":{"1min":5.9870000000000001,"5min":14.803000000000001,"15min":14.803000000000001},"last":0.49399999999999999},{"interface":"front","average":{"1min":1.6639999999999999,"5min":1.389,"15min":0.81699999999999995},"min":{"1min":0.33500000000000002,"5min":0.31,"15min":0.30299999999999999},"max":{"1min":8.4779999999999998,"5min":8.6479999999999997,"15min":8.6479999999999997},"last":0.68300000000000005}]},{"osd":7,"last update":"Mon Aug 28 02:26:19 2023","interfaces":[{"interface":"back","average":{"1min":2.8159999999999998,"5min":1.75,"15min":0.95499999999999996},"min":{"1min":0.33500000000000002,"5min":0.31,"15min":0.31},"max":{"1min":9.9949999999999992,"5min":9.9949999999999992,"15min":9.9949999999999992},"last":0.70599999999999996},{"interface":"front","average":{"1min":1.54,"5min":1.5229999999999999,"15min":0.874},"min":{"1min":0.37,"5min":0.32100000000000001,"15min":0.29799999999999999},"max":{"1min":4.9889999999999999,"5min":12.787000000000001,"15min":12.787000000000001},"last":0.63700000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":398196736,"data_stored":398196736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":41549824,"data_stored":41549824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-08-28T02:26:48.596 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:26:48.602 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:48 smithi143 ceph-mon[183059]: pgmap v434: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 264 op/s; 1665/84 objects degraded (1982.143%); 0 B/s, 111 objects/s recovering 2023-08-28T02:26:48.602 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:48 smithi143 ceph-mon[189871]: pgmap v434: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 264 op/s; 1665/84 objects degraded (1982.143%); 0 B/s, 111 objects/s recovering 2023-08-28T02:26:48.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:48.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:26:48.665 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:26:48.705 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:26:48.736 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:48.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:48 smithi169 ceph-mon[159415]: pgmap v434: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 264 op/s; 1665/84 objects degraded (1982.143%); 0 B/s, 111 objects/s recovering 2023-08-28T02:26:48.793 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:26:48.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:48.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:26:48.898 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:26:48.930 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:48.936 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:26:48.978 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:26:48.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:48.985 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:26:49.086 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:26:49.114 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:49.158 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:26:49.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:49.163 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:26:49.264 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:26:49.298 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:49.363 INFO:teuthology.orchestra.run.smithi143.stderr:pool 'unique_pool_0' removed 2023-08-28T02:26:49.392 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:49 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1312351130' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-08-28T02:26:49.393 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:49 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1312351130' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-08-28T02:26:49.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:49 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1312351130' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-08-28T02:26:49.773 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:26:49.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:49.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:26:49.877 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:26:49.905 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:49.948 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:26:49.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:49.952 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:26:50.053 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:26:50.080 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:50.121 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:26:50.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:50.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:26:50.226 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:26:50.252 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:50.294 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:26:50.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:50.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:26:50.349 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is unset 2023-08-28T02:26:50.400 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:26:50.427 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:50.469 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:26:50.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:50.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:26:50.576 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:26:50.601 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:50 smithi143 ceph-mon[189871]: pgmap v435: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 30 op/s; 748/84 objects degraded (890.476%); 0 B/s, 172 objects/s recovering 2023-08-28T02:26:50.602 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:50 smithi143 ceph-mon[189871]: failed to encode map e185 with expected crc 2023-08-28T02:26:50.602 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:50 smithi143 ceph-mon[189871]: failed to encode map e185 with expected crc 2023-08-28T02:26:50.602 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:50 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1312351130' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-08-28T02:26:50.602 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:50 smithi143 ceph-mon[189871]: osdmap e185: 8 total, 8 up, 8 in 2023-08-28T02:26:50.602 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:50 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/733343122' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:26:50.602 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:50 smithi143 ceph-mon[183059]: pgmap v435: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 30 op/s; 748/84 objects degraded (890.476%); 0 B/s, 172 objects/s recovering 2023-08-28T02:26:50.602 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:50 smithi143 ceph-mon[183059]: failed to encode map e185 with expected crc 2023-08-28T02:26:50.602 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:50 smithi143 ceph-mon[183059]: failed to encode map e185 with expected crc 2023-08-28T02:26:50.602 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:50 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1312351130' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-08-28T02:26:50.603 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:50 smithi143 ceph-mon[183059]: osdmap e185: 8 total, 8 up, 8 in 2023-08-28T02:26:50.603 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:50 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/733343122' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:26:50.604 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:50.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:50 smithi169 ceph-mon[159415]: pgmap v435: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 4.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 30 op/s; 748/84 objects degraded (890.476%); 0 B/s, 172 objects/s recovering 2023-08-28T02:26:50.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:50 smithi169 ceph-mon[159415]: failed to encode map e185 with expected crc 2023-08-28T02:26:50.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:50 smithi169 ceph-mon[159415]: failed to encode map e185 with expected crc 2023-08-28T02:26:50.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:50 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1312351130' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-08-28T02:26:50.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:50 smithi169 ceph-mon[159415]: osdmap e185: 8 total, 8 up, 8 in 2023-08-28T02:26:50.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:50 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/733343122' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:26:51.258 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:26:51.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:51.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:26:51.361 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:26:51.389 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:51.425 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph pg dump --format=json 2023-08-28T02:26:51.448 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:51 smithi143 ceph-mon[183059]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:26:51.448 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:51 smithi143 ceph-mon[183059]: failed to encode map e186 with expected crc 2023-08-28T02:26:51.448 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:51 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/733343122' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:26:51.448 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:51 smithi143 ceph-mon[183059]: osdmap e186: 8 total, 8 up, 8 in 2023-08-28T02:26:51.448 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:51 smithi143 ceph-mon[183059]: failed to encode map e185 with expected crc 2023-08-28T02:26:51.448 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:51 smithi143 ceph-mon[183059]: failed to encode map e186 with expected crc 2023-08-28T02:26:51.448 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:51 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 748/84 objects degraded (890.476%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:26:51.449 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:51 smithi143 ceph-mon[189871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:26:51.449 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:51 smithi143 ceph-mon[189871]: failed to encode map e186 with expected crc 2023-08-28T02:26:51.449 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:51 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/733343122' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:26:51.449 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:51 smithi143 ceph-mon[189871]: osdmap e186: 8 total, 8 up, 8 in 2023-08-28T02:26:51.449 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:51 smithi143 ceph-mon[189871]: failed to encode map e185 with expected crc 2023-08-28T02:26:51.449 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:51 smithi143 ceph-mon[189871]: failed to encode map e186 with expected crc 2023-08-28T02:26:51.449 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:51 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 748/84 objects degraded (890.476%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:26:51.571 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-08-28T02:26:51.572 INFO:tasks.radosbench:Beginning radosbench... 2023-08-28T02:26:51.572 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2023-08-28T02:26:51.572 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd pool create unique_pool_1 16 2023-08-28T02:26:51.585 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:26:51.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:51.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:26:51.690 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:26:51.719 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:51 smithi169 ceph-mon[159415]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:26:51.719 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:51 smithi169 ceph-mon[159415]: failed to encode map e186 with expected crc 2023-08-28T02:26:51.719 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:51 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/733343122' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:26:51.719 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:51 smithi169 ceph-mon[159415]: osdmap e186: 8 total, 8 up, 8 in 2023-08-28T02:26:51.719 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:51 smithi169 ceph-mon[159415]: failed to encode map e185 with expected crc 2023-08-28T02:26:51.719 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:51 smithi169 ceph-mon[159415]: failed to encode map e186 with expected crc 2023-08-28T02:26:51.719 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:51 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 748/84 objects degraded (890.476%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:26:51.722 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:51.771 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:26:51.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:51.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:26:51.877 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:26:51.903 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:52.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:26:51 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:26:51] "GET /metrics HTTP/1.1" 200 36499 "" "Prometheus/2.33.4" 2023-08-28T02:26:52.376 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:26:52.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:52.379 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:26:52.479 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:26:52.511 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:52.555 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:26:52.599 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:26:52.599 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:26:52.607 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:52 smithi143 ceph-mon[183059]: failed to encode map e185 with expected crc 2023-08-28T02:26:52.607 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:52 smithi143 ceph-mon[183059]: pgmap v438: 73 pgs: 73 active+clean; 4.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:26:52.607 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:52 smithi143 ceph-mon[183059]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 748/84 objects degraded (890.476%), 1 pg degraded, 1 pg undersized) 2023-08-28T02:26:52.607 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:52 smithi143 ceph-mon[189871]: failed to encode map e185 with expected crc 2023-08-28T02:26:52.607 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:52 smithi143 ceph-mon[189871]: pgmap v438: 73 pgs: 73 active+clean; 4.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:26:52.607 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:52 smithi143 ceph-mon[189871]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 748/84 objects degraded (890.476%), 1 pg degraded, 1 pg undersized) 2023-08-28T02:26:52.617 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:26:52.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:52.620 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:26:52.721 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:26:52.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:52 smithi169 ceph-mon[159415]: failed to encode map e185 with expected crc 2023-08-28T02:26:52.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:52 smithi169 ceph-mon[159415]: pgmap v438: 73 pgs: 73 active+clean; 4.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:26:52.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:52 smithi169 ceph-mon[159415]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 748/84 objects degraded (890.476%), 1 pg degraded, 1 pg undersized) 2023-08-28T02:26:52.749 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:52.770 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:26:52.861 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:26:52.872 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:26:52.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:52.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:26:52.883 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:26:52.903 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:26:52.923 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:26:52.942 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:26:52.960 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:26:52.976 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:26:52.979 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:26:52.998 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:26:52.999 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:26:52.999 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:26:53.000 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:26:53.000 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:26:53.000 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:26:53.000 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:26:53.000 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:26:53.000 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:26:53.019 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:53.066 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:26:53.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:53.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:26:53.174 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:26:53.201 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:53.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:53 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3456182780' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:26:53.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:53 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3456182780' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:26:53.509 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset nodeep-scrub 2023-08-28T02:26:53.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:53 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3456182780' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:26:54.049 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:26:54.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:54.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:26:54.154 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:26:54.186 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:54.236 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:26:54.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:54.240 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:26:54.263 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.0 dump_historic_ops 2023-08-28T02:26:54.341 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:26:54.370 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:54.415 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:26:54.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:54.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:26:54.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:54 smithi169 ceph-mon[159415]: pgmap v439: 73 pgs: 73 active+clean; 4.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:26:54.521 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:26:54.547 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:54.699 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:54 smithi143 ceph-mon[183059]: pgmap v439: 73 pgs: 73 active+clean; 4.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:26:54.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:54 smithi143 ceph-mon[189871]: pgmap v439: 73 pgs: 73 active+clean; 4.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:26:55.394 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:26:55.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:55.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:26:55.500 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:26:55.505 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:26:55.532 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:55.580 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:26:55.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:55.584 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:26:55.685 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:26:55.715 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:55.728 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:26:55.820 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:26:55.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:55.825 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:26:55.926 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:26:55.957 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:56.005 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:26:56.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:56.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:26:56.112 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:26:56.140 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:56.185 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:26:56.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:56.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:26:56.271 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:26:56.272 INFO:teuthology.orchestra.run.smithi143.stderr:dumped all 2023-08-28T02:26:56.290 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:26:56.323 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:56.383 INFO:teuthology.orchestra.run.smithi143.stderr:pool 'unique_pool_1' created 2023-08-28T02:26:56.476 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:26:56.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:56.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:26:56.580 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:26:56.604 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:56 smithi143 ceph-mon[183059]: pgmap v440: 73 pgs: 73 active+clean; 4.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:26:56.605 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:56 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/4117489696' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-08-28T02:26:56.605 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:56 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:26:56.605 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:56 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:26:56.605 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:56 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:26:56.606 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:56 smithi143 ceph-mon[189871]: pgmap v440: 73 pgs: 73 active+clean; 4.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:26:56.606 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:56 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/4117489696' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-08-28T02:26:56.606 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:56 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:26:56.606 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:56 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:26:56.606 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:56 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:26:56.607 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:56.701 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:26:56.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:56 smithi169 ceph-mon[159415]: pgmap v440: 73 pgs: 73 active+clean; 4.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:26:56.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:56 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/4117489696' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-08-28T02:26:56.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:56 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:26:56.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:56 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:26:56.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:56 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:26:57.694 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:57 smithi143 ceph-mon[183059]: from='client.44987 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:26:57.694 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:57 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/4117489696' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-08-28T02:26:57.694 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:57 smithi143 ceph-mon[183059]: osdmap e187: 8 total, 8 up, 8 in 2023-08-28T02:26:57.694 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:57 smithi143 ceph-mon[183059]: failed to encode map e187 with expected crc 2023-08-28T02:26:57.694 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:57 smithi143 ceph-mon[183059]: failed to encode map e187 with expected crc 2023-08-28T02:26:57.694 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:57 smithi143 ceph-mon[183059]: failed to encode map e187 with expected crc 2023-08-28T02:26:57.695 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:57 smithi143 ceph-mon[183059]: failed to encode map e187 with expected crc 2023-08-28T02:26:57.695 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:57 smithi143 ceph-mon[183059]: failed to encode map e187 with expected crc 2023-08-28T02:26:57.695 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:57 smithi143 ceph-mon[183059]: failed to encode map e187 with expected crc 2023-08-28T02:26:57.695 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:57 smithi143 ceph-mon[183059]: failed to encode map e187 with expected crc 2023-08-28T02:26:57.695 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:57 smithi143 ceph-mon[189871]: from='client.44987 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:26:57.695 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:57 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/4117489696' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-08-28T02:26:57.696 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:57 smithi143 ceph-mon[189871]: osdmap e187: 8 total, 8 up, 8 in 2023-08-28T02:26:57.696 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:57 smithi143 ceph-mon[189871]: failed to encode map e187 with expected crc 2023-08-28T02:26:57.696 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:57 smithi143 ceph-mon[189871]: failed to encode map e187 with expected crc 2023-08-28T02:26:57.696 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:57 smithi143 ceph-mon[189871]: failed to encode map e187 with expected crc 2023-08-28T02:26:57.696 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:57 smithi143 ceph-mon[189871]: failed to encode map e187 with expected crc 2023-08-28T02:26:57.696 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:57 smithi143 ceph-mon[189871]: failed to encode map e187 with expected crc 2023-08-28T02:26:57.696 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:57 smithi143 ceph-mon[189871]: failed to encode map e187 with expected crc 2023-08-28T02:26:57.696 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:57 smithi143 ceph-mon[189871]: failed to encode map e187 with expected crc 2023-08-28T02:26:57.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:57 smithi169 ceph-mon[159415]: from='client.44987 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:26:57.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:57 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/4117489696' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-08-28T02:26:57.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:57 smithi169 ceph-mon[159415]: osdmap e187: 8 total, 8 up, 8 in 2023-08-28T02:26:57.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:57 smithi169 ceph-mon[159415]: failed to encode map e187 with expected crc 2023-08-28T02:26:57.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:57 smithi169 ceph-mon[159415]: failed to encode map e187 with expected crc 2023-08-28T02:26:57.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:57 smithi169 ceph-mon[159415]: failed to encode map e187 with expected crc 2023-08-28T02:26:57.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:57 smithi169 ceph-mon[159415]: failed to encode map e187 with expected crc 2023-08-28T02:26:57.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:57 smithi169 ceph-mon[159415]: failed to encode map e187 with expected crc 2023-08-28T02:26:57.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:57 smithi169 ceph-mon[159415]: failed to encode map e187 with expected crc 2023-08-28T02:26:57.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:57 smithi169 ceph-mon[159415]: failed to encode map e187 with expected crc 2023-08-28T02:26:57.753 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2023-08-28T02:26:58.047 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:26:58.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:58.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:26:58.077 INFO:teuthology.orchestra.run.smithi143.stdout:{"pg_ready":true,"pg_map":{"version":440,"stamp":"2023-08-28T02:26:55.181877+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6274,"num_read_kb":13707,"num_write":1106,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":5384814,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":234,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2213324,"kb_used_data":451276,"kb_used_omap":29,"kb_used_meta":1761954,"kb_avail":747747892,"statfs":{"total":767960285184,"available":765693841408,"internally_reserved":0,"allocated":462106624,"data_stored":456471227,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30674,"internal_metadata":1804240942},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":58,"apply_latency_ms":58,"commit_latency_ns":58000000,"apply_latency_ns":58000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1665,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":-220282,"num_write_kb":-7048961,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-8227,"num_bytes_recovered":-464453656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002022"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":169,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-08-28T02:26:49.357984+0000","last_change":"2023-08-28T02:20:19.849807+0000","last_active":"2023-08-28T02:26:49.357984+0000","last_peered":"2023-08-28T02:26:49.357984+0000","last_clean":"2023-08-28T02:26:49.357984+0000","last_became_active":"2023-08-28T02:20:19.849464+0000","last_became_peered":"2023-08-28T02:20:19.849464+0000","last_unstale":"2023-08-28T02:26:49.357984+0000","last_undegraded":"2023-08-28T02:26:49.357984+0000","last_fullsized":"2023-08-28T02:26:49.357984+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":44,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-08-28T02:26:49.999290+0000","last_change":"2023-08-28T02:23:59.204957+0000","last_active":"2023-08-28T02:26:49.999290+0000","last_peered":"2023-08-28T02:26:49.999290+0000","last_clean":"2023-08-28T02:26:49.999290+0000","last_became_active":"2023-08-28T02:23:59.204717+0000","last_became_peered":"2023-08-28T02:23:59.204717+0000","last_unstale":"2023-08-28T02:26:49.999290+0000","last_undegraded":"2023-08-28T02:26:49.999290+0000","last_fullsized":"2023-08-28T02:26:49.999290+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:34:58.977269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":144,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-08-28T02:26:49.343615+0000","last_change":"2023-08-28T02:23:59.212721+0000","last_active":"2023-08-28T02:26:49.343615+0000","last_peered":"2023-08-28T02:26:49.343615+0000","last_clean":"2023-08-28T02:26:49.343615+0000","last_became_active":"2023-08-28T02:23:59.212446+0000","last_became_peered":"2023-08-28T02:23:59.212446+0000","last_unstale":"2023-08-28T02:26:49.343615+0000","last_undegraded":"2023-08-28T02:26:49.343615+0000","last_fullsized":"2023-08-28T02:26:49.343615+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:03:54.701399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":172,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-08-28T02:26:39.768232+0000","last_change":"2023-08-28T02:23:59.223634+0000","last_active":"2023-08-28T02:26:39.768232+0000","last_peered":"2023-08-28T02:26:39.768232+0000","last_clean":"2023-08-28T02:26:39.768232+0000","last_became_active":"2023-08-28T02:23:59.222704+0000","last_became_peered":"2023-08-28T02:23:59.222704+0000","last_unstale":"2023-08-28T02:26:39.768232+0000","last_undegraded":"2023-08-28T02:26:39.768232+0000","last_fullsized":"2023-08-28T02:26:39.768232+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:51:52.752361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":44,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-08-28T02:26:49.998645+0000","last_change":"2023-08-28T02:23:59.220459+0000","last_active":"2023-08-28T02:26:49.998645+0000","last_peered":"2023-08-28T02:26:49.998645+0000","last_clean":"2023-08-28T02:26:49.998645+0000","last_became_active":"2023-08-28T02:23:59.219274+0000","last_became_peered":"2023-08-28T02:23:59.219274+0000","last_unstale":"2023-08-28T02:26:49.998645+0000","last_undegraded":"2023-08-28T02:26:49.998645+0000","last_fullsized":"2023-08-28T02:26:49.998645+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:31:53.889311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":180,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-08-28T02:26:49.357984+0000","last_change":"2023-08-28T02:21:59.183009+0000","last_active":"2023-08-28T02:26:49.357984+0000","last_peered":"2023-08-28T02:26:49.357984+0000","last_clean":"2023-08-28T02:26:49.357984+0000","last_became_active":"2023-08-28T02:21:59.182621+0000","last_became_peered":"2023-08-28T02:21:59.182621+0000","last_unstale":"2023-08-28T02:26:49.357984+0000","last_undegraded":"2023-08-28T02:26:49.357984+0000","last_fullsized":"2023-08-28T02:26:49.357984+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":152,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-08-28T02:26:39.768306+0000","last_change":"2023-08-28T02:23:59.227592+0000","last_active":"2023-08-28T02:26:39.768306+0000","last_peered":"2023-08-28T02:26:39.768306+0000","last_clean":"2023-08-28T02:26:39.768306+0000","last_became_active":"2023-08-28T02:23:59.227342+0000","last_became_peered":"2023-08-28T02:23:59.227342+0000","last_unstale":"2023-08-28T02:26:39.768306+0000","last_undegraded":"2023-08-28T02:26:39.768306+0000","last_fullsized":"2023-08-28T02:26:39.768306+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:08:19.874396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":44,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-08-28T02:26:49.998555+0000","last_change":"2023-08-28T02:23:59.220634+0000","last_active":"2023-08-28T02:26:49.998555+0000","last_peered":"2023-08-28T02:26:49.998555+0000","last_clean":"2023-08-28T02:26:49.998555+0000","last_became_active":"2023-08-28T02:23:59.220177+0000","last_became_peered":"2023-08-28T02:23:59.220177+0000","last_unstale":"2023-08-28T02:26:49.998555+0000","last_undegraded":"2023-08-28T02:26:49.998555+0000","last_fullsized":"2023-08-28T02:26:49.998555+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:16:26.906064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":170,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-08-28T02:26:39.714966+0000","last_change":"2023-08-28T02:23:38.339581+0000","last_active":"2023-08-28T02:26:39.714966+0000","last_peered":"2023-08-28T02:26:39.714966+0000","last_clean":"2023-08-28T02:26:39.714966+0000","last_became_active":"2023-08-28T02:23:38.339244+0000","last_became_peered":"2023-08-28T02:23:38.339244+0000","last_unstale":"2023-08-28T02:26:39.714966+0000","last_undegraded":"2023-08-28T02:26:39.714966+0000","last_fullsized":"2023-08-28T02:26:39.714966+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":44,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-08-28T02:26:49.998943+0000","last_change":"2023-08-28T02:23:59.220551+0000","last_active":"2023-08-28T02:26:49.998943+0000","last_peered":"2023-08-28T02:26:49.998943+0000","last_clean":"2023-08-28T02:26:49.998943+0000","last_became_active":"2023-08-28T02:23:59.219681+0000","last_became_peered":"2023-08-28T02:23:59.219681+0000","last_unstale":"2023-08-28T02:26:49.998943+0000","last_undegraded":"2023-08-28T02:26:49.998943+0000","last_fullsized":"2023-08-28T02:26:49.998943+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:39:58.811256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":71,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-08-28T02:26:39.867049+0000","last_change":"2023-08-28T02:23:59.316561+0000","last_active":"2023-08-28T02:26:39.867049+0000","last_peered":"2023-08-28T02:26:39.867049+0000","last_clean":"2023-08-28T02:26:39.867049+0000","last_became_active":"2023-08-28T02:23:59.227433+0000","last_became_peered":"2023-08-28T02:23:59.227433+0000","last_unstale":"2023-08-28T02:26:39.867049+0000","last_undegraded":"2023-08-28T02:26:39.867049+0000","last_fullsized":"2023-08-28T02:26:39.867049+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":128,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-08-28T02:26:49.344233+0000","last_change":"2023-08-28T02:20:19.851616+0000","last_active":"2023-08-28T02:26:49.344233+0000","last_peered":"2023-08-28T02:26:49.344233+0000","last_clean":"2023-08-28T02:26:49.344233+0000","last_became_active":"2023-08-28T02:20:19.851322+0000","last_became_peered":"2023-08-28T02:20:19.851322+0000","last_unstale":"2023-08-28T02:26:49.344233+0000","last_undegraded":"2023-08-28T02:26:49.344233+0000","last_fullsized":"2023-08-28T02:26:49.344233+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:03:37.565646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"86'22","reported_seq":198,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-08-28T02:26:49.359233+0000","last_change":"2023-08-28T02:23:38.368275+0000","last_active":"2023-08-28T02:26:49.359233+0000","last_peered":"2023-08-28T02:26:49.359233+0000","last_clean":"2023-08-28T02:26:49.359233+0000","last_became_active":"2023-08-28T02:23:38.342097+0000","last_became_peered":"2023-08-28T02:23:38.342097+0000","last_unstale":"2023-08-28T02:26:49.359233+0000","last_undegraded":"2023-08-28T02:26:49.359233+0000","last_fullsized":"2023-08-28T02:26:49.359233+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":166,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-08-28T02:26:49.357583+0000","last_change":"2023-08-28T02:19:51.204118+0000","last_active":"2023-08-28T02:26:49.357583+0000","last_peered":"2023-08-28T02:26:49.357583+0000","last_clean":"2023-08-28T02:26:49.357583+0000","last_became_active":"2023-08-28T02:19:51.203745+0000","last_became_peered":"2023-08-28T02:19:51.203745+0000","last_unstale":"2023-08-28T02:26:49.357583+0000","last_undegraded":"2023-08-28T02:26:49.357583+0000","last_fullsized":"2023-08-28T02:26:49.357583+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":56,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-08-28T02:26:39.866638+0000","last_change":"2023-08-28T02:23:38.983634+0000","last_active":"2023-08-28T02:26:39.866638+0000","last_peered":"2023-08-28T02:26:39.866638+0000","last_clean":"2023-08-28T02:26:39.866638+0000","last_became_active":"2023-08-28T02:23:38.344566+0000","last_became_peered":"2023-08-28T02:23:38.344566+0000","last_unstale":"2023-08-28T02:26:39.866638+0000","last_undegraded":"2023-08-28T02:26:39.866638+0000","last_fullsized":"2023-08-28T02:26:39.866638+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":44,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-08-28T02:26:49.999070+0000","last_change":"2023-08-28T02:23:59.216960+0000","last_active":"2023-08-28T02:26:49.999070+0000","last_peered":"2023-08-28T02:26:49.999070+0000","last_clean":"2023-08-28T02:26:49.999070+0000","last_became_active":"2023-08-28T02:23:59.215657+0000","last_became_peered":"2023-08-28T02:23:59.215657+0000","last_unstale":"2023-08-28T02:26:49.999070+0000","last_undegraded":"2023-08-28T02:26:49.999070+0000","last_fullsized":"2023-08-28T02:26:49.999070+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:31:14.407108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":170,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-08-28T02:26:39.767314+0000","last_change":"2023-08-28T02:23:59.223066+0000","last_active":"2023-08-28T02:26:39.767314+0000","last_peered":"2023-08-28T02:26:39.767314+0000","last_clean":"2023-08-28T02:26:39.767314+0000","last_became_active":"2023-08-28T02:23:59.222731+0000","last_became_peered":"2023-08-28T02:23:59.222731+0000","last_unstale":"2023-08-28T02:26:39.767314+0000","last_undegraded":"2023-08-28T02:26:39.767314+0000","last_fullsized":"2023-08-28T02:26:39.767314+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:48:37.558266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":212,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-08-28T02:26:39.714751+0000","last_change":"2023-08-28T02:23:59.210469+0000","last_active":"2023-08-28T02:26:39.714751+0000","last_peered":"2023-08-28T02:26:39.714751+0000","last_clean":"2023-08-28T02:26:39.714751+0000","last_became_active":"2023-08-28T02:23:59.210031+0000","last_became_peered":"2023-08-28T02:23:59.210031+0000","last_unstale":"2023-08-28T02:26:39.714751+0000","last_undegraded":"2023-08-28T02:26:39.714751+0000","last_fullsized":"2023-08-28T02:26:39.714751+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":50,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-08-28T02:26:39.866581+0000","last_change":"2023-08-28T02:23:38.370183+0000","last_active":"2023-08-28T02:26:39.866581+0000","last_peered":"2023-08-28T02:26:39.866581+0000","last_clean":"2023-08-28T02:26:39.866581+0000","last_became_active":"2023-08-28T02:23:38.368415+0000","last_became_peered":"2023-08-28T02:23:38.368415+0000","last_unstale":"2023-08-28T02:26:39.866581+0000","last_undegraded":"2023-08-28T02:26:39.866581+0000","last_fullsized":"2023-08-28T02:26:39.866581+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":171,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-08-28T02:26:39.768246+0000","last_change":"2023-08-28T02:23:59.220932+0000","last_active":"2023-08-28T02:26:39.768246+0000","last_peered":"2023-08-28T02:26:39.768246+0000","last_clean":"2023-08-28T02:26:39.768246+0000","last_became_active":"2023-08-28T02:23:59.220488+0000","last_became_peered":"2023-08-28T02:23:59.220488+0000","last_unstale":"2023-08-28T02:26:39.768246+0000","last_undegraded":"2023-08-28T02:26:39.768246+0000","last_fullsized":"2023-08-28T02:26:39.768246+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:53:09.216340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":160,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-08-28T02:26:49.357453+0000","last_change":"2023-08-28T02:18:39.778728+0000","last_active":"2023-08-28T02:26:49.357453+0000","last_peered":"2023-08-28T02:26:49.357453+0000","last_clean":"2023-08-28T02:26:49.357453+0000","last_became_active":"2023-08-28T02:18:39.778283+0000","last_became_peered":"2023-08-28T02:18:39.778283+0000","last_unstale":"2023-08-28T02:26:49.357453+0000","last_undegraded":"2023-08-28T02:26:49.357453+0000","last_fullsized":"2023-08-28T02:26:49.357453+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":44,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-08-28T02:26:49.999395+0000","last_change":"2023-08-28T02:23:59.216743+0000","last_active":"2023-08-28T02:26:49.999395+0000","last_peered":"2023-08-28T02:26:49.999395+0000","last_clean":"2023-08-28T02:26:49.999395+0000","last_became_active":"2023-08-28T02:23:59.216416+0000","last_became_peered":"2023-08-28T02:23:59.216416+0000","last_unstale":"2023-08-28T02:26:49.999395+0000","last_undegraded":"2023-08-28T02:26:49.999395+0000","last_fullsized":"2023-08-28T02:26:49.999395+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:25:13.056747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":54,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-08-28T02:26:39.867090+0000","last_change":"2023-08-28T02:23:38.657185+0000","last_active":"2023-08-28T02:26:39.867090+0000","last_peered":"2023-08-28T02:26:39.867090+0000","last_clean":"2023-08-28T02:26:39.867090+0000","last_became_active":"2023-08-28T02:23:38.339016+0000","last_became_peered":"2023-08-28T02:23:38.339016+0000","last_unstale":"2023-08-28T02:26:39.867090+0000","last_undegraded":"2023-08-28T02:26:39.867090+0000","last_fullsized":"2023-08-28T02:26:39.867090+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":154,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-08-28T02:26:39.714309+0000","last_change":"2023-08-28T02:23:38.339886+0000","last_active":"2023-08-28T02:26:39.714309+0000","last_peered":"2023-08-28T02:26:39.714309+0000","last_clean":"2023-08-28T02:26:39.714309+0000","last_became_active":"2023-08-28T02:23:38.339514+0000","last_became_peered":"2023-08-28T02:23:38.339514+0000","last_unstale":"2023-08-28T02:26:39.714309+0000","last_undegraded":"2023-08-28T02:26:39.714309+0000","last_fullsized":"2023-08-28T02:26:39.714309+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":54,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-08-28T02:26:39.867057+0000","last_change":"2023-08-28T02:23:38.555339+0000","last_active":"2023-08-28T02:26:39.867057+0000","last_peered":"2023-08-28T02:26:39.867057+0000","last_clean":"2023-08-28T02:26:39.867057+0000","last_became_active":"2023-08-28T02:23:38.337938+0000","last_became_peered":"2023-08-28T02:23:38.337938+0000","last_unstale":"2023-08-28T02:26:39.867057+0000","last_undegraded":"2023-08-28T02:26:39.867057+0000","last_fullsized":"2023-08-28T02:26:39.867057+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":194,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-08-28T02:26:49.361411+0000","last_change":"2023-08-28T02:23:59.208685+0000","last_active":"2023-08-28T02:26:49.361411+0000","last_peered":"2023-08-28T02:26:49.361411+0000","last_clean":"2023-08-28T02:26:49.361411+0000","last_became_active":"2023-08-28T02:23:59.208377+0000","last_became_peered":"2023-08-28T02:23:59.208377+0000","last_unstale":"2023-08-28T02:26:49.361411+0000","last_undegraded":"2023-08-28T02:26:49.361411+0000","last_fullsized":"2023-08-28T02:26:49.361411+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":54,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-08-28T02:26:39.866848+0000","last_change":"2023-08-28T02:23:38.749428+0000","last_active":"2023-08-28T02:26:39.866848+0000","last_peered":"2023-08-28T02:26:39.866848+0000","last_clean":"2023-08-28T02:26:39.866848+0000","last_became_active":"2023-08-28T02:23:38.339587+0000","last_became_peered":"2023-08-28T02:23:38.339587+0000","last_unstale":"2023-08-28T02:26:39.866848+0000","last_undegraded":"2023-08-28T02:26:39.866848+0000","last_fullsized":"2023-08-28T02:26:39.866848+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":212,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-08-28T02:26:49.359979+0000","last_change":"2023-08-28T02:23:38.340065+0000","last_active":"2023-08-28T02:26:49.359979+0000","last_peered":"2023-08-28T02:26:49.359979+0000","last_clean":"2023-08-28T02:26:49.359979+0000","last_became_active":"2023-08-28T02:23:38.328689+0000","last_became_peered":"2023-08-28T02:23:38.328689+0000","last_unstale":"2023-08-28T02:26:49.359979+0000","last_undegraded":"2023-08-28T02:26:49.359979+0000","last_fullsized":"2023-08-28T02:26:49.359979+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":170,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-08-28T02:26:39.714703+0000","last_change":"2023-08-28T02:23:59.211466+0000","last_active":"2023-08-28T02:26:39.714703+0000","last_peered":"2023-08-28T02:26:39.714703+0000","last_clean":"2023-08-28T02:26:39.714703+0000","last_became_active":"2023-08-28T02:23:59.211175+0000","last_became_peered":"2023-08-28T02:23:59.211175+0000","last_unstale":"2023-08-28T02:26:39.714703+0000","last_undegraded":"2023-08-28T02:26:39.714703+0000","last_fullsized":"2023-08-28T02:26:39.714703+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":146,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-08-28T02:26:39.766948+0000","last_change":"2023-08-28T02:19:51.205245+0000","last_active":"2023-08-28T02:26:39.766948+0000","last_peered":"2023-08-28T02:26:39.766948+0000","last_clean":"2023-08-28T02:26:39.766948+0000","last_became_active":"2023-08-28T02:19:51.204896+0000","last_became_peered":"2023-08-28T02:19:51.204896+0000","last_unstale":"2023-08-28T02:26:39.766948+0000","last_undegraded":"2023-08-28T02:26:39.766948+0000","last_fullsized":"2023-08-28T02:26:39.766948+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:51:24.634702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":48,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-08-28T02:26:49.998761+0000","last_change":"2023-08-28T02:23:59.230059+0000","last_active":"2023-08-28T02:26:49.998761+0000","last_peered":"2023-08-28T02:26:49.998761+0000","last_clean":"2023-08-28T02:26:49.998761+0000","last_became_active":"2023-08-28T02:23:59.217783+0000","last_became_peered":"2023-08-28T02:23:59.217783+0000","last_unstale":"2023-08-28T02:26:49.998761+0000","last_undegraded":"2023-08-28T02:26:49.998761+0000","last_fullsized":"2023-08-28T02:26:49.998761+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:00:12.145395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":179,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-08-28T02:26:49.358899+0000","last_change":"2023-08-28T02:23:59.211286+0000","last_active":"2023-08-28T02:26:49.358899+0000","last_peered":"2023-08-28T02:26:49.358899+0000","last_clean":"2023-08-28T02:26:49.358899+0000","last_became_active":"2023-08-28T02:23:59.210942+0000","last_became_peered":"2023-08-28T02:23:59.210942+0000","last_unstale":"2023-08-28T02:26:49.358899+0000","last_undegraded":"2023-08-28T02:26:49.358899+0000","last_fullsized":"2023-08-28T02:26:49.358899+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":181,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-08-28T02:26:49.360127+0000","last_change":"2023-08-28T02:21:59.179699+0000","last_active":"2023-08-28T02:26:49.360127+0000","last_peered":"2023-08-28T02:26:49.360127+0000","last_clean":"2023-08-28T02:26:49.360127+0000","last_became_active":"2023-08-28T02:21:59.179345+0000","last_became_peered":"2023-08-28T02:21:59.179345+0000","last_unstale":"2023-08-28T02:26:49.360127+0000","last_undegraded":"2023-08-28T02:26:49.360127+0000","last_fullsized":"2023-08-28T02:26:49.360127+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":205,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-08-28T02:26:49.358982+0000","last_change":"2023-08-28T02:22:44.083121+0000","last_active":"2023-08-28T02:26:49.358982+0000","last_peered":"2023-08-28T02:26:49.358982+0000","last_clean":"2023-08-28T02:26:49.358982+0000","last_became_active":"2023-08-28T02:22:44.082776+0000","last_became_peered":"2023-08-28T02:22:44.082776+0000","last_unstale":"2023-08-28T02:26:49.358982+0000","last_undegraded":"2023-08-28T02:26:49.358982+0000","last_fullsized":"2023-08-28T02:26:49.358982+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":95,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-08-28T02:26:49.361559+0000","last_change":"2023-08-28T02:23:38.347043+0000","last_active":"2023-08-28T02:26:49.361559+0000","last_peered":"2023-08-28T02:26:49.361559+0000","last_clean":"2023-08-28T02:26:49.361559+0000","last_became_active":"2023-08-28T02:23:38.346680+0000","last_became_peered":"2023-08-28T02:23:38.346680+0000","last_unstale":"2023-08-28T02:26:49.361559+0000","last_undegraded":"2023-08-28T02:26:49.361559+0000","last_fullsized":"2023-08-28T02:26:49.361559+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":115,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-08-28T02:26:39.766963+0000","last_change":"2023-08-28T02:19:51.204017+0000","last_active":"2023-08-28T02:26:39.766963+0000","last_peered":"2023-08-28T02:26:39.766963+0000","last_clean":"2023-08-28T02:26:39.766963+0000","last_became_active":"2023-08-28T02:19:51.203627+0000","last_became_peered":"2023-08-28T02:19:51.203627+0000","last_unstale":"2023-08-28T02:26:39.766963+0000","last_undegraded":"2023-08-28T02:26:39.766963+0000","last_fullsized":"2023-08-28T02:26:39.766963+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:14:06.231701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":44,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-08-28T02:26:49.998475+0000","last_change":"2023-08-28T02:23:59.216950+0000","last_active":"2023-08-28T02:26:49.998475+0000","last_peered":"2023-08-28T02:26:49.998475+0000","last_clean":"2023-08-28T02:26:49.998475+0000","last_became_active":"2023-08-28T02:23:59.216373+0000","last_became_peered":"2023-08-28T02:23:59.216373+0000","last_unstale":"2023-08-28T02:26:49.998475+0000","last_undegraded":"2023-08-28T02:26:49.998475+0000","last_fullsized":"2023-08-28T02:26:49.998475+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:13:14.736771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":164,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-08-28T02:26:49.344474+0000","last_change":"2023-08-28T02:23:59.220269+0000","last_active":"2023-08-28T02:26:49.344474+0000","last_peered":"2023-08-28T02:26:49.344474+0000","last_clean":"2023-08-28T02:26:49.344474+0000","last_became_active":"2023-08-28T02:23:59.219966+0000","last_became_peered":"2023-08-28T02:23:59.219966+0000","last_unstale":"2023-08-28T02:26:49.344474+0000","last_undegraded":"2023-08-28T02:26:49.344474+0000","last_fullsized":"2023-08-28T02:26:49.344474+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:16:08.119909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"78'126","reported_seq":663,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-08-28T02:26:49.347704+0000","last_change":"2023-08-28T02:23:39.351975+0000","last_active":"2023-08-28T02:26:49.347704+0000","last_peered":"2023-08-28T02:26:49.347704+0000","last_clean":"2023-08-28T02:26:49.347704+0000","last_became_active":"2023-08-28T02:23:39.351437+0000","last_became_peered":"2023-08-28T02:23:39.351437+0000","last_unstale":"2023-08-28T02:26:49.347704+0000","last_undegraded":"2023-08-28T02:26:49.347704+0000","last_fullsized":"2023-08-28T02:26:49.347704+0000","mapping_epoch":145,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":146,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:25:22.203120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":743,"num_read_kb":1331,"num_write":125,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":185,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-08-28T02:26:39.714873+0000","last_change":"2023-08-28T02:23:59.207148+0000","last_active":"2023-08-28T02:26:39.714873+0000","last_peered":"2023-08-28T02:26:39.714873+0000","last_clean":"2023-08-28T02:26:39.714873+0000","last_became_active":"2023-08-28T02:23:59.206802+0000","last_became_peered":"2023-08-28T02:23:59.206802+0000","last_unstale":"2023-08-28T02:26:39.714873+0000","last_undegraded":"2023-08-28T02:26:39.714873+0000","last_fullsized":"2023-08-28T02:26:39.714873+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":83,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-08-28T02:26:49.344546+0000","last_change":"2023-08-28T02:21:43.238044+0000","last_active":"2023-08-28T02:26:49.344546+0000","last_peered":"2023-08-28T02:26:49.344546+0000","last_clean":"2023-08-28T02:26:49.344546+0000","last_became_active":"2023-08-28T02:21:43.237792+0000","last_became_peered":"2023-08-28T02:21:43.237792+0000","last_unstale":"2023-08-28T02:26:49.344546+0000","last_undegraded":"2023-08-28T02:26:49.344546+0000","last_fullsized":"2023-08-28T02:26:49.344546+0000","mapping_epoch":115,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":116,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:11:35.488503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"78'204","reported_seq":1101,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-08-28T02:26:39.714478+0000","last_change":"2023-08-28T02:19:50.219949+0000","last_active":"2023-08-28T02:26:39.714478+0000","last_peered":"2023-08-28T02:26:39.714478+0000","last_clean":"2023-08-28T02:26:39.714478+0000","last_became_active":"2023-08-28T02:19:50.219356+0000","last_became_peered":"2023-08-28T02:19:50.219356+0000","last_unstale":"2023-08-28T02:26:39.714478+0000","last_undegraded":"2023-08-28T02:26:39.714478+0000","last_fullsized":"2023-08-28T02:26:39.714478+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":204,"log_dups_size":0,"ondisk_log_size":204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1521,"num_read_kb":2511,"num_write":178,"num_write_kb":1175,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":153,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-08-28T02:26:49.357749+0000","last_change":"2023-08-28T02:18:39.779607+0000","last_active":"2023-08-28T02:26:49.357749+0000","last_peered":"2023-08-28T02:26:49.357749+0000","last_clean":"2023-08-28T02:26:49.357749+0000","last_became_active":"2023-08-28T02:18:39.779303+0000","last_became_peered":"2023-08-28T02:18:39.779303+0000","last_unstale":"2023-08-28T02:26:49.357749+0000","last_undegraded":"2023-08-28T02:26:49.357749+0000","last_fullsized":"2023-08-28T02:26:49.357749+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":144,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-08-28T02:26:49.347632+0000","last_change":"2023-08-28T02:23:59.221979+0000","last_active":"2023-08-28T02:26:49.347632+0000","last_peered":"2023-08-28T02:26:49.347632+0000","last_clean":"2023-08-28T02:26:49.347632+0000","last_became_active":"2023-08-28T02:23:59.221619+0000","last_became_peered":"2023-08-28T02:23:59.221619+0000","last_unstale":"2023-08-28T02:26:49.347632+0000","last_undegraded":"2023-08-28T02:26:49.347632+0000","last_fullsized":"2023-08-28T02:26:49.347632+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:54:45.892284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"78'47","reported_seq":401,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-08-28T02:26:50.000376+0000","last_change":"2023-08-28T02:24:05.799647+0000","last_active":"2023-08-28T02:26:50.000376+0000","last_peered":"2023-08-28T02:26:50.000376+0000","last_clean":"2023-08-28T02:26:50.000376+0000","last_became_active":"2023-08-28T02:23:59.210481+0000","last_became_peered":"2023-08-28T02:23:59.210481+0000","last_unstale":"2023-08-28T02:26:50.000376+0000","last_undegraded":"2023-08-28T02:26:50.000376+0000","last_fullsized":"2023-08-28T02:26:50.000376+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":47,"log_dups_size":0,"ondisk_log_size":47,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:40:18.542375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":162,"num_read_kb":1281,"num_write":48,"num_write_kb":1153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":127,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-08-28T02:26:49.344073+0000","last_change":"2023-08-28T02:20:19.851591+0000","last_active":"2023-08-28T02:26:49.344073+0000","last_peered":"2023-08-28T02:26:49.344073+0000","last_clean":"2023-08-28T02:26:49.344073+0000","last_became_active":"2023-08-28T02:20:19.851291+0000","last_became_peered":"2023-08-28T02:20:19.851291+0000","last_unstale":"2023-08-28T02:26:49.344073+0000","last_undegraded":"2023-08-28T02:26:49.344073+0000","last_fullsized":"2023-08-28T02:26:49.344073+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:33:51.259707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":159,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-08-28T02:26:49.344404+0000","last_change":"2023-08-28T02:19:50.211156+0000","last_active":"2023-08-28T02:26:49.344404+0000","last_peered":"2023-08-28T02:26:49.344404+0000","last_clean":"2023-08-28T02:26:49.344404+0000","last_became_active":"2023-08-28T02:19:50.210398+0000","last_became_peered":"2023-08-28T02:19:50.210398+0000","last_unstale":"2023-08-28T02:26:49.344404+0000","last_undegraded":"2023-08-28T02:26:49.344404+0000","last_fullsized":"2023-08-28T02:26:49.344404+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:14:41.748197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"78'114","reported_seq":549,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-08-28T02:26:39.714646+0000","last_change":"2023-08-28T02:24:01.210739+0000","last_active":"2023-08-28T02:26:39.714646+0000","last_peered":"2023-08-28T02:26:39.714646+0000","last_clean":"2023-08-28T02:26:39.714646+0000","last_became_active":"2023-08-28T02:24:01.210386+0000","last_became_peered":"2023-08-28T02:24:01.210386+0000","last_unstale":"2023-08-28T02:26:39.714646+0000","last_undegraded":"2023-08-28T02:26:39.714646+0000","last_fullsized":"2023-08-28T02:26:39.714646+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":114,"log_dups_size":0,"ondisk_log_size":114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":358,"num_read_kb":1352,"num_write":102,"num_write_kb":1104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"86'139","reported_seq":343,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-08-28T02:26:39.866414+0000","last_change":"2023-08-28T02:24:04.276519+0000","last_active":"2023-08-28T02:26:39.866414+0000","last_peered":"2023-08-28T02:26:39.866414+0000","last_clean":"2023-08-28T02:26:39.866414+0000","last_became_active":"2023-08-28T02:24:04.276104+0000","last_became_peered":"2023-08-28T02:24:04.276104+0000","last_unstale":"2023-08-28T02:26:39.866414+0000","last_undegraded":"2023-08-28T02:26:39.866414+0000","last_fullsized":"2023-08-28T02:26:39.866414+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_clean_scrub_stamp":"2023-08-28T02:08:02.043718+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":44,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-08-28T02:26:49.998687+0000","last_change":"2023-08-28T02:23:59.219981+0000","last_active":"2023-08-28T02:26:49.998687+0000","last_peered":"2023-08-28T02:26:49.998687+0000","last_clean":"2023-08-28T02:26:49.998687+0000","last_became_active":"2023-08-28T02:23:59.219633+0000","last_became_peered":"2023-08-28T02:23:59.219633+0000","last_unstale":"2023-08-28T02:26:49.998687+0000","last_undegraded":"2023-08-28T02:26:49.998687+0000","last_fullsized":"2023-08-28T02:26:49.998687+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:56:14.172518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":154,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-08-28T02:26:49.357505+0000","last_change":"2023-08-28T02:23:38.341351+0000","last_active":"2023-08-28T02:26:49.357505+0000","last_peered":"2023-08-28T02:26:49.357505+0000","last_clean":"2023-08-28T02:26:49.357505+0000","last_became_active":"2023-08-28T02:23:38.340997+0000","last_became_peered":"2023-08-28T02:23:38.340997+0000","last_unstale":"2023-08-28T02:26:49.357505+0000","last_undegraded":"2023-08-28T02:26:49.357505+0000","last_fullsized":"2023-08-28T02:26:49.357505+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"78'148","reported_seq":634,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-08-28T02:26:39.866391+0000","last_change":"2023-08-28T02:24:01.211494+0000","last_active":"2023-08-28T02:26:39.866391+0000","last_peered":"2023-08-28T02:26:39.866391+0000","last_clean":"2023-08-28T02:26:39.866391+0000","last_became_active":"2023-08-28T02:24:01.210957+0000","last_became_peered":"2023-08-28T02:24:01.210957+0000","last_unstale":"2023-08-28T02:26:39.866391+0000","last_undegraded":"2023-08-28T02:26:39.866391+0000","last_fullsized":"2023-08-28T02:26:39.866391+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1092,"num_read_kb":1955,"num_write":127,"num_write_kb":5177,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":152,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-08-28T02:26:39.767118+0000","last_change":"2023-08-28T02:23:59.226986+0000","last_active":"2023-08-28T02:26:39.767118+0000","last_peered":"2023-08-28T02:26:39.767118+0000","last_clean":"2023-08-28T02:26:39.767118+0000","last_became_active":"2023-08-28T02:23:59.226529+0000","last_became_peered":"2023-08-28T02:23:59.226529+0000","last_unstale":"2023-08-28T02:26:39.767118+0000","last_undegraded":"2023-08-28T02:26:39.767118+0000","last_fullsized":"2023-08-28T02:26:39.767118+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:22:09.193619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":179,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-08-28T02:26:39.768153+0000","last_change":"2023-08-28T02:21:59.167858+0000","last_active":"2023-08-28T02:26:39.768153+0000","last_peered":"2023-08-28T02:26:39.768153+0000","last_clean":"2023-08-28T02:26:39.768153+0000","last_became_active":"2023-08-28T02:21:59.167436+0000","last_became_peered":"2023-08-28T02:21:59.167436+0000","last_unstale":"2023-08-28T02:26:39.768153+0000","last_undegraded":"2023-08-28T02:26:39.768153+0000","last_fullsized":"2023-08-28T02:26:39.768153+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:17:50.717282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"78'103","reported_seq":561,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-08-28T02:26:39.866905+0000","last_change":"2023-08-28T02:24:07.212128+0000","last_active":"2023-08-28T02:26:39.866905+0000","last_peered":"2023-08-28T02:26:39.866905+0000","last_clean":"2023-08-28T02:26:39.866905+0000","last_became_active":"2023-08-28T02:24:07.211562+0000","last_became_peered":"2023-08-28T02:24:07.211562+0000","last_unstale":"2023-08-28T02:26:39.866905+0000","last_undegraded":"2023-08-28T02:26:39.866905+0000","last_fullsized":"2023-08-28T02:26:39.866905+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":803,"num_read_kb":2080,"num_write":73,"num_write_kb":1244,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":127,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-08-28T02:26:49.347831+0000","last_change":"2023-08-28T02:20:19.850916+0000","last_active":"2023-08-28T02:26:49.347831+0000","last_peered":"2023-08-28T02:26:49.347831+0000","last_clean":"2023-08-28T02:26:49.347831+0000","last_became_active":"2023-08-28T02:20:19.850627+0000","last_became_peered":"2023-08-28T02:20:19.850627+0000","last_unstale":"2023-08-28T02:26:49.347831+0000","last_undegraded":"2023-08-28T02:26:49.347831+0000","last_fullsized":"2023-08-28T02:26:49.347831+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:19:14.346189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":138,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-08-28T02:26:39.768069+0000","last_change":"2023-08-28T02:23:38.329667+0000","last_active":"2023-08-28T02:26:39.768069+0000","last_peered":"2023-08-28T02:26:39.768069+0000","last_clean":"2023-08-28T02:26:39.768069+0000","last_became_active":"2023-08-28T02:23:38.329175+0000","last_became_peered":"2023-08-28T02:23:38.329175+0000","last_unstale":"2023-08-28T02:26:39.768069+0000","last_undegraded":"2023-08-28T02:26:39.768069+0000","last_fullsized":"2023-08-28T02:26:39.768069+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:21:34.815067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":48,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-08-28T02:26:49.999464+0000","last_change":"2023-08-28T02:23:59.232099+0000","last_active":"2023-08-28T02:26:49.999464+0000","last_peered":"2023-08-28T02:26:49.999464+0000","last_clean":"2023-08-28T02:26:49.999464+0000","last_became_active":"2023-08-28T02:23:59.219318+0000","last_became_peered":"2023-08-28T02:23:59.219318+0000","last_unstale":"2023-08-28T02:26:49.999464+0000","last_undegraded":"2023-08-28T02:26:49.999464+0000","last_fullsized":"2023-08-28T02:26:49.999464+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:59:30.093289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'112","reported_seq":431,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-08-28T02:26:49.344217+0000","last_change":"2023-08-28T02:19:50.218206+0000","last_active":"2023-08-28T02:26:49.344217+0000","last_peered":"2023-08-28T02:26:49.344217+0000","last_clean":"2023-08-28T02:26:49.344217+0000","last_became_active":"2023-08-28T02:19:50.217637+0000","last_became_peered":"2023-08-28T02:19:50.217637+0000","last_unstale":"2023-08-28T02:26:49.344217+0000","last_undegraded":"2023-08-28T02:26:49.344217+0000","last_fullsized":"2023-08-28T02:26:49.344217+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:12:06.224907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":444,"num_read_kb":473,"num_write":94,"num_write_kb":127,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":44,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-08-28T02:26:49.998473+0000","last_change":"2023-08-28T02:23:59.221034+0000","last_active":"2023-08-28T02:26:49.998473+0000","last_peered":"2023-08-28T02:26:49.998473+0000","last_clean":"2023-08-28T02:26:49.998473+0000","last_became_active":"2023-08-28T02:23:59.220730+0000","last_became_peered":"2023-08-28T02:23:59.220730+0000","last_unstale":"2023-08-28T02:26:49.998473+0000","last_undegraded":"2023-08-28T02:26:49.998473+0000","last_fullsized":"2023-08-28T02:26:49.998473+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:25:52.364370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"78'122","reported_seq":715,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-08-28T02:26:49.357898+0000","last_change":"2023-08-28T02:24:04.276850+0000","last_active":"2023-08-28T02:26:49.357898+0000","last_peered":"2023-08-28T02:26:49.357898+0000","last_clean":"2023-08-28T02:26:49.357898+0000","last_became_active":"2023-08-28T02:24:04.276367+0000","last_became_peered":"2023-08-28T02:24:04.276367+0000","last_unstale":"2023-08-28T02:26:49.357898+0000","last_undegraded":"2023-08-28T02:26:49.357898+0000","last_fullsized":"2023-08-28T02:26:49.357898+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":857,"num_read_kb":2224,"num_write":99,"num_write_kb":2312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":138,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-08-28T02:26:39.768167+0000","last_change":"2023-08-28T02:19:50.221001+0000","last_active":"2023-08-28T02:26:39.768167+0000","last_peered":"2023-08-28T02:26:39.768167+0000","last_clean":"2023-08-28T02:26:39.768167+0000","last_became_active":"2023-08-28T02:19:50.220611+0000","last_became_peered":"2023-08-28T02:19:50.220611+0000","last_unstale":"2023-08-28T02:26:39.768167+0000","last_undegraded":"2023-08-28T02:26:39.768167+0000","last_fullsized":"2023-08-28T02:26:39.768167+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:23:10.637630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":137,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-08-28T02:26:49.359509+0000","last_change":"2023-08-28T02:11:22.437146+0000","last_active":"2023-08-28T02:26:49.359509+0000","last_peered":"2023-08-28T02:26:49.359509+0000","last_clean":"2023-08-28T02:26:49.359509+0000","last_became_active":"2023-08-28T02:11:22.436798+0000","last_became_peered":"2023-08-28T02:11:22.436798+0000","last_unstale":"2023-08-28T02:26:49.359509+0000","last_undegraded":"2023-08-28T02:26:49.359509+0000","last_fullsized":"2023-08-28T02:26:49.359509+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":141,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-08-28T02:26:39.768075+0000","last_change":"2023-08-28T02:23:38.329474+0000","last_active":"2023-08-28T02:26:39.768075+0000","last_peered":"2023-08-28T02:26:39.768075+0000","last_clean":"2023-08-28T02:26:39.768075+0000","last_became_active":"2023-08-28T02:23:38.328505+0000","last_became_peered":"2023-08-28T02:23:38.328505+0000","last_unstale":"2023-08-28T02:26:39.768075+0000","last_undegraded":"2023-08-28T02:26:39.768075+0000","last_fullsized":"2023-08-28T02:26:39.768075+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:12:14.835809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":207,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-08-28T02:26:50.001496+0000","last_change":"2023-08-28T02:23:59.212036+0000","last_active":"2023-08-28T02:26:50.001496+0000","last_peered":"2023-08-28T02:26:50.001496+0000","last_clean":"2023-08-28T02:26:50.001496+0000","last_became_active":"2023-08-28T02:23:59.211709+0000","last_became_peered":"2023-08-28T02:23:59.211709+0000","last_unstale":"2023-08-28T02:26:50.001496+0000","last_undegraded":"2023-08-28T02:26:50.001496+0000","last_fullsized":"2023-08-28T02:26:50.001496+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T14:04:25.046626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":154,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-08-28T02:26:39.714897+0000","last_change":"2023-08-28T02:19:50.218760+0000","last_active":"2023-08-28T02:26:39.714897+0000","last_peered":"2023-08-28T02:26:39.714897+0000","last_clean":"2023-08-28T02:26:39.714897+0000","last_became_active":"2023-08-28T02:19:50.218376+0000","last_became_peered":"2023-08-28T02:19:50.218376+0000","last_unstale":"2023-08-28T02:26:39.714897+0000","last_undegraded":"2023-08-28T02:26:39.714897+0000","last_fullsized":"2023-08-28T02:26:39.714897+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":185,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-08-28T02:26:49.358369+0000","last_change":"2023-08-28T02:23:59.207822+0000","last_active":"2023-08-28T02:26:49.358369+0000","last_peered":"2023-08-28T02:26:49.358369+0000","last_clean":"2023-08-28T02:26:49.358369+0000","last_became_active":"2023-08-28T02:23:59.207475+0000","last_became_peered":"2023-08-28T02:23:59.207475+0000","last_unstale":"2023-08-28T02:26:49.358369+0000","last_undegraded":"2023-08-28T02:26:49.358369+0000","last_fullsized":"2023-08-28T02:26:49.358369+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":139,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-08-28T02:26:39.714957+0000","last_change":"2023-08-28T02:11:22.186024+0000","last_active":"2023-08-28T02:26:39.714957+0000","last_peered":"2023-08-28T02:26:39.714957+0000","last_clean":"2023-08-28T02:26:39.714957+0000","last_became_active":"2023-08-28T02:11:22.185500+0000","last_became_peered":"2023-08-28T02:11:22.185500+0000","last_unstale":"2023-08-28T02:26:39.714957+0000","last_undegraded":"2023-08-28T02:26:39.714957+0000","last_fullsized":"2023-08-28T02:26:39.714957+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":185,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-08-28T02:26:49.358129+0000","last_change":"2023-08-28T02:23:59.207929+0000","last_active":"2023-08-28T02:26:49.358129+0000","last_peered":"2023-08-28T02:26:49.358129+0000","last_clean":"2023-08-28T02:26:49.358129+0000","last_became_active":"2023-08-28T02:23:59.207589+0000","last_became_peered":"2023-08-28T02:23:59.207589+0000","last_unstale":"2023-08-28T02:26:49.358129+0000","last_undegraded":"2023-08-28T02:26:49.358129+0000","last_fullsized":"2023-08-28T02:26:49.358129+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":44,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-08-28T02:26:49.999352+0000","last_change":"2023-08-28T02:23:59.220437+0000","last_active":"2023-08-28T02:26:49.999352+0000","last_peered":"2023-08-28T02:26:49.999352+0000","last_clean":"2023-08-28T02:26:49.999352+0000","last_became_active":"2023-08-28T02:23:59.220091+0000","last_became_peered":"2023-08-28T02:23:59.220091+0000","last_unstale":"2023-08-28T02:26:49.999352+0000","last_undegraded":"2023-08-28T02:26:49.999352+0000","last_fullsized":"2023-08-28T02:26:49.999352+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:51:21.695566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":44,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-08-28T02:26:49.999031+0000","last_change":"2023-08-28T02:23:59.217794+0000","last_active":"2023-08-28T02:26:49.999031+0000","last_peered":"2023-08-28T02:26:49.999031+0000","last_clean":"2023-08-28T02:26:49.999031+0000","last_became_active":"2023-08-28T02:23:59.217231+0000","last_became_peered":"2023-08-28T02:23:59.217231+0000","last_unstale":"2023-08-28T02:26:49.999031+0000","last_undegraded":"2023-08-28T02:26:49.999031+0000","last_fullsized":"2023-08-28T02:26:49.999031+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:36:26.297410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":138,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-08-28T02:26:39.768280+0000","last_change":"2023-08-28T02:20:19.850252+0000","last_active":"2023-08-28T02:26:39.768280+0000","last_peered":"2023-08-28T02:26:39.768280+0000","last_clean":"2023-08-28T02:26:39.768280+0000","last_became_active":"2023-08-28T02:20:19.849913+0000","last_became_peered":"2023-08-28T02:20:19.849913+0000","last_unstale":"2023-08-28T02:26:39.768280+0000","last_undegraded":"2023-08-28T02:26:39.768280+0000","last_fullsized":"2023-08-28T02:26:39.768280+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:48:47.713465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":181,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-08-28T02:26:49.358825+0000","last_change":"2023-08-28T02:21:59.180201+0000","last_active":"2023-08-28T02:26:49.358825+0000","last_peered":"2023-08-28T02:26:49.358825+0000","last_clean":"2023-08-28T02:26:49.358825+0000","last_became_active":"2023-08-28T02:21:59.179863+0000","last_became_peered":"2023-08-28T02:21:59.179863+0000","last_unstale":"2023-08-28T02:26:49.358825+0000","last_undegraded":"2023-08-28T02:26:49.358825+0000","last_fullsized":"2023-08-28T02:26:49.358825+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":9774,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26810,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430415,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307776,"kb_used_data":6456,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93437376,"statfs":{"total":95995035648,"available":95679873024,"internally_reserved":0,"allocated":6610944,"data_stored":6026514,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:25:56 2023","interfaces":[{"interface":"back","average":{"1min":2.536,"5min":3.7240000000000002,"15min":1.5980000000000001},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":9.5679999999999996,"5min":116.755,"15min":116.755},"last":0.46600000000000003},{"interface":"front","average":{"1min":3.1320000000000001,"5min":4.1520000000000001,"15min":1.7889999999999999},"min":{"1min":0.28000000000000003,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":12.206,"5min":116.971,"15min":116.971},"last":0.33000000000000002}]},{"osd":1,"last update":"Mon Aug 28 02:26:03 2023","interfaces":[{"interface":"back","average":{"1min":1.5800000000000001,"5min":1.143,"15min":0.73299999999999998},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":9.0139999999999993,"5min":9.8710000000000004,"15min":9.8710000000000004},"last":0.49099999999999999},{"interface":"front","average":{"1min":3.0169999999999999,"5min":1.7889999999999999,"15min":0.98799999999999999},"min":{"1min":0.27700000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":11.140000000000001,"5min":11.696999999999999,"15min":11.696999999999999},"last":0.36499999999999999}]},{"osd":2,"last update":"Mon Aug 28 02:26:28 2023","interfaces":[{"interface":"back","average":{"1min":2.5600000000000001,"5min":4.2670000000000003,"15min":1.8049999999999999},"min":{"1min":0.29499999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":8.4540000000000006,"5min":119.571,"15min":119.571},"last":0.41599999999999998},{"interface":"front","average":{"1min":5.4619999999999997,"5min":5.2939999999999996,"15min":2.1549999999999998},"min":{"1min":0.35799999999999998,"5min":0.312,"15min":0.312},"max":{"1min":54.191000000000003,"5min":119.554,"15min":119.554},"last":0.48299999999999998}]},{"osd":3,"last update":"Mon Aug 28 02:26:03 2023","interfaces":[{"interface":"back","average":{"1min":2.2250000000000001,"5min":2.3260000000000001,"15min":1.2210000000000001},"min":{"1min":0.36799999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":6.5650000000000004,"5min":107.916,"15min":107.916},"last":0.35199999999999998},{"interface":"front","average":{"1min":2.548,"5min":2.9990000000000001,"15min":1.446},"min":{"1min":0.378,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":9.8460000000000001,"5min":112.00700000000001,"15min":112.00700000000001},"last":0.45400000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:26:28 2023","interfaces":[{"interface":"back","average":{"1min":2.9569999999999999,"5min":2.5219999999999998,"15min":1.163},"min":{"1min":0.32100000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":11.696,"5min":63.048000000000002,"15min":63.048000000000002},"last":0.42099999999999999},{"interface":"front","average":{"1min":2.4009999999999998,"5min":2.2559999999999998,"15min":1.0900000000000001},"min":{"1min":0.28899999999999998,"5min":0.247,"15min":0.247},"max":{"1min":10.840999999999999,"5min":62.912999999999997,"15min":62.912999999999997},"last":0.376}]},{"osd":5,"last update":"Mon Aug 28 02:26:28 2023","interfaces":[{"interface":"back","average":{"1min":2.3809999999999998,"5min":1.919,"15min":0.96699999999999997},"min":{"1min":0.28499999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":13.788,"5min":63.021000000000001,"15min":63.021000000000001},"last":0.434},{"interface":"front","average":{"1min":2.855,"5min":2.6259999999999999,"15min":1.244},"min":{"1min":0.36299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":7.4690000000000003,"5min":63.082999999999998,"15min":63.082999999999998},"last":0.39400000000000002}]},{"osd":6,"last update":"Mon Aug 28 02:26:28 2023","interfaces":[{"interface":"back","average":{"1min":2.5219999999999998,"5min":2.5779999999999998,"15min":1.2190000000000001},"min":{"1min":0.32100000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":9.7070000000000007,"5min":101.61,"15min":101.61},"last":0.38600000000000001},{"interface":"front","average":{"1min":2.4609999999999999,"5min":2.7519999999999998,"15min":1.3160000000000001},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":7.6239999999999997,"5min":101.621,"15min":101.621},"last":0.51200000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626642,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307764,"kb_used_data":2348,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93437388,"statfs":{"total":95995035648,"available":95679885312,"internally_reserved":0,"allocated":2404352,"data_stored":1831768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:26:02 2023","interfaces":[{"interface":"back","average":{"1min":3.48,"5min":4.3949999999999996,"15min":1.845},"min":{"1min":0.245,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":13.260999999999999,"5min":121.247,"15min":121.247},"last":0.45000000000000001},{"interface":"front","average":{"1min":4.4560000000000004,"5min":4.6369999999999996,"15min":1.9430000000000001},"min":{"1min":0.371,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":13.298999999999999,"5min":118.977,"15min":118.977},"last":0.54500000000000004}]},{"osd":1,"last update":"Mon Aug 28 02:26:02 2023","interfaces":[{"interface":"back","average":{"1min":2.0910000000000002,"5min":2.258,"15min":1.1100000000000001},"min":{"1min":0.307,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":6.7050000000000001,"5min":34.677999999999997,"15min":34.677999999999997},"last":0.57499999999999996},{"interface":"front","average":{"1min":2.7850000000000001,"5min":2.5009999999999999,"15min":1.2130000000000001},"min":{"1min":0.317,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":9.6799999999999997,"5min":34.348999999999997,"15min":34.348999999999997},"last":0.71199999999999997}]},{"osd":2,"last update":"Mon Aug 28 02:26:39 2023","interfaces":[{"interface":"back","average":{"1min":9.5429999999999993,"5min":4.0819999999999999,"15min":1.7669999999999999},"min":{"1min":0.34000000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":79.787000000000006,"5min":79.787000000000006,"15min":79.787000000000006},"last":0.64700000000000002},{"interface":"front","average":{"1min":10.275,"5min":4.056,"15min":1.736},"min":{"1min":0.35799999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":79.935000000000002,"5min":79.935000000000002,"15min":79.935000000000002},"last":0.53200000000000003}]},{"osd":3,"last update":"Mon Aug 28 02:26:02 2023","interfaces":[{"interface":"back","average":{"1min":2.2869999999999999,"5min":1.885,"15min":1.075},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":6.4699999999999998,"5min":13.103,"15min":13.103},"last":0.627},{"interface":"front","average":{"1min":2.508,"5min":2.0649999999999999,"15min":1.143},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":11.395,"5min":13.09,"15min":13.09},"last":0.754}]},{"osd":4,"last update":"Mon Aug 28 02:26:06 2023","interfaces":[{"interface":"back","average":{"1min":1.5880000000000001,"5min":1.669,"15min":0.86699999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":8.9580000000000002,"5min":10.637,"15min":10.637},"last":0.66600000000000004},{"interface":"front","average":{"1min":2.0819999999999999,"5min":1.8100000000000001,"15min":0.94299999999999995},"min":{"1min":0.29899999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":9.7690000000000001,"5min":10.055,"15min":10.055},"last":0.59899999999999998}]},{"osd":5,"last update":"Mon Aug 28 02:26:06 2023","interfaces":[{"interface":"back","average":{"1min":1.982,"5min":1.9399999999999999,"15min":0.96499999999999997},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":9.9809999999999999,"5min":20.759,"15min":20.759},"last":0.61599999999999999},{"interface":"front","average":{"1min":1.8220000000000001,"5min":1.484,"15min":0.85099999999999998},"min":{"1min":0.32200000000000001,"5min":0.311,"15min":0.28799999999999998},"max":{"1min":5.5030000000000001,"5min":7.3780000000000001,"15min":7.3780000000000001},"last":0.72899999999999998}]},{"osd":7,"last update":"Mon Aug 28 02:26:35 2023","interfaces":[{"interface":"back","average":{"1min":1.7490000000000001,"5min":1.5529999999999999,"15min":0.88700000000000001},"min":{"1min":0.34200000000000003,"5min":0.314,"15min":0.314},"max":{"1min":9.0830000000000002,"5min":9.0830000000000002,"15min":9.0830000000000002},"last":0.69999999999999996},{"interface":"front","average":{"1min":1.915,"5min":1.742,"15min":0.96999999999999997},"min":{"1min":0.38700000000000001,"5min":0.33500000000000002,"15min":0.29799999999999999},"max":{"1min":9.7110000000000003,"5min":9.7110000000000003,"15min":9.7110000000000003},"last":0.78600000000000003}]}]},{"osd":1,"up_from":98,"seq":420906795095,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":128160,"kb_used_data":6232,"kb_used_omap":7,"kb_used_meta":121912,"kb_avail":93616992,"statfs":{"total":95995035648,"available":95863799808,"internally_reserved":0,"allocated":6381568,"data_stored":5490216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":124838219},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:26:01 2023","interfaces":[{"interface":"back","average":{"1min":14.807,"5min":4.4260000000000002,"15min":1.6990000000000001},"min":{"1min":0.216,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":128.60599999999999,"5min":128.60599999999999,"15min":128.60599999999999},"last":0.36499999999999999},{"interface":"front","average":{"1min":14.028,"5min":4.4489999999999998,"15min":1.7190000000000001},"min":{"1min":0.23699999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":128.685,"5min":128.685,"15min":128.685},"last":0.27000000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:26:31 2023","interfaces":[{"interface":"back","average":{"1min":16.405999999999999,"5min":5.4660000000000002,"15min":2.0499999999999998},"min":{"1min":0.27600000000000002,"5min":0.26200000000000001,"15min":0.22700000000000001},"max":{"1min":135.25999999999999,"5min":135.25999999999999,"15min":135.25999999999999},"last":0.28899999999999998},{"interface":"front","average":{"1min":16.736000000000001,"5min":5.0179999999999998,"15min":1.9259999999999999},"min":{"1min":0.34300000000000003,"5min":0.221,"15min":0.221},"max":{"1min":135.10300000000001,"5min":135.10300000000001,"15min":135.10300000000001},"last":0.309}]},{"osd":3,"last update":"Mon Aug 28 02:26:01 2023","interfaces":[{"interface":"back","average":{"1min":2.1200000000000001,"5min":1.8140000000000001,"15min":0.84799999999999998},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":12.321999999999999,"5min":15.395,"15min":15.395},"last":0.34399999999999997},{"interface":"front","average":{"1min":3.7250000000000001,"5min":2.1480000000000001,"15min":1.0229999999999999},"min":{"1min":0.26900000000000002,"5min":0.254,"15min":0.254},"max":{"1min":22.361999999999998,"5min":22.361999999999998,"15min":22.361999999999998},"last":0.35199999999999998}]},{"osd":4,"last update":"Mon Aug 28 02:26:01 2023","interfaces":[{"interface":"back","average":{"1min":3.3570000000000002,"5min":1.9339999999999999,"15min":1.0620000000000001},"min":{"1min":0.32800000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":17.600000000000001,"5min":17.600000000000001,"15min":17.600000000000001},"last":0.57599999999999996},{"interface":"front","average":{"1min":2.5499999999999998,"5min":1.556,"15min":0.93200000000000005},"min":{"1min":0.36299999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":19.010000000000002,"5min":19.010000000000002,"15min":19.010000000000002},"last":0.58499999999999996}]},{"osd":5,"last update":"Mon Aug 28 02:26:01 2023","interfaces":[{"interface":"back","average":{"1min":1.754,"5min":2.0089999999999999,"15min":1.0940000000000001},"min":{"1min":0.312,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.0579999999999998,"5min":15.544,"15min":15.544},"last":0.495},{"interface":"front","average":{"1min":3.4529999999999998,"5min":1.839,"15min":1.04},"min":{"1min":0.38800000000000001,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":17.646999999999998,"5min":17.646999999999998,"15min":17.646999999999998},"last":0.64600000000000002}]},{"osd":6,"last update":"Mon Aug 28 02:26:01 2023","interfaces":[{"interface":"back","average":{"1min":1.992,"5min":2.3359999999999999,"15min":1.1910000000000001},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":6.0869999999999997,"5min":15.512,"15min":15.512},"last":0.59399999999999997},{"interface":"front","average":{"1min":3.3450000000000002,"5min":2.2810000000000001,"15min":1.169},"min":{"1min":0.39600000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":17.673999999999999,"5min":17.673999999999999,"15min":17.673999999999999},"last":0.53100000000000003}]},{"osd":7,"last update":"Mon Aug 28 02:26:01 2023","interfaces":[{"interface":"back","average":{"1min":3.1930000000000001,"5min":1.8440000000000001,"15min":1.0369999999999999},"min":{"1min":0.39900000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":19.061,"5min":19.061,"15min":19.061},"last":0.66100000000000003},{"interface":"front","average":{"1min":2.3500000000000001,"5min":2.1349999999999998,"15min":1.137},"min":{"1min":0.41899999999999998,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":6.1100000000000003,"5min":15.476000000000001,"15min":15.476000000000001},"last":0.54400000000000004}]}]},{"osd":0,"up_from":90,"seq":386547056741,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":618936,"kb_used_data":389936,"kb_used_omap":8,"kb_used_meta":228983,"kb_avail":93126216,"statfs":{"total":95995035648,"available":95361245184,"internally_reserved":0,"allocated":399294464,"data_stored":398438542,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":234478953},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Aug 28 02:26:09 2023","interfaces":[{"interface":"back","average":{"1min":2.516,"5min":2.754,"15min":1.153},"min":{"1min":0.184,"5min":0.184,"15min":0.184},"max":{"1min":11.282,"5min":40.313000000000002,"15min":40.313000000000002},"last":0.374},{"interface":"front","average":{"1min":3.8370000000000002,"5min":2.5899999999999999,"15min":1.111},"min":{"1min":0.23899999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":19.555,"5min":36.655000000000001,"15min":36.655000000000001},"last":0.35299999999999998}]},{"osd":2,"last update":"Mon Aug 28 02:26:28 2023","interfaces":[{"interface":"back","average":{"1min":3.492,"5min":2.5720000000000001,"15min":1.0920000000000001},"min":{"1min":0.32100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":8.4169999999999998,"5min":15.33,"15min":15.33},"last":0.40100000000000002},{"interface":"front","average":{"1min":6.0800000000000001,"5min":3.621,"15min":1.468},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":39.877000000000002,"5min":39.877000000000002,"15min":39.877000000000002},"last":0.311}]},{"osd":3,"last update":"Mon Aug 28 02:26:09 2023","interfaces":[{"interface":"back","average":{"1min":3.77,"5min":2.2090000000000001,"15min":0.97799999999999998},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":12.272,"5min":18.545999999999999,"15min":18.545999999999999},"last":2.3479999999999999},{"interface":"front","average":{"1min":3.3300000000000001,"5min":2.1309999999999998,"15min":0.97299999999999998},"min":{"1min":0.28999999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":12.474,"5min":14.534000000000001,"15min":14.534000000000001},"last":0.33700000000000002}]},{"osd":4,"last update":"Mon Aug 28 02:26:04 2023","interfaces":[{"interface":"back","average":{"1min":2.3700000000000001,"5min":1.8520000000000001,"15min":1.0600000000000001},"min":{"1min":0.29599999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":7.5979999999999999,"5min":9.6649999999999991,"15min":9.6649999999999991},"last":0.62},{"interface":"front","average":{"1min":2.7679999999999998,"5min":2.0299999999999998,"15min":1.1419999999999999},"min":{"1min":0.45700000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":12.305,"5min":12.305,"15min":12.305},"last":0.59399999999999997}]},{"osd":5,"last update":"Mon Aug 28 02:26:04 2023","interfaces":[{"interface":"back","average":{"1min":3.0529999999999999,"5min":2.508,"15min":1.268},"min":{"1min":0.41299999999999998,"5min":0.313,"15min":0.313},"max":{"1min":12.337999999999999,"5min":44.000999999999998,"15min":44.000999999999998},"last":0.51600000000000001},{"interface":"front","average":{"1min":3.177,"5min":2.423,"15min":1.238},"min":{"1min":0.438,"5min":0.33100000000000002,"15min":0.30599999999999999},"max":{"1min":12.334,"5min":43.953000000000003,"15min":43.953000000000003},"last":0.55400000000000005}]},{"osd":6,"last update":"Mon Aug 28 02:26:04 2023","interfaces":[{"interface":"back","average":{"1min":3.1970000000000001,"5min":2.5409999999999999,"15min":1.2789999999999999},"min":{"1min":0.42799999999999999,"5min":0.36299999999999999,"15min":0.32000000000000001},"max":{"1min":12.361000000000001,"5min":32.417999999999999,"15min":32.417999999999999},"last":0.54500000000000004},{"interface":"front","average":{"1min":3.0499999999999998,"5min":2.286,"15min":1.23},"min":{"1min":0.378,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":8.3829999999999991,"5min":32.130000000000003,"15min":32.130000000000003},"last":0.57699999999999996}]},{"osd":7,"last update":"Mon Aug 28 02:26:04 2023","interfaces":[{"interface":"back","average":{"1min":3.2690000000000001,"5min":2.1019999999999999,"15min":1.1619999999999999},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40000000000000002},"max":{"1min":8.9870000000000001,"5min":9.6020000000000003,"15min":9.6020000000000003},"last":0.65400000000000003},{"interface":"front","average":{"1min":3.649,"5min":2.3570000000000002,"15min":1.25},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":19.478999999999999,"5min":19.478999999999999,"15min":19.478999999999999},"last":0.60799999999999998}]}]},{"osd":2,"up_from":104,"seq":446676598865,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":132096,"kb_used_data":2040,"kb_used_omap":10,"kb_used_meta":130037,"kb_avail":93613056,"statfs":{"total":95995035648,"available":95859769344,"internally_reserved":0,"allocated":2088960,"data_stored":1280520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10792,"internal_metadata":133158360},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:26:33 2023","interfaces":[{"interface":"back","average":{"1min":8.6509999999999998,"5min":3.9180000000000001,"15min":1.5189999999999999},"min":{"1min":0.17699999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":51.920000000000002,"5min":51.920000000000002,"15min":51.920000000000002},"last":0.32100000000000001},{"interface":"front","average":{"1min":22.567,"5min":6.899,"15min":2.5419999999999998},"min":{"1min":0.45000000000000001,"5min":0.214,"15min":0.19900000000000001},"max":{"1min":147.286,"5min":147.286,"15min":147.286},"last":0.36899999999999999}]},{"osd":1,"last update":"Mon Aug 28 02:26:33 2023","interfaces":[{"interface":"back","average":{"1min":4.452,"5min":3.2450000000000001,"15min":1.321},"min":{"1min":0.23599999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":11.031000000000001,"5min":30.888000000000002,"15min":30.888000000000002},"last":0.28999999999999998},{"interface":"front","average":{"1min":3.9380000000000002,"5min":2.6509999999999998,"15min":1.1060000000000001},"min":{"1min":0.25800000000000001,"5min":0.21299999999999999,"15min":0.20999999999999999},"max":{"1min":20.259,"5min":31.648,"15min":31.648},"last":0.34399999999999997}]},{"osd":3,"last update":"Mon Aug 28 02:26:02 2023","interfaces":[{"interface":"back","average":{"1min":10.356,"5min":3.331,"15min":1.3819999999999999},"min":{"1min":0.309,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":128.63900000000001,"5min":128.63900000000001,"15min":128.63900000000001},"last":0.39500000000000002},{"interface":"front","average":{"1min":10.528,"5min":3.6179999999999999,"15min":1.464},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":128.75700000000001,"5min":128.75700000000001,"15min":128.75700000000001},"last":0.40799999999999997}]},{"osd":4,"last update":"Mon Aug 28 02:26:33 2023","interfaces":[{"interface":"back","average":{"1min":3.6869999999999998,"5min":2.391,"15min":1.1930000000000001},"min":{"1min":0.32200000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":17.024999999999999,"5min":17.024999999999999,"15min":17.024999999999999},"last":0.44700000000000001},{"interface":"front","average":{"1min":3.351,"5min":1.8700000000000001,"15min":1.056},"min":{"1min":0.28999999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":16.936,"5min":16.936,"15min":16.936},"last":0.58299999999999996}]},{"osd":5,"last update":"Mon Aug 28 02:26:33 2023","interfaces":[{"interface":"back","average":{"1min":10.657,"5min":3.5259999999999998,"15min":1.5860000000000001},"min":{"1min":0.38400000000000001,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":128.696,"5min":128.696,"15min":128.696},"last":0.48199999999999998},{"interface":"front","average":{"1min":9.4239999999999995,"5min":3.819,"15min":1.645},"min":{"1min":0.32100000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":128.84100000000001,"5min":128.84100000000001,"15min":128.84100000000001},"last":0.52600000000000002}]},{"osd":6,"last update":"Mon Aug 28 02:26:33 2023","interfaces":[{"interface":"back","average":{"1min":2.7410000000000001,"5min":2.0870000000000002,"15min":1.0960000000000001},"min":{"1min":0.307,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":11.285,"5min":11.285,"15min":11.285},"last":0.49099999999999999},{"interface":"front","average":{"1min":3.7509999999999999,"5min":2.3620000000000001,"15min":1.198},"min":{"1min":0.377,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":11.055999999999999,"5min":11.055999999999999,"15min":11.055999999999999},"last":0.57499999999999996}]},{"osd":7,"last update":"Mon Aug 28 02:26:33 2023","interfaces":[{"interface":"back","average":{"1min":9.8529999999999998,"5min":3.4609999999999999,"15min":1.5529999999999999},"min":{"1min":0.34499999999999997,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":128.911,"5min":128.911,"15min":128.911},"last":0.54400000000000004},{"interface":"front","average":{"1min":11.334,"5min":3.819,"15min":1.6759999999999999},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":128.72399999999999,"5min":128.72399999999999,"15min":128.72399999999999},"last":0.51500000000000001}]}]},{"osd":3,"up_from":112,"seq":481036337227,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":77800,"kb_used_data":2016,"kb_used_omap":3,"kb_used_meta":75772,"kb_avail":93667352,"statfs":{"total":95995035648,"available":95915368448,"internally_reserved":0,"allocated":2064384,"data_stored":1275732,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":77591458},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:26:00 2023","interfaces":[{"interface":"back","average":{"1min":1.96,"5min":5.8419999999999996,"15min":2.1800000000000002},"min":{"1min":0.25600000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":8.1150000000000002,"5min":164.25299999999999,"15min":164.25299999999999},"last":0.26200000000000001},{"interface":"front","average":{"1min":2.085,"5min":6.0650000000000004,"15min":2.2429999999999999},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":9.0530000000000008,"5min":164.55000000000001,"15min":164.55000000000001},"last":0.29699999999999999}]},{"osd":1,"last update":"Mon Aug 28 02:26:00 2023","interfaces":[{"interface":"back","average":{"1min":3.25,"5min":3.9390000000000001,"15min":1.5389999999999999},"min":{"1min":0.251,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":13.936,"5min":117.34699999999999,"15min":117.34699999999999},"last":0.313},{"interface":"front","average":{"1min":1.8089999999999999,"5min":3.6619999999999999,"15min":1.474},"min":{"1min":0.28100000000000003,"5min":0.224,"15min":0.224},"max":{"1min":7.2679999999999998,"5min":118.012,"15min":118.012},"last":0.34799999999999998}]},{"osd":2,"last update":"Mon Aug 28 02:26:00 2023","interfaces":[{"interface":"back","average":{"1min":3.6829999999999998,"5min":5.5730000000000004,"15min":2.1070000000000002},"min":{"1min":0.314,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":18.718,"5min":120.43300000000001,"15min":120.43300000000001},"last":0.32300000000000001},{"interface":"front","average":{"1min":3.2949999999999999,"5min":5.5949999999999998,"15min":2.1160000000000001},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":18.774000000000001,"5min":120.44499999999999,"15min":120.44499999999999},"last":0.33500000000000002}]},{"osd":4,"last update":"Mon Aug 28 02:26:00 2023","interfaces":[{"interface":"back","average":{"1min":2.7559999999999998,"5min":1.681,"15min":0.97899999999999998},"min":{"1min":0.35199999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":9.016,"5min":9.016,"15min":9.016},"last":0.42099999999999999},{"interface":"front","average":{"1min":3.0409999999999999,"5min":1.669,"15min":0.97999999999999998},"min":{"1min":0.37,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":10.428000000000001,"5min":10.428000000000001,"15min":10.428000000000001},"last":0.51900000000000002}]},{"osd":5,"last update":"Mon Aug 28 02:26:00 2023","interfaces":[{"interface":"back","average":{"1min":2.4460000000000002,"5min":1.488,"15min":0.92000000000000004},"min":{"1min":0.41399999999999998,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":8.9789999999999992,"5min":8.9789999999999992,"15min":8.9789999999999992},"last":0.59999999999999998},{"interface":"front","average":{"1min":1.772,"5min":1.333,"15min":0.86399999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":13.331,"5min":13.331,"15min":13.331},"last":0.60999999999999999}]},{"osd":6,"last update":"Mon Aug 28 02:26:00 2023","interfaces":[{"interface":"back","average":{"1min":2.427,"5min":2.1440000000000001,"15min":1.1379999999999999},"min":{"1min":0.36799999999999999,"5min":0.314,"15min":0.314},"max":{"1min":9.4529999999999994,"5min":72.116,"15min":72.116},"last":0.46000000000000002},{"interface":"front","average":{"1min":2.0459999999999998,"5min":2.2210000000000001,"15min":1.139},"min":{"1min":0.38,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":6.7110000000000003,"5min":72.030000000000001,"15min":72.030000000000001},"last":0.40300000000000002}]},{"osd":7,"last update":"Mon Aug 28 02:26:00 2023","interfaces":[{"interface":"back","average":{"1min":2.9329999999999998,"5min":1.7250000000000001,"15min":1.0129999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":11.949999999999999,"5min":11.949999999999999,"15min":11.949999999999999},"last":0.49399999999999999},{"interface":"front","average":{"1min":3.702,"5min":1.907,"15min":1.0660000000000001},"min":{"1min":0.438,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":13.890000000000001,"5min":13.890000000000001,"15min":13.890000000000001},"last":0.5}]}]},{"osd":4,"up_from":30,"seq":128849019100,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":303092,"kb_used_data":1772,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93442060,"statfs":{"total":95995035648,"available":95684669440,"internally_reserved":0,"allocated":1814528,"data_stored":1241750,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":12,"apply_latency_ms":12,"commit_latency_ns":12000000,"apply_latency_ns":12000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:25:59 2023","interfaces":[{"interface":"back","average":{"1min":6.2859999999999996,"5min":5.8010000000000002,"15min":2.3399999999999999},"min":{"1min":0.26500000000000001,"5min":0.223,"15min":0.223},"max":{"1min":43.548000000000002,"5min":138.67400000000001,"15min":138.67400000000001},"last":0.432},{"interface":"front","average":{"1min":5.8789999999999996,"5min":5.6669999999999998,"15min":2.2839999999999998},"min":{"1min":0.25,"5min":0.248,"15min":0.248},"max":{"1min":43.515000000000001,"5min":134.14599999999999,"15min":134.14599999999999},"last":0.38200000000000001}]},{"osd":1,"last update":"Mon Aug 28 02:25:59 2023","interfaces":[{"interface":"back","average":{"1min":2.6509999999999998,"5min":1.915,"15min":0.98699999999999999},"min":{"1min":0.32700000000000001,"5min":0.255,"15min":0.255},"max":{"1min":8.9570000000000007,"5min":53.622999999999998,"15min":53.622999999999998},"last":0.40400000000000003},{"interface":"front","average":{"1min":2.1339999999999999,"5min":1.8999999999999999,"15min":1.01},"min":{"1min":0.28000000000000003,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":7.1580000000000004,"5min":53.774000000000001,"15min":53.774000000000001},"last":0.35999999999999999}]},{"osd":2,"last update":"Mon Aug 28 02:26:30 2023","interfaces":[{"interface":"back","average":{"1min":6.0960000000000001,"5min":4.7309999999999999,"15min":1.9430000000000001},"min":{"1min":0.30499999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":38.488999999999997,"5min":74.099000000000004,"15min":74.099000000000004},"last":0.36899999999999999},{"interface":"front","average":{"1min":4.6980000000000004,"5min":4.3129999999999997,"15min":1.7969999999999999},"min":{"1min":0.45500000000000002,"5min":0.25,"15min":0.25},"max":{"1min":35.134999999999998,"5min":74.081999999999994,"15min":74.081999999999994},"last":0.33000000000000002}]},{"osd":3,"last update":"Mon Aug 28 02:25:59 2023","interfaces":[{"interface":"back","average":{"1min":3.3359999999999999,"5min":1.5149999999999999,"15min":0.83899999999999997},"min":{"1min":0.32400000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":8.6349999999999998,"5min":8.6349999999999998,"15min":8.6349999999999998},"last":0.42299999999999999},{"interface":"front","average":{"1min":2.9430000000000001,"5min":1.6870000000000001,"15min":0.95099999999999996},"min":{"1min":0.39900000000000002,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":10.956,"5min":10.956,"15min":10.956},"last":0.45400000000000001}]},{"osd":5,"last update":"Mon Aug 28 02:26:41 2023","interfaces":[{"interface":"back","average":{"1min":2.5350000000000001,"5min":1.383,"15min":0.77900000000000003},"min":{"1min":0.34899999999999998,"5min":0.249,"15min":0.249},"max":{"1min":7.7350000000000003,"5min":8.2560000000000002,"15min":8.2560000000000002},"last":0.26000000000000001},{"interface":"front","average":{"1min":3.2109999999999999,"5min":1.875,"15min":0.94999999999999996},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":16.497,"5min":16.497,"15min":16.497},"last":0.29399999999999998}]},{"osd":6,"last update":"Mon Aug 28 02:25:59 2023","interfaces":[{"interface":"back","average":{"1min":2.1059999999999999,"5min":1.373,"15min":0.79600000000000004},"min":{"1min":0.32300000000000001,"5min":0.29199999999999998,"15min":0.27500000000000002},"max":{"1min":11.170999999999999,"5min":11.170999999999999,"15min":11.170999999999999},"last":0.314},{"interface":"front","average":{"1min":1.403,"5min":0.97199999999999998,"15min":0.69199999999999995},"min":{"1min":0.33700000000000002,"5min":0.32200000000000001,"15min":0.318},"max":{"1min":7.3170000000000002,"5min":7.3170000000000002,"15min":7.3170000000000002},"last":0.41299999999999998}]},{"osd":7,"last update":"Mon Aug 28 02:26:24 2023","interfaces":[{"interface":"back","average":{"1min":4.2800000000000002,"5min":1.8400000000000001,"15min":0.97599999999999998},"min":{"1min":0.41399999999999998,"5min":0.33900000000000002,"15min":0.312},"max":{"1min":16.584,"5min":16.584,"15min":16.584},"last":0.33900000000000002},{"interface":"front","average":{"1min":2.3399999999999999,"5min":1.79,"15min":0.97199999999999998},"min":{"1min":0.371,"5min":0.32500000000000001,"15min":0.32100000000000001},"max":{"1min":7.6360000000000001,"5min":8.7569999999999997,"15min":8.7569999999999997},"last":0.47199999999999998}]}]},{"osd":5,"up_from":36,"seq":154618822872,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":337700,"kb_used_data":40476,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":93407452,"statfs":{"total":95995035648,"available":95649230848,"internally_reserved":0,"allocated":41447424,"data_stored":40886185,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:25:52 2023","interfaces":[{"interface":"back","average":{"1min":3.5249999999999999,"5min":5.8579999999999997,"15min":2.3639999999999999},"min":{"1min":0.24099999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":12.874000000000001,"5min":131.06100000000001,"15min":131.06100000000001},"last":0.61199999999999999},{"interface":"front","average":{"1min":3.758,"5min":6.1749999999999998,"15min":2.4740000000000002},"min":{"1min":0.24199999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":15.691000000000001,"5min":131.262,"15min":131.262},"last":2.9390000000000001}]},{"osd":1,"last update":"Mon Aug 28 02:26:03 2023","interfaces":[{"interface":"back","average":{"1min":1.298,"5min":5.0019999999999998,"15min":2.0190000000000001},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":5.7240000000000002,"5min":183.97900000000001,"15min":183.97900000000001},"last":0.53000000000000003},{"interface":"front","average":{"1min":2.4929999999999999,"5min":5.4960000000000004,"15min":2.1920000000000002},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":8.5640000000000001,"5min":184.33699999999999,"15min":184.33699999999999},"last":0.45700000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:26:33 2023","interfaces":[{"interface":"back","average":{"1min":15.742000000000001,"5min":8.2479999999999993,"15min":3.1419999999999999},"min":{"1min":0.315,"5min":0.252,"15min":0.252},"max":{"1min":176.69499999999999,"5min":176.69499999999999,"15min":176.69499999999999},"last":3.02},{"interface":"front","average":{"1min":16.274999999999999,"5min":8.6199999999999992,"15min":3.2650000000000001},"min":{"1min":1.492,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":176.71199999999999,"5min":176.71199999999999,"15min":176.71199999999999},"last":3.012}]},{"osd":3,"last update":"Mon Aug 28 02:26:03 2023","interfaces":[{"interface":"back","average":{"1min":2.004,"5min":1.649,"15min":0.95699999999999996},"min":{"1min":0.28899999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.0039999999999996,"5min":10.576000000000001,"15min":10.576000000000001},"last":0.496},{"interface":"front","average":{"1min":3.5499999999999998,"5min":2.177,"15min":1.143},"min":{"1min":0.34300000000000003,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":12.118,"5min":12.118,"15min":12.118},"last":0.51000000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:26:45 2023","interfaces":[{"interface":"back","average":{"1min":1.3779999999999999,"5min":1.522,"15min":0.82399999999999995},"min":{"1min":0.30299999999999999,"5min":0.246,"15min":0.246},"max":{"1min":5.8159999999999998,"5min":10.759,"15min":10.759},"last":0.47699999999999998},{"interface":"front","average":{"1min":1.069,"5min":1.5720000000000001,"15min":0.85999999999999999},"min":{"1min":0.312,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.4859999999999998,"5min":8.6769999999999996,"15min":8.6769999999999996},"last":0.64100000000000001}]},{"osd":6,"last update":"Mon Aug 28 02:26:03 2023","interfaces":[{"interface":"back","average":{"1min":1.968,"5min":1.694,"15min":0.91900000000000004},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.28399999999999997},"max":{"1min":5.9870000000000001,"5min":14.803000000000001,"15min":14.803000000000001},"last":0.56699999999999995},{"interface":"front","average":{"1min":1.6639999999999999,"5min":1.389,"15min":0.81699999999999995},"min":{"1min":0.33500000000000002,"5min":0.31,"15min":0.30299999999999999},"max":{"1min":8.4779999999999998,"5min":8.6479999999999997,"15min":8.6479999999999997},"last":0.57899999999999996}]},{"osd":7,"last update":"Mon Aug 28 02:26:19 2023","interfaces":[{"interface":"back","average":{"1min":2.8159999999999998,"5min":1.75,"15min":0.95499999999999996},"min":{"1min":0.33500000000000002,"5min":0.31,"15min":0.31},"max":{"1min":9.9949999999999992,"5min":9.9949999999999992,"15min":9.9949999999999992},"last":0.54300000000000004},{"interface":"front","average":{"1min":1.54,"5min":1.5229999999999999,"15min":0.874},"min":{"1min":0.37,"5min":0.32100000000000001,"15min":0.29799999999999999},"max":{"1min":4.9889999999999999,"5min":12.787000000000001,"15min":12.787000000000001},"last":0.55500000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-08-28T02:26:58.083 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-08-28T02:26:58.083 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-08-28T02:26:58.083 INFO:tasks.thrashosds.thrasher:Removing osd 1, in_osds are: [0, 1, 2, 4, 5, 6, 7, 3] 2023-08-28T02:26:58.083 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd out 1 2023-08-28T02:26:58.151 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:26:58.183 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:58.231 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:26:58.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:58.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:26:58.339 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:26:58.365 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:58.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:58 smithi169 ceph-mon[159415]: pgmap v442: 89 pgs: 3 creating+peering, 13 unknown, 73 active+clean; 4.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:26:58.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:58 smithi169 ceph-mon[159415]: osdmap e188: 8 total, 8 up, 8 in 2023-08-28T02:26:58.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:58 smithi169 ceph-mon[159415]: failed to encode map e188 with expected crc 2023-08-28T02:26:58.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:58 smithi169 ceph-mon[159415]: failed to encode map e188 with expected crc 2023-08-28T02:26:58.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:58 smithi169 ceph-mon[159415]: failed to encode map e188 with expected crc 2023-08-28T02:26:58.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:58 smithi169 ceph-mon[159415]: failed to encode map e188 with expected crc 2023-08-28T02:26:58.564 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:26:58.568 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:58 smithi143 ceph-mon[183059]: pgmap v442: 89 pgs: 3 creating+peering, 13 unknown, 73 active+clean; 4.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:26:58.568 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:58 smithi143 ceph-mon[183059]: osdmap e188: 8 total, 8 up, 8 in 2023-08-28T02:26:58.568 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:58 smithi143 ceph-mon[183059]: failed to encode map e188 with expected crc 2023-08-28T02:26:58.568 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:58 smithi143 ceph-mon[183059]: failed to encode map e188 with expected crc 2023-08-28T02:26:58.568 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:58 smithi143 ceph-mon[183059]: failed to encode map e188 with expected crc 2023-08-28T02:26:58.568 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:58 smithi143 ceph-mon[183059]: failed to encode map e188 with expected crc 2023-08-28T02:26:58.568 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:58 smithi143 ceph-mon[189871]: pgmap v442: 89 pgs: 3 creating+peering, 13 unknown, 73 active+clean; 4.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:26:58.568 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:58 smithi143 ceph-mon[189871]: osdmap e188: 8 total, 8 up, 8 in 2023-08-28T02:26:58.568 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:58 smithi143 ceph-mon[189871]: failed to encode map e188 with expected crc 2023-08-28T02:26:58.569 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:58 smithi143 ceph-mon[189871]: failed to encode map e188 with expected crc 2023-08-28T02:26:58.569 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:58 smithi143 ceph-mon[189871]: failed to encode map e188 with expected crc 2023-08-28T02:26:58.569 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:58 smithi143 ceph-mon[189871]: failed to encode map e188 with expected crc 2023-08-28T02:26:58.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:58.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:26:58.670 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:26:58.703 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:58.757 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:26:58.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:58.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:26:58.861 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:26:58.893 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:58.941 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:26:58.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:58.947 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:26:59.020 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:26:59.048 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:26:59.084 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:59.208 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:26:59.236 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:26:59.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:26:59.241 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:26:59.312 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:26:59.312 INFO:teuthology.orchestra.run.smithi143.stdout: "size": 20, 2023-08-28T02:26:59.312 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 600, 2023-08-28T02:26:59.312 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [ 2023-08-28T02:26:59.312 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.312 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:123521 5.0 5:0f371aae:::benchmark_data_smithi143_236200_object13378:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e154)", 2023-08-28T02:26:59.312 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:59.196550+0000", 2023-08-28T02:26:59.313 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 180.11393552000001, 2023-08-28T02:26:59.313 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.128653905, 2023-08-28T02:26:59.313 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:26:59.313 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:59.313 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:26:59.313 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:26:59.313 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:59.313 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 123521 2023-08-28T02:26:59.313 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.313 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:26:59.313 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.313 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:26:59.314 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196550+0000", 2023-08-28T02:26:59.314 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:26:59.314 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.314 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.314 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:26:59.314 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196550+0000", 2023-08-28T02:26:59.314 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:26:59.314 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.314 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.314 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:26:59.314 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196548+0000", 2023-08-28T02:26:59.314 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:26:59.315 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.315 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.315 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:26:59.315 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196553+0000", 2023-08-28T02:26:59.315 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.6439999999999999e-06 2023-08-28T02:26:59.315 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.315 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.315 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:26:59.315 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196564+0000", 2023-08-28T02:26:59.315 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0550999999999999e-05 2023-08-28T02:26:59.315 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.315 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.315 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:26:59.316 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196573+0000", 2023-08-28T02:26:59.316 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.8619999999999998e-06 2023-08-28T02:26:59.316 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.316 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.316 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:26:59.316 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196799+0000", 2023-08-28T02:26:59.316 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000225913 2023-08-28T02:26:59.316 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.316 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.316 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op must wait for map", 2023-08-28T02:26:59.316 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196806+0000", 2023-08-28T02:26:59.316 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.0720000000000002e-06 2023-08-28T02:26:59.317 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.317 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.317 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:26:59.317 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.208800+0000", 2023-08-28T02:26:59.317 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.011993996999999999 2023-08-28T02:26:59.317 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.317 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.317 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:26:59.317 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.208811+0000", 2023-08-28T02:26:59.317 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.048e-05 2023-08-28T02:26:59.317 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.317 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.318 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:26:59.318 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.305571+0000", 2023-08-28T02:26:59.318 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.096759813 2023-08-28T02:26:59.318 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.318 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.318 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:26:59.318 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.307358+0000", 2023-08-28T02:26:59.318 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0017876599999999999 2023-08-28T02:26:59.318 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.318 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.318 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,3,4", 2023-08-28T02:26:59.318 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.307473+0000", 2023-08-28T02:26:59.318 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000114498 2023-08-28T02:26:59.319 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.319 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.319 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.319 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.317779+0000", 2023-08-28T02:26:59.319 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.010306031 2023-08-28T02:26:59.319 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.319 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.319 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.319 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.317817+0000", 2023-08-28T02:26:59.319 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.7766999999999999e-05 2023-08-28T02:26:59.319 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.319 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.320 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.320 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.324897+0000", 2023-08-28T02:26:59.320 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0070804420000000002 2023-08-28T02:26:59.320 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.320 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.320 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:26:59.320 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325027+0000", 2023-08-28T02:26:59.320 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00012982799999999999 2023-08-28T02:26:59.320 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.320 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.320 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:26:59.320 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325088+0000", 2023-08-28T02:26:59.320 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.1532999999999995e-05 2023-08-28T02:26:59.321 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.321 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.321 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:26:59.321 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325204+0000", 2023-08-28T02:26:59.321 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00011503299999999999 2023-08-28T02:26:59.321 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:26:59.321 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:26:59.321 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:26:59.321 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.321 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.321 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:123522 5.0 5:0bc2203e:::benchmark_data_smithi143_236200_object13379:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e154)", 2023-08-28T02:26:59.321 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:59.196585+0000", 2023-08-28T02:26:59.321 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 180.113900468, 2023-08-28T02:26:59.321 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1287665410000001, 2023-08-28T02:26:59.322 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:26:59.322 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:59.322 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:26:59.322 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:26:59.322 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:59.322 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 123522 2023-08-28T02:26:59.322 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.322 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:26:59.322 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.322 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:26:59.322 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196585+0000", 2023-08-28T02:26:59.322 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:26:59.322 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.323 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.323 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:26:59.323 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196585+0000", 2023-08-28T02:26:59.323 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:26:59.323 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.323 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.323 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:26:59.323 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196584+0000", 2023-08-28T02:26:59.323 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:26:59.323 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.323 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.323 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:26:59.324 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196587+0000", 2023-08-28T02:26:59.324 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.238e-06 2023-08-28T02:26:59.324 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.324 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.324 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:26:59.324 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196593+0000", 2023-08-28T02:26:59.324 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.9710000000000001e-06 2023-08-28T02:26:59.324 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.324 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.324 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:26:59.324 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196598+0000", 2023-08-28T02:26:59.324 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.1120000000000004e-06 2023-08-28T02:26:59.325 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.325 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.325 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:26:59.325 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196858+0000", 2023-08-28T02:26:59.325 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00026082200000000002 2023-08-28T02:26:59.325 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.325 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.325 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting_for_map not empty", 2023-08-28T02:26:59.325 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196863+0000", 2023-08-28T02:26:59.325 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.8999999999999997e-06 2023-08-28T02:26:59.325 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.325 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.325 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:26:59.326 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.209793+0000", 2023-08-28T02:26:59.326 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.012929248000000001 2023-08-28T02:26:59.326 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.326 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.326 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:26:59.326 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.209798+0000", 2023-08-28T02:26:59.326 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.7250000000000002e-06 2023-08-28T02:26:59.326 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.326 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.326 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:26:59.326 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.310606+0000", 2023-08-28T02:26:59.326 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.10080757 2023-08-28T02:26:59.326 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.327 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.327 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:26:59.327 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.310743+0000", 2023-08-28T02:26:59.327 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00013726600000000001 2023-08-28T02:26:59.327 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.327 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.327 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,3,4", 2023-08-28T02:26:59.327 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.310871+0000", 2023-08-28T02:26:59.327 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00012777599999999999 2023-08-28T02:26:59.327 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.327 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.328 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.328 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.317856+0000", 2023-08-28T02:26:59.328 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0069848460000000003 2023-08-28T02:26:59.328 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.328 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.328 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.328 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.317896+0000", 2023-08-28T02:26:59.328 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.9743999999999998e-05 2023-08-28T02:26:59.328 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.328 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.328 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.328 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.324952+0000", 2023-08-28T02:26:59.328 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0070567060000000003 2023-08-28T02:26:59.329 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.329 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.329 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:26:59.329 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325214+0000", 2023-08-28T02:26:59.329 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00026183400000000001 2023-08-28T02:26:59.329 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.329 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.329 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:26:59.329 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325264+0000", 2023-08-28T02:26:59.329 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.9496999999999997e-05 2023-08-28T02:26:59.329 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.329 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.329 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:26:59.330 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325351+0000", 2023-08-28T02:26:59.330 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.763e-05 2023-08-28T02:26:59.330 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:26:59.330 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:26:59.330 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:26:59.330 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.330 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.330 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:123553 5.0 5:03eb21c5:::benchmark_data_smithi143_236200_object13410:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e154)", 2023-08-28T02:26:59.330 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:59.196602+0000", 2023-08-28T02:26:59.330 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 180.113883313, 2023-08-28T02:26:59.330 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.128974272, 2023-08-28T02:26:59.330 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:26:59.331 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:59.331 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:26:59.331 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:26:59.331 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:59.331 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 123553 2023-08-28T02:26:59.331 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.331 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:26:59.331 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.331 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:26:59.331 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196602+0000", 2023-08-28T02:26:59.331 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:26:59.331 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.331 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.332 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:26:59.332 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196602+0000", 2023-08-28T02:26:59.332 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:26:59.332 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.332 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.332 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:26:59.332 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196602+0000", 2023-08-28T02:26:59.332 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:26:59.332 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.332 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.332 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:26:59.332 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196604+0000", 2023-08-28T02:26:59.332 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.0990000000000001e-06 2023-08-28T02:26:59.333 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.333 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.333 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:26:59.333 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196609+0000", 2023-08-28T02:26:59.333 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.3920000000000002e-06 2023-08-28T02:26:59.333 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.333 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.333 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:26:59.333 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196613+0000", 2023-08-28T02:26:59.333 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.1679999999999997e-06 2023-08-28T02:26:59.333 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.333 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.334 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:26:59.334 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196883+0000", 2023-08-28T02:26:59.334 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00026973699999999998 2023-08-28T02:26:59.334 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.334 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.334 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting_for_map not empty", 2023-08-28T02:26:59.334 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196888+0000", 2023-08-28T02:26:59.334 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.3399999999999997e-06 2023-08-28T02:26:59.334 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.334 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.334 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:26:59.334 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.209815+0000", 2023-08-28T02:26:59.335 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.012926936 2023-08-28T02:26:59.335 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.335 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.335 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:26:59.335 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.209821+0000", 2023-08-28T02:26:59.335 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.049e-06 2023-08-28T02:26:59.335 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.335 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.335 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:26:59.335 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.314325+0000", 2023-08-28T02:26:59.335 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.10450329 2023-08-28T02:26:59.335 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.336 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.336 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:26:59.336 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.314416+0000", 2023-08-28T02:26:59.336 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.1209e-05 2023-08-28T02:26:59.336 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.336 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.336 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,3,4", 2023-08-28T02:26:59.336 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.314497+0000", 2023-08-28T02:26:59.336 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.0943000000000001e-05 2023-08-28T02:26:59.336 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.336 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.336 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.336 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.317937+0000", 2023-08-28T02:26:59.337 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.003440088 2023-08-28T02:26:59.337 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.337 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.337 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:26:59.337 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325361+0000", 2023-08-28T02:26:59.337 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0074246319999999996 2023-08-28T02:26:59.337 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.337 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.337 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.337 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325398+0000", 2023-08-28T02:26:59.337 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.6105000000000001e-05 2023-08-28T02:26:59.338 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.338 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.338 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.338 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325471+0000", 2023-08-28T02:26:59.338 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.3779e-05 2023-08-28T02:26:59.338 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.338 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.338 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:26:59.338 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325499+0000", 2023-08-28T02:26:59.338 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.7585000000000001e-05 2023-08-28T02:26:59.338 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.338 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.338 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:26:59.338 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325576+0000", 2023-08-28T02:26:59.339 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.7211000000000003e-05 2023-08-28T02:26:59.339 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:26:59.339 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:26:59.339 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:26:59.339 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.339 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.339 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:123563 5.0 5:0dabf8cb:::benchmark_data_smithi143_236200_object13420:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e154)", 2023-08-28T02:26:59.339 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:59.196617+0000", 2023-08-28T02:26:59.339 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 180.113868422, 2023-08-28T02:26:59.339 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1291040370000001, 2023-08-28T02:26:59.339 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:26:59.339 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:59.340 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:26:59.340 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:26:59.340 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:59.340 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 123563 2023-08-28T02:26:59.340 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.340 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:26:59.340 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.340 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:26:59.340 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196617+0000", 2023-08-28T02:26:59.340 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:26:59.340 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.340 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.340 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:26:59.341 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196617+0000", 2023-08-28T02:26:59.341 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:26:59.341 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.341 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.341 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:26:59.341 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196616+0000", 2023-08-28T02:26:59.341 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:26:59.341 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.341 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.341 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:26:59.341 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196618+0000", 2023-08-28T02:26:59.341 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.9970000000000001e-06 2023-08-28T02:26:59.341 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.342 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.342 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:26:59.342 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196623+0000", 2023-08-28T02:26:59.342 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.8509999999999998e-06 2023-08-28T02:26:59.342 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.342 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.342 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:26:59.342 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196628+0000", 2023-08-28T02:26:59.342 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.6149999999999999e-06 2023-08-28T02:26:59.342 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.342 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.342 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:26:59.342 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196908+0000", 2023-08-28T02:26:59.343 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000279788 2023-08-28T02:26:59.343 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.343 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.343 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting_for_map not empty", 2023-08-28T02:26:59.343 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196912+0000", 2023-08-28T02:26:59.343 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.544e-06 2023-08-28T02:26:59.343 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.343 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.343 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:26:59.343 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.209839+0000", 2023-08-28T02:26:59.343 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.012926524999999999 2023-08-28T02:26:59.343 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.344 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.344 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:26:59.344 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.209845+0000", 2023-08-28T02:26:59.344 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.579e-06 2023-08-28T02:26:59.344 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.344 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.344 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:26:59.344 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.317968+0000", 2023-08-28T02:26:59.344 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.108122914 2023-08-28T02:26:59.344 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.344 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.344 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:26:59.344 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.318146+0000", 2023-08-28T02:26:59.345 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00017759300000000001 2023-08-28T02:26:59.345 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.345 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.345 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,3,4", 2023-08-28T02:26:59.345 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.318278+0000", 2023-08-28T02:26:59.345 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00013184000000000001 2023-08-28T02:26:59.345 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.345 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.345 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.345 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325014+0000", 2023-08-28T02:26:59.345 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0067361549999999997 2023-08-28T02:26:59.345 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.345 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.346 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:26:59.346 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325368+0000", 2023-08-28T02:26:59.346 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00035389200000000002 2023-08-28T02:26:59.346 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.346 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.346 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.346 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325421+0000", 2023-08-28T02:26:59.346 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.2902000000000003e-05 2023-08-28T02:26:59.346 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.346 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.346 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.346 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325612+0000", 2023-08-28T02:26:59.346 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00019104500000000001 2023-08-28T02:26:59.347 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.347 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.347 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:26:59.347 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325640+0000", 2023-08-28T02:26:59.347 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.8425999999999999e-05 2023-08-28T02:26:59.347 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.347 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.347 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:26:59.347 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325721+0000", 2023-08-28T02:26:59.347 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.0642000000000005e-05 2023-08-28T02:26:59.347 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:26:59.347 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:26:59.348 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:26:59.348 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.348 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.348 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:123565 5.0 5:0ec9f19d:::benchmark_data_smithi143_236200_object13422:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e154)", 2023-08-28T02:26:59.348 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:59.196632+0000", 2023-08-28T02:26:59.348 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 180.11385336699999, 2023-08-28T02:26:59.348 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.129315396, 2023-08-28T02:26:59.348 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:26:59.348 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:59.348 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:26:59.348 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:26:59.348 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:59.349 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 123565 2023-08-28T02:26:59.349 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.349 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:26:59.349 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.349 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:26:59.349 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196632+0000", 2023-08-28T02:26:59.349 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:26:59.349 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.349 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.349 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:26:59.349 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196632+0000", 2023-08-28T02:26:59.349 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:26:59.349 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.349 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.350 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:26:59.350 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196631+0000", 2023-08-28T02:26:59.350 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:26:59.350 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.350 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.350 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:26:59.350 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196634+0000", 2023-08-28T02:26:59.350 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.0650000000000001e-06 2023-08-28T02:26:59.350 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.350 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.350 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:26:59.350 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196638+0000", 2023-08-28T02:26:59.351 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.6700000000000002e-06 2023-08-28T02:26:59.351 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.351 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.351 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:26:59.351 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196643+0000", 2023-08-28T02:26:59.351 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.7859999999999996e-06 2023-08-28T02:26:59.351 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.351 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.351 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:26:59.351 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196931+0000", 2023-08-28T02:26:59.351 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000288044 2023-08-28T02:26:59.351 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.352 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.352 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting_for_map not empty", 2023-08-28T02:26:59.352 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196935+0000", 2023-08-28T02:26:59.352 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.3930000000000001e-06 2023-08-28T02:26:59.352 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.352 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.352 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:26:59.352 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.209868+0000", 2023-08-28T02:26:59.352 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.012932073000000001 2023-08-28T02:26:59.352 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.352 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.352 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:26:59.352 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.209874+0000", 2023-08-28T02:26:59.353 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.6989999999999997e-06 2023-08-28T02:26:59.353 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.353 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.353 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:26:59.353 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.321422+0000", 2023-08-28T02:26:59.353 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.111547675 2023-08-28T02:26:59.353 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.353 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.353 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:26:59.353 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.321549+0000", 2023-08-28T02:26:59.353 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000127028 2023-08-28T02:26:59.353 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.353 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.354 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,3,4", 2023-08-28T02:26:59.354 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.321630+0000", 2023-08-28T02:26:59.354 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.0882000000000003e-05 2023-08-28T02:26:59.354 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.354 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.354 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:26:59.354 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325428+0000", 2023-08-28T02:26:59.354 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0037977200000000001 2023-08-28T02:26:59.354 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.354 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.354 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.354 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325447+0000", 2023-08-28T02:26:59.354 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.906e-05 2023-08-28T02:26:59.355 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.355 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.355 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.355 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325752+0000", 2023-08-28T02:26:59.355 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00030524300000000002 2023-08-28T02:26:59.355 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.355 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.355 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.355 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325801+0000", 2023-08-28T02:26:59.355 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.9429999999999999e-05 2023-08-28T02:26:59.355 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.355 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.355 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:26:59.356 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325845+0000", 2023-08-28T02:26:59.356 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.3900000000000003e-05 2023-08-28T02:26:59.356 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.356 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.356 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:26:59.356 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325947+0000", 2023-08-28T02:26:59.356 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000102007 2023-08-28T02:26:59.356 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:26:59.356 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:26:59.356 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:26:59.356 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.356 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.356 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:123578 5.0 5:0d4223f2:::benchmark_data_smithi143_236200_object13435:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e154)", 2023-08-28T02:26:59.357 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:59.196647+0000", 2023-08-28T02:26:59.357 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 180.11383827200001, 2023-08-28T02:26:59.357 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.14071996, 2023-08-28T02:26:59.357 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:26:59.357 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:59.357 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:26:59.357 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:26:59.357 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:59.357 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 123578 2023-08-28T02:26:59.357 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.357 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:26:59.357 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.357 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:26:59.358 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196647+0000", 2023-08-28T02:26:59.358 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:26:59.358 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.358 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.358 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:26:59.358 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196647+0000", 2023-08-28T02:26:59.358 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:26:59.359 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.359 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.359 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:26:59.359 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196647+0000", 2023-08-28T02:26:59.359 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:26:59.359 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.359 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.359 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:26:59.359 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196649+0000", 2023-08-28T02:26:59.359 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.4449999999999999e-06 2023-08-28T02:26:59.359 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.359 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.360 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:26:59.360 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196654+0000", 2023-08-28T02:26:59.360 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.1139999999999997e-06 2023-08-28T02:26:59.360 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.360 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.360 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:26:59.360 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196658+0000", 2023-08-28T02:26:59.360 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.2420000000000002e-06 2023-08-28T02:26:59.360 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.360 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.360 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:26:59.361 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196950+0000", 2023-08-28T02:26:59.361 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00029176700000000001 2023-08-28T02:26:59.361 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.361 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.361 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting_for_map not empty", 2023-08-28T02:26:59.361 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196953+0000", 2023-08-28T02:26:59.361 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.5359999999999999e-06 2023-08-28T02:26:59.361 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.361 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.361 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:26:59.361 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.209894+0000", 2023-08-28T02:26:59.361 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.012941413000000001 2023-08-28T02:26:59.362 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.362 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.362 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:26:59.362 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.209898+0000", 2023-08-28T02:26:59.362 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.484e-06 2023-08-28T02:26:59.362 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.362 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.362 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:26:59.362 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325997+0000", 2023-08-28T02:26:59.362 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.116099921 2023-08-28T02:26:59.362 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.362 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.362 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:26:59.363 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.326137+0000", 2023-08-28T02:26:59.363 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000139696 2023-08-28T02:26:59.363 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.363 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.363 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,3,4", 2023-08-28T02:26:59.363 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.326268+0000", 2023-08-28T02:26:59.363 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00013096399999999999 2023-08-28T02:26:59.363 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.363 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.363 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.363 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.335576+0000", 2023-08-28T02:26:59.363 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0093078999999999992 2023-08-28T02:26:59.364 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.364 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.364 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.364 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.335620+0000", 2023-08-28T02:26:59.364 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.3868000000000001e-05 2023-08-28T02:26:59.364 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.364 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.364 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:26:59.364 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.335661+0000", 2023-08-28T02:26:59.364 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.0908999999999997e-05 2023-08-28T02:26:59.364 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.365 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.365 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.365 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.337193+0000", 2023-08-28T02:26:59.365 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0015320399999999999 2023-08-28T02:26:59.365 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.365 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.365 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:26:59.365 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.337245+0000", 2023-08-28T02:26:59.365 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.2092999999999997e-05 2023-08-28T02:26:59.365 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.365 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.365 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:26:59.365 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.337367+0000", 2023-08-28T02:26:59.366 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000121902 2023-08-28T02:26:59.366 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:26:59.366 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:26:59.366 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:26:59.366 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.366 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.366 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:123581 5.0 5:06f44fcd:::benchmark_data_smithi143_236200_object13438:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e154)", 2023-08-28T02:26:59.366 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:59.196662+0000", 2023-08-28T02:26:59.366 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 180.113822696, 2023-08-28T02:26:59.366 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.14107384, 2023-08-28T02:26:59.366 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:26:59.366 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:59.367 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:26:59.367 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:26:59.367 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:59.367 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 123581 2023-08-28T02:26:59.367 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.367 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:26:59.367 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.367 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:26:59.367 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196662+0000", 2023-08-28T02:26:59.367 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:26:59.367 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.367 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.368 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:26:59.368 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196662+0000", 2023-08-28T02:26:59.368 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:26:59.368 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.368 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.368 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:26:59.368 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196662+0000", 2023-08-28T02:26:59.368 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:26:59.368 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.368 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.368 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:26:59.368 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196664+0000", 2023-08-28T02:26:59.368 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.6810000000000001e-06 2023-08-28T02:26:59.369 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.369 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.369 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:26:59.369 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196669+0000", 2023-08-28T02:26:59.369 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.0610000000000003e-06 2023-08-28T02:26:59.369 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.369 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.369 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:26:59.369 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196673+0000", 2023-08-28T02:26:59.369 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.2080000000000002e-06 2023-08-28T02:26:59.369 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.369 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.369 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:26:59.369 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196963+0000", 2023-08-28T02:26:59.370 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00029004399999999999 2023-08-28T02:26:59.370 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.370 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.370 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting_for_map not empty", 2023-08-28T02:26:59.370 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196966+0000", 2023-08-28T02:26:59.370 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.362e-06 2023-08-28T02:26:59.370 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.370 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.370 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:26:59.370 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.209987+0000", 2023-08-28T02:26:59.370 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.013021624000000001 2023-08-28T02:26:59.370 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.370 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.370 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:26:59.371 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.209994+0000", 2023-08-28T02:26:59.371 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.6440000000000003e-06 2023-08-28T02:26:59.371 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.371 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.371 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:26:59.371 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.332359+0000", 2023-08-28T02:26:59.371 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1223654620000001 2023-08-28T02:26:59.371 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.371 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.371 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:26:59.371 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.332502+0000", 2023-08-28T02:26:59.371 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00014240999999999999 2023-08-28T02:26:59.372 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.372 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.372 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,3,4", 2023-08-28T02:26:59.372 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.332652+0000", 2023-08-28T02:26:59.372 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000149853 2023-08-28T02:26:59.372 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.372 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.372 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.372 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.335652+0000", 2023-08-28T02:26:59.372 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0030004179999999999 2023-08-28T02:26:59.372 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.372 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.373 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:26:59.373 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.335671+0000", 2023-08-28T02:26:59.373 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.9049e-05 2023-08-28T02:26:59.373 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.373 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.373 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.373 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.336202+0000", 2023-08-28T02:26:59.373 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00053052899999999996 2023-08-28T02:26:59.373 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.373 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.373 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.373 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.337555+0000", 2023-08-28T02:26:59.373 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0013538479999999999 2023-08-28T02:26:59.374 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.374 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.374 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:26:59.374 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.337610+0000", 2023-08-28T02:26:59.374 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.4595000000000002e-05 2023-08-28T02:26:59.374 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.374 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.374 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:26:59.374 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.337736+0000", 2023-08-28T02:26:59.374 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00012634199999999999 2023-08-28T02:26:59.374 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:26:59.374 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:26:59.375 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:26:59.375 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.375 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.375 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:124089 5.0 5:06d6dacb:::benchmark_data_smithi143_236200_object13946:head [delete] snapc 0=[] ondisk+write+known_if_redirected e155)", 2023-08-28T02:26:59.375 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:24:01.180357+0000", 2023-08-28T02:26:59.375 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 178.13012805, 2023-08-28T02:26:59.375 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.22593939499999999, 2023-08-28T02:26:59.375 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:26:59.375 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:59.375 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:26:59.375 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:26:59.375 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:59.376 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 124089 2023-08-28T02:26:59.376 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.376 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:26:59.376 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.376 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:26:59.376 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.180357+0000", 2023-08-28T02:26:59.376 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:26:59.376 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.376 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.376 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:26:59.376 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.180357+0000", 2023-08-28T02:26:59.376 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:26:59.376 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.377 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.377 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:26:59.377 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.180356+0000", 2023-08-28T02:26:59.377 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:26:59.377 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.377 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.377 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:26:59.377 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.180363+0000", 2023-08-28T02:26:59.377 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.0899999999999999e-06 2023-08-28T02:26:59.377 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.377 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.377 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:26:59.377 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.180381+0000", 2023-08-28T02:26:59.377 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.8519e-05 2023-08-28T02:26:59.378 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.378 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.378 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:26:59.378 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.180391+0000", 2023-08-28T02:26:59.378 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0192e-05 2023-08-28T02:26:59.378 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.378 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.378 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:26:59.378 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.180456+0000", 2023-08-28T02:26:59.378 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.4084999999999994e-05 2023-08-28T02:26:59.378 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.378 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.378 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:26:59.379 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.180607+0000", 2023-08-28T02:26:59.379 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000151232 2023-08-28T02:26:59.379 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.379 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.379 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,3,4", 2023-08-28T02:26:59.379 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.181058+0000", 2023-08-28T02:26:59.379 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00045131499999999998 2023-08-28T02:26:59.379 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.379 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.379 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:26:59.379 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.187145+0000", 2023-08-28T02:26:59.379 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0060865600000000004 2023-08-28T02:26:59.380 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.380 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.380 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.380 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.187725+0000", 2023-08-28T02:26:59.380 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00058020299999999995 2023-08-28T02:26:59.380 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.380 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.380 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.380 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.189158+0000", 2023-08-28T02:26:59.380 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.001433134 2023-08-28T02:26:59.380 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.380 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.381 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.381 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.406130+0000", 2023-08-28T02:26:59.381 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.216971527 2023-08-28T02:26:59.381 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.381 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.381 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:26:59.381 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.406181+0000", 2023-08-28T02:26:59.381 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.1264999999999998e-05 2023-08-28T02:26:59.381 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.381 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.381 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:26:59.381 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.406296+0000", 2023-08-28T02:26:59.381 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000115686 2023-08-28T02:26:59.382 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:26:59.382 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:26:59.382 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:26:59.382 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.382 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.382 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:130455 5.3 5:c62e95e0:::benchmark_data_smithi143_236200_object20312:head [delete] snapc 0=[] ondisk+write+known_if_redirected e163)", 2023-08-28T02:26:59.382 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:24:13.094164+0000", 2023-08-28T02:26:59.382 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 166.216321604, 2023-08-28T02:26:59.382 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.23304668100000001, 2023-08-28T02:26:59.382 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:26:59.382 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:59.382 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:26:59.382 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:26:59.383 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:59.383 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 130455 2023-08-28T02:26:59.383 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.383 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:26:59.383 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.383 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:26:59.383 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.094164+0000", 2023-08-28T02:26:59.383 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:26:59.383 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.383 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.383 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:26:59.383 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.094164+0000", 2023-08-28T02:26:59.383 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:26:59.384 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.384 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.384 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:26:59.384 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.094162+0000", 2023-08-28T02:26:59.384 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:26:59.384 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.384 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.384 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:26:59.384 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.094168+0000", 2023-08-28T02:26:59.384 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.5480000000000001e-06 2023-08-28T02:26:59.384 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.384 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.385 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:26:59.385 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.200430+0000", 2023-08-28T02:26:59.385 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.106262407 2023-08-28T02:26:59.385 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.385 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.385 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:26:59.385 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.200446+0000", 2023-08-28T02:26:59.385 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.5396e-05 2023-08-28T02:26:59.385 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.385 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.385 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:26:59.385 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.208625+0000", 2023-08-28T02:26:59.385 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0081796609999999995 2023-08-28T02:26:59.386 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.386 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.386 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:26:59.386 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.208750+0000", 2023-08-28T02:26:59.386 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000124692 2023-08-28T02:26:59.386 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.386 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.386 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 5,6", 2023-08-28T02:26:59.386 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.208854+0000", 2023-08-28T02:26:59.386 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000103959 2023-08-28T02:26:59.386 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.386 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.387 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:26:59.387 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.212292+0000", 2023-08-28T02:26:59.387 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0034382129999999999 2023-08-28T02:26:59.387 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.387 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.387 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.387 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.327100+0000", 2023-08-28T02:26:59.387 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.114808038 2023-08-28T02:26:59.387 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.387 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.387 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.387 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.327129+0000", 2023-08-28T02:26:59.387 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.8972e-05 2023-08-28T02:26:59.388 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.388 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.388 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:26:59.388 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.327172+0000", 2023-08-28T02:26:59.388 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.3093000000000002e-05 2023-08-28T02:26:59.388 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.388 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.388 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:26:59.388 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.327210+0000", 2023-08-28T02:26:59.388 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.7783000000000001e-05 2023-08-28T02:26:59.388 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:26:59.388 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:26:59.388 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:26:59.389 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.389 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.389 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:140334 5.0 5:0466772d:::benchmark_data_smithi143_236200_object30191:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-08-28T02:26:59.389 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:24:28.055937+0000", 2023-08-28T02:26:59.389 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 151.254548298, 2023-08-28T02:26:59.389 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.25254626600000002, 2023-08-28T02:26:59.389 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:26:59.389 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:59.389 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:26:59.389 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:26:59.389 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:59.389 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 140334 2023-08-28T02:26:59.390 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.390 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:26:59.390 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.390 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:26:59.390 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.055937+0000", 2023-08-28T02:26:59.390 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:26:59.390 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.390 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.390 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:26:59.390 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.055937+0000", 2023-08-28T02:26:59.390 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:26:59.390 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.391 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.391 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:26:59.391 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.055936+0000", 2023-08-28T02:26:59.391 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:26:59.391 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.391 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.391 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:26:59.391 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.055942+0000", 2023-08-28T02:26:59.391 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.9519999999999999e-06 2023-08-28T02:26:59.391 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.392 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.392 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:26:59.392 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.240512+0000", 2023-08-28T02:26:59.392 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.18457072699999999 2023-08-28T02:26:59.392 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.392 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.392 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:26:59.392 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.240525+0000", 2023-08-28T02:26:59.392 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.2515e-05 2023-08-28T02:26:59.392 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.392 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.392 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:26:59.392 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.240582+0000", 2023-08-28T02:26:59.393 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.6839000000000002e-05 2023-08-28T02:26:59.393 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.393 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.393 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:26:59.393 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.240726+0000", 2023-08-28T02:26:59.393 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0001448 2023-08-28T02:26:59.393 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.393 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.393 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,3,4", 2023-08-28T02:26:59.393 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.240865+0000", 2023-08-28T02:26:59.393 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000138601 2023-08-28T02:26:59.393 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.393 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.394 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:26:59.394 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.245256+0000", 2023-08-28T02:26:59.394 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0043909229999999997 2023-08-28T02:26:59.394 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.394 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.394 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.394 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.247591+0000", 2023-08-28T02:26:59.394 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0023347799999999998 2023-08-28T02:26:59.394 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.394 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.394 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.394 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.256080+0000", 2023-08-28T02:26:59.395 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0084893469999999995 2023-08-28T02:26:59.395 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.395 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.395 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.395 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.308312+0000", 2023-08-28T02:26:59.395 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.052232369000000001 2023-08-28T02:26:59.395 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.395 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.395 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:26:59.395 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.308369+0000", 2023-08-28T02:26:59.395 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.6746000000000002e-05 2023-08-28T02:26:59.395 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.395 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.396 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:26:59.396 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.308483+0000", 2023-08-28T02:26:59.396 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00011390700000000001 2023-08-28T02:26:59.396 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:26:59.396 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:26:59.396 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:26:59.396 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.396 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.396 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:154038 5.0 5:081df89e:::benchmark_data_smithi143_236200_object43895:head [delete] snapc 0=[] ondisk+write+known_if_redirected e169)", 2023-08-28T02:26:59.396 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:24:50.334001+0000", 2023-08-28T02:26:59.396 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 128.97648375399999, 2023-08-28T02:26:59.396 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.21960570900000001, 2023-08-28T02:26:59.396 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:26:59.397 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:59.397 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:26:59.397 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:26:59.397 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:59.397 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 154038 2023-08-28T02:26:59.397 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.397 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:26:59.397 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.397 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:26:59.397 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.334001+0000", 2023-08-28T02:26:59.397 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:26:59.397 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.397 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.397 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:26:59.397 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.334001+0000", 2023-08-28T02:26:59.398 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:26:59.398 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.398 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.398 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:26:59.398 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.333999+0000", 2023-08-28T02:26:59.398 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:26:59.398 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.398 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.398 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:26:59.398 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.334011+0000", 2023-08-28T02:26:59.398 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1799e-05 2023-08-28T02:26:59.398 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.399 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.399 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:26:59.399 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.334024+0000", 2023-08-28T02:26:59.399 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.2565000000000001e-05 2023-08-28T02:26:59.399 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.399 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.399 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:26:59.399 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.334031+0000", 2023-08-28T02:26:59.399 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.5789999999999998e-06 2023-08-28T02:26:59.399 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.399 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.400 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:26:59.400 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.334084+0000", 2023-08-28T02:26:59.400 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.2589000000000001e-05 2023-08-28T02:26:59.400 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.400 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.400 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:26:59.400 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.535051+0000", 2023-08-28T02:26:59.400 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.20096666199999999 2023-08-28T02:26:59.400 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.400 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.400 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,3,4", 2023-08-28T02:26:59.400 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.535201+0000", 2023-08-28T02:26:59.400 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000150109 2023-08-28T02:26:59.401 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.401 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.401 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:26:59.401 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.545453+0000", 2023-08-28T02:26:59.401 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.010252297 2023-08-28T02:26:59.401 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.401 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.401 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.401 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.546595+0000", 2023-08-28T02:26:59.401 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0011422209999999999 2023-08-28T02:26:59.401 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.401 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.401 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.402 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.553365+0000", 2023-08-28T02:26:59.402 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0067698059999999997 2023-08-28T02:26:59.402 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.402 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.402 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.402 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.553404+0000", 2023-08-28T02:26:59.402 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.8495000000000003e-05 2023-08-28T02:26:59.402 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.402 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.402 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:26:59.402 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.553465+0000", 2023-08-28T02:26:59.402 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.1643999999999995e-05 2023-08-28T02:26:59.402 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.403 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.403 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:26:59.403 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.553607+0000", 2023-08-28T02:26:59.403 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00014188199999999999 2023-08-28T02:26:59.403 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:26:59.403 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:26:59.403 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:26:59.403 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.403 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.403 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:185292 5.0 5:00fb3be2:::benchmark_data_smithi143_236200_object75149:head [delete] snapc 0=[] ondisk+write+known_if_redirected e176)", 2023-08-28T02:26:59.403 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:25:41.081250+0000", 2023-08-28T02:26:59.404 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 78.229234980000001, 2023-08-28T02:26:59.404 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.22154157599999999, 2023-08-28T02:26:59.404 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:26:59.404 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:59.404 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:26:59.404 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:26:59.404 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:59.404 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 185292 2023-08-28T02:26:59.404 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.404 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:26:59.404 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.404 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:26:59.405 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:41.081250+0000", 2023-08-28T02:26:59.405 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:26:59.405 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.405 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.405 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:26:59.405 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:41.081250+0000", 2023-08-28T02:26:59.405 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:26:59.405 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.405 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.405 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:26:59.405 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:41.081250+0000", 2023-08-28T02:26:59.405 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:26:59.406 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.406 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.406 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:26:59.406 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:41.081252+0000", 2023-08-28T02:26:59.406 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.8930000000000001e-06 2023-08-28T02:26:59.406 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.406 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.406 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:26:59.406 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:41.081258+0000", 2023-08-28T02:26:59.406 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.9440000000000002e-06 2023-08-28T02:26:59.406 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.406 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.406 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:26:59.407 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:41.081264+0000", 2023-08-28T02:26:59.407 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.8189999999999997e-06 2023-08-28T02:26:59.407 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.407 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.407 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:26:59.407 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:41.284634+0000", 2023-08-28T02:26:59.407 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.203369884 2023-08-28T02:26:59.407 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.407 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.407 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:26:59.407 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:41.284801+0000", 2023-08-28T02:26:59.407 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00016733300000000001 2023-08-28T02:26:59.408 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.408 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.408 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,3,4", 2023-08-28T02:26:59.408 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:41.284962+0000", 2023-08-28T02:26:59.408 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00016062800000000001 2023-08-28T02:26:59.408 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.408 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.408 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.408 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:41.293941+0000", 2023-08-28T02:26:59.408 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0089789069999999995 2023-08-28T02:26:59.408 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.408 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.409 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:26:59.409 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:41.293949+0000", 2023-08-28T02:26:59.409 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.3340000000000002e-06 2023-08-28T02:26:59.409 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.409 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.409 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.409 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:41.294023+0000", 2023-08-28T02:26:59.409 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.3842000000000002e-05 2023-08-28T02:26:59.409 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.409 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.409 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.409 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:41.302636+0000", 2023-08-28T02:26:59.409 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0086131060000000006 2023-08-28T02:26:59.410 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.410 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.410 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:26:59.410 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:41.302685+0000", 2023-08-28T02:26:59.410 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.8588999999999998e-05 2023-08-28T02:26:59.410 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.410 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.410 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:26:59.410 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:41.302792+0000", 2023-08-28T02:26:59.410 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000106884 2023-08-28T02:26:59.410 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:26:59.411 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:26:59.411 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:26:59.411 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.411 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.411 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.44594.0:185289 5.b e176/119 5:d66ebebd:::benchmark_data_smithi143_236200_object75146:head v 176'11543, mlcod=176'11541)", 2023-08-28T02:26:59.411 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:25:41.090390+0000", 2023-08-28T02:26:59.411 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 78.220095573999998, 2023-08-28T02:26:59.411 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.21198209300000001, 2023-08-28T02:26:59.411 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:26:59.411 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:59.411 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:26:59.411 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.411 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:26:59.412 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:41.090390+0000", 2023-08-28T02:26:59.412 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:26:59.412 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.412 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.412 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:26:59.412 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:41.090390+0000", 2023-08-28T02:26:59.412 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:26:59.412 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.412 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.412 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:26:59.412 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:41.090389+0000", 2023-08-28T02:26:59.413 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:26:59.413 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.413 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.413 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:26:59.413 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:41.090394+0000", 2023-08-28T02:26:59.413 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.527e-06 2023-08-28T02:26:59.413 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.413 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.413 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:26:59.413 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:41.090400+0000", 2023-08-28T02:26:59.413 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.7490000000000001e-06 2023-08-28T02:26:59.413 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.414 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.414 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:26:59.414 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:41.090404+0000", 2023-08-28T02:26:59.414 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.8330000000000004e-06 2023-08-28T02:26:59.414 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.414 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.414 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:26:59.414 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:41.094934+0000", 2023-08-28T02:26:59.414 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0045296679999999997 2023-08-28T02:26:59.414 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.414 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.414 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:26:59.415 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:41.094955+0000", 2023-08-28T02:26:59.415 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.0877e-05 2023-08-28T02:26:59.415 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.415 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.415 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:26:59.415 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:41.302336+0000", 2023-08-28T02:26:59.415 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.20738100200000001 2023-08-28T02:26:59.415 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.415 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.415 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:26:59.415 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:41.302372+0000", 2023-08-28T02:26:59.415 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.5917999999999997e-05 2023-08-28T02:26:59.416 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:26:59.416 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:26:59.416 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:26:59.416 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.416 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.416 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:196479 5.0 5:09cd934e:::benchmark_data_smithi143_236200_object86336:head [delete] snapc 0=[] ondisk+write+known_if_redirected e179)", 2023-08-28T02:26:59.416 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:25:58.967224+0000", 2023-08-28T02:26:59.416 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 60.343260751000003, 2023-08-28T02:26:59.416 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.23017068800000001, 2023-08-28T02:26:59.416 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:26:59.416 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:59.416 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:26:59.417 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:26:59.417 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:59.417 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 196479 2023-08-28T02:26:59.417 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.417 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:26:59.417 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.417 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:26:59.417 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.967224+0000", 2023-08-28T02:26:59.417 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:26:59.417 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.417 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.417 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:26:59.418 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.967224+0000", 2023-08-28T02:26:59.418 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:26:59.418 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.418 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.418 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:26:59.418 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.967223+0000", 2023-08-28T02:26:59.418 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:26:59.418 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.418 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.418 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:26:59.418 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.967231+0000", 2023-08-28T02:26:59.418 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.5250000000000005e-06 2023-08-28T02:26:59.418 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.419 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.419 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:26:59.419 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.967245+0000", 2023-08-28T02:26:59.419 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3628000000000001e-05 2023-08-28T02:26:59.419 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.419 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.419 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:26:59.419 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.967254+0000", 2023-08-28T02:26:59.419 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.5920000000000001e-06 2023-08-28T02:26:59.419 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.419 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.420 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:26:59.420 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.967321+0000", 2023-08-28T02:26:59.420 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.7597999999999999e-05 2023-08-28T02:26:59.420 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.420 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.420 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:26:59.420 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.967453+0000", 2023-08-28T02:26:59.420 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00013172800000000001 2023-08-28T02:26:59.420 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.420 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.420 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,3,4", 2023-08-28T02:26:59.421 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.967581+0000", 2023-08-28T02:26:59.421 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00012768499999999999 2023-08-28T02:26:59.421 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.421 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.421 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:26:59.421 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.976035+0000", 2023-08-28T02:26:59.421 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0084540080000000007 2023-08-28T02:26:59.421 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.421 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.421 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.421 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.990429+0000", 2023-08-28T02:26:59.421 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.014394239 2023-08-28T02:26:59.422 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.422 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.422 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.422 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:59.193123+0000", 2023-08-28T02:26:59.422 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.202694387 2023-08-28T02:26:59.422 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.422 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.422 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.422 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:59.197240+0000", 2023-08-28T02:26:59.422 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0041169079999999999 2023-08-28T02:26:59.422 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.423 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.423 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:26:59.423 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:59.197286+0000", 2023-08-28T02:26:59.423 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.5537000000000001e-05 2023-08-28T02:26:59.423 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.423 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.423 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:26:59.423 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:59.197395+0000", 2023-08-28T02:26:59.423 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00010937 2023-08-28T02:26:59.423 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:26:59.423 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:26:59.423 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:26:59.424 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.424 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.424 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:196481 5.0 5:02ad625a:::benchmark_data_smithi143_236200_object86338:head [delete] snapc 0=[] ondisk+write+known_if_redirected e179)", 2023-08-28T02:26:59.424 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:25:58.967272+0000", 2023-08-28T02:26:59.424 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 60.343212952999998, 2023-08-28T02:26:59.424 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.23031311300000001, 2023-08-28T02:26:59.424 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:26:59.424 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:59.424 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:26:59.424 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:26:59.424 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:59.424 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 196481 2023-08-28T02:26:59.424 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.425 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:26:59.425 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.425 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:26:59.425 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.967272+0000", 2023-08-28T02:26:59.425 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:26:59.425 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.425 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.425 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:26:59.425 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.967272+0000", 2023-08-28T02:26:59.425 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:26:59.425 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.425 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.426 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:26:59.426 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.967272+0000", 2023-08-28T02:26:59.426 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:26:59.426 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.426 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.426 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:26:59.426 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.967275+0000", 2023-08-28T02:26:59.426 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.7020000000000001e-06 2023-08-28T02:26:59.426 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.426 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.426 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:26:59.426 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.967280+0000", 2023-08-28T02:26:59.427 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.9220000000000002e-06 2023-08-28T02:26:59.427 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.427 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.427 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:26:59.427 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.967285+0000", 2023-08-28T02:26:59.427 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.4689999999999999e-06 2023-08-28T02:26:59.427 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.427 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.427 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:26:59.427 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.971611+0000", 2023-08-28T02:26:59.427 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0043258719999999997 2023-08-28T02:26:59.427 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.427 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.428 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:26:59.428 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.971747+0000", 2023-08-28T02:26:59.428 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00013653999999999999 2023-08-28T02:26:59.428 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.428 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.428 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,3,4", 2023-08-28T02:26:59.428 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.971883+0000", 2023-08-28T02:26:59.428 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00013597699999999999 2023-08-28T02:26:59.428 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.428 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.428 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:26:59.428 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.976324+0000", 2023-08-28T02:26:59.429 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0044409469999999998 2023-08-28T02:26:59.429 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.429 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.429 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.429 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:59.193077+0000", 2023-08-28T02:26:59.429 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.216752376 2023-08-28T02:26:59.429 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.429 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.429 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.429 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:59.193216+0000", 2023-08-28T02:26:59.429 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00013949899999999999 2023-08-28T02:26:59.429 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.429 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.430 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.430 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:59.197441+0000", 2023-08-28T02:26:59.430 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0042246360000000004 2023-08-28T02:26:59.430 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.430 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.430 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:26:59.430 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:59.197483+0000", 2023-08-28T02:26:59.430 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.1931999999999998e-05 2023-08-28T02:26:59.430 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.430 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.430 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:26:59.430 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:59.197585+0000", 2023-08-28T02:26:59.431 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000102631 2023-08-28T02:26:59.431 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:26:59.431 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:26:59.431 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:26:59.431 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.431 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.431 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:196489 5.0 5:0bfcd855:::benchmark_data_smithi143_236200_object86346:head [delete] snapc 0=[] ondisk+write+known_if_redirected e179)", 2023-08-28T02:26:59.431 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:25:58.977293+0000", 2023-08-28T02:26:59.431 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 60.333192502999999, 2023-08-28T02:26:59.431 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.22063449600000001, 2023-08-28T02:26:59.431 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:26:59.431 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:59.432 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:26:59.432 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:26:59.432 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:59.432 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 196489 2023-08-28T02:26:59.432 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.432 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:26:59.432 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.432 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:26:59.432 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.977293+0000", 2023-08-28T02:26:59.432 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:26:59.432 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.432 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.433 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:26:59.433 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.977293+0000", 2023-08-28T02:26:59.433 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:26:59.433 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.433 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.433 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:26:59.433 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.977292+0000", 2023-08-28T02:26:59.433 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:26:59.433 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.433 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.433 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:26:59.433 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.977295+0000", 2023-08-28T02:26:59.434 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.8760000000000001e-06 2023-08-28T02:26:59.434 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.434 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.434 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:26:59.434 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.977302+0000", 2023-08-28T02:26:59.434 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.6869999999999997e-06 2023-08-28T02:26:59.434 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.434 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.434 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:26:59.434 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.977307+0000", 2023-08-28T02:26:59.434 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.9239999999999998e-06 2023-08-28T02:26:59.434 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.434 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.434 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:26:59.435 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.977386+0000", 2023-08-28T02:26:59.435 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.9243e-05 2023-08-28T02:26:59.435 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.435 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.435 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:26:59.435 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.977507+0000", 2023-08-28T02:26:59.435 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00012143599999999999 2023-08-28T02:26:59.435 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.435 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.435 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,3,4", 2023-08-28T02:26:59.435 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.977642+0000", 2023-08-28T02:26:59.436 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000134795 2023-08-28T02:26:59.436 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.436 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.436 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:26:59.436 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:59.193137+0000", 2023-08-28T02:26:59.436 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.21549442399999999 2023-08-28T02:26:59.436 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.436 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.436 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.436 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:59.197200+0000", 2023-08-28T02:26:59.436 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0040637410000000001 2023-08-28T02:26:59.436 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.436 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.437 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.437 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:59.197632+0000", 2023-08-28T02:26:59.437 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00043185499999999998 2023-08-28T02:26:59.437 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.437 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.437 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.437 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:59.197768+0000", 2023-08-28T02:26:59.437 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000136172 2023-08-28T02:26:59.437 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.437 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.437 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:26:59.437 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:59.197819+0000", 2023-08-28T02:26:59.437 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.0305e-05 2023-08-28T02:26:59.438 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.438 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.438 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:26:59.438 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:59.197927+0000", 2023-08-28T02:26:59.438 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00010841900000000001 2023-08-28T02:26:59.438 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:26:59.438 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:26:59.438 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:26:59.438 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.438 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.438 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:196490 5.3 5:c185802a:::benchmark_data_smithi143_236200_object86347:head [delete] snapc 0=[] ondisk+write+known_if_redirected e179)", 2023-08-28T02:26:59.438 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:25:58.977318+0000", 2023-08-28T02:26:59.438 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 60.333166802999997, 2023-08-28T02:26:59.439 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.21579271899999999, 2023-08-28T02:26:59.439 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:26:59.439 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:59.439 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:26:59.439 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:26:59.439 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:59.439 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 196490 2023-08-28T02:26:59.439 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.439 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:26:59.439 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.439 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:26:59.439 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.977318+0000", 2023-08-28T02:26:59.439 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:26:59.440 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.440 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.440 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:26:59.440 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.977318+0000", 2023-08-28T02:26:59.440 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:26:59.440 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.440 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.440 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:26:59.440 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.977318+0000", 2023-08-28T02:26:59.440 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:26:59.440 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.440 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.440 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:26:59.441 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.977321+0000", 2023-08-28T02:26:59.441 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.7520000000000001e-06 2023-08-28T02:26:59.441 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.441 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.441 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:26:59.441 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.977327+0000", 2023-08-28T02:26:59.441 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.4130000000000003e-06 2023-08-28T02:26:59.441 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.441 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.441 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:26:59.441 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.977333+0000", 2023-08-28T02:26:59.442 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.4149999999999998e-06 2023-08-28T02:26:59.442 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.442 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.442 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:26:59.442 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.981235+0000", 2023-08-28T02:26:59.442 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0039020349999999999 2023-08-28T02:26:59.442 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.442 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.442 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:26:59.442 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.981349+0000", 2023-08-28T02:26:59.442 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000114398 2023-08-28T02:26:59.442 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.442 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.443 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 5,6", 2023-08-28T02:26:59.443 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.981456+0000", 2023-08-28T02:26:59.443 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000107396 2023-08-28T02:26:59.443 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.443 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.443 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.443 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.990283+0000", 2023-08-28T02:26:59.443 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0088262280000000002 2023-08-28T02:26:59.443 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.443 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.443 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:26:59.444 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.990409+0000", 2023-08-28T02:26:59.444 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00012678799999999999 2023-08-28T02:26:59.444 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.444 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.444 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.444 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:59.193024+0000", 2023-08-28T02:26:59.444 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.20261434 2023-08-28T02:26:59.444 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.444 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.444 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:26:59.444 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:59.193069+0000", 2023-08-28T02:26:59.444 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.5643000000000003e-05 2023-08-28T02:26:59.445 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.445 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.445 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:26:59.445 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:59.193111+0000", 2023-08-28T02:26:59.445 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.1695e-05 2023-08-28T02:26:59.445 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:26:59.445 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:26:59.445 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:26:59.445 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.445 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.445 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:196496 5.0 5:08dff0bd:::benchmark_data_smithi143_236200_object86353:head [delete] snapc 0=[] ondisk+write+known_if_redirected e179)", 2023-08-28T02:26:59.445 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:25:58.977338+0000", 2023-08-28T02:26:59.445 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 60.333147199000003, 2023-08-28T02:26:59.445 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.220942423, 2023-08-28T02:26:59.446 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:26:59.446 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:59.446 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:26:59.446 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:26:59.446 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:26:59.446 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 196496 2023-08-28T02:26:59.446 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.446 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:26:59.446 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.446 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:26:59.446 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.977338+0000", 2023-08-28T02:26:59.446 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:26:59.447 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.447 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.447 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:26:59.447 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.977338+0000", 2023-08-28T02:26:59.447 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:26:59.447 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.447 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.447 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:26:59.447 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.977338+0000", 2023-08-28T02:26:59.447 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:26:59.447 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.447 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.448 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:26:59.448 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.977340+0000", 2023-08-28T02:26:59.448 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.8370000000000001e-06 2023-08-28T02:26:59.448 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.448 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.448 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:26:59.448 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.977346+0000", 2023-08-28T02:26:59.448 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.8300000000000001e-06 2023-08-28T02:26:59.448 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.448 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.448 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:26:59.448 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.977351+0000", 2023-08-28T02:26:59.449 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.0180000000000001e-06 2023-08-28T02:26:59.449 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.449 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.449 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:26:59.449 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.986297+0000", 2023-08-28T02:26:59.449 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0089462469999999992 2023-08-28T02:26:59.449 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.449 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.449 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:26:59.449 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.986435+0000", 2023-08-28T02:26:59.449 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000137854 2023-08-28T02:26:59.449 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.449 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.449 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,3,4", 2023-08-28T02:26:59.450 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.986573+0000", 2023-08-28T02:26:59.450 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000137658 2023-08-28T02:26:59.450 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.450 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.450 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:26:59.450 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:59.193180+0000", 2023-08-28T02:26:59.450 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.20660690900000001 2023-08-28T02:26:59.450 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.450 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.450 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.450 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:59.197731+0000", 2023-08-28T02:26:59.450 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0045515269999999997 2023-08-28T02:26:59.451 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.451 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.451 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.451 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:59.198111+0000", 2023-08-28T02:26:59.451 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000379334 2023-08-28T02:26:59.451 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.451 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.451 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:26:59.451 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:59.198143+0000", 2023-08-28T02:26:59.451 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.2155e-05 2023-08-28T02:26:59.451 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.451 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.452 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:26:59.452 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:59.198186+0000", 2023-08-28T02:26:59.452 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.2800999999999999e-05 2023-08-28T02:26:59.452 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.452 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.452 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:26:59.452 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:59.198280+0000", 2023-08-28T02:26:59.452 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.4678000000000004e-05 2023-08-28T02:26:59.452 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:26:59.452 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:26:59.452 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:26:59.452 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.452 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.453 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.44594.0:196486 5.8 e179/119 5:1c5ba9b5:::benchmark_data_smithi143_236200_object86343:head v 179'12320, mlcod=179'12318)", 2023-08-28T02:26:59.453 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:25:58.977544+0000", 2023-08-28T02:26:59.453 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 60.332940694000001, 2023-08-28T02:26:59.453 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.21562642900000001, 2023-08-28T02:26:59.453 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:26:59.453 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:26:59.453 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:26:59.453 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.453 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:26:59.453 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.977544+0000", 2023-08-28T02:26:59.453 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:26:59.453 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.453 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.454 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:26:59.454 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.977544+0000", 2023-08-28T02:26:59.454 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:26:59.454 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.454 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.454 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:26:59.454 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.977544+0000", 2023-08-28T02:26:59.454 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:26:59.454 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.454 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.454 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:26:59.454 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.977551+0000", 2023-08-28T02:26:59.454 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.9609999999999999e-06 2023-08-28T02:26:59.455 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.455 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.455 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:26:59.455 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.977560+0000", 2023-08-28T02:26:59.455 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.0170000000000002e-06 2023-08-28T02:26:59.455 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.455 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.455 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:26:59.455 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.977566+0000", 2023-08-28T02:26:59.455 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.015e-06 2023-08-28T02:26:59.455 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.455 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.455 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:26:59.456 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.981719+0000", 2023-08-28T02:26:59.456 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0041529369999999998 2023-08-28T02:26:59.456 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.456 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.456 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:26:59.456 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.981738+0000", 2023-08-28T02:26:59.456 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.9598000000000001e-05 2023-08-28T02:26:59.456 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.456 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.456 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:26:59.456 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:59.193139+0000", 2023-08-28T02:26:59.456 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.211400478 2023-08-28T02:26:59.456 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.457 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.457 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:26:59.457 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:59.193171+0000", 2023-08-28T02:26:59.457 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.2020999999999997e-05 2023-08-28T02:26:59.457 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:26:59.457 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:26:59.457 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:26:59.457 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.457 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.457 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop_reply(client.44594.0:196489 5.0 e179/154 ondisk, result = 0)", 2023-08-28T02:26:59.457 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:25:58.984389+0000", 2023-08-28T02:26:59.457 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 60.326096331999999, 2023-08-28T02:26:59.458 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.212816752, 2023-08-28T02:26:59.458 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:26:59.458 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "started", 2023-08-28T02:26:59.458 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:26:59.458 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.458 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:26:59.458 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.984389+0000", 2023-08-28T02:26:59.458 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:26:59.458 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.458 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.458 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:26:59.459 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.984389+0000", 2023-08-28T02:26:59.459 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:26:59.459 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.459 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.459 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:26:59.459 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.984388+0000", 2023-08-28T02:26:59.459 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:26:59.459 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.459 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.459 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:26:59.459 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.984393+0000", 2023-08-28T02:26:59.459 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.0000000000000004e-06 2023-08-28T02:26:59.459 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.460 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.460 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:26:59.460 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.984403+0000", 2023-08-28T02:26:59.460 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0233e-05 2023-08-28T02:26:59.460 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.460 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.460 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:26:59.460 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.984410+0000", 2023-08-28T02:26:59.460 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.562e-06 2023-08-28T02:26:59.460 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.460 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.460 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:26:59.460 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:59.197185+0000", 2023-08-28T02:26:59.461 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.21277526399999999 2023-08-28T02:26:59.461 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.461 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.461 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:26:59.461 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:59.197194+0000", 2023-08-28T02:26:59.461 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.4440000000000008e-06 2023-08-28T02:26:59.461 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:26:59.461 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:26:59.461 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:26:59.461 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:59.197206+0000", 2023-08-28T02:26:59.461 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1912e-05 2023-08-28T02:26:59.461 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:26:59.462 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:26:59.462 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:26:59.462 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:26:59.462 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:26:59.462 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:26:59.462 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:26:59.463 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:26:59.463 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is unset 2023-08-28T02:26:59.491 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:59 smithi143 ceph-mon[183059]: failed to encode map e189 with expected crc 2023-08-28T02:26:59.492 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:59 smithi143 ceph-mon[183059]: osdmap e189: 8 total, 8 up, 8 in 2023-08-28T02:26:59.492 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:59 smithi143 ceph-mon[183059]: failed to encode map e189 with expected crc 2023-08-28T02:26:59.492 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:59 smithi143 ceph-mon[183059]: failed to encode map e189 with expected crc 2023-08-28T02:26:59.493 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:59 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3427626795' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:26:59.493 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:26:59 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:26:59.493 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:59 smithi143 ceph-mon[189871]: failed to encode map e189 with expected crc 2023-08-28T02:26:59.493 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:59 smithi143 ceph-mon[189871]: osdmap e189: 8 total, 8 up, 8 in 2023-08-28T02:26:59.494 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:59 smithi143 ceph-mon[189871]: failed to encode map e189 with expected crc 2023-08-28T02:26:59.494 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:59 smithi143 ceph-mon[189871]: failed to encode map e189 with expected crc 2023-08-28T02:26:59.494 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:59 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3427626795' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:26:59.494 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:26:59 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:26:59.500 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:26:59.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:59 smithi169 ceph-mon[159415]: failed to encode map e189 with expected crc 2023-08-28T02:26:59.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:59 smithi169 ceph-mon[159415]: osdmap e189: 8 total, 8 up, 8 in 2023-08-28T02:26:59.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:59 smithi169 ceph-mon[159415]: failed to encode map e189 with expected crc 2023-08-28T02:26:59.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:59 smithi169 ceph-mon[159415]: failed to encode map e189 with expected crc 2023-08-28T02:26:59.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:59 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3427626795' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:26:59.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:26:59 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:27:00.046 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:27:00.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:00.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:27:00.149 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:27:00.184 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:00.235 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:27:00.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:00.240 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:27:00.341 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:27:00.368 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:00.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:00 smithi169 ceph-mon[159415]: failed to encode map e189 with expected crc 2023-08-28T02:27:00.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:00 smithi169 ceph-mon[159415]: pgmap v445: 89 pgs: 8 creating+peering, 81 active+clean; 4.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:27:00.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:00 smithi169 ceph-mon[159415]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:27:00.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:00 smithi169 ceph-mon[159415]: Cluster is now healthy 2023-08-28T02:27:00.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:00 smithi169 ceph-mon[159415]: failed to encode map e190 with expected crc 2023-08-28T02:27:00.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:00 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:27:00.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:00 smithi169 ceph-mon[159415]: osdmap e190: 8 total, 8 up, 8 in 2023-08-28T02:27:00.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:00 smithi169 ceph-mon[159415]: failed to encode map e190 with expected crc 2023-08-28T02:27:00.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:00 smithi169 ceph-mon[159415]: failed to encode map e190 with expected crc 2023-08-28T02:27:00.505 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.1 dump_ops_in_flight 2023-08-28T02:27:00.662 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:00 smithi143 ceph-mon[183059]: failed to encode map e189 with expected crc 2023-08-28T02:27:00.662 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:00 smithi143 ceph-mon[183059]: pgmap v445: 89 pgs: 8 creating+peering, 81 active+clean; 4.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:27:00.662 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:00 smithi143 ceph-mon[183059]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:27:00.662 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:00 smithi143 ceph-mon[183059]: Cluster is now healthy 2023-08-28T02:27:00.663 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:00 smithi143 ceph-mon[183059]: failed to encode map e190 with expected crc 2023-08-28T02:27:00.663 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:00 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:27:00.663 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:00 smithi143 ceph-mon[183059]: osdmap e190: 8 total, 8 up, 8 in 2023-08-28T02:27:00.663 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:00 smithi143 ceph-mon[183059]: failed to encode map e190 with expected crc 2023-08-28T02:27:00.663 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:00 smithi143 ceph-mon[183059]: failed to encode map e190 with expected crc 2023-08-28T02:27:00.663 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:00 smithi143 ceph-mon[189871]: failed to encode map e189 with expected crc 2023-08-28T02:27:00.663 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:00 smithi143 ceph-mon[189871]: pgmap v445: 89 pgs: 8 creating+peering, 81 active+clean; 4.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:27:00.663 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:00 smithi143 ceph-mon[189871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:27:00.663 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:00 smithi143 ceph-mon[189871]: Cluster is now healthy 2023-08-28T02:27:00.664 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:00 smithi143 ceph-mon[189871]: failed to encode map e190 with expected crc 2023-08-28T02:27:00.664 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:00 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:27:00.664 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:00 smithi143 ceph-mon[189871]: osdmap e190: 8 total, 8 up, 8 in 2023-08-28T02:27:00.664 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:00 smithi143 ceph-mon[189871]: failed to encode map e190 with expected crc 2023-08-28T02:27:00.664 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:00 smithi143 ceph-mon[189871]: failed to encode map e190 with expected crc 2023-08-28T02:27:01.468 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:27:01.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:01.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:27:01.572 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:27:01.600 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:01 smithi143 ceph-mon[183059]: failed to encode map e190 with expected crc 2023-08-28T02:27:01.600 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:01 smithi143 ceph-mon[183059]: pgmap v447: 89 pgs: 8 creating+peering, 81 active+clean; 4.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:27:01.601 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:01 smithi143 ceph-mon[189871]: failed to encode map e190 with expected crc 2023-08-28T02:27:01.601 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:01 smithi143 ceph-mon[189871]: pgmap v447: 89 pgs: 8 creating+peering, 81 active+clean; 4.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:27:01.604 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:01.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:01 smithi169 ceph-mon[159415]: failed to encode map e190 with expected crc 2023-08-28T02:27:01.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:01 smithi169 ceph-mon[159415]: pgmap v447: 89 pgs: 8 creating+peering, 81 active+clean; 4.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:27:02.067 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:27:02.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:02.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:27:02.172 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:27:02.199 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:27:01 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:27:01] "GET /metrics HTTP/1.1" 200 34120 "" "Prometheus/2.33.4" 2023-08-28T02:27:02.205 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:02.254 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:27:02.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:02.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:27:02.360 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:27:02.388 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:02.432 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:27:02.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:02.436 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:27:02.537 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:27:02.564 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:02.606 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:27:02.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:02.610 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:27:02.651 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:27:02.652 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:27:02.652 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:27:02.652 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:27:02.670 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:27:02.687 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:27:02.705 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:27:02.710 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:27:02.723 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:27:02.724 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:27:02.724 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:27:02.724 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:27:02.725 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:27:02.725 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:27:02.725 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:27:02.725 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:27:02.725 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:27:02.735 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:27:02.736 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:02.778 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:27:02.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:02.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:27:02.885 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:27:02.908 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:02 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2923421808' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:27:02.911 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:02.928 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:02 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2923421808' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:27:02.929 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:02 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2923421808' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:27:02.953 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:27:02.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:02.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:27:02.992 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set noscrub 2023-08-28T02:27:03.058 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:27:03.084 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:03.125 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:27:03.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:03.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:27:03.230 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:27:03.255 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:03.297 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:27:03.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:03.305 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:27:03.406 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:27:03.432 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:03.474 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:27:03.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:03.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:27:03.580 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:27:03.605 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:03.648 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:27:03.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:03.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:27:03.683 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:03 smithi143 ceph-mon[183059]: pgmap v448: 89 pgs: 89 active+clean; 4.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:27:03.683 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:03 smithi143 ceph-mon[189871]: pgmap v448: 89 pgs: 89 active+clean; 4.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:27:03.756 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:27:03.781 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:03.824 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:27:03.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:03.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:27:03.929 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:27:03.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:03 smithi143 conmon[183035]: 2023-08-28T02:27:03.912+0000 7fadc3887700 -1 mon.a@0(leader).osd e190 definitely_dead 0 2023-08-28T02:27:03.952 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:03 smithi169 ceph-mon[159415]: pgmap v448: 89 pgs: 89 active+clean; 4.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2023-08-28T02:27:03.954 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:03.996 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:27:03.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:03.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:27:04.100 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:27:04.130 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:04.172 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:27:04.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:04.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:27:04.276 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:27:04.303 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:04.343 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:27:04.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:04.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:27:04.447 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:27:04.475 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:04.655 INFO:teuthology.orchestra.run.smithi143.stderr:marked out osd.1. 2023-08-28T02:27:04.659 INFO:teuthology.orchestra.run.smithi143.stderr:enabled application 'rados' on pool 'unique_pool_1' 2023-08-28T02:27:04.669 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:27:04.682 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:27:04.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:04.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:27:04.786 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:27:04.811 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:04 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1419305356' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-08-28T02:27:04.811 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:04 smithi143 ceph-mon[183059]: Client client.admin marked osd.1 out, while it was still marked up 2023-08-28T02:27:04.811 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:04 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3978737866' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-28T02:27:04.812 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:04 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1419305356' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-08-28T02:27:04.812 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:04 smithi143 ceph-mon[189871]: Client client.admin marked osd.1 out, while it was still marked up 2023-08-28T02:27:04.812 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:04 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3978737866' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-28T02:27:04.814 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:04.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:04 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1419305356' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-08-28T02:27:04.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:04 smithi169 ceph-mon[159415]: Client client.admin marked osd.1 out, while it was still marked up 2023-08-28T02:27:04.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:04 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3978737866' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-28T02:27:05.261 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:27:05.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:05.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:27:05.364 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:27:05.397 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:05.445 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:27:05.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:05.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:27:05.551 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:27:05.579 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:05.623 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:27:05.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:05.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:27:05.708 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:27:05.728 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:27:05.753 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:05.795 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:27:05.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:05.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:27:05.901 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:27:05.924 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:05 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1419305356' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-08-28T02:27:05.924 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:05 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3978737866' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-08-28T02:27:05.925 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:05 smithi169 ceph-mon[159415]: osdmap e191: 8 total, 8 up, 7 in 2023-08-28T02:27:05.925 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:05 smithi169 ceph-mon[159415]: failed to encode map e191 with expected crc 2023-08-28T02:27:05.925 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:05 smithi169 ceph-mon[159415]: failed to encode map e191 with expected crc 2023-08-28T02:27:05.925 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:05 smithi169 ceph-mon[159415]: failed to encode map e191 with expected crc 2023-08-28T02:27:05.925 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:05 smithi169 ceph-mon[159415]: failed to encode map e191 with expected crc 2023-08-28T02:27:05.925 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:05 smithi169 ceph-mon[159415]: failed to encode map e191 with expected crc 2023-08-28T02:27:05.925 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:05 smithi169 ceph-mon[159415]: failed to encode map e191 with expected crc 2023-08-28T02:27:05.925 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:05 smithi169 ceph-mon[159415]: failed to encode map e191 with expected crc 2023-08-28T02:27:05.925 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:05 smithi169 ceph-mon[159415]: failed to encode map e191 with expected crc 2023-08-28T02:27:05.925 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:05 smithi169 ceph-mon[159415]: failed to encode map e191 with expected crc 2023-08-28T02:27:05.925 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:05 smithi169 ceph-mon[159415]: failed to encode map e191 with expected crc 2023-08-28T02:27:05.925 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:05 smithi169 ceph-mon[159415]: pgmap v450: 89 pgs: 89 active+clean; 4.6 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-28T02:27:05.926 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:05.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:05 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1419305356' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-08-28T02:27:05.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:05 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3978737866' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-08-28T02:27:05.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:05 smithi143 ceph-mon[189871]: osdmap e191: 8 total, 8 up, 7 in 2023-08-28T02:27:05.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:05 smithi143 ceph-mon[189871]: failed to encode map e191 with expected crc 2023-08-28T02:27:05.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:05 smithi143 ceph-mon[189871]: failed to encode map e191 with expected crc 2023-08-28T02:27:05.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:05 smithi143 ceph-mon[189871]: failed to encode map e191 with expected crc 2023-08-28T02:27:05.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:05 smithi143 ceph-mon[189871]: failed to encode map e191 with expected crc 2023-08-28T02:27:05.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:05 smithi143 ceph-mon[189871]: failed to encode map e191 with expected crc 2023-08-28T02:27:05.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:05 smithi143 ceph-mon[189871]: failed to encode map e191 with expected crc 2023-08-28T02:27:05.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:05 smithi143 ceph-mon[189871]: failed to encode map e191 with expected crc 2023-08-28T02:27:05.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:05 smithi143 ceph-mon[189871]: failed to encode map e191 with expected crc 2023-08-28T02:27:05.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:05 smithi143 ceph-mon[189871]: failed to encode map e191 with expected crc 2023-08-28T02:27:05.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:05 smithi143 ceph-mon[189871]: failed to encode map e191 with expected crc 2023-08-28T02:27:05.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:05 smithi143 ceph-mon[189871]: pgmap v450: 89 pgs: 89 active+clean; 4.6 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-28T02:27:05.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:05 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1419305356' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-08-28T02:27:05.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:05 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3978737866' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-08-28T02:27:05.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:05 smithi143 ceph-mon[183059]: osdmap e191: 8 total, 8 up, 7 in 2023-08-28T02:27:05.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:05 smithi143 ceph-mon[183059]: failed to encode map e191 with expected crc 2023-08-28T02:27:05.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:05 smithi143 ceph-mon[183059]: failed to encode map e191 with expected crc 2023-08-28T02:27:05.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:05 smithi143 ceph-mon[183059]: failed to encode map e191 with expected crc 2023-08-28T02:27:05.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:05 smithi143 ceph-mon[183059]: failed to encode map e191 with expected crc 2023-08-28T02:27:05.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:05 smithi143 ceph-mon[183059]: failed to encode map e191 with expected crc 2023-08-28T02:27:05.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:05 smithi143 ceph-mon[183059]: failed to encode map e191 with expected crc 2023-08-28T02:27:05.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:05 smithi143 ceph-mon[183059]: failed to encode map e191 with expected crc 2023-08-28T02:27:05.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:05 smithi143 ceph-mon[183059]: failed to encode map e191 with expected crc 2023-08-28T02:27:05.953 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:05 smithi143 ceph-mon[183059]: failed to encode map e191 with expected crc 2023-08-28T02:27:05.953 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:05 smithi143 ceph-mon[183059]: failed to encode map e191 with expected crc 2023-08-28T02:27:05.953 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:05 smithi143 ceph-mon[183059]: pgmap v450: 89 pgs: 89 active+clean; 4.6 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-08-28T02:27:05.968 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:27:05.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:05.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:27:06.074 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:27:06.102 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:06.204 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:27:06.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:06.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:27:06.264 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:27:06.264 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:27:06.308 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:27:06.344 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:06.443 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:27:06.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:06.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:27:06.547 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:27:06.573 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:06.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:06 smithi143 ceph-mon[189871]: failed to encode map e192 with expected crc 2023-08-28T02:27:06.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:06 smithi143 ceph-mon[189871]: failed to encode map e192 with expected crc 2023-08-28T02:27:06.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:06 smithi143 ceph-mon[189871]: failed to encode map e192 with expected crc 2023-08-28T02:27:06.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:06 smithi143 ceph-mon[189871]: failed to encode map e192 with expected crc 2023-08-28T02:27:06.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:06 smithi143 ceph-mon[189871]: failed to encode map e192 with expected crc 2023-08-28T02:27:06.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:06 smithi143 ceph-mon[189871]: failed to encode map e192 with expected crc 2023-08-28T02:27:06.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:06 smithi143 ceph-mon[189871]: failed to encode map e192 with expected crc 2023-08-28T02:27:06.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:06 smithi143 ceph-mon[189871]: failed to encode map e192 with expected crc 2023-08-28T02:27:06.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:06 smithi143 ceph-mon[189871]: failed to encode map e192 with expected crc 2023-08-28T02:27:06.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:06 smithi143 ceph-mon[189871]: failed to encode map e192 with expected crc 2023-08-28T02:27:06.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:06 smithi143 ceph-mon[189871]: failed to encode map e192 with expected crc 2023-08-28T02:27:06.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:06 smithi143 ceph-mon[189871]: osdmap e192: 8 total, 8 up, 7 in 2023-08-28T02:27:06.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:06 smithi143 ceph-mon[189871]: failed to encode map e192 with expected crc 2023-08-28T02:27:06.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:06 smithi143 ceph-mon[183059]: failed to encode map e192 with expected crc 2023-08-28T02:27:06.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:06 smithi143 ceph-mon[183059]: failed to encode map e192 with expected crc 2023-08-28T02:27:06.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:06 smithi143 ceph-mon[183059]: failed to encode map e192 with expected crc 2023-08-28T02:27:06.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:06 smithi143 ceph-mon[183059]: failed to encode map e192 with expected crc 2023-08-28T02:27:06.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:06 smithi143 ceph-mon[183059]: failed to encode map e192 with expected crc 2023-08-28T02:27:06.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:06 smithi143 ceph-mon[183059]: failed to encode map e192 with expected crc 2023-08-28T02:27:06.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:06 smithi143 ceph-mon[183059]: failed to encode map e192 with expected crc 2023-08-28T02:27:06.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:06 smithi143 ceph-mon[183059]: failed to encode map e192 with expected crc 2023-08-28T02:27:06.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:06 smithi143 ceph-mon[183059]: failed to encode map e192 with expected crc 2023-08-28T02:27:06.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:06 smithi143 ceph-mon[183059]: failed to encode map e192 with expected crc 2023-08-28T02:27:06.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:06 smithi143 ceph-mon[183059]: failed to encode map e192 with expected crc 2023-08-28T02:27:06.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:06 smithi143 ceph-mon[183059]: osdmap e192: 8 total, 8 up, 7 in 2023-08-28T02:27:06.953 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:06 smithi143 ceph-mon[183059]: failed to encode map e192 with expected crc 2023-08-28T02:27:06.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:06 smithi169 ceph-mon[159415]: failed to encode map e192 with expected crc 2023-08-28T02:27:06.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:06 smithi169 ceph-mon[159415]: failed to encode map e192 with expected crc 2023-08-28T02:27:06.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:06 smithi169 ceph-mon[159415]: failed to encode map e192 with expected crc 2023-08-28T02:27:06.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:06 smithi169 ceph-mon[159415]: failed to encode map e192 with expected crc 2023-08-28T02:27:06.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:06 smithi169 ceph-mon[159415]: failed to encode map e192 with expected crc 2023-08-28T02:27:06.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:06 smithi169 ceph-mon[159415]: failed to encode map e192 with expected crc 2023-08-28T02:27:06.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:06 smithi169 ceph-mon[159415]: failed to encode map e192 with expected crc 2023-08-28T02:27:06.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:06 smithi169 ceph-mon[159415]: failed to encode map e192 with expected crc 2023-08-28T02:27:06.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:06 smithi169 ceph-mon[159415]: failed to encode map e192 with expected crc 2023-08-28T02:27:06.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:06 smithi169 ceph-mon[159415]: failed to encode map e192 with expected crc 2023-08-28T02:27:06.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:06 smithi169 ceph-mon[159415]: failed to encode map e192 with expected crc 2023-08-28T02:27:06.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:06 smithi169 ceph-mon[159415]: osdmap e192: 8 total, 8 up, 7 in 2023-08-28T02:27:06.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:06 smithi169 ceph-mon[159415]: failed to encode map e192 with expected crc 2023-08-28T02:27:07.043 DEBUG:teuthology.orchestra.run.smithi143:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2023-08-28T02:27:07.045 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-08-28T02:27:07.045 INFO:teuthology.orchestra.run:waiting for 3000 2023-08-28T02:27:07.090 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:hints = 1 2023-08-28T02:27:07.090 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-08-28T02:27:07.091 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Object prefix: benchmark_data_smithi143_305742 2023-08-28T02:27:07.091 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-08-28T02:27:07.091 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 0 0 0 0 0 0 - 0 2023-08-28T02:27:07.119 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:27:07.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:07.122 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:27:07.223 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:27:07.250 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:07.331 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:27:07.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:07.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:27:07.441 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:27:07.491 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:07.589 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:27:07.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:07.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:27:07.600 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.1 dump_blocked_ops 2023-08-28T02:27:07.620 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:27:07.700 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:27:07.779 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:07 smithi143 ceph-mon[189871]: osdmap e193: 8 total, 8 up, 7 in 2023-08-28T02:27:07.780 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:07 smithi143 ceph-mon[189871]: failed to encode map e193 with expected crc 2023-08-28T02:27:07.780 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:07 smithi143 ceph-mon[189871]: failed to encode map e193 with expected crc 2023-08-28T02:27:07.780 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:07 smithi143 ceph-mon[189871]: failed to encode map e193 with expected crc 2023-08-28T02:27:07.780 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:07 smithi143 ceph-mon[189871]: failed to encode map e193 with expected crc 2023-08-28T02:27:07.780 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:07 smithi143 ceph-mon[189871]: failed to encode map e193 with expected crc 2023-08-28T02:27:07.780 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:07 smithi143 ceph-mon[189871]: failed to encode map e193 with expected crc 2023-08-28T02:27:07.780 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:07 smithi143 ceph-mon[189871]: failed to encode map e193 with expected crc 2023-08-28T02:27:07.780 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:07 smithi143 ceph-mon[189871]: pgmap v453: 89 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.2 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 107 KiB/s wr, 1 op/s; 1/114 objects misplaced (0.877%); 0 B/s, 3 objects/s recovering 2023-08-28T02:27:07.782 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:07.783 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:07 smithi143 ceph-mon[183059]: osdmap e193: 8 total, 8 up, 7 in 2023-08-28T02:27:07.783 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:07 smithi143 ceph-mon[183059]: failed to encode map e193 with expected crc 2023-08-28T02:27:07.783 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:07 smithi143 ceph-mon[183059]: failed to encode map e193 with expected crc 2023-08-28T02:27:07.783 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:07 smithi143 ceph-mon[183059]: failed to encode map e193 with expected crc 2023-08-28T02:27:07.783 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:07 smithi143 ceph-mon[183059]: failed to encode map e193 with expected crc 2023-08-28T02:27:07.783 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:07 smithi143 ceph-mon[183059]: failed to encode map e193 with expected crc 2023-08-28T02:27:07.783 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:07 smithi143 ceph-mon[183059]: failed to encode map e193 with expected crc 2023-08-28T02:27:07.783 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:07 smithi143 ceph-mon[183059]: failed to encode map e193 with expected crc 2023-08-28T02:27:07.784 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:07 smithi143 ceph-mon[183059]: pgmap v453: 89 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.2 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 107 KiB/s wr, 1 op/s; 1/114 objects misplaced (0.877%); 0 B/s, 3 objects/s recovering 2023-08-28T02:27:07.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:07 smithi169 ceph-mon[159415]: osdmap e193: 8 total, 8 up, 7 in 2023-08-28T02:27:07.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:07 smithi169 ceph-mon[159415]: failed to encode map e193 with expected crc 2023-08-28T02:27:07.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:07 smithi169 ceph-mon[159415]: failed to encode map e193 with expected crc 2023-08-28T02:27:07.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:07 smithi169 ceph-mon[159415]: failed to encode map e193 with expected crc 2023-08-28T02:27:07.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:07 smithi169 ceph-mon[159415]: failed to encode map e193 with expected crc 2023-08-28T02:27:07.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:07 smithi169 ceph-mon[159415]: failed to encode map e193 with expected crc 2023-08-28T02:27:07.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:07 smithi169 ceph-mon[159415]: failed to encode map e193 with expected crc 2023-08-28T02:27:07.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:07 smithi169 ceph-mon[159415]: failed to encode map e193 with expected crc 2023-08-28T02:27:07.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:07 smithi169 ceph-mon[159415]: pgmap v453: 89 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 5.2 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 107 KiB/s wr, 1 op/s; 1/114 objects misplaced (0.877%); 0 B/s, 3 objects/s recovering 2023-08-28T02:27:08.038 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:27:08.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:08.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:27:08.091 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 1 16 2813 2797 174.794 174.812 0.00292476 0.00569765 2023-08-28T02:27:08.144 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:27:08.206 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:08.350 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:27:08.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:08.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:27:08.462 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:27:08.520 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:08.611 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:27:08.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:08.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:27:08.698 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is set 2023-08-28T02:27:08.717 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:27:08.772 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:08.862 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:27:08.866 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:08 smithi143 ceph-mon[189871]: failed to encode map e194 with expected crc 2023-08-28T02:27:08.866 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:08 smithi143 ceph-mon[189871]: failed to encode map e194 with expected crc 2023-08-28T02:27:08.866 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:08 smithi143 ceph-mon[189871]: failed to encode map e194 with expected crc 2023-08-28T02:27:08.866 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:08 smithi143 ceph-mon[189871]: failed to encode map e194 with expected crc 2023-08-28T02:27:08.866 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:08 smithi143 ceph-mon[189871]: failed to encode map e194 with expected crc 2023-08-28T02:27:08.866 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:08 smithi143 ceph-mon[189871]: failed to encode map e194 with expected crc 2023-08-28T02:27:08.866 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:08 smithi143 ceph-mon[189871]: osdmap e194: 8 total, 8 up, 7 in 2023-08-28T02:27:08.866 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:08 smithi143 ceph-mon[189871]: failed to encode map e194 with expected crc 2023-08-28T02:27:08.866 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:08 smithi143 ceph-mon[189871]: failed to encode map e194 with expected crc 2023-08-28T02:27:08.866 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:08 smithi143 ceph-mon[189871]: failed to encode map e194 with expected crc 2023-08-28T02:27:08.867 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:08 smithi143 ceph-mon[189871]: failed to encode map e194 with expected crc 2023-08-28T02:27:08.867 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:08 smithi143 ceph-mon[189871]: failed to encode map e194 with expected crc 2023-08-28T02:27:08.867 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:08 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3552911537' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:27:08.867 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:08 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:27:08.867 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:08 smithi143 ceph-mon[183059]: failed to encode map e194 with expected crc 2023-08-28T02:27:08.867 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:08 smithi143 ceph-mon[183059]: failed to encode map e194 with expected crc 2023-08-28T02:27:08.867 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:08 smithi143 ceph-mon[183059]: failed to encode map e194 with expected crc 2023-08-28T02:27:08.867 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:08 smithi143 ceph-mon[183059]: failed to encode map e194 with expected crc 2023-08-28T02:27:08.867 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:08 smithi143 ceph-mon[183059]: failed to encode map e194 with expected crc 2023-08-28T02:27:08.867 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:08 smithi143 ceph-mon[183059]: failed to encode map e194 with expected crc 2023-08-28T02:27:08.868 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:08 smithi143 ceph-mon[183059]: osdmap e194: 8 total, 8 up, 7 in 2023-08-28T02:27:08.868 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:08 smithi143 ceph-mon[183059]: failed to encode map e194 with expected crc 2023-08-28T02:27:08.868 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:08 smithi143 ceph-mon[183059]: failed to encode map e194 with expected crc 2023-08-28T02:27:08.868 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:08 smithi143 ceph-mon[183059]: failed to encode map e194 with expected crc 2023-08-28T02:27:08.868 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:08 smithi143 ceph-mon[183059]: failed to encode map e194 with expected crc 2023-08-28T02:27:08.868 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:08 smithi143 ceph-mon[183059]: failed to encode map e194 with expected crc 2023-08-28T02:27:08.868 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:08 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3552911537' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:27:08.868 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:08 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:27:08.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:08.869 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:27:08.970 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:27:08.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:08 smithi169 ceph-mon[159415]: failed to encode map e194 with expected crc 2023-08-28T02:27:08.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:08 smithi169 ceph-mon[159415]: failed to encode map e194 with expected crc 2023-08-28T02:27:08.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:08 smithi169 ceph-mon[159415]: failed to encode map e194 with expected crc 2023-08-28T02:27:08.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:08 smithi169 ceph-mon[159415]: failed to encode map e194 with expected crc 2023-08-28T02:27:08.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:08 smithi169 ceph-mon[159415]: failed to encode map e194 with expected crc 2023-08-28T02:27:08.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:08 smithi169 ceph-mon[159415]: failed to encode map e194 with expected crc 2023-08-28T02:27:08.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:08 smithi169 ceph-mon[159415]: osdmap e194: 8 total, 8 up, 7 in 2023-08-28T02:27:08.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:08 smithi169 ceph-mon[159415]: failed to encode map e194 with expected crc 2023-08-28T02:27:08.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:08 smithi169 ceph-mon[159415]: failed to encode map e194 with expected crc 2023-08-28T02:27:08.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:08 smithi169 ceph-mon[159415]: failed to encode map e194 with expected crc 2023-08-28T02:27:08.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:08 smithi169 ceph-mon[159415]: failed to encode map e194 with expected crc 2023-08-28T02:27:08.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:08 smithi169 ceph-mon[159415]: failed to encode map e194 with expected crc 2023-08-28T02:27:08.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:08 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3552911537' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:27:08.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:08 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:27:09.020 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:09.091 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 2 16 6130 6114 191.044 207.312 0.00165261 0.00521748 2023-08-28T02:27:09.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:09 smithi143 ceph-mon[183059]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:27:09.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:09 smithi143 ceph-mon[183059]: failed to encode map e195 with expected crc 2023-08-28T02:27:09.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:09 smithi143 ceph-mon[183059]: failed to encode map e195 with expected crc 2023-08-28T02:27:09.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:09 smithi143 ceph-mon[183059]: failed to encode map e195 with expected crc 2023-08-28T02:27:09.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:09 smithi143 ceph-mon[183059]: failed to encode map e195 with expected crc 2023-08-28T02:27:09.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:09 smithi143 ceph-mon[183059]: failed to encode map e195 with expected crc 2023-08-28T02:27:09.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:09 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:27:09.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:09 smithi143 ceph-mon[183059]: osdmap e195: 8 total, 8 up, 7 in 2023-08-28T02:27:09.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:09 smithi143 ceph-mon[183059]: failed to encode map e195 with expected crc 2023-08-28T02:27:09.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:09 smithi143 ceph-mon[183059]: failed to encode map e195 with expected crc 2023-08-28T02:27:09.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:09 smithi143 ceph-mon[183059]: failed to encode map e195 with expected crc 2023-08-28T02:27:09.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:09 smithi143 ceph-mon[183059]: failed to encode map e195 with expected crc 2023-08-28T02:27:09.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:09 smithi143 ceph-mon[183059]: failed to encode map e195 with expected crc 2023-08-28T02:27:09.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:09 smithi143 ceph-mon[183059]: failed to encode map e195 with expected crc 2023-08-28T02:27:09.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:09 smithi143 ceph-mon[183059]: failed to encode map e195 with expected crc 2023-08-28T02:27:09.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:09 smithi143 ceph-mon[183059]: failed to encode map e195 with expected crc 2023-08-28T02:27:09.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:09 smithi143 ceph-mon[183059]: failed to encode map e195 with expected crc 2023-08-28T02:27:09.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:09 smithi143 ceph-mon[183059]: failed to encode map e195 with expected crc 2023-08-28T02:27:09.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:09 smithi143 ceph-mon[183059]: pgmap v456: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 86 active+clean; 132 MiB data, 2.3 GiB used, 624 GiB / 626 GiB avail; 32 MiB/s wr, 509 op/s; 16/6201 objects degraded (0.258%); 1/6201 objects misplaced (0.016%); 0 B/s, 7 objects/s recovering 2023-08-28T02:27:09.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:09 smithi143 ceph-mon[189871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:27:09.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:09 smithi143 ceph-mon[189871]: failed to encode map e195 with expected crc 2023-08-28T02:27:09.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:09 smithi143 ceph-mon[189871]: failed to encode map e195 with expected crc 2023-08-28T02:27:09.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:09 smithi143 ceph-mon[189871]: failed to encode map e195 with expected crc 2023-08-28T02:27:09.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:09 smithi143 ceph-mon[189871]: failed to encode map e195 with expected crc 2023-08-28T02:27:09.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:09 smithi143 ceph-mon[189871]: failed to encode map e195 with expected crc 2023-08-28T02:27:09.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:09 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:27:09.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:09 smithi143 ceph-mon[189871]: osdmap e195: 8 total, 8 up, 7 in 2023-08-28T02:27:09.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:09 smithi143 ceph-mon[189871]: failed to encode map e195 with expected crc 2023-08-28T02:27:09.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:09 smithi143 ceph-mon[189871]: failed to encode map e195 with expected crc 2023-08-28T02:27:09.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:09 smithi143 ceph-mon[189871]: failed to encode map e195 with expected crc 2023-08-28T02:27:09.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:09 smithi143 ceph-mon[189871]: failed to encode map e195 with expected crc 2023-08-28T02:27:09.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:09 smithi143 ceph-mon[189871]: failed to encode map e195 with expected crc 2023-08-28T02:27:09.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:09 smithi143 ceph-mon[189871]: failed to encode map e195 with expected crc 2023-08-28T02:27:09.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:09 smithi143 ceph-mon[189871]: failed to encode map e195 with expected crc 2023-08-28T02:27:09.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:09 smithi143 ceph-mon[189871]: failed to encode map e195 with expected crc 2023-08-28T02:27:09.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:09 smithi143 ceph-mon[189871]: failed to encode map e195 with expected crc 2023-08-28T02:27:09.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:09 smithi143 ceph-mon[189871]: failed to encode map e195 with expected crc 2023-08-28T02:27:09.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:09 smithi143 ceph-mon[189871]: pgmap v456: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 86 active+clean; 132 MiB data, 2.3 GiB used, 624 GiB / 626 GiB avail; 32 MiB/s wr, 509 op/s; 16/6201 objects degraded (0.258%); 1/6201 objects misplaced (0.016%); 0 B/s, 7 objects/s recovering 2023-08-28T02:27:09.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:09 smithi169 ceph-mon[159415]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:27:09.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:09 smithi169 ceph-mon[159415]: failed to encode map e195 with expected crc 2023-08-28T02:27:09.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:09 smithi169 ceph-mon[159415]: failed to encode map e195 with expected crc 2023-08-28T02:27:09.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:09 smithi169 ceph-mon[159415]: failed to encode map e195 with expected crc 2023-08-28T02:27:09.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:09 smithi169 ceph-mon[159415]: failed to encode map e195 with expected crc 2023-08-28T02:27:09.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:09 smithi169 ceph-mon[159415]: failed to encode map e195 with expected crc 2023-08-28T02:27:09.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:09 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:27:09.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:09 smithi169 ceph-mon[159415]: osdmap e195: 8 total, 8 up, 7 in 2023-08-28T02:27:09.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:09 smithi169 ceph-mon[159415]: failed to encode map e195 with expected crc 2023-08-28T02:27:09.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:09 smithi169 ceph-mon[159415]: failed to encode map e195 with expected crc 2023-08-28T02:27:09.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:09 smithi169 ceph-mon[159415]: failed to encode map e195 with expected crc 2023-08-28T02:27:09.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:09 smithi169 ceph-mon[159415]: failed to encode map e195 with expected crc 2023-08-28T02:27:09.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:09 smithi169 ceph-mon[159415]: failed to encode map e195 with expected crc 2023-08-28T02:27:09.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:09 smithi169 ceph-mon[159415]: failed to encode map e195 with expected crc 2023-08-28T02:27:09.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:09 smithi169 ceph-mon[159415]: failed to encode map e195 with expected crc 2023-08-28T02:27:09.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:09 smithi169 ceph-mon[159415]: failed to encode map e195 with expected crc 2023-08-28T02:27:09.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:09 smithi169 ceph-mon[159415]: failed to encode map e195 with expected crc 2023-08-28T02:27:09.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:09 smithi169 ceph-mon[159415]: failed to encode map e195 with expected crc 2023-08-28T02:27:09.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:09 smithi169 ceph-mon[159415]: pgmap v456: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 86 active+clean; 132 MiB data, 2.3 GiB used, 624 GiB / 626 GiB avail; 32 MiB/s wr, 509 op/s; 16/6201 objects degraded (0.258%); 1/6201 objects misplaced (0.016%); 0 B/s, 7 objects/s recovering 2023-08-28T02:27:10.091 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 3 15 9657 9642 200.835 220.5 0.00515384 0.00497067 2023-08-28T02:27:10.385 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 6, 7, 3] out_osds: [1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-08-28T02:27:10.385 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-08-28T02:27:10.386 INFO:tasks.thrashosds.thrasher:Removing osd 6, in_osds are: [0, 2, 4, 5, 6, 7, 3] 2023-08-28T02:27:10.386 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd out 6 2023-08-28T02:27:10.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:10 smithi143 ceph-mon[183059]: Health check failed: Degraded data redundancy: 16/6201 objects degraded (0.258%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:27:10.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:10 smithi143 ceph-mon[189871]: Health check failed: Degraded data redundancy: 16/6201 objects degraded (0.258%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:27:10.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:10 smithi169 ceph-mon[159415]: Health check failed: Degraded data redundancy: 16/6201 objects degraded (0.258%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:27:11.004 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:27:11.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:11.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:27:11.091 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 4 16 12457 12441 194.357 174.938 0.0086401 0.00513718 2023-08-28T02:27:11.101 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:27:11.109 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:27:11.150 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:11.242 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:27:11.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:11.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:27:11.347 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:27:11.406 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:11.497 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:27:11.562 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:27:11.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:11.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:27:11.648 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:27:11.670 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:27:11.695 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:27:11.732 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:27:11.735 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:11.770 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:27:11.808 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:27:11.841 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:27:11.877 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:27:11.909 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:27:11.917 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:27:11.918 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:27:11.918 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:27:11.918 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:27:11.918 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:27:11.918 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:27:11.918 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:27:11.918 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:27:11.996 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:27:11 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:27:11] "GET /metrics HTTP/1.1" 200 36480 "" "Prometheus/2.33.4" 2023-08-28T02:27:12.091 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 5 16 14983 14967 187.059 157.875 0.00484367 0.00534133 2023-08-28T02:27:12.248 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:12 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:12.249 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:12 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:27:12.249 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:12 smithi143 ceph-mon[183059]: pgmap v457: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 86 active+clean; 132 MiB data, 2.3 GiB used, 624 GiB / 626 GiB avail; 23 MiB/s wr, 368 op/s; 16/6201 objects degraded (0.258%); 1/6201 objects misplaced (0.016%); 0 B/s, 5 objects/s recovering 2023-08-28T02:27:12.249 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:12 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:12.249 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:12 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1388102548' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:27:12.249 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:12 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:12.249 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:12 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:27:12.249 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:12 smithi143 ceph-mon[189871]: pgmap v457: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 86 active+clean; 132 MiB data, 2.3 GiB used, 624 GiB / 626 GiB avail; 23 MiB/s wr, 368 op/s; 16/6201 objects degraded (0.258%); 1/6201 objects misplaced (0.016%); 0 B/s, 5 objects/s recovering 2023-08-28T02:27:12.249 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:12 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:12.249 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:12 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1388102548' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:27:12.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:12 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:12.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:12 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:27:12.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:12 smithi169 ceph-mon[159415]: pgmap v457: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 86 active+clean; 132 MiB data, 2.3 GiB used, 624 GiB / 626 GiB avail; 23 MiB/s wr, 368 op/s; 16/6201 objects degraded (0.258%); 1/6201 objects misplaced (0.016%); 0 B/s, 5 objects/s recovering 2023-08-28T02:27:12.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:12 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:12.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:12 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1388102548' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:27:12.543 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:27:12.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:12.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:27:12.652 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:27:12.703 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:13.092 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 6 16 17763 17747 184.807 173.75 0.00842213 0.00540519 2023-08-28T02:27:13.148 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set nodeep-scrub 2023-08-28T02:27:13.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:13 smithi143 ceph-mon[183059]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 16/6201 objects degraded (0.258%), 1 pg degraded) 2023-08-28T02:27:13.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:13 smithi143 ceph-mon[189871]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 16/6201 objects degraded (0.258%), 1 pg degraded) 2023-08-28T02:27:13.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:13 smithi169 ceph-mon[159415]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 16/6201 objects degraded (0.258%), 1 pg degraded) 2023-08-28T02:27:14.092 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 7 16 20093 20077 179.209 145.625 0.0064693 0.00557496 2023-08-28T02:27:14.201 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:27:14.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:14.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:27:14.306 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:27:14.332 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:14.386 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:27:14.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:14.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:27:14.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:14 smithi143 ceph-mon[183059]: pgmap v458: 89 pgs: 89 active+clean; 726 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 110 MiB/s wr, 1.76k op/s; 626 KiB/s, 7 objects/s recovering 2023-08-28T02:27:14.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:14 smithi143 ceph-mon[189871]: pgmap v458: 89 pgs: 89 active+clean; 726 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 110 MiB/s wr, 1.76k op/s; 626 KiB/s, 7 objects/s recovering 2023-08-28T02:27:14.493 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:27:14.541 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:14.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:14 smithi169 ceph-mon[159415]: pgmap v458: 89 pgs: 89 active+clean; 726 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 110 MiB/s wr, 1.76k op/s; 626 KiB/s, 7 objects/s recovering 2023-08-28T02:27:14.817 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:27:14.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:14.825 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:27:14.926 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:27:15.003 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:15.093 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 8 16 21899 21883 170.918 112.875 0.00719581 0.00584591 2023-08-28T02:27:15.169 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:27:15.315 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:27:15.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:15.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:27:15.421 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:27:15.462 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:15.549 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:27:15.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:15.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:27:15.654 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:27:15.655 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [], 2023-08-28T02:27:15.655 INFO:teuthology.orchestra.run.smithi143.stdout: "complaint_time": 30, 2023-08-28T02:27:15.655 INFO:teuthology.orchestra.run.smithi143.stdout: "num_blocked_ops": 0 2023-08-28T02:27:15.655 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:27:15.655 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:27:15.705 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:15.794 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:27:15.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:15.805 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:27:15.906 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:27:15.965 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:16.061 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:27:16.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:16.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:27:16.096 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 9 16 24017 24001 166.634 132.375 0.00562771 0.00599615 2023-08-28T02:27:16.168 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:27:16.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:15 smithi143 conmon[183035]: 2023-08-28T02:27:15.911+0000 7fadc3887700 -1 mon.a@0(leader).osd e195 definitely_dead 0 2023-08-28T02:27:16.224 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:16.314 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:27:16.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:16.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:27:16.333 INFO:teuthology.orchestra.run.smithi143.stderr:marked out osd.6. 2023-08-28T02:27:16.419 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:27:16.449 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:16.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:16 smithi143 ceph-mon[183059]: pgmap v459: 89 pgs: 89 active+clean; 1.0 GiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 132 MiB/s wr, 2.11k op/s; 512 KiB/s, 3 objects/s recovering 2023-08-28T02:27:16.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:16 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1996907245' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-08-28T02:27:16.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:16 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-08-28T02:27:16.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:16 smithi143 ceph-mon[183059]: Client client.admin marked osd.6 out, while it was still marked up 2023-08-28T02:27:16.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:16 smithi143 ceph-mon[189871]: pgmap v459: 89 pgs: 89 active+clean; 1.0 GiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 132 MiB/s wr, 2.11k op/s; 512 KiB/s, 3 objects/s recovering 2023-08-28T02:27:16.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:16 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1996907245' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-08-28T02:27:16.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:16 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-08-28T02:27:16.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:16 smithi143 ceph-mon[189871]: Client client.admin marked osd.6 out, while it was still marked up 2023-08-28T02:27:16.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:16 smithi169 ceph-mon[159415]: pgmap v459: 89 pgs: 89 active+clean; 1.0 GiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 132 MiB/s wr, 2.11k op/s; 512 KiB/s, 3 objects/s recovering 2023-08-28T02:27:16.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:16 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1996907245' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-08-28T02:27:16.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:16 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-08-28T02:27:16.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:16 smithi169 ceph-mon[159415]: Client client.admin marked osd.6 out, while it was still marked up 2023-08-28T02:27:16.921 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:27:17.093 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 10 16 24559 24543 153.36 33.875 0.00192805 0.00602046 2023-08-28T02:27:17.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:17 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-08-28T02:27:17.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:17 smithi143 ceph-mon[183059]: osdmap e196: 8 total, 8 up, 6 in 2023-08-28T02:27:17.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:17 smithi143 ceph-mon[183059]: failed to encode map e196 with expected crc 2023-08-28T02:27:17.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:17 smithi143 ceph-mon[183059]: failed to encode map e196 with expected crc 2023-08-28T02:27:17.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:17 smithi143 ceph-mon[183059]: failed to encode map e196 with expected crc 2023-08-28T02:27:17.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:17 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:27:17.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:17 smithi143 ceph-mon[183059]: osdmap e197: 8 total, 8 up, 6 in 2023-08-28T02:27:17.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:17 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-08-28T02:27:17.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:17 smithi143 ceph-mon[189871]: osdmap e196: 8 total, 8 up, 6 in 2023-08-28T02:27:17.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:17 smithi143 ceph-mon[189871]: failed to encode map e196 with expected crc 2023-08-28T02:27:17.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:17 smithi143 ceph-mon[189871]: failed to encode map e196 with expected crc 2023-08-28T02:27:17.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:17 smithi143 ceph-mon[189871]: failed to encode map e196 with expected crc 2023-08-28T02:27:17.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:17 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:27:17.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:17 smithi143 ceph-mon[189871]: osdmap e197: 8 total, 8 up, 6 in 2023-08-28T02:27:17.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:17 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-08-28T02:27:17.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:17 smithi169 ceph-mon[159415]: osdmap e196: 8 total, 8 up, 6 in 2023-08-28T02:27:17.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:17 smithi169 ceph-mon[159415]: failed to encode map e196 with expected crc 2023-08-28T02:27:17.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:17 smithi169 ceph-mon[159415]: failed to encode map e196 with expected crc 2023-08-28T02:27:17.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:17 smithi169 ceph-mon[159415]: failed to encode map e196 with expected crc 2023-08-28T02:27:17.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:17 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:27:17.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:17 smithi169 ceph-mon[159415]: osdmap e197: 8 total, 8 up, 6 in 2023-08-28T02:27:18.058 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.1 dump_historic_ops 2023-08-28T02:27:18.093 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 11 16 24559 24543 139.42 0 - 0.00602046 2023-08-28T02:27:18.191 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:27:18.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:18.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:27:18.296 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:27:18.348 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:18.494 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:27:18.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:18.506 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:27:18.539 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:27:18.606 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:27:18.658 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:18 smithi169 ceph-mon[159415]: pgmap v461: 89 pgs: 18 peering, 71 active+clean; 1.2 GiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 146 MiB/s wr, 2.33k op/s; 482 KiB/s, 1 objects/s recovering 2023-08-28T02:27:18.658 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:18 smithi169 ceph-mon[159415]: failed to encode map e197 with expected crc 2023-08-28T02:27:18.658 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:18 smithi169 ceph-mon[159415]: failed to encode map e197 with expected crc 2023-08-28T02:27:18.658 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:18 smithi169 ceph-mon[159415]: failed to encode map e197 with expected crc 2023-08-28T02:27:18.658 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:18 smithi169 ceph-mon[159415]: failed to encode map e197 with expected crc 2023-08-28T02:27:18.658 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:18 smithi169 ceph-mon[159415]: failed to encode map e197 with expected crc 2023-08-28T02:27:18.658 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:18 smithi169 ceph-mon[159415]: failed to encode map e197 with expected crc 2023-08-28T02:27:18.658 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:18 smithi169 ceph-mon[159415]: failed to encode map e197 with expected crc 2023-08-28T02:27:18.659 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:18 smithi169 ceph-mon[159415]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-08-28T02:27:18.659 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:18 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:18.659 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:18 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:18.659 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:18 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:18.659 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:18 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:18.659 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:18 smithi169 ceph-mon[159415]: osdmap e198: 8 total, 8 up, 6 in 2023-08-28T02:27:18.664 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:18.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:18 smithi143 ceph-mon[183059]: pgmap v461: 89 pgs: 18 peering, 71 active+clean; 1.2 GiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 146 MiB/s wr, 2.33k op/s; 482 KiB/s, 1 objects/s recovering 2023-08-28T02:27:18.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:18 smithi143 ceph-mon[183059]: failed to encode map e197 with expected crc 2023-08-28T02:27:18.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:18 smithi143 ceph-mon[183059]: failed to encode map e197 with expected crc 2023-08-28T02:27:18.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:18 smithi143 ceph-mon[183059]: failed to encode map e197 with expected crc 2023-08-28T02:27:18.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:18 smithi143 ceph-mon[183059]: failed to encode map e197 with expected crc 2023-08-28T02:27:18.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:18 smithi143 ceph-mon[183059]: failed to encode map e197 with expected crc 2023-08-28T02:27:18.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:18 smithi143 ceph-mon[183059]: failed to encode map e197 with expected crc 2023-08-28T02:27:18.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:18 smithi143 ceph-mon[183059]: failed to encode map e197 with expected crc 2023-08-28T02:27:18.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:18 smithi143 ceph-mon[183059]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-08-28T02:27:18.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:18 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:18.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:18 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:18.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:18 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:18.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:18 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:18.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:18 smithi143 ceph-mon[183059]: osdmap e198: 8 total, 8 up, 6 in 2023-08-28T02:27:18.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:18 smithi143 ceph-mon[189871]: pgmap v461: 89 pgs: 18 peering, 71 active+clean; 1.2 GiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 146 MiB/s wr, 2.33k op/s; 482 KiB/s, 1 objects/s recovering 2023-08-28T02:27:18.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:18 smithi143 ceph-mon[189871]: failed to encode map e197 with expected crc 2023-08-28T02:27:18.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:18 smithi143 ceph-mon[189871]: failed to encode map e197 with expected crc 2023-08-28T02:27:18.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:18 smithi143 ceph-mon[189871]: failed to encode map e197 with expected crc 2023-08-28T02:27:18.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:18 smithi143 ceph-mon[189871]: failed to encode map e197 with expected crc 2023-08-28T02:27:18.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:18 smithi143 ceph-mon[189871]: failed to encode map e197 with expected crc 2023-08-28T02:27:18.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:18 smithi143 ceph-mon[189871]: failed to encode map e197 with expected crc 2023-08-28T02:27:18.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:18 smithi143 ceph-mon[189871]: failed to encode map e197 with expected crc 2023-08-28T02:27:18.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:18 smithi143 ceph-mon[189871]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-08-28T02:27:18.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:18 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:18.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:18 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:18.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:18 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:18.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:18 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:18.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:18 smithi143 ceph-mon[189871]: osdmap e198: 8 total, 8 up, 6 in 2023-08-28T02:27:18.756 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:27:18.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:18.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:27:18.866 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:27:18.924 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:18.984 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:27:19.065 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:27:19.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:19.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:27:19.096 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 12 16 25419 25403 132.274 26.875 0.0155512 0.0075537 2023-08-28T02:27:19.171 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:27:19.241 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:19.366 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:19 smithi143 ceph-mon[183059]: failed to encode map e198 with expected crc 2023-08-28T02:27:19.366 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:19 smithi143 ceph-mon[183059]: failed to encode map e198 with expected crc 2023-08-28T02:27:19.366 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:19 smithi143 ceph-mon[183059]: failed to encode map e198 with expected crc 2023-08-28T02:27:19.366 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:19 smithi143 ceph-mon[183059]: failed to encode map e198 with expected crc 2023-08-28T02:27:19.366 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:19 smithi143 ceph-mon[183059]: failed to encode map e198 with expected crc 2023-08-28T02:27:19.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:19 smithi143 ceph-mon[189871]: failed to encode map e198 with expected crc 2023-08-28T02:27:19.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:19 smithi143 ceph-mon[189871]: failed to encode map e198 with expected crc 2023-08-28T02:27:19.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:19 smithi143 ceph-mon[189871]: failed to encode map e198 with expected crc 2023-08-28T02:27:19.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:19 smithi143 ceph-mon[189871]: failed to encode map e198 with expected crc 2023-08-28T02:27:19.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:19 smithi143 ceph-mon[189871]: failed to encode map e198 with expected crc 2023-08-28T02:27:19.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:19 smithi169 ceph-mon[159415]: failed to encode map e198 with expected crc 2023-08-28T02:27:19.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:19 smithi169 ceph-mon[159415]: failed to encode map e198 with expected crc 2023-08-28T02:27:19.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:19 smithi169 ceph-mon[159415]: failed to encode map e198 with expected crc 2023-08-28T02:27:19.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:19 smithi169 ceph-mon[159415]: failed to encode map e198 with expected crc 2023-08-28T02:27:19.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:19 smithi169 ceph-mon[159415]: failed to encode map e198 with expected crc 2023-08-28T02:27:20.094 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 13 16 26853 26837 128.993 89.625 0.00955964 0.00774544 2023-08-28T02:27:20.391 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is set 2023-08-28T02:27:20.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:20 smithi143 ceph-mon[183059]: pgmap v464: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 26 peering, 60 active+clean; 1.5 GiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 138 MiB/s wr, 2.21k op/s; 3142/74556 objects degraded (4.214%); 1532/74556 objects misplaced (2.055%); 469 KiB/s, 4 objects/s recovering 2023-08-28T02:27:20.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:20 smithi143 ceph-mon[183059]: Health check failed: Degraded data redundancy: 3142/74556 objects degraded (4.214%), 2 pgs degraded (PG_DEGRADED) 2023-08-28T02:27:20.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:20 smithi143 ceph-mon[183059]: failed to encode map e199 with expected crc 2023-08-28T02:27:20.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:20 smithi143 ceph-mon[183059]: failed to encode map e199 with expected crc 2023-08-28T02:27:20.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:20 smithi143 ceph-mon[183059]: failed to encode map e199 with expected crc 2023-08-28T02:27:20.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:20 smithi143 ceph-mon[183059]: osdmap e199: 8 total, 8 up, 6 in 2023-08-28T02:27:20.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:20 smithi143 ceph-mon[183059]: failed to encode map e199 with expected crc 2023-08-28T02:27:20.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:20 smithi143 ceph-mon[183059]: failed to encode map e199 with expected crc 2023-08-28T02:27:20.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:20 smithi143 ceph-mon[183059]: failed to encode map e199 with expected crc 2023-08-28T02:27:20.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:20 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2340362505' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:27:20.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:20 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:27:20.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:20 smithi143 ceph-mon[189871]: pgmap v464: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 26 peering, 60 active+clean; 1.5 GiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 138 MiB/s wr, 2.21k op/s; 3142/74556 objects degraded (4.214%); 1532/74556 objects misplaced (2.055%); 469 KiB/s, 4 objects/s recovering 2023-08-28T02:27:20.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:20 smithi143 ceph-mon[189871]: Health check failed: Degraded data redundancy: 3142/74556 objects degraded (4.214%), 2 pgs degraded (PG_DEGRADED) 2023-08-28T02:27:20.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:20 smithi143 ceph-mon[189871]: failed to encode map e199 with expected crc 2023-08-28T02:27:20.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:20 smithi143 ceph-mon[189871]: failed to encode map e199 with expected crc 2023-08-28T02:27:20.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:20 smithi143 ceph-mon[189871]: failed to encode map e199 with expected crc 2023-08-28T02:27:20.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:20 smithi143 ceph-mon[189871]: osdmap e199: 8 total, 8 up, 6 in 2023-08-28T02:27:20.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:20 smithi143 ceph-mon[189871]: failed to encode map e199 with expected crc 2023-08-28T02:27:20.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:20 smithi143 ceph-mon[189871]: failed to encode map e199 with expected crc 2023-08-28T02:27:20.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:20 smithi143 ceph-mon[189871]: failed to encode map e199 with expected crc 2023-08-28T02:27:20.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:20 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2340362505' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:27:20.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:20 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:27:20.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:20 smithi169 ceph-mon[159415]: pgmap v464: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 26 peering, 60 active+clean; 1.5 GiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 138 MiB/s wr, 2.21k op/s; 3142/74556 objects degraded (4.214%); 1532/74556 objects misplaced (2.055%); 469 KiB/s, 4 objects/s recovering 2023-08-28T02:27:20.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:20 smithi169 ceph-mon[159415]: Health check failed: Degraded data redundancy: 3142/74556 objects degraded (4.214%), 2 pgs degraded (PG_DEGRADED) 2023-08-28T02:27:20.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:20 smithi169 ceph-mon[159415]: failed to encode map e199 with expected crc 2023-08-28T02:27:20.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:20 smithi169 ceph-mon[159415]: failed to encode map e199 with expected crc 2023-08-28T02:27:20.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:20 smithi169 ceph-mon[159415]: failed to encode map e199 with expected crc 2023-08-28T02:27:20.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:20 smithi169 ceph-mon[159415]: osdmap e199: 8 total, 8 up, 6 in 2023-08-28T02:27:20.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:20 smithi169 ceph-mon[159415]: failed to encode map e199 with expected crc 2023-08-28T02:27:20.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:20 smithi169 ceph-mon[159415]: failed to encode map e199 with expected crc 2023-08-28T02:27:20.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:20 smithi169 ceph-mon[159415]: failed to encode map e199 with expected crc 2023-08-28T02:27:20.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:20 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2340362505' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:27:20.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:20 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:27:21.094 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 14 15 27992 27977 124.869 71.25 0.00975611 0.00800165 2023-08-28T02:27:21.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:21 smithi169 ceph-mon[159415]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:27:21.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:21 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:27:21.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:21 smithi169 ceph-mon[159415]: osdmap e200: 8 total, 8 up, 6 in 2023-08-28T02:27:21.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:21 smithi169 ceph-mon[159415]: failed to encode map e200 with expected crc 2023-08-28T02:27:21.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:21 smithi169 ceph-mon[159415]: failed to encode map e200 with expected crc 2023-08-28T02:27:21.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:21 smithi169 ceph-mon[159415]: failed to encode map e200 with expected crc 2023-08-28T02:27:21.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:21 smithi169 ceph-mon[159415]: failed to encode map e200 with expected crc 2023-08-28T02:27:21.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:21 smithi169 ceph-mon[159415]: failed to encode map e200 with expected crc 2023-08-28T02:27:21.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:21 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:21.619 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:27:21.623 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:21 smithi143 ceph-mon[189871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:27:21.623 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:21 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:27:21.623 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:21 smithi143 ceph-mon[189871]: osdmap e200: 8 total, 8 up, 6 in 2023-08-28T02:27:21.623 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:21 smithi143 ceph-mon[189871]: failed to encode map e200 with expected crc 2023-08-28T02:27:21.624 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:21 smithi143 ceph-mon[189871]: failed to encode map e200 with expected crc 2023-08-28T02:27:21.624 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:21 smithi143 ceph-mon[189871]: failed to encode map e200 with expected crc 2023-08-28T02:27:21.624 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:21 smithi143 ceph-mon[189871]: failed to encode map e200 with expected crc 2023-08-28T02:27:21.624 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:21 smithi143 ceph-mon[189871]: failed to encode map e200 with expected crc 2023-08-28T02:27:21.624 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:21 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:21.624 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:21 smithi143 ceph-mon[183059]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:27:21.624 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:21 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:27:21.625 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:21 smithi143 ceph-mon[183059]: osdmap e200: 8 total, 8 up, 6 in 2023-08-28T02:27:21.625 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:21 smithi143 ceph-mon[183059]: failed to encode map e200 with expected crc 2023-08-28T02:27:21.625 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:21 smithi143 ceph-mon[183059]: failed to encode map e200 with expected crc 2023-08-28T02:27:21.625 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:21 smithi143 ceph-mon[183059]: failed to encode map e200 with expected crc 2023-08-28T02:27:21.625 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:21 smithi143 ceph-mon[183059]: failed to encode map e200 with expected crc 2023-08-28T02:27:21.625 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:21 smithi143 ceph-mon[183059]: failed to encode map e200 with expected crc 2023-08-28T02:27:21.625 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:21 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:21.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:21.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:27:21.727 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:27:21.759 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:22.094 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 15 16 29260 29244 121.823 79.1875 0.0122012 0.00820183 2023-08-28T02:27:22.201 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:27:21 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:27:21] "GET /metrics HTTP/1.1" 200 36480 "" "Prometheus/2.33.4" 2023-08-28T02:27:22.537 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:27:22.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:22.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:27:22.646 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:27:22.653 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:22 smithi143 ceph-mon[183059]: pgmap v467: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 26 peering, 60 active+clean; 1.5 GiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 78 MiB/s wr, 1.24k op/s; 3142/74556 objects degraded (4.214%); 1532/74556 objects misplaced (2.055%); 704 KiB/s, 11 objects/s recovering 2023-08-28T02:27:22.653 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:22 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:22.654 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:22 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:22.654 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:22 smithi143 ceph-mon[189871]: pgmap v467: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 26 peering, 60 active+clean; 1.5 GiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 78 MiB/s wr, 1.24k op/s; 3142/74556 objects degraded (4.214%); 1532/74556 objects misplaced (2.055%); 704 KiB/s, 11 objects/s recovering 2023-08-28T02:27:22.654 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:22.654 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:22.687 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:22 smithi169 ceph-mon[159415]: pgmap v467: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 26 peering, 60 active+clean; 1.5 GiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 78 MiB/s wr, 1.24k op/s; 3142/74556 objects degraded (4.214%); 1532/74556 objects misplaced (2.055%); 704 KiB/s, 11 objects/s recovering 2023-08-28T02:27:22.687 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:22 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:22.688 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:22 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:22.692 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:22.774 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:27:22.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:22.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:27:22.882 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:27:22.924 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:22.992 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:27:22.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:22.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:27:23.094 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 16 16 30629 30613 119.557 85.5625 0.00690186 0.00835544 2023-08-28T02:27:23.098 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:27:23.100 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 7, 3] out_osds: [1, 6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-08-28T02:27:23.100 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-08-28T02:27:23.101 INFO:tasks.thrashosds.thrasher:Adding osd 1 2023-08-28T02:27:23.101 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd in 1 2023-08-28T02:27:23.139 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:23.210 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:27:23.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:23.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:27:23.316 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:27:23.347 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:27:23.348 INFO:teuthology.orchestra.run.smithi143.stdout: "size": 20, 2023-08-28T02:27:23.348 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 600, 2023-08-28T02:27:23.348 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [ 2023-08-28T02:27:23.348 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.348 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:45857 5.0 5:02266a25:::benchmark_data_smithi143_236200_object45856:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-08-28T02:27:23.348 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:22:29.926706+0000", 2023-08-28T02:27:23.348 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 293.41814049200002, 2023-08-28T02:27:23.348 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.20099894200000001, 2023-08-28T02:27:23.348 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:27:23.348 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:27:23.348 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:27:23.348 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:27:23.349 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:27:23.349 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 45857 2023-08-28T02:27:23.349 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.349 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:27:23.349 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.349 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:27:23.349 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.926706+0000", 2023-08-28T02:27:23.349 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:27:23.349 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.349 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.349 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:27:23.349 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.926706+0000", 2023-08-28T02:27:23.350 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:27:23.350 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.350 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.350 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:27:23.350 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.926706+0000", 2023-08-28T02:27:23.350 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:27:23.350 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.350 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.350 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:27:23.350 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.926725+0000", 2023-08-28T02:27:23.350 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.9286000000000001e-05 2023-08-28T02:27:23.350 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.351 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.351 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:27:23.351 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.926744+0000", 2023-08-28T02:27:23.351 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.9125000000000001e-05 2023-08-28T02:27:23.351 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.351 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.351 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:27:23.351 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.926751+0000", 2023-08-28T02:27:23.351 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.289e-06 2023-08-28T02:27:23.351 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.351 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.351 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:27:23.352 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.928156+0000", 2023-08-28T02:27:23.352 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.001404195 2023-08-28T02:27:23.352 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.352 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.352 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:27:23.352 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.928240+0000", 2023-08-28T02:27:23.352 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.4497000000000001e-05 2023-08-28T02:27:23.352 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.352 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.352 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,4", 2023-08-28T02:27:23.352 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.928342+0000", 2023-08-28T02:27:23.352 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000101699 2023-08-28T02:27:23.353 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.353 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.353 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:27:23.353 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.929633+0000", 2023-08-28T02:27:23.353 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0012916589999999999 2023-08-28T02:27:23.353 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.353 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.353 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:27:23.353 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.930440+0000", 2023-08-28T02:27:23.353 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00080684300000000001 2023-08-28T02:27:23.353 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.353 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.353 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:27:23.354 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.127618+0000", 2023-08-28T02:27:23.354 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.19717718100000001 2023-08-28T02:27:23.354 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.354 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.354 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:27:23.354 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.127662+0000", 2023-08-28T02:27:23.354 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.4737000000000002e-05 2023-08-28T02:27:23.354 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.354 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.354 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:27:23.354 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.127705+0000", 2023-08-28T02:27:23.354 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.3130000000000002e-05 2023-08-28T02:27:23.354 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:27:23.355 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:27:23.355 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:27:23.355 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.355 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.355 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:45862 5.c 5:3fd0b482:::benchmark_data_smithi143_236200_object45861:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-08-28T02:27:23.355 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:22:29.930174+0000", 2023-08-28T02:27:23.355 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 293.41467328599998, 2023-08-28T02:27:23.355 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.194363008, 2023-08-28T02:27:23.355 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:27:23.355 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:27:23.355 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:27:23.355 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:27:23.356 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:27:23.356 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 45862 2023-08-28T02:27:23.356 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.356 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:27:23.356 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.356 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:27:23.356 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.930174+0000", 2023-08-28T02:27:23.356 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:27:23.356 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.356 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.356 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:27:23.356 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.930174+0000", 2023-08-28T02:27:23.357 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:27:23.357 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.357 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.357 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:27:23.357 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.930173+0000", 2023-08-28T02:27:23.357 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:27:23.357 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.357 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.357 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:27:23.357 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.930192+0000", 2023-08-28T02:27:23.357 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.9391000000000001e-05 2023-08-28T02:27:23.357 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.357 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.358 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:27:23.358 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.930203+0000", 2023-08-28T02:27:23.358 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1134999999999999e-05 2023-08-28T02:27:23.358 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.358 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.358 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:27:23.358 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.930211+0000", 2023-08-28T02:27:23.358 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.6459999999999993e-06 2023-08-28T02:27:23.358 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.358 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.358 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:27:23.358 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.930254+0000", 2023-08-28T02:27:23.359 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.3362999999999998e-05 2023-08-28T02:27:23.359 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.359 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.359 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:27:23.359 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.930374+0000", 2023-08-28T02:27:23.359 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000120337 2023-08-28T02:27:23.359 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.359 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.359 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,4", 2023-08-28T02:27:23.359 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.930474+0000", 2023-08-28T02:27:23.359 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.9928999999999998e-05 2023-08-28T02:27:23.359 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.360 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.360 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:27:23.360 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.932207+0000", 2023-08-28T02:27:23.360 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.001733006 2023-08-28T02:27:23.360 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.360 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.360 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:27:23.360 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.932341+0000", 2023-08-28T02:27:23.360 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000133645 2023-08-28T02:27:23.360 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.360 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.360 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:27:23.360 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.124452+0000", 2023-08-28T02:27:23.361 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.19211102599999999 2023-08-28T02:27:23.361 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.361 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.361 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:27:23.361 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.124497+0000", 2023-08-28T02:27:23.361 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.5141e-05 2023-08-28T02:27:23.361 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.361 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.361 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:27:23.361 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.124537+0000", 2023-08-28T02:27:23.361 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.9440000000000002e-05 2023-08-28T02:27:23.361 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:27:23.361 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:27:23.361 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:27:23.362 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.362 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.362 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:45867 5.0 5:0bc76487:::benchmark_data_smithi143_236200_object45866:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-08-28T02:27:23.362 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:22:29.933114+0000", 2023-08-28T02:27:23.362 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 293.411732852, 2023-08-28T02:27:23.362 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.19481781000000001, 2023-08-28T02:27:23.362 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:27:23.362 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:27:23.362 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:27:23.362 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:27:23.362 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:27:23.362 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 45867 2023-08-28T02:27:23.363 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.363 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:27:23.363 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.363 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:27:23.363 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.933114+0000", 2023-08-28T02:27:23.363 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:27:23.363 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.363 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.363 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:27:23.363 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.933114+0000", 2023-08-28T02:27:23.363 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:27:23.363 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.364 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.364 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:27:23.364 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.933113+0000", 2023-08-28T02:27:23.364 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:27:23.364 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.364 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.364 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:27:23.364 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.933136+0000", 2023-08-28T02:27:23.364 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.3587e-05 2023-08-28T02:27:23.364 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.364 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.364 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:27:23.364 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.933148+0000", 2023-08-28T02:27:23.364 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1691999999999999e-05 2023-08-28T02:27:23.364 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.365 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.365 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:27:23.365 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.933155+0000", 2023-08-28T02:27:23.365 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.8920000000000003e-06 2023-08-28T02:27:23.365 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.365 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.365 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:27:23.365 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.933193+0000", 2023-08-28T02:27:23.365 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.8355999999999997e-05 2023-08-28T02:27:23.365 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.365 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.365 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:27:23.365 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.933297+0000", 2023-08-28T02:27:23.366 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000104149 2023-08-28T02:27:23.366 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.366 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.366 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,4", 2023-08-28T02:27:23.366 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.933434+0000", 2023-08-28T02:27:23.366 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000136491 2023-08-28T02:27:23.366 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.366 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.366 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:27:23.366 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.127755+0000", 2023-08-28T02:27:23.366 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.19432112100000001 2023-08-28T02:27:23.366 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.366 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.367 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:27:23.367 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.127763+0000", 2023-08-28T02:27:23.367 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.892e-06 2023-08-28T02:27:23.367 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.367 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.367 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:27:23.367 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.127851+0000", 2023-08-28T02:27:23.367 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.8060999999999996e-05 2023-08-28T02:27:23.367 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.367 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.367 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:27:23.367 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.127897+0000", 2023-08-28T02:27:23.368 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.6443000000000002e-05 2023-08-28T02:27:23.368 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.368 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.368 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:27:23.368 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.127932+0000", 2023-08-28T02:27:23.368 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.4514000000000002e-05 2023-08-28T02:27:23.368 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:27:23.368 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:27:23.368 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:27:23.368 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.368 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.368 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:45869 5.0 5:0a0258a1:::benchmark_data_smithi143_236200_object45868:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e128)", 2023-08-28T02:27:23.369 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:22:29.933277+0000", 2023-08-28T02:27:23.369 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 293.41156984700001, 2023-08-28T02:27:23.369 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.19651866300000001, 2023-08-28T02:27:23.369 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:27:23.369 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:27:23.369 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:27:23.369 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:27:23.369 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:27:23.369 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 45869 2023-08-28T02:27:23.369 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.369 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:27:23.369 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.369 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:27:23.370 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.933277+0000", 2023-08-28T02:27:23.370 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:27:23.370 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.370 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.370 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:27:23.370 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.933277+0000", 2023-08-28T02:27:23.370 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:27:23.370 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.370 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.370 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:27:23.370 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.933276+0000", 2023-08-28T02:27:23.370 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:27:23.370 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.371 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.371 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:27:23.371 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.933299+0000", 2023-08-28T02:27:23.371 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.266e-05 2023-08-28T02:27:23.371 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.371 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.371 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:27:23.371 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.933311+0000", 2023-08-28T02:27:23.371 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.2713e-05 2023-08-28T02:27:23.371 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.371 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.371 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:27:23.371 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.933319+0000", 2023-08-28T02:27:23.372 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.9100000000000005e-06 2023-08-28T02:27:23.372 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.372 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.372 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:27:23.372 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.125884+0000", 2023-08-28T02:27:23.372 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.192564764 2023-08-28T02:27:23.372 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.372 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.372 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:27:23.372 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.126020+0000", 2023-08-28T02:27:23.372 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00013563200000000001 2023-08-28T02:27:23.372 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.372 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.373 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,4", 2023-08-28T02:27:23.373 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.126160+0000", 2023-08-28T02:27:23.373 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000140283 2023-08-28T02:27:23.373 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.373 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.373 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:27:23.373 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.129656+0000", 2023-08-28T02:27:23.373 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.003496337 2023-08-28T02:27:23.373 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.373 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.373 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:27:23.373 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.129697+0000", 2023-08-28T02:27:23.374 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.0803999999999998e-05 2023-08-28T02:27:23.374 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.374 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.374 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:27:23.374 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.129717+0000", 2023-08-28T02:27:23.375 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.0239000000000001e-05 2023-08-28T02:27:23.375 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.375 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.375 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:27:23.375 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.129761+0000", 2023-08-28T02:27:23.375 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.4002999999999999e-05 2023-08-28T02:27:23.375 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.375 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.375 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:27:23.375 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.129796+0000", 2023-08-28T02:27:23.375 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.4294000000000001e-05 2023-08-28T02:27:23.375 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:27:23.376 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:27:23.376 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:27:23.376 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.376 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.376 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop_reply(client.44594.0:45857 5.0 e128/119 ondisk, result = 0)", 2023-08-28T02:27:23.376 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:22:29.935059+0000", 2023-08-28T02:27:23.376 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 293.40978755100002, 2023-08-28T02:27:23.376 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.19265895199999999, 2023-08-28T02:27:23.376 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:27:23.376 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "started", 2023-08-28T02:27:23.376 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:27:23.376 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.376 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:27:23.377 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.935059+0000", 2023-08-28T02:27:23.377 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:27:23.377 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.377 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.377 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:27:23.377 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.935059+0000", 2023-08-28T02:27:23.377 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:27:23.377 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.377 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.377 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:27:23.377 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.935059+0000", 2023-08-28T02:27:23.377 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:27:23.378 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.378 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.378 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:27:23.378 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.935063+0000", 2023-08-28T02:27:23.378 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.6709999999999998e-06 2023-08-28T02:27:23.378 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.378 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.378 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:27:23.378 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.124274+0000", 2023-08-28T02:27:23.378 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.189210354 2023-08-28T02:27:23.378 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.378 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.379 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:27:23.379 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.124284+0000", 2023-08-28T02:27:23.379 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0553999999999999e-05 2023-08-28T02:27:23.379 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.379 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.379 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:27:23.379 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.127604+0000", 2023-08-28T02:27:23.379 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0033197869999999998 2023-08-28T02:27:23.379 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.379 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.379 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:27:23.379 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.127612+0000", 2023-08-28T02:27:23.379 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.6459999999999993e-06 2023-08-28T02:27:23.380 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.380 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.380 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:27:23.380 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.127718+0000", 2023-08-28T02:27:23.380 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00010658999999999999 2023-08-28T02:27:23.380 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:27:23.380 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:27:23.380 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:27:23.380 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.380 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.380 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.44594.0:45874 5.8 e128/119 5:1f0c8adb:::benchmark_data_smithi143_236200_object45873:head v 128'2824, mlcod=128'2822)", 2023-08-28T02:27:23.380 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:22:29.935903+0000", 2023-08-28T02:27:23.380 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 293.40894432800002, 2023-08-28T02:27:23.381 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.193952238, 2023-08-28T02:27:23.381 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:27:23.381 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:27:23.381 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:27:23.381 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.381 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:27:23.381 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.935903+0000", 2023-08-28T02:27:23.381 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:27:23.381 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.381 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.381 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:27:23.381 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.935903+0000", 2023-08-28T02:27:23.382 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:27:23.382 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.382 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.382 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:27:23.382 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.935902+0000", 2023-08-28T02:27:23.382 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:27:23.382 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.382 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.382 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:27:23.382 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:29.935925+0000", 2023-08-28T02:27:23.382 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.3197000000000002e-05 2023-08-28T02:27:23.382 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.382 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.383 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:27:23.383 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.124267+0000", 2023-08-28T02:27:23.383 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.188342173 2023-08-28T02:27:23.383 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.383 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.383 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:27:23.383 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.124279+0000", 2023-08-28T02:27:23.383 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.208e-05 2023-08-28T02:27:23.383 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.383 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.383 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:27:23.383 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.127786+0000", 2023-08-28T02:27:23.383 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0035070019999999999 2023-08-28T02:27:23.384 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.384 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.384 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:27:23.384 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.127809+0000", 2023-08-28T02:27:23.384 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.2206999999999999e-05 2023-08-28T02:27:23.384 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.384 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.384 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:27:23.384 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.129826+0000", 2023-08-28T02:27:23.384 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.002017146 2023-08-28T02:27:23.384 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.384 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.384 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:27:23.385 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:22:30.129855+0000", 2023-08-28T02:27:23.385 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.9145e-05 2023-08-28T02:27:23.385 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:27:23.385 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:27:23.385 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:27:23.385 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.385 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.385 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:122091 5.0 5:0d3d7eef:::benchmark_data_smithi143_236200_object11948:head [delete] snapc 0=[] ondisk+write+known_if_redirected e152)", 2023-08-28T02:27:23.385 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:56.357099+0000", 2023-08-28T02:27:23.385 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 206.987748337, 2023-08-28T02:27:23.385 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.19015922599999999, 2023-08-28T02:27:23.385 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:27:23.385 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:27:23.386 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:27:23.386 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:27:23.386 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:27:23.386 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 122091 2023-08-28T02:27:23.386 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.386 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:27:23.386 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.386 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:27:23.386 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.357099+0000", 2023-08-28T02:27:23.386 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:27:23.386 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.386 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.387 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:27:23.387 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.357099+0000", 2023-08-28T02:27:23.387 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:27:23.387 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.387 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.387 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:27:23.387 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.357097+0000", 2023-08-28T02:27:23.387 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:27:23.387 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.387 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.387 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:27:23.387 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.357104+0000", 2023-08-28T02:27:23.387 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.759e-06 2023-08-28T02:27:23.388 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.388 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.388 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:27:23.388 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.357116+0000", 2023-08-28T02:27:23.388 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.2728000000000001e-05 2023-08-28T02:27:23.388 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.388 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.388 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:27:23.388 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.357126+0000", 2023-08-28T02:27:23.388 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.2080000000000006e-06 2023-08-28T02:27:23.388 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.388 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.388 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:27:23.389 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.357171+0000", 2023-08-28T02:27:23.389 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.5358000000000002e-05 2023-08-28T02:27:23.389 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.389 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.389 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:27:23.389 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.357301+0000", 2023-08-28T02:27:23.389 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00012978299999999999 2023-08-28T02:27:23.389 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.389 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.389 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,4", 2023-08-28T02:27:23.389 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.357427+0000", 2023-08-28T02:27:23.389 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000126487 2023-08-28T02:27:23.389 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.390 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.390 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:27:23.390 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.361063+0000", 2023-08-28T02:27:23.390 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0036359949999999999 2023-08-28T02:27:23.390 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.390 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.390 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:27:23.390 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.361171+0000", 2023-08-28T02:27:23.390 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000107351 2023-08-28T02:27:23.390 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.390 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.390 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:27:23.390 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.547178+0000", 2023-08-28T02:27:23.391 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.18600725600000001 2023-08-28T02:27:23.391 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.391 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.391 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:27:23.391 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.547222+0000", 2023-08-28T02:27:23.391 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.4178000000000003e-05 2023-08-28T02:27:23.391 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.391 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.391 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:27:23.391 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:56.547258+0000", 2023-08-28T02:27:23.391 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.5800999999999998e-05 2023-08-28T02:27:23.391 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:27:23.391 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:27:23.392 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:27:23.392 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.392 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.392 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:124097 5.c 5:363ea2bd:::benchmark_data_smithi143_236200_object13954:head [delete] snapc 0=[] ondisk+write+known_if_redirected e155)", 2023-08-28T02:27:23.392 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:24:01.195135+0000", 2023-08-28T02:27:23.392 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 202.14971175700001, 2023-08-28T02:27:23.392 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.21136323500000001, 2023-08-28T02:27:23.392 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:27:23.392 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:27:23.392 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:27:23.392 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:27:23.392 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:27:23.393 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 124097 2023-08-28T02:27:23.393 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.393 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:27:23.393 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.393 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:27:23.393 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.195135+0000", 2023-08-28T02:27:23.393 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:27:23.393 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.393 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.393 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:27:23.393 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.195135+0000", 2023-08-28T02:27:23.393 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:27:23.393 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.394 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.394 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:27:23.394 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.195133+0000", 2023-08-28T02:27:23.394 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:27:23.394 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.394 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.394 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:27:23.394 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.195144+0000", 2023-08-28T02:27:23.394 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0613e-05 2023-08-28T02:27:23.394 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.394 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.394 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:27:23.394 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.195160+0000", 2023-08-28T02:27:23.394 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.5760000000000002e-05 2023-08-28T02:27:23.395 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.395 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.395 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:27:23.395 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.195170+0000", 2023-08-28T02:27:23.395 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0282000000000001e-05 2023-08-28T02:27:23.395 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.395 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.395 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:27:23.395 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.195232+0000", 2023-08-28T02:27:23.395 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.2101999999999996e-05 2023-08-28T02:27:23.395 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.395 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.395 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:27:23.396 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.195382+0000", 2023-08-28T02:27:23.396 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00014973499999999999 2023-08-28T02:27:23.396 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.396 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.396 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,4", 2023-08-28T02:27:23.396 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.195531+0000", 2023-08-28T02:27:23.396 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00014867600000000001 2023-08-28T02:27:23.396 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.396 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.396 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:27:23.396 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.198673+0000", 2023-08-28T02:27:23.396 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0031423190000000002 2023-08-28T02:27:23.396 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.397 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.397 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:27:23.397 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.198720+0000", 2023-08-28T02:27:23.397 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.7618999999999999e-05 2023-08-28T02:27:23.397 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.397 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.397 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:27:23.397 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.406413+0000", 2023-08-28T02:27:23.397 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.20769302000000001 2023-08-28T02:27:23.397 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.397 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.397 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:27:23.397 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.406459+0000", 2023-08-28T02:27:23.398 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.5889000000000001e-05 2023-08-28T02:27:23.398 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.398 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.398 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:27:23.398 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.406498+0000", 2023-08-28T02:27:23.398 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.9017000000000003e-05 2023-08-28T02:27:23.398 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:27:23.398 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:27:23.398 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:27:23.398 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.398 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.398 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:130442 5.c 5:3d69323a:::benchmark_data_smithi143_236200_object20299:head [delete] snapc 0=[] ondisk+write+known_if_redirected e163)", 2023-08-28T02:27:23.399 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:24:13.079137+0000", 2023-08-28T02:27:23.399 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 190.265709644, 2023-08-28T02:27:23.399 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.244954062, 2023-08-28T02:27:23.399 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:27:23.399 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:27:23.399 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:27:23.399 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:27:23.399 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:27:23.399 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 130442 2023-08-28T02:27:23.399 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.399 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:27:23.399 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.399 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:27:23.399 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.079137+0000", 2023-08-28T02:27:23.400 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:27:23.400 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.400 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.400 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:27:23.400 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.079137+0000", 2023-08-28T02:27:23.400 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:27:23.400 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.400 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.400 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:27:23.400 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.079137+0000", 2023-08-28T02:27:23.400 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:27:23.400 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.400 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.400 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:27:23.401 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.079140+0000", 2023-08-28T02:27:23.401 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.621e-06 2023-08-28T02:27:23.401 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.401 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.401 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:27:23.401 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.079146+0000", 2023-08-28T02:27:23.401 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.9669999999999998e-06 2023-08-28T02:27:23.401 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.401 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.401 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:27:23.401 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.079150+0000", 2023-08-28T02:27:23.401 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.826e-06 2023-08-28T02:27:23.401 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.402 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.402 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:27:23.402 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.083017+0000", 2023-08-28T02:27:23.402 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0038665069999999999 2023-08-28T02:27:23.402 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.402 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.402 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:27:23.402 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.083142+0000", 2023-08-28T02:27:23.402 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00012473700000000001 2023-08-28T02:27:23.402 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.402 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.402 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,4", 2023-08-28T02:27:23.402 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.083252+0000", 2023-08-28T02:27:23.403 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000110258 2023-08-28T02:27:23.403 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.403 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.403 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:27:23.403 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.090235+0000", 2023-08-28T02:27:23.403 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0069833969999999997 2023-08-28T02:27:23.403 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.403 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.403 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:27:23.403 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.092255+0000", 2023-08-28T02:27:23.403 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0020200410000000002 2023-08-28T02:27:23.403 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.403 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.404 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:27:23.404 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.324000+0000", 2023-08-28T02:27:23.404 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.231744278 2023-08-28T02:27:23.404 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.404 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.404 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:27:23.404 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.324050+0000", 2023-08-28T02:27:23.404 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.0912000000000003e-05 2023-08-28T02:27:23.404 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.404 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.404 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:27:23.404 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.324091+0000", 2023-08-28T02:27:23.404 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.0877000000000002e-05 2023-08-28T02:27:23.405 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:27:23.405 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:27:23.405 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:27:23.405 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.405 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.405 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:130453 5.c 5:31ecd121:::benchmark_data_smithi143_236200_object20310:head [delete] snapc 0=[] ondisk+write+known_if_redirected e163)", 2023-08-28T02:27:23.405 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:24:13.092231+0000", 2023-08-28T02:27:23.405 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 190.25261552000001, 2023-08-28T02:27:23.405 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.231980675, 2023-08-28T02:27:23.405 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:27:23.405 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:27:23.405 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:27:23.405 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:27:23.406 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:27:23.406 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 130453 2023-08-28T02:27:23.406 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.406 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:27:23.406 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.406 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:27:23.406 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.092231+0000", 2023-08-28T02:27:23.406 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:27:23.406 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.406 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.406 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:27:23.406 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.092231+0000", 2023-08-28T02:27:23.407 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:27:23.407 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.407 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.407 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:27:23.407 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.092230+0000", 2023-08-28T02:27:23.407 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:27:23.407 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.407 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.407 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:27:23.407 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.092234+0000", 2023-08-28T02:27:23.407 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.5109999999999997e-06 2023-08-28T02:27:23.407 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.407 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.408 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:27:23.408 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.092243+0000", 2023-08-28T02:27:23.408 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.0650000000000005e-06 2023-08-28T02:27:23.408 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.408 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.408 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:27:23.408 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.092250+0000", 2023-08-28T02:27:23.408 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.0589999999999999e-06 2023-08-28T02:27:23.408 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.408 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.408 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:27:23.408 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.092281+0000", 2023-08-28T02:27:23.409 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.1158000000000002e-05 2023-08-28T02:27:23.409 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.409 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.409 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:27:23.409 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.092404+0000", 2023-08-28T02:27:23.409 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00012336899999999999 2023-08-28T02:27:23.409 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.409 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.409 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,4", 2023-08-28T02:27:23.409 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.092524+0000", 2023-08-28T02:27:23.409 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000119855 2023-08-28T02:27:23.409 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.409 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.410 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:27:23.410 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.096216+0000", 2023-08-28T02:27:23.410 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0036923099999999999 2023-08-28T02:27:23.410 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.410 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.410 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:27:23.410 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.099817+0000", 2023-08-28T02:27:23.410 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0036007690000000002 2023-08-28T02:27:23.410 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.410 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.410 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:27:23.410 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.324138+0000", 2023-08-28T02:27:23.410 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.224321459 2023-08-28T02:27:23.411 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.411 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.411 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:27:23.411 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.324180+0000", 2023-08-28T02:27:23.411 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.1684999999999998e-05 2023-08-28T02:27:23.411 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.411 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.411 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:27:23.411 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.324212+0000", 2023-08-28T02:27:23.411 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.1937999999999998e-05 2023-08-28T02:27:23.411 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:27:23.411 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:27:23.412 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:27:23.412 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.412 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.412 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:140332 5.c 5:38d68a55:::benchmark_data_smithi143_236200_object30189:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-08-28T02:27:23.412 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:24:28.055909+0000", 2023-08-28T02:27:23.412 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 175.288938382, 2023-08-28T02:27:23.412 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.198230936, 2023-08-28T02:27:23.412 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:27:23.412 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:27:23.412 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:27:23.412 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:27:23.412 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:27:23.412 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 140332 2023-08-28T02:27:23.413 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.413 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:27:23.413 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.413 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:27:23.413 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.055909+0000", 2023-08-28T02:27:23.413 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:27:23.413 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.413 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.413 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:27:23.413 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.055909+0000", 2023-08-28T02:27:23.413 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:27:23.413 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.413 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.414 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:27:23.414 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.055907+0000", 2023-08-28T02:27:23.414 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:27:23.414 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.414 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.414 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:27:23.414 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.055913+0000", 2023-08-28T02:27:23.414 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.3630000000000002e-06 2023-08-28T02:27:23.414 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.414 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.414 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:27:23.414 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.055922+0000", 2023-08-28T02:27:23.414 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.0999999999999993e-06 2023-08-28T02:27:23.414 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.415 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.415 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:27:23.415 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.055927+0000", 2023-08-28T02:27:23.415 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.2499999999999997e-06 2023-08-28T02:27:23.415 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.415 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.415 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:27:23.415 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.055968+0000", 2023-08-28T02:27:23.415 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.1242999999999998e-05 2023-08-28T02:27:23.415 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.415 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.415 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:27:23.416 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.056090+0000", 2023-08-28T02:27:23.416 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000121289 2023-08-28T02:27:23.416 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.416 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.416 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,4", 2023-08-28T02:27:23.416 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.056196+0000", 2023-08-28T02:27:23.416 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000106216 2023-08-28T02:27:23.416 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.416 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.416 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:27:23.416 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.059202+0000", 2023-08-28T02:27:23.416 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0030063199999999998 2023-08-28T02:27:23.416 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.417 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.417 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:27:23.417 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.064571+0000", 2023-08-28T02:27:23.417 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0053685110000000003 2023-08-28T02:27:23.417 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.417 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.417 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:27:23.417 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.254065+0000", 2023-08-28T02:27:23.417 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.18949487400000001 2023-08-28T02:27:23.417 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.417 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.417 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:27:23.417 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.254107+0000", 2023-08-28T02:27:23.418 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.1387000000000002e-05 2023-08-28T02:27:23.418 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.418 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.418 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:27:23.418 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.254139+0000", 2023-08-28T02:27:23.418 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.2654999999999998e-05 2023-08-28T02:27:23.418 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:27:23.418 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:27:23.418 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:27:23.418 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.418 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.418 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:140339 5.c 5:3390e4f0:::benchmark_data_smithi143_236200_object30196:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-08-28T02:27:23.419 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:24:28.065603+0000", 2023-08-28T02:27:23.419 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 175.279243463, 2023-08-28T02:27:23.419 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.188624818, 2023-08-28T02:27:23.419 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:27:23.419 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:27:23.419 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:27:23.419 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:27:23.419 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:27:23.419 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 140339 2023-08-28T02:27:23.419 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.419 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:27:23.419 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.419 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:27:23.420 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.065603+0000", 2023-08-28T02:27:23.420 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:27:23.420 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.420 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.420 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:27:23.420 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.065603+0000", 2023-08-28T02:27:23.420 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:27:23.420 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.420 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.420 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:27:23.420 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.065602+0000", 2023-08-28T02:27:23.420 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:27:23.420 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.421 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.421 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:27:23.421 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.065608+0000", 2023-08-28T02:27:23.421 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.9819999999999996e-06 2023-08-28T02:27:23.421 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.421 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.421 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:27:23.421 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.065618+0000", 2023-08-28T02:27:23.421 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.4779999999999997e-06 2023-08-28T02:27:23.421 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.421 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.421 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:27:23.421 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.065623+0000", 2023-08-28T02:27:23.422 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.383e-06 2023-08-28T02:27:23.422 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.422 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.422 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:27:23.422 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.065663+0000", 2023-08-28T02:27:23.422 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.0012000000000002e-05 2023-08-28T02:27:23.422 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.422 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.422 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:27:23.422 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.065752+0000", 2023-08-28T02:27:23.422 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.9171000000000004e-05 2023-08-28T02:27:23.422 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.422 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.423 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,4", 2023-08-28T02:27:23.423 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.065826+0000", 2023-08-28T02:27:23.423 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.3498999999999995e-05 2023-08-28T02:27:23.423 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.423 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.423 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:27:23.423 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.070482+0000", 2023-08-28T02:27:23.423 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0046563660000000003 2023-08-28T02:27:23.423 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.423 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.423 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:27:23.423 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.071688+0000", 2023-08-28T02:27:23.424 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.001206223 2023-08-28T02:27:23.424 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.424 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.424 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:27:23.424 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.254168+0000", 2023-08-28T02:27:23.424 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.182479893 2023-08-28T02:27:23.424 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.424 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.424 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:27:23.424 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.254202+0000", 2023-08-28T02:27:23.424 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.3467999999999999e-05 2023-08-28T02:27:23.424 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.425 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.425 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:27:23.425 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.254228+0000", 2023-08-28T02:27:23.425 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.6676999999999999e-05 2023-08-28T02:27:23.425 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:27:23.425 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:27:23.425 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:27:23.425 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.425 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.425 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:154041 5.c 5:38b53cd8:::benchmark_data_smithi143_236200_object43898:head [delete] snapc 0=[] ondisk+write+known_if_redirected e169)", 2023-08-28T02:27:23.425 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:24:50.334265+0000", 2023-08-28T02:27:23.425 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 153.010582035, 2023-08-28T02:27:23.425 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.21132811000000001, 2023-08-28T02:27:23.426 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:27:23.426 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:27:23.426 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:27:23.426 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:27:23.426 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:27:23.426 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 154041 2023-08-28T02:27:23.426 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.426 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:27:23.426 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.426 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:27:23.426 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.334265+0000", 2023-08-28T02:27:23.426 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:27:23.426 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.427 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.427 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:27:23.427 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.334265+0000", 2023-08-28T02:27:23.427 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:27:23.427 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.427 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.427 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:27:23.427 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.334263+0000", 2023-08-28T02:27:23.427 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:27:23.427 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.427 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.427 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:27:23.427 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.334271+0000", 2023-08-28T02:27:23.428 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.9780000000000005e-06 2023-08-28T02:27:23.428 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.428 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.428 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:27:23.428 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.334284+0000", 2023-08-28T02:27:23.428 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3158e-05 2023-08-28T02:27:23.428 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.428 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.428 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:27:23.428 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.334293+0000", 2023-08-28T02:27:23.428 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.0550000000000005e-06 2023-08-28T02:27:23.428 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.428 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.429 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:27:23.429 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.334349+0000", 2023-08-28T02:27:23.429 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.5129e-05 2023-08-28T02:27:23.429 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.429 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.429 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:27:23.429 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.334465+0000", 2023-08-28T02:27:23.429 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00011665199999999999 2023-08-28T02:27:23.429 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.429 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.429 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,4", 2023-08-28T02:27:23.429 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.334584+0000", 2023-08-28T02:27:23.429 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00011909 2023-08-28T02:27:23.430 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.430 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.430 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:27:23.430 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.340936+0000", 2023-08-28T02:27:23.430 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0063517970000000002 2023-08-28T02:27:23.430 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.430 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.430 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:27:23.430 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.352299+0000", 2023-08-28T02:27:23.430 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.011362921 2023-08-28T02:27:23.430 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.430 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.430 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:27:23.431 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.545477+0000", 2023-08-28T02:27:23.431 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.193178145 2023-08-28T02:27:23.431 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.431 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.431 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:27:23.431 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.545543+0000", 2023-08-28T02:27:23.431 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.5420999999999996e-05 2023-08-28T02:27:23.431 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.431 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.431 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:27:23.431 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.545593+0000", 2023-08-28T02:27:23.431 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.0386000000000002e-05 2023-08-28T02:27:23.432 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:27:23.432 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:27:23.432 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:27:23.432 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.432 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.432 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:154043 5.c 5:393c5f9d:::benchmark_data_smithi143_236200_object43900:head [delete] snapc 0=[] ondisk+write+known_if_redirected e169)", 2023-08-28T02:27:23.432 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:24:50.337310+0000", 2023-08-28T02:27:23.432 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 153.00753695200001, 2023-08-28T02:27:23.432 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.210709331, 2023-08-28T02:27:23.432 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:27:23.432 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:27:23.432 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:27:23.432 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:27:23.433 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:27:23.433 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 154043 2023-08-28T02:27:23.433 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.433 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:27:23.433 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.433 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:27:23.433 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.337310+0000", 2023-08-28T02:27:23.433 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:27:23.433 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.433 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.433 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:27:23.433 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.337310+0000", 2023-08-28T02:27:23.433 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:27:23.434 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.434 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.434 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:27:23.434 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.337309+0000", 2023-08-28T02:27:23.434 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:27:23.434 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.434 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.434 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:27:23.434 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.337314+0000", 2023-08-28T02:27:23.434 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.626e-06 2023-08-28T02:27:23.434 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.434 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.435 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:27:23.435 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.337324+0000", 2023-08-28T02:27:23.435 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.5869999999999999e-06 2023-08-28T02:27:23.435 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.435 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.435 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:27:23.435 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.337331+0000", 2023-08-28T02:27:23.435 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.5599999999999999e-06 2023-08-28T02:27:23.435 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.435 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.435 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:27:23.435 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.337896+0000", 2023-08-28T02:27:23.436 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00056541399999999998 2023-08-28T02:27:23.436 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.436 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.436 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:27:23.436 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.337971+0000", 2023-08-28T02:27:23.436 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.5228999999999994e-05 2023-08-28T02:27:23.436 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.436 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.436 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,4", 2023-08-28T02:27:23.436 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.338049+0000", 2023-08-28T02:27:23.436 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.7358e-05 2023-08-28T02:27:23.436 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.436 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.437 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:27:23.437 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.341053+0000", 2023-08-28T02:27:23.437 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.003004039 2023-08-28T02:27:23.437 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.437 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.437 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:27:23.437 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.352329+0000", 2023-08-28T02:27:23.437 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.011276833999999999 2023-08-28T02:27:23.437 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.437 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.437 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:27:23.437 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.547929+0000", 2023-08-28T02:27:23.437 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.19559933900000001 2023-08-28T02:27:23.438 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.438 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.438 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:27:23.438 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.547975+0000", 2023-08-28T02:27:23.438 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.5739999999999999e-05 2023-08-28T02:27:23.438 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.438 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.438 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:27:23.438 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.548019+0000", 2023-08-28T02:27:23.438 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.4749999999999997e-05 2023-08-28T02:27:23.438 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:27:23.438 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:27:23.439 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:27:23.439 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.439 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.439 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:154052 5.c 5:30744441:::benchmark_data_smithi143_236200_object43909:head [delete] snapc 0=[] ondisk+write+known_if_redirected e169)", 2023-08-28T02:27:23.439 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:24:50.355115+0000", 2023-08-28T02:27:23.439 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 152.98973169800001, 2023-08-28T02:27:23.439 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.193283705, 2023-08-28T02:27:23.439 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:27:23.439 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:27:23.439 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:27:23.439 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:27:23.439 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:27:23.439 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 154052 2023-08-28T02:27:23.440 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.440 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:27:23.440 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.440 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:27:23.440 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.355115+0000", 2023-08-28T02:27:23.440 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:27:23.440 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.440 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.440 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:27:23.440 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.355115+0000", 2023-08-28T02:27:23.440 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:27:23.440 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.440 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.441 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:27:23.441 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.355113+0000", 2023-08-28T02:27:23.441 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:27:23.441 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.441 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.441 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:27:23.441 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.355122+0000", 2023-08-28T02:27:23.441 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.3869999999999995e-06 2023-08-28T02:27:23.441 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.441 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.441 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:27:23.441 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.355134+0000", 2023-08-28T02:27:23.442 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1772e-05 2023-08-28T02:27:23.442 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.442 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.442 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:27:23.442 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.355142+0000", 2023-08-28T02:27:23.442 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.9619999999999993e-06 2023-08-28T02:27:23.442 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.442 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.442 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:27:23.442 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.355189+0000", 2023-08-28T02:27:23.442 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.7345999999999997e-05 2023-08-28T02:27:23.442 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.443 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.443 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:27:23.443 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.355323+0000", 2023-08-28T02:27:23.443 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00013421900000000001 2023-08-28T02:27:23.443 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.443 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.443 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,4", 2023-08-28T02:27:23.443 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.355444+0000", 2023-08-28T02:27:23.443 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00012092200000000001 2023-08-28T02:27:23.443 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.443 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.443 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:27:23.443 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.358770+0000", 2023-08-28T02:27:23.444 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.003326308 2023-08-28T02:27:23.444 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.444 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.444 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:27:23.444 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.359196+0000", 2023-08-28T02:27:23.444 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00042524199999999999 2023-08-28T02:27:23.444 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.444 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.444 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:27:23.444 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.548328+0000", 2023-08-28T02:27:23.444 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.189132252 2023-08-28T02:27:23.444 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.444 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.444 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:27:23.445 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.548367+0000", 2023-08-28T02:27:23.445 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.9487000000000003e-05 2023-08-28T02:27:23.445 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.445 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.445 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:27:23.445 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:50.548399+0000", 2023-08-28T02:27:23.445 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.163e-05 2023-08-28T02:27:23.445 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:27:23.445 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:27:23.445 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:27:23.445 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.445 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.445 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:185300 5.c 5:395d3b15:::benchmark_data_smithi143_236200_object75157:head [delete] snapc 0=[] ondisk+write+known_if_redirected e176)", 2023-08-28T02:27:23.446 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:25:41.092098+0000", 2023-08-28T02:27:23.446 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 102.252749005, 2023-08-28T02:27:23.446 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.19304016900000001, 2023-08-28T02:27:23.446 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:27:23.446 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:27:23.446 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:27:23.446 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:27:23.446 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:27:23.446 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 185300 2023-08-28T02:27:23.446 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.446 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:27:23.446 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.447 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:27:23.447 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:41.092098+0000", 2023-08-28T02:27:23.447 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:27:23.447 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.447 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.447 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:27:23.447 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:41.092098+0000", 2023-08-28T02:27:23.447 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:27:23.447 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.447 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.447 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:27:23.447 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:41.092096+0000", 2023-08-28T02:27:23.447 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:27:23.448 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.448 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.448 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:27:23.448 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:41.092101+0000", 2023-08-28T02:27:23.448 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.5390000000000001e-06 2023-08-28T02:27:23.448 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.448 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.448 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:27:23.448 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:41.092109+0000", 2023-08-28T02:27:23.448 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.2029999999999992e-06 2023-08-28T02:27:23.448 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.448 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.449 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:27:23.449 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:41.092115+0000", 2023-08-28T02:27:23.449 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.7180000000000001e-06 2023-08-28T02:27:23.449 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.449 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.449 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:27:23.449 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:41.092187+0000", 2023-08-28T02:27:23.449 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.2105000000000002e-05 2023-08-28T02:27:23.449 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.449 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.449 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:27:23.449 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:41.092314+0000", 2023-08-28T02:27:23.449 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000126823 2023-08-28T02:27:23.450 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.450 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.450 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,4", 2023-08-28T02:27:23.450 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:41.092435+0000", 2023-08-28T02:27:23.450 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000120733 2023-08-28T02:27:23.450 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.450 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.450 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:27:23.450 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:41.097646+0000", 2023-08-28T02:27:23.450 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0052111400000000004 2023-08-28T02:27:23.450 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.450 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.450 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:27:23.451 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:41.103277+0000", 2023-08-28T02:27:23.451 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0056310780000000003 2023-08-28T02:27:23.451 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.451 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.451 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:27:23.451 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:41.285053+0000", 2023-08-28T02:27:23.451 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.181776681 2023-08-28T02:27:23.451 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.451 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.451 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:27:23.451 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:41.285101+0000", 2023-08-28T02:27:23.451 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.7843999999999998e-05 2023-08-28T02:27:23.452 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.452 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.452 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:27:23.452 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:41.285138+0000", 2023-08-28T02:27:23.452 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.6755000000000003e-05 2023-08-28T02:27:23.452 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:27:23.452 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:27:23.452 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:27:23.452 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.452 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.452 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:195786 5.c 5:3ad0355e:::benchmark_data_smithi143_236200_object85643:head [delete] snapc 0=[] ondisk+write+known_if_redirected e179)", 2023-08-28T02:27:23.452 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:25:57.762119+0000", 2023-08-28T02:27:23.452 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 85.582727836999993, 2023-08-28T02:27:23.453 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.19277117499999999, 2023-08-28T02:27:23.453 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:27:23.453 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:27:23.453 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:27:23.453 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:27:23.453 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:27:23.453 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 195786 2023-08-28T02:27:23.453 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.453 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:27:23.453 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.453 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:27:23.453 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:57.762119+0000", 2023-08-28T02:27:23.453 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:27:23.454 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.454 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.454 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:27:23.454 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:57.762119+0000", 2023-08-28T02:27:23.454 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:27:23.454 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.454 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.454 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:27:23.454 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:57.762118+0000", 2023-08-28T02:27:23.454 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:27:23.454 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.454 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.455 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:27:23.455 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:57.762122+0000", 2023-08-28T02:27:23.455 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.7890000000000002e-06 2023-08-28T02:27:23.455 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.455 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.455 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:27:23.455 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:57.762134+0000", 2023-08-28T02:27:23.455 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1151e-05 2023-08-28T02:27:23.455 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.455 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.455 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:27:23.455 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:57.762141+0000", 2023-08-28T02:27:23.456 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.9179999999999997e-06 2023-08-28T02:27:23.456 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.456 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.456 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:27:23.456 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:57.762420+0000", 2023-08-28T02:27:23.456 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00027898899999999999 2023-08-28T02:27:23.456 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.456 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.456 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:27:23.456 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:57.762543+0000", 2023-08-28T02:27:23.456 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00012358099999999999 2023-08-28T02:27:23.456 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.456 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.457 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,4", 2023-08-28T02:27:23.457 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:57.762654+0000", 2023-08-28T02:27:23.457 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000110503 2023-08-28T02:27:23.457 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.457 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.457 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:27:23.457 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:57.766948+0000", 2023-08-28T02:27:23.457 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0042941910000000002 2023-08-28T02:27:23.457 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.457 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.457 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:27:23.457 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:57.769502+0000", 2023-08-28T02:27:23.458 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.002554111 2023-08-28T02:27:23.458 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.458 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.458 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:27:23.458 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:57.954807+0000", 2023-08-28T02:27:23.458 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.18530535100000001 2023-08-28T02:27:23.458 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.458 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.458 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:27:23.458 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:57.954854+0000", 2023-08-28T02:27:23.458 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.6755000000000002e-05 2023-08-28T02:27:23.458 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.458 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.459 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:27:23.459 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:57.954890+0000", 2023-08-28T02:27:23.459 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.6217000000000002e-05 2023-08-28T02:27:23.459 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:27:23.459 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:27:23.459 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:27:23.459 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.459 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.459 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:195796 5.c 5:3a69d3c6:::benchmark_data_smithi143_236200_object85653:head [delete] snapc 0=[] ondisk+write+known_if_redirected e179)", 2023-08-28T02:27:23.459 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:25:57.768145+0000", 2023-08-28T02:27:23.459 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 85.576701456999999, 2023-08-28T02:27:23.459 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.19309088499999999, 2023-08-28T02:27:23.460 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:27:23.460 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:27:23.460 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:27:23.460 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:27:23.460 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:27:23.460 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 195796 2023-08-28T02:27:23.460 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.460 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:27:23.460 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.460 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:27:23.460 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:57.768145+0000", 2023-08-28T02:27:23.460 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:27:23.460 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.461 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.461 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:27:23.461 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:57.768145+0000", 2023-08-28T02:27:23.461 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:27:23.461 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.461 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.461 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:27:23.461 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:57.768145+0000", 2023-08-28T02:27:23.461 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:27:23.461 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.461 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.461 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:27:23.461 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:57.768148+0000", 2023-08-28T02:27:23.462 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.9979999999999999e-06 2023-08-28T02:27:23.462 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.462 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.462 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:27:23.462 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:57.768156+0000", 2023-08-28T02:27:23.462 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.4109999999999996e-06 2023-08-28T02:27:23.462 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.462 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.462 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:27:23.462 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:57.768162+0000", 2023-08-28T02:27:23.462 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.0780000000000003e-06 2023-08-28T02:27:23.462 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.462 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.463 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:27:23.463 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:57.773941+0000", 2023-08-28T02:27:23.463 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0057793580000000001 2023-08-28T02:27:23.463 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.463 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.463 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:27:23.463 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:57.774054+0000", 2023-08-28T02:27:23.463 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000112908 2023-08-28T02:27:23.463 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.463 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.463 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,4", 2023-08-28T02:27:23.463 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:57.774155+0000", 2023-08-28T02:27:23.463 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000101598 2023-08-28T02:27:23.464 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.464 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.464 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:27:23.464 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:57.778199+0000", 2023-08-28T02:27:23.464 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0040438660000000001 2023-08-28T02:27:23.464 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.464 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.464 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:27:23.464 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:57.785300+0000", 2023-08-28T02:27:23.464 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0071003849999999999 2023-08-28T02:27:23.464 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.464 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.464 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:27:23.464 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:57.961146+0000", 2023-08-28T02:27:23.465 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.17584677900000001 2023-08-28T02:27:23.465 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.465 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.465 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:27:23.465 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:57.961197+0000", 2023-08-28T02:27:23.465 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.0392000000000001e-05 2023-08-28T02:27:23.465 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.465 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.465 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:27:23.465 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:57.961236+0000", 2023-08-28T02:27:23.465 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.9533999999999998e-05 2023-08-28T02:27:23.465 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:27:23.465 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:27:23.466 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:27:23.466 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.466 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.466 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:196493 5.c 5:3e662ce2:::benchmark_data_smithi143_236200_object86350:head [delete] snapc 0=[] ondisk+write+known_if_redirected e179)", 2023-08-28T02:27:23.466 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:25:58.977172+0000", 2023-08-28T02:27:23.466 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 84.367674694000002, 2023-08-28T02:27:23.466 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.21636216999999999, 2023-08-28T02:27:23.466 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:27:23.466 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:27:23.466 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:27:23.466 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:27:23.466 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:27:23.466 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 196493 2023-08-28T02:27:23.467 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.467 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:27:23.467 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.467 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:27:23.467 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.977172+0000", 2023-08-28T02:27:23.467 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:27:23.467 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.467 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.467 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:27:23.467 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.977172+0000", 2023-08-28T02:27:23.467 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:27:23.467 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.467 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.468 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:27:23.468 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.977171+0000", 2023-08-28T02:27:23.468 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:27:23.468 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.468 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.468 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:27:23.468 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.977177+0000", 2023-08-28T02:27:23.468 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.268e-06 2023-08-28T02:27:23.468 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.468 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.468 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:27:23.468 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.977190+0000", 2023-08-28T02:27:23.468 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3040999999999999e-05 2023-08-28T02:27:23.469 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.469 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.469 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:27:23.469 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.977199+0000", 2023-08-28T02:27:23.469 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.0059999999999998e-06 2023-08-28T02:27:23.469 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.469 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.469 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:27:23.469 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.977251+0000", 2023-08-28T02:27:23.469 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.2232999999999999e-05 2023-08-28T02:27:23.469 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.469 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.469 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:27:23.470 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.977357+0000", 2023-08-28T02:27:23.470 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000105381 2023-08-28T02:27:23.470 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.470 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.470 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,4", 2023-08-28T02:27:23.470 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.977958+0000", 2023-08-28T02:27:23.470 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00060142599999999996 2023-08-28T02:27:23.470 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.470 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.470 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:27:23.470 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.982392+0000", 2023-08-28T02:27:23.470 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0044334489999999999 2023-08-28T02:27:23.470 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.471 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.471 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:27:23.471 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.991198+0000", 2023-08-28T02:27:23.471 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0088061770000000001 2023-08-28T02:27:23.471 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.471 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.471 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:27:23.471 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:59.193443+0000", 2023-08-28T02:27:23.471 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.20224545499999999 2023-08-28T02:27:23.471 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.471 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.471 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:27:23.471 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:59.193495+0000", 2023-08-28T02:27:23.472 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.2216999999999998e-05 2023-08-28T02:27:23.472 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.472 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.472 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:27:23.472 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:59.193534+0000", 2023-08-28T02:27:23.472 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.8906000000000003e-05 2023-08-28T02:27:23.472 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:27:23.472 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:27:23.472 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:27:23.472 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.472 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.472 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:196499 5.c 5:3dd6a684:::benchmark_data_smithi143_236200_object86356:head [delete] snapc 0=[] ondisk+write+known_if_redirected e179)", 2023-08-28T02:27:23.472 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:25:58.995207+0000", 2023-08-28T02:27:23.473 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 84.349639908, 2023-08-28T02:27:23.473 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.202743008, 2023-08-28T02:27:23.473 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:27:23.473 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:27:23.473 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:27:23.473 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:27:23.473 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:27:23.473 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 196499 2023-08-28T02:27:23.473 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.473 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:27:23.473 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.473 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:27:23.473 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.995207+0000", 2023-08-28T02:27:23.473 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:27:23.474 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.474 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.474 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:27:23.474 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.995207+0000", 2023-08-28T02:27:23.474 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:27:23.474 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.474 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.474 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:27:23.474 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.995206+0000", 2023-08-28T02:27:23.474 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:27:23.474 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.474 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.474 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:27:23.475 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.995212+0000", 2023-08-28T02:27:23.475 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.1179999999999997e-06 2023-08-28T02:27:23.475 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.475 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.475 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:27:23.475 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.995221+0000", 2023-08-28T02:27:23.475 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.9879999999999993e-06 2023-08-28T02:27:23.475 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.475 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.475 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:27:23.475 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.995227+0000", 2023-08-28T02:27:23.475 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.8200000000000002e-06 2023-08-28T02:27:23.475 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.476 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.476 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:27:23.476 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.995275+0000", 2023-08-28T02:27:23.476 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.8383000000000001e-05 2023-08-28T02:27:23.476 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.476 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.476 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:27:23.476 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.995410+0000", 2023-08-28T02:27:23.476 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00013449600000000001 2023-08-28T02:27:23.476 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.476 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.476 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,4", 2023-08-28T02:27:23.476 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.995532+0000", 2023-08-28T02:27:23.477 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00012282499999999999 2023-08-28T02:27:23.477 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.477 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.477 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:27:23.477 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.999308+0000", 2023-08-28T02:27:23.477 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0037755660000000002 2023-08-28T02:27:23.477 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.477 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.477 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:27:23.477 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:59.000320+0000", 2023-08-28T02:27:23.477 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.001011798 2023-08-28T02:27:23.477 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.477 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.478 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:27:23.478 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:59.197871+0000", 2023-08-28T02:27:23.478 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.19755096999999999 2023-08-28T02:27:23.478 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.478 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.478 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:27:23.478 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:59.197915+0000", 2023-08-28T02:27:23.478 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.3844999999999998e-05 2023-08-28T02:27:23.478 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:23.478 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:23.478 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:27:23.478 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:59.197950+0000", 2023-08-28T02:27:23.478 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.5479999999999999e-05 2023-08-28T02:27:23.479 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:27:23.479 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:27:23.479 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:27:23.479 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:27:23.479 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:27:23.479 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:27:23.480 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:23 smithi169 ceph-mon[159415]: pgmap v468: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.7 GiB data, 5.9 GiB used, 531 GiB / 536 GiB avail; 57 MiB/s wr, 917 op/s; 7787/82533 objects degraded (9.435%); 2627/82533 objects misplaced (3.183%); 4.7 MiB/s, 81 objects/s recovering 2023-08-28T02:27:23.480 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:23 smithi169 ceph-mon[159415]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2023-08-28T02:27:23.480 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:23.480 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:27:23.511 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:27:23.555 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:27:23.594 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:27:23.614 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:27:23.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:23.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:27:23.630 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:27:23.663 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:27:23.695 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:27:23.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:23 smithi143 ceph-mon[183059]: pgmap v468: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.7 GiB data, 5.9 GiB used, 531 GiB / 536 GiB avail; 57 MiB/s wr, 917 op/s; 7787/82533 objects degraded (9.435%); 2627/82533 objects misplaced (3.183%); 4.7 MiB/s, 81 objects/s recovering 2023-08-28T02:27:23.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:23 smithi143 ceph-mon[183059]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2023-08-28T02:27:23.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:23 smithi143 ceph-mon[189871]: pgmap v468: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.7 GiB data, 5.9 GiB used, 531 GiB / 536 GiB avail; 57 MiB/s wr, 917 op/s; 7787/82533 objects degraded (9.435%); 2627/82533 objects misplaced (3.183%); 4.7 MiB/s, 81 objects/s recovering 2023-08-28T02:27:23.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:23 smithi143 ceph-mon[189871]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2023-08-28T02:27:23.720 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:27:23.730 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:27:23.732 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:27:23.732 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:27:23.732 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:27:23.732 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:27:23.732 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:27:23.732 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:27:23.732 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:27:23.732 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:27:23.766 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:23.850 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:27:23.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:23.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:27:23.955 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:27:24.013 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:24.094 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 17 16 31923 31907 117.281 80.875 0.0108057 0.00851954 2023-08-28T02:27:24.698 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset noscrub 2023-08-28T02:27:24.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:24 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/463104852' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:27:24.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:24 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/463104852' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:27:24.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:24 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/463104852' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:27:24.967 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:27:24.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:24.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:27:25.075 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:27:25.094 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 18 16 33072 33056 114.755 71.8125 0.00780939 0.00870803 2023-08-28T02:27:25.126 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:25.190 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:27:25.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:25.194 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:27:25.295 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:27:25.321 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:25.412 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:27:25.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:25.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:27:25.513 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.2 dump_ops_in_flight 2023-08-28T02:27:25.523 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:27:25.569 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:25.644 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:27:25.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:25.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:27:25.721 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:25 smithi143 ceph-mon[183059]: pgmap v469: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.9 GiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 75 MiB/s wr, 1.20k op/s; 7787/91218 objects degraded (8.537%); 2160/91218 objects misplaced (2.368%); 8.3 MiB/s, 137 objects/s recovering 2023-08-28T02:27:25.721 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:25 smithi143 ceph-mon[189871]: pgmap v469: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.9 GiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 75 MiB/s wr, 1.20k op/s; 7787/91218 objects degraded (8.537%); 2160/91218 objects misplaced (2.368%); 8.3 MiB/s, 137 objects/s recovering 2023-08-28T02:27:25.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:25 smithi169 ceph-mon[159415]: pgmap v469: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.9 GiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 75 MiB/s wr, 1.20k op/s; 7787/91218 objects degraded (8.537%); 2160/91218 objects misplaced (2.368%); 8.3 MiB/s, 137 objects/s recovering 2023-08-28T02:27:25.756 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:27:25.854 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:27:25.872 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:25.988 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:27:25.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:25.994 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:27:26.085 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:27:26.095 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:27:26.098 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 19 16 34130 34114 112.177 66.125 0.0100223 0.00890803 2023-08-28T02:27:26.139 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:26.205 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:27:26.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:26.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:27:26.316 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:27:26.389 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:26.463 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:26 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 7787/91218 objects degraded (8.537%), 5 pgs degraded (PG_DEGRADED) 2023-08-28T02:27:26.463 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:26 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:27:26.540 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:27:26.543 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:26 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 7787/91218 objects degraded (8.537%), 5 pgs degraded (PG_DEGRADED) 2023-08-28T02:27:26.543 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:26 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:27:26.543 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:26 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 7787/91218 objects degraded (8.537%), 5 pgs degraded (PG_DEGRADED) 2023-08-28T02:27:26.544 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:26 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:27:26.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:26.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:27:26.645 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:27:26.723 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:27.098 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:2023-08-28T02:27:27.099109+0000 min lat: 0.00119624 max lat: 2.12752 avg lat: 0.00904919 2023-08-28T02:27:27.098 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-08-28T02:27:27.098 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 20 16 35367 35351 110.433 77.3125 0.0065939 0.00904919 2023-08-28T02:27:27.723 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:27:27.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:27 smithi169 ceph-mon[159415]: pgmap v470: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 58 MiB/s wr, 935 op/s; 7787/97017 objects degraded (8.026%); 2160/97017 objects misplaced (2.226%); 6.8 MiB/s, 112 objects/s recovering 2023-08-28T02:27:27.747 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:27 smithi143 ceph-mon[183059]: pgmap v470: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 58 MiB/s wr, 935 op/s; 7787/97017 objects degraded (8.026%); 2160/97017 objects misplaced (2.226%); 6.8 MiB/s, 112 objects/s recovering 2023-08-28T02:27:27.747 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:27 smithi143 ceph-mon[189871]: pgmap v470: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 58 MiB/s wr, 935 op/s; 7787/97017 objects degraded (8.026%); 2160/97017 objects misplaced (2.226%); 6.8 MiB/s, 112 objects/s recovering 2023-08-28T02:27:27.945 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:27:27.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:27.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:27:28.051 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:27:28.084 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:28.097 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 21 16 36558 36542 108.719 74.4375 0.0186958 0.00919229 2023-08-28T02:27:28.160 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:27:28.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:28.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:27:28.271 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:27:28.306 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:28.487 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:27:28.754 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:27:28.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:28.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:27:28.860 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:27:28.885 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:28.928 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:27:28.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:28.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:27:29.032 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:27:29.058 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:29.098 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 22 16 37057 37041 105.196 31.1875 0.00516486 0.00927377 2023-08-28T02:27:29.102 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:27:29.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:29.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:27:29.208 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:27:29.240 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:29.346 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:27:29.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:29.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:27:29.449 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:27:29.475 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:29.554 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:27:29.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:29.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:27:29.665 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:27:29.722 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:29.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:29 smithi169 ceph-mon[159415]: osdmap e201: 8 total, 8 up, 6 in 2023-08-28T02:27:29.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:29 smithi169 ceph-mon[159415]: failed to encode map e201 with expected crc 2023-08-28T02:27:29.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:29 smithi169 ceph-mon[159415]: failed to encode map e201 with expected crc 2023-08-28T02:27:29.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:29 smithi169 ceph-mon[159415]: failed to encode map e201 with expected crc 2023-08-28T02:27:29.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:29 smithi169 ceph-mon[159415]: pgmap v472: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 80 MiB/s wr, 1.29k op/s; 7787/108582 objects degraded (7.172%); 574/108582 objects misplaced (0.529%); 17 MiB/s, 281 objects/s recovering 2023-08-28T02:27:29.892 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:29 smithi143 ceph-mon[183059]: osdmap e201: 8 total, 8 up, 6 in 2023-08-28T02:27:29.892 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:29 smithi143 ceph-mon[183059]: failed to encode map e201 with expected crc 2023-08-28T02:27:29.893 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:29 smithi143 ceph-mon[183059]: failed to encode map e201 with expected crc 2023-08-28T02:27:29.893 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:29 smithi143 ceph-mon[183059]: failed to encode map e201 with expected crc 2023-08-28T02:27:29.893 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:29 smithi143 ceph-mon[183059]: pgmap v472: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 80 MiB/s wr, 1.29k op/s; 7787/108582 objects degraded (7.172%); 574/108582 objects misplaced (0.529%); 17 MiB/s, 281 objects/s recovering 2023-08-28T02:27:29.893 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:29 smithi143 conmon[183035]: 2023-08-28T02:27:29.532+0000 7fadc3887700 -1 mon.a@0(leader).osd e202 definitely_dead 0 2023-08-28T02:27:29.894 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:29 smithi143 ceph-mon[189871]: osdmap e201: 8 total, 8 up, 6 in 2023-08-28T02:27:29.894 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:29 smithi143 ceph-mon[189871]: failed to encode map e201 with expected crc 2023-08-28T02:27:29.894 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:29 smithi143 ceph-mon[189871]: failed to encode map e201 with expected crc 2023-08-28T02:27:29.894 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:29 smithi143 ceph-mon[189871]: failed to encode map e201 with expected crc 2023-08-28T02:27:29.894 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:29 smithi143 ceph-mon[189871]: pgmap v472: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 80 MiB/s wr, 1.29k op/s; 7787/108582 objects degraded (7.172%); 574/108582 objects misplaced (0.529%); 17 MiB/s, 281 objects/s recovering 2023-08-28T02:27:30.098 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 23 16 37898 37882 102.908 52.5625 0.00260838 0.00968275 2023-08-28T02:27:30.647 INFO:teuthology.orchestra.run.smithi143.stderr:marked in osd.1. 2023-08-28T02:27:30.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:30 smithi169 ceph-mon[159415]: osdmap e202: 8 total, 8 up, 6 in 2023-08-28T02:27:30.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:30 smithi169 ceph-mon[159415]: failed to encode map e202 with expected crc 2023-08-28T02:27:30.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:30 smithi169 ceph-mon[159415]: failed to encode map e202 with expected crc 2023-08-28T02:27:30.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:30 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3165828045' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-08-28T02:27:30.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:30 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-08-28T02:27:30.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:30 smithi169 ceph-mon[159415]: failed to encode map e202 with expected crc 2023-08-28T02:27:30.842 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:30 smithi143 ceph-mon[189871]: osdmap e202: 8 total, 8 up, 6 in 2023-08-28T02:27:30.842 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:30 smithi143 ceph-mon[189871]: failed to encode map e202 with expected crc 2023-08-28T02:27:30.842 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:30 smithi143 ceph-mon[189871]: failed to encode map e202 with expected crc 2023-08-28T02:27:30.842 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:30 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3165828045' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-08-28T02:27:30.842 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:30 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-08-28T02:27:30.843 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:30 smithi143 ceph-mon[189871]: failed to encode map e202 with expected crc 2023-08-28T02:27:30.843 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:30 smithi143 ceph-mon[183059]: osdmap e202: 8 total, 8 up, 6 in 2023-08-28T02:27:30.843 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:30 smithi143 ceph-mon[183059]: failed to encode map e202 with expected crc 2023-08-28T02:27:30.843 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:30 smithi143 ceph-mon[183059]: failed to encode map e202 with expected crc 2023-08-28T02:27:30.843 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:30 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3165828045' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-08-28T02:27:30.843 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:30 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-08-28T02:27:30.843 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:30 smithi143 ceph-mon[183059]: failed to encode map e202 with expected crc 2023-08-28T02:27:31.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:31.091 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:27:31.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:27:31.098 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 24 16 38528 38512 100.261 39.375 0.00382521 0.00974112 2023-08-28T02:27:31.192 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:27:31.218 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:31.264 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:27:31.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:31.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:27:31.326 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:27:31.358 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is unset 2023-08-28T02:27:31.371 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:27:31.399 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:31.443 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:27:31.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:31.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:27:31.549 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:27:31.574 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:31.617 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:27:31.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:31.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:27:31.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:31 smithi143 ceph-mon[183059]: failed to encode map e202 with expected crc 2023-08-28T02:27:31.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:31 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-08-28T02:27:31.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:31 smithi143 ceph-mon[183059]: osdmap e203: 8 total, 8 up, 7 in 2023-08-28T02:27:31.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:31 smithi143 ceph-mon[183059]: failed to encode map e203 with expected crc 2023-08-28T02:27:31.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:31 smithi143 ceph-mon[183059]: failed to encode map e203 with expected crc 2023-08-28T02:27:31.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:31 smithi143 ceph-mon[183059]: failed to encode map e203 with expected crc 2023-08-28T02:27:31.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:31 smithi143 ceph-mon[183059]: failed to encode map e203 with expected crc 2023-08-28T02:27:31.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:31 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3808166135' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:27:31.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:31 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:27:31.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:31 smithi143 ceph-mon[183059]: pgmap v475: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 60 MiB/s wr, 964 op/s; 7787/108582 objects degraded (7.172%); 574/108582 objects misplaced (0.529%); 17 MiB/s, 264 objects/s recovering 2023-08-28T02:27:31.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:31 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 7787/108582 objects degraded (7.172%), 5 pgs degraded (PG_DEGRADED) 2023-08-28T02:27:31.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:31 smithi143 ceph-mon[183059]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:27:31.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:31 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:27:31.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:31 smithi143 ceph-mon[183059]: failed to encode map e204 with expected crc 2023-08-28T02:27:31.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:31 smithi143 ceph-mon[183059]: osdmap e204: 8 total, 8 up, 7 in 2023-08-28T02:27:31.704 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:31 smithi143 ceph-mon[183059]: failed to encode map e204 with expected crc 2023-08-28T02:27:31.704 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:31 smithi143 ceph-mon[183059]: failed to encode map e204 with expected crc 2023-08-28T02:27:31.704 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:31 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:31.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:31 smithi143 ceph-mon[189871]: failed to encode map e202 with expected crc 2023-08-28T02:27:31.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:31 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-08-28T02:27:31.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:31 smithi143 ceph-mon[189871]: osdmap e203: 8 total, 8 up, 7 in 2023-08-28T02:27:31.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:31 smithi143 ceph-mon[189871]: failed to encode map e203 with expected crc 2023-08-28T02:27:31.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:31 smithi143 ceph-mon[189871]: failed to encode map e203 with expected crc 2023-08-28T02:27:31.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:31 smithi143 ceph-mon[189871]: failed to encode map e203 with expected crc 2023-08-28T02:27:31.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:31 smithi143 ceph-mon[189871]: failed to encode map e203 with expected crc 2023-08-28T02:27:31.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:31 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3808166135' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:27:31.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:31 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:27:31.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:31 smithi143 ceph-mon[189871]: pgmap v475: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 60 MiB/s wr, 964 op/s; 7787/108582 objects degraded (7.172%); 574/108582 objects misplaced (0.529%); 17 MiB/s, 264 objects/s recovering 2023-08-28T02:27:31.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:31 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 7787/108582 objects degraded (7.172%), 5 pgs degraded (PG_DEGRADED) 2023-08-28T02:27:31.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:31 smithi143 ceph-mon[189871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:27:31.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:31 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:27:31.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:31 smithi143 ceph-mon[189871]: failed to encode map e204 with expected crc 2023-08-28T02:27:31.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:31 smithi143 ceph-mon[189871]: osdmap e204: 8 total, 8 up, 7 in 2023-08-28T02:27:31.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:31 smithi143 ceph-mon[189871]: failed to encode map e204 with expected crc 2023-08-28T02:27:31.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:31 smithi143 ceph-mon[189871]: failed to encode map e204 with expected crc 2023-08-28T02:27:31.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:31 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:31.724 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:27:31.747 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:31 smithi169 ceph-mon[159415]: failed to encode map e202 with expected crc 2023-08-28T02:27:31.747 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:31 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-08-28T02:27:31.747 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:31 smithi169 ceph-mon[159415]: osdmap e203: 8 total, 8 up, 7 in 2023-08-28T02:27:31.747 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:31 smithi169 ceph-mon[159415]: failed to encode map e203 with expected crc 2023-08-28T02:27:31.747 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:31 smithi169 ceph-mon[159415]: failed to encode map e203 with expected crc 2023-08-28T02:27:31.747 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:31 smithi169 ceph-mon[159415]: failed to encode map e203 with expected crc 2023-08-28T02:27:31.747 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:31 smithi169 ceph-mon[159415]: failed to encode map e203 with expected crc 2023-08-28T02:27:31.748 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:31 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3808166135' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:27:31.748 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:31 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:27:31.748 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:31 smithi169 ceph-mon[159415]: pgmap v475: 89 pgs: 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 60 MiB/s wr, 964 op/s; 7787/108582 objects degraded (7.172%); 574/108582 objects misplaced (0.529%); 17 MiB/s, 264 objects/s recovering 2023-08-28T02:27:31.748 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:31 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 7787/108582 objects degraded (7.172%), 5 pgs degraded (PG_DEGRADED) 2023-08-28T02:27:31.748 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:31 smithi169 ceph-mon[159415]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:27:31.748 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:31 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:27:31.748 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:31 smithi169 ceph-mon[159415]: failed to encode map e204 with expected crc 2023-08-28T02:27:31.748 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:31 smithi169 ceph-mon[159415]: osdmap e204: 8 total, 8 up, 7 in 2023-08-28T02:27:31.748 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:31 smithi169 ceph-mon[159415]: failed to encode map e204 with expected crc 2023-08-28T02:27:31.748 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:31 smithi169 ceph-mon[159415]: failed to encode map e204 with expected crc 2023-08-28T02:27:31.748 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:31 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:31.749 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:31.791 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:27:31.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:31.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:27:31.898 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:27:31.923 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:31.968 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:27:31.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:31.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:27:32.075 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:27:32.098 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 25 16 38571 38555 96.359 2.6875 0.00265171 0.00998075 2023-08-28T02:27:32.100 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:32.142 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:27:32.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:32.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:27:32.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:27:31 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:27:31] "GET /metrics HTTP/1.1" 200 36499 "" "Prometheus/2.33.4" 2023-08-28T02:27:32.247 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:27:32.277 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:32.377 INFO:tasks.thrashosds.thrasher:Added osd 1 2023-08-28T02:27:32.634 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:32 smithi143 ceph-mon[183059]: failed to encode map e203 with expected crc 2023-08-28T02:27:32.634 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:32 smithi143 ceph-mon[183059]: failed to encode map e203 with expected crc 2023-08-28T02:27:32.634 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:32 smithi143 ceph-mon[183059]: failed to encode map e203 with expected crc 2023-08-28T02:27:32.635 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:32 smithi143 ceph-mon[183059]: failed to encode map e204 with expected crc 2023-08-28T02:27:32.635 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:32 smithi143 ceph-mon[183059]: failed to encode map e204 with expected crc 2023-08-28T02:27:32.635 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:32 smithi143 ceph-mon[183059]: failed to encode map e204 with expected crc 2023-08-28T02:27:32.635 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:32 smithi143 ceph-mon[183059]: failed to encode map e204 with expected crc 2023-08-28T02:27:32.635 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:32 smithi143 ceph-mon[183059]: failed to encode map e204 with expected crc 2023-08-28T02:27:32.635 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:32 smithi143 ceph-mon[183059]: failed to encode map e204 with expected crc 2023-08-28T02:27:32.635 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:32 smithi143 ceph-mon[183059]: failed to encode map e204 with expected crc 2023-08-28T02:27:32.635 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:32 smithi143 ceph-mon[183059]: osdmap e205: 8 total, 8 up, 7 in 2023-08-28T02:27:32.635 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:32 smithi143 ceph-mon[183059]: failed to encode map e205 with expected crc 2023-08-28T02:27:32.635 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:32 smithi143 ceph-mon[183059]: failed to encode map e205 with expected crc 2023-08-28T02:27:32.635 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:32 smithi143 ceph-mon[183059]: failed to encode map e205 with expected crc 2023-08-28T02:27:32.636 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:32 smithi143 ceph-mon[189871]: failed to encode map e203 with expected crc 2023-08-28T02:27:32.636 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:32 smithi143 ceph-mon[189871]: failed to encode map e203 with expected crc 2023-08-28T02:27:32.636 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:32 smithi143 ceph-mon[189871]: failed to encode map e203 with expected crc 2023-08-28T02:27:32.636 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:32 smithi143 ceph-mon[189871]: failed to encode map e204 with expected crc 2023-08-28T02:27:32.636 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:32 smithi143 ceph-mon[189871]: failed to encode map e204 with expected crc 2023-08-28T02:27:32.636 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:32 smithi143 ceph-mon[189871]: failed to encode map e204 with expected crc 2023-08-28T02:27:32.636 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:32 smithi143 ceph-mon[189871]: failed to encode map e204 with expected crc 2023-08-28T02:27:32.636 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:32 smithi143 ceph-mon[189871]: failed to encode map e204 with expected crc 2023-08-28T02:27:32.636 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:32 smithi143 ceph-mon[189871]: failed to encode map e204 with expected crc 2023-08-28T02:27:32.636 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:32 smithi143 ceph-mon[189871]: failed to encode map e204 with expected crc 2023-08-28T02:27:32.636 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:32 smithi143 ceph-mon[189871]: osdmap e205: 8 total, 8 up, 7 in 2023-08-28T02:27:32.637 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:32 smithi143 ceph-mon[189871]: failed to encode map e205 with expected crc 2023-08-28T02:27:32.637 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:32 smithi143 ceph-mon[189871]: failed to encode map e205 with expected crc 2023-08-28T02:27:32.637 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:32 smithi143 ceph-mon[189871]: failed to encode map e205 with expected crc 2023-08-28T02:27:32.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:32 smithi169 ceph-mon[159415]: failed to encode map e203 with expected crc 2023-08-28T02:27:32.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:32 smithi169 ceph-mon[159415]: failed to encode map e203 with expected crc 2023-08-28T02:27:32.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:32 smithi169 ceph-mon[159415]: failed to encode map e203 with expected crc 2023-08-28T02:27:32.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:32 smithi169 ceph-mon[159415]: failed to encode map e204 with expected crc 2023-08-28T02:27:32.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:32 smithi169 ceph-mon[159415]: failed to encode map e204 with expected crc 2023-08-28T02:27:32.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:32 smithi169 ceph-mon[159415]: failed to encode map e204 with expected crc 2023-08-28T02:27:32.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:32 smithi169 ceph-mon[159415]: failed to encode map e204 with expected crc 2023-08-28T02:27:32.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:32 smithi169 ceph-mon[159415]: failed to encode map e204 with expected crc 2023-08-28T02:27:32.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:32 smithi169 ceph-mon[159415]: failed to encode map e204 with expected crc 2023-08-28T02:27:32.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:32 smithi169 ceph-mon[159415]: failed to encode map e204 with expected crc 2023-08-28T02:27:32.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:32 smithi169 ceph-mon[159415]: osdmap e205: 8 total, 8 up, 7 in 2023-08-28T02:27:32.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:32 smithi169 ceph-mon[159415]: failed to encode map e205 with expected crc 2023-08-28T02:27:32.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:32 smithi169 ceph-mon[159415]: failed to encode map e205 with expected crc 2023-08-28T02:27:32.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:32 smithi169 ceph-mon[159415]: failed to encode map e205 with expected crc 2023-08-28T02:27:32.998 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:27:33.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:33.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:27:33.098 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 26 16 39247 39231 94.2782 42.25 0.00530194 0.0106019 2023-08-28T02:27:33.111 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:27:33.170 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:27:33.170 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:27:33.203 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:33.263 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:27:33.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:33.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:27:33.370 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:27:33.459 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:33.704 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:33 smithi143 ceph-mon[183059]: failed to encode map e205 with expected crc 2023-08-28T02:27:33.704 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:33 smithi143 ceph-mon[183059]: failed to encode map e205 with expected crc 2023-08-28T02:27:33.704 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:33 smithi143 ceph-mon[183059]: failed to encode map e205 with expected crc 2023-08-28T02:27:33.704 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:33 smithi143 ceph-mon[183059]: failed to encode map e205 with expected crc 2023-08-28T02:27:33.704 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:33 smithi143 ceph-mon[183059]: failed to encode map e205 with expected crc 2023-08-28T02:27:33.704 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:33 smithi143 ceph-mon[183059]: failed to encode map e205 with expected crc 2023-08-28T02:27:33.704 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:33 smithi143 ceph-mon[183059]: pgmap v478: 89 pgs: 5 remapped+peering, 3 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.3 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 24 MiB/s wr, 385 op/s; 6956/113211 objects degraded (6.144%); 2094/113211 objects misplaced (1.850%); 3.0 MiB/s, 48 objects/s recovering 2023-08-28T02:27:33.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:33 smithi143 ceph-mon[189871]: failed to encode map e205 with expected crc 2023-08-28T02:27:33.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:33 smithi143 ceph-mon[189871]: failed to encode map e205 with expected crc 2023-08-28T02:27:33.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:33 smithi143 ceph-mon[189871]: failed to encode map e205 with expected crc 2023-08-28T02:27:33.706 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:33 smithi143 ceph-mon[189871]: failed to encode map e205 with expected crc 2023-08-28T02:27:33.706 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:33 smithi143 ceph-mon[189871]: failed to encode map e205 with expected crc 2023-08-28T02:27:33.706 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:33 smithi143 ceph-mon[189871]: failed to encode map e205 with expected crc 2023-08-28T02:27:33.706 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:33 smithi143 ceph-mon[189871]: pgmap v478: 89 pgs: 5 remapped+peering, 3 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.3 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 24 MiB/s wr, 385 op/s; 6956/113211 objects degraded (6.144%); 2094/113211 objects misplaced (1.850%); 3.0 MiB/s, 48 objects/s recovering 2023-08-28T02:27:33.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:33 smithi169 ceph-mon[159415]: failed to encode map e205 with expected crc 2023-08-28T02:27:33.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:33 smithi169 ceph-mon[159415]: failed to encode map e205 with expected crc 2023-08-28T02:27:33.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:33 smithi169 ceph-mon[159415]: failed to encode map e205 with expected crc 2023-08-28T02:27:33.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:33 smithi169 ceph-mon[159415]: failed to encode map e205 with expected crc 2023-08-28T02:27:33.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:33 smithi169 ceph-mon[159415]: failed to encode map e205 with expected crc 2023-08-28T02:27:33.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:33 smithi169 ceph-mon[159415]: failed to encode map e205 with expected crc 2023-08-28T02:27:33.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:33 smithi169 ceph-mon[159415]: pgmap v478: 89 pgs: 5 remapped+peering, 3 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.3 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 24 MiB/s wr, 385 op/s; 6956/113211 objects degraded (6.144%); 2094/113211 objects misplaced (1.850%); 3.0 MiB/s, 48 objects/s recovering 2023-08-28T02:27:34.100 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 27 16 40256 40240 93.1219 63.0625 0.0103489 0.0107316 2023-08-28T02:27:34.832 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset nodeep-scrub 2023-08-28T02:27:35.098 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 28 16 41006 40990 91.4703 46.875 0.0150812 0.0109255 2023-08-28T02:27:35.432 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.2 dump_blocked_ops 2023-08-28T02:27:35.441 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:27:35.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:35.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:27:35.547 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:27:35.660 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:35.856 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:27:36.020 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:27:36.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:36.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:27:36.100 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 29 16 41854 41838 90.1392 53 0.031979 0.0110867 2023-08-28T02:27:36.124 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:27:36.155 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:36.209 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:27:36.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:36.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:27:36.279 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:36 smithi143 ceph-mon[183059]: pgmap v479: 89 pgs: 1 active+recovery_wait+remapped, 5 remapped+peering, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 2.4 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 34 MiB/s wr, 550 op/s; 8617/118020 objects degraded (7.301%); 3782/118020 objects misplaced (3.205%); 11 MiB/s, 171 objects/s recovering 2023-08-28T02:27:36.279 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:36 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:36.279 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:36 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:36.279 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:36 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3499117244' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:27:36.279 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:36 smithi143 ceph-mon[189871]: pgmap v479: 89 pgs: 1 active+recovery_wait+remapped, 5 remapped+peering, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 2.4 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 34 MiB/s wr, 550 op/s; 8617/118020 objects degraded (7.301%); 3782/118020 objects misplaced (3.205%); 11 MiB/s, 171 objects/s recovering 2023-08-28T02:27:36.280 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:36 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:36.280 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:36 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:36.280 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:36 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3499117244' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:27:36.282 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:27:36.314 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:27:36.323 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:27:36.345 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:36.375 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:27:36.400 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:27:36.404 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:36 smithi169 ceph-mon[159415]: pgmap v479: 89 pgs: 1 active+recovery_wait+remapped, 5 remapped+peering, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 2.4 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 34 MiB/s wr, 550 op/s; 8617/118020 objects degraded (7.301%); 3782/118020 objects misplaced (3.205%); 11 MiB/s, 171 objects/s recovering 2023-08-28T02:27:36.404 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:36 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:36.404 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:36 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:36.404 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:36 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3499117244' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:27:36.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:36.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:27:36.446 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:27:36.483 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:27:36.506 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:27:36.525 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:27:36.572 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:27:36.606 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:36.606 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:27:36.620 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:27:36.620 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:27:36.620 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:27:36.621 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:27:36.621 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:27:36.621 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:27:36.621 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:27:36.621 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:27:37.100 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 30 16 42706 42690 88.9097 53.25 0.0126223 0.0112406 2023-08-28T02:27:37.306 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:27:37.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:37.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:27:37.378 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 7, 3, 1] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-08-28T02:27:37.378 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-08-28T02:27:37.378 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_1 2023-08-28T02:27:37.378 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph pg dump --format=json 2023-08-28T02:27:37.413 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:27:37.530 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:37 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 8617/118020 objects degraded (7.301%), 6 pgs degraded (PG_DEGRADED) 2023-08-28T02:27:37.531 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:37 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:27:37.531 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:37 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:27:37.531 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:37 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:37.531 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:37 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 8617/118020 objects degraded (7.301%), 6 pgs degraded (PG_DEGRADED) 2023-08-28T02:27:37.531 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:37 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:27:37.531 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:37 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:27:37.531 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:37 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:37.532 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:37.672 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:27:37.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:37.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:27:37.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:37 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 8617/118020 objects degraded (7.301%), 6 pgs degraded (PG_DEGRADED) 2023-08-28T02:27:37.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:37 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:27:37.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:37 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:27:37.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:37 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:37.793 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:27:37.818 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:37.884 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:27:37.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:37.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:27:37.992 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:27:38.027 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:38.099 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:27:38.100 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 31 16 43639 43623 87.9227 58.3125 0.0112848 0.0113665 2023-08-28T02:27:38.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:38.105 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:27:38.206 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:27:38.277 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:38.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:38 smithi169 ceph-mon[159415]: pgmap v480: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 2.5 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 41 MiB/s wr, 663 op/s; 11039/121896 objects degraded (9.056%); 3668/121896 objects misplaced (3.009%); 10 MiB/s, 163 objects/s recovering 2023-08-28T02:27:38.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:38 smithi143 ceph-mon[183059]: pgmap v480: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 2.5 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 41 MiB/s wr, 663 op/s; 11039/121896 objects degraded (9.056%); 3668/121896 objects misplaced (3.009%); 10 MiB/s, 163 objects/s recovering 2023-08-28T02:27:38.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:38 smithi143 ceph-mon[189871]: pgmap v480: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 2.5 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 41 MiB/s wr, 663 op/s; 11039/121896 objects degraded (9.056%); 3668/121896 objects misplaced (3.009%); 10 MiB/s, 163 objects/s recovering 2023-08-28T02:27:38.716 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:27:38.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:38.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:27:38.824 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:27:38.859 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:38.919 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:27:38.919 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:27:38.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:38.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:27:39.021 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:27:39.025 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:27:39.057 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:39.100 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 32 16 44625 44609 87.1009 61.625 0.0147422 0.0114747 2023-08-28T02:27:39.117 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:27:39.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:39.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:27:39.224 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:27:39.257 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:39.319 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:27:39.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:39.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:27:39.425 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:27:39.509 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:39.624 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:27:39.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:39.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:27:39.740 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:27:39.772 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:39.829 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:27:39.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:39.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:27:39.937 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:27:40.028 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:40.100 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 33 16 45533 45517 86.1812 56.75 0.010048 0.0115961 2023-08-28T02:27:40.358 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is unset 2023-08-28T02:27:40.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:40 smithi143 ceph-mon[189871]: pgmap v481: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 2.7 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 58 MiB/s wr, 928 op/s; 13366/130887 objects degraded (10.212%); 3417/130887 objects misplaced (2.611%); 11 MiB/s, 168 objects/s recovering 2023-08-28T02:27:40.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:40 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1383661350' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:27:40.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:40 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:27:40.705 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:40 smithi143 ceph-mon[183059]: pgmap v481: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 2.7 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 58 MiB/s wr, 928 op/s; 13366/130887 objects degraded (10.212%); 3417/130887 objects misplaced (2.611%); 11 MiB/s, 168 objects/s recovering 2023-08-28T02:27:40.705 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:40 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1383661350' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:27:40.706 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:40 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:27:40.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:40 smithi169 ceph-mon[159415]: pgmap v481: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 2.7 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 58 MiB/s wr, 928 op/s; 13366/130887 objects degraded (10.212%); 3417/130887 objects misplaced (2.611%); 11 MiB/s, 168 objects/s recovering 2023-08-28T02:27:40.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:40 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1383661350' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:27:40.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:40 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:27:40.817 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:27:41.103 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 34 16 46512 46496 85.4439 61.1875 0.0121607 0.0116965 2023-08-28T02:27:41.706 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:41 smithi143 ceph-mon[183059]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:27:41.706 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:41 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:27:41.706 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:41 smithi143 ceph-mon[183059]: osdmap e206: 8 total, 8 up, 7 in 2023-08-28T02:27:41.707 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:41 smithi143 ceph-mon[183059]: failed to encode map e206 with expected crc 2023-08-28T02:27:41.707 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:41 smithi143 ceph-mon[183059]: failed to encode map e206 with expected crc 2023-08-28T02:27:41.707 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:41 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:41.707 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:41 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:27:41.707 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:41 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 13366/130887 objects degraded (10.212%), 11 pgs degraded (PG_DEGRADED) 2023-08-28T02:27:41.707 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:41 smithi143 ceph-mon[189871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:27:41.707 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:41 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:27:41.707 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:41 smithi143 ceph-mon[189871]: osdmap e206: 8 total, 8 up, 7 in 2023-08-28T02:27:41.707 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:41 smithi143 ceph-mon[189871]: failed to encode map e206 with expected crc 2023-08-28T02:27:41.707 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:41 smithi143 ceph-mon[189871]: failed to encode map e206 with expected crc 2023-08-28T02:27:41.708 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:41 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:41.708 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:41 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:27:41.708 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:41 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 13366/130887 objects degraded (10.212%), 11 pgs degraded (PG_DEGRADED) 2023-08-28T02:27:41.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:41 smithi169 ceph-mon[159415]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:27:41.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:41 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:27:41.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:41 smithi169 ceph-mon[159415]: osdmap e206: 8 total, 8 up, 7 in 2023-08-28T02:27:41.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:41 smithi169 ceph-mon[159415]: failed to encode map e206 with expected crc 2023-08-28T02:27:41.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:41 smithi169 ceph-mon[159415]: failed to encode map e206 with expected crc 2023-08-28T02:27:41.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:41 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:41.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:41 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:27:41.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:41 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 13366/130887 objects degraded (10.212%), 11 pgs degraded (PG_DEGRADED) 2023-08-28T02:27:42.101 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 35 16 47287 47271 84.3867 48.4375 0.00329445 0.0118014 2023-08-28T02:27:42.205 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:27:41 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:27:41] "GET /metrics HTTP/1.1" 200 36508 "" "Prometheus/2.33.4" 2023-08-28T02:27:42.704 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:42 smithi143 ceph-mon[183059]: pgmap v483: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 2.7 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 41 MiB/s wr, 663 op/s; 13366/130887 objects degraded (10.212%); 3417/130887 objects misplaced (2.611%); 8.2 MiB/s, 129 objects/s recovering 2023-08-28T02:27:42.704 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:42 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:42.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:42 smithi143 ceph-mon[189871]: pgmap v483: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 2.7 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 41 MiB/s wr, 663 op/s; 13366/130887 objects degraded (10.212%); 3417/130887 objects misplaced (2.611%); 8.2 MiB/s, 129 objects/s recovering 2023-08-28T02:27:42.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:42 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:42.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:42 smithi169 ceph-mon[159415]: pgmap v483: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 2.7 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 41 MiB/s wr, 663 op/s; 13366/130887 objects degraded (10.212%); 3417/130887 objects misplaced (2.611%); 8.2 MiB/s, 129 objects/s recovering 2023-08-28T02:27:42.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:42 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:42.802 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:27:42.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:42.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:27:42.908 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:27:42.939 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:42.998 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:27:43.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:43.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:27:43.040 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:27:43.101 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 36 16 48150 48134 83.541 53.9375 0.0130136 0.0119629 2023-08-28T02:27:43.104 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:27:43.211 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:43.455 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:27:43.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:43.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:27:43.563 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:27:43.595 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:43.623 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:27:43.623 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [], 2023-08-28T02:27:43.623 INFO:teuthology.orchestra.run.smithi143.stdout: "complaint_time": 30, 2023-08-28T02:27:43.623 INFO:teuthology.orchestra.run.smithi143.stdout: "num_blocked_ops": 0 2023-08-28T02:27:43.623 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:27:43.653 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:27:43.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:43.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:27:43.760 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:27:43.836 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:44.090 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:27:44.101 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 37 16 49015 48999 82.7443 54.0625 0.0249997 0.0120796 2023-08-28T02:27:44.111 INFO:teuthology.orchestra.run.smithi143.stderr:dumped all 2023-08-28T02:27:44.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:44 smithi143 ceph-mon[183059]: pgmap v484: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 2.8 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 51 MiB/s wr, 819 op/s; 13366/137808 objects degraded (9.699%); 3304/137808 objects misplaced (2.398%); 7.9 MiB/s, 126 objects/s recovering 2023-08-28T02:27:44.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:44 smithi143 ceph-mon[189871]: pgmap v484: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 2.8 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 51 MiB/s wr, 819 op/s; 13366/137808 objects degraded (9.699%); 3304/137808 objects misplaced (2.398%); 7.9 MiB/s, 126 objects/s recovering 2023-08-28T02:27:44.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:44 smithi169 ceph-mon[159415]: pgmap v484: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 2.8 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 51 MiB/s wr, 819 op/s; 13366/137808 objects degraded (9.699%); 3304/137808 objects misplaced (2.398%); 7.9 MiB/s, 126 objects/s recovering 2023-08-28T02:27:44.770 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set noscrub 2023-08-28T02:27:45.110 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 38 16 49710 49694 81.693 43.4375 0.00973389 0.0122331 2023-08-28T02:27:45.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:45 smithi143 ceph-mon[183059]: from='client.45074 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:27:45.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:45 smithi143 ceph-mon[189871]: from='client.45074 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:27:45.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:45 smithi169 ceph-mon[159415]: from='client.45074 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:27:45.802 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:27:45.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:45.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:27:45.909 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:27:45.934 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:45.981 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:27:45.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:45.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:27:46.089 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:27:46.098 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.2 dump_historic_ops 2023-08-28T02:27:46.109 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 39 16 50540 50524 80.9286 51.875 0.00547687 0.0123466 2023-08-28T02:27:46.117 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:46.159 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:27:46.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:46.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:27:46.264 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:27:46.294 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:46.354 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:27:46.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:46.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:27:46.389 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:46 smithi143 ceph-mon[183059]: pgmap v485: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 2.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 55 MiB/s wr, 878 op/s; 13366/144381 objects degraded (9.257%); 3081/144381 objects misplaced (2.134%); 4.4 MiB/s, 70 objects/s recovering 2023-08-28T02:27:46.389 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:46 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 13366/144381 objects degraded (9.257%), 11 pgs degraded (PG_DEGRADED) 2023-08-28T02:27:46.389 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:46 smithi143 ceph-mon[189871]: pgmap v485: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 2.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 55 MiB/s wr, 878 op/s; 13366/144381 objects degraded (9.257%); 3081/144381 objects misplaced (2.134%); 4.4 MiB/s, 70 objects/s recovering 2023-08-28T02:27:46.389 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:46 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 13366/144381 objects degraded (9.257%), 11 pgs degraded (PG_DEGRADED) 2023-08-28T02:27:46.459 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:27:46.493 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:46.559 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:27:46.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:46.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:27:46.568 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:46 smithi169 ceph-mon[159415]: pgmap v485: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 2.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 55 MiB/s wr, 878 op/s; 13366/144381 objects degraded (9.257%); 3081/144381 objects misplaced (2.134%); 4.4 MiB/s, 70 objects/s recovering 2023-08-28T02:27:46.568 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:46 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 13366/144381 objects degraded (9.257%), 11 pgs degraded (PG_DEGRADED) 2023-08-28T02:27:46.666 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:27:46.695 INFO:teuthology.orchestra.run.smithi143.stdout:{"pg_ready":true,"pg_map":{"version":484,"stamp":"2023-08-28T02:27:43.201225+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3013416883,"num_objects":45936,"num_object_clones":0,"num_object_copies":137808,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13366,"num_objects_misplaced":3304,"num_objects_unfound":0,"num_objects_dirty":45936,"num_whiteouts":0,"num_read":6274,"num_read_kb":13707,"num_write":47014,"num_write_kb":2953253,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4128,"num_bytes_recovered":269172600,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":47078,"ondisk_log_size":47078,"up":267,"acting":255,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":286,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":10152144,"kb_used_data":8551128,"kb_used_omap":29,"kb_used_meta":1600930,"kb_avail":646063920,"statfs":{"total":671965249536,"available":661569454080,"internally_reserved":0,"allocated":8756355072,"data_stored":8750932202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30674,"internal_metadata":1639352366},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[6,5,5,2,3,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":537329664,"num_objects":8199,"num_object_clones":0,"num_object_copies":24597,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6410,"num_objects_misplaced":1210,"num_objects_unfound":0,"num_objects_dirty":8199,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8199,"num_write_kb":524736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1264,"num_bytes_recovered":83361792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.007380"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":207,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-08-28T02:27:32.320279+0000","last_change":"2023-08-28T02:27:31.367469+0000","last_active":"2023-08-28T02:27:32.320279+0000","last_peered":"2023-08-28T02:27:32.320279+0000","last_clean":"2023-08-28T02:27:32.320279+0000","last_became_active":"2023-08-28T02:27:31.366946+0000","last_became_peered":"2023-08-28T02:27:31.366946+0000","last_unstale":"2023-08-28T02:27:32.320279+0000","last_undegraded":"2023-08-28T02:27:32.320279+0000","last_fullsized":"2023-08-28T02:27:32.320279+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":90,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-08-28T02:27:31.332823+0000","last_change":"2023-08-28T02:27:31.324452+0000","last_active":"2023-08-28T02:27:31.332823+0000","last_peered":"2023-08-28T02:27:31.332823+0000","last_clean":"2023-08-28T02:27:31.332823+0000","last_became_active":"2023-08-28T02:27:31.323749+0000","last_became_peered":"2023-08-28T02:27:31.323749+0000","last_unstale":"2023-08-28T02:27:31.332823+0000","last_undegraded":"2023-08-28T02:27:31.332823+0000","last_fullsized":"2023-08-28T02:27:31.332823+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:18:10.904127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":12,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-08-28T02:27:32.315046+0000","last_change":"2023-08-28T02:27:31.337338+0000","last_active":"2023-08-28T02:27:32.315046+0000","last_peered":"2023-08-28T02:27:32.315046+0000","last_clean":"2023-08-28T02:27:32.315046+0000","last_became_active":"2023-08-28T02:27:31.337018+0000","last_became_peered":"2023-08-28T02:27:31.337018+0000","last_unstale":"2023-08-28T02:27:32.315046+0000","last_undegraded":"2023-08-28T02:27:32.315046+0000","last_fullsized":"2023-08-28T02:27:32.315046+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:46:34.845663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":191,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-08-28T02:27:31.313147+0000","last_change":"2023-08-28T02:23:59.223634+0000","last_active":"2023-08-28T02:27:31.313147+0000","last_peered":"2023-08-28T02:27:31.313147+0000","last_clean":"2023-08-28T02:27:31.313147+0000","last_became_active":"2023-08-28T02:23:59.222704+0000","last_became_peered":"2023-08-28T02:23:59.222704+0000","last_unstale":"2023-08-28T02:27:31.313147+0000","last_undegraded":"2023-08-28T02:27:31.313147+0000","last_fullsized":"2023-08-28T02:27:31.313147+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:51:52.752361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":81,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-08-28T02:27:31.319853+0000","last_change":"2023-08-28T02:27:31.318153+0000","last_active":"2023-08-28T02:27:31.319853+0000","last_peered":"2023-08-28T02:27:31.319853+0000","last_clean":"2023-08-28T02:27:31.319853+0000","last_became_active":"2023-08-28T02:27:31.317732+0000","last_became_peered":"2023-08-28T02:27:31.317732+0000","last_unstale":"2023-08-28T02:27:31.319853+0000","last_undegraded":"2023-08-28T02:27:31.319853+0000","last_fullsized":"2023-08-28T02:27:31.319853+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:06:13.421674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":216,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-08-28T02:27:32.322916+0000","last_change":"2023-08-28T02:27:31.333055+0000","last_active":"2023-08-28T02:27:32.322916+0000","last_peered":"2023-08-28T02:27:32.322916+0000","last_clean":"2023-08-28T02:27:32.322916+0000","last_became_active":"2023-08-28T02:27:31.332770+0000","last_became_peered":"2023-08-28T02:27:31.332770+0000","last_unstale":"2023-08-28T02:27:32.322916+0000","last_undegraded":"2023-08-28T02:27:32.322916+0000","last_fullsized":"2023-08-28T02:27:32.322916+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":171,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-08-28T02:27:31.310074+0000","last_change":"2023-08-28T02:23:59.227592+0000","last_active":"2023-08-28T02:27:31.310074+0000","last_peered":"2023-08-28T02:27:31.310074+0000","last_clean":"2023-08-28T02:27:31.310074+0000","last_became_active":"2023-08-28T02:23:59.227342+0000","last_became_peered":"2023-08-28T02:23:59.227342+0000","last_unstale":"2023-08-28T02:27:31.310074+0000","last_undegraded":"2023-08-28T02:27:31.310074+0000","last_fullsized":"2023-08-28T02:27:31.310074+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:08:19.874396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":90,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-08-28T02:27:31.332938+0000","last_change":"2023-08-28T02:27:31.323728+0000","last_active":"2023-08-28T02:27:31.332938+0000","last_peered":"2023-08-28T02:27:31.332938+0000","last_clean":"2023-08-28T02:27:31.332938+0000","last_became_active":"2023-08-28T02:27:31.323375+0000","last_became_peered":"2023-08-28T02:27:31.323375+0000","last_unstale":"2023-08-28T02:27:31.332938+0000","last_undegraded":"2023-08-28T02:27:31.332938+0000","last_fullsized":"2023-08-28T02:27:31.332938+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:47:13.141692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":208,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-08-28T02:27:32.322861+0000","last_change":"2023-08-28T02:27:31.328499+0000","last_active":"2023-08-28T02:27:32.322861+0000","last_peered":"2023-08-28T02:27:32.322861+0000","last_clean":"2023-08-28T02:27:32.322861+0000","last_became_active":"2023-08-28T02:27:31.328164+0000","last_became_peered":"2023-08-28T02:27:31.328164+0000","last_unstale":"2023-08-28T02:27:32.322861+0000","last_undegraded":"2023-08-28T02:27:32.322861+0000","last_fullsized":"2023-08-28T02:27:32.322861+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":72,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-08-28T02:27:31.303774+0000","last_change":"2023-08-28T02:27:17.334402+0000","last_active":"2023-08-28T02:27:31.303774+0000","last_peered":"2023-08-28T02:27:31.303774+0000","last_clean":"2023-08-28T02:27:31.303774+0000","last_became_active":"2023-08-28T02:27:17.334148+0000","last_became_peered":"2023-08-28T02:27:17.334148+0000","last_unstale":"2023-08-28T02:27:31.303774+0000","last_undegraded":"2023-08-28T02:27:31.303774+0000","last_fullsized":"2023-08-28T02:27:31.303774+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:55:16.705215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":91,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-08-28T02:27:32.324384+0000","last_change":"2023-08-28T02:23:59.316561+0000","last_active":"2023-08-28T02:27:32.324384+0000","last_peered":"2023-08-28T02:27:32.324384+0000","last_clean":"2023-08-28T02:27:32.324384+0000","last_became_active":"2023-08-28T02:23:59.227433+0000","last_became_peered":"2023-08-28T02:23:59.227433+0000","last_unstale":"2023-08-28T02:27:32.324384+0000","last_undegraded":"2023-08-28T02:27:32.324384+0000","last_fullsized":"2023-08-28T02:27:32.324384+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":12,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-08-28T02:27:32.315228+0000","last_change":"2023-08-28T02:27:31.382991+0000","last_active":"2023-08-28T02:27:32.315228+0000","last_peered":"2023-08-28T02:27:32.315228+0000","last_clean":"2023-08-28T02:27:32.315228+0000","last_became_active":"2023-08-28T02:27:31.382673+0000","last_became_peered":"2023-08-28T02:27:31.382673+0000","last_unstale":"2023-08-28T02:27:32.315228+0000","last_undegraded":"2023-08-28T02:27:32.315228+0000","last_fullsized":"2023-08-28T02:27:32.315228+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:43:52.960169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"86'22","reported_seq":237,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-08-28T02:27:32.320566+0000","last_change":"2023-08-28T02:27:17.358194+0000","last_active":"2023-08-28T02:27:32.320566+0000","last_peered":"2023-08-28T02:27:32.320566+0000","last_clean":"2023-08-28T02:27:32.320566+0000","last_became_active":"2023-08-28T02:27:17.335089+0000","last_became_peered":"2023-08-28T02:27:17.335089+0000","last_unstale":"2023-08-28T02:27:32.320566+0000","last_undegraded":"2023-08-28T02:27:32.320566+0000","last_fullsized":"2023-08-28T02:27:32.320566+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":213,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-08-28T02:27:32.320436+0000","last_change":"2023-08-28T02:27:31.319631+0000","last_active":"2023-08-28T02:27:32.320436+0000","last_peered":"2023-08-28T02:27:32.320436+0000","last_clean":"2023-08-28T02:27:32.320436+0000","last_became_active":"2023-08-28T02:27:31.318882+0000","last_became_peered":"2023-08-28T02:27:31.318882+0000","last_unstale":"2023-08-28T02:27:32.320436+0000","last_undegraded":"2023-08-28T02:27:32.320436+0000","last_fullsized":"2023-08-28T02:27:32.320436+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":76,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-08-28T02:27:32.326372+0000","last_change":"2023-08-28T02:23:38.983634+0000","last_active":"2023-08-28T02:27:32.326372+0000","last_peered":"2023-08-28T02:27:32.326372+0000","last_clean":"2023-08-28T02:27:32.326372+0000","last_became_active":"2023-08-28T02:23:38.344566+0000","last_became_peered":"2023-08-28T02:23:38.344566+0000","last_unstale":"2023-08-28T02:27:32.326372+0000","last_undegraded":"2023-08-28T02:27:32.326372+0000","last_fullsized":"2023-08-28T02:27:32.326372+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":81,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-08-28T02:27:31.351506+0000","last_change":"2023-08-28T02:27:31.344974+0000","last_active":"2023-08-28T02:27:31.351506+0000","last_peered":"2023-08-28T02:27:31.351506+0000","last_clean":"2023-08-28T02:27:31.351506+0000","last_became_active":"2023-08-28T02:27:31.344281+0000","last_became_peered":"2023-08-28T02:27:31.344281+0000","last_unstale":"2023-08-28T02:27:31.351506+0000","last_undegraded":"2023-08-28T02:27:31.351506+0000","last_fullsized":"2023-08-28T02:27:31.351506+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:04:02.821752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":189,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-08-28T02:27:31.314173+0000","last_change":"2023-08-28T02:23:59.223066+0000","last_active":"2023-08-28T02:27:31.314173+0000","last_peered":"2023-08-28T02:27:31.314173+0000","last_clean":"2023-08-28T02:27:31.314173+0000","last_became_active":"2023-08-28T02:23:59.222731+0000","last_became_peered":"2023-08-28T02:23:59.222731+0000","last_unstale":"2023-08-28T02:27:31.314173+0000","last_undegraded":"2023-08-28T02:27:31.314173+0000","last_fullsized":"2023-08-28T02:27:31.314173+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:48:37.558266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":232,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-08-28T02:27:32.378787+0000","last_change":"2023-08-28T02:23:59.210469+0000","last_active":"2023-08-28T02:27:32.378787+0000","last_peered":"2023-08-28T02:27:32.378787+0000","last_clean":"2023-08-28T02:27:32.378787+0000","last_became_active":"2023-08-28T02:23:59.210031+0000","last_became_peered":"2023-08-28T02:23:59.210031+0000","last_unstale":"2023-08-28T02:27:32.378787+0000","last_undegraded":"2023-08-28T02:27:32.378787+0000","last_fullsized":"2023-08-28T02:27:32.378787+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":79,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-08-28T02:27:32.324365+0000","last_change":"2023-08-28T02:27:17.336945+0000","last_active":"2023-08-28T02:27:32.324365+0000","last_peered":"2023-08-28T02:27:32.324365+0000","last_clean":"2023-08-28T02:27:32.324365+0000","last_became_active":"2023-08-28T02:27:17.336574+0000","last_became_peered":"2023-08-28T02:27:17.336574+0000","last_unstale":"2023-08-28T02:27:32.324365+0000","last_undegraded":"2023-08-28T02:27:32.324365+0000","last_fullsized":"2023-08-28T02:27:32.324365+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":190,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-08-28T02:27:31.313194+0000","last_change":"2023-08-28T02:23:59.220932+0000","last_active":"2023-08-28T02:27:31.313194+0000","last_peered":"2023-08-28T02:27:31.313194+0000","last_clean":"2023-08-28T02:27:31.313194+0000","last_became_active":"2023-08-28T02:23:59.220488+0000","last_became_peered":"2023-08-28T02:23:59.220488+0000","last_unstale":"2023-08-28T02:27:31.313194+0000","last_undegraded":"2023-08-28T02:27:31.313194+0000","last_fullsized":"2023-08-28T02:27:31.313194+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:53:09.216340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":41,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-08-28T02:27:31.309455+0000","last_change":"2023-08-28T02:27:17.340932+0000","last_active":"2023-08-28T02:27:31.309455+0000","last_peered":"2023-08-28T02:27:31.309455+0000","last_clean":"2023-08-28T02:27:31.309455+0000","last_became_active":"2023-08-28T02:27:17.325514+0000","last_became_peered":"2023-08-28T02:27:17.325514+0000","last_unstale":"2023-08-28T02:27:31.309455+0000","last_undegraded":"2023-08-28T02:27:31.309455+0000","last_fullsized":"2023-08-28T02:27:31.309455+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:45:08.412118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":81,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-08-28T02:27:31.330566+0000","last_change":"2023-08-28T02:27:31.330059+0000","last_active":"2023-08-28T02:27:31.330566+0000","last_peered":"2023-08-28T02:27:31.330566+0000","last_clean":"2023-08-28T02:27:31.330566+0000","last_became_active":"2023-08-28T02:27:31.329769+0000","last_became_peered":"2023-08-28T02:27:31.329769+0000","last_unstale":"2023-08-28T02:27:31.330566+0000","last_undegraded":"2023-08-28T02:27:31.330566+0000","last_fullsized":"2023-08-28T02:27:31.330566+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:14:54.349909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"206'2962","reported_seq":3349,"reported_epoch":206,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:27:42.218069+0000","last_change":"2023-08-28T02:27:31.330530+0000","last_active":"2023-08-28T02:27:42.218069+0000","last_peered":"2023-08-28T02:27:42.218069+0000","last_clean":"2023-08-28T02:27:16.305418+0000","last_became_active":"2023-08-28T02:27:18.435652+0000","last_became_peered":"2023-08-28T02:27:18.435652+0000","last_unstale":"2023-08-28T02:27:42.218069+0000","last_undegraded":"2023-08-28T02:27:31.329359+0000","last_fullsized":"2023-08-28T02:27:18.320372+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":2962,"log_dups_size":0,"ondisk_log_size":2962,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194117632,"num_objects":2962,"num_object_clones":0,"num_object_copies":8886,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1370,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2962,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2962,"num_write_kb":189568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":176,"num_bytes_recovered":11534336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0],"avail_no_missing":["5","0","6"],"object_location_counts":[{"shards":"0,5","objects":1592},{"shards":"0,5,6","objects":1370}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":74,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-08-28T02:27:32.324216+0000","last_change":"2023-08-28T02:23:38.657185+0000","last_active":"2023-08-28T02:27:32.324216+0000","last_peered":"2023-08-28T02:27:32.324216+0000","last_clean":"2023-08-28T02:27:32.324216+0000","last_became_active":"2023-08-28T02:23:38.339016+0000","last_became_peered":"2023-08-28T02:23:38.339016+0000","last_unstale":"2023-08-28T02:27:32.324216+0000","last_undegraded":"2023-08-28T02:27:32.324216+0000","last_fullsized":"2023-08-28T02:27:32.324216+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":183,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-08-28T02:27:32.322705+0000","last_change":"2023-08-28T02:27:17.332232+0000","last_active":"2023-08-28T02:27:32.322705+0000","last_peered":"2023-08-28T02:27:32.322705+0000","last_clean":"2023-08-28T02:27:32.322705+0000","last_became_active":"2023-08-28T02:27:17.331892+0000","last_became_peered":"2023-08-28T02:27:17.331892+0000","last_unstale":"2023-08-28T02:27:32.322705+0000","last_undegraded":"2023-08-28T02:27:32.322705+0000","last_fullsized":"2023-08-28T02:27:32.322705+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"205'2751","reported_seq":2807,"reported_epoch":205,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:27:38.958327+0000","last_change":"2023-08-28T02:27:31.476755+0000","last_active":"2023-08-28T02:27:38.958327+0000","last_peered":"2023-08-28T02:27:38.958327+0000","last_clean":"2023-08-28T02:27:16.307202+0000","last_became_active":"2023-08-28T02:27:31.472816+0000","last_became_peered":"2023-08-28T02:27:31.472816+0000","last_unstale":"2023-08-28T02:27:38.958327+0000","last_undegraded":"2023-08-28T02:27:31.324882+0000","last_fullsized":"2023-08-28T02:27:31.303720+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":2751,"log_dups_size":0,"ondisk_log_size":2751,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:50:24.871641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180289536,"num_objects":2751,"num_object_clones":0,"num_object_copies":8253,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2406,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2751,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2751,"num_write_kb":176064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7],"avail_no_missing":["3","6","7"],"object_location_counts":[{"shards":"3,4,6,7","objects":1},{"shards":"3,4,7","objects":843},{"shards":"3,6,7","objects":1562},{"shards":"3,7","objects":345}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":100,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-08-28T02:27:32.324655+0000","last_change":"2023-08-28T02:27:31.395328+0000","last_active":"2023-08-28T02:27:32.324655+0000","last_peered":"2023-08-28T02:27:32.324655+0000","last_clean":"2023-08-28T02:27:32.324655+0000","last_became_active":"2023-08-28T02:27:31.328228+0000","last_became_peered":"2023-08-28T02:27:31.328228+0000","last_unstale":"2023-08-28T02:27:32.324655+0000","last_undegraded":"2023-08-28T02:27:32.324655+0000","last_fullsized":"2023-08-28T02:27:32.324655+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":241,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-08-28T02:27:32.320226+0000","last_change":"2023-08-28T02:27:31.344915+0000","last_active":"2023-08-28T02:27:32.320226+0000","last_peered":"2023-08-28T02:27:32.320226+0000","last_clean":"2023-08-28T02:27:32.320226+0000","last_became_active":"2023-08-28T02:27:31.344577+0000","last_became_peered":"2023-08-28T02:27:31.344577+0000","last_unstale":"2023-08-28T02:27:32.320226+0000","last_undegraded":"2023-08-28T02:27:32.320226+0000","last_fullsized":"2023-08-28T02:27:32.320226+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"206'2930","reported_seq":2957,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-08-28T02:27:41.965580+0000","last_change":"2023-08-28T02:26:57.392828+0000","last_active":"2023-08-28T02:27:41.965580+0000","last_peered":"2023-08-28T02:27:41.965580+0000","last_clean":"2023-08-28T02:27:41.965580+0000","last_became_active":"2023-08-28T02:26:57.392485+0000","last_became_peered":"2023-08-28T02:26:57.392485+0000","last_unstale":"2023-08-28T02:27:41.965580+0000","last_undegraded":"2023-08-28T02:27:41.965580+0000","last_fullsized":"2023-08-28T02:27:41.965580+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":2930,"log_dups_size":0,"ondisk_log_size":2930,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192020480,"num_objects":2930,"num_object_clones":0,"num_object_copies":8790,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2930,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2930,"num_write_kb":187520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":87,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-08-28T02:27:32.324630+0000","last_change":"2023-08-28T02:27:17.390708+0000","last_active":"2023-08-28T02:27:32.324630+0000","last_peered":"2023-08-28T02:27:32.324630+0000","last_clean":"2023-08-28T02:27:32.324630+0000","last_became_active":"2023-08-28T02:27:17.340738+0000","last_became_peered":"2023-08-28T02:27:17.340738+0000","last_unstale":"2023-08-28T02:27:32.324630+0000","last_undegraded":"2023-08-28T02:27:32.324630+0000","last_fullsized":"2023-08-28T02:27:32.324630+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":12,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-08-28T02:27:32.315320+0000","last_change":"2023-08-28T02:27:31.327251+0000","last_active":"2023-08-28T02:27:32.315320+0000","last_peered":"2023-08-28T02:27:32.315320+0000","last_clean":"2023-08-28T02:27:32.315320+0000","last_became_active":"2023-08-28T02:27:31.326910+0000","last_became_peered":"2023-08-28T02:27:31.326910+0000","last_unstale":"2023-08-28T02:27:32.315320+0000","last_undegraded":"2023-08-28T02:27:32.315320+0000","last_fullsized":"2023-08-28T02:27:32.315320+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:46:06.080137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.9","version":"205'2774","reported_seq":2801,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-08-28T02:27:38.828190+0000","last_change":"2023-08-28T02:26:57.392442+0000","last_active":"2023-08-28T02:27:38.828190+0000","last_peered":"2023-08-28T02:27:38.828190+0000","last_clean":"2023-08-28T02:27:38.828190+0000","last_became_active":"2023-08-28T02:26:57.392139+0000","last_became_peered":"2023-08-28T02:26:57.392139+0000","last_unstale":"2023-08-28T02:27:38.828190+0000","last_undegraded":"2023-08-28T02:27:38.828190+0000","last_fullsized":"2023-08-28T02:27:38.828190+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":2774,"log_dups_size":0,"ondisk_log_size":2774,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:49:35.095582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181796864,"num_objects":2774,"num_object_clones":0,"num_object_copies":8322,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2774,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2774,"num_write_kb":177536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":208,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-08-28T02:27:32.323383+0000","last_change":"2023-08-28T02:27:31.344348+0000","last_active":"2023-08-28T02:27:32.323383+0000","last_peered":"2023-08-28T02:27:32.323383+0000","last_clean":"2023-08-28T02:27:32.323383+0000","last_became_active":"2023-08-28T02:27:31.344003+0000","last_became_peered":"2023-08-28T02:27:31.344003+0000","last_unstale":"2023-08-28T02:27:32.323383+0000","last_undegraded":"2023-08-28T02:27:32.323383+0000","last_fullsized":"2023-08-28T02:27:32.323383+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":184,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-08-28T02:27:32.317729+0000","last_change":"2023-08-28T02:27:31.319783+0000","last_active":"2023-08-28T02:27:32.317729+0000","last_peered":"2023-08-28T02:27:32.317729+0000","last_clean":"2023-08-28T02:27:32.317729+0000","last_became_active":"2023-08-28T02:27:31.319451+0000","last_became_peered":"2023-08-28T02:27:31.319451+0000","last_unstale":"2023-08-28T02:27:32.317729+0000","last_undegraded":"2023-08-28T02:27:32.317729+0000","last_fullsized":"2023-08-28T02:27:32.317729+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:20:37.097889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"206'2943","reported_seq":2987,"reported_epoch":206,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:27:42.172288+0000","last_change":"2023-08-28T02:27:32.432780+0000","last_active":"2023-08-28T02:27:42.172288+0000","last_peered":"2023-08-28T02:27:42.172288+0000","last_clean":"2023-08-28T02:27:30.498067+0000","last_became_active":"2023-08-28T02:27:32.427260+0000","last_became_peered":"2023-08-28T02:27:32.427260+0000","last_unstale":"2023-08-28T02:27:42.172288+0000","last_undegraded":"2023-08-28T02:27:32.333766+0000","last_fullsized":"2023-08-28T02:27:32.320843+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":2943,"log_dups_size":0,"ondisk_log_size":2943,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192872448,"num_objects":2943,"num_object_clones":0,"num_object_copies":8829,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2409,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2943,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2943,"num_write_kb":188352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,2],"avail_no_missing":["4","2","5"],"object_location_counts":[{"shards":"2,4","objects":534},{"shards":"2,4,5","objects":2409}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":67,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-08-28T02:27:31.306200+0000","last_change":"2023-08-28T02:23:59.230059+0000","last_active":"2023-08-28T02:27:31.306200+0000","last_peered":"2023-08-28T02:27:31.306200+0000","last_clean":"2023-08-28T02:27:31.306200+0000","last_became_active":"2023-08-28T02:23:59.217783+0000","last_became_peered":"2023-08-28T02:23:59.217783+0000","last_unstale":"2023-08-28T02:27:31.306200+0000","last_undegraded":"2023-08-28T02:27:31.306200+0000","last_fullsized":"2023-08-28T02:27:31.306200+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:00:12.145395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":217,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-08-28T02:27:32.320426+0000","last_change":"2023-08-28T02:27:31.344966+0000","last_active":"2023-08-28T02:27:32.320426+0000","last_peered":"2023-08-28T02:27:32.320426+0000","last_clean":"2023-08-28T02:27:32.320426+0000","last_became_active":"2023-08-28T02:27:31.344667+0000","last_became_peered":"2023-08-28T02:27:31.344667+0000","last_unstale":"2023-08-28T02:27:32.320426+0000","last_undegraded":"2023-08-28T02:27:32.320426+0000","last_fullsized":"2023-08-28T02:27:32.320426+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"205'2783","reported_seq":2810,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-08-28T02:27:39.036366+0000","last_change":"2023-08-28T02:26:57.390417+0000","last_active":"2023-08-28T02:27:39.036366+0000","last_peered":"2023-08-28T02:27:39.036366+0000","last_clean":"2023-08-28T02:27:39.036366+0000","last_became_active":"2023-08-28T02:26:57.390124+0000","last_became_peered":"2023-08-28T02:26:57.390124+0000","last_unstale":"2023-08-28T02:27:39.036366+0000","last_undegraded":"2023-08-28T02:27:39.036366+0000","last_fullsized":"2023-08-28T02:27:39.036366+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":2783,"log_dups_size":0,"ondisk_log_size":2783,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:38:48.957967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182386688,"num_objects":2783,"num_object_clones":0,"num_object_copies":8349,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2783,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2783,"num_write_kb":178112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":217,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-08-28T02:27:32.320451+0000","last_change":"2023-08-28T02:27:31.344854+0000","last_active":"2023-08-28T02:27:32.320451+0000","last_peered":"2023-08-28T02:27:32.320451+0000","last_clean":"2023-08-28T02:27:32.320451+0000","last_became_active":"2023-08-28T02:27:31.344535+0000","last_became_peered":"2023-08-28T02:27:31.344535+0000","last_unstale":"2023-08-28T02:27:32.320451+0000","last_undegraded":"2023-08-28T02:27:32.320451+0000","last_fullsized":"2023-08-28T02:27:32.320451+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":252,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-08-28T02:27:32.320446+0000","last_change":"2023-08-28T02:27:31.358436+0000","last_active":"2023-08-28T02:27:32.320446+0000","last_peered":"2023-08-28T02:27:32.320446+0000","last_clean":"2023-08-28T02:27:32.320446+0000","last_became_active":"2023-08-28T02:27:31.358185+0000","last_became_peered":"2023-08-28T02:27:31.358185+0000","last_unstale":"2023-08-28T02:27:32.320446+0000","last_undegraded":"2023-08-28T02:27:32.320446+0000","last_fullsized":"2023-08-28T02:27:32.320446+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"205'2855","reported_seq":6049,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-08-28T02:27:38.915322+0000","last_change":"2023-08-28T02:27:32.338654+0000","last_active":"2023-08-28T02:27:38.915322+0000","last_peered":"2023-08-28T02:27:38.915322+0000","last_clean":"2023-08-28T02:27:38.915322+0000","last_became_active":"2023-08-28T02:27:32.336003+0000","last_became_peered":"2023-08-28T02:27:32.336003+0000","last_unstale":"2023-08-28T02:27:38.915322+0000","last_undegraded":"2023-08-28T02:27:38.915322+0000","last_fullsized":"2023-08-28T02:27:38.915322+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":2855,"log_dups_size":0,"ondisk_log_size":2855,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T14:06:21.191989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187105280,"num_objects":2855,"num_object_clones":0,"num_object_copies":8565,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2855,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2855,"num_write_kb":182720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1574,"num_bytes_recovered":103153664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":115,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-08-28T02:27:32.320324+0000","last_change":"2023-08-28T02:23:38.347043+0000","last_active":"2023-08-28T02:27:32.320324+0000","last_peered":"2023-08-28T02:27:32.320324+0000","last_clean":"2023-08-28T02:27:32.320324+0000","last_became_active":"2023-08-28T02:23:38.346680+0000","last_became_peered":"2023-08-28T02:23:38.346680+0000","last_unstale":"2023-08-28T02:27:32.320324+0000","last_undegraded":"2023-08-28T02:27:32.320324+0000","last_fullsized":"2023-08-28T02:27:32.320324+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":161,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-08-28T02:27:31.349323+0000","last_change":"2023-08-28T02:27:31.336931+0000","last_active":"2023-08-28T02:27:31.349323+0000","last_peered":"2023-08-28T02:27:31.349323+0000","last_clean":"2023-08-28T02:27:31.349323+0000","last_became_active":"2023-08-28T02:27:31.336619+0000","last_became_peered":"2023-08-28T02:27:31.336619+0000","last_unstale":"2023-08-28T02:27:31.349323+0000","last_undegraded":"2023-08-28T02:27:31.349323+0000","last_fullsized":"2023-08-28T02:27:31.349323+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:38:49.935193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.d","version":"206'2875","reported_seq":2922,"reported_epoch":206,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:27:42.202371+0000","last_change":"2023-08-28T02:27:32.436317+0000","last_active":"2023-08-28T02:27:42.202371+0000","last_peered":"2023-08-28T02:27:42.202371+0000","last_clean":"2023-08-28T02:27:30.522858+0000","last_became_active":"2023-08-28T02:27:32.431857+0000","last_became_peered":"2023-08-28T02:27:32.431857+0000","last_unstale":"2023-08-28T02:27:42.202371+0000","last_undegraded":"2023-08-28T02:27:32.377766+0000","last_fullsized":"2023-08-28T02:27:32.321737+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":2875,"log_dups_size":0,"ondisk_log_size":2875,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188416000,"num_objects":2875,"num_object_clones":0,"num_object_copies":8625,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2311,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2875,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2875,"num_write_kb":184000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,0],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"0,2,5","objects":2311},{"shards":"0,5","objects":564}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":81,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-08-28T02:27:31.332976+0000","last_change":"2023-08-28T02:27:31.325260+0000","last_active":"2023-08-28T02:27:31.332976+0000","last_peered":"2023-08-28T02:27:31.332976+0000","last_clean":"2023-08-28T02:27:31.332976+0000","last_became_active":"2023-08-28T02:27:31.324950+0000","last_became_peered":"2023-08-28T02:27:31.324950+0000","last_unstale":"2023-08-28T02:27:31.332976+0000","last_undegraded":"2023-08-28T02:27:31.332976+0000","last_fullsized":"2023-08-28T02:27:31.332976+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:21:33.833261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":12,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-08-28T02:27:32.315504+0000","last_change":"2023-08-28T02:27:31.322118+0000","last_active":"2023-08-28T02:27:32.315504+0000","last_peered":"2023-08-28T02:27:32.315504+0000","last_clean":"2023-08-28T02:27:32.315504+0000","last_became_active":"2023-08-28T02:27:31.321338+0000","last_became_peered":"2023-08-28T02:27:31.321338+0000","last_unstale":"2023-08-28T02:27:32.315504+0000","last_undegraded":"2023-08-28T02:27:32.315504+0000","last_fullsized":"2023-08-28T02:27:32.315504+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:56:14.210571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"205'2724","reported_seq":2751,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-08-28T02:27:38.830958+0000","last_change":"2023-08-28T02:26:57.389358+0000","last_active":"2023-08-28T02:27:38.830958+0000","last_peered":"2023-08-28T02:27:38.830958+0000","last_clean":"2023-08-28T02:27:38.830958+0000","last_became_active":"2023-08-28T02:26:57.389088+0000","last_became_peered":"2023-08-28T02:26:57.389088+0000","last_unstale":"2023-08-28T02:27:38.830958+0000","last_undegraded":"2023-08-28T02:27:38.830958+0000","last_fullsized":"2023-08-28T02:27:38.830958+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":2724,"log_dups_size":0,"ondisk_log_size":2724,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:28:16.594323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178520064,"num_objects":2724,"num_object_clones":0,"num_object_copies":8172,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2724,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2724,"num_write_kb":174336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"78'126","reported_seq":586,"reported_epoch":205,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:27:32.326632+0000","last_change":"2023-08-28T02:27:32.326812+0000","last_active":"2023-08-28T02:27:32.326632+0000","last_peered":"2023-08-28T02:27:32.326632+0000","last_clean":"2023-08-28T02:27:29.502868+0000","last_became_active":"2023-08-28T02:27:32.326341+0000","last_became_peered":"2023-08-28T02:27:32.326341+0000","last_unstale":"2023-08-28T02:27:32.326632+0000","last_undegraded":"2023-08-28T02:27:32.317152+0000","last_fullsized":"2023-08-28T02:27:32.316588+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:33:05.480437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":743,"num_read_kb":1331,"num_write":125,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[0,7],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":205,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-08-28T02:27:32.378744+0000","last_change":"2023-08-28T02:23:59.207148+0000","last_active":"2023-08-28T02:27:32.378744+0000","last_peered":"2023-08-28T02:27:32.378744+0000","last_clean":"2023-08-28T02:27:32.378744+0000","last_became_active":"2023-08-28T02:23:59.206802+0000","last_became_peered":"2023-08-28T02:23:59.206802+0000","last_unstale":"2023-08-28T02:27:32.378744+0000","last_undegraded":"2023-08-28T02:27:32.378744+0000","last_fullsized":"2023-08-28T02:27:32.378744+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":12,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-08-28T02:27:32.315134+0000","last_change":"2023-08-28T02:27:31.328027+0000","last_active":"2023-08-28T02:27:32.315134+0000","last_peered":"2023-08-28T02:27:32.315134+0000","last_clean":"2023-08-28T02:27:32.315134+0000","last_became_active":"2023-08-28T02:27:31.326529+0000","last_became_peered":"2023-08-28T02:27:31.326529+0000","last_unstale":"2023-08-28T02:27:32.315134+0000","last_undegraded":"2023-08-28T02:27:32.315134+0000","last_fullsized":"2023-08-28T02:27:32.315134+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:56:57.331355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"205'2788","reported_seq":2815,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-08-28T02:27:38.940575+0000","last_change":"2023-08-28T02:26:57.392533+0000","last_active":"2023-08-28T02:27:38.940575+0000","last_peered":"2023-08-28T02:27:38.940575+0000","last_clean":"2023-08-28T02:27:38.940575+0000","last_became_active":"2023-08-28T02:26:57.392242+0000","last_became_peered":"2023-08-28T02:26:57.392242+0000","last_unstale":"2023-08-28T02:27:38.940575+0000","last_undegraded":"2023-08-28T02:27:38.940575+0000","last_fullsized":"2023-08-28T02:27:38.940575+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":2788,"log_dups_size":0,"ondisk_log_size":2788,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:46:20.394577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182714368,"num_objects":2788,"num_object_clones":0,"num_object_copies":8364,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2788,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2788,"num_write_kb":178432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"78'204","reported_seq":1231,"reported_epoch":205,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:27:32.336234+0000","last_change":"2023-08-28T02:27:32.336234+0000","last_active":"2023-08-28T02:27:32.336234+0000","last_peered":"2023-08-28T02:27:32.336234+0000","last_clean":"2023-08-28T02:27:29.493374+0000","last_became_active":"2023-08-28T02:27:32.335554+0000","last_became_peered":"2023-08-28T02:27:32.335554+0000","last_unstale":"2023-08-28T02:27:32.336234+0000","last_undegraded":"2023-08-28T02:27:32.322576+0000","last_fullsized":"2023-08-28T02:27:32.321714+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":204,"log_dups_size":0,"ondisk_log_size":204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1521,"num_read_kb":2511,"num_write":178,"num_write_kb":1175,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,0],"avail_no_missing":["5","0","3"],"object_location_counts":[{"shards":"0,3,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":191,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-08-28T02:27:32.320594+0000","last_change":"2023-08-28T02:27:31.319815+0000","last_active":"2023-08-28T02:27:32.320594+0000","last_peered":"2023-08-28T02:27:32.320594+0000","last_clean":"2023-08-28T02:27:32.320594+0000","last_became_active":"2023-08-28T02:27:31.319328+0000","last_became_peered":"2023-08-28T02:27:31.319328+0000","last_unstale":"2023-08-28T02:27:32.320594+0000","last_undegraded":"2023-08-28T02:27:32.320594+0000","last_fullsized":"2023-08-28T02:27:32.320594+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":12,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-08-28T02:27:32.315396+0000","last_change":"2023-08-28T02:27:31.376755+0000","last_active":"2023-08-28T02:27:32.315396+0000","last_peered":"2023-08-28T02:27:32.315396+0000","last_clean":"2023-08-28T02:27:32.315396+0000","last_became_active":"2023-08-28T02:27:31.373279+0000","last_became_peered":"2023-08-28T02:27:31.373279+0000","last_unstale":"2023-08-28T02:27:32.315396+0000","last_undegraded":"2023-08-28T02:27:32.315396+0000","last_fullsized":"2023-08-28T02:27:32.315396+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:26:14.628941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"206'2998","reported_seq":3520,"reported_epoch":206,"state":"active+recovering+undersized+remapped","last_fresh":"2023-08-28T02:27:41.974261+0000","last_change":"2023-08-28T02:27:31.426947+0000","last_active":"2023-08-28T02:27:41.974261+0000","last_peered":"2023-08-28T02:27:41.974261+0000","last_clean":"2023-08-28T02:27:16.315756+0000","last_became_active":"2023-08-28T02:27:18.427477+0000","last_became_peered":"2023-08-28T02:27:18.427477+0000","last_unstale":"2023-08-28T02:27:41.974261+0000","last_undegraded":"2023-08-28T02:27:41.974261+0000","last_fullsized":"2023-08-28T02:27:18.318144+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":2998,"log_dups_size":0,"ondisk_log_size":2998,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196476928,"num_objects":2998,"num_object_clones":0,"num_object_copies":8994,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1293,"num_objects_unfound":0,"num_objects_dirty":2998,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2998,"num_write_kb":191872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":244,"num_bytes_recovered":15990784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3],"avail_no_missing":["7","3","6"],"object_location_counts":[{"shards":"3,6,7","objects":1293},{"shards":"3,7","objects":1705}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'47","reported_seq":420,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-08-28T02:27:31.366298+0000","last_change":"2023-08-28T02:24:05.799647+0000","last_active":"2023-08-28T02:27:31.366298+0000","last_peered":"2023-08-28T02:27:31.366298+0000","last_clean":"2023-08-28T02:27:31.366298+0000","last_became_active":"2023-08-28T02:23:59.210481+0000","last_became_peered":"2023-08-28T02:23:59.210481+0000","last_unstale":"2023-08-28T02:27:31.366298+0000","last_undegraded":"2023-08-28T02:27:31.366298+0000","last_fullsized":"2023-08-28T02:27:31.366298+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":47,"log_dups_size":0,"ondisk_log_size":47,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:40:18.542375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":162,"num_read_kb":1281,"num_write":48,"num_write_kb":1153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":12,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-08-28T02:27:32.315077+0000","last_change":"2023-08-28T02:27:31.381307+0000","last_active":"2023-08-28T02:27:32.315077+0000","last_peered":"2023-08-28T02:27:32.315077+0000","last_clean":"2023-08-28T02:27:32.315077+0000","last_became_active":"2023-08-28T02:27:31.380937+0000","last_became_peered":"2023-08-28T02:27:31.380937+0000","last_unstale":"2023-08-28T02:27:32.315077+0000","last_undegraded":"2023-08-28T02:27:32.315077+0000","last_fullsized":"2023-08-28T02:27:32.315077+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:30:16.960846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":12,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-08-28T02:27:32.315580+0000","last_change":"2023-08-28T02:27:31.322967+0000","last_active":"2023-08-28T02:27:32.315580+0000","last_peered":"2023-08-28T02:27:32.315580+0000","last_clean":"2023-08-28T02:27:32.315580+0000","last_became_active":"2023-08-28T02:27:31.322655+0000","last_became_peered":"2023-08-28T02:27:31.322655+0000","last_unstale":"2023-08-28T02:27:32.315580+0000","last_undegraded":"2023-08-28T02:27:32.315580+0000","last_fullsized":"2023-08-28T02:27:32.315580+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:05:15.868115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"206'2939","reported_seq":2965,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-08-28T02:27:42.212128+0000","last_change":"2023-08-28T02:26:57.390571+0000","last_active":"2023-08-28T02:27:42.212128+0000","last_peered":"2023-08-28T02:27:42.212128+0000","last_clean":"2023-08-28T02:27:42.212128+0000","last_became_active":"2023-08-28T02:26:57.390276+0000","last_became_peered":"2023-08-28T02:26:57.390276+0000","last_unstale":"2023-08-28T02:27:42.212128+0000","last_undegraded":"2023-08-28T02:27:42.212128+0000","last_fullsized":"2023-08-28T02:27:42.212128+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":2939,"log_dups_size":0,"ondisk_log_size":2939,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192610304,"num_objects":2939,"num_object_clones":0,"num_object_copies":8817,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2939,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2939,"num_write_kb":188096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"78'114","reported_seq":569,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-08-28T02:27:32.322659+0000","last_change":"2023-08-28T02:24:01.210739+0000","last_active":"2023-08-28T02:27:32.322659+0000","last_peered":"2023-08-28T02:27:32.322659+0000","last_clean":"2023-08-28T02:27:32.322659+0000","last_became_active":"2023-08-28T02:24:01.210386+0000","last_became_peered":"2023-08-28T02:24:01.210386+0000","last_unstale":"2023-08-28T02:27:32.322659+0000","last_undegraded":"2023-08-28T02:27:32.322659+0000","last_fullsized":"2023-08-28T02:27:32.322659+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":114,"log_dups_size":0,"ondisk_log_size":114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":358,"num_read_kb":1352,"num_write":102,"num_write_kb":1104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"86'139","reported_seq":394,"reported_epoch":205,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:27:32.344560+0000","last_change":"2023-08-28T02:27:32.344560+0000","last_active":"2023-08-28T02:27:32.344560+0000","last_peered":"2023-08-28T02:27:32.344560+0000","last_clean":"2023-08-28T02:27:29.492330+0000","last_became_active":"2023-08-28T02:27:32.344137+0000","last_became_peered":"2023-08-28T02:27:32.344137+0000","last_unstale":"2023-08-28T02:27:32.344560+0000","last_undegraded":"2023-08-28T02:27:32.330982+0000","last_fullsized":"2023-08-28T02:27:32.330358+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_clean_scrub_stamp":"2023-08-28T02:08:02.043718+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0],"avail_no_missing":["7","0","2"],"object_location_counts":[{"shards":"0,2,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":72,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-08-28T02:27:31.304983+0000","last_change":"2023-08-28T02:27:17.325562+0000","last_active":"2023-08-28T02:27:31.304983+0000","last_peered":"2023-08-28T02:27:31.304983+0000","last_clean":"2023-08-28T02:27:31.304983+0000","last_became_active":"2023-08-28T02:27:17.324666+0000","last_became_peered":"2023-08-28T02:27:17.324666+0000","last_unstale":"2023-08-28T02:27:31.304983+0000","last_undegraded":"2023-08-28T02:27:31.304983+0000","last_fullsized":"2023-08-28T02:27:31.304983+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:02:33.330219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":125,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-08-28T02:27:32.320583+0000","last_change":"2023-08-28T02:27:17.333753+0000","last_active":"2023-08-28T02:27:32.320583+0000","last_peered":"2023-08-28T02:27:32.320583+0000","last_clean":"2023-08-28T02:27:32.320583+0000","last_became_active":"2023-08-28T02:27:17.333436+0000","last_became_peered":"2023-08-28T02:27:17.333436+0000","last_unstale":"2023-08-28T02:27:32.320583+0000","last_undegraded":"2023-08-28T02:27:32.320583+0000","last_fullsized":"2023-08-28T02:27:32.320583+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.4","version":"205'2781","reported_seq":5795,"reported_epoch":205,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:27:38.881554+0000","last_change":"2023-08-28T02:27:32.429684+0000","last_active":"2023-08-28T02:27:38.881554+0000","last_peered":"2023-08-28T02:27:38.881554+0000","last_clean":"2023-08-28T02:27:30.461946+0000","last_became_active":"2023-08-28T02:27:32.423303+0000","last_became_peered":"2023-08-28T02:27:32.423303+0000","last_unstale":"2023-08-28T02:27:38.881554+0000","last_undegraded":"2023-08-28T02:27:32.331232+0000","last_fullsized":"2023-08-28T02:27:32.317727+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":2781,"log_dups_size":0,"ondisk_log_size":2781,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:53:53.647889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182255616,"num_objects":2781,"num_object_clones":0,"num_object_copies":8343,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2385,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2781,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2781,"num_write_kb":177984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1485,"num_bytes_recovered":97320960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[3,5],"avail_no_missing":["3","4","5","6"],"object_location_counts":[{"shards":"3,4,5","objects":2385},{"shards":"3,5","objects":396}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"78'148","reported_seq":703,"reported_epoch":205,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:27:32.344664+0000","last_change":"2023-08-28T02:27:32.344664+0000","last_active":"2023-08-28T02:27:32.344664+0000","last_peered":"2023-08-28T02:27:32.344664+0000","last_clean":"2023-08-28T02:27:29.492264+0000","last_became_active":"2023-08-28T02:27:32.344175+0000","last_became_peered":"2023-08-28T02:27:32.344175+0000","last_unstale":"2023-08-28T02:27:32.344664+0000","last_undegraded":"2023-08-28T02:27:32.331999+0000","last_fullsized":"2023-08-28T02:27:32.331438+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1092,"num_read_kb":1955,"num_write":127,"num_write_kb":5177,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,0],"avail_no_missing":["7","0","3"],"object_location_counts":[{"shards":"0,3,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":172,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-08-28T02:27:31.309871+0000","last_change":"2023-08-28T02:23:59.226986+0000","last_active":"2023-08-28T02:27:31.309871+0000","last_peered":"2023-08-28T02:27:31.309871+0000","last_clean":"2023-08-28T02:27:31.309871+0000","last_became_active":"2023-08-28T02:23:59.226529+0000","last_became_peered":"2023-08-28T02:23:59.226529+0000","last_unstale":"2023-08-28T02:27:31.309871+0000","last_undegraded":"2023-08-28T02:27:31.309871+0000","last_fullsized":"2023-08-28T02:27:31.309871+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:22:09.193619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":217,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-08-28T02:27:31.372418+0000","last_change":"2023-08-28T02:27:31.361298+0000","last_active":"2023-08-28T02:27:31.372418+0000","last_peered":"2023-08-28T02:27:31.372418+0000","last_clean":"2023-08-28T02:27:31.372418+0000","last_became_active":"2023-08-28T02:27:31.360939+0000","last_became_peered":"2023-08-28T02:27:31.360939+0000","last_unstale":"2023-08-28T02:27:31.372418+0000","last_undegraded":"2023-08-28T02:27:31.372418+0000","last_fullsized":"2023-08-28T02:27:31.372418+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:48:42.024920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"205'2827","reported_seq":3805,"reported_epoch":205,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-08-28T02:27:39.016468+0000","last_change":"2023-08-28T02:27:31.478787+0000","last_active":"2023-08-28T02:27:39.016468+0000","last_peered":"2023-08-28T02:27:39.016468+0000","last_clean":"2023-08-28T02:27:16.286032+0000","last_became_active":"2023-08-28T02:27:31.457399+0000","last_became_peered":"2023-08-28T02:27:31.457399+0000","last_unstale":"2023-08-28T02:27:39.016468+0000","last_undegraded":"2023-08-28T02:27:31.364363+0000","last_fullsized":"2023-08-28T02:27:31.325243+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":2827,"log_dups_size":0,"ondisk_log_size":2827,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:21:22.811563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185270272,"num_objects":2827,"num_object_clones":0,"num_object_copies":8481,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":2002,"num_objects_unfound":0,"num_objects_dirty":2827,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2827,"num_write_kb":180928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":470,"num_bytes_recovered":30801920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[2,7],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"0,2,7","objects":892},{"shards":"2,6,7","objects":1110},{"shards":"2,7","objects":824}],"blocked_by":[],"up_primary":1,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"78'103","reported_seq":581,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-08-28T02:27:32.324526+0000","last_change":"2023-08-28T02:24:07.212128+0000","last_active":"2023-08-28T02:27:32.324526+0000","last_peered":"2023-08-28T02:27:32.324526+0000","last_clean":"2023-08-28T02:27:32.324526+0000","last_became_active":"2023-08-28T02:24:07.211562+0000","last_became_peered":"2023-08-28T02:24:07.211562+0000","last_unstale":"2023-08-28T02:27:32.324526+0000","last_undegraded":"2023-08-28T02:27:32.324526+0000","last_fullsized":"2023-08-28T02:27:32.324526+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":803,"num_read_kb":2080,"num_write":73,"num_write_kb":1244,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":12,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-08-28T02:27:32.319107+0000","last_change":"2023-08-28T02:27:31.377042+0000","last_active":"2023-08-28T02:27:32.319107+0000","last_peered":"2023-08-28T02:27:32.319107+0000","last_clean":"2023-08-28T02:27:32.319107+0000","last_became_active":"2023-08-28T02:27:31.373709+0000","last_became_peered":"2023-08-28T02:27:31.373709+0000","last_unstale":"2023-08-28T02:27:32.319107+0000","last_undegraded":"2023-08-28T02:27:32.319107+0000","last_fullsized":"2023-08-28T02:27:32.319107+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:12:38.286332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":158,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-08-28T02:27:31.312875+0000","last_change":"2023-08-28T02:23:38.329667+0000","last_active":"2023-08-28T02:27:31.312875+0000","last_peered":"2023-08-28T02:27:31.312875+0000","last_clean":"2023-08-28T02:27:31.312875+0000","last_became_active":"2023-08-28T02:23:38.329175+0000","last_became_peered":"2023-08-28T02:23:38.329175+0000","last_unstale":"2023-08-28T02:27:31.312875+0000","last_undegraded":"2023-08-28T02:27:31.312875+0000","last_fullsized":"2023-08-28T02:27:31.312875+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:21:34.815067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"206'2972","reported_seq":2999,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-08-28T02:27:42.153756+0000","last_change":"2023-08-28T02:26:57.390608+0000","last_active":"2023-08-28T02:27:42.153756+0000","last_peered":"2023-08-28T02:27:42.153756+0000","last_clean":"2023-08-28T02:27:42.153756+0000","last_became_active":"2023-08-28T02:26:57.390279+0000","last_became_peered":"2023-08-28T02:26:57.390279+0000","last_unstale":"2023-08-28T02:27:42.153756+0000","last_undegraded":"2023-08-28T02:27:42.153756+0000","last_fullsized":"2023-08-28T02:27:42.153756+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":2972,"log_dups_size":0,"ondisk_log_size":2972,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194772992,"num_objects":2972,"num_object_clones":0,"num_object_copies":8916,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2972,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2972,"num_write_kb":190208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":67,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-08-28T02:27:31.306788+0000","last_change":"2023-08-28T02:23:59.232099+0000","last_active":"2023-08-28T02:27:31.306788+0000","last_peered":"2023-08-28T02:27:31.306788+0000","last_clean":"2023-08-28T02:27:31.306788+0000","last_became_active":"2023-08-28T02:23:59.219318+0000","last_became_peered":"2023-08-28T02:23:59.219318+0000","last_unstale":"2023-08-28T02:27:31.306788+0000","last_undegraded":"2023-08-28T02:27:31.306788+0000","last_fullsized":"2023-08-28T02:27:31.306788+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:59:30.093289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'112","reported_seq":335,"reported_epoch":205,"state":"active+recovery_wait+remapped","last_fresh":"2023-08-28T02:27:32.336743+0000","last_change":"2023-08-28T02:27:32.336743+0000","last_active":"2023-08-28T02:27:32.336743+0000","last_peered":"2023-08-28T02:27:32.336743+0000","last_clean":"2023-08-28T02:12:45.636147+0000","last_became_active":"2023-08-28T02:27:32.336342+0000","last_became_peered":"2023-08-28T02:27:32.336342+0000","last_unstale":"2023-08-28T02:27:32.336743+0000","last_undegraded":"2023-08-28T02:27:32.336743+0000","last_fullsized":"2023-08-28T02:27:32.336743+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:27:30.669474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":9,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":444,"num_read_kb":473,"num_write":94,"num_write_kb":127,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[0,4,3],"avail_no_missing":["0","3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":63,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-08-28T02:27:31.303935+0000","last_change":"2023-08-28T02:23:59.221034+0000","last_active":"2023-08-28T02:27:31.303935+0000","last_peered":"2023-08-28T02:27:31.303935+0000","last_clean":"2023-08-28T02:27:31.303935+0000","last_became_active":"2023-08-28T02:23:59.220730+0000","last_became_peered":"2023-08-28T02:23:59.220730+0000","last_unstale":"2023-08-28T02:27:31.303935+0000","last_undegraded":"2023-08-28T02:27:31.303935+0000","last_fullsized":"2023-08-28T02:27:31.303935+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:25:52.364370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"206'3006","reported_seq":3047,"reported_epoch":206,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:27:41.964308+0000","last_change":"2023-08-28T02:27:31.475333+0000","last_active":"2023-08-28T02:27:41.964308+0000","last_peered":"2023-08-28T02:27:41.964308+0000","last_clean":"2023-08-28T02:27:16.315150+0000","last_became_active":"2023-08-28T02:27:31.471964+0000","last_became_peered":"2023-08-28T02:27:31.471964+0000","last_unstale":"2023-08-28T02:27:41.964308+0000","last_undegraded":"2023-08-28T02:27:31.329293+0000","last_fullsized":"2023-08-28T02:27:31.313675+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":3006,"log_dups_size":0,"ondisk_log_size":3006,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197001216,"num_objects":3006,"num_object_clones":0,"num_object_copies":9018,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2444,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3006,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3006,"num_write_kb":192384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2],"avail_no_missing":["7","2","6"],"object_location_counts":[{"shards":"0,2,7","objects":882},{"shards":"2,6,7","objects":1562},{"shards":"2,7","objects":562}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"78'122","reported_seq":688,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-08-28T02:27:32.324053+0000","last_change":"2023-08-28T02:27:20.402737+0000","last_active":"2023-08-28T02:27:32.324053+0000","last_peered":"2023-08-28T02:27:32.324053+0000","last_clean":"2023-08-28T02:27:32.324053+0000","last_became_active":"2023-08-28T02:27:20.402278+0000","last_became_peered":"2023-08-28T02:27:20.402278+0000","last_unstale":"2023-08-28T02:27:32.324053+0000","last_undegraded":"2023-08-28T02:27:32.324053+0000","last_fullsized":"2023-08-28T02:27:32.324053+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":857,"num_read_kb":2224,"num_write":99,"num_write_kb":2312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":184,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-08-28T02:27:31.425200+0000","last_change":"2023-08-28T02:27:31.424373+0000","last_active":"2023-08-28T02:27:31.425200+0000","last_peered":"2023-08-28T02:27:31.425200+0000","last_clean":"2023-08-28T02:27:31.425200+0000","last_became_active":"2023-08-28T02:27:31.324505+0000","last_became_peered":"2023-08-28T02:27:31.324505+0000","last_unstale":"2023-08-28T02:27:31.425200+0000","last_undegraded":"2023-08-28T02:27:31.425200+0000","last_fullsized":"2023-08-28T02:27:31.425200+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:27:57.837524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":166,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-08-28T02:27:32.320096+0000","last_change":"2023-08-28T02:27:17.329102+0000","last_active":"2023-08-28T02:27:32.320096+0000","last_peered":"2023-08-28T02:27:32.320096+0000","last_clean":"2023-08-28T02:27:32.320096+0000","last_became_active":"2023-08-28T02:27:17.328738+0000","last_became_peered":"2023-08-28T02:27:17.328738+0000","last_unstale":"2023-08-28T02:27:32.320096+0000","last_undegraded":"2023-08-28T02:27:32.320096+0000","last_fullsized":"2023-08-28T02:27:32.320096+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":160,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-08-28T02:27:31.312794+0000","last_change":"2023-08-28T02:23:38.329474+0000","last_active":"2023-08-28T02:27:31.312794+0000","last_peered":"2023-08-28T02:27:31.312794+0000","last_clean":"2023-08-28T02:27:31.312794+0000","last_became_active":"2023-08-28T02:23:38.328505+0000","last_became_peered":"2023-08-28T02:23:38.328505+0000","last_unstale":"2023-08-28T02:27:31.312794+0000","last_undegraded":"2023-08-28T02:27:31.312794+0000","last_fullsized":"2023-08-28T02:27:31.312794+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:12:14.835809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":244,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-08-28T02:27:31.398702+0000","last_change":"2023-08-28T02:27:31.384467+0000","last_active":"2023-08-28T02:27:31.398702+0000","last_peered":"2023-08-28T02:27:31.398702+0000","last_clean":"2023-08-28T02:27:31.398702+0000","last_became_active":"2023-08-28T02:27:31.384183+0000","last_became_peered":"2023-08-28T02:27:31.384183+0000","last_unstale":"2023-08-28T02:27:31.398702+0000","last_undegraded":"2023-08-28T02:27:31.398702+0000","last_fullsized":"2023-08-28T02:27:31.398702+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:59:47.551892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":213,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-08-28T02:27:32.323571+0000","last_change":"2023-08-28T02:27:31.427323+0000","last_active":"2023-08-28T02:27:32.323571+0000","last_peered":"2023-08-28T02:27:32.323571+0000","last_clean":"2023-08-28T02:27:32.323571+0000","last_became_active":"2023-08-28T02:27:31.328152+0000","last_became_peered":"2023-08-28T02:27:31.328152+0000","last_unstale":"2023-08-28T02:27:32.323571+0000","last_undegraded":"2023-08-28T02:27:32.323571+0000","last_fullsized":"2023-08-28T02:27:32.323571+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":223,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-08-28T02:27:32.320734+0000","last_change":"2023-08-28T02:27:31.343538+0000","last_active":"2023-08-28T02:27:32.320734+0000","last_peered":"2023-08-28T02:27:32.320734+0000","last_clean":"2023-08-28T02:27:32.320734+0000","last_became_active":"2023-08-28T02:27:31.343324+0000","last_became_peered":"2023-08-28T02:27:31.343324+0000","last_unstale":"2023-08-28T02:27:32.320734+0000","last_undegraded":"2023-08-28T02:27:32.320734+0000","last_fullsized":"2023-08-28T02:27:32.320734+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":185,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-08-28T02:27:32.378724+0000","last_change":"2023-08-28T02:27:31.346525+0000","last_active":"2023-08-28T02:27:32.378724+0000","last_peered":"2023-08-28T02:27:32.378724+0000","last_clean":"2023-08-28T02:27:32.378724+0000","last_became_active":"2023-08-28T02:27:31.327945+0000","last_became_peered":"2023-08-28T02:27:31.327945+0000","last_unstale":"2023-08-28T02:27:32.378724+0000","last_undegraded":"2023-08-28T02:27:32.378724+0000","last_fullsized":"2023-08-28T02:27:32.378724+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":205,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-08-28T02:27:32.320399+0000","last_change":"2023-08-28T02:23:59.207929+0000","last_active":"2023-08-28T02:27:32.320399+0000","last_peered":"2023-08-28T02:27:32.320399+0000","last_clean":"2023-08-28T02:27:32.320399+0000","last_became_active":"2023-08-28T02:23:59.207589+0000","last_became_peered":"2023-08-28T02:23:59.207589+0000","last_unstale":"2023-08-28T02:27:32.320399+0000","last_undegraded":"2023-08-28T02:27:32.320399+0000","last_fullsized":"2023-08-28T02:27:32.320399+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":81,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-08-28T02:27:31.370958+0000","last_change":"2023-08-28T02:27:31.361242+0000","last_active":"2023-08-28T02:27:31.370958+0000","last_peered":"2023-08-28T02:27:31.370958+0000","last_clean":"2023-08-28T02:27:31.370958+0000","last_became_active":"2023-08-28T02:27:31.360903+0000","last_became_peered":"2023-08-28T02:27:31.360903+0000","last_unstale":"2023-08-28T02:27:31.370958+0000","last_undegraded":"2023-08-28T02:27:31.370958+0000","last_fullsized":"2023-08-28T02:27:31.370958+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:50:54.882883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":63,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-08-28T02:27:31.303850+0000","last_change":"2023-08-28T02:23:59.217794+0000","last_active":"2023-08-28T02:27:31.303850+0000","last_peered":"2023-08-28T02:27:31.303850+0000","last_clean":"2023-08-28T02:27:31.303850+0000","last_became_active":"2023-08-28T02:23:59.217231+0000","last_became_peered":"2023-08-28T02:23:59.217231+0000","last_unstale":"2023-08-28T02:27:31.303850+0000","last_undegraded":"2023-08-28T02:27:31.303850+0000","last_fullsized":"2023-08-28T02:27:31.303850+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:36:26.297410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":158,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-08-28T02:27:31.309853+0000","last_change":"2023-08-28T02:20:19.850252+0000","last_active":"2023-08-28T02:27:31.309853+0000","last_peered":"2023-08-28T02:27:31.309853+0000","last_clean":"2023-08-28T02:27:31.309853+0000","last_became_active":"2023-08-28T02:20:19.849913+0000","last_became_peered":"2023-08-28T02:20:19.849913+0000","last_unstale":"2023-08-28T02:27:31.309853+0000","last_undegraded":"2023-08-28T02:27:31.309853+0000","last_fullsized":"2023-08-28T02:27:31.309853+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:48:47.713465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":228,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-08-28T02:27:32.320095+0000","last_change":"2023-08-28T02:27:31.325339+0000","last_active":"2023-08-28T02:27:32.320095+0000","last_peered":"2023-08-28T02:27:32.320095+0000","last_clean":"2023-08-28T02:27:32.320095+0000","last_became_active":"2023-08-28T02:27:31.324954+0000","last_became_peered":"2023-08-28T02:27:31.324954+0000","last_unstale":"2023-08-28T02:27:32.320095+0000","last_undegraded":"2023-08-28T02:27:32.320095+0000","last_fullsized":"2023-08-28T02:27:32.320095+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":3008626688,"num_objects":45908,"num_object_clones":0,"num_object_copies":137724,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13326,"num_objects_misplaced":3295,"num_objects_unfound":0,"num_objects_dirty":45908,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45908,"num_write_kb":2938112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3952,"num_bytes_recovered":258998272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9178513408,"data_stored":9178513408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":45908,"ondisk_log_size":45908,"up":48,"acting":40,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":14578,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21316,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26810,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":38,"num_objects_misplaced":9,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":131,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12599296,"data_stored":12582988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":21,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430425,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1811692,"kb_used_data":1514468,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":91933460,"statfs":{"total":95995035648,"available":94139863040,"internally_reserved":0,"allocated":1550815232,"data_stored":1550214767,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,3,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:26:58 2023","interfaces":[{"interface":"back","average":{"1min":4.923,"5min":4.5999999999999996,"15min":1.891},"min":{"1min":0.22800000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":59.753999999999998,"5min":116.755,"15min":116.755},"last":2.77},{"interface":"front","average":{"1min":4.8010000000000002,"5min":5.0119999999999996,"15min":2.069},"min":{"1min":0.248,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":59.765999999999998,"5min":116.971,"15min":116.971},"last":0.42699999999999999}]},{"osd":1,"last update":"Mon Aug 28 02:27:06 2023","interfaces":[{"interface":"back","average":{"1min":2.0219999999999998,"5min":1.4359999999999999,"15min":0.83199999999999996},"min":{"1min":0.26000000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":6.8970000000000002,"5min":9.8710000000000004,"15min":9.8710000000000004},"last":2.9830000000000001},{"interface":"front","average":{"1min":2.4239999999999999,"5min":2.1600000000000001,"15min":1.1100000000000001},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":9.6270000000000007,"5min":11.696999999999999,"15min":11.696999999999999},"last":2.2469999999999999}]},{"osd":2,"last update":"Mon Aug 28 02:27:33 2023","interfaces":[{"interface":"back","average":{"1min":2.3769999999999998,"5min":4.5970000000000004,"15min":1.925},"min":{"1min":0.29499999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":12.558999999999999,"5min":119.571,"15min":119.571},"last":0.89700000000000002},{"interface":"front","average":{"1min":3.3839999999999999,"5min":5.8129999999999997,"15min":2.3420000000000001},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":15.951000000000001,"5min":119.554,"15min":119.554},"last":0.88600000000000001}]},{"osd":3,"last update":"Mon Aug 28 02:27:06 2023","interfaces":[{"interface":"back","average":{"1min":2.2269999999999999,"5min":2.637,"15min":1.325},"min":{"1min":0.34599999999999997,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":11.023,"5min":107.916,"15min":107.916},"last":0.41899999999999998},{"interface":"front","average":{"1min":2.1749999999999998,"5min":3.2999999999999998,"15min":1.546},"min":{"1min":0.311,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":9.6489999999999991,"5min":112.00700000000001,"15min":112.00700000000001},"last":3.415}]},{"osd":4,"last update":"Mon Aug 28 02:27:33 2023","interfaces":[{"interface":"back","average":{"1min":1.1879999999999999,"5min":2.6360000000000001,"15min":1.208},"min":{"1min":0.29099999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":9.6709999999999994,"5min":63.048000000000002,"15min":63.048000000000002},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.173,"5min":2.359,"15min":1.133},"min":{"1min":0.31,"5min":0.247,"15min":0.247},"max":{"1min":5.3739999999999997,"5min":62.912999999999997,"15min":62.912999999999997},"last":0.34599999999999997}]},{"osd":5,"last update":"Mon Aug 28 02:27:33 2023","interfaces":[{"interface":"back","average":{"1min":1.248,"5min":2.0430000000000001,"15min":1.016},"min":{"1min":0.34999999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":6.9169999999999998,"5min":63.021000000000001,"15min":63.021000000000001},"last":0.38200000000000001},{"interface":"front","average":{"1min":1.4530000000000001,"5min":2.7320000000000002,"15min":1.304},"min":{"1min":0.36599999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":9.6929999999999996,"5min":63.082999999999998,"15min":63.082999999999998},"last":0.501}]},{"osd":6,"last update":"Mon Aug 28 02:27:33 2023","interfaces":[{"interface":"back","average":{"1min":1.2629999999999999,"5min":2.7010000000000001,"15min":1.266},"min":{"1min":0.38600000000000001,"5min":0.317,"15min":0.29699999999999999},"max":{"1min":8.827,"5min":101.61,"15min":101.61},"last":0.44700000000000001},{"interface":"front","average":{"1min":1.71,"5min":2.948,"15min":1.3899999999999999},"min":{"1min":0.40300000000000002,"5min":0.308,"15min":0.308},"max":{"1min":14.423999999999999,"5min":101.621,"15min":101.621},"last":0.48499999999999999}]}]},{"osd":6,"up_from":0,"seq":180388626652,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":98,"seq":420906795104,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":265224,"kb_used_data":135104,"kb_used_omap":7,"kb_used_meta":130104,"kb_avail":93479928,"statfs":{"total":95995035648,"available":95723446272,"internally_reserved":0,"allocated":138346496,"data_stored":137381388,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":133226827},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:27:01 2023","interfaces":[{"interface":"back","average":{"1min":1.282,"5min":4.6189999999999998,"15min":1.762},"min":{"1min":0.23100000000000001,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":7.0019999999999998,"5min":128.60599999999999,"15min":128.60599999999999},"last":3.3180000000000001},{"interface":"front","average":{"1min":1.9770000000000001,"5min":4.7770000000000001,"15min":1.827},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":9.5299999999999994,"5min":128.685,"15min":128.685},"last":0.23699999999999999}]},{"osd":2,"last update":"Mon Aug 28 02:27:31 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":5.4569999999999999,"15min":2.0710000000000002},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":2.7170000000000001,"5min":135.25999999999999,"15min":135.25999999999999},"last":9.4469999999999992},{"interface":"front","average":{"1min":0.96199999999999997,"5min":5.0460000000000003,"15min":1.9650000000000001},"min":{"1min":0.245,"5min":0.23200000000000001,"15min":0.221},"max":{"1min":2.968,"5min":135.10300000000001,"15min":135.10300000000001},"last":0.85799999999999998}]},{"osd":3,"last update":"Mon Aug 28 02:27:01 2023","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":1.9450000000000001,"15min":0.89100000000000001},"min":{"1min":0.25700000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":6.0979999999999999,"5min":15.395,"15min":15.395},"last":6.7779999999999996},{"interface":"front","average":{"1min":0.77400000000000002,"5min":2.2109999999999999,"15min":1.044},"min":{"1min":0.30399999999999999,"5min":0.254,"15min":0.254},"max":{"1min":6.3760000000000003,"5min":22.361999999999998,"15min":22.361999999999998},"last":9.3249999999999993}]},{"osd":4,"last update":"Mon Aug 28 02:27:01 2023","interfaces":[{"interface":"back","average":{"1min":1.1220000000000001,"5min":2.02,"15min":1.095},"min":{"1min":0.39700000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":4.6040000000000001,"5min":17.600000000000001,"15min":17.600000000000001},"last":3.645},{"interface":"front","average":{"1min":1.27,"5min":1.6910000000000001,"15min":0.97499999999999998},"min":{"1min":0.36699999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":5.9909999999999997,"5min":19.010000000000002,"15min":19.010000000000002},"last":0.58699999999999997}]},{"osd":5,"last update":"Mon Aug 28 02:27:01 2023","interfaces":[{"interface":"back","average":{"1min":1.6180000000000001,"5min":2.2130000000000001,"15min":1.159},"min":{"1min":0.36499999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":7.4359999999999999,"5min":15.544,"15min":15.544},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.77500000000000002,"5min":1.859,"15min":1.0489999999999999},"min":{"1min":0.36699999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":3.2599999999999998,"5min":17.646999999999998,"15min":17.646999999999998},"last":4.008}]},{"osd":6,"last update":"Mon Aug 28 02:27:01 2023","interfaces":[{"interface":"back","average":{"1min":1.8620000000000001,"5min":2.5960000000000001,"15min":1.274},"min":{"1min":0.39700000000000002,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":8.0210000000000008,"5min":15.512,"15min":15.512},"last":0.49199999999999999},{"interface":"front","average":{"1min":1.246,"5min":2.399,"15min":1.212},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.806,"5min":17.673999999999999,"15min":17.673999999999999},"last":2.0009999999999999}]},{"osd":7,"last update":"Mon Aug 28 02:27:01 2023","interfaces":[{"interface":"back","average":{"1min":0.89000000000000001,"5min":1.9039999999999999,"15min":1.054},"min":{"1min":0.38300000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":5.9470000000000001,"5min":19.061,"15min":19.061},"last":0.82699999999999996},{"interface":"front","average":{"1min":1.661,"5min":2.3439999999999999,"15min":1.2050000000000001},"min":{"1min":0.46100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":7.3979999999999997,"5min":15.476000000000001,"15min":15.476000000000001},"last":0.52600000000000002}]}]},{"osd":0,"up_from":90,"seq":386547056750,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1406628,"kb_used_data":1113628,"kb_used_omap":8,"kb_used_meta":292983,"kb_avail":92338524,"statfs":{"total":95995035648,"available":94554648576,"internally_reserved":0,"allocated":1140355072,"data_stored":1139420965,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":300014953},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Aug 28 02:27:12 2023","interfaces":[{"interface":"back","average":{"1min":0.93200000000000005,"5min":2.8690000000000002,"15min":1.1919999999999999},"min":{"1min":0.23300000000000001,"5min":0.184,"15min":0.184},"max":{"1min":3.677,"5min":40.313000000000002,"15min":40.313000000000002},"last":1.0940000000000001},{"interface":"front","average":{"1min":0.73999999999999999,"5min":2.673,"15min":1.1359999999999999},"min":{"1min":0.20999999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":4.5940000000000003,"5min":36.655000000000001,"15min":36.655000000000001},"last":9.577}]},{"osd":2,"last update":"Mon Aug 28 02:27:31 2023","interfaces":[{"interface":"back","average":{"1min":5.0019999999999998,"5min":3.411,"15min":1.4019999999999999},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":71.984999999999999,"5min":71.984999999999999,"15min":71.984999999999999},"last":1.3300000000000001},{"interface":"front","average":{"1min":5.0910000000000002,"5min":4.4260000000000002,"15min":1.7809999999999999},"min":{"1min":0.23400000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":72.816999999999993,"5min":72.816999999999993,"15min":72.816999999999993},"last":9.6600000000000001}]},{"osd":3,"last update":"Mon Aug 28 02:27:12 2023","interfaces":[{"interface":"back","average":{"1min":2.4830000000000001,"5min":2.633,"15min":1.119},"min":{"1min":0.23499999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":11.718999999999999,"5min":18.545999999999999,"15min":18.545999999999999},"last":1.3620000000000001},{"interface":"front","average":{"1min":2.3109999999999999,"5min":2.5139999999999998,"15min":1.1000000000000001},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27200000000000002},"max":{"1min":11.864000000000001,"5min":14.534000000000001,"15min":14.534000000000001},"last":0.41699999999999998}]},{"osd":4,"last update":"Mon Aug 28 02:27:05 2023","interfaces":[{"interface":"back","average":{"1min":1.1379999999999999,"5min":1.9570000000000001,"15min":1.091},"min":{"1min":0.33800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":5.0300000000000002,"5min":9.6649999999999991,"15min":9.6649999999999991},"last":1.216},{"interface":"front","average":{"1min":2.3119999999999998,"5min":2.367,"15min":1.2490000000000001},"min":{"1min":0.436,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":11.805,"5min":12.305,"15min":12.305},"last":0.45100000000000001}]},{"osd":5,"last update":"Mon Aug 28 02:27:05 2023","interfaces":[{"interface":"back","average":{"1min":2.1000000000000001,"5min":2.8210000000000002,"15min":1.365},"min":{"1min":0.32600000000000001,"5min":0.313,"15min":0.313},"max":{"1min":11.754,"5min":44.000999999999998,"15min":44.000999999999998},"last":0.47699999999999998},{"interface":"front","average":{"1min":1.2030000000000001,"5min":2.5390000000000001,"15min":1.274},"min":{"1min":0.376,"5min":0.33100000000000002,"15min":0.30599999999999999},"max":{"1min":5.306,"5min":43.953000000000003,"15min":43.953000000000003},"last":1.252}]},{"osd":6,"last update":"Mon Aug 28 02:27:05 2023","interfaces":[{"interface":"back","average":{"1min":2.1739999999999999,"5min":2.8570000000000002,"15min":1.3799999999999999},"min":{"1min":0.373,"5min":0.36299999999999999,"15min":0.32000000000000001},"max":{"1min":11.831,"5min":32.417999999999999,"15min":32.417999999999999},"last":0.498},{"interface":"front","average":{"1min":1.4690000000000001,"5min":2.448,"15min":1.28},"min":{"1min":0.39000000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":4.9880000000000004,"5min":32.130000000000003,"15min":32.130000000000003},"last":1.3029999999999999}]},{"osd":7,"last update":"Mon Aug 28 02:27:05 2023","interfaces":[{"interface":"back","average":{"1min":1.202,"5min":2.2149999999999999,"15min":1.1950000000000001},"min":{"1min":0.41599999999999998,"5min":0.40799999999999997,"15min":0.40000000000000002},"max":{"1min":4.9560000000000004,"5min":9.6020000000000003,"15min":9.6020000000000003},"last":1.278},{"interface":"front","average":{"1min":1.466,"5min":2.528,"15min":1.3009999999999999},"min":{"1min":0.40300000000000002,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":7.4699999999999998,"5min":19.478999999999999,"15min":19.478999999999999},"last":9.6310000000000002}]}]},{"osd":2,"up_from":104,"seq":446676598874,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1926456,"kb_used_data":1760304,"kb_used_omap":10,"kb_used_meta":166133,"kb_avail":91818696,"statfs":{"total":95995035648,"available":94022344704,"internally_reserved":0,"allocated":1802551296,"data_stored":1801657658,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10792,"internal_metadata":170120664},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,2,1,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:27:34 2023","interfaces":[{"interface":"back","average":{"1min":7.0460000000000003,"5min":5.1909999999999998,"15min":1.9670000000000001},"min":{"1min":0.246,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":113.89100000000001,"5min":113.89100000000001,"15min":113.89100000000001},"last":2.2770000000000001},{"interface":"front","average":{"1min":8.2379999999999995,"5min":8.3840000000000003,"15min":3.0670000000000002},"min":{"1min":0.252,"5min":0.214,"15min":0.19900000000000001},"max":{"1min":114.291,"5min":147.286,"15min":147.286},"last":11.744}]},{"osd":1,"last update":"Mon Aug 28 02:27:34 2023","interfaces":[{"interface":"back","average":{"1min":1.127,"5min":2.9940000000000002,"15min":1.3720000000000001},"min":{"1min":0.26500000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":5.8490000000000002,"5min":16.372,"15min":30.888000000000002},"last":0.371},{"interface":"front","average":{"1min":2.0110000000000001,"5min":2.6099999999999999,"15min":1.218},"min":{"1min":0.252,"5min":0.21299999999999999,"15min":0.20999999999999999},"max":{"1min":11.827999999999999,"5min":20.259,"15min":31.648},"last":3.6840000000000002}]},{"osd":3,"last update":"Mon Aug 28 02:27:04 2023","interfaces":[{"interface":"back","average":{"1min":3.8170000000000002,"5min":4.0129999999999999,"15min":1.609},"min":{"1min":0.27400000000000002,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":14.827999999999999,"5min":128.63900000000001,"15min":128.63900000000001},"last":0.25},{"interface":"front","average":{"1min":3.532,"5min":4.2469999999999999,"15min":1.673},"min":{"1min":0.26500000000000001,"5min":0.245,"15min":0.245},"max":{"1min":14.787000000000001,"5min":128.75700000000001,"15min":128.75700000000001},"last":3.6059999999999999}]},{"osd":4,"last update":"Mon Aug 28 02:27:34 2023","interfaces":[{"interface":"back","average":{"1min":1.9119999999999999,"5min":2.6139999999999999,"15min":1.28},"min":{"1min":0.29799999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":14.254,"5min":17.024999999999999,"15min":17.024999999999999},"last":3.6389999999999998},{"interface":"front","average":{"1min":1.1220000000000001,"5min":1.881,"15min":1.0880000000000001},"min":{"1min":0.33000000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.2969999999999997,"5min":16.936,"15min":16.936},"last":2.323}]},{"osd":5,"last update":"Mon Aug 28 02:27:34 2023","interfaces":[{"interface":"back","average":{"1min":1.744,"5min":3.6869999999999998,"15min":1.661},"min":{"1min":0.38300000000000001,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":10.050000000000001,"5min":128.696,"15min":128.696},"last":0.34300000000000003},{"interface":"front","average":{"1min":1.6779999999999999,"5min":4.0140000000000002,"15min":1.72},"min":{"1min":0.34999999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":10.787000000000001,"5min":128.84100000000001,"15min":128.84100000000001},"last":3.6619999999999999}]},{"osd":6,"last update":"Mon Aug 28 02:27:34 2023","interfaces":[{"interface":"back","average":{"1min":1.1459999999999999,"5min":2.1230000000000002,"15min":1.1319999999999999},"min":{"1min":0.33200000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":6.2679999999999998,"5min":11.285,"15min":11.285},"last":3.4660000000000002},{"interface":"front","average":{"1min":1.3320000000000001,"5min":2.4180000000000001,"15min":1.2450000000000001},"min":{"1min":0.42699999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":6.008,"5min":11.055999999999999,"15min":11.055999999999999},"last":0.39400000000000002}]},{"osd":7,"last update":"Mon Aug 28 02:27:34 2023","interfaces":[{"interface":"back","average":{"1min":1.466,"5min":3.5510000000000002,"15min":1.611},"min":{"1min":0.36399999999999999,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":7.2389999999999999,"5min":128.911,"15min":128.911},"last":2.3519999999999999},{"interface":"front","average":{"1min":2.004,"5min":4.0010000000000003,"15min":1.7689999999999999},"min":{"1min":0.45500000000000002,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":13.281000000000001,"5min":128.72399999999999,"15min":128.72399999999999},"last":0.41399999999999998}]}]},{"osd":3,"up_from":112,"seq":481036337236,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1897808,"kb_used_data":1785928,"kb_used_omap":3,"kb_used_meta":111868,"kb_avail":91847344,"statfs":{"total":95995035648,"available":94051680256,"internally_reserved":0,"allocated":1828790272,"data_stored":1827926807,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":114553762},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:27:02 2023","interfaces":[{"interface":"back","average":{"1min":1.78,"5min":6.1280000000000001,"15min":2.2759999999999998},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":11.845000000000001,"5min":164.25299999999999,"15min":164.25299999999999},"last":0.29399999999999998},{"interface":"front","average":{"1min":1.2809999999999999,"5min":6.2549999999999999,"15min":2.3069999999999999},"min":{"1min":0.22500000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":11.718,"5min":164.55000000000001,"15min":164.55000000000001},"last":1.714}]},{"osd":1,"last update":"Mon Aug 28 02:27:02 2023","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":4.0579999999999998,"15min":1.579},"min":{"1min":0.249,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":3.1640000000000001,"5min":117.34699999999999,"15min":117.34699999999999},"last":2.2509999999999999},{"interface":"front","average":{"1min":1.917,"5min":3.9689999999999999,"15min":1.5760000000000001},"min":{"1min":0.30299999999999999,"5min":0.224,"15min":0.224},"max":{"1min":14.300000000000001,"5min":118.012,"15min":118.012},"last":2.2349999999999999}]},{"osd":2,"last update":"Mon Aug 28 02:27:02 2023","interfaces":[{"interface":"back","average":{"1min":2.1120000000000001,"5min":5.9199999999999999,"15min":2.2229999999999999},"min":{"1min":0.249,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":18.678000000000001,"5min":120.43300000000001,"15min":120.43300000000001},"last":2.2050000000000001},{"interface":"front","average":{"1min":1.8540000000000001,"5min":5.891,"15min":2.2149999999999999},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":18.629999999999999,"5min":120.44499999999999,"15min":120.44499999999999},"last":1.8149999999999999}]},{"osd":4,"last update":"Mon Aug 28 02:27:02 2023","interfaces":[{"interface":"back","average":{"1min":1.272,"5min":1.8100000000000001,"15min":1.022},"min":{"1min":0.32300000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":6.694,"5min":9.016,"15min":9.016},"last":1.7929999999999999},{"interface":"front","average":{"1min":1.827,"5min":1.907,"15min":1.0589999999999999},"min":{"1min":0.41299999999999998,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":11.768000000000001,"5min":11.768000000000001,"15min":11.768000000000001},"last":0.96199999999999997}]},{"osd":5,"last update":"Mon Aug 28 02:27:02 2023","interfaces":[{"interface":"back","average":{"1min":7.4900000000000002,"5min":2.8580000000000001,"15min":1.377},"min":{"1min":0.376,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":102.904,"5min":102.904,"15min":102.904},"last":0.92700000000000005},{"interface":"front","average":{"1min":7.4000000000000004,"5min":2.6869999999999998,"15min":1.3149999999999999},"min":{"1min":0.37,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":103.283,"5min":103.283,"15min":103.283},"last":2.1709999999999998}]},{"osd":6,"last update":"Mon Aug 28 02:27:02 2023","interfaces":[{"interface":"back","average":{"1min":1.298,"5min":2.2759999999999998,"15min":1.1819999999999999},"min":{"1min":0.39700000000000002,"5min":0.314,"15min":0.314},"max":{"1min":6.8369999999999997,"5min":72.116,"15min":72.116},"last":2.1349999999999998},{"interface":"front","average":{"1min":0.59599999999999997,"5min":2.2210000000000001,"15min":1.139},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.8759999999999999,"5min":72.030000000000001,"15min":72.030000000000001},"last":1.956}]},{"osd":7,"last update":"Mon Aug 28 02:27:02 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":1.786,"15min":1.034},"min":{"1min":0.34300000000000003,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":3.0640000000000001,"5min":11.949999999999999,"15min":11.949999999999999},"last":1.917},{"interface":"front","average":{"1min":1.6910000000000001,"5min":2.1160000000000001,"15min":1.1359999999999999},"min":{"1min":0.40999999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":7.2409999999999997,"5min":13.890000000000001,"15min":13.890000000000001},"last":0.99399999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019110,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1446304,"kb_used_data":1149080,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":92298848,"statfs":{"total":95995035648,"available":94514020352,"internally_reserved":0,"allocated":1176657920,"data_stored":1176069107,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:27:00 2023","interfaces":[{"interface":"back","average":{"1min":2.665,"5min":6.2320000000000002,"15min":2.4769999999999999},"min":{"1min":0.252,"5min":0.223,"15min":0.223},"max":{"1min":13.374000000000001,"5min":138.67400000000001,"15min":138.67400000000001},"last":0.28599999999999998},{"interface":"front","average":{"1min":2.2010000000000001,"5min":6.0129999999999999,"15min":2.391},"min":{"1min":0.34300000000000003,"5min":0.248,"15min":0.248},"max":{"1min":13.335000000000001,"5min":134.14599999999999,"15min":134.14599999999999},"last":3.1800000000000002}]},{"osd":1,"last update":"Mon Aug 28 02:27:00 2023","interfaces":[{"interface":"back","average":{"1min":2.0209999999999999,"5min":2.218,"15min":1.0860000000000001},"min":{"1min":0.29399999999999998,"5min":0.255,"15min":0.255},"max":{"1min":8.6790000000000003,"5min":53.622999999999998,"15min":53.622999999999998},"last":0.72299999999999998},{"interface":"front","average":{"1min":1.919,"5min":2.181,"15min":1.1000000000000001},"min":{"1min":0.34999999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":7.008,"5min":53.774000000000001,"15min":53.774000000000001},"last":4.7439999999999998}]},{"osd":2,"last update":"Mon Aug 28 02:27:33 2023","interfaces":[{"interface":"back","average":{"1min":5.8869999999999996,"5min":5.7910000000000004,"15min":2.2989999999999999},"min":{"1min":0.36899999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":64.063000000000002,"5min":74.099000000000004,"15min":74.099000000000004},"last":1.4970000000000001},{"interface":"front","average":{"1min":5.968,"5min":5.3780000000000001,"15min":2.1589999999999998},"min":{"1min":0.29299999999999998,"5min":0.28599999999999998,"15min":0.25},"max":{"1min":64.073999999999998,"5min":74.081999999999994,"15min":74.081999999999994},"last":4.258}]},{"osd":3,"last update":"Mon Aug 28 02:27:00 2023","interfaces":[{"interface":"back","average":{"1min":2.3719999999999999,"5min":1.889,"15min":0.96299999999999997},"min":{"1min":0.36299999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":8.5960000000000001,"5min":8.6349999999999998,"15min":8.6349999999999998},"last":1.0860000000000001},{"interface":"front","average":{"1min":2.2730000000000001,"5min":2.0249999999999999,"15min":1.0640000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":8.7319999999999993,"5min":10.956,"15min":10.956},"last":2.4399999999999999}]},{"osd":5,"last update":"Mon Aug 28 02:26:41 2023","interfaces":[{"interface":"back","average":{"1min":2.5350000000000001,"5min":1.383,"15min":0.77900000000000003},"min":{"1min":0.34899999999999998,"5min":0.249,"15min":0.249},"max":{"1min":7.7350000000000003,"5min":8.2560000000000002,"15min":8.2560000000000002},"last":3.1360000000000001},{"interface":"front","average":{"1min":3.2109999999999999,"5min":1.875,"15min":0.94999999999999996},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":16.497,"5min":16.497,"15min":16.497},"last":0.48999999999999999}]},{"osd":6,"last update":"Mon Aug 28 02:27:00 2023","interfaces":[{"interface":"back","average":{"1min":2.1709999999999998,"5min":1.6950000000000001,"15min":0.90900000000000003},"min":{"1min":0.314,"5min":0.29199999999999998,"15min":0.27500000000000002},"max":{"1min":8.6359999999999992,"5min":11.170999999999999,"15min":11.170999999999999},"last":0.46600000000000003},{"interface":"front","average":{"1min":1.7350000000000001,"5min":1.204,"15min":0.77400000000000002},"min":{"1min":0.38,"5min":0.32200000000000001,"15min":0.318},"max":{"1min":5.6900000000000004,"5min":7.3170000000000002,"15min":7.3170000000000002},"last":3.2149999999999999}]},{"osd":7,"last update":"Mon Aug 28 02:27:24 2023","interfaces":[{"interface":"back","average":{"1min":1.1719999999999999,"5min":1.931,"15min":1.02},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.312},"max":{"1min":6.21,"5min":16.584,"15min":16.584},"last":1.9370000000000001},{"interface":"front","average":{"1min":1.7789999999999999,"5min":2.0350000000000001,"15min":1.054},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.32100000000000001},"max":{"1min":10.826000000000001,"5min":10.826000000000001,"15min":10.826000000000001},"last":1.804}]}]},{"osd":5,"up_from":36,"seq":154618822882,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1398032,"kb_used_data":1092616,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":92347120,"statfs":{"total":95995035648,"available":94563450880,"internally_reserved":0,"allocated":1118838784,"data_stored":1118261510,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:26:53 2023","interfaces":[{"interface":"back","average":{"1min":1.496,"5min":6.0540000000000003,"15min":2.4220000000000002},"min":{"1min":0.24299999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":7.2869999999999999,"5min":131.06100000000001,"15min":131.06100000000001},"last":0.251},{"interface":"front","average":{"1min":2.0190000000000001,"5min":6.4690000000000003,"15min":2.5670000000000002},"min":{"1min":0.27300000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":7.2380000000000004,"5min":131.262,"15min":131.262},"last":0.41099999999999998}]},{"osd":1,"last update":"Mon Aug 28 02:27:04 2023","interfaces":[{"interface":"back","average":{"1min":1.988,"5min":5.3019999999999996,"15min":2.1160000000000001},"min":{"1min":0.24399999999999999,"5min":0.216,"15min":0.216},"max":{"1min":10.074,"5min":183.97900000000001,"15min":183.97900000000001},"last":0.71599999999999997},{"interface":"front","average":{"1min":2.0019999999999998,"5min":5.7960000000000003,"15min":2.29},"min":{"1min":0.33000000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":8.8000000000000007,"5min":184.33699999999999,"15min":184.33699999999999},"last":0.38800000000000001}]},{"osd":2,"last update":"Mon Aug 28 02:27:37 2023","interfaces":[{"interface":"back","average":{"1min":1.1919999999999999,"5min":7.415,"15min":3.1819999999999999},"min":{"1min":0.35599999999999998,"5min":0.28499999999999998,"15min":0.252},"max":{"1min":5.8470000000000004,"5min":176.69499999999999,"15min":176.69499999999999},"last":0.311},{"interface":"front","average":{"1min":1.2350000000000001,"5min":7.7930000000000001,"15min":3.3079999999999998},"min":{"1min":0.36699999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":5.8360000000000003,"5min":176.71199999999999,"15min":176.71199999999999},"last":0.434}]},{"osd":3,"last update":"Mon Aug 28 02:27:04 2023","interfaces":[{"interface":"back","average":{"1min":1.744,"5min":1.875,"15min":1.0329999999999999},"min":{"1min":0.32700000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":7.3449999999999998,"5min":10.576000000000001,"15min":10.576000000000001},"last":0.33800000000000002},{"interface":"front","average":{"1min":2.8969999999999998,"5min":2.6309999999999998,"15min":1.2949999999999999},"min":{"1min":0.43099999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":8.8780000000000001,"5min":12.118,"15min":12.118},"last":0.47699999999999998}]},{"osd":4,"last update":"Mon Aug 28 02:26:45 2023","interfaces":[{"interface":"back","average":{"1min":1.3779999999999999,"5min":1.522,"15min":0.82399999999999995},"min":{"1min":0.30299999999999999,"5min":0.246,"15min":0.246},"max":{"1min":5.8159999999999998,"5min":10.759,"15min":10.759},"last":0.372},{"interface":"front","average":{"1min":1.069,"5min":1.5720000000000001,"15min":0.85999999999999999},"min":{"1min":0.312,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.4859999999999998,"5min":8.6769999999999996,"15min":8.6769999999999996},"last":0.35799999999999998}]},{"osd":6,"last update":"Mon Aug 28 02:27:04 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":1.794,"15min":0.95799999999999996},"min":{"1min":0.29899999999999999,"5min":0.29699999999999999,"15min":0.28399999999999997},"max":{"1min":5.7089999999999996,"5min":14.803000000000001,"15min":14.803000000000001},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.89700000000000002,"5min":1.444,"15min":0.84299999999999997},"min":{"1min":0.32200000000000001,"5min":0.31,"15min":0.30299999999999999},"max":{"1min":6.601,"5min":8.6479999999999997,"15min":8.6479999999999997},"last":0.60999999999999999}]},{"osd":7,"last update":"Mon Aug 28 02:27:22 2023","interfaces":[{"interface":"back","average":{"1min":1.0620000000000001,"5min":1.8440000000000001,"15min":0.98999999999999999},"min":{"1min":0.35299999999999998,"5min":0.31,"15min":0.31},"max":{"1min":10.737,"5min":10.737,"15min":10.737},"last":0.73099999999999998},{"interface":"front","average":{"1min":1.143,"5min":1.639,"15min":0.91400000000000003},"min":{"1min":0.35799999999999998,"5min":0.32100000000000001,"15min":0.29799999999999999},"max":{"1min":5.3579999999999997,"5min":12.787000000000001,"15min":12.787000000000001},"last":0.67100000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":834142208,"data_stored":834142208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":162856960,"data_stored":162856960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1801977856,"data_stored":1801977856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1823604736,"data_stored":1823604736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1180827648,"data_stored":1180827648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1125449728,"data_stored":1125449728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":689373184,"data_stored":689373184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1560281088,"data_stored":1560281088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-08-28T02:27:46.702 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-08-28T02:27:46.703 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd pool set unique_pool_1 pg_num 26 2023-08-28T02:27:46.799 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:46.969 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:27:47.031 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:27:47.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:47.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:27:47.117 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:2023-08-28T02:27:47.118351+0000 min lat: 0.00119624 max lat: 2.12752 avg lat: 0.0124861 2023-08-28T02:27:47.117 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-08-28T02:27:47.117 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 40 16 51278 51262 80.0445 46.125 0.0141697 0.0124861 2023-08-28T02:27:47.140 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:27:47.222 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:47.400 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:27:47.422 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:27:47.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:47.433 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:27:47.445 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:27:47.497 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:27:47.532 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:27:47.534 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:27:47.573 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:47 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/919769056' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:27:47.576 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:47 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/919769056' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:27:47.576 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:27:47.609 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:27:47.615 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:47.664 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:27:47.712 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:27:47.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:47 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/919769056' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:27:47.763 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:27:47.792 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:27:47.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:47.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:27:47.813 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:27:47.813 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:27:47.813 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:27:47.813 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:27:47.813 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:27:47.814 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:27:47.814 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:27:47.814 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:27:47.898 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:27:47.985 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:48.117 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 41 16 52109 52093 79.3591 51.9375 0.00506857 0.0125941 2023-08-28T02:27:48.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:48 smithi143 ceph-mon[183059]: pgmap v486: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 3.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 55 MiB/s wr, 874 op/s; 13366/148143 objects degraded (9.022%); 2969/148143 objects misplaced (2.004%); 4.4 MiB/s, 69 objects/s recovering 2023-08-28T02:27:48.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:48 smithi143 ceph-mon[189871]: pgmap v486: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 3.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 55 MiB/s wr, 874 op/s; 13366/148143 objects degraded (9.022%); 2969/148143 objects misplaced (2.004%); 4.4 MiB/s, 69 objects/s recovering 2023-08-28T02:27:48.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:48 smithi169 ceph-mon[159415]: pgmap v486: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 3.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 55 MiB/s wr, 874 op/s; 13366/148143 objects degraded (9.022%); 2969/148143 objects misplaced (2.004%); 4.4 MiB/s, 69 objects/s recovering 2023-08-28T02:27:49.119 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 42 16 53008 52992 78.803 56.1875 0.0145484 0.012684 2023-08-28T02:27:49.624 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:27:49.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:49.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:27:49.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:49 smithi143 ceph-mon[183059]: pgmap v487: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 53 MiB/s wr, 847 op/s; 13366/156342 objects degraded (8.549%); 2662/156342 objects misplaced (1.703%); 4.7 MiB/s, 75 objects/s recovering 2023-08-28T02:27:49.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:49 smithi143 ceph-mon[189871]: pgmap v487: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 53 MiB/s wr, 847 op/s; 13366/156342 objects degraded (8.549%); 2662/156342 objects misplaced (1.703%); 4.7 MiB/s, 75 objects/s recovering 2023-08-28T02:27:49.730 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:27:49.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:49 smithi169 ceph-mon[159415]: pgmap v487: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 53 MiB/s wr, 847 op/s; 13366/156342 objects degraded (8.549%); 2662/156342 objects misplaced (1.703%); 4.7 MiB/s, 75 objects/s recovering 2023-08-28T02:27:49.764 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:49.826 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:27:49.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:49.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:27:49.934 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:27:49.972 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:50.037 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:27:50.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:50.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:27:50.120 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 43 16 53899 53883 78.2656 55.6875 0.0143992 0.0127714 2023-08-28T02:27:50.144 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:27:50.212 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:50.337 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:27:50.489 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:27:50.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:50.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:27:50.604 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:27:50.680 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:51.015 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:27:51.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:51.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:27:51.120 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 44 16 54777 54761 77.7342 54.875 0.0127119 0.0128574 2023-08-28T02:27:51.129 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:27:51.224 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:51.370 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:27:51.370 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:27:51.441 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:27:51.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:51.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:27:51.554 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:27:51.617 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:51.953 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:27:51 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:27:51] "GET /metrics HTTP/1.1" 200 36508 "" "Prometheus/2.33.4" 2023-08-28T02:27:52.120 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 45 16 55596 55580 77.1445 51.1875 0.0026809 0.0129384 2023-08-28T02:27:52.350 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is set 2023-08-28T02:27:52.373 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:27:52.642 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:52 smithi143 ceph-mon[189871]: pgmap v488: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 49 MiB/s wr, 779 op/s; 13366/156342 objects degraded (8.549%); 2662/156342 objects misplaced (1.703%); 4.3 MiB/s, 69 objects/s recovering 2023-08-28T02:27:52.643 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:52 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 13366/156342 objects degraded (8.549%), 11 pgs degraded (PG_DEGRADED) 2023-08-28T02:27:52.643 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:52 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2221526056' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:27:52.643 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:52 smithi143 ceph-mon[183059]: pgmap v488: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 49 MiB/s wr, 779 op/s; 13366/156342 objects degraded (8.549%); 2662/156342 objects misplaced (1.703%); 4.3 MiB/s, 69 objects/s recovering 2023-08-28T02:27:52.643 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:52 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 13366/156342 objects degraded (8.549%), 11 pgs degraded (PG_DEGRADED) 2023-08-28T02:27:52.643 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:52 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2221526056' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:27:52.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:52 smithi169 ceph-mon[159415]: pgmap v488: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 49 MiB/s wr, 779 op/s; 13366/156342 objects degraded (8.549%); 2662/156342 objects misplaced (1.703%); 4.3 MiB/s, 69 objects/s recovering 2023-08-28T02:27:52.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:52 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 13366/156342 objects degraded (8.549%), 11 pgs degraded (PG_DEGRADED) 2023-08-28T02:27:52.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:52 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2221526056' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:27:53.120 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 46 16 56331 56315 76.4664 45.9375 0.0203913 0.0130716 2023-08-28T02:27:53.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:53 smithi143 ceph-mon[189871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:27:53.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:53 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2221526056' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:27:53.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:53 smithi143 ceph-mon[189871]: osdmap e207: 8 total, 8 up, 7 in 2023-08-28T02:27:53.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:53 smithi143 ceph-mon[189871]: failed to encode map e207 with expected crc 2023-08-28T02:27:53.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:53 smithi143 ceph-mon[189871]: failed to encode map e207 with expected crc 2023-08-28T02:27:53.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:53 smithi143 ceph-mon[189871]: failed to encode map e207 with expected crc 2023-08-28T02:27:53.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:53 smithi143 ceph-mon[189871]: failed to encode map e207 with expected crc 2023-08-28T02:27:53.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:53 smithi143 ceph-mon[189871]: failed to encode map e207 with expected crc 2023-08-28T02:27:53.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:53 smithi143 ceph-mon[183059]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:27:53.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:53 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2221526056' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:27:53.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:53 smithi143 ceph-mon[183059]: osdmap e207: 8 total, 8 up, 7 in 2023-08-28T02:27:53.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:53 smithi143 ceph-mon[183059]: failed to encode map e207 with expected crc 2023-08-28T02:27:53.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:53 smithi143 ceph-mon[183059]: failed to encode map e207 with expected crc 2023-08-28T02:27:53.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:53 smithi143 ceph-mon[183059]: failed to encode map e207 with expected crc 2023-08-28T02:27:53.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:53 smithi143 ceph-mon[183059]: failed to encode map e207 with expected crc 2023-08-28T02:27:53.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:53 smithi143 ceph-mon[183059]: failed to encode map e207 with expected crc 2023-08-28T02:27:53.729 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:27:53.731 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.3 dump_ops_in_flight 2023-08-28T02:27:53.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:53 smithi169 ceph-mon[159415]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:27:53.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:53 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2221526056' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:27:53.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:53 smithi169 ceph-mon[159415]: osdmap e207: 8 total, 8 up, 7 in 2023-08-28T02:27:53.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:53 smithi169 ceph-mon[159415]: failed to encode map e207 with expected crc 2023-08-28T02:27:53.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:53 smithi169 ceph-mon[159415]: failed to encode map e207 with expected crc 2023-08-28T02:27:53.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:53 smithi169 ceph-mon[159415]: failed to encode map e207 with expected crc 2023-08-28T02:27:53.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:53 smithi169 ceph-mon[159415]: failed to encode map e207 with expected crc 2023-08-28T02:27:53.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:53 smithi169 ceph-mon[159415]: failed to encode map e207 with expected crc 2023-08-28T02:27:53.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:53.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:27:53.851 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:27:54.013 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:54.121 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 47 16 57118 57102 75.8835 49.1875 0.0229856 0.0131711 2023-08-28T02:27:54.217 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:27:54.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:54.223 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:27:54.228 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:27:54.324 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:27:54.325 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:54 smithi143 ceph-mon[189871]: pgmap v490: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 52 MiB/s wr, 836 op/s; 13366/162915 objects degraded (8.204%); 2549/162915 objects misplaced (1.565%); 4.7 MiB/s, 75 objects/s recovering 2023-08-28T02:27:54.357 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:54.415 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:27:54.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:54.419 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:27:54.520 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:27:54.584 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:54 smithi143 ceph-mon[183059]: pgmap v490: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 52 MiB/s wr, 836 op/s; 13366/162915 objects degraded (8.204%); 2549/162915 objects misplaced (1.565%); 4.7 MiB/s, 75 objects/s recovering 2023-08-28T02:27:54.634 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:54.727 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:27:54.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:54.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:27:54.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:54 smithi169 ceph-mon[159415]: pgmap v490: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 52 MiB/s wr, 836 op/s; 13366/162915 objects degraded (8.204%); 2549/162915 objects misplaced (1.565%); 4.7 MiB/s, 75 objects/s recovering 2023-08-28T02:27:54.832 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:27:54.866 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:54.871 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:27:54.928 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:27:54.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:54.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:27:55.034 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:27:55.069 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:55.125 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 48 16 57937 57921 75.3634 51.1875 0.00896745 0.0132638 2023-08-28T02:27:55.126 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:27:55.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:55.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:27:55.233 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:27:55.274 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:55.344 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:27:55.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:55.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:27:55.370 INFO:teuthology.orchestra.run.smithi143.stderr:set pool 6 pg_num to 26 2023-08-28T02:27:55.449 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:27:55.476 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:55 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/818714865' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]: dispatch 2023-08-28T02:27:55.478 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:55.521 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:27:55.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:55.525 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:27:55.625 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:27:55.662 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:55.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:55 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/818714865' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]: dispatch 2023-08-28T02:27:55.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:55 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/818714865' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]: dispatch 2023-08-28T02:27:55.717 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:27:55.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:55.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:27:55.755 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set nodeep-scrub 2023-08-28T02:27:55.823 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:27:55.862 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:55.917 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:27:55.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:55.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:27:56.026 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:27:56.126 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 49 16 58684 58668 74.7783 46.6875 0.0142138 0.0133647 2023-08-28T02:27:56.189 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:56.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:56 smithi143 ceph-mon[183059]: pgmap v491: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 51 MiB/s wr, 818 op/s; 13366/168960 objects degraded (7.911%); 2282/168960 objects misplaced (1.351%); 5.0 MiB/s, 79 objects/s recovering 2023-08-28T02:27:56.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:56 smithi143 ceph-mon[183059]: failed to encode map e208 with expected crc 2023-08-28T02:27:56.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:56 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/818714865' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]': finished 2023-08-28T02:27:56.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:56 smithi143 ceph-mon[183059]: osdmap e208: 8 total, 8 up, 7 in 2023-08-28T02:27:56.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:56 smithi143 ceph-mon[183059]: failed to encode map e208 with expected crc 2023-08-28T02:27:56.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:56 smithi143 ceph-mon[183059]: failed to encode map e208 with expected crc 2023-08-28T02:27:56.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:56 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:56.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:56 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:27:56.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:56 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 13366/168960 objects degraded (7.911%), 11 pgs degraded (PG_DEGRADED) 2023-08-28T02:27:56.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:56 smithi143 ceph-mon[189871]: pgmap v491: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 51 MiB/s wr, 818 op/s; 13366/168960 objects degraded (7.911%); 2282/168960 objects misplaced (1.351%); 5.0 MiB/s, 79 objects/s recovering 2023-08-28T02:27:56.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:56 smithi143 ceph-mon[189871]: failed to encode map e208 with expected crc 2023-08-28T02:27:56.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:56 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/818714865' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]': finished 2023-08-28T02:27:56.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:56 smithi143 ceph-mon[189871]: osdmap e208: 8 total, 8 up, 7 in 2023-08-28T02:27:56.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:56 smithi143 ceph-mon[189871]: failed to encode map e208 with expected crc 2023-08-28T02:27:56.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:56 smithi143 ceph-mon[189871]: failed to encode map e208 with expected crc 2023-08-28T02:27:56.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:56 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:56.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:56 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:27:56.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:56 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 13366/168960 objects degraded (7.911%), 11 pgs degraded (PG_DEGRADED) 2023-08-28T02:27:56.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:56 smithi169 ceph-mon[159415]: pgmap v491: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 51 MiB/s wr, 818 op/s; 13366/168960 objects degraded (7.911%); 2282/168960 objects misplaced (1.351%); 5.0 MiB/s, 79 objects/s recovering 2023-08-28T02:27:56.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:56 smithi169 ceph-mon[159415]: failed to encode map e208 with expected crc 2023-08-28T02:27:56.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:56 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/818714865' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]': finished 2023-08-28T02:27:56.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:56 smithi169 ceph-mon[159415]: osdmap e208: 8 total, 8 up, 7 in 2023-08-28T02:27:56.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:56 smithi169 ceph-mon[159415]: failed to encode map e208 with expected crc 2023-08-28T02:27:56.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:56 smithi169 ceph-mon[159415]: failed to encode map e208 with expected crc 2023-08-28T02:27:56.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:56 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:27:56.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:56 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:27:56.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:56 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 13366/168960 objects degraded (7.911%), 11 pgs degraded (PG_DEGRADED) 2023-08-28T02:27:56.949 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:27:56.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:56.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:27:57.061 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:27:57.099 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:57.125 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 50 16 59506 59490 74.3104 51.375 0.0131433 0.0134497 2023-08-28T02:27:57.156 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:27:57.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:57.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:27:57.263 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:27:57.328 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:57.407 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:57 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-08-28T02:27:57.407 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:57 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-08-28T02:27:57.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:57 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-08-28T02:27:58.126 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 51 16 59730 59714 73.1286 14 0.0101356 0.0134933 2023-08-28T02:27:58.255 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:27:58.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:58.258 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:27:58.325 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:27:58.359 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:27:58.396 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:58.509 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:58 smithi143 ceph-mon[183059]: pgmap v493: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 42 MiB/s wr, 678 op/s; 13366/172647 objects degraded (7.742%); 2169/172647 objects misplaced (1.256%); 3.8 MiB/s, 61 objects/s recovering 2023-08-28T02:27:58.509 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:58 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]': finished 2023-08-28T02:27:58.509 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:58 smithi143 ceph-mon[183059]: osdmap e209: 8 total, 8 up, 7 in 2023-08-28T02:27:58.509 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:58 smithi143 ceph-mon[183059]: failed to encode map e209 with expected crc 2023-08-28T02:27:58.510 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:58 smithi143 ceph-mon[183059]: failed to encode map e209 with expected crc 2023-08-28T02:27:58.512 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:58 smithi143 ceph-mon[189871]: pgmap v493: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 42 MiB/s wr, 678 op/s; 13366/172647 objects degraded (7.742%); 2169/172647 objects misplaced (1.256%); 3.8 MiB/s, 61 objects/s recovering 2023-08-28T02:27:58.512 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:58 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]': finished 2023-08-28T02:27:58.513 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:58 smithi143 ceph-mon[189871]: osdmap e209: 8 total, 8 up, 7 in 2023-08-28T02:27:58.513 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:58 smithi143 ceph-mon[189871]: failed to encode map e209 with expected crc 2023-08-28T02:27:58.513 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:58 smithi143 ceph-mon[189871]: failed to encode map e209 with expected crc 2023-08-28T02:27:58.521 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:27:58.564 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:27:58.565 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:27:58.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:58.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:27:58.593 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:27:58.616 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:27:58.639 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:27:58.663 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:27:58.674 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:27:58.684 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:27:58.707 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:27:58.709 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:58.709 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:27:58.709 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:27:58.709 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:27:58.709 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:27:58.709 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:27:58.709 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:27:58.710 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:27:58.710 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:27:58.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:58 smithi169 ceph-mon[159415]: pgmap v493: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 42 MiB/s wr, 678 op/s; 13366/172647 objects degraded (7.742%); 2169/172647 objects misplaced (1.256%); 3.8 MiB/s, 61 objects/s recovering 2023-08-28T02:27:58.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:58 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]': finished 2023-08-28T02:27:58.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:58 smithi169 ceph-mon[159415]: osdmap e209: 8 total, 8 up, 7 in 2023-08-28T02:27:58.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:58 smithi169 ceph-mon[159415]: failed to encode map e209 with expected crc 2023-08-28T02:27:58.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:58 smithi169 ceph-mon[159415]: failed to encode map e209 with expected crc 2023-08-28T02:27:58.828 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:27:58.828 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [ 2023-08-28T02:27:58.829 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:58.829 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "MOSDPGPushReply(6.b 210/203 [PushReplyOp(6:d29c183f:::benchmark_data_smithi143_305742_object2363:head)])", 2023-08-28T02:27:58.829 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:27:58.828065+0000", 2023-08-28T02:27:58.829 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.0013459500000000001, 2023-08-28T02:27:58.829 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0013665249999999999, 2023-08-28T02:27:58.829 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:27:58.829 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "reached pg", 2023-08-28T02:27:58.829 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:27:58.829 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:58.829 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:27:58.829 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:58.828065+0000", 2023-08-28T02:27:58.829 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:27:58.830 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:58.830 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:58.830 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:27:58.830 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:58.828065+0000", 2023-08-28T02:27:58.830 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:27:58.830 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:58.830 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:58.830 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:27:58.830 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:58.828064+0000", 2023-08-28T02:27:58.830 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:27:58.830 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:58.830 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:58.831 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:27:58.831 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:58.828070+0000", 2023-08-28T02:27:58.831 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.7039999999999999e-06 2023-08-28T02:27:58.831 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:58.831 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:58.831 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:27:58.831 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:58.828084+0000", 2023-08-28T02:27:58.831 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3648999999999999e-05 2023-08-28T02:27:58.831 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:58.831 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:58.831 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:27:58.831 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:58.828093+0000", 2023-08-28T02:27:58.832 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.0799999999999995e-06 2023-08-28T02:27:58.832 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:27:58.832 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:27:58.832 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:27:58.832 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:58.828138+0000", 2023-08-28T02:27:58.832 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.4802000000000002e-05 2023-08-28T02:27:58.832 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:27:58.832 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:27:58.832 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:27:58.832 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:27:58.832 INFO:teuthology.orchestra.run.smithi143.stdout: ], 2023-08-28T02:27:58.832 INFO:teuthology.orchestra.run.smithi143.stdout: "num_ops": 1 2023-08-28T02:27:58.833 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:27:59.126 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 52 16 59801 59785 71.8083 4.4375 0.00802811 0.0137265 2023-08-28T02:27:59.326 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:27:59.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:59.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:27:59.430 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:27:59.464 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:59.515 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:27:59.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:59.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:27:59.621 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:27:59.650 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:27:59.695 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:27:59.699 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:59 smithi169 ceph-mon[159415]: failed to encode map e210 with expected crc 2023-08-28T02:27:59.699 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:59 smithi169 ceph-mon[159415]: failed to encode map e210 with expected crc 2023-08-28T02:27:59.699 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:59 smithi169 ceph-mon[159415]: failed to encode map e210 with expected crc 2023-08-28T02:27:59.699 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:59 smithi169 ceph-mon[159415]: failed to encode map e210 with expected crc 2023-08-28T02:27:59.699 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:59 smithi169 ceph-mon[159415]: failed to encode map e210 with expected crc 2023-08-28T02:27:59.699 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:59 smithi169 ceph-mon[159415]: osdmap e210: 8 total, 8 up, 7 in 2023-08-28T02:27:59.699 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:59 smithi169 ceph-mon[159415]: failed to encode map e210 with expected crc 2023-08-28T02:27:59.699 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:59 smithi169 ceph-mon[159415]: failed to encode map e210 with expected crc 2023-08-28T02:27:59.700 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:59 smithi169 ceph-mon[159415]: failed to encode map e210 with expected crc 2023-08-28T02:27:59.700 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:59 smithi169 ceph-mon[159415]: failed to encode map e210 with expected crc 2023-08-28T02:27:59.700 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:59 smithi169 ceph-mon[159415]: failed to encode map e210 with expected crc 2023-08-28T02:27:59.700 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:59 smithi169 ceph-mon[159415]: failed to encode map e210 with expected crc 2023-08-28T02:27:59.700 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:59 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3024657010' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:27:59.700 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:59 smithi169 ceph-mon[159415]: pgmap v496: 99 pgs: 1 creating, 1 peering, 1 active+recovery_wait+remapped, 1 creating+peering, 5 unknown, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.6 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 56 MiB/s wr, 895 op/s; 9075/179037 objects degraded (5.069%); 3219/179037 objects misplaced (1.798%); 7.8 MiB/s, 124 objects/s recovering 2023-08-28T02:27:59.700 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:27:59 smithi169 ceph-mon[159415]: osdmap e211: 8 total, 8 up, 7 in 2023-08-28T02:27:59.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:59 smithi143 ceph-mon[189871]: failed to encode map e210 with expected crc 2023-08-28T02:27:59.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:59 smithi143 ceph-mon[189871]: failed to encode map e210 with expected crc 2023-08-28T02:27:59.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:59 smithi143 ceph-mon[189871]: failed to encode map e210 with expected crc 2023-08-28T02:27:59.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:59 smithi143 ceph-mon[189871]: failed to encode map e210 with expected crc 2023-08-28T02:27:59.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:59 smithi143 ceph-mon[189871]: failed to encode map e210 with expected crc 2023-08-28T02:27:59.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:59 smithi143 ceph-mon[189871]: osdmap e210: 8 total, 8 up, 7 in 2023-08-28T02:27:59.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:59 smithi143 ceph-mon[189871]: failed to encode map e210 with expected crc 2023-08-28T02:27:59.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:59 smithi143 ceph-mon[189871]: failed to encode map e210 with expected crc 2023-08-28T02:27:59.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:59 smithi143 ceph-mon[189871]: failed to encode map e210 with expected crc 2023-08-28T02:27:59.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:59 smithi143 ceph-mon[189871]: failed to encode map e210 with expected crc 2023-08-28T02:27:59.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:59 smithi143 ceph-mon[189871]: failed to encode map e210 with expected crc 2023-08-28T02:27:59.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:59 smithi143 ceph-mon[189871]: failed to encode map e210 with expected crc 2023-08-28T02:27:59.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:59 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3024657010' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:27:59.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:59 smithi143 ceph-mon[189871]: pgmap v496: 99 pgs: 1 creating, 1 peering, 1 active+recovery_wait+remapped, 1 creating+peering, 5 unknown, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.6 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 56 MiB/s wr, 895 op/s; 9075/179037 objects degraded (5.069%); 3219/179037 objects misplaced (1.798%); 7.8 MiB/s, 124 objects/s recovering 2023-08-28T02:27:59.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:27:59 smithi143 ceph-mon[189871]: osdmap e211: 8 total, 8 up, 7 in 2023-08-28T02:27:59.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:59 smithi143 ceph-mon[183059]: failed to encode map e210 with expected crc 2023-08-28T02:27:59.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:59 smithi143 ceph-mon[183059]: failed to encode map e210 with expected crc 2023-08-28T02:27:59.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:59 smithi143 ceph-mon[183059]: failed to encode map e210 with expected crc 2023-08-28T02:27:59.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:59 smithi143 ceph-mon[183059]: failed to encode map e210 with expected crc 2023-08-28T02:27:59.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:59 smithi143 ceph-mon[183059]: failed to encode map e210 with expected crc 2023-08-28T02:27:59.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:59 smithi143 ceph-mon[183059]: osdmap e210: 8 total, 8 up, 7 in 2023-08-28T02:27:59.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:59 smithi143 ceph-mon[183059]: failed to encode map e210 with expected crc 2023-08-28T02:27:59.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:59 smithi143 ceph-mon[183059]: failed to encode map e210 with expected crc 2023-08-28T02:27:59.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:59 smithi143 ceph-mon[183059]: failed to encode map e210 with expected crc 2023-08-28T02:27:59.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:59 smithi143 ceph-mon[183059]: failed to encode map e210 with expected crc 2023-08-28T02:27:59.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:59 smithi143 ceph-mon[183059]: failed to encode map e210 with expected crc 2023-08-28T02:27:59.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:59 smithi143 ceph-mon[183059]: failed to encode map e210 with expected crc 2023-08-28T02:27:59.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:59 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3024657010' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:27:59.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:59 smithi143 ceph-mon[183059]: pgmap v496: 99 pgs: 1 creating, 1 peering, 1 active+recovery_wait+remapped, 1 creating+peering, 5 unknown, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.6 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 56 MiB/s wr, 895 op/s; 9075/179037 objects degraded (5.069%); 3219/179037 objects misplaced (1.798%); 7.8 MiB/s, 124 objects/s recovering 2023-08-28T02:27:59.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:27:59 smithi143 ceph-mon[183059]: osdmap e211: 8 total, 8 up, 7 in 2023-08-28T02:27:59.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:27:59.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:27:59.805 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:27:59.838 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:00.126 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 53 16 59811 59795 70.466 0.625 0.00259688 0.013739 2023-08-28T02:28:00.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:00 smithi143 ceph-mon[183059]: failed to encode map e211 with expected crc 2023-08-28T02:28:00.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:00 smithi143 ceph-mon[183059]: failed to encode map e211 with expected crc 2023-08-28T02:28:00.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:00 smithi143 ceph-mon[183059]: failed to encode map e211 with expected crc 2023-08-28T02:28:00.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:00 smithi143 ceph-mon[183059]: failed to encode map e211 with expected crc 2023-08-28T02:28:00.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:00 smithi143 ceph-mon[183059]: failed to encode map e211 with expected crc 2023-08-28T02:28:00.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:00 smithi143 ceph-mon[183059]: osdmap e212: 8 total, 8 up, 7 in 2023-08-28T02:28:00.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:00 smithi143 ceph-mon[189871]: failed to encode map e211 with expected crc 2023-08-28T02:28:00.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:00 smithi143 ceph-mon[189871]: failed to encode map e211 with expected crc 2023-08-28T02:28:00.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:00 smithi143 ceph-mon[189871]: failed to encode map e211 with expected crc 2023-08-28T02:28:00.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:00 smithi143 ceph-mon[189871]: failed to encode map e211 with expected crc 2023-08-28T02:28:00.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:00 smithi143 ceph-mon[189871]: failed to encode map e211 with expected crc 2023-08-28T02:28:00.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:00 smithi143 ceph-mon[189871]: osdmap e212: 8 total, 8 up, 7 in 2023-08-28T02:28:00.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:00 smithi169 ceph-mon[159415]: failed to encode map e211 with expected crc 2023-08-28T02:28:00.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:00 smithi169 ceph-mon[159415]: failed to encode map e211 with expected crc 2023-08-28T02:28:00.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:00 smithi169 ceph-mon[159415]: failed to encode map e211 with expected crc 2023-08-28T02:28:00.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:00 smithi169 ceph-mon[159415]: failed to encode map e211 with expected crc 2023-08-28T02:28:00.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:00 smithi169 ceph-mon[159415]: failed to encode map e211 with expected crc 2023-08-28T02:28:00.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:00 smithi169 ceph-mon[159415]: osdmap e212: 8 total, 8 up, 7 in 2023-08-28T02:28:00.960 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.3 dump_blocked_ops 2023-08-28T02:28:00.967 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:28:00.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:00.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:28:01.072 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:28:01.126 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 54 16 60418 60402 69.8639 37.9375 0.0157797 0.0143066 2023-08-28T02:28:01.169 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:01.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:01 smithi143 ceph-mon[183059]: failed to encode map e212 with expected crc 2023-08-28T02:28:01.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:01 smithi143 ceph-mon[183059]: failed to encode map e212 with expected crc 2023-08-28T02:28:01.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:01 smithi143 ceph-mon[183059]: failed to encode map e212 with expected crc 2023-08-28T02:28:01.704 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:01 smithi143 ceph-mon[183059]: failed to encode map e212 with expected crc 2023-08-28T02:28:01.704 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:01 smithi143 ceph-mon[183059]: failed to encode map e212 with expected crc 2023-08-28T02:28:01.704 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:01 smithi143 ceph-mon[183059]: failed to encode map e212 with expected crc 2023-08-28T02:28:01.704 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:01 smithi143 ceph-mon[183059]: pgmap v499: 99 pgs: 1 creating, 1 peering, 1 active+recovery_wait+remapped, 1 creating+peering, 5 unknown, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.6 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 33 MiB/s wr, 532 op/s; 9075/179037 objects degraded (5.069%); 3219/179037 objects misplaced (1.798%); 5.8 MiB/s, 92 objects/s recovering 2023-08-28T02:28:01.704 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:01 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 9075/179037 objects degraded (5.069%), 9 pgs degraded (PG_DEGRADED) 2023-08-28T02:28:01.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:01 smithi143 ceph-mon[189871]: failed to encode map e212 with expected crc 2023-08-28T02:28:01.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:01 smithi143 ceph-mon[189871]: failed to encode map e212 with expected crc 2023-08-28T02:28:01.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:01 smithi143 ceph-mon[189871]: failed to encode map e212 with expected crc 2023-08-28T02:28:01.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:01 smithi143 ceph-mon[189871]: failed to encode map e212 with expected crc 2023-08-28T02:28:01.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:01 smithi143 ceph-mon[189871]: failed to encode map e212 with expected crc 2023-08-28T02:28:01.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:01 smithi143 ceph-mon[189871]: failed to encode map e212 with expected crc 2023-08-28T02:28:01.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:01 smithi143 ceph-mon[189871]: pgmap v499: 99 pgs: 1 creating, 1 peering, 1 active+recovery_wait+remapped, 1 creating+peering, 5 unknown, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.6 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 33 MiB/s wr, 532 op/s; 9075/179037 objects degraded (5.069%); 3219/179037 objects misplaced (1.798%); 5.8 MiB/s, 92 objects/s recovering 2023-08-28T02:28:01.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:01 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 9075/179037 objects degraded (5.069%), 9 pgs degraded (PG_DEGRADED) 2023-08-28T02:28:01.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:01 smithi169 ceph-mon[159415]: failed to encode map e212 with expected crc 2023-08-28T02:28:01.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:01 smithi169 ceph-mon[159415]: failed to encode map e212 with expected crc 2023-08-28T02:28:01.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:01 smithi169 ceph-mon[159415]: failed to encode map e212 with expected crc 2023-08-28T02:28:01.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:01 smithi169 ceph-mon[159415]: failed to encode map e212 with expected crc 2023-08-28T02:28:01.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:01 smithi169 ceph-mon[159415]: failed to encode map e212 with expected crc 2023-08-28T02:28:01.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:01 smithi169 ceph-mon[159415]: failed to encode map e212 with expected crc 2023-08-28T02:28:01.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:01 smithi169 ceph-mon[159415]: pgmap v499: 99 pgs: 1 creating, 1 peering, 1 active+recovery_wait+remapped, 1 creating+peering, 5 unknown, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 3.6 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 33 MiB/s wr, 532 op/s; 9075/179037 objects degraded (5.069%); 3219/179037 objects misplaced (1.798%); 5.8 MiB/s, 92 objects/s recovering 2023-08-28T02:28:01.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:01 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 9075/179037 objects degraded (5.069%), 9 pgs degraded (PG_DEGRADED) 2023-08-28T02:28:01.758 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:28:01.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:01.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:28:01.869 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:28:01.970 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 7, 3, 1] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-08-28T02:28:01.970 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-08-28T02:28:01.971 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-08-28T02:28:01.971 INFO:tasks.thrashosds.thrasher:Setting osd 5 primary_affinity to 0.738213 2023-08-28T02:28:01.971 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd primary-affinity 5 0.7382125605420777 2023-08-28T02:28:01.973 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:28:01 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:28:01] "GET /metrics HTTP/1.1" 200 36510 "" "Prometheus/2.33.4" 2023-08-28T02:28:01.981 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:02.135 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 55 16 61249 61233 69.5262 51.9375 0.0521354 0.0143748 2023-08-28T02:28:02.185 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:28:02.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:02.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:28:02.299 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:28:02.403 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:28:02.427 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:02.542 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:28:02.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:02.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:28:02.655 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:28:02.771 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:02.884 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:28:02.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:02.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:28:03.002 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:28:03.040 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:03.058 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is set 2023-08-28T02:28:03.096 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:28:03.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:03.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:28:03.141 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 56 16 62006 61990 69.1225 47.3125 0.0352498 0.0144614 2023-08-28T02:28:03.202 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:28:03.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:02 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1122196168' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:28:03.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:02 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:28:03.272 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:02 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1122196168' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:28:03.273 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:02 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:28:03.273 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:02 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1122196168' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:28:03.273 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:02 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:28:03.277 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:03.763 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:28:04.141 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 57 16 62784 62768 68.7631 48.625 0.0142816 0.0145344 2023-08-28T02:28:04.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:03 smithi143 ceph-mon[183059]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:28:04.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:03 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:28:04.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:03 smithi143 ceph-mon[183059]: osdmap e213: 8 total, 8 up, 7 in 2023-08-28T02:28:04.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:03 smithi143 ceph-mon[183059]: failed to encode map e213 with expected crc 2023-08-28T02:28:04.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:03 smithi143 ceph-mon[183059]: failed to encode map e213 with expected crc 2023-08-28T02:28:04.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:03 smithi143 ceph-mon[183059]: failed to encode map e213 with expected crc 2023-08-28T02:28:04.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:03 smithi143 ceph-mon[183059]: failed to encode map e213 with expected crc 2023-08-28T02:28:04.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:03 smithi143 ceph-mon[183059]: pgmap v501: 99 pgs: 1 creating, 1 peering, 2 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 creating+peering, 2 active+recovering+undersized+remapped, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 30 MiB/s wr, 480 op/s; 8654/181755 objects degraded (4.761%); 2770/181755 objects misplaced (1.524%); 16 MiB/s, 258 objects/s recovering 2023-08-28T02:28:04.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:03 smithi143 ceph-mon[189871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:28:04.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:03 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:28:04.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:03 smithi143 ceph-mon[189871]: osdmap e213: 8 total, 8 up, 7 in 2023-08-28T02:28:04.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:03 smithi143 ceph-mon[189871]: failed to encode map e213 with expected crc 2023-08-28T02:28:04.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:03 smithi143 ceph-mon[189871]: failed to encode map e213 with expected crc 2023-08-28T02:28:04.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:03 smithi143 ceph-mon[189871]: failed to encode map e213 with expected crc 2023-08-28T02:28:04.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:03 smithi143 ceph-mon[189871]: failed to encode map e213 with expected crc 2023-08-28T02:28:04.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:03 smithi143 ceph-mon[189871]: pgmap v501: 99 pgs: 1 creating, 1 peering, 2 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 creating+peering, 2 active+recovering+undersized+remapped, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 30 MiB/s wr, 480 op/s; 8654/181755 objects degraded (4.761%); 2770/181755 objects misplaced (1.524%); 16 MiB/s, 258 objects/s recovering 2023-08-28T02:28:04.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:03 smithi169 ceph-mon[159415]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:28:04.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:03 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:28:04.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:03 smithi169 ceph-mon[159415]: osdmap e213: 8 total, 8 up, 7 in 2023-08-28T02:28:04.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:03 smithi169 ceph-mon[159415]: failed to encode map e213 with expected crc 2023-08-28T02:28:04.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:03 smithi169 ceph-mon[159415]: failed to encode map e213 with expected crc 2023-08-28T02:28:04.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:03 smithi169 ceph-mon[159415]: failed to encode map e213 with expected crc 2023-08-28T02:28:04.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:03 smithi169 ceph-mon[159415]: failed to encode map e213 with expected crc 2023-08-28T02:28:04.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:03 smithi169 ceph-mon[159415]: pgmap v501: 99 pgs: 1 creating, 1 peering, 2 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 creating+peering, 2 active+recovering+undersized+remapped, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 30 MiB/s wr, 480 op/s; 8654/181755 objects degraded (4.761%); 2770/181755 objects misplaced (1.524%); 16 MiB/s, 258 objects/s recovering 2023-08-28T02:28:04.780 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:28:04.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:04.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:28:04.887 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:28:04.915 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:28:04.916 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:04.962 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:28:04.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:04.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:28:05.068 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:28:05.101 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:05.141 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 58 16 63515 63499 68.3656 45.6875 0.0315516 0.0146198 2023-08-28T02:28:05.154 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:28:05.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:05.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:28:05.261 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:28:05.294 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:05.353 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:28:05.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:05.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:28:05.461 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:28:05.499 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:28:05.568 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:05.671 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:28:05.671 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [], 2023-08-28T02:28:05.671 INFO:teuthology.orchestra.run.smithi143.stdout: "complaint_time": 30, 2023-08-28T02:28:05.671 INFO:teuthology.orchestra.run.smithi143.stdout: "num_blocked_ops": 0 2023-08-28T02:28:05.671 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:28:05.906 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:28:05.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:05.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:28:06.014 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:28:06.100 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:06.141 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 59 16 64396 64380 68.1402 55.0625 0.00709008 0.0146664 2023-08-28T02:28:06.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:06 smithi143 ceph-mon[183059]: pgmap v502: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovering, 2 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 24 MiB/s wr, 385 op/s; 10308/185979 objects degraded (5.543%); 3596/185979 objects misplaced (1.934%); 13 MiB/s, 210 objects/s recovering 2023-08-28T02:28:06.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:06 smithi143 ceph-mon[189871]: pgmap v502: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovering, 2 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 24 MiB/s wr, 385 op/s; 10308/185979 objects degraded (5.543%); 3596/185979 objects misplaced (1.934%); 13 MiB/s, 210 objects/s recovering 2023-08-28T02:28:06.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:06 smithi169 ceph-mon[159415]: pgmap v502: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovering, 2 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 24 MiB/s wr, 385 op/s; 10308/185979 objects degraded (5.543%); 3596/185979 objects misplaced (1.934%); 13 MiB/s, 210 objects/s recovering 2023-08-28T02:28:06.963 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset noscrub 2023-08-28T02:28:07.142 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:2023-08-28T02:28:07.143302+0000 min lat: 0.00119624 max lat: 3.09266 avg lat: 0.0147422 2023-08-28T02:28:07.142 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-08-28T02:28:07.142 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 60 16 65167 65151 67.8078 48.1875 0.0105675 0.0147422 2023-08-28T02:28:07.224 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:28:07.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:07.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:28:07.328 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:28:07.361 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:07.421 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:28:07.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:07.430 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:28:07.490 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:07 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 10308/185979 objects degraded (5.543%), 13 pgs degraded (PG_DEGRADED) 2023-08-28T02:28:07.491 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:07 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 10308/185979 objects degraded (5.543%), 13 pgs degraded (PG_DEGRADED) 2023-08-28T02:28:07.531 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:28:07.556 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:07 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 10308/185979 objects degraded (5.543%), 13 pgs degraded (PG_DEGRADED) 2023-08-28T02:28:07.558 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:07.614 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:28:07.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:07.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:28:07.657 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.3 dump_historic_ops 2023-08-28T02:28:07.719 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:28:07.803 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:07.942 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:28:07.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:07.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:28:08.047 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:28:08.074 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:08.142 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 61 16 65654 65638 67.1956 30.4375 0.00402711 0.014759 2023-08-28T02:28:08.519 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:28:08.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:08.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:28:08.622 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:08 smithi143 ceph-mon[183059]: pgmap v503: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovering, 2 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 28 MiB/s wr, 452 op/s; 10308/189645 objects degraded (5.435%); 3596/189645 objects misplaced (1.896%); 10 MiB/s, 161 objects/s recovering 2023-08-28T02:28:08.622 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:08 smithi143 ceph-mon[183059]: osdmap e214: 8 total, 8 up, 7 in 2023-08-28T02:28:08.622 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:08 smithi143 ceph-mon[183059]: failed to encode map e214 with expected crc 2023-08-28T02:28:08.622 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:08 smithi143 ceph-mon[183059]: failed to encode map e214 with expected crc 2023-08-28T02:28:08.622 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:08 smithi143 ceph-mon[189871]: pgmap v503: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovering, 2 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 28 MiB/s wr, 452 op/s; 10308/189645 objects degraded (5.435%); 3596/189645 objects misplaced (1.896%); 10 MiB/s, 161 objects/s recovering 2023-08-28T02:28:08.622 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:08 smithi143 ceph-mon[189871]: osdmap e214: 8 total, 8 up, 7 in 2023-08-28T02:28:08.623 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:08 smithi143 ceph-mon[189871]: failed to encode map e214 with expected crc 2023-08-28T02:28:08.623 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:08 smithi143 ceph-mon[189871]: failed to encode map e214 with expected crc 2023-08-28T02:28:08.628 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:28:08.654 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:08 smithi169 ceph-mon[159415]: pgmap v503: 99 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovering, 2 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 28 MiB/s wr, 452 op/s; 10308/189645 objects degraded (5.435%); 3596/189645 objects misplaced (1.896%); 10 MiB/s, 161 objects/s recovering 2023-08-28T02:28:08.655 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:08 smithi169 ceph-mon[159415]: osdmap e214: 8 total, 8 up, 7 in 2023-08-28T02:28:08.655 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:08 smithi169 ceph-mon[159415]: failed to encode map e214 with expected crc 2023-08-28T02:28:08.655 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:08 smithi169 ceph-mon[159415]: failed to encode map e214 with expected crc 2023-08-28T02:28:08.661 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:08.710 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:28:08.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:08.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:28:08.819 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:28:08.893 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:08.985 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:28:09.016 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:28:09.142 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 62 16 66286 66270 66.7491 39.5 0.0823406 0.0149744 2023-08-28T02:28:09.172 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:28:09.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:09.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:28:09.277 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:28:09.305 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:09.359 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:28:09.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:09.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:28:09.445 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:09 smithi143 ceph-mon[183059]: failed to encode map e214 with expected crc 2023-08-28T02:28:09.445 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:09 smithi143 ceph-mon[183059]: failed to encode map e215 with expected crc 2023-08-28T02:28:09.445 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:09 smithi143 ceph-mon[183059]: failed to encode map e215 with expected crc 2023-08-28T02:28:09.445 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:09 smithi143 ceph-mon[183059]: osdmap e215: 8 total, 8 up, 7 in 2023-08-28T02:28:09.446 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:09 smithi143 ceph-mon[183059]: failed to encode map e215 with expected crc 2023-08-28T02:28:09.446 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:09 smithi143 ceph-mon[183059]: failed to encode map e215 with expected crc 2023-08-28T02:28:09.446 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:09 smithi143 ceph-mon[183059]: failed to encode map e215 with expected crc 2023-08-28T02:28:09.447 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:09 smithi143 ceph-mon[189871]: failed to encode map e214 with expected crc 2023-08-28T02:28:09.448 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:09 smithi143 ceph-mon[189871]: failed to encode map e215 with expected crc 2023-08-28T02:28:09.448 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:09 smithi143 ceph-mon[189871]: failed to encode map e215 with expected crc 2023-08-28T02:28:09.448 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:09 smithi143 ceph-mon[189871]: osdmap e215: 8 total, 8 up, 7 in 2023-08-28T02:28:09.448 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:09 smithi143 ceph-mon[189871]: failed to encode map e215 with expected crc 2023-08-28T02:28:09.448 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:09 smithi143 ceph-mon[189871]: failed to encode map e215 with expected crc 2023-08-28T02:28:09.448 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:09 smithi143 ceph-mon[189871]: failed to encode map e215 with expected crc 2023-08-28T02:28:09.466 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:28:09.495 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:09.548 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:28:09.554 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:09 smithi169 ceph-mon[159415]: failed to encode map e214 with expected crc 2023-08-28T02:28:09.554 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:09 smithi169 ceph-mon[159415]: failed to encode map e215 with expected crc 2023-08-28T02:28:09.554 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:09 smithi169 ceph-mon[159415]: failed to encode map e215 with expected crc 2023-08-28T02:28:09.554 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:09 smithi169 ceph-mon[159415]: osdmap e215: 8 total, 8 up, 7 in 2023-08-28T02:28:09.554 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:09 smithi169 ceph-mon[159415]: failed to encode map e215 with expected crc 2023-08-28T02:28:09.554 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:09 smithi169 ceph-mon[159415]: failed to encode map e215 with expected crc 2023-08-28T02:28:09.554 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:09 smithi169 ceph-mon[159415]: failed to encode map e215 with expected crc 2023-08-28T02:28:09.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:09.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:28:09.569 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:28:09.605 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:28:09.653 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:28:09.656 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:28:09.706 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:28:09.746 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:28:09.778 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:28:09.788 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:09.822 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:28:09.858 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:28:09.916 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:28:09.916 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:28:09.916 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:28:09.916 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:28:09.916 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:28:09.916 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:28:09.916 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:28:09.916 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:28:10.142 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 63 16 66996 66980 66.3942 44.375 0.00710553 0.0150537 2023-08-28T02:28:10.398 INFO:teuthology.orchestra.run.smithi143.stderr:set osd.5 primary-affinity to 0.738213 (8483792) 2023-08-28T02:28:10.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:10 smithi143 ceph-mon[183059]: pgmap v506: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering, 1 active+undersized+remapped, 2 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 51 MiB/s wr, 822 op/s; 10308/197097 objects degraded (5.230%); 2740/197097 objects misplaced (1.390%); 13 MiB/s, 200 objects/s recovering 2023-08-28T02:28:10.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:10 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/498557188' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.7382125605420777}]: dispatch 2023-08-28T02:28:10.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:10 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.7382125605420777}]: dispatch 2023-08-28T02:28:10.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:10 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1748906482' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:28:10.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:10 smithi143 ceph-mon[189871]: pgmap v506: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering, 1 active+undersized+remapped, 2 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 51 MiB/s wr, 822 op/s; 10308/197097 objects degraded (5.230%); 2740/197097 objects misplaced (1.390%); 13 MiB/s, 200 objects/s recovering 2023-08-28T02:28:10.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:10 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/498557188' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.7382125605420777}]: dispatch 2023-08-28T02:28:10.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:10 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.7382125605420777}]: dispatch 2023-08-28T02:28:10.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:10 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1748906482' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:28:10.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:10 smithi169 ceph-mon[159415]: pgmap v506: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering, 1 active+undersized+remapped, 2 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 51 MiB/s wr, 822 op/s; 10308/197097 objects degraded (5.230%); 2740/197097 objects misplaced (1.390%); 13 MiB/s, 200 objects/s recovering 2023-08-28T02:28:10.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:10 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/498557188' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.7382125605420777}]: dispatch 2023-08-28T02:28:10.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:10 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.7382125605420777}]: dispatch 2023-08-28T02:28:10.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:10 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1748906482' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:28:11.142 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 64 16 67532 67516 65.8805 33.5 0.00234257 0.015036 2023-08-28T02:28:11.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:11 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.7382125605420777}]': finished 2023-08-28T02:28:11.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:11 smithi143 ceph-mon[183059]: osdmap e216: 8 total, 8 up, 7 in 2023-08-28T02:28:11.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:11 smithi143 ceph-mon[183059]: failed to encode map e216 with expected crc 2023-08-28T02:28:11.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:11 smithi143 ceph-mon[183059]: failed to encode map e216 with expected crc 2023-08-28T02:28:11.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:11 smithi143 ceph-mon[183059]: failed to encode map e216 with expected crc 2023-08-28T02:28:11.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:11 smithi143 ceph-mon[183059]: failed to encode map e216 with expected crc 2023-08-28T02:28:11.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:11 smithi143 ceph-mon[183059]: failed to encode map e216 with expected crc 2023-08-28T02:28:11.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:11 smithi143 ceph-mon[183059]: failed to encode map e216 with expected crc 2023-08-28T02:28:11.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:11 smithi143 ceph-mon[183059]: failed to encode map e216 with expected crc 2023-08-28T02:28:11.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:11 smithi143 ceph-mon[183059]: failed to encode map e216 with expected crc 2023-08-28T02:28:11.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:11 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:28:11.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:11 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 10308/197097 objects degraded (5.230%), 13 pgs degraded (PG_DEGRADED) 2023-08-28T02:28:11.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:11 smithi143 ceph-mon[183059]: osdmap e217: 8 total, 8 up, 7 in 2023-08-28T02:28:11.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:11 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.7382125605420777}]': finished 2023-08-28T02:28:11.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:11 smithi143 ceph-mon[189871]: osdmap e216: 8 total, 8 up, 7 in 2023-08-28T02:28:11.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:11 smithi143 ceph-mon[189871]: failed to encode map e216 with expected crc 2023-08-28T02:28:11.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:11 smithi143 ceph-mon[189871]: failed to encode map e216 with expected crc 2023-08-28T02:28:11.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:11 smithi143 ceph-mon[189871]: failed to encode map e216 with expected crc 2023-08-28T02:28:11.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:11 smithi143 ceph-mon[189871]: failed to encode map e216 with expected crc 2023-08-28T02:28:11.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:11 smithi143 ceph-mon[189871]: failed to encode map e216 with expected crc 2023-08-28T02:28:11.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:11 smithi143 ceph-mon[189871]: failed to encode map e216 with expected crc 2023-08-28T02:28:11.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:11 smithi143 ceph-mon[189871]: failed to encode map e216 with expected crc 2023-08-28T02:28:11.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:11 smithi143 ceph-mon[189871]: failed to encode map e216 with expected crc 2023-08-28T02:28:11.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:11 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:28:11.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:11 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 10308/197097 objects degraded (5.230%), 13 pgs degraded (PG_DEGRADED) 2023-08-28T02:28:11.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:11 smithi143 ceph-mon[189871]: osdmap e217: 8 total, 8 up, 7 in 2023-08-28T02:28:11.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:11 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.7382125605420777}]': finished 2023-08-28T02:28:11.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:11 smithi169 ceph-mon[159415]: osdmap e216: 8 total, 8 up, 7 in 2023-08-28T02:28:11.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:11 smithi169 ceph-mon[159415]: failed to encode map e216 with expected crc 2023-08-28T02:28:11.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:11 smithi169 ceph-mon[159415]: failed to encode map e216 with expected crc 2023-08-28T02:28:11.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:11 smithi169 ceph-mon[159415]: failed to encode map e216 with expected crc 2023-08-28T02:28:11.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:11 smithi169 ceph-mon[159415]: failed to encode map e216 with expected crc 2023-08-28T02:28:11.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:11 smithi169 ceph-mon[159415]: failed to encode map e216 with expected crc 2023-08-28T02:28:11.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:11 smithi169 ceph-mon[159415]: failed to encode map e216 with expected crc 2023-08-28T02:28:11.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:11 smithi169 ceph-mon[159415]: failed to encode map e216 with expected crc 2023-08-28T02:28:11.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:11 smithi169 ceph-mon[159415]: failed to encode map e216 with expected crc 2023-08-28T02:28:11.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:11 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:28:11.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:11 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 10308/197097 objects degraded (5.230%), 13 pgs degraded (PG_DEGRADED) 2023-08-28T02:28:11.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:11 smithi169 ceph-mon[159415]: osdmap e217: 8 total, 8 up, 7 in 2023-08-28T02:28:11.862 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:28:11.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:11.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:28:11.975 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:28:12.056 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:12.143 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 65 16 68376 68360 65.6786 52.75 0.0119702 0.0152178 2023-08-28T02:28:12.201 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:28:11 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:28:11] "GET /metrics HTTP/1.1" 200 36511 "" "Prometheus/2.33.4" 2023-08-28T02:28:12.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:12 smithi143 ceph-mon[189871]: failed to encode map e216 with expected crc 2023-08-28T02:28:12.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:12 smithi143 ceph-mon[189871]: failed to encode map e216 with expected crc 2023-08-28T02:28:12.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:12 smithi143 ceph-mon[189871]: pgmap v508: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering, 1 active+undersized+remapped, 2 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 39 MiB/s wr, 617 op/s; 10308/197097 objects degraded (5.230%); 2740/197097 objects misplaced (1.390%); 8.9 MiB/s, 142 objects/s recovering 2023-08-28T02:28:12.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:12 smithi143 ceph-mon[189871]: failed to encode map e217 with expected crc 2023-08-28T02:28:12.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:12 smithi143 ceph-mon[189871]: failed to encode map e217 with expected crc 2023-08-28T02:28:12.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:12 smithi143 ceph-mon[189871]: failed to encode map e217 with expected crc 2023-08-28T02:28:12.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:12 smithi143 ceph-mon[189871]: failed to encode map e217 with expected crc 2023-08-28T02:28:12.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:12 smithi143 ceph-mon[189871]: failed to encode map e217 with expected crc 2023-08-28T02:28:12.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:12 smithi143 ceph-mon[189871]: failed to encode map e217 with expected crc 2023-08-28T02:28:12.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:12 smithi143 ceph-mon[189871]: failed to encode map e217 with expected crc 2023-08-28T02:28:12.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:12 smithi143 ceph-mon[189871]: osdmap e218: 8 total, 8 up, 7 in 2023-08-28T02:28:12.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:12 smithi143 ceph-mon[189871]: failed to encode map e218 with expected crc 2023-08-28T02:28:12.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:12 smithi143 ceph-mon[189871]: failed to encode map e218 with expected crc 2023-08-28T02:28:12.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:12 smithi143 ceph-mon[183059]: failed to encode map e216 with expected crc 2023-08-28T02:28:12.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:12 smithi143 ceph-mon[183059]: failed to encode map e216 with expected crc 2023-08-28T02:28:12.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:12 smithi143 ceph-mon[183059]: pgmap v508: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering, 1 active+undersized+remapped, 2 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 39 MiB/s wr, 617 op/s; 10308/197097 objects degraded (5.230%); 2740/197097 objects misplaced (1.390%); 8.9 MiB/s, 142 objects/s recovering 2023-08-28T02:28:12.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:12 smithi143 ceph-mon[183059]: failed to encode map e217 with expected crc 2023-08-28T02:28:12.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:12 smithi143 ceph-mon[183059]: failed to encode map e217 with expected crc 2023-08-28T02:28:12.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:12 smithi143 ceph-mon[183059]: failed to encode map e217 with expected crc 2023-08-28T02:28:12.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:12 smithi143 ceph-mon[183059]: failed to encode map e217 with expected crc 2023-08-28T02:28:12.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:12 smithi143 ceph-mon[183059]: failed to encode map e217 with expected crc 2023-08-28T02:28:12.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:12 smithi143 ceph-mon[183059]: failed to encode map e217 with expected crc 2023-08-28T02:28:12.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:12 smithi143 ceph-mon[183059]: failed to encode map e217 with expected crc 2023-08-28T02:28:12.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:12 smithi143 ceph-mon[183059]: osdmap e218: 8 total, 8 up, 7 in 2023-08-28T02:28:12.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:12 smithi143 ceph-mon[183059]: failed to encode map e218 with expected crc 2023-08-28T02:28:12.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:12 smithi143 ceph-mon[183059]: failed to encode map e218 with expected crc 2023-08-28T02:28:12.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:12 smithi169 ceph-mon[159415]: failed to encode map e216 with expected crc 2023-08-28T02:28:12.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:12 smithi169 ceph-mon[159415]: failed to encode map e216 with expected crc 2023-08-28T02:28:12.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:12 smithi169 ceph-mon[159415]: pgmap v508: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering, 1 active+undersized+remapped, 2 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 39 MiB/s wr, 617 op/s; 10308/197097 objects degraded (5.230%); 2740/197097 objects misplaced (1.390%); 8.9 MiB/s, 142 objects/s recovering 2023-08-28T02:28:12.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:12 smithi169 ceph-mon[159415]: failed to encode map e217 with expected crc 2023-08-28T02:28:12.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:12 smithi169 ceph-mon[159415]: failed to encode map e217 with expected crc 2023-08-28T02:28:12.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:12 smithi169 ceph-mon[159415]: failed to encode map e217 with expected crc 2023-08-28T02:28:12.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:12 smithi169 ceph-mon[159415]: failed to encode map e217 with expected crc 2023-08-28T02:28:12.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:12 smithi169 ceph-mon[159415]: failed to encode map e217 with expected crc 2023-08-28T02:28:12.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:12 smithi169 ceph-mon[159415]: failed to encode map e217 with expected crc 2023-08-28T02:28:12.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:12 smithi169 ceph-mon[159415]: failed to encode map e217 with expected crc 2023-08-28T02:28:12.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:12 smithi169 ceph-mon[159415]: osdmap e218: 8 total, 8 up, 7 in 2023-08-28T02:28:12.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:12 smithi169 ceph-mon[159415]: failed to encode map e218 with expected crc 2023-08-28T02:28:12.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:12 smithi169 ceph-mon[159415]: failed to encode map e218 with expected crc 2023-08-28T02:28:12.817 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:28:12.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:12.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:28:12.926 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:28:12.957 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:13.012 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:28:13.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:13.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:28:13.116 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:28:13.144 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 66 16 69196 69180 65.4579 51.25 0.0292812 0.015271 2023-08-28T02:28:13.193 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:13.314 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:28:13.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:13.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:28:13.428 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:28:13.439 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:13 smithi143 ceph-mon[183059]: failed to encode map e218 with expected crc 2023-08-28T02:28:13.439 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:13 smithi143 ceph-mon[183059]: failed to encode map e218 with expected crc 2023-08-28T02:28:13.439 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:13 smithi143 ceph-mon[183059]: failed to encode map e218 with expected crc 2023-08-28T02:28:13.457 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:13.508 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:28:13.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:13.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:28:13.616 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:28:13.672 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:13.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:13 smithi143 ceph-mon[189871]: failed to encode map e218 with expected crc 2023-08-28T02:28:13.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:13 smithi143 ceph-mon[189871]: failed to encode map e218 with expected crc 2023-08-28T02:28:13.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:13 smithi143 ceph-mon[189871]: failed to encode map e218 with expected crc 2023-08-28T02:28:13.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:13 smithi169 ceph-mon[159415]: failed to encode map e218 with expected crc 2023-08-28T02:28:13.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:13 smithi169 ceph-mon[159415]: failed to encode map e218 with expected crc 2023-08-28T02:28:13.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:13 smithi169 ceph-mon[159415]: failed to encode map e218 with expected crc 2023-08-28T02:28:13.941 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:28:13.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:13.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:28:14.064 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:28:14.065 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:28:14.065 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:28:14.096 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:14.144 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 67 16 69984 69968 65.2161 49.25 0.0157379 0.0153268 2023-08-28T02:28:14.145 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:28:14.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:14.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:28:14.249 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:28:14.278 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:14.326 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:28:14.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:14.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:28:14.434 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:28:14.474 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:14.479 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is unset 2023-08-28T02:28:14.532 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:28:14.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:14.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:28:14.639 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:28:14.672 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:14.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:14 smithi143 ceph-mon[183059]: pgmap v511: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering, 1 active+undersized+remapped, 2 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 41 MiB/s wr, 651 op/s; 10308/201306 objects degraded (5.121%); 2740/201306 objects misplaced (1.361%); 4.5 MiB/s, 70 objects/s recovering 2023-08-28T02:28:14.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:14 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/291869554' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:28:14.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:14 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:28:14.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:14 smithi143 ceph-mon[189871]: pgmap v511: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering, 1 active+undersized+remapped, 2 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 41 MiB/s wr, 651 op/s; 10308/201306 objects degraded (5.121%); 2740/201306 objects misplaced (1.361%); 4.5 MiB/s, 70 objects/s recovering 2023-08-28T02:28:14.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:14 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/291869554' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:28:14.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:14 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:28:14.730 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:28:14.734 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:14 smithi169 ceph-mon[159415]: pgmap v511: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 1 active+recovering, 1 active+undersized+remapped, 2 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 41 MiB/s wr, 651 op/s; 10308/201306 objects degraded (5.121%); 2740/201306 objects misplaced (1.361%); 4.5 MiB/s, 70 objects/s recovering 2023-08-28T02:28:14.734 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:14 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/291869554' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:28:14.734 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:14 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:28:14.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:14.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:28:14.836 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:28:14.865 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:15.145 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 68 16 70731 70715 64.9438 46.6875 0.0187333 0.0153921 2023-08-28T02:28:15.165 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:28:15.451 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:28:15.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:15.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:28:15.565 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:28:15.612 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:15 smithi143 ceph-mon[183059]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:28:15.612 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:15 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:28:15.612 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:15 smithi143 ceph-mon[183059]: osdmap e219: 8 total, 8 up, 7 in 2023-08-28T02:28:15.612 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:15 smithi143 ceph-mon[183059]: failed to encode map e219 with expected crc 2023-08-28T02:28:15.612 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:15 smithi143 ceph-mon[183059]: failed to encode map e219 with expected crc 2023-08-28T02:28:15.612 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:15 smithi143 ceph-mon[183059]: failed to encode map e219 with expected crc 2023-08-28T02:28:15.612 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:15 smithi143 ceph-mon[183059]: failed to encode map e219 with expected crc 2023-08-28T02:28:15.612 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:15 smithi143 ceph-mon[183059]: failed to encode map e219 with expected crc 2023-08-28T02:28:15.612 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:15 smithi143 ceph-mon[183059]: failed to encode map e219 with expected crc 2023-08-28T02:28:15.613 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:15 smithi143 ceph-mon[183059]: failed to encode map e219 with expected crc 2023-08-28T02:28:15.613 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:15 smithi143 ceph-mon[183059]: pgmap v513: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+recovering, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 46 MiB/s wr, 734 op/s; 10308/207768 objects degraded (4.961%); 2288/207768 objects misplaced (1.101%); 5.8 MiB/s, 85 objects/s recovering 2023-08-28T02:28:15.613 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:15 smithi143 ceph-mon[189871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:28:15.613 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:15 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:28:15.613 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:15 smithi143 ceph-mon[189871]: osdmap e219: 8 total, 8 up, 7 in 2023-08-28T02:28:15.613 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:15 smithi143 ceph-mon[189871]: failed to encode map e219 with expected crc 2023-08-28T02:28:15.613 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:15 smithi143 ceph-mon[189871]: failed to encode map e219 with expected crc 2023-08-28T02:28:15.613 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:15 smithi143 ceph-mon[189871]: failed to encode map e219 with expected crc 2023-08-28T02:28:15.614 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:15 smithi143 ceph-mon[189871]: failed to encode map e219 with expected crc 2023-08-28T02:28:15.614 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:15 smithi143 ceph-mon[189871]: failed to encode map e219 with expected crc 2023-08-28T02:28:15.614 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:15 smithi143 ceph-mon[189871]: failed to encode map e219 with expected crc 2023-08-28T02:28:15.614 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:15 smithi143 ceph-mon[189871]: failed to encode map e219 with expected crc 2023-08-28T02:28:15.614 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:15 smithi143 ceph-mon[189871]: pgmap v513: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+recovering, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 46 MiB/s wr, 734 op/s; 10308/207768 objects degraded (4.961%); 2288/207768 objects misplaced (1.101%); 5.8 MiB/s, 85 objects/s recovering 2023-08-28T02:28:15.614 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:15.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:15 smithi169 ceph-mon[159415]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:28:15.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:15 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:28:15.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:15 smithi169 ceph-mon[159415]: osdmap e219: 8 total, 8 up, 7 in 2023-08-28T02:28:15.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:15 smithi169 ceph-mon[159415]: failed to encode map e219 with expected crc 2023-08-28T02:28:15.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:15 smithi169 ceph-mon[159415]: failed to encode map e219 with expected crc 2023-08-28T02:28:15.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:15 smithi169 ceph-mon[159415]: failed to encode map e219 with expected crc 2023-08-28T02:28:15.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:15 smithi169 ceph-mon[159415]: failed to encode map e219 with expected crc 2023-08-28T02:28:15.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:15 smithi169 ceph-mon[159415]: failed to encode map e219 with expected crc 2023-08-28T02:28:15.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:15 smithi169 ceph-mon[159415]: failed to encode map e219 with expected crc 2023-08-28T02:28:15.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:15 smithi169 ceph-mon[159415]: failed to encode map e219 with expected crc 2023-08-28T02:28:15.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:15 smithi169 ceph-mon[159415]: pgmap v513: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+recovering, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 46 MiB/s wr, 734 op/s; 10308/207768 objects degraded (4.961%); 2288/207768 objects misplaced (1.101%); 5.8 MiB/s, 85 objects/s recovering 2023-08-28T02:28:15.881 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.4 dump_ops_in_flight 2023-08-28T02:28:16.015 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:28:16.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:16.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:28:16.126 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:28:16.145 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 69 16 71520 71504 64.7173 49.3125 0.0242453 0.0154441 2023-08-28T02:28:16.241 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:16.305 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:28:16.309 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:28:16.376 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:28:16.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:16.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:28:16.498 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:28:16.541 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:16.609 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:28:16.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:16.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:28:16.714 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:28:16.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:16 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 10308/207768 objects degraded (4.961%), 13 pgs degraded (PG_DEGRADED) 2023-08-28T02:28:16.783 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:16 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 10308/207768 objects degraded (4.961%), 13 pgs degraded (PG_DEGRADED) 2023-08-28T02:28:16.786 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:16 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 10308/207768 objects degraded (4.961%), 13 pgs degraded (PG_DEGRADED) 2023-08-28T02:28:16.796 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:17.069 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:28:17.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:17.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:28:17.145 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 70 16 72473 72457 64.6437 59.5625 0.0113078 0.0154637 2023-08-28T02:28:17.180 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:28:17.244 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:17.365 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:28:17.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:17.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:28:17.470 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:28:17.507 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:17.569 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:28:17.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:17.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:28:17.674 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:28:17.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:17 smithi143 ceph-mon[189871]: pgmap v514: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+recovering, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 50 MiB/s wr, 795 op/s; 10308/211434 objects degraded (4.875%); 2288/211434 objects misplaced (1.082%); 4.7 MiB/s, 68 objects/s recovering 2023-08-28T02:28:17.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:17 smithi143 ceph-mon[183059]: pgmap v514: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+recovering, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 50 MiB/s wr, 795 op/s; 10308/211434 objects degraded (4.875%); 2288/211434 objects misplaced (1.082%); 4.7 MiB/s, 68 objects/s recovering 2023-08-28T02:28:17.710 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:17.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:17 smithi169 ceph-mon[159415]: pgmap v514: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+recovering, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 50 MiB/s wr, 795 op/s; 10308/211434 objects degraded (4.875%); 2288/211434 objects misplaced (1.082%); 4.7 MiB/s, 68 objects/s recovering 2023-08-28T02:28:17.766 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:28:17.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:17.769 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:28:17.870 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:28:17.907 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:17.963 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:28:17.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:17.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:28:18.061 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 7, 3, 1] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-08-28T02:28:18.061 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-08-28T02:28:18.061 INFO:tasks.thrashosds.thrasher:Adding osd 6 2023-08-28T02:28:18.062 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd in 6 2023-08-28T02:28:18.068 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:28:18.097 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:18.145 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 71 16 73446 73430 64.5895 60.8125 0.0128996 0.0154763 2023-08-28T02:28:18.147 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset nodeep-scrub 2023-08-28T02:28:18.150 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:28:18.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:18.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:28:18.254 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:28:18.287 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:18.499 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:28:18.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:18.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:28:18.603 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:28:18.636 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:18.720 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:28:18.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:18.723 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:28:18.824 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:28:18.856 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:18.867 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:28:18.914 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:28:18.915 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:18 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2780996739' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:28:18.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:18.916 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:28:18.928 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:28:18.946 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:28:18.974 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:28:19.017 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:28:19.025 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:28:19.056 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:28:19.062 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:28:19.116 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:28:19.139 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:18 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2780996739' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:28:19.140 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:28:19.140 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:18 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2780996739' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:28:19.144 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:19.146 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 72 16 74110 74094 64.2681 41.5 0.0145608 0.0155547 2023-08-28T02:28:19.174 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:28:19.186 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:28:19.186 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:28:19.186 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:28:19.186 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:28:19.186 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:28:19.186 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:28:19.186 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:28:19.186 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:28:19.297 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:28:19.298 INFO:teuthology.orchestra.run.smithi169.stdout: "ops": [ 2023-08-28T02:28:19.298 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:19.298 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:74228 6.e 6:7e9bce94:::benchmark_data_smithi143_305742_object74227:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e219)", 2023-08-28T02:28:19.298 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:28:19.287987+0000", 2023-08-28T02:28:19.299 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 0.010682373, 2023-08-28T02:28:19.299 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.010704991000000001, 2023-08-28T02:28:19.299 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:28:19.299 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "waiting for sub ops", 2023-08-28T02:28:19.299 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:28:19.299 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:28:19.299 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:28:19.299 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 74228 2023-08-28T02:28:19.300 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:19.300 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:28:19.300 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:19.300 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:28:19.300 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:19.287987+0000", 2023-08-28T02:28:19.300 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:28:19.300 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:19.300 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:19.300 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:28:19.300 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:19.287987+0000", 2023-08-28T02:28:19.301 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:28:19.301 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:19.301 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:19.301 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:28:19.301 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:19.287986+0000", 2023-08-28T02:28:19.302 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.3417e-05 2023-08-28T02:28:19.302 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:19.302 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:19.302 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:28:19.302 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:19.288009+0000", 2023-08-28T02:28:19.302 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0492999999999999e-05 2023-08-28T02:28:19.302 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:19.302 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:19.302 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:28:19.302 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:19.288020+0000", 2023-08-28T02:28:19.302 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.8839999999999999e-06 2023-08-28T02:28:19.302 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:19.303 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:19.303 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:28:19.303 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:19.288026+0000", 2023-08-28T02:28:19.303 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.2679999999999997e-05 2023-08-28T02:28:19.303 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:19.303 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:19.303 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:19.303 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:19.288078+0000", 2023-08-28T02:28:19.303 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000187288 2023-08-28T02:28:19.303 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:19.303 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:19.303 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:28:19.304 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:19.288266+0000", 2023-08-28T02:28:19.304 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00018443800000000001 2023-08-28T02:28:19.304 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:19.304 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:19.304 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 1,2", 2023-08-28T02:28:19.304 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:19.288450+0000", 2023-08-28T02:28:19.304 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0025637580000000002 2023-08-28T02:28:19.304 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:19.304 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:19.304 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:28:19.304 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:19.291014+0000", 2023-08-28T02:28:19.304 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0030267710000000001 2023-08-28T02:28:19.305 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:19.305 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:28:19.305 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:19.305 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:19.305 INFO:teuthology.orchestra.run.smithi169.stdout: ], 2023-08-28T02:28:19.305 INFO:teuthology.orchestra.run.smithi169.stdout: "num_ops": 1 2023-08-28T02:28:19.305 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:28:20.146 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 73 16 74938 74922 64.0967 51.75 0.00708272 0.0155936 2023-08-28T02:28:20.178 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:19 smithi169 ceph-mon[159415]: pgmap v515: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+recovering, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 61 MiB/s wr, 969 op/s; 10308/220095 objects degraded (4.683%); 1946/220095 objects misplaced (0.884%); 6.3 MiB/s, 95 objects/s recovering 2023-08-28T02:28:20.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:19 smithi143 ceph-mon[183059]: pgmap v515: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+recovering, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 61 MiB/s wr, 969 op/s; 10308/220095 objects degraded (4.683%); 1946/220095 objects misplaced (0.884%); 6.3 MiB/s, 95 objects/s recovering 2023-08-28T02:28:20.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:19 smithi143 ceph-mon[189871]: pgmap v515: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+recovering, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 61 MiB/s wr, 969 op/s; 10308/220095 objects degraded (4.683%); 1946/220095 objects misplaced (0.884%); 6.3 MiB/s, 95 objects/s recovering 2023-08-28T02:28:20.208 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.4 dump_blocked_ops 2023-08-28T02:28:20.613 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:28:20.846 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:28:20.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:20.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:28:20.955 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:28:20.990 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:21.146 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 74 16 75730 75714 63.8995 49.5 0.0603795 0.0156436 2023-08-28T02:28:21.251 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:28:21.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:21.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:28:21.355 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:28:21.385 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:21.435 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:28:21.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:21.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:28:21.539 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:28:21.568 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:21.610 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:28:21.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:21.613 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:28:21.713 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:28:21.745 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:22.040 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:28:22.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:22.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:28:22.146 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 75 16 76546 76530 63.7275 51 0.00182518 0.0156746 2023-08-28T02:28:22.147 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:28:22.187 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:28:21 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:28:21] "GET /metrics HTTP/1.1" 200 36511 "" "Prometheus/2.33.4" 2023-08-28T02:28:22.190 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:22.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:22 smithi143 ceph-mon[183059]: pgmap v516: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+recovering, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 44 MiB/s wr, 703 op/s; 10308/220095 objects degraded (4.683%); 1946/220095 objects misplaced (0.884%); 5.6 MiB/s, 85 objects/s recovering 2023-08-28T02:28:22.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:22 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 10308/220095 objects degraded (4.683%), 13 pgs degraded (PG_DEGRADED) 2023-08-28T02:28:22.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:22 smithi143 ceph-mon[189871]: pgmap v516: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+recovering, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 44 MiB/s wr, 703 op/s; 10308/220095 objects degraded (4.683%); 1946/220095 objects misplaced (0.884%); 5.6 MiB/s, 85 objects/s recovering 2023-08-28T02:28:22.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:22 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 10308/220095 objects degraded (4.683%), 13 pgs degraded (PG_DEGRADED) 2023-08-28T02:28:22.528 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:28:22.529 INFO:teuthology.orchestra.run.smithi169.stdout: "ops": [], 2023-08-28T02:28:22.529 INFO:teuthology.orchestra.run.smithi169.stdout: "complaint_time": 30, 2023-08-28T02:28:22.529 INFO:teuthology.orchestra.run.smithi169.stdout: "num_blocked_ops": 0 2023-08-28T02:28:22.529 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:28:22.558 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:22 smithi169 ceph-mon[159415]: pgmap v516: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+recovering, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 44 MiB/s wr, 703 op/s; 10308/220095 objects degraded (4.683%); 1946/220095 objects misplaced (0.884%); 5.6 MiB/s, 85 objects/s recovering 2023-08-28T02:28:22.558 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:22 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 10308/220095 objects degraded (4.683%), 13 pgs degraded (PG_DEGRADED) 2023-08-28T02:28:22.636 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:28:22.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:22.640 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:28:22.741 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:28:22.824 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:22.992 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:28:22.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:22.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:28:23.093 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:28:23.099 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:28:23.147 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 76 16 77238 77222 63.4582 43.25 0.012685 0.0157525 2023-08-28T02:28:23.211 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:23.367 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.4 dump_historic_ops 2023-08-28T02:28:23.381 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:28:23.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:23.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:28:23.496 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:28:23.529 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:23.585 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:28:23.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:23.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:28:23.692 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:28:23.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:23 smithi143 conmon[183035]: 2023-08-28T02:28:23.561+0000 7fadc3887700 -1 mon.a@0(leader).osd e219 definitely_dead 0 2023-08-28T02:28:23.800 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:28:23.819 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:23.873 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:28:23.943 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:28:23.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:23.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:28:24.052 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:28:24.091 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:24.150 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:28:24.150 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 77 16 77982 77966 63.238 46.5 0.0221896 0.0158079 2023-08-28T02:28:24.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:24.153 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:28:24.254 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:28:24.334 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:24.340 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:24 smithi169 ceph-mon[159415]: pgmap v517: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+recovering, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 50 MiB/s wr, 799 op/s; 10308/225339 objects degraded (4.574%); 1946/225339 objects misplaced (0.864%); 5.0 MiB/s, 76 objects/s recovering 2023-08-28T02:28:24.341 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:24 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/333409677' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-08-28T02:28:24.342 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:24 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3822106670' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:28:24.342 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:24 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:28:24.351 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is unset 2023-08-28T02:28:24.367 INFO:teuthology.orchestra.run.smithi143.stderr:marked in osd.6. 2023-08-28T02:28:24.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:24 smithi143 ceph-mon[183059]: pgmap v517: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+recovering, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 50 MiB/s wr, 799 op/s; 10308/225339 objects degraded (4.574%); 1946/225339 objects misplaced (0.864%); 5.0 MiB/s, 76 objects/s recovering 2023-08-28T02:28:24.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:24 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/333409677' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-08-28T02:28:24.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:24 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3822106670' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:28:24.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:24 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:28:24.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:24 smithi143 ceph-mon[189871]: pgmap v517: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+recovering, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 50 MiB/s wr, 799 op/s; 10308/225339 objects degraded (4.574%); 1946/225339 objects misplaced (0.864%); 5.0 MiB/s, 76 objects/s recovering 2023-08-28T02:28:24.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:24 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/333409677' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-08-28T02:28:24.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:24 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3822106670' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:28:24.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:24 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:28:25.147 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 78 16 78184 78168 62.5895 12.625 0.0789006 0.0158137 2023-08-28T02:28:25.408 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:25 smithi169 ceph-mon[159415]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:28:25.408 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:25 smithi169 ceph-mon[159415]: failed to encode map e220 with expected crc 2023-08-28T02:28:25.408 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:25 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/333409677' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-08-28T02:28:25.408 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:25 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:28:25.409 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:25 smithi169 ceph-mon[159415]: osdmap e220: 8 total, 8 up, 8 in 2023-08-28T02:28:25.409 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:25 smithi169 ceph-mon[159415]: failed to encode map e220 with expected crc 2023-08-28T02:28:25.409 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:25 smithi169 ceph-mon[159415]: osdmap e221: 8 total, 8 up, 8 in 2023-08-28T02:28:25.640 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:28:25.641 INFO:teuthology.orchestra.run.smithi169.stdout: "size": 20, 2023-08-28T02:28:25.641 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 600, 2023-08-28T02:28:25.641 INFO:teuthology.orchestra.run.smithi169.stdout: "ops": [ 2023-08-28T02:28:25.641 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.641 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:114254 5.1 5:8f7eaa26:::benchmark_data_smithi143_236200_object4111:head [delete] snapc 0=[] ondisk+write+known_if_redirected e147)", 2023-08-28T02:28:25.641 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:43.862931+0000", 2023-08-28T02:28:25.641 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 281.77552857400002, 2023-08-28T02:28:25.641 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0947160739999999, 2023-08-28T02:28:25.641 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:28:25.641 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "reached pg", 2023-08-28T02:28:25.642 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:28:25.642 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:28:25.642 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:28:25.642 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 114254 2023-08-28T02:28:25.642 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.642 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:28:25.642 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.642 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:28:25.642 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.862931+0000", 2023-08-28T02:28:25.642 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:28:25.642 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.642 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.642 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:28:25.643 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.862931+0000", 2023-08-28T02:28:25.643 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:28:25.643 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.643 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.643 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:28:25.643 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.862929+0000", 2023-08-28T02:28:25.643 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.157e-06 2023-08-28T02:28:25.643 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.643 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.643 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:28:25.644 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.862937+0000", 2023-08-28T02:28:25.644 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1687e-05 2023-08-28T02:28:25.644 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.644 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.644 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:28:25.644 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.862949+0000", 2023-08-28T02:28:25.644 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.3419999999999996e-06 2023-08-28T02:28:25.645 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.645 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.645 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:28:25.645 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.862955+0000", 2023-08-28T02:28:25.645 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0034338319999999999 2023-08-28T02:28:25.645 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.645 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.645 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:25.646 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.866389+0000", 2023-08-28T02:28:25.646 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00011907499999999999 2023-08-28T02:28:25.646 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.646 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.646 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:28:25.646 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.866508+0000", 2023-08-28T02:28:25.647 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00014797899999999999 2023-08-28T02:28:25.647 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.647 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.647 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,7", 2023-08-28T02:28:25.647 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.866656+0000", 2023-08-28T02:28:25.647 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0075255089999999997 2023-08-28T02:28:25.648 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.648 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.648 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:28:25.648 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.874182+0000", 2023-08-28T02:28:25.648 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0046194599999999997 2023-08-28T02:28:25.648 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.648 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.648 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:25.649 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.878801+0000", 2023-08-28T02:28:25.649 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1385e-05 2023-08-28T02:28:25.649 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.649 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.649 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:28:25.649 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.878813+0000", 2023-08-28T02:28:25.649 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.078743335 2023-08-28T02:28:25.649 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.649 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.650 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:25.650 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.957556+0000", 2023-08-28T02:28:25.650 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.1067000000000001e-05 2023-08-28T02:28:25.650 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.650 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.650 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:28:25.650 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.957647+0000", 2023-08-28T02:28:25.650 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0947160739999999 2023-08-28T02:28:25.650 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:25.650 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:28:25.650 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:25.650 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.650 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.651 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:114255 5.1 5:8ac446c3:::benchmark_data_smithi143_236200_object4112:head [delete] snapc 0=[] ondisk+write+known_if_redirected e147)", 2023-08-28T02:28:25.651 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:43.862970+0000", 2023-08-28T02:28:25.651 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 281.77548959500001, 2023-08-28T02:28:25.651 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.10295312, 2023-08-28T02:28:25.651 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:28:25.651 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "reached pg", 2023-08-28T02:28:25.651 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:28:25.651 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:28:25.651 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:28:25.651 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 114255 2023-08-28T02:28:25.651 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.651 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:28:25.652 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.652 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:28:25.652 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.862970+0000", 2023-08-28T02:28:25.652 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:28:25.652 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.652 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.652 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:28:25.652 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.862970+0000", 2023-08-28T02:28:25.652 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999995 2023-08-28T02:28:25.652 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.652 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.652 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:28:25.652 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.862969+0000", 2023-08-28T02:28:25.653 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.7919999999999999e-06 2023-08-28T02:28:25.653 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.653 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.653 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:28:25.653 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.862973+0000", 2023-08-28T02:28:25.653 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.6030000000000001e-06 2023-08-28T02:28:25.653 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.653 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.653 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:28:25.653 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.862980+0000", 2023-08-28T02:28:25.653 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.2400000000000001e-06 2023-08-28T02:28:25.653 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.653 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.654 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:28:25.654 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.862984+0000", 2023-08-28T02:28:25.654 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0072338599999999999 2023-08-28T02:28:25.654 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.654 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.654 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:25.654 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.870218+0000", 2023-08-28T02:28:25.654 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000128653 2023-08-28T02:28:25.654 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.654 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.654 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:28:25.654 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.870347+0000", 2023-08-28T02:28:25.654 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000162411 2023-08-28T02:28:25.655 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.655 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.655 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,7", 2023-08-28T02:28:25.655 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.870509+0000", 2023-08-28T02:28:25.655 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0083309839999999996 2023-08-28T02:28:25.655 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.655 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.655 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:25.655 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.878840+0000", 2023-08-28T02:28:25.655 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.272e-06 2023-08-28T02:28:25.655 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.655 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.656 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:28:25.656 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.878847+0000", 2023-08-28T02:28:25.656 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.079164749 2023-08-28T02:28:25.656 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.656 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.656 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:25.656 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.958012+0000", 2023-08-28T02:28:25.656 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00015897099999999999 2023-08-28T02:28:25.656 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.656 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.656 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for missing object", 2023-08-28T02:28:25.656 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.958171+0000", 2023-08-28T02:28:25.656 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0076846830000000003 2023-08-28T02:28:25.657 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.657 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.657 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:25.657 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.965856+0000", 2023-08-28T02:28:25.657 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.7471000000000005e-05 2023-08-28T02:28:25.657 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.657 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.657 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:28:25.657 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.965923+0000", 2023-08-28T02:28:25.657 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.10295312 2023-08-28T02:28:25.657 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:25.657 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:28:25.658 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:25.658 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.658 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.658 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:114271 5.1 5:8ca8421e:::benchmark_data_smithi143_236200_object4128:head [delete] snapc 0=[] ondisk+write+known_if_redirected e148)", 2023-08-28T02:28:25.658 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:43.884816+0000", 2023-08-28T02:28:25.658 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 281.75364313199998, 2023-08-28T02:28:25.658 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0813256689999999, 2023-08-28T02:28:25.658 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:28:25.658 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:28:25.658 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:28:25.658 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:28:25.658 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:28:25.658 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 114271 2023-08-28T02:28:25.659 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.659 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:28:25.659 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.659 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:28:25.659 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.884816+0000", 2023-08-28T02:28:25.659 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:28:25.659 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.659 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.659 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:28:25.659 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.884816+0000", 2023-08-28T02:28:25.659 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:28:25.659 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.660 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.660 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:28:25.660 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.884815+0000", 2023-08-28T02:28:25.660 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.8809999999999998e-06 2023-08-28T02:28:25.660 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.660 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.660 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:28:25.660 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.884822+0000", 2023-08-28T02:28:25.660 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1715999999999999e-05 2023-08-28T02:28:25.660 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.660 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.660 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:28:25.661 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.884833+0000", 2023-08-28T02:28:25.661 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.3779999999999998e-06 2023-08-28T02:28:25.661 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.661 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.661 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:28:25.661 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.884840+0000", 2023-08-28T02:28:25.661 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0016300780000000001 2023-08-28T02:28:25.661 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.661 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.661 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:25.661 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.886470+0000", 2023-08-28T02:28:25.661 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.6500000000000002e-06 2023-08-28T02:28:25.661 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.662 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.662 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:28:25.662 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.886478+0000", 2023-08-28T02:28:25.662 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0717474890000001 2023-08-28T02:28:25.662 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.662 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.662 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:25.662 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.958226+0000", 2023-08-28T02:28:25.662 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012793900000000001 2023-08-28T02:28:25.662 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.662 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.662 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:28:25.662 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.958354+0000", 2023-08-28T02:28:25.663 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000149421 2023-08-28T02:28:25.663 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.663 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.663 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,7", 2023-08-28T02:28:25.663 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.958503+0000", 2023-08-28T02:28:25.663 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0070383570000000003 2023-08-28T02:28:25.663 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.663 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.663 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:25.663 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.965542+0000", 2023-08-28T02:28:25.664 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.3148e-05 2023-08-28T02:28:25.664 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.664 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.664 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:28:25.664 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.965555+0000", 2023-08-28T02:28:25.664 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00044809299999999999 2023-08-28T02:28:25.664 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.664 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.664 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:25.664 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.966003+0000", 2023-08-28T02:28:25.665 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.1020999999999998e-05 2023-08-28T02:28:25.665 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.665 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.665 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:28:25.665 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.966054+0000", 2023-08-28T02:28:25.665 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.8335e-05 2023-08-28T02:28:25.665 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.665 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.666 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:28:25.666 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.966142+0000", 2023-08-28T02:28:25.666 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0813256689999999 2023-08-28T02:28:25.666 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:25.666 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:28:25.666 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:25.666 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.666 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.667 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:114280 5.1 5:85701856:::benchmark_data_smithi143_236200_object4137:head [delete] snapc 0=[] ondisk+write+known_if_redirected e148)", 2023-08-28T02:28:25.667 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:43.900301+0000", 2023-08-28T02:28:25.667 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 281.73815825299999, 2023-08-28T02:28:25.667 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0660135610000001, 2023-08-28T02:28:25.667 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:28:25.667 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:28:25.667 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:28:25.667 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:28:25.668 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:28:25.668 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 114280 2023-08-28T02:28:25.668 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.668 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:28:25.668 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.668 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:28:25.668 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.900301+0000", 2023-08-28T02:28:25.668 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:28:25.668 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.668 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.669 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:28:25.669 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.900301+0000", 2023-08-28T02:28:25.669 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:28:25.669 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.669 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.669 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:28:25.669 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.900299+0000", 2023-08-28T02:28:25.669 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.8669999999999999e-06 2023-08-28T02:28:25.670 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.670 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.670 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:28:25.670 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.900305+0000", 2023-08-28T02:28:25.670 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1285e-05 2023-08-28T02:28:25.670 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.670 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.670 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:28:25.671 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.900317+0000", 2023-08-28T02:28:25.671 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.5029999999999997e-06 2023-08-28T02:28:25.671 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.671 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.671 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:28:25.671 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.900322+0000", 2023-08-28T02:28:25.671 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.002364484 2023-08-28T02:28:25.671 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.672 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.672 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:25.672 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.902687+0000", 2023-08-28T02:28:25.672 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2643000000000001e-05 2023-08-28T02:28:25.672 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.672 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.672 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:28:25.672 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.902699+0000", 2023-08-28T02:28:25.673 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0592813210000001 2023-08-28T02:28:25.673 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.673 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.673 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:25.673 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.961981+0000", 2023-08-28T02:28:25.673 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000131493 2023-08-28T02:28:25.673 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.673 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.674 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:28:25.674 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.962112+0000", 2023-08-28T02:28:25.674 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00016603899999999999 2023-08-28T02:28:25.674 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.674 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.674 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,7", 2023-08-28T02:28:25.674 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.962278+0000", 2023-08-28T02:28:25.674 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0036423079999999999 2023-08-28T02:28:25.675 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.675 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.675 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:28:25.675 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.965920+0000", 2023-08-28T02:28:25.675 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00028787099999999998 2023-08-28T02:28:25.675 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.675 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.675 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:25.676 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.966208+0000", 2023-08-28T02:28:25.676 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.5313e-05 2023-08-28T02:28:25.676 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.676 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.676 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:25.676 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.966234+0000", 2023-08-28T02:28:25.676 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.6737000000000002e-05 2023-08-28T02:28:25.676 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.677 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.677 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:28:25.677 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.966260+0000", 2023-08-28T02:28:25.677 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.4552999999999998e-05 2023-08-28T02:28:25.677 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.677 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.677 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:28:25.677 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.966315+0000", 2023-08-28T02:28:25.677 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0660135610000001 2023-08-28T02:28:25.678 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:25.678 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:28:25.678 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:25.678 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.678 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.678 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:114309 5.1 5:87b49cb9:::benchmark_data_smithi143_236200_object4166:head [delete] snapc 0=[] ondisk+write+known_if_redirected e148)", 2023-08-28T02:28:25.678 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:43.938203+0000", 2023-08-28T02:28:25.678 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 281.70025709800001, 2023-08-28T02:28:25.679 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0461147120000001, 2023-08-28T02:28:25.679 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:28:25.679 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:28:25.679 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:28:25.679 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:28:25.679 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:28:25.679 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 114309 2023-08-28T02:28:25.679 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.679 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:28:25.680 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.680 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:28:25.680 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.938203+0000", 2023-08-28T02:28:25.680 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:28:25.680 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.680 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.680 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:28:25.680 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.938203+0000", 2023-08-28T02:28:25.680 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999976 2023-08-28T02:28:25.681 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.681 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.682 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:28:25.683 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.938200+0000", 2023-08-28T02:28:25.683 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.6900000000000004e-06 2023-08-28T02:28:25.683 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.683 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.683 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:28:25.683 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.938210+0000", 2023-08-28T02:28:25.683 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.3209e-05 2023-08-28T02:28:25.683 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.683 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.683 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:28:25.683 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.938223+0000", 2023-08-28T02:28:25.683 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.7580000000000003e-06 2023-08-28T02:28:25.684 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.684 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.684 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:28:25.684 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.938230+0000", 2023-08-28T02:28:25.684 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.0442999999999998e-05 2023-08-28T02:28:25.684 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.685 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.685 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:25.685 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.938280+0000", 2023-08-28T02:28:25.685 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0271e-05 2023-08-28T02:28:25.685 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.685 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.686 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:28:25.686 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.938291+0000", 2023-08-28T02:28:25.686 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0280542239999999 2023-08-28T02:28:25.686 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.686 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.686 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:25.687 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.966345+0000", 2023-08-28T02:28:25.687 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.8124e-05 2023-08-28T02:28:25.687 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.687 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.687 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:28:25.687 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.966413+0000", 2023-08-28T02:28:25.687 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000117788 2023-08-28T02:28:25.687 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.688 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.688 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,7", 2023-08-28T02:28:25.688 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.966531+0000", 2023-08-28T02:28:25.688 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.010700407 2023-08-28T02:28:25.688 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.688 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.688 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:25.688 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.977231+0000", 2023-08-28T02:28:25.689 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0069568429999999999 2023-08-28T02:28:25.689 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.689 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.689 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:25.689 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.984188+0000", 2023-08-28T02:28:25.689 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.8620000000000003e-06 2023-08-28T02:28:25.689 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.690 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.690 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:28:25.690 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.984196+0000", 2023-08-28T02:28:25.690 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.7914999999999999e-05 2023-08-28T02:28:25.690 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.690 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.690 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:28:25.690 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.984234+0000", 2023-08-28T02:28:25.691 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.3529999999999995e-05 2023-08-28T02:28:25.691 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.691 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.691 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:28:25.691 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.984317+0000", 2023-08-28T02:28:25.691 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0461147120000001 2023-08-28T02:28:25.691 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:25.691 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:28:25.691 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:25.691 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.691 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.691 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:114345 5.1 5:8edcacfd:::benchmark_data_smithi143_236200_object4202:head [delete] snapc 0=[] ondisk+write+known_if_redirected e148)", 2023-08-28T02:28:25.692 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:43.981100+0000", 2023-08-28T02:28:25.692 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 281.65735991100001, 2023-08-28T02:28:25.692 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0099165400000001, 2023-08-28T02:28:25.692 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:28:25.692 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:28:25.692 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:28:25.692 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:28:25.692 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:28:25.692 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 114345 2023-08-28T02:28:25.692 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.692 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:28:25.692 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.692 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:28:25.693 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.981100+0000", 2023-08-28T02:28:25.693 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:28:25.693 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.693 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.693 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:28:25.693 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.981100+0000", 2023-08-28T02:28:25.693 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:28:25.693 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.693 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.693 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:28:25.693 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.981098+0000", 2023-08-28T02:28:25.693 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.0620000000000007e-06 2023-08-28T02:28:25.693 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.693 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.694 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:28:25.694 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.981107+0000", 2023-08-28T02:28:25.694 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.3488e-05 2023-08-28T02:28:25.694 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.694 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.694 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:28:25.694 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.981120+0000", 2023-08-28T02:28:25.694 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.5180000000000001e-06 2023-08-28T02:28:25.694 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.694 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.694 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:28:25.694 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.981127+0000", 2023-08-28T02:28:25.694 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.5450000000000004e-05 2023-08-28T02:28:25.695 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.695 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.695 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:25.695 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.981202+0000", 2023-08-28T02:28:25.695 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1759999999999999e-05 2023-08-28T02:28:25.695 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.695 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.695 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:28:25.695 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.981214+0000", 2023-08-28T02:28:25.695 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.98907669099999995 2023-08-28T02:28:25.695 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.695 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.695 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:25.696 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.970291+0000", 2023-08-28T02:28:25.696 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.2393999999999998e-05 2023-08-28T02:28:25.696 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.696 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.696 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:28:25.696 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.970373+0000", 2023-08-28T02:28:25.696 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00024490299999999998 2023-08-28T02:28:25.696 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.696 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.697 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,7", 2023-08-28T02:28:25.697 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.970618+0000", 2023-08-28T02:28:25.697 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0066488270000000004 2023-08-28T02:28:25.697 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.697 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.697 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:25.697 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.977267+0000", 2023-08-28T02:28:25.697 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0070592379999999998 2023-08-28T02:28:25.697 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.697 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.697 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:28:25.697 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.984326+0000", 2023-08-28T02:28:25.697 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.006583144 2023-08-28T02:28:25.698 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.698 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.698 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:25.698 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.990909+0000", 2023-08-28T02:28:25.698 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.7954000000000003e-05 2023-08-28T02:28:25.698 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.698 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.698 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:28:25.698 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.990947+0000", 2023-08-28T02:28:25.698 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.9080999999999998e-05 2023-08-28T02:28:25.698 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.698 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.698 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:28:25.699 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.991016+0000", 2023-08-28T02:28:25.699 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0099165400000001 2023-08-28T02:28:25.699 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:25.699 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:28:25.699 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:25.699 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.699 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.699 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:114347 5.1 5:8f8588e4:::benchmark_data_smithi143_236200_object4204:head [delete] snapc 0=[] ondisk+write+known_if_redirected e148)", 2023-08-28T02:28:25.699 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:43.984527+0000", 2023-08-28T02:28:25.699 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 281.65393302799998, 2023-08-28T02:28:25.699 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.006606983, 2023-08-28T02:28:25.699 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:28:25.700 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:28:25.700 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:28:25.700 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:28:25.700 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:28:25.700 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 114347 2023-08-28T02:28:25.700 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.700 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:28:25.700 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.700 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:28:25.700 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.984527+0000", 2023-08-28T02:28:25.700 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:28:25.700 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.700 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.701 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:28:25.701 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.984527+0000", 2023-08-28T02:28:25.701 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:28:25.701 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.701 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.701 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:28:25.701 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.984525+0000", 2023-08-28T02:28:25.701 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.2899999999999997e-06 2023-08-28T02:28:25.701 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.701 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.702 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:28:25.702 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.984532+0000", 2023-08-28T02:28:25.702 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.3574000000000001e-05 2023-08-28T02:28:25.702 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.702 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.702 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:28:25.702 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.984545+0000", 2023-08-28T02:28:25.702 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.204e-06 2023-08-28T02:28:25.702 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.703 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.703 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:28:25.703 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.984553+0000", 2023-08-28T02:28:25.703 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0026792869999999998 2023-08-28T02:28:25.703 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.703 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.703 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:25.703 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.987232+0000", 2023-08-28T02:28:25.703 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2547e-05 2023-08-28T02:28:25.703 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.703 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.703 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:28:25.704 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.987244+0000", 2023-08-28T02:28:25.704 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.98630143800000003 2023-08-28T02:28:25.704 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.704 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.704 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:25.704 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.973546+0000", 2023-08-28T02:28:25.704 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.5401000000000004e-05 2023-08-28T02:28:25.704 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.704 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.704 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:28:25.704 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.973641+0000", 2023-08-28T02:28:25.705 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012306799999999999 2023-08-28T02:28:25.705 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.705 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.705 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,7", 2023-08-28T02:28:25.705 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.973764+0000", 2023-08-28T02:28:25.705 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.010567417000000001 2023-08-28T02:28:25.705 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.705 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.705 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:28:25.705 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.984332+0000", 2023-08-28T02:28:25.705 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.0612999999999999e-05 2023-08-28T02:28:25.705 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.705 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.706 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:25.706 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.984372+0000", 2023-08-28T02:28:25.706 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0066857319999999998 2023-08-28T02:28:25.706 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.706 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.706 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:25.706 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.991058+0000", 2023-08-28T02:28:25.706 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.2064000000000001e-05 2023-08-28T02:28:25.706 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.706 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.706 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:28:25.706 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.991080+0000", 2023-08-28T02:28:25.707 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.3366999999999999e-05 2023-08-28T02:28:25.707 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.707 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.707 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:28:25.707 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.991134+0000", 2023-08-28T02:28:25.707 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.006606983 2023-08-28T02:28:25.707 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:25.707 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:28:25.707 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:25.707 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.707 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.707 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:114352 5.1 5:89a4ee05:::benchmark_data_smithi143_236200_object4209:head [delete] snapc 0=[] ondisk+write+known_if_redirected e148)", 2023-08-28T02:28:25.707 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:43.989451+0000", 2023-08-28T02:28:25.708 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 281.64900814800001, 2023-08-28T02:28:25.708 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0017969799999999, 2023-08-28T02:28:25.708 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:28:25.708 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:28:25.708 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:28:25.708 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:28:25.708 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:28:25.708 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 114352 2023-08-28T02:28:25.708 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.708 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:28:25.708 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.708 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:28:25.708 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.989451+0000", 2023-08-28T02:28:25.709 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:28:25.709 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.709 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.709 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:28:25.709 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.989451+0000", 2023-08-28T02:28:25.709 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:28:25.709 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.709 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.709 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:28:25.709 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.989450+0000", 2023-08-28T02:28:25.709 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.8460000000000002e-06 2023-08-28T02:28:25.709 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.709 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.710 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:28:25.710 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.989457+0000", 2023-08-28T02:28:25.710 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.8840000000000006e-06 2023-08-28T02:28:25.710 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.710 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.710 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:28:25.710 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.989467+0000", 2023-08-28T02:28:25.710 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.9309999999999996e-06 2023-08-28T02:28:25.710 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.710 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.710 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:28:25.710 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.989473+0000", 2023-08-28T02:28:25.710 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.9379999999999998e-05 2023-08-28T02:28:25.711 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.711 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.711 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:25.711 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.989522+0000", 2023-08-28T02:28:25.711 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.7560000000000005e-06 2023-08-28T02:28:25.711 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.711 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.711 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:28:25.711 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:43.989531+0000", 2023-08-28T02:28:25.711 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.98783754999999995 2023-08-28T02:28:25.711 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.711 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.712 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:25.712 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.977368+0000", 2023-08-28T02:28:25.712 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000131094 2023-08-28T02:28:25.712 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.712 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.712 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:28:25.712 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.977499+0000", 2023-08-28T02:28:25.712 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00014025499999999999 2023-08-28T02:28:25.712 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.712 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.712 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,7", 2023-08-28T02:28:25.712 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.977640+0000", 2023-08-28T02:28:25.712 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0067410400000000002 2023-08-28T02:28:25.713 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.713 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.713 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:28:25.713 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.984381+0000", 2023-08-28T02:28:25.713 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0066601270000000001 2023-08-28T02:28:25.713 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.713 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.713 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:25.713 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.991041+0000", 2023-08-28T02:28:25.713 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00013248099999999999 2023-08-28T02:28:25.713 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.713 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.713 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:25.714 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.991173+0000", 2023-08-28T02:28:25.714 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.2561999999999998e-05 2023-08-28T02:28:25.714 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.714 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.714 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:28:25.714 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.991196+0000", 2023-08-28T02:28:25.714 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.2604e-05 2023-08-28T02:28:25.714 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.714 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.714 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:28:25.714 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.991248+0000", 2023-08-28T02:28:25.714 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0017969799999999 2023-08-28T02:28:25.714 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:25.715 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:28:25.715 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:25.715 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.715 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.715 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:114363 5.1 5:890fd397:::benchmark_data_smithi143_236200_object4220:head [delete] snapc 0=[] ondisk+write+known_if_redirected e148)", 2023-08-28T02:28:25.715 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:44.004622+0000", 2023-08-28T02:28:25.715 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 281.63383751499998, 2023-08-28T02:28:25.715 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.99759494800000004, 2023-08-28T02:28:25.715 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:28:25.715 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:28:25.715 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:28:25.715 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:28:25.716 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:28:25.716 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 114363 2023-08-28T02:28:25.716 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.716 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:28:25.716 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.716 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:28:25.716 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.004622+0000", 2023-08-28T02:28:25.716 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:28:25.716 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.716 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.716 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:28:25.716 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.004622+0000", 2023-08-28T02:28:25.717 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:28:25.717 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.717 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.717 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:28:25.717 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.004621+0000", 2023-08-28T02:28:25.717 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.4139999999999997e-06 2023-08-28T02:28:25.717 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.717 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.717 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:28:25.717 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.004628+0000", 2023-08-28T02:28:25.717 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.3670000000000003e-06 2023-08-28T02:28:25.717 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.717 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.718 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:28:25.718 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.004637+0000", 2023-08-28T02:28:25.718 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.3399999999999997e-06 2023-08-28T02:28:25.718 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.718 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.718 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:28:25.718 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.004643+0000", 2023-08-28T02:28:25.718 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.3288999999999998e-05 2023-08-28T02:28:25.718 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.718 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.718 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:25.718 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.004696+0000", 2023-08-28T02:28:25.719 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2778e-05 2023-08-28T02:28:25.719 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.719 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.719 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:28:25.719 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.004709+0000", 2023-08-28T02:28:25.719 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.97640097000000003 2023-08-28T02:28:25.719 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.719 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.719 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:25.719 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.981110+0000", 2023-08-28T02:28:25.719 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00014084300000000001 2023-08-28T02:28:25.719 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.720 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.720 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:28:25.720 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.981251+0000", 2023-08-28T02:28:25.720 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.3012999999999999e-05 2023-08-28T02:28:25.720 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.720 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.720 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,7", 2023-08-28T02:28:25.720 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.981344+0000", 2023-08-28T02:28:25.720 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.009928203 2023-08-28T02:28:25.720 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.720 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.720 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:25.720 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.991272+0000", 2023-08-28T02:28:25.721 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0073894390000000003 2023-08-28T02:28:25.721 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.721 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.721 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:25.721 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.998661+0000", 2023-08-28T02:28:25.721 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0033621649999999999 2023-08-28T02:28:25.721 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.721 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.721 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:28:25.721 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.002023+0000", 2023-08-28T02:28:25.721 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.8695e-05 2023-08-28T02:28:25.721 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.721 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.722 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:28:25.722 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.002082+0000", 2023-08-28T02:28:25.722 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00013499700000000001 2023-08-28T02:28:25.722 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.722 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.722 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:28:25.722 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.002217+0000", 2023-08-28T02:28:25.722 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.99759494800000004 2023-08-28T02:28:25.722 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:25.722 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:28:25.722 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:25.722 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.723 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.723 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:114367 5.1 5:8b417cad:::benchmark_data_smithi143_236200_object4224:head [delete] snapc 0=[] ondisk+write+known_if_redirected e148)", 2023-08-28T02:28:25.723 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:44.007552+0000", 2023-08-28T02:28:25.723 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 281.63090743200001, 2023-08-28T02:28:25.723 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.99493006100000003, 2023-08-28T02:28:25.723 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:28:25.723 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:28:25.723 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:28:25.723 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:28:25.723 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:28:25.723 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 114367 2023-08-28T02:28:25.723 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.723 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:28:25.724 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.724 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:28:25.724 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.007552+0000", 2023-08-28T02:28:25.724 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:28:25.724 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.724 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.724 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:28:25.724 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.007552+0000", 2023-08-28T02:28:25.724 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:28:25.724 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.724 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.724 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:28:25.724 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.007551+0000", 2023-08-28T02:28:25.725 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.8379999999999996e-06 2023-08-28T02:28:25.725 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.725 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.725 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:28:25.725 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.007557+0000", 2023-08-28T02:28:25.725 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.4807000000000001e-05 2023-08-28T02:28:25.725 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.725 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.725 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:28:25.725 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.007572+0000", 2023-08-28T02:28:25.725 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.2530000000000001e-06 2023-08-28T02:28:25.725 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.725 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.726 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:28:25.726 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.007578+0000", 2023-08-28T02:28:25.726 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.3174000000000002e-05 2023-08-28T02:28:25.726 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.726 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.726 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:25.726 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.007642+0000", 2023-08-28T02:28:25.726 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1406000000000001e-05 2023-08-28T02:28:25.726 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.726 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.726 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:28:25.726 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.007653+0000", 2023-08-28T02:28:25.727 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.97676323499999995 2023-08-28T02:28:25.727 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.727 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.727 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:25.727 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.984416+0000", 2023-08-28T02:28:25.727 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012243200000000001 2023-08-28T02:28:25.727 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.727 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.727 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:28:25.727 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.984539+0000", 2023-08-28T02:28:25.727 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012780699999999999 2023-08-28T02:28:25.727 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.728 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.728 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,7", 2023-08-28T02:28:25.728 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.984667+0000", 2023-08-28T02:28:25.728 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.014060197999999999 2023-08-28T02:28:25.728 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.728 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.728 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:25.728 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.998727+0000", 2023-08-28T02:28:25.728 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.003498091 2023-08-28T02:28:25.728 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.728 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.728 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:28:25.728 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.002225+0000", 2023-08-28T02:28:25.729 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.8292e-05 2023-08-28T02:28:25.729 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.729 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.729 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:25.729 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.002323+0000", 2023-08-28T02:28:25.729 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.0098e-05 2023-08-28T02:28:25.729 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.729 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.729 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:28:25.729 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.002373+0000", 2023-08-28T02:28:25.729 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000109016 2023-08-28T02:28:25.729 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.730 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.730 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:28:25.730 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.002482+0000", 2023-08-28T02:28:25.730 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.99493006100000003 2023-08-28T02:28:25.730 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:25.730 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:28:25.730 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:25.730 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.730 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.730 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:114373 5.1 5:8d61f17b:::benchmark_data_smithi143_236200_object4230:head [delete] snapc 0=[] ondisk+write+known_if_redirected e148)", 2023-08-28T02:28:25.730 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:44.020572+0000", 2023-08-28T02:28:25.730 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 281.617887989, 2023-08-28T02:28:25.731 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.98890386600000002, 2023-08-28T02:28:25.731 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:28:25.731 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:28:25.731 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:28:25.731 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:28:25.731 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:28:25.731 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 114373 2023-08-28T02:28:25.731 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.731 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:28:25.731 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.731 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:28:25.731 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.020572+0000", 2023-08-28T02:28:25.732 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:28:25.732 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.732 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.732 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:28:25.732 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.020572+0000", 2023-08-28T02:28:25.732 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:28:25.732 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.732 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.732 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:28:25.732 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.020570+0000", 2023-08-28T02:28:25.732 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0434999999999999e-05 2023-08-28T02:28:25.732 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.732 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.733 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:28:25.733 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.020580+0000", 2023-08-28T02:28:25.733 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.1787e-05 2023-08-28T02:28:25.733 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.733 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.733 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:28:25.733 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.020602+0000", 2023-08-28T02:28:25.733 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.2979999999999992e-06 2023-08-28T02:28:25.733 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.733 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.733 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:28:25.733 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.020610+0000", 2023-08-28T02:28:25.733 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.1524000000000002e-05 2023-08-28T02:28:25.734 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.734 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.734 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:25.734 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.020682+0000", 2023-08-28T02:28:25.734 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1493e-05 2023-08-28T02:28:25.734 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.734 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.734 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:28:25.734 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.020693+0000", 2023-08-28T02:28:25.734 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.96700611599999997 2023-08-28T02:28:25.734 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.734 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.735 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:25.735 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.987699+0000", 2023-08-28T02:28:25.735 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000114647 2023-08-28T02:28:25.735 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.735 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.735 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:28:25.735 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.987814+0000", 2023-08-28T02:28:25.735 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.221e-05 2023-08-28T02:28:25.735 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.735 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.735 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,7", 2023-08-28T02:28:25.735 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.987906+0000", 2023-08-28T02:28:25.736 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.014324356999999999 2023-08-28T02:28:25.736 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.736 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.736 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:28:25.736 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.002231+0000", 2023-08-28T02:28:25.736 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.9576000000000001e-05 2023-08-28T02:28:25.736 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.736 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.736 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:25.736 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.002280+0000", 2023-08-28T02:28:25.736 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0070311599999999998 2023-08-28T02:28:25.736 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.736 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.737 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:25.737 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.009311+0000", 2023-08-28T02:28:25.737 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.0016999999999999e-05 2023-08-28T02:28:25.737 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.737 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.737 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:28:25.737 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.009361+0000", 2023-08-28T02:28:25.737 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000114168 2023-08-28T02:28:25.737 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.737 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.737 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:28:25.737 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.009475+0000", 2023-08-28T02:28:25.737 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.98890386600000002 2023-08-28T02:28:25.737 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:25.738 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:28:25.738 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:25.738 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.738 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.738 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:114383 5.1 5:81a4d21f:::benchmark_data_smithi143_236200_object4240:head [delete] snapc 0=[] ondisk+write+known_if_redirected e148)", 2023-08-28T02:28:25.738 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:44.031002+0000", 2023-08-28T02:28:25.738 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 281.60745736199999, 2023-08-28T02:28:25.738 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.97863153999999997, 2023-08-28T02:28:25.738 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:28:25.738 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:28:25.738 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:28:25.738 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:28:25.738 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:28:25.739 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 114383 2023-08-28T02:28:25.739 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.739 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:28:25.739 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.739 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:28:25.739 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.031002+0000", 2023-08-28T02:28:25.739 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:28:25.739 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.739 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.739 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:28:25.739 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.031002+0000", 2023-08-28T02:28:25.739 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999995 2023-08-28T02:28:25.740 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.740 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.740 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:28:25.740 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.031002+0000", 2023-08-28T02:28:25.740 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.5300000000000001e-06 2023-08-28T02:28:25.740 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.740 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.740 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:28:25.740 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.031005+0000", 2023-08-28T02:28:25.740 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.1270000000000002e-06 2023-08-28T02:28:25.740 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.740 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.740 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:28:25.741 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.031010+0000", 2023-08-28T02:28:25.741 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.0199999999999999e-06 2023-08-28T02:28:25.741 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.741 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.741 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:28:25.741 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.031013+0000", 2023-08-28T02:28:25.741 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.9960000000000003e-05 2023-08-28T02:28:25.741 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.741 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.741 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:25.741 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.031063+0000", 2023-08-28T02:28:25.741 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.8900000000000002e-06 2023-08-28T02:28:25.742 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.742 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.742 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:28:25.742 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.031073+0000", 2023-08-28T02:28:25.742 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.96021787599999997 2023-08-28T02:28:25.742 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.742 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.742 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:25.742 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.991291+0000", 2023-08-28T02:28:25.742 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000106072 2023-08-28T02:28:25.742 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.742 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.742 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:28:25.743 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.991397+0000", 2023-08-28T02:28:25.743 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000111848 2023-08-28T02:28:25.743 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.743 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.743 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,7", 2023-08-28T02:28:25.743 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.991509+0000", 2023-08-28T02:28:25.743 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.010727258 2023-08-28T02:28:25.743 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.743 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.743 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:28:25.743 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.002236+0000", 2023-08-28T02:28:25.743 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00029985 2023-08-28T02:28:25.744 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.744 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.744 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:25.744 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.002536+0000", 2023-08-28T02:28:25.744 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.006984929 2023-08-28T02:28:25.744 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.744 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.744 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:25.744 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.009521+0000", 2023-08-28T02:28:25.744 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.3491000000000001e-05 2023-08-28T02:28:25.744 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.744 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.744 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:28:25.745 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.009564+0000", 2023-08-28T02:28:25.745 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.9277999999999996e-05 2023-08-28T02:28:25.745 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.745 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.745 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:28:25.745 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.009634+0000", 2023-08-28T02:28:25.745 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.97863153999999997 2023-08-28T02:28:25.745 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:25.745 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:28:25.745 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:25.745 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.745 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.745 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:114398 5.1 5:8f550161:::benchmark_data_smithi143_236200_object4255:head [delete] snapc 0=[] ondisk+write+known_if_redirected e148)", 2023-08-28T02:28:25.746 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:44.053922+0000", 2023-08-28T02:28:25.746 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 281.58453718200002, 2023-08-28T02:28:25.746 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.96977096500000004, 2023-08-28T02:28:25.746 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:28:25.746 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:28:25.746 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:28:25.746 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:28:25.746 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:28:25.746 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 114398 2023-08-28T02:28:25.746 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.746 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:28:25.746 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.746 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:28:25.747 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.053922+0000", 2023-08-28T02:28:25.747 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:28:25.747 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.747 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.747 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:28:25.747 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.053922+0000", 2023-08-28T02:28:25.747 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:28:25.747 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.747 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.747 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:28:25.747 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.053921+0000", 2023-08-28T02:28:25.747 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.3129999999999999e-06 2023-08-28T02:28:25.747 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.748 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.748 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:28:25.748 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.053929+0000", 2023-08-28T02:28:25.748 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.738e-05 2023-08-28T02:28:25.748 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.748 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.748 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:28:25.748 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.053947+0000", 2023-08-28T02:28:25.748 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.8860000000000004e-06 2023-08-28T02:28:25.748 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.748 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.748 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:28:25.748 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.053955+0000", 2023-08-28T02:28:25.749 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.8510999999999997e-05 2023-08-28T02:28:25.749 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.749 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.749 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:25.749 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.054003+0000", 2023-08-28T02:28:25.749 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.252e-05 2023-08-28T02:28:25.749 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.749 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.749 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:28:25.749 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.054016+0000", 2023-08-28T02:28:25.749 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.94065110100000005 2023-08-28T02:28:25.749 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.750 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.750 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:25.750 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.994667+0000", 2023-08-28T02:28:25.750 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000114616 2023-08-28T02:28:25.750 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.750 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.750 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:28:25.750 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.994781+0000", 2023-08-28T02:28:25.750 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00013167800000000001 2023-08-28T02:28:25.750 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.750 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.750 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,7", 2023-08-28T02:28:25.750 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.994913+0000", 2023-08-28T02:28:25.750 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0073757720000000001 2023-08-28T02:28:25.751 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.751 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.751 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:28:25.751 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.002289+0000", 2023-08-28T02:28:25.751 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0069236580000000001 2023-08-28T02:28:25.751 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.751 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.751 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:25.751 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.009213+0000", 2023-08-28T02:28:25.751 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.014330872999999999 2023-08-28T02:28:25.751 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.752 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.752 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:25.752 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.023543+0000", 2023-08-28T02:28:25.752 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.5627000000000002e-05 2023-08-28T02:28:25.752 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.752 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.752 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:28:25.752 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.023589+0000", 2023-08-28T02:28:25.752 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000104381 2023-08-28T02:28:25.752 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.752 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.752 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:28:25.752 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.023693+0000", 2023-08-28T02:28:25.753 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.96977096500000004 2023-08-28T02:28:25.753 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:25.753 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:28:25.753 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:25.753 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.753 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.753 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:114399 5.1 5:821d8672:::benchmark_data_smithi143_236200_object4256:head [delete] snapc 0=[] ondisk+write+known_if_redirected e148)", 2023-08-28T02:28:25.753 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:44.054511+0000", 2023-08-28T02:28:25.753 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 281.58394838499999, 2023-08-28T02:28:25.753 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.96934294700000001, 2023-08-28T02:28:25.753 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:28:25.753 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:28:25.753 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:28:25.753 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:28:25.754 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:28:25.754 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 114399 2023-08-28T02:28:25.754 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.754 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:28:25.754 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.754 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:28:25.754 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.054511+0000", 2023-08-28T02:28:25.754 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:28:25.754 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.754 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.754 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:28:25.754 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.054511+0000", 2023-08-28T02:28:25.755 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:28:25.755 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.755 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.755 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:28:25.755 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.054510+0000", 2023-08-28T02:28:25.755 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.6059999999999996e-06 2023-08-28T02:28:25.755 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.755 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.755 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:28:25.755 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.054515+0000", 2023-08-28T02:28:25.755 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.6590000000000002e-06 2023-08-28T02:28:25.755 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.756 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.756 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:28:25.756 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.054525+0000", 2023-08-28T02:28:25.756 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.8189999999999997e-06 2023-08-28T02:28:25.756 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.756 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.756 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:28:25.756 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.054531+0000", 2023-08-28T02:28:25.756 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.8594999999999999e-05 2023-08-28T02:28:25.756 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.756 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.756 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:25.756 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.054569+0000", 2023-08-28T02:28:25.757 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.7869999999999996e-06 2023-08-28T02:28:25.757 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.757 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.757 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:28:25.757 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.054579+0000", 2023-08-28T02:28:25.757 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.94418591900000004 2023-08-28T02:28:25.757 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.757 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.757 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:25.757 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.998765+0000", 2023-08-28T02:28:25.757 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000119363 2023-08-28T02:28:25.757 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.757 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.758 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:28:25.758 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.998884+0000", 2023-08-28T02:28:25.758 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000133683 2023-08-28T02:28:25.758 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.758 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.758 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,7", 2023-08-28T02:28:25.758 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.999018+0000", 2023-08-28T02:28:25.758 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.010199753000000001 2023-08-28T02:28:25.758 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.758 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.758 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:28:25.758 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.009218+0000", 2023-08-28T02:28:25.758 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00044989900000000002 2023-08-28T02:28:25.759 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.759 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.759 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:25.759 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.009668+0000", 2023-08-28T02:28:25.759 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.014060081 2023-08-28T02:28:25.759 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.759 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.759 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:25.759 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.023728+0000", 2023-08-28T02:28:25.759 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.6545000000000003e-05 2023-08-28T02:28:25.759 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.759 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.760 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:28:25.760 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.023764+0000", 2023-08-28T02:28:25.760 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.9993000000000004e-05 2023-08-28T02:28:25.760 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.760 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.760 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:28:25.760 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.023854+0000", 2023-08-28T02:28:25.760 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.96934294700000001 2023-08-28T02:28:25.760 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:25.760 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:28:25.760 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:25.760 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.760 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.761 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:114406 5.1 5:8f9a1570:::benchmark_data_smithi143_236200_object4263:head [delete] snapc 0=[] ondisk+write+known_if_redirected e148)", 2023-08-28T02:28:25.761 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:44.071870+0000", 2023-08-28T02:28:25.761 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 281.56658961900001, 2023-08-28T02:28:25.761 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.95214170300000001, 2023-08-28T02:28:25.761 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:28:25.761 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:28:25.761 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:28:25.761 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:28:25.761 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:28:25.761 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 114406 2023-08-28T02:28:25.761 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.761 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:28:25.761 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.762 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:28:25.762 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.071870+0000", 2023-08-28T02:28:25.762 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:28:25.762 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.762 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.762 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:28:25.762 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.071870+0000", 2023-08-28T02:28:25.762 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:28:25.762 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.762 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.762 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:28:25.762 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.071869+0000", 2023-08-28T02:28:25.762 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.541e-06 2023-08-28T02:28:25.763 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.763 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.763 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:28:25.763 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.071874+0000", 2023-08-28T02:28:25.763 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1824999999999999e-05 2023-08-28T02:28:25.763 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.763 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.763 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:28:25.763 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.071886+0000", 2023-08-28T02:28:25.763 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.9069999999999998e-06 2023-08-28T02:28:25.763 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.763 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.763 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:28:25.764 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.071891+0000", 2023-08-28T02:28:25.764 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.7457999999999998e-05 2023-08-28T02:28:25.764 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.764 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.764 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:25.764 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.071948+0000", 2023-08-28T02:28:25.764 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1953000000000001e-05 2023-08-28T02:28:25.764 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.764 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.764 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:28:25.764 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.071960+0000", 2023-08-28T02:28:25.764 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.93060808500000003 2023-08-28T02:28:25.764 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.765 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.765 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:25.765 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.002568+0000", 2023-08-28T02:28:25.765 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000125628 2023-08-28T02:28:25.765 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.765 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.765 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:28:25.765 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.002694+0000", 2023-08-28T02:28:25.765 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00013300900000000001 2023-08-28T02:28:25.765 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.765 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.766 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,7", 2023-08-28T02:28:25.766 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.002827+0000", 2023-08-28T02:28:25.766 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0064272879999999998 2023-08-28T02:28:25.767 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.767 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.767 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:28:25.767 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.009254+0000", 2023-08-28T02:28:25.767 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0036058919999999999 2023-08-28T02:28:25.767 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.767 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.767 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:25.767 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.012860+0000", 2023-08-28T02:28:25.767 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.011026055 2023-08-28T02:28:25.767 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.767 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.768 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:25.768 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.023886+0000", 2023-08-28T02:28:25.768 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.5678e-05 2023-08-28T02:28:25.768 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.768 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.768 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:28:25.768 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.023922+0000", 2023-08-28T02:28:25.768 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.9739000000000003e-05 2023-08-28T02:28:25.768 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.768 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.768 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:28:25.768 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.024012+0000", 2023-08-28T02:28:25.769 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.95214170300000001 2023-08-28T02:28:25.769 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:25.769 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:28:25.769 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:25.769 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.769 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.769 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:114414 5.1 5:82f807f5:::benchmark_data_smithi143_236200_object4271:head [delete] snapc 0=[] ondisk+write+known_if_redirected e148)", 2023-08-28T02:28:25.769 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:44.107189+0000", 2023-08-28T02:28:25.769 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 281.531270979, 2023-08-28T02:28:25.769 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.91698741100000003, 2023-08-28T02:28:25.769 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:28:25.769 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:28:25.769 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:28:25.770 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:28:25.770 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:28:25.770 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 114414 2023-08-28T02:28:25.770 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.770 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:28:25.770 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.770 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:28:25.770 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.107189+0000", 2023-08-28T02:28:25.770 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:28:25.770 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.770 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.770 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:28:25.771 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.107189+0000", 2023-08-28T02:28:25.771 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:28:25.771 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.771 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.771 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:28:25.771 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.107187+0000", 2023-08-28T02:28:25.771 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.9719999999999997e-06 2023-08-28T02:28:25.771 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.771 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.771 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:28:25.771 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.107193+0000", 2023-08-28T02:28:25.771 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2877000000000001e-05 2023-08-28T02:28:25.771 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.772 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.772 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:28:25.772 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.107206+0000", 2023-08-28T02:28:25.772 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.6389999999999997e-06 2023-08-28T02:28:25.772 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.772 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.772 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:28:25.772 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.107212+0000", 2023-08-28T02:28:25.772 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.7942000000000003e-05 2023-08-28T02:28:25.772 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.772 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.772 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:25.772 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.107260+0000", 2023-08-28T02:28:25.772 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.8300000000000008e-06 2023-08-28T02:28:25.773 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.773 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.773 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:28:25.773 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:44.107270+0000", 2023-08-28T02:28:25.773 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.89864702200000002 2023-08-28T02:28:25.773 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.773 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.773 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:25.773 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.005917+0000", 2023-08-28T02:28:25.773 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00014625200000000001 2023-08-28T02:28:25.773 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.773 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.774 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:28:25.774 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.006063+0000", 2023-08-28T02:28:25.774 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000161736 2023-08-28T02:28:25.774 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.774 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.774 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,7", 2023-08-28T02:28:25.774 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.006225+0000", 2023-08-28T02:28:25.774 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.003415029 2023-08-28T02:28:25.774 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.774 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.774 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:28:25.774 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.009640+0000", 2023-08-28T02:28:25.774 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.013842596 2023-08-28T02:28:25.775 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.775 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.775 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:25.775 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.023482+0000", 2023-08-28T02:28:25.775 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00056224300000000003 2023-08-28T02:28:25.775 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.775 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.775 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:25.775 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.024044+0000", 2023-08-28T02:28:25.775 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.5420999999999998e-05 2023-08-28T02:28:25.775 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.775 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.776 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:28:25.776 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.024080+0000", 2023-08-28T02:28:25.776 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.6132999999999996e-05 2023-08-28T02:28:25.776 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.776 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.776 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:28:25.776 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:45.024176+0000", 2023-08-28T02:28:25.776 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.91698741100000003 2023-08-28T02:28:25.776 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:25.776 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:28:25.776 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:25.776 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.777 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.777 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:123484 5.1 5:85bec03d:::benchmark_data_smithi143_236200_object13341:head [delete] snapc 0=[] ondisk+write+known_if_redirected e152)", 2023-08-28T02:28:25.777 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:58.166108+0000", 2023-08-28T02:28:25.777 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 267.47235189899999, 2023-08-28T02:28:25.777 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.1640762979999999, 2023-08-28T02:28:25.777 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:28:25.777 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "reached pg", 2023-08-28T02:28:25.777 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:28:25.777 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:28:25.777 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:28:25.777 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 123484 2023-08-28T02:28:25.777 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.777 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:28:25.778 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.778 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:28:25.778 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.166108+0000", 2023-08-28T02:28:25.778 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:28:25.778 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.778 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.778 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:28:25.778 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.166108+0000", 2023-08-28T02:28:25.778 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:28:25.778 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.778 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.778 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:28:25.778 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.166106+0000", 2023-08-28T02:28:25.779 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.661e-06 2023-08-28T02:28:25.779 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.779 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.779 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:28:25.779 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.166114+0000", 2023-08-28T02:28:25.779 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.537e-05 2023-08-28T02:28:25.779 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.779 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.779 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:28:25.779 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.166129+0000", 2023-08-28T02:28:25.779 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0088000000000001e-05 2023-08-28T02:28:25.779 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.780 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.780 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:28:25.780 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.166139+0000", 2023-08-28T02:28:25.780 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.003e-05 2023-08-28T02:28:25.780 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.780 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.780 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:25.780 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.166199+0000", 2023-08-28T02:28:25.780 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00014851600000000001 2023-08-28T02:28:25.780 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.780 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.780 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:28:25.781 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.166348+0000", 2023-08-28T02:28:25.781 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000186997 2023-08-28T02:28:25.781 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.781 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.781 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,7", 2023-08-28T02:28:25.781 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.166535+0000", 2023-08-28T02:28:25.781 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0066866920000000002 2023-08-28T02:28:25.781 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.781 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.781 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:28:25.781 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.173221+0000", 2023-08-28T02:28:25.781 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.1798000000000001e-05 2023-08-28T02:28:25.781 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.782 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.782 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:25.782 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.173273+0000", 2023-08-28T02:28:25.782 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.032931016 2023-08-28T02:28:25.782 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.782 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.782 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:25.782 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.206204+0000", 2023-08-28T02:28:25.782 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.2499999999999995e-06 2023-08-28T02:28:25.782 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.782 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.782 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:28:25.782 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.206214+0000", 2023-08-28T02:28:25.783 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.994048975 2023-08-28T02:28:25.783 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.783 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.783 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:25.783 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:59.200263+0000", 2023-08-28T02:28:25.783 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1748e-05 2023-08-28T02:28:25.783 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.783 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.783 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:28:25.783 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:59.200274+0000", 2023-08-28T02:28:25.783 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1298252950000001 2023-08-28T02:28:25.783 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.783 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.784 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:25.784 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:00.330100+0000", 2023-08-28T02:28:25.784 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.4437999999999994e-05 2023-08-28T02:28:25.784 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.784 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.784 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:28:25.784 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:00.330184+0000", 2023-08-28T02:28:25.784 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.1640762979999999 2023-08-28T02:28:25.784 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:25.784 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:28:25.784 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:25.784 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.785 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.785 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:123485 5.1 5:8d110a8f:::benchmark_data_smithi143_236200_object13342:head [delete] snapc 0=[] ondisk+write+known_if_redirected e152)", 2023-08-28T02:28:25.785 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:58.166177+0000", 2023-08-28T02:28:25.785 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 267.47228227800002, 2023-08-28T02:28:25.785 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.1643501280000002, 2023-08-28T02:28:25.785 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:28:25.785 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "reached pg", 2023-08-28T02:28:25.785 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:28:25.785 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:28:25.785 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:28:25.785 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 123485 2023-08-28T02:28:25.785 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.785 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:28:25.785 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.786 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:28:25.786 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.166177+0000", 2023-08-28T02:28:25.786 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:28:25.786 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.786 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.786 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:28:25.786 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.166177+0000", 2023-08-28T02:28:25.786 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:28:25.786 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.786 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.786 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:28:25.786 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.166177+0000", 2023-08-28T02:28:25.786 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.5520000000000005e-06 2023-08-28T02:28:25.787 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.787 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.787 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:28:25.787 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.166185+0000", 2023-08-28T02:28:25.787 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.7098000000000001e-05 2023-08-28T02:28:25.787 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.787 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.787 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:28:25.787 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.166202+0000", 2023-08-28T02:28:25.787 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.2490000000000001e-06 2023-08-28T02:28:25.787 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.787 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.788 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:28:25.788 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.166207+0000", 2023-08-28T02:28:25.788 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0040287659999999996 2023-08-28T02:28:25.788 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.788 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.788 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:25.788 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.170236+0000", 2023-08-28T02:28:25.788 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012299300000000001 2023-08-28T02:28:25.788 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.788 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.788 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:28:25.788 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.170359+0000", 2023-08-28T02:28:25.788 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000106042 2023-08-28T02:28:25.789 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.789 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.789 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,7", 2023-08-28T02:28:25.789 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.170465+0000", 2023-08-28T02:28:25.789 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0029179420000000002 2023-08-28T02:28:25.789 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.789 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.789 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:28:25.789 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.173383+0000", 2023-08-28T02:28:25.789 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00065289099999999995 2023-08-28T02:28:25.789 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.789 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.789 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:25.789 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.174036+0000", 2023-08-28T02:28:25.790 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.032203163999999999 2023-08-28T02:28:25.790 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.790 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.790 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:25.790 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.206239+0000", 2023-08-28T02:28:25.790 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.2450000000000001e-06 2023-08-28T02:28:25.790 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.790 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.790 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:28:25.790 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.206246+0000", 2023-08-28T02:28:25.790 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.99406602799999999 2023-08-28T02:28:25.790 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.790 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.791 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:25.791 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:59.200313+0000", 2023-08-28T02:28:25.791 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.7479999999999993e-06 2023-08-28T02:28:25.791 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.791 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.791 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:28:25.791 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:59.200320+0000", 2023-08-28T02:28:25.791 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.130151983 2023-08-28T02:28:25.791 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.791 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.791 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:25.791 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:00.330472+0000", 2023-08-28T02:28:25.792 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.5208999999999999e-05 2023-08-28T02:28:25.792 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.792 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.792 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:28:25.792 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:00.330527+0000", 2023-08-28T02:28:25.792 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.1643501280000002 2023-08-28T02:28:25.792 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:25.792 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:28:25.792 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:25.792 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.792 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.792 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:123528 5.1 5:826b0b9b:::benchmark_data_smithi143_236200_object13385:head [delete] snapc 0=[] ondisk+write+known_if_redirected e153)", 2023-08-28T02:28:25.792 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:58.314486+0000", 2023-08-28T02:28:25.793 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 267.32397381800001, 2023-08-28T02:28:25.793 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.0278054889999999, 2023-08-28T02:28:25.793 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:28:25.793 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:28:25.793 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:28:25.793 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:28:25.793 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:28:25.793 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 123528 2023-08-28T02:28:25.793 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.793 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:28:25.793 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.793 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:28:25.793 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.314486+0000", 2023-08-28T02:28:25.794 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:28:25.794 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.794 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.794 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:28:25.794 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.314486+0000", 2023-08-28T02:28:25.794 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:28:25.794 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.794 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.794 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:28:25.794 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.314484+0000", 2023-08-28T02:28:25.794 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.5689999999999999e-06 2023-08-28T02:28:25.794 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.795 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.795 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:28:25.795 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.314492+0000", 2023-08-28T02:28:25.795 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2577e-05 2023-08-28T02:28:25.795 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.795 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.795 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:28:25.795 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.314504+0000", 2023-08-28T02:28:25.795 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.9720000000000003e-06 2023-08-28T02:28:25.795 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.795 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.795 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:28:25.796 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.314511+0000", 2023-08-28T02:28:25.796 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.4861000000000001e-05 2023-08-28T02:28:25.796 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.796 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.796 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:25.796 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.314576+0000", 2023-08-28T02:28:25.796 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1195e-05 2023-08-28T02:28:25.796 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.796 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.796 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:28:25.796 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:58.314587+0000", 2023-08-28T02:28:25.796 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.88650818399999998 2023-08-28T02:28:25.796 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.797 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.797 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:25.797 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:59.201095+0000", 2023-08-28T02:28:25.797 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.064e-06 2023-08-28T02:28:25.797 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.797 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.797 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:28:25.797 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:59.201105+0000", 2023-08-28T02:28:25.797 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.129516108 2023-08-28T02:28:25.797 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.797 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.797 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:25.798 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:00.330621+0000", 2023-08-28T02:28:25.798 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000122156 2023-08-28T02:28:25.798 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.798 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.798 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:28:25.798 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:00.330743+0000", 2023-08-28T02:28:25.798 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00014396 2023-08-28T02:28:25.798 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.798 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.798 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,3,7", 2023-08-28T02:28:25.798 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:00.330887+0000", 2023-08-28T02:28:25.798 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0064385120000000004 2023-08-28T02:28:25.798 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.799 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.799 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:28:25.799 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:00.337325+0000", 2023-08-28T02:28:25.799 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00024257599999999999 2023-08-28T02:28:25.799 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.799 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.799 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:25.799 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:00.337568+0000", 2023-08-28T02:28:25.799 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00046788599999999999 2023-08-28T02:28:25.799 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.799 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.799 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:25.799 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:00.338036+0000", 2023-08-28T02:28:25.800 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0041302170000000003 2023-08-28T02:28:25.800 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.800 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.800 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:25.800 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:00.342166+0000", 2023-08-28T02:28:25.800 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.6479e-05 2023-08-28T02:28:25.800 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.800 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.800 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:28:25.800 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:00.342212+0000", 2023-08-28T02:28:25.800 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.8874000000000004e-05 2023-08-28T02:28:25.800 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.800 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.801 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:28:25.801 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:00.342291+0000", 2023-08-28T02:28:25.801 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.0278054889999999 2023-08-28T02:28:25.801 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:25.801 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:28:25.801 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:25.801 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.801 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.801 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:49468 6.2 6:4bd48029:::benchmark_data_smithi143_305742_object49467:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e206)", 2023-08-28T02:28:25.801 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:27:44.521913+0000", 2023-08-28T02:28:25.801 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 41.116546663000001, 2023-08-28T02:28:25.801 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.326332187, 2023-08-28T02:28:25.801 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:28:25.802 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:28:25.802 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:28:25.802 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:28:25.802 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:28:25.802 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 49468 2023-08-28T02:28:25.802 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.802 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:28:25.802 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.802 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:28:25.803 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:44.521913+0000", 2023-08-28T02:28:25.803 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:28:25.803 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.803 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.803 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:28:25.803 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:44.521913+0000", 2023-08-28T02:28:25.803 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:28:25.803 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.803 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.803 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:28:25.803 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:44.521911+0000", 2023-08-28T02:28:25.803 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.8035000000000003e-05 2023-08-28T02:28:25.803 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.804 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.804 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:28:25.804 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:44.521969+0000", 2023-08-28T02:28:25.804 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.3923e-05 2023-08-28T02:28:25.804 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.804 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.804 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:28:25.804 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:44.521983+0000", 2023-08-28T02:28:25.804 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.9910000000000002e-06 2023-08-28T02:28:25.804 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.804 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.804 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:28:25.804 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:44.521988+0000", 2023-08-28T02:28:25.805 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.2017000000000002e-05 2023-08-28T02:28:25.805 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.805 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.805 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:25.805 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:44.522020+0000", 2023-08-28T02:28:25.805 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.5396999999999995e-05 2023-08-28T02:28:25.805 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.805 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.805 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:28:25.805 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:44.522116+0000", 2023-08-28T02:28:25.805 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00015915300000000001 2023-08-28T02:28:25.805 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.805 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.806 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 2,3", 2023-08-28T02:28:25.806 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:44.522275+0000", 2023-08-28T02:28:25.806 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0019057080000000001 2023-08-28T02:28:25.806 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.806 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.806 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:28:25.806 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:44.524181+0000", 2023-08-28T02:28:25.806 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.011793755 2023-08-28T02:28:25.806 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.806 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.806 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:25.806 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:44.535974+0000", 2023-08-28T02:28:25.806 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.31216330799999997 2023-08-28T02:28:25.807 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.807 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.807 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:25.807 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:44.848138+0000", 2023-08-28T02:28:25.807 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.5228000000000006e-05 2023-08-28T02:28:25.807 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.807 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.807 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:28:25.807 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:44.848203+0000", 2023-08-28T02:28:25.807 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.2190000000000001e-05 2023-08-28T02:28:25.807 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:25.807 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:25.808 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:28:25.808 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:44.848245+0000", 2023-08-28T02:28:25.808 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.326332187 2023-08-28T02:28:25.808 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:25.808 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:28:25.808 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:25.808 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:25.808 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:28:25.808 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:28:25.809 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:25 smithi143 ceph-mon[189871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:28:25.809 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:25 smithi143 ceph-mon[189871]: failed to encode map e220 with expected crc 2023-08-28T02:28:25.809 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:25 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/333409677' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-08-28T02:28:25.809 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:25 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:28:25.809 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:25 smithi143 ceph-mon[189871]: osdmap e220: 8 total, 8 up, 8 in 2023-08-28T02:28:25.809 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:25 smithi143 ceph-mon[189871]: failed to encode map e220 with expected crc 2023-08-28T02:28:25.809 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:25 smithi143 ceph-mon[189871]: osdmap e221: 8 total, 8 up, 8 in 2023-08-28T02:28:25.809 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:25 smithi143 ceph-mon[183059]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:28:25.810 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:25 smithi143 ceph-mon[183059]: failed to encode map e220 with expected crc 2023-08-28T02:28:25.810 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:25 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/333409677' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-08-28T02:28:25.810 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:25 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:28:25.810 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:25 smithi143 ceph-mon[183059]: osdmap e220: 8 total, 8 up, 8 in 2023-08-28T02:28:25.810 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:25 smithi143 ceph-mon[183059]: failed to encode map e220 with expected crc 2023-08-28T02:28:25.810 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:25 smithi143 ceph-mon[183059]: osdmap e221: 8 total, 8 up, 8 in 2023-08-28T02:28:26.147 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 79 16 78192 78176 61.8041 0.5 0.0159676 0.0158547 2023-08-28T02:28:26.466 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.5 dump_ops_in_flight 2023-08-28T02:28:26.525 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:28:26.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:26.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:28:26.621 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:26 smithi143 ceph-mon[183059]: pgmap v519: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+recovering, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 50 MiB/s wr, 807 op/s; 10308/232041 objects degraded (4.442%); 1544/232041 objects misplaced (0.665%); 4.6 MiB/s, 74 objects/s recovering 2023-08-28T02:28:26.622 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:26 smithi143 ceph-mon[183059]: failed to encode map e221 with expected crc 2023-08-28T02:28:26.622 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:26 smithi143 ceph-mon[183059]: failed to encode map e221 with expected crc 2023-08-28T02:28:26.622 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:26 smithi143 ceph-mon[183059]: failed to encode map e221 with expected crc 2023-08-28T02:28:26.622 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:26 smithi143 ceph-mon[183059]: failed to encode map e221 with expected crc 2023-08-28T02:28:26.622 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:26 smithi143 ceph-mon[183059]: failed to encode map e221 with expected crc 2023-08-28T02:28:26.622 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:26 smithi143 ceph-mon[183059]: failed to encode map e221 with expected crc 2023-08-28T02:28:26.622 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:26 smithi143 ceph-mon[183059]: failed to encode map e221 with expected crc 2023-08-28T02:28:26.622 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:26 smithi143 ceph-mon[183059]: failed to encode map e221 with expected crc 2023-08-28T02:28:26.622 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:26 smithi143 ceph-mon[183059]: 6.2 scrub starts 2023-08-28T02:28:26.622 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:26 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:28:26.623 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:26 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:28:26.623 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:26 smithi143 ceph-mon[183059]: osdmap e222: 8 total, 8 up, 8 in 2023-08-28T02:28:26.623 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:26 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 10308/232041 objects degraded (4.442%), 13 pgs degraded (PG_DEGRADED) 2023-08-28T02:28:26.623 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:26 smithi143 ceph-mon[189871]: pgmap v519: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+recovering, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 50 MiB/s wr, 807 op/s; 10308/232041 objects degraded (4.442%); 1544/232041 objects misplaced (0.665%); 4.6 MiB/s, 74 objects/s recovering 2023-08-28T02:28:26.623 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:26 smithi143 ceph-mon[189871]: failed to encode map e221 with expected crc 2023-08-28T02:28:26.623 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:26 smithi143 ceph-mon[189871]: failed to encode map e221 with expected crc 2023-08-28T02:28:26.623 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:26 smithi143 ceph-mon[189871]: failed to encode map e221 with expected crc 2023-08-28T02:28:26.623 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:26 smithi143 ceph-mon[189871]: failed to encode map e221 with expected crc 2023-08-28T02:28:26.624 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:26 smithi143 ceph-mon[189871]: failed to encode map e221 with expected crc 2023-08-28T02:28:26.624 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:26 smithi143 ceph-mon[189871]: failed to encode map e221 with expected crc 2023-08-28T02:28:26.624 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:26 smithi143 ceph-mon[189871]: failed to encode map e221 with expected crc 2023-08-28T02:28:26.624 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:26 smithi143 ceph-mon[189871]: failed to encode map e221 with expected crc 2023-08-28T02:28:26.624 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:26 smithi143 ceph-mon[189871]: 6.2 scrub starts 2023-08-28T02:28:26.624 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:26 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:28:26.624 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:26 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:28:26.624 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:26 smithi143 ceph-mon[189871]: osdmap e222: 8 total, 8 up, 8 in 2023-08-28T02:28:26.624 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:26 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 10308/232041 objects degraded (4.442%), 13 pgs degraded (PG_DEGRADED) 2023-08-28T02:28:26.640 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:28:26.672 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:26 smithi169 ceph-mon[159415]: pgmap v519: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 1 active+recovering, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 50 MiB/s wr, 807 op/s; 10308/232041 objects degraded (4.442%); 1544/232041 objects misplaced (0.665%); 4.6 MiB/s, 74 objects/s recovering 2023-08-28T02:28:26.672 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:26 smithi169 ceph-mon[159415]: failed to encode map e221 with expected crc 2023-08-28T02:28:26.672 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:26 smithi169 ceph-mon[159415]: failed to encode map e221 with expected crc 2023-08-28T02:28:26.672 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:26 smithi169 ceph-mon[159415]: failed to encode map e221 with expected crc 2023-08-28T02:28:26.672 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:26 smithi169 ceph-mon[159415]: failed to encode map e221 with expected crc 2023-08-28T02:28:26.672 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:26 smithi169 ceph-mon[159415]: failed to encode map e221 with expected crc 2023-08-28T02:28:26.672 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:26 smithi169 ceph-mon[159415]: failed to encode map e221 with expected crc 2023-08-28T02:28:26.672 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:26 smithi169 ceph-mon[159415]: failed to encode map e221 with expected crc 2023-08-28T02:28:26.672 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:26 smithi169 ceph-mon[159415]: failed to encode map e221 with expected crc 2023-08-28T02:28:26.673 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:26 smithi169 ceph-mon[159415]: 6.2 scrub starts 2023-08-28T02:28:26.673 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:26 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:28:26.673 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:26 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:28:26.673 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:26 smithi169 ceph-mon[159415]: osdmap e222: 8 total, 8 up, 8 in 2023-08-28T02:28:26.673 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:26 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 10308/232041 objects degraded (4.442%), 13 pgs degraded (PG_DEGRADED) 2023-08-28T02:28:26.675 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:26.733 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:28:26.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:26.737 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:28:26.766 INFO:tasks.thrashosds.thrasher:Added osd 6 2023-08-28T02:28:26.837 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:28:26.872 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:26.882 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:28:26.931 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:28:26.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:26.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:28:27.036 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:28:27.062 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:27.147 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:2023-08-28T02:28:27.148515+0000 min lat: 0.00119624 max lat: 3.09266 avg lat: 0.0160952 2023-08-28T02:28:27.147 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-08-28T02:28:27.147 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 80 16 78362 78346 61.1647 10.625 0.00371462 0.0160952 2023-08-28T02:28:27.154 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:28:27.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:27.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:28:27.259 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:28:27.282 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:28:27.287 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:27.317 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:28:27.339 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:28:27.362 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:28:27.405 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:28:27.436 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:28:27.455 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:28:27.476 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:28:27.478 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:28:27.478 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:28:27.478 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:28:27.478 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:28:27.478 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:28:27.479 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:28:27.479 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:28:27.479 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:28:27.533 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:28:27.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:27.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:28:27.636 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:28:27.660 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:27 smithi169 ceph-mon[159415]: failed to encode map e222 with expected crc 2023-08-28T02:28:27.661 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:27 smithi169 ceph-mon[159415]: failed to encode map e222 with expected crc 2023-08-28T02:28:27.661 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:27 smithi169 ceph-mon[159415]: failed to encode map e222 with expected crc 2023-08-28T02:28:27.661 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:27 smithi169 ceph-mon[159415]: failed to encode map e222 with expected crc 2023-08-28T02:28:27.661 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:27 smithi169 ceph-mon[159415]: failed to encode map e222 with expected crc 2023-08-28T02:28:27.661 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:27 smithi169 ceph-mon[159415]: failed to encode map e222 with expected crc 2023-08-28T02:28:27.662 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:27 smithi169 ceph-mon[159415]: 6.c starting backfill to osd.6 from (0'0,195'1574] MIN to 219'4991 2023-08-28T02:28:27.662 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:27 smithi169 ceph-mon[159415]: 6.a starting backfill to osd.6 from (0'0,195'1546] MIN to 219'4904 2023-08-28T02:28:27.662 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:27 smithi169 ceph-mon[159415]: 6.2 scrub starts 2023-08-28T02:28:27.662 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:27 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/16614397' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:28:27.663 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:27.664 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:27 smithi143 ceph-mon[189871]: failed to encode map e222 with expected crc 2023-08-28T02:28:27.664 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:27 smithi143 ceph-mon[189871]: failed to encode map e222 with expected crc 2023-08-28T02:28:27.665 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:27 smithi143 ceph-mon[189871]: failed to encode map e222 with expected crc 2023-08-28T02:28:27.665 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:27 smithi143 ceph-mon[189871]: failed to encode map e222 with expected crc 2023-08-28T02:28:27.665 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:27 smithi143 ceph-mon[189871]: failed to encode map e222 with expected crc 2023-08-28T02:28:27.665 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:27 smithi143 ceph-mon[189871]: failed to encode map e222 with expected crc 2023-08-28T02:28:27.665 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:27 smithi143 ceph-mon[189871]: 6.c starting backfill to osd.6 from (0'0,195'1574] MIN to 219'4991 2023-08-28T02:28:27.665 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:27 smithi143 ceph-mon[189871]: 6.a starting backfill to osd.6 from (0'0,195'1546] MIN to 219'4904 2023-08-28T02:28:27.665 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:27 smithi143 ceph-mon[189871]: 6.2 scrub starts 2023-08-28T02:28:27.665 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:27 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/16614397' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:28:27.666 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:27 smithi143 ceph-mon[183059]: failed to encode map e222 with expected crc 2023-08-28T02:28:27.666 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:27 smithi143 ceph-mon[183059]: failed to encode map e222 with expected crc 2023-08-28T02:28:27.666 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:27 smithi143 ceph-mon[183059]: failed to encode map e222 with expected crc 2023-08-28T02:28:27.666 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:27 smithi143 ceph-mon[183059]: failed to encode map e222 with expected crc 2023-08-28T02:28:27.666 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:27 smithi143 ceph-mon[183059]: failed to encode map e222 with expected crc 2023-08-28T02:28:27.666 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:27 smithi143 ceph-mon[183059]: failed to encode map e222 with expected crc 2023-08-28T02:28:27.666 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:27 smithi143 ceph-mon[183059]: 6.c starting backfill to osd.6 from (0'0,195'1574] MIN to 219'4991 2023-08-28T02:28:27.666 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:27 smithi143 ceph-mon[183059]: 6.a starting backfill to osd.6 from (0'0,195'1546] MIN to 219'4904 2023-08-28T02:28:27.666 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:27 smithi143 ceph-mon[183059]: 6.2 scrub starts 2023-08-28T02:28:27.666 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:27 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/16614397' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:28:27.706 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:28:27.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:27.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:28:27.810 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:28:27.836 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:28.132 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:28:28.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:28.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:28:28.147 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 81 16 78362 78346 60.4101 0 - 0.0160952 2023-08-28T02:28:28.238 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:28:28.270 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:28.330 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:28:28.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:28.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:28:28.373 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:28 smithi169 ceph-mon[159415]: pgmap v522: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 49 MiB/s wr, 786 op/s; 13638/234264 objects degraded (5.822%); 1544/234264 objects misplaced (0.659%); 4.2 MiB/s, 68 objects/s recovering 2023-08-28T02:28:28.373 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:28 smithi169 ceph-mon[159415]: failed to encode map e223 with expected crc 2023-08-28T02:28:28.373 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:28 smithi169 ceph-mon[159415]: osdmap e223: 8 total, 8 up, 8 in 2023-08-28T02:28:28.374 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:28 smithi169 ceph-mon[159415]: failed to encode map e223 with expected crc 2023-08-28T02:28:28.374 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:28 smithi169 ceph-mon[159415]: failed to encode map e223 with expected crc 2023-08-28T02:28:28.374 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:28 smithi169 ceph-mon[159415]: failed to encode map e223 with expected crc 2023-08-28T02:28:28.374 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:28 smithi169 ceph-mon[159415]: 6.2 scrub starts 2023-08-28T02:28:28.437 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:28:28.466 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:28.525 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:28:28.530 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:28 smithi143 ceph-mon[183059]: pgmap v522: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 49 MiB/s wr, 786 op/s; 13638/234264 objects degraded (5.822%); 1544/234264 objects misplaced (0.659%); 4.2 MiB/s, 68 objects/s recovering 2023-08-28T02:28:28.530 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:28 smithi143 ceph-mon[183059]: failed to encode map e223 with expected crc 2023-08-28T02:28:28.530 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:28 smithi143 ceph-mon[183059]: osdmap e223: 8 total, 8 up, 8 in 2023-08-28T02:28:28.530 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:28 smithi143 ceph-mon[183059]: failed to encode map e223 with expected crc 2023-08-28T02:28:28.530 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:28 smithi143 ceph-mon[183059]: failed to encode map e223 with expected crc 2023-08-28T02:28:28.531 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:28 smithi143 ceph-mon[183059]: failed to encode map e223 with expected crc 2023-08-28T02:28:28.531 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:28 smithi143 ceph-mon[183059]: 6.2 scrub starts 2023-08-28T02:28:28.531 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:28 smithi143 ceph-mon[189871]: pgmap v522: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 49 MiB/s wr, 786 op/s; 13638/234264 objects degraded (5.822%); 1544/234264 objects misplaced (0.659%); 4.2 MiB/s, 68 objects/s recovering 2023-08-28T02:28:28.531 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:28 smithi143 ceph-mon[189871]: failed to encode map e223 with expected crc 2023-08-28T02:28:28.531 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:28 smithi143 ceph-mon[189871]: osdmap e223: 8 total, 8 up, 8 in 2023-08-28T02:28:28.531 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:28 smithi143 ceph-mon[189871]: failed to encode map e223 with expected crc 2023-08-28T02:28:28.531 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:28 smithi143 ceph-mon[189871]: failed to encode map e223 with expected crc 2023-08-28T02:28:28.531 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:28 smithi143 ceph-mon[189871]: failed to encode map e223 with expected crc 2023-08-28T02:28:28.531 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:28 smithi143 ceph-mon[189871]: 6.2 scrub starts 2023-08-28T02:28:28.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:28.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:28:28.598 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:28:28.598 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:28:28.633 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:28:28.636 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set noscrub 2023-08-28T02:28:28.667 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:28.762 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:28:28.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:28.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:28:28.866 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:28:28.897 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:28.957 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:28:28.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:28.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:28:29.064 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:28:29.147 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 82 16 78480 78464 59.7637 3.6875 0.0104926 0.0166286 2023-08-28T02:28:29.148 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:29.171 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:28:29.259 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:28:29.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:29.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:28:29.310 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.5 dump_blocked_ops 2023-08-28T02:28:29.368 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:28:29.410 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:29.488 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:28:29.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:29.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:28:29.531 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:29 smithi143 ceph-mon[189871]: failed to encode map e224 with expected crc 2023-08-28T02:28:29.531 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:29 smithi143 ceph-mon[189871]: osdmap e224: 8 total, 8 up, 8 in 2023-08-28T02:28:29.531 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:29 smithi143 ceph-mon[189871]: failed to encode map e224 with expected crc 2023-08-28T02:28:29.531 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:29 smithi143 ceph-mon[189871]: failed to encode map e224 with expected crc 2023-08-28T02:28:29.531 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:29 smithi143 ceph-mon[189871]: failed to encode map e224 with expected crc 2023-08-28T02:28:29.531 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:29 smithi143 ceph-mon[189871]: failed to encode map e224 with expected crc 2023-08-28T02:28:29.531 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:29 smithi143 ceph-mon[183059]: failed to encode map e224 with expected crc 2023-08-28T02:28:29.532 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:29 smithi143 ceph-mon[183059]: osdmap e224: 8 total, 8 up, 8 in 2023-08-28T02:28:29.532 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:29 smithi143 ceph-mon[183059]: failed to encode map e224 with expected crc 2023-08-28T02:28:29.532 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:29 smithi143 ceph-mon[183059]: failed to encode map e224 with expected crc 2023-08-28T02:28:29.532 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:29 smithi143 ceph-mon[183059]: failed to encode map e224 with expected crc 2023-08-28T02:28:29.532 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:29 smithi143 ceph-mon[183059]: failed to encode map e224 with expected crc 2023-08-28T02:28:29.596 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:28:29.631 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:29.635 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:29 smithi169 ceph-mon[159415]: failed to encode map e224 with expected crc 2023-08-28T02:28:29.635 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:29 smithi169 ceph-mon[159415]: osdmap e224: 8 total, 8 up, 8 in 2023-08-28T02:28:29.635 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:29 smithi169 ceph-mon[159415]: failed to encode map e224 with expected crc 2023-08-28T02:28:29.635 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:29 smithi169 ceph-mon[159415]: failed to encode map e224 with expected crc 2023-08-28T02:28:29.635 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:29 smithi169 ceph-mon[159415]: failed to encode map e224 with expected crc 2023-08-28T02:28:29.635 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:29 smithi169 ceph-mon[159415]: failed to encode map e224 with expected crc 2023-08-28T02:28:29.732 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:28:29.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:29.740 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:28:29.766 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:28:29.841 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:28:29.911 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:30.018 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:28:30.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:30.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:28:30.127 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:28:30.147 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 83 16 79131 79115 59.5339 40.6875 0.00448155 0.0167698 2023-08-28T02:28:30.155 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:30.306 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:28:30.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:30.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:28:30.350 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:28:30.413 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:28:30.476 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:30.598 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:28:30.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:30.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:28:30.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:30 smithi143 ceph-mon[183059]: pgmap v525: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait, 1 active+recovering, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 16 MiB/s wr, 262 op/s; 16988/235197 objects degraded (7.223%); 10822/235197 objects misplaced (4.601%); 11 MiB/s, 173 objects/s recovering 2023-08-28T02:28:30.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:30 smithi143 ceph-mon[183059]: 6.2 scrub starts 2023-08-28T02:28:30.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:30 smithi143 ceph-mon[189871]: pgmap v525: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait, 1 active+recovering, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 16 MiB/s wr, 262 op/s; 16988/235197 objects degraded (7.223%); 10822/235197 objects misplaced (4.601%); 11 MiB/s, 173 objects/s recovering 2023-08-28T02:28:30.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:30 smithi143 ceph-mon[189871]: 6.2 scrub starts 2023-08-28T02:28:30.723 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:28:30.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:30 smithi169 ceph-mon[159415]: pgmap v525: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait, 1 active+recovering, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 16 MiB/s wr, 262 op/s; 16988/235197 objects degraded (7.223%); 10822/235197 objects misplaced (4.601%); 11 MiB/s, 173 objects/s recovering 2023-08-28T02:28:30.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:30 smithi169 ceph-mon[159415]: 6.2 scrub starts 2023-08-28T02:28:30.762 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:30.949 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:28:30.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:30.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:28:31.061 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:28:31.103 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:31.147 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 84 16 79730 79714 59.2709 37.4375 0.127798 0.0168631 2023-08-28T02:28:31.157 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:28:31.158 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:28:31.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:31.162 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:28:31.263 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:28:31.296 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:31.455 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:28:31.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:31.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:28:31.560 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:28:31.597 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:31.664 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:28:31.666 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:31 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 16988/235197 objects degraded (7.223%), 14 pgs degraded (PG_DEGRADED) 2023-08-28T02:28:31.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:31.668 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:28:31.697 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:31 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 16988/235197 objects degraded (7.223%), 14 pgs degraded (PG_DEGRADED) 2023-08-28T02:28:31.697 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:31 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 16988/235197 objects degraded (7.223%), 14 pgs degraded (PG_DEGRADED) 2023-08-28T02:28:31.703 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:28:31.704 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:28:31.767 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 7, 3, 1, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-08-28T02:28:31.767 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-08-28T02:28:31.768 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_1 2023-08-28T02:28:31.768 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph pg dump --format=json 2023-08-28T02:28:31.769 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:28:31.844 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:31.955 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:28:31 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:28:31] "GET /metrics HTTP/1.1" 200 36511 "" "Prometheus/2.33.4" 2023-08-28T02:28:31.969 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:28:31.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:31.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:28:32.078 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:28:32.105 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:32.147 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 85 16 80341 80325 59.023 38.1875 0.0948391 0.0169251 2023-08-28T02:28:32.148 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:28:32.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:32.150 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:28:32.251 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:28:32.276 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:32.429 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:32 smithi169 ceph-mon[159415]: pgmap v526: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait, 1 active+recovering, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 11 MiB/s wr, 177 op/s; 16988/235197 objects degraded (7.223%); 10822/235197 objects misplaced (4.601%); 7.1 MiB/s, 117 objects/s recovering 2023-08-28T02:28:32.453 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is set 2023-08-28T02:28:32.579 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.5 dump_historic_ops 2023-08-28T02:28:32.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:32 smithi143 ceph-mon[183059]: pgmap v526: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait, 1 active+recovering, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 11 MiB/s wr, 177 op/s; 16988/235197 objects degraded (7.223%); 10822/235197 objects misplaced (4.601%); 7.1 MiB/s, 117 objects/s recovering 2023-08-28T02:28:32.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:32 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2073857709' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:28:32.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:32 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:28:32.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:32 smithi143 ceph-mon[183059]: 6.2 deep-scrub starts 2023-08-28T02:28:32.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:32 smithi143 ceph-mon[189871]: pgmap v526: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait, 1 active+recovering, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 11 MiB/s wr, 177 op/s; 16988/235197 objects degraded (7.223%); 10822/235197 objects misplaced (4.601%); 7.1 MiB/s, 117 objects/s recovering 2023-08-28T02:28:32.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:32 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2073857709' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:28:32.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:32 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:28:32.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:32 smithi143 ceph-mon[189871]: 6.2 deep-scrub starts 2023-08-28T02:28:32.741 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:32 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2073857709' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:28:32.741 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:32 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:28:32.741 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:32 smithi169 ceph-mon[159415]: 6.2 deep-scrub starts 2023-08-28T02:28:32.925 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:28:32.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:32.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:28:33.032 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:28:33.040 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:28:33.059 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:33.103 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:28:33.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:33.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:28:33.147 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 86 16 81004 80988 58.8186 41.4375 0.0052167 0.0169886 2023-08-28T02:28:33.207 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:28:33.275 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:33.484 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:28:33.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:33.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:28:33.512 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:33 smithi143 ceph-mon[183059]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:28:33.512 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:33 smithi143 ceph-mon[183059]: failed to encode map e225 with expected crc 2023-08-28T02:28:33.512 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:33 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:28:33.512 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:33 smithi143 ceph-mon[183059]: osdmap e225: 8 total, 8 up, 8 in 2023-08-28T02:28:33.512 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:33 smithi143 ceph-mon[183059]: failed to encode map e225 with expected crc 2023-08-28T02:28:33.512 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:33 smithi143 ceph-mon[183059]: failed to encode map e225 with expected crc 2023-08-28T02:28:33.513 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:33 smithi143 ceph-mon[189871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:28:33.513 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:33 smithi143 ceph-mon[189871]: failed to encode map e225 with expected crc 2023-08-28T02:28:33.513 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:33 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:28:33.513 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:33 smithi143 ceph-mon[189871]: osdmap e225: 8 total, 8 up, 8 in 2023-08-28T02:28:33.513 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:33 smithi143 ceph-mon[189871]: failed to encode map e225 with expected crc 2023-08-28T02:28:33.513 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:33 smithi143 ceph-mon[189871]: failed to encode map e225 with expected crc 2023-08-28T02:28:33.597 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:28:33.618 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:33 smithi169 ceph-mon[159415]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:28:33.618 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:33 smithi169 ceph-mon[159415]: failed to encode map e225 with expected crc 2023-08-28T02:28:33.618 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:33 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:28:33.619 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:33 smithi169 ceph-mon[159415]: osdmap e225: 8 total, 8 up, 8 in 2023-08-28T02:28:33.619 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:33 smithi169 ceph-mon[159415]: failed to encode map e225 with expected crc 2023-08-28T02:28:33.619 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:33 smithi169 ceph-mon[159415]: failed to encode map e225 with expected crc 2023-08-28T02:28:33.688 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:33.862 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:28:33.903 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:28:33.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:33.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:28:33.941 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:28:33.959 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:28:33.994 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:28:34.008 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:28:34.033 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:28:34.076 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:28:34.083 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:34.132 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:28:34.148 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 87 16 81656 81640 58.611 40.75 0.050824 0.0170552 2023-08-28T02:28:34.205 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:28:34.205 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:28:34.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:34.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:28:34.244 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:28:34.248 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:28:34.248 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:28:34.248 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:28:34.248 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:28:34.248 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:28:34.248 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:28:34.248 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:28:34.248 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:28:34.319 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:28:34.409 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:34.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:34 smithi143 ceph-mon[183059]: pgmap v528: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait, 1 active+recovering, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 12 MiB/s wr, 196 op/s; 16988/237795 objects degraded (7.144%); 10822/237795 objects misplaced (4.551%); 7.0 MiB/s, 112 objects/s recovering 2023-08-28T02:28:34.456 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:34 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/608522856' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:28:34.629 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:34 smithi169 ceph-mon[159415]: pgmap v528: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait, 1 active+recovering, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 12 MiB/s wr, 196 op/s; 16988/237795 objects degraded (7.144%); 10822/237795 objects misplaced (4.551%); 7.0 MiB/s, 112 objects/s recovering 2023-08-28T02:28:34.630 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:34 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/608522856' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:28:34.742 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:28:34.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:34.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:28:34.758 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:34 smithi143 ceph-mon[189871]: pgmap v528: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait, 1 active+recovering, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 12 MiB/s wr, 196 op/s; 16988/237795 objects degraded (7.144%); 10822/237795 objects misplaced (4.551%); 7.0 MiB/s, 112 objects/s recovering 2023-08-28T02:28:34.758 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:34 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/608522856' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:28:34.847 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:28:34.865 INFO:teuthology.orchestra.run.smithi169.stdout: "size": 20, 2023-08-28T02:28:34.866 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 600, 2023-08-28T02:28:34.866 INFO:teuthology.orchestra.run.smithi169.stdout: "ops": [ 2023-08-28T02:28:34.866 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.867 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:24529 6.a 6:5d60f11f:::benchmark_data_smithi143_305742_object24528:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e195)", 2023-08-28T02:28:34.867 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:27:16.316338+0000", 2023-08-28T02:28:34.867 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 78.527597177000004, 2023-08-28T02:28:34.867 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.1223833000000001, 2023-08-28T02:28:34.868 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:28:34.868 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "reached pg", 2023-08-28T02:28:34.868 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:28:34.868 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:28:34.868 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:28:34.868 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 24529 2023-08-28T02:28:34.868 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.868 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:28:34.868 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.869 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:28:34.869 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.316338+0000", 2023-08-28T02:28:34.869 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:28:34.870 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.870 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.870 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:28:34.870 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.316338+0000", 2023-08-28T02:28:34.870 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999995 2023-08-28T02:28:34.870 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.870 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.870 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:28:34.871 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.316338+0000", 2023-08-28T02:28:34.871 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.0701000000000003e-05 2023-08-28T02:28:34.871 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.871 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.871 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:28:34.871 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.316368+0000", 2023-08-28T02:28:34.871 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.1279999999999996e-06 2023-08-28T02:28:34.871 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.872 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.873 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:28:34.873 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.316374+0000", 2023-08-28T02:28:34.873 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.1609999999999996e-06 2023-08-28T02:28:34.873 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.873 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.873 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:28:34.873 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.316379+0000", 2023-08-28T02:28:34.873 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.7463000000000002e-05 2023-08-28T02:28:34.873 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.874 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.875 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:34.875 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.316436+0000", 2023-08-28T02:28:34.875 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000108937 2023-08-28T02:28:34.875 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.875 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.875 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:28:34.875 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.316545+0000", 2023-08-28T02:28:34.875 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00015960400000000001 2023-08-28T02:28:34.875 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.875 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.877 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,6", 2023-08-28T02:28:34.877 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.316705+0000", 2023-08-28T02:28:34.877 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.002000862 2023-08-28T02:28:34.877 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.877 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.878 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:28:34.878 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.318705+0000", 2023-08-28T02:28:34.878 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00033078399999999998 2023-08-28T02:28:34.878 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.878 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.878 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:34.878 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.319036+0000", 2023-08-28T02:28:34.878 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0036924560000000002 2023-08-28T02:28:34.878 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.878 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.878 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:34.879 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.322729+0000", 2023-08-28T02:28:34.879 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0696999999999999e-05 2023-08-28T02:28:34.879 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.879 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.879 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:28:34.879 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.322739+0000", 2023-08-28T02:28:34.879 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.997422806 2023-08-28T02:28:34.879 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.879 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.879 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:34.879 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:17.320162+0000", 2023-08-28T02:28:34.879 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1532000000000001e-05 2023-08-28T02:28:34.879 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.880 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.880 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:28:34.880 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:17.320174+0000", 2023-08-28T02:28:34.880 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1184618099999999 2023-08-28T02:28:34.880 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.880 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.880 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:34.880 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.438635+0000", 2023-08-28T02:28:34.880 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.6031000000000004e-05 2023-08-28T02:28:34.880 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.880 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.880 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:28:34.881 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.438722+0000", 2023-08-28T02:28:34.881 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.1223833000000001 2023-08-28T02:28:34.881 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:34.881 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:28:34.881 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:34.881 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.881 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.881 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:24537 6.4 6:28e59a94:::benchmark_data_smithi143_305742_object24536:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e195)", 2023-08-28T02:28:34.881 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:27:16.321295+0000", 2023-08-28T02:28:34.881 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 78.522639967000003, 2023-08-28T02:28:34.881 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.0777337359999999, 2023-08-28T02:28:34.881 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:28:34.881 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:28:34.882 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:28:34.882 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:28:34.882 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:28:34.882 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 24537 2023-08-28T02:28:34.882 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.882 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:28:34.882 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.882 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:28:34.882 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.321295+0000", 2023-08-28T02:28:34.882 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:28:34.882 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.882 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.883 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:28:34.883 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.321295+0000", 2023-08-28T02:28:34.883 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999995 2023-08-28T02:28:34.883 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.883 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.883 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:28:34.883 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.321295+0000", 2023-08-28T02:28:34.883 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.9675999999999998e-05 2023-08-28T02:28:34.883 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.883 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.883 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:28:34.883 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.321325+0000", 2023-08-28T02:28:34.883 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.5349999999999999e-06 2023-08-28T02:28:34.884 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.884 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.884 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:28:34.884 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.321332+0000", 2023-08-28T02:28:34.884 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.3789999999999999e-06 2023-08-28T02:28:34.884 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.884 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.884 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:28:34.884 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.321336+0000", 2023-08-28T02:28:34.884 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0038322999999999999 2023-08-28T02:28:34.884 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.884 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.885 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:34.885 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.325169+0000", 2023-08-28T02:28:34.885 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.5019999999999998e-06 2023-08-28T02:28:34.885 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.885 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.885 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:28:34.885 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.325178+0000", 2023-08-28T02:28:34.885 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.99164512999999999 2023-08-28T02:28:34.885 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.885 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.885 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:34.885 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:17.316823+0000", 2023-08-28T02:28:34.885 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2853999999999999e-05 2023-08-28T02:28:34.886 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.886 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.886 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:28:34.886 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:17.316836+0000", 2023-08-28T02:28:34.886 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0684663839999999 2023-08-28T02:28:34.886 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.886 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.886 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:34.886 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.385303+0000", 2023-08-28T02:28:34.886 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00014184199999999999 2023-08-28T02:28:34.886 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.886 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.886 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:28:34.887 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.385444+0000", 2023-08-28T02:28:34.887 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00014312299999999999 2023-08-28T02:28:34.887 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.887 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.887 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 3,4", 2023-08-28T02:28:34.887 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.385588+0000", 2023-08-28T02:28:34.887 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0019724999999999999 2023-08-28T02:28:34.887 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.887 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.887 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:28:34.887 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.387560+0000", 2023-08-28T02:28:34.887 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0033467089999999998 2023-08-28T02:28:34.888 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.888 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.888 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:34.888 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.390907+0000", 2023-08-28T02:28:34.888 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0067727239999999999 2023-08-28T02:28:34.888 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.888 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.888 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:34.888 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.397680+0000", 2023-08-28T02:28:34.888 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.6778000000000002e-05 2023-08-28T02:28:34.888 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.888 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.888 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:28:34.889 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.397766+0000", 2023-08-28T02:28:34.889 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.001262846 2023-08-28T02:28:34.889 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.889 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.889 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:28:34.889 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.399029+0000", 2023-08-28T02:28:34.889 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.0777337359999999 2023-08-28T02:28:34.889 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:34.889 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:28:34.889 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:34.889 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.889 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.890 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:24538 6.4 6:2aa5ab67:::benchmark_data_smithi143_305742_object24537:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e195)", 2023-08-28T02:28:34.890 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:27:16.332135+0000", 2023-08-28T02:28:34.890 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 78.511799948000004, 2023-08-28T02:28:34.890 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.06786067, 2023-08-28T02:28:34.890 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:28:34.890 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:28:34.890 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:28:34.890 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:28:34.890 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:28:34.890 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 24538 2023-08-28T02:28:34.890 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.890 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:28:34.891 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.891 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:28:34.891 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.332135+0000", 2023-08-28T02:28:34.891 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:28:34.891 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.891 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.891 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:28:34.891 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.332135+0000", 2023-08-28T02:28:34.891 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:28:34.891 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.891 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.891 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:28:34.892 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.332134+0000", 2023-08-28T02:28:34.892 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.9920999999999999e-05 2023-08-28T02:28:34.892 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.892 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.892 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:28:34.892 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.332184+0000", 2023-08-28T02:28:34.892 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.4516999999999999e-05 2023-08-28T02:28:34.892 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.892 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.892 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:28:34.892 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.332208+0000", 2023-08-28T02:28:34.892 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.9049999999999997e-06 2023-08-28T02:28:34.892 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.893 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.893 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:28:34.893 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.332216+0000", 2023-08-28T02:28:34.893 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0016499900000000001 2023-08-28T02:28:34.893 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.893 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.893 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:34.893 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.333866+0000", 2023-08-28T02:28:34.893 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.4530000000000008e-06 2023-08-28T02:28:34.893 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.893 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.893 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:28:34.894 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.333876+0000", 2023-08-28T02:28:34.894 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.98298540099999998 2023-08-28T02:28:34.894 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.894 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.894 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:34.894 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:17.316861+0000", 2023-08-28T02:28:34.894 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.3659999999999992e-06 2023-08-28T02:28:34.894 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.894 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.894 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:28:34.894 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:17.316869+0000", 2023-08-28T02:28:34.894 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0707238489999999 2023-08-28T02:28:34.894 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.895 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.895 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:34.895 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.387593+0000", 2023-08-28T02:28:34.895 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00013506600000000001 2023-08-28T02:28:34.895 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.895 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.895 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:28:34.895 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.387728+0000", 2023-08-28T02:28:34.895 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000163043 2023-08-28T02:28:34.895 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.895 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.895 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 3,4", 2023-08-28T02:28:34.896 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.387891+0000", 2023-08-28T02:28:34.896 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.001260326 2023-08-28T02:28:34.896 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.896 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.896 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:28:34.896 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.389152+0000", 2023-08-28T02:28:34.896 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0028075629999999999 2023-08-28T02:28:34.896 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.896 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.897 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:34.897 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.391959+0000", 2023-08-28T02:28:34.897 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0071406500000000001 2023-08-28T02:28:34.897 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.897 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.897 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:34.897 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.399100+0000", 2023-08-28T02:28:34.897 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.2137000000000004e-05 2023-08-28T02:28:34.897 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.897 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.897 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:28:34.897 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.399172+0000", 2023-08-28T02:28:34.898 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00082417600000000003 2023-08-28T02:28:34.898 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.898 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.898 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:28:34.898 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.399996+0000", 2023-08-28T02:28:34.898 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.06786067 2023-08-28T02:28:34.898 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:34.898 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:28:34.898 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:34.898 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.898 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.898 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:24542 6.a 6:57630bd8:::benchmark_data_smithi143_305742_object24541:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e196)", 2023-08-28T02:28:34.899 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:27:16.332962+0000", 2023-08-28T02:28:34.899 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 78.510973304999993, 2023-08-28T02:28:34.899 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.1171907980000002, 2023-08-28T02:28:34.899 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:28:34.899 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:28:34.899 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:28:34.899 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:28:34.899 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:28:34.899 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 24542 2023-08-28T02:28:34.899 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.899 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:28:34.899 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.900 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:28:34.900 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.332962+0000", 2023-08-28T02:28:34.900 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:28:34.900 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.900 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.900 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:28:34.900 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.332962+0000", 2023-08-28T02:28:34.900 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:28:34.900 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.900 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.900 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:28:34.900 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.332961+0000", 2023-08-28T02:28:34.901 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00094618000000000002 2023-08-28T02:28:34.901 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.901 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.901 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:28:34.901 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.333907+0000", 2023-08-28T02:28:34.901 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.7800000000000006e-06 2023-08-28T02:28:34.901 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.901 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.901 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:28:34.901 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.333916+0000", 2023-08-28T02:28:34.901 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.9120000000000003e-06 2023-08-28T02:28:34.901 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.901 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.902 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:28:34.902 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.333922+0000", 2023-08-28T02:28:34.902 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00015135300000000001 2023-08-28T02:28:34.902 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.902 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.902 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:34.902 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.334073+0000", 2023-08-28T02:28:34.902 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.8440000000000004e-06 2023-08-28T02:28:34.902 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.902 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.902 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:28:34.902 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.334082+0000", 2023-08-28T02:28:34.903 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.98611508999999997 2023-08-28T02:28:34.903 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.903 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.903 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:34.903 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:17.320197+0000", 2023-08-28T02:28:34.903 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.3210000000000005e-06 2023-08-28T02:28:34.903 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.903 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.903 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:28:34.903 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:17.320206+0000", 2023-08-28T02:28:34.903 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.119032764 2023-08-28T02:28:34.903 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.904 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.904 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:34.904 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.439238+0000", 2023-08-28T02:28:34.904 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00014970000000000001 2023-08-28T02:28:34.904 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.904 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.904 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:28:34.904 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.439388+0000", 2023-08-28T02:28:34.904 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00021345199999999999 2023-08-28T02:28:34.904 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.904 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.904 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,2", 2023-08-28T02:28:34.905 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.439602+0000", 2023-08-28T02:28:34.905 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0019913299999999999 2023-08-28T02:28:34.905 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.905 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.905 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:28:34.905 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.441593+0000", 2023-08-28T02:28:34.905 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0018547699999999999 2023-08-28T02:28:34.905 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.905 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.905 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:34.905 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.443448+0000", 2023-08-28T02:28:34.905 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0050988140000000001 2023-08-28T02:28:34.906 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.906 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.906 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:34.906 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.448547+0000", 2023-08-28T02:28:34.906 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.4356000000000003e-05 2023-08-28T02:28:34.906 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.906 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.906 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:28:34.906 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.448641+0000", 2023-08-28T02:28:34.906 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.001511992 2023-08-28T02:28:34.906 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.906 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.907 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:28:34.907 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.450153+0000", 2023-08-28T02:28:34.907 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.1171907980000002 2023-08-28T02:28:34.907 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:34.907 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:28:34.907 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:34.907 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.907 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.907 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:65345 6.a 6:5cafef95:::benchmark_data_smithi143_305742_object65344:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-08-28T02:28:34.907 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:28:07.393170+0000", 2023-08-28T02:28:34.907 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 27.450765235999999, 2023-08-28T02:28:34.907 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.97206194499999998, 2023-08-28T02:28:34.908 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:28:34.908 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:28:34.908 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:28:34.908 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:28:34.908 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:28:34.908 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 65345 2023-08-28T02:28:34.908 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.908 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:28:34.908 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.908 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:28:34.908 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.393170+0000", 2023-08-28T02:28:34.908 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:28:34.909 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.909 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.909 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:28:34.909 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.393170+0000", 2023-08-28T02:28:34.909 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999976 2023-08-28T02:28:34.909 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.909 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.909 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:28:34.909 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.393168+0000", 2023-08-28T02:28:34.909 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.392e-05 2023-08-28T02:28:34.909 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.909 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.910 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:28:34.910 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.393212+0000", 2023-08-28T02:28:34.910 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1738e-05 2023-08-28T02:28:34.910 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.910 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.910 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:28:34.910 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.393224+0000", 2023-08-28T02:28:34.910 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.5649999999999998e-06 2023-08-28T02:28:34.910 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.910 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.910 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:28:34.910 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.393230+0000", 2023-08-28T02:28:34.910 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.4586000000000002e-05 2023-08-28T02:28:34.911 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.911 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.911 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:34.911 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.393285+0000", 2023-08-28T02:28:34.911 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.8400000000000007e-06 2023-08-28T02:28:34.911 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.911 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.911 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:28:34.911 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.393295+0000", 2023-08-28T02:28:34.911 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.96419019500000003 2023-08-28T02:28:34.911 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.911 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.912 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:34.912 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.357485+0000", 2023-08-28T02:28:34.912 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00013377699999999999 2023-08-28T02:28:34.912 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.912 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.912 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:28:34.912 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.357619+0000", 2023-08-28T02:28:34.912 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012924900000000001 2023-08-28T02:28:34.912 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.912 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.912 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,2", 2023-08-28T02:28:34.912 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.357748+0000", 2023-08-28T02:28:34.913 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.006065857 2023-08-28T02:28:34.913 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.913 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.913 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:28:34.913 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.363814+0000", 2023-08-28T02:28:34.913 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0012912659999999999 2023-08-28T02:28:34.913 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.913 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.913 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:34.913 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.365105+0000", 2023-08-28T02:28:34.913 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.3149000000000003e-05 2023-08-28T02:28:34.913 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.913 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.914 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:34.914 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.365148+0000", 2023-08-28T02:28:34.914 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.8458999999999997e-05 2023-08-28T02:28:34.914 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.914 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.914 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:28:34.914 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.365207+0000", 2023-08-28T02:28:34.914 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.5528e-05 2023-08-28T02:28:34.914 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.914 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.914 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:28:34.914 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.365232+0000", 2023-08-28T02:28:34.915 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.97206194499999998 2023-08-28T02:28:34.915 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:34.915 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:28:34.915 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:34.915 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.915 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.915 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:65370 6.a 6:5b0efaa9:::benchmark_data_smithi143_305742_object65369:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-08-28T02:28:34.915 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:28:07.421258+0000", 2023-08-28T02:28:34.915 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 27.422677264000001, 2023-08-28T02:28:34.915 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.95615907099999997, 2023-08-28T02:28:34.915 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:28:34.915 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:28:34.915 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:28:34.916 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:28:34.916 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:28:34.916 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 65370 2023-08-28T02:28:34.916 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.916 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:28:34.916 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.916 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:28:34.916 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.421258+0000", 2023-08-28T02:28:34.916 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:28:34.916 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.916 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.916 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:28:34.917 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.421258+0000", 2023-08-28T02:28:34.917 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:28:34.917 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.917 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.917 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:28:34.917 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.421257+0000", 2023-08-28T02:28:34.917 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.1618000000000004e-05 2023-08-28T02:28:34.917 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.917 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.917 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:28:34.917 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.421338+0000", 2023-08-28T02:28:34.917 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.7480000000000005e-06 2023-08-28T02:28:34.917 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.918 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.918 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:28:34.918 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.421348+0000", 2023-08-28T02:28:34.918 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.7060000000000003e-06 2023-08-28T02:28:34.918 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.918 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.918 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:28:34.918 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.421353+0000", 2023-08-28T02:28:34.918 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.9262000000000003e-05 2023-08-28T02:28:34.918 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.918 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.918 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:34.919 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.421412+0000", 2023-08-28T02:28:34.919 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0037000000000001e-05 2023-08-28T02:28:34.919 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.919 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.919 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:28:34.919 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.421422+0000", 2023-08-28T02:28:34.919 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.93841324500000001 2023-08-28T02:28:34.919 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.919 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.919 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:34.919 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.359835+0000", 2023-08-28T02:28:34.919 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00010686199999999999 2023-08-28T02:28:34.919 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.920 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.920 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:28:34.920 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.359942+0000", 2023-08-28T02:28:34.920 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000114066 2023-08-28T02:28:34.920 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.920 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.920 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,2", 2023-08-28T02:28:34.920 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.360056+0000", 2023-08-28T02:28:34.920 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0051866639999999997 2023-08-28T02:28:34.920 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.920 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.920 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:28:34.920 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.365243+0000", 2023-08-28T02:28:34.921 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0065147149999999999 2023-08-28T02:28:34.921 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.921 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.921 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:34.921 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.371758+0000", 2023-08-28T02:28:34.921 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0055873479999999998 2023-08-28T02:28:34.921 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.921 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.921 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:34.921 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377345+0000", 2023-08-28T02:28:34.921 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.3902000000000001e-05 2023-08-28T02:28:34.921 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.922 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.922 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:28:34.922 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377389+0000", 2023-08-28T02:28:34.922 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.8276000000000002e-05 2023-08-28T02:28:34.922 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.922 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.922 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:28:34.922 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377417+0000", 2023-08-28T02:28:34.922 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.95615907099999997 2023-08-28T02:28:34.922 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:34.922 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:28:34.922 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:34.922 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.923 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.923 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:65382 6.a 6:591f19d2:::benchmark_data_smithi143_305742_object65381:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-08-28T02:28:34.923 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:28:07.430975+0000", 2023-08-28T02:28:34.923 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 27.412960103, 2023-08-28T02:28:34.923 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.94658987900000002, 2023-08-28T02:28:34.923 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:28:34.923 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:28:34.923 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:28:34.923 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:28:34.923 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:28:34.923 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 65382 2023-08-28T02:28:34.923 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.924 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:28:34.924 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.924 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:28:34.924 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.430975+0000", 2023-08-28T02:28:34.924 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:28:34.924 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.924 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.924 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:28:34.924 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.430975+0000", 2023-08-28T02:28:34.924 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:28:34.924 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.924 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.924 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:28:34.925 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.430974+0000", 2023-08-28T02:28:34.925 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.9980000000000004e-05 2023-08-28T02:28:34.925 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.925 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.925 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:28:34.925 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.431044+0000", 2023-08-28T02:28:34.925 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.115e-06 2023-08-28T02:28:34.925 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.925 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.925 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:28:34.925 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.431053+0000", 2023-08-28T02:28:34.925 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.6990000000000002e-06 2023-08-28T02:28:34.925 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.926 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.926 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:28:34.926 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.431058+0000", 2023-08-28T02:28:34.926 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.4104999999999994e-05 2023-08-28T02:28:34.926 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.926 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.926 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:34.926 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.431152+0000", 2023-08-28T02:28:34.926 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.5532999999999998e-05 2023-08-28T02:28:34.926 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.926 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.926 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:28:34.926 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.431168+0000", 2023-08-28T02:28:34.927 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.92994798499999998 2023-08-28T02:28:34.927 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.927 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.927 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:34.927 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.361116+0000", 2023-08-28T02:28:34.927 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000128291 2023-08-28T02:28:34.927 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.927 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.927 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:28:34.927 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.361244+0000", 2023-08-28T02:28:34.927 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00016667299999999999 2023-08-28T02:28:34.927 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.928 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.928 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,2", 2023-08-28T02:28:34.928 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.361411+0000", 2023-08-28T02:28:34.928 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0038359240000000001 2023-08-28T02:28:34.928 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.928 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.928 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:28:34.928 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.365247+0000", 2023-08-28T02:28:34.928 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0093736270000000007 2023-08-28T02:28:34.928 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.928 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.928 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:34.928 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.374620+0000", 2023-08-28T02:28:34.929 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0028494010000000001 2023-08-28T02:28:34.929 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.929 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.929 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:34.929 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377470+0000", 2023-08-28T02:28:34.929 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.486e-05 2023-08-28T02:28:34.929 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.929 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.929 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:28:34.929 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377525+0000", 2023-08-28T02:28:34.929 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.0658999999999998e-05 2023-08-28T02:28:34.929 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.929 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.930 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:28:34.930 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377565+0000", 2023-08-28T02:28:34.930 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.94658987900000002 2023-08-28T02:28:34.930 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:34.930 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:28:34.930 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:34.930 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.930 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.930 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:65411 6.a 6:59a52dc1:::benchmark_data_smithi143_305742_object65410:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-08-28T02:28:34.930 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:28:07.493832+0000", 2023-08-28T02:28:34.930 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 27.350103474000001, 2023-08-28T02:28:34.930 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.88382154300000004, 2023-08-28T02:28:34.931 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:28:34.931 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:28:34.931 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:28:34.931 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:28:34.931 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:28:34.931 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 65411 2023-08-28T02:28:34.931 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.931 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:28:34.931 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.931 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:28:34.931 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493832+0000", 2023-08-28T02:28:34.931 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:28:34.931 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.932 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.932 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:28:34.932 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493832+0000", 2023-08-28T02:28:34.932 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999995 2023-08-28T02:28:34.932 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.932 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.932 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:28:34.932 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493831+0000", 2023-08-28T02:28:34.932 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.7842999999999997e-05 2023-08-28T02:28:34.932 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.932 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.932 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:28:34.933 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493869+0000", 2023-08-28T02:28:34.933 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.6300000000000003e-06 2023-08-28T02:28:34.933 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.933 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.933 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:28:34.933 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493875+0000", 2023-08-28T02:28:34.933 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.6119999999999999e-06 2023-08-28T02:28:34.933 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.933 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.933 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:28:34.933 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493879+0000", 2023-08-28T02:28:34.933 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.4725999999999997e-05 2023-08-28T02:28:34.934 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.934 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.934 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:34.934 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493933+0000", 2023-08-28T02:28:34.934 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.5399999999999996e-06 2023-08-28T02:28:34.934 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.934 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.934 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:28:34.934 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493942+0000", 2023-08-28T02:28:34.934 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.868725466 2023-08-28T02:28:34.934 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.934 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.935 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:34.935 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.362667+0000", 2023-08-28T02:28:34.935 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012190500000000001 2023-08-28T02:28:34.935 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.935 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.935 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:28:34.935 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.362789+0000", 2023-08-28T02:28:34.935 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000101784 2023-08-28T02:28:34.935 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.935 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.935 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,2", 2023-08-28T02:28:34.935 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.362891+0000", 2023-08-28T02:28:34.936 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0023590709999999999 2023-08-28T02:28:34.936 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.936 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.936 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:28:34.936 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.365250+0000", 2023-08-28T02:28:34.936 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0094366910000000005 2023-08-28T02:28:34.936 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.936 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.936 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:34.936 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.374687+0000", 2023-08-28T02:28:34.936 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0029174880000000002 2023-08-28T02:28:34.936 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.937 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.937 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:34.937 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377604+0000", 2023-08-28T02:28:34.937 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.8928000000000002e-05 2023-08-28T02:28:34.937 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.937 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.937 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:28:34.937 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377633+0000", 2023-08-28T02:28:34.937 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.0352000000000001e-05 2023-08-28T02:28:34.937 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.937 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.937 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:28:34.937 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377653+0000", 2023-08-28T02:28:34.938 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.88382154300000004 2023-08-28T02:28:34.938 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:34.938 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:28:34.938 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:34.938 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.938 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.938 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:65413 6.a 6:51617f2c:::benchmark_data_smithi143_305742_object65412:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-08-28T02:28:34.938 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:28:07.493890+0000", 2023-08-28T02:28:34.938 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 27.350045408, 2023-08-28T02:28:34.938 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.88389938400000001, 2023-08-28T02:28:34.938 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:28:34.938 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:28:34.939 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:28:34.939 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:28:34.939 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:28:34.939 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 65413 2023-08-28T02:28:34.939 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.939 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:28:34.939 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.939 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:28:34.939 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493890+0000", 2023-08-28T02:28:34.939 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:28:34.939 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.939 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.940 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:28:34.940 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493890+0000", 2023-08-28T02:28:34.940 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999995 2023-08-28T02:28:34.940 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.940 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.940 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:28:34.940 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493889+0000", 2023-08-28T02:28:34.940 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.5259999999999998e-05 2023-08-28T02:28:34.940 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.940 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.940 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:28:34.940 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493925+0000", 2023-08-28T02:28:34.940 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.6899999999999997e-06 2023-08-28T02:28:34.940 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.941 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.941 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:28:34.941 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493930+0000", 2023-08-28T02:28:34.941 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.9160000000000003e-06 2023-08-28T02:28:34.941 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.941 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.941 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:28:34.941 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493934+0000", 2023-08-28T02:28:34.941 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.5824e-05 2023-08-28T02:28:34.941 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.941 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.941 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:34.942 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493980+0000", 2023-08-28T02:28:34.942 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0889e-05 2023-08-28T02:28:34.942 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.942 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.942 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:28:34.942 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493991+0000", 2023-08-28T02:28:34.942 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.86984277300000001 2023-08-28T02:28:34.942 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.942 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.942 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:34.942 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.363834+0000", 2023-08-28T02:28:34.943 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.9104999999999996e-05 2023-08-28T02:28:34.943 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.943 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.943 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:28:34.943 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.363913+0000", 2023-08-28T02:28:34.943 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.9914999999999995e-05 2023-08-28T02:28:34.943 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.943 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.943 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,2", 2023-08-28T02:28:34.943 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.364013+0000", 2023-08-28T02:28:34.943 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.003811933 2023-08-28T02:28:34.943 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.944 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.944 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:28:34.944 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.367825+0000", 2023-08-28T02:28:34.944 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0069041399999999996 2023-08-28T02:28:34.944 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.944 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.944 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:34.944 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.374729+0000", 2023-08-28T02:28:34.944 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0029736039999999999 2023-08-28T02:28:34.944 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.944 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.944 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:34.945 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377703+0000", 2023-08-28T02:28:34.945 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.8238999999999997e-05 2023-08-28T02:28:34.945 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.945 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.945 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:28:34.945 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377751+0000", 2023-08-28T02:28:34.945 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.8593000000000001e-05 2023-08-28T02:28:34.945 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.945 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.945 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:28:34.945 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377789+0000", 2023-08-28T02:28:34.945 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.88389938400000001 2023-08-28T02:28:34.945 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:34.946 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:28:34.946 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:34.946 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.946 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.946 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:65445 6.a 6:554d4ed8:::benchmark_data_smithi143_305742_object65444:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-08-28T02:28:34.946 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:28:07.533277+0000", 2023-08-28T02:28:34.946 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 27.310658338, 2023-08-28T02:28:34.946 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.84459700800000004, 2023-08-28T02:28:34.946 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:28:34.946 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:28:34.946 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:28:34.946 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:28:34.947 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:28:34.947 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 65445 2023-08-28T02:28:34.947 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.947 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:28:34.947 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.947 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:28:34.947 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533277+0000", 2023-08-28T02:28:34.947 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:28:34.947 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.947 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.947 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:28:34.947 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533277+0000", 2023-08-28T02:28:34.948 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:28:34.948 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.948 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.948 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:28:34.948 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533276+0000", 2023-08-28T02:28:34.948 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.8158999999999996e-05 2023-08-28T02:28:34.948 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.948 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.948 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:28:34.948 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533344+0000", 2023-08-28T02:28:34.948 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.1166e-05 2023-08-28T02:28:34.949 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:28:34.950 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.950 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.950 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:28:34.950 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533365+0000", 2023-08-28T02:28:34.950 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.2080000000000002e-06 2023-08-28T02:28:34.951 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.951 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.951 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:28:34.951 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533369+0000", 2023-08-28T02:28:34.951 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.5699999999999998e-05 2023-08-28T02:28:34.951 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.951 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.951 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:34.951 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533435+0000", 2023-08-28T02:28:34.951 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0674e-05 2023-08-28T02:28:34.951 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.951 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.952 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:28:34.952 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533446+0000", 2023-08-28T02:28:34.952 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.83182787300000005 2023-08-28T02:28:34.952 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.952 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.952 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:34.952 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.365274+0000", 2023-08-28T02:28:34.952 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000119156 2023-08-28T02:28:34.952 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.952 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.952 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:28:34.952 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.365393+0000", 2023-08-28T02:28:34.952 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000164434 2023-08-28T02:28:34.953 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.953 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.953 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,2", 2023-08-28T02:28:34.953 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.365557+0000", 2023-08-28T02:28:34.953 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0062156579999999998 2023-08-28T02:28:34.953 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.953 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.953 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:28:34.953 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.371773+0000", 2023-08-28T02:28:34.953 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0055333500000000002 2023-08-28T02:28:34.953 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.953 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.954 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:34.954 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377306+0000", 2023-08-28T02:28:34.954 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00051990900000000004 2023-08-28T02:28:34.954 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.954 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.954 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:34.954 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377826+0000", 2023-08-28T02:28:34.954 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.7752000000000001e-05 2023-08-28T02:28:34.954 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.954 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.954 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:28:34.954 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377854+0000", 2023-08-28T02:28:34.954 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.012e-05 2023-08-28T02:28:34.955 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.955 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.955 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:28:34.955 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377874+0000", 2023-08-28T02:28:34.955 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.84459700800000004 2023-08-28T02:28:34.955 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:34.955 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:28:34.955 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:34.955 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.955 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.955 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:65447 6.a 6:5c3c3d34:::benchmark_data_smithi143_305742_object65446:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-08-28T02:28:34.955 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:28:07.533387+0000", 2023-08-28T02:28:34.956 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 27.310548425, 2023-08-28T02:28:34.956 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.858966224, 2023-08-28T02:28:34.956 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:28:34.956 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:28:34.956 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:28:34.956 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:28:34.956 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:28:34.956 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 65447 2023-08-28T02:28:34.956 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.956 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:28:34.956 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.956 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:28:34.956 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533387+0000", 2023-08-28T02:28:34.957 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:28:34.957 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.957 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.957 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:28:34.957 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533387+0000", 2023-08-28T02:28:34.957 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:28:34.957 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.957 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.957 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:28:34.957 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533386+0000", 2023-08-28T02:28:34.957 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.5514999999999999e-05 2023-08-28T02:28:34.957 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.958 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.958 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:28:34.958 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533452+0000", 2023-08-28T02:28:34.958 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.9199999999999998e-06 2023-08-28T02:28:34.958 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.958 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.958 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:28:34.958 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533459+0000", 2023-08-28T02:28:34.958 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.9409999999999998e-06 2023-08-28T02:28:34.958 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.958 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.958 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:28:34.959 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533464+0000", 2023-08-28T02:28:34.959 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.6830999999999999e-05 2023-08-28T02:28:34.959 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.959 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.959 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:34.959 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533510+0000", 2023-08-28T02:28:34.959 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.3380999999999999e-05 2023-08-28T02:28:34.959 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.959 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.959 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:28:34.959 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533524+0000", 2023-08-28T02:28:34.959 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.83310036300000001 2023-08-28T02:28:34.960 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.960 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.960 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:34.960 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.366624+0000", 2023-08-28T02:28:34.960 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012930600000000001 2023-08-28T02:28:34.960 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.960 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.960 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:28:34.960 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.366753+0000", 2023-08-28T02:28:34.960 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000102413 2023-08-28T02:28:34.960 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.960 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.960 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,2", 2023-08-28T02:28:34.961 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.366856+0000", 2023-08-28T02:28:34.961 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0077729440000000004 2023-08-28T02:28:34.961 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.961 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.961 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:28:34.961 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.374629+0000", 2023-08-28T02:28:34.961 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0032932349999999998 2023-08-28T02:28:34.961 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.961 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.961 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:34.961 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377922+0000", 2023-08-28T02:28:34.961 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.01433544 2023-08-28T02:28:34.962 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.962 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.962 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:34.962 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.392257+0000", 2023-08-28T02:28:34.962 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.2425e-05 2023-08-28T02:28:34.962 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.962 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.962 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:28:34.962 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.392310+0000", 2023-08-28T02:28:34.962 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.3294999999999998e-05 2023-08-28T02:28:34.962 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.962 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.963 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:28:34.963 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.392353+0000", 2023-08-28T02:28:34.963 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.858966224 2023-08-28T02:28:34.963 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:34.963 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:28:34.963 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:34.963 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.963 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.963 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:65449 6.a 6:53253814:::benchmark_data_smithi143_305742_object65448:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-08-28T02:28:34.963 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:28:07.533542+0000", 2023-08-28T02:28:34.963 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 27.310392995000001, 2023-08-28T02:28:34.963 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.85898421800000002, 2023-08-28T02:28:34.964 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:28:34.964 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:28:34.964 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:28:34.964 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:28:34.964 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:28:34.964 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 65449 2023-08-28T02:28:34.964 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.964 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:28:34.964 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.964 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:28:34.964 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533542+0000", 2023-08-28T02:28:34.964 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:28:34.965 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.965 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.965 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:28:34.965 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533542+0000", 2023-08-28T02:28:34.965 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999995 2023-08-28T02:28:34.965 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.965 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.965 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:28:34.965 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533542+0000", 2023-08-28T02:28:34.965 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.8541999999999999e-05 2023-08-28T02:28:34.965 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.965 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.966 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:28:34.966 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533570+0000", 2023-08-28T02:28:34.966 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.267e-06 2023-08-28T02:28:34.966 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.966 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.966 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:28:34.966 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533578+0000", 2023-08-28T02:28:34.966 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.6210000000000003e-06 2023-08-28T02:28:34.966 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.966 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.966 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:28:34.966 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533582+0000", 2023-08-28T02:28:34.967 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.1579000000000002e-05 2023-08-28T02:28:34.967 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.967 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.967 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:34.967 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533634+0000", 2023-08-28T02:28:34.967 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.8409999999999995e-06 2023-08-28T02:28:34.967 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.967 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.967 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:28:34.967 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533644+0000", 2023-08-28T02:28:34.967 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.83422247699999996 2023-08-28T02:28:34.967 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.968 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.968 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:34.968 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.367866+0000", 2023-08-28T02:28:34.968 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00013198600000000001 2023-08-28T02:28:34.968 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.968 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.968 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:28:34.968 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.367998+0000", 2023-08-28T02:28:34.968 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00016983100000000001 2023-08-28T02:28:34.968 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.968 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.968 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,2", 2023-08-28T02:28:34.969 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.368168+0000", 2023-08-28T02:28:34.969 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0064664470000000002 2023-08-28T02:28:34.969 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.969 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.969 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:28:34.969 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.374634+0000", 2023-08-28T02:28:34.969 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0074225960000000001 2023-08-28T02:28:34.969 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.969 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.969 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:34.969 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.382057+0000", 2023-08-28T02:28:34.969 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.010359019000000001 2023-08-28T02:28:34.969 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.970 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.970 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:34.970 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.392416+0000", 2023-08-28T02:28:34.970 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.0843e-05 2023-08-28T02:28:34.970 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.970 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.970 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:28:34.970 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.392477+0000", 2023-08-28T02:28:34.970 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.9842999999999997e-05 2023-08-28T02:28:34.970 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.970 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.970 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:28:34.971 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.392527+0000", 2023-08-28T02:28:34.971 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.85898421800000002 2023-08-28T02:28:34.971 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:34.971 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:28:34.971 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:34.971 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.971 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.971 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:65517 6.a 6:57043958:::benchmark_data_smithi143_305742_object65516:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-08-28T02:28:34.971 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:28:07.651846+0000", 2023-08-28T02:28:34.971 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 27.192089062000001, 2023-08-28T02:28:34.971 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.74079570299999997, 2023-08-28T02:28:34.971 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:28:34.972 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:28:34.972 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:28:34.972 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:28:34.972 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:28:34.972 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 65517 2023-08-28T02:28:34.972 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.972 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:28:34.972 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.972 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:28:34.972 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.651846+0000", 2023-08-28T02:28:34.972 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:28:34.972 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.972 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.973 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:28:34.973 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.651846+0000", 2023-08-28T02:28:34.973 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:28:34.973 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.973 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.973 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:28:34.973 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.651845+0000", 2023-08-28T02:28:34.973 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.3328000000000002e-05 2023-08-28T02:28:34.973 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.973 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.973 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:28:34.974 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.651899+0000", 2023-08-28T02:28:34.974 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1822e-05 2023-08-28T02:28:34.974 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.974 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.974 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:28:34.974 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.651910+0000", 2023-08-28T02:28:34.974 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.1270000000000002e-06 2023-08-28T02:28:34.974 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.974 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.974 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:28:34.974 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.651915+0000", 2023-08-28T02:28:34.974 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.2360000000000005e-05 2023-08-28T02:28:34.974 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.975 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.975 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:34.975 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.651988+0000", 2023-08-28T02:28:34.975 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.1109999999999997e-06 2023-08-28T02:28:34.975 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.975 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.975 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:28:34.975 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.651997+0000", 2023-08-28T02:28:34.975 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.71717974100000004 2023-08-28T02:28:34.975 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.975 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.975 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:34.976 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.369177+0000", 2023-08-28T02:28:34.976 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012856099999999999 2023-08-28T02:28:34.976 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.976 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.976 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:28:34.976 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.369305+0000", 2023-08-28T02:28:34.976 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00010799200000000001 2023-08-28T02:28:34.976 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.976 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.976 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,2", 2023-08-28T02:28:34.976 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.369413+0000", 2023-08-28T02:28:34.976 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.005225861 2023-08-28T02:28:34.976 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.977 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.977 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:28:34.977 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.374639+0000", 2023-08-28T02:28:34.977 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.013565384999999999 2023-08-28T02:28:34.977 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.977 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.977 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:34.977 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.388205+0000", 2023-08-28T02:28:34.977 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0043731919999999997 2023-08-28T02:28:34.977 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.977 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.977 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:34.978 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.392578+0000", 2023-08-28T02:28:34.978 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.9850000000000001e-05 2023-08-28T02:28:34.978 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.978 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.978 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:28:34.978 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.392608+0000", 2023-08-28T02:28:34.978 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.4477999999999998e-05 2023-08-28T02:28:34.978 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.978 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.978 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:28:34.978 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.392642+0000", 2023-08-28T02:28:34.978 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.74079570299999997 2023-08-28T02:28:34.978 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:34.979 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:28:34.979 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:34.979 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.979 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.979 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:65528 6.a 6:584fd7a4:::benchmark_data_smithi143_305742_object65527:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-08-28T02:28:34.979 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:28:07.678223+0000", 2023-08-28T02:28:34.979 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 27.165712051, 2023-08-28T02:28:34.979 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.71583466399999995, 2023-08-28T02:28:34.979 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:28:34.979 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:28:34.980 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:28:34.980 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:28:34.980 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:28:34.980 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 65528 2023-08-28T02:28:34.980 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.980 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:28:34.980 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.980 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:28:34.980 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.678223+0000", 2023-08-28T02:28:34.980 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:28:34.981 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.981 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.981 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:28:34.981 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.678223+0000", 2023-08-28T02:28:34.981 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:28:34.981 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.981 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.981 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:28:34.981 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.678222+0000", 2023-08-28T02:28:34.981 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.5943999999999999e-05 2023-08-28T02:28:34.981 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.981 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.982 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:28:34.982 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.678268+0000", 2023-08-28T02:28:34.982 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0148e-05 2023-08-28T02:28:34.982 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.982 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.982 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:28:34.982 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.678278+0000", 2023-08-28T02:28:34.982 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.0350000000000001e-06 2023-08-28T02:28:34.982 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.982 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.982 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:28:34.982 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.678286+0000", 2023-08-28T02:28:34.982 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.7000000000000003e-05 2023-08-28T02:28:34.983 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.983 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.983 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:34.983 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.678343+0000", 2023-08-28T02:28:34.983 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.8419999999999994e-06 2023-08-28T02:28:34.983 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.983 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.983 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:28:34.983 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.678352+0000", 2023-08-28T02:28:34.983 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.69213458500000002 2023-08-28T02:28:34.983 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.983 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.984 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:34.984 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.370487+0000", 2023-08-28T02:28:34.984 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012174400000000001 2023-08-28T02:28:34.984 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.984 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.984 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:28:34.984 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.370608+0000", 2023-08-28T02:28:34.984 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00016430399999999999 2023-08-28T02:28:34.984 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.984 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.984 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,2", 2023-08-28T02:28:34.984 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.370773+0000", 2023-08-28T02:28:34.984 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0039615850000000001 2023-08-28T02:28:34.985 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.985 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.985 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:28:34.985 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.374734+0000", 2023-08-28T02:28:34.985 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.017946527 2023-08-28T02:28:34.985 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.985 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.985 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:34.985 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.392681+0000", 2023-08-28T02:28:34.985 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.001296632 2023-08-28T02:28:34.985 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.985 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.986 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:34.986 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.393977+0000", 2023-08-28T02:28:34.986 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.4098999999999999e-05 2023-08-28T02:28:34.986 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.986 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.986 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:28:34.986 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.394022+0000", 2023-08-28T02:28:34.986 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.6463e-05 2023-08-28T02:28:34.986 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.986 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.986 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:28:34.986 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.394058+0000", 2023-08-28T02:28:34.987 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.71583466399999995 2023-08-28T02:28:34.987 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:34.987 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:28:34.987 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:34.987 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.987 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.987 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:65590 6.a 6:52c121f0:::benchmark_data_smithi143_305742_object65589:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-08-28T02:28:34.987 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:28:07.793373+0000", 2023-08-28T02:28:34.987 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 27.050562532000001, 2023-08-28T02:28:34.987 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.60446993100000002, 2023-08-28T02:28:34.987 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:28:34.987 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:28:34.987 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:28:34.988 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:28:34.988 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:28:34.988 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 65590 2023-08-28T02:28:34.988 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.988 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:28:34.988 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.988 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:28:34.988 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.793373+0000", 2023-08-28T02:28:34.988 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:28:34.988 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.988 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.988 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:28:34.989 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.793373+0000", 2023-08-28T02:28:34.989 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:28:34.989 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.989 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.989 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:28:34.989 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.793372+0000", 2023-08-28T02:28:34.989 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.9734000000000002e-05 2023-08-28T02:28:34.989 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.989 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.989 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:28:34.989 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.793421+0000", 2023-08-28T02:28:34.989 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2872e-05 2023-08-28T02:28:34.989 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.990 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.990 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:28:34.990 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.793434+0000", 2023-08-28T02:28:34.990 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.4409999999999998e-06 2023-08-28T02:28:34.990 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.990 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.990 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:28:34.990 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.793441+0000", 2023-08-28T02:28:34.990 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.2967000000000003e-05 2023-08-28T02:28:34.990 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.990 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.990 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:34.991 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.793494+0000", 2023-08-28T02:28:34.991 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.2439999999999996e-06 2023-08-28T02:28:34.991 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.991 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.991 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:28:34.991 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.793501+0000", 2023-08-28T02:28:34.991 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.57831116699999996 2023-08-28T02:28:34.991 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.991 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.991 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:34.991 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.371812+0000", 2023-08-28T02:28:34.991 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000123433 2023-08-28T02:28:34.991 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.992 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.992 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:28:34.992 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.371936+0000", 2023-08-28T02:28:34.992 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00016441700000000001 2023-08-28T02:28:34.992 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.992 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.992 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,2", 2023-08-28T02:28:34.992 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.372100+0000", 2023-08-28T02:28:34.992 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0026377420000000002 2023-08-28T02:28:34.992 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.992 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.992 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:28:34.992 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.374738+0000", 2023-08-28T02:28:34.993 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.019368895000000001 2023-08-28T02:28:34.993 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.993 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.993 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:34.993 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.394107+0000", 2023-08-28T02:28:34.993 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.003625634 2023-08-28T02:28:34.993 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.993 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.993 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:34.993 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.397732+0000", 2023-08-28T02:28:34.993 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.1490999999999998e-05 2023-08-28T02:28:34.993 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.994 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.994 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:28:34.994 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.397794+0000", 2023-08-28T02:28:34.994 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.9030000000000003e-05 2023-08-28T02:28:34.994 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.994 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.994 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:28:34.994 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.397843+0000", 2023-08-28T02:28:34.994 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.60446993100000002 2023-08-28T02:28:34.994 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:34.994 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:28:34.994 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:34.995 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.995 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.995 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:65630 6.a 6:54d1eb7a:::benchmark_data_smithi143_305742_object65629:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-08-28T02:28:34.995 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:28:07.879482+0000", 2023-08-28T02:28:34.995 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 26.964453834, 2023-08-28T02:28:34.995 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.51854038599999996, 2023-08-28T02:28:34.995 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:28:34.995 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:28:34.995 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:28:34.995 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:28:34.995 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:28:34.996 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 65630 2023-08-28T02:28:34.996 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.996 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:28:34.996 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.996 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:28:34.996 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.879482+0000", 2023-08-28T02:28:34.996 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:28:34.996 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.996 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.996 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:28:34.996 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.879482+0000", 2023-08-28T02:28:34.996 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999976 2023-08-28T02:28:34.997 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.997 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.997 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:28:34.997 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.879479+0000", 2023-08-28T02:28:34.997 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.5662999999999999e-05 2023-08-28T02:28:34.997 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.997 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.997 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:28:34.997 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.879535+0000", 2023-08-28T02:28:34.997 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.6812000000000001e-05 2023-08-28T02:28:34.997 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.997 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.998 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:28:34.998 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.879552+0000", 2023-08-28T02:28:34.998 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.8879999999999997e-06 2023-08-28T02:28:34.998 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.998 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.998 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:28:34.998 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.879559+0000", 2023-08-28T02:28:34.998 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.8687000000000002e-05 2023-08-28T02:28:34.998 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.998 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.998 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:34.998 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.879628+0000", 2023-08-28T02:28:34.999 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2177999999999999e-05 2023-08-28T02:28:34.999 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.999 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.999 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:28:34.999 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.879640+0000", 2023-08-28T02:28:34.999 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.49364018300000001 2023-08-28T02:28:34.999 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.999 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:34.999 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:34.999 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.373281+0000", 2023-08-28T02:28:34.999 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000123545 2023-08-28T02:28:34.999 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:34.999 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.000 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:28:35.000 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.373404+0000", 2023-08-28T02:28:35.000 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00016205899999999999 2023-08-28T02:28:35.000 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.000 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.000 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,2", 2023-08-28T02:28:35.000 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.373566+0000", 2023-08-28T02:28:35.000 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.003690292 2023-08-28T02:28:35.000 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.000 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.000 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:28:35.000 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377256+0000", 2023-08-28T02:28:35.001 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.016889149999999999 2023-08-28T02:28:35.001 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.001 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.001 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:35.001 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.394146+0000", 2023-08-28T02:28:35.001 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0037648989999999999 2023-08-28T02:28:35.001 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.001 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.001 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:35.001 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.397910+0000", 2023-08-28T02:28:35.001 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.1008999999999998e-05 2023-08-28T02:28:35.001 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.001 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.002 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:28:35.002 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.397971+0000", 2023-08-28T02:28:35.002 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.0457000000000002e-05 2023-08-28T02:28:35.002 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.002 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.002 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:28:35.002 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.398022+0000", 2023-08-28T02:28:35.002 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.51854038599999996 2023-08-28T02:28:35.002 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:35.002 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:28:35.002 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:35.002 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.003 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.003 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:65632 6.a 6:5613ae6c:::benchmark_data_smithi143_305742_object65631:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-08-28T02:28:35.003 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:28:07.884330+0000", 2023-08-28T02:28:35.003 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 26.959605750000001, 2023-08-28T02:28:35.003 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.51386843999999998, 2023-08-28T02:28:35.003 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:28:35.003 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:28:35.003 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:28:35.003 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:28:35.003 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:28:35.003 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 65632 2023-08-28T02:28:35.003 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.003 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:28:35.004 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.004 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:28:35.004 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.884330+0000", 2023-08-28T02:28:35.004 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:28:35.004 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.004 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.004 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:28:35.004 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.884330+0000", 2023-08-28T02:28:35.004 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:28:35.004 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.004 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.004 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:28:35.004 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.884328+0000", 2023-08-28T02:28:35.005 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.0673000000000001e-05 2023-08-28T02:28:35.005 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.005 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.005 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:28:35.005 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.884379+0000", 2023-08-28T02:28:35.005 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.296e-05 2023-08-28T02:28:35.005 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.005 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.005 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:28:35.005 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.884392+0000", 2023-08-28T02:28:35.005 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.7640000000000002e-06 2023-08-28T02:28:35.005 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.006 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.006 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:28:35.006 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.884398+0000", 2023-08-28T02:28:35.006 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.6813000000000001e-05 2023-08-28T02:28:35.006 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.006 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.006 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:35.006 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.884445+0000", 2023-08-28T02:28:35.006 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.9770000000000003e-06 2023-08-28T02:28:35.006 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.006 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.006 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:28:35.007 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.884451+0000", 2023-08-28T02:28:35.007 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.490316737 2023-08-28T02:28:35.007 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.007 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.007 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:35.007 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.374768+0000", 2023-08-28T02:28:35.007 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000120143 2023-08-28T02:28:35.007 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.007 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.007 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:28:35.007 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.374888+0000", 2023-08-28T02:28:35.007 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000175883 2023-08-28T02:28:35.008 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.008 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.008 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,2", 2023-08-28T02:28:35.008 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.375064+0000", 2023-08-28T02:28:35.008 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0023626849999999998 2023-08-28T02:28:35.008 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.008 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.008 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:28:35.008 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377427+0000", 2023-08-28T02:28:35.008 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.016763871999999999 2023-08-28T02:28:35.008 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.008 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.008 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:35.009 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.394191+0000", 2023-08-28T02:28:35.009 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0038951279999999999 2023-08-28T02:28:35.009 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.009 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.009 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:35.009 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.398086+0000", 2023-08-28T02:28:35.009 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.7018000000000002e-05 2023-08-28T02:28:35.009 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.009 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.009 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:28:35.009 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.398143+0000", 2023-08-28T02:28:35.009 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.5121000000000003e-05 2023-08-28T02:28:35.010 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.010 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.010 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:28:35.010 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.398198+0000", 2023-08-28T02:28:35.010 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.51386843999999998 2023-08-28T02:28:35.010 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:35.010 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:28:35.010 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:35.010 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.010 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.010 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:65634 6.a 6:59a0c1a8:::benchmark_data_smithi143_305742_object65633:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-08-28T02:28:35.010 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:28:07.885805+0000", 2023-08-28T02:28:35.011 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 26.958130738000001, 2023-08-28T02:28:35.011 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.51256355899999995, 2023-08-28T02:28:35.011 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:28:35.011 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:28:35.011 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:28:35.011 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:28:35.011 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:28:35.011 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 65634 2023-08-28T02:28:35.011 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.011 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:28:35.011 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.011 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:28:35.012 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.885805+0000", 2023-08-28T02:28:35.012 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:28:35.012 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.012 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.012 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:28:35.012 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.885805+0000", 2023-08-28T02:28:35.012 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:28:35.012 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.012 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.012 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:28:35.012 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.885804+0000", 2023-08-28T02:28:35.012 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.6968000000000001e-05 2023-08-28T02:28:35.013 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.013 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.013 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:28:35.013 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.885861+0000", 2023-08-28T02:28:35.013 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0698e-05 2023-08-28T02:28:35.013 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.013 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.013 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:28:35.013 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.885871+0000", 2023-08-28T02:28:35.013 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.2269999999999996e-06 2023-08-28T02:28:35.013 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.013 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.013 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:28:35.014 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.885878+0000", 2023-08-28T02:28:35.014 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.7559999999999998e-05 2023-08-28T02:28:35.014 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.014 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.014 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:35.014 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.885936+0000", 2023-08-28T02:28:35.014 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2449000000000001e-05 2023-08-28T02:28:35.014 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.014 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.014 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:28:35.014 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.885948+0000", 2023-08-28T02:28:35.014 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.49012758200000001 2023-08-28T02:28:35.015 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.015 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.015 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:35.015 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.376076+0000", 2023-08-28T02:28:35.015 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.6649000000000003e-05 2023-08-28T02:28:35.015 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.015 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.015 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:28:35.015 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.376163+0000", 2023-08-28T02:28:35.015 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000115367 2023-08-28T02:28:35.015 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.015 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.016 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,2", 2023-08-28T02:28:35.016 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.376278+0000", 2023-08-28T02:28:35.016 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.001382323 2023-08-28T02:28:35.016 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.016 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.016 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:28:35.016 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377660+0000", 2023-08-28T02:28:35.016 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.016561135000000001 2023-08-28T02:28:35.016 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.016 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.016 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:35.016 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.394222+0000", 2023-08-28T02:28:35.016 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0040408889999999998 2023-08-28T02:28:35.017 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.017 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.017 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:35.017 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.398262+0000", 2023-08-28T02:28:35.017 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.0809e-05 2023-08-28T02:28:35.017 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.017 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.017 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:28:35.017 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.398323+0000", 2023-08-28T02:28:35.017 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.4972999999999998e-05 2023-08-28T02:28:35.017 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.017 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.018 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:28:35.018 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.398368+0000", 2023-08-28T02:28:35.018 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.51256355899999995 2023-08-28T02:28:35.018 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:35.018 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:28:35.018 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:35.018 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.018 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.018 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:65644 6.a 6:5ea91d10:::benchmark_data_smithi143_305742_object65643:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-08-28T02:28:35.018 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:28:07.919016+0000", 2023-08-28T02:28:35.018 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 26.924919863, 2023-08-28T02:28:35.018 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.47950173400000001, 2023-08-28T02:28:35.019 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:28:35.019 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:28:35.019 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:28:35.019 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:28:35.019 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:28:35.019 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 65644 2023-08-28T02:28:35.019 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.019 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:28:35.019 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.019 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:28:35.019 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.919016+0000", 2023-08-28T02:28:35.019 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:28:35.019 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.020 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.020 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:28:35.020 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.919016+0000", 2023-08-28T02:28:35.020 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:28:35.020 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.020 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.020 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:28:35.020 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.919014+0000", 2023-08-28T02:28:35.020 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.3714999999999996e-05 2023-08-28T02:28:35.020 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.020 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.020 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:28:35.021 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.919078+0000", 2023-08-28T02:28:35.021 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.4093999999999999e-05 2023-08-28T02:28:35.021 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.021 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.021 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:28:35.021 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.919092+0000", 2023-08-28T02:28:35.021 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.1810000000000004e-06 2023-08-28T02:28:35.021 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.021 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.021 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:28:35.021 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.919099+0000", 2023-08-28T02:28:35.021 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.5120000000000002e-05 2023-08-28T02:28:35.021 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.022 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.022 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:35.022 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.919144+0000", 2023-08-28T02:28:35.022 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.6199999999999994e-06 2023-08-28T02:28:35.022 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.022 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.022 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:28:35.022 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.919154+0000", 2023-08-28T02:28:35.022 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.45879626000000001 2023-08-28T02:28:35.022 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.022 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.022 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:35.023 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377950+0000", 2023-08-28T02:28:35.023 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.5245000000000002e-05 2023-08-28T02:28:35.023 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.023 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.023 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:28:35.023 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.378025+0000", 2023-08-28T02:28:35.023 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.7627000000000006e-05 2023-08-28T02:28:35.023 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.023 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.023 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,2", 2023-08-28T02:28:35.023 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.378123+0000", 2023-08-28T02:28:35.023 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0021671680000000001 2023-08-28T02:28:35.023 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.024 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.024 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:28:35.024 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.380290+0000", 2023-08-28T02:28:35.024 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.013962753 2023-08-28T02:28:35.024 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.024 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.024 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:35.024 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.394253+0000", 2023-08-28T02:28:35.024 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0041639750000000003 2023-08-28T02:28:35.024 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.024 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.024 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:35.024 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.398417+0000", 2023-08-28T02:28:35.025 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.6158e-05 2023-08-28T02:28:35.025 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.025 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.025 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:28:35.025 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.398473+0000", 2023-08-28T02:28:35.025 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.4319e-05 2023-08-28T02:28:35.025 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.025 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.025 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:28:35.025 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.398517+0000", 2023-08-28T02:28:35.025 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.47950173400000001 2023-08-28T02:28:35.025 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:35.026 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:28:35.026 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:35.026 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.026 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.026 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:65654 6.a 6:5d7fc7b1:::benchmark_data_smithi143_305742_object65653:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-08-28T02:28:35.026 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:28:07.966725+0000", 2023-08-28T02:28:35.026 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 26.877210476999998, 2023-08-28T02:28:35.026 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.43370553099999998, 2023-08-28T02:28:35.026 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:28:35.026 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:28:35.026 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:28:35.026 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:28:35.027 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:28:35.027 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 65654 2023-08-28T02:28:35.027 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.027 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:28:35.027 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.027 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:28:35.027 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.966725+0000", 2023-08-28T02:28:35.027 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:28:35.027 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.027 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.027 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:28:35.027 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.966725+0000", 2023-08-28T02:28:35.028 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999976 2023-08-28T02:28:35.028 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.028 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.028 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:28:35.028 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.966723+0000", 2023-08-28T02:28:35.028 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.8591999999999998e-05 2023-08-28T02:28:35.028 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.028 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.028 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:28:35.028 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.966771+0000", 2023-08-28T02:28:35.028 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.2523e-05 2023-08-28T02:28:35.028 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.028 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.029 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:28:35.029 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.966804+0000", 2023-08-28T02:28:35.029 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.3426e-05 2023-08-28T02:28:35.029 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.029 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.029 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:28:35.029 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.966817+0000", 2023-08-28T02:28:35.029 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000114685 2023-08-28T02:28:35.029 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.029 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.029 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:35.029 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.966932+0000", 2023-08-28T02:28:35.030 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.7608999999999999e-05 2023-08-28T02:28:35.030 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.030 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.030 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:28:35.030 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.966959+0000", 2023-08-28T02:28:35.030 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.41208789600000001 2023-08-28T02:28:35.030 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.030 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.030 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:28:35.030 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.379047+0000", 2023-08-28T02:28:35.030 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000123454 2023-08-28T02:28:35.030 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.031 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.031 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:28:35.031 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.379171+0000", 2023-08-28T02:28:35.031 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000160791 2023-08-28T02:28:35.031 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.031 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.031 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,2", 2023-08-28T02:28:35.031 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.379332+0000", 2023-08-28T02:28:35.031 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0012247479999999999 2023-08-28T02:28:35.031 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.031 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.031 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:28:35.032 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.380556+0000", 2023-08-28T02:28:35.032 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.013728103 2023-08-28T02:28:35.032 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.032 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.032 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:35.032 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.394284+0000", 2023-08-28T02:28:35.032 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0060570850000000002 2023-08-28T02:28:35.032 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.032 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.032 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:28:35.032 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.400341+0000", 2023-08-28T02:28:35.032 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.1956000000000002e-05 2023-08-28T02:28:35.032 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.033 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.033 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:28:35.033 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.400393+0000", 2023-08-28T02:28:35.033 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.7042999999999998e-05 2023-08-28T02:28:35.033 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:28:35.033 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:28:35.033 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:28:35.033 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.400430+0000", 2023-08-28T02:28:35.033 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.43370553099999998 2023-08-28T02:28:35.033 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:35.033 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:28:35.033 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:35.034 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:28:35.034 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:28:35.034 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:28:35.068 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:35.148 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 88 16 82235 82219 58.3563 36.1875 0.00451746 0.0171299 2023-08-28T02:28:35.465 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:35 smithi169 ceph-mon[159415]: pgmap v529: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering, 1 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 24 MiB/s wr, 378 op/s; 16987/243198 objects degraded (6.985%); 11004/243198 objects misplaced (4.525%); 6.8 MiB/s, 109 objects/s recovering 2023-08-28T02:28:35.544 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set nodeep-scrub 2023-08-28T02:28:35.647 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.6 dump_ops_in_flight 2023-08-28T02:28:35.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:35 smithi143 ceph-mon[183059]: pgmap v529: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering, 1 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 24 MiB/s wr, 378 op/s; 16987/243198 objects degraded (6.985%); 11004/243198 objects misplaced (4.525%); 6.8 MiB/s, 109 objects/s recovering 2023-08-28T02:28:35.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:35 smithi143 ceph-mon[189871]: pgmap v529: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering, 1 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 24 MiB/s wr, 378 op/s; 16987/243198 objects degraded (6.985%); 11004/243198 objects misplaced (4.525%); 6.8 MiB/s, 109 objects/s recovering 2023-08-28T02:28:36.005 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:28:36.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:36.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:28:36.107 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:28:36.113 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:28:36.148 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 89 16 82974 82958 58.2196 46.1875 0.00884038 0.0171666 2023-08-28T02:28:36.187 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:36.360 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:28:36.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:36.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:28:36.467 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:28:36.502 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:36.662 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:28:36.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:36 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 16987/243198 objects degraded (6.985%), 14 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:28:36.756 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:28:36.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:36.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:28:36.860 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:28:36.944 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:36 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 16987/243198 objects degraded (6.985%), 14 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:28:36.946 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:36 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 16987/243198 objects degraded (6.985%), 14 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:28:36.954 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:37.050 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:28:37.058 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:28:37.062 INFO:teuthology.orchestra.run.smithi143.stderr:dumped all 2023-08-28T02:28:37.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:37.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:28:37.148 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Total time run: 90.0242 2023-08-28T02:28:37.148 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Total writes made: 83616 2023-08-28T02:28:37.148 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Write size: 65536 2023-08-28T02:28:37.149 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Object size: 65536 2023-08-28T02:28:37.149 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Bandwidth (MB/sec): 58.0511 2023-08-28T02:28:37.149 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Stddev Bandwidth: 42.4092 2023-08-28T02:28:37.149 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Max bandwidth (MB/sec): 220.5 2023-08-28T02:28:37.149 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Min bandwidth (MB/sec): 0 2023-08-28T02:28:37.149 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Average IOPS: 928 2023-08-28T02:28:37.149 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Stddev IOPS: 678.548 2023-08-28T02:28:37.149 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Max IOPS: 3528 2023-08-28T02:28:37.149 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Min IOPS: 0 2023-08-28T02:28:37.149 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Average Latency(s): 0.0172214 2023-08-28T02:28:37.149 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Stddev Latency(s): 0.0741785 2023-08-28T02:28:37.149 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Max latency(s): 4.61609 2023-08-28T02:28:37.149 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Min latency(s): 0.00119624 2023-08-28T02:28:37.158 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Cleaning up (deleting benchmark objects) 2023-08-28T02:28:37.167 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:28:37.258 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:37.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:37 smithi143 ceph-mon[183059]: from='client.45143 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:28:37.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:37 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:28:37.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:37 smithi143 ceph-mon[183059]: pgmap v530: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering, 1 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 115 B/s rd, 31 MiB/s wr, 495 op/s; 16987/247404 objects degraded (6.866%); 11004/247404 objects misplaced (4.448%); 6.0 MiB/s, 96 objects/s recovering 2023-08-28T02:28:37.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:37 smithi143 ceph-mon[189871]: from='client.45143 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:28:37.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:37 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:28:37.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:37 smithi143 ceph-mon[189871]: pgmap v530: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering, 1 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 115 B/s rd, 31 MiB/s wr, 495 op/s; 16987/247404 objects degraded (6.866%); 11004/247404 objects misplaced (4.448%); 6.0 MiB/s, 96 objects/s recovering 2023-08-28T02:28:37.822 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:37 smithi169 ceph-mon[159415]: from='client.45143 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:28:37.822 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:37 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:28:37.822 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:37 smithi169 ceph-mon[159415]: pgmap v530: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering, 1 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 115 B/s rd, 31 MiB/s wr, 495 op/s; 16987/247404 objects degraded (6.866%); 11004/247404 objects misplaced (4.448%); 6.0 MiB/s, 96 objects/s recovering 2023-08-28T02:28:37.957 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:28:38.066 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:28:38.066 INFO:teuthology.orchestra.run.smithi169.stdout: "ops": [], 2023-08-28T02:28:38.066 INFO:teuthology.orchestra.run.smithi169.stdout: "num_ops": 0 2023-08-28T02:28:38.067 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:28:38.651 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:28:38.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:38.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:28:38.691 INFO:teuthology.orchestra.run.smithi143.stdout:{"pg_ready":true,"pg_map":{"version":529,"stamp":"2023-08-28T02:28:35.243146+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5315696563,"num_objects":81066,"num_object_clones":0,"num_object_copies":243198,"num_objects_missing_on_primary":2401,"num_objects_missing":2401,"num_objects_degraded":16987,"num_objects_misplaced":11004,"num_objects_unfound":0,"num_objects_dirty":81066,"num_whiteouts":0,"num_read":6274,"num_read_kb":13707,"num_write":82144,"num_write_kb":5201573,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9780,"num_bytes_recovered":641225238,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":119546,"ondisk_log_size":119546,"up":297,"acting":284,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":350,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":18377448,"kb_used_data":16192104,"kb_used_omap":29,"kb_used_meta":2185250,"kb_avail":731583768,"statfs":{"total":767960285184,"available":749141778432,"internally_reserved":0,"allocated":16580714496,"data_stored":16574084840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30674,"internal_metadata":2237696046},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":195166208,"num_objects":2978,"num_object_clones":0,"num_object_copies":8934,"num_objects_missing_on_primary":418,"num_objects_missing":418,"num_objects_degraded":3349,"num_objects_misplaced":9460,"num_objects_unfound":0,"num_objects_dirty":2978,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2978,"num_write_kb":190592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":859,"num_bytes_recovered":56164352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.875621"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":225,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-08-28T02:28:27.409601+0000","last_change":"2023-08-28T02:27:31.367469+0000","last_active":"2023-08-28T02:28:27.409601+0000","last_peered":"2023-08-28T02:28:27.409601+0000","last_clean":"2023-08-28T02:28:27.409601+0000","last_became_active":"2023-08-28T02:27:31.366946+0000","last_became_peered":"2023-08-28T02:27:31.366946+0000","last_unstale":"2023-08-28T02:28:27.409601+0000","last_undegraded":"2023-08-28T02:28:27.409601+0000","last_fullsized":"2023-08-28T02:28:27.409601+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":110,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-08-28T02:28:28.369708+0000","last_change":"2023-08-28T02:27:31.324452+0000","last_active":"2023-08-28T02:28:28.369708+0000","last_peered":"2023-08-28T02:28:28.369708+0000","last_clean":"2023-08-28T02:28:28.369708+0000","last_became_active":"2023-08-28T02:27:31.323749+0000","last_became_peered":"2023-08-28T02:27:31.323749+0000","last_unstale":"2023-08-28T02:28:28.369708+0000","last_undegraded":"2023-08-28T02:28:28.369708+0000","last_fullsized":"2023-08-28T02:28:28.369708+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:18:10.904127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":31,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-08-28T02:28:28.891083+0000","last_change":"2023-08-28T02:27:31.337338+0000","last_active":"2023-08-28T02:28:28.891083+0000","last_peered":"2023-08-28T02:28:28.891083+0000","last_clean":"2023-08-28T02:28:28.891083+0000","last_became_active":"2023-08-28T02:27:31.337018+0000","last_became_peered":"2023-08-28T02:27:31.337018+0000","last_unstale":"2023-08-28T02:28:28.891083+0000","last_undegraded":"2023-08-28T02:28:28.891083+0000","last_fullsized":"2023-08-28T02:28:28.891083+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:46:34.845663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":211,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-08-28T02:28:28.761133+0000","last_change":"2023-08-28T02:23:59.223634+0000","last_active":"2023-08-28T02:28:28.761133+0000","last_peered":"2023-08-28T02:28:28.761133+0000","last_clean":"2023-08-28T02:28:28.761133+0000","last_became_active":"2023-08-28T02:23:59.222704+0000","last_became_peered":"2023-08-28T02:23:59.222704+0000","last_unstale":"2023-08-28T02:28:28.761133+0000","last_undegraded":"2023-08-28T02:28:28.761133+0000","last_fullsized":"2023-08-28T02:28:28.761133+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:51:52.752361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":101,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-08-28T02:28:28.368503+0000","last_change":"2023-08-28T02:27:31.318153+0000","last_active":"2023-08-28T02:28:28.368503+0000","last_peered":"2023-08-28T02:28:28.368503+0000","last_clean":"2023-08-28T02:28:28.368503+0000","last_became_active":"2023-08-28T02:27:31.317732+0000","last_became_peered":"2023-08-28T02:27:31.317732+0000","last_unstale":"2023-08-28T02:28:28.368503+0000","last_undegraded":"2023-08-28T02:28:28.368503+0000","last_fullsized":"2023-08-28T02:28:28.368503+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:06:13.421674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-08-28T02:28:27.374774+0000","last_change":"2023-08-28T02:28:25.348941+0000","last_active":"2023-08-28T02:28:27.374774+0000","last_peered":"2023-08-28T02:28:27.374774+0000","last_clean":"2023-08-28T02:28:27.374774+0000","last_became_active":"2023-08-28T02:28:25.348650+0000","last_became_peered":"2023-08-28T02:28:25.348650+0000","last_unstale":"2023-08-28T02:28:27.374774+0000","last_undegraded":"2023-08-28T02:28:27.374774+0000","last_fullsized":"2023-08-28T02:28:27.374774+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.12","version":"224'4373","reported_seq":4430,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-08-28T02:28:32.018340+0000","last_change":"2023-08-28T02:28:25.453536+0000","last_active":"2023-08-28T02:28:32.018340+0000","last_peered":"2023-08-28T02:28:32.018340+0000","last_clean":"2023-08-28T02:28:32.018340+0000","last_became_active":"2023-08-28T02:27:58.479936+0000","last_became_peered":"2023-08-28T02:27:58.479936+0000","last_unstale":"2023-08-28T02:28:32.018340+0000","last_undegraded":"2023-08-28T02:28:32.018340+0000","last_fullsized":"2023-08-28T02:28:32.018340+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":4373,"log_dups_size":0,"ondisk_log_size":4373,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164364288,"num_objects":2508,"num_object_clones":0,"num_object_copies":7524,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2508,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2508,"num_write_kb":160512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":192,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-08-28T02:28:28.761163+0000","last_change":"2023-08-28T02:23:59.227592+0000","last_active":"2023-08-28T02:28:28.761163+0000","last_peered":"2023-08-28T02:28:28.761163+0000","last_clean":"2023-08-28T02:28:28.761163+0000","last_became_active":"2023-08-28T02:23:59.227342+0000","last_became_peered":"2023-08-28T02:23:59.227342+0000","last_unstale":"2023-08-28T02:28:28.761163+0000","last_undegraded":"2023-08-28T02:28:28.761163+0000","last_fullsized":"2023-08-28T02:28:28.761163+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:08:19.874396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":119,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-08-28T02:28:28.368581+0000","last_change":"2023-08-28T02:28:25.348233+0000","last_active":"2023-08-28T02:28:28.368581+0000","last_peered":"2023-08-28T02:28:28.368581+0000","last_clean":"2023-08-28T02:28:28.368581+0000","last_became_active":"2023-08-28T02:28:25.347896+0000","last_became_peered":"2023-08-28T02:28:25.347896+0000","last_unstale":"2023-08-28T02:28:28.368581+0000","last_undegraded":"2023-08-28T02:28:28.368581+0000","last_fullsized":"2023-08-28T02:28:28.368581+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:50:17.674564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.13","version":"224'4454","reported_seq":4529,"reported_epoch":224,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:28:31.893531+0000","last_change":"2023-08-28T02:28:26.468919+0000","last_active":"2023-08-28T02:28:31.893531+0000","last_peered":"2023-08-28T02:28:31.893531+0000","last_clean":"2023-08-28T02:27:16.315150+0000","last_became_active":"2023-08-28T02:28:26.463939+0000","last_became_peered":"2023-08-28T02:28:26.463939+0000","last_unstale":"2023-08-28T02:28:31.893531+0000","last_undegraded":"2023-08-28T02:28:26.355850+0000","last_fullsized":"2023-08-28T02:28:26.347877+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":4454,"log_dups_size":0,"ondisk_log_size":4454,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168165376,"num_objects":2566,"num_object_clones":0,"num_object_copies":7698,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1703,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2566,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2566,"num_write_kb":164224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"0,2,7","objects":446},{"shards":"1,2,7","objects":1257},{"shards":"2,7","objects":863}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":226,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-08-28T02:28:27.390224+0000","last_change":"2023-08-28T02:27:31.328499+0000","last_active":"2023-08-28T02:28:27.390224+0000","last_peered":"2023-08-28T02:28:27.390224+0000","last_clean":"2023-08-28T02:28:27.390224+0000","last_became_active":"2023-08-28T02:27:31.328164+0000","last_became_peered":"2023-08-28T02:27:31.328164+0000","last_unstale":"2023-08-28T02:28:27.390224+0000","last_undegraded":"2023-08-28T02:28:27.390224+0000","last_fullsized":"2023-08-28T02:28:27.390224+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":101,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-08-28T02:28:28.369214+0000","last_change":"2023-08-28T02:28:25.348760+0000","last_active":"2023-08-28T02:28:28.369214+0000","last_peered":"2023-08-28T02:28:28.369214+0000","last_clean":"2023-08-28T02:28:28.369214+0000","last_became_active":"2023-08-28T02:28:25.348465+0000","last_became_peered":"2023-08-28T02:28:25.348465+0000","last_unstale":"2023-08-28T02:28:28.369214+0000","last_undegraded":"2023-08-28T02:28:28.369214+0000","last_fullsized":"2023-08-28T02:28:28.369214+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:43:05.659999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.10","version":"225'4338","reported_seq":4394,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-08-28T02:28:33.840591+0000","last_change":"2023-08-28T02:27:58.495599+0000","last_active":"2023-08-28T02:28:33.840591+0000","last_peered":"2023-08-28T02:28:33.840591+0000","last_clean":"2023-08-28T02:28:33.840591+0000","last_became_active":"2023-08-28T02:27:58.494108+0000","last_became_peered":"2023-08-28T02:27:58.494108+0000","last_unstale":"2023-08-28T02:28:33.840591+0000","last_undegraded":"2023-08-28T02:28:33.840591+0000","last_fullsized":"2023-08-28T02:28:33.840591+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":4338,"log_dups_size":0,"ondisk_log_size":4338,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163905536,"num_objects":2501,"num_object_clones":0,"num_object_copies":7503,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2501,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2501,"num_write_kb":160064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":109,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-08-28T02:28:27.377708+0000","last_change":"2023-08-28T02:23:59.316561+0000","last_active":"2023-08-28T02:28:27.377708+0000","last_peered":"2023-08-28T02:28:27.377708+0000","last_clean":"2023-08-28T02:28:27.377708+0000","last_became_active":"2023-08-28T02:23:59.227433+0000","last_became_peered":"2023-08-28T02:23:59.227433+0000","last_unstale":"2023-08-28T02:28:27.377708+0000","last_undegraded":"2023-08-28T02:28:27.377708+0000","last_fullsized":"2023-08-28T02:28:27.377708+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":41,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-08-28T02:28:28.369840+0000","last_change":"2023-08-28T02:28:25.347771+0000","last_active":"2023-08-28T02:28:28.369840+0000","last_peered":"2023-08-28T02:28:28.369840+0000","last_clean":"2023-08-28T02:28:28.369840+0000","last_became_active":"2023-08-28T02:28:25.347320+0000","last_became_peered":"2023-08-28T02:28:25.347320+0000","last_unstale":"2023-08-28T02:28:28.369840+0000","last_undegraded":"2023-08-28T02:28:28.369840+0000","last_fullsized":"2023-08-28T02:28:28.369840+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:20:27.350131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.11","version":"225'4461","reported_seq":7500,"reported_epoch":225,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:28:33.868665+0000","last_change":"2023-08-28T02:28:28.917276+0000","last_active":"2023-08-28T02:28:33.868665+0000","last_peered":"2023-08-28T02:28:33.868665+0000","last_clean":"2023-08-28T02:27:16.286032+0000","last_became_active":"2023-08-28T02:28:28.496820+0000","last_became_peered":"2023-08-28T02:28:28.496820+0000","last_unstale":"2023-08-28T02:28:33.868665+0000","last_undegraded":"2023-08-28T02:28:27.454055+0000","last_fullsized":"2023-08-28T02:28:26.327749+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":4461,"log_dups_size":0,"ondisk_log_size":4461,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169476096,"num_objects":2586,"num_object_clones":0,"num_object_copies":7758,"num_objects_missing_on_primary":486,"num_objects_missing":486,"num_objects_degraded":2140,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2586,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2586,"num_write_kb":165504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":743,"num_bytes_recovered":48660480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,2],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"0,2,7","objects":444},{"shards":"1,2","objects":890},{"shards":"1,2,7","objects":1210},{"shards":"2,6,7","objects":42}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"86'22","reported_seq":274,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-08-28T02:28:27.410287+0000","last_change":"2023-08-28T02:28:25.361783+0000","last_active":"2023-08-28T02:28:27.410287+0000","last_peered":"2023-08-28T02:28:27.410287+0000","last_clean":"2023-08-28T02:28:27.410287+0000","last_became_active":"2023-08-28T02:28:25.351418+0000","last_became_peered":"2023-08-28T02:28:25.351418+0000","last_unstale":"2023-08-28T02:28:27.410287+0000","last_undegraded":"2023-08-28T02:28:27.410287+0000","last_fullsized":"2023-08-28T02:28:27.410287+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":240,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-08-28T02:28:27.409937+0000","last_change":"2023-08-28T02:28:25.352395+0000","last_active":"2023-08-28T02:28:27.409937+0000","last_peered":"2023-08-28T02:28:27.409937+0000","last_clean":"2023-08-28T02:28:27.409937+0000","last_became_active":"2023-08-28T02:28:25.352189+0000","last_became_peered":"2023-08-28T02:28:25.352189+0000","last_unstale":"2023-08-28T02:28:27.409937+0000","last_undegraded":"2023-08-28T02:28:27.409937+0000","last_fullsized":"2023-08-28T02:28:27.409937+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.16","version":"225'4414","reported_seq":4471,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-08-28T02:28:34.107208+0000","last_change":"2023-08-28T02:27:58.492359+0000","last_active":"2023-08-28T02:28:34.107208+0000","last_peered":"2023-08-28T02:28:34.107208+0000","last_clean":"2023-08-28T02:28:34.107208+0000","last_became_active":"2023-08-28T02:27:58.479842+0000","last_became_peered":"2023-08-28T02:27:58.479842+0000","last_unstale":"2023-08-28T02:28:34.107208+0000","last_undegraded":"2023-08-28T02:28:34.107208+0000","last_fullsized":"2023-08-28T02:28:34.107208+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":4414,"log_dups_size":0,"ondisk_log_size":4414,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165707776,"num_objects":2529,"num_object_clones":0,"num_object_copies":7587,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2529,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2529,"num_write_kb":161824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":94,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-08-28T02:28:27.377204+0000","last_change":"2023-08-28T02:23:38.983634+0000","last_active":"2023-08-28T02:28:27.377204+0000","last_peered":"2023-08-28T02:28:27.377204+0000","last_clean":"2023-08-28T02:28:27.377204+0000","last_became_active":"2023-08-28T02:23:38.344566+0000","last_became_peered":"2023-08-28T02:23:38.344566+0000","last_unstale":"2023-08-28T02:28:27.377204+0000","last_undegraded":"2023-08-28T02:28:27.377204+0000","last_fullsized":"2023-08-28T02:28:27.377204+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":101,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-08-28T02:28:28.371607+0000","last_change":"2023-08-28T02:27:31.344974+0000","last_active":"2023-08-28T02:28:28.371607+0000","last_peered":"2023-08-28T02:28:28.371607+0000","last_clean":"2023-08-28T02:28:28.371607+0000","last_became_active":"2023-08-28T02:27:31.344281+0000","last_became_peered":"2023-08-28T02:27:31.344281+0000","last_unstale":"2023-08-28T02:28:28.371607+0000","last_undegraded":"2023-08-28T02:28:28.371607+0000","last_fullsized":"2023-08-28T02:28:28.371607+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:04:02.821752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.17","version":"224'4337","reported_seq":4391,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-08-28T02:28:32.161675+0000","last_change":"2023-08-28T02:27:58.474771+0000","last_active":"2023-08-28T02:28:32.161675+0000","last_peered":"2023-08-28T02:28:32.161675+0000","last_clean":"2023-08-28T02:28:32.161675+0000","last_became_active":"2023-08-28T02:27:58.472976+0000","last_became_peered":"2023-08-28T02:27:58.472976+0000","last_unstale":"2023-08-28T02:28:32.161675+0000","last_undegraded":"2023-08-28T02:28:32.161675+0000","last_fullsized":"2023-08-28T02:28:32.161675+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":4337,"log_dups_size":0,"ondisk_log_size":4337,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162594816,"num_objects":2481,"num_object_clones":0,"num_object_copies":7443,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2481,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2481,"num_write_kb":158784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":210,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-08-28T02:28:28.760871+0000","last_change":"2023-08-28T02:23:59.223066+0000","last_active":"2023-08-28T02:28:28.760871+0000","last_peered":"2023-08-28T02:28:28.760871+0000","last_clean":"2023-08-28T02:28:28.760871+0000","last_became_active":"2023-08-28T02:23:59.222731+0000","last_became_peered":"2023-08-28T02:23:59.222731+0000","last_unstale":"2023-08-28T02:28:28.760871+0000","last_undegraded":"2023-08-28T02:28:28.760871+0000","last_fullsized":"2023-08-28T02:28:28.760871+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:48:37.558266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":250,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-08-28T02:28:27.390258+0000","last_change":"2023-08-28T02:23:59.210469+0000","last_active":"2023-08-28T02:28:27.390258+0000","last_peered":"2023-08-28T02:28:27.390258+0000","last_clean":"2023-08-28T02:28:27.390258+0000","last_became_active":"2023-08-28T02:23:59.210031+0000","last_became_peered":"2023-08-28T02:23:59.210031+0000","last_unstale":"2023-08-28T02:28:27.390258+0000","last_undegraded":"2023-08-28T02:28:27.390258+0000","last_fullsized":"2023-08-28T02:28:27.390258+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.14","version":"225'4425","reported_seq":8542,"reported_epoch":225,"state":"active+recovery_wait+degraded","last_fresh":"2023-08-28T02:28:33.714572+0000","last_change":"2023-08-28T02:28:25.482653+0000","last_active":"2023-08-28T02:28:33.714572+0000","last_peered":"2023-08-28T02:28:33.714572+0000","last_clean":"2023-08-28T02:27:30.461946+0000","last_became_active":"2023-08-28T02:28:00.020960+0000","last_became_peered":"2023-08-28T02:28:00.020960+0000","last_unstale":"2023-08-28T02:28:33.714572+0000","last_undegraded":"2023-08-28T02:28:25.451731+0000","last_fullsized":"2023-08-28T02:28:33.714572+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":4425,"log_dups_size":0,"ondisk_log_size":4425,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169902080,"num_objects":2593,"num_object_clones":0,"num_object_copies":7779,"num_objects_missing_on_primary":682,"num_objects_missing":682,"num_objects_degraded":682,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2593,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2593,"num_write_kb":165920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1279,"num_bytes_recovered":83787776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":["3","4","5","6"],"object_location_counts":[{"shards":"1,3,5","objects":1911},{"shards":"3,4,5","objects":682}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":106,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-08-28T02:28:27.375546+0000","last_change":"2023-08-28T02:28:25.354526+0000","last_active":"2023-08-28T02:28:27.375546+0000","last_peered":"2023-08-28T02:28:27.375546+0000","last_clean":"2023-08-28T02:28:27.375546+0000","last_became_active":"2023-08-28T02:28:25.350991+0000","last_became_peered":"2023-08-28T02:28:25.350991+0000","last_unstale":"2023-08-28T02:28:27.375546+0000","last_undegraded":"2023-08-28T02:28:27.375546+0000","last_fullsized":"2023-08-28T02:28:27.375546+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":211,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-08-28T02:28:28.761220+0000","last_change":"2023-08-28T02:23:59.220932+0000","last_active":"2023-08-28T02:28:28.761220+0000","last_peered":"2023-08-28T02:28:28.761220+0000","last_clean":"2023-08-28T02:28:28.761220+0000","last_became_active":"2023-08-28T02:23:59.220488+0000","last_became_peered":"2023-08-28T02:23:59.220488+0000","last_unstale":"2023-08-28T02:28:28.761220+0000","last_undegraded":"2023-08-28T02:28:28.761220+0000","last_fullsized":"2023-08-28T02:28:28.761220+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:53:09.216340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.15","version":"224'4409","reported_seq":5655,"reported_epoch":224,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:28:31.987789+0000","last_change":"2023-08-28T02:28:25.456367+0000","last_active":"2023-08-28T02:28:31.987789+0000","last_peered":"2023-08-28T02:28:31.987789+0000","last_clean":"2023-08-28T02:27:16.315756+0000","last_became_active":"2023-08-28T02:28:25.449323+0000","last_became_peered":"2023-08-28T02:28:25.449323+0000","last_unstale":"2023-08-28T02:28:31.987789+0000","last_undegraded":"2023-08-28T02:28:25.349175+0000","last_fullsized":"2023-08-28T02:28:25.327058+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":4409,"log_dups_size":0,"ondisk_log_size":4409,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165478400,"num_objects":2525,"num_object_clones":0,"num_object_copies":7575,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1690,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2525,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2525,"num_write_kb":161600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":294,"num_bytes_recovered":19267584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"2,3,7","objects":1690},{"shards":"3,7","objects":835}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":23,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-08-28T02:28:27.374196+0000","last_change":"2023-08-28T02:28:25.703333+0000","last_active":"2023-08-28T02:28:27.374196+0000","last_peered":"2023-08-28T02:28:27.374196+0000","last_clean":"2023-08-28T02:28:27.374196+0000","last_became_active":"2023-08-28T02:28:25.343515+0000","last_became_peered":"2023-08-28T02:28:25.343515+0000","last_unstale":"2023-08-28T02:28:27.374196+0000","last_undegraded":"2023-08-28T02:28:27.374196+0000","last_fullsized":"2023-08-28T02:28:27.374196+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":101,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-08-28T02:28:28.371074+0000","last_change":"2023-08-28T02:27:31.330059+0000","last_active":"2023-08-28T02:28:28.371074+0000","last_peered":"2023-08-28T02:28:28.371074+0000","last_clean":"2023-08-28T02:28:28.371074+0000","last_became_active":"2023-08-28T02:27:31.329769+0000","last_became_peered":"2023-08-28T02:27:31.329769+0000","last_unstale":"2023-08-28T02:28:28.371074+0000","last_undegraded":"2023-08-28T02:28:28.371074+0000","last_fullsized":"2023-08-28T02:28:28.371074+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:14:54.349909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"225'5108","reported_seq":8282,"reported_epoch":225,"state":"active+recovery_wait+remapped","last_fresh":"2023-08-28T02:28:33.846367+0000","last_change":"2023-08-28T02:28:26.487426+0000","last_active":"2023-08-28T02:28:33.846367+0000","last_peered":"2023-08-28T02:28:33.846367+0000","last_clean":"2023-08-28T02:28:24.280158+0000","last_became_active":"2023-08-28T02:28:26.465041+0000","last_became_peered":"2023-08-28T02:28:26.465041+0000","last_unstale":"2023-08-28T02:28:33.846367+0000","last_undegraded":"2023-08-28T02:28:33.846367+0000","last_fullsized":"2023-08-28T02:28:33.846367+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":5108,"log_dups_size":0,"ondisk_log_size":5108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:30:34.822957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334757888,"num_objects":5108,"num_object_clones":0,"num_object_copies":15324,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5108,"num_objects_unfound":0,"num_objects_dirty":5108,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5108,"num_write_kb":326912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1547,"num_bytes_recovered":101384192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[0,5,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":5108}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":92,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-08-28T02:28:27.376172+0000","last_change":"2023-08-28T02:23:38.657185+0000","last_active":"2023-08-28T02:28:27.376172+0000","last_peered":"2023-08-28T02:28:27.376172+0000","last_clean":"2023-08-28T02:28:27.376172+0000","last_became_active":"2023-08-28T02:23:38.339016+0000","last_became_peered":"2023-08-28T02:23:38.339016+0000","last_unstale":"2023-08-28T02:28:27.376172+0000","last_undegraded":"2023-08-28T02:28:27.376172+0000","last_fullsized":"2023-08-28T02:28:27.376172+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":210,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-08-28T02:28:27.389472+0000","last_change":"2023-08-28T02:28:25.348106+0000","last_active":"2023-08-28T02:28:27.389472+0000","last_peered":"2023-08-28T02:28:27.389472+0000","last_clean":"2023-08-28T02:28:27.389472+0000","last_became_active":"2023-08-28T02:28:25.347786+0000","last_became_peered":"2023-08-28T02:28:25.347786+0000","last_unstale":"2023-08-28T02:28:27.389472+0000","last_undegraded":"2023-08-28T02:28:27.389472+0000","last_fullsized":"2023-08-28T02:28:27.389472+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"225'5119","reported_seq":9091,"reported_epoch":225,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:28:34.135169+0000","last_change":"2023-08-28T02:28:25.445263+0000","last_active":"2023-08-28T02:28:34.135169+0000","last_peered":"2023-08-28T02:28:34.135169+0000","last_clean":"2023-08-28T02:27:16.307202+0000","last_became_active":"2023-08-28T02:27:31.472816+0000","last_became_peered":"2023-08-28T02:27:31.472816+0000","last_unstale":"2023-08-28T02:28:34.135169+0000","last_undegraded":"2023-08-28T02:28:25.444621+0000","last_fullsized":"2023-08-28T02:27:31.303720+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":5119,"log_dups_size":0,"ondisk_log_size":5119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:50:24.871641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335478784,"num_objects":5119,"num_object_clones":0,"num_object_copies":15357,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":459,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5119,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5119,"num_write_kb":327616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1949,"num_bytes_recovered":127729664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7],"avail_no_missing":["3","6","7"],"object_location_counts":[{"shards":"3,4,7","objects":459},{"shards":"3,7","objects":4660}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":118,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-08-28T02:28:27.375940+0000","last_change":"2023-08-28T02:27:31.395328+0000","last_active":"2023-08-28T02:28:27.375940+0000","last_peered":"2023-08-28T02:28:27.375940+0000","last_clean":"2023-08-28T02:28:27.375940+0000","last_became_active":"2023-08-28T02:27:31.328228+0000","last_became_peered":"2023-08-28T02:27:31.328228+0000","last_unstale":"2023-08-28T02:28:27.375940+0000","last_undegraded":"2023-08-28T02:28:27.375940+0000","last_fullsized":"2023-08-28T02:28:27.375940+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":259,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-08-28T02:28:27.409756+0000","last_change":"2023-08-28T02:27:31.344915+0000","last_active":"2023-08-28T02:28:27.409756+0000","last_peered":"2023-08-28T02:28:27.409756+0000","last_clean":"2023-08-28T02:28:27.409756+0000","last_became_active":"2023-08-28T02:27:31.344577+0000","last_became_peered":"2023-08-28T02:27:31.344577+0000","last_unstale":"2023-08-28T02:28:27.409756+0000","last_undegraded":"2023-08-28T02:28:27.409756+0000","last_fullsized":"2023-08-28T02:28:27.409756+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"224'4398","reported_seq":4453,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-08-28T02:28:32.006601+0000","last_change":"2023-08-28T02:27:58.457913+0000","last_active":"2023-08-28T02:28:32.006601+0000","last_peered":"2023-08-28T02:28:32.006601+0000","last_clean":"2023-08-28T02:28:32.006601+0000","last_became_active":"2023-08-28T02:27:58.451333+0000","last_became_peered":"2023-08-28T02:27:58.451333+0000","last_unstale":"2023-08-28T02:28:32.006601+0000","last_undegraded":"2023-08-28T02:28:32.006601+0000","last_fullsized":"2023-08-28T02:28:32.006601+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":4398,"log_dups_size":0,"ondisk_log_size":4398,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165249024,"num_objects":2521,"num_object_clones":0,"num_object_copies":7563,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2521,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2521,"num_write_kb":161376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":118,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-08-28T02:28:27.377905+0000","last_change":"2023-08-28T02:28:25.749249+0000","last_active":"2023-08-28T02:28:27.377905+0000","last_peered":"2023-08-28T02:28:27.377905+0000","last_clean":"2023-08-28T02:28:27.377905+0000","last_became_active":"2023-08-28T02:28:25.397684+0000","last_became_peered":"2023-08-28T02:28:25.397684+0000","last_unstale":"2023-08-28T02:28:27.377905+0000","last_undegraded":"2023-08-28T02:28:27.377905+0000","last_fullsized":"2023-08-28T02:28:27.377905+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-08-28T02:28:27.374550+0000","last_change":"2023-08-28T02:28:25.350802+0000","last_active":"2023-08-28T02:28:27.374550+0000","last_peered":"2023-08-28T02:28:27.374550+0000","last_clean":"2023-08-28T02:28:27.374550+0000","last_became_active":"2023-08-28T02:28:25.350474+0000","last_became_peered":"2023-08-28T02:28:25.350474+0000","last_unstale":"2023-08-28T02:28:27.374550+0000","last_undegraded":"2023-08-28T02:28:27.374550+0000","last_fullsized":"2023-08-28T02:28:27.374550+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"225'4384","reported_seq":4441,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-08-28T02:28:33.795299+0000","last_change":"2023-08-28T02:27:58.435120+0000","last_active":"2023-08-28T02:28:33.795299+0000","last_peered":"2023-08-28T02:28:33.795299+0000","last_clean":"2023-08-28T02:28:33.795299+0000","last_became_active":"2023-08-28T02:27:58.433161+0000","last_became_peered":"2023-08-28T02:27:58.433161+0000","last_unstale":"2023-08-28T02:28:33.795299+0000","last_undegraded":"2023-08-28T02:28:33.795299+0000","last_fullsized":"2023-08-28T02:28:33.795299+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":4384,"log_dups_size":0,"ondisk_log_size":4384,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164790272,"num_objects":2514,"num_object_clones":0,"num_object_copies":7542,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2514,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2514,"num_write_kb":160928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":226,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-08-28T02:28:28.371743+0000","last_change":"2023-08-28T02:28:25.445982+0000","last_active":"2023-08-28T02:28:28.371743+0000","last_peered":"2023-08-28T02:28:28.371743+0000","last_clean":"2023-08-28T02:28:28.371743+0000","last_became_active":"2023-08-28T02:28:25.443577+0000","last_became_peered":"2023-08-28T02:28:25.443577+0000","last_unstale":"2023-08-28T02:28:28.371743+0000","last_undegraded":"2023-08-28T02:28:28.371743+0000","last_fullsized":"2023-08-28T02:28:28.371743+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:24:03.381751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":204,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-08-28T02:28:28.760344+0000","last_change":"2023-08-28T02:27:31.319783+0000","last_active":"2023-08-28T02:28:28.760344+0000","last_peered":"2023-08-28T02:28:28.760344+0000","last_clean":"2023-08-28T02:28:28.760344+0000","last_became_active":"2023-08-28T02:27:31.319451+0000","last_became_peered":"2023-08-28T02:27:31.319451+0000","last_unstale":"2023-08-28T02:28:28.760344+0000","last_undegraded":"2023-08-28T02:28:28.760344+0000","last_fullsized":"2023-08-28T02:28:28.760344+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:20:37.097889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"224'4971","reported_seq":5033,"reported_epoch":224,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:28:32.179958+0000","last_change":"2023-08-28T02:27:32.432780+0000","last_active":"2023-08-28T02:28:32.179958+0000","last_peered":"2023-08-28T02:28:32.179958+0000","last_clean":"2023-08-28T02:27:30.498067+0000","last_became_active":"2023-08-28T02:27:32.427260+0000","last_became_peered":"2023-08-28T02:27:32.427260+0000","last_unstale":"2023-08-28T02:28:32.179958+0000","last_undegraded":"2023-08-28T02:27:32.333766+0000","last_fullsized":"2023-08-28T02:27:32.320843+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":4971,"log_dups_size":0,"ondisk_log_size":4971,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":325779456,"num_objects":4971,"num_object_clones":0,"num_object_copies":14913,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2409,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4971,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4971,"num_write_kb":318144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,2],"avail_no_missing":["4","2","5"],"object_location_counts":[{"shards":"2,4","objects":2562},{"shards":"2,4,5","objects":2409}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":88,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-08-28T02:28:29.149560+0000","last_change":"2023-08-28T02:23:59.230059+0000","last_active":"2023-08-28T02:28:29.149560+0000","last_peered":"2023-08-28T02:28:29.149560+0000","last_clean":"2023-08-28T02:28:29.149560+0000","last_became_active":"2023-08-28T02:23:59.217783+0000","last_became_peered":"2023-08-28T02:23:59.217783+0000","last_unstale":"2023-08-28T02:28:29.149560+0000","last_undegraded":"2023-08-28T02:28:29.149560+0000","last_fullsized":"2023-08-28T02:28:29.149560+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:00:12.145395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":244,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-08-28T02:28:27.410171+0000","last_change":"2023-08-28T02:28:25.351698+0000","last_active":"2023-08-28T02:28:27.410171+0000","last_peered":"2023-08-28T02:28:27.410171+0000","last_clean":"2023-08-28T02:28:27.410171+0000","last_became_active":"2023-08-28T02:28:25.351349+0000","last_became_peered":"2023-08-28T02:28:25.351349+0000","last_unstale":"2023-08-28T02:28:27.410171+0000","last_undegraded":"2023-08-28T02:28:27.410171+0000","last_fullsized":"2023-08-28T02:28:27.410171+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"225'5112","reported_seq":5159,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-08-28T02:28:34.041949+0000","last_change":"2023-08-28T02:26:57.390417+0000","last_active":"2023-08-28T02:28:34.041949+0000","last_peered":"2023-08-28T02:28:34.041949+0000","last_clean":"2023-08-28T02:28:34.041949+0000","last_became_active":"2023-08-28T02:26:57.390124+0000","last_became_peered":"2023-08-28T02:26:57.390124+0000","last_unstale":"2023-08-28T02:28:34.041949+0000","last_undegraded":"2023-08-28T02:28:34.041949+0000","last_fullsized":"2023-08-28T02:28:34.041949+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":5112,"log_dups_size":0,"ondisk_log_size":5112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:38:48.957967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335020032,"num_objects":5112,"num_object_clones":0,"num_object_copies":15336,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5112,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5112,"num_write_kb":327168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":13,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-08-28T02:28:27.374649+0000","last_change":"2023-08-28T02:28:25.345274+0000","last_active":"2023-08-28T02:28:27.374649+0000","last_peered":"2023-08-28T02:28:27.374649+0000","last_clean":"2023-08-28T02:28:27.374649+0000","last_became_active":"2023-08-28T02:28:25.344885+0000","last_became_peered":"2023-08-28T02:28:25.344885+0000","last_unstale":"2023-08-28T02:28:27.374649+0000","last_undegraded":"2023-08-28T02:28:27.374649+0000","last_fullsized":"2023-08-28T02:28:27.374649+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":270,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-08-28T02:28:27.410267+0000","last_change":"2023-08-28T02:27:31.358436+0000","last_active":"2023-08-28T02:28:27.410267+0000","last_peered":"2023-08-28T02:28:27.410267+0000","last_clean":"2023-08-28T02:28:27.410267+0000","last_became_active":"2023-08-28T02:27:31.358185+0000","last_became_peered":"2023-08-28T02:27:31.358185+0000","last_unstale":"2023-08-28T02:28:27.410267+0000","last_undegraded":"2023-08-28T02:28:27.410267+0000","last_fullsized":"2023-08-28T02:28:27.410267+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"225'5189","reported_seq":8412,"reported_epoch":225,"state":"active+remapped+backfill_wait","last_fresh":"2023-08-28T02:28:34.118546+0000","last_change":"2023-08-28T02:28:26.455411+0000","last_active":"2023-08-28T02:28:34.118546+0000","last_peered":"2023-08-28T02:28:34.118546+0000","last_clean":"2023-08-28T02:28:24.332252+0000","last_became_active":"2023-08-28T02:28:26.447621+0000","last_became_peered":"2023-08-28T02:28:26.447621+0000","last_unstale":"2023-08-28T02:28:34.118546+0000","last_undegraded":"2023-08-28T02:28:34.118546+0000","last_fullsized":"2023-08-28T02:28:34.118546+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":5189,"log_dups_size":0,"ondisk_log_size":5189,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:28:52.578588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340066304,"num_objects":5189,"num_object_clones":0,"num_object_copies":15567,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5189,"num_objects_unfound":0,"num_objects_dirty":5189,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5189,"num_write_kb":332096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1574,"num_bytes_recovered":103153664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,5,7],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":5189}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":133,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-08-28T02:28:27.409839+0000","last_change":"2023-08-28T02:23:38.347043+0000","last_active":"2023-08-28T02:28:27.409839+0000","last_peered":"2023-08-28T02:28:27.409839+0000","last_clean":"2023-08-28T02:28:27.409839+0000","last_became_active":"2023-08-28T02:23:38.346680+0000","last_became_peered":"2023-08-28T02:23:38.346680+0000","last_unstale":"2023-08-28T02:28:27.409839+0000","last_undegraded":"2023-08-28T02:28:27.409839+0000","last_fullsized":"2023-08-28T02:28:27.409839+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":190,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-08-28T02:28:28.761301+0000","last_change":"2023-08-28T02:28:25.347494+0000","last_active":"2023-08-28T02:28:28.761301+0000","last_peered":"2023-08-28T02:28:28.761301+0000","last_clean":"2023-08-28T02:28:28.761301+0000","last_became_active":"2023-08-28T02:28:25.347142+0000","last_became_peered":"2023-08-28T02:28:25.347142+0000","last_unstale":"2023-08-28T02:28:28.761301+0000","last_undegraded":"2023-08-28T02:28:28.761301+0000","last_fullsized":"2023-08-28T02:28:28.761301+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:37:14.384884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.d","version":"224'4866","reported_seq":4931,"reported_epoch":224,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:28:32.178310+0000","last_change":"2023-08-28T02:27:32.436317+0000","last_active":"2023-08-28T02:28:32.178310+0000","last_peered":"2023-08-28T02:28:32.178310+0000","last_clean":"2023-08-28T02:27:30.522858+0000","last_became_active":"2023-08-28T02:27:32.431857+0000","last_became_peered":"2023-08-28T02:27:32.431857+0000","last_unstale":"2023-08-28T02:28:32.178310+0000","last_undegraded":"2023-08-28T02:27:32.377766+0000","last_fullsized":"2023-08-28T02:27:32.321737+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":4866,"log_dups_size":0,"ondisk_log_size":4866,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318898176,"num_objects":4866,"num_object_clones":0,"num_object_copies":14598,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2311,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4866,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4866,"num_write_kb":311424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,0],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"0,2,5","objects":2311},{"shards":"0,5","objects":2555}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":102,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-08-28T02:28:28.368862+0000","last_change":"2023-08-28T02:27:31.325260+0000","last_active":"2023-08-28T02:28:28.368862+0000","last_peered":"2023-08-28T02:28:28.368862+0000","last_clean":"2023-08-28T02:28:28.368862+0000","last_became_active":"2023-08-28T02:27:31.324950+0000","last_became_peered":"2023-08-28T02:27:31.324950+0000","last_unstale":"2023-08-28T02:28:28.368862+0000","last_undegraded":"2023-08-28T02:28:28.368862+0000","last_fullsized":"2023-08-28T02:28:28.368862+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:21:33.833261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":32,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-08-28T02:28:28.370333+0000","last_change":"2023-08-28T02:27:31.322118+0000","last_active":"2023-08-28T02:28:28.370333+0000","last_peered":"2023-08-28T02:28:28.370333+0000","last_clean":"2023-08-28T02:28:28.370333+0000","last_became_active":"2023-08-28T02:27:31.321338+0000","last_became_peered":"2023-08-28T02:27:31.321338+0000","last_unstale":"2023-08-28T02:28:28.370333+0000","last_undegraded":"2023-08-28T02:28:28.370333+0000","last_fullsized":"2023-08-28T02:28:28.370333+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:56:14.210571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"225'4326","reported_seq":4382,"reported_epoch":225,"state":"active+recovery_wait","last_fresh":"2023-08-28T02:28:33.692367+0000","last_change":"2023-08-28T02:27:58.580806+0000","last_active":"2023-08-28T02:28:33.692367+0000","last_peered":"2023-08-28T02:28:33.692367+0000","last_clean":"2023-08-28T02:27:57.350390+0000","last_became_active":"2023-08-28T02:27:58.437499+0000","last_became_peered":"2023-08-28T02:27:58.437499+0000","last_unstale":"2023-08-28T02:28:33.692367+0000","last_undegraded":"2023-08-28T02:28:33.692367+0000","last_fullsized":"2023-08-28T02:28:33.692367+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":4326,"log_dups_size":0,"ondisk_log_size":4326,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163119104,"num_objects":2489,"num_object_clones":0,"num_object_copies":7467,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2489,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2489,"num_write_kb":159296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":2489}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"78'126","reported_seq":594,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-08-28T02:28:28.967235+0000","last_change":"2023-08-28T02:28:27.383137+0000","last_active":"2023-08-28T02:28:28.967235+0000","last_peered":"2023-08-28T02:28:28.967235+0000","last_clean":"2023-08-28T02:28:28.967235+0000","last_became_active":"2023-08-28T02:28:27.382488+0000","last_became_peered":"2023-08-28T02:28:27.382488+0000","last_unstale":"2023-08-28T02:28:28.967235+0000","last_undegraded":"2023-08-28T02:28:28.967235+0000","last_fullsized":"2023-08-28T02:28:28.967235+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:14:48.414426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":743,"num_read_kb":1331,"num_write":125,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":223,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-08-28T02:28:27.390181+0000","last_change":"2023-08-28T02:23:59.207148+0000","last_active":"2023-08-28T02:28:27.390181+0000","last_peered":"2023-08-28T02:28:27.390181+0000","last_clean":"2023-08-28T02:28:27.390181+0000","last_became_active":"2023-08-28T02:23:59.206802+0000","last_became_peered":"2023-08-28T02:23:59.206802+0000","last_unstale":"2023-08-28T02:28:27.390181+0000","last_undegraded":"2023-08-28T02:28:27.390181+0000","last_fullsized":"2023-08-28T02:28:27.390181+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":32,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-08-28T02:28:28.369634+0000","last_change":"2023-08-28T02:27:31.328027+0000","last_active":"2023-08-28T02:28:28.369634+0000","last_peered":"2023-08-28T02:28:28.369634+0000","last_clean":"2023-08-28T02:28:28.369634+0000","last_became_active":"2023-08-28T02:27:31.326529+0000","last_became_peered":"2023-08-28T02:27:31.326529+0000","last_unstale":"2023-08-28T02:28:28.369634+0000","last_undegraded":"2023-08-28T02:28:28.369634+0000","last_fullsized":"2023-08-28T02:28:28.369634+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:56:57.331355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"225'4478","reported_seq":4535,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-08-28T02:28:34.103029+0000","last_change":"2023-08-28T02:27:58.462974+0000","last_active":"2023-08-28T02:28:34.103029+0000","last_peered":"2023-08-28T02:28:34.103029+0000","last_clean":"2023-08-28T02:28:34.103029+0000","last_became_active":"2023-08-28T02:27:58.460293+0000","last_became_peered":"2023-08-28T02:27:58.460293+0000","last_unstale":"2023-08-28T02:28:34.103029+0000","last_undegraded":"2023-08-28T02:28:34.103029+0000","last_fullsized":"2023-08-28T02:28:34.103029+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":4478,"log_dups_size":0,"ondisk_log_size":4478,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169902080,"num_objects":2592,"num_object_clones":0,"num_object_copies":7776,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2592,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2592,"num_write_kb":165920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"78'204","reported_seq":1250,"reported_epoch":224,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:28:28.369291+0000","last_change":"2023-08-28T02:28:26.356651+0000","last_active":"2023-08-28T02:28:28.369291+0000","last_peered":"2023-08-28T02:28:28.369291+0000","last_clean":"2023-08-28T02:26:59.547026+0000","last_became_active":"2023-08-28T02:28:26.356177+0000","last_became_peered":"2023-08-28T02:28:26.356177+0000","last_unstale":"2023-08-28T02:28:28.369291+0000","last_undegraded":"2023-08-28T02:28:26.341394+0000","last_fullsized":"2023-08-28T02:28:26.327240+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":204,"log_dups_size":0,"ondisk_log_size":204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:27:09.894553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":16,"num_objects_missing":16,"num_objects_degraded":32,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1521,"num_read_kb":2511,"num_write":178,"num_write_kb":1175,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,3,5","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":218,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-08-28T02:28:27.409988+0000","last_change":"2023-08-28T02:28:25.352417+0000","last_active":"2023-08-28T02:28:27.409988+0000","last_peered":"2023-08-28T02:28:27.409988+0000","last_clean":"2023-08-28T02:28:27.409988+0000","last_became_active":"2023-08-28T02:28:25.352216+0000","last_became_peered":"2023-08-28T02:28:25.352216+0000","last_unstale":"2023-08-28T02:28:27.409988+0000","last_undegraded":"2023-08-28T02:28:27.409988+0000","last_fullsized":"2023-08-28T02:28:27.409988+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":32,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-08-28T02:28:28.373727+0000","last_change":"2023-08-28T02:27:31.376755+0000","last_active":"2023-08-28T02:28:28.373727+0000","last_peered":"2023-08-28T02:28:28.373727+0000","last_clean":"2023-08-28T02:28:28.373727+0000","last_became_active":"2023-08-28T02:27:31.373279+0000","last_became_peered":"2023-08-28T02:27:31.373279+0000","last_unstale":"2023-08-28T02:28:28.373727+0000","last_undegraded":"2023-08-28T02:28:28.373727+0000","last_fullsized":"2023-08-28T02:28:28.373727+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:26:14.628941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"224'4396","reported_seq":5642,"reported_epoch":224,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:28:31.957092+0000","last_change":"2023-08-28T02:28:25.405536+0000","last_active":"2023-08-28T02:28:31.957092+0000","last_peered":"2023-08-28T02:28:31.957092+0000","last_clean":"2023-08-28T02:27:16.315756+0000","last_became_active":"2023-08-28T02:28:25.400914+0000","last_became_peered":"2023-08-28T02:28:25.400914+0000","last_unstale":"2023-08-28T02:28:31.957092+0000","last_undegraded":"2023-08-28T02:28:25.339881+0000","last_fullsized":"2023-08-28T02:28:25.329514+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":4396,"log_dups_size":0,"ondisk_log_size":4396,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164626432,"num_objects":2512,"num_object_clones":0,"num_object_copies":7536,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1670,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2512,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2512,"num_write_kb":160768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":294,"num_bytes_recovered":19267584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"2,3,7","objects":1670},{"shards":"3,7","objects":842}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'47","reported_seq":440,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-08-28T02:28:28.379581+0000","last_change":"2023-08-28T02:24:05.799647+0000","last_active":"2023-08-28T02:28:28.379581+0000","last_peered":"2023-08-28T02:28:28.379581+0000","last_clean":"2023-08-28T02:28:28.379581+0000","last_became_active":"2023-08-28T02:23:59.210481+0000","last_became_peered":"2023-08-28T02:23:59.210481+0000","last_unstale":"2023-08-28T02:28:28.379581+0000","last_undegraded":"2023-08-28T02:28:28.379581+0000","last_fullsized":"2023-08-28T02:28:28.379581+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":47,"log_dups_size":0,"ondisk_log_size":47,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:40:18.542375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":162,"num_read_kb":1281,"num_write":48,"num_write_kb":1153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":41,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-08-28T02:28:28.370225+0000","last_change":"2023-08-28T02:28:25.343406+0000","last_active":"2023-08-28T02:28:28.370225+0000","last_peered":"2023-08-28T02:28:28.370225+0000","last_clean":"2023-08-28T02:28:28.370225+0000","last_became_active":"2023-08-28T02:28:25.343065+0000","last_became_peered":"2023-08-28T02:28:25.343065+0000","last_unstale":"2023-08-28T02:28:28.370225+0000","last_undegraded":"2023-08-28T02:28:28.370225+0000","last_fullsized":"2023-08-28T02:28:28.370225+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:20:31.893609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":32,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-08-28T02:28:28.370406+0000","last_change":"2023-08-28T02:27:31.322967+0000","last_active":"2023-08-28T02:28:28.370406+0000","last_peered":"2023-08-28T02:28:28.370406+0000","last_clean":"2023-08-28T02:28:28.370406+0000","last_became_active":"2023-08-28T02:27:31.322655+0000","last_became_peered":"2023-08-28T02:27:31.322655+0000","last_unstale":"2023-08-28T02:28:28.370406+0000","last_undegraded":"2023-08-28T02:28:28.370406+0000","last_fullsized":"2023-08-28T02:28:28.370406+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:05:15.868115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"224'4395","reported_seq":4450,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-08-28T02:28:32.073870+0000","last_change":"2023-08-28T02:27:58.455961+0000","last_active":"2023-08-28T02:28:32.073870+0000","last_peered":"2023-08-28T02:28:32.073870+0000","last_clean":"2023-08-28T02:28:32.073870+0000","last_became_active":"2023-08-28T02:27:58.452334+0000","last_became_peered":"2023-08-28T02:27:58.452334+0000","last_unstale":"2023-08-28T02:28:32.073870+0000","last_undegraded":"2023-08-28T02:28:32.073870+0000","last_fullsized":"2023-08-28T02:28:32.073870+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":4395,"log_dups_size":0,"ondisk_log_size":4395,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166395904,"num_objects":2539,"num_object_clones":0,"num_object_copies":7617,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2539,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2539,"num_write_kb":162496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"78'114","reported_seq":587,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-08-28T02:28:27.389737+0000","last_change":"2023-08-28T02:24:01.210739+0000","last_active":"2023-08-28T02:28:27.389737+0000","last_peered":"2023-08-28T02:28:27.389737+0000","last_clean":"2023-08-28T02:28:27.389737+0000","last_became_active":"2023-08-28T02:24:01.210386+0000","last_became_peered":"2023-08-28T02:24:01.210386+0000","last_unstale":"2023-08-28T02:28:27.389737+0000","last_undegraded":"2023-08-28T02:28:27.389737+0000","last_fullsized":"2023-08-28T02:28:27.389737+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":114,"log_dups_size":0,"ondisk_log_size":114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":358,"num_read_kb":1352,"num_write":102,"num_write_kb":1104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"86'139","reported_seq":419,"reported_epoch":223,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:28:27.375421+0000","last_change":"2023-08-28T02:28:25.354428+0000","last_active":"2023-08-28T02:28:27.375421+0000","last_peered":"2023-08-28T02:28:27.375421+0000","last_clean":"2023-08-28T02:27:29.492330+0000","last_became_active":"2023-08-28T02:28:25.350352+0000","last_became_peered":"2023-08-28T02:28:25.350352+0000","last_unstale":"2023-08-28T02:28:27.375421+0000","last_undegraded":"2023-08-28T02:28:25.326135+0000","last_fullsized":"2023-08-28T02:28:25.325547+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_clean_scrub_stamp":"2023-08-28T02:08:02.043718+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0],"avail_no_missing":["7","0","2"],"object_location_counts":[{"shards":"0,2,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":102,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-08-28T02:28:28.370598+0000","last_change":"2023-08-28T02:28:25.347600+0000","last_active":"2023-08-28T02:28:28.370598+0000","last_peered":"2023-08-28T02:28:28.370598+0000","last_clean":"2023-08-28T02:28:28.370598+0000","last_became_active":"2023-08-28T02:28:25.347237+0000","last_became_peered":"2023-08-28T02:28:25.347237+0000","last_unstale":"2023-08-28T02:28:28.370598+0000","last_undegraded":"2023-08-28T02:28:28.370598+0000","last_fullsized":"2023-08-28T02:28:28.370598+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:29:26.767559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-08-28T02:28:27.374717+0000","last_change":"2023-08-28T02:28:25.398594+0000","last_active":"2023-08-28T02:28:27.374717+0000","last_peered":"2023-08-28T02:28:27.374717+0000","last_clean":"2023-08-28T02:28:27.374717+0000","last_became_active":"2023-08-28T02:28:25.397058+0000","last_became_peered":"2023-08-28T02:28:25.397058+0000","last_unstale":"2023-08-28T02:28:27.374717+0000","last_undegraded":"2023-08-28T02:28:27.374717+0000","last_fullsized":"2023-08-28T02:28:27.374717+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"225'4374","reported_seq":8335,"reported_epoch":225,"state":"active+recovering","last_fresh":"2023-08-28T02:28:33.884888+0000","last_change":"2023-08-28T02:28:25.759316+0000","last_active":"2023-08-28T02:28:33.884888+0000","last_peered":"2023-08-28T02:28:33.884888+0000","last_clean":"2023-08-28T02:27:30.461946+0000","last_became_active":"2023-08-28T02:28:00.019510+0000","last_became_peered":"2023-08-28T02:28:00.019510+0000","last_unstale":"2023-08-28T02:28:33.884888+0000","last_undegraded":"2023-08-28T02:28:33.884888+0000","last_fullsized":"2023-08-28T02:28:33.884888+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":4374,"log_dups_size":0,"ondisk_log_size":4374,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166559744,"num_objects":2541,"num_object_clones":0,"num_object_copies":7623,"num_objects_missing_on_primary":707,"num_objects_missing":707,"num_objects_degraded":0,"num_objects_misplaced":707,"num_objects_unfound":0,"num_objects_dirty":2541,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2541,"num_write_kb":162656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1202,"num_bytes_recovered":78807040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":["3","4","5","6"],"object_location_counts":[{"shards":"1,3,4,5","objects":1},{"shards":"1,3,5","objects":1833},{"shards":"3,4,5","objects":707}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'148","reported_seq":721,"reported_epoch":223,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:28:27.376113+0000","last_change":"2023-08-28T02:27:32.344664+0000","last_active":"2023-08-28T02:28:27.376113+0000","last_peered":"2023-08-28T02:28:27.376113+0000","last_clean":"2023-08-28T02:27:29.492264+0000","last_became_active":"2023-08-28T02:27:32.344175+0000","last_became_peered":"2023-08-28T02:27:32.344175+0000","last_unstale":"2023-08-28T02:28:27.376113+0000","last_undegraded":"2023-08-28T02:27:32.331999+0000","last_fullsized":"2023-08-28T02:27:32.331438+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1092,"num_read_kb":1955,"num_write":127,"num_write_kb":5177,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,0],"avail_no_missing":["7","0","3"],"object_location_counts":[{"shards":"0,3,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":192,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-08-28T02:28:28.760697+0000","last_change":"2023-08-28T02:23:59.226986+0000","last_active":"2023-08-28T02:28:28.760697+0000","last_peered":"2023-08-28T02:28:28.760697+0000","last_clean":"2023-08-28T02:28:28.760697+0000","last_became_active":"2023-08-28T02:23:59.226529+0000","last_became_peered":"2023-08-28T02:23:59.226529+0000","last_unstale":"2023-08-28T02:28:28.760697+0000","last_undegraded":"2023-08-28T02:28:28.760697+0000","last_fullsized":"2023-08-28T02:28:28.760697+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:22:09.193619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":237,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-08-28T02:28:28.760830+0000","last_change":"2023-08-28T02:27:31.361298+0000","last_active":"2023-08-28T02:28:28.760830+0000","last_peered":"2023-08-28T02:28:28.760830+0000","last_clean":"2023-08-28T02:28:28.760830+0000","last_became_active":"2023-08-28T02:27:31.360939+0000","last_became_peered":"2023-08-28T02:27:31.360939+0000","last_unstale":"2023-08-28T02:28:28.760830+0000","last_undegraded":"2023-08-28T02:28:28.760830+0000","last_fullsized":"2023-08-28T02:28:28.760830+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:48:42.024920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"225'4477","reported_seq":7520,"reported_epoch":225,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:28:33.889818+0000","last_change":"2023-08-28T02:28:28.918500+0000","last_active":"2023-08-28T02:28:33.889818+0000","last_peered":"2023-08-28T02:28:33.889818+0000","last_clean":"2023-08-28T02:27:16.286032+0000","last_became_active":"2023-08-28T02:28:28.897473+0000","last_became_peered":"2023-08-28T02:28:28.897473+0000","last_unstale":"2023-08-28T02:28:33.889818+0000","last_undegraded":"2023-08-28T02:28:27.476267+0000","last_fullsized":"2023-08-28T02:28:26.328174+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":192,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":4477,"log_dups_size":0,"ondisk_log_size":4477,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170524672,"num_objects":2602,"num_object_clones":0,"num_object_copies":7806,"num_objects_missing_on_primary":501,"num_objects_missing":501,"num_objects_degraded":2200,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2602,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2602,"num_write_kb":166528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":743,"num_bytes_recovered":48726016,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,2],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"0,2,7","objects":448},{"shards":"1,2","objects":851},{"shards":"1,2,7","objects":1250},{"shards":"2,6,7","objects":52},{"shards":"2,7","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'103","reported_seq":599,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-08-28T02:28:27.377886+0000","last_change":"2023-08-28T02:24:07.212128+0000","last_active":"2023-08-28T02:28:27.377886+0000","last_peered":"2023-08-28T02:28:27.377886+0000","last_clean":"2023-08-28T02:28:27.377886+0000","last_became_active":"2023-08-28T02:24:07.211562+0000","last_became_peered":"2023-08-28T02:24:07.211562+0000","last_unstale":"2023-08-28T02:28:27.377886+0000","last_undegraded":"2023-08-28T02:28:27.377886+0000","last_fullsized":"2023-08-28T02:28:27.377886+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":803,"num_read_kb":2080,"num_write":73,"num_write_kb":1244,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":32,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-08-28T02:28:28.375368+0000","last_change":"2023-08-28T02:27:31.377042+0000","last_active":"2023-08-28T02:28:28.375368+0000","last_peered":"2023-08-28T02:28:28.375368+0000","last_clean":"2023-08-28T02:28:28.375368+0000","last_became_active":"2023-08-28T02:27:31.373709+0000","last_became_peered":"2023-08-28T02:27:31.373709+0000","last_unstale":"2023-08-28T02:28:28.375368+0000","last_undegraded":"2023-08-28T02:28:28.375368+0000","last_fullsized":"2023-08-28T02:28:28.375368+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:12:38.286332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":178,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-08-28T02:28:28.760578+0000","last_change":"2023-08-28T02:23:38.329667+0000","last_active":"2023-08-28T02:28:28.760578+0000","last_peered":"2023-08-28T02:28:28.760578+0000","last_clean":"2023-08-28T02:28:28.760578+0000","last_became_active":"2023-08-28T02:23:38.329175+0000","last_became_peered":"2023-08-28T02:23:38.329175+0000","last_unstale":"2023-08-28T02:28:28.760578+0000","last_undegraded":"2023-08-28T02:28:28.760578+0000","last_fullsized":"2023-08-28T02:28:28.760578+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:21:34.815067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"224'4391","reported_seq":4455,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-08-28T02:28:32.044427+0000","last_change":"2023-08-28T02:28:31.770660+0000","last_active":"2023-08-28T02:28:32.044427+0000","last_peered":"2023-08-28T02:28:32.044427+0000","last_clean":"2023-08-28T02:28:32.044427+0000","last_became_active":"2023-08-28T02:27:58.451860+0000","last_became_peered":"2023-08-28T02:27:58.451860+0000","last_unstale":"2023-08-28T02:28:32.044427+0000","last_undegraded":"2023-08-28T02:28:32.044427+0000","last_fullsized":"2023-08-28T02:28:32.044427+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":4391,"log_dups_size":0,"ondisk_log_size":4391,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165543936,"num_objects":2526,"num_object_clones":0,"num_object_copies":7578,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2526,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2526,"num_write_kb":161664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":88,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-08-28T02:28:28.371158+0000","last_change":"2023-08-28T02:23:59.232099+0000","last_active":"2023-08-28T02:28:28.371158+0000","last_peered":"2023-08-28T02:28:28.371158+0000","last_clean":"2023-08-28T02:28:28.371158+0000","last_became_active":"2023-08-28T02:23:59.219318+0000","last_became_peered":"2023-08-28T02:23:59.219318+0000","last_unstale":"2023-08-28T02:28:28.371158+0000","last_undegraded":"2023-08-28T02:28:28.371158+0000","last_fullsized":"2023-08-28T02:28:28.371158+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:59:30.093289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'112","reported_seq":340,"reported_epoch":224,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:28:28.967258+0000","last_change":"2023-08-28T02:28:26.354231+0000","last_active":"2023-08-28T02:28:28.967258+0000","last_peered":"2023-08-28T02:28:28.967258+0000","last_clean":"2023-08-28T02:12:45.636147+0000","last_became_active":"2023-08-28T02:28:26.353929+0000","last_became_peered":"2023-08-28T02:28:26.353929+0000","last_unstale":"2023-08-28T02:28:28.967258+0000","last_undegraded":"2023-08-28T02:28:26.332044+0000","last_fullsized":"2023-08-28T02:28:26.331305+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:21:35.988693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":9,"num_objects_missing":9,"num_objects_degraded":18,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":444,"num_read_kb":473,"num_write":94,"num_write_kb":127,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,4],"avail_no_missing":["0","3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":84,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-08-28T02:28:28.368715+0000","last_change":"2023-08-28T02:23:59.221034+0000","last_active":"2023-08-28T02:28:28.368715+0000","last_peered":"2023-08-28T02:28:28.368715+0000","last_clean":"2023-08-28T02:28:28.368715+0000","last_became_active":"2023-08-28T02:23:59.220730+0000","last_became_peered":"2023-08-28T02:23:59.220730+0000","last_unstale":"2023-08-28T02:28:28.368715+0000","last_undegraded":"2023-08-28T02:28:28.368715+0000","last_fullsized":"2023-08-28T02:28:28.368715+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:25:52.364370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"224'4398","reported_seq":4473,"reported_epoch":224,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:28:32.052861+0000","last_change":"2023-08-28T02:28:26.460507+0000","last_active":"2023-08-28T02:28:32.052861+0000","last_peered":"2023-08-28T02:28:32.052861+0000","last_clean":"2023-08-28T02:27:16.315150+0000","last_became_active":"2023-08-28T02:28:26.432234+0000","last_became_peered":"2023-08-28T02:28:26.432234+0000","last_unstale":"2023-08-28T02:28:32.052861+0000","last_undegraded":"2023-08-28T02:28:26.346974+0000","last_fullsized":"2023-08-28T02:28:26.332755+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":4398,"log_dups_size":0,"ondisk_log_size":4398,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164495360,"num_objects":2510,"num_object_clones":0,"num_object_copies":7530,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1660,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2510,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2510,"num_write_kb":160640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"0,2,7","objects":436},{"shards":"1,2,7","objects":1224},{"shards":"2,7","objects":850}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"78'122","reported_seq":531,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-08-28T02:28:28.383306+0000","last_change":"2023-08-28T02:28:28.383306+0000","last_active":"2023-08-28T02:28:28.383306+0000","last_peered":"2023-08-28T02:28:28.383306+0000","last_clean":"2023-08-28T02:28:28.383306+0000","last_became_active":"2023-08-28T02:28:28.382672+0000","last_became_peered":"2023-08-28T02:28:28.382672+0000","last_unstale":"2023-08-28T02:28:28.383306+0000","last_undegraded":"2023-08-28T02:28:28.383306+0000","last_fullsized":"2023-08-28T02:28:28.383306+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":857,"num_read_kb":2224,"num_write":99,"num_write_kb":2312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":204,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-08-28T02:28:28.761043+0000","last_change":"2023-08-28T02:27:31.424373+0000","last_active":"2023-08-28T02:28:28.761043+0000","last_peered":"2023-08-28T02:28:28.761043+0000","last_clean":"2023-08-28T02:28:28.761043+0000","last_became_active":"2023-08-28T02:27:31.324505+0000","last_became_peered":"2023-08-28T02:27:31.324505+0000","last_unstale":"2023-08-28T02:28:28.761043+0000","last_undegraded":"2023-08-28T02:28:28.761043+0000","last_fullsized":"2023-08-28T02:28:28.761043+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:27:57.837524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":193,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-08-28T02:28:27.409656+0000","last_change":"2023-08-28T02:28:25.351840+0000","last_active":"2023-08-28T02:28:27.409656+0000","last_peered":"2023-08-28T02:28:27.409656+0000","last_clean":"2023-08-28T02:28:27.409656+0000","last_became_active":"2023-08-28T02:28:25.351399+0000","last_became_peered":"2023-08-28T02:28:25.351399+0000","last_unstale":"2023-08-28T02:28:27.409656+0000","last_undegraded":"2023-08-28T02:28:27.409656+0000","last_fullsized":"2023-08-28T02:28:27.409656+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":180,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-08-28T02:28:28.760497+0000","last_change":"2023-08-28T02:23:38.329474+0000","last_active":"2023-08-28T02:28:28.760497+0000","last_peered":"2023-08-28T02:28:28.760497+0000","last_clean":"2023-08-28T02:28:28.760497+0000","last_became_active":"2023-08-28T02:23:38.328505+0000","last_became_peered":"2023-08-28T02:23:38.328505+0000","last_unstale":"2023-08-28T02:28:28.760497+0000","last_undegraded":"2023-08-28T02:28:28.760497+0000","last_fullsized":"2023-08-28T02:28:28.760497+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:12:14.835809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":265,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-08-28T02:28:28.380209+0000","last_change":"2023-08-28T02:27:31.384467+0000","last_active":"2023-08-28T02:28:28.380209+0000","last_peered":"2023-08-28T02:28:28.380209+0000","last_clean":"2023-08-28T02:28:28.380209+0000","last_became_active":"2023-08-28T02:27:31.384183+0000","last_became_peered":"2023-08-28T02:27:31.384183+0000","last_unstale":"2023-08-28T02:28:28.380209+0000","last_undegraded":"2023-08-28T02:28:28.380209+0000","last_fullsized":"2023-08-28T02:28:28.380209+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:59:47.551892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.19","version":"225'4424","reported_seq":4481,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-08-28T02:28:33.724107+0000","last_change":"2023-08-28T02:27:58.492065+0000","last_active":"2023-08-28T02:28:33.724107+0000","last_peered":"2023-08-28T02:28:33.724107+0000","last_clean":"2023-08-28T02:28:33.724107+0000","last_became_active":"2023-08-28T02:27:58.490323+0000","last_became_peered":"2023-08-28T02:27:58.490323+0000","last_unstale":"2023-08-28T02:28:33.724107+0000","last_undegraded":"2023-08-28T02:28:33.724107+0000","last_fullsized":"2023-08-28T02:28:33.724107+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":4424,"log_dups_size":0,"ondisk_log_size":4424,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167411712,"num_objects":2555,"num_object_clones":0,"num_object_copies":7665,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2555,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2555,"num_write_kb":163488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":30,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-08-28T02:28:27.410165+0000","last_change":"2023-08-28T02:28:11.333622+0000","last_active":"2023-08-28T02:28:27.410165+0000","last_peered":"2023-08-28T02:28:27.410165+0000","last_clean":"2023-08-28T02:28:27.410165+0000","last_became_active":"2023-08-28T02:28:11.333321+0000","last_became_peered":"2023-08-28T02:28:11.333321+0000","last_unstale":"2023-08-28T02:28:27.410165+0000","last_undegraded":"2023-08-28T02:28:27.410165+0000","last_fullsized":"2023-08-28T02:28:27.410165+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":241,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-08-28T02:28:27.410030+0000","last_change":"2023-08-28T02:27:31.343538+0000","last_active":"2023-08-28T02:28:27.410030+0000","last_peered":"2023-08-28T02:28:27.410030+0000","last_clean":"2023-08-28T02:28:27.410030+0000","last_became_active":"2023-08-28T02:27:31.343324+0000","last_became_peered":"2023-08-28T02:27:31.343324+0000","last_unstale":"2023-08-28T02:28:27.410030+0000","last_undegraded":"2023-08-28T02:28:27.410030+0000","last_fullsized":"2023-08-28T02:28:27.410030+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.18","version":"224'4359","reported_seq":4413,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-08-28T02:28:31.976498+0000","last_change":"2023-08-28T02:27:58.496324+0000","last_active":"2023-08-28T02:28:31.976498+0000","last_peered":"2023-08-28T02:28:31.976498+0000","last_clean":"2023-08-28T02:28:31.976498+0000","last_became_active":"2023-08-28T02:27:58.494507+0000","last_became_peered":"2023-08-28T02:27:58.494507+0000","last_unstale":"2023-08-28T02:28:31.976498+0000","last_undegraded":"2023-08-28T02:28:31.976498+0000","last_fullsized":"2023-08-28T02:28:31.976498+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":4359,"log_dups_size":0,"ondisk_log_size":4359,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162693120,"num_objects":2483,"num_object_clones":0,"num_object_copies":7449,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2483,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2483,"num_write_kb":158880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":43,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-08-28T02:28:27.410307+0000","last_change":"2023-08-28T02:28:25.705331+0000","last_active":"2023-08-28T02:28:27.410307+0000","last_peered":"2023-08-28T02:28:27.410307+0000","last_clean":"2023-08-28T02:28:27.410307+0000","last_became_active":"2023-08-28T02:28:25.397480+0000","last_became_peered":"2023-08-28T02:28:25.397480+0000","last_unstale":"2023-08-28T02:28:27.410307+0000","last_undegraded":"2023-08-28T02:28:27.410307+0000","last_fullsized":"2023-08-28T02:28:27.410307+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":223,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-08-28T02:28:27.409729+0000","last_change":"2023-08-28T02:23:59.207929+0000","last_active":"2023-08-28T02:28:27.409729+0000","last_peered":"2023-08-28T02:28:27.409729+0000","last_clean":"2023-08-28T02:28:27.409729+0000","last_became_active":"2023-08-28T02:23:59.207589+0000","last_became_peered":"2023-08-28T02:23:59.207589+0000","last_unstale":"2023-08-28T02:28:27.409729+0000","last_undegraded":"2023-08-28T02:28:27.409729+0000","last_fullsized":"2023-08-28T02:28:27.409729+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":111,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-08-28T02:28:28.370312+0000","last_change":"2023-08-28T02:28:25.347988+0000","last_active":"2023-08-28T02:28:28.370312+0000","last_peered":"2023-08-28T02:28:28.370312+0000","last_clean":"2023-08-28T02:28:28.370312+0000","last_became_active":"2023-08-28T02:28:25.347632+0000","last_became_peered":"2023-08-28T02:28:25.347632+0000","last_unstale":"2023-08-28T02:28:28.370312+0000","last_undegraded":"2023-08-28T02:28:28.370312+0000","last_fullsized":"2023-08-28T02:28:28.370312+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:56:20.202662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":83,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-08-28T02:28:28.369284+0000","last_change":"2023-08-28T02:23:59.217794+0000","last_active":"2023-08-28T02:28:28.369284+0000","last_peered":"2023-08-28T02:28:28.369284+0000","last_clean":"2023-08-28T02:28:28.369284+0000","last_became_active":"2023-08-28T02:23:59.217231+0000","last_became_peered":"2023-08-28T02:23:59.217231+0000","last_unstale":"2023-08-28T02:28:28.369284+0000","last_undegraded":"2023-08-28T02:28:28.369284+0000","last_fullsized":"2023-08-28T02:28:28.369284+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:36:26.297410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":178,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-08-28T02:28:28.761110+0000","last_change":"2023-08-28T02:20:19.850252+0000","last_active":"2023-08-28T02:28:28.761110+0000","last_peered":"2023-08-28T02:28:28.761110+0000","last_clean":"2023-08-28T02:28:28.761110+0000","last_became_active":"2023-08-28T02:20:19.849913+0000","last_became_peered":"2023-08-28T02:20:19.849913+0000","last_unstale":"2023-08-28T02:28:28.761110+0000","last_undegraded":"2023-08-28T02:28:28.761110+0000","last_fullsized":"2023-08-28T02:28:28.761110+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:48:47.713465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":255,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-08-28T02:28:27.409645+0000","last_change":"2023-08-28T02:28:25.347845+0000","last_active":"2023-08-28T02:28:27.409645+0000","last_peered":"2023-08-28T02:28:27.409645+0000","last_clean":"2023-08-28T02:28:27.409645+0000","last_became_active":"2023-08-28T02:28:25.347502+0000","last_became_peered":"2023-08-28T02:28:25.347502+0000","last_unstale":"2023-08-28T02:28:27.409645+0000","last_undegraded":"2023-08-28T02:28:27.409645+0000","last_fullsized":"2023-08-28T02:28:27.409645+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":26,"stat_sum":{"num_bytes":5310906368,"num_objects":81038,"num_object_clones":0,"num_object_copies":243114,"num_objects_missing_on_primary":2376,"num_objects_missing":2376,"num_objects_degraded":16924,"num_objects_misplaced":11004,"num_objects_unfound":0,"num_objects_dirty":81038,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":81038,"num_write_kb":5186432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9629,"num_bytes_recovered":631046144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16436756480,"data_stored":16436756480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":118376,"ondisk_log_size":118376,"up":78,"acting":69,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":19382,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26810,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":25,"num_objects_missing":25,"num_objects_degraded":61,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":101,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":21,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430435,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2879592,"kb_used_data":2570080,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":90865560,"statfs":{"total":95995035648,"available":93046333440,"internally_reserved":0,"allocated":2631761920,"data_stored":2631117103,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:27:59 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":4.6600000000000001,"15min":1.9119999999999999},"min":{"1min":0.27300000000000002,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":3.1019999999999999,"5min":116.755,"15min":116.755},"last":0.317},{"interface":"front","average":{"1min":1.3740000000000001,"5min":5.0869999999999997,"15min":2.1190000000000002},"min":{"1min":0.30299999999999999,"5min":0.248,"15min":0.23999999999999999},"max":{"1min":7.109,"5min":116.971,"15min":116.971},"last":0.36599999999999999}]},{"osd":1,"last update":"Mon Aug 28 02:28:09 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":1.407,"15min":0.86199999999999999},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":3.4169999999999998,"5min":9.8710000000000004,"15min":9.8710000000000004},"last":0.27900000000000003},{"interface":"front","average":{"1min":2.2490000000000001,"5min":2.4009999999999998,"15min":1.2210000000000001},"min":{"1min":0.28100000000000003,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":11.205,"5min":11.696999999999999,"15min":11.696999999999999},"last":0.47399999999999998}]},{"osd":2,"last update":"Mon Aug 28 02:27:33 2023","interfaces":[{"interface":"back","average":{"1min":2.3769999999999998,"5min":4.5970000000000004,"15min":1.925},"min":{"1min":0.29499999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":12.558999999999999,"5min":119.571,"15min":119.571},"last":0.34399999999999997},{"interface":"front","average":{"1min":3.3839999999999999,"5min":5.8129999999999997,"15min":2.3420000000000001},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":15.951000000000001,"5min":119.554,"15min":119.554},"last":0.39600000000000002}]},{"osd":3,"last update":"Mon Aug 28 02:28:09 2023","interfaces":[{"interface":"back","average":{"1min":7.4169999999999998,"5min":3.899,"15min":1.7749999999999999},"min":{"1min":0.32300000000000001,"5min":0.30599999999999999,"15min":0.30199999999999999},"max":{"1min":99.459000000000003,"5min":107.916,"15min":107.916},"last":0.44400000000000001},{"interface":"front","average":{"1min":6.6479999999999997,"5min":4.4240000000000004,"15min":1.9450000000000001},"min":{"1min":0.373,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":98.084000000000003,"5min":112.00700000000001,"15min":112.00700000000001},"last":0.82099999999999995}]},{"osd":4,"last update":"Mon Aug 28 02:27:33 2023","interfaces":[{"interface":"back","average":{"1min":1.1879999999999999,"5min":2.6360000000000001,"15min":1.208},"min":{"1min":0.29099999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":9.6709999999999994,"5min":63.048000000000002,"15min":63.048000000000002},"last":0.32900000000000001},{"interface":"front","average":{"1min":1.173,"5min":2.359,"15min":1.133},"min":{"1min":0.31,"5min":0.247,"15min":0.247},"max":{"1min":5.3739999999999997,"5min":62.912999999999997,"15min":62.912999999999997},"last":0.49299999999999999}]},{"osd":5,"last update":"Mon Aug 28 02:27:33 2023","interfaces":[{"interface":"back","average":{"1min":1.248,"5min":2.0430000000000001,"15min":1.016},"min":{"1min":0.34999999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":6.9169999999999998,"5min":63.021000000000001,"15min":63.021000000000001},"last":0.38},{"interface":"front","average":{"1min":1.4530000000000001,"5min":2.7320000000000002,"15min":1.304},"min":{"1min":0.36599999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":9.6929999999999996,"5min":63.082999999999998,"15min":63.082999999999998},"last":0.433}]},{"osd":6,"last update":"Mon Aug 28 02:27:33 2023","interfaces":[{"interface":"back","average":{"1min":1.2629999999999999,"5min":2.7010000000000001,"15min":1.266},"min":{"1min":0.38600000000000001,"5min":0.317,"15min":0.29699999999999999},"max":{"1min":8.827,"5min":101.61,"15min":101.61},"last":0.48199999999999998},{"interface":"front","average":{"1min":1.71,"5min":2.948,"15min":1.3899999999999999},"min":{"1min":0.40300000000000002,"5min":0.308,"15min":0.308},"max":{"1min":14.423999999999999,"5min":101.621,"15min":101.621},"last":0.45800000000000002}]}]},{"osd":6,"up_from":42,"seq":180388626662,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":902804,"kb_used_data":593292,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92842348,"statfs":{"total":95995035648,"available":95070564352,"internally_reserved":0,"allocated":607531008,"data_stored":606905666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:28:07 2023","interfaces":[{"interface":"back","average":{"1min":1.242,"5min":6.3150000000000004,"15min":2.5489999999999999},"min":{"1min":0.246,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":10.819000000000001,"5min":121.247,"15min":121.247},"last":1.153},{"interface":"front","average":{"1min":1.8620000000000001,"5min":6.2510000000000003,"15min":2.4929999999999999},"min":{"1min":0.26100000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":15.864000000000001,"5min":118.977,"15min":118.977},"last":1.5}]},{"osd":1,"last update":"Mon Aug 28 02:28:07 2023","interfaces":[{"interface":"back","average":{"1min":1.3999999999999999,"5min":2.8159999999999998,"15min":1.5249999999999999},"min":{"1min":0.32400000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":10.93,"5min":99.998999999999995,"15min":99.998999999999995},"last":4.415},{"interface":"front","average":{"1min":1.6060000000000001,"5min":3.1560000000000001,"15min":1.6619999999999999},"min":{"1min":0.377,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":8.3300000000000001,"5min":95.040000000000006,"15min":95.040000000000006},"last":2.0110000000000001}]},{"osd":2,"last update":"Mon Aug 28 02:27:43 2023","interfaces":[{"interface":"back","average":{"1min":1.181,"5min":4.2060000000000004,"15min":1.8049999999999999},"min":{"1min":0.34899999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":5.7750000000000004,"5min":79.787000000000006,"15min":79.787000000000006},"last":1.5760000000000001},{"interface":"front","average":{"1min":0.76900000000000002,"5min":4.1059999999999999,"15min":1.748},"min":{"1min":0.32700000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.9650000000000001,"5min":79.935000000000002,"15min":79.935000000000002},"last":1.746}]},{"osd":3,"last update":"Mon Aug 28 02:28:07 2023","interfaces":[{"interface":"back","average":{"1min":5.3789999999999996,"5min":3.3149999999999999,"15min":1.5740000000000001},"min":{"1min":0.35199999999999998,"5min":0.317,"15min":0.317},"max":{"1min":39.587000000000003,"5min":39.587000000000003,"15min":39.587000000000003},"last":2.9529999999999998},{"interface":"front","average":{"1min":5.8239999999999998,"5min":3.4729999999999999,"15min":1.6699999999999999},"min":{"1min":0.44500000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":39.593000000000004,"5min":39.593000000000004,"15min":39.593000000000004},"last":2.9649999999999999}]},{"osd":4,"last update":"Mon Aug 28 02:28:10 2023","interfaces":[{"interface":"back","average":{"1min":0.73899999999999999,"5min":1.7370000000000001,"15min":0.92500000000000004},"min":{"1min":0.30199999999999999,"5min":0.251,"15min":0.251},"max":{"1min":4.1340000000000003,"5min":10.637,"15min":10.637},"last":1.212},{"interface":"front","average":{"1min":1.1879999999999999,"5min":1.9339999999999999,"15min":1.018},"min":{"1min":0.31900000000000001,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":8.109,"5min":10.055,"15min":10.055},"last":1.534}]},{"osd":5,"last update":"Mon Aug 28 02:28:10 2023","interfaces":[{"interface":"back","average":{"1min":1.0069999999999999,"5min":1.962,"15min":1.0609999999999999},"min":{"1min":0.30299999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":2.8130000000000002,"5min":20.759,"15min":20.759},"last":1.718},{"interface":"front","average":{"1min":0.80200000000000005,"5min":1.492,"15min":0.89900000000000002},"min":{"1min":0.34200000000000003,"5min":0.311,"15min":0.28799999999999998},"max":{"1min":4.0590000000000002,"5min":7.2880000000000003,"15min":7.3780000000000001},"last":1.1879999999999999}]},{"osd":7,"last update":"Mon Aug 28 02:27:38 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":1.5409999999999999,"15min":0.88800000000000001},"min":{"1min":0.34599999999999997,"5min":0.314,"15min":0.314},"max":{"1min":1.3640000000000001,"5min":9.0830000000000002,"15min":9.0830000000000002},"last":1.2390000000000001},{"interface":"front","average":{"1min":0.95699999999999996,"5min":1.744,"15min":0.98999999999999999},"min":{"1min":0.38400000000000001,"5min":0.33500000000000002,"15min":0.29799999999999999},"max":{"1min":4.843,"5min":9.7110000000000003,"15min":9.7110000000000003},"last":1.595}]}]},{"osd":1,"up_from":98,"seq":420906795115,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1465964,"kb_used_data":1289892,"kb_used_omap":7,"kb_used_meta":176056,"kb_avail":92279188,"statfs":{"total":95995035648,"available":94493888512,"internally_reserved":0,"allocated":1320849408,"data_stored":1319758680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":180281675},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:28:04 2023","interfaces":[{"interface":"back","average":{"1min":3.4169999999999998,"5min":5.0090000000000003,"15min":1.9670000000000001},"min":{"1min":0.191,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":34.588999999999999,"5min":128.60599999999999,"15min":128.60599999999999},"last":3.2909999999999999},{"interface":"front","average":{"1min":2.5249999999999999,"5min":4.8250000000000002,"15min":1.972},"min":{"1min":0.20499999999999999,"5min":0.188,"15min":0.188},"max":{"1min":34.628999999999998,"5min":128.685,"15min":128.685},"last":0.216}]},{"osd":2,"last update":"Mon Aug 28 02:28:32 2023","interfaces":[{"interface":"back","average":{"1min":10.458,"5min":7.0570000000000004,"15min":2.746},"min":{"1min":0.26400000000000001,"5min":0.222,"15min":0.222},"max":{"1min":90.177000000000007,"5min":135.25999999999999,"15min":135.25999999999999},"last":5.3540000000000001},{"interface":"front","average":{"1min":9.6669999999999998,"5min":6.524,"15min":2.5840000000000001},"min":{"1min":0.308,"5min":0.23200000000000001,"15min":0.221},"max":{"1min":89.587999999999994,"5min":135.10300000000001,"15min":135.10300000000001},"last":3.1539999999999999}]},{"osd":3,"last update":"Mon Aug 28 02:28:04 2023","interfaces":[{"interface":"back","average":{"1min":11.666,"5min":3.9489999999999998,"15min":1.645},"min":{"1min":0.29199999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":123.741,"5min":123.741,"15min":123.741},"last":1.829},{"interface":"front","average":{"1min":11.526,"5min":4.1929999999999996,"15min":1.782},"min":{"1min":0.29999999999999999,"5min":0.254,"15min":0.254},"max":{"1min":123.642,"5min":123.642,"15min":123.642},"last":5.2530000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:28:04 2023","interfaces":[{"interface":"back","average":{"1min":1.379,"5min":2.0710000000000002,"15min":1.145},"min":{"1min":0.32800000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":6.4400000000000004,"5min":17.600000000000001,"15min":17.600000000000001},"last":5.1539999999999999},{"interface":"front","average":{"1min":1.5860000000000001,"5min":1.827,"15min":1.04},"min":{"1min":0.33200000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":8.0419999999999998,"5min":19.010000000000002,"15min":19.010000000000002},"last":3.242}]},{"osd":5,"last update":"Mon Aug 28 02:28:04 2023","interfaces":[{"interface":"back","average":{"1min":1.1679999999999999,"5min":2.0950000000000002,"15min":1.1950000000000001},"min":{"1min":0.42699999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":3.2170000000000001,"5min":15.544,"15min":15.544},"last":1.0389999999999999},{"interface":"front","average":{"1min":1.496,"5min":1.9139999999999999,"15min":1.1060000000000001},"min":{"1min":0.40400000000000003,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":6.3929999999999998,"5min":17.646999999999998,"15min":17.646999999999998},"last":5.1970000000000001}]},{"osd":6,"last update":"Mon Aug 28 02:28:04 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":2.2930000000000001,"15min":1.304},"min":{"1min":0.371,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":3.1549999999999998,"5min":15.512,"15min":15.512},"last":1.0700000000000001},{"interface":"front","average":{"1min":1.3089999999999999,"5min":2.4220000000000002,"15min":1.258},"min":{"1min":0.39600000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":5.4649999999999999,"5min":17.673999999999999,"15min":17.673999999999999},"last":5.226}]},{"osd":7,"last update":"Mon Aug 28 02:28:04 2023","interfaces":[{"interface":"back","average":{"1min":1.5900000000000001,"5min":2.0329999999999999,"15min":1.1180000000000001},"min":{"1min":0.38200000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":8.1129999999999995,"5min":19.061,"15min":19.061},"last":3.194},{"interface":"front","average":{"1min":1.1830000000000001,"5min":2.2309999999999999,"15min":1.2410000000000001},"min":{"1min":0.40500000000000003,"5min":0.39900000000000002,"15min":0.33100000000000002},"max":{"1min":3.1890000000000001,"5min":15.476000000000001,"15min":15.476000000000001},"last":1.0940000000000001}]}]},{"osd":0,"up_from":90,"seq":386547056761,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1909172,"kb_used_data":1598124,"kb_used_omap":8,"kb_used_meta":311031,"kb_avail":91835980,"statfs":{"total":95995035648,"available":94040043520,"internally_reserved":0,"allocated":1636478976,"data_stored":1635436719,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":318496105},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Aug 28 02:28:14 2023","interfaces":[{"interface":"back","average":{"1min":3.5409999999999999,"5min":3.2130000000000001,"15min":1.4039999999999999},"min":{"1min":0.19400000000000001,"5min":0.184,"15min":0.184},"max":{"1min":54.106999999999999,"5min":54.106999999999999,"15min":54.106999999999999},"last":0.96799999999999997},{"interface":"front","average":{"1min":4.0179999999999998,"5min":3.282,"15min":1.379},"min":{"1min":0.223,"5min":0.17699999999999999,"15min":0.17000000000000001},"max":{"1min":54.354999999999997,"5min":54.354999999999997,"15min":54.354999999999997},"last":4.5570000000000004}]},{"osd":2,"last update":"Mon Aug 28 02:28:31 2023","interfaces":[{"interface":"back","average":{"1min":7.843,"5min":4.6470000000000002,"15min":1.9019999999999999},"min":{"1min":0.24299999999999999,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":100.718,"5min":100.718,"15min":100.718},"last":0.64000000000000001},{"interface":"front","average":{"1min":7.5890000000000004,"5min":5.5709999999999997,"15min":2.2610000000000001},"min":{"1min":0.24199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":100.962,"5min":100.962,"15min":100.962},"last":4.484}]},{"osd":3,"last update":"Mon Aug 28 02:28:14 2023","interfaces":[{"interface":"back","average":{"1min":24.100000000000001,"5min":7.1529999999999996,"15min":2.702},"min":{"1min":0.26800000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":237.43899999999999,"5min":237.43899999999999,"15min":237.43899999999999},"last":0.93100000000000005},{"interface":"front","average":{"1min":23.928999999999998,"5min":6.9980000000000002,"15min":2.669},"min":{"1min":0.27500000000000002,"5min":0.27300000000000002,"15min":0.27200000000000002},"max":{"1min":237.48099999999999,"5min":237.48099999999999,"15min":237.48099999999999},"last":0.53000000000000003}]},{"osd":4,"last update":"Mon Aug 28 02:28:06 2023","interfaces":[{"interface":"back","average":{"1min":0.75900000000000001,"5min":1.8660000000000001,"15min":1.097},"min":{"1min":0.29799999999999999,"5min":0.29599999999999999,"15min":0.26800000000000002},"max":{"1min":2.8719999999999999,"5min":9.6649999999999991,"15min":9.6649999999999991},"last":0.58599999999999997},{"interface":"front","average":{"1min":2.5350000000000001,"5min":2.5449999999999999,"15min":1.371},"min":{"1min":0.34000000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":27.271999999999998,"5min":27.271999999999998,"15min":27.271999999999998},"last":0.503}]},{"osd":5,"last update":"Mon Aug 28 02:28:06 2023","interfaces":[{"interface":"back","average":{"1min":2.625,"5min":2.9940000000000002,"15min":1.496},"min":{"1min":0.33200000000000002,"5min":0.313,"15min":0.313},"max":{"1min":27.367000000000001,"5min":44.000999999999998,"15min":44.000999999999998},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.79400000000000004,"5min":2.3959999999999999,"15min":1.284},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.30599999999999999},"max":{"1min":2.8420000000000001,"5min":43.953000000000003,"15min":43.953000000000003},"last":4.2850000000000001}]},{"osd":6,"last update":"Mon Aug 28 02:28:06 2023","interfaces":[{"interface":"back","average":{"1min":2.6160000000000001,"5min":3.008,"15min":1.5109999999999999},"min":{"1min":0.378,"5min":0.373,"15min":0.32000000000000001},"max":{"1min":27.331,"5min":32.417999999999999,"15min":32.417999999999999},"last":0.55400000000000005},{"interface":"front","average":{"1min":2.1819999999999999,"5min":2.653,"15min":1.379},"min":{"1min":0.36099999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":27.247,"5min":32.130000000000003,"15min":32.130000000000003},"last":4.218}]},{"osd":7,"last update":"Mon Aug 28 02:28:06 2023","interfaces":[{"interface":"back","average":{"1min":2.2450000000000001,"5min":2.3740000000000001,"15min":1.298},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":27.385000000000002,"5min":27.385000000000002,"15min":27.385000000000002},"last":0.61799999999999999},{"interface":"front","average":{"1min":2.8690000000000002,"5min":2.9249999999999998,"15min":1.4450000000000001},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":27.213999999999999,"5min":27.213999999999999,"15min":27.213999999999999},"last":4.524}]}]},{"osd":2,"up_from":104,"seq":446676598885,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3595192,"kb_used_data":3319088,"kb_used_omap":10,"kb_used_meta":276085,"kb_avail":90149960,"statfs":{"total":95995035648,"available":92313559040,"internally_reserved":0,"allocated":3398746112,"data_stored":3397756286,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10792,"internal_metadata":282711512},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:27:34 2023","interfaces":[{"interface":"back","average":{"1min":7.0460000000000003,"5min":5.1909999999999998,"15min":1.9670000000000001},"min":{"1min":0.246,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":113.89100000000001,"5min":113.89100000000001,"15min":113.89100000000001},"last":0.187},{"interface":"front","average":{"1min":8.2379999999999995,"5min":8.3840000000000003,"15min":3.0670000000000002},"min":{"1min":0.252,"5min":0.214,"15min":0.19900000000000001},"max":{"1min":114.291,"5min":147.286,"15min":147.286},"last":1.952}]},{"osd":1,"last update":"Mon Aug 28 02:27:34 2023","interfaces":[{"interface":"back","average":{"1min":1.127,"5min":2.9940000000000002,"15min":1.3720000000000001},"min":{"1min":0.26500000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":5.8490000000000002,"5min":16.372,"15min":30.888000000000002},"last":0.56699999999999995},{"interface":"front","average":{"1min":2.0110000000000001,"5min":2.6099999999999999,"15min":1.218},"min":{"1min":0.252,"5min":0.21299999999999999,"15min":0.20999999999999999},"max":{"1min":11.827999999999999,"5min":20.259,"15min":31.648},"last":1.0880000000000001}]},{"osd":3,"last update":"Mon Aug 28 02:28:06 2023","interfaces":[{"interface":"back","average":{"1min":6.9980000000000002,"5min":5.1669999999999998,"15min":2.0489999999999999},"min":{"1min":0.25,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":71.983999999999995,"5min":128.63900000000001,"15min":128.63900000000001},"last":0.58899999999999997},{"interface":"front","average":{"1min":6.4100000000000001,"5min":5.3879999999999999,"15min":2.0750000000000002},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":71.944999999999993,"5min":128.75700000000001,"15min":128.75700000000001},"last":1.4239999999999999}]},{"osd":4,"last update":"Mon Aug 28 02:27:34 2023","interfaces":[{"interface":"back","average":{"1min":1.9119999999999999,"5min":2.6139999999999999,"15min":1.28},"min":{"1min":0.29799999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":14.254,"5min":17.024999999999999,"15min":17.024999999999999},"last":1.125},{"interface":"front","average":{"1min":1.1220000000000001,"5min":1.881,"15min":1.0880000000000001},"min":{"1min":0.33000000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.2969999999999997,"5min":16.936,"15min":16.936},"last":0.42699999999999999}]},{"osd":5,"last update":"Mon Aug 28 02:27:34 2023","interfaces":[{"interface":"back","average":{"1min":1.744,"5min":3.6869999999999998,"15min":1.661},"min":{"1min":0.38300000000000001,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":10.050000000000001,"5min":128.696,"15min":128.696},"last":0.63300000000000001},{"interface":"front","average":{"1min":1.6779999999999999,"5min":4.0140000000000002,"15min":1.72},"min":{"1min":0.34999999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":10.787000000000001,"5min":128.84100000000001,"15min":128.84100000000001},"last":1.1539999999999999}]},{"osd":6,"last update":"Mon Aug 28 02:27:34 2023","interfaces":[{"interface":"back","average":{"1min":1.1459999999999999,"5min":2.1230000000000002,"15min":1.1319999999999999},"min":{"1min":0.33200000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":6.2679999999999998,"5min":11.285,"15min":11.285},"last":0.39900000000000002},{"interface":"front","average":{"1min":1.3320000000000001,"5min":2.4180000000000001,"15min":1.2450000000000001},"min":{"1min":0.42699999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":6.008,"5min":11.055999999999999,"15min":11.055999999999999},"last":0.61199999999999999}]},{"osd":7,"last update":"Mon Aug 28 02:27:34 2023","interfaces":[{"interface":"back","average":{"1min":1.466,"5min":3.5510000000000002,"15min":1.611},"min":{"1min":0.36399999999999999,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":7.2389999999999999,"5min":128.911,"15min":128.911},"last":108.904},{"interface":"front","average":{"1min":2.004,"5min":4.0010000000000003,"15min":1.7689999999999999},"min":{"1min":0.45500000000000002,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":13.281000000000001,"5min":128.72399999999999,"15min":128.72399999999999},"last":108.217}]}]},{"osd":3,"up_from":112,"seq":481036337247,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3441644,"kb_used_data":3257572,"kb_used_omap":3,"kb_used_meta":184060,"kb_avail":90303508,"statfs":{"total":95995035648,"available":92470792192,"internally_reserved":0,"allocated":3335753728,"data_stored":3334771833,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":188478370},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:28:03 2023","interfaces":[{"interface":"back","average":{"1min":1.9390000000000001,"5min":6.2990000000000004,"15min":2.3820000000000001},"min":{"1min":0.216,"5min":0.193,"15min":0.193},"max":{"1min":6.2279999999999998,"5min":164.25299999999999,"15min":164.25299999999999},"last":0.39300000000000002},{"interface":"front","average":{"1min":1.603,"5min":6.3630000000000004,"15min":2.391},"min":{"1min":0.192,"5min":0.192,"15min":0.192},"max":{"1min":5.6900000000000004,"5min":164.55000000000001,"15min":164.55000000000001},"last":3.7050000000000001}]},{"osd":1,"last update":"Mon Aug 28 02:28:03 2023","interfaces":[{"interface":"back","average":{"1min":1.2889999999999999,"5min":4.1529999999999996,"15min":1.6419999999999999},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":6.2380000000000004,"5min":117.34699999999999,"15min":117.34699999999999},"last":0.373},{"interface":"front","average":{"1min":1.784,"5min":4.1349999999999998,"15min":1.6699999999999999},"min":{"1min":0.28999999999999998,"5min":0.224,"15min":0.224},"max":{"1min":4.9349999999999996,"5min":118.012,"15min":118.012},"last":0.436}]},{"osd":2,"last update":"Mon Aug 28 02:28:03 2023","interfaces":[{"interface":"back","average":{"1min":4.9480000000000004,"5min":6.7000000000000002,"15min":2.528},"min":{"1min":0.34399999999999997,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":65.308999999999997,"5min":120.43300000000001,"15min":120.43300000000001},"last":0.313},{"interface":"front","average":{"1min":4.7359999999999998,"5min":6.6699999999999999,"15min":2.5049999999999999},"min":{"1min":0.32200000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":65.120999999999995,"5min":120.44499999999999,"15min":120.44499999999999},"last":0.58499999999999996}]},{"osd":4,"last update":"Mon Aug 28 02:28:03 2023","interfaces":[{"interface":"back","average":{"1min":1.327,"5min":1.8700000000000001,"15min":1.069},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":4.1959999999999997,"5min":9.016,"15min":9.016},"last":0.52500000000000002},{"interface":"front","average":{"1min":1.171,"5min":1.9359999999999999,"15min":1.095},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":3.5680000000000001,"5min":11.768000000000001,"15min":11.768000000000001},"last":0.41199999999999998}]},{"osd":5,"last update":"Mon Aug 28 02:28:03 2023","interfaces":[{"interface":"back","average":{"1min":1.2509999999999999,"5min":2.9319999999999999,"15min":1.4179999999999999},"min":{"1min":0.38800000000000001,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":3.6459999999999999,"5min":102.904,"15min":102.904},"last":34.060000000000002},{"interface":"front","average":{"1min":1.621,"5min":2.802,"15min":1.381},"min":{"1min":0.40799999999999997,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":4.883,"5min":103.283,"15min":103.283},"last":34.079999999999998}]},{"osd":6,"last update":"Mon Aug 28 02:28:03 2023","interfaces":[{"interface":"back","average":{"1min":1.7909999999999999,"5min":2.4020000000000001,"15min":1.2589999999999999},"min":{"1min":0.41099999999999998,"5min":0.314,"15min":0.314},"max":{"1min":4.9109999999999996,"5min":72.116,"15min":72.116},"last":0.47799999999999998},{"interface":"front","average":{"1min":1.643,"5min":2.3610000000000002,"15min":1.208},"min":{"1min":0.46400000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":5.569,"5min":72.030000000000001,"15min":72.030000000000001},"last":0.56000000000000005}]},{"osd":7,"last update":"Mon Aug 28 02:28:03 2023","interfaces":[{"interface":"back","average":{"1min":1.859,"5min":1.915,"15min":1.1140000000000001},"min":{"1min":0.46500000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":5.7290000000000001,"5min":11.949999999999999,"15min":11.949999999999999},"last":38.564},{"interface":"front","average":{"1min":1.587,"5min":2.1840000000000002,"15min":1.198},"min":{"1min":0.36399999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":3.9940000000000002,"5min":13.890000000000001,"15min":13.890000000000001},"last":38.823999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019120,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2116252,"kb_used_data":1810836,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":91628900,"statfs":{"total":95995035648,"available":93827993600,"internally_reserved":0,"allocated":1854296064,"data_stored":1853662899,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:28:05 2023","interfaces":[{"interface":"back","average":{"1min":1.208,"5min":6.3230000000000004,"15min":2.516},"min":{"1min":0.25900000000000001,"5min":0.223,"15min":0.223},"max":{"1min":4.8250000000000002,"5min":138.67400000000001,"15min":138.67400000000001},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.95899999999999996,"5min":5.9930000000000003,"15min":2.415},"min":{"1min":0.25,"5min":0.248,"15min":0.248},"max":{"1min":3.899,"5min":134.14599999999999,"15min":134.14599999999999},"last":0.38300000000000001}]},{"osd":1,"last update":"Mon Aug 28 02:28:05 2023","interfaces":[{"interface":"back","average":{"1min":1.238,"5min":2.2170000000000001,"15min":1.1339999999999999},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":4.9459999999999997,"5min":53.622999999999998,"15min":53.622999999999998},"last":0.34999999999999998},{"interface":"front","average":{"1min":1.4850000000000001,"5min":2.278,"15min":1.161},"min":{"1min":0.28000000000000003,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":4.7439999999999998,"5min":53.774000000000001,"15min":53.774000000000001},"last":0.42099999999999999}]},{"osd":2,"last update":"Mon Aug 28 02:27:33 2023","interfaces":[{"interface":"back","average":{"1min":5.8869999999999996,"5min":5.7910000000000004,"15min":2.2989999999999999},"min":{"1min":0.36899999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":64.063000000000002,"5min":74.099000000000004,"15min":74.099000000000004},"last":0.437},{"interface":"front","average":{"1min":5.968,"5min":5.3780000000000001,"15min":2.1589999999999998},"min":{"1min":0.29299999999999998,"5min":0.28599999999999998,"15min":0.25},"max":{"1min":64.073999999999998,"5min":74.081999999999994,"15min":74.081999999999994},"last":0.45200000000000001}]},{"osd":3,"last update":"Mon Aug 28 02:28:05 2023","interfaces":[{"interface":"back","average":{"1min":10.048,"5min":3.6920000000000002,"15min":1.6000000000000001},"min":{"1min":0.372,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":117.264,"5min":117.264,"15min":117.264},"last":1.647},{"interface":"front","average":{"1min":10.414,"5min":3.8210000000000002,"15min":1.7190000000000001},"min":{"1min":0.38,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":113.876,"5min":113.876,"15min":113.876},"last":0.48999999999999999}]},{"osd":5,"last update":"Mon Aug 28 02:27:43 2023","interfaces":[{"interface":"back","average":{"1min":0.77900000000000003,"5min":1.365,"15min":0.79900000000000004},"min":{"1min":0.26000000000000001,"5min":0.249,"15min":0.249},"max":{"1min":3.1360000000000001,"5min":8.2560000000000002,"15min":8.2560000000000002},"last":0.496},{"interface":"front","average":{"1min":0.53400000000000003,"5min":1.776,"15min":0.95299999999999996},"min":{"1min":0.29399999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":1.599,"5min":16.497,"15min":16.497},"last":0.36799999999999999}]},{"osd":6,"last update":"Mon Aug 28 02:28:05 2023","interfaces":[{"interface":"back","average":{"1min":0.78000000000000003,"5min":1.599,"15min":0.92500000000000004},"min":{"1min":0.33000000000000002,"5min":0.29199999999999998,"15min":0.27500000000000002},"max":{"1min":4.883,"5min":11.170999999999999,"15min":11.170999999999999},"last":0.314},{"interface":"front","average":{"1min":0.78000000000000003,"5min":1.1639999999999999,"15min":0.78800000000000003},"min":{"1min":0.38200000000000001,"5min":0.32200000000000001,"15min":0.318},"max":{"1min":3.2149999999999999,"5min":7.3170000000000002,"15min":7.3170000000000002},"last":0.47399999999999998}]},{"osd":7,"last update":"Mon Aug 28 02:28:26 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":1.7070000000000001,"15min":1.0189999999999999},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.312},"max":{"1min":1.9370000000000001,"5min":16.584,"15min":16.584},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.622,"5min":1.7010000000000001,"15min":1.0569999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":1.804,"5min":10.826000000000001,"15min":10.826000000000001},"last":0.46800000000000003}]}]},{"osd":5,"up_from":36,"seq":154618822892,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2066828,"kb_used_data":1753220,"kb_used_omap":0,"kb_used_meta":313600,"kb_avail":91678324,"statfs":{"total":95995035648,"available":93878603776,"internally_reserved":0,"allocated":1795297280,"data_stored":1794675654,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321126400},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:27:55 2023","interfaces":[{"interface":"back","average":{"1min":1.171,"5min":4.5309999999999997,"15min":2.4590000000000001},"min":{"1min":0.219,"5min":0.219,"15min":0.20300000000000001},"max":{"1min":4.7329999999999997,"5min":101.014,"15min":131.06100000000001},"last":0.223},{"interface":"front","average":{"1min":1.218,"5min":4.9249999999999998,"15min":2.6059999999999999},"min":{"1min":0.28000000000000003,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":4.7240000000000002,"5min":101.03,"15min":131.262},"last":0.36899999999999999}]},{"osd":1,"last update":"Mon Aug 28 02:28:06 2023","interfaces":[{"interface":"back","average":{"1min":1.264,"5min":2.8559999999999999,"15min":2.165},"min":{"1min":0.28999999999999998,"5min":0.216,"15min":0.216},"max":{"1min":4.0940000000000003,"5min":104.748,"15min":183.97900000000001},"last":0.314},{"interface":"front","average":{"1min":0.88900000000000001,"5min":3.3159999999999998,"15min":2.3130000000000002},"min":{"1min":0.29799999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.1609999999999996,"5min":105.666,"15min":184.33699999999999},"last":0.33400000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:27:37 2023","interfaces":[{"interface":"back","average":{"1min":1.1919999999999999,"5min":7.415,"15min":3.1819999999999999},"min":{"1min":0.35599999999999998,"5min":0.28499999999999998,"15min":0.252},"max":{"1min":5.8470000000000004,"5min":176.69499999999999,"15min":176.69499999999999},"last":0.46400000000000002},{"interface":"front","average":{"1min":1.2350000000000001,"5min":7.7930000000000001,"15min":3.3079999999999998},"min":{"1min":0.36699999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":5.8360000000000003,"5min":176.71199999999999,"15min":176.71199999999999},"last":0.45700000000000002}]},{"osd":3,"last update":"Mon Aug 28 02:28:06 2023","interfaces":[{"interface":"back","average":{"1min":1.0660000000000001,"5min":1.956,"15min":1.0629999999999999},"min":{"1min":0.307,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":4.1929999999999996,"5min":10.576000000000001,"15min":10.576000000000001},"last":0.29899999999999999},{"interface":"front","average":{"1min":1.635,"5min":2.714,"15min":1.3620000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6.5650000000000004,"5min":12.118,"15min":12.118},"last":1}]},{"osd":4,"last update":"Mon Aug 28 02:27:46 2023","interfaces":[{"interface":"back","average":{"1min":0.70699999999999996,"5min":1.4399999999999999,"15min":0.83799999999999997},"min":{"1min":0.317,"5min":0.246,"15min":0.246},"max":{"1min":3.8380000000000001,"5min":10.759,"15min":10.759},"last":0.27900000000000003},{"interface":"front","average":{"1min":0.626,"5min":1.504,"15min":0.86899999999999999},"min":{"1min":0.307,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":2.9169999999999998,"5min":8.6769999999999996,"15min":8.6769999999999996},"last":0.29299999999999998}]},{"osd":6,"last update":"Mon Aug 28 02:28:06 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":1.6220000000000001,"15min":0.97799999999999998},"min":{"1min":0.34699999999999998,"5min":0.29699999999999999,"15min":0.28399999999999997},"max":{"1min":4.109,"5min":14.803000000000001,"15min":14.803000000000001},"last":0.34999999999999998},{"interface":"front","average":{"1min":1.046,"5min":1.476,"15min":0.875},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30299999999999999},"max":{"1min":4.3540000000000001,"5min":8.6479999999999997,"15min":8.6479999999999997},"last":0.32200000000000001}]},{"osd":7,"last update":"Mon Aug 28 02:28:25 2023","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":1.734,"15min":1.0109999999999999},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.31},"max":{"1min":4.319,"5min":10.737,"15min":10.737},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.68400000000000005,"5min":1.4510000000000001,"15min":0.92200000000000004},"min":{"1min":0.33700000000000002,"5min":0.32200000000000001,"15min":0.29799999999999999},"max":{"1min":5.4900000000000002,"5min":12.787000000000001,"15min":12.787000000000001},"last":0.39900000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1433862144,"data_stored":1433862144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1332609024,"data_stored":1332609024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3417833472,"data_stored":3417833472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3348692992,"data_stored":3348692992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1857486848,"data_stored":1857486848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1801977856,"data_stored":1801977856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":607453184,"data_stored":607453184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2636840960,"data_stored":2636840960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-08-28T02:28:38.698 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd pool set unique_pool_1 pgp_num 26 2023-08-28T02:28:38.759 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:28:38.906 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:39.119 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:28:39.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:39.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:28:39.222 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:28:39.302 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:39.356 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:28:39.440 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.6 dump_blocked_ops 2023-08-28T02:28:39.491 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:28:39.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:39.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:28:39.595 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:28:39.661 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:39.733 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:28:39.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:39.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:28:39.772 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:28:39.839 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:28:39.958 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:40.101 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:28:40.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:40.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:28:40.126 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:28:40.209 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:28:40.246 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:40.353 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is set 2023-08-28T02:28:40.368 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:28:40.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:40.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:28:40.475 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:28:40.521 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:40 smithi169 ceph-mon[159415]: pgmap v531: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering, 1 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 102 B/s rd, 33 MiB/s wr, 600 op/s; 16987/248640 objects degraded (6.832%); 10903/248640 objects misplaced (4.385%); 2.4 MiB/s, 38 objects/s recovering 2023-08-28T02:28:40.521 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:40 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1427819385' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:28:40.521 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:40 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:28:40.526 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:40.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:40 smithi143 ceph-mon[183059]: pgmap v531: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering, 1 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 102 B/s rd, 33 MiB/s wr, 600 op/s; 16987/248640 objects degraded (6.832%); 10903/248640 objects misplaced (4.385%); 2.4 MiB/s, 38 objects/s recovering 2023-08-28T02:28:40.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:40 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1427819385' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:28:40.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:40 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:28:40.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:40 smithi143 ceph-mon[189871]: pgmap v531: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering, 1 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 102 B/s rd, 33 MiB/s wr, 600 op/s; 16987/248640 objects degraded (6.832%); 10903/248640 objects misplaced (4.385%); 2.4 MiB/s, 38 objects/s recovering 2023-08-28T02:28:40.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:40 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1427819385' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:28:40.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:40 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:28:40.813 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:28:40.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:40.817 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:28:40.918 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:28:40.967 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:41.652 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:41 smithi143 ceph-mon[189871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:28:41.652 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:41 smithi143 ceph-mon[189871]: failed to encode map e226 with expected crc 2023-08-28T02:28:41.652 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:41 smithi143 ceph-mon[189871]: failed to encode map e226 with expected crc 2023-08-28T02:28:41.653 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:41 smithi143 ceph-mon[189871]: failed to encode map e226 with expected crc 2023-08-28T02:28:41.653 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:41 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:28:41.653 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:41 smithi143 ceph-mon[189871]: osdmap e226: 8 total, 8 up, 8 in 2023-08-28T02:28:41.653 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:41 smithi143 ceph-mon[189871]: failed to encode map e226 with expected crc 2023-08-28T02:28:41.654 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:41 smithi143 ceph-mon[189871]: failed to encode map e226 with expected crc 2023-08-28T02:28:41.654 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:41 smithi143 ceph-mon[189871]: failed to encode map e226 with expected crc 2023-08-28T02:28:41.654 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:41 smithi143 ceph-mon[189871]: failed to encode map e226 with expected crc 2023-08-28T02:28:41.654 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:41 smithi143 ceph-mon[189871]: failed to encode map e226 with expected crc 2023-08-28T02:28:41.654 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:41 smithi143 ceph-mon[189871]: failed to encode map e226 with expected crc 2023-08-28T02:28:41.654 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:41 smithi143 ceph-mon[189871]: failed to encode map e226 with expected crc 2023-08-28T02:28:41.655 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:41 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:28:41.655 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:41 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:28:41.655 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:41 smithi143 ceph-mon[183059]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:28:41.655 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:41 smithi143 ceph-mon[183059]: failed to encode map e226 with expected crc 2023-08-28T02:28:41.655 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:41 smithi143 ceph-mon[183059]: failed to encode map e226 with expected crc 2023-08-28T02:28:41.656 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:41 smithi143 ceph-mon[183059]: failed to encode map e226 with expected crc 2023-08-28T02:28:41.656 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:41 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:28:41.656 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:41 smithi143 ceph-mon[183059]: osdmap e226: 8 total, 8 up, 8 in 2023-08-28T02:28:41.656 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:41 smithi143 ceph-mon[183059]: failed to encode map e226 with expected crc 2023-08-28T02:28:41.656 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:41 smithi143 ceph-mon[183059]: failed to encode map e226 with expected crc 2023-08-28T02:28:41.656 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:41 smithi143 ceph-mon[183059]: failed to encode map e226 with expected crc 2023-08-28T02:28:41.657 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:41 smithi143 ceph-mon[183059]: failed to encode map e226 with expected crc 2023-08-28T02:28:41.657 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:41 smithi143 ceph-mon[183059]: failed to encode map e226 with expected crc 2023-08-28T02:28:41.657 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:41 smithi143 ceph-mon[183059]: failed to encode map e226 with expected crc 2023-08-28T02:28:41.657 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:41 smithi143 ceph-mon[183059]: failed to encode map e226 with expected crc 2023-08-28T02:28:41.657 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:41 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:28:41.657 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:41 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:28:41.693 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:28:41.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:41.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:28:41.700 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:41 smithi169 ceph-mon[159415]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:28:41.700 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:41 smithi169 ceph-mon[159415]: failed to encode map e226 with expected crc 2023-08-28T02:28:41.700 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:41 smithi169 ceph-mon[159415]: failed to encode map e226 with expected crc 2023-08-28T02:28:41.700 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:41 smithi169 ceph-mon[159415]: failed to encode map e226 with expected crc 2023-08-28T02:28:41.701 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:41 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:28:41.701 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:41 smithi169 ceph-mon[159415]: osdmap e226: 8 total, 8 up, 8 in 2023-08-28T02:28:41.701 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:41 smithi169 ceph-mon[159415]: failed to encode map e226 with expected crc 2023-08-28T02:28:41.701 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:41 smithi169 ceph-mon[159415]: failed to encode map e226 with expected crc 2023-08-28T02:28:41.701 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:41 smithi169 ceph-mon[159415]: failed to encode map e226 with expected crc 2023-08-28T02:28:41.701 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:41 smithi169 ceph-mon[159415]: failed to encode map e226 with expected crc 2023-08-28T02:28:41.701 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:41 smithi169 ceph-mon[159415]: failed to encode map e226 with expected crc 2023-08-28T02:28:41.701 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:41 smithi169 ceph-mon[159415]: failed to encode map e226 with expected crc 2023-08-28T02:28:41.701 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:41 smithi169 ceph-mon[159415]: failed to encode map e226 with expected crc 2023-08-28T02:28:41.701 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:41 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:28:41.701 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:41 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:28:41.801 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:28:41.842 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:41.933 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:28:41 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:28:41] "GET /metrics HTTP/1.1" 200 36517 "" "Prometheus/2.33.4" 2023-08-28T02:28:41.981 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:28:41.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:41.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:28:42.090 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:28:42.169 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:42.268 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:28:42.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:42.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:28:42.375 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:28:42.459 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:42 smithi143 ceph-mon[183059]: pgmap v533: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering, 1 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 115 B/s rd, 31 MiB/s wr, 582 op/s; 16987/248640 objects degraded (6.832%); 10903/248640 objects misplaced (4.385%); 2.8 MiB/s, 44 objects/s recovering 2023-08-28T02:28:42.459 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:42 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 16987/248640 objects degraded (6.832%), 14 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:28:42.459 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:42 smithi143 ceph-mon[189871]: pgmap v533: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering, 1 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 115 B/s rd, 31 MiB/s wr, 582 op/s; 16987/248640 objects degraded (6.832%); 10903/248640 objects misplaced (4.385%); 2.8 MiB/s, 44 objects/s recovering 2023-08-28T02:28:42.459 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:42 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 16987/248640 objects degraded (6.832%), 14 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:28:42.461 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:42.621 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:28:42.621 INFO:teuthology.orchestra.run.smithi169.stdout: "ops": [], 2023-08-28T02:28:42.621 INFO:teuthology.orchestra.run.smithi169.stdout: "complaint_time": 30, 2023-08-28T02:28:42.621 INFO:teuthology.orchestra.run.smithi169.stdout: "num_blocked_ops": 0 2023-08-28T02:28:42.622 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:28:42.648 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:42 smithi169 ceph-mon[159415]: pgmap v533: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering, 1 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 115 B/s rd, 31 MiB/s wr, 582 op/s; 16987/248640 objects degraded (6.832%); 10903/248640 objects misplaced (4.385%); 2.8 MiB/s, 44 objects/s recovering 2023-08-28T02:28:42.648 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:42 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 16987/248640 objects degraded (6.832%), 14 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:28:42.808 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:28:42.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:42.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:28:42.912 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:28:42.957 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:43.378 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:28:43.411 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:28:43.476 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:28:43.523 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:28:43.561 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:28:43.620 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:28:43.680 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:28:43.715 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:28:43.744 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:28:43.744 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:28:43.745 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:28:43.745 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:28:43.745 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:28:43.745 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:28:43.745 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:28:43.745 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:28:43.820 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.6 dump_historic_ops 2023-08-28T02:28:43.920 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:28:43.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:43.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:28:44.039 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:28:44.046 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset noscrub 2023-08-28T02:28:44.090 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:44.174 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:28:44.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:44.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:28:44.286 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:28:44.346 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:28:44.373 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:44 smithi143 ceph-mon[183059]: pgmap v534: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering, 1 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 102 B/s rd, 27 MiB/s wr, 680 op/s; 16987/243651 objects degraded (6.972%); 10903/243651 objects misplaced (4.475%); 2.4 MiB/s, 38 objects/s recovering 2023-08-28T02:28:44.373 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:44 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2759085034' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:28:44.373 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:44 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1311774722' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "26"}]: dispatch 2023-08-28T02:28:44.373 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:44 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:28:44.374 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:44 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:28:44.374 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:44 smithi143 ceph-mon[189871]: pgmap v534: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering, 1 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 102 B/s rd, 27 MiB/s wr, 680 op/s; 16987/243651 objects degraded (6.972%); 10903/243651 objects misplaced (4.475%); 2.4 MiB/s, 38 objects/s recovering 2023-08-28T02:28:44.374 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:44 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2759085034' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:28:44.374 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:44 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1311774722' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "26"}]: dispatch 2023-08-28T02:28:44.374 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:44 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:28:44.374 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:44 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:28:44.375 INFO:teuthology.orchestra.run.smithi143.stderr:set pool 6 pgp_num to 26 2023-08-28T02:28:44.377 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:44.614 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:44 smithi169 ceph-mon[159415]: pgmap v534: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering, 1 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 102 B/s rd, 27 MiB/s wr, 680 op/s; 16987/243651 objects degraded (6.972%); 10903/243651 objects misplaced (4.475%); 2.4 MiB/s, 38 objects/s recovering 2023-08-28T02:28:44.615 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:44 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2759085034' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:28:44.615 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:44 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1311774722' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "26"}]: dispatch 2023-08-28T02:28:44.615 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:44 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:28:44.615 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:44 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:28:45.175 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:28:45.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:45.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:28:45.280 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:28:45.306 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:45.433 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:45 smithi143 ceph-mon[183059]: failed to encode map e227 with expected crc 2023-08-28T02:28:45.433 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:45 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1311774722' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "26"}]': finished 2023-08-28T02:28:45.433 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:45 smithi143 ceph-mon[183059]: osdmap e227: 8 total, 8 up, 8 in 2023-08-28T02:28:45.433 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:45 smithi143 ceph-mon[183059]: failed to encode map e227 with expected crc 2023-08-28T02:28:45.433 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:45 smithi143 ceph-mon[183059]: failed to encode map e227 with expected crc 2023-08-28T02:28:45.433 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:45 smithi143 ceph-mon[183059]: failed to encode map e227 with expected crc 2023-08-28T02:28:45.434 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:45 smithi143 ceph-mon[189871]: failed to encode map e227 with expected crc 2023-08-28T02:28:45.434 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:45 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1311774722' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "26"}]': finished 2023-08-28T02:28:45.434 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:45 smithi143 ceph-mon[189871]: osdmap e227: 8 total, 8 up, 8 in 2023-08-28T02:28:45.434 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:45 smithi143 ceph-mon[189871]: failed to encode map e227 with expected crc 2023-08-28T02:28:45.434 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:45 smithi143 ceph-mon[189871]: failed to encode map e227 with expected crc 2023-08-28T02:28:45.434 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:45 smithi143 ceph-mon[189871]: failed to encode map e227 with expected crc 2023-08-28T02:28:45.563 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:28:45.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:45.574 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:28:45.669 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:28:45.675 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:28:45.707 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:45 smithi169 ceph-mon[159415]: failed to encode map e227 with expected crc 2023-08-28T02:28:45.707 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:45 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1311774722' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "26"}]': finished 2023-08-28T02:28:45.707 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:45 smithi169 ceph-mon[159415]: osdmap e227: 8 total, 8 up, 8 in 2023-08-28T02:28:45.707 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:45 smithi169 ceph-mon[159415]: failed to encode map e227 with expected crc 2023-08-28T02:28:45.707 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:45 smithi169 ceph-mon[159415]: failed to encode map e227 with expected crc 2023-08-28T02:28:45.707 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:45 smithi169 ceph-mon[159415]: failed to encode map e227 with expected crc 2023-08-28T02:28:45.731 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:45.805 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:28:45.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:45.809 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:28:45.910 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:28:45.946 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:46.063 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:28:46.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:46.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:28:46.167 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:28:46.245 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:46.279 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:28:46.279 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:28:46.695 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:46 smithi143 ceph-mon[183059]: pgmap v536: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering, 1 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 9.1 MiB/s wr, 717 op/s; 16987/237183 objects degraded (7.162%); 10121/237183 objects misplaced (4.267%); 3.8 MiB/s, 60 objects/s recovering 2023-08-28T02:28:46.695 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:46 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 16987/237183 objects degraded (7.162%), 14 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:28:46.695 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:46 smithi143 ceph-mon[189871]: pgmap v536: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering, 1 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 9.1 MiB/s wr, 717 op/s; 16987/237183 objects degraded (7.162%); 10121/237183 objects misplaced (4.267%); 3.8 MiB/s, 60 objects/s recovering 2023-08-28T02:28:46.695 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:46 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 16987/237183 objects degraded (7.162%), 14 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:28:46.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:46 smithi169 ceph-mon[159415]: pgmap v536: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering, 1 active+recovery_wait, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 9.1 MiB/s wr, 717 op/s; 16987/237183 objects degraded (7.162%); 10121/237183 objects misplaced (4.267%); 3.8 MiB/s, 60 objects/s recovering 2023-08-28T02:28:46.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:46 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 16987/237183 objects degraded (7.162%), 14 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:28:47.228 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:28:47.242 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.7 dump_ops_in_flight 2023-08-28T02:28:47.643 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:28:47.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:47.649 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:28:47.711 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:28:47.750 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:28:47.841 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:48.223 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:28:48.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:48.235 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:28:48.335 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:28:48.392 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:48.572 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:28:48.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:48.600 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:28:48.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:28:48.601 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:48 smithi143 ceph-mon[183059]: pgmap v537: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 15033/232086 objects degraded (6.477%); 12045/232086 objects misplaced (5.190%); 2.4 MiB/s, 38 objects/s recovering 2023-08-28T02:28:48.601 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:48 smithi143 ceph-mon[189871]: pgmap v537: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 15033/232086 objects degraded (6.477%); 12045/232086 objects misplaced (5.190%); 2.4 MiB/s, 38 objects/s recovering 2023-08-28T02:28:48.701 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:28:48.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:48 smithi169 ceph-mon[159415]: pgmap v537: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering, 1 active+recovery_wait, 12 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 15033/232086 objects degraded (6.477%); 12045/232086 objects misplaced (5.190%); 2.4 MiB/s, 38 objects/s recovering 2023-08-28T02:28:48.731 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:48.778 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:28:48.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:48.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:28:48.881 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:28:48.921 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:48.987 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:28:48.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:48.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:28:49.092 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:28:49.124 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:49.326 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:28:49.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:49.333 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:28:49.434 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:28:49.471 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:49.553 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:28:49.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:49.558 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:28:49.607 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is unset 2023-08-28T02:28:49.637 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:49 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:28:49.637 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:49 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:28:49.638 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:49 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/4193923451' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:28:49.638 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:49 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:28:49.638 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:49 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:28:49.638 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:49 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:28:49.638 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:49 smithi143 ceph-mon[183059]: pgmap v538: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 832 op/s; 14351/226332 objects degraded (6.341%); 11991/226332 objects misplaced (5.298%); 4.3 MiB/s, 69 objects/s recovering 2023-08-28T02:28:49.638 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:49 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:28:49.638 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:49 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:28:49.638 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:49 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:28:49.638 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:49 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/4193923451' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:28:49.638 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:49 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:28:49.638 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:49 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:28:49.639 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:49 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:28:49.639 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:49 smithi143 ceph-mon[189871]: pgmap v538: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 832 op/s; 14351/226332 objects degraded (6.341%); 11991/226332 objects misplaced (5.298%); 4.3 MiB/s, 69 objects/s recovering 2023-08-28T02:28:49.639 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:49 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:28:49.658 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:28:49.700 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:49.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:49 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:28:49.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:49 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:28:49.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:49 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/4193923451' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:28:49.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:49 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:28:49.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:49 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:28:49.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:49 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:28:49.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:49 smithi169 ceph-mon[159415]: pgmap v538: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 832 op/s; 14351/226332 objects degraded (6.341%); 11991/226332 objects misplaced (5.298%); 4.3 MiB/s, 69 objects/s recovering 2023-08-28T02:28:49.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:49 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:28:49.883 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:28:49.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:49.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:28:49.964 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:28:49.965 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:28:49.988 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:28:50.023 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:50.193 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:28:50.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:50.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:28:50.300 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:28:50.349 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:50.409 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:28:50.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:50.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:28:50.466 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 7, 3, 1, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-08-28T02:28:50.466 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-08-28T02:28:50.466 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-08-28T02:28:50.466 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-08-28T02:28:50.466 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-08-28T02:28:50.466 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-08-28T02:28:50.513 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:28:50.542 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:50.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:50 smithi143 ceph-mon[183059]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:28:50.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:50 smithi143 ceph-mon[183059]: failed to encode map e228 with expected crc 2023-08-28T02:28:50.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:50 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:28:50.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:50 smithi143 ceph-mon[183059]: osdmap e228: 8 total, 8 up, 8 in 2023-08-28T02:28:50.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:50 smithi143 ceph-mon[183059]: failed to encode map e228 with expected crc 2023-08-28T02:28:50.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:50 smithi143 ceph-mon[183059]: failed to encode map e228 with expected crc 2023-08-28T02:28:50.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:50 smithi143 ceph-mon[183059]: failed to encode map e228 with expected crc 2023-08-28T02:28:50.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:50 smithi143 ceph-mon[183059]: failed to encode map e228 with expected crc 2023-08-28T02:28:50.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:50 smithi143 ceph-mon[189871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:28:50.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:50 smithi143 ceph-mon[189871]: failed to encode map e228 with expected crc 2023-08-28T02:28:50.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:50 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:28:50.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:50 smithi143 ceph-mon[189871]: osdmap e228: 8 total, 8 up, 8 in 2023-08-28T02:28:50.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:50 smithi143 ceph-mon[189871]: failed to encode map e228 with expected crc 2023-08-28T02:28:50.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:50 smithi143 ceph-mon[189871]: failed to encode map e228 with expected crc 2023-08-28T02:28:50.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:50 smithi143 ceph-mon[189871]: failed to encode map e228 with expected crc 2023-08-28T02:28:50.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:50 smithi143 ceph-mon[189871]: failed to encode map e228 with expected crc 2023-08-28T02:28:50.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:50 smithi169 ceph-mon[159415]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:28:50.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:50 smithi169 ceph-mon[159415]: failed to encode map e228 with expected crc 2023-08-28T02:28:50.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:50 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:28:50.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:50 smithi169 ceph-mon[159415]: osdmap e228: 8 total, 8 up, 8 in 2023-08-28T02:28:50.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:50 smithi169 ceph-mon[159415]: failed to encode map e228 with expected crc 2023-08-28T02:28:50.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:50 smithi169 ceph-mon[159415]: failed to encode map e228 with expected crc 2023-08-28T02:28:50.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:50 smithi169 ceph-mon[159415]: failed to encode map e228 with expected crc 2023-08-28T02:28:50.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:50 smithi169 ceph-mon[159415]: failed to encode map e228 with expected crc 2023-08-28T02:28:51.004 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:28:51.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:51.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:28:51.039 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.7 dump_blocked_ops 2023-08-28T02:28:51.112 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:28:51.166 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:51.177 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:28:51.253 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:28:51.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:51.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:28:51.364 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:28:51.405 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:51.523 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:28:51.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:51.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:28:51.587 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:28:51.629 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:28:51.630 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:28:51.683 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:28:51.709 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:51 smithi169 ceph-mon[159415]: failed to encode map e228 with expected crc 2023-08-28T02:28:51.710 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:51 smithi169 ceph-mon[159415]: pgmap v540: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 14351/226332 objects degraded (6.341%); 11991/226332 objects misplaced (5.298%); 4.8 MiB/s, 77 objects/s recovering 2023-08-28T02:28:51.710 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:51 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 14351/226332 objects degraded (6.341%), 13 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:28:51.710 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:51 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3612540893' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:28:51.730 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:51.735 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:28:51.775 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:28:51.808 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:28:51.844 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:28:51.855 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:28:51.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:51.868 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:28:51.868 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:51 smithi143 ceph-mon[189871]: failed to encode map e228 with expected crc 2023-08-28T02:28:51.868 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:51 smithi143 ceph-mon[189871]: pgmap v540: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 14351/226332 objects degraded (6.341%); 11991/226332 objects misplaced (5.298%); 4.8 MiB/s, 77 objects/s recovering 2023-08-28T02:28:51.868 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:51 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 14351/226332 objects degraded (6.341%), 13 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:28:51.869 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:51 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3612540893' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:28:51.870 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:51 smithi143 ceph-mon[183059]: failed to encode map e228 with expected crc 2023-08-28T02:28:51.870 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:51 smithi143 ceph-mon[183059]: pgmap v540: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 14351/226332 objects degraded (6.341%); 11991/226332 objects misplaced (5.298%); 4.8 MiB/s, 77 objects/s recovering 2023-08-28T02:28:51.870 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:51 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 14351/226332 objects degraded (6.341%), 13 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:28:51.870 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:51 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3612540893' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:28:51.880 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:28:51.894 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:28:51.894 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:28:51.894 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:28:51.895 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:28:51.895 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:28:51.895 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:28:51.895 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:28:51.895 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:28:51.969 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:28:52.046 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:52.174 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:28:52.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:52.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:28:52.185 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:28:51 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:28:51] "GET /metrics HTTP/1.1" 200 36517 "" "Prometheus/2.33.4" 2023-08-28T02:28:52.286 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:28:52.350 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:52.409 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:28:52.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:52.412 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:28:52.512 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:28:52.540 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:52.599 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:28:52.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:52.601 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:28:52.702 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:28:52.777 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:52.924 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:28:52.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:52.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:28:52.952 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:28:53.012 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset nodeep-scrub 2023-08-28T02:28:53.040 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:28:53.123 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:53.252 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:28:53.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:53.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:28:53.362 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:28:53.455 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:53.573 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:28:53.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:53.582 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:28:53.683 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:28:53.722 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:28:53.722 INFO:teuthology.orchestra.run.smithi169.stdout: "success": "" 2023-08-28T02:28:53.722 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:28:53.723 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:28:53.742 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:54.329 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:54 smithi143 ceph-mon[189871]: pgmap v541: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 14359/221757 objects degraded (6.475%); 11742/221757 objects misplaced (5.295%); 4.1 MiB/s, 65 objects/s recovering 2023-08-28T02:28:54.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:54 smithi169 ceph-mon[159415]: pgmap v541: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 14359/221757 objects degraded (6.475%); 11742/221757 objects misplaced (5.295%); 4.1 MiB/s, 65 objects/s recovering 2023-08-28T02:28:54.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:54 smithi143 ceph-mon[183059]: pgmap v541: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 14359/221757 objects degraded (6.475%); 11742/221757 objects misplaced (5.295%); 4.1 MiB/s, 65 objects/s recovering 2023-08-28T02:28:54.971 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:28:55.124 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:28:55.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:55.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:28:55.229 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:28:55.259 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:55.638 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:28:55.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:55.641 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:28:55.708 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:28:55.742 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:28:55.840 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:56.127 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:28:56.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:56.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:28:56.154 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:28:56.154 INFO:teuthology.orchestra.run.smithi169.stdout: "ops": [], 2023-08-28T02:28:56.154 INFO:teuthology.orchestra.run.smithi169.stdout: "complaint_time": 30, 2023-08-28T02:28:56.154 INFO:teuthology.orchestra.run.smithi169.stdout: "num_blocked_ops": 0 2023-08-28T02:28:56.154 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:28:56.235 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:28:56.315 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:56.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:56 smithi143 ceph-mon[183059]: pgmap v542: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 14359/215406 objects degraded (6.666%); 10978/215406 objects misplaced (5.096%); 5.4 MiB/s, 85 objects/s recovering 2023-08-28T02:28:56.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:56 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:28:56.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:56 smithi143 ceph-mon[189871]: pgmap v542: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 14359/215406 objects degraded (6.666%); 10978/215406 objects misplaced (5.096%); 5.4 MiB/s, 85 objects/s recovering 2023-08-28T02:28:56.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:56 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:28:56.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:56 smithi169 ceph-mon[159415]: pgmap v542: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 14359/215406 objects degraded (6.666%); 10978/215406 objects misplaced (5.096%); 5.4 MiB/s, 85 objects/s recovering 2023-08-28T02:28:56.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:56 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:28:56.526 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:28:56.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:56.532 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:28:56.633 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:28:56.663 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:56.706 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:28:56.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:56.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:28:56.810 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:28:56.850 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:57.070 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:28:57.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:57.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:28:57.174 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:28:57.195 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.7 dump_historic_ops 2023-08-28T02:28:57.273 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:57.340 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:57 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 14359/215406 objects degraded (6.666%), 13 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:28:57.340 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:57 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/216972512' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:28:57.340 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:57 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:28:57.376 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:28:57.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:57.388 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is unset 2023-08-28T02:28:57.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:28:57.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:57 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 14359/215406 objects degraded (6.666%), 13 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:28:57.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:57 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/216972512' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:28:57.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:57 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:28:57.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:57 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 14359/215406 objects degraded (6.666%), 13 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:28:57.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:57 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/216972512' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:28:57.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:57 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:28:57.489 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:28:57.563 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:57.685 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:28:58.347 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:28:58.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:58.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:28:58.453 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:28:58.527 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:58.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:58.650 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:28:58.650 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:58 smithi143 ceph-mon[183059]: pgmap v543: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 14359/210624 objects degraded (6.817%); 10695/210624 objects misplaced (5.078%); 6.9 MiB/s, 111 objects/s recovering 2023-08-28T02:28:58.650 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:58 smithi143 ceph-mon[183059]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:28:58.650 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:58 smithi143 ceph-mon[183059]: failed to encode map e229 with expected crc 2023-08-28T02:28:58.650 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:58 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:28:58.650 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:58 smithi143 ceph-mon[183059]: osdmap e229: 8 total, 8 up, 8 in 2023-08-28T02:28:58.651 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:58 smithi143 ceph-mon[183059]: failed to encode map e229 with expected crc 2023-08-28T02:28:58.651 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:28:58.651 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:58 smithi143 ceph-mon[189871]: pgmap v543: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 14359/210624 objects degraded (6.817%); 10695/210624 objects misplaced (5.078%); 6.9 MiB/s, 111 objects/s recovering 2023-08-28T02:28:58.652 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:58 smithi143 ceph-mon[189871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:28:58.652 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:58 smithi143 ceph-mon[189871]: failed to encode map e229 with expected crc 2023-08-28T02:28:58.652 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:58 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:28:58.652 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:58 smithi143 ceph-mon[189871]: osdmap e229: 8 total, 8 up, 8 in 2023-08-28T02:28:58.652 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:58 smithi143 ceph-mon[189871]: failed to encode map e229 with expected crc 2023-08-28T02:28:58.719 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:28:58.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:58 smithi169 ceph-mon[159415]: pgmap v543: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 1 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 14359/210624 objects degraded (6.817%); 10695/210624 objects misplaced (5.078%); 6.9 MiB/s, 111 objects/s recovering 2023-08-28T02:28:58.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:58 smithi169 ceph-mon[159415]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:28:58.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:58 smithi169 ceph-mon[159415]: failed to encode map e229 with expected crc 2023-08-28T02:28:58.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:58 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:28:58.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:58 smithi169 ceph-mon[159415]: osdmap e229: 8 total, 8 up, 8 in 2023-08-28T02:28:58.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:58 smithi169 ceph-mon[159415]: failed to encode map e229 with expected crc 2023-08-28T02:28:58.741 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:28:58.752 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:28:58.764 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:28:58.781 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:58.786 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:28:58.806 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:28:58.826 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:28:58.826 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:28:58.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:58.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:28:58.843 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:28:58.930 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:28:58.955 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:28:58.970 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:58.970 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:28:58.970 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:28:58.970 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:28:58.970 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:28:58.970 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:28:58.971 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:28:58.971 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:28:58.971 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:28:59.187 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:28:59.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:59.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:28:59.312 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:28:59.343 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:59.573 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:59 smithi169 ceph-mon[159415]: failed to encode map e230 with expected crc 2023-08-28T02:28:59.573 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:59 smithi169 ceph-mon[159415]: osdmap e230: 8 total, 8 up, 8 in 2023-08-28T02:28:59.573 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:59 smithi169 ceph-mon[159415]: failed to encode map e230 with expected crc 2023-08-28T02:28:59.573 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:59 smithi169 ceph-mon[159415]: failed to encode map e230 with expected crc 2023-08-28T02:28:59.573 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:59 smithi169 ceph-mon[159415]: failed to encode map e230 with expected crc 2023-08-28T02:28:59.573 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:28:59 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2414139832' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:28:59.592 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:28:59.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:59.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:28:59.700 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:28:59.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:59 smithi143 ceph-mon[189871]: failed to encode map e230 with expected crc 2023-08-28T02:28:59.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:59 smithi143 ceph-mon[189871]: osdmap e230: 8 total, 8 up, 8 in 2023-08-28T02:28:59.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:59 smithi143 ceph-mon[189871]: failed to encode map e230 with expected crc 2023-08-28T02:28:59.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:59 smithi143 ceph-mon[189871]: failed to encode map e230 with expected crc 2023-08-28T02:28:59.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:59 smithi143 ceph-mon[189871]: failed to encode map e230 with expected crc 2023-08-28T02:28:59.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:28:59 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2414139832' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:28:59.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:59 smithi143 ceph-mon[183059]: failed to encode map e230 with expected crc 2023-08-28T02:28:59.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:59 smithi143 ceph-mon[183059]: osdmap e230: 8 total, 8 up, 8 in 2023-08-28T02:28:59.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:59 smithi143 ceph-mon[183059]: failed to encode map e230 with expected crc 2023-08-28T02:28:59.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:59 smithi143 ceph-mon[183059]: failed to encode map e230 with expected crc 2023-08-28T02:28:59.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:59 smithi143 ceph-mon[183059]: failed to encode map e230 with expected crc 2023-08-28T02:28:59.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:28:59 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2414139832' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:28:59.766 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:28:59.860 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:28:59.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:28:59.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:28:59.873 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:28:59.873 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:28:59.964 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:29:00.010 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:00.102 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:29:00.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:00.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:29:00.206 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:29:00.249 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:00.326 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:29:00.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:00.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:29:00.428 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set noscrub 2023-08-28T02:29:00.435 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:29:00.496 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:00 smithi143 ceph-mon[189871]: 6.2 scrub starts 2023-08-28T02:29:00.496 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:00 smithi143 ceph-mon[189871]: pgmap v546: 99 pgs: 1 active+recovery_wait+remapped, 2 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 897 op/s; 14309/204783 objects degraded (6.987%); 9941/204783 objects misplaced (4.854%); 8.7 MiB/s, 142 objects/s recovering 2023-08-28T02:29:00.496 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:00 smithi143 ceph-mon[189871]: failed to encode map e231 with expected crc 2023-08-28T02:29:00.496 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:00 smithi143 ceph-mon[189871]: failed to encode map e231 with expected crc 2023-08-28T02:29:00.496 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:00 smithi143 ceph-mon[189871]: osdmap e231: 8 total, 8 up, 8 in 2023-08-28T02:29:00.497 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:00 smithi143 ceph-mon[183059]: 6.2 scrub starts 2023-08-28T02:29:00.497 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:00 smithi143 ceph-mon[183059]: pgmap v546: 99 pgs: 1 active+recovery_wait+remapped, 2 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 897 op/s; 14309/204783 objects degraded (6.987%); 9941/204783 objects misplaced (4.854%); 8.7 MiB/s, 142 objects/s recovering 2023-08-28T02:29:00.497 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:00 smithi143 ceph-mon[183059]: failed to encode map e231 with expected crc 2023-08-28T02:29:00.497 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:00 smithi143 ceph-mon[183059]: failed to encode map e231 with expected crc 2023-08-28T02:29:00.497 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:00 smithi143 ceph-mon[183059]: osdmap e231: 8 total, 8 up, 8 in 2023-08-28T02:29:00.498 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:00.616 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:29:00.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:00.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:29:00.732 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:29:00.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:00 smithi169 ceph-mon[159415]: 6.2 scrub starts 2023-08-28T02:29:00.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:00 smithi169 ceph-mon[159415]: pgmap v546: 99 pgs: 1 active+recovery_wait+remapped, 2 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 897 op/s; 14309/204783 objects degraded (6.987%); 9941/204783 objects misplaced (4.854%); 8.7 MiB/s, 142 objects/s recovering 2023-08-28T02:29:00.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:00 smithi169 ceph-mon[159415]: failed to encode map e231 with expected crc 2023-08-28T02:29:00.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:00 smithi169 ceph-mon[159415]: failed to encode map e231 with expected crc 2023-08-28T02:29:00.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:00 smithi169 ceph-mon[159415]: osdmap e231: 8 total, 8 up, 8 in 2023-08-28T02:29:00.755 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 7, 3, 1, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-08-28T02:29:00.755 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-08-28T02:29:00.755 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-08-28T02:29:00.755 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-08-28T02:29:00.755 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-08-28T02:29:00.755 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-08-28T02:29:00.772 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:00.953 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:29:00.954 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.0 dump_ops_in_flight 2023-08-28T02:29:00.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:00.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:29:01.066 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:29:01.069 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:29:01.106 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:01.172 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:29:01.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:01.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:29:01.277 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:29:01.330 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:29:01.341 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:01.445 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:29:01.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:01.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:29:01.562 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:29:01.634 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:01 smithi143 ceph-mon[183059]: 6.2 scrub starts 2023-08-28T02:29:01.634 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:01 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 14309/204783 objects degraded (6.987%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:29:01.635 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:01 smithi143 ceph-mon[189871]: 6.2 scrub starts 2023-08-28T02:29:01.635 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:01 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 14309/204783 objects degraded (6.987%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:29:01.640 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:01.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:01 smithi169 ceph-mon[159415]: 6.2 scrub starts 2023-08-28T02:29:01.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:01 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 14309/204783 objects degraded (6.987%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:29:01.804 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:29:01.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:01.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:29:01.848 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:29:01.913 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:29:01.958 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:02.020 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:29:02.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:02.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:29:02.126 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:29:02.163 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:02.201 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:29:01 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:29:01] "GET /metrics HTTP/1.1" 200 36515 "" "Prometheus/2.33.4" 2023-08-28T02:29:02.235 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:29:02.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:02.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:29:02.339 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:29:02.364 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:02.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:02 smithi143 ceph-mon[183059]: pgmap v548: 99 pgs: 1 active+recovery_wait+remapped, 2 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 14309/204783 objects degraded (6.987%); 9941/204783 objects misplaced (4.854%); 6.2 MiB/s, 104 objects/s recovering 2023-08-28T02:29:02.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:02 smithi143 ceph-mon[189871]: pgmap v548: 99 pgs: 1 active+recovery_wait+remapped, 2 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 14309/204783 objects degraded (6.987%); 9941/204783 objects misplaced (4.854%); 6.2 MiB/s, 104 objects/s recovering 2023-08-28T02:29:02.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:02 smithi169 ceph-mon[159415]: pgmap v548: 99 pgs: 1 active+recovery_wait+remapped, 2 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 14309/204783 objects degraded (6.987%); 9941/204783 objects misplaced (4.854%); 6.2 MiB/s, 104 objects/s recovering 2023-08-28T02:29:02.744 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:29:02.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:02.754 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:29:02.833 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:29:02.855 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:29:02.938 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:03.081 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:29:03.136 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:29:03.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:03.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:29:03.241 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:29:03.324 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:03.490 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:29:03.491 INFO:teuthology.orchestra.run.smithi169.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-08-28T02:29:03.491 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:29:03.516 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:03 smithi169 ceph-mon[159415]: 6.2 scrub starts 2023-08-28T02:29:03.617 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:29:03.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:03.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:29:03.706 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:03 smithi143 ceph-mon[183059]: 6.2 scrub starts 2023-08-28T02:29:03.707 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:03 smithi143 ceph-mon[189871]: 6.2 scrub starts 2023-08-28T02:29:03.726 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:29:03.771 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:03.835 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:29:03.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:03.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:29:03.939 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:29:04.017 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:04.437 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is set 2023-08-28T02:29:04.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:04 smithi143 ceph-mon[189871]: pgmap v549: 99 pgs: 1 active+recovery_wait+remapped, 2 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 14309/200145 objects degraded (7.149%); 9610/200145 objects misplaced (4.802%); 6.7 MiB/s, 112 objects/s recovering 2023-08-28T02:29:04.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:04 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3387841287' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:29:04.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:04 smithi143 ceph-mon[183059]: pgmap v549: 99 pgs: 1 active+recovery_wait+remapped, 2 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 14309/200145 objects degraded (7.149%); 9610/200145 objects misplaced (4.802%); 6.7 MiB/s, 112 objects/s recovering 2023-08-28T02:29:04.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:04 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3387841287' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:29:04.743 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:04 smithi169 ceph-mon[159415]: pgmap v549: 99 pgs: 1 active+recovery_wait+remapped, 2 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 14309/200145 objects degraded (7.149%); 9610/200145 objects misplaced (4.802%); 6.7 MiB/s, 112 objects/s recovering 2023-08-28T02:29:04.743 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:04 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3387841287' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:29:05.377 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:29:05.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:05.383 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:29:05.483 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:29:05.553 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:05.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:05 smithi143 ceph-mon[189871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:29:05.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:05 smithi143 ceph-mon[189871]: failed to encode map e232 with expected crc 2023-08-28T02:29:05.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:05 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3387841287' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:29:05.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:05 smithi143 ceph-mon[189871]: osdmap e232: 8 total, 8 up, 8 in 2023-08-28T02:29:05.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:05 smithi143 ceph-mon[189871]: failed to encode map e232 with expected crc 2023-08-28T02:29:05.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:05 smithi143 ceph-mon[189871]: failed to encode map e232 with expected crc 2023-08-28T02:29:05.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:05 smithi143 ceph-mon[189871]: failed to encode map e232 with expected crc 2023-08-28T02:29:05.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:05 smithi143 ceph-mon[183059]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:29:05.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:05 smithi143 ceph-mon[183059]: failed to encode map e232 with expected crc 2023-08-28T02:29:05.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:05 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3387841287' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:29:05.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:05 smithi143 ceph-mon[183059]: osdmap e232: 8 total, 8 up, 8 in 2023-08-28T02:29:05.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:05 smithi143 ceph-mon[183059]: failed to encode map e232 with expected crc 2023-08-28T02:29:05.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:05 smithi143 ceph-mon[183059]: failed to encode map e232 with expected crc 2023-08-28T02:29:05.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:05 smithi143 ceph-mon[183059]: failed to encode map e232 with expected crc 2023-08-28T02:29:05.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:05 smithi169 ceph-mon[159415]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:29:05.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:05 smithi169 ceph-mon[159415]: failed to encode map e232 with expected crc 2023-08-28T02:29:05.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:05 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3387841287' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:29:05.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:05 smithi169 ceph-mon[159415]: osdmap e232: 8 total, 8 up, 8 in 2023-08-28T02:29:05.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:05 smithi169 ceph-mon[159415]: failed to encode map e232 with expected crc 2023-08-28T02:29:05.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:05 smithi169 ceph-mon[159415]: failed to encode map e232 with expected crc 2023-08-28T02:29:05.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:05 smithi169 ceph-mon[159415]: failed to encode map e232 with expected crc 2023-08-28T02:29:05.749 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:29:05.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:05.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:29:05.856 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:29:05.898 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:05.955 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:29:05.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:05.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:29:06.060 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:29:06.144 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:06.204 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:29:06.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:06.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:29:06.309 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:29:06.331 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:29:06.380 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:29:06.404 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:06.429 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:29:06.466 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:29:06.519 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:29:06.559 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:29:06.564 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:29:06.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:06.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:29:06.580 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:06 smithi143 ceph-mon[183059]: pgmap v551: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 815 op/s; 14309/193752 objects degraded (7.385%); 9140/193752 objects misplaced (4.717%); 5.9 MiB/s, 97 objects/s recovering 2023-08-28T02:29:06.581 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:06 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/146389160' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:29:06.581 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:06 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 14309/193752 objects degraded (7.385%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:29:06.581 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:06 smithi143 ceph-mon[189871]: pgmap v551: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 815 op/s; 14309/193752 objects degraded (7.385%); 9140/193752 objects misplaced (4.717%); 5.9 MiB/s, 97 objects/s recovering 2023-08-28T02:29:06.581 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:06 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/146389160' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:29:06.581 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:06 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 14309/193752 objects degraded (7.385%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:29:06.599 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:29:06.621 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:29:06.621 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [ 2023-08-28T02:29:06.621 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:06.621 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:105093 6.a 6:5782af69:::benchmark_data_smithi143_305742_object21474:head [delete] snapc 0=[] ondisk+write+known_if_redirected e232)", 2023-08-28T02:29:06.621 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:29:06.613689+0000", 2023-08-28T02:29:06.621 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.006229409, 2023-08-28T02:29:06.621 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0062621250000000003, 2023-08-28T02:29:06.621 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:29:06.621 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "waiting for sub ops", 2023-08-28T02:29:06.622 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:29:06.622 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:29:06.622 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:29:06.622 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 105093 2023-08-28T02:29:06.622 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:06.622 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:29:06.622 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:06.622 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:29:06.622 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:06.613689+0000", 2023-08-28T02:29:06.622 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:29:06.622 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:06.623 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:06.623 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:29:06.623 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:06.613689+0000", 2023-08-28T02:29:06.623 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:29:06.623 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:06.623 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:06.623 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:29:06.623 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:06.613687+0000", 2023-08-28T02:29:06.623 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:29:06.623 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:06.623 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:06.623 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:29:06.623 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:06.613695+0000", 2023-08-28T02:29:06.624 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.7039999999999994e-06 2023-08-28T02:29:06.624 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:06.624 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:06.624 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:29:06.624 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:06.613714+0000", 2023-08-28T02:29:06.624 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.8961e-05 2023-08-28T02:29:06.624 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:06.624 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:06.624 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:29:06.624 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:06.613724+0000", 2023-08-28T02:29:06.624 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0637e-05 2023-08-28T02:29:06.624 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:06.625 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:06.625 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:29:06.625 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:06.613785+0000", 2023-08-28T02:29:06.625 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.0523000000000003e-05 2023-08-28T02:29:06.625 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:06.625 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:06.625 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:29:06.625 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:06.613966+0000", 2023-08-28T02:29:06.625 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000181688 2023-08-28T02:29:06.625 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:06.625 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:06.625 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 2,5,6", 2023-08-28T02:29:06.625 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:06.614149+0000", 2023-08-28T02:29:06.626 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000182533 2023-08-28T02:29:06.626 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:29:06.626 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:29:06.626 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:29:06.626 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:06.626 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:06.626 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:105096 6.a 6:51bc389b:::benchmark_data_smithi143_305742_object21477:head [delete] snapc 0=[] ondisk+write+known_if_redirected e232)", 2023-08-28T02:29:06.626 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:29:06.613744+0000", 2023-08-28T02:29:06.626 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.0061734440000000002, 2023-08-28T02:29:06.626 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0062947910000000001, 2023-08-28T02:29:06.626 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:29:06.626 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "waiting for sub ops", 2023-08-28T02:29:06.626 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:29:06.627 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:29:06.627 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:29:06.627 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 105096 2023-08-28T02:29:06.627 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:06.627 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:29:06.627 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:06.627 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:29:06.627 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:06.613744+0000", 2023-08-28T02:29:06.627 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:29:06.627 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:06.627 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:06.627 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:29:06.628 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:06.613744+0000", 2023-08-28T02:29:06.628 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:29:06.628 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:06.628 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:06.628 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:29:06.628 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:06.613744+0000", 2023-08-28T02:29:06.628 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:29:06.628 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:06.628 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:06.628 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:29:06.628 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:06.613747+0000", 2023-08-28T02:29:06.628 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.2329999999999998e-06 2023-08-28T02:29:06.629 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:06.629 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:06.629 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:29:06.629 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:06.613768+0000", 2023-08-28T02:29:06.629 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.0432e-05 2023-08-28T02:29:06.629 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:06.629 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:06.629 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:29:06.629 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:06.613773+0000", 2023-08-28T02:29:06.629 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.6339999999999998e-06 2023-08-28T02:29:06.629 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:06.629 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:06.629 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:29:06.630 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:06.617863+0000", 2023-08-28T02:29:06.630 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0040897440000000002 2023-08-28T02:29:06.630 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:06.630 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:06.630 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:29:06.630 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:06.618034+0000", 2023-08-28T02:29:06.630 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00017136399999999999 2023-08-28T02:29:06.630 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:06.630 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:06.630 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 2,5,6", 2023-08-28T02:29:06.630 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:06.618181+0000", 2023-08-28T02:29:06.630 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00014669600000000001 2023-08-28T02:29:06.631 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:29:06.631 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:29:06.631 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:29:06.631 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:06.631 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:06.631 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop_reply(client.35076.0:105093 6.a e232/221)", 2023-08-28T02:29:06.631 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:29:06.617862+0000", 2023-08-28T02:29:06.631 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.0020563819999999998, 2023-08-28T02:29:06.631 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0022462879999999999, 2023-08-28T02:29:06.631 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:29:06.631 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "queued for pg", 2023-08-28T02:29:06.631 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:29:06.632 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:06.632 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:29:06.632 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:06.617862+0000", 2023-08-28T02:29:06.632 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:29:06.632 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:06.632 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:06.632 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:29:06.632 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:06.617862+0000", 2023-08-28T02:29:06.632 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:29:06.632 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:06.632 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:06.632 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:29:06.632 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:06.617861+0000", 2023-08-28T02:29:06.633 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:29:06.633 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:06.633 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:06.633 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:29:06.633 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:06.617866+0000", 2023-08-28T02:29:06.633 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.8350000000000003e-06 2023-08-28T02:29:06.633 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:06.633 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:06.633 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:29:06.633 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:06.617875+0000", 2023-08-28T02:29:06.633 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.5929999999999995e-06 2023-08-28T02:29:06.633 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:06.634 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:06.634 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:29:06.634 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:06.617882+0000", 2023-08-28T02:29:06.634 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.7220000000000002e-06 2023-08-28T02:29:06.634 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:29:06.634 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:29:06.634 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:29:06.634 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:29:06.634 INFO:teuthology.orchestra.run.smithi143.stdout: ], 2023-08-28T02:29:06.634 INFO:teuthology.orchestra.run.smithi143.stdout: "num_ops": 3 2023-08-28T02:29:06.634 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:29:06.640 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:29:06.646 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:29:06.646 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:29:06.646 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:29:06.646 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:29:06.647 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:29:06.647 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:29:06.647 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:29:06.647 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:29:06.671 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:29:06.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:06 smithi169 ceph-mon[159415]: pgmap v551: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 815 op/s; 14309/193752 objects degraded (7.385%); 9140/193752 objects misplaced (4.717%); 5.9 MiB/s, 97 objects/s recovering 2023-08-28T02:29:06.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:06 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/146389160' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:29:06.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:06 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 14309/193752 objects degraded (7.385%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:29:06.745 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:07.736 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set nodeep-scrub 2023-08-28T02:29:07.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:07 smithi169 ceph-mon[159415]: pgmap v552: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 14309/189072 objects degraded (7.568%); 8774/189072 objects misplaced (4.641%); 5.4 MiB/s, 87 objects/s recovering 2023-08-28T02:29:07.761 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:07 smithi143 ceph-mon[183059]: pgmap v552: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 14309/189072 objects degraded (7.568%); 8774/189072 objects misplaced (4.641%); 5.4 MiB/s, 87 objects/s recovering 2023-08-28T02:29:07.761 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:07 smithi143 ceph-mon[189871]: pgmap v552: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 14309/189072 objects degraded (7.568%); 8774/189072 objects misplaced (4.641%); 5.4 MiB/s, 87 objects/s recovering 2023-08-28T02:29:08.044 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:29:08.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:08.055 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:29:08.153 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.0 dump_blocked_ops 2023-08-28T02:29:08.155 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:29:08.242 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:08.430 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:29:08.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:08.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:29:08.499 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:29:08.537 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:29:08.573 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:08.636 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:29:08.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:08.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:29:08.743 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:29:08.777 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:08.836 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:29:08.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:08.842 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:29:08.943 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:29:08.996 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:09.068 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:29:09.224 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:29:09.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:09.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:29:09.331 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:29:09.360 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:09.408 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:29:09.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:09.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:29:09.513 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:29:09.526 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 7, 3, 1, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-08-28T02:29:09.526 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd reweight 0 1 2023-08-28T02:29:09.540 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:09.599 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:29:09.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:09.604 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:29:09.705 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:29:09.775 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:10.134 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:29:10.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:10.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:29:10.226 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:29:10.259 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:29:10.319 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:10.549 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:29:10.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:10.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:29:10.568 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:10 smithi143 ceph-mon[189871]: pgmap v553: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 14309/184359 objects degraded (7.761%); 8330/184359 objects misplaced (4.518%); 4.4 MiB/s, 70 objects/s recovering 2023-08-28T02:29:10.569 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:10 smithi143 ceph-mon[183059]: pgmap v553: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 14309/184359 objects degraded (7.761%); 8330/184359 objects misplaced (4.518%); 4.4 MiB/s, 70 objects/s recovering 2023-08-28T02:29:10.666 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:29:10.699 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:10 smithi169 ceph-mon[159415]: pgmap v553: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 14309/184359 objects degraded (7.761%); 8330/184359 objects misplaced (4.518%); 4.4 MiB/s, 70 objects/s recovering 2023-08-28T02:29:10.701 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:10.760 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:29:10.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:10.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:29:10.869 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:29:10.898 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:11.344 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is set 2023-08-28T02:29:11.406 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:29:11.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:11.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:29:11.511 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:29:11.553 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:11.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:11 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:29:11.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:11 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2782506654' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:29:11.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:11 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:29:11.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:11 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2782506654' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:29:11.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:11 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:29:11.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:11 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2782506654' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:29:11.998 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:29:12.024 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:29:12.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:12.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:29:12.129 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:29:12.170 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:29:11 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:29:11] "GET /metrics HTTP/1.1" 200 36500 "" "Prometheus/2.33.4" 2023-08-28T02:29:12.173 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:12.281 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:29:12.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:12.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:29:12.388 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:29:12.421 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:12.435 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:29:12.439 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:12 smithi143 ceph-mon[183059]: pgmap v554: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 724 op/s; 14286/183021 objects degraded (7.806%); 8330/183021 objects misplaced (4.551%); 4.4 MiB/s, 71 objects/s recovering 2023-08-28T02:29:12.439 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:12 smithi143 ceph-mon[183059]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:29:12.439 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:12 smithi143 ceph-mon[183059]: failed to encode map e233 with expected crc 2023-08-28T02:29:12.439 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:12 smithi143 ceph-mon[183059]: failed to encode map e233 with expected crc 2023-08-28T02:29:12.439 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:12 smithi143 ceph-mon[183059]: failed to encode map e233 with expected crc 2023-08-28T02:29:12.439 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:12 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2782506654' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:29:12.439 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:12 smithi143 ceph-mon[183059]: osdmap e233: 8 total, 8 up, 8 in 2023-08-28T02:29:12.439 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:12 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 14286/183021 objects degraded (7.806%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:29:12.439 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:12 smithi143 ceph-mon[183059]: failed to encode map e233 with expected crc 2023-08-28T02:29:12.442 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:12 smithi143 ceph-mon[189871]: pgmap v554: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 724 op/s; 14286/183021 objects degraded (7.806%); 8330/183021 objects misplaced (4.551%); 4.4 MiB/s, 71 objects/s recovering 2023-08-28T02:29:12.442 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:12 smithi143 ceph-mon[189871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:29:12.443 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:12 smithi143 ceph-mon[189871]: failed to encode map e233 with expected crc 2023-08-28T02:29:12.443 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:12 smithi143 ceph-mon[189871]: failed to encode map e233 with expected crc 2023-08-28T02:29:12.443 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:12 smithi143 ceph-mon[189871]: failed to encode map e233 with expected crc 2023-08-28T02:29:12.443 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:12 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2782506654' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:29:12.443 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:12 smithi143 ceph-mon[189871]: osdmap e233: 8 total, 8 up, 8 in 2023-08-28T02:29:12.443 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:12 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 14286/183021 objects degraded (7.806%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:29:12.443 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:12 smithi143 ceph-mon[189871]: failed to encode map e233 with expected crc 2023-08-28T02:29:12.469 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:29:12.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:12.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:29:12.507 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:29:12.507 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [], 2023-08-28T02:29:12.507 INFO:teuthology.orchestra.run.smithi143.stdout: "complaint_time": 30, 2023-08-28T02:29:12.507 INFO:teuthology.orchestra.run.smithi143.stdout: "num_blocked_ops": 0 2023-08-28T02:29:12.507 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:29:12.576 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:29:12.602 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:12 smithi169 ceph-mon[159415]: pgmap v554: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 724 op/s; 14286/183021 objects degraded (7.806%); 8330/183021 objects misplaced (4.551%); 4.4 MiB/s, 71 objects/s recovering 2023-08-28T02:29:12.602 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:12 smithi169 ceph-mon[159415]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:29:12.602 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:12 smithi169 ceph-mon[159415]: failed to encode map e233 with expected crc 2023-08-28T02:29:12.603 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:12 smithi169 ceph-mon[159415]: failed to encode map e233 with expected crc 2023-08-28T02:29:12.603 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:12 smithi169 ceph-mon[159415]: failed to encode map e233 with expected crc 2023-08-28T02:29:12.603 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:12 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2782506654' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:29:12.603 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:12 smithi169 ceph-mon[159415]: osdmap e233: 8 total, 8 up, 8 in 2023-08-28T02:29:12.603 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:12 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 14286/183021 objects degraded (7.806%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:29:12.603 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:12 smithi169 ceph-mon[159415]: failed to encode map e233 with expected crc 2023-08-28T02:29:12.605 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:12.653 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:29:12.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:12.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:29:12.758 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:29:12.806 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:14.037 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset noscrub 2023-08-28T02:29:14.689 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:29:14.692 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:14 smithi143 ceph-mon[183059]: pgmap v556: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 14286/179331 objects degraded (7.966%); 7767/179331 objects misplaced (4.331%); 6.7 MiB/s, 106 objects/s recovering 2023-08-28T02:29:14.692 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:14 smithi143 ceph-mon[189871]: pgmap v556: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 14286/179331 objects degraded (7.966%); 7767/179331 objects misplaced (4.331%); 6.7 MiB/s, 106 objects/s recovering 2023-08-28T02:29:14.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:14.693 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:29:14.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:14 smithi169 ceph-mon[159415]: pgmap v556: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 14286/179331 objects degraded (7.966%); 7767/179331 objects misplaced (4.331%); 6.7 MiB/s, 106 objects/s recovering 2023-08-28T02:29:14.794 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:29:14.813 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.0 dump_historic_ops 2023-08-28T02:29:14.865 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:15.012 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:29:15.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:15.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:29:15.120 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:29:15.227 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:15.316 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:29:15.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:15.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:29:15.432 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:29:15.448 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:29:15.552 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:15.609 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:29:15.648 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:29:15.664 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:29:15.686 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:29:15.726 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:29:15.764 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:29:15.765 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:29:15.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:15.773 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:29:15.803 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:29:15.833 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:29:15.865 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:29:15.871 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:29:15.871 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:29:15.871 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:29:15.871 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:29:15.871 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:29:15.871 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:29:15.871 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:29:15.872 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:29:15.874 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:29:15.916 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:16.446 INFO:teuthology.orchestra.run.smithi143.stderr:reweighted osd.0 to 1 (10000) 2023-08-28T02:29:16.661 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:29:16.663 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:16 smithi143 ceph-mon[183059]: pgmap v557: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 14096/175521 objects degraded (8.031%); 7471/175521 objects misplaced (4.256%); 6.7 MiB/s, 107 objects/s recovering 2023-08-28T02:29:16.663 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:16 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/4242943027' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:29:16.663 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:16 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2096499163' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-08-28T02:29:16.663 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:16 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-08-28T02:29:16.664 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:16 smithi143 ceph-mon[189871]: pgmap v557: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 14096/175521 objects degraded (8.031%); 7471/175521 objects misplaced (4.256%); 6.7 MiB/s, 107 objects/s recovering 2023-08-28T02:29:16.664 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:16 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/4242943027' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:29:16.664 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:16 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2096499163' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-08-28T02:29:16.664 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:16 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-08-28T02:29:16.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:16.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:29:16.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:16 smithi169 ceph-mon[159415]: pgmap v557: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 14096/175521 objects degraded (8.031%); 7471/175521 objects misplaced (4.256%); 6.7 MiB/s, 107 objects/s recovering 2023-08-28T02:29:16.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:16 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/4242943027' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:29:16.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:16 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2096499163' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-08-28T02:29:16.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:16 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-08-28T02:29:16.766 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:29:16.793 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:16.837 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:29:16.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:16.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:29:16.943 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:29:16.972 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:17.016 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:29:17.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:17.023 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:29:17.124 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:29:17.153 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:17.197 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:29:17.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:17.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:29:17.306 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:29:17.333 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:17.394 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:29:17.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:17.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:29:17.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:17 smithi169 ceph-mon[159415]: failed to encode map e234 with expected crc 2023-08-28T02:29:17.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:17 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-08-28T02:29:17.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:17 smithi169 ceph-mon[159415]: osdmap e234: 8 total, 8 up, 8 in 2023-08-28T02:29:17.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:17 smithi169 ceph-mon[159415]: failed to encode map e234 with expected crc 2023-08-28T02:29:17.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:17 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 14096/175521 objects degraded (8.031%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:29:17.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:17 smithi169 ceph-mon[159415]: failed to encode map e234 with expected crc 2023-08-28T02:29:17.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:17 smithi169 ceph-mon[159415]: failed to encode map e234 with expected crc 2023-08-28T02:29:17.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:17 smithi169 ceph-mon[159415]: failed to encode map e234 with expected crc 2023-08-28T02:29:17.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:17 smithi169 ceph-mon[159415]: failed to encode map e234 with expected crc 2023-08-28T02:29:17.501 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:29:17.536 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:17.604 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:29:17.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:17.613 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:29:17.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:17 smithi143 ceph-mon[183059]: failed to encode map e234 with expected crc 2023-08-28T02:29:17.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:17 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-08-28T02:29:17.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:17 smithi143 ceph-mon[183059]: osdmap e234: 8 total, 8 up, 8 in 2023-08-28T02:29:17.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:17 smithi143 ceph-mon[183059]: failed to encode map e234 with expected crc 2023-08-28T02:29:17.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:17 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 14096/175521 objects degraded (8.031%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:29:17.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:17 smithi143 ceph-mon[183059]: failed to encode map e234 with expected crc 2023-08-28T02:29:17.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:17 smithi143 ceph-mon[183059]: failed to encode map e234 with expected crc 2023-08-28T02:29:17.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:17 smithi143 ceph-mon[183059]: failed to encode map e234 with expected crc 2023-08-28T02:29:17.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:17 smithi143 ceph-mon[183059]: failed to encode map e234 with expected crc 2023-08-28T02:29:17.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:17 smithi143 ceph-mon[189871]: failed to encode map e234 with expected crc 2023-08-28T02:29:17.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:17 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-08-28T02:29:17.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:17 smithi143 ceph-mon[189871]: osdmap e234: 8 total, 8 up, 8 in 2023-08-28T02:29:17.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:17 smithi143 ceph-mon[189871]: failed to encode map e234 with expected crc 2023-08-28T02:29:17.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:17 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 14096/175521 objects degraded (8.031%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:29:17.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:17 smithi143 ceph-mon[189871]: failed to encode map e234 with expected crc 2023-08-28T02:29:17.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:17 smithi143 ceph-mon[189871]: failed to encode map e234 with expected crc 2023-08-28T02:29:17.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:17 smithi143 ceph-mon[189871]: failed to encode map e234 with expected crc 2023-08-28T02:29:17.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:17 smithi143 ceph-mon[189871]: failed to encode map e234 with expected crc 2023-08-28T02:29:17.714 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:29:17.767 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:18.231 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd reweight 2 1 2023-08-28T02:29:18.308 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:29:18.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:18.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:29:18.416 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:29:18.443 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:18.489 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:29:18.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:18.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:29:18.513 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:18 smithi143 ceph-mon[189871]: pgmap v559: 99 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 466 op/s; 12905/173166 objects degraded (7.452%); 7914/173166 objects misplaced (4.570%); 7.8 MiB/s, 182 objects/s recovering 2023-08-28T02:29:18.513 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:18 smithi143 ceph-mon[189871]: failed to encode map e235 with expected crc 2023-08-28T02:29:18.513 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:18 smithi143 ceph-mon[189871]: osdmap e235: 8 total, 8 up, 8 in 2023-08-28T02:29:18.513 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:18 smithi143 ceph-mon[189871]: failed to encode map e235 with expected crc 2023-08-28T02:29:18.513 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:18 smithi143 ceph-mon[189871]: failed to encode map e235 with expected crc 2023-08-28T02:29:18.513 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:18 smithi143 ceph-mon[189871]: failed to encode map e235 with expected crc 2023-08-28T02:29:18.513 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:18 smithi143 ceph-mon[189871]: failed to encode map e235 with expected crc 2023-08-28T02:29:18.513 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:18 smithi143 ceph-mon[189871]: failed to encode map e235 with expected crc 2023-08-28T02:29:18.513 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:18 smithi143 ceph-mon[183059]: pgmap v559: 99 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 466 op/s; 12905/173166 objects degraded (7.452%); 7914/173166 objects misplaced (4.570%); 7.8 MiB/s, 182 objects/s recovering 2023-08-28T02:29:18.514 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:18 smithi143 ceph-mon[183059]: failed to encode map e235 with expected crc 2023-08-28T02:29:18.514 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:18 smithi143 ceph-mon[183059]: osdmap e235: 8 total, 8 up, 8 in 2023-08-28T02:29:18.514 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:18 smithi143 ceph-mon[183059]: failed to encode map e235 with expected crc 2023-08-28T02:29:18.514 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:18 smithi143 ceph-mon[183059]: failed to encode map e235 with expected crc 2023-08-28T02:29:18.514 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:18 smithi143 ceph-mon[183059]: failed to encode map e235 with expected crc 2023-08-28T02:29:18.514 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:18 smithi143 ceph-mon[183059]: failed to encode map e235 with expected crc 2023-08-28T02:29:18.514 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:18 smithi143 ceph-mon[183059]: failed to encode map e235 with expected crc 2023-08-28T02:29:18.595 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:29:18.620 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:18 smithi169 ceph-mon[159415]: pgmap v559: 99 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 466 op/s; 12905/173166 objects degraded (7.452%); 7914/173166 objects misplaced (4.570%); 7.8 MiB/s, 182 objects/s recovering 2023-08-28T02:29:18.620 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:18 smithi169 ceph-mon[159415]: failed to encode map e235 with expected crc 2023-08-28T02:29:18.620 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:18 smithi169 ceph-mon[159415]: osdmap e235: 8 total, 8 up, 8 in 2023-08-28T02:29:18.620 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:18 smithi169 ceph-mon[159415]: failed to encode map e235 with expected crc 2023-08-28T02:29:18.620 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:18 smithi169 ceph-mon[159415]: failed to encode map e235 with expected crc 2023-08-28T02:29:18.620 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:18 smithi169 ceph-mon[159415]: failed to encode map e235 with expected crc 2023-08-28T02:29:18.620 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:18 smithi169 ceph-mon[159415]: failed to encode map e235 with expected crc 2023-08-28T02:29:18.621 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:18 smithi169 ceph-mon[159415]: failed to encode map e235 with expected crc 2023-08-28T02:29:18.621 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:18.674 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:29:18.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:18.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:29:18.783 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:29:18.811 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:18.863 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:29:18.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:18.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:29:18.968 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:29:18.995 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:19.029 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:29:19.042 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:29:19.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:19.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:29:19.147 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:29:19.202 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:19.290 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:29:19.290 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:29:19.469 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:29:19.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:19.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:29:19.579 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:29:19.618 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:20.131 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:29:20.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:20.135 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:29:20.236 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:29:20.263 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:20.264 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:29:20.309 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:29:20.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:20.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:29:20.390 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is unset 2023-08-28T02:29:20.413 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:20 smithi143 ceph-mon[189871]: pgmap v561: 99 pgs: 1 peering, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 511 op/s; 12905/170838 objects degraded (7.554%); 7684/170838 objects misplaced (4.498%); 7.7 MiB/s, 181 objects/s recovering 2023-08-28T02:29:20.413 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:20 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3795027449' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:29:20.414 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:20 smithi143 ceph-mon[183059]: pgmap v561: 99 pgs: 1 peering, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 511 op/s; 12905/170838 objects degraded (7.554%); 7684/170838 objects misplaced (4.498%); 7.7 MiB/s, 181 objects/s recovering 2023-08-28T02:29:20.414 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:20 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3795027449' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:29:20.416 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:29:20.485 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:20.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:20 smithi169 ceph-mon[159415]: pgmap v561: 99 pgs: 1 peering, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 511 op/s; 12905/170838 objects degraded (7.554%); 7684/170838 objects misplaced (4.498%); 7.7 MiB/s, 181 objects/s recovering 2023-08-28T02:29:20.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:20 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3795027449' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:29:21.272 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.1 dump_ops_in_flight 2023-08-28T02:29:21.559 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:21 smithi143 ceph-mon[189871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:29:21.559 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:21 smithi143 ceph-mon[189871]: failed to encode map e236 with expected crc 2023-08-28T02:29:21.560 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:21 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3795027449' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:29:21.560 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:21 smithi143 ceph-mon[189871]: osdmap e236: 8 total, 8 up, 8 in 2023-08-28T02:29:21.560 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:21 smithi143 ceph-mon[189871]: failed to encode map e236 with expected crc 2023-08-28T02:29:21.560 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:21 smithi143 ceph-mon[189871]: failed to encode map e236 with expected crc 2023-08-28T02:29:21.560 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:21 smithi143 ceph-mon[189871]: failed to encode map e236 with expected crc 2023-08-28T02:29:21.560 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:21 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 12905/170838 objects degraded (7.554%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-08-28T02:29:21.560 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:21 smithi143 ceph-mon[183059]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:29:21.560 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:21 smithi143 ceph-mon[183059]: failed to encode map e236 with expected crc 2023-08-28T02:29:21.560 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:21 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3795027449' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:29:21.560 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:21 smithi143 ceph-mon[183059]: osdmap e236: 8 total, 8 up, 8 in 2023-08-28T02:29:21.561 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:21 smithi143 ceph-mon[183059]: failed to encode map e236 with expected crc 2023-08-28T02:29:21.561 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:21 smithi143 ceph-mon[183059]: failed to encode map e236 with expected crc 2023-08-28T02:29:21.561 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:21 smithi143 ceph-mon[183059]: failed to encode map e236 with expected crc 2023-08-28T02:29:21.561 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:21 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 12905/170838 objects degraded (7.554%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-08-28T02:29:21.621 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:29:21.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:21.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:29:21.691 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:29:21.729 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:29:21.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:21 smithi169 ceph-mon[159415]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:29:21.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:21 smithi169 ceph-mon[159415]: failed to encode map e236 with expected crc 2023-08-28T02:29:21.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:21 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3795027449' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:29:21.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:21 smithi169 ceph-mon[159415]: osdmap e236: 8 total, 8 up, 8 in 2023-08-28T02:29:21.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:21 smithi169 ceph-mon[159415]: failed to encode map e236 with expected crc 2023-08-28T02:29:21.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:21 smithi169 ceph-mon[159415]: failed to encode map e236 with expected crc 2023-08-28T02:29:21.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:21 smithi169 ceph-mon[159415]: failed to encode map e236 with expected crc 2023-08-28T02:29:21.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:21 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 12905/170838 objects degraded (7.554%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-08-28T02:29:21.795 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:21.916 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:29:21.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:21.920 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:29:22.020 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:29:22.096 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:22.201 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:29:21 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:29:21] "GET /metrics HTTP/1.1" 200 36500 "" "Prometheus/2.33.4" 2023-08-28T02:29:22.288 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:29:22.289 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:29:22.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:22.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:29:22.404 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:29:22.466 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:22.552 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:29:22.554 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:22 smithi143 ceph-mon[183059]: pgmap v563: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 303 op/s; 10745/170058 objects degraded (6.318%); 9594/170058 objects misplaced (5.642%); 4.3 MiB/s, 145 objects/s recovering 2023-08-28T02:29:22.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:22.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:29:22.557 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:22 smithi143 ceph-mon[189871]: pgmap v563: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 303 op/s; 10745/170058 objects degraded (6.318%); 9594/170058 objects misplaced (5.642%); 4.3 MiB/s, 145 objects/s recovering 2023-08-28T02:29:22.656 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:29:22.695 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:22.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:22 smithi169 ceph-mon[159415]: pgmap v563: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 303 op/s; 10745/170058 objects degraded (6.318%); 9594/170058 objects misplaced (5.642%); 4.3 MiB/s, 145 objects/s recovering 2023-08-28T02:29:22.763 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:29:22.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:22.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:29:22.867 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:29:22.907 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:23.083 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:29:23.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:23.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:29:23.191 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:29:23.242 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:23.309 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:29:23.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:23.313 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:29:23.414 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:29:23.426 INFO:teuthology.orchestra.run.smithi143.stderr:reweighted osd.2 to 1 (10000) 2023-08-28T02:29:23.465 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:23.587 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset nodeep-scrub 2023-08-28T02:29:23.615 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:23 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/582655291' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-08-28T02:29:23.615 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:23 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-08-28T02:29:23.616 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:23 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/582655291' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-08-28T02:29:23.616 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:23 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-08-28T02:29:23.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:23 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/582655291' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-08-28T02:29:23.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:23 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-08-28T02:29:23.793 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:29:23.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:23.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:29:23.900 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:29:23.925 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:24.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:24 smithi143 ceph-mon[183059]: pgmap v564: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 378 op/s; 10745/167658 objects degraded (6.409%); 9082/167658 objects misplaced (5.417%); 8.3 MiB/s, 200 objects/s recovering 2023-08-28T02:29:24.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:24 smithi143 ceph-mon[183059]: failed to encode map e237 with expected crc 2023-08-28T02:29:24.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:24 smithi143 ceph-mon[183059]: failed to encode map e237 with expected crc 2023-08-28T02:29:24.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:24 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-08-28T02:29:24.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:24 smithi143 ceph-mon[183059]: osdmap e237: 8 total, 8 up, 8 in 2023-08-28T02:29:24.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:24 smithi143 ceph-mon[183059]: failed to encode map e237 with expected crc 2023-08-28T02:29:24.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:24 smithi143 ceph-mon[183059]: failed to encode map e237 with expected crc 2023-08-28T02:29:24.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:24 smithi143 ceph-mon[183059]: failed to encode map e237 with expected crc 2023-08-28T02:29:24.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:24 smithi143 ceph-mon[189871]: pgmap v564: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 378 op/s; 10745/167658 objects degraded (6.409%); 9082/167658 objects misplaced (5.417%); 8.3 MiB/s, 200 objects/s recovering 2023-08-28T02:29:24.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:24 smithi143 ceph-mon[189871]: failed to encode map e237 with expected crc 2023-08-28T02:29:24.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:24 smithi143 ceph-mon[189871]: failed to encode map e237 with expected crc 2023-08-28T02:29:24.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:24 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-08-28T02:29:24.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:24 smithi143 ceph-mon[189871]: osdmap e237: 8 total, 8 up, 8 in 2023-08-28T02:29:24.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:24 smithi143 ceph-mon[189871]: failed to encode map e237 with expected crc 2023-08-28T02:29:24.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:24 smithi143 ceph-mon[189871]: failed to encode map e237 with expected crc 2023-08-28T02:29:24.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:24 smithi143 ceph-mon[189871]: failed to encode map e237 with expected crc 2023-08-28T02:29:24.708 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:29:24.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:24.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:29:24.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:24 smithi169 ceph-mon[159415]: pgmap v564: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 378 op/s; 10745/167658 objects degraded (6.409%); 9082/167658 objects misplaced (5.417%); 8.3 MiB/s, 200 objects/s recovering 2023-08-28T02:29:24.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:24 smithi169 ceph-mon[159415]: failed to encode map e237 with expected crc 2023-08-28T02:29:24.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:24 smithi169 ceph-mon[159415]: failed to encode map e237 with expected crc 2023-08-28T02:29:24.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:24 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-08-28T02:29:24.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:24 smithi169 ceph-mon[159415]: osdmap e237: 8 total, 8 up, 8 in 2023-08-28T02:29:24.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:24 smithi169 ceph-mon[159415]: failed to encode map e237 with expected crc 2023-08-28T02:29:24.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:24 smithi169 ceph-mon[159415]: failed to encode map e237 with expected crc 2023-08-28T02:29:24.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:24 smithi169 ceph-mon[159415]: failed to encode map e237 with expected crc 2023-08-28T02:29:24.812 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:29:24.869 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:24.871 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd reweight 4 1 2023-08-28T02:29:25.051 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:29:25.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:25.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:29:25.158 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:29:25.164 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:29:25.170 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:29:25.186 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:25.198 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:29:25.236 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:29:25.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:25.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:29:25.254 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:29:25.302 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:29:25.335 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:29:25.342 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:29:25.369 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:25.401 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:29:25.404 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:29:25.424 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:29:25.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:25.428 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:29:25.435 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:29:25.447 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:29:25.447 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:29:25.447 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:29:25.447 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:29:25.447 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:29:25.447 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:29:25.448 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:29:25.448 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:29:25.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:25 smithi169 ceph-mon[159415]: 6.2 scrub starts 2023-08-28T02:29:25.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:25 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/200623174' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:29:25.529 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:29:25.557 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:25.616 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:29:25.619 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:29:25.619 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [ 2023-08-28T02:29:25.620 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.620 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.35076.0:113457 6.d e237/204)", 2023-08-28T02:29:25.620 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:29:25.617001+0000", 2023-08-28T02:29:25.620 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.0019308909999999999, 2023-08-28T02:29:25.620 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0019557979999999999, 2023-08-28T02:29:25.620 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:29:25.620 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "queued for pg", 2023-08-28T02:29:25.620 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:29:25.620 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.620 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:29:25.620 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.617001+0000", 2023-08-28T02:29:25.621 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:29:25.621 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.621 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.621 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:29:25.621 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.617001+0000", 2023-08-28T02:29:25.621 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:29:25.621 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.621 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.621 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:29:25.621 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.617000+0000", 2023-08-28T02:29:25.621 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:29:25.621 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.622 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.622 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:29:25.622 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.617011+0000", 2023-08-28T02:29:25.622 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1371e-05 2023-08-28T02:29:25.622 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.622 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.622 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:29:25.622 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.617022+0000", 2023-08-28T02:29:25.622 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0719e-05 2023-08-28T02:29:25.622 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.622 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.622 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:29:25.623 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.617029+0000", 2023-08-28T02:29:25.623 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.8210000000000004e-06 2023-08-28T02:29:25.623 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:29:25.623 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:29:25.623 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:29:25.623 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.623 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.623 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:113352 6.1 6:85fefc42:::benchmark_data_smithi143_305742_object29733:head [delete] snapc 0=[] ondisk+write+known_if_redirected e237)", 2023-08-28T02:29:25.623 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:29:25.362002+0000", 2023-08-28T02:29:25.623 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.25692897100000001, 2023-08-28T02:29:25.623 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.25703090200000001, 2023-08-28T02:29:25.623 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:29:25.624 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "delayed", 2023-08-28T02:29:25.624 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:29:25.624 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:29:25.624 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:29:25.624 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 113352 2023-08-28T02:29:25.624 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.624 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:29:25.624 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.624 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:29:25.624 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.362002+0000", 2023-08-28T02:29:25.624 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:29:25.624 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.624 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.625 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:29:25.625 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.362002+0000", 2023-08-28T02:29:25.625 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:29:25.625 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.625 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.625 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:29:25.625 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.362001+0000", 2023-08-28T02:29:25.625 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:29:25.625 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.625 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.625 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:29:25.626 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.362021+0000", 2023-08-28T02:29:25.626 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.0044999999999999e-05 2023-08-28T02:29:25.626 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.626 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.626 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:29:25.626 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.362033+0000", 2023-08-28T02:29:25.626 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1919999999999999e-05 2023-08-28T02:29:25.626 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.626 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.626 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:29:25.626 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.362040+0000", 2023-08-28T02:29:25.626 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.9820000000000002e-06 2023-08-28T02:29:25.626 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.627 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.627 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:29:25.627 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.404071+0000", 2023-08-28T02:29:25.627 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.042030966000000003 2023-08-28T02:29:25.627 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.627 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.627 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for missing object", 2023-08-28T02:29:25.627 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.406512+0000", 2023-08-28T02:29:25.627 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0024411340000000002 2023-08-28T02:29:25.627 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.627 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.627 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:29:25.628 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.503252+0000", 2023-08-28T02:29:25.628 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.096739304999999998 2023-08-28T02:29:25.628 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.628 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.628 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for rw locks", 2023-08-28T02:29:25.628 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.506032+0000", 2023-08-28T02:29:25.628 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0027799819999999999 2023-08-28T02:29:25.628 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:29:25.628 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:29:25.628 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:29:25.628 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.628 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.629 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:113294 6.1 6:83e9ac59:::benchmark_data_smithi143_305742_object29675:head [delete] snapc 0=[] ondisk+write+known_if_redirected e237)", 2023-08-28T02:29:25.629 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:29:25.227186+0000", 2023-08-28T02:29:25.629 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.39174509200000002, 2023-08-28T02:29:25.629 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.39193008600000001, 2023-08-28T02:29:25.629 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:29:25.629 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "delayed", 2023-08-28T02:29:25.629 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:29:25.629 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:29:25.629 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:29:25.629 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 113294 2023-08-28T02:29:25.629 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.629 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:29:25.629 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.630 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:29:25.630 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.227186+0000", 2023-08-28T02:29:25.630 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:29:25.630 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.630 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.630 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:29:25.630 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.227186+0000", 2023-08-28T02:29:25.630 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:29:25.630 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.630 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.630 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:29:25.630 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.227185+0000", 2023-08-28T02:29:25.630 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:29:25.631 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.631 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.631 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:29:25.631 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.227192+0000", 2023-08-28T02:29:25.631 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.5549999999999997e-06 2023-08-28T02:29:25.631 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.631 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.631 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:29:25.631 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.227206+0000", 2023-08-28T02:29:25.631 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.4166e-05 2023-08-28T02:29:25.631 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.631 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.631 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:29:25.632 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.227216+0000", 2023-08-28T02:29:25.632 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.8139999999999995e-06 2023-08-28T02:29:25.632 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.632 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.632 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:29:25.632 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.230126+0000", 2023-08-28T02:29:25.632 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0029101610000000001 2023-08-28T02:29:25.632 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.632 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.632 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for missing object", 2023-08-28T02:29:25.632 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.232954+0000", 2023-08-28T02:29:25.632 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0028279070000000002 2023-08-28T02:29:25.632 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.633 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.633 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:29:25.633 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.248031+0000", 2023-08-28T02:29:25.633 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.015077165 2023-08-28T02:29:25.633 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.633 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.633 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for rw locks", 2023-08-28T02:29:25.633 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.250985+0000", 2023-08-28T02:29:25.633 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0029540149999999999 2023-08-28T02:29:25.633 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:29:25.633 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:29:25.633 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:29:25.634 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.634 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.634 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:113295 6.11 6:89df84c1:::benchmark_data_smithi143_305742_object29676:head [delete] snapc 0=[] ondisk+write+known_if_redirected e237)", 2023-08-28T02:29:25.634 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:29:25.227225+0000", 2023-08-28T02:29:25.634 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.39170616899999999, 2023-08-28T02:29:25.634 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.391968541, 2023-08-28T02:29:25.634 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:29:25.634 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "waiting for sub ops", 2023-08-28T02:29:25.634 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:29:25.634 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:29:25.634 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:29:25.634 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 113295 2023-08-28T02:29:25.634 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.635 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:29:25.635 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.635 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:29:25.635 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.227225+0000", 2023-08-28T02:29:25.635 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:29:25.635 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.635 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.635 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:29:25.635 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.227225+0000", 2023-08-28T02:29:25.635 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:29:25.635 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.636 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.636 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:29:25.636 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.227225+0000", 2023-08-28T02:29:25.636 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:29:25.636 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.636 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.636 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:29:25.636 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.227228+0000", 2023-08-28T02:29:25.636 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.1499999999999999e-06 2023-08-28T02:29:25.636 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.636 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.636 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:29:25.636 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.227235+0000", 2023-08-28T02:29:25.637 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.7089999999999996e-06 2023-08-28T02:29:25.637 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.637 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.637 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:29:25.637 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.227246+0000", 2023-08-28T02:29:25.637 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.094e-05 2023-08-28T02:29:25.637 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.637 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.637 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:29:25.637 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.233852+0000", 2023-08-28T02:29:25.637 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0066068059999999998 2023-08-28T02:29:25.637 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.638 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.638 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for rw locks", 2023-08-28T02:29:25.638 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.236634+0000", 2023-08-28T02:29:25.638 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0027819419999999999 2023-08-28T02:29:25.638 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.638 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.638 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:29:25.638 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.596890+0000", 2023-08-28T02:29:25.638 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.36025575599999998 2023-08-28T02:29:25.638 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.638 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.638 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:29:25.639 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.599059+0000", 2023-08-28T02:29:25.639 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0021692159999999999 2023-08-28T02:29:25.639 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.639 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.639 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 2,6", 2023-08-28T02:29:25.639 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.602430+0000", 2023-08-28T02:29:25.639 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.003370942 2023-08-28T02:29:25.639 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.639 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.639 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:29:25.639 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.608048+0000", 2023-08-28T02:29:25.639 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0056177019999999996 2023-08-28T02:29:25.639 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.640 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.640 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:29:25.640 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.610500+0000", 2023-08-28T02:29:25.640 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0024520700000000002 2023-08-28T02:29:25.640 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:29:25.640 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:29:25.640 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:29:25.640 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.640 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.640 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:113332 6.11 6:88f568bd:::benchmark_data_smithi143_305742_object29713:head [delete] snapc 0=[] ondisk+write+known_if_redirected e237)", 2023-08-28T02:29:25.640 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:29:25.321210+0000", 2023-08-28T02:29:25.640 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.29772116399999998, 2023-08-28T02:29:25.641 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.298078275, 2023-08-28T02:29:25.641 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:29:25.641 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "delayed", 2023-08-28T02:29:25.641 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:29:25.641 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:29:25.641 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:29:25.641 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 113332 2023-08-28T02:29:25.641 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.641 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:29:25.641 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.641 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:29:25.641 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.321210+0000", 2023-08-28T02:29:25.641 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:29:25.642 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.642 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.642 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:29:25.642 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.321210+0000", 2023-08-28T02:29:25.642 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:29:25.642 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.642 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.642 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:29:25.642 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.321208+0000", 2023-08-28T02:29:25.642 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:29:25.642 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.642 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.643 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:29:25.643 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.321217+0000", 2023-08-28T02:29:25.643 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.0110000000000006e-06 2023-08-28T02:29:25.643 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.643 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.643 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:29:25.643 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.321232+0000", 2023-08-28T02:29:25.643 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.4137e-05 2023-08-28T02:29:25.643 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.643 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.643 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:29:25.643 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.321241+0000", 2023-08-28T02:29:25.644 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.8530000000000004e-06 2023-08-28T02:29:25.644 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.644 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.644 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:29:25.644 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.347562+0000", 2023-08-28T02:29:25.644 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.026320178 2023-08-28T02:29:25.644 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.644 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.644 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for missing object", 2023-08-28T02:29:25.644 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.350429+0000", 2023-08-28T02:29:25.645 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0028676349999999999 2023-08-28T02:29:25.645 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.645 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.645 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:29:25.645 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.366278+0000", 2023-08-28T02:29:25.645 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.015848936000000001 2023-08-28T02:29:25.645 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.645 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.645 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for rw locks", 2023-08-28T02:29:25.645 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.368912+0000", 2023-08-28T02:29:25.645 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0026341300000000002 2023-08-28T02:29:25.646 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:29:25.646 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:29:25.646 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:29:25.646 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.646 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.646 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:113405 6.1 6:811ce3d7:::benchmark_data_smithi143_305742_object29786:head [delete] snapc 0=[] ondisk+write+known_if_redirected e237)", 2023-08-28T02:29:25.646 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:29:25.479222+0000", 2023-08-28T02:29:25.646 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.13970929700000001, 2023-08-28T02:29:25.646 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.14015717699999999, 2023-08-28T02:29:25.646 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:29:25.646 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "delayed", 2023-08-28T02:29:25.646 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:29:25.646 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:29:25.647 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:29:25.647 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 113405 2023-08-28T02:29:25.647 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.647 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:29:25.647 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.647 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:29:25.647 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.479222+0000", 2023-08-28T02:29:25.647 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:29:25.647 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.647 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.647 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:29:25.647 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.479222+0000", 2023-08-28T02:29:25.648 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:29:25.648 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.648 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.648 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:29:25.648 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.479220+0000", 2023-08-28T02:29:25.648 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:29:25.648 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.648 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.648 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:29:25.648 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.479229+0000", 2023-08-28T02:29:25.648 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.1990000000000006e-06 2023-08-28T02:29:25.649 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.649 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.649 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:29:25.649 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.479243+0000", 2023-08-28T02:29:25.649 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.4107999999999999e-05 2023-08-28T02:29:25.650 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.650 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.650 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:29:25.650 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.479251+0000", 2023-08-28T02:29:25.650 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.5310000000000001e-06 2023-08-28T02:29:25.650 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.650 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.650 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:29:25.650 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.506327+0000", 2023-08-28T02:29:25.650 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.027075625999999998 2023-08-28T02:29:25.650 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.650 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.651 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for missing object", 2023-08-28T02:29:25.651 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.509232+0000", 2023-08-28T02:29:25.651 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0029054129999999999 2023-08-28T02:29:25.651 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.651 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.651 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:29:25.651 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.567920+0000", 2023-08-28T02:29:25.651 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.058687611000000001 2023-08-28T02:29:25.651 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.651 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.651 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for rw locks", 2023-08-28T02:29:25.651 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.570845+0000", 2023-08-28T02:29:25.651 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0029253299999999999 2023-08-28T02:29:25.652 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:29:25.652 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:29:25.652 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:29:25.652 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.652 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.652 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:113333 6.11 6:8e1d139e:::benchmark_data_smithi143_305742_object29714:head [delete] snapc 0=[] ondisk+write+known_if_redirected e237)", 2023-08-28T02:29:25.652 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:29:25.324986+0000", 2023-08-28T02:29:25.652 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.29394506199999998, 2023-08-28T02:29:25.652 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.29447424799999999, 2023-08-28T02:29:25.652 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:29:25.652 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "delayed", 2023-08-28T02:29:25.652 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:29:25.653 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:29:25.653 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:29:25.653 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 113333 2023-08-28T02:29:25.653 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.653 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:29:25.653 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.653 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:29:25.653 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.324986+0000", 2023-08-28T02:29:25.653 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:29:25.653 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.653 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.653 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:29:25.654 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.324986+0000", 2023-08-28T02:29:25.654 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:29:25.654 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.654 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.654 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:29:25.654 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.324985+0000", 2023-08-28T02:29:25.654 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:29:25.654 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.654 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.654 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:29:25.654 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.324993+0000", 2023-08-28T02:29:25.654 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.4929999999999998e-06 2023-08-28T02:29:25.654 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.655 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.655 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:29:25.655 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.325020+0000", 2023-08-28T02:29:25.655 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.6546999999999998e-05 2023-08-28T02:29:25.655 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.655 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.655 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:29:25.655 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.325029+0000", 2023-08-28T02:29:25.655 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.1700000000000003e-06 2023-08-28T02:29:25.656 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.656 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.656 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:29:25.656 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.350709+0000", 2023-08-28T02:29:25.656 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.025679865 2023-08-28T02:29:25.656 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.656 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.656 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for missing object", 2023-08-28T02:29:25.656 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.353580+0000", 2023-08-28T02:29:25.656 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.002870909 2023-08-28T02:29:25.656 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.656 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.656 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:29:25.657 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.398812+0000", 2023-08-28T02:29:25.657 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.045231942999999997 2023-08-28T02:29:25.657 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.657 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.657 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for rw locks", 2023-08-28T02:29:25.657 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.401208+0000", 2023-08-28T02:29:25.657 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0023968100000000001 2023-08-28T02:29:25.657 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:29:25.657 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:29:25.657 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:29:25.657 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.657 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.658 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:113406 6.11 6:8dd993f0:::benchmark_data_smithi143_305742_object29787:head [delete] snapc 0=[] ondisk+write+known_if_redirected e237)", 2023-08-28T02:29:25.658 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:29:25.481046+0000", 2023-08-28T02:29:25.658 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.13788556799999999, 2023-08-28T02:29:25.658 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.138501442, 2023-08-28T02:29:25.658 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:29:25.658 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "delayed", 2023-08-28T02:29:25.658 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:29:25.658 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:29:25.658 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:29:25.658 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 113406 2023-08-28T02:29:25.658 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.658 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:29:25.659 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.659 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:29:25.659 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.481046+0000", 2023-08-28T02:29:25.659 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:29:25.659 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.659 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.659 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:29:25.659 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.481046+0000", 2023-08-28T02:29:25.659 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:29:25.659 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.659 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.659 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:29:25.660 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.481044+0000", 2023-08-28T02:29:25.660 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:29:25.660 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.660 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.660 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:29:25.660 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.481051+0000", 2023-08-28T02:29:25.660 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.6900000000000003e-06 2023-08-28T02:29:25.660 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.660 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.660 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:29:25.660 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.481064+0000", 2023-08-28T02:29:25.660 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.2488e-05 2023-08-28T02:29:25.661 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.661 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.661 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:29:25.661 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.481073+0000", 2023-08-28T02:29:25.661 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.0080000000000008e-06 2023-08-28T02:29:25.661 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.661 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.661 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:29:25.661 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.510929+0000", 2023-08-28T02:29:25.661 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.029856160999999999 2023-08-28T02:29:25.661 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.661 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.661 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for missing object", 2023-08-28T02:29:25.662 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.513881+0000", 2023-08-28T02:29:25.662 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0029519049999999999 2023-08-28T02:29:25.662 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.662 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.662 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:29:25.662 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.567632+0000", 2023-08-28T02:29:25.662 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.053750774000000001 2023-08-28T02:29:25.662 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.662 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.662 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for rw locks", 2023-08-28T02:29:25.662 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.569762+0000", 2023-08-28T02:29:25.662 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.002130014 2023-08-28T02:29:25.663 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:29:25.663 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:29:25.663 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:29:25.663 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.663 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.663 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:113408 6.1 6:83bc20ce:::benchmark_data_smithi143_305742_object29789:head [delete] snapc 0=[] ondisk+write+known_if_redirected e237)", 2023-08-28T02:29:25.663 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:29:25.485282+0000", 2023-08-28T02:29:25.663 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.13364968299999999, 2023-08-28T02:29:25.663 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.134343183, 2023-08-28T02:29:25.663 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:29:25.663 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "delayed", 2023-08-28T02:29:25.663 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:29:25.664 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:29:25.664 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:29:25.664 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 113408 2023-08-28T02:29:25.664 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.664 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:29:25.664 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.664 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:29:25.664 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.485282+0000", 2023-08-28T02:29:25.664 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:29:25.664 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.664 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.664 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:29:25.664 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.485282+0000", 2023-08-28T02:29:25.665 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:29:25.665 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.665 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.665 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:29:25.665 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.485280+0000", 2023-08-28T02:29:25.665 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:29:25.665 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.665 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.665 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:29:25.665 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.485288+0000", 2023-08-28T02:29:25.665 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.2690000000000002e-06 2023-08-28T02:29:25.665 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.665 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.666 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:29:25.666 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.485301+0000", 2023-08-28T02:29:25.666 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3427999999999999e-05 2023-08-28T02:29:25.666 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.666 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.666 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:29:25.666 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.485309+0000", 2023-08-28T02:29:25.666 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.7549999999999994e-06 2023-08-28T02:29:25.666 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.666 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.666 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:29:25.666 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.520382+0000", 2023-08-28T02:29:25.666 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.035073393000000001 2023-08-28T02:29:25.667 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.667 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.667 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for missing object", 2023-08-28T02:29:25.667 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.523438+0000", 2023-08-28T02:29:25.667 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0030554250000000001 2023-08-28T02:29:25.667 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.667 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.667 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:29:25.667 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.546540+0000", 2023-08-28T02:29:25.667 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.023102230000000001 2023-08-28T02:29:25.667 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.667 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.668 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for rw locks", 2023-08-28T02:29:25.668 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.548835+0000", 2023-08-28T02:29:25.668 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0022952290000000002 2023-08-28T02:29:25.668 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:29:25.668 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:29:25.668 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:29:25.668 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.668 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.668 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:113340 6.1 6:87fa9f46:::benchmark_data_smithi143_305742_object29721:head [delete] snapc 0=[] ondisk+write+known_if_redirected e237)", 2023-08-28T02:29:25.668 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:29:25.334546+0000", 2023-08-28T02:29:25.668 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.28438508499999998, 2023-08-28T02:29:25.669 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.28515496000000001, 2023-08-28T02:29:25.669 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:29:25.669 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "delayed", 2023-08-28T02:29:25.669 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:29:25.669 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:29:25.669 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:29:25.669 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 113340 2023-08-28T02:29:25.669 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.669 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:29:25.669 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.669 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:29:25.669 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.334546+0000", 2023-08-28T02:29:25.670 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:29:25.670 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.670 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.670 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:29:25.670 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.334546+0000", 2023-08-28T02:29:25.670 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:29:25.670 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.670 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.670 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:29:25.670 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.334544+0000", 2023-08-28T02:29:25.670 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999976 2023-08-28T02:29:25.670 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.671 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.671 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:29:25.671 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.334553+0000", 2023-08-28T02:29:25.671 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.9819999999999997e-06 2023-08-28T02:29:25.671 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.671 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.671 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:29:25.671 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.334564+0000", 2023-08-28T02:29:25.671 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0349999999999999e-05 2023-08-28T02:29:25.671 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.671 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.671 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:29:25.671 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.334570+0000", 2023-08-28T02:29:25.672 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.4400000000000002e-06 2023-08-28T02:29:25.672 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.672 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.672 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:29:25.672 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.353890+0000", 2023-08-28T02:29:25.672 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.019320158 2023-08-28T02:29:25.672 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.672 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.672 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for missing object", 2023-08-28T02:29:25.672 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.356932+0000", 2023-08-28T02:29:25.672 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0030417780000000002 2023-08-28T02:29:25.672 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.672 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.673 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:29:25.673 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.397982+0000", 2023-08-28T02:29:25.673 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.041049655999999997 2023-08-28T02:29:25.673 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.673 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.673 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for rw locks", 2023-08-28T02:29:25.673 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.400495+0000", 2023-08-28T02:29:25.673 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0025135629999999999 2023-08-28T02:29:25.673 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:29:25.673 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:29:25.673 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:29:25.673 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.674 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.674 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "MOSDPGPush(6.11 237/221 [PushOp(6:8f442da6:::benchmark_data_smithi143_305742_object29827:head, version: 200'1918, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(6:8f442da6:::benchmark_data_smithi143_305742_object29827:head@200'1918, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-08-28T02:29:25.674 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:29:25.588188+0000", 2023-08-28T02:29:25.674 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.030743935999999999, 2023-08-28T02:29:25.674 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.031612164999999998, 2023-08-28T02:29:25.674 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:29:25.674 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "started", 2023-08-28T02:29:25.674 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:29:25.674 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.674 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:29:25.674 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.588188+0000", 2023-08-28T02:29:25.674 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:29:25.674 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.675 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.675 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:29:25.675 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.588188+0000", 2023-08-28T02:29:25.675 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:29:25.675 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.675 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.675 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:29:25.675 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.588187+0000", 2023-08-28T02:29:25.675 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:29:25.675 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.675 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.675 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:29:25.676 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.588214+0000", 2023-08-28T02:29:25.676 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.7191000000000001e-05 2023-08-28T02:29:25.676 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.676 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.676 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:29:25.676 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.588232+0000", 2023-08-28T02:29:25.676 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.8249000000000001e-05 2023-08-28T02:29:25.676 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.676 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.676 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:29:25.676 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.588248+0000", 2023-08-28T02:29:25.676 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.6092000000000001e-05 2023-08-28T02:29:25.676 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.677 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.677 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:29:25.677 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.613120+0000", 2023-08-28T02:29:25.677 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.024871880999999998 2023-08-28T02:29:25.677 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.677 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.677 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:29:25.677 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.613383+0000", 2023-08-28T02:29:25.677 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00026260200000000002 2023-08-28T02:29:25.677 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:29:25.677 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:29:25.677 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:29:25.678 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.678 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.678 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:113446 6.11 6:8f442da6:::benchmark_data_smithi143_305742_object29827:head [delete] snapc 0=[] ondisk+write+known_if_redirected e237)", 2023-08-28T02:29:25.678 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:29:25.589775+0000", 2023-08-28T02:29:25.678 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.029156349000000002, 2023-08-28T02:29:25.678 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.030088179999999999, 2023-08-28T02:29:25.678 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:29:25.678 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "delayed", 2023-08-28T02:29:25.678 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:29:25.678 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:29:25.678 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:29:25.678 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 113446 2023-08-28T02:29:25.678 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.679 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:29:25.679 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.679 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:29:25.679 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.589775+0000", 2023-08-28T02:29:25.679 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:29:25.679 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.679 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.679 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:29:25.679 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.589775+0000", 2023-08-28T02:29:25.679 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:29:25.679 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.679 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.680 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:29:25.680 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.589773+0000", 2023-08-28T02:29:25.680 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:29:25.680 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.680 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.680 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:29:25.680 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.589782+0000", 2023-08-28T02:29:25.680 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.1689999999999992e-06 2023-08-28T02:29:25.680 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.680 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.680 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:29:25.680 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.589825+0000", 2023-08-28T02:29:25.680 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.3785000000000002e-05 2023-08-28T02:29:25.681 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.681 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.681 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:29:25.681 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.589836+0000", 2023-08-28T02:29:25.681 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0264e-05 2023-08-28T02:29:25.681 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.681 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.681 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:29:25.681 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.606021+0000", 2023-08-28T02:29:25.681 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.016185377000000001 2023-08-28T02:29:25.681 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.681 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.681 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for missing object", 2023-08-28T02:29:25.682 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.607298+0000", 2023-08-28T02:29:25.682 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0012766570000000001 2023-08-28T02:29:25.682 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:29:25.682 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:29:25.682 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:29:25.682 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.682 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.682 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:113350 6.1 6:80dd58cb:::benchmark_data_smithi143_305742_object29731:head [delete] snapc 0=[] ondisk+write+known_if_redirected e237)", 2023-08-28T02:29:25.682 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:29:25.353765+0000", 2023-08-28T02:29:25.682 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.26516674400000001, 2023-08-28T02:29:25.682 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.26616827799999998, 2023-08-28T02:29:25.682 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:29:25.683 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "delayed", 2023-08-28T02:29:25.683 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:29:25.683 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:29:25.683 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:29:25.683 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 113350 2023-08-28T02:29:25.683 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.683 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:29:25.683 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.683 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:29:25.683 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.353765+0000", 2023-08-28T02:29:25.683 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:29:25.683 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.684 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.684 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:29:25.684 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.353765+0000", 2023-08-28T02:29:25.684 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:29:25.684 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.684 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.684 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:29:25.684 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.353763+0000", 2023-08-28T02:29:25.684 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:29:25.684 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.684 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.684 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:29:25.684 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.353770+0000", 2023-08-28T02:29:25.685 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.421e-06 2023-08-28T02:29:25.685 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.685 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.685 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:29:25.685 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.353784+0000", 2023-08-28T02:29:25.685 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.4263e-05 2023-08-28T02:29:25.685 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.685 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.685 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:29:25.685 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.353792+0000", 2023-08-28T02:29:25.685 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.089e-06 2023-08-28T02:29:25.685 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.685 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.686 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:29:25.686 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.357929+0000", 2023-08-28T02:29:25.686 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0041366659999999998 2023-08-28T02:29:25.686 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.686 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.686 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for missing object", 2023-08-28T02:29:25.686 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.360870+0000", 2023-08-28T02:29:25.686 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0029417359999999999 2023-08-28T02:29:25.686 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.686 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.686 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:29:25.686 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.400700+0000", 2023-08-28T02:29:25.686 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.039829477000000002 2023-08-28T02:29:25.687 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.687 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.687 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for rw locks", 2023-08-28T02:29:25.687 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.402944+0000", 2023-08-28T02:29:25.687 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.002244256 2023-08-28T02:29:25.687 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:29:25.687 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:29:25.687 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:29:25.687 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.687 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.687 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:113459 6.11 6.715a8d71 (undecoded) ondisk+write+known_if_redirected e237)", 2023-08-28T02:29:25.687 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:29:25.616188+0000", 2023-08-28T02:29:25.687 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.002743522, 2023-08-28T02:29:25.688 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0038314349999999998, 2023-08-28T02:29:25.688 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:29:25.688 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "queued for pg", 2023-08-28T02:29:25.688 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:29:25.688 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:29:25.688 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:29:25.688 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 113459 2023-08-28T02:29:25.688 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.688 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:29:25.688 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.688 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:29:25.688 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.616188+0000", 2023-08-28T02:29:25.688 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:29:25.689 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.689 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.689 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:29:25.689 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.616188+0000", 2023-08-28T02:29:25.689 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:29:25.689 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.689 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.689 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:29:25.689 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.616187+0000", 2023-08-28T02:29:25.689 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:29:25.689 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.689 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.689 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:29:25.690 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.616193+0000", 2023-08-28T02:29:25.690 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.1419999999999998e-06 2023-08-28T02:29:25.690 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.690 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.690 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:29:25.690 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.616207+0000", 2023-08-28T02:29:25.690 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3684e-05 2023-08-28T02:29:25.690 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:25.690 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:25.690 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:29:25.690 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:25.616215+0000", 2023-08-28T02:29:25.690 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.5129999999999996e-06 2023-08-28T02:29:25.690 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:29:25.691 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:29:25.691 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:29:25.691 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:29:25.691 INFO:teuthology.orchestra.run.smithi143.stdout: ], 2023-08-28T02:29:25.691 INFO:teuthology.orchestra.run.smithi143.stdout: "num_ops": 14 2023-08-28T02:29:25.691 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:29:25.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:25.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:29:25.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:25 smithi143 ceph-mon[189871]: 6.2 scrub starts 2023-08-28T02:29:25.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:25 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/200623174' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:29:25.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:25 smithi143 ceph-mon[183059]: 6.2 scrub starts 2023-08-28T02:29:25.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:25 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/200623174' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:29:25.793 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:29:25.819 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:25.859 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:29:25.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:25.864 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:29:25.965 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:29:25.995 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:26.045 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:29:26.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:26.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:29:26.153 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:29:26.202 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:26.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:26 smithi143 ceph-mon[183059]: pgmap v566: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 373 op/s; 10652/164301 objects degraded (6.483%); 8691/164301 objects misplaced (5.290%); 6.0 MiB/s, 95 objects/s recovering 2023-08-28T02:29:26.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:26 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:29:26.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:26 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 10652/164301 objects degraded (6.483%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-08-28T02:29:26.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:26 smithi143 ceph-mon[189871]: pgmap v566: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 373 op/s; 10652/164301 objects degraded (6.483%); 8691/164301 objects misplaced (5.290%); 6.0 MiB/s, 95 objects/s recovering 2023-08-28T02:29:26.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:26 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:29:26.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:26 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 10652/164301 objects degraded (6.483%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-08-28T02:29:26.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:26 smithi169 ceph-mon[159415]: pgmap v566: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 373 op/s; 10652/164301 objects degraded (6.483%); 8691/164301 objects misplaced (5.290%); 6.0 MiB/s, 95 objects/s recovering 2023-08-28T02:29:26.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:26 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:29:26.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:26 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 10652/164301 objects degraded (6.483%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-08-28T02:29:27.477 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:29:27.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:27.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:29:27.585 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:29:27.607 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.1 dump_blocked_ops 2023-08-28T02:29:27.615 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:27.662 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:29:27.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:27.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:29:27.766 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:29:27.812 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:27.889 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:29:27.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:27.899 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:29:27.961 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:29:27.999 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:29:28.070 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:28.209 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:29:28.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:28.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:29:28.328 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:29:28.390 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:28.480 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:29:28.508 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:29:28.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:28.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:29:28.619 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:29:28.651 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:28.700 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:29:28.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:28 smithi143 ceph-mon[183059]: 6.2 scrub starts 2023-08-28T02:29:28.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:28 smithi143 ceph-mon[183059]: pgmap v567: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 378 op/s; 10131/161754 objects degraded (6.263%); 8628/161754 objects misplaced (5.334%); 6.4 MiB/s, 167 objects/s recovering 2023-08-28T02:29:28.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:28 smithi143 ceph-mon[189871]: 6.2 scrub starts 2023-08-28T02:29:28.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:28 smithi143 ceph-mon[189871]: pgmap v567: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 378 op/s; 10131/161754 objects degraded (6.263%); 8628/161754 objects misplaced (5.334%); 6.4 MiB/s, 167 objects/s recovering 2023-08-28T02:29:28.705 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:28 smithi169 ceph-mon[159415]: 6.2 scrub starts 2023-08-28T02:29:28.705 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:28 smithi169 ceph-mon[159415]: pgmap v567: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 378 op/s; 10131/161754 objects degraded (6.263%); 8628/161754 objects misplaced (5.334%); 6.4 MiB/s, 167 objects/s recovering 2023-08-28T02:29:28.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:28.706 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:29:28.807 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:29:28.835 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:28.877 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:29:28.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:28.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:29:28.983 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:29:29.010 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:29.310 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:29:29.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:29.316 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:29:29.417 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:29:29.456 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:29.456 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is unset 2023-08-28T02:29:29.513 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:29:29.593 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:29 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2865868745' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:29:29.593 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:29 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:29:29.593 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:29 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2865868745' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:29:29.593 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:29 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:29:29.641 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:29:29.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:29.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:29:29.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:29 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2865868745' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:29:29.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:29 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:29:29.751 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:29:29.776 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:29.817 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:29:29.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:29.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:29:29.923 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:29:29.966 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:30.472 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:29:30.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:30.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:29:30.578 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:29:30.605 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:30.651 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:29:30.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:30.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:29:30.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:30 smithi143 ceph-mon[183059]: pgmap v568: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 405 op/s; 10131/159240 objects degraded (6.362%); 8381/159240 objects misplaced (5.263%); 4.9 MiB/s, 136 objects/s recovering 2023-08-28T02:29:30.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:30 smithi143 ceph-mon[183059]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:29:30.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:30 smithi143 ceph-mon[183059]: failed to encode map e238 with expected crc 2023-08-28T02:29:30.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:30 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:29:30.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:30 smithi143 ceph-mon[183059]: osdmap e238: 8 total, 8 up, 8 in 2023-08-28T02:29:30.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:30 smithi143 ceph-mon[183059]: failed to encode map e238 with expected crc 2023-08-28T02:29:30.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:30 smithi143 ceph-mon[183059]: failed to encode map e238 with expected crc 2023-08-28T02:29:30.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:30 smithi143 ceph-mon[189871]: pgmap v568: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 405 op/s; 10131/159240 objects degraded (6.362%); 8381/159240 objects misplaced (5.263%); 4.9 MiB/s, 136 objects/s recovering 2023-08-28T02:29:30.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:30 smithi143 ceph-mon[189871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:29:30.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:30 smithi143 ceph-mon[189871]: failed to encode map e238 with expected crc 2023-08-28T02:29:30.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:30 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:29:30.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:30 smithi143 ceph-mon[189871]: osdmap e238: 8 total, 8 up, 8 in 2023-08-28T02:29:30.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:30 smithi143 ceph-mon[189871]: failed to encode map e238 with expected crc 2023-08-28T02:29:30.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:30 smithi143 ceph-mon[189871]: failed to encode map e238 with expected crc 2023-08-28T02:29:30.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:30 smithi169 ceph-mon[159415]: pgmap v568: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 405 op/s; 10131/159240 objects degraded (6.362%); 8381/159240 objects misplaced (5.263%); 4.9 MiB/s, 136 objects/s recovering 2023-08-28T02:29:30.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:30 smithi169 ceph-mon[159415]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:29:30.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:30 smithi169 ceph-mon[159415]: failed to encode map e238 with expected crc 2023-08-28T02:29:30.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:30 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:29:30.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:30 smithi169 ceph-mon[159415]: osdmap e238: 8 total, 8 up, 8 in 2023-08-28T02:29:30.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:30 smithi169 ceph-mon[159415]: failed to encode map e238 with expected crc 2023-08-28T02:29:30.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:30 smithi169 ceph-mon[159415]: failed to encode map e238 with expected crc 2023-08-28T02:29:30.756 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:29:30.798 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:31.163 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:29:31.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:31.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:29:31.268 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:29:31.296 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:31.339 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:29:31.342 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:29:31.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:31.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:29:31.431 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:31 smithi143 ceph-mon[189871]: 6.2 scrub starts 2023-08-28T02:29:31.432 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:31 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 10131/159240 objects degraded (6.362%), 9 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-08-28T02:29:31.448 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:29:31.478 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:31.525 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:29:31.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:31.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:29:31.633 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:29:31.683 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:31 smithi143 ceph-mon[183059]: 6.2 scrub starts 2023-08-28T02:29:31.683 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:31 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 10131/159240 objects degraded (6.362%), 9 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-08-28T02:29:31.694 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:31.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:31 smithi169 ceph-mon[159415]: 6.2 scrub starts 2023-08-28T02:29:31.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:31 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 10131/159240 objects degraded (6.362%), 9 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-08-28T02:29:31.765 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:29:31.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:31.772 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:29:31.806 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:29:31.807 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [], 2023-08-28T02:29:31.807 INFO:teuthology.orchestra.run.smithi143.stdout: "complaint_time": 30, 2023-08-28T02:29:31.807 INFO:teuthology.orchestra.run.smithi143.stdout: "num_blocked_ops": 0 2023-08-28T02:29:31.807 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:29:31.873 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:29:31.928 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:31.951 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:29:31 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:29:31] "GET /metrics HTTP/1.1" 200 36489 "" "Prometheus/2.33.4" 2023-08-28T02:29:32.439 INFO:teuthology.orchestra.run.smithi143.stderr:reweighted osd.4 to 1 (10000) 2023-08-28T02:29:32.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:32 smithi143 ceph-mon[189871]: 6.2 scrub starts 2023-08-28T02:29:32.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:32 smithi143 ceph-mon[189871]: pgmap v570: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 382 op/s; 9996/158472 objects degraded (6.308%); 8254/158472 objects misplaced (5.208%); 2.4 MiB/s, 104 objects/s recovering 2023-08-28T02:29:32.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:32 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/4220392139' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-08-28T02:29:32.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:32 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-08-28T02:29:32.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:32 smithi143 ceph-mon[183059]: 6.2 scrub starts 2023-08-28T02:29:32.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:32 smithi143 ceph-mon[183059]: pgmap v570: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 382 op/s; 9996/158472 objects degraded (6.308%); 8254/158472 objects misplaced (5.208%); 2.4 MiB/s, 104 objects/s recovering 2023-08-28T02:29:32.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:32 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/4220392139' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-08-28T02:29:32.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:32 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-08-28T02:29:32.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:32 smithi169 ceph-mon[159415]: 6.2 scrub starts 2023-08-28T02:29:32.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:32 smithi169 ceph-mon[159415]: pgmap v570: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 382 op/s; 9996/158472 objects degraded (6.308%); 8254/158472 objects misplaced (5.208%); 2.4 MiB/s, 104 objects/s recovering 2023-08-28T02:29:32.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:32 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/4220392139' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-08-28T02:29:32.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:32 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-08-28T02:29:32.828 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set noscrub 2023-08-28T02:29:33.316 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:29:33.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:33.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:29:33.418 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:29:33.448 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:33.506 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:29:33.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:33.513 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:29:33.612 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:33 smithi143 ceph-mon[183059]: failed to encode map e239 with expected crc 2023-08-28T02:29:33.612 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:33 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-08-28T02:29:33.613 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:33 smithi143 ceph-mon[183059]: osdmap e239: 8 total, 8 up, 8 in 2023-08-28T02:29:33.613 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:33 smithi143 ceph-mon[183059]: failed to encode map e239 with expected crc 2023-08-28T02:29:33.613 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:33 smithi143 ceph-mon[183059]: failed to encode map e239 with expected crc 2023-08-28T02:29:33.614 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:29:33.617 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:33 smithi143 ceph-mon[189871]: failed to encode map e239 with expected crc 2023-08-28T02:29:33.617 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:33 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-08-28T02:29:33.618 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:33 smithi143 ceph-mon[189871]: osdmap e239: 8 total, 8 up, 8 in 2023-08-28T02:29:33.618 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:33 smithi143 ceph-mon[189871]: failed to encode map e239 with expected crc 2023-08-28T02:29:33.618 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:33 smithi143 ceph-mon[189871]: failed to encode map e239 with expected crc 2023-08-28T02:29:33.642 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:33.648 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.1 dump_historic_ops 2023-08-28T02:29:33.700 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:29:33.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:33.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:29:33.705 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:33 smithi169 ceph-mon[159415]: failed to encode map e239 with expected crc 2023-08-28T02:29:33.706 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:33 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-08-28T02:29:33.706 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:33 smithi169 ceph-mon[159415]: osdmap e239: 8 total, 8 up, 8 in 2023-08-28T02:29:33.706 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:33 smithi169 ceph-mon[159415]: failed to encode map e239 with expected crc 2023-08-28T02:29:33.706 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:33 smithi169 ceph-mon[159415]: failed to encode map e239 with expected crc 2023-08-28T02:29:33.806 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:29:33.840 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:33.968 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd reweight 5 1 2023-08-28T02:29:34.016 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:29:34.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:34.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:29:34.053 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:29:34.133 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:29:34.163 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:34.170 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:29:34.210 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:29:34.211 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:29:34.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:34.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:29:34.248 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:29:34.300 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:29:34.317 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:29:34.330 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:29:34.360 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:29:34.388 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:34.399 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:29:34.440 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:29:34.465 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:29:34.465 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:29:34.465 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:29:34.465 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:29:34.465 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:29:34.465 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:29:34.465 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:29:34.465 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:29:34.475 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:29:34.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:34.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:29:34.581 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:29:34.607 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:34.607 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:29:34.654 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:29:34.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:34.659 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:29:34.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:34 smithi143 ceph-mon[183059]: pgmap v572: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 347 op/s; 9253/155967 objects degraded (5.933%); 8254/155967 objects misplaced (5.292%); 1.5 MiB/s, 182 objects/s recovering 2023-08-28T02:29:34.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:34 smithi143 ceph-mon[183059]: 6.2 scrub starts 2023-08-28T02:29:34.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:34 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1900806772' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:29:34.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:34 smithi143 ceph-mon[189871]: pgmap v572: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 347 op/s; 9253/155967 objects degraded (5.933%); 8254/155967 objects misplaced (5.292%); 1.5 MiB/s, 182 objects/s recovering 2023-08-28T02:29:34.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:34 smithi143 ceph-mon[189871]: 6.2 scrub starts 2023-08-28T02:29:34.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:34 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1900806772' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:29:34.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:34 smithi169 ceph-mon[159415]: pgmap v572: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 347 op/s; 9253/155967 objects degraded (5.933%); 8254/155967 objects misplaced (5.292%); 1.5 MiB/s, 182 objects/s recovering 2023-08-28T02:29:34.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:34 smithi169 ceph-mon[159415]: 6.2 scrub starts 2023-08-28T02:29:34.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:34 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1900806772' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:29:34.760 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:29:34.786 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:34.829 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:29:34.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:34.834 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:29:34.935 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:29:34.991 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:35.055 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:29:35.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:35.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:29:35.159 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:29:35.190 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:35.236 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:29:35.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:35.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:29:35.340 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:29:35.366 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:35.408 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:29:35.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:35.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:29:35.514 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:29:35.543 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:35.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:35 smithi169 ceph-mon[159415]: osdmap e240: 8 total, 8 up, 8 in 2023-08-28T02:29:35.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:35 smithi169 ceph-mon[159415]: failed to encode map e240 with expected crc 2023-08-28T02:29:35.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:35 smithi169 ceph-mon[159415]: failed to encode map e240 with expected crc 2023-08-28T02:29:35.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:35 smithi169 ceph-mon[159415]: failed to encode map e240 with expected crc 2023-08-28T02:29:35.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:35 smithi143 ceph-mon[183059]: osdmap e240: 8 total, 8 up, 8 in 2023-08-28T02:29:35.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:35 smithi143 ceph-mon[183059]: failed to encode map e240 with expected crc 2023-08-28T02:29:35.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:35 smithi143 ceph-mon[183059]: failed to encode map e240 with expected crc 2023-08-28T02:29:35.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:35 smithi143 ceph-mon[183059]: failed to encode map e240 with expected crc 2023-08-28T02:29:35.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:35 smithi143 ceph-mon[189871]: osdmap e240: 8 total, 8 up, 8 in 2023-08-28T02:29:35.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:35 smithi143 ceph-mon[189871]: failed to encode map e240 with expected crc 2023-08-28T02:29:35.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:35 smithi143 ceph-mon[189871]: failed to encode map e240 with expected crc 2023-08-28T02:29:35.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:35 smithi143 ceph-mon[189871]: failed to encode map e240 with expected crc 2023-08-28T02:29:35.977 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:29:35.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:35.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:29:36.088 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:29:36.120 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:36.167 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:29:36.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:36.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:29:36.273 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:29:36.305 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:36.352 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:29:36.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:36.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:29:36.419 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:29:36.460 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:29:36.490 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:36.545 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:29:36.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:36.550 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:29:36.651 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:29:36.680 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:36.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:36 smithi143 ceph-mon[183059]: pgmap v574: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 361 op/s; 8791/152739 objects degraded (5.756%); 8223/152739 objects misplaced (5.384%); 2.8 MiB/s, 168 objects/s recovering 2023-08-28T02:29:36.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:36 smithi143 ceph-mon[183059]: failed to encode map e241 with expected crc 2023-08-28T02:29:36.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:36 smithi143 ceph-mon[183059]: osdmap e241: 8 total, 8 up, 8 in 2023-08-28T02:29:36.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:36 smithi143 ceph-mon[183059]: failed to encode map e241 with expected crc 2023-08-28T02:29:36.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:36 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 8791/152739 objects degraded (5.756%), 9 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-08-28T02:29:36.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:36 smithi143 ceph-mon[189871]: pgmap v574: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 361 op/s; 8791/152739 objects degraded (5.756%); 8223/152739 objects misplaced (5.384%); 2.8 MiB/s, 168 objects/s recovering 2023-08-28T02:29:36.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:36 smithi143 ceph-mon[189871]: failed to encode map e241 with expected crc 2023-08-28T02:29:36.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:36 smithi143 ceph-mon[189871]: osdmap e241: 8 total, 8 up, 8 in 2023-08-28T02:29:36.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:36 smithi143 ceph-mon[189871]: failed to encode map e241 with expected crc 2023-08-28T02:29:36.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:36 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 8791/152739 objects degraded (5.756%), 9 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-08-28T02:29:36.733 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:29:36.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:36.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:36 smithi169 ceph-mon[159415]: pgmap v574: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 361 op/s; 8791/152739 objects degraded (5.756%); 8223/152739 objects misplaced (5.384%); 2.8 MiB/s, 168 objects/s recovering 2023-08-28T02:29:36.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:36 smithi169 ceph-mon[159415]: failed to encode map e241 with expected crc 2023-08-28T02:29:36.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:36 smithi169 ceph-mon[159415]: osdmap e241: 8 total, 8 up, 8 in 2023-08-28T02:29:36.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:36 smithi169 ceph-mon[159415]: failed to encode map e241 with expected crc 2023-08-28T02:29:36.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:36 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 8791/152739 objects degraded (5.756%), 9 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-08-28T02:29:36.740 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:29:36.841 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:29:36.871 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:36.917 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:29:36.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:36.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:29:37.024 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:29:37.151 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:37.406 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:29:37.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:37.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:29:37.505 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is set 2023-08-28T02:29:37.510 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:29:37.588 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:37.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:37 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1333738446' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:29:37.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:37 smithi143 ceph-mon[183059]: pgmap v576: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 476 op/s; 8472/149889 objects degraded (5.652%); 8223/149889 objects misplaced (5.486%); 1.4 MiB/s, 200 objects/s recovering 2023-08-28T02:29:37.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:37 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1333738446' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:29:37.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:37 smithi143 ceph-mon[189871]: pgmap v576: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 476 op/s; 8472/149889 objects degraded (5.652%); 8223/149889 objects misplaced (5.486%); 1.4 MiB/s, 200 objects/s recovering 2023-08-28T02:29:37.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:37 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1333738446' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:29:37.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:37 smithi169 ceph-mon[159415]: pgmap v576: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 476 op/s; 8472/149889 objects degraded (5.652%); 8223/149889 objects misplaced (5.486%); 1.4 MiB/s, 200 objects/s recovering 2023-08-28T02:29:38.058 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:29:38.101 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:29:38.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:38.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:29:38.206 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:29:38.240 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:38.674 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:29:38.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:38.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:29:38.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:38 smithi169 ceph-mon[159415]: 6.7 deep-scrub starts 2023-08-28T02:29:38.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:38 smithi169 ceph-mon[159415]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:29:38.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:38 smithi169 ceph-mon[159415]: failed to encode map e242 with expected crc 2023-08-28T02:29:38.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:38 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1333738446' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:29:38.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:38 smithi169 ceph-mon[159415]: osdmap e242: 8 total, 8 up, 8 in 2023-08-28T02:29:38.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:38 smithi169 ceph-mon[159415]: failed to encode map e242 with expected crc 2023-08-28T02:29:38.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:38 smithi169 ceph-mon[159415]: failed to encode map e242 with expected crc 2023-08-28T02:29:38.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:38 smithi169 ceph-mon[159415]: failed to encode map e242 with expected crc 2023-08-28T02:29:38.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:38 smithi169 ceph-mon[159415]: failed to encode map e242 with expected crc 2023-08-28T02:29:38.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:38 smithi169 ceph-mon[159415]: failed to encode map e242 with expected crc 2023-08-28T02:29:38.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:38 smithi169 ceph-mon[159415]: failed to encode map e242 with expected crc 2023-08-28T02:29:38.772 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:38 smithi143 ceph-mon[183059]: 6.7 deep-scrub starts 2023-08-28T02:29:38.772 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:38 smithi143 ceph-mon[183059]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:29:38.773 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:38 smithi143 ceph-mon[183059]: failed to encode map e242 with expected crc 2023-08-28T02:29:38.773 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:38 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1333738446' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:29:38.773 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:38 smithi143 ceph-mon[183059]: osdmap e242: 8 total, 8 up, 8 in 2023-08-28T02:29:38.773 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:38 smithi143 ceph-mon[183059]: failed to encode map e242 with expected crc 2023-08-28T02:29:38.773 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:38 smithi143 ceph-mon[183059]: failed to encode map e242 with expected crc 2023-08-28T02:29:38.773 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:38 smithi143 ceph-mon[183059]: failed to encode map e242 with expected crc 2023-08-28T02:29:38.773 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:38 smithi143 ceph-mon[183059]: failed to encode map e242 with expected crc 2023-08-28T02:29:38.773 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:38 smithi143 ceph-mon[183059]: failed to encode map e242 with expected crc 2023-08-28T02:29:38.773 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:38 smithi143 ceph-mon[183059]: failed to encode map e242 with expected crc 2023-08-28T02:29:38.774 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:38 smithi143 ceph-mon[189871]: 6.7 deep-scrub starts 2023-08-28T02:29:38.774 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:38 smithi143 ceph-mon[189871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:29:38.774 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:38 smithi143 ceph-mon[189871]: failed to encode map e242 with expected crc 2023-08-28T02:29:38.774 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:38 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1333738446' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:29:38.774 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:38 smithi143 ceph-mon[189871]: osdmap e242: 8 total, 8 up, 8 in 2023-08-28T02:29:38.774 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:38 smithi143 ceph-mon[189871]: failed to encode map e242 with expected crc 2023-08-28T02:29:38.774 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:38 smithi143 ceph-mon[189871]: failed to encode map e242 with expected crc 2023-08-28T02:29:38.774 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:38 smithi143 ceph-mon[189871]: failed to encode map e242 with expected crc 2023-08-28T02:29:38.774 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:38 smithi143 ceph-mon[189871]: failed to encode map e242 with expected crc 2023-08-28T02:29:38.774 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:38 smithi143 ceph-mon[189871]: failed to encode map e242 with expected crc 2023-08-28T02:29:38.774 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:38 smithi143 ceph-mon[189871]: failed to encode map e242 with expected crc 2023-08-28T02:29:38.792 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:29:38.843 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:38.899 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:29:38.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:38.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:29:39.007 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:29:39.041 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:39.147 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:29:39.147 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:29:39.542 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:29:39.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:39.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:29:39.646 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:29:39.672 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:39.713 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:29:39.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:39.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:29:39.821 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:29:39.846 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:39 smithi169 ceph-mon[159415]: failed to encode map e243 with expected crc 2023-08-28T02:29:39.846 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:39 smithi169 ceph-mon[159415]: osdmap e243: 8 total, 8 up, 8 in 2023-08-28T02:29:39.846 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:39 smithi169 ceph-mon[159415]: failed to encode map e243 with expected crc 2023-08-28T02:29:39.846 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:39 smithi169 ceph-mon[159415]: failed to encode map e243 with expected crc 2023-08-28T02:29:39.846 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:39 smithi169 ceph-mon[159415]: failed to encode map e243 with expected crc 2023-08-28T02:29:39.846 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:39 smithi169 ceph-mon[159415]: failed to encode map e243 with expected crc 2023-08-28T02:29:39.846 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:39 smithi169 ceph-mon[159415]: pgmap v579: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 455 op/s; 8342/146895 objects degraded (5.679%); 7609/146895 objects misplaced (5.180%); 5.9 MiB/s, 187 objects/s recovering 2023-08-28T02:29:39.848 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:39.890 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:29:39.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:39.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:29:39.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:39 smithi143 ceph-mon[183059]: failed to encode map e243 with expected crc 2023-08-28T02:29:39.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:39 smithi143 ceph-mon[183059]: osdmap e243: 8 total, 8 up, 8 in 2023-08-28T02:29:39.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:39 smithi143 ceph-mon[183059]: failed to encode map e243 with expected crc 2023-08-28T02:29:39.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:39 smithi143 ceph-mon[183059]: failed to encode map e243 with expected crc 2023-08-28T02:29:39.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:39 smithi143 ceph-mon[183059]: failed to encode map e243 with expected crc 2023-08-28T02:29:39.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:39 smithi143 ceph-mon[183059]: failed to encode map e243 with expected crc 2023-08-28T02:29:39.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:39 smithi143 ceph-mon[183059]: pgmap v579: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 455 op/s; 8342/146895 objects degraded (5.679%); 7609/146895 objects misplaced (5.180%); 5.9 MiB/s, 187 objects/s recovering 2023-08-28T02:29:39.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:39 smithi143 ceph-mon[189871]: failed to encode map e243 with expected crc 2023-08-28T02:29:39.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:39 smithi143 ceph-mon[189871]: osdmap e243: 8 total, 8 up, 8 in 2023-08-28T02:29:39.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:39 smithi143 ceph-mon[189871]: failed to encode map e243 with expected crc 2023-08-28T02:29:39.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:39 smithi143 ceph-mon[189871]: failed to encode map e243 with expected crc 2023-08-28T02:29:39.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:39 smithi143 ceph-mon[189871]: failed to encode map e243 with expected crc 2023-08-28T02:29:39.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:39 smithi143 ceph-mon[189871]: failed to encode map e243 with expected crc 2023-08-28T02:29:39.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:39 smithi143 ceph-mon[189871]: pgmap v579: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 455 op/s; 8342/146895 objects degraded (5.679%); 7609/146895 objects misplaced (5.180%); 5.9 MiB/s, 187 objects/s recovering 2023-08-28T02:29:39.995 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:29:40.025 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:40.074 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:29:40.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:40.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:29:40.092 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.2 dump_ops_in_flight 2023-08-28T02:29:40.153 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:29:40.183 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:29:40.239 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:40.313 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:29:40.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:40.324 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:29:40.425 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:29:40.452 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:40.507 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:29:40.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:40.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:29:40.553 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:40 smithi143 ceph-mon[183059]: failed to encode map e244 with expected crc 2023-08-28T02:29:40.554 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:40 smithi143 ceph-mon[183059]: failed to encode map e244 with expected crc 2023-08-28T02:29:40.554 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:40 smithi143 ceph-mon[183059]: osdmap e244: 8 total, 8 up, 8 in 2023-08-28T02:29:40.554 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:40 smithi143 ceph-mon[183059]: failed to encode map e244 with expected crc 2023-08-28T02:29:40.554 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:40 smithi143 ceph-mon[183059]: failed to encode map e244 with expected crc 2023-08-28T02:29:40.554 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:40 smithi143 ceph-mon[183059]: failed to encode map e244 with expected crc 2023-08-28T02:29:40.556 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:40 smithi143 ceph-mon[189871]: failed to encode map e244 with expected crc 2023-08-28T02:29:40.556 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:40 smithi143 ceph-mon[189871]: failed to encode map e244 with expected crc 2023-08-28T02:29:40.556 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:40 smithi143 ceph-mon[189871]: osdmap e244: 8 total, 8 up, 8 in 2023-08-28T02:29:40.556 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:40 smithi143 ceph-mon[189871]: failed to encode map e244 with expected crc 2023-08-28T02:29:40.556 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:40 smithi143 ceph-mon[189871]: failed to encode map e244 with expected crc 2023-08-28T02:29:40.556 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:40 smithi143 ceph-mon[189871]: failed to encode map e244 with expected crc 2023-08-28T02:29:40.613 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:29:40.640 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:40.688 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:29:40.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:40.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:29:40.699 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set nodeep-scrub 2023-08-28T02:29:40.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:40 smithi169 ceph-mon[159415]: failed to encode map e244 with expected crc 2023-08-28T02:29:40.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:40 smithi169 ceph-mon[159415]: failed to encode map e244 with expected crc 2023-08-28T02:29:40.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:40 smithi169 ceph-mon[159415]: osdmap e244: 8 total, 8 up, 8 in 2023-08-28T02:29:40.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:40 smithi169 ceph-mon[159415]: failed to encode map e244 with expected crc 2023-08-28T02:29:40.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:40 smithi169 ceph-mon[159415]: failed to encode map e244 with expected crc 2023-08-28T02:29:40.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:40 smithi169 ceph-mon[159415]: failed to encode map e244 with expected crc 2023-08-28T02:29:40.796 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:29:40.824 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:40.879 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:29:40.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:40.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:29:40.987 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:29:41.039 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:41.413 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:29:41.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:41.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:29:41.470 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:29:41.521 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:29:41.547 INFO:teuthology.orchestra.run.smithi143.stderr:reweighted osd.5 to 1 (10000) 2023-08-28T02:29:41.614 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:41.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:41 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1097706118' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-08-28T02:29:41.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:41 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-08-28T02:29:41.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:41 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:29:41.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:41 smithi143 ceph-mon[183059]: pgmap v581: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 384 op/s; 8209/146046 objects degraded (5.621%); 7480/146046 objects misplaced (5.122%); 4.9 MiB/s, 155 objects/s recovering 2023-08-28T02:29:41.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:41 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 8342/146895 objects degraded (5.679%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-08-28T02:29:41.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:41 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1097706118' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-08-28T02:29:41.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:41 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-08-28T02:29:41.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:41 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:29:41.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:41 smithi143 ceph-mon[189871]: pgmap v581: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 384 op/s; 8209/146046 objects degraded (5.621%); 7480/146046 objects misplaced (5.122%); 4.9 MiB/s, 155 objects/s recovering 2023-08-28T02:29:41.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:41 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 8342/146895 objects degraded (5.679%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-08-28T02:29:41.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:41 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1097706118' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-08-28T02:29:41.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:41 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-08-28T02:29:41.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:41 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:29:41.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:41 smithi169 ceph-mon[159415]: pgmap v581: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 384 op/s; 8209/146046 objects degraded (5.621%); 7480/146046 objects misplaced (5.122%); 4.9 MiB/s, 155 objects/s recovering 2023-08-28T02:29:41.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:41 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 8342/146895 objects degraded (5.679%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-08-28T02:29:42.156 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:29:42.157 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:29:41 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:29:41] "GET /metrics HTTP/1.1" 200 36495 "" "Prometheus/2.33.4" 2023-08-28T02:29:42.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:42.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:29:42.259 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:29:42.294 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:42.351 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:29:42.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:42.361 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:29:42.462 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:29:42.514 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:42.583 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd reweight 7 1 2023-08-28T02:29:42.612 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:29:42.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:42.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:29:42.716 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:29:42.724 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:29:42.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:42 smithi169 ceph-mon[159415]: failed to encode map e245 with expected crc 2023-08-28T02:29:42.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:42 smithi169 ceph-mon[159415]: failed to encode map e245 with expected crc 2023-08-28T02:29:42.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:42 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-08-28T02:29:42.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:42 smithi169 ceph-mon[159415]: osdmap e245: 8 total, 8 up, 8 in 2023-08-28T02:29:42.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:42 smithi169 ceph-mon[159415]: failed to encode map e245 with expected crc 2023-08-28T02:29:42.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:42 smithi169 ceph-mon[159415]: failed to encode map e245 with expected crc 2023-08-28T02:29:42.756 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:42.817 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:42 smithi143 ceph-mon[183059]: failed to encode map e245 with expected crc 2023-08-28T02:29:42.817 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:42 smithi143 ceph-mon[183059]: failed to encode map e245 with expected crc 2023-08-28T02:29:42.817 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:42 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-08-28T02:29:42.817 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:42 smithi143 ceph-mon[183059]: osdmap e245: 8 total, 8 up, 8 in 2023-08-28T02:29:42.817 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:42 smithi143 ceph-mon[183059]: failed to encode map e245 with expected crc 2023-08-28T02:29:42.817 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:42 smithi143 ceph-mon[183059]: failed to encode map e245 with expected crc 2023-08-28T02:29:42.819 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:42 smithi143 ceph-mon[189871]: failed to encode map e245 with expected crc 2023-08-28T02:29:42.819 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:42 smithi143 ceph-mon[189871]: failed to encode map e245 with expected crc 2023-08-28T02:29:42.820 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:42 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-08-28T02:29:42.820 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:42 smithi143 ceph-mon[189871]: osdmap e245: 8 total, 8 up, 8 in 2023-08-28T02:29:42.820 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:42 smithi143 ceph-mon[189871]: failed to encode map e245 with expected crc 2023-08-28T02:29:42.820 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:42 smithi143 ceph-mon[189871]: failed to encode map e245 with expected crc 2023-08-28T02:29:42.845 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:29:42.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:42.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:29:42.899 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:29:42.935 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:29:42.960 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:29:42.965 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:29:42.989 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:43.006 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:29:43.037 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:29:43.049 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:29:43.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:43.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:29:43.086 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:29:43.101 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:29:43.126 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:29:43.128 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:29:43.128 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:29:43.128 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:29:43.128 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:29:43.128 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:29:43.128 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:29:43.129 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:29:43.129 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:29:43.155 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:29:43.186 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:43.245 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:29:43.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:43.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:29:43.349 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:29:43.377 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:43.440 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:29:43.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:43.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:29:43.546 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:29:43.574 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:43.629 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:29:43.632 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:43 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/4107064621' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:29:43.633 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:43 smithi169 ceph-mon[159415]: pgmap v583: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 395 op/s; 8198/143022 objects degraded (5.732%); 7480/143022 objects misplaced (5.230%); 5.6 MiB/s, 102 objects/s recovering 2023-08-28T02:29:43.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:43.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:29:43.734 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:29:43.763 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:43.807 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:29:43.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:43.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:29:43.915 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:29:43.945 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:43.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:43 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/4107064621' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:29:43.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:43 smithi143 ceph-mon[183059]: pgmap v583: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 395 op/s; 8198/143022 objects degraded (5.732%); 7480/143022 objects misplaced (5.230%); 5.6 MiB/s, 102 objects/s recovering 2023-08-28T02:29:43.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:43 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/4107064621' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:29:43.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:43 smithi143 ceph-mon[189871]: pgmap v583: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 395 op/s; 8198/143022 objects degraded (5.732%); 7480/143022 objects misplaced (5.230%); 5.6 MiB/s, 102 objects/s recovering 2023-08-28T02:29:43.997 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:29:44.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:44.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:29:44.103 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:29:44.134 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:44.371 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:29:44.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:44.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:29:44.481 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:29:44.510 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:44.554 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:29:44.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:44.560 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:29:44.661 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:29:44.713 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:44.809 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:29:44.836 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:29:44.836 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [ 2023-08-28T02:29:44.836 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:44.836 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.35076.0:121688 6.10 e245/209)", 2023-08-28T02:29:44.836 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:29:44.834337+0000", 2023-08-28T02:29:44.836 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.0025986189999999999, 2023-08-28T02:29:44.837 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.002620275, 2023-08-28T02:29:44.837 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:29:44.837 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "queued for pg", 2023-08-28T02:29:44.837 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:29:44.837 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:44.837 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:29:44.837 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:44.834337+0000", 2023-08-28T02:29:44.837 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:29:44.837 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:44.837 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:44.837 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:29:44.838 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:44.834337+0000", 2023-08-28T02:29:44.838 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:29:44.838 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:44.838 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:44.838 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:29:44.838 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:44.834336+0000", 2023-08-28T02:29:44.838 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:29:44.838 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:44.838 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:44.838 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:29:44.838 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:44.834346+0000", 2023-08-28T02:29:44.839 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0193999999999999e-05 2023-08-28T02:29:44.839 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:44.839 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:44.839 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:29:44.839 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:44.834367+0000", 2023-08-28T02:29:44.839 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.0615000000000002e-05 2023-08-28T02:29:44.839 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:29:44.839 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:29:44.839 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:29:44.839 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:44.834376+0000", 2023-08-28T02:29:44.839 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.7479999999999999e-06 2023-08-28T02:29:44.840 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:29:44.840 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:29:44.840 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:29:44.840 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:29:44.840 INFO:teuthology.orchestra.run.smithi143.stdout: ], 2023-08-28T02:29:44.840 INFO:teuthology.orchestra.run.smithi143.stdout: "num_ops": 1 2023-08-28T02:29:44.840 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:29:46.054 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:29:46.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:46.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:29:46.086 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.2 dump_blocked_ops 2023-08-28T02:29:46.167 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:29:46.205 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:46.292 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:29:46.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:46.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:29:46.381 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:29:46.398 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:29:46.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:46 smithi143 ceph-mon[183059]: pgmap v584: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 2.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 531 op/s; 8034/139056 objects degraded (5.778%); 7061/139056 objects misplaced (5.078%); 6.1 MiB/s, 109 objects/s recovering 2023-08-28T02:29:46.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:46 smithi143 ceph-mon[189871]: pgmap v584: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 2.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 531 op/s; 8034/139056 objects degraded (5.778%); 7061/139056 objects misplaced (5.078%); 6.1 MiB/s, 109 objects/s recovering 2023-08-28T02:29:46.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:46 smithi169 ceph-mon[159415]: pgmap v584: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 2.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 531 op/s; 8034/139056 objects degraded (5.778%); 7061/139056 objects misplaced (5.078%); 6.1 MiB/s, 109 objects/s recovering 2023-08-28T02:29:46.541 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:46.672 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:29:46.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:46.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:29:46.779 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:29:46.810 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:47.412 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:29:47.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:47.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:29:47.428 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is set 2023-08-28T02:29:47.520 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:29:47.607 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:47 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 8034/139056 objects degraded (5.778%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-08-28T02:29:47.607 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:47 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1695701403' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:29:47.607 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:47 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:29:47.608 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:47 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 8034/139056 objects degraded (5.778%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-08-28T02:29:47.608 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:47 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1695701403' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:29:47.608 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:47 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:29:47.614 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:47.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:47 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 8034/139056 objects degraded (5.778%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-08-28T02:29:47.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:47 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1695701403' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:29:47.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:47 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:29:48.100 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:29:48.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:48.105 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:29:48.206 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:29:48.259 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:48.302 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:29:48.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:48.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:29:48.407 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:29:48.437 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:48 smithi143 ceph-mon[189871]: pgmap v585: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 486 op/s; 8034/135210 objects degraded (5.942%); 7061/135210 objects misplaced (5.222%); 2.7 MiB/s, 36 objects/s recovering 2023-08-28T02:29:48.437 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:48 smithi143 ceph-mon[189871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:29:48.437 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:48 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:29:48.438 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:48 smithi143 ceph-mon[189871]: osdmap e246: 8 total, 8 up, 8 in 2023-08-28T02:29:48.438 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:48 smithi143 ceph-mon[189871]: failed to encode map e246 with expected crc 2023-08-28T02:29:48.438 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:48 smithi143 ceph-mon[189871]: failed to encode map e246 with expected crc 2023-08-28T02:29:48.438 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:48 smithi143 ceph-mon[183059]: pgmap v585: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 486 op/s; 8034/135210 objects degraded (5.942%); 7061/135210 objects misplaced (5.222%); 2.7 MiB/s, 36 objects/s recovering 2023-08-28T02:29:48.438 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:48 smithi143 ceph-mon[183059]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:29:48.438 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:48 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:29:48.438 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:48 smithi143 ceph-mon[183059]: osdmap e246: 8 total, 8 up, 8 in 2023-08-28T02:29:48.439 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:48 smithi143 ceph-mon[183059]: failed to encode map e246 with expected crc 2023-08-28T02:29:48.439 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:48 smithi143 ceph-mon[183059]: failed to encode map e246 with expected crc 2023-08-28T02:29:48.445 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:48.672 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:29:48.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:48.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:29:48.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:48 smithi169 ceph-mon[159415]: pgmap v585: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 486 op/s; 8034/135210 objects degraded (5.942%); 7061/135210 objects misplaced (5.222%); 2.7 MiB/s, 36 objects/s recovering 2023-08-28T02:29:48.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:48 smithi169 ceph-mon[159415]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:29:48.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:48 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:29:48.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:48 smithi169 ceph-mon[159415]: osdmap e246: 8 total, 8 up, 8 in 2023-08-28T02:29:48.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:48 smithi169 ceph-mon[159415]: failed to encode map e246 with expected crc 2023-08-28T02:29:48.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:48 smithi169 ceph-mon[159415]: failed to encode map e246 with expected crc 2023-08-28T02:29:48.781 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:29:48.890 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:48.937 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:29:49.013 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:29:49.044 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:29:49.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:49.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:29:49.149 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:29:49.212 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:49.567 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:29:49.577 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:49 smithi143 ceph-mon[183059]: failed to encode map e246 with expected crc 2023-08-28T02:29:49.578 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:49 smithi143 ceph-mon[183059]: failed to encode map e246 with expected crc 2023-08-28T02:29:49.578 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:49 smithi143 ceph-mon[189871]: failed to encode map e246 with expected crc 2023-08-28T02:29:49.578 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:49 smithi143 ceph-mon[189871]: failed to encode map e246 with expected crc 2023-08-28T02:29:49.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:49.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:29:49.680 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:29:49.731 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:49 smithi169 ceph-mon[159415]: failed to encode map e246 with expected crc 2023-08-28T02:29:49.731 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:49 smithi169 ceph-mon[159415]: failed to encode map e246 with expected crc 2023-08-28T02:29:49.735 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:49.825 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:29:49.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:49.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:29:49.932 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:29:49.999 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:50.122 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:29:50.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:50.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:29:50.233 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:29:50.322 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:50.398 INFO:teuthology.orchestra.run.smithi143.stderr:reweighted osd.7 to 1 (10000) 2023-08-28T02:29:50.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:50 smithi143 ceph-mon[183059]: pgmap v587: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 8034/130365 objects degraded (6.163%); 6603/130365 objects misplaced (5.065%); 1.7 MiB/s, 19 objects/s recovering 2023-08-28T02:29:50.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:50 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3027006319' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-08-28T02:29:50.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:50 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:29:50.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:50 smithi143 ceph-mon[189871]: pgmap v587: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 8034/130365 objects degraded (6.163%); 6603/130365 objects misplaced (5.065%); 1.7 MiB/s, 19 objects/s recovering 2023-08-28T02:29:50.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:50 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3027006319' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-08-28T02:29:50.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:50 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:29:50.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:50 smithi169 ceph-mon[159415]: pgmap v587: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 8034/130365 objects degraded (6.163%); 6603/130365 objects misplaced (5.065%); 1.7 MiB/s, 19 objects/s recovering 2023-08-28T02:29:50.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:50 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3027006319' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-08-28T02:29:50.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:50 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:29:50.832 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset noscrub 2023-08-28T02:29:51.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:51 smithi143 ceph-mon[183059]: failed to encode map e247 with expected crc 2023-08-28T02:29:51.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:51 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3027006319' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-08-28T02:29:51.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:51 smithi143 ceph-mon[183059]: osdmap e247: 8 total, 8 up, 8 in 2023-08-28T02:29:51.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:51 smithi143 ceph-mon[183059]: failed to encode map e247 with expected crc 2023-08-28T02:29:51.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:51 smithi143 ceph-mon[183059]: failed to encode map e247 with expected crc 2023-08-28T02:29:51.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:51 smithi143 ceph-mon[183059]: failed to encode map e247 with expected crc 2023-08-28T02:29:51.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:51 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 8034/130365 objects degraded (6.163%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-08-28T02:29:51.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:51 smithi143 ceph-mon[189871]: failed to encode map e247 with expected crc 2023-08-28T02:29:51.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:51 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3027006319' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-08-28T02:29:51.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:51 smithi143 ceph-mon[189871]: osdmap e247: 8 total, 8 up, 8 in 2023-08-28T02:29:51.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:51 smithi143 ceph-mon[189871]: failed to encode map e247 with expected crc 2023-08-28T02:29:51.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:51 smithi143 ceph-mon[189871]: failed to encode map e247 with expected crc 2023-08-28T02:29:51.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:51 smithi143 ceph-mon[189871]: failed to encode map e247 with expected crc 2023-08-28T02:29:51.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:51 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 8034/130365 objects degraded (6.163%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-08-28T02:29:51.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:51 smithi169 ceph-mon[159415]: failed to encode map e247 with expected crc 2023-08-28T02:29:51.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:51 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3027006319' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-08-28T02:29:51.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:51 smithi169 ceph-mon[159415]: osdmap e247: 8 total, 8 up, 8 in 2023-08-28T02:29:51.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:51 smithi169 ceph-mon[159415]: failed to encode map e247 with expected crc 2023-08-28T02:29:51.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:51 smithi169 ceph-mon[159415]: failed to encode map e247 with expected crc 2023-08-28T02:29:51.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:51 smithi169 ceph-mon[159415]: failed to encode map e247 with expected crc 2023-08-28T02:29:51.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:51 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 8034/130365 objects degraded (6.163%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-08-28T02:29:52.101 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:29:52.107 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:29:51 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:29:51] "GET /metrics HTTP/1.1" 200 36495 "" "Prometheus/2.33.4" 2023-08-28T02:29:52.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:52.109 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:29:52.210 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:29:52.283 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:52.389 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:52 smithi143 ceph-mon[189871]: pgmap v589: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 7977/128934 objects degraded (6.187%); 6204/128934 objects misplaced (4.812%); 4.4 MiB/s, 70 objects/s recovering 2023-08-28T02:29:52.579 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:29:52.598 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd reweight 3 1 2023-08-28T02:29:52.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:52.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:29:52.660 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:52 smithi143 ceph-mon[183059]: pgmap v589: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 7977/128934 objects degraded (6.187%); 6204/128934 objects misplaced (4.812%); 4.4 MiB/s, 70 objects/s recovering 2023-08-28T02:29:52.700 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:29:52.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:52 smithi169 ceph-mon[159415]: pgmap v589: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 7977/128934 objects degraded (6.187%); 6204/128934 objects misplaced (4.812%); 4.4 MiB/s, 70 objects/s recovering 2023-08-28T02:29:52.756 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:52.881 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:29:52.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:52.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:29:52.985 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:29:53.058 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:29:53.088 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:53.094 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:29:53.144 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:29:53.192 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:29:53.198 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:29:53.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:53.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:29:53.204 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:29:53.239 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:29:53.294 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:29:53.303 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:29:53.327 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:29:53.359 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:29:53.359 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:29:53.370 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:29:53.383 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:29:53.383 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:29:53.383 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:29:53.383 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:29:53.383 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:29:53.383 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:29:53.383 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:29:53.383 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:29:53.413 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:53.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:53 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2529794659' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:29:53.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:53 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2529794659' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:29:53.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:53 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2529794659' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:29:54.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:54 smithi143 ceph-mon[189871]: pgmap v590: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 7977/123564 objects degraded (6.456%); 6204/123564 objects misplaced (5.021%); 3.2 MiB/s, 51 objects/s recovering 2023-08-28T02:29:54.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:54 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:29:54.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:54 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:29:54.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:54 smithi143 ceph-mon[183059]: pgmap v590: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 7977/123564 objects degraded (6.456%); 6204/123564 objects misplaced (5.021%); 3.2 MiB/s, 51 objects/s recovering 2023-08-28T02:29:54.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:54 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:29:54.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:54 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:29:54.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:54 smithi169 ceph-mon[159415]: pgmap v590: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 7977/123564 objects degraded (6.456%); 6204/123564 objects misplaced (5.021%); 3.2 MiB/s, 51 objects/s recovering 2023-08-28T02:29:54.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:54 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:29:54.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:54 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:29:54.779 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:29:54.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:54.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:29:54.884 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:29:54.914 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:55.457 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:29:55.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:55.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:29:55.514 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.2 dump_historic_ops 2023-08-28T02:29:55.564 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:29:55.636 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:29:55.646 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:55.765 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:29:55.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:55.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:29:55.875 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:29:55.923 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:55.967 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:29:55.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:55.973 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:29:56.073 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:29:56.100 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:56.435 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:56 smithi143 ceph-mon[183059]: pgmap v591: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 7977/116679 objects degraded (6.837%); 5157/116679 objects misplaced (4.420%); 7.4 MiB/s, 118 objects/s recovering 2023-08-28T02:29:56.436 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:56 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:29:56.436 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:56 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 7977/116679 objects degraded (6.837%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-08-28T02:29:56.436 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:56 smithi143 ceph-mon[189871]: pgmap v591: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 7977/116679 objects degraded (6.837%); 5157/116679 objects misplaced (4.420%); 7.4 MiB/s, 118 objects/s recovering 2023-08-28T02:29:56.436 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:56 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:29:56.436 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:56 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 7977/116679 objects degraded (6.837%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-08-28T02:29:56.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:56 smithi169 ceph-mon[159415]: pgmap v591: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 7977/116679 objects degraded (6.837%); 5157/116679 objects misplaced (4.420%); 7.4 MiB/s, 118 objects/s recovering 2023-08-28T02:29:56.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:56 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:29:56.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:56 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 7977/116679 objects degraded (6.837%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-08-28T02:29:56.994 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:29:57.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:57.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:29:57.118 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:29:57.204 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:57.300 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:29:57.504 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:29:57.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:57.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:29:57.612 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:29:57.658 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:57.702 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:29:57.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:57.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:29:57.807 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:29:57.889 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:58.474 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:29:58.478 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is unset 2023-08-28T02:29:58.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:58.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:29:58.580 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:29:58.594 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:29:58.719 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:58 smithi143 ceph-mon[183059]: pgmap v592: 99 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 7975/111228 objects degraded (7.170%); 5157/111228 objects misplaced (4.636%); 6.0 MiB/s, 95 objects/s recovering 2023-08-28T02:29:58.719 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:58 smithi143 ceph-mon[183059]: failed to encode map e248 with expected crc 2023-08-28T02:29:58.719 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:58 smithi143 ceph-mon[183059]: osdmap e248: 8 total, 8 up, 8 in 2023-08-28T02:29:58.720 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:58 smithi143 ceph-mon[183059]: failed to encode map e248 with expected crc 2023-08-28T02:29:58.720 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:58 smithi143 ceph-mon[183059]: failed to encode map e248 with expected crc 2023-08-28T02:29:58.720 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:58 smithi143 ceph-mon[183059]: failed to encode map e248 with expected crc 2023-08-28T02:29:58.720 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:58 smithi143 ceph-mon[183059]: failed to encode map e248 with expected crc 2023-08-28T02:29:58.720 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:58 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1564026911' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:29:58.720 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:58 smithi143 ceph-mon[189871]: pgmap v592: 99 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 7975/111228 objects degraded (7.170%); 5157/111228 objects misplaced (4.636%); 6.0 MiB/s, 95 objects/s recovering 2023-08-28T02:29:58.720 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:58 smithi143 ceph-mon[189871]: failed to encode map e248 with expected crc 2023-08-28T02:29:58.720 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:58 smithi143 ceph-mon[189871]: osdmap e248: 8 total, 8 up, 8 in 2023-08-28T02:29:58.721 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:58 smithi143 ceph-mon[189871]: failed to encode map e248 with expected crc 2023-08-28T02:29:58.721 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:58 smithi143 ceph-mon[189871]: failed to encode map e248 with expected crc 2023-08-28T02:29:58.721 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:58 smithi143 ceph-mon[189871]: failed to encode map e248 with expected crc 2023-08-28T02:29:58.721 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:58 smithi143 ceph-mon[189871]: failed to encode map e248 with expected crc 2023-08-28T02:29:58.721 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:58 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1564026911' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:29:58.721 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:58.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:58 smithi169 ceph-mon[159415]: pgmap v592: 99 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 7975/111228 objects degraded (7.170%); 5157/111228 objects misplaced (4.636%); 6.0 MiB/s, 95 objects/s recovering 2023-08-28T02:29:58.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:58 smithi169 ceph-mon[159415]: failed to encode map e248 with expected crc 2023-08-28T02:29:58.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:58 smithi169 ceph-mon[159415]: osdmap e248: 8 total, 8 up, 8 in 2023-08-28T02:29:58.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:58 smithi169 ceph-mon[159415]: failed to encode map e248 with expected crc 2023-08-28T02:29:58.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:58 smithi169 ceph-mon[159415]: failed to encode map e248 with expected crc 2023-08-28T02:29:58.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:58 smithi169 ceph-mon[159415]: failed to encode map e248 with expected crc 2023-08-28T02:29:58.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:58 smithi169 ceph-mon[159415]: failed to encode map e248 with expected crc 2023-08-28T02:29:58.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:58 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1564026911' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:29:58.936 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:29:58.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:58.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:29:59.041 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:29:59.118 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:29:59.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:59 smithi169 ceph-mon[159415]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:29:59.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:59 smithi169 ceph-mon[159415]: failed to encode map e249 with expected crc 2023-08-28T02:29:59.741 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:59 smithi169 ceph-mon[159415]: failed to encode map e249 with expected crc 2023-08-28T02:29:59.741 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:59 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1564026911' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:29:59.741 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:59 smithi169 ceph-mon[159415]: osdmap e249: 8 total, 8 up, 8 in 2023-08-28T02:29:59.741 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:59 smithi169 ceph-mon[159415]: failed to encode map e249 with expected crc 2023-08-28T02:29:59.741 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:59 smithi169 ceph-mon[159415]: failed to encode map e249 with expected crc 2023-08-28T02:29:59.741 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:59 smithi169 ceph-mon[159415]: failed to encode map e249 with expected crc 2023-08-28T02:29:59.741 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:59 smithi169 ceph-mon[159415]: failed to encode map e249 with expected crc 2023-08-28T02:29:59.741 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:29:59 smithi169 ceph-mon[159415]: failed to encode map e249 with expected crc 2023-08-28T02:29:59.792 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:29:59.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:29:59.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:29:59.804 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:59 smithi143 ceph-mon[183059]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:29:59.804 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:59 smithi143 ceph-mon[183059]: failed to encode map e249 with expected crc 2023-08-28T02:29:59.805 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:59 smithi143 ceph-mon[183059]: failed to encode map e249 with expected crc 2023-08-28T02:29:59.805 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:59 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1564026911' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:29:59.805 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:59 smithi143 ceph-mon[183059]: osdmap e249: 8 total, 8 up, 8 in 2023-08-28T02:29:59.805 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:59 smithi143 ceph-mon[183059]: failed to encode map e249 with expected crc 2023-08-28T02:29:59.805 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:59 smithi143 ceph-mon[183059]: failed to encode map e249 with expected crc 2023-08-28T02:29:59.805 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:59 smithi143 ceph-mon[183059]: failed to encode map e249 with expected crc 2023-08-28T02:29:59.805 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:59 smithi143 ceph-mon[183059]: failed to encode map e249 with expected crc 2023-08-28T02:29:59.805 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:29:59 smithi143 ceph-mon[183059]: failed to encode map e249 with expected crc 2023-08-28T02:29:59.805 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:59 smithi143 ceph-mon[189871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:29:59.805 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:59 smithi143 ceph-mon[189871]: failed to encode map e249 with expected crc 2023-08-28T02:29:59.806 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:59 smithi143 ceph-mon[189871]: failed to encode map e249 with expected crc 2023-08-28T02:29:59.806 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:59 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1564026911' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:29:59.806 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:59 smithi143 ceph-mon[189871]: osdmap e249: 8 total, 8 up, 8 in 2023-08-28T02:29:59.806 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:59 smithi143 ceph-mon[189871]: failed to encode map e249 with expected crc 2023-08-28T02:29:59.806 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:59 smithi143 ceph-mon[189871]: failed to encode map e249 with expected crc 2023-08-28T02:29:59.806 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:59 smithi143 ceph-mon[189871]: failed to encode map e249 with expected crc 2023-08-28T02:29:59.806 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:59 smithi143 ceph-mon[189871]: failed to encode map e249 with expected crc 2023-08-28T02:29:59.806 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:29:59 smithi143 ceph-mon[189871]: failed to encode map e249 with expected crc 2023-08-28T02:29:59.905 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:29:59.998 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:00.168 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:30:00.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:00.176 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:30:00.277 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:30:00.312 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:30:00.380 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:00.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:00 smithi143 ceph-mon[183059]: pgmap v595: 99 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 2.2 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 948 op/s; 7975/106161 objects degraded (7.512%); 4671/106161 objects misplaced (4.400%); 4.2 MiB/s, 66 objects/s recovering 2023-08-28T02:30:00.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:00 smithi143 ceph-mon[183059]: 6.2 scrub starts 2023-08-28T02:30:00.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:00 smithi143 ceph-mon[183059]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Degraded data redundancy: 7975/106161 objects degraded (7.512%), 6 pgs degraded, 7 pgs undersized 2023-08-28T02:30:00.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:00 smithi143 ceph-mon[183059]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-08-28T02:30:00.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:00 smithi143 ceph-mon[183059]: [WRN] PG_DEGRADED: Degraded data redundancy: 7975/106161 objects degraded (7.512%), 6 pgs degraded, 7 pgs undersized 2023-08-28T02:30:00.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:00 smithi143 ceph-mon[183059]: pg 1.0 is stuck undersized for 93s, current state active+undersized+remapped, last acting [7,0] 2023-08-28T02:30:00.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:00 smithi143 ceph-mon[183059]: pg 6.1 is stuck undersized for 92s, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,2] 2023-08-28T02:30:00.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:00 smithi143 ceph-mon[183059]: pg 6.3 is stuck undersized for 92s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,2] 2023-08-28T02:30:00.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:00 smithi143 ceph-mon[183059]: pg 6.5 is stuck undersized for 93s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,3] 2023-08-28T02:30:00.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:00 smithi143 ceph-mon[183059]: pg 6.11 is stuck undersized for 92s, current state active+recovering+undersized+degraded+remapped, last acting [1,2] 2023-08-28T02:30:00.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:00 smithi143 ceph-mon[183059]: pg 6.13 is stuck undersized for 92s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,2] 2023-08-28T02:30:00.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:00 smithi143 ceph-mon[183059]: pg 6.15 is stuck undersized for 93s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,3] 2023-08-28T02:30:00.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:00 smithi143 ceph-mon[189871]: pgmap v595: 99 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 2.2 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 948 op/s; 7975/106161 objects degraded (7.512%); 4671/106161 objects misplaced (4.400%); 4.2 MiB/s, 66 objects/s recovering 2023-08-28T02:30:00.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:00 smithi143 ceph-mon[189871]: 6.2 scrub starts 2023-08-28T02:30:00.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:00 smithi143 ceph-mon[189871]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Degraded data redundancy: 7975/106161 objects degraded (7.512%), 6 pgs degraded, 7 pgs undersized 2023-08-28T02:30:00.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:00 smithi143 ceph-mon[189871]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-08-28T02:30:00.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:00 smithi143 ceph-mon[189871]: [WRN] PG_DEGRADED: Degraded data redundancy: 7975/106161 objects degraded (7.512%), 6 pgs degraded, 7 pgs undersized 2023-08-28T02:30:00.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:00 smithi143 ceph-mon[189871]: pg 1.0 is stuck undersized for 93s, current state active+undersized+remapped, last acting [7,0] 2023-08-28T02:30:00.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:00 smithi143 ceph-mon[189871]: pg 6.1 is stuck undersized for 92s, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,2] 2023-08-28T02:30:00.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:00 smithi143 ceph-mon[189871]: pg 6.3 is stuck undersized for 92s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,2] 2023-08-28T02:30:00.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:00 smithi143 ceph-mon[189871]: pg 6.5 is stuck undersized for 93s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,3] 2023-08-28T02:30:00.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:00 smithi143 ceph-mon[189871]: pg 6.11 is stuck undersized for 92s, current state active+recovering+undersized+degraded+remapped, last acting [1,2] 2023-08-28T02:30:00.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:00 smithi143 ceph-mon[189871]: pg 6.13 is stuck undersized for 92s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,2] 2023-08-28T02:30:00.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:00 smithi143 ceph-mon[189871]: pg 6.15 is stuck undersized for 93s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,3] 2023-08-28T02:30:00.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:00 smithi169 ceph-mon[159415]: pgmap v595: 99 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 2.2 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 948 op/s; 7975/106161 objects degraded (7.512%); 4671/106161 objects misplaced (4.400%); 4.2 MiB/s, 66 objects/s recovering 2023-08-28T02:30:00.741 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:00 smithi169 ceph-mon[159415]: 6.2 scrub starts 2023-08-28T02:30:00.741 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:00 smithi169 ceph-mon[159415]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Degraded data redundancy: 7975/106161 objects degraded (7.512%), 6 pgs degraded, 7 pgs undersized 2023-08-28T02:30:00.741 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:00 smithi169 ceph-mon[159415]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-08-28T02:30:00.741 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:00 smithi169 ceph-mon[159415]: [WRN] PG_DEGRADED: Degraded data redundancy: 7975/106161 objects degraded (7.512%), 6 pgs degraded, 7 pgs undersized 2023-08-28T02:30:00.741 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:00 smithi169 ceph-mon[159415]: pg 1.0 is stuck undersized for 93s, current state active+undersized+remapped, last acting [7,0] 2023-08-28T02:30:00.741 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:00 smithi169 ceph-mon[159415]: pg 6.1 is stuck undersized for 92s, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,2] 2023-08-28T02:30:00.741 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:00 smithi169 ceph-mon[159415]: pg 6.3 is stuck undersized for 92s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,2] 2023-08-28T02:30:00.741 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:00 smithi169 ceph-mon[159415]: pg 6.5 is stuck undersized for 93s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,3] 2023-08-28T02:30:00.741 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:00 smithi169 ceph-mon[159415]: pg 6.11 is stuck undersized for 92s, current state active+recovering+undersized+degraded+remapped, last acting [1,2] 2023-08-28T02:30:00.741 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:00 smithi169 ceph-mon[159415]: pg 6.13 is stuck undersized for 92s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,2] 2023-08-28T02:30:00.742 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:00 smithi169 ceph-mon[159415]: pg 6.15 is stuck undersized for 93s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,3] 2023-08-28T02:30:01.393 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:30:01.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:01.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:30:01.507 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:30:01.533 INFO:teuthology.orchestra.run.smithi143.stderr:reweighted osd.3 to 1 (10000) 2023-08-28T02:30:01.606 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:01.690 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:01 smithi143 ceph-mon[183059]: 6.7 scrub starts 2023-08-28T02:30:01.690 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:01 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3743912350' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-08-28T02:30:01.690 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:01 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-08-28T02:30:01.690 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:01 smithi143 ceph-mon[183059]: 6.9 scrub starts 2023-08-28T02:30:01.690 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:01 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 7975/106161 objects degraded (7.512%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-08-28T02:30:01.693 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:01 smithi143 ceph-mon[189871]: 6.7 scrub starts 2023-08-28T02:30:01.693 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:01 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3743912350' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-08-28T02:30:01.693 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:01 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-08-28T02:30:01.693 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:01 smithi143 ceph-mon[189871]: 6.9 scrub starts 2023-08-28T02:30:01.693 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:01 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 7975/106161 objects degraded (7.512%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-08-28T02:30:01.831 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:30:01.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:01.841 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:30:01.942 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:30:01.950 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:30:01 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:30:01] "GET /metrics HTTP/1.1" 200 36490 "" "Prometheus/2.33.4" 2023-08-28T02:30:01.991 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:01 smithi169 ceph-mon[159415]: 6.7 scrub starts 2023-08-28T02:30:01.991 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:01 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3743912350' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-08-28T02:30:01.991 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:01 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-08-28T02:30:01.991 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:01 smithi169 ceph-mon[159415]: 6.9 scrub starts 2023-08-28T02:30:01.991 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:01 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 7975/106161 objects degraded (7.512%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-08-28T02:30:02.021 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:02.055 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset nodeep-scrub 2023-08-28T02:30:02.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:02 smithi143 ceph-mon[183059]: pgmap v596: 99 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 2.1 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 778 op/s; 7009/104877 objects degraded (6.683%); 5026/104877 objects misplaced (4.792%); 9.0 MiB/s, 142 objects/s recovering 2023-08-28T02:30:02.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:02 smithi143 ceph-mon[183059]: 6.2 scrub starts 2023-08-28T02:30:02.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:02 smithi143 ceph-mon[183059]: failed to encode map e250 with expected crc 2023-08-28T02:30:02.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:02 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-08-28T02:30:02.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:02 smithi143 ceph-mon[183059]: osdmap e250: 8 total, 8 up, 8 in 2023-08-28T02:30:02.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:02 smithi143 ceph-mon[183059]: failed to encode map e250 with expected crc 2023-08-28T02:30:02.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:02 smithi143 ceph-mon[183059]: failed to encode map e250 with expected crc 2023-08-28T02:30:02.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:02 smithi143 ceph-mon[183059]: failed to encode map e250 with expected crc 2023-08-28T02:30:02.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:02 smithi143 ceph-mon[183059]: failed to encode map e250 with expected crc 2023-08-28T02:30:02.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:02 smithi143 ceph-mon[183059]: failed to encode map e250 with expected crc 2023-08-28T02:30:02.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:02 smithi143 ceph-mon[189871]: pgmap v596: 99 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 2.1 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 778 op/s; 7009/104877 objects degraded (6.683%); 5026/104877 objects misplaced (4.792%); 9.0 MiB/s, 142 objects/s recovering 2023-08-28T02:30:02.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:02 smithi143 ceph-mon[189871]: 6.2 scrub starts 2023-08-28T02:30:02.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:02 smithi143 ceph-mon[189871]: failed to encode map e250 with expected crc 2023-08-28T02:30:02.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:02 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-08-28T02:30:02.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:02 smithi143 ceph-mon[189871]: osdmap e250: 8 total, 8 up, 8 in 2023-08-28T02:30:02.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:02 smithi143 ceph-mon[189871]: failed to encode map e250 with expected crc 2023-08-28T02:30:02.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:02 smithi143 ceph-mon[189871]: failed to encode map e250 with expected crc 2023-08-28T02:30:02.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:02 smithi143 ceph-mon[189871]: failed to encode map e250 with expected crc 2023-08-28T02:30:02.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:02 smithi143 ceph-mon[189871]: failed to encode map e250 with expected crc 2023-08-28T02:30:02.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:02 smithi143 ceph-mon[189871]: failed to encode map e250 with expected crc 2023-08-28T02:30:02.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:02 smithi169 ceph-mon[159415]: pgmap v596: 99 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 2.1 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 778 op/s; 7009/104877 objects degraded (6.683%); 5026/104877 objects misplaced (4.792%); 9.0 MiB/s, 142 objects/s recovering 2023-08-28T02:30:02.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:02 smithi169 ceph-mon[159415]: 6.2 scrub starts 2023-08-28T02:30:02.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:02 smithi169 ceph-mon[159415]: failed to encode map e250 with expected crc 2023-08-28T02:30:02.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:02 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-08-28T02:30:02.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:02 smithi169 ceph-mon[159415]: osdmap e250: 8 total, 8 up, 8 in 2023-08-28T02:30:02.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:02 smithi169 ceph-mon[159415]: failed to encode map e250 with expected crc 2023-08-28T02:30:02.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:02 smithi169 ceph-mon[159415]: failed to encode map e250 with expected crc 2023-08-28T02:30:02.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:02 smithi169 ceph-mon[159415]: failed to encode map e250 with expected crc 2023-08-28T02:30:02.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:02 smithi169 ceph-mon[159415]: failed to encode map e250 with expected crc 2023-08-28T02:30:02.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:02 smithi169 ceph-mon[159415]: failed to encode map e250 with expected crc 2023-08-28T02:30:03.036 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:30:03.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:03.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:30:03.147 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:30:03.191 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:03.748 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:03 smithi143 ceph-mon[183059]: pgmap v598: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.0 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 629 op/s; 7009/99897 objects degraded (7.016%); 5026/99897 objects misplaced (5.031%); 6.3 MiB/s, 101 objects/s recovering 2023-08-28T02:30:03.749 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:03 smithi143 ceph-mon[189871]: pgmap v598: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.0 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 629 op/s; 7009/99897 objects degraded (7.016%); 5026/99897 objects misplaced (5.031%); 6.3 MiB/s, 101 objects/s recovering 2023-08-28T02:30:03.749 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd reweight 1 1 2023-08-28T02:30:03.898 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:30:03.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:03.909 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:30:03.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:03 smithi169 ceph-mon[159415]: pgmap v598: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.0 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 629 op/s; 7009/99897 objects degraded (7.016%); 5026/99897 objects misplaced (5.031%); 6.3 MiB/s, 101 objects/s recovering 2023-08-28T02:30:04.010 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:30:04.056 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:04.117 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:30:04.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:04.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:30:04.184 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:30:04.223 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:30:04.314 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:30:04.314 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:30:04.340 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:04.572 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:30:04.587 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:04 smithi143 ceph-mon[183059]: 6.0 scrub starts 2023-08-28T02:30:04.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:04.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:30:04.588 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:04 smithi143 ceph-mon[189871]: 6.0 scrub starts 2023-08-28T02:30:04.689 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:30:04.730 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:04.789 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:30:04.792 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:04 smithi169 ceph-mon[159415]: 6.0 scrub starts 2023-08-28T02:30:04.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:04.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:30:04.894 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:30:04.929 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:05.029 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:30:05.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:05.038 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:30:05.139 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:30:05.187 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:05.247 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:30:05.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:05.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:30:05.309 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:30:05.310 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:30:05.310 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:30:05.310 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:30:05.310 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:30:05.310 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:30:05.311 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:30:05.311 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:30:05.311 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:30:05.311 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:30:05.311 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:30:05.311 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:30:05.312 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:30:05.312 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:30:05.312 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:30:05.312 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:30:05.356 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:30:05.385 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:05.609 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:05 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1083238168' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:30:05.609 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:05 smithi143 ceph-mon[183059]: pgmap v599: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.9 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 7009/92979 objects degraded (7.538%); 3960/92979 objects misplaced (4.259%); 8.8 MiB/s, 141 objects/s recovering 2023-08-28T02:30:05.609 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:05 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1083238168' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:30:05.609 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:05 smithi143 ceph-mon[189871]: pgmap v599: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.9 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 7009/92979 objects degraded (7.538%); 3960/92979 objects misplaced (4.259%); 8.8 MiB/s, 141 objects/s recovering 2023-08-28T02:30:05.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:05 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1083238168' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:30:05.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:05 smithi169 ceph-mon[159415]: pgmap v599: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.9 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 7009/92979 objects degraded (7.538%); 3960/92979 objects misplaced (4.259%); 8.8 MiB/s, 141 objects/s recovering 2023-08-28T02:30:06.814 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:30:06.822 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:06 smithi143 ceph-mon[189871]: 6.19 scrub starts 2023-08-28T02:30:06.822 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:06 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 7009/92979 objects degraded (7.538%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-08-28T02:30:06.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:06.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:30:06.827 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:06 smithi143 ceph-mon[183059]: 6.19 scrub starts 2023-08-28T02:30:06.827 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:06 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 7009/92979 objects degraded (7.538%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-08-28T02:30:06.853 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.3 dump_ops_in_flight 2023-08-28T02:30:06.926 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:30:06.974 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:06 smithi169 ceph-mon[159415]: 6.19 scrub starts 2023-08-28T02:30:06.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:06 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 7009/92979 objects degraded (7.538%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-08-28T02:30:06.979 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:07.040 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:30:07.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:07.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:30:07.148 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:30:07.197 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:07.426 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:30:07.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:07.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:30:07.534 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:30:07.598 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:30:07.601 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:07.666 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:30:07.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:07.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:30:07.781 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:30:07.837 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:07 smithi169 ceph-mon[159415]: pgmap v600: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.8 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 899 op/s; 7009/87405 objects degraded (8.019%); 3960/87405 objects misplaced (4.531%); 7.8 MiB/s, 125 objects/s recovering 2023-08-28T02:30:07.839 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:07.909 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:30:07.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:07.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:30:07.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:07 smithi143 ceph-mon[183059]: pgmap v600: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.8 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 899 op/s; 7009/87405 objects degraded (8.019%); 3960/87405 objects misplaced (4.531%); 7.8 MiB/s, 125 objects/s recovering 2023-08-28T02:30:07.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:07 smithi143 ceph-mon[189871]: pgmap v600: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 1.8 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 899 op/s; 7009/87405 objects degraded (8.019%); 3960/87405 objects misplaced (4.531%); 7.8 MiB/s, 125 objects/s recovering 2023-08-28T02:30:08.021 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:30:08.119 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:08.295 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:30:08.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:08.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:30:08.401 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:30:08.445 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:08.499 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:30:08.507 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:30:08.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:08.512 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:30:08.613 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:30:08.658 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:08.751 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:30:08.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:08.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:30:08.865 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:30:08.920 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:09.004 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:30:09.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:09.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:30:09.110 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:30:09.141 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:09.186 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:30:09.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:09.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:30:09.292 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:30:09.320 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:09.442 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:30:09.575 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is unset 2023-08-28T02:30:09.602 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:09 smithi143 ceph-mon[183059]: 6.7 scrub starts 2023-08-28T02:30:09.602 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:09 smithi143 ceph-mon[183059]: osdmap e251: 8 total, 8 up, 8 in 2023-08-28T02:30:09.602 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:09 smithi143 ceph-mon[183059]: failed to encode map e251 with expected crc 2023-08-28T02:30:09.602 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:09 smithi143 ceph-mon[183059]: failed to encode map e251 with expected crc 2023-08-28T02:30:09.602 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:09 smithi143 ceph-mon[183059]: failed to encode map e251 with expected crc 2023-08-28T02:30:09.602 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:09 smithi143 ceph-mon[183059]: failed to encode map e251 with expected crc 2023-08-28T02:30:09.602 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:09 smithi143 ceph-mon[183059]: failed to encode map e251 with expected crc 2023-08-28T02:30:09.602 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:09 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2136292013' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:30:09.603 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:09 smithi143 ceph-mon[183059]: 6.10 scrub starts 2023-08-28T02:30:09.603 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:09 smithi143 ceph-mon[183059]: pgmap v602: 99 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 1.7 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 948 op/s; 7009/82104 objects degraded (8.537%); 3462/82104 objects misplaced (4.217%); 3.9 MiB/s, 62 objects/s recovering 2023-08-28T02:30:09.603 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:09 smithi143 ceph-mon[189871]: 6.7 scrub starts 2023-08-28T02:30:09.603 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:09 smithi143 ceph-mon[189871]: osdmap e251: 8 total, 8 up, 8 in 2023-08-28T02:30:09.603 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:09 smithi143 ceph-mon[189871]: failed to encode map e251 with expected crc 2023-08-28T02:30:09.603 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:09 smithi143 ceph-mon[189871]: failed to encode map e251 with expected crc 2023-08-28T02:30:09.603 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:09 smithi143 ceph-mon[189871]: failed to encode map e251 with expected crc 2023-08-28T02:30:09.603 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:09 smithi143 ceph-mon[189871]: failed to encode map e251 with expected crc 2023-08-28T02:30:09.603 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:09 smithi143 ceph-mon[189871]: failed to encode map e251 with expected crc 2023-08-28T02:30:09.603 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:09 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2136292013' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:30:09.604 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:09 smithi143 ceph-mon[189871]: 6.10 scrub starts 2023-08-28T02:30:09.604 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:09 smithi143 ceph-mon[189871]: pgmap v602: 99 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 1.7 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 948 op/s; 7009/82104 objects degraded (8.537%); 3462/82104 objects misplaced (4.217%); 3.9 MiB/s, 62 objects/s recovering 2023-08-28T02:30:09.854 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:30:09.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:09.862 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:30:09.963 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:30:09.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:09 smithi169 ceph-mon[159415]: 6.7 scrub starts 2023-08-28T02:30:09.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:09 smithi169 ceph-mon[159415]: osdmap e251: 8 total, 8 up, 8 in 2023-08-28T02:30:09.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:09 smithi169 ceph-mon[159415]: failed to encode map e251 with expected crc 2023-08-28T02:30:09.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:09 smithi169 ceph-mon[159415]: failed to encode map e251 with expected crc 2023-08-28T02:30:09.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:09 smithi169 ceph-mon[159415]: failed to encode map e251 with expected crc 2023-08-28T02:30:09.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:09 smithi169 ceph-mon[159415]: failed to encode map e251 with expected crc 2023-08-28T02:30:09.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:09 smithi169 ceph-mon[159415]: failed to encode map e251 with expected crc 2023-08-28T02:30:09.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:09 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2136292013' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:30:09.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:09 smithi169 ceph-mon[159415]: 6.10 scrub starts 2023-08-28T02:30:09.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:09 smithi169 ceph-mon[159415]: pgmap v602: 99 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 1.7 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 948 op/s; 7009/82104 objects degraded (8.537%); 3462/82104 objects misplaced (4.217%); 3.9 MiB/s, 62 objects/s recovering 2023-08-28T02:30:10.009 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:10.088 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:30:10.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:10.094 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:30:10.195 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:30:10.220 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:10.266 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:30:10.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:10.269 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:30:10.370 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:30:10.425 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:10.520 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:30:10.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:10.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:30:10.633 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:30:10.750 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:10.829 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:10 smithi143 ceph-mon[189871]: 6.7 scrub starts 2023-08-28T02:30:10.829 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:10 smithi143 ceph-mon[189871]: 6.4 scrub starts 2023-08-28T02:30:10.829 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:10 smithi143 ceph-mon[189871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:30:10.829 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:10 smithi143 ceph-mon[189871]: failed to encode map e252 with expected crc 2023-08-28T02:30:10.829 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:10 smithi143 ceph-mon[189871]: failed to encode map e252 with expected crc 2023-08-28T02:30:10.829 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:10 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2136292013' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:30:10.829 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:10 smithi143 ceph-mon[189871]: osdmap e252: 8 total, 8 up, 8 in 2023-08-28T02:30:10.830 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:10 smithi143 ceph-mon[189871]: failed to encode map e252 with expected crc 2023-08-28T02:30:10.830 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:10 smithi143 ceph-mon[189871]: failed to encode map e252 with expected crc 2023-08-28T02:30:10.830 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:10 smithi143 ceph-mon[189871]: 6.2 scrub starts 2023-08-28T02:30:10.830 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:10 smithi143 ceph-mon[189871]: failed to encode map e252 with expected crc 2023-08-28T02:30:10.830 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:10 smithi143 ceph-mon[189871]: 6.9 scrub starts 2023-08-28T02:30:10.830 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:10 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:30:10.830 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:10 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:30:10.830 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:10 smithi143 ceph-mon[183059]: 6.7 scrub starts 2023-08-28T02:30:10.831 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:10 smithi143 ceph-mon[183059]: 6.4 scrub starts 2023-08-28T02:30:10.831 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:10 smithi143 ceph-mon[183059]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:30:10.831 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:10 smithi143 ceph-mon[183059]: failed to encode map e252 with expected crc 2023-08-28T02:30:10.831 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:10 smithi143 ceph-mon[183059]: failed to encode map e252 with expected crc 2023-08-28T02:30:10.831 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:10 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2136292013' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:30:10.831 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:10 smithi143 ceph-mon[183059]: osdmap e252: 8 total, 8 up, 8 in 2023-08-28T02:30:10.831 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:10 smithi143 ceph-mon[183059]: failed to encode map e252 with expected crc 2023-08-28T02:30:10.831 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:10 smithi143 ceph-mon[183059]: failed to encode map e252 with expected crc 2023-08-28T02:30:10.831 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:10 smithi143 ceph-mon[183059]: 6.2 scrub starts 2023-08-28T02:30:10.831 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:10 smithi143 ceph-mon[183059]: failed to encode map e252 with expected crc 2023-08-28T02:30:10.831 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:10 smithi143 ceph-mon[183059]: 6.9 scrub starts 2023-08-28T02:30:10.832 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:10 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:30:10.832 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:10 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:30:10.877 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:30:10.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:10.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:30:10.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:10 smithi169 ceph-mon[159415]: 6.7 scrub starts 2023-08-28T02:30:10.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:10 smithi169 ceph-mon[159415]: 6.4 scrub starts 2023-08-28T02:30:10.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:10 smithi169 ceph-mon[159415]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:30:10.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:10 smithi169 ceph-mon[159415]: failed to encode map e252 with expected crc 2023-08-28T02:30:10.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:10 smithi169 ceph-mon[159415]: failed to encode map e252 with expected crc 2023-08-28T02:30:10.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:10 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2136292013' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:30:10.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:10 smithi169 ceph-mon[159415]: osdmap e252: 8 total, 8 up, 8 in 2023-08-28T02:30:10.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:10 smithi169 ceph-mon[159415]: failed to encode map e252 with expected crc 2023-08-28T02:30:10.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:10 smithi169 ceph-mon[159415]: failed to encode map e252 with expected crc 2023-08-28T02:30:10.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:10 smithi169 ceph-mon[159415]: 6.2 scrub starts 2023-08-28T02:30:10.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:10 smithi169 ceph-mon[159415]: failed to encode map e252 with expected crc 2023-08-28T02:30:10.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:10 smithi169 ceph-mon[159415]: 6.9 scrub starts 2023-08-28T02:30:10.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:10 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:30:10.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:10 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:30:10.991 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:30:11.037 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:11.118 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:30:11.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:11.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:30:11.225 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:30:11.291 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:11.385 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:30:11.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:11.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:30:11.491 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:30:11.552 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:11.596 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:30:11.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:11.598 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:30:11.699 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:30:11.764 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:12.050 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:30:11 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:30:11] "GET /metrics HTTP/1.1" 200 36487 "" "Prometheus/2.33.4" 2023-08-28T02:30:12.072 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:30:12.114 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:30:12.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:12.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:30:12.219 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:30:12.276 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:12.349 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:12 smithi143 ceph-mon[183059]: 6.14 scrub starts 2023-08-28T02:30:12.349 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:12 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:30:12.349 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:12 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:30:12.350 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:12 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:30:12.350 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:12 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:30:12.350 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:12 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:30:12.350 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:12 smithi143 ceph-mon[183059]: 6.7 scrub starts 2023-08-28T02:30:12.350 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:12 smithi143 ceph-mon[183059]: pgmap v604: 99 pgs: 1 peering, 2 active+clean+scrubbing, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 1.6 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 6723/80754 objects degraded (8.325%); 3289/80754 objects misplaced (4.073%); 5.2 MiB/s, 119 objects/s recovering 2023-08-28T02:30:12.350 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:12 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 7009/82104 objects degraded (8.537%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-08-28T02:30:12.350 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:12 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/973640015' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-08-28T02:30:12.350 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:12 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-08-28T02:30:12.350 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:12 smithi143 ceph-mon[189871]: 6.14 scrub starts 2023-08-28T02:30:12.350 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:12 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:30:12.351 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:12 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:30:12.351 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:12 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:30:12.351 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:12 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:30:12.351 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:12 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:30:12.351 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:12 smithi143 ceph-mon[189871]: 6.7 scrub starts 2023-08-28T02:30:12.351 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:12 smithi143 ceph-mon[189871]: pgmap v604: 99 pgs: 1 peering, 2 active+clean+scrubbing, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 1.6 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 6723/80754 objects degraded (8.325%); 3289/80754 objects misplaced (4.073%); 5.2 MiB/s, 119 objects/s recovering 2023-08-28T02:30:12.351 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:12 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 7009/82104 objects degraded (8.537%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-08-28T02:30:12.351 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:12 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/973640015' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-08-28T02:30:12.351 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:12 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-08-28T02:30:12.357 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:30:12.361 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:12 smithi169 ceph-mon[159415]: 6.14 scrub starts 2023-08-28T02:30:12.361 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:12 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:30:12.361 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:12 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:30:12.361 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:12 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:30:12.362 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:12 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:30:12.362 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:12 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:30:12.362 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:12 smithi169 ceph-mon[159415]: 6.7 scrub starts 2023-08-28T02:30:12.362 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:12 smithi169 ceph-mon[159415]: pgmap v604: 99 pgs: 1 peering, 2 active+clean+scrubbing, 2 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 1.6 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 797 op/s; 6723/80754 objects degraded (8.325%); 3289/80754 objects misplaced (4.073%); 5.2 MiB/s, 119 objects/s recovering 2023-08-28T02:30:12.362 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:12 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 7009/82104 objects degraded (8.537%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-08-28T02:30:12.362 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:12 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/973640015' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-08-28T02:30:12.362 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:12 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-08-28T02:30:12.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:12.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:30:12.425 INFO:teuthology.orchestra.run.smithi143.stderr:reweighted osd.1 to 1 (10000) 2023-08-28T02:30:12.464 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:30:12.541 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:12.743 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:30:12.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:12.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:30:12.850 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:30:12.857 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set noscrub 2023-08-28T02:30:12.906 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:12.937 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:30:12.937 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [ 2023-08-28T02:30:12.938 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:12.938 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:143682 6.c 6:3663a5cb:::benchmark_data_smithi143_305742_object60063:head [delete] snapc 0=[] ondisk+write+known_if_redirected e253)", 2023-08-28T02:30:12.938 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:30:12.918045+0000", 2023-08-28T02:30:12.938 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.017793410999999999, 2023-08-28T02:30:12.938 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.017824858999999998, 2023-08-28T02:30:12.938 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:30:12.938 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "waiting for sub ops", 2023-08-28T02:30:12.938 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:30:12.938 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:30:12.938 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:30:12.938 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 143682 2023-08-28T02:30:12.939 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:12.939 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:30:12.939 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:12.939 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:30:12.939 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:12.918045+0000", 2023-08-28T02:30:12.939 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:30:12.939 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:12.939 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:12.939 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:30:12.939 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:12.918045+0000", 2023-08-28T02:30:12.939 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:30:12.940 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:12.940 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:12.940 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:30:12.940 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:12.918044+0000", 2023-08-28T02:30:12.940 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:30:12.940 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:12.940 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:12.940 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:30:12.940 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:12.918050+0000", 2023-08-28T02:30:12.940 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.9610000000000002e-06 2023-08-28T02:30:12.940 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:12.940 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:12.940 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:30:12.941 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:12.918067+0000", 2023-08-28T02:30:12.941 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.6997e-05 2023-08-28T02:30:12.941 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:12.941 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:12.941 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:30:12.941 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:12.918075+0000", 2023-08-28T02:30:12.941 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.7919999999999998e-06 2023-08-28T02:30:12.941 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:12.941 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:12.941 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:12.941 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:12.918185+0000", 2023-08-28T02:30:12.942 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00010999500000000001 2023-08-28T02:30:12.942 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:12.942 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:12.942 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:30:12.942 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:12.918317+0000", 2023-08-28T02:30:12.942 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000132065 2023-08-28T02:30:12.942 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:12.942 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:12.942 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 5,6,7", 2023-08-28T02:30:12.942 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:12.918432+0000", 2023-08-28T02:30:12.942 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000114535 2023-08-28T02:30:12.942 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:12.942 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:12.942 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:30:12.943 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:12.932128+0000", 2023-08-28T02:30:12.943 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.013696221999999999 2023-08-28T02:30:12.943 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:12.943 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:12.943 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:30:12.943 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:12.932192+0000", 2023-08-28T02:30:12.943 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.3336999999999993e-05 2023-08-28T02:30:12.943 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:12.943 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:12.943 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:30:12.943 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:12.932443+0000", 2023-08-28T02:30:12.943 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00025167000000000001 2023-08-28T02:30:12.943 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:12.944 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:30:12.944 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:12.944 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:12.944 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:12.944 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:143692 6.c 6:3257b537:::benchmark_data_smithi143_305742_object60073:head [delete] snapc 0=[] ondisk+write+known_if_redirected e253)", 2023-08-28T02:30:12.944 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:30:12.918092+0000", 2023-08-28T02:30:12.944 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.017746809999999998, 2023-08-28T02:30:12.944 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.017874956000000001, 2023-08-28T02:30:12.944 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:30:12.944 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "waiting for sub ops", 2023-08-28T02:30:12.944 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:30:12.944 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:30:12.945 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:30:12.945 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 143692 2023-08-28T02:30:12.945 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:12.945 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:30:12.945 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:12.945 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:30:12.945 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:12.918092+0000", 2023-08-28T02:30:12.945 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:30:12.945 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:12.945 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:12.945 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:30:12.945 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:12.918092+0000", 2023-08-28T02:30:12.945 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:30:12.946 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:12.946 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:12.946 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:30:12.946 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:12.918091+0000", 2023-08-28T02:30:12.946 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:30:12.946 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:12.946 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:12.946 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:30:12.946 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:12.918094+0000", 2023-08-28T02:30:12.946 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.9340000000000002e-06 2023-08-28T02:30:12.946 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:12.946 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:12.947 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:30:12.947 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:12.918101+0000", 2023-08-28T02:30:12.947 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.421e-06 2023-08-28T02:30:12.947 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:12.947 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:12.947 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:30:12.947 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:12.918106+0000", 2023-08-28T02:30:12.947 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.0640000000000001e-06 2023-08-28T02:30:12.947 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:12.947 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:12.947 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:12.947 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:12.921228+0000", 2023-08-28T02:30:12.948 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0031218729999999998 2023-08-28T02:30:12.948 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:12.948 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:12.948 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:30:12.948 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:12.921346+0000", 2023-08-28T02:30:12.948 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000118866 2023-08-28T02:30:12.948 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:12.948 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:12.948 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 5,6,7", 2023-08-28T02:30:12.948 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:12.921457+0000", 2023-08-28T02:30:12.948 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000110639 2023-08-28T02:30:12.948 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:12.949 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:12.949 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:30:12.949 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:12.932161+0000", 2023-08-28T02:30:12.949 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.010704345000000001 2023-08-28T02:30:12.949 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:12.949 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:12.949 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:30:12.949 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:12.932221+0000", 2023-08-28T02:30:12.949 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.9744000000000003e-05 2023-08-28T02:30:12.949 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:12.949 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:12.950 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:30:12.950 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:12.932450+0000", 2023-08-28T02:30:12.950 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000228994 2023-08-28T02:30:12.950 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:12.950 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:30:12.950 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:12.950 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:12.950 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:12.950 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.35076.0:143698 6.2 e253/209 6:41092396:::benchmark_data_smithi143_305742_object60079:head v 253'6401, mlcod=253'6399)", 2023-08-28T02:30:12.950 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:30:12.935049+0000", 2023-08-28T02:30:12.950 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.00078943000000000004, 2023-08-28T02:30:12.950 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00101201, 2023-08-28T02:30:12.951 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:30:12.951 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "started", 2023-08-28T02:30:12.951 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:30:12.951 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:12.951 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:30:12.951 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:12.935049+0000", 2023-08-28T02:30:12.951 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:30:12.951 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:12.951 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:12.951 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:30:12.951 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:12.935049+0000", 2023-08-28T02:30:12.951 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:30:12.951 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:12.952 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:12.952 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:30:12.952 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:12.935048+0000", 2023-08-28T02:30:12.952 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:30:12.952 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:12.952 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:12.952 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:30:12.952 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:12.935055+0000", 2023-08-28T02:30:12.952 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.2700000000000001e-06 2023-08-28T02:30:12.953 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:12.953 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:12.953 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:30:12.953 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:12.935068+0000", 2023-08-28T02:30:12.953 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3587999999999999e-05 2023-08-28T02:30:12.953 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:12.953 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:12.953 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:30:12.953 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:12.935076+0000", 2023-08-28T02:30:12.953 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.7190000000000001e-06 2023-08-28T02:30:12.953 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:12.953 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:12.954 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:12.954 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:12.935268+0000", 2023-08-28T02:30:12.954 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00019198199999999999 2023-08-28T02:30:12.954 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:12.954 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:12.954 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:30:12.954 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:12.935293+0000", 2023-08-28T02:30:12.954 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.4856000000000001e-05 2023-08-28T02:30:12.954 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:12.954 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:30:12.954 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:12.955 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:12.955 INFO:teuthology.orchestra.run.smithi143.stdout: ], 2023-08-28T02:30:12.955 INFO:teuthology.orchestra.run.smithi143.stdout: "num_ops": 3 2023-08-28T02:30:12.955 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:30:12.988 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:30:12.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:12.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:30:13.099 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:30:13.179 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:13 smithi143 ceph-mon[183059]: 6.11 deep-scrub starts 2023-08-28T02:30:13.179 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:13 smithi143 ceph-mon[183059]: 6.0 scrub starts 2023-08-28T02:30:13.180 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:13 smithi143 ceph-mon[183059]: failed to encode map e253 with expected crc 2023-08-28T02:30:13.180 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:13 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-08-28T02:30:13.180 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:13 smithi143 ceph-mon[183059]: osdmap e253: 8 total, 8 up, 8 in 2023-08-28T02:30:13.180 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:13 smithi143 ceph-mon[183059]: failed to encode map e253 with expected crc 2023-08-28T02:30:13.180 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:13 smithi143 ceph-mon[189871]: 6.11 deep-scrub starts 2023-08-28T02:30:13.180 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:13 smithi143 ceph-mon[189871]: 6.0 scrub starts 2023-08-28T02:30:13.180 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:13 smithi143 ceph-mon[189871]: failed to encode map e253 with expected crc 2023-08-28T02:30:13.181 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:13 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-08-28T02:30:13.181 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:13 smithi143 ceph-mon[189871]: osdmap e253: 8 total, 8 up, 8 in 2023-08-28T02:30:13.181 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:13 smithi143 ceph-mon[189871]: failed to encode map e253 with expected crc 2023-08-28T02:30:13.182 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:13.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:13 smithi169 ceph-mon[159415]: 6.11 deep-scrub starts 2023-08-28T02:30:13.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:13 smithi169 ceph-mon[159415]: 6.0 scrub starts 2023-08-28T02:30:13.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:13 smithi169 ceph-mon[159415]: failed to encode map e253 with expected crc 2023-08-28T02:30:13.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:13 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-08-28T02:30:13.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:13 smithi169 ceph-mon[159415]: osdmap e253: 8 total, 8 up, 8 in 2023-08-28T02:30:13.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:13 smithi169 ceph-mon[159415]: failed to encode map e253 with expected crc 2023-08-28T02:30:14.077 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:14 smithi143 ceph-mon[183059]: 6.1 scrub starts 2023-08-28T02:30:14.077 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:14 smithi143 ceph-mon[183059]: pgmap v606: 99 pgs: 1 peering, 2 active+clean+scrubbing, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 1.6 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 6078/76098 objects degraded (7.987%); 3839/76098 objects misplaced (5.045%); 2.8 MiB/s, 92 objects/s recovering 2023-08-28T02:30:14.077 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:14 smithi143 ceph-mon[183059]: 6.2 scrub starts 2023-08-28T02:30:14.232 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:30:14.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:14.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:30:14.339 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:30:14.367 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:14 smithi143 ceph-mon[189871]: 6.1 scrub starts 2023-08-28T02:30:14.367 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:14 smithi143 ceph-mon[189871]: pgmap v606: 99 pgs: 1 peering, 2 active+clean+scrubbing, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 1.6 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 6078/76098 objects degraded (7.987%); 3839/76098 objects misplaced (5.045%); 2.8 MiB/s, 92 objects/s recovering 2023-08-28T02:30:14.367 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:14 smithi143 ceph-mon[189871]: 6.2 scrub starts 2023-08-28T02:30:14.387 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:14 smithi169 ceph-mon[159415]: 6.1 scrub starts 2023-08-28T02:30:14.388 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:14 smithi169 ceph-mon[159415]: pgmap v606: 99 pgs: 1 peering, 2 active+clean+scrubbing, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 1.6 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 6078/76098 objects degraded (7.987%); 3839/76098 objects misplaced (5.045%); 2.8 MiB/s, 92 objects/s recovering 2023-08-28T02:30:14.388 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:14 smithi169 ceph-mon[159415]: 6.2 scrub starts 2023-08-28T02:30:14.389 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:14.392 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd reweight 6 1 2023-08-28T02:30:14.493 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:30:14.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:14.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:30:14.602 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:30:14.639 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:14.681 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:30:14.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:14.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:30:14.789 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:30:14.881 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:15.004 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:30:15.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:15.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:30:15.058 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.3 dump_blocked_ops 2023-08-28T02:30:15.082 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:15 smithi143 ceph-mon[189871]: 6.7 scrub starts 2023-08-28T02:30:15.083 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:15 smithi143 ceph-mon[189871]: 6.4 scrub starts 2023-08-28T02:30:15.116 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:30:15.252 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:15.333 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:15 smithi143 ceph-mon[183059]: 6.7 scrub starts 2023-08-28T02:30:15.333 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:15 smithi143 ceph-mon[183059]: 6.4 scrub starts 2023-08-28T02:30:15.471 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:30:15.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:15.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:30:15.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:15 smithi169 ceph-mon[159415]: 6.7 scrub starts 2023-08-28T02:30:15.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:15 smithi169 ceph-mon[159415]: 6.4 scrub starts 2023-08-28T02:30:15.510 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:30:15.585 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:30:15.690 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:15.691 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:30:15.732 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:30:15.737 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:30:15.780 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:30:15.855 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:30:15.879 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:30:15.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:15.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:30:15.889 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:30:15.936 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:30:15.968 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:30:15.984 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:30:15.997 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:30:16.007 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:30:16.007 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:30:16.008 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:30:16.008 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:30:16.008 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:30:16.008 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:30:16.008 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:30:16.008 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:30:16.046 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:16.380 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:30:16.428 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:16 smithi143 ceph-mon[189871]: pgmap v607: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 1.4 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 6078/69915 objects degraded (8.693%); 3405/69915 objects misplaced (4.870%); 2.5 MiB/s, 82 objects/s recovering 2023-08-28T02:30:16.428 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:16 smithi143 ceph-mon[189871]: 6.2 scrub starts 2023-08-28T02:30:16.428 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:16 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3455252326' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:30:16.429 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:16 smithi143 ceph-mon[183059]: pgmap v607: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 1.4 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 6078/69915 objects degraded (8.693%); 3405/69915 objects misplaced (4.870%); 2.5 MiB/s, 82 objects/s recovering 2023-08-28T02:30:16.429 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:16 smithi143 ceph-mon[183059]: 6.2 scrub starts 2023-08-28T02:30:16.429 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:16 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3455252326' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:30:16.458 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:30:16.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:16.467 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:30:16.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:16 smithi169 ceph-mon[159415]: pgmap v607: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 1.4 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 6078/69915 objects degraded (8.693%); 3405/69915 objects misplaced (4.870%); 2.5 MiB/s, 82 objects/s recovering 2023-08-28T02:30:16.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:16 smithi169 ceph-mon[159415]: 6.2 scrub starts 2023-08-28T02:30:16.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:16 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3455252326' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:30:16.568 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:30:16.625 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:16.668 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:30:16.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:16.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:30:16.777 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:30:16.831 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:16.932 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:30:16.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:16.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:30:17.041 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:30:17.073 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:17.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:17 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 6078/69915 objects degraded (8.693%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:30:17.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:17 smithi169 ceph-mon[159415]: 6.2 scrub starts 2023-08-28T02:30:17.339 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:17 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 6078/69915 objects degraded (8.693%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:30:17.339 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:17 smithi143 ceph-mon[183059]: 6.2 scrub starts 2023-08-28T02:30:17.340 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:17 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 6078/69915 objects degraded (8.693%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:30:17.340 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:17 smithi143 ceph-mon[189871]: 6.2 scrub starts 2023-08-28T02:30:18.235 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:30:18.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:18.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:30:18.344 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:30:18.394 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:18 smithi169 ceph-mon[159415]: 6.19 scrub starts 2023-08-28T02:30:18.395 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:18 smithi169 ceph-mon[159415]: pgmap v608: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 1.3 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 6078/64689 objects degraded (9.396%); 3290/64689 objects misplaced (5.086%); 3.0 MiB/s, 83 objects/s recovering 2023-08-28T02:30:18.395 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:18 smithi169 ceph-mon[159415]: 6.10 scrub starts 2023-08-28T02:30:18.398 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:18.435 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:18 smithi143 ceph-mon[183059]: 6.19 scrub starts 2023-08-28T02:30:18.435 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:18 smithi143 ceph-mon[183059]: pgmap v608: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 1.3 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 6078/64689 objects degraded (9.396%); 3290/64689 objects misplaced (5.086%); 3.0 MiB/s, 83 objects/s recovering 2023-08-28T02:30:18.435 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:18 smithi143 ceph-mon[183059]: 6.10 scrub starts 2023-08-28T02:30:18.438 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:18 smithi143 ceph-mon[189871]: 6.19 scrub starts 2023-08-28T02:30:18.438 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:18 smithi143 ceph-mon[189871]: pgmap v608: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 1.3 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 6078/64689 objects degraded (9.396%); 3290/64689 objects misplaced (5.086%); 3.0 MiB/s, 83 objects/s recovering 2023-08-28T02:30:18.439 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:18 smithi143 ceph-mon[189871]: 6.10 scrub starts 2023-08-28T02:30:18.483 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:30:18.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:18.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:30:18.589 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:30:18.682 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:19.240 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:30:19.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:19.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:30:19.327 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:19 smithi143 ceph-mon[183059]: 6.2 scrub starts 2023-08-28T02:30:19.327 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:19 smithi143 ceph-mon[189871]: 6.2 scrub starts 2023-08-28T02:30:19.345 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:30:19.390 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:19 smithi169 ceph-mon[159415]: 6.2 scrub starts 2023-08-28T02:30:19.397 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:19.496 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:30:19.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:19.507 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:30:19.607 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:30:19.672 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:20.073 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:30:20.180 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is set 2023-08-28T02:30:20.409 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:20 smithi143 ceph-mon[183059]: pgmap v609: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 1.2 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 6078/59487 objects degraded (10.217%); 2795/59487 objects misplaced (4.699%); 1.3 MiB/s, 21 objects/s recovering 2023-08-28T02:30:20.409 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:20 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3864242250' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:30:20.409 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:20 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:30:20.409 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:20 smithi143 ceph-mon[183059]: 6.9 scrub starts 2023-08-28T02:30:20.410 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:20 smithi143 ceph-mon[189871]: pgmap v609: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 1.2 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 6078/59487 objects degraded (10.217%); 2795/59487 objects misplaced (4.699%); 1.3 MiB/s, 21 objects/s recovering 2023-08-28T02:30:20.410 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:20 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3864242250' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:30:20.410 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:20 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:30:20.410 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:20 smithi143 ceph-mon[189871]: 6.9 scrub starts 2023-08-28T02:30:20.449 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:30:20.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:20.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:30:20.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:20 smithi169 ceph-mon[159415]: pgmap v609: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 1.2 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 6078/59487 objects degraded (10.217%); 2795/59487 objects misplaced (4.699%); 1.3 MiB/s, 21 objects/s recovering 2023-08-28T02:30:20.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:20 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3864242250' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:30:20.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:20 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:30:20.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:20 smithi169 ceph-mon[159415]: 6.9 scrub starts 2023-08-28T02:30:20.562 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:30:20.589 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:21.006 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:30:21.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:21.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:30:21.113 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:30:21.157 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:21.198 INFO:teuthology.orchestra.run.smithi143.stderr:reweighted osd.6 to 1 (10000) 2023-08-28T02:30:21.245 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:30:21.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:21.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:30:21.356 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:30:21.361 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:21 smithi143 ceph-mon[183059]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:30:21.361 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:21 smithi143 ceph-mon[183059]: failed to encode map e254 with expected crc 2023-08-28T02:30:21.361 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:21 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:30:21.361 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:21 smithi143 ceph-mon[183059]: osdmap e254: 8 total, 8 up, 8 in 2023-08-28T02:30:21.361 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:21 smithi143 ceph-mon[183059]: failed to encode map e254 with expected crc 2023-08-28T02:30:21.361 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:21 smithi143 ceph-mon[183059]: failed to encode map e254 with expected crc 2023-08-28T02:30:21.362 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:21 smithi143 ceph-mon[183059]: failed to encode map e254 with expected crc 2023-08-28T02:30:21.362 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:21 smithi143 ceph-mon[183059]: failed to encode map e254 with expected crc 2023-08-28T02:30:21.362 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:21 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/768022442' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-08-28T02:30:21.362 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:21 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-08-28T02:30:21.362 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:21 smithi143 ceph-mon[189871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:30:21.362 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:21 smithi143 ceph-mon[189871]: failed to encode map e254 with expected crc 2023-08-28T02:30:21.362 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:21 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:30:21.362 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:21 smithi143 ceph-mon[189871]: osdmap e254: 8 total, 8 up, 8 in 2023-08-28T02:30:21.362 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:21 smithi143 ceph-mon[189871]: failed to encode map e254 with expected crc 2023-08-28T02:30:21.362 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:21 smithi143 ceph-mon[189871]: failed to encode map e254 with expected crc 2023-08-28T02:30:21.363 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:21 smithi143 ceph-mon[189871]: failed to encode map e254 with expected crc 2023-08-28T02:30:21.363 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:21 smithi143 ceph-mon[189871]: failed to encode map e254 with expected crc 2023-08-28T02:30:21.363 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:21 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/768022442' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-08-28T02:30:21.363 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:21 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-08-28T02:30:21.399 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:21.489 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:30:21.495 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:21 smithi169 ceph-mon[159415]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:30:21.496 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:21 smithi169 ceph-mon[159415]: failed to encode map e254 with expected crc 2023-08-28T02:30:21.496 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:21 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:30:21.496 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:21 smithi169 ceph-mon[159415]: osdmap e254: 8 total, 8 up, 8 in 2023-08-28T02:30:21.496 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:21 smithi169 ceph-mon[159415]: failed to encode map e254 with expected crc 2023-08-28T02:30:21.496 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:21 smithi169 ceph-mon[159415]: failed to encode map e254 with expected crc 2023-08-28T02:30:21.496 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:21 smithi169 ceph-mon[159415]: failed to encode map e254 with expected crc 2023-08-28T02:30:21.496 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:21 smithi169 ceph-mon[159415]: failed to encode map e254 with expected crc 2023-08-28T02:30:21.496 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:21 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/768022442' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-08-28T02:30:21.497 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:21 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-08-28T02:30:21.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:21.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:30:21.608 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:30:21.634 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:21.676 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:30:21.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:21.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:30:21.781 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:30:21.891 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:21.992 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:30:21.992 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:30:22.154 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-08-28T02:30:22.154 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph pg dump --format=json 2023-08-28T02:30:22.155 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:30:21 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:30:21] "GET /metrics HTTP/1.1" 200 36487 "" "Prometheus/2.33.4" 2023-08-28T02:30:22.257 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:30:22.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:22.267 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:30:22.367 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:30:22.428 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:22 smithi143 ceph-mon[183059]: failed to encode map e255 with expected crc 2023-08-28T02:30:22.428 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:22 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-08-28T02:30:22.428 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:22 smithi143 ceph-mon[183059]: osdmap e255: 8 total, 8 up, 8 in 2023-08-28T02:30:22.429 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:22 smithi143 ceph-mon[183059]: failed to encode map e255 with expected crc 2023-08-28T02:30:22.430 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:22 smithi143 ceph-mon[183059]: failed to encode map e255 with expected crc 2023-08-28T02:30:22.430 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:22 smithi143 ceph-mon[183059]: failed to encode map e255 with expected crc 2023-08-28T02:30:22.430 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:22 smithi143 ceph-mon[183059]: pgmap v612: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 1.2 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 6078/57900 objects degraded (10.497%); 2795/57900 objects misplaced (4.827%); 919 KiB/s, 14 objects/s recovering 2023-08-28T02:30:22.430 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:22 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 6078/59487 objects degraded (10.217%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:30:22.430 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:22 smithi143 ceph-mon[189871]: failed to encode map e255 with expected crc 2023-08-28T02:30:22.430 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:22 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-08-28T02:30:22.430 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:22 smithi143 ceph-mon[189871]: osdmap e255: 8 total, 8 up, 8 in 2023-08-28T02:30:22.430 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:22 smithi143 ceph-mon[189871]: failed to encode map e255 with expected crc 2023-08-28T02:30:22.430 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:22 smithi143 ceph-mon[189871]: failed to encode map e255 with expected crc 2023-08-28T02:30:22.430 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:22 smithi143 ceph-mon[189871]: failed to encode map e255 with expected crc 2023-08-28T02:30:22.431 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:22 smithi143 ceph-mon[189871]: pgmap v612: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 1.2 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 6078/57900 objects degraded (10.497%); 2795/57900 objects misplaced (4.827%); 919 KiB/s, 14 objects/s recovering 2023-08-28T02:30:22.431 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:22 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 6078/59487 objects degraded (10.217%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:30:22.432 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:22.492 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:22 smithi169 ceph-mon[159415]: failed to encode map e255 with expected crc 2023-08-28T02:30:22.492 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:22 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-08-28T02:30:22.492 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:22 smithi169 ceph-mon[159415]: osdmap e255: 8 total, 8 up, 8 in 2023-08-28T02:30:22.492 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:22 smithi169 ceph-mon[159415]: failed to encode map e255 with expected crc 2023-08-28T02:30:22.492 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:22 smithi169 ceph-mon[159415]: failed to encode map e255 with expected crc 2023-08-28T02:30:22.492 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:22 smithi169 ceph-mon[159415]: failed to encode map e255 with expected crc 2023-08-28T02:30:22.492 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:22 smithi169 ceph-mon[159415]: pgmap v612: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 1.2 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 6078/57900 objects degraded (10.497%); 2795/57900 objects misplaced (4.827%); 919 KiB/s, 14 objects/s recovering 2023-08-28T02:30:22.492 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:22 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 6078/59487 objects degraded (10.217%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:30:22.783 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:30:22.785 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:30:22.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:22.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:30:22.800 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.3 dump_historic_ops 2023-08-28T02:30:22.900 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:30:22.933 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:23.003 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:30:23.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:23.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:30:23.111 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:30:23.134 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set nodeep-scrub 2023-08-28T02:30:23.199 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:23.371 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:30:23.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:23.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:30:23.477 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:30:23.516 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:30:23.522 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:23.593 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:30:23.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:23.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:30:23.701 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:30:23.761 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:24.470 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:30:24.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:24.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:30:24.577 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:30:24.639 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:24 smithi143 ceph-mon[183059]: pgmap v613: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 1.1 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 6078/52479 objects degraded (11.582%); 2680/52479 objects misplaced (5.107%); 1.8 MiB/s, 28 objects/s recovering 2023-08-28T02:30:24.641 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:24 smithi143 ceph-mon[189871]: pgmap v613: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 1.1 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 6078/52479 objects degraded (11.582%); 2680/52479 objects misplaced (5.107%); 1.8 MiB/s, 28 objects/s recovering 2023-08-28T02:30:24.666 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:24.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:24 smithi169 ceph-mon[159415]: pgmap v613: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 1.1 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 6078/52479 objects degraded (11.582%); 2680/52479 objects misplaced (5.107%); 1.8 MiB/s, 28 objects/s recovering 2023-08-28T02:30:24.826 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:30:25.043 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:30:25.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:25.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:30:25.158 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:30:25.240 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:25.585 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:30:25.623 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:30:25.662 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:30:25.699 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:30:25.746 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:30:25.790 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:30:25.819 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:30:25.820 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:30:25.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:25.830 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:30:25.860 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:30:25.867 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:30:25.867 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:30:25.867 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:30:25.868 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:30:25.868 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:30:25.868 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:30:25.868 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:30:25.868 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:30:25.930 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:30:25.983 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:25.998 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:30:26.045 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:30:26.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:26.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:30:26.156 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:30:26.200 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:26.268 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:30:26.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:26.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:30:26.380 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:30:26.406 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:26.451 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:30:26.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:26.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:30:26.557 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:30:26.606 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:26 smithi169 ceph-mon[159415]: pgmap v614: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 952 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 796 op/s; 6078/45567 objects degraded (13.339%); 2121/45567 objects misplaced (4.655%); 919 KiB/s, 14 objects/s recovering 2023-08-28T02:30:26.607 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:26 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3911184853' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:30:26.607 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:26 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:30:26.607 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:26 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:30:26.610 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:26.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:26 smithi143 ceph-mon[189871]: pgmap v614: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 952 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 796 op/s; 6078/45567 objects degraded (13.339%); 2121/45567 objects misplaced (4.655%); 919 KiB/s, 14 objects/s recovering 2023-08-28T02:30:26.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:26 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3911184853' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:30:26.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:26 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:30:26.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:26 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:30:26.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:26 smithi143 ceph-mon[183059]: pgmap v614: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 952 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 796 op/s; 6078/45567 objects degraded (13.339%); 2121/45567 objects misplaced (4.655%); 919 KiB/s, 14 objects/s recovering 2023-08-28T02:30:26.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:26 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3911184853' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:30:26.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:26 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:30:26.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:26 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:30:26.712 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:30:26.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:26.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:30:26.824 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:30:26.895 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:27.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:27 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 6078/45567 objects degraded (13.339%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:30:27.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:27 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 6078/45567 objects degraded (13.339%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:30:27.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:27 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 6078/45567 objects degraded (13.339%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:30:27.799 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:30:27.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:27.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:30:27.910 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:30:28.000 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:28.176 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:30:28.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:28.181 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:30:28.282 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:30:28.358 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:28.590 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:30:28.612 INFO:teuthology.orchestra.run.smithi143.stderr:dumped all 2023-08-28T02:30:28.648 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:28 smithi143 ceph-mon[183059]: pgmap v615: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 842 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 6078/40305 objects degraded (15.080%); 2005/40305 objects misplaced (4.975%); 1.8 MiB/s, 28 objects/s recovering 2023-08-28T02:30:28.649 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:28 smithi143 ceph-mon[189871]: pgmap v615: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 842 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 6078/40305 objects degraded (15.080%); 2005/40305 objects misplaced (4.975%); 1.8 MiB/s, 28 objects/s recovering 2023-08-28T02:30:28.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:28 smithi169 ceph-mon[159415]: pgmap v615: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 842 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 6078/40305 objects degraded (15.080%); 2005/40305 objects misplaced (4.975%); 1.8 MiB/s, 28 objects/s recovering 2023-08-28T02:30:28.793 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:30:28.793 INFO:teuthology.orchestra.run.smithi143.stdout: "size": 20, 2023-08-28T02:30:28.793 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 600, 2023-08-28T02:30:28.793 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [ 2023-08-28T02:30:28.793 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.793 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:123521 5.0 5.7558ecf0 (undecoded) ondisk+write+known_if_redirected e153)", 2023-08-28T02:30:28.793 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:58.301375+0000", 2023-08-28T02:30:28.793 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 390.39101666599998, 2023-08-28T02:30:28.793 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.89313394000000002, 2023-08-28T02:30:28.794 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:30:28.794 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "reached pg", 2023-08-28T02:30:28.794 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:30:28.794 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:30:28.794 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:30:28.794 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 123521 2023-08-28T02:30:28.794 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.794 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:30:28.794 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.794 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:30:28.794 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.301375+0000", 2023-08-28T02:30:28.794 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:30:28.794 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.795 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.795 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:30:28.795 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.301375+0000", 2023-08-28T02:30:28.795 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:30:28.795 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.795 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.795 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:30:28.795 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.301374+0000", 2023-08-28T02:30:28.795 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:30:28.795 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.795 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.795 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:30:28.796 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.301380+0000", 2023-08-28T02:30:28.796 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.2729999999999999e-06 2023-08-28T02:30:28.796 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.796 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.796 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:30:28.796 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.301390+0000", 2023-08-28T02:30:28.796 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0088000000000001e-05 2023-08-28T02:30:28.796 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.796 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.796 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:30:28.796 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.301402+0000", 2023-08-28T02:30:28.796 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.2335e-05 2023-08-28T02:30:28.796 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.797 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.797 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.797 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.301463+0000", 2023-08-28T02:30:28.797 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.0585999999999999e-05 2023-08-28T02:30:28.797 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.797 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.797 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:30:28.797 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.301478+0000", 2023-08-28T02:30:28.797 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.4983e-05 2023-08-28T02:30:28.797 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.797 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.797 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.797 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.194498+0000", 2023-08-28T02:30:28.798 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.89302048199999995 2023-08-28T02:30:28.798 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.798 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.798 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:30:28.798 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.194509+0000", 2023-08-28T02:30:28.798 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0518e-05 2023-08-28T02:30:28.798 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:28.798 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:30:28.798 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:28.798 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.798 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.798 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:123522 5.0 5.7c0443d0 (undecoded) ondisk+write+known_if_redirected e153)", 2023-08-28T02:30:28.798 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:58.306232+0000", 2023-08-28T02:30:28.798 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 390.386159582, 2023-08-28T02:30:28.799 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.88830339700000005, 2023-08-28T02:30:28.799 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:30:28.799 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "reached pg", 2023-08-28T02:30:28.799 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:30:28.799 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:30:28.799 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:30:28.799 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 123522 2023-08-28T02:30:28.799 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.799 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:30:28.799 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.799 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:30:28.799 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.306232+0000", 2023-08-28T02:30:28.799 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:30:28.800 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.800 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.800 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:30:28.800 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.306232+0000", 2023-08-28T02:30:28.800 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:30:28.800 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.800 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.800 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:30:28.800 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.306230+0000", 2023-08-28T02:30:28.800 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999976 2023-08-28T02:30:28.800 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.800 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.800 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:30:28.801 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.306243+0000", 2023-08-28T02:30:28.801 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3753e-05 2023-08-28T02:30:28.801 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.801 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.801 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:30:28.801 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.306285+0000", 2023-08-28T02:30:28.801 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.1990999999999998e-05 2023-08-28T02:30:28.801 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.801 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.801 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:30:28.801 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.306295+0000", 2023-08-28T02:30:28.801 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.6299999999999993e-06 2023-08-28T02:30:28.801 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.802 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.802 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.802 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.306347+0000", 2023-08-28T02:30:28.802 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.2488999999999998e-05 2023-08-28T02:30:28.802 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.802 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.802 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:30:28.802 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.306356+0000", 2023-08-28T02:30:28.802 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.1270000000000003e-06 2023-08-28T02:30:28.802 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.802 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.802 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.802 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.194528+0000", 2023-08-28T02:30:28.803 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.88817234 2023-08-28T02:30:28.803 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.803 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.803 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:30:28.803 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.194535+0000", 2023-08-28T02:30:28.803 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.537e-06 2023-08-28T02:30:28.803 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:28.803 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:30:28.803 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:28.803 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.803 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.803 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:123553 5.0 5.a384d7c0 (undecoded) ondisk+write+known_if_redirected e153)", 2023-08-28T02:30:28.803 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:58.342227+0000", 2023-08-28T02:30:28.804 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 390.35016458500002, 2023-08-28T02:30:28.804 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.852330637, 2023-08-28T02:30:28.804 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:30:28.804 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "reached pg", 2023-08-28T02:30:28.804 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:30:28.804 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:30:28.804 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:30:28.804 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 123553 2023-08-28T02:30:28.804 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.804 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:30:28.804 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.804 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:30:28.805 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.342227+0000", 2023-08-28T02:30:28.805 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:30:28.805 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.805 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.805 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:30:28.805 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.342227+0000", 2023-08-28T02:30:28.805 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:30:28.805 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.805 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.805 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:30:28.805 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.342225+0000", 2023-08-28T02:30:28.805 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:30:28.805 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.805 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.806 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:30:28.806 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.342236+0000", 2023-08-28T02:30:28.806 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0883999999999999e-05 2023-08-28T02:30:28.806 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.806 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.806 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:30:28.806 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.342261+0000", 2023-08-28T02:30:28.806 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.5318000000000001e-05 2023-08-28T02:30:28.806 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.806 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.806 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:30:28.806 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.342282+0000", 2023-08-28T02:30:28.807 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.0397000000000001e-05 2023-08-28T02:30:28.807 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.807 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.807 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.807 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.342338+0000", 2023-08-28T02:30:28.807 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.6208000000000001e-05 2023-08-28T02:30:28.807 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.807 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.807 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:30:28.807 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.342349+0000", 2023-08-28T02:30:28.807 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0902e-05 2023-08-28T02:30:28.807 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.807 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.807 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.808 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.194551+0000", 2023-08-28T02:30:28.808 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.85220246499999996 2023-08-28T02:30:28.808 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.808 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.808 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:30:28.808 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.194558+0000", 2023-08-28T02:30:28.808 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.533e-06 2023-08-28T02:30:28.808 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:28.808 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:30:28.808 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:28.808 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.808 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.808 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:123563 5.0 5.d31fd5b0 (undecoded) ondisk+write+known_if_redirected e153)", 2023-08-28T02:30:28.809 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:58.354660+0000", 2023-08-28T02:30:28.809 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 390.33773211099998, 2023-08-28T02:30:28.809 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.83992108399999998, 2023-08-28T02:30:28.809 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:30:28.809 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "reached pg", 2023-08-28T02:30:28.809 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:30:28.809 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:30:28.809 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:30:28.809 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 123563 2023-08-28T02:30:28.809 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.809 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:30:28.809 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.809 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:30:28.810 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.354660+0000", 2023-08-28T02:30:28.810 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:30:28.810 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.810 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.810 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:30:28.810 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.354660+0000", 2023-08-28T02:30:28.810 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:30:28.810 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.810 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.810 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:30:28.810 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.354657+0000", 2023-08-28T02:30:28.810 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999976 2023-08-28T02:30:28.810 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.811 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.811 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:30:28.811 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.354667+0000", 2023-08-28T02:30:28.811 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.008e-05 2023-08-28T02:30:28.811 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.811 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.811 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:30:28.811 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.354691+0000", 2023-08-28T02:30:28.811 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.3669999999999999e-05 2023-08-28T02:30:28.811 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.811 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.811 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:30:28.812 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.354702+0000", 2023-08-28T02:30:28.812 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.135e-05 2023-08-28T02:30:28.812 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.812 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.812 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.812 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.354771+0000", 2023-08-28T02:30:28.812 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.8449999999999997e-05 2023-08-28T02:30:28.812 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.812 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.812 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:30:28.812 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.354784+0000", 2023-08-28T02:30:28.812 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3247999999999999e-05 2023-08-28T02:30:28.812 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.813 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.813 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.813 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.194573+0000", 2023-08-28T02:30:28.813 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.83978885700000006 2023-08-28T02:30:28.813 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.813 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.813 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:30:28.813 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.194581+0000", 2023-08-28T02:30:28.813 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.8059999999999995e-06 2023-08-28T02:30:28.813 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:28.813 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:30:28.813 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:28.813 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.814 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.814 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:123565 5.0 5.b98f9370 (undecoded) ondisk+write+known_if_redirected e153)", 2023-08-28T02:30:28.814 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:58.361101+0000", 2023-08-28T02:30:28.814 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 390.331290368, 2023-08-28T02:30:28.814 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.83350137999999996, 2023-08-28T02:30:28.814 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:30:28.814 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "reached pg", 2023-08-28T02:30:28.814 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:30:28.814 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:30:28.814 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:30:28.814 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 123565 2023-08-28T02:30:28.814 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.814 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:30:28.815 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.815 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:30:28.815 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.361101+0000", 2023-08-28T02:30:28.815 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:30:28.815 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.815 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.815 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:30:28.815 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.361101+0000", 2023-08-28T02:30:28.815 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:30:28.815 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.815 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.815 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:30:28.815 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.361099+0000", 2023-08-28T02:30:28.815 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:30:28.816 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.816 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.816 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:30:28.816 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.361108+0000", 2023-08-28T02:30:28.816 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.7749999999999998e-06 2023-08-28T02:30:28.816 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.816 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.816 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:30:28.816 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.361129+0000", 2023-08-28T02:30:28.816 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.1199000000000002e-05 2023-08-28T02:30:28.816 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.816 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.816 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:30:28.817 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.361139+0000", 2023-08-28T02:30:28.817 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.9960000000000005e-06 2023-08-28T02:30:28.817 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.817 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.817 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.817 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.361195+0000", 2023-08-28T02:30:28.817 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.5661000000000001e-05 2023-08-28T02:30:28.817 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.817 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.817 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:30:28.817 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.361207+0000", 2023-08-28T02:30:28.817 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1593999999999999e-05 2023-08-28T02:30:28.817 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.818 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.818 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.818 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.194596+0000", 2023-08-28T02:30:28.818 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.83338983600000005 2023-08-28T02:30:28.818 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.818 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.818 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:30:28.818 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.194603+0000", 2023-08-28T02:30:28.818 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.319e-06 2023-08-28T02:30:28.818 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:28.818 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:30:28.818 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:28.819 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.819 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.819 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:123578 5.0 5.4fc442b0 (undecoded) ondisk+write+known_if_redirected e153)", 2023-08-28T02:30:28.819 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:58.384080+0000", 2023-08-28T02:30:28.819 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 390.30831201500001, 2023-08-28T02:30:28.819 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.81054503600000005, 2023-08-28T02:30:28.819 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:30:28.819 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "reached pg", 2023-08-28T02:30:28.819 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:30:28.819 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:30:28.819 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:30:28.819 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 123578 2023-08-28T02:30:28.819 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.820 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:30:28.820 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.820 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:30:28.820 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.384080+0000", 2023-08-28T02:30:28.820 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:30:28.820 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.820 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.820 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:30:28.820 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.384080+0000", 2023-08-28T02:30:28.820 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:30:28.820 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.820 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.820 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:30:28.821 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.384078+0000", 2023-08-28T02:30:28.821 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:30:28.821 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.821 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.821 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:30:28.821 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.384102+0000", 2023-08-28T02:30:28.821 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.4003e-05 2023-08-28T02:30:28.821 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.821 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.821 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:30:28.821 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.384126+0000", 2023-08-28T02:30:28.821 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.4209000000000001e-05 2023-08-28T02:30:28.821 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.822 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.822 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:30:28.822 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.384137+0000", 2023-08-28T02:30:28.822 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1518000000000001e-05 2023-08-28T02:30:28.822 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.822 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.822 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.822 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.384202+0000", 2023-08-28T02:30:28.822 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.478e-05 2023-08-28T02:30:28.822 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.822 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.822 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:30:28.822 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.384214+0000", 2023-08-28T02:30:28.823 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1683e-05 2023-08-28T02:30:28.823 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.823 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.823 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.823 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.194618+0000", 2023-08-28T02:30:28.823 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.81040443399999995 2023-08-28T02:30:28.823 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.823 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.823 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:30:28.823 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.194625+0000", 2023-08-28T02:30:28.823 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.5309999999999998e-06 2023-08-28T02:30:28.823 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:28.823 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:30:28.824 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:28.824 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.824 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.824 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:123581 5.0 5.b3f22f60 (undecoded) ondisk+write+known_if_redirected e153)", 2023-08-28T02:30:28.824 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:58.396533+0000", 2023-08-28T02:30:28.824 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 390.29585819900001, 2023-08-28T02:30:28.824 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.79811518599999998, 2023-08-28T02:30:28.824 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:30:28.824 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "reached pg", 2023-08-28T02:30:28.824 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:30:28.824 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:30:28.824 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:30:28.824 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 123581 2023-08-28T02:30:28.825 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.825 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:30:28.825 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.825 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:30:28.825 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.396533+0000", 2023-08-28T02:30:28.825 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:30:28.825 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.825 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.825 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:30:28.825 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.396533+0000", 2023-08-28T02:30:28.825 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:30:28.825 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.825 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.826 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:30:28.826 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.396532+0000", 2023-08-28T02:30:28.826 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:30:28.826 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.826 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.826 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:30:28.826 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.396540+0000", 2023-08-28T02:30:28.826 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.568e-06 2023-08-28T02:30:28.826 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.826 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.826 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:30:28.826 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.396560+0000", 2023-08-28T02:30:28.827 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.9976e-05 2023-08-28T02:30:28.827 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.827 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.827 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:30:28.827 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.396571+0000", 2023-08-28T02:30:28.827 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0594000000000001e-05 2023-08-28T02:30:28.827 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.827 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.827 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.827 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.396625+0000", 2023-08-28T02:30:28.827 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.4197999999999998e-05 2023-08-28T02:30:28.827 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.827 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.828 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:30:28.828 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.396636+0000", 2023-08-28T02:30:28.828 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1113e-05 2023-08-28T02:30:28.828 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.828 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.828 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.828 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.194641+0000", 2023-08-28T02:30:28.828 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.798005407 2023-08-28T02:30:28.828 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.828 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.828 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:30:28.828 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.194649+0000", 2023-08-28T02:30:28.828 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.2389999999999996e-06 2023-08-28T02:30:28.829 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:28.829 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:30:28.829 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:28.829 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.829 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.829 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:24531 6.b 6:df250847:::benchmark_data_smithi143_305742_object24530:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e195)", 2023-08-28T02:30:28.829 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:27:16.320361+0000", 2023-08-28T02:30:28.829 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 192.37203034500001, 2023-08-28T02:30:28.829 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.1070281099999999, 2023-08-28T02:30:28.829 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:30:28.829 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "reached pg", 2023-08-28T02:30:28.829 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:30:28.829 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:30:28.830 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:30:28.830 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 24531 2023-08-28T02:30:28.830 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.830 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:30:28.830 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.830 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:30:28.830 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.320361+0000", 2023-08-28T02:30:28.830 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:30:28.830 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.830 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.830 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:30:28.830 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.320361+0000", 2023-08-28T02:30:28.830 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:30:28.831 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.831 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.831 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:30:28.831 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.320360+0000", 2023-08-28T02:30:28.831 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:30:28.831 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.831 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.831 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:30:28.831 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.320379+0000", 2023-08-28T02:30:28.831 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.9511999999999999e-05 2023-08-28T02:30:28.831 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.831 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.832 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:30:28.832 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.320390+0000", 2023-08-28T02:30:28.832 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.045e-05 2023-08-28T02:30:28.832 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.832 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.832 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:30:28.832 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.320398+0000", 2023-08-28T02:30:28.832 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.1170000000000004e-06 2023-08-28T02:30:28.832 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.832 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.832 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.832 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.320457+0000", 2023-08-28T02:30:28.832 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.8777999999999999e-05 2023-08-28T02:30:28.833 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.833 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.833 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:30:28.833 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.320637+0000", 2023-08-28T02:30:28.833 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000180027 2023-08-28T02:30:28.833 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.833 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.833 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 6,7", 2023-08-28T02:30:28.833 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.320816+0000", 2023-08-28T02:30:28.833 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000178896 2023-08-28T02:30:28.833 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.833 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.833 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.834 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.326752+0000", 2023-08-28T02:30:28.834 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0059362620000000003 2023-08-28T02:30:28.834 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.834 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.834 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:30:28.834 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.326764+0000", 2023-08-28T02:30:28.834 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.2121e-05 2023-08-28T02:30:28.834 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.834 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.834 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.834 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:17.317026+0000", 2023-08-28T02:30:28.834 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.99026151500000004 2023-08-28T02:30:28.835 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.835 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.835 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:30:28.835 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:17.317041+0000", 2023-08-28T02:30:28.835 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.5868000000000001e-05 2023-08-28T02:30:28.835 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.835 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.835 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.835 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.411484+0000", 2023-08-28T02:30:28.835 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.094442685 2023-08-28T02:30:28.835 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.835 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.835 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for degraded object", 2023-08-28T02:30:28.836 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.412525+0000", 2023-08-28T02:30:28.836 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0010407999999999999 2023-08-28T02:30:28.836 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.836 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.836 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.836 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.427277+0000", 2023-08-28T02:30:28.836 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.014751874999999999 2023-08-28T02:30:28.836 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.836 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.836 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:30:28.836 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.427389+0000", 2023-08-28T02:30:28.836 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00011259700000000001 2023-08-28T02:30:28.836 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:28.836 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:30:28.837 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:28.837 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.837 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.837 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:24535 6.b 6:d57aa623:::benchmark_data_smithi143_305742_object24534:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e195)", 2023-08-28T02:30:28.837 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:27:16.320888+0000", 2023-08-28T02:30:28.837 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 192.37150383599999, 2023-08-28T02:30:28.837 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.1085812970000002, 2023-08-28T02:30:28.837 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:30:28.837 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "reached pg", 2023-08-28T02:30:28.837 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:30:28.837 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:30:28.837 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:30:28.837 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 24535 2023-08-28T02:30:28.838 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.838 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:30:28.838 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.838 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:30:28.838 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.320888+0000", 2023-08-28T02:30:28.838 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:30:28.838 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.838 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.838 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:30:28.838 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.320888+0000", 2023-08-28T02:30:28.838 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:30:28.838 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.838 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.839 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:30:28.839 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.320887+0000", 2023-08-28T02:30:28.839 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:30:28.839 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.839 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.839 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:30:28.839 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.320908+0000", 2023-08-28T02:30:28.839 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.124e-05 2023-08-28T02:30:28.839 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.839 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.839 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:30:28.839 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.320921+0000", 2023-08-28T02:30:28.839 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.259e-05 2023-08-28T02:30:28.840 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.840 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.840 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:30:28.840 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.320929+0000", 2023-08-28T02:30:28.840 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.0539999999999995e-06 2023-08-28T02:30:28.840 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.840 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.840 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.840 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.321877+0000", 2023-08-28T02:30:28.840 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00094830700000000001 2023-08-28T02:30:28.840 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.840 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.841 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:30:28.841 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.322041+0000", 2023-08-28T02:30:28.841 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00016411900000000001 2023-08-28T02:30:28.841 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.841 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.841 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 6,7", 2023-08-28T02:30:28.841 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.322212+0000", 2023-08-28T02:30:28.841 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00017074500000000001 2023-08-28T02:30:28.841 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.841 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.841 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.841 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.326837+0000", 2023-08-28T02:30:28.841 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0046252649999999999 2023-08-28T02:30:28.841 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.842 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.842 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:30:28.842 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.326845+0000", 2023-08-28T02:30:28.842 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.1589999999999993e-06 2023-08-28T02:30:28.842 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.842 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.842 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.842 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:17.318726+0000", 2023-08-28T02:30:28.842 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.99188029 2023-08-28T02:30:28.842 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.842 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.842 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:30:28.842 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:17.318739+0000", 2023-08-28T02:30:28.843 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3410999999999999e-05 2023-08-28T02:30:28.843 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.843 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.843 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.843 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.413991+0000", 2023-08-28T02:30:28.843 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.095252347 2023-08-28T02:30:28.843 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.843 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.843 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for degraded object", 2023-08-28T02:30:28.843 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.414851+0000", 2023-08-28T02:30:28.843 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00085915000000000004 2023-08-28T02:30:28.843 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.844 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.844 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.844 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.429365+0000", 2023-08-28T02:30:28.844 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.01451469 2023-08-28T02:30:28.844 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.844 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.844 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:30:28.844 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.429469+0000", 2023-08-28T02:30:28.844 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00010385100000000001 2023-08-28T02:30:28.844 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:28.844 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:30:28.844 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:28.844 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.845 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.845 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:24539 6.c 6:37ee2adc:::benchmark_data_smithi143_305742_object24538:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e196)", 2023-08-28T02:30:28.845 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:27:16.328855+0000", 2023-08-28T02:30:28.845 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 192.36353714399999, 2023-08-28T02:30:28.845 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.0684156310000001, 2023-08-28T02:30:28.845 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:30:28.845 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:30:28.845 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:30:28.845 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:30:28.845 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:30:28.845 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 24539 2023-08-28T02:30:28.845 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.845 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:30:28.846 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.846 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:30:28.846 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.328855+0000", 2023-08-28T02:30:28.846 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:30:28.846 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.846 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.846 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:30:28.846 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.328855+0000", 2023-08-28T02:30:28.846 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:30:28.846 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.846 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.846 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:30:28.846 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.328853+0000", 2023-08-28T02:30:28.847 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:30:28.847 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.847 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.847 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:30:28.847 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.328886+0000", 2023-08-28T02:30:28.847 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.3365000000000003e-05 2023-08-28T02:30:28.847 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.847 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.847 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:30:28.847 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.328899+0000", 2023-08-28T02:30:28.847 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.2480000000000001e-05 2023-08-28T02:30:28.847 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.847 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.848 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:30:28.848 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.328907+0000", 2023-08-28T02:30:28.848 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.4090000000000003e-06 2023-08-28T02:30:28.848 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.848 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.848 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.848 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.328959+0000", 2023-08-28T02:30:28.848 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.2089999999999998e-05 2023-08-28T02:30:28.848 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.848 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.848 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:30:28.848 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.328970+0000", 2023-08-28T02:30:28.848 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0574000000000001e-05 2023-08-28T02:30:28.849 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.849 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.849 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.849 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:17.318763+0000", 2023-08-28T02:30:28.849 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.98979299099999996 2023-08-28T02:30:28.849 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.849 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.849 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:30:28.849 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:17.318773+0000", 2023-08-28T02:30:28.849 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.9199999999999999e-06 2023-08-28T02:30:28.849 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.849 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.849 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.850 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.389871+0000", 2023-08-28T02:30:28.850 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0710983110000001 2023-08-28T02:30:28.850 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.850 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.850 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:30:28.850 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.390037+0000", 2023-08-28T02:30:28.850 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000166016 2023-08-28T02:30:28.850 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.850 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.850 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 5,7", 2023-08-28T02:30:28.850 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.390215+0000", 2023-08-28T02:30:28.850 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00017822000000000001 2023-08-28T02:30:28.850 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.851 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.851 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:30:28.851 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.392419+0000", 2023-08-28T02:30:28.851 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0022036859999999998 2023-08-28T02:30:28.851 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.851 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.851 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:30:28.851 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.395920+0000", 2023-08-28T02:30:28.851 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0035014930000000001 2023-08-28T02:30:28.851 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.851 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.851 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:30:28.851 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.396023+0000", 2023-08-28T02:30:28.852 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00010273899999999999 2023-08-28T02:30:28.852 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.852 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.852 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:30:28.852 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.396082+0000", 2023-08-28T02:30:28.852 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.8977999999999997e-05 2023-08-28T02:30:28.852 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.852 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.852 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:30:28.852 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.397270+0000", 2023-08-28T02:30:28.852 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0011880510000000001 2023-08-28T02:30:28.852 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:28.853 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:30:28.853 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:28.853 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.853 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.853 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:24543 6.c 6:3872c264:::benchmark_data_smithi143_305742_object24542:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e196)", 2023-08-28T02:30:28.853 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:27:16.333172+0000", 2023-08-28T02:30:28.853 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 192.35921952699999, 2023-08-28T02:30:28.853 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.0692718540000001, 2023-08-28T02:30:28.853 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:30:28.853 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:30:28.853 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:30:28.853 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:30:28.853 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:30:28.854 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 24543 2023-08-28T02:30:28.854 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.854 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:30:28.854 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.854 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:30:28.854 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.333172+0000", 2023-08-28T02:30:28.854 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:30:28.854 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.854 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.854 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:30:28.854 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.333172+0000", 2023-08-28T02:30:28.854 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:30:28.855 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.855 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.855 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:30:28.855 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.333170+0000", 2023-08-28T02:30:28.855 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:30:28.855 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.855 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.855 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:30:28.855 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.333197+0000", 2023-08-28T02:30:28.855 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.6242e-05 2023-08-28T02:30:28.855 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.855 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.855 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:30:28.856 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.333212+0000", 2023-08-28T02:30:28.856 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.5457000000000001e-05 2023-08-28T02:30:28.856 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.856 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.856 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:30:28.856 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.333221+0000", 2023-08-28T02:30:28.856 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.0110000000000006e-06 2023-08-28T02:30:28.856 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.856 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.856 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.856 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.333266+0000", 2023-08-28T02:30:28.856 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.5185000000000002e-05 2023-08-28T02:30:28.856 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.857 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.857 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:30:28.857 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.333277+0000", 2023-08-28T02:30:28.857 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1113e-05 2023-08-28T02:30:28.857 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.857 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.857 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.857 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:17.318796+0000", 2023-08-28T02:30:28.857 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.98551865400000005 2023-08-28T02:30:28.857 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.857 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.857 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:30:28.857 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:17.318803+0000", 2023-08-28T02:30:28.858 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.8669999999999996e-06 2023-08-28T02:30:28.858 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.858 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.858 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.858 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.392449+0000", 2023-08-28T02:30:28.858 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0736462950000001 2023-08-28T02:30:28.858 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.858 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.858 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:30:28.858 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.392620+0000", 2023-08-28T02:30:28.858 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00017026000000000001 2023-08-28T02:30:28.858 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.858 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.859 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 5,7", 2023-08-28T02:30:28.859 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.392829+0000", 2023-08-28T02:30:28.859 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000209471 2023-08-28T02:30:28.859 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.859 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.859 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:30:28.859 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.397281+0000", 2023-08-28T02:30:28.859 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0044518149999999996 2023-08-28T02:30:28.859 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.859 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.859 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:30:28.859 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.397661+0000", 2023-08-28T02:30:28.859 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00038041300000000002 2023-08-28T02:30:28.860 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.860 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.860 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:30:28.860 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.401154+0000", 2023-08-28T02:30:28.860 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0034931839999999999 2023-08-28T02:30:28.860 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.860 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.860 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:30:28.860 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.401223+0000", 2023-08-28T02:30:28.860 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.8558000000000003e-05 2023-08-28T02:30:28.860 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.860 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.860 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:30:28.861 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.402444+0000", 2023-08-28T02:30:28.861 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.001220989 2023-08-28T02:30:28.861 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:28.861 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:30:28.861 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:28.861 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.861 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.861 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:24547 6.b 6:d3c1f3d7:::benchmark_data_smithi143_305742_object24546:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e196)", 2023-08-28T02:30:28.861 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:27:16.334218+0000", 2023-08-28T02:30:28.861 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 192.35817339499999, 2023-08-28T02:30:28.861 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.0963319720000002, 2023-08-28T02:30:28.861 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:30:28.861 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:30:28.862 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:30:28.862 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:30:28.862 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:30:28.862 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 24547 2023-08-28T02:30:28.862 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.862 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:30:28.862 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.862 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:30:28.862 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.334218+0000", 2023-08-28T02:30:28.862 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:30:28.862 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.862 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.862 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:30:28.862 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.334218+0000", 2023-08-28T02:30:28.863 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:30:28.863 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.863 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.863 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:30:28.863 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.334218+0000", 2023-08-28T02:30:28.863 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:30:28.863 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.863 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.863 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:30:28.863 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.334239+0000", 2023-08-28T02:30:28.863 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.1067999999999999e-05 2023-08-28T02:30:28.863 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.863 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.864 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:30:28.864 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.334246+0000", 2023-08-28T02:30:28.864 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.99e-06 2023-08-28T02:30:28.864 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.864 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.864 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:30:28.864 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.334251+0000", 2023-08-28T02:30:28.864 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.57e-06 2023-08-28T02:30:28.864 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.864 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.864 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.864 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.334304+0000", 2023-08-28T02:30:28.864 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.2512999999999997e-05 2023-08-28T02:30:28.865 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.865 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.865 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:30:28.865 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.334311+0000", 2023-08-28T02:30:28.865 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.9480000000000002e-06 2023-08-28T02:30:28.865 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.865 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.865 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.865 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:17.318776+0000", 2023-08-28T02:30:28.865 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.98446481100000005 2023-08-28T02:30:28.865 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.865 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.865 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:30:28.866 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:17.318784+0000", 2023-08-28T02:30:28.866 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.5399999999999996e-06 2023-08-28T02:30:28.866 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.866 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.866 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.866 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.414890+0000", 2023-08-28T02:30:28.866 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0961061059999999 2023-08-28T02:30:28.866 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.866 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.866 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:30:28.866 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.415073+0000", 2023-08-28T02:30:28.866 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000182981 2023-08-28T02:30:28.866 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.866 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.867 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 4,7", 2023-08-28T02:30:28.867 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.415266+0000", 2023-08-28T02:30:28.867 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00019221099999999999 2023-08-28T02:30:28.867 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.867 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.867 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:30:28.867 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.419976+0000", 2023-08-28T02:30:28.867 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0047105030000000004 2023-08-28T02:30:28.867 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.867 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.867 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:30:28.867 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.429512+0000", 2023-08-28T02:30:28.867 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0095358749999999992 2023-08-28T02:30:28.868 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.868 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.868 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:30:28.868 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.429591+0000", 2023-08-28T02:30:28.868 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.9344000000000005e-05 2023-08-28T02:30:28.868 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.868 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.868 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:30:28.868 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.429643+0000", 2023-08-28T02:30:28.868 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.1643e-05 2023-08-28T02:30:28.868 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.868 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.868 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:30:28.869 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.430550+0000", 2023-08-28T02:30:28.869 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00090736799999999998 2023-08-28T02:30:28.869 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:28.869 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:30:28.869 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:28.869 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.869 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.869 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:24550 6.c 6:3ac8366e:::benchmark_data_smithi143_305742_object24549:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e196)", 2023-08-28T02:30:28.869 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:27:16.341304+0000", 2023-08-28T02:30:28.869 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 192.35108751999999, 2023-08-28T02:30:28.869 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.0622437910000002, 2023-08-28T02:30:28.869 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:30:28.869 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:30:28.870 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:30:28.870 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:30:28.870 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:30:28.870 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 24550 2023-08-28T02:30:28.870 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.870 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:30:28.870 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.870 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:30:28.870 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.341304+0000", 2023-08-28T02:30:28.870 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:30:28.870 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.870 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.870 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:30:28.871 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.341304+0000", 2023-08-28T02:30:28.871 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:30:28.871 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.871 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.871 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:30:28.871 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.341302+0000", 2023-08-28T02:30:28.871 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:30:28.871 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.871 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.871 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:30:28.871 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.341327+0000", 2023-08-28T02:30:28.871 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.4898000000000001e-05 2023-08-28T02:30:28.871 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.872 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.872 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:30:28.872 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.341344+0000", 2023-08-28T02:30:28.872 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.649e-05 2023-08-28T02:30:28.872 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.872 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.872 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:30:28.872 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.341353+0000", 2023-08-28T02:30:28.872 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.8079999999999994e-06 2023-08-28T02:30:28.872 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.872 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.872 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.872 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.341409+0000", 2023-08-28T02:30:28.873 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.6702e-05 2023-08-28T02:30:28.873 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.873 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.873 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:30:28.873 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.341418+0000", 2023-08-28T02:30:28.873 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.3480000000000004e-06 2023-08-28T02:30:28.873 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.873 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.873 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.873 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:17.318825+0000", 2023-08-28T02:30:28.873 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.977407523 2023-08-28T02:30:28.873 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.873 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.874 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:30:28.874 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:17.318833+0000", 2023-08-28T02:30:28.874 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.2609999999999996e-06 2023-08-28T02:30:28.874 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.874 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.874 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.874 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.394062+0000", 2023-08-28T02:30:28.874 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.075229827 2023-08-28T02:30:28.874 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.874 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.874 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:30:28.874 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.394241+0000", 2023-08-28T02:30:28.874 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00017895699999999999 2023-08-28T02:30:28.875 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.875 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.875 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 5,7", 2023-08-28T02:30:28.875 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.394429+0000", 2023-08-28T02:30:28.875 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00018816500000000001 2023-08-28T02:30:28.875 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.875 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.875 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:30:28.875 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.397287+0000", 2023-08-28T02:30:28.875 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00285746 2023-08-28T02:30:28.875 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.875 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.875 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:30:28.876 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.397696+0000", 2023-08-28T02:30:28.876 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00040932400000000003 2023-08-28T02:30:28.876 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.876 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.876 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:30:28.876 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.402502+0000", 2023-08-28T02:30:28.876 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0048053749999999998 2023-08-28T02:30:28.876 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.876 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.876 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:30:28.876 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.402561+0000", 2023-08-28T02:30:28.876 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.8909000000000001e-05 2023-08-28T02:30:28.876 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.877 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.877 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:30:28.877 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.403548+0000", 2023-08-28T02:30:28.877 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00098740299999999998 2023-08-28T02:30:28.877 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:28.877 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:30:28.877 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:28.877 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.877 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.877 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:24552 6.b 6:df693f16:::benchmark_data_smithi143_305742_object24551:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e196)", 2023-08-28T02:30:28.877 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:27:16.342014+0000", 2023-08-28T02:30:28.877 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 192.35037780499999, 2023-08-28T02:30:28.877 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.089540714, 2023-08-28T02:30:28.878 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:30:28.878 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:30:28.878 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:30:28.878 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:30:28.878 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:30:28.878 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 24552 2023-08-28T02:30:28.878 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.878 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:30:28.878 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.878 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:30:28.878 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.342014+0000", 2023-08-28T02:30:28.878 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:30:28.878 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.879 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.879 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:30:28.879 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.342014+0000", 2023-08-28T02:30:28.879 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:30:28.879 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.879 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.879 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:30:28.879 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.342013+0000", 2023-08-28T02:30:28.879 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:30:28.879 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.879 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.879 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:30:28.879 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.342033+0000", 2023-08-28T02:30:28.880 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.9669000000000001e-05 2023-08-28T02:30:28.880 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.880 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.880 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:30:28.880 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.342053+0000", 2023-08-28T02:30:28.880 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.9826999999999998e-05 2023-08-28T02:30:28.880 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.880 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.880 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:30:28.880 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.342065+0000", 2023-08-28T02:30:28.880 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.2617e-05 2023-08-28T02:30:28.880 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.880 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.880 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.881 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.342125+0000", 2023-08-28T02:30:28.881 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.9490000000000001e-05 2023-08-28T02:30:28.881 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.881 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.881 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:30:28.881 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.342133+0000", 2023-08-28T02:30:28.881 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.6810000000000004e-06 2023-08-28T02:30:28.881 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.881 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.881 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.881 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:17.318812+0000", 2023-08-28T02:30:28.881 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.97667820100000002 2023-08-28T02:30:28.881 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.882 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.882 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:30:28.882 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:17.318820+0000", 2023-08-28T02:30:28.882 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.2409999999999995e-06 2023-08-28T02:30:28.882 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.882 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.882 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.882 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.416769+0000", 2023-08-28T02:30:28.882 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.097949235 2023-08-28T02:30:28.882 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.882 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.882 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:30:28.883 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.416955+0000", 2023-08-28T02:30:28.883 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00018632300000000001 2023-08-28T02:30:28.883 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.883 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.883 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 4,7", 2023-08-28T02:30:28.883 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.417157+0000", 2023-08-28T02:30:28.883 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00020180999999999999 2023-08-28T02:30:28.883 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.883 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.883 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:30:28.883 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.419984+0000", 2023-08-28T02:30:28.883 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.002826781 2023-08-28T02:30:28.883 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.884 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.884 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:30:28.884 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.429553+0000", 2023-08-28T02:30:28.884 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0095693670000000005 2023-08-28T02:30:28.884 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.884 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.884 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:30:28.884 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.430600+0000", 2023-08-28T02:30:28.884 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.001047217 2023-08-28T02:30:28.884 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.884 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.884 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:30:28.885 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.430650+0000", 2023-08-28T02:30:28.885 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.9903e-05 2023-08-28T02:30:28.885 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.885 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.885 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:30:28.885 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.431555+0000", 2023-08-28T02:30:28.885 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00090419100000000004 2023-08-28T02:30:28.885 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:28.885 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:30:28.885 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:28.885 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.885 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.885 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:24559 6.b 6:d824c343:::benchmark_data_smithi143_305742_object24558:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e196)", 2023-08-28T02:30:28.886 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:27:16.354268+0000", 2023-08-28T02:30:28.886 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 192.338124142, 2023-08-28T02:30:28.886 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.0782777179999998, 2023-08-28T02:30:28.886 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:30:28.886 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:30:28.886 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:30:28.886 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:30:28.886 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:30:28.886 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 24559 2023-08-28T02:30:28.886 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.886 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:30:28.886 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.886 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:30:28.887 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.354268+0000", 2023-08-28T02:30:28.887 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:30:28.887 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.887 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.887 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:30:28.887 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.354268+0000", 2023-08-28T02:30:28.887 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:30:28.887 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.887 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.887 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:30:28.887 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.354267+0000", 2023-08-28T02:30:28.887 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:30:28.887 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.888 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.888 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:30:28.888 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.354286+0000", 2023-08-28T02:30:28.888 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.9307e-05 2023-08-28T02:30:28.888 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.888 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.888 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:30:28.888 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.354297+0000", 2023-08-28T02:30:28.888 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1544e-05 2023-08-28T02:30:28.888 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.888 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.888 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:30:28.888 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.354303+0000", 2023-08-28T02:30:28.889 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.6860000000000003e-06 2023-08-28T02:30:28.889 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.889 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.889 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.889 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.354363+0000", 2023-08-28T02:30:28.889 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.0337999999999997e-05 2023-08-28T02:30:28.889 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.889 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.889 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:30:28.889 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.354376+0000", 2023-08-28T02:30:28.889 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.2218999999999999e-05 2023-08-28T02:30:28.889 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.889 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.890 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.890 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:17.318866+0000", 2023-08-28T02:30:28.890 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.96449035000000005 2023-08-28T02:30:28.890 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.890 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.890 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:30:28.890 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:17.318874+0000", 2023-08-28T02:30:28.890 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.4039999999999995e-06 2023-08-28T02:30:28.890 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.890 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.890 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.890 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.418477+0000", 2023-08-28T02:30:28.890 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.099602873 2023-08-28T02:30:28.891 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.891 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.891 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:30:28.891 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.418650+0000", 2023-08-28T02:30:28.891 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000173134 2023-08-28T02:30:28.891 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.891 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.891 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 4,7", 2023-08-28T02:30:28.891 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.418804+0000", 2023-08-28T02:30:28.891 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00015363400000000001 2023-08-28T02:30:28.891 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.891 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.891 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:30:28.892 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.420430+0000", 2023-08-28T02:30:28.892 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0016257139999999999 2023-08-28T02:30:28.892 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.892 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.892 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:30:28.892 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.431611+0000", 2023-08-28T02:30:28.892 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.011181590999999999 2023-08-28T02:30:28.892 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.892 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.892 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:30:28.892 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.431651+0000", 2023-08-28T02:30:28.892 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.9746000000000002e-05 2023-08-28T02:30:28.892 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.893 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.893 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:30:28.893 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.431700+0000", 2023-08-28T02:30:28.893 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.9177e-05 2023-08-28T02:30:28.893 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.893 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.893 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:30:28.893 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.432545+0000", 2023-08-28T02:30:28.893 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00084502299999999998 2023-08-28T02:30:28.893 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:28.893 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:30:28.893 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:28.893 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.894 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.894 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:78158 6.c 6:3c2b9257:::benchmark_data_smithi143_305742_object78157:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e220)", 2023-08-28T02:30:28.894 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:28:24.351475+0000", 2023-08-28T02:30:28.894 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 124.34091642600001, 2023-08-28T02:30:28.894 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.115024359, 2023-08-28T02:30:28.894 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:30:28.894 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:30:28.894 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:30:28.894 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:30:28.894 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:30:28.894 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 78158 2023-08-28T02:30:28.894 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.894 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:30:28.895 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.895 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:30:28.895 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.351475+0000", 2023-08-28T02:30:28.895 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:30:28.895 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.895 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.895 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:30:28.895 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.351475+0000", 2023-08-28T02:30:28.895 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:30:28.895 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.895 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.895 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:30:28.895 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.351473+0000", 2023-08-28T02:30:28.896 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999976 2023-08-28T02:30:28.896 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.896 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.896 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:30:28.896 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.351505+0000", 2023-08-28T02:30:28.896 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.2289999999999997e-05 2023-08-28T02:30:28.896 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.896 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.896 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:30:28.896 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.351530+0000", 2023-08-28T02:30:28.896 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.4946000000000001e-05 2023-08-28T02:30:28.896 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.896 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.897 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:30:28.897 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.351544+0000", 2023-08-28T02:30:28.897 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3298e-05 2023-08-28T02:30:28.897 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.897 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.897 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.897 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.352280+0000", 2023-08-28T02:30:28.897 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00073618100000000003 2023-08-28T02:30:28.897 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.897 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.897 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:30:28.897 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.352291+0000", 2023-08-28T02:30:28.898 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0964e-05 2023-08-28T02:30:28.898 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.898 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.898 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.898 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.331636+0000", 2023-08-28T02:30:28.898 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.97934522000000002 2023-08-28T02:30:28.898 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.898 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.898 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:30:28.898 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.331647+0000", 2023-08-28T02:30:28.898 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1392e-05 2023-08-28T02:30:28.898 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.898 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.899 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.899 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.452208+0000", 2023-08-28T02:30:28.899 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.120560625 2023-08-28T02:30:28.899 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.899 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.899 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:30:28.899 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.452384+0000", 2023-08-28T02:30:28.899 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00017561200000000001 2023-08-28T02:30:28.899 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.899 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.899 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 5,6,7", 2023-08-28T02:30:28.899 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.452555+0000", 2023-08-28T02:30:28.899 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00017118799999999999 2023-08-28T02:30:28.900 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.900 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.900 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:30:28.900 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.455618+0000", 2023-08-28T02:30:28.900 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0030633460000000002 2023-08-28T02:30:28.900 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.900 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.900 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:30:28.900 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.455688+0000", 2023-08-28T02:30:28.900 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.9611999999999997e-05 2023-08-28T02:30:28.900 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.900 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.900 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:30:28.901 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.458168+0000", 2023-08-28T02:30:28.901 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0024801239999999998 2023-08-28T02:30:28.901 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.901 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.901 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:30:28.901 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.466293+0000", 2023-08-28T02:30:28.901 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0081255110000000002 2023-08-28T02:30:28.901 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.901 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.901 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:30:28.901 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.466366+0000", 2023-08-28T02:30:28.901 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.2871000000000001e-05 2023-08-28T02:30:28.901 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.902 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.902 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:30:28.902 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.466500+0000", 2023-08-28T02:30:28.902 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00013335899999999999 2023-08-28T02:30:28.902 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:28.902 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:30:28.902 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:28.902 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.902 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.902 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:78162 6.c 6:318d1535:::benchmark_data_smithi143_305742_object78161:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e220)", 2023-08-28T02:30:28.902 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:28:24.353558+0000", 2023-08-28T02:30:28.902 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 124.338834017, 2023-08-28T02:30:28.902 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.1131857620000001, 2023-08-28T02:30:28.903 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:30:28.903 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:30:28.903 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:30:28.903 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:30:28.903 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:30:28.903 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 78162 2023-08-28T02:30:28.903 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.903 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:30:28.903 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.903 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:30:28.903 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.353558+0000", 2023-08-28T02:30:28.903 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:30:28.904 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.904 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.904 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:30:28.904 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.353558+0000", 2023-08-28T02:30:28.904 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:30:28.904 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.904 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.904 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:30:28.904 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.353556+0000", 2023-08-28T02:30:28.904 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:30:28.904 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.904 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.904 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:30:28.905 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.353594+0000", 2023-08-28T02:30:28.905 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.7595999999999997e-05 2023-08-28T02:30:28.905 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.905 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.905 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:30:28.905 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.353608+0000", 2023-08-28T02:30:28.905 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.4416000000000001e-05 2023-08-28T02:30:28.905 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.905 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.905 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:30:28.905 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.353617+0000", 2023-08-28T02:30:28.905 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.9069999999999996e-06 2023-08-28T02:30:28.905 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.906 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.906 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.906 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.353671+0000", 2023-08-28T02:30:28.906 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.3871e-05 2023-08-28T02:30:28.906 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.906 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.906 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:30:28.906 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.353681+0000", 2023-08-28T02:30:28.906 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0713999999999999e-05 2023-08-28T02:30:28.906 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.906 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.906 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.906 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.331885+0000", 2023-08-28T02:30:28.907 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.97820372 2023-08-28T02:30:28.907 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.907 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.907 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:30:28.907 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.331895+0000", 2023-08-28T02:30:28.907 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.9920000000000002e-06 2023-08-28T02:30:28.907 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.907 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.907 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.907 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.455726+0000", 2023-08-28T02:30:28.907 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1238307970000001 2023-08-28T02:30:28.907 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.907 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.908 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:30:28.908 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.455870+0000", 2023-08-28T02:30:28.908 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000143814 2023-08-28T02:30:28.908 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.908 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.908 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 5,6,7", 2023-08-28T02:30:28.908 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.456053+0000", 2023-08-28T02:30:28.908 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000183234 2023-08-28T02:30:28.908 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.908 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.908 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:30:28.908 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.461802+0000", 2023-08-28T02:30:28.908 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0057491449999999998 2023-08-28T02:30:28.909 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.909 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.909 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:30:28.909 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.464367+0000", 2023-08-28T02:30:28.909 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0025651480000000002 2023-08-28T02:30:28.909 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.909 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.909 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:30:28.909 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.464470+0000", 2023-08-28T02:30:28.909 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00010294 2023-08-28T02:30:28.909 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.909 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.909 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:30:28.910 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.466571+0000", 2023-08-28T02:30:28.910 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0021002669999999998 2023-08-28T02:30:28.910 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.910 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.910 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:30:28.910 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.466630+0000", 2023-08-28T02:30:28.910 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.9827000000000002e-05 2023-08-28T02:30:28.910 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.910 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.910 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:30:28.910 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.466743+0000", 2023-08-28T02:30:28.910 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000113047 2023-08-28T02:30:28.910 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:28.910 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:30:28.911 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:28.911 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.911 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.911 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:78191 6.c 6:388e80c8:::benchmark_data_smithi143_305742_object78190:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e221)", 2023-08-28T02:30:28.911 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:28:25.485179+0000", 2023-08-28T02:30:28.911 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 123.20721285800001, 2023-08-28T02:30:28.911 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.98181897100000004, 2023-08-28T02:30:28.911 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:30:28.911 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:30:28.911 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:30:28.911 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:30:28.911 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:30:28.912 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 78191 2023-08-28T02:30:28.912 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.912 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:30:28.912 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.912 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:30:28.912 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.485179+0000", 2023-08-28T02:30:28.912 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:30:28.912 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.912 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.912 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:30:28.912 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.485179+0000", 2023-08-28T02:30:28.912 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:30:28.912 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.913 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.913 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:30:28.913 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.485177+0000", 2023-08-28T02:30:28.913 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:30:28.913 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.913 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.913 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:30:28.913 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.485203+0000", 2023-08-28T02:30:28.913 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.6021e-05 2023-08-28T02:30:28.913 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.913 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.913 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:30:28.913 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.485220+0000", 2023-08-28T02:30:28.913 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.7220000000000001e-05 2023-08-28T02:30:28.914 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.914 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.914 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:30:28.914 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.485230+0000", 2023-08-28T02:30:28.914 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.3940000000000002e-06 2023-08-28T02:30:28.914 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.914 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.914 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.914 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.485298+0000", 2023-08-28T02:30:28.914 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.8418999999999997e-05 2023-08-28T02:30:28.914 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.914 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.915 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:30:28.915 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.485312+0000", 2023-08-28T02:30:28.915 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.4382000000000001e-05 2023-08-28T02:30:28.915 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.915 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.915 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.915 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.458189+0000", 2023-08-28T02:30:28.915 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.97287682399999997 2023-08-28T02:30:28.915 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.915 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.915 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:30:28.915 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.458284+0000", 2023-08-28T02:30:28.915 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.4900000000000003e-05 2023-08-28T02:30:28.916 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.916 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.916 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 5,6,7", 2023-08-28T02:30:28.916 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.458384+0000", 2023-08-28T02:30:28.916 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.9763e-05 2023-08-28T02:30:28.916 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.916 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.916 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:30:28.916 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.461806+0000", 2023-08-28T02:30:28.916 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0034222649999999999 2023-08-28T02:30:28.916 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.916 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.916 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:30:28.917 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.464511+0000", 2023-08-28T02:30:28.917 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0027045929999999999 2023-08-28T02:30:28.917 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.917 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.917 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:30:28.917 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.464548+0000", 2023-08-28T02:30:28.917 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.7626999999999997e-05 2023-08-28T02:30:28.917 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.917 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.917 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:30:28.917 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.466803+0000", 2023-08-28T02:30:28.917 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0022543279999999999 2023-08-28T02:30:28.917 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.917 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.918 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:30:28.918 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.466866+0000", 2023-08-28T02:30:28.918 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.3870999999999999e-05 2023-08-28T02:30:28.918 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.918 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.918 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:30:28.918 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.466998+0000", 2023-08-28T02:30:28.918 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00013130699999999999 2023-08-28T02:30:28.918 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:28.918 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:30:28.918 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:28.918 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.918 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.919 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:78192 6.c 6:36cee94b:::benchmark_data_smithi143_305742_object78191:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e221)", 2023-08-28T02:30:28.919 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:28:25.494860+0000", 2023-08-28T02:30:28.919 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 123.19753160499999, 2023-08-28T02:30:28.919 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.97873924599999995, 2023-08-28T02:30:28.919 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:30:28.919 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:30:28.919 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:30:28.919 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:30:28.919 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:30:28.919 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 78192 2023-08-28T02:30:28.919 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.919 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:30:28.919 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.920 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:30:28.920 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.494860+0000", 2023-08-28T02:30:28.920 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:30:28.920 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.920 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.920 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:30:28.920 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.494860+0000", 2023-08-28T02:30:28.920 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:30:28.920 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.920 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.920 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:30:28.920 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.494859+0000", 2023-08-28T02:30:28.921 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:30:28.921 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.921 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.921 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:30:28.921 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.494878+0000", 2023-08-28T02:30:28.921 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.8918000000000001e-05 2023-08-28T02:30:28.921 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.921 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.921 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:30:28.921 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.494891+0000", 2023-08-28T02:30:28.921 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3122000000000001e-05 2023-08-28T02:30:28.921 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.921 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.922 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:30:28.922 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.494897+0000", 2023-08-28T02:30:28.922 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.1929999999999998e-06 2023-08-28T02:30:28.922 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.922 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.922 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.922 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.494945+0000", 2023-08-28T02:30:28.922 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.8053999999999998e-05 2023-08-28T02:30:28.922 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.922 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.922 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:30:28.922 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.494956+0000", 2023-08-28T02:30:28.922 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0399e-05 2023-08-28T02:30:28.923 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.923 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.923 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.923 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.459931+0000", 2023-08-28T02:30:28.923 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.96497512699999999 2023-08-28T02:30:28.923 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.923 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.923 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:30:28.923 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.460075+0000", 2023-08-28T02:30:28.923 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000144571 2023-08-28T02:30:28.923 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.923 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.924 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 5,6,7", 2023-08-28T02:30:28.924 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.460181+0000", 2023-08-28T02:30:28.924 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000105993 2023-08-28T02:30:28.924 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.924 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.924 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:30:28.924 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.461924+0000", 2023-08-28T02:30:28.924 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.001743047 2023-08-28T02:30:28.924 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.924 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.924 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:30:28.924 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.464585+0000", 2023-08-28T02:30:28.924 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0026605359999999998 2023-08-28T02:30:28.925 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.925 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.925 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:30:28.925 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.464630+0000", 2023-08-28T02:30:28.925 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.4801e-05 2023-08-28T02:30:28.925 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.925 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.925 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:30:28.925 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.473424+0000", 2023-08-28T02:30:28.925 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0087939469999999999 2023-08-28T02:30:28.925 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.925 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.925 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:30:28.925 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.473484+0000", 2023-08-28T02:30:28.926 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.0270999999999999e-05 2023-08-28T02:30:28.926 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.926 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.926 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:30:28.926 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.473599+0000", 2023-08-28T02:30:28.926 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000115362 2023-08-28T02:30:28.926 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:28.926 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:30:28.926 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:28.926 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.926 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.926 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:141072 6.6 6:617ed47d:::benchmark_data_smithi143_305742_object57453:head [delete] snapc 0=[] ondisk+write+known_if_redirected e251)", 2023-08-28T02:30:28.927 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:30:09.241579+0000", 2023-08-28T02:30:28.927 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 19.450812320000001, 2023-08-28T02:30:28.927 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.40119437200000002, 2023-08-28T02:30:28.927 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:30:28.927 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:30:28.927 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:30:28.927 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:30:28.927 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:30:28.927 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 141072 2023-08-28T02:30:28.927 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.927 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:30:28.927 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.927 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:30:28.928 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.241579+0000", 2023-08-28T02:30:28.928 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:30:28.928 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.928 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.928 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:30:28.928 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.241579+0000", 2023-08-28T02:30:28.928 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:30:28.928 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.928 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.928 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:30:28.928 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.241578+0000", 2023-08-28T02:30:28.928 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:30:28.928 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.929 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.929 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:30:28.929 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.241584+0000", 2023-08-28T02:30:28.929 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.613e-06 2023-08-28T02:30:28.929 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.929 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.929 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:30:28.929 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.241600+0000", 2023-08-28T02:30:28.929 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.6050999999999999e-05 2023-08-28T02:30:28.929 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.929 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.929 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:30:28.929 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.241609+0000", 2023-08-28T02:30:28.930 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.1210000000000007e-06 2023-08-28T02:30:28.930 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.930 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.930 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:30:28.930 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.241658+0000", 2023-08-28T02:30:28.930 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.9273000000000001e-05 2023-08-28T02:30:28.930 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.930 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.930 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:30:28.930 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.241795+0000", 2023-08-28T02:30:28.930 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00013767099999999999 2023-08-28T02:30:28.930 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.931 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.931 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 4,7", 2023-08-28T02:30:28.931 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.241926+0000", 2023-08-28T02:30:28.931 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000130524 2023-08-28T02:30:28.931 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.931 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.931 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:30:28.931 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.244258+0000", 2023-08-28T02:30:28.931 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0023322629999999998 2023-08-28T02:30:28.931 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.931 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.931 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:30:28.931 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.247680+0000", 2023-08-28T02:30:28.932 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0034219189999999998 2023-08-28T02:30:28.932 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.932 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.932 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:30:28.932 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.642673+0000", 2023-08-28T02:30:28.932 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.39499316299999998 2023-08-28T02:30:28.932 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.932 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.932 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:30:28.932 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.642726+0000", 2023-08-28T02:30:28.932 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.2147000000000001e-05 2023-08-28T02:30:28.932 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:30:28.932 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:30:28.933 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:30:28.933 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.642774+0000", 2023-08-28T02:30:28.933 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.8208999999999999e-05 2023-08-28T02:30:28.933 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:28.933 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:30:28.933 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:28.933 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:30:28.933 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:30:28.933 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:30:28.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:28.934 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:30:28.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:30:29.035 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:30:29.079 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:29.521 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:30:29.911 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:30:30.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:30.234 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:30:30.334 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:30:30.395 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:30.467 INFO:teuthology.orchestra.run.smithi143.stdout:{"pg_ready":true,"pg_map":{"version":615,"stamp":"2023-08-28T02:30:27.303052+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":883365835,"num_objects":13435,"num_object_clones":0,"num_object_copies":40305,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6078,"num_objects_misplaced":2005,"num_objects_unfound":0,"num_objects_dirty":13435,"num_whiteouts":0,"num_read":6275,"num_read_kb":13708,"num_write":154933,"num_write_kb":5366566,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20275,"num_bytes_recovered":1169839177,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":181535,"ondisk_log_size":181535,"up":297,"acting":293,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":311,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5958964,"kb_used_data":3584564,"kb_used_omap":29,"kb_used_meta":2374050,"kb_avail":744002252,"statfs":{"total":767960285184,"available":761858306048,"internally_reserved":0,"allocated":3670593536,"data_stored":3663094125,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30674,"internal_metadata":2431027246},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-419037184,"num_objects":-6394,"num_object_clones":0,"num_object_copies":-19182,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-790,"num_objects_unfound":0,"num_objects_dirty":-6394,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6394,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":231,"num_bytes_recovered":15138816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.005706"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":256,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.172817+0000","last_change":"2023-08-28T02:27:31.367469+0000","last_active":"2023-08-28T02:30:20.172817+0000","last_peered":"2023-08-28T02:30:20.172817+0000","last_clean":"2023-08-28T02:30:20.172817+0000","last_became_active":"2023-08-28T02:27:31.366946+0000","last_became_peered":"2023-08-28T02:27:31.366946+0000","last_unstale":"2023-08-28T02:30:20.172817+0000","last_undegraded":"2023-08-28T02:30:20.172817+0000","last_fullsized":"2023-08-28T02:30:20.172817+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":141,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.163141+0000","last_change":"2023-08-28T02:27:31.324452+0000","last_active":"2023-08-28T02:30:20.163141+0000","last_peered":"2023-08-28T02:30:20.163141+0000","last_clean":"2023-08-28T02:30:20.163141+0000","last_became_active":"2023-08-28T02:27:31.323749+0000","last_became_peered":"2023-08-28T02:27:31.323749+0000","last_unstale":"2023-08-28T02:30:20.163141+0000","last_undegraded":"2023-08-28T02:30:20.163141+0000","last_fullsized":"2023-08-28T02:30:20.163141+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:18:10.904127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":62,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.158364+0000","last_change":"2023-08-28T02:27:31.337338+0000","last_active":"2023-08-28T02:30:20.158364+0000","last_peered":"2023-08-28T02:30:20.158364+0000","last_clean":"2023-08-28T02:30:20.158364+0000","last_became_active":"2023-08-28T02:27:31.337018+0000","last_became_peered":"2023-08-28T02:27:31.337018+0000","last_unstale":"2023-08-28T02:30:20.158364+0000","last_undegraded":"2023-08-28T02:30:20.158364+0000","last_fullsized":"2023-08-28T02:30:20.158364+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:46:34.845663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":242,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.147929+0000","last_change":"2023-08-28T02:23:59.223634+0000","last_active":"2023-08-28T02:30:20.147929+0000","last_peered":"2023-08-28T02:30:20.147929+0000","last_clean":"2023-08-28T02:30:20.147929+0000","last_became_active":"2023-08-28T02:23:59.222704+0000","last_became_peered":"2023-08-28T02:23:59.222704+0000","last_unstale":"2023-08-28T02:30:20.147929+0000","last_undegraded":"2023-08-28T02:30:20.147929+0000","last_fullsized":"2023-08-28T02:30:20.147929+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:51:52.752361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":132,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.162592+0000","last_change":"2023-08-28T02:27:31.318153+0000","last_active":"2023-08-28T02:30:20.162592+0000","last_peered":"2023-08-28T02:30:20.162592+0000","last_clean":"2023-08-28T02:30:20.162592+0000","last_became_active":"2023-08-28T02:27:31.317732+0000","last_became_peered":"2023-08-28T02:27:31.317732+0000","last_unstale":"2023-08-28T02:30:20.162592+0000","last_undegraded":"2023-08-28T02:30:20.162592+0000","last_fullsized":"2023-08-28T02:30:20.162592+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:06:13.421674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":44,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.160159+0000","last_change":"2023-08-28T02:28:25.348941+0000","last_active":"2023-08-28T02:30:20.160159+0000","last_peered":"2023-08-28T02:30:20.160159+0000","last_clean":"2023-08-28T02:30:20.160159+0000","last_became_active":"2023-08-28T02:28:25.348650+0000","last_became_peered":"2023-08-28T02:28:25.348650+0000","last_unstale":"2023-08-28T02:30:20.160159+0000","last_undegraded":"2023-08-28T02:30:20.160159+0000","last_fullsized":"2023-08-28T02:30:20.160159+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.12","version":"255'6697","reported_seq":6785,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-08-28T02:30:27.135741+0000","last_change":"2023-08-28T02:28:25.453536+0000","last_active":"2023-08-28T02:30:27.135741+0000","last_peered":"2023-08-28T02:30:27.135741+0000","last_clean":"2023-08-28T02:30:27.135741+0000","last_became_active":"2023-08-28T02:27:58.479936+0000","last_became_peered":"2023-08-28T02:27:58.479936+0000","last_unstale":"2023-08-28T02:30:27.135741+0000","last_undegraded":"2023-08-28T02:30:27.135741+0000","last_fullsized":"2023-08-28T02:30:27.135741+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6697,"log_dups_size":0,"ondisk_log_size":6697,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26214400,"num_objects":400,"num_object_clones":0,"num_object_copies":1200,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":400,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4832,"num_write_kb":167424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":222,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.148309+0000","last_change":"2023-08-28T02:23:59.227592+0000","last_active":"2023-08-28T02:30:20.148309+0000","last_peered":"2023-08-28T02:30:20.148309+0000","last_clean":"2023-08-28T02:30:20.148309+0000","last_became_active":"2023-08-28T02:23:59.227342+0000","last_became_peered":"2023-08-28T02:23:59.227342+0000","last_unstale":"2023-08-28T02:30:20.148309+0000","last_undegraded":"2023-08-28T02:30:20.148309+0000","last_fullsized":"2023-08-28T02:30:20.148309+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:08:19.874396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":150,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.162666+0000","last_change":"2023-08-28T02:28:25.348233+0000","last_active":"2023-08-28T02:30:20.162666+0000","last_peered":"2023-08-28T02:30:20.162666+0000","last_clean":"2023-08-28T02:30:20.162666+0000","last_became_active":"2023-08-28T02:28:25.347896+0000","last_became_peered":"2023-08-28T02:28:25.347896+0000","last_unstale":"2023-08-28T02:30:20.162666+0000","last_undegraded":"2023-08-28T02:30:20.162666+0000","last_fullsized":"2023-08-28T02:30:20.162666+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:50:17.674564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.13","version":"255'6819","reported_seq":6925,"reported_epoch":255,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:30:27.057677+0000","last_change":"2023-08-28T02:28:26.468919+0000","last_active":"2023-08-28T02:30:27.057677+0000","last_peered":"2023-08-28T02:30:27.057677+0000","last_clean":"2023-08-28T02:27:16.315150+0000","last_became_active":"2023-08-28T02:28:26.463939+0000","last_became_peered":"2023-08-28T02:28:26.463939+0000","last_unstale":"2023-08-28T02:30:27.057677+0000","last_undegraded":"2023-08-28T02:28:26.355850+0000","last_fullsized":"2023-08-28T02:28:26.347877+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6819,"log_dups_size":0,"ondisk_log_size":6819,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25100288,"num_objects":383,"num_object_clones":0,"num_object_copies":1149,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1703,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":383,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4931,"num_write_kb":170048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"1,2,7","objects":215},{"shards":"2,7","objects":-1320}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":257,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.157339+0000","last_change":"2023-08-28T02:27:31.328499+0000","last_active":"2023-08-28T02:30:20.157339+0000","last_peered":"2023-08-28T02:30:20.157339+0000","last_clean":"2023-08-28T02:30:20.157339+0000","last_became_active":"2023-08-28T02:27:31.328164+0000","last_became_peered":"2023-08-28T02:27:31.328164+0000","last_unstale":"2023-08-28T02:30:20.157339+0000","last_undegraded":"2023-08-28T02:30:20.157339+0000","last_fullsized":"2023-08-28T02:30:20.157339+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":132,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.162796+0000","last_change":"2023-08-28T02:28:25.348760+0000","last_active":"2023-08-28T02:30:20.162796+0000","last_peered":"2023-08-28T02:30:20.162796+0000","last_clean":"2023-08-28T02:30:20.162796+0000","last_became_active":"2023-08-28T02:28:25.348465+0000","last_became_peered":"2023-08-28T02:28:25.348465+0000","last_unstale":"2023-08-28T02:30:20.162796+0000","last_undegraded":"2023-08-28T02:30:20.162796+0000","last_fullsized":"2023-08-28T02:30:20.162796+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:43:05.659999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.10","version":"255'6504","reported_seq":6593,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-08-28T02:30:23.894344+0000","last_change":"2023-08-28T02:30:17.869105+0000","last_active":"2023-08-28T02:30:23.894344+0000","last_peered":"2023-08-28T02:30:23.894344+0000","last_clean":"2023-08-28T02:30:23.894344+0000","last_became_active":"2023-08-28T02:27:58.494108+0000","last_became_peered":"2023-08-28T02:27:58.494108+0000","last_unstale":"2023-08-28T02:30:23.894344+0000","last_undegraded":"2023-08-28T02:30:23.894344+0000","last_fullsized":"2023-08-28T02:30:23.894344+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6504,"log_dups_size":0,"ondisk_log_size":6504,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30736384,"num_objects":469,"num_object_clones":0,"num_object_copies":1407,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":469,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4667,"num_write_kb":164352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":140,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.168856+0000","last_change":"2023-08-28T02:23:59.316561+0000","last_active":"2023-08-28T02:30:20.168856+0000","last_peered":"2023-08-28T02:30:20.168856+0000","last_clean":"2023-08-28T02:30:20.168856+0000","last_became_active":"2023-08-28T02:23:59.227433+0000","last_became_peered":"2023-08-28T02:23:59.227433+0000","last_unstale":"2023-08-28T02:30:20.168856+0000","last_undegraded":"2023-08-28T02:30:20.168856+0000","last_fullsized":"2023-08-28T02:30:20.168856+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":72,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.157855+0000","last_change":"2023-08-28T02:28:25.347771+0000","last_active":"2023-08-28T02:30:20.157855+0000","last_peered":"2023-08-28T02:30:20.157855+0000","last_clean":"2023-08-28T02:30:20.157855+0000","last_became_active":"2023-08-28T02:28:25.347320+0000","last_became_peered":"2023-08-28T02:28:25.347320+0000","last_unstale":"2023-08-28T02:30:20.157855+0000","last_undegraded":"2023-08-28T02:30:20.157855+0000","last_fullsized":"2023-08-28T02:30:20.157855+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:20:27.350131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.11","version":"255'6700","reported_seq":13617,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-08-28T02:30:24.531912+0000","last_change":"2023-08-28T02:30:11.322180+0000","last_active":"2023-08-28T02:30:24.531912+0000","last_peered":"2023-08-28T02:30:24.531912+0000","last_clean":"2023-08-28T02:30:24.531912+0000","last_became_active":"2023-08-28T02:29:58.475435+0000","last_became_peered":"2023-08-28T02:29:58.475435+0000","last_unstale":"2023-08-28T02:30:24.531912+0000","last_undegraded":"2023-08-28T02:30:24.531912+0000","last_fullsized":"2023-08-28T02:30:24.531912+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6700,"log_dups_size":0,"ondisk_log_size":6700,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30605312,"num_objects":467,"num_object_clones":0,"num_object_copies":1401,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":467,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4825,"num_write_kb":169344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2439,"num_bytes_recovered":159678464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"86'22","reported_seq":305,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.173493+0000","last_change":"2023-08-28T02:28:25.361783+0000","last_active":"2023-08-28T02:30:20.173493+0000","last_peered":"2023-08-28T02:30:20.173493+0000","last_clean":"2023-08-28T02:30:20.173493+0000","last_became_active":"2023-08-28T02:28:25.351418+0000","last_became_peered":"2023-08-28T02:28:25.351418+0000","last_unstale":"2023-08-28T02:30:20.173493+0000","last_undegraded":"2023-08-28T02:30:20.173493+0000","last_fullsized":"2023-08-28T02:30:20.173493+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":271,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.172185+0000","last_change":"2023-08-28T02:28:25.352395+0000","last_active":"2023-08-28T02:30:20.172185+0000","last_peered":"2023-08-28T02:30:20.172185+0000","last_clean":"2023-08-28T02:30:20.172185+0000","last_became_active":"2023-08-28T02:28:25.352189+0000","last_became_peered":"2023-08-28T02:28:25.352189+0000","last_unstale":"2023-08-28T02:30:20.172185+0000","last_undegraded":"2023-08-28T02:30:20.172185+0000","last_fullsized":"2023-08-28T02:30:20.172185+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.16","version":"255'6640","reported_seq":6727,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-08-28T02:30:24.187539+0000","last_change":"2023-08-28T02:27:58.492359+0000","last_active":"2023-08-28T02:30:24.187539+0000","last_peered":"2023-08-28T02:30:24.187539+0000","last_clean":"2023-08-28T02:30:24.187539+0000","last_became_active":"2023-08-28T02:27:58.479842+0000","last_became_peered":"2023-08-28T02:27:58.479842+0000","last_unstale":"2023-08-28T02:30:24.187539+0000","last_undegraded":"2023-08-28T02:30:24.187539+0000","last_fullsized":"2023-08-28T02:30:24.187539+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6640,"log_dups_size":0,"ondisk_log_size":6640,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27033600,"num_objects":413,"num_object_clones":0,"num_object_copies":1239,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":413,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4755,"num_write_kb":165344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":125,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.163803+0000","last_change":"2023-08-28T02:23:38.983634+0000","last_active":"2023-08-28T02:30:20.163803+0000","last_peered":"2023-08-28T02:30:20.163803+0000","last_clean":"2023-08-28T02:30:20.163803+0000","last_became_active":"2023-08-28T02:23:38.344566+0000","last_became_peered":"2023-08-28T02:23:38.344566+0000","last_unstale":"2023-08-28T02:30:20.163803+0000","last_undegraded":"2023-08-28T02:30:20.163803+0000","last_fullsized":"2023-08-28T02:30:20.163803+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":132,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.163484+0000","last_change":"2023-08-28T02:27:31.344974+0000","last_active":"2023-08-28T02:30:20.163484+0000","last_peered":"2023-08-28T02:30:20.163484+0000","last_clean":"2023-08-28T02:30:20.163484+0000","last_became_active":"2023-08-28T02:27:31.344281+0000","last_became_peered":"2023-08-28T02:27:31.344281+0000","last_unstale":"2023-08-28T02:30:20.163484+0000","last_undegraded":"2023-08-28T02:30:20.163484+0000","last_fullsized":"2023-08-28T02:30:20.163484+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:04:02.821752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.17","version":"255'6646","reported_seq":6731,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-08-28T02:30:27.188984+0000","last_change":"2023-08-28T02:27:58.474771+0000","last_active":"2023-08-28T02:30:27.188984+0000","last_peered":"2023-08-28T02:30:27.188984+0000","last_clean":"2023-08-28T02:30:27.188984+0000","last_became_active":"2023-08-28T02:27:58.472976+0000","last_became_peered":"2023-08-28T02:27:58.472976+0000","last_unstale":"2023-08-28T02:30:27.188984+0000","last_undegraded":"2023-08-28T02:30:27.188984+0000","last_fullsized":"2023-08-28T02:30:27.188984+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6646,"log_dups_size":0,"ondisk_log_size":6646,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23855104,"num_objects":364,"num_object_clones":0,"num_object_copies":1092,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":364,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4790,"num_write_kb":164928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":241,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.148668+0000","last_change":"2023-08-28T02:23:59.223066+0000","last_active":"2023-08-28T02:30:20.148668+0000","last_peered":"2023-08-28T02:30:20.148668+0000","last_clean":"2023-08-28T02:30:20.148668+0000","last_became_active":"2023-08-28T02:23:59.222731+0000","last_became_peered":"2023-08-28T02:23:59.222731+0000","last_unstale":"2023-08-28T02:30:20.148668+0000","last_undegraded":"2023-08-28T02:30:20.148668+0000","last_fullsized":"2023-08-28T02:30:20.148668+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:48:37.558266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":281,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.157322+0000","last_change":"2023-08-28T02:23:59.210469+0000","last_active":"2023-08-28T02:30:20.157322+0000","last_peered":"2023-08-28T02:30:20.157322+0000","last_clean":"2023-08-28T02:30:20.157322+0000","last_became_active":"2023-08-28T02:23:59.210031+0000","last_became_peered":"2023-08-28T02:23:59.210031+0000","last_unstale":"2023-08-28T02:30:20.157322+0000","last_undegraded":"2023-08-28T02:30:20.157322+0000","last_fullsized":"2023-08-28T02:30:20.157322+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.14","version":"255'6673","reported_seq":12190,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-08-28T02:30:24.497653+0000","last_change":"2023-08-28T02:30:10.332493+0000","last_active":"2023-08-28T02:30:24.497653+0000","last_peered":"2023-08-28T02:30:24.497653+0000","last_clean":"2023-08-28T02:30:24.497653+0000","last_became_active":"2023-08-28T02:28:00.020960+0000","last_became_peered":"2023-08-28T02:28:00.020960+0000","last_unstale":"2023-08-28T02:30:24.497653+0000","last_undegraded":"2023-08-28T02:30:24.497653+0000","last_fullsized":"2023-08-28T02:30:24.497653+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6673,"log_dups_size":0,"ondisk_log_size":6673,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32669696,"num_objects":499,"num_object_clones":0,"num_object_copies":1497,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":499,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4841,"num_write_kb":170848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1961,"num_bytes_recovered":128483328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":137,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.163329+0000","last_change":"2023-08-28T02:28:25.354526+0000","last_active":"2023-08-28T02:30:20.163329+0000","last_peered":"2023-08-28T02:30:20.163329+0000","last_clean":"2023-08-28T02:30:20.163329+0000","last_became_active":"2023-08-28T02:28:25.350991+0000","last_became_peered":"2023-08-28T02:28:25.350991+0000","last_unstale":"2023-08-28T02:30:20.163329+0000","last_undegraded":"2023-08-28T02:30:20.163329+0000","last_fullsized":"2023-08-28T02:30:20.163329+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":241,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.148001+0000","last_change":"2023-08-28T02:23:59.220932+0000","last_active":"2023-08-28T02:30:20.148001+0000","last_peered":"2023-08-28T02:30:20.148001+0000","last_clean":"2023-08-28T02:30:20.148001+0000","last_became_active":"2023-08-28T02:23:59.220488+0000","last_became_peered":"2023-08-28T02:23:59.220488+0000","last_unstale":"2023-08-28T02:30:20.148001+0000","last_undegraded":"2023-08-28T02:30:20.148001+0000","last_fullsized":"2023-08-28T02:30:20.148001+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:53:09.216340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.15","version":"255'6794","reported_seq":8071,"reported_epoch":255,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:30:27.150745+0000","last_change":"2023-08-28T02:28:25.456367+0000","last_active":"2023-08-28T02:30:27.150745+0000","last_peered":"2023-08-28T02:30:27.150745+0000","last_clean":"2023-08-28T02:27:16.315756+0000","last_became_active":"2023-08-28T02:28:25.449323+0000","last_became_peered":"2023-08-28T02:28:25.449323+0000","last_unstale":"2023-08-28T02:30:27.150745+0000","last_undegraded":"2023-08-28T02:28:25.349175+0000","last_fullsized":"2023-08-28T02:28:25.327058+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6794,"log_dups_size":0,"ondisk_log_size":6794,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22675456,"num_objects":346,"num_object_clones":0,"num_object_copies":1038,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1690,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":346,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4910,"num_write_kb":168192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":294,"num_bytes_recovered":19267584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"2,3,7","objects":192},{"shards":"3,7","objects":-1344}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":54,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.159813+0000","last_change":"2023-08-28T02:28:25.703333+0000","last_active":"2023-08-28T02:30:20.159813+0000","last_peered":"2023-08-28T02:30:20.159813+0000","last_clean":"2023-08-28T02:30:20.159813+0000","last_became_active":"2023-08-28T02:28:25.343515+0000","last_became_peered":"2023-08-28T02:28:25.343515+0000","last_unstale":"2023-08-28T02:30:20.159813+0000","last_undegraded":"2023-08-28T02:30:20.159813+0000","last_fullsized":"2023-08-28T02:30:20.159813+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":132,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.164059+0000","last_change":"2023-08-28T02:27:31.330059+0000","last_active":"2023-08-28T02:30:20.164059+0000","last_peered":"2023-08-28T02:30:20.164059+0000","last_clean":"2023-08-28T02:30:20.164059+0000","last_became_active":"2023-08-28T02:27:31.329769+0000","last_became_peered":"2023-08-28T02:27:31.329769+0000","last_unstale":"2023-08-28T02:30:20.164059+0000","last_undegraded":"2023-08-28T02:30:20.164059+0000","last_fullsized":"2023-08-28T02:30:20.164059+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:14:54.349909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"255'9582","reported_seq":12792,"reported_epoch":255,"state":"active+remapped+backfill_wait","last_fresh":"2023-08-28T02:30:23.907789+0000","last_change":"2023-08-28T02:29:15.939050+0000","last_active":"2023-08-28T02:30:23.907789+0000","last_peered":"2023-08-28T02:30:23.907789+0000","last_clean":"2023-08-28T02:28:24.280158+0000","last_became_active":"2023-08-28T02:28:26.465041+0000","last_became_peered":"2023-08-28T02:28:26.465041+0000","last_unstale":"2023-08-28T02:30:23.907789+0000","last_undegraded":"2023-08-28T02:30:23.907789+0000","last_fullsized":"2023-08-28T02:30:23.907789+0000","mapping_epoch":221,"log_start":"195'1200","ondisk_log_start":"195'1200","created":187,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":8382,"log_dups_size":0,"ondisk_log_size":8382,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:30:34.822957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61079552,"num_objects":932,"num_object_clones":0,"num_object_copies":2796,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":932,"num_objects_unfound":0,"num_objects_dirty":932,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9582,"num_write_kb":336448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1547,"num_bytes_recovered":101384192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[0,5,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":932}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":123,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.168859+0000","last_change":"2023-08-28T02:23:38.657185+0000","last_active":"2023-08-28T02:30:20.168859+0000","last_peered":"2023-08-28T02:30:20.168859+0000","last_clean":"2023-08-28T02:30:20.168859+0000","last_became_active":"2023-08-28T02:23:38.339016+0000","last_became_peered":"2023-08-28T02:23:38.339016+0000","last_unstale":"2023-08-28T02:30:20.168859+0000","last_undegraded":"2023-08-28T02:30:20.168859+0000","last_fullsized":"2023-08-28T02:30:20.168859+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":241,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.156027+0000","last_change":"2023-08-28T02:28:25.348106+0000","last_active":"2023-08-28T02:30:20.156027+0000","last_peered":"2023-08-28T02:30:20.156027+0000","last_clean":"2023-08-28T02:30:20.156027+0000","last_became_active":"2023-08-28T02:28:25.347786+0000","last_became_peered":"2023-08-28T02:28:25.347786+0000","last_unstale":"2023-08-28T02:30:20.156027+0000","last_undegraded":"2023-08-28T02:30:20.156027+0000","last_fullsized":"2023-08-28T02:30:20.156027+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"255'9548","reported_seq":14481,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-08-28T02:30:24.236836+0000","last_change":"2023-08-28T02:29:38.520992+0000","last_active":"2023-08-28T02:30:24.236836+0000","last_peered":"2023-08-28T02:30:24.236836+0000","last_clean":"2023-08-28T02:30:24.236836+0000","last_became_active":"2023-08-28T02:29:38.501413+0000","last_became_peered":"2023-08-28T02:29:38.501413+0000","last_unstale":"2023-08-28T02:30:24.236836+0000","last_undegraded":"2023-08-28T02:30:24.236836+0000","last_fullsized":"2023-08-28T02:30:24.236836+0000","mapping_epoch":242,"log_start":"206'3200","ondisk_log_start":"206'3200","created":187,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6348,"log_dups_size":0,"ondisk_log_size":6348,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:45:22.978036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61997056,"num_objects":946,"num_object_clones":0,"num_object_copies":2838,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":946,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9548,"num_write_kb":335808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2408,"num_bytes_recovered":149291008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":149,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.169203+0000","last_change":"2023-08-28T02:27:31.395328+0000","last_active":"2023-08-28T02:30:20.169203+0000","last_peered":"2023-08-28T02:30:20.169203+0000","last_clean":"2023-08-28T02:30:20.169203+0000","last_became_active":"2023-08-28T02:27:31.328228+0000","last_became_peered":"2023-08-28T02:27:31.328228+0000","last_unstale":"2023-08-28T02:30:20.169203+0000","last_undegraded":"2023-08-28T02:30:20.169203+0000","last_fullsized":"2023-08-28T02:30:20.169203+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":290,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.172385+0000","last_change":"2023-08-28T02:27:31.344915+0000","last_active":"2023-08-28T02:30:20.172385+0000","last_peered":"2023-08-28T02:30:20.172385+0000","last_clean":"2023-08-28T02:30:20.172385+0000","last_became_active":"2023-08-28T02:27:31.344577+0000","last_became_peered":"2023-08-28T02:27:31.344577+0000","last_unstale":"2023-08-28T02:30:20.172385+0000","last_undegraded":"2023-08-28T02:30:20.172385+0000","last_fullsized":"2023-08-28T02:30:20.172385+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"255'6815","reported_seq":6901,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-08-28T02:30:27.134212+0000","last_change":"2023-08-28T02:27:58.457913+0000","last_active":"2023-08-28T02:30:27.134212+0000","last_peered":"2023-08-28T02:30:27.134212+0000","last_clean":"2023-08-28T02:30:27.134212+0000","last_became_active":"2023-08-28T02:27:58.451333+0000","last_became_peered":"2023-08-28T02:27:58.451333+0000","last_unstale":"2023-08-28T02:30:27.134212+0000","last_undegraded":"2023-08-28T02:30:27.134212+0000","last_fullsized":"2023-08-28T02:30:27.134212+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6815,"log_dups_size":0,"ondisk_log_size":6815,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19038208,"num_objects":290,"num_object_clones":0,"num_object_copies":870,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":290,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4938,"num_write_kb":167328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":149,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.169056+0000","last_change":"2023-08-28T02:28:25.749249+0000","last_active":"2023-08-28T02:30:20.169056+0000","last_peered":"2023-08-28T02:30:20.169056+0000","last_clean":"2023-08-28T02:30:20.169056+0000","last_became_active":"2023-08-28T02:28:25.397684+0000","last_became_peered":"2023-08-28T02:28:25.397684+0000","last_unstale":"2023-08-28T02:30:20.169056+0000","last_undegraded":"2023-08-28T02:30:20.169056+0000","last_fullsized":"2023-08-28T02:30:20.169056+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":44,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.159920+0000","last_change":"2023-08-28T02:28:25.350802+0000","last_active":"2023-08-28T02:30:20.159920+0000","last_peered":"2023-08-28T02:30:20.159920+0000","last_clean":"2023-08-28T02:30:20.159920+0000","last_became_active":"2023-08-28T02:28:25.350474+0000","last_became_peered":"2023-08-28T02:28:25.350474+0000","last_unstale":"2023-08-28T02:30:20.159920+0000","last_undegraded":"2023-08-28T02:30:20.159920+0000","last_fullsized":"2023-08-28T02:30:20.159920+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"255'6560","reported_seq":6651,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-08-28T02:30:23.808331+0000","last_change":"2023-08-28T02:30:19.887172+0000","last_active":"2023-08-28T02:30:23.808331+0000","last_peered":"2023-08-28T02:30:23.808331+0000","last_clean":"2023-08-28T02:30:23.808331+0000","last_became_active":"2023-08-28T02:27:58.433161+0000","last_became_peered":"2023-08-28T02:27:58.433161+0000","last_unstale":"2023-08-28T02:30:23.808331+0000","last_undegraded":"2023-08-28T02:30:23.808331+0000","last_fullsized":"2023-08-28T02:30:23.808331+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6560,"log_dups_size":0,"ondisk_log_size":6560,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30572544,"num_objects":466,"num_object_clones":0,"num_object_copies":1398,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":466,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4690,"num_write_kb":165024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":256,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.163613+0000","last_change":"2023-08-28T02:28:25.445982+0000","last_active":"2023-08-28T02:30:20.163613+0000","last_peered":"2023-08-28T02:30:20.163613+0000","last_clean":"2023-08-28T02:30:20.163613+0000","last_became_active":"2023-08-28T02:28:25.443577+0000","last_became_peered":"2023-08-28T02:28:25.443577+0000","last_unstale":"2023-08-28T02:30:20.163613+0000","last_undegraded":"2023-08-28T02:30:20.163613+0000","last_fullsized":"2023-08-28T02:30:20.163613+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:24:03.381751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":235,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.150195+0000","last_change":"2023-08-28T02:27:31.319783+0000","last_active":"2023-08-28T02:30:20.150195+0000","last_peered":"2023-08-28T02:30:20.150195+0000","last_clean":"2023-08-28T02:30:20.150195+0000","last_became_active":"2023-08-28T02:27:31.319451+0000","last_became_peered":"2023-08-28T02:27:31.319451+0000","last_unstale":"2023-08-28T02:30:20.150195+0000","last_undegraded":"2023-08-28T02:30:20.150195+0000","last_fullsized":"2023-08-28T02:30:20.150195+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:20:37.097889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"255'9614","reported_seq":14535,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-08-28T02:30:27.148825+0000","last_change":"2023-08-28T02:29:17.383106+0000","last_active":"2023-08-28T02:30:27.148825+0000","last_peered":"2023-08-28T02:30:27.148825+0000","last_clean":"2023-08-28T02:30:27.148825+0000","last_became_active":"2023-08-28T02:29:17.367223+0000","last_became_peered":"2023-08-28T02:29:17.367223+0000","last_unstale":"2023-08-28T02:30:27.148825+0000","last_undegraded":"2023-08-28T02:30:27.148825+0000","last_fullsized":"2023-08-28T02:30:27.148825+0000","mapping_epoch":234,"log_start":"199'1700","ondisk_log_start":"199'1700","created":187,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":7914,"log_dups_size":0,"ondisk_log_size":7914,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47316992,"num_objects":722,"num_object_clones":0,"num_object_copies":2166,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":722,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9614,"num_write_kb":330752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2409,"num_bytes_recovered":127533056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":118,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.163206+0000","last_change":"2023-08-28T02:23:59.230059+0000","last_active":"2023-08-28T02:30:20.163206+0000","last_peered":"2023-08-28T02:30:20.163206+0000","last_clean":"2023-08-28T02:30:20.163206+0000","last_became_active":"2023-08-28T02:23:59.217783+0000","last_became_peered":"2023-08-28T02:23:59.217783+0000","last_unstale":"2023-08-28T02:30:20.163206+0000","last_undegraded":"2023-08-28T02:30:20.163206+0000","last_fullsized":"2023-08-28T02:30:20.163206+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:00:12.145395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":275,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.173416+0000","last_change":"2023-08-28T02:28:25.351698+0000","last_active":"2023-08-28T02:30:20.173416+0000","last_peered":"2023-08-28T02:30:20.173416+0000","last_clean":"2023-08-28T02:30:20.173416+0000","last_became_active":"2023-08-28T02:28:25.351349+0000","last_became_peered":"2023-08-28T02:28:25.351349+0000","last_unstale":"2023-08-28T02:30:20.173416+0000","last_undegraded":"2023-08-28T02:30:20.173416+0000","last_fullsized":"2023-08-28T02:30:20.173416+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"255'9553","reported_seq":9631,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-08-28T02:30:24.116079+0000","last_change":"2023-08-28T02:26:57.390417+0000","last_active":"2023-08-28T02:30:24.116079+0000","last_peered":"2023-08-28T02:30:24.116079+0000","last_clean":"2023-08-28T02:30:24.116079+0000","last_became_active":"2023-08-28T02:26:57.390124+0000","last_became_peered":"2023-08-28T02:26:57.390124+0000","last_unstale":"2023-08-28T02:30:24.116079+0000","last_undegraded":"2023-08-28T02:30:24.116079+0000","last_fullsized":"2023-08-28T02:30:24.116079+0000","mapping_epoch":187,"log_start":"195'1200","ondisk_log_start":"195'1200","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":8353,"log_dups_size":0,"ondisk_log_size":8353,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:38:48.957967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62455808,"num_objects":953,"num_object_clones":0,"num_object_copies":2859,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":953,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9553,"num_write_kb":336192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":44,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.160065+0000","last_change":"2023-08-28T02:28:25.345274+0000","last_active":"2023-08-28T02:30:20.160065+0000","last_peered":"2023-08-28T02:30:20.160065+0000","last_clean":"2023-08-28T02:30:20.160065+0000","last_became_active":"2023-08-28T02:28:25.344885+0000","last_became_peered":"2023-08-28T02:28:25.344885+0000","last_unstale":"2023-08-28T02:30:20.160065+0000","last_undegraded":"2023-08-28T02:30:20.160065+0000","last_fullsized":"2023-08-28T02:30:20.160065+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":301,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.172652+0000","last_change":"2023-08-28T02:27:31.358436+0000","last_active":"2023-08-28T02:30:20.172652+0000","last_peered":"2023-08-28T02:30:20.172652+0000","last_clean":"2023-08-28T02:30:20.172652+0000","last_became_active":"2023-08-28T02:27:31.358185+0000","last_became_peered":"2023-08-28T02:27:31.358185+0000","last_unstale":"2023-08-28T02:30:20.172652+0000","last_undegraded":"2023-08-28T02:30:20.172652+0000","last_fullsized":"2023-08-28T02:30:20.172652+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"255'9769","reported_seq":13024,"reported_epoch":255,"state":"active+remapped+backfill_wait","last_fresh":"2023-08-28T02:30:24.234009+0000","last_change":"2023-08-28T02:28:26.455411+0000","last_active":"2023-08-28T02:30:24.234009+0000","last_peered":"2023-08-28T02:30:24.234009+0000","last_clean":"2023-08-28T02:28:24.332252+0000","last_became_active":"2023-08-28T02:28:26.447621+0000","last_became_peered":"2023-08-28T02:28:26.447621+0000","last_unstale":"2023-08-28T02:30:24.234009+0000","last_undegraded":"2023-08-28T02:30:24.234009+0000","last_fullsized":"2023-08-28T02:30:24.234009+0000","mapping_epoch":221,"log_start":"206'3500","ondisk_log_start":"206'3500","created":187,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6269,"log_dups_size":0,"ondisk_log_size":6269,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:28:52.578588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56950784,"num_objects":869,"num_object_clones":0,"num_object_copies":2607,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":869,"num_objects_unfound":0,"num_objects_dirty":869,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9769,"num_write_kb":340416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1574,"num_bytes_recovered":103153664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,5,7],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":869}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":164,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.172512+0000","last_change":"2023-08-28T02:23:38.347043+0000","last_active":"2023-08-28T02:30:20.172512+0000","last_peered":"2023-08-28T02:30:20.172512+0000","last_clean":"2023-08-28T02:30:20.172512+0000","last_became_active":"2023-08-28T02:23:38.346680+0000","last_became_peered":"2023-08-28T02:23:38.346680+0000","last_unstale":"2023-08-28T02:30:20.172512+0000","last_undegraded":"2023-08-28T02:30:20.172512+0000","last_fullsized":"2023-08-28T02:30:20.172512+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":221,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.148081+0000","last_change":"2023-08-28T02:28:25.347494+0000","last_active":"2023-08-28T02:30:20.148081+0000","last_peered":"2023-08-28T02:30:20.148081+0000","last_clean":"2023-08-28T02:30:20.148081+0000","last_became_active":"2023-08-28T02:28:25.347142+0000","last_became_peered":"2023-08-28T02:28:25.347142+0000","last_unstale":"2023-08-28T02:30:20.148081+0000","last_undegraded":"2023-08-28T02:30:20.148081+0000","last_fullsized":"2023-08-28T02:30:20.148081+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:37:14.384884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.d","version":"255'9468","reported_seq":14197,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-08-28T02:30:27.149094+0000","last_change":"2023-08-28T02:29:35.551357+0000","last_active":"2023-08-28T02:30:27.149094+0000","last_peered":"2023-08-28T02:30:27.149094+0000","last_clean":"2023-08-28T02:30:27.149094+0000","last_became_active":"2023-08-28T02:29:35.530821+0000","last_became_peered":"2023-08-28T02:29:35.530821+0000","last_unstale":"2023-08-28T02:30:27.149094+0000","last_undegraded":"2023-08-28T02:30:27.149094+0000","last_fullsized":"2023-08-28T02:30:27.149094+0000","mapping_epoch":240,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":9468,"log_dups_size":0,"ondisk_log_size":9468,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45023256,"num_objects":688,"num_object_clones":0,"num_object_copies":2064,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":688,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9468,"num_write_kb":324929,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2312,"num_bytes_recovered":47775744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":132,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.162870+0000","last_change":"2023-08-28T02:27:31.325260+0000","last_active":"2023-08-28T02:30:20.162870+0000","last_peered":"2023-08-28T02:30:20.162870+0000","last_clean":"2023-08-28T02:30:20.162870+0000","last_became_active":"2023-08-28T02:27:31.324950+0000","last_became_peered":"2023-08-28T02:27:31.324950+0000","last_unstale":"2023-08-28T02:30:20.162870+0000","last_undegraded":"2023-08-28T02:30:20.162870+0000","last_fullsized":"2023-08-28T02:30:20.162870+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:21:33.833261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":63,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.160282+0000","last_change":"2023-08-28T02:27:31.322118+0000","last_active":"2023-08-28T02:30:20.160282+0000","last_peered":"2023-08-28T02:30:20.160282+0000","last_clean":"2023-08-28T02:30:20.160282+0000","last_became_active":"2023-08-28T02:27:31.321338+0000","last_became_peered":"2023-08-28T02:27:31.321338+0000","last_unstale":"2023-08-28T02:30:20.160282+0000","last_undegraded":"2023-08-28T02:30:20.160282+0000","last_fullsized":"2023-08-28T02:30:20.160282+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:56:14.210571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"255'6516","reported_seq":6609,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-08-28T02:30:23.828514+0000","last_change":"2023-08-28T02:30:11.788028+0000","last_active":"2023-08-28T02:30:23.828514+0000","last_peered":"2023-08-28T02:30:23.828514+0000","last_clean":"2023-08-28T02:30:23.828514+0000","last_became_active":"2023-08-28T02:27:58.437499+0000","last_became_peered":"2023-08-28T02:27:58.437499+0000","last_unstale":"2023-08-28T02:30:23.828514+0000","last_undegraded":"2023-08-28T02:30:23.828514+0000","last_fullsized":"2023-08-28T02:30:23.828514+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6516,"log_dups_size":0,"ondisk_log_size":6516,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28246016,"num_objects":431,"num_object_clones":0,"num_object_copies":1293,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":431,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4679,"num_write_kb":163520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"78'126","reported_seq":624,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.158384+0000","last_change":"2023-08-28T02:28:27.383137+0000","last_active":"2023-08-28T02:30:20.158384+0000","last_peered":"2023-08-28T02:30:20.158384+0000","last_clean":"2023-08-28T02:30:20.158384+0000","last_became_active":"2023-08-28T02:28:27.382488+0000","last_became_peered":"2023-08-28T02:28:27.382488+0000","last_unstale":"2023-08-28T02:30:20.158384+0000","last_undegraded":"2023-08-28T02:30:20.158384+0000","last_fullsized":"2023-08-28T02:30:20.158384+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:14:48.414426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":743,"num_read_kb":1331,"num_write":125,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":254,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.157250+0000","last_change":"2023-08-28T02:23:59.207148+0000","last_active":"2023-08-28T02:30:20.157250+0000","last_peered":"2023-08-28T02:30:20.157250+0000","last_clean":"2023-08-28T02:30:20.157250+0000","last_became_active":"2023-08-28T02:23:59.206802+0000","last_became_peered":"2023-08-28T02:23:59.206802+0000","last_unstale":"2023-08-28T02:30:20.157250+0000","last_undegraded":"2023-08-28T02:30:20.157250+0000","last_fullsized":"2023-08-28T02:30:20.157250+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":63,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.157794+0000","last_change":"2023-08-28T02:27:31.328027+0000","last_active":"2023-08-28T02:30:20.157794+0000","last_peered":"2023-08-28T02:30:20.157794+0000","last_clean":"2023-08-28T02:30:20.157794+0000","last_became_active":"2023-08-28T02:27:31.326529+0000","last_became_peered":"2023-08-28T02:27:31.326529+0000","last_unstale":"2023-08-28T02:30:20.157794+0000","last_undegraded":"2023-08-28T02:30:20.157794+0000","last_fullsized":"2023-08-28T02:30:20.157794+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:56:57.331355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"255'6697","reported_seq":6785,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-08-28T02:30:24.108152+0000","last_change":"2023-08-28T02:27:58.462974+0000","last_active":"2023-08-28T02:30:24.108152+0000","last_peered":"2023-08-28T02:30:24.108152+0000","last_clean":"2023-08-28T02:30:24.108152+0000","last_became_active":"2023-08-28T02:27:58.460293+0000","last_became_peered":"2023-08-28T02:27:58.460293+0000","last_unstale":"2023-08-28T02:30:24.108152+0000","last_undegraded":"2023-08-28T02:30:24.108152+0000","last_fullsized":"2023-08-28T02:30:24.108152+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6697,"log_dups_size":0,"ondisk_log_size":6697,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30244864,"num_objects":461,"num_object_clones":0,"num_object_copies":1383,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":461,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4811,"num_write_kb":168736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"78'204","reported_seq":1340,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.157629+0000","last_change":"2023-08-28T02:28:59.480561+0000","last_active":"2023-08-28T02:30:20.157629+0000","last_peered":"2023-08-28T02:30:20.157629+0000","last_clean":"2023-08-28T02:30:20.157629+0000","last_became_active":"2023-08-28T02:28:59.479868+0000","last_became_peered":"2023-08-28T02:28:59.479868+0000","last_unstale":"2023-08-28T02:30:20.157629+0000","last_undegraded":"2023-08-28T02:30:20.157629+0000","last_fullsized":"2023-08-28T02:30:20.157629+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":204,"log_dups_size":0,"ondisk_log_size":204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:39:50.444629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1521,"num_read_kb":2511,"num_write":178,"num_write_kb":1175,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":249,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.172337+0000","last_change":"2023-08-28T02:28:25.352417+0000","last_active":"2023-08-28T02:30:20.172337+0000","last_peered":"2023-08-28T02:30:20.172337+0000","last_clean":"2023-08-28T02:30:20.172337+0000","last_became_active":"2023-08-28T02:28:25.352216+0000","last_became_peered":"2023-08-28T02:28:25.352216+0000","last_unstale":"2023-08-28T02:30:20.172337+0000","last_undegraded":"2023-08-28T02:30:20.172337+0000","last_fullsized":"2023-08-28T02:30:20.172337+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":63,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.158228+0000","last_change":"2023-08-28T02:27:31.376755+0000","last_active":"2023-08-28T02:30:20.158228+0000","last_peered":"2023-08-28T02:30:20.158228+0000","last_clean":"2023-08-28T02:30:20.158228+0000","last_became_active":"2023-08-28T02:27:31.373279+0000","last_became_peered":"2023-08-28T02:27:31.373279+0000","last_unstale":"2023-08-28T02:30:20.158228+0000","last_undegraded":"2023-08-28T02:30:20.158228+0000","last_fullsized":"2023-08-28T02:30:20.158228+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:26:14.628941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"255'6700","reported_seq":8860,"reported_epoch":255,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-08-28T02:30:27.180705+0000","last_change":"2023-08-28T02:30:08.028893+0000","last_active":"2023-08-28T02:30:27.180705+0000","last_peered":"2023-08-28T02:30:27.180705+0000","last_clean":"2023-08-28T02:27:16.315756+0000","last_became_active":"2023-08-28T02:28:25.400914+0000","last_became_peered":"2023-08-28T02:28:25.400914+0000","last_unstale":"2023-08-28T02:30:27.180705+0000","last_undegraded":"2023-08-28T02:28:25.339881+0000","last_fullsized":"2023-08-28T02:28:25.329514+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6700,"log_dups_size":0,"ondisk_log_size":6700,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26476544,"num_objects":404,"num_object_clones":0,"num_object_copies":1212,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1025,"num_objects_misplaced":204,"num_objects_unfound":0,"num_objects_dirty":404,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4816,"num_write_kb":167040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":735,"num_bytes_recovered":48168960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"2,3,7","objects":204},{"shards":"3,7","objects":-825}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'47","reported_seq":471,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.172543+0000","last_change":"2023-08-28T02:24:05.799647+0000","last_active":"2023-08-28T02:30:20.172543+0000","last_peered":"2023-08-28T02:30:20.172543+0000","last_clean":"2023-08-28T02:30:20.172543+0000","last_became_active":"2023-08-28T02:23:59.210481+0000","last_became_peered":"2023-08-28T02:23:59.210481+0000","last_unstale":"2023-08-28T02:30:20.172543+0000","last_undegraded":"2023-08-28T02:30:20.172543+0000","last_fullsized":"2023-08-28T02:30:20.172543+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":47,"log_dups_size":0,"ondisk_log_size":47,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:40:18.542375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":162,"num_read_kb":1281,"num_write":48,"num_write_kb":1153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":72,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.157536+0000","last_change":"2023-08-28T02:28:25.343406+0000","last_active":"2023-08-28T02:30:20.157536+0000","last_peered":"2023-08-28T02:30:20.157536+0000","last_clean":"2023-08-28T02:30:20.157536+0000","last_became_active":"2023-08-28T02:28:25.343065+0000","last_became_peered":"2023-08-28T02:28:25.343065+0000","last_unstale":"2023-08-28T02:30:20.157536+0000","last_undegraded":"2023-08-28T02:30:20.157536+0000","last_fullsized":"2023-08-28T02:30:20.157536+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:20:31.893609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":63,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.160362+0000","last_change":"2023-08-28T02:27:31.322967+0000","last_active":"2023-08-28T02:30:20.160362+0000","last_peered":"2023-08-28T02:30:20.160362+0000","last_clean":"2023-08-28T02:30:20.160362+0000","last_became_active":"2023-08-28T02:27:31.322655+0000","last_became_peered":"2023-08-28T02:27:31.322655+0000","last_unstale":"2023-08-28T02:30:20.160362+0000","last_undegraded":"2023-08-28T02:30:20.160362+0000","last_fullsized":"2023-08-28T02:30:20.160362+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:05:15.868115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"255'6755","reported_seq":6853,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-08-28T02:30:27.160300+0000","last_change":"2023-08-28T02:30:14.118676+0000","last_active":"2023-08-28T02:30:27.160300+0000","last_peered":"2023-08-28T02:30:27.160300+0000","last_clean":"2023-08-28T02:30:27.160300+0000","last_became_active":"2023-08-28T02:27:58.452334+0000","last_became_peered":"2023-08-28T02:27:58.452334+0000","last_unstale":"2023-08-28T02:30:27.160300+0000","last_undegraded":"2023-08-28T02:30:27.160300+0000","last_fullsized":"2023-08-28T02:30:27.160300+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6755,"log_dups_size":0,"ondisk_log_size":6755,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23527424,"num_objects":359,"num_object_clones":0,"num_object_copies":1077,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":359,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4899,"num_write_kb":168256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"78'114","reported_seq":618,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.156199+0000","last_change":"2023-08-28T02:24:01.210739+0000","last_active":"2023-08-28T02:30:20.156199+0000","last_peered":"2023-08-28T02:30:20.156199+0000","last_clean":"2023-08-28T02:30:20.156199+0000","last_became_active":"2023-08-28T02:24:01.210386+0000","last_became_peered":"2023-08-28T02:24:01.210386+0000","last_unstale":"2023-08-28T02:30:20.156199+0000","last_undegraded":"2023-08-28T02:30:20.156199+0000","last_fullsized":"2023-08-28T02:30:20.156199+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":114,"log_dups_size":0,"ondisk_log_size":114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":358,"num_read_kb":1352,"num_write":102,"num_write_kb":1104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"86'139","reported_seq":465,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.163327+0000","last_change":"2023-08-28T02:29:58.484566+0000","last_active":"2023-08-28T02:30:20.163327+0000","last_peered":"2023-08-28T02:30:20.163327+0000","last_clean":"2023-08-28T02:30:20.163327+0000","last_became_active":"2023-08-28T02:29:58.484046+0000","last_became_peered":"2023-08-28T02:29:58.484046+0000","last_unstale":"2023-08-28T02:30:20.163327+0000","last_undegraded":"2023-08-28T02:30:20.163327+0000","last_fullsized":"2023-08-28T02:30:20.163327+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_clean_scrub_stamp":"2023-08-28T02:08:02.043718+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":133,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.162879+0000","last_change":"2023-08-28T02:28:25.347600+0000","last_active":"2023-08-28T02:30:20.162879+0000","last_peered":"2023-08-28T02:30:20.162879+0000","last_clean":"2023-08-28T02:30:20.162879+0000","last_became_active":"2023-08-28T02:28:25.347237+0000","last_became_peered":"2023-08-28T02:28:25.347237+0000","last_unstale":"2023-08-28T02:30:20.162879+0000","last_undegraded":"2023-08-28T02:30:20.162879+0000","last_fullsized":"2023-08-28T02:30:20.162879+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:29:26.767559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":44,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.160079+0000","last_change":"2023-08-28T02:28:25.398594+0000","last_active":"2023-08-28T02:30:20.160079+0000","last_peered":"2023-08-28T02:30:20.160079+0000","last_clean":"2023-08-28T02:30:20.160079+0000","last_became_active":"2023-08-28T02:28:25.397058+0000","last_became_peered":"2023-08-28T02:28:25.397058+0000","last_unstale":"2023-08-28T02:30:20.160079+0000","last_undegraded":"2023-08-28T02:30:20.160079+0000","last_fullsized":"2023-08-28T02:30:20.160079+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"255'6571","reported_seq":11984,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-08-28T02:30:24.555488+0000","last_change":"2023-08-28T02:30:14.359869+0000","last_active":"2023-08-28T02:30:24.555488+0000","last_peered":"2023-08-28T02:30:24.555488+0000","last_clean":"2023-08-28T02:30:24.555488+0000","last_became_active":"2023-08-28T02:28:00.019510+0000","last_became_peered":"2023-08-28T02:28:00.019510+0000","last_unstale":"2023-08-28T02:30:24.555488+0000","last_undegraded":"2023-08-28T02:30:24.555488+0000","last_fullsized":"2023-08-28T02:30:24.555488+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6571,"log_dups_size":0,"ondisk_log_size":6571,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30441472,"num_objects":464,"num_object_clones":0,"num_object_copies":1392,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":464,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4738,"num_write_kb":166496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1910,"num_bytes_recovered":125206528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'148","reported_seq":785,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.163376+0000","last_change":"2023-08-28T02:29:39.524927+0000","last_active":"2023-08-28T02:30:20.163376+0000","last_peered":"2023-08-28T02:30:20.163376+0000","last_clean":"2023-08-28T02:30:20.163376+0000","last_became_active":"2023-08-28T02:29:39.524297+0000","last_became_peered":"2023-08-28T02:29:39.524297+0000","last_unstale":"2023-08-28T02:30:20.163376+0000","last_undegraded":"2023-08-28T02:30:20.163376+0000","last_fullsized":"2023-08-28T02:30:20.163376+0000","mapping_epoch":243,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":244,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1092,"num_read_kb":1955,"num_write":127,"num_write_kb":5177,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":223,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.149815+0000","last_change":"2023-08-28T02:23:59.226986+0000","last_active":"2023-08-28T02:30:20.149815+0000","last_peered":"2023-08-28T02:30:20.149815+0000","last_clean":"2023-08-28T02:30:20.149815+0000","last_became_active":"2023-08-28T02:23:59.226529+0000","last_became_peered":"2023-08-28T02:23:59.226529+0000","last_unstale":"2023-08-28T02:30:20.149815+0000","last_undegraded":"2023-08-28T02:30:20.149815+0000","last_fullsized":"2023-08-28T02:30:20.149815+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:22:09.193619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":268,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.147758+0000","last_change":"2023-08-28T02:27:31.361298+0000","last_active":"2023-08-28T02:30:20.147758+0000","last_peered":"2023-08-28T02:30:20.147758+0000","last_clean":"2023-08-28T02:30:20.147758+0000","last_became_active":"2023-08-28T02:27:31.360939+0000","last_became_peered":"2023-08-28T02:27:31.360939+0000","last_unstale":"2023-08-28T02:30:20.147758+0000","last_undegraded":"2023-08-28T02:30:20.147758+0000","last_fullsized":"2023-08-28T02:30:20.147758+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:48:42.024920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"255'6726","reported_seq":13762,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-08-28T02:30:24.504145+0000","last_change":"2023-08-28T02:30:12.290655+0000","last_active":"2023-08-28T02:30:24.504145+0000","last_peered":"2023-08-28T02:30:24.504145+0000","last_clean":"2023-08-28T02:30:24.504145+0000","last_became_active":"2023-08-28T02:30:09.573625+0000","last_became_peered":"2023-08-28T02:30:09.573625+0000","last_unstale":"2023-08-28T02:30:24.504145+0000","last_undegraded":"2023-08-28T02:30:24.504145+0000","last_fullsized":"2023-08-28T02:30:24.504145+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6726,"log_dups_size":0,"ondisk_log_size":6726,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30998528,"num_objects":473,"num_object_clones":0,"num_object_copies":1419,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":473,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4851,"num_write_kb":170368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2494,"num_bytes_recovered":144736256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'103","reported_seq":630,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.168992+0000","last_change":"2023-08-28T02:24:07.212128+0000","last_active":"2023-08-28T02:30:20.168992+0000","last_peered":"2023-08-28T02:30:20.168992+0000","last_clean":"2023-08-28T02:30:20.168992+0000","last_became_active":"2023-08-28T02:24:07.211562+0000","last_became_peered":"2023-08-28T02:24:07.211562+0000","last_unstale":"2023-08-28T02:30:20.168992+0000","last_undegraded":"2023-08-28T02:30:20.168992+0000","last_fullsized":"2023-08-28T02:30:20.168992+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":803,"num_read_kb":2080,"num_write":73,"num_write_kb":1244,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":63,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.158505+0000","last_change":"2023-08-28T02:27:31.377042+0000","last_active":"2023-08-28T02:30:20.158505+0000","last_peered":"2023-08-28T02:30:20.158505+0000","last_clean":"2023-08-28T02:30:20.158505+0000","last_became_active":"2023-08-28T02:27:31.373709+0000","last_became_peered":"2023-08-28T02:27:31.373709+0000","last_unstale":"2023-08-28T02:30:20.158505+0000","last_undegraded":"2023-08-28T02:30:20.158505+0000","last_fullsized":"2023-08-28T02:30:20.158505+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:12:38.286332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":209,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.150385+0000","last_change":"2023-08-28T02:23:38.329667+0000","last_active":"2023-08-28T02:30:20.150385+0000","last_peered":"2023-08-28T02:30:20.150385+0000","last_clean":"2023-08-28T02:30:20.150385+0000","last_became_active":"2023-08-28T02:23:38.329175+0000","last_became_peered":"2023-08-28T02:23:38.329175+0000","last_unstale":"2023-08-28T02:30:20.150385+0000","last_undegraded":"2023-08-28T02:30:20.150385+0000","last_fullsized":"2023-08-28T02:30:20.150385+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:21:34.815067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"255'6769","reported_seq":6894,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-08-28T02:30:27.221308+0000","last_change":"2023-08-28T02:30:18.579772+0000","last_active":"2023-08-28T02:30:27.221308+0000","last_peered":"2023-08-28T02:30:27.221308+0000","last_clean":"2023-08-28T02:30:27.221308+0000","last_became_active":"2023-08-28T02:27:58.451860+0000","last_became_peered":"2023-08-28T02:27:58.451860+0000","last_unstale":"2023-08-28T02:30:27.221308+0000","last_undegraded":"2023-08-28T02:30:27.221308+0000","last_fullsized":"2023-08-28T02:30:27.221308+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6769,"log_dups_size":0,"ondisk_log_size":6769,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22675456,"num_objects":346,"num_object_clones":0,"num_object_copies":1038,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":346,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4904,"num_write_kb":168000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":118,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.164131+0000","last_change":"2023-08-28T02:23:59.232099+0000","last_active":"2023-08-28T02:30:20.164131+0000","last_peered":"2023-08-28T02:30:20.164131+0000","last_clean":"2023-08-28T02:30:20.164131+0000","last_became_active":"2023-08-28T02:23:59.219318+0000","last_became_peered":"2023-08-28T02:23:59.219318+0000","last_unstale":"2023-08-28T02:30:20.164131+0000","last_undegraded":"2023-08-28T02:30:20.164131+0000","last_fullsized":"2023-08-28T02:30:20.164131+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:59:30.093289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'112","reported_seq":408,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.160413+0000","last_change":"2023-08-28T02:28:59.477917+0000","last_active":"2023-08-28T02:30:20.160413+0000","last_peered":"2023-08-28T02:30:20.160413+0000","last_clean":"2023-08-28T02:30:20.160413+0000","last_became_active":"2023-08-28T02:28:59.477383+0000","last_became_peered":"2023-08-28T02:28:59.477383+0000","last_unstale":"2023-08-28T02:30:20.160413+0000","last_undegraded":"2023-08-28T02:30:20.160413+0000","last_fullsized":"2023-08-28T02:30:20.160413+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:28:45.045742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":444,"num_read_kb":473,"num_write":94,"num_write_kb":127,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":115,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.162796+0000","last_change":"2023-08-28T02:23:59.221034+0000","last_active":"2023-08-28T02:30:20.162796+0000","last_peered":"2023-08-28T02:30:20.162796+0000","last_clean":"2023-08-28T02:30:20.162796+0000","last_became_active":"2023-08-28T02:23:59.220730+0000","last_became_peered":"2023-08-28T02:23:59.220730+0000","last_unstale":"2023-08-28T02:30:20.162796+0000","last_undegraded":"2023-08-28T02:30:20.162796+0000","last_fullsized":"2023-08-28T02:30:20.162796+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:25:52.364370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"255'6771","reported_seq":6877,"reported_epoch":255,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:30:26.956719+0000","last_change":"2023-08-28T02:28:26.460507+0000","last_active":"2023-08-28T02:30:26.956719+0000","last_peered":"2023-08-28T02:30:26.956719+0000","last_clean":"2023-08-28T02:27:16.315150+0000","last_became_active":"2023-08-28T02:28:26.432234+0000","last_became_peered":"2023-08-28T02:28:26.432234+0000","last_unstale":"2023-08-28T02:30:26.956719+0000","last_undegraded":"2023-08-28T02:28:26.346974+0000","last_fullsized":"2023-08-28T02:28:26.332755+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6771,"log_dups_size":0,"ondisk_log_size":6771,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25755648,"num_objects":393,"num_object_clones":0,"num_object_copies":1179,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1660,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":393,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4883,"num_write_kb":168832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"1,2,7","objects":191},{"shards":"2,7","objects":-1267}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"78'122","reported_seq":562,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.160115+0000","last_change":"2023-08-28T02:28:28.383306+0000","last_active":"2023-08-28T02:30:20.160115+0000","last_peered":"2023-08-28T02:30:20.160115+0000","last_clean":"2023-08-28T02:30:20.160115+0000","last_became_active":"2023-08-28T02:28:28.382672+0000","last_became_peered":"2023-08-28T02:28:28.382672+0000","last_unstale":"2023-08-28T02:30:20.160115+0000","last_undegraded":"2023-08-28T02:30:20.160115+0000","last_fullsized":"2023-08-28T02:30:20.160115+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":857,"num_read_kb":2224,"num_write":99,"num_write_kb":2312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":235,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.147852+0000","last_change":"2023-08-28T02:27:31.424373+0000","last_active":"2023-08-28T02:30:20.147852+0000","last_peered":"2023-08-28T02:30:20.147852+0000","last_clean":"2023-08-28T02:30:20.147852+0000","last_became_active":"2023-08-28T02:27:31.324505+0000","last_became_peered":"2023-08-28T02:27:31.324505+0000","last_unstale":"2023-08-28T02:30:20.147852+0000","last_undegraded":"2023-08-28T02:30:20.147852+0000","last_fullsized":"2023-08-28T02:30:20.147852+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:27:57.837524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":224,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.172185+0000","last_change":"2023-08-28T02:28:25.351840+0000","last_active":"2023-08-28T02:30:20.172185+0000","last_peered":"2023-08-28T02:30:20.172185+0000","last_clean":"2023-08-28T02:30:20.172185+0000","last_became_active":"2023-08-28T02:28:25.351399+0000","last_became_peered":"2023-08-28T02:28:25.351399+0000","last_unstale":"2023-08-28T02:30:20.172185+0000","last_undegraded":"2023-08-28T02:30:20.172185+0000","last_fullsized":"2023-08-28T02:30:20.172185+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":211,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.150330+0000","last_change":"2023-08-28T02:23:38.329474+0000","last_active":"2023-08-28T02:30:20.150330+0000","last_peered":"2023-08-28T02:30:20.150330+0000","last_clean":"2023-08-28T02:30:20.150330+0000","last_became_active":"2023-08-28T02:23:38.328505+0000","last_became_peered":"2023-08-28T02:23:38.328505+0000","last_unstale":"2023-08-28T02:30:20.150330+0000","last_undegraded":"2023-08-28T02:30:20.150330+0000","last_fullsized":"2023-08-28T02:30:20.150330+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:12:14.835809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":296,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.173047+0000","last_change":"2023-08-28T02:27:31.384467+0000","last_active":"2023-08-28T02:30:20.173047+0000","last_peered":"2023-08-28T02:30:20.173047+0000","last_clean":"2023-08-28T02:30:20.173047+0000","last_became_active":"2023-08-28T02:27:31.384183+0000","last_became_peered":"2023-08-28T02:27:31.384183+0000","last_unstale":"2023-08-28T02:30:20.173047+0000","last_undegraded":"2023-08-28T02:30:20.173047+0000","last_fullsized":"2023-08-28T02:30:20.173047+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:59:47.551892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.19","version":"255'6651","reported_seq":6741,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-08-28T02:30:23.899123+0000","last_change":"2023-08-28T02:30:16.900410+0000","last_active":"2023-08-28T02:30:23.899123+0000","last_peered":"2023-08-28T02:30:23.899123+0000","last_clean":"2023-08-28T02:30:23.899123+0000","last_became_active":"2023-08-28T02:27:58.490323+0000","last_became_peered":"2023-08-28T02:27:58.490323+0000","last_unstale":"2023-08-28T02:30:23.899123+0000","last_undegraded":"2023-08-28T02:30:23.899123+0000","last_fullsized":"2023-08-28T02:30:23.899123+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6651,"log_dups_size":0,"ondisk_log_size":6651,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30113792,"num_objects":460,"num_object_clones":0,"num_object_copies":1380,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":460,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4782,"num_write_kb":167712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":61,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.173347+0000","last_change":"2023-08-28T02:28:11.333622+0000","last_active":"2023-08-28T02:30:20.173347+0000","last_peered":"2023-08-28T02:30:20.173347+0000","last_clean":"2023-08-28T02:30:20.173347+0000","last_became_active":"2023-08-28T02:28:11.333321+0000","last_became_peered":"2023-08-28T02:28:11.333321+0000","last_unstale":"2023-08-28T02:30:20.173347+0000","last_undegraded":"2023-08-28T02:30:20.173347+0000","last_fullsized":"2023-08-28T02:30:20.173347+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":272,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.172467+0000","last_change":"2023-08-28T02:27:31.343538+0000","last_active":"2023-08-28T02:30:20.172467+0000","last_peered":"2023-08-28T02:30:20.172467+0000","last_clean":"2023-08-28T02:30:20.172467+0000","last_became_active":"2023-08-28T02:27:31.343324+0000","last_became_peered":"2023-08-28T02:27:31.343324+0000","last_unstale":"2023-08-28T02:30:20.172467+0000","last_undegraded":"2023-08-28T02:30:20.172467+0000","last_fullsized":"2023-08-28T02:30:20.172467+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.18","version":"255'6627","reported_seq":6712,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-08-28T02:30:27.160640+0000","last_change":"2023-08-28T02:27:58.496324+0000","last_active":"2023-08-28T02:30:27.160640+0000","last_peered":"2023-08-28T02:30:27.160640+0000","last_clean":"2023-08-28T02:30:27.160640+0000","last_became_active":"2023-08-28T02:27:58.494507+0000","last_became_peered":"2023-08-28T02:27:58.494507+0000","last_unstale":"2023-08-28T02:30:27.160640+0000","last_undegraded":"2023-08-28T02:30:27.160640+0000","last_fullsized":"2023-08-28T02:30:27.160640+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6627,"log_dups_size":0,"ondisk_log_size":6627,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26771456,"num_objects":409,"num_object_clones":0,"num_object_copies":1227,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":409,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4751,"num_write_kb":165088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":74,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.172738+0000","last_change":"2023-08-28T02:28:25.705331+0000","last_active":"2023-08-28T02:30:20.172738+0000","last_peered":"2023-08-28T02:30:20.172738+0000","last_clean":"2023-08-28T02:30:20.172738+0000","last_became_active":"2023-08-28T02:28:25.397480+0000","last_became_peered":"2023-08-28T02:28:25.397480+0000","last_unstale":"2023-08-28T02:30:20.172738+0000","last_undegraded":"2023-08-28T02:30:20.172738+0000","last_fullsized":"2023-08-28T02:30:20.172738+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":254,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.172686+0000","last_change":"2023-08-28T02:23:59.207929+0000","last_active":"2023-08-28T02:30:20.172686+0000","last_peered":"2023-08-28T02:30:20.172686+0000","last_clean":"2023-08-28T02:30:20.172686+0000","last_became_active":"2023-08-28T02:23:59.207589+0000","last_became_peered":"2023-08-28T02:23:59.207589+0000","last_unstale":"2023-08-28T02:30:20.172686+0000","last_undegraded":"2023-08-28T02:30:20.172686+0000","last_fullsized":"2023-08-28T02:30:20.172686+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":142,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.163560+0000","last_change":"2023-08-28T02:28:25.347988+0000","last_active":"2023-08-28T02:30:20.163560+0000","last_peered":"2023-08-28T02:30:20.163560+0000","last_clean":"2023-08-28T02:30:20.163560+0000","last_became_active":"2023-08-28T02:28:25.347632+0000","last_became_peered":"2023-08-28T02:28:25.347632+0000","last_unstale":"2023-08-28T02:30:20.163560+0000","last_undegraded":"2023-08-28T02:30:20.163560+0000","last_fullsized":"2023-08-28T02:30:20.163560+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:56:20.202662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":114,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.162870+0000","last_change":"2023-08-28T02:23:59.217794+0000","last_active":"2023-08-28T02:30:20.162870+0000","last_peered":"2023-08-28T02:30:20.162870+0000","last_clean":"2023-08-28T02:30:20.162870+0000","last_became_active":"2023-08-28T02:23:59.217231+0000","last_became_peered":"2023-08-28T02:23:59.217231+0000","last_unstale":"2023-08-28T02:30:20.162870+0000","last_undegraded":"2023-08-28T02:30:20.162870+0000","last_fullsized":"2023-08-28T02:30:20.162870+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:36:26.297410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":209,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.148238+0000","last_change":"2023-08-28T02:20:19.850252+0000","last_active":"2023-08-28T02:30:20.148238+0000","last_peered":"2023-08-28T02:30:20.148238+0000","last_clean":"2023-08-28T02:30:20.148238+0000","last_became_active":"2023-08-28T02:20:19.849913+0000","last_became_peered":"2023-08-28T02:20:19.849913+0000","last_unstale":"2023-08-28T02:30:20.148238+0000","last_undegraded":"2023-08-28T02:30:20.148238+0000","last_fullsized":"2023-08-28T02:30:20.148238+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:48:47.713465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":286,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.173349+0000","last_change":"2023-08-28T02:28:25.347845+0000","last_active":"2023-08-28T02:30:20.173349+0000","last_peered":"2023-08-28T02:30:20.173349+0000","last_clean":"2023-08-28T02:30:20.173349+0000","last_became_active":"2023-08-28T02:28:25.347502+0000","last_became_peered":"2023-08-28T02:28:25.347502+0000","last_unstale":"2023-08-28T02:30:20.173349+0000","last_undegraded":"2023-08-28T02:30:20.173349+0000","last_fullsized":"2023-08-28T02:30:20.173349+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":26,"stat_sum":{"num_bytes":878575640,"num_objects":13407,"num_object_clones":0,"num_object_copies":40221,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6078,"num_objects_misplaced":2005,"num_objects_unfound":0,"num_objects_dirty":13407,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":153827,"num_write_kb":5351425,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20086,"num_bytes_recovered":1154875392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3568709632,"data_stored":3568697416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":180365,"ondisk_log_size":180365,"up":78,"acting":74,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":19382,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26810,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":137,"num_bytes_recovered":12582931,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430458,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":973860,"kb_used_data":672540,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92771292,"statfs":{"total":95995035648,"available":94997803008,"internally_reserved":0,"allocated":688680960,"data_stored":687966198,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:30:03 2023","interfaces":[{"interface":"back","average":{"1min":2.194,"5min":2.3999999999999999,"15min":2.0859999999999999},"min":{"1min":0.25600000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":24.379999999999999,"5min":59.753999999999998,"15min":116.755},"last":7.4240000000000004},{"interface":"front","average":{"1min":2.7120000000000002,"5min":2.7559999999999998,"15min":2.3359999999999999},"min":{"1min":0.26500000000000001,"5min":0.245,"15min":0.23999999999999999},"max":{"1min":24.213999999999999,"5min":59.765999999999998,"15min":116.971},"last":0.42599999999999999}]},{"osd":1,"last update":"Mon Aug 28 02:30:11 2023","interfaces":[{"interface":"back","average":{"1min":2.496,"5min":2.2509999999999999,"15min":1.2370000000000001},"min":{"1min":0.27900000000000003,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":24.681000000000001,"5min":44.287999999999997,"15min":44.287999999999997},"last":7.3769999999999998},{"interface":"front","average":{"1min":2.7570000000000001,"5min":3.109,"15min":1.667},"min":{"1min":0.28399999999999997,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":24.189,"5min":44.393999999999998,"15min":44.393999999999998},"last":0.39500000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:29:35 2023","interfaces":[{"interface":"back","average":{"1min":2.1309999999999998,"5min":4.3079999999999998,"15min":2.3759999999999999},"min":{"1min":0.28000000000000003,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":8.2680000000000007,"5min":119.571,"15min":119.571},"last":7.4820000000000002},{"interface":"front","average":{"1min":1.843,"5min":5.2800000000000002,"15min":2.7869999999999999},"min":{"1min":0.35099999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":8.2789999999999999,"5min":119.554,"15min":119.554},"last":0.48999999999999999}]},{"osd":3,"last update":"Mon Aug 28 02:30:11 2023","interfaces":[{"interface":"back","average":{"1min":2.0110000000000001,"5min":4.0620000000000003,"15min":2.2480000000000002},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30199999999999999},"max":{"1min":10.686999999999999,"5min":99.459000000000003,"15min":107.916},"last":7.4539999999999997},{"interface":"front","average":{"1min":1.794,"5min":3.6789999999999998,"15min":2.3239999999999998},"min":{"1min":0.34300000000000003,"5min":0.311,"15min":0.27700000000000002},"max":{"1min":10.140000000000001,"5min":98.084000000000003,"15min":112.00700000000001},"last":0.47099999999999997}]},{"osd":4,"last update":"Mon Aug 28 02:29:35 2023","interfaces":[{"interface":"back","average":{"1min":0.75900000000000001,"5min":1.3360000000000001,"15min":1.2390000000000001},"min":{"1min":0.26500000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.4209999999999998,"5min":11.696,"15min":63.048000000000002},"last":1.121},{"interface":"front","average":{"1min":1.179,"5min":1.4059999999999999,"15min":1.1830000000000001},"min":{"1min":0.29499999999999998,"5min":0.247,"15min":0.247},"max":{"1min":4.2359999999999998,"5min":10.840999999999999,"15min":62.912999999999997},"last":6.1219999999999999}]},{"osd":5,"last update":"Mon Aug 28 02:29:35 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":1.1759999999999999,"15min":1.0269999999999999},"min":{"1min":0.315,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":1.8939999999999999,"5min":13.788,"15min":63.021000000000001},"last":7.5110000000000001},{"interface":"front","average":{"1min":0.71299999999999997,"5min":1.4299999999999999,"15min":1.3260000000000001},"min":{"1min":0.29999999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.5449999999999999,"5min":9.6929999999999996,"15min":63.082999999999998},"last":6.7110000000000003}]},{"osd":6,"last update":"Mon Aug 28 02:29:35 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":1.3360000000000001,"15min":1.2769999999999999},"min":{"1min":0.33700000000000002,"5min":0.317,"15min":0.29699999999999999},"max":{"1min":2.6949999999999998,"5min":9.7070000000000007,"15min":101.61},"last":7.5369999999999999},{"interface":"front","average":{"1min":0.85599999999999998,"5min":1.395,"15min":1.413},"min":{"1min":0.33900000000000002,"5min":0.308,"15min":0.308},"max":{"1min":2.3969999999999998,"5min":14.423999999999999,"15min":101.621},"last":0.44900000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626685,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":484504,"kb_used_data":183184,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93260648,"statfs":{"total":95995035648,"available":95498903552,"internally_reserved":0,"allocated":187580416,"data_stored":186877500,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:30:11 2023","interfaces":[{"interface":"back","average":{"1min":3.246,"5min":3.911,"15min":2.7629999999999999},"min":{"1min":0.22700000000000001,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":44.512999999999998,"5min":60.625999999999998,"15min":121.247},"last":2.7269999999999999},{"interface":"front","average":{"1min":3.3300000000000001,"5min":3.7080000000000002,"15min":2.722},"min":{"1min":0.26000000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":46.637999999999998,"5min":54.273000000000003,"15min":118.977},"last":0.77700000000000002}]},{"osd":1,"last update":"Mon Aug 28 02:30:11 2023","interfaces":[{"interface":"back","average":{"1min":1.78,"5min":2.524,"15min":1.669},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23699999999999999},"max":{"1min":7.2270000000000003,"5min":99.998999999999995,"15min":99.998999999999995},"last":2.5510000000000002},{"interface":"front","average":{"1min":1.9550000000000001,"5min":2.8010000000000002,"15min":1.8100000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":8.6029999999999998,"5min":95.040000000000006,"15min":95.040000000000006},"last":3.1099999999999999}]},{"osd":2,"last update":"Mon Aug 28 02:29:53 2023","interfaces":[{"interface":"back","average":{"1min":1.4650000000000001,"5min":4.258,"15min":2.2250000000000001},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":4.3259999999999996,"5min":79.787000000000006,"15min":79.787000000000006},"last":4.7939999999999996},{"interface":"front","average":{"1min":1.3959999999999999,"5min":4.4470000000000001,"15min":2.2200000000000002},"min":{"1min":0.28899999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":7.8490000000000002,"5min":79.935000000000002,"15min":79.935000000000002},"last":2.694}]},{"osd":3,"last update":"Mon Aug 28 02:30:11 2023","interfaces":[{"interface":"back","average":{"1min":1.1419999999999999,"5min":2.7850000000000001,"15min":1.6719999999999999},"min":{"1min":0.32000000000000001,"5min":0.317,"15min":0.317},"max":{"1min":3.968,"5min":39.587000000000003,"15min":39.587000000000003},"last":3.2130000000000001},{"interface":"front","average":{"1min":1.0069999999999999,"5min":2.8940000000000001,"15min":1.7589999999999999},"min":{"1min":0.35299999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":3.9780000000000002,"5min":39.593000000000004,"15min":39.593000000000004},"last":4.7350000000000003}]},{"osd":4,"last update":"Mon Aug 28 02:30:18 2023","interfaces":[{"interface":"back","average":{"1min":1.0069999999999999,"5min":1.224,"15min":1.04},"min":{"1min":0.27100000000000002,"5min":0.251,"15min":0.251},"max":{"1min":3.6499999999999999,"5min":18.308,"15min":18.308},"last":2.5190000000000001},{"interface":"front","average":{"1min":0.77400000000000002,"5min":1.3899999999999999,"15min":1.131},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.26700000000000002},"max":{"1min":3.5070000000000001,"5min":18.263000000000002,"15min":18.263000000000002},"last":4.7690000000000001}]},{"osd":5,"last update":"Mon Aug 28 02:30:18 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":1.216,"15min":1.1080000000000001},"min":{"1min":0.29699999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":2.8359999999999999,"5min":9.9809999999999999,"15min":20.759},"last":6.0049999999999999},{"interface":"front","average":{"1min":0.98999999999999999,"5min":1.113,"15min":0.95599999999999996},"min":{"1min":0.314,"5min":0.314,"15min":0.28799999999999998},"max":{"1min":3.1549999999999998,"5min":5.9500000000000002,"15min":7.3780000000000001},"last":2.7869999999999999}]},{"osd":7,"last update":"Mon Aug 28 02:29:40 2023","interfaces":[{"interface":"back","average":{"1min":0.80600000000000005,"5min":1.161,"15min":0.92700000000000005},"min":{"1min":0.34799999999999998,"5min":0.34200000000000003,"15min":0.314},"max":{"1min":3.8490000000000002,"5min":9.0830000000000002,"15min":9.0830000000000002},"last":2.7599999999999998},{"interface":"front","average":{"1min":0.68400000000000005,"5min":1.3320000000000001,"15min":1.0389999999999999},"min":{"1min":0.35599999999999998,"5min":0.33500000000000002,"15min":0.29799999999999999},"max":{"1min":3.0329999999999999,"5min":9.7110000000000003,"15min":9.7110000000000003},"last":0.44800000000000001}]}]},{"osd":1,"up_from":98,"seq":420906795137,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":724432,"kb_used_data":457736,"kb_used_omap":7,"kb_used_meta":266680,"kb_avail":93020720,"statfs":{"total":95995035648,"available":95253217280,"internally_reserved":0,"allocated":468721664,"data_stored":467455840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":273080651},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:30:09 2023","interfaces":[{"interface":"back","average":{"1min":1.534,"5min":4.548,"15min":2.1379999999999999},"min":{"1min":0.20799999999999999,"5min":0.191,"15min":0.17499999999999999},"max":{"1min":8.3230000000000004,"5min":128.60599999999999,"15min":128.60599999999999},"last":4.6740000000000004},{"interface":"front","average":{"1min":1.1930000000000001,"5min":4.3760000000000003,"15min":2.1480000000000001},"min":{"1min":0.221,"5min":0.188,"15min":0.188},"max":{"1min":3.8220000000000001,"5min":128.685,"15min":128.685},"last":3.573}]},{"osd":2,"last update":"Mon Aug 28 02:29:33 2023","interfaces":[{"interface":"back","average":{"1min":1.379,"5min":6.8920000000000003,"15min":2.8149999999999999},"min":{"1min":0.33600000000000002,"5min":0.222,"15min":0.222},"max":{"1min":5.3490000000000002,"5min":135.25999999999999,"15min":135.25999999999999},"last":0.34399999999999997},{"interface":"front","average":{"1min":1.4610000000000001,"5min":6.4219999999999997,"15min":2.6560000000000001},"min":{"1min":0.23499999999999999,"5min":0.23200000000000001,"15min":0.221},"max":{"1min":4.3150000000000004,"5min":135.10300000000001,"15min":135.10300000000001},"last":4.7270000000000003}]},{"osd":3,"last update":"Mon Aug 28 02:30:09 2023","interfaces":[{"interface":"back","average":{"1min":1.6899999999999999,"5min":3.7010000000000001,"15min":1.843},"min":{"1min":0.33600000000000002,"5min":0.22,"15min":0.22},"max":{"1min":3.867,"5min":123.741,"15min":123.741},"last":3.625},{"interface":"front","average":{"1min":3.9060000000000001,"5min":4.4829999999999997,"15min":2.1459999999999999},"min":{"1min":0.501,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":13.340999999999999,"5min":123.642,"15min":123.642},"last":4.5309999999999997}]},{"osd":4,"last update":"Mon Aug 28 02:30:09 2023","interfaces":[{"interface":"back","average":{"1min":1.6930000000000001,"5min":1.7789999999999999,"15min":1.264},"min":{"1min":0.29999999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.2939999999999996,"5min":17.600000000000001,"15min":17.600000000000001},"last":0.40000000000000002},{"interface":"front","average":{"1min":1.2170000000000001,"5min":1.6220000000000001,"15min":1.137},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.25600000000000001},"max":{"1min":3.5920000000000001,"5min":19.010000000000002,"15min":19.010000000000002},"last":4.7839999999999998}]},{"osd":5,"last update":"Mon Aug 28 02:30:09 2023","interfaces":[{"interface":"back","average":{"1min":2.242,"5min":1.6910000000000001,"15min":1.371},"min":{"1min":0.32100000000000001,"5min":0.312,"15min":0.29399999999999998},"max":{"1min":11.167999999999999,"5min":11.167999999999999,"15min":15.544},"last":4.4729999999999999},{"interface":"front","average":{"1min":2.2440000000000002,"5min":1.847,"15min":1.2549999999999999},"min":{"1min":0.34599999999999997,"5min":0.33600000000000002,"15min":0.33200000000000002},"max":{"1min":10.084,"5min":17.646999999999998,"15min":17.646999999999998},"last":0.376}]},{"osd":6,"last update":"Mon Aug 28 02:30:09 2023","interfaces":[{"interface":"back","average":{"1min":1.7170000000000001,"5min":1.7090000000000001,"15min":1.4630000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":3.923,"5min":9.2989999999999995,"15min":15.512},"last":3.2549999999999999},{"interface":"front","average":{"1min":1.8160000000000001,"5min":1.823,"15min":1.3899999999999999},"min":{"1min":0.39600000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":6.3140000000000001,"5min":17.673999999999999,"15min":17.673999999999999},"last":0.41899999999999998}]},{"osd":7,"last update":"Mon Aug 28 02:30:09 2023","interfaces":[{"interface":"back","average":{"1min":1.4059999999999999,"5min":1.7909999999999999,"15min":1.252},"min":{"1min":0.38500000000000001,"5min":0.32500000000000001,"15min":0.30199999999999999},"max":{"1min":4.0460000000000003,"5min":19.061,"15min":19.061},"last":4.758},{"interface":"front","average":{"1min":1.7889999999999999,"5min":1.905,"15min":1.4450000000000001},"min":{"1min":0.437,"5min":0.40500000000000003,"15min":0.33100000000000002},"max":{"1min":5.8890000000000002,"5min":9.3239999999999998,"15min":15.476000000000001},"last":2.093}]}]},{"osd":0,"up_from":90,"seq":386547056783,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":637796,"kb_used_data":308508,"kb_used_omap":8,"kb_used_meta":329079,"kb_avail":93107356,"statfs":{"total":95995035648,"available":95341932544,"internally_reserved":0,"allocated":315912192,"data_stored":314743946,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":336977257},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Aug 28 02:30:17 2023","interfaces":[{"interface":"back","average":{"1min":5.1619999999999999,"5min":3.004,"15min":1.8919999999999999},"min":{"1min":0.20000000000000001,"5min":0.184,"15min":0.184},"max":{"1min":56.576999999999998,"5min":56.576999999999998,"15min":56.576999999999998},"last":6.9370000000000003},{"interface":"front","average":{"1min":6.1689999999999996,"5min":3.524,"15min":1.931},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.17000000000000001},"max":{"1min":56.552999999999997,"5min":56.552999999999997,"15min":56.552999999999997},"last":2.5390000000000001}]},{"osd":2,"last update":"Mon Aug 28 02:29:32 2023","interfaces":[{"interface":"back","average":{"1min":1.9750000000000001,"5min":4.5430000000000001,"15min":2.0099999999999998},"min":{"1min":0.23200000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":8.8629999999999995,"5min":100.718,"15min":100.718},"last":2.6869999999999998},{"interface":"front","average":{"1min":1.7749999999999999,"5min":5.3769999999999998,"15min":2.3530000000000002},"min":{"1min":0.254,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":5.766,"5min":100.962,"15min":100.962},"last":2.0720000000000001}]},{"osd":3,"last update":"Mon Aug 28 02:30:17 2023","interfaces":[{"interface":"back","average":{"1min":2.6459999999999999,"5min":6.9390000000000001,"15min":2.9430000000000001},"min":{"1min":0.24299999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":19.327000000000002,"5min":237.43899999999999,"15min":237.43899999999999},"last":1.4630000000000001},{"interface":"front","average":{"1min":1.7390000000000001,"5min":6.6399999999999997,"15min":2.859},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":6.0010000000000003,"5min":237.48099999999999,"15min":237.48099999999999},"last":1.496}]},{"osd":4,"last update":"Mon Aug 28 02:30:12 2023","interfaces":[{"interface":"back","average":{"1min":1.7709999999999999,"5min":1.6859999999999999,"15min":1.2849999999999999},"min":{"1min":0.29699999999999999,"5min":0.29599999999999999,"15min":0.26800000000000002},"max":{"1min":9.3680000000000003,"5min":9.3680000000000003,"15min":9.6649999999999991},"last":2.7450000000000001},{"interface":"front","average":{"1min":1.536,"5min":2.0870000000000002,"15min":1.4650000000000001},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":8.7880000000000003,"5min":27.271999999999998,"15min":27.271999999999998},"last":1.528}]},{"osd":5,"last update":"Mon Aug 28 02:30:12 2023","interfaces":[{"interface":"back","average":{"1min":1.153,"5min":2.0099999999999998,"15min":1.5609999999999999},"min":{"1min":0.34000000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":3.0230000000000001,"5min":27.367000000000001,"15min":44.000999999999998},"last":1.548},{"interface":"front","average":{"1min":1.671,"5min":1.9079999999999999,"15min":1.488},"min":{"1min":0.34300000000000003,"5min":0.33000000000000002,"15min":0.30599999999999999},"max":{"1min":9.4489999999999998,"5min":12.334,"15min":43.953000000000003},"last":2.7839999999999998}]},{"osd":6,"last update":"Mon Aug 28 02:30:12 2023","interfaces":[{"interface":"back","average":{"1min":1.276,"5min":2.0819999999999999,"15min":1.585},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":3.0510000000000002,"5min":27.331,"15min":32.417999999999999},"last":6.8250000000000002},{"interface":"front","average":{"1min":1.6930000000000001,"5min":2.173,"15min":1.5620000000000001},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.30399999999999999},"max":{"1min":9.2789999999999999,"5min":27.247,"15min":32.130000000000003},"last":2.8119999999999998}]},{"osd":7,"last update":"Mon Aug 28 02:30:12 2023","interfaces":[{"interface":"back","average":{"1min":1.802,"5min":2.4769999999999999,"15min":1.583},"min":{"1min":0.36799999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":9.4209999999999994,"5min":27.385000000000002,"15min":27.385000000000002},"last":3.0680000000000001},{"interface":"front","average":{"1min":1.5069999999999999,"5min":2.339,"15min":1.599},"min":{"1min":0.38900000000000001,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":5.7300000000000004,"5min":27.213999999999999,"15min":27.213999999999999},"last":0.33000000000000002}]}]},{"osd":2,"up_from":104,"seq":446676598907,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1158452,"kb_used_data":842156,"kb_used_omap":10,"kb_used_meta":316277,"kb_avail":92586700,"statfs":{"total":95995035648,"available":94808780800,"internally_reserved":0,"allocated":862367744,"data_stored":861240759,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10792,"internal_metadata":323868120},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:29:36 2023","interfaces":[{"interface":"back","average":{"1min":1.3680000000000001,"5min":4.4729999999999999,"15min":2.1280000000000001},"min":{"1min":0.185,"5min":0.17699999999999999,"15min":0.17599999999999999},"max":{"1min":8.577,"5min":113.89100000000001,"15min":113.89100000000001},"last":4.5640000000000001},{"interface":"front","average":{"1min":1.399,"5min":8.0090000000000003,"15min":3.3260000000000001},"min":{"1min":0.25900000000000001,"5min":0.214,"15min":0.19900000000000001},"max":{"1min":7.0129999999999999,"5min":147.286,"15min":147.286},"last":0.96499999999999997}]},{"osd":1,"last update":"Mon Aug 28 02:29:36 2023","interfaces":[{"interface":"back","average":{"1min":1.288,"5min":2.798,"15min":1.585},"min":{"1min":0.23499999999999999,"5min":0.21099999999999999,"15min":0.20999999999999999},"max":{"1min":7.0919999999999996,"5min":16.372,"15min":30.888000000000002},"last":1.867},{"interface":"front","average":{"1min":1.653,"5min":2.9119999999999999,"15min":1.488},"min":{"1min":0.29199999999999998,"5min":0.21299999999999999,"15min":0.20999999999999999},"max":{"1min":5.7400000000000002,"5min":20.259,"15min":31.648},"last":0.308}]},{"osd":3,"last update":"Mon Aug 28 02:30:09 2023","interfaces":[{"interface":"back","average":{"1min":1.6799999999999999,"5min":5.1100000000000003,"15min":2.286},"min":{"1min":0.314,"5min":0.25,"15min":0.22700000000000001},"max":{"1min":9.7629999999999999,"5min":128.63900000000001,"15min":128.63900000000001},"last":2.0659999999999998},{"interface":"front","average":{"1min":2.1070000000000002,"5min":4.9269999999999996,"15min":2.3010000000000002},"min":{"1min":0.375,"5min":0.223,"15min":0.223},"max":{"1min":8.7029999999999994,"5min":128.75700000000001,"15min":128.75700000000001},"last":0.39100000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:29:36 2023","interfaces":[{"interface":"back","average":{"1min":7.9359999999999999,"5min":3.839,"15min":1.8700000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27100000000000002},"max":{"1min":130.27199999999999,"5min":130.27199999999999,"15min":130.27199999999999},"last":0.36899999999999999},{"interface":"front","average":{"1min":8.1839999999999993,"5min":3.1859999999999999,"15min":1.651},"min":{"1min":0.32600000000000001,"5min":0.28999999999999998,"15min":0.27400000000000002},"max":{"1min":132.21600000000001,"5min":132.21600000000001,"15min":132.21600000000001},"last":4.6260000000000003}]},{"osd":5,"last update":"Mon Aug 28 02:29:36 2023","interfaces":[{"interface":"back","average":{"1min":10.651,"5min":5.4619999999999997,"15min":2.4460000000000002},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":94.614000000000004,"5min":128.696,"15min":128.696},"last":1.254},{"interface":"front","average":{"1min":10.534000000000001,"5min":5.585,"15min":2.496},"min":{"1min":0.32300000000000001,"5min":0.32100000000000001,"15min":0.30299999999999999},"max":{"1min":94.534000000000006,"5min":128.84100000000001,"15min":128.84100000000001},"last":0.34499999999999997}]},{"osd":6,"last update":"Mon Aug 28 02:29:36 2023","interfaces":[{"interface":"back","average":{"1min":1.472,"5min":1.8480000000000001,"15min":1.2609999999999999},"min":{"1min":0.33000000000000002,"5min":0.307,"15min":0.29299999999999998},"max":{"1min":4.798,"5min":11.285,"15min":11.285},"last":4.6660000000000004},{"interface":"front","average":{"1min":1.333,"5min":2.1749999999999998,"15min":1.4079999999999999},"min":{"1min":0.39500000000000002,"5min":0.377,"15min":0.35899999999999999},"max":{"1min":7.4130000000000003,"5min":11.055999999999999,"15min":11.055999999999999},"last":1.018}]},{"osd":7,"last update":"Mon Aug 28 02:29:36 2023","interfaces":[{"interface":"back","average":{"1min":6.5789999999999997,"5min":5.3780000000000001,"15min":2.431},"min":{"1min":0.40400000000000003,"5min":0.34499999999999997,"15min":0.32100000000000001},"max":{"1min":100.65000000000001,"5min":128.911,"15min":128.911},"last":4.7030000000000003},{"interface":"front","average":{"1min":6.9000000000000004,"5min":5.9660000000000002,"15min":2.637},"min":{"1min":0.54400000000000004,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":101.53100000000001,"5min":128.72399999999999,"15min":128.72399999999999},"last":1.1100000000000001}]}]},{"osd":3,"up_from":112,"seq":481036337269,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":840812,"kb_used_data":583972,"kb_used_omap":3,"kb_used_meta":256764,"kb_avail":92904340,"statfs":{"total":95995035648,"available":95134044160,"internally_reserved":0,"allocated":597987328,"data_stored":596865403,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":262927266},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:30:05 2023","interfaces":[{"interface":"back","average":{"1min":7.0190000000000001,"5min":3.1560000000000001,"15min":3.0089999999999999},"min":{"1min":0.20000000000000001,"5min":0.193,"15min":0.193},"max":{"1min":79.793000000000006,"5min":79.793000000000006,"15min":164.25299999999999},"last":1.3420000000000001},{"interface":"front","average":{"1min":6.7839999999999998,"5min":2.8380000000000001,"15min":2.9620000000000002},"min":{"1min":0.27200000000000002,"5min":0.192,"15min":0.192},"max":{"1min":78.555000000000007,"5min":78.555000000000007,"15min":164.55000000000001},"last":3.702}]},{"osd":1,"last update":"Mon Aug 28 02:30:05 2023","interfaces":[{"interface":"back","average":{"1min":1.669,"5min":1.907,"15min":1.8680000000000001},"min":{"1min":0.245,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":6.1500000000000004,"5min":13.936,"15min":117.34699999999999},"last":0.51900000000000002},{"interface":"front","average":{"1min":2.5310000000000001,"5min":2.1499999999999999,"15min":1.968},"min":{"1min":0.27300000000000002,"5min":0.245,"15min":0.224},"max":{"1min":10.125,"5min":14.300000000000001,"15min":118.012},"last":1.236}]},{"osd":2,"last update":"Mon Aug 28 02:30:05 2023","interfaces":[{"interface":"back","average":{"1min":2.4199999999999999,"5min":3.157,"15min":2.8140000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.19900000000000001},"max":{"1min":10.163,"5min":65.308999999999997,"15min":120.43300000000001},"last":4.1980000000000004},{"interface":"front","average":{"1min":2.3999999999999999,"5min":2.891,"15min":2.7599999999999998},"min":{"1min":0.33500000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":7.3620000000000001,"5min":65.120999999999995,"15min":120.44499999999999},"last":3.7589999999999999}]},{"osd":4,"last update":"Mon Aug 28 02:30:05 2023","interfaces":[{"interface":"back","average":{"1min":1.403,"5min":2.4980000000000002,"15min":1.4610000000000001},"min":{"1min":0.32800000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":4.7240000000000002,"5min":58.067999999999998,"15min":58.067999999999998},"last":3.0819999999999999},{"interface":"front","average":{"1min":1.6279999999999999,"5min":2.7589999999999999,"15min":1.5269999999999999},"min":{"1min":0.40400000000000003,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":4.6280000000000001,"5min":58.088000000000001,"15min":58.088000000000001},"last":3.0470000000000002}]},{"osd":5,"last update":"Mon Aug 28 02:30:05 2023","interfaces":[{"interface":"back","average":{"1min":1.359,"5min":3.2109999999999999,"15min":1.6579999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.29599999999999999},"max":{"1min":2.923,"5min":102.904,"15min":102.904},"last":0.46999999999999997},{"interface":"front","average":{"1min":1.556,"5min":3.2509999999999999,"15min":1.6619999999999999},"min":{"1min":0.36699999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":7.0039999999999996,"5min":103.283,"15min":103.283},"last":1.276}]},{"osd":6,"last update":"Mon Aug 28 02:30:05 2023","interfaces":[{"interface":"back","average":{"1min":1.5880000000000001,"5min":1.877,"15min":1.4330000000000001},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.314},"max":{"1min":7.0300000000000002,"5min":9.4529999999999994,"15min":72.116},"last":1.3029999999999999},{"interface":"front","average":{"1min":1.7929999999999999,"5min":1.538,"15min":1.3560000000000001},"min":{"1min":0.35599999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":7.2990000000000004,"5min":7.2990000000000004,"15min":72.030000000000001},"last":2.1970000000000001}]},{"osd":7,"last update":"Mon Aug 28 02:30:05 2023","interfaces":[{"interface":"back","average":{"1min":1.6879999999999999,"5min":2.9660000000000002,"15min":1.631},"min":{"1min":0.38300000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":4.7640000000000002,"5min":54.427999999999997,"15min":54.427999999999997},"last":2.1749999999999998},{"interface":"front","average":{"1min":1.855,"5min":3.3399999999999999,"15min":1.76},"min":{"1min":0.433,"5min":0.36399999999999999,"15min":0.34799999999999998},"max":{"1min":3.8100000000000001,"5min":54.445,"15min":54.445},"last":0.496}]}]},{"osd":4,"up_from":30,"seq":128849019143,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":546200,"kb_used_data":248976,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":93198952,"statfs":{"total":95995035648,"available":95435726848,"internally_reserved":0,"allocated":254951424,"data_stored":254248314,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:30:10 2023","interfaces":[{"interface":"back","average":{"1min":1.528,"5min":2.8479999999999999,"15min":2.706},"min":{"1min":0.222,"5min":0.219,"15min":0.219},"max":{"1min":13.492000000000001,"5min":43.548000000000002,"15min":138.67400000000001},"last":7.4080000000000004},{"interface":"front","average":{"1min":1.4850000000000001,"5min":2.7149999999999999,"15min":2.637},"min":{"1min":0.251,"5min":0.248,"15min":0.248},"max":{"1min":8.0060000000000002,"5min":43.515000000000001,"15min":134.14599999999999},"last":2.9089999999999998}]},{"osd":1,"last update":"Mon Aug 28 02:30:10 2023","interfaces":[{"interface":"back","average":{"1min":1.5629999999999999,"5min":2.1640000000000001,"15min":1.3919999999999999},"min":{"1min":0.23599999999999999,"5min":0.221,"15min":0.221},"max":{"1min":9.75,"5min":34.945999999999998,"15min":53.622999999999998},"last":7.1429999999999998},{"interface":"front","average":{"1min":1.4039999999999999,"5min":2.073,"15min":1.4079999999999999},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":5.2919999999999998,"5min":34.869999999999997,"15min":53.774000000000001},"last":4.4400000000000004}]},{"osd":2,"last update":"Mon Aug 28 02:29:34 2023","interfaces":[{"interface":"back","average":{"1min":3.3130000000000002,"5min":4.3470000000000004,"15min":2.5369999999999999},"min":{"1min":0.27400000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":40.226999999999997,"5min":64.063000000000002,"15min":74.099000000000004},"last":3.5089999999999999},{"interface":"front","average":{"1min":3.4729999999999999,"5min":4.0979999999999999,"15min":2.4279999999999999},"min":{"1min":0.28999999999999998,"5min":0.28599999999999998,"15min":0.25},"max":{"1min":39.039000000000001,"5min":64.073999999999998,"15min":74.081999999999994},"last":3.206}]},{"osd":3,"last update":"Mon Aug 28 02:30:10 2023","interfaces":[{"interface":"back","average":{"1min":11.220000000000001,"5min":6.0819999999999999,"15min":2.5099999999999998},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":159.10400000000001,"5min":159.10400000000001,"15min":159.10400000000001},"last":7.8209999999999997},{"interface":"front","average":{"1min":11.119999999999999,"5min":5.9939999999999998,"15min":2.5979999999999999},"min":{"1min":0.33800000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":159.232,"5min":159.232,"15min":159.232},"last":1.821}]},{"osd":5,"last update":"Mon Aug 28 02:29:50 2023","interfaces":[{"interface":"back","average":{"1min":0.78100000000000003,"5min":2.8140000000000001,"15min":1.397},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":4.3090000000000002,"5min":124.718,"15min":124.718},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.73599999999999999,"5min":3.1949999999999998,"15min":1.554},"min":{"1min":0.27800000000000002,"5min":0.247,"15min":0.247},"max":{"1min":4.665,"5min":124.694,"15min":124.694},"last":3.4489999999999998}]},{"osd":6,"last update":"Mon Aug 28 02:30:10 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":1.264,"15min":0.94699999999999995},"min":{"1min":0.28499999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":2.6659999999999999,"5min":11.170999999999999,"15min":11.170999999999999},"last":6.9390000000000001},{"interface":"front","average":{"1min":0.91500000000000004,"5min":1.1180000000000001,"15min":0.83299999999999996},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":5.2679999999999998,"5min":7.3170000000000002,"15min":7.3170000000000002},"last":0.41399999999999998}]},{"osd":7,"last update":"Mon Aug 28 02:29:26 2023","interfaces":[{"interface":"back","average":{"1min":0.85099999999999998,"5min":1.734,"15min":1.044},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.5550000000000002,"5min":16.584,"15min":16.584},"last":3.4159999999999999},{"interface":"front","average":{"1min":0.51700000000000002,"5min":1.4199999999999999,"15min":1.0600000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":1.25,"5min":10.826000000000001,"15min":10.826000000000001},"last":6.9829999999999997}]}]},{"osd":5,"up_from":36,"seq":154618822915,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":592908,"kb_used_data":287492,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93152244,"statfs":{"total":95995035648,"available":95387897856,"internally_reserved":0,"allocated":294391808,"data_stored":293696165,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:30:02 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":2.052,"15min":2.6469999999999998},"min":{"1min":0.214,"5min":0.214,"15min":0.20300000000000001},"max":{"1min":4.548,"5min":42.229999999999997,"15min":131.06100000000001},"last":0.23400000000000001},{"interface":"front","average":{"1min":1.4710000000000001,"5min":2.371,"15min":2.8460000000000001},"min":{"1min":0.27500000000000002,"5min":0.24199999999999999,"15min":0.22900000000000001},"max":{"1min":5.8129999999999997,"5min":42.213999999999999,"15min":131.262},"last":3.4700000000000002}]},{"osd":1,"last update":"Mon Aug 28 02:30:12 2023","interfaces":[{"interface":"back","average":{"1min":1.8819999999999999,"5min":1.7869999999999999,"15min":2.387},"min":{"1min":0.27400000000000002,"5min":0.216,"15min":0.216},"max":{"1min":7.7809999999999997,"5min":30.593,"15min":183.97900000000001},"last":0.33500000000000002},{"interface":"front","average":{"1min":1.1220000000000001,"5min":2.0070000000000001,"15min":2.5510000000000002},"min":{"1min":0.23499999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":5.8810000000000002,"5min":34.616999999999997,"15min":184.33699999999999},"last":3.5089999999999999}]},{"osd":2,"last update":"Mon Aug 28 02:29:39 2023","interfaces":[{"interface":"back","average":{"1min":1.6240000000000001,"5min":6.2750000000000004,"15min":3.831},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.252},"max":{"1min":8.9600000000000009,"5min":176.69499999999999,"15min":176.69499999999999},"last":0.34599999999999997},{"interface":"front","average":{"1min":2.2440000000000002,"5min":6.5730000000000004,"15min":4.0019999999999998},"min":{"1min":0.34000000000000002,"5min":0.29799999999999999,"15min":0.28299999999999997},"max":{"1min":8.9489999999999998,"5min":176.71199999999999,"15min":176.71199999999999},"last":3.8889999999999998}]},{"osd":3,"last update":"Mon Aug 28 02:30:12 2023","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":2.8730000000000002,"15min":1.6180000000000001},"min":{"1min":0.28799999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.1070000000000002,"5min":102.64400000000001,"15min":102.64400000000001},"last":0.32100000000000001},{"interface":"front","average":{"1min":1.8919999999999999,"5min":3.6509999999999998,"15min":1.956},"min":{"1min":0.28699999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":8.1660000000000004,"5min":103.771,"15min":103.771},"last":2.1419999999999999}]},{"osd":4,"last update":"Mon Aug 28 02:29:50 2023","interfaces":[{"interface":"back","average":{"1min":1.8300000000000001,"5min":1.157,"15min":0.93100000000000005},"min":{"1min":0.26700000000000002,"5min":0.246,"15min":0.246},"max":{"1min":25.306999999999999,"5min":25.306999999999999,"15min":25.306999999999999},"last":0.36299999999999999},{"interface":"front","average":{"1min":1.7250000000000001,"5min":1.2430000000000001,"15min":0.95899999999999996},"min":{"1min":0.29399999999999998,"5min":0.28999999999999998,"15min":0.28000000000000003},"max":{"1min":25.347000000000001,"5min":25.347000000000001,"15min":25.347000000000001},"last":1.2310000000000001}]},{"osd":6,"last update":"Mon Aug 28 02:30:12 2023","interfaces":[{"interface":"back","average":{"1min":1.0429999999999999,"5min":1.099,"15min":1.002},"min":{"1min":0.30399999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.6210000000000004,"5min":5.9870000000000001,"15min":14.803000000000001},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.95499999999999996,"5min":1.04,"15min":0.91600000000000004},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":4.9509999999999996,"5min":8.4779999999999998,"15min":8.6479999999999997},"last":0.39600000000000002}]},{"osd":7,"last update":"Mon Aug 28 02:29:27 2023","interfaces":[{"interface":"back","average":{"1min":1.6399999999999999,"5min":1.6839999999999999,"15min":1.0860000000000001},"min":{"1min":0.34499999999999997,"5min":0.33300000000000002,"15min":0.31},"max":{"1min":25.379000000000001,"5min":25.379000000000001,"15min":25.379000000000001},"last":1.1619999999999999},{"interface":"front","average":{"1min":1.706,"5min":1.3879999999999999,"15min":1.004},"min":{"1min":0.36299999999999999,"5min":0.32200000000000001,"15min":0.32100000000000001},"max":{"1min":25.437000000000001,"5min":25.437000000000001,"15min":25.437000000000001},"last":1.7889999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":294981632,"data_stored":294977560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":457641984,"data_stored":457637912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":852164608,"data_stored":852164608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":590741504,"data_stored":590741504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":239075328,"data_stored":239075328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":291508224,"data_stored":291504152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":180158464,"data_stored":180158464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":662437888,"data_stored":662437888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-08-28T02:30:30.474 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph pg dump --format=json 2023-08-28T02:30:30.474 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:30:30.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:30.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:30:30.577 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:30:30.640 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:30 smithi143 ceph-mon[183059]: from='client.55168 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:30:30.640 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:30 smithi143 ceph-mon[183059]: pgmap v616: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 729 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 6078/34839 objects degraded (17.446%); 1489/34839 objects misplaced (4.274%); 1.6 MiB/s, 25 objects/s recovering 2023-08-28T02:30:30.641 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:30 smithi143 ceph-mon[189871]: from='client.55168 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:30:30.641 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:30 smithi143 ceph-mon[189871]: pgmap v616: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 729 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 6078/34839 objects degraded (17.446%); 1489/34839 objects misplaced (4.274%); 1.6 MiB/s, 25 objects/s recovering 2023-08-28T02:30:30.654 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:30.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:30 smithi169 ceph-mon[159415]: from='client.55168 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:30:30.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:30 smithi169 ceph-mon[159415]: pgmap v616: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 729 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 6078/34839 objects degraded (17.446%); 1489/34839 objects misplaced (4.274%); 1.6 MiB/s, 25 objects/s recovering 2023-08-28T02:30:30.805 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.4 dump_ops_in_flight 2023-08-28T02:30:30.859 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:30:30.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:30.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:30:30.968 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:30:31.082 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:31.188 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:30:31.224 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:30:31.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:31.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:30:31.273 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:30:31.299 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:30:31.331 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:30:31.374 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:31.507 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:30:31.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:31.510 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:30:31.547 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:31 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 6078/34839 objects degraded (17.446%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:30:31.554 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:31 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 6078/34839 objects degraded (17.446%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:30:31.611 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:30:31.636 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:31 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 6078/34839 objects degraded (17.446%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:30:31.671 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:31.780 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:30:31.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:31.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:30:31.886 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:30:31.934 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:30:31 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:30:31] "GET /metrics HTTP/1.1" 200 36484 "" "Prometheus/2.33.4" 2023-08-28T02:30:31.954 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:32.351 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:30:32.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:32.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:30:32.443 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is set 2023-08-28T02:30:32.453 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:32 smithi169 ceph-mon[159415]: pgmap v617: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 695 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 6078/33246 objects degraded (18.282%); 1489/33246 objects misplaced (4.479%); 1.4 MiB/s, 22 objects/s recovering 2023-08-28T02:30:32.454 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:32 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2486025301' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:30:32.454 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:32 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:30:32.458 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:30:32.554 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:32.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:32 smithi143 ceph-mon[183059]: pgmap v617: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 695 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 6078/33246 objects degraded (18.282%); 1489/33246 objects misplaced (4.479%); 1.4 MiB/s, 22 objects/s recovering 2023-08-28T02:30:32.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:32 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2486025301' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:30:32.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:32 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:30:32.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:32 smithi143 ceph-mon[189871]: pgmap v617: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 695 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 6078/33246 objects degraded (18.282%); 1489/33246 objects misplaced (4.479%); 1.4 MiB/s, 22 objects/s recovering 2023-08-28T02:30:32.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:32 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2486025301' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:30:32.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:32 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:30:33.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:33 smithi169 ceph-mon[159415]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:30:33.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:33 smithi169 ceph-mon[159415]: failed to encode map e256 with expected crc 2023-08-28T02:30:33.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:33 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:30:33.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:33 smithi169 ceph-mon[159415]: osdmap e256: 8 total, 8 up, 8 in 2023-08-28T02:30:33.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:33 smithi169 ceph-mon[159415]: failed to encode map e256 with expected crc 2023-08-28T02:30:33.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:33 smithi169 ceph-mon[159415]: failed to encode map e256 with expected crc 2023-08-28T02:30:33.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:33 smithi169 ceph-mon[159415]: failed to encode map e256 with expected crc 2023-08-28T02:30:33.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:33 smithi169 ceph-mon[159415]: failed to encode map e256 with expected crc 2023-08-28T02:30:33.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:33 smithi169 ceph-mon[159415]: failed to encode map e256 with expected crc 2023-08-28T02:30:33.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:33 smithi169 ceph-mon[159415]: failed to encode map e256 with expected crc 2023-08-28T02:30:33.889 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:30:33.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:33.891 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:33 smithi143 ceph-mon[183059]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:30:33.891 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:33 smithi143 ceph-mon[183059]: failed to encode map e256 with expected crc 2023-08-28T02:30:33.891 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:33 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:30:33.891 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:33 smithi143 ceph-mon[183059]: osdmap e256: 8 total, 8 up, 8 in 2023-08-28T02:30:33.891 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:33 smithi143 ceph-mon[183059]: failed to encode map e256 with expected crc 2023-08-28T02:30:33.891 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:33 smithi143 ceph-mon[183059]: failed to encode map e256 with expected crc 2023-08-28T02:30:33.891 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:33 smithi143 ceph-mon[183059]: failed to encode map e256 with expected crc 2023-08-28T02:30:33.891 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:33 smithi143 ceph-mon[183059]: failed to encode map e256 with expected crc 2023-08-28T02:30:33.892 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:33 smithi143 ceph-mon[183059]: failed to encode map e256 with expected crc 2023-08-28T02:30:33.892 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:33 smithi143 ceph-mon[183059]: failed to encode map e256 with expected crc 2023-08-28T02:30:33.892 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:33 smithi143 ceph-mon[189871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:30:33.892 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:33 smithi143 ceph-mon[189871]: failed to encode map e256 with expected crc 2023-08-28T02:30:33.892 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:33 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:30:33.892 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:33 smithi143 ceph-mon[189871]: osdmap e256: 8 total, 8 up, 8 in 2023-08-28T02:30:33.892 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:33 smithi143 ceph-mon[189871]: failed to encode map e256 with expected crc 2023-08-28T02:30:33.892 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:33 smithi143 ceph-mon[189871]: failed to encode map e256 with expected crc 2023-08-28T02:30:33.892 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:33 smithi143 ceph-mon[189871]: failed to encode map e256 with expected crc 2023-08-28T02:30:33.892 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:33 smithi143 ceph-mon[189871]: failed to encode map e256 with expected crc 2023-08-28T02:30:33.893 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:33 smithi143 ceph-mon[189871]: failed to encode map e256 with expected crc 2023-08-28T02:30:33.893 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:33 smithi143 ceph-mon[189871]: failed to encode map e256 with expected crc 2023-08-28T02:30:33.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:30:33.994 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:30:34.082 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:34.266 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:30:34.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:34.274 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:30:34.375 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:30:34.475 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:34.629 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:30:34.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:34.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:30:34.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:34 smithi143 ceph-mon[183059]: pgmap v619: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 591 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 6088/28212 objects degraded (21.579%); 1363/28212 objects misplaced (4.831%); 1.4 MiB/s, 23 objects/s recovering 2023-08-28T02:30:34.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:34 smithi143 ceph-mon[189871]: pgmap v619: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 591 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 6088/28212 objects degraded (21.579%); 1363/28212 objects misplaced (4.831%); 1.4 MiB/s, 23 objects/s recovering 2023-08-28T02:30:34.737 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:30:34.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:34 smithi169 ceph-mon[159415]: pgmap v619: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 591 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 6088/28212 objects degraded (21.579%); 1363/28212 objects misplaced (4.831%); 1.4 MiB/s, 23 objects/s recovering 2023-08-28T02:30:34.742 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:30:34.780 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:30:34.819 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:30:34.820 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:34.864 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:30:34.910 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:30:34.951 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:30:34.957 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:30:34.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:34.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:30:34.995 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:30:35.027 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:30:35.030 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:30:35.031 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:30:35.031 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:30:35.031 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:30:35.031 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:30:35.031 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:30:35.031 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:30:35.031 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:30:35.070 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:30:35.111 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:30:35.113 INFO:teuthology.orchestra.run.smithi143.stderr:dumped all 2023-08-28T02:30:35.126 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:35.225 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:30:35.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:35.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:30:35.328 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:30:35.383 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:35.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:35 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3108135818' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:30:35.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:35 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3108135818' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:30:35.726 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:30:35.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:35.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:30:35.742 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:35 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3108135818' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:30:35.830 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:30:35.918 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:36.191 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset noscrub 2023-08-28T02:30:36.354 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:30:36.354 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:30:36.608 INFO:teuthology.orchestra.run.smithi143.stdout:{"pg_ready":true,"pg_map":{"version":619,"stamp":"2023-08-28T02:30:33.304729+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":619190219,"num_objects":9404,"num_object_clones":0,"num_object_copies":28212,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6088,"num_objects_misplaced":1363,"num_objects_unfound":0,"num_objects_dirty":9404,"num_whiteouts":0,"num_read":6275,"num_read_kb":13708,"num_write":158964,"num_write_kb":5366566,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20391,"num_bytes_recovered":1177441353,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":184566,"ondisk_log_size":184566,"up":297,"acting":293,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":310,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5212468,"kb_used_data":2813620,"kb_used_omap":29,"kb_used_meta":2398626,"kb_avail":744748748,"statfs":{"total":767960285184,"available":762622717952,"internally_reserved":0,"allocated":2881146880,"data_stored":2873647469,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30674,"internal_metadata":2456193070},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,4,1,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-530120704,"num_objects":-8089,"num_object_clones":0,"num_object_copies":-24267,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":-1317,"num_objects_unfound":0,"num_objects_dirty":-8089,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8089,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":232,"num_bytes_recovered":15204352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003000"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":256,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.172817+0000","last_change":"2023-08-28T02:27:31.367469+0000","last_active":"2023-08-28T02:30:20.172817+0000","last_peered":"2023-08-28T02:30:20.172817+0000","last_clean":"2023-08-28T02:30:20.172817+0000","last_became_active":"2023-08-28T02:27:31.366946+0000","last_became_peered":"2023-08-28T02:27:31.366946+0000","last_unstale":"2023-08-28T02:30:20.172817+0000","last_undegraded":"2023-08-28T02:30:20.172817+0000","last_fullsized":"2023-08-28T02:30:20.172817+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":141,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.163141+0000","last_change":"2023-08-28T02:27:31.324452+0000","last_active":"2023-08-28T02:30:20.163141+0000","last_peered":"2023-08-28T02:30:20.163141+0000","last_clean":"2023-08-28T02:30:20.163141+0000","last_became_active":"2023-08-28T02:27:31.323749+0000","last_became_peered":"2023-08-28T02:27:31.323749+0000","last_unstale":"2023-08-28T02:30:20.163141+0000","last_undegraded":"2023-08-28T02:30:20.163141+0000","last_fullsized":"2023-08-28T02:30:20.163141+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:18:10.904127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":62,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.158364+0000","last_change":"2023-08-28T02:27:31.337338+0000","last_active":"2023-08-28T02:30:20.158364+0000","last_peered":"2023-08-28T02:30:20.158364+0000","last_clean":"2023-08-28T02:30:20.158364+0000","last_became_active":"2023-08-28T02:27:31.337018+0000","last_became_peered":"2023-08-28T02:27:31.337018+0000","last_unstale":"2023-08-28T02:30:20.158364+0000","last_undegraded":"2023-08-28T02:30:20.158364+0000","last_fullsized":"2023-08-28T02:30:20.158364+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:46:34.845663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":242,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.147929+0000","last_change":"2023-08-28T02:23:59.223634+0000","last_active":"2023-08-28T02:30:20.147929+0000","last_peered":"2023-08-28T02:30:20.147929+0000","last_clean":"2023-08-28T02:30:20.147929+0000","last_became_active":"2023-08-28T02:23:59.222704+0000","last_became_peered":"2023-08-28T02:23:59.222704+0000","last_unstale":"2023-08-28T02:30:20.147929+0000","last_undegraded":"2023-08-28T02:30:20.147929+0000","last_fullsized":"2023-08-28T02:30:20.147929+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:51:52.752361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":132,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.162592+0000","last_change":"2023-08-28T02:27:31.318153+0000","last_active":"2023-08-28T02:30:20.162592+0000","last_peered":"2023-08-28T02:30:20.162592+0000","last_clean":"2023-08-28T02:30:20.162592+0000","last_became_active":"2023-08-28T02:27:31.317732+0000","last_became_peered":"2023-08-28T02:27:31.317732+0000","last_unstale":"2023-08-28T02:30:20.162592+0000","last_undegraded":"2023-08-28T02:30:20.162592+0000","last_fullsized":"2023-08-28T02:30:20.162592+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:06:13.421674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":44,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.160159+0000","last_change":"2023-08-28T02:28:25.348941+0000","last_active":"2023-08-28T02:30:20.160159+0000","last_peered":"2023-08-28T02:30:20.160159+0000","last_clean":"2023-08-28T02:30:20.160159+0000","last_became_active":"2023-08-28T02:28:25.348650+0000","last_became_peered":"2023-08-28T02:28:25.348650+0000","last_unstale":"2023-08-28T02:30:20.160159+0000","last_undegraded":"2023-08-28T02:30:20.160159+0000","last_fullsized":"2023-08-28T02:30:20.160159+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.12","version":"255'6821","reported_seq":6909,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.137246+0000","last_change":"2023-08-28T02:28:25.453536+0000","last_active":"2023-08-28T02:30:32.137246+0000","last_peered":"2023-08-28T02:30:32.137246+0000","last_clean":"2023-08-28T02:30:32.137246+0000","last_became_active":"2023-08-28T02:27:58.479936+0000","last_became_peered":"2023-08-28T02:27:58.479936+0000","last_unstale":"2023-08-28T02:30:32.137246+0000","last_undegraded":"2023-08-28T02:30:32.137246+0000","last_fullsized":"2023-08-28T02:30:32.137246+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6821,"log_dups_size":0,"ondisk_log_size":6821,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18087936,"num_objects":276,"num_object_clones":0,"num_object_copies":828,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":276,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4956,"num_write_kb":167424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":222,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.148309+0000","last_change":"2023-08-28T02:23:59.227592+0000","last_active":"2023-08-28T02:30:20.148309+0000","last_peered":"2023-08-28T02:30:20.148309+0000","last_clean":"2023-08-28T02:30:20.148309+0000","last_became_active":"2023-08-28T02:23:59.227342+0000","last_became_peered":"2023-08-28T02:23:59.227342+0000","last_unstale":"2023-08-28T02:30:20.148309+0000","last_undegraded":"2023-08-28T02:30:20.148309+0000","last_fullsized":"2023-08-28T02:30:20.148309+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:08:19.874396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":150,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.162666+0000","last_change":"2023-08-28T02:28:25.348233+0000","last_active":"2023-08-28T02:30:20.162666+0000","last_peered":"2023-08-28T02:30:20.162666+0000","last_clean":"2023-08-28T02:30:20.162666+0000","last_became_active":"2023-08-28T02:28:25.347896+0000","last_became_peered":"2023-08-28T02:28:25.347896+0000","last_unstale":"2023-08-28T02:30:20.162666+0000","last_undegraded":"2023-08-28T02:30:20.162666+0000","last_fullsized":"2023-08-28T02:30:20.162666+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:50:17.674564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.13","version":"255'6949","reported_seq":7055,"reported_epoch":255,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:30:32.129603+0000","last_change":"2023-08-28T02:28:26.468919+0000","last_active":"2023-08-28T02:30:32.129603+0000","last_peered":"2023-08-28T02:30:32.129603+0000","last_clean":"2023-08-28T02:27:16.315150+0000","last_became_active":"2023-08-28T02:28:26.463939+0000","last_became_peered":"2023-08-28T02:28:26.463939+0000","last_unstale":"2023-08-28T02:30:32.129603+0000","last_undegraded":"2023-08-28T02:28:26.355850+0000","last_fullsized":"2023-08-28T02:28:26.347877+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6949,"log_dups_size":0,"ondisk_log_size":6949,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16580608,"num_objects":253,"num_object_clones":0,"num_object_copies":759,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1703,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":253,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5061,"num_write_kb":170048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"1,2,7","objects":85},{"shards":"2,7","objects":-1450}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":257,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.157339+0000","last_change":"2023-08-28T02:27:31.328499+0000","last_active":"2023-08-28T02:30:20.157339+0000","last_peered":"2023-08-28T02:30:20.157339+0000","last_clean":"2023-08-28T02:30:20.157339+0000","last_became_active":"2023-08-28T02:27:31.328164+0000","last_became_peered":"2023-08-28T02:27:31.328164+0000","last_unstale":"2023-08-28T02:30:20.157339+0000","last_undegraded":"2023-08-28T02:30:20.157339+0000","last_fullsized":"2023-08-28T02:30:20.157339+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":132,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.162796+0000","last_change":"2023-08-28T02:28:25.348760+0000","last_active":"2023-08-28T02:30:20.162796+0000","last_peered":"2023-08-28T02:30:20.162796+0000","last_clean":"2023-08-28T02:30:20.162796+0000","last_became_active":"2023-08-28T02:28:25.348465+0000","last_became_peered":"2023-08-28T02:28:25.348465+0000","last_unstale":"2023-08-28T02:30:20.162796+0000","last_undegraded":"2023-08-28T02:30:20.162796+0000","last_fullsized":"2023-08-28T02:30:20.162796+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:43:05.659999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.10","version":"255'6635","reported_seq":6724,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-08-28T02:30:28.878445+0000","last_change":"2023-08-28T02:30:17.869105+0000","last_active":"2023-08-28T02:30:28.878445+0000","last_peered":"2023-08-28T02:30:28.878445+0000","last_clean":"2023-08-28T02:30:28.878445+0000","last_became_active":"2023-08-28T02:27:58.494108+0000","last_became_peered":"2023-08-28T02:27:58.494108+0000","last_unstale":"2023-08-28T02:30:28.878445+0000","last_undegraded":"2023-08-28T02:30:28.878445+0000","last_fullsized":"2023-08-28T02:30:28.878445+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6635,"log_dups_size":0,"ondisk_log_size":6635,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22151168,"num_objects":338,"num_object_clones":0,"num_object_copies":1014,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":338,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4798,"num_write_kb":164352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":140,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.168856+0000","last_change":"2023-08-28T02:23:59.316561+0000","last_active":"2023-08-28T02:30:20.168856+0000","last_peered":"2023-08-28T02:30:20.168856+0000","last_clean":"2023-08-28T02:30:20.168856+0000","last_became_active":"2023-08-28T02:23:59.227433+0000","last_became_peered":"2023-08-28T02:23:59.227433+0000","last_unstale":"2023-08-28T02:30:20.168856+0000","last_undegraded":"2023-08-28T02:30:20.168856+0000","last_fullsized":"2023-08-28T02:30:20.168856+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":72,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.157855+0000","last_change":"2023-08-28T02:28:25.347771+0000","last_active":"2023-08-28T02:30:20.157855+0000","last_peered":"2023-08-28T02:30:20.157855+0000","last_clean":"2023-08-28T02:30:20.157855+0000","last_became_active":"2023-08-28T02:28:25.347320+0000","last_became_peered":"2023-08-28T02:28:25.347320+0000","last_unstale":"2023-08-28T02:30:20.157855+0000","last_undegraded":"2023-08-28T02:30:20.157855+0000","last_fullsized":"2023-08-28T02:30:20.157855+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:20:27.350131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.11","version":"255'6813","reported_seq":13730,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-08-28T02:30:29.526175+0000","last_change":"2023-08-28T02:30:11.322180+0000","last_active":"2023-08-28T02:30:29.526175+0000","last_peered":"2023-08-28T02:30:29.526175+0000","last_clean":"2023-08-28T02:30:29.526175+0000","last_became_active":"2023-08-28T02:29:58.475435+0000","last_became_peered":"2023-08-28T02:29:58.475435+0000","last_unstale":"2023-08-28T02:30:29.526175+0000","last_undegraded":"2023-08-28T02:30:29.526175+0000","last_fullsized":"2023-08-28T02:30:29.526175+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6813,"log_dups_size":0,"ondisk_log_size":6813,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23199744,"num_objects":354,"num_object_clones":0,"num_object_copies":1062,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":354,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4938,"num_write_kb":169344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2439,"num_bytes_recovered":159678464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"86'22","reported_seq":305,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.173493+0000","last_change":"2023-08-28T02:28:25.361783+0000","last_active":"2023-08-28T02:30:20.173493+0000","last_peered":"2023-08-28T02:30:20.173493+0000","last_clean":"2023-08-28T02:30:20.173493+0000","last_became_active":"2023-08-28T02:28:25.351418+0000","last_became_peered":"2023-08-28T02:28:25.351418+0000","last_unstale":"2023-08-28T02:30:20.173493+0000","last_undegraded":"2023-08-28T02:30:20.173493+0000","last_fullsized":"2023-08-28T02:30:20.173493+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":271,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.172185+0000","last_change":"2023-08-28T02:28:25.352395+0000","last_active":"2023-08-28T02:30:20.172185+0000","last_peered":"2023-08-28T02:30:20.172185+0000","last_clean":"2023-08-28T02:30:20.172185+0000","last_became_active":"2023-08-28T02:28:25.352189+0000","last_became_peered":"2023-08-28T02:28:25.352189+0000","last_unstale":"2023-08-28T02:30:20.172185+0000","last_undegraded":"2023-08-28T02:30:20.172185+0000","last_fullsized":"2023-08-28T02:30:20.172185+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.16","version":"255'6753","reported_seq":6841,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-08-28T02:30:29.111027+0000","last_change":"2023-08-28T02:27:58.492359+0000","last_active":"2023-08-28T02:30:29.111027+0000","last_peered":"2023-08-28T02:30:29.111027+0000","last_clean":"2023-08-28T02:30:29.111027+0000","last_became_active":"2023-08-28T02:27:58.479842+0000","last_became_peered":"2023-08-28T02:27:58.479842+0000","last_unstale":"2023-08-28T02:30:29.111027+0000","last_undegraded":"2023-08-28T02:30:29.111027+0000","last_fullsized":"2023-08-28T02:30:29.111027+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6753,"log_dups_size":0,"ondisk_log_size":6753,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19628032,"num_objects":300,"num_object_clones":0,"num_object_copies":900,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":300,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4868,"num_write_kb":165344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":125,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.163803+0000","last_change":"2023-08-28T02:23:38.983634+0000","last_active":"2023-08-28T02:30:20.163803+0000","last_peered":"2023-08-28T02:30:20.163803+0000","last_clean":"2023-08-28T02:30:20.163803+0000","last_became_active":"2023-08-28T02:23:38.344566+0000","last_became_peered":"2023-08-28T02:23:38.344566+0000","last_unstale":"2023-08-28T02:30:20.163803+0000","last_undegraded":"2023-08-28T02:30:20.163803+0000","last_fullsized":"2023-08-28T02:30:20.163803+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":132,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.163484+0000","last_change":"2023-08-28T02:27:31.344974+0000","last_active":"2023-08-28T02:30:20.163484+0000","last_peered":"2023-08-28T02:30:20.163484+0000","last_clean":"2023-08-28T02:30:20.163484+0000","last_became_active":"2023-08-28T02:27:31.344281+0000","last_became_peered":"2023-08-28T02:27:31.344281+0000","last_unstale":"2023-08-28T02:30:20.163484+0000","last_undegraded":"2023-08-28T02:30:20.163484+0000","last_fullsized":"2023-08-28T02:30:20.163484+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:04:02.821752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.17","version":"255'6769","reported_seq":6854,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.238792+0000","last_change":"2023-08-28T02:27:58.474771+0000","last_active":"2023-08-28T02:30:32.238792+0000","last_peered":"2023-08-28T02:30:32.238792+0000","last_clean":"2023-08-28T02:30:32.238792+0000","last_became_active":"2023-08-28T02:27:58.472976+0000","last_became_peered":"2023-08-28T02:27:58.472976+0000","last_unstale":"2023-08-28T02:30:32.238792+0000","last_undegraded":"2023-08-28T02:30:32.238792+0000","last_fullsized":"2023-08-28T02:30:32.238792+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6769,"log_dups_size":0,"ondisk_log_size":6769,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15794176,"num_objects":241,"num_object_clones":0,"num_object_copies":723,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":241,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4913,"num_write_kb":164928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":241,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.148668+0000","last_change":"2023-08-28T02:23:59.223066+0000","last_active":"2023-08-28T02:30:20.148668+0000","last_peered":"2023-08-28T02:30:20.148668+0000","last_clean":"2023-08-28T02:30:20.148668+0000","last_became_active":"2023-08-28T02:23:59.222731+0000","last_became_peered":"2023-08-28T02:23:59.222731+0000","last_unstale":"2023-08-28T02:30:20.148668+0000","last_undegraded":"2023-08-28T02:30:20.148668+0000","last_fullsized":"2023-08-28T02:30:20.148668+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:48:37.558266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":281,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.157322+0000","last_change":"2023-08-28T02:23:59.210469+0000","last_active":"2023-08-28T02:30:20.157322+0000","last_peered":"2023-08-28T02:30:20.157322+0000","last_clean":"2023-08-28T02:30:20.157322+0000","last_became_active":"2023-08-28T02:23:59.210031+0000","last_became_peered":"2023-08-28T02:23:59.210031+0000","last_unstale":"2023-08-28T02:30:20.157322+0000","last_undegraded":"2023-08-28T02:30:20.157322+0000","last_fullsized":"2023-08-28T02:30:20.157322+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.14","version":"255'6825","reported_seq":12342,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-08-28T02:30:29.465193+0000","last_change":"2023-08-28T02:30:10.332493+0000","last_active":"2023-08-28T02:30:29.465193+0000","last_peered":"2023-08-28T02:30:29.465193+0000","last_clean":"2023-08-28T02:30:29.465193+0000","last_became_active":"2023-08-28T02:28:00.020960+0000","last_became_peered":"2023-08-28T02:28:00.020960+0000","last_unstale":"2023-08-28T02:30:29.465193+0000","last_undegraded":"2023-08-28T02:30:29.465193+0000","last_fullsized":"2023-08-28T02:30:29.465193+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6825,"log_dups_size":0,"ondisk_log_size":6825,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22708224,"num_objects":347,"num_object_clones":0,"num_object_copies":1041,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":347,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4993,"num_write_kb":170848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1961,"num_bytes_recovered":128483328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":137,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.163329+0000","last_change":"2023-08-28T02:28:25.354526+0000","last_active":"2023-08-28T02:30:20.163329+0000","last_peered":"2023-08-28T02:30:20.163329+0000","last_clean":"2023-08-28T02:30:20.163329+0000","last_became_active":"2023-08-28T02:28:25.350991+0000","last_became_peered":"2023-08-28T02:28:25.350991+0000","last_unstale":"2023-08-28T02:30:20.163329+0000","last_undegraded":"2023-08-28T02:30:20.163329+0000","last_fullsized":"2023-08-28T02:30:20.163329+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":241,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.148001+0000","last_change":"2023-08-28T02:23:59.220932+0000","last_active":"2023-08-28T02:30:20.148001+0000","last_peered":"2023-08-28T02:30:20.148001+0000","last_clean":"2023-08-28T02:30:20.148001+0000","last_became_active":"2023-08-28T02:23:59.220488+0000","last_became_peered":"2023-08-28T02:23:59.220488+0000","last_unstale":"2023-08-28T02:30:20.148001+0000","last_undegraded":"2023-08-28T02:30:20.148001+0000","last_fullsized":"2023-08-28T02:30:20.148001+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:53:09.216340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.15","version":"255'6904","reported_seq":8181,"reported_epoch":255,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:30:32.039049+0000","last_change":"2023-08-28T02:28:25.456367+0000","last_active":"2023-08-28T02:30:32.039049+0000","last_peered":"2023-08-28T02:30:32.039049+0000","last_clean":"2023-08-28T02:27:16.315756+0000","last_became_active":"2023-08-28T02:28:25.449323+0000","last_became_peered":"2023-08-28T02:28:25.449323+0000","last_unstale":"2023-08-28T02:30:32.039049+0000","last_undegraded":"2023-08-28T02:28:25.349175+0000","last_fullsized":"2023-08-28T02:28:25.327058+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6904,"log_dups_size":0,"ondisk_log_size":6904,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15466496,"num_objects":236,"num_object_clones":0,"num_object_copies":708,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1690,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":236,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5020,"num_write_kb":168192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":294,"num_bytes_recovered":19267584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"2,3,7","objects":82},{"shards":"3,7","objects":-1454}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":54,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.159813+0000","last_change":"2023-08-28T02:28:25.703333+0000","last_active":"2023-08-28T02:30:20.159813+0000","last_peered":"2023-08-28T02:30:20.159813+0000","last_clean":"2023-08-28T02:30:20.159813+0000","last_became_active":"2023-08-28T02:28:25.343515+0000","last_became_peered":"2023-08-28T02:28:25.343515+0000","last_unstale":"2023-08-28T02:30:20.159813+0000","last_undegraded":"2023-08-28T02:30:20.159813+0000","last_fullsized":"2023-08-28T02:30:20.159813+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":132,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.164059+0000","last_change":"2023-08-28T02:27:31.330059+0000","last_active":"2023-08-28T02:30:20.164059+0000","last_peered":"2023-08-28T02:30:20.164059+0000","last_clean":"2023-08-28T02:30:20.164059+0000","last_became_active":"2023-08-28T02:27:31.329769+0000","last_became_peered":"2023-08-28T02:27:31.329769+0000","last_unstale":"2023-08-28T02:30:20.164059+0000","last_undegraded":"2023-08-28T02:30:20.164059+0000","last_fullsized":"2023-08-28T02:30:20.164059+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:14:54.349909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"255'9837","reported_seq":13049,"reported_epoch":255,"state":"active+remapped+backfill_wait","last_fresh":"2023-08-28T02:30:28.910320+0000","last_change":"2023-08-28T02:29:15.939050+0000","last_active":"2023-08-28T02:30:28.910320+0000","last_peered":"2023-08-28T02:30:28.910320+0000","last_clean":"2023-08-28T02:28:24.280158+0000","last_became_active":"2023-08-28T02:28:26.465041+0000","last_became_peered":"2023-08-28T02:28:26.465041+0000","last_unstale":"2023-08-28T02:30:28.910320+0000","last_undegraded":"2023-08-28T02:30:28.910320+0000","last_fullsized":"2023-08-28T02:30:28.910320+0000","mapping_epoch":221,"log_start":"195'1200","ondisk_log_start":"195'1200","created":187,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":8637,"log_dups_size":0,"ondisk_log_size":8637,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:30:34.822957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44367872,"num_objects":677,"num_object_clones":0,"num_object_copies":2031,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":677,"num_objects_unfound":0,"num_objects_dirty":677,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9837,"num_write_kb":336448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1547,"num_bytes_recovered":101384192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[0,5,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":677}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":123,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.168859+0000","last_change":"2023-08-28T02:23:38.657185+0000","last_active":"2023-08-28T02:30:20.168859+0000","last_peered":"2023-08-28T02:30:20.168859+0000","last_clean":"2023-08-28T02:30:20.168859+0000","last_became_active":"2023-08-28T02:23:38.339016+0000","last_became_peered":"2023-08-28T02:23:38.339016+0000","last_unstale":"2023-08-28T02:30:20.168859+0000","last_undegraded":"2023-08-28T02:30:20.168859+0000","last_fullsized":"2023-08-28T02:30:20.168859+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":241,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.156027+0000","last_change":"2023-08-28T02:28:25.348106+0000","last_active":"2023-08-28T02:30:20.156027+0000","last_peered":"2023-08-28T02:30:20.156027+0000","last_clean":"2023-08-28T02:30:20.156027+0000","last_became_active":"2023-08-28T02:28:25.347786+0000","last_became_peered":"2023-08-28T02:28:25.347786+0000","last_unstale":"2023-08-28T02:30:20.156027+0000","last_undegraded":"2023-08-28T02:30:20.156027+0000","last_fullsized":"2023-08-28T02:30:20.156027+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"255'9822","reported_seq":14755,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-08-28T02:30:29.196197+0000","last_change":"2023-08-28T02:29:38.520992+0000","last_active":"2023-08-28T02:30:29.196197+0000","last_peered":"2023-08-28T02:30:29.196197+0000","last_clean":"2023-08-28T02:30:29.196197+0000","last_became_active":"2023-08-28T02:29:38.501413+0000","last_became_peered":"2023-08-28T02:29:38.501413+0000","last_unstale":"2023-08-28T02:30:29.196197+0000","last_undegraded":"2023-08-28T02:30:29.196197+0000","last_fullsized":"2023-08-28T02:30:29.196197+0000","mapping_epoch":242,"log_start":"207'3500","ondisk_log_start":"207'3500","created":187,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6322,"log_dups_size":0,"ondisk_log_size":6322,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:45:22.978036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44040192,"num_objects":672,"num_object_clones":0,"num_object_copies":2016,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":672,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9822,"num_write_kb":335808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2408,"num_bytes_recovered":149291008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":149,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.169203+0000","last_change":"2023-08-28T02:27:31.395328+0000","last_active":"2023-08-28T02:30:20.169203+0000","last_peered":"2023-08-28T02:30:20.169203+0000","last_clean":"2023-08-28T02:30:20.169203+0000","last_became_active":"2023-08-28T02:27:31.328228+0000","last_became_peered":"2023-08-28T02:27:31.328228+0000","last_unstale":"2023-08-28T02:30:20.169203+0000","last_undegraded":"2023-08-28T02:30:20.169203+0000","last_fullsized":"2023-08-28T02:30:20.169203+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":290,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.172385+0000","last_change":"2023-08-28T02:27:31.344915+0000","last_active":"2023-08-28T02:30:20.172385+0000","last_peered":"2023-08-28T02:30:20.172385+0000","last_clean":"2023-08-28T02:30:20.172385+0000","last_became_active":"2023-08-28T02:27:31.344577+0000","last_became_peered":"2023-08-28T02:27:31.344577+0000","last_unstale":"2023-08-28T02:30:20.172385+0000","last_undegraded":"2023-08-28T02:30:20.172385+0000","last_fullsized":"2023-08-28T02:30:20.172385+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"255'6921","reported_seq":7007,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-08-28T02:30:31.987986+0000","last_change":"2023-08-28T02:27:58.457913+0000","last_active":"2023-08-28T02:30:31.987986+0000","last_peered":"2023-08-28T02:30:31.987986+0000","last_clean":"2023-08-28T02:30:31.987986+0000","last_became_active":"2023-08-28T02:27:58.451333+0000","last_became_peered":"2023-08-28T02:27:58.451333+0000","last_unstale":"2023-08-28T02:30:31.987986+0000","last_undegraded":"2023-08-28T02:30:31.987986+0000","last_fullsized":"2023-08-28T02:30:31.987986+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6921,"log_dups_size":0,"ondisk_log_size":6921,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12091392,"num_objects":184,"num_object_clones":0,"num_object_copies":552,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":184,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5044,"num_write_kb":167328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":149,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.169056+0000","last_change":"2023-08-28T02:28:25.749249+0000","last_active":"2023-08-28T02:30:20.169056+0000","last_peered":"2023-08-28T02:30:20.169056+0000","last_clean":"2023-08-28T02:30:20.169056+0000","last_became_active":"2023-08-28T02:28:25.397684+0000","last_became_peered":"2023-08-28T02:28:25.397684+0000","last_unstale":"2023-08-28T02:30:20.169056+0000","last_undegraded":"2023-08-28T02:30:20.169056+0000","last_fullsized":"2023-08-28T02:30:20.169056+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":44,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.159920+0000","last_change":"2023-08-28T02:28:25.350802+0000","last_active":"2023-08-28T02:30:20.159920+0000","last_peered":"2023-08-28T02:30:20.159920+0000","last_clean":"2023-08-28T02:30:20.159920+0000","last_became_active":"2023-08-28T02:28:25.350474+0000","last_became_peered":"2023-08-28T02:28:25.350474+0000","last_unstale":"2023-08-28T02:30:20.159920+0000","last_undegraded":"2023-08-28T02:30:20.159920+0000","last_fullsized":"2023-08-28T02:30:20.159920+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"255'6679","reported_seq":6770,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-08-28T02:30:28.702300+0000","last_change":"2023-08-28T02:30:19.887172+0000","last_active":"2023-08-28T02:30:28.702300+0000","last_peered":"2023-08-28T02:30:28.702300+0000","last_clean":"2023-08-28T02:30:28.702300+0000","last_became_active":"2023-08-28T02:27:58.433161+0000","last_became_peered":"2023-08-28T02:27:58.433161+0000","last_unstale":"2023-08-28T02:30:28.702300+0000","last_undegraded":"2023-08-28T02:30:28.702300+0000","last_fullsized":"2023-08-28T02:30:28.702300+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6679,"log_dups_size":0,"ondisk_log_size":6679,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22773760,"num_objects":347,"num_object_clones":0,"num_object_copies":1041,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":347,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4809,"num_write_kb":165024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":257,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-08-28T02:30:21.180721+0000","last_change":"2023-08-28T02:28:25.445982+0000","last_active":"2023-08-28T02:30:21.180721+0000","last_peered":"2023-08-28T02:30:21.180721+0000","last_clean":"2023-08-28T02:30:21.180721+0000","last_became_active":"2023-08-28T02:28:25.443577+0000","last_became_peered":"2023-08-28T02:28:25.443577+0000","last_unstale":"2023-08-28T02:30:21.180721+0000","last_undegraded":"2023-08-28T02:30:21.180721+0000","last_fullsized":"2023-08-28T02:30:21.180721+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:24:03.381751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":235,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.150195+0000","last_change":"2023-08-28T02:27:31.319783+0000","last_active":"2023-08-28T02:30:20.150195+0000","last_peered":"2023-08-28T02:30:20.150195+0000","last_clean":"2023-08-28T02:30:20.150195+0000","last_became_active":"2023-08-28T02:27:31.319451+0000","last_became_peered":"2023-08-28T02:27:31.319451+0000","last_unstale":"2023-08-28T02:30:20.150195+0000","last_undegraded":"2023-08-28T02:30:20.150195+0000","last_fullsized":"2023-08-28T02:30:20.150195+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:20:37.097889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"255'9849","reported_seq":14770,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.229149+0000","last_change":"2023-08-28T02:29:17.383106+0000","last_active":"2023-08-28T02:30:32.229149+0000","last_peered":"2023-08-28T02:30:32.229149+0000","last_clean":"2023-08-28T02:30:32.229149+0000","last_became_active":"2023-08-28T02:29:17.367223+0000","last_became_peered":"2023-08-28T02:29:17.367223+0000","last_unstale":"2023-08-28T02:30:32.229149+0000","last_undegraded":"2023-08-28T02:30:32.229149+0000","last_fullsized":"2023-08-28T02:30:32.229149+0000","mapping_epoch":234,"log_start":"200'1900","ondisk_log_start":"200'1900","created":187,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":7949,"log_dups_size":0,"ondisk_log_size":7949,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31916032,"num_objects":487,"num_object_clones":0,"num_object_copies":1461,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":487,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9849,"num_write_kb":330752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2409,"num_bytes_recovered":127533056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":118,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.163206+0000","last_change":"2023-08-28T02:23:59.230059+0000","last_active":"2023-08-28T02:30:20.163206+0000","last_peered":"2023-08-28T02:30:20.163206+0000","last_clean":"2023-08-28T02:30:20.163206+0000","last_became_active":"2023-08-28T02:23:59.217783+0000","last_became_peered":"2023-08-28T02:23:59.217783+0000","last_unstale":"2023-08-28T02:30:20.163206+0000","last_undegraded":"2023-08-28T02:30:20.163206+0000","last_fullsized":"2023-08-28T02:30:20.163206+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:00:12.145395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":275,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.173416+0000","last_change":"2023-08-28T02:28:25.351698+0000","last_active":"2023-08-28T02:30:20.173416+0000","last_peered":"2023-08-28T02:30:20.173416+0000","last_clean":"2023-08-28T02:30:20.173416+0000","last_became_active":"2023-08-28T02:28:25.351349+0000","last_became_peered":"2023-08-28T02:28:25.351349+0000","last_unstale":"2023-08-28T02:30:20.173416+0000","last_undegraded":"2023-08-28T02:30:20.173416+0000","last_fullsized":"2023-08-28T02:30:20.173416+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"255'9836","reported_seq":9914,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-08-28T02:30:28.979530+0000","last_change":"2023-08-28T02:26:57.390417+0000","last_active":"2023-08-28T02:30:28.979530+0000","last_peered":"2023-08-28T02:30:28.979530+0000","last_clean":"2023-08-28T02:30:28.979530+0000","last_became_active":"2023-08-28T02:26:57.390124+0000","last_became_peered":"2023-08-28T02:26:57.390124+0000","last_unstale":"2023-08-28T02:30:28.979530+0000","last_undegraded":"2023-08-28T02:30:28.979530+0000","last_fullsized":"2023-08-28T02:30:28.979530+0000","mapping_epoch":187,"log_start":"195'1500","ondisk_log_start":"195'1500","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":8336,"log_dups_size":0,"ondisk_log_size":8336,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:38:48.957967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43909120,"num_objects":670,"num_object_clones":0,"num_object_copies":2010,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":670,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9836,"num_write_kb":336192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":44,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.160065+0000","last_change":"2023-08-28T02:28:25.345274+0000","last_active":"2023-08-28T02:30:20.160065+0000","last_peered":"2023-08-28T02:30:20.160065+0000","last_clean":"2023-08-28T02:30:20.160065+0000","last_became_active":"2023-08-28T02:28:25.344885+0000","last_became_peered":"2023-08-28T02:28:25.344885+0000","last_unstale":"2023-08-28T02:30:20.160065+0000","last_undegraded":"2023-08-28T02:30:20.160065+0000","last_fullsized":"2023-08-28T02:30:20.160065+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":301,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.172652+0000","last_change":"2023-08-28T02:27:31.358436+0000","last_active":"2023-08-28T02:30:20.172652+0000","last_peered":"2023-08-28T02:30:20.172652+0000","last_clean":"2023-08-28T02:30:20.172652+0000","last_became_active":"2023-08-28T02:27:31.358185+0000","last_became_peered":"2023-08-28T02:27:31.358185+0000","last_unstale":"2023-08-28T02:30:20.172652+0000","last_undegraded":"2023-08-28T02:30:20.172652+0000","last_fullsized":"2023-08-28T02:30:20.172652+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"255'10030","reported_seq":13285,"reported_epoch":255,"state":"active+remapped+backfill_wait","last_fresh":"2023-08-28T02:30:29.169102+0000","last_change":"2023-08-28T02:28:26.455411+0000","last_active":"2023-08-28T02:30:29.169102+0000","last_peered":"2023-08-28T02:30:29.169102+0000","last_clean":"2023-08-28T02:28:24.332252+0000","last_became_active":"2023-08-28T02:28:26.447621+0000","last_became_peered":"2023-08-28T02:28:26.447621+0000","last_unstale":"2023-08-28T02:30:29.169102+0000","last_undegraded":"2023-08-28T02:30:29.169102+0000","last_fullsized":"2023-08-28T02:30:29.169102+0000","mapping_epoch":221,"log_start":"207'3700","ondisk_log_start":"207'3700","created":187,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6330,"log_dups_size":0,"ondisk_log_size":6330,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:28:52.578588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39845888,"num_objects":608,"num_object_clones":0,"num_object_copies":1824,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":608,"num_objects_unfound":0,"num_objects_dirty":608,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10030,"num_write_kb":340416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1574,"num_bytes_recovered":103153664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,5,7],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":608}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":164,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.172512+0000","last_change":"2023-08-28T02:23:38.347043+0000","last_active":"2023-08-28T02:30:20.172512+0000","last_peered":"2023-08-28T02:30:20.172512+0000","last_clean":"2023-08-28T02:30:20.172512+0000","last_became_active":"2023-08-28T02:23:38.346680+0000","last_became_peered":"2023-08-28T02:23:38.346680+0000","last_unstale":"2023-08-28T02:30:20.172512+0000","last_undegraded":"2023-08-28T02:30:20.172512+0000","last_fullsized":"2023-08-28T02:30:20.172512+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":221,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.148081+0000","last_change":"2023-08-28T02:28:25.347494+0000","last_active":"2023-08-28T02:30:20.148081+0000","last_peered":"2023-08-28T02:30:20.148081+0000","last_clean":"2023-08-28T02:30:20.148081+0000","last_became_active":"2023-08-28T02:28:25.347142+0000","last_became_peered":"2023-08-28T02:28:25.347142+0000","last_unstale":"2023-08-28T02:30:20.148081+0000","last_undegraded":"2023-08-28T02:30:20.148081+0000","last_fullsized":"2023-08-28T02:30:20.148081+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:37:14.384884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.d","version":"255'9679","reported_seq":14405,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.247114+0000","last_change":"2023-08-28T02:29:35.551357+0000","last_active":"2023-08-28T02:30:32.247114+0000","last_peered":"2023-08-28T02:30:32.247114+0000","last_clean":"2023-08-28T02:30:32.247114+0000","last_became_active":"2023-08-28T02:29:35.530821+0000","last_became_peered":"2023-08-28T02:29:35.530821+0000","last_unstale":"2023-08-28T02:30:32.247114+0000","last_undegraded":"2023-08-28T02:30:32.247114+0000","last_fullsized":"2023-08-28T02:30:32.247114+0000","mapping_epoch":240,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":9679,"log_dups_size":0,"ondisk_log_size":9679,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31195160,"num_objects":477,"num_object_clones":0,"num_object_copies":1431,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":477,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9679,"num_write_kb":324929,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2312,"num_bytes_recovered":47775744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":133,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-08-28T02:30:21.174845+0000","last_change":"2023-08-28T02:27:31.325260+0000","last_active":"2023-08-28T02:30:21.174845+0000","last_peered":"2023-08-28T02:30:21.174845+0000","last_clean":"2023-08-28T02:30:21.174845+0000","last_became_active":"2023-08-28T02:27:31.324950+0000","last_became_peered":"2023-08-28T02:27:31.324950+0000","last_unstale":"2023-08-28T02:30:21.174845+0000","last_undegraded":"2023-08-28T02:30:21.174845+0000","last_fullsized":"2023-08-28T02:30:21.174845+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:21:33.833261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":63,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.160282+0000","last_change":"2023-08-28T02:27:31.322118+0000","last_active":"2023-08-28T02:30:20.160282+0000","last_peered":"2023-08-28T02:30:20.160282+0000","last_clean":"2023-08-28T02:30:20.160282+0000","last_became_active":"2023-08-28T02:27:31.321338+0000","last_became_peered":"2023-08-28T02:27:31.321338+0000","last_unstale":"2023-08-28T02:30:20.160282+0000","last_undegraded":"2023-08-28T02:30:20.160282+0000","last_fullsized":"2023-08-28T02:30:20.160282+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:56:14.210571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"255'6643","reported_seq":6736,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-08-28T02:30:28.881275+0000","last_change":"2023-08-28T02:30:11.788028+0000","last_active":"2023-08-28T02:30:28.881275+0000","last_peered":"2023-08-28T02:30:28.881275+0000","last_clean":"2023-08-28T02:30:28.881275+0000","last_became_active":"2023-08-28T02:27:58.437499+0000","last_became_peered":"2023-08-28T02:27:58.437499+0000","last_unstale":"2023-08-28T02:30:28.881275+0000","last_undegraded":"2023-08-28T02:30:28.881275+0000","last_fullsized":"2023-08-28T02:30:28.881275+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6643,"log_dups_size":0,"ondisk_log_size":6643,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19922944,"num_objects":304,"num_object_clones":0,"num_object_copies":912,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":304,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4806,"num_write_kb":163520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"78'126","reported_seq":624,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.158384+0000","last_change":"2023-08-28T02:28:27.383137+0000","last_active":"2023-08-28T02:30:20.158384+0000","last_peered":"2023-08-28T02:30:20.158384+0000","last_clean":"2023-08-28T02:30:20.158384+0000","last_became_active":"2023-08-28T02:28:27.382488+0000","last_became_peered":"2023-08-28T02:28:27.382488+0000","last_unstale":"2023-08-28T02:30:20.158384+0000","last_undegraded":"2023-08-28T02:30:20.158384+0000","last_fullsized":"2023-08-28T02:30:20.158384+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:14:48.414426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":743,"num_read_kb":1331,"num_write":125,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":254,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.157250+0000","last_change":"2023-08-28T02:23:59.207148+0000","last_active":"2023-08-28T02:30:20.157250+0000","last_peered":"2023-08-28T02:30:20.157250+0000","last_clean":"2023-08-28T02:30:20.157250+0000","last_became_active":"2023-08-28T02:23:59.206802+0000","last_became_peered":"2023-08-28T02:23:59.206802+0000","last_unstale":"2023-08-28T02:30:20.157250+0000","last_undegraded":"2023-08-28T02:30:20.157250+0000","last_fullsized":"2023-08-28T02:30:20.157250+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":63,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.157794+0000","last_change":"2023-08-28T02:27:31.328027+0000","last_active":"2023-08-28T02:30:20.157794+0000","last_peered":"2023-08-28T02:30:20.157794+0000","last_clean":"2023-08-28T02:30:20.157794+0000","last_became_active":"2023-08-28T02:27:31.326529+0000","last_became_peered":"2023-08-28T02:27:31.326529+0000","last_unstale":"2023-08-28T02:30:20.157794+0000","last_undegraded":"2023-08-28T02:30:20.157794+0000","last_fullsized":"2023-08-28T02:30:20.157794+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:56:57.331355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"255'6834","reported_seq":6922,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-08-28T02:30:29.209385+0000","last_change":"2023-08-28T02:27:58.462974+0000","last_active":"2023-08-28T02:30:29.209385+0000","last_peered":"2023-08-28T02:30:29.209385+0000","last_clean":"2023-08-28T02:30:29.209385+0000","last_became_active":"2023-08-28T02:27:58.460293+0000","last_became_peered":"2023-08-28T02:27:58.460293+0000","last_unstale":"2023-08-28T02:30:29.209385+0000","last_undegraded":"2023-08-28T02:30:29.209385+0000","last_fullsized":"2023-08-28T02:30:29.209385+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6834,"log_dups_size":0,"ondisk_log_size":6834,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21266432,"num_objects":324,"num_object_clones":0,"num_object_copies":972,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":324,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4948,"num_write_kb":168736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"78'204","reported_seq":1340,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.157629+0000","last_change":"2023-08-28T02:28:59.480561+0000","last_active":"2023-08-28T02:30:20.157629+0000","last_peered":"2023-08-28T02:30:20.157629+0000","last_clean":"2023-08-28T02:30:20.157629+0000","last_became_active":"2023-08-28T02:28:59.479868+0000","last_became_peered":"2023-08-28T02:28:59.479868+0000","last_unstale":"2023-08-28T02:30:20.157629+0000","last_undegraded":"2023-08-28T02:30:20.157629+0000","last_fullsized":"2023-08-28T02:30:20.157629+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":204,"log_dups_size":0,"ondisk_log_size":204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:39:50.444629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1521,"num_read_kb":2511,"num_write":178,"num_write_kb":1175,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":249,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.172337+0000","last_change":"2023-08-28T02:28:25.352417+0000","last_active":"2023-08-28T02:30:20.172337+0000","last_peered":"2023-08-28T02:30:20.172337+0000","last_clean":"2023-08-28T02:30:20.172337+0000","last_became_active":"2023-08-28T02:28:25.352216+0000","last_became_peered":"2023-08-28T02:28:25.352216+0000","last_unstale":"2023-08-28T02:30:20.172337+0000","last_undegraded":"2023-08-28T02:30:20.172337+0000","last_fullsized":"2023-08-28T02:30:20.172337+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":63,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.158228+0000","last_change":"2023-08-28T02:27:31.376755+0000","last_active":"2023-08-28T02:30:20.158228+0000","last_peered":"2023-08-28T02:30:20.158228+0000","last_clean":"2023-08-28T02:30:20.158228+0000","last_became_active":"2023-08-28T02:27:31.373279+0000","last_became_peered":"2023-08-28T02:27:31.373279+0000","last_unstale":"2023-08-28T02:30:20.158228+0000","last_undegraded":"2023-08-28T02:30:20.158228+0000","last_fullsized":"2023-08-28T02:30:20.158228+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:26:14.628941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"255'6839","reported_seq":9231,"reported_epoch":255,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-08-28T02:30:32.147500+0000","last_change":"2023-08-28T02:30:08.028893+0000","last_active":"2023-08-28T02:30:32.147500+0000","last_peered":"2023-08-28T02:30:32.147500+0000","last_clean":"2023-08-28T02:27:16.315756+0000","last_became_active":"2023-08-28T02:28:25.400914+0000","last_became_peered":"2023-08-28T02:28:25.400914+0000","last_unstale":"2023-08-28T02:30:32.147500+0000","last_undegraded":"2023-08-28T02:28:25.339881+0000","last_fullsized":"2023-08-28T02:28:25.329514+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6839,"log_dups_size":0,"ondisk_log_size":6839,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17367040,"num_objects":265,"num_object_clones":0,"num_object_copies":795,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1035,"num_objects_misplaced":78,"num_objects_unfound":0,"num_objects_dirty":265,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4955,"num_write_kb":167040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":851,"num_bytes_recovered":55771136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"2,3,7","objects":78},{"shards":"3,7","objects":-848}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'47","reported_seq":471,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.172543+0000","last_change":"2023-08-28T02:24:05.799647+0000","last_active":"2023-08-28T02:30:20.172543+0000","last_peered":"2023-08-28T02:30:20.172543+0000","last_clean":"2023-08-28T02:30:20.172543+0000","last_became_active":"2023-08-28T02:23:59.210481+0000","last_became_peered":"2023-08-28T02:23:59.210481+0000","last_unstale":"2023-08-28T02:30:20.172543+0000","last_undegraded":"2023-08-28T02:30:20.172543+0000","last_fullsized":"2023-08-28T02:30:20.172543+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":47,"log_dups_size":0,"ondisk_log_size":47,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:40:18.542375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":162,"num_read_kb":1281,"num_write":48,"num_write_kb":1153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":72,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.157536+0000","last_change":"2023-08-28T02:28:25.343406+0000","last_active":"2023-08-28T02:30:20.157536+0000","last_peered":"2023-08-28T02:30:20.157536+0000","last_clean":"2023-08-28T02:30:20.157536+0000","last_became_active":"2023-08-28T02:28:25.343065+0000","last_became_peered":"2023-08-28T02:28:25.343065+0000","last_unstale":"2023-08-28T02:30:20.157536+0000","last_undegraded":"2023-08-28T02:30:20.157536+0000","last_fullsized":"2023-08-28T02:30:20.157536+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:20:31.893609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":63,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.160362+0000","last_change":"2023-08-28T02:27:31.322967+0000","last_active":"2023-08-28T02:30:20.160362+0000","last_peered":"2023-08-28T02:30:20.160362+0000","last_clean":"2023-08-28T02:30:20.160362+0000","last_became_active":"2023-08-28T02:27:31.322655+0000","last_became_peered":"2023-08-28T02:27:31.322655+0000","last_unstale":"2023-08-28T02:30:20.160362+0000","last_undegraded":"2023-08-28T02:30:20.160362+0000","last_fullsized":"2023-08-28T02:30:20.160362+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:05:15.868115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"255'6881","reported_seq":6979,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.255811+0000","last_change":"2023-08-28T02:30:14.118676+0000","last_active":"2023-08-28T02:30:32.255811+0000","last_peered":"2023-08-28T02:30:32.255811+0000","last_clean":"2023-08-28T02:30:32.255811+0000","last_became_active":"2023-08-28T02:27:58.452334+0000","last_became_peered":"2023-08-28T02:27:58.452334+0000","last_unstale":"2023-08-28T02:30:32.255811+0000","last_undegraded":"2023-08-28T02:30:32.255811+0000","last_fullsized":"2023-08-28T02:30:32.255811+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6881,"log_dups_size":0,"ondisk_log_size":6881,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15269888,"num_objects":233,"num_object_clones":0,"num_object_copies":699,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":233,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5025,"num_write_kb":168256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"78'114","reported_seq":618,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.156199+0000","last_change":"2023-08-28T02:24:01.210739+0000","last_active":"2023-08-28T02:30:20.156199+0000","last_peered":"2023-08-28T02:30:20.156199+0000","last_clean":"2023-08-28T02:30:20.156199+0000","last_became_active":"2023-08-28T02:24:01.210386+0000","last_became_peered":"2023-08-28T02:24:01.210386+0000","last_unstale":"2023-08-28T02:30:20.156199+0000","last_undegraded":"2023-08-28T02:30:20.156199+0000","last_fullsized":"2023-08-28T02:30:20.156199+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":114,"log_dups_size":0,"ondisk_log_size":114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":358,"num_read_kb":1352,"num_write":102,"num_write_kb":1104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"86'139","reported_seq":465,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.163327+0000","last_change":"2023-08-28T02:29:58.484566+0000","last_active":"2023-08-28T02:30:20.163327+0000","last_peered":"2023-08-28T02:30:20.163327+0000","last_clean":"2023-08-28T02:30:20.163327+0000","last_became_active":"2023-08-28T02:29:58.484046+0000","last_became_peered":"2023-08-28T02:29:58.484046+0000","last_unstale":"2023-08-28T02:30:20.163327+0000","last_undegraded":"2023-08-28T02:30:20.163327+0000","last_fullsized":"2023-08-28T02:30:20.163327+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_clean_scrub_stamp":"2023-08-28T02:08:02.043718+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":133,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.162879+0000","last_change":"2023-08-28T02:28:25.347600+0000","last_active":"2023-08-28T02:30:20.162879+0000","last_peered":"2023-08-28T02:30:20.162879+0000","last_clean":"2023-08-28T02:30:20.162879+0000","last_became_active":"2023-08-28T02:28:25.347237+0000","last_became_peered":"2023-08-28T02:28:25.347237+0000","last_unstale":"2023-08-28T02:30:20.162879+0000","last_undegraded":"2023-08-28T02:30:20.162879+0000","last_fullsized":"2023-08-28T02:30:20.162879+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:29:26.767559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":44,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.160079+0000","last_change":"2023-08-28T02:28:25.398594+0000","last_active":"2023-08-28T02:30:20.160079+0000","last_peered":"2023-08-28T02:30:20.160079+0000","last_clean":"2023-08-28T02:30:20.160079+0000","last_became_active":"2023-08-28T02:28:25.397058+0000","last_became_peered":"2023-08-28T02:28:25.397058+0000","last_unstale":"2023-08-28T02:30:20.160079+0000","last_undegraded":"2023-08-28T02:30:20.160079+0000","last_fullsized":"2023-08-28T02:30:20.160079+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"255'6698","reported_seq":12111,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-08-28T02:30:29.512926+0000","last_change":"2023-08-28T02:30:14.359869+0000","last_active":"2023-08-28T02:30:29.512926+0000","last_peered":"2023-08-28T02:30:29.512926+0000","last_clean":"2023-08-28T02:30:29.512926+0000","last_became_active":"2023-08-28T02:28:00.019510+0000","last_became_peered":"2023-08-28T02:28:00.019510+0000","last_unstale":"2023-08-28T02:30:29.512926+0000","last_undegraded":"2023-08-28T02:30:29.512926+0000","last_fullsized":"2023-08-28T02:30:29.512926+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6698,"log_dups_size":0,"ondisk_log_size":6698,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22118400,"num_objects":337,"num_object_clones":0,"num_object_copies":1011,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":337,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4865,"num_write_kb":166496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1910,"num_bytes_recovered":125206528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'148","reported_seq":785,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.163376+0000","last_change":"2023-08-28T02:29:39.524927+0000","last_active":"2023-08-28T02:30:20.163376+0000","last_peered":"2023-08-28T02:30:20.163376+0000","last_clean":"2023-08-28T02:30:20.163376+0000","last_became_active":"2023-08-28T02:29:39.524297+0000","last_became_peered":"2023-08-28T02:29:39.524297+0000","last_unstale":"2023-08-28T02:30:20.163376+0000","last_undegraded":"2023-08-28T02:30:20.163376+0000","last_fullsized":"2023-08-28T02:30:20.163376+0000","mapping_epoch":243,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":244,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1092,"num_read_kb":1955,"num_write":127,"num_write_kb":5177,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":223,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.149815+0000","last_change":"2023-08-28T02:23:59.226986+0000","last_active":"2023-08-28T02:30:20.149815+0000","last_peered":"2023-08-28T02:30:20.149815+0000","last_clean":"2023-08-28T02:30:20.149815+0000","last_became_active":"2023-08-28T02:23:59.226529+0000","last_became_peered":"2023-08-28T02:23:59.226529+0000","last_unstale":"2023-08-28T02:30:20.149815+0000","last_undegraded":"2023-08-28T02:30:20.149815+0000","last_fullsized":"2023-08-28T02:30:20.149815+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:22:09.193619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":268,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.147758+0000","last_change":"2023-08-28T02:27:31.361298+0000","last_active":"2023-08-28T02:30:20.147758+0000","last_peered":"2023-08-28T02:30:20.147758+0000","last_clean":"2023-08-28T02:30:20.147758+0000","last_became_active":"2023-08-28T02:27:31.360939+0000","last_became_peered":"2023-08-28T02:27:31.360939+0000","last_unstale":"2023-08-28T02:30:20.147758+0000","last_undegraded":"2023-08-28T02:30:20.147758+0000","last_fullsized":"2023-08-28T02:30:20.147758+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:48:42.024920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"255'6865","reported_seq":13901,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-08-28T02:30:29.504986+0000","last_change":"2023-08-28T02:30:12.290655+0000","last_active":"2023-08-28T02:30:29.504986+0000","last_peered":"2023-08-28T02:30:29.504986+0000","last_clean":"2023-08-28T02:30:29.504986+0000","last_became_active":"2023-08-28T02:30:09.573625+0000","last_became_peered":"2023-08-28T02:30:09.573625+0000","last_unstale":"2023-08-28T02:30:29.504986+0000","last_undegraded":"2023-08-28T02:30:29.504986+0000","last_fullsized":"2023-08-28T02:30:29.504986+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6865,"log_dups_size":0,"ondisk_log_size":6865,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21889024,"num_objects":334,"num_object_clones":0,"num_object_copies":1002,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":334,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4990,"num_write_kb":170368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2494,"num_bytes_recovered":144736256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'103","reported_seq":630,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.168992+0000","last_change":"2023-08-28T02:24:07.212128+0000","last_active":"2023-08-28T02:30:20.168992+0000","last_peered":"2023-08-28T02:30:20.168992+0000","last_clean":"2023-08-28T02:30:20.168992+0000","last_became_active":"2023-08-28T02:24:07.211562+0000","last_became_peered":"2023-08-28T02:24:07.211562+0000","last_unstale":"2023-08-28T02:30:20.168992+0000","last_undegraded":"2023-08-28T02:30:20.168992+0000","last_fullsized":"2023-08-28T02:30:20.168992+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":803,"num_read_kb":2080,"num_write":73,"num_write_kb":1244,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":63,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.158505+0000","last_change":"2023-08-28T02:27:31.377042+0000","last_active":"2023-08-28T02:30:20.158505+0000","last_peered":"2023-08-28T02:30:20.158505+0000","last_clean":"2023-08-28T02:30:20.158505+0000","last_became_active":"2023-08-28T02:27:31.373709+0000","last_became_peered":"2023-08-28T02:27:31.373709+0000","last_unstale":"2023-08-28T02:30:20.158505+0000","last_undegraded":"2023-08-28T02:30:20.158505+0000","last_fullsized":"2023-08-28T02:30:20.158505+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:12:38.286332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":209,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.150385+0000","last_change":"2023-08-28T02:23:38.329667+0000","last_active":"2023-08-28T02:30:20.150385+0000","last_peered":"2023-08-28T02:30:20.150385+0000","last_clean":"2023-08-28T02:30:20.150385+0000","last_became_active":"2023-08-28T02:23:38.329175+0000","last_became_peered":"2023-08-28T02:23:38.329175+0000","last_unstale":"2023-08-28T02:30:20.150385+0000","last_undegraded":"2023-08-28T02:30:20.150385+0000","last_fullsized":"2023-08-28T02:30:20.150385+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:21:34.815067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"255'6909","reported_seq":7034,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.048567+0000","last_change":"2023-08-28T02:30:18.579772+0000","last_active":"2023-08-28T02:30:32.048567+0000","last_peered":"2023-08-28T02:30:32.048567+0000","last_clean":"2023-08-28T02:30:32.048567+0000","last_became_active":"2023-08-28T02:27:58.451860+0000","last_became_peered":"2023-08-28T02:27:58.451860+0000","last_unstale":"2023-08-28T02:30:32.048567+0000","last_undegraded":"2023-08-28T02:30:32.048567+0000","last_fullsized":"2023-08-28T02:30:32.048567+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6909,"log_dups_size":0,"ondisk_log_size":6909,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13500416,"num_objects":206,"num_object_clones":0,"num_object_copies":618,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":206,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5044,"num_write_kb":168000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":119,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-08-28T02:30:21.181118+0000","last_change":"2023-08-28T02:23:59.232099+0000","last_active":"2023-08-28T02:30:21.181118+0000","last_peered":"2023-08-28T02:30:21.181118+0000","last_clean":"2023-08-28T02:30:21.181118+0000","last_became_active":"2023-08-28T02:23:59.219318+0000","last_became_peered":"2023-08-28T02:23:59.219318+0000","last_unstale":"2023-08-28T02:30:21.181118+0000","last_undegraded":"2023-08-28T02:30:21.181118+0000","last_fullsized":"2023-08-28T02:30:21.181118+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:59:30.093289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'112","reported_seq":408,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.160413+0000","last_change":"2023-08-28T02:28:59.477917+0000","last_active":"2023-08-28T02:30:20.160413+0000","last_peered":"2023-08-28T02:30:20.160413+0000","last_clean":"2023-08-28T02:30:20.160413+0000","last_became_active":"2023-08-28T02:28:59.477383+0000","last_became_peered":"2023-08-28T02:28:59.477383+0000","last_unstale":"2023-08-28T02:30:20.160413+0000","last_undegraded":"2023-08-28T02:30:20.160413+0000","last_fullsized":"2023-08-28T02:30:20.160413+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:28:45.045742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":444,"num_read_kb":473,"num_write":94,"num_write_kb":127,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":115,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.162796+0000","last_change":"2023-08-28T02:23:59.221034+0000","last_active":"2023-08-28T02:30:20.162796+0000","last_peered":"2023-08-28T02:30:20.162796+0000","last_clean":"2023-08-28T02:30:20.162796+0000","last_became_active":"2023-08-28T02:23:59.220730+0000","last_became_peered":"2023-08-28T02:23:59.220730+0000","last_unstale":"2023-08-28T02:30:20.162796+0000","last_undegraded":"2023-08-28T02:30:20.162796+0000","last_fullsized":"2023-08-28T02:30:20.162796+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:25:52.364370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"255'6889","reported_seq":6995,"reported_epoch":255,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:30:32.115257+0000","last_change":"2023-08-28T02:28:26.460507+0000","last_active":"2023-08-28T02:30:32.115257+0000","last_peered":"2023-08-28T02:30:32.115257+0000","last_clean":"2023-08-28T02:27:16.315150+0000","last_became_active":"2023-08-28T02:28:26.432234+0000","last_became_peered":"2023-08-28T02:28:26.432234+0000","last_unstale":"2023-08-28T02:30:32.115257+0000","last_undegraded":"2023-08-28T02:28:26.346974+0000","last_fullsized":"2023-08-28T02:28:26.332755+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6889,"log_dups_size":0,"ondisk_log_size":6889,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18022400,"num_objects":275,"num_object_clones":0,"num_object_copies":825,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1660,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":275,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5001,"num_write_kb":168832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"1,2,7","objects":73},{"shards":"2,7","objects":-1385}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"78'122","reported_seq":562,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.160115+0000","last_change":"2023-08-28T02:28:28.383306+0000","last_active":"2023-08-28T02:30:20.160115+0000","last_peered":"2023-08-28T02:30:20.160115+0000","last_clean":"2023-08-28T02:30:20.160115+0000","last_became_active":"2023-08-28T02:28:28.382672+0000","last_became_peered":"2023-08-28T02:28:28.382672+0000","last_unstale":"2023-08-28T02:30:20.160115+0000","last_undegraded":"2023-08-28T02:30:20.160115+0000","last_fullsized":"2023-08-28T02:30:20.160115+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":857,"num_read_kb":2224,"num_write":99,"num_write_kb":2312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":235,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.147852+0000","last_change":"2023-08-28T02:27:31.424373+0000","last_active":"2023-08-28T02:30:20.147852+0000","last_peered":"2023-08-28T02:30:20.147852+0000","last_clean":"2023-08-28T02:30:20.147852+0000","last_became_active":"2023-08-28T02:27:31.324505+0000","last_became_peered":"2023-08-28T02:27:31.324505+0000","last_unstale":"2023-08-28T02:30:20.147852+0000","last_undegraded":"2023-08-28T02:30:20.147852+0000","last_fullsized":"2023-08-28T02:30:20.147852+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:27:57.837524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":224,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.172185+0000","last_change":"2023-08-28T02:28:25.351840+0000","last_active":"2023-08-28T02:30:20.172185+0000","last_peered":"2023-08-28T02:30:20.172185+0000","last_clean":"2023-08-28T02:30:20.172185+0000","last_became_active":"2023-08-28T02:28:25.351399+0000","last_became_peered":"2023-08-28T02:28:25.351399+0000","last_unstale":"2023-08-28T02:30:20.172185+0000","last_undegraded":"2023-08-28T02:30:20.172185+0000","last_fullsized":"2023-08-28T02:30:20.172185+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":211,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.150330+0000","last_change":"2023-08-28T02:23:38.329474+0000","last_active":"2023-08-28T02:30:20.150330+0000","last_peered":"2023-08-28T02:30:20.150330+0000","last_clean":"2023-08-28T02:30:20.150330+0000","last_became_active":"2023-08-28T02:23:38.328505+0000","last_became_peered":"2023-08-28T02:23:38.328505+0000","last_unstale":"2023-08-28T02:30:20.150330+0000","last_undegraded":"2023-08-28T02:30:20.150330+0000","last_fullsized":"2023-08-28T02:30:20.150330+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:12:14.835809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":296,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.173047+0000","last_change":"2023-08-28T02:27:31.384467+0000","last_active":"2023-08-28T02:30:20.173047+0000","last_peered":"2023-08-28T02:30:20.173047+0000","last_clean":"2023-08-28T02:30:20.173047+0000","last_became_active":"2023-08-28T02:27:31.384183+0000","last_became_peered":"2023-08-28T02:27:31.384183+0000","last_unstale":"2023-08-28T02:30:20.173047+0000","last_undegraded":"2023-08-28T02:30:20.173047+0000","last_fullsized":"2023-08-28T02:30:20.173047+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:59:47.551892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.19","version":"255'6773","reported_seq":6863,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-08-28T02:30:28.803328+0000","last_change":"2023-08-28T02:30:16.900410+0000","last_active":"2023-08-28T02:30:28.803328+0000","last_peered":"2023-08-28T02:30:28.803328+0000","last_clean":"2023-08-28T02:30:28.803328+0000","last_became_active":"2023-08-28T02:27:58.490323+0000","last_became_peered":"2023-08-28T02:27:58.490323+0000","last_unstale":"2023-08-28T02:30:28.803328+0000","last_undegraded":"2023-08-28T02:30:28.803328+0000","last_fullsized":"2023-08-28T02:30:28.803328+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6773,"log_dups_size":0,"ondisk_log_size":6773,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22118400,"num_objects":338,"num_object_clones":0,"num_object_copies":1014,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":338,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4904,"num_write_kb":167712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":61,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.173347+0000","last_change":"2023-08-28T02:28:11.333622+0000","last_active":"2023-08-28T02:30:20.173347+0000","last_peered":"2023-08-28T02:30:20.173347+0000","last_clean":"2023-08-28T02:30:20.173347+0000","last_became_active":"2023-08-28T02:28:11.333321+0000","last_became_peered":"2023-08-28T02:28:11.333321+0000","last_unstale":"2023-08-28T02:30:20.173347+0000","last_undegraded":"2023-08-28T02:30:20.173347+0000","last_fullsized":"2023-08-28T02:30:20.173347+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":272,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.172467+0000","last_change":"2023-08-28T02:27:31.343538+0000","last_active":"2023-08-28T02:30:20.172467+0000","last_peered":"2023-08-28T02:30:20.172467+0000","last_clean":"2023-08-28T02:30:20.172467+0000","last_became_active":"2023-08-28T02:27:31.343324+0000","last_became_peered":"2023-08-28T02:27:31.343324+0000","last_unstale":"2023-08-28T02:30:20.172467+0000","last_undegraded":"2023-08-28T02:30:20.172467+0000","last_fullsized":"2023-08-28T02:30:20.172467+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.18","version":"255'6743","reported_seq":6828,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.118346+0000","last_change":"2023-08-28T02:27:58.496324+0000","last_active":"2023-08-28T02:30:32.118346+0000","last_peered":"2023-08-28T02:30:32.118346+0000","last_clean":"2023-08-28T02:30:32.118346+0000","last_became_active":"2023-08-28T02:27:58.494507+0000","last_became_peered":"2023-08-28T02:27:58.494507+0000","last_unstale":"2023-08-28T02:30:32.118346+0000","last_undegraded":"2023-08-28T02:30:32.118346+0000","last_fullsized":"2023-08-28T02:30:32.118346+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6743,"log_dups_size":0,"ondisk_log_size":6743,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19169280,"num_objects":293,"num_object_clones":0,"num_object_copies":879,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":293,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4867,"num_write_kb":165088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":74,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.172738+0000","last_change":"2023-08-28T02:28:25.705331+0000","last_active":"2023-08-28T02:30:20.172738+0000","last_peered":"2023-08-28T02:30:20.172738+0000","last_clean":"2023-08-28T02:30:20.172738+0000","last_became_active":"2023-08-28T02:28:25.397480+0000","last_became_peered":"2023-08-28T02:28:25.397480+0000","last_unstale":"2023-08-28T02:30:20.172738+0000","last_undegraded":"2023-08-28T02:30:20.172738+0000","last_fullsized":"2023-08-28T02:30:20.172738+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":254,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.172686+0000","last_change":"2023-08-28T02:23:59.207929+0000","last_active":"2023-08-28T02:30:20.172686+0000","last_peered":"2023-08-28T02:30:20.172686+0000","last_clean":"2023-08-28T02:30:20.172686+0000","last_became_active":"2023-08-28T02:23:59.207589+0000","last_became_peered":"2023-08-28T02:23:59.207589+0000","last_unstale":"2023-08-28T02:30:20.172686+0000","last_undegraded":"2023-08-28T02:30:20.172686+0000","last_fullsized":"2023-08-28T02:30:20.172686+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":142,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.163560+0000","last_change":"2023-08-28T02:28:25.347988+0000","last_active":"2023-08-28T02:30:20.163560+0000","last_peered":"2023-08-28T02:30:20.163560+0000","last_clean":"2023-08-28T02:30:20.163560+0000","last_became_active":"2023-08-28T02:28:25.347632+0000","last_became_peered":"2023-08-28T02:28:25.347632+0000","last_unstale":"2023-08-28T02:30:20.163560+0000","last_undegraded":"2023-08-28T02:30:20.163560+0000","last_fullsized":"2023-08-28T02:30:20.163560+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:56:20.202662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":114,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.162870+0000","last_change":"2023-08-28T02:23:59.217794+0000","last_active":"2023-08-28T02:30:20.162870+0000","last_peered":"2023-08-28T02:30:20.162870+0000","last_clean":"2023-08-28T02:30:20.162870+0000","last_became_active":"2023-08-28T02:23:59.217231+0000","last_became_peered":"2023-08-28T02:23:59.217231+0000","last_unstale":"2023-08-28T02:30:20.162870+0000","last_undegraded":"2023-08-28T02:30:20.162870+0000","last_fullsized":"2023-08-28T02:30:20.162870+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:36:26.297410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":209,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.148238+0000","last_change":"2023-08-28T02:20:19.850252+0000","last_active":"2023-08-28T02:30:20.148238+0000","last_peered":"2023-08-28T02:30:20.148238+0000","last_clean":"2023-08-28T02:30:20.148238+0000","last_became_active":"2023-08-28T02:20:19.849913+0000","last_became_peered":"2023-08-28T02:20:19.849913+0000","last_unstale":"2023-08-28T02:30:20.148238+0000","last_undegraded":"2023-08-28T02:30:20.148238+0000","last_fullsized":"2023-08-28T02:30:20.148238+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:48:47.713465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":286,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-08-28T02:30:20.173349+0000","last_change":"2023-08-28T02:28:25.347845+0000","last_active":"2023-08-28T02:30:20.173349+0000","last_peered":"2023-08-28T02:30:20.173349+0000","last_clean":"2023-08-28T02:30:20.173349+0000","last_became_active":"2023-08-28T02:28:25.347502+0000","last_became_peered":"2023-08-28T02:28:25.347502+0000","last_unstale":"2023-08-28T02:30:20.173349+0000","last_undegraded":"2023-08-28T02:30:20.173349+0000","last_fullsized":"2023-08-28T02:30:20.173349+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":26,"stat_sum":{"num_bytes":614400024,"num_objects":9376,"num_object_clones":0,"num_object_copies":28128,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6088,"num_objects_misplaced":1363,"num_objects_unfound":0,"num_objects_dirty":9376,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":157858,"num_write_kb":5351425,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20202,"num_bytes_recovered":1162477568,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2787389440,"data_stored":2787377224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":183396,"ondisk_log_size":183396,"up":78,"acting":74,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":19382,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26810,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":137,"num_bytes_recovered":12582931,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430459,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":872868,"kb_used_data":563356,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92872284,"statfs":{"total":95995035648,"available":95101218816,"internally_reserved":0,"allocated":576876544,"data_stored":576161782,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:30:03 2023","interfaces":[{"interface":"back","average":{"1min":2.194,"5min":2.3999999999999999,"15min":2.0859999999999999},"min":{"1min":0.25600000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":24.379999999999999,"5min":59.753999999999998,"15min":116.755},"last":0.59399999999999997},{"interface":"front","average":{"1min":2.7120000000000002,"5min":2.7559999999999998,"15min":2.3359999999999999},"min":{"1min":0.26500000000000001,"5min":0.245,"15min":0.23999999999999999},"max":{"1min":24.213999999999999,"5min":59.765999999999998,"15min":116.971},"last":0.40400000000000003}]},{"osd":1,"last update":"Mon Aug 28 02:30:11 2023","interfaces":[{"interface":"back","average":{"1min":2.496,"5min":2.2509999999999999,"15min":1.2370000000000001},"min":{"1min":0.27900000000000003,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":24.681000000000001,"5min":44.287999999999997,"15min":44.287999999999997},"last":0.625},{"interface":"front","average":{"1min":2.7570000000000001,"5min":3.109,"15min":1.667},"min":{"1min":0.28399999999999997,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":24.189,"5min":44.393999999999998,"15min":44.393999999999998},"last":0.45700000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:29:35 2023","interfaces":[{"interface":"back","average":{"1min":2.1309999999999998,"5min":4.3079999999999998,"15min":2.3759999999999999},"min":{"1min":0.28000000000000003,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":8.2680000000000007,"5min":119.571,"15min":119.571},"last":0.81799999999999995},{"interface":"front","average":{"1min":1.843,"5min":5.2800000000000002,"15min":2.7869999999999999},"min":{"1min":0.35099999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":8.2789999999999999,"5min":119.554,"15min":119.554},"last":0.51300000000000001}]},{"osd":3,"last update":"Mon Aug 28 02:30:11 2023","interfaces":[{"interface":"back","average":{"1min":2.0110000000000001,"5min":4.0620000000000003,"15min":2.2480000000000002},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30199999999999999},"max":{"1min":10.686999999999999,"5min":99.459000000000003,"15min":107.916},"last":0.84199999999999997},{"interface":"front","average":{"1min":1.794,"5min":3.6789999999999998,"15min":2.3239999999999998},"min":{"1min":0.34300000000000003,"5min":0.311,"15min":0.27700000000000002},"max":{"1min":10.140000000000001,"5min":98.084000000000003,"15min":112.00700000000001},"last":0.48099999999999998}]},{"osd":4,"last update":"Mon Aug 28 02:29:35 2023","interfaces":[{"interface":"back","average":{"1min":0.75900000000000001,"5min":1.3360000000000001,"15min":1.2390000000000001},"min":{"1min":0.26500000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.4209999999999998,"5min":11.696,"15min":63.048000000000002},"last":0.43099999999999999},{"interface":"front","average":{"1min":1.179,"5min":1.4059999999999999,"15min":1.1830000000000001},"min":{"1min":0.29499999999999998,"5min":0.247,"15min":0.247},"max":{"1min":4.2359999999999998,"5min":10.840999999999999,"15min":62.912999999999997},"last":0.34899999999999998}]},{"osd":5,"last update":"Mon Aug 28 02:29:35 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":1.1759999999999999,"15min":1.0269999999999999},"min":{"1min":0.315,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":1.8939999999999999,"5min":13.788,"15min":63.021000000000001},"last":0.79000000000000004},{"interface":"front","average":{"1min":0.71299999999999997,"5min":1.4299999999999999,"15min":1.3260000000000001},"min":{"1min":0.29999999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.5449999999999999,"5min":9.6929999999999996,"15min":63.082999999999998},"last":2.6099999999999999}]},{"osd":6,"last update":"Mon Aug 28 02:29:35 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":1.3360000000000001,"15min":1.2769999999999999},"min":{"1min":0.33700000000000002,"5min":0.317,"15min":0.29699999999999999},"max":{"1min":2.6949999999999998,"5min":9.7070000000000007,"15min":101.61},"last":0.86199999999999999},{"interface":"front","average":{"1min":0.85599999999999998,"5min":1.395,"15min":1.413},"min":{"1min":0.33900000000000002,"5min":0.308,"15min":0.308},"max":{"1min":2.3969999999999998,"5min":14.423999999999999,"15min":101.621},"last":0.53500000000000003}]}]},{"osd":6,"up_from":42,"seq":180388626686,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":463832,"kb_used_data":158416,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93281320,"statfs":{"total":95995035648,"available":95520071680,"internally_reserved":0,"allocated":162217984,"data_stored":161515068,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:30:11 2023","interfaces":[{"interface":"back","average":{"1min":3.246,"5min":3.911,"15min":2.7629999999999999},"min":{"1min":0.22700000000000001,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":44.512999999999998,"5min":60.625999999999998,"15min":121.247},"last":4.3280000000000003},{"interface":"front","average":{"1min":3.3300000000000001,"5min":3.7080000000000002,"15min":2.722},"min":{"1min":0.26000000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":46.637999999999998,"5min":54.273000000000003,"15min":118.977},"last":0.26700000000000002}]},{"osd":1,"last update":"Mon Aug 28 02:30:11 2023","interfaces":[{"interface":"back","average":{"1min":1.78,"5min":2.524,"15min":1.669},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23699999999999999},"max":{"1min":7.2270000000000003,"5min":99.998999999999995,"15min":99.998999999999995},"last":2.9569999999999999},{"interface":"front","average":{"1min":1.9550000000000001,"5min":2.8010000000000002,"15min":1.8100000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":8.6029999999999998,"5min":95.040000000000006,"15min":95.040000000000006},"last":3.0910000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:29:53 2023","interfaces":[{"interface":"back","average":{"1min":1.4650000000000001,"5min":4.258,"15min":2.2250000000000001},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":4.3259999999999996,"5min":79.787000000000006,"15min":79.787000000000006},"last":1.351},{"interface":"front","average":{"1min":1.3959999999999999,"5min":4.4470000000000001,"15min":2.2200000000000002},"min":{"1min":0.28899999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":7.8490000000000002,"5min":79.935000000000002,"15min":79.935000000000002},"last":3.2549999999999999}]},{"osd":3,"last update":"Mon Aug 28 02:30:11 2023","interfaces":[{"interface":"back","average":{"1min":1.1419999999999999,"5min":2.7850000000000001,"15min":1.6719999999999999},"min":{"1min":0.32000000000000001,"5min":0.317,"15min":0.317},"max":{"1min":3.968,"5min":39.587000000000003,"15min":39.587000000000003},"last":3.012},{"interface":"front","average":{"1min":1.0069999999999999,"5min":2.8940000000000001,"15min":1.7589999999999999},"min":{"1min":0.35299999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":3.9780000000000002,"5min":39.593000000000004,"15min":39.593000000000004},"last":3.028}]},{"osd":4,"last update":"Mon Aug 28 02:30:18 2023","interfaces":[{"interface":"back","average":{"1min":1.0069999999999999,"5min":1.224,"15min":1.04},"min":{"1min":0.27100000000000002,"5min":0.251,"15min":0.251},"max":{"1min":3.6499999999999999,"5min":18.308,"15min":18.308},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.77400000000000002,"5min":1.3899999999999999,"15min":1.131},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.26700000000000002},"max":{"1min":3.5070000000000001,"5min":18.263000000000002,"15min":18.263000000000002},"last":0.32300000000000001}]},{"osd":5,"last update":"Mon Aug 28 02:30:18 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":1.216,"15min":1.1080000000000001},"min":{"1min":0.29699999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":2.8359999999999999,"5min":9.9809999999999999,"15min":20.759},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.98999999999999999,"5min":1.113,"15min":0.95599999999999996},"min":{"1min":0.314,"5min":0.314,"15min":0.28799999999999998},"max":{"1min":3.1549999999999998,"5min":5.9500000000000002,"15min":7.3780000000000001},"last":0.49399999999999999}]},{"osd":7,"last update":"Mon Aug 28 02:29:40 2023","interfaces":[{"interface":"back","average":{"1min":0.80600000000000005,"5min":1.161,"15min":0.92700000000000005},"min":{"1min":0.34799999999999998,"5min":0.34200000000000003,"15min":0.314},"max":{"1min":3.8490000000000002,"5min":9.0830000000000002,"15min":9.0830000000000002},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.68400000000000005,"5min":1.3320000000000001,"15min":1.0389999999999999},"min":{"1min":0.35599999999999998,"5min":0.33500000000000002,"15min":0.29799999999999999},"max":{"1min":3.0329999999999999,"5min":9.7110000000000003,"15min":9.7110000000000003},"last":0.47299999999999998}]}]},{"osd":1,"up_from":98,"seq":420906795138,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":645840,"kb_used_data":375048,"kb_used_omap":7,"kb_used_meta":270776,"kb_avail":93099312,"statfs":{"total":95995035648,"available":95333695488,"internally_reserved":0,"allocated":384049152,"data_stored":382783328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":277274955},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:30:09 2023","interfaces":[{"interface":"back","average":{"1min":1.534,"5min":4.548,"15min":2.1379999999999999},"min":{"1min":0.20799999999999999,"5min":0.191,"15min":0.17499999999999999},"max":{"1min":8.3230000000000004,"5min":128.60599999999999,"15min":128.60599999999999},"last":0.48799999999999999},{"interface":"front","average":{"1min":1.1930000000000001,"5min":4.3760000000000003,"15min":2.1480000000000001},"min":{"1min":0.221,"5min":0.188,"15min":0.188},"max":{"1min":3.8220000000000001,"5min":128.685,"15min":128.685},"last":0.98799999999999999}]},{"osd":2,"last update":"Mon Aug 28 02:29:33 2023","interfaces":[{"interface":"back","average":{"1min":1.379,"5min":6.8920000000000003,"15min":2.8149999999999999},"min":{"1min":0.33600000000000002,"5min":0.222,"15min":0.222},"max":{"1min":5.3490000000000002,"5min":135.25999999999999,"15min":135.25999999999999},"last":0.22},{"interface":"front","average":{"1min":1.4610000000000001,"5min":6.4219999999999997,"15min":2.6560000000000001},"min":{"1min":0.23499999999999999,"5min":0.23200000000000001,"15min":0.221},"max":{"1min":4.3150000000000004,"5min":135.10300000000001,"15min":135.10300000000001},"last":0.26700000000000002}]},{"osd":3,"last update":"Mon Aug 28 02:30:09 2023","interfaces":[{"interface":"back","average":{"1min":1.6899999999999999,"5min":3.7010000000000001,"15min":1.843},"min":{"1min":0.33600000000000002,"5min":0.22,"15min":0.22},"max":{"1min":3.867,"5min":123.741,"15min":123.741},"last":1.016},{"interface":"front","average":{"1min":3.9060000000000001,"5min":4.4829999999999997,"15min":2.1459999999999999},"min":{"1min":0.501,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":13.340999999999999,"5min":123.642,"15min":123.642},"last":0.30099999999999999}]},{"osd":4,"last update":"Mon Aug 28 02:30:09 2023","interfaces":[{"interface":"back","average":{"1min":1.6930000000000001,"5min":1.7789999999999999,"15min":1.264},"min":{"1min":0.29999999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.2939999999999996,"5min":17.600000000000001,"15min":17.600000000000001},"last":0.80800000000000005},{"interface":"front","average":{"1min":1.2170000000000001,"5min":1.6220000000000001,"15min":1.137},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.25600000000000001},"max":{"1min":3.5920000000000001,"5min":19.010000000000002,"15min":19.010000000000002},"last":0.752}]},{"osd":5,"last update":"Mon Aug 28 02:30:09 2023","interfaces":[{"interface":"back","average":{"1min":2.242,"5min":1.6910000000000001,"15min":1.371},"min":{"1min":0.32100000000000001,"5min":0.312,"15min":0.29399999999999998},"max":{"1min":11.167999999999999,"5min":11.167999999999999,"15min":15.544},"last":0.78600000000000003},{"interface":"front","average":{"1min":2.2440000000000002,"5min":1.847,"15min":1.2549999999999999},"min":{"1min":0.34599999999999997,"5min":0.33600000000000002,"15min":0.33200000000000002},"max":{"1min":10.084,"5min":17.646999999999998,"15min":17.646999999999998},"last":0.40699999999999997}]},{"osd":6,"last update":"Mon Aug 28 02:30:09 2023","interfaces":[{"interface":"back","average":{"1min":1.7170000000000001,"5min":1.7090000000000001,"15min":1.4630000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":3.923,"5min":9.2989999999999995,"15min":15.512},"last":0.96299999999999997},{"interface":"front","average":{"1min":1.8160000000000001,"5min":1.823,"15min":1.3899999999999999},"min":{"1min":0.39600000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":6.3140000000000001,"5min":17.673999999999999,"15min":17.673999999999999},"last":4.3550000000000004}]},{"osd":7,"last update":"Mon Aug 28 02:30:09 2023","interfaces":[{"interface":"back","average":{"1min":1.4059999999999999,"5min":1.7909999999999999,"15min":1.252},"min":{"1min":0.38500000000000001,"5min":0.32500000000000001,"15min":0.30199999999999999},"max":{"1min":4.0460000000000003,"5min":19.061,"15min":19.061},"last":0.56499999999999995},{"interface":"front","average":{"1min":1.7889999999999999,"5min":1.905,"15min":1.4450000000000001},"min":{"1min":0.437,"5min":0.40500000000000003,"15min":0.33100000000000002},"max":{"1min":5.8890000000000002,"5min":9.3239999999999998,"15min":15.476000000000001},"last":0.92700000000000005}]}]},{"osd":0,"up_from":90,"seq":386547056784,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":572900,"kb_used_data":243804,"kb_used_omap":8,"kb_used_meta":329079,"kb_avail":93172252,"statfs":{"total":95995035648,"available":95408386048,"internally_reserved":0,"allocated":249655296,"data_stored":248487050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":336977257},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Aug 28 02:30:17 2023","interfaces":[{"interface":"back","average":{"1min":5.1619999999999999,"5min":3.004,"15min":1.8919999999999999},"min":{"1min":0.20000000000000001,"5min":0.184,"15min":0.184},"max":{"1min":56.576999999999998,"5min":56.576999999999998,"15min":56.576999999999998},"last":0.86199999999999999},{"interface":"front","average":{"1min":6.1689999999999996,"5min":3.524,"15min":1.931},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.17000000000000001},"max":{"1min":56.552999999999997,"5min":56.552999999999997,"15min":56.552999999999997},"last":0.64000000000000001}]},{"osd":2,"last update":"Mon Aug 28 02:29:32 2023","interfaces":[{"interface":"back","average":{"1min":1.9750000000000001,"5min":4.5430000000000001,"15min":2.0099999999999998},"min":{"1min":0.23200000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":8.8629999999999995,"5min":100.718,"15min":100.718},"last":0.83999999999999997},{"interface":"front","average":{"1min":1.7749999999999999,"5min":5.3769999999999998,"15min":2.3530000000000002},"min":{"1min":0.254,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":5.766,"5min":100.962,"15min":100.962},"last":1.855}]},{"osd":3,"last update":"Mon Aug 28 02:30:17 2023","interfaces":[{"interface":"back","average":{"1min":2.6459999999999999,"5min":6.9390000000000001,"15min":2.9430000000000001},"min":{"1min":0.24299999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":19.327000000000002,"5min":237.43899999999999,"15min":237.43899999999999},"last":0.88900000000000001},{"interface":"front","average":{"1min":1.7390000000000001,"5min":6.6399999999999997,"15min":2.859},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":6.0010000000000003,"5min":237.48099999999999,"15min":237.48099999999999},"last":0.78200000000000003}]},{"osd":4,"last update":"Mon Aug 28 02:30:12 2023","interfaces":[{"interface":"back","average":{"1min":1.7709999999999999,"5min":1.6859999999999999,"15min":1.2849999999999999},"min":{"1min":0.29699999999999999,"5min":0.29599999999999999,"15min":0.26800000000000002},"max":{"1min":9.3680000000000003,"5min":9.3680000000000003,"15min":9.6649999999999991},"last":0.79500000000000004},{"interface":"front","average":{"1min":1.536,"5min":2.0870000000000002,"15min":1.4650000000000001},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":8.7880000000000003,"5min":27.271999999999998,"15min":27.271999999999998},"last":0.71699999999999997}]},{"osd":5,"last update":"Mon Aug 28 02:30:12 2023","interfaces":[{"interface":"back","average":{"1min":1.153,"5min":2.0099999999999998,"15min":1.5609999999999999},"min":{"1min":0.34000000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":3.0230000000000001,"5min":27.367000000000001,"15min":44.000999999999998},"last":0.754},{"interface":"front","average":{"1min":1.671,"5min":1.9079999999999999,"15min":1.488},"min":{"1min":0.34300000000000003,"5min":0.33000000000000002,"15min":0.30599999999999999},"max":{"1min":9.4489999999999998,"5min":12.334,"15min":43.953000000000003},"last":0.68100000000000005}]},{"osd":6,"last update":"Mon Aug 28 02:30:12 2023","interfaces":[{"interface":"back","average":{"1min":1.276,"5min":2.0819999999999999,"15min":1.585},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":3.0510000000000002,"5min":27.331,"15min":32.417999999999999},"last":1.8959999999999999},{"interface":"front","average":{"1min":1.6930000000000001,"5min":2.173,"15min":1.5620000000000001},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.30399999999999999},"max":{"1min":9.2789999999999999,"5min":27.247,"15min":32.130000000000003},"last":0.70199999999999996}]},{"osd":7,"last update":"Mon Aug 28 02:30:12 2023","interfaces":[{"interface":"back","average":{"1min":1.802,"5min":2.4769999999999999,"15min":1.583},"min":{"1min":0.36799999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":9.4209999999999994,"5min":27.385000000000002,"15min":27.385000000000002},"last":0.81899999999999995},{"interface":"front","average":{"1min":1.5069999999999999,"5min":2.339,"15min":1.599},"min":{"1min":0.38900000000000001,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":5.7300000000000004,"5min":27.213999999999999,"15min":27.213999999999999},"last":0.61499999999999999}]}]},{"osd":2,"up_from":104,"seq":446676598908,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1008884,"kb_used_data":684268,"kb_used_omap":10,"kb_used_meta":324469,"kb_avail":92736268,"statfs":{"total":95995035648,"available":94961938432,"internally_reserved":0,"allocated":700690432,"data_stored":699563447,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10792,"internal_metadata":332256728},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,3,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:29:36 2023","interfaces":[{"interface":"back","average":{"1min":1.3680000000000001,"5min":4.4729999999999999,"15min":2.1280000000000001},"min":{"1min":0.185,"5min":0.17699999999999999,"15min":0.17599999999999999},"max":{"1min":8.577,"5min":113.89100000000001,"15min":113.89100000000001},"last":16.48},{"interface":"front","average":{"1min":1.399,"5min":8.0090000000000003,"15min":3.3260000000000001},"min":{"1min":0.25900000000000001,"5min":0.214,"15min":0.19900000000000001},"max":{"1min":7.0129999999999999,"5min":147.286,"15min":147.286},"last":1.7669999999999999}]},{"osd":1,"last update":"Mon Aug 28 02:29:36 2023","interfaces":[{"interface":"back","average":{"1min":1.288,"5min":2.798,"15min":1.585},"min":{"1min":0.23499999999999999,"5min":0.21099999999999999,"15min":0.20999999999999999},"max":{"1min":7.0919999999999996,"5min":16.372,"15min":30.888000000000002},"last":1.625},{"interface":"front","average":{"1min":1.653,"5min":2.9119999999999999,"15min":1.488},"min":{"1min":0.29199999999999998,"5min":0.21299999999999999,"15min":0.20999999999999999},"max":{"1min":5.7400000000000002,"5min":20.259,"15min":31.648},"last":6.7359999999999998}]},{"osd":3,"last update":"Mon Aug 28 02:30:09 2023","interfaces":[{"interface":"back","average":{"1min":1.6799999999999999,"5min":5.1100000000000003,"15min":2.286},"min":{"1min":0.314,"5min":0.25,"15min":0.22700000000000001},"max":{"1min":9.7629999999999999,"5min":128.63900000000001,"15min":128.63900000000001},"last":1.663},{"interface":"front","average":{"1min":2.1070000000000002,"5min":4.9269999999999996,"15min":2.3010000000000002},"min":{"1min":0.375,"5min":0.223,"15min":0.223},"max":{"1min":8.7029999999999994,"5min":128.75700000000001,"15min":128.75700000000001},"last":6.7190000000000003}]},{"osd":4,"last update":"Mon Aug 28 02:29:36 2023","interfaces":[{"interface":"back","average":{"1min":7.9359999999999999,"5min":3.839,"15min":1.8700000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27100000000000002},"max":{"1min":130.27199999999999,"5min":130.27199999999999,"15min":130.27199999999999},"last":6.6740000000000004},{"interface":"front","average":{"1min":8.1839999999999993,"5min":3.1859999999999999,"15min":1.651},"min":{"1min":0.32600000000000001,"5min":0.28999999999999998,"15min":0.27400000000000002},"max":{"1min":132.21600000000001,"5min":132.21600000000001,"15min":132.21600000000001},"last":6.2039999999999997}]},{"osd":5,"last update":"Mon Aug 28 02:29:36 2023","interfaces":[{"interface":"back","average":{"1min":10.651,"5min":5.4619999999999997,"15min":2.4460000000000002},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":94.614000000000004,"5min":128.696,"15min":128.696},"last":1.6919999999999999},{"interface":"front","average":{"1min":10.534000000000001,"5min":5.585,"15min":2.496},"min":{"1min":0.32300000000000001,"5min":0.32100000000000001,"15min":0.30299999999999999},"max":{"1min":94.534000000000006,"5min":128.84100000000001,"15min":128.84100000000001},"last":6.6980000000000004}]},{"osd":6,"last update":"Mon Aug 28 02:29:36 2023","interfaces":[{"interface":"back","average":{"1min":1.472,"5min":1.8480000000000001,"15min":1.2609999999999999},"min":{"1min":0.33000000000000002,"5min":0.307,"15min":0.29299999999999998},"max":{"1min":4.798,"5min":11.285,"15min":11.285},"last":11.997},{"interface":"front","average":{"1min":1.333,"5min":2.1749999999999998,"15min":1.4079999999999999},"min":{"1min":0.39500000000000002,"5min":0.377,"15min":0.35899999999999999},"max":{"1min":7.4130000000000003,"5min":11.055999999999999,"15min":11.055999999999999},"last":1.72}]},{"osd":7,"last update":"Mon Aug 28 02:29:36 2023","interfaces":[{"interface":"back","average":{"1min":6.5789999999999997,"5min":5.3780000000000001,"15min":2.431},"min":{"1min":0.40400000000000003,"5min":0.34499999999999997,"15min":0.32100000000000001},"max":{"1min":100.65000000000001,"5min":128.911,"15min":128.911},"last":6.29},{"interface":"front","average":{"1min":6.9000000000000004,"5min":5.9660000000000002,"15min":2.637},"min":{"1min":0.54400000000000004,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":101.53100000000001,"5min":128.72399999999999,"15min":128.72399999999999},"last":3.415}]}]},{"osd":3,"up_from":112,"seq":481036337270,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":666796,"kb_used_data":418212,"kb_used_omap":3,"kb_used_meta":248572,"kb_avail":93078356,"statfs":{"total":95995035648,"available":95312236544,"internally_reserved":0,"allocated":428249088,"data_stored":427127163,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":254538658},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:30:05 2023","interfaces":[{"interface":"back","average":{"1min":7.0190000000000001,"5min":3.1560000000000001,"15min":3.0089999999999999},"min":{"1min":0.20000000000000001,"5min":0.193,"15min":0.193},"max":{"1min":79.793000000000006,"5min":79.793000000000006,"15min":164.25299999999999},"last":9.3670000000000009},{"interface":"front","average":{"1min":6.7839999999999998,"5min":2.8380000000000001,"15min":2.9620000000000002},"min":{"1min":0.27200000000000002,"5min":0.192,"15min":0.192},"max":{"1min":78.555000000000007,"5min":78.555000000000007,"15min":164.55000000000001},"last":7.407}]},{"osd":1,"last update":"Mon Aug 28 02:30:05 2023","interfaces":[{"interface":"back","average":{"1min":1.669,"5min":1.907,"15min":1.8680000000000001},"min":{"1min":0.245,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":6.1500000000000004,"5min":13.936,"15min":117.34699999999999},"last":2.3050000000000002},{"interface":"front","average":{"1min":2.5310000000000001,"5min":2.1499999999999999,"15min":1.968},"min":{"1min":0.27300000000000002,"5min":0.245,"15min":0.224},"max":{"1min":10.125,"5min":14.300000000000001,"15min":118.012},"last":9.2989999999999995}]},{"osd":2,"last update":"Mon Aug 28 02:30:05 2023","interfaces":[{"interface":"back","average":{"1min":2.4199999999999999,"5min":3.157,"15min":2.8140000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.19900000000000001},"max":{"1min":10.163,"5min":65.308999999999997,"15min":120.43300000000001},"last":6.9379999999999997},{"interface":"front","average":{"1min":2.3999999999999999,"5min":2.891,"15min":2.7599999999999998},"min":{"1min":0.33500000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":7.3620000000000001,"5min":65.120999999999995,"15min":120.44499999999999},"last":1.776}]},{"osd":4,"last update":"Mon Aug 28 02:30:05 2023","interfaces":[{"interface":"back","average":{"1min":1.403,"5min":2.4980000000000002,"15min":1.4610000000000001},"min":{"1min":0.32800000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":4.7240000000000002,"5min":58.067999999999998,"15min":58.067999999999998},"last":1.738},{"interface":"front","average":{"1min":1.6279999999999999,"5min":2.7589999999999999,"15min":1.5269999999999999},"min":{"1min":0.40400000000000003,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":4.6280000000000001,"5min":58.088000000000001,"15min":58.088000000000001},"last":2.2370000000000001}]},{"osd":5,"last update":"Mon Aug 28 02:30:05 2023","interfaces":[{"interface":"back","average":{"1min":1.359,"5min":3.2109999999999999,"15min":1.6579999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.29599999999999999},"max":{"1min":2.923,"5min":102.904,"15min":102.904},"last":2.2799999999999998},{"interface":"front","average":{"1min":1.556,"5min":3.2509999999999999,"15min":1.6619999999999999},"min":{"1min":0.36699999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":7.0039999999999996,"5min":103.283,"15min":103.283},"last":5.0069999999999997}]},{"osd":6,"last update":"Mon Aug 28 02:30:05 2023","interfaces":[{"interface":"back","average":{"1min":1.5880000000000001,"5min":1.877,"15min":1.4330000000000001},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.314},"max":{"1min":7.0300000000000002,"5min":9.4529999999999994,"15min":72.116},"last":4.9740000000000002},{"interface":"front","average":{"1min":1.7929999999999999,"5min":1.538,"15min":1.3560000000000001},"min":{"1min":0.35599999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":7.2990000000000004,"5min":7.2990000000000004,"15min":72.030000000000001},"last":7.4710000000000001}]},{"osd":7,"last update":"Mon Aug 28 02:30:05 2023","interfaces":[{"interface":"back","average":{"1min":1.6879999999999999,"5min":2.9660000000000002,"15min":1.631},"min":{"1min":0.38300000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":4.7640000000000002,"5min":54.427999999999997,"15min":54.427999999999997},"last":7.4379999999999997},{"interface":"front","average":{"1min":1.855,"5min":3.3399999999999999,"15min":1.76},"min":{"1min":0.433,"5min":0.36399999999999999,"15min":0.34799999999999998},"max":{"1min":3.8100000000000001,"5min":54.445,"15min":54.445},"last":2.625}]}]},{"osd":4,"up_from":30,"seq":128849019144,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":468824,"kb_used_data":167504,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93276328,"statfs":{"total":95995035648,"available":95514959872,"internally_reserved":0,"allocated":171524096,"data_stored":170820986,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:30:10 2023","interfaces":[{"interface":"back","average":{"1min":1.528,"5min":2.8479999999999999,"15min":2.706},"min":{"1min":0.222,"5min":0.219,"15min":0.219},"max":{"1min":13.492000000000001,"5min":43.548000000000002,"15min":138.67400000000001},"last":3.996},{"interface":"front","average":{"1min":1.4850000000000001,"5min":2.7149999999999999,"15min":2.637},"min":{"1min":0.251,"5min":0.248,"15min":0.248},"max":{"1min":8.0060000000000002,"5min":43.515000000000001,"15min":134.14599999999999},"last":0.38300000000000001}]},{"osd":1,"last update":"Mon Aug 28 02:30:10 2023","interfaces":[{"interface":"back","average":{"1min":1.5629999999999999,"5min":2.1640000000000001,"15min":1.3919999999999999},"min":{"1min":0.23599999999999999,"5min":0.221,"15min":0.221},"max":{"1min":9.75,"5min":34.945999999999998,"15min":53.622999999999998},"last":1.381},{"interface":"front","average":{"1min":1.4039999999999999,"5min":2.073,"15min":1.4079999999999999},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":5.2919999999999998,"5min":34.869999999999997,"15min":53.774000000000001},"last":1.278}]},{"osd":2,"last update":"Mon Aug 28 02:29:34 2023","interfaces":[{"interface":"back","average":{"1min":3.3130000000000002,"5min":4.3470000000000004,"15min":2.5369999999999999},"min":{"1min":0.27400000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":40.226999999999997,"5min":64.063000000000002,"15min":74.099000000000004},"last":1.861},{"interface":"front","average":{"1min":3.4729999999999999,"5min":4.0979999999999999,"15min":2.4279999999999999},"min":{"1min":0.28999999999999998,"5min":0.28599999999999998,"15min":0.25},"max":{"1min":39.039000000000001,"5min":64.073999999999998,"15min":74.081999999999994},"last":0.64900000000000002}]},{"osd":3,"last update":"Mon Aug 28 02:30:10 2023","interfaces":[{"interface":"back","average":{"1min":11.220000000000001,"5min":6.0819999999999999,"15min":2.5099999999999998},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":159.10400000000001,"5min":159.10400000000001,"15min":159.10400000000001},"last":4.4690000000000003},{"interface":"front","average":{"1min":11.119999999999999,"5min":5.9939999999999998,"15min":2.5979999999999999},"min":{"1min":0.33800000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":159.232,"5min":159.232,"15min":159.232},"last":0.73399999999999999}]},{"osd":5,"last update":"Mon Aug 28 02:29:50 2023","interfaces":[{"interface":"back","average":{"1min":0.78100000000000003,"5min":2.8140000000000001,"15min":1.397},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":4.3090000000000002,"5min":124.718,"15min":124.718},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.73599999999999999,"5min":3.1949999999999998,"15min":1.554},"min":{"1min":0.27800000000000002,"5min":0.247,"15min":0.247},"max":{"1min":4.665,"5min":124.694,"15min":124.694},"last":0.39900000000000002}]},{"osd":6,"last update":"Mon Aug 28 02:30:10 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":1.264,"15min":0.94699999999999995},"min":{"1min":0.28499999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":2.6659999999999999,"5min":11.170999999999999,"15min":11.170999999999999},"last":0.498},{"interface":"front","average":{"1min":0.91500000000000004,"5min":1.1180000000000001,"15min":0.83299999999999996},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":5.2679999999999998,"5min":7.3170000000000002,"15min":7.3170000000000002},"last":0.46600000000000003}]},{"osd":7,"last update":"Mon Aug 28 02:30:27 2023","interfaces":[{"interface":"back","average":{"1min":1.651,"5min":1.6990000000000001,"15min":1.1180000000000001},"min":{"1min":0.312,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":8.3490000000000002,"5min":16.584,"15min":16.584},"last":0.51400000000000001},{"interface":"front","average":{"1min":1.706,"5min":1.3919999999999999,"15min":1.137},"min":{"1min":0.378,"5min":0.311,"15min":0.311},"max":{"1min":7.4619999999999997,"5min":10.826000000000001,"15min":10.826000000000001},"last":0.53700000000000003}]}]},{"osd":5,"up_from":36,"seq":154618822916,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":512524,"kb_used_data":203012,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93232628,"statfs":{"total":95995035648,"available":95470211072,"internally_reserved":0,"allocated":207884288,"data_stored":207188645,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:30:02 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":2.052,"15min":2.6469999999999998},"min":{"1min":0.214,"5min":0.214,"15min":0.20300000000000001},"max":{"1min":4.548,"5min":42.229999999999997,"15min":131.06100000000001},"last":2.6640000000000001},{"interface":"front","average":{"1min":1.4710000000000001,"5min":2.371,"15min":2.8460000000000001},"min":{"1min":0.27500000000000002,"5min":0.24199999999999999,"15min":0.22900000000000001},"max":{"1min":5.8129999999999997,"5min":42.213999999999999,"15min":131.262},"last":2.6739999999999999}]},{"osd":1,"last update":"Mon Aug 28 02:30:12 2023","interfaces":[{"interface":"back","average":{"1min":1.8819999999999999,"5min":1.7869999999999999,"15min":2.387},"min":{"1min":0.27400000000000002,"5min":0.216,"15min":0.216},"max":{"1min":7.7809999999999997,"5min":30.593,"15min":183.97900000000001},"last":2.3159999999999998},{"interface":"front","average":{"1min":1.1220000000000001,"5min":2.0070000000000001,"15min":2.5510000000000002},"min":{"1min":0.23499999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":5.8810000000000002,"5min":34.616999999999997,"15min":184.33699999999999},"last":3.2330000000000001}]},{"osd":2,"last update":"Mon Aug 28 02:29:39 2023","interfaces":[{"interface":"back","average":{"1min":1.6240000000000001,"5min":6.2750000000000004,"15min":3.831},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.252},"max":{"1min":8.9600000000000009,"5min":176.69499999999999,"15min":176.69499999999999},"last":0.47699999999999998},{"interface":"front","average":{"1min":2.2440000000000002,"5min":6.5730000000000004,"15min":4.0019999999999998},"min":{"1min":0.34000000000000002,"5min":0.29799999999999999,"15min":0.28299999999999997},"max":{"1min":8.9489999999999998,"5min":176.71199999999999,"15min":176.71199999999999},"last":0.42099999999999999}]},{"osd":3,"last update":"Mon Aug 28 02:30:12 2023","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":2.8730000000000002,"15min":1.6180000000000001},"min":{"1min":0.28799999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.1070000000000002,"5min":102.64400000000001,"15min":102.64400000000001},"last":0.623},{"interface":"front","average":{"1min":1.8919999999999999,"5min":3.6509999999999998,"15min":1.956},"min":{"1min":0.28699999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":8.1660000000000004,"5min":103.771,"15min":103.771},"last":0.73299999999999998}]},{"osd":4,"last update":"Mon Aug 28 02:29:50 2023","interfaces":[{"interface":"back","average":{"1min":1.8300000000000001,"5min":1.157,"15min":0.93100000000000005},"min":{"1min":0.26700000000000002,"5min":0.246,"15min":0.246},"max":{"1min":25.306999999999999,"5min":25.306999999999999,"15min":25.306999999999999},"last":0.35699999999999998},{"interface":"front","average":{"1min":1.7250000000000001,"5min":1.2430000000000001,"15min":0.95899999999999996},"min":{"1min":0.29399999999999998,"5min":0.28999999999999998,"15min":0.28000000000000003},"max":{"1min":25.347000000000001,"5min":25.347000000000001,"15min":25.347000000000001},"last":0.32900000000000001}]},{"osd":6,"last update":"Mon Aug 28 02:30:12 2023","interfaces":[{"interface":"back","average":{"1min":1.0429999999999999,"5min":1.099,"15min":1.002},"min":{"1min":0.30399999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.6210000000000004,"5min":5.9870000000000001,"15min":14.803000000000001},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.95499999999999996,"5min":1.04,"15min":0.91600000000000004},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":4.9509999999999996,"5min":8.4779999999999998,"15min":8.6479999999999997},"last":0.54800000000000004}]},{"osd":7,"last update":"Mon Aug 28 02:30:29 2023","interfaces":[{"interface":"back","average":{"1min":1.1419999999999999,"5min":1.5109999999999999,"15min":1.127},"min":{"1min":0.34300000000000003,"5min":0.33300000000000002,"15min":0.31},"max":{"1min":4.0060000000000002,"5min":25.379000000000001,"15min":25.379000000000001},"last":0.57399999999999995},{"interface":"front","average":{"1min":1.3160000000000001,"5min":1.2769999999999999,"15min":1.0569999999999999},"min":{"1min":0.38300000000000001,"5min":0.33700000000000002,"15min":0.32100000000000001},"max":{"1min":5.931,"5min":25.437000000000001,"15min":25.437000000000001},"last":0.39100000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":229576704,"data_stored":229572632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":374083584,"data_stored":374079512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":694157312,"data_stored":694157312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":420347904,"data_stored":420347904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":159252480,"data_stored":159252480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":204083200,"data_stored":204079128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":155582464,"data_stored":155582464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":550305792,"data_stored":550305792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-08-28T02:30:36.615 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph status --format=json 2023-08-28T02:30:36.656 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:30:36.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:36.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:30:36.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:36 smithi169 ceph-mon[159415]: from='client.55183 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:30:36.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:36 smithi169 ceph-mon[159415]: pgmap v620: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 443 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 813 op/s; 6088/21144 objects degraded (28.793%); 883/21144 objects misplaced (4.176%); 1.4 MiB/s, 23 objects/s recovering 2023-08-28T02:30:36.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:36 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 6088/21144 objects degraded (28.793%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:30:36.781 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:30:36.873 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:36 smithi143 ceph-mon[183059]: from='client.55183 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:30:36.873 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:36 smithi143 ceph-mon[183059]: pgmap v620: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 443 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 813 op/s; 6088/21144 objects degraded (28.793%); 883/21144 objects misplaced (4.176%); 1.4 MiB/s, 23 objects/s recovering 2023-08-28T02:30:36.873 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:36 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 6088/21144 objects degraded (28.793%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:30:36.873 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:36 smithi143 ceph-mon[189871]: from='client.55183 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:30:36.873 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:36 smithi143 ceph-mon[189871]: pgmap v620: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 443 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 813 op/s; 6088/21144 objects degraded (28.793%); 883/21144 objects misplaced (4.176%); 1.4 MiB/s, 23 objects/s recovering 2023-08-28T02:30:36.873 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:36 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 6088/21144 objects degraded (28.793%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:30:36.876 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:36.947 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:30:37.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:37.040 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:30:37.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:30:37.141 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:30:37.199 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:37.361 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:30:37.372 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.4 dump_blocked_ops 2023-08-28T02:30:37.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:37.373 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:30:37.475 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:30:37.546 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:37.812 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:30:37.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:37.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:30:37.918 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:30:38.000 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:38.074 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:30:38.092 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:30:38.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:38.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:30:38.198 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:30:38.260 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:38.420 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:30:38.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:38.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:30:38.507 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:38 smithi169 ceph-mon[159415]: pgmap v621: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 338 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 6050/16083 objects degraded (37.617%); 805/16083 objects misplaced (5.005%); 1.1 MiB/s, 23 objects/s recovering 2023-08-28T02:30:38.526 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:30:38.599 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:38.637 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:30:38.671 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:30:38.797 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:30:38.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:38.802 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:30:38.903 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:30:38.948 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:38 smithi143 ceph-mon[183059]: pgmap v621: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 338 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 6050/16083 objects degraded (37.617%); 805/16083 objects misplaced (5.005%); 1.1 MiB/s, 23 objects/s recovering 2023-08-28T02:30:38.949 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:38 smithi143 ceph-mon[189871]: pgmap v621: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 338 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 6050/16083 objects degraded (37.617%); 805/16083 objects misplaced (5.005%); 1.1 MiB/s, 23 objects/s recovering 2023-08-28T02:30:39.025 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:39.148 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:30:39.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:39.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:30:39.252 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:30:39.298 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:39.567 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:39 smithi143 ceph-mon[183059]: pgmap v622: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 237 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 6050/11235 objects degraded (53.850%); 346/11235 objects misplaced (3.080%); 1.1 MiB/s, 23 objects/s recovering 2023-08-28T02:30:39.570 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:30:39.572 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:39 smithi169 ceph-mon[159415]: pgmap v622: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 237 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 6050/11235 objects degraded (53.850%); 346/11235 objects misplaced (3.080%); 1.1 MiB/s, 23 objects/s recovering 2023-08-28T02:30:39.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:39.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:30:39.679 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:30:39.713 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:39.855 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:30:39.862 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:39 smithi143 ceph-mon[189871]: pgmap v622: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 237 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 6050/11235 objects degraded (53.850%); 346/11235 objects misplaced (3.080%); 1.1 MiB/s, 23 objects/s recovering 2023-08-28T02:30:39.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:39.866 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:30:39.967 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:30:40.066 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:40.130 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:30:40.130 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:30:40.182 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:30:40.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:40.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:30:40.289 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:30:40.346 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:40.425 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:30:40.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:40.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:30:40.444 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:30:40.529 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:30:40.605 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is unset 2023-08-28T02:30:40.614 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:40.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:40 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1840597132' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:30:40.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:40 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:30:40.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:40 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1840597132' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:30:40.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:40 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:30:40.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:40 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1840597132' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:30:40.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:40 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:30:40.747 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:30:40.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:40.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:30:40.853 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:30:40.924 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:41.138 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.4 dump_historic_ops 2023-08-28T02:30:41.513 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:30:41.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:41.522 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:30:41.623 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:30:41.690 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:41.750 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:30:41.813 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:30:41.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:41.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:30:41.848 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:41 smithi143 ceph-mon[189871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:30:41.848 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:41 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:30:41.848 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:41 smithi143 ceph-mon[189871]: osdmap e257: 8 total, 8 up, 8 in 2023-08-28T02:30:41.848 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:41 smithi143 ceph-mon[189871]: failed to encode map e257 with expected crc 2023-08-28T02:30:41.848 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:41 smithi143 ceph-mon[189871]: failed to encode map e257 with expected crc 2023-08-28T02:30:41.849 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:41 smithi143 ceph-mon[189871]: failed to encode map e257 with expected crc 2023-08-28T02:30:41.849 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:41 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:30:41.849 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:41 smithi143 ceph-mon[189871]: pgmap v624: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 203 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 6050/9597 objects degraded (63.041%); 346/9597 objects misplaced (3.605%); 475 KiB/s, 13 objects/s recovering 2023-08-28T02:30:41.849 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:41 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 6050/11235 objects degraded (53.850%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:30:41.849 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:41 smithi143 ceph-mon[183059]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:30:41.849 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:41 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:30:41.849 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:41 smithi143 ceph-mon[183059]: osdmap e257: 8 total, 8 up, 8 in 2023-08-28T02:30:41.849 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:41 smithi143 ceph-mon[183059]: failed to encode map e257 with expected crc 2023-08-28T02:30:41.849 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:41 smithi143 ceph-mon[183059]: failed to encode map e257 with expected crc 2023-08-28T02:30:41.849 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:41 smithi143 ceph-mon[183059]: failed to encode map e257 with expected crc 2023-08-28T02:30:41.850 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:41 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:30:41.850 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:41 smithi143 ceph-mon[183059]: pgmap v624: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 203 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 6050/9597 objects degraded (63.041%); 346/9597 objects misplaced (3.605%); 475 KiB/s, 13 objects/s recovering 2023-08-28T02:30:41.850 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:41 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 6050/11235 objects degraded (53.850%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:30:41.925 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:30:41.964 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:41 smithi169 ceph-mon[159415]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:30:41.965 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:41 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:30:41.965 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:41 smithi169 ceph-mon[159415]: osdmap e257: 8 total, 8 up, 8 in 2023-08-28T02:30:41.965 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:41 smithi169 ceph-mon[159415]: failed to encode map e257 with expected crc 2023-08-28T02:30:41.965 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:41 smithi169 ceph-mon[159415]: failed to encode map e257 with expected crc 2023-08-28T02:30:41.965 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:41 smithi169 ceph-mon[159415]: failed to encode map e257 with expected crc 2023-08-28T02:30:41.965 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:41 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:30:41.965 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:41 smithi169 ceph-mon[159415]: pgmap v624: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 203 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 6050/9597 objects degraded (63.041%); 346/9597 objects misplaced (3.605%); 475 KiB/s, 13 objects/s recovering 2023-08-28T02:30:41.965 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:41 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 6050/11235 objects degraded (53.850%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:30:41.969 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:42.053 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:30:42.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:42.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:30:42.161 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:30:42.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:30:41 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:30:41] "GET /metrics HTTP/1.1" 200 36484 "" "Prometheus/2.33.4" 2023-08-28T02:30:42.238 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:42.288 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:30:42.289 INFO:teuthology.orchestra.run.smithi143.stdout:{"fsid":"2fcb0bfa-4547-11ee-817c-1b7a5f1ba315","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 6050/9597 objects degraded (63.041%), 4 pgs degraded, 4 pgs undersized","count":8},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":799,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":257,"num_osds":8,"num_up_osds":8,"osd_up_since":1693189249,"num_in_osds":8,"osd_in_since":1693189704,"num_remapped_pgs":6},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":93},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":3},{"state_name":"active+remapped+backfill_wait","count":2},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":99,"num_pools":5,"num_objects":3199,"data_bytes":212539339,"bytes_used":4107796480,"bytes_avail":763852488704,"bytes_total":767960285184,"degraded_objects":6050,"degraded_total":9597,"degraded_ratio":0.63040533500052098,"misplaced_objects":346,"misplaced_total":9597,"misplaced_ratio":0.036052933208294259,"recovering_objects_per_sec":13,"recovering_bytes_per_sec":486452,"recovering_keys_per_sec":0,"num_objects_recovered":116,"num_bytes_recovered":4325376,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":697},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi143.spifrj","status":"up:active","gid":14637}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.143:8443/","prometheus":"http://172.21.15.143:9283/"}},"servicemap":{"epoch":22,"modified":"2023-08-28T02:21:49.036237+0000","services":{}},"progress_events":{"3f0d3518-bd25-4431-af15-5a1c1658296d":{"message":"Global Recovery Event (2m)\n [==========================..] (remaining: 11s)","progress":0.93939393758773804,"add_to_ceph_s":true},"8e5f590e-829c-445f-9a5b-24676b2537ad":{"message":"Upgrade to 18.2.0-187-g9cf20820 (3m)\n [============................] (remaining: 4m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-08-28T02:30:42.293 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:30:42.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:42.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:30:42.397 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:30:42.445 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:42.538 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:30:42.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:42.544 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:30:42.645 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:30:42.684 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:42.911 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Removed 83616 objects 2023-08-28T02:30:42.912 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Clean up completed and total clean up time :125.754 2023-08-28T02:30:42.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:42 smithi143 ceph-mon[183059]: 6.9 scrub starts 2023-08-28T02:30:42.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:42 smithi143 ceph-mon[183059]: 6.7 scrub starts 2023-08-28T02:30:42.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:42 smithi143 ceph-mon[183059]: 6.2 scrub starts 2023-08-28T02:30:42.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:42 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/575069895' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-08-28T02:30:42.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:42 smithi143 ceph-mon[189871]: 6.9 scrub starts 2023-08-28T02:30:42.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:42 smithi143 ceph-mon[189871]: 6.7 scrub starts 2023-08-28T02:30:42.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:42 smithi143 ceph-mon[189871]: 6.2 scrub starts 2023-08-28T02:30:42.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:42 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/575069895' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-08-28T02:30:42.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:42 smithi169 ceph-mon[159415]: 6.9 scrub starts 2023-08-28T02:30:42.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:42 smithi169 ceph-mon[159415]: 6.7 scrub starts 2023-08-28T02:30:42.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:42 smithi169 ceph-mon[159415]: 6.2 scrub starts 2023-08-28T02:30:42.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:42 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/575069895' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-08-28T02:30:43.278 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:30:43.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:43.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:30:43.364 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-08-28T02:30:43.364 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph pg dump --format=json 2023-08-28T02:30:43.381 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:30:43.410 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:43.509 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:30:43.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:43.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:30:43.595 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset nodeep-scrub 2023-08-28T02:30:43.613 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:30:43.643 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:43.696 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:30:43.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:43.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:30:43.740 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:30:43.740 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:30:43.762 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:43 smithi169 ceph-mon[159415]: 6.0 scrub starts 2023-08-28T02:30:43.763 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:43 smithi169 ceph-mon[159415]: 6.4 scrub starts 2023-08-28T02:30:43.763 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:43 smithi169 ceph-mon[159415]: 6.19 scrub starts 2023-08-28T02:30:43.763 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:43 smithi169 ceph-mon[159415]: pgmap v625: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 109 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 5941/5115 objects degraded (116.149%); 346/5115 objects misplaced (6.764%); 422 KiB/s, 22 objects/s recovering 2023-08-28T02:30:43.801 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:30:43.826 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:43.839 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:30:43.858 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:30:43.889 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:30:43.910 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:30:43.929 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:30:43.948 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:30:43.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:43 smithi143 ceph-mon[189871]: 6.0 scrub starts 2023-08-28T02:30:43.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:43 smithi143 ceph-mon[189871]: 6.4 scrub starts 2023-08-28T02:30:43.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:43 smithi143 ceph-mon[189871]: 6.19 scrub starts 2023-08-28T02:30:43.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:43 smithi143 ceph-mon[189871]: pgmap v625: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 109 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 5941/5115 objects degraded (116.149%); 346/5115 objects misplaced (6.764%); 422 KiB/s, 22 objects/s recovering 2023-08-28T02:30:43.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:43 smithi143 ceph-mon[183059]: 6.0 scrub starts 2023-08-28T02:30:43.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:43 smithi143 ceph-mon[183059]: 6.4 scrub starts 2023-08-28T02:30:43.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:43 smithi143 ceph-mon[183059]: 6.19 scrub starts 2023-08-28T02:30:43.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:43 smithi143 ceph-mon[183059]: pgmap v625: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 109 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 5941/5115 objects degraded (116.149%); 346/5115 objects misplaced (6.764%); 422 KiB/s, 22 objects/s recovering 2023-08-28T02:30:43.965 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:30:43.983 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:30:44.001 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:30:44.002 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:30:44.002 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:30:44.002 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:30:44.002 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:30:44.003 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:30:44.003 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:30:44.003 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:30:44.003 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:30:44.024 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:30:44.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:44.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:30:44.128 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:30:44.153 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:44.201 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:30:44.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:44.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:30:44.305 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:30:44.331 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:44.554 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:30:44.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:44.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:30:44.613 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.5 dump_ops_in_flight 2023-08-28T02:30:44.657 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:30:44.687 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:44.882 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:30:44.883 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:44 smithi143 ceph-mon[183059]: 6.7 scrub starts 2023-08-28T02:30:44.884 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:44 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2797829106' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:30:44.884 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:44 smithi143 ceph-mon[189871]: 6.7 scrub starts 2023-08-28T02:30:44.884 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:44 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2797829106' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:30:44.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:44.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:30:44.970 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:30:44.985 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:30:44.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:44 smithi169 ceph-mon[159415]: 6.7 scrub starts 2023-08-28T02:30:44.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:44 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2797829106' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:30:45.014 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:45.134 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:30:45.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:45.136 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:30:45.237 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:30:45.248 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:30:45.265 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:45.314 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:30:45.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:45.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:30:45.417 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:30:45.450 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:45.646 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:30:45.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:45.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:30:45.750 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:30:45.780 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:45.856 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:30:45.858 INFO:teuthology.orchestra.run.smithi143.stderr:dumped all 2023-08-28T02:30:45.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:45 smithi143 ceph-mon[183059]: 6.10 scrub starts 2023-08-28T02:30:45.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:45 smithi143 ceph-mon[183059]: 6.14 scrub starts 2023-08-28T02:30:45.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:45 smithi143 ceph-mon[183059]: 6.2 scrub starts 2023-08-28T02:30:45.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:45 smithi143 ceph-mon[183059]: pgmap v626: 99 pgs: 2 active+clean+scrubbing, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 23 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 5941/984 objects degraded (603.760%); 422 KiB/s, 22 objects/s recovering 2023-08-28T02:30:45.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:45 smithi143 ceph-mon[189871]: 6.10 scrub starts 2023-08-28T02:30:45.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:45 smithi143 ceph-mon[189871]: 6.14 scrub starts 2023-08-28T02:30:45.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:45 smithi143 ceph-mon[189871]: 6.2 scrub starts 2023-08-28T02:30:45.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:45 smithi143 ceph-mon[189871]: pgmap v626: 99 pgs: 2 active+clean+scrubbing, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 23 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 5941/984 objects degraded (603.760%); 422 KiB/s, 22 objects/s recovering 2023-08-28T02:30:45.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:45 smithi169 ceph-mon[159415]: 6.10 scrub starts 2023-08-28T02:30:45.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:45 smithi169 ceph-mon[159415]: 6.14 scrub starts 2023-08-28T02:30:45.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:45 smithi169 ceph-mon[159415]: 6.2 scrub starts 2023-08-28T02:30:45.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:45 smithi169 ceph-mon[159415]: pgmap v626: 99 pgs: 2 active+clean+scrubbing, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 23 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 5941/984 objects degraded (603.760%); 422 KiB/s, 22 objects/s recovering 2023-08-28T02:30:46.485 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:30:46.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:46.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:30:46.589 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:30:46.617 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:30:46.617 INFO:teuthology.orchestra.run.smithi169.stdout: "ops": [], 2023-08-28T02:30:46.617 INFO:teuthology.orchestra.run.smithi169.stdout: "num_ops": 0 2023-08-28T02:30:46.618 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:30:46.618 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:46.666 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:30:46.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:46.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:30:46.694 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2023-08-28T02:30:46.695 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2023-08-28T02:30:46.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:46 smithi169 ceph-mon[159415]: from='client.35301 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:30:46.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:46 smithi169 ceph-mon[159415]: 6.9 scrub starts 2023-08-28T02:30:46.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:46 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 5941/984 objects degraded (603.760%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:30:46.770 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:30:46.797 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:46.842 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:30:46.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:46.844 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:30:46.881 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:46 smithi143 ceph-mon[183059]: from='client.35301 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:30:46.881 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:46 smithi143 ceph-mon[183059]: 6.9 scrub starts 2023-08-28T02:30:46.881 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:46 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 5941/984 objects degraded (603.760%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:30:46.881 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:46 smithi143 ceph-mon[189871]: from='client.35301 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:30:46.881 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:46 smithi143 ceph-mon[189871]: 6.9 scrub starts 2023-08-28T02:30:46.882 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:46 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 5941/984 objects degraded (603.760%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:30:46.945 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:30:46.964 INFO:teuthology.orchestra.run.smithi143.stdout:{"pg_ready":true,"pg_map":{"version":626,"stamp":"2023-08-28T02:30:45.320178+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":24451019,"num_objects":328,"num_object_clones":0,"num_object_copies":984,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5941,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":328,"num_whiteouts":0,"num_read":6275,"num_read_kb":13708,"num_write":168171,"num_write_kb":5366566,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20616,"num_bytes_recovered":1181766729,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":190573,"ondisk_log_size":190573,"up":297,"acting":293,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":309,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":3371900,"kb_used_data":1005692,"kb_used_omap":29,"kb_used_meta":2365858,"kb_avail":746589316,"statfs":{"total":767960285184,"available":764507459584,"internally_reserved":0,"allocated":1029828608,"data_stored":1022320009,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30674,"internal_metadata":2422638638},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-440336384,"num_objects":-6720,"num_object_clones":0,"num_object_copies":-20160,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-147,"num_objects_misplaced":-883,"num_objects_unfound":0,"num_objects_dirty":-6720,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6851,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":225,"num_bytes_recovered":4325376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.014772"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":258,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.432476+0000","last_change":"2023-08-28T02:27:31.367469+0000","last_active":"2023-08-28T02:30:32.432476+0000","last_peered":"2023-08-28T02:30:32.432476+0000","last_clean":"2023-08-28T02:30:32.432476+0000","last_became_active":"2023-08-28T02:27:31.366946+0000","last_became_peered":"2023-08-28T02:27:31.366946+0000","last_unstale":"2023-08-28T02:30:32.432476+0000","last_undegraded":"2023-08-28T02:30:32.432476+0000","last_fullsized":"2023-08-28T02:30:32.432476+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":143,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.432150+0000","last_change":"2023-08-28T02:27:31.324452+0000","last_active":"2023-08-28T02:30:32.432150+0000","last_peered":"2023-08-28T02:30:32.432150+0000","last_clean":"2023-08-28T02:30:32.432150+0000","last_became_active":"2023-08-28T02:27:31.323749+0000","last_became_peered":"2023-08-28T02:27:31.323749+0000","last_unstale":"2023-08-28T02:30:32.432150+0000","last_undegraded":"2023-08-28T02:30:32.432150+0000","last_fullsized":"2023-08-28T02:30:32.432150+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:18:10.904127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":64,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.438237+0000","last_change":"2023-08-28T02:27:31.337338+0000","last_active":"2023-08-28T02:30:32.438237+0000","last_peered":"2023-08-28T02:30:32.438237+0000","last_clean":"2023-08-28T02:30:32.438237+0000","last_became_active":"2023-08-28T02:27:31.337018+0000","last_became_peered":"2023-08-28T02:27:31.337018+0000","last_unstale":"2023-08-28T02:30:32.438237+0000","last_undegraded":"2023-08-28T02:30:32.438237+0000","last_fullsized":"2023-08-28T02:30:32.438237+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:46:34.845663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":245,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:38.924961+0000","last_change":"2023-08-28T02:23:59.223634+0000","last_active":"2023-08-28T02:30:38.924961+0000","last_peered":"2023-08-28T02:30:38.924961+0000","last_clean":"2023-08-28T02:30:38.924961+0000","last_became_active":"2023-08-28T02:23:59.222704+0000","last_became_peered":"2023-08-28T02:23:59.222704+0000","last_unstale":"2023-08-28T02:30:38.924961+0000","last_undegraded":"2023-08-28T02:30:38.924961+0000","last_fullsized":"2023-08-28T02:30:38.924961+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:51:52.752361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":134,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.431599+0000","last_change":"2023-08-28T02:27:31.318153+0000","last_active":"2023-08-28T02:30:32.431599+0000","last_peered":"2023-08-28T02:30:32.431599+0000","last_clean":"2023-08-28T02:30:32.431599+0000","last_became_active":"2023-08-28T02:27:31.317732+0000","last_became_peered":"2023-08-28T02:27:31.317732+0000","last_unstale":"2023-08-28T02:30:32.431599+0000","last_undegraded":"2023-08-28T02:30:32.431599+0000","last_fullsized":"2023-08-28T02:30:32.431599+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:06:13.421674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":46,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.457565+0000","last_change":"2023-08-28T02:28:25.348941+0000","last_active":"2023-08-28T02:30:32.457565+0000","last_peered":"2023-08-28T02:30:32.457565+0000","last_clean":"2023-08-28T02:30:32.457565+0000","last_became_active":"2023-08-28T02:28:25.348650+0000","last_became_peered":"2023-08-28T02:28:25.348650+0000","last_unstale":"2023-08-28T02:30:32.457565+0000","last_undegraded":"2023-08-28T02:30:32.457565+0000","last_fullsized":"2023-08-28T02:30:32.457565+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.12","version":"257'7048","reported_seq":7138,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:41.883404+0000","last_change":"2023-08-28T02:28:25.453536+0000","last_active":"2023-08-28T02:30:41.883404+0000","last_peered":"2023-08-28T02:30:41.883404+0000","last_clean":"2023-08-28T02:30:41.883404+0000","last_became_active":"2023-08-28T02:27:58.479936+0000","last_became_peered":"2023-08-28T02:27:58.479936+0000","last_unstale":"2023-08-28T02:30:41.883404+0000","last_undegraded":"2023-08-28T02:30:41.883404+0000","last_fullsized":"2023-08-28T02:30:41.883404+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":7048,"log_dups_size":0,"ondisk_log_size":7048,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3211264,"num_objects":49,"num_object_clones":0,"num_object_copies":147,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":49,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5183,"num_write_kb":167424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":225,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:40.609931+0000","last_change":"2023-08-28T02:23:59.227592+0000","last_active":"2023-08-28T02:30:40.609931+0000","last_peered":"2023-08-28T02:30:40.609931+0000","last_clean":"2023-08-28T02:30:40.609931+0000","last_became_active":"2023-08-28T02:23:59.227342+0000","last_became_peered":"2023-08-28T02:23:59.227342+0000","last_unstale":"2023-08-28T02:30:40.609931+0000","last_undegraded":"2023-08-28T02:30:40.609931+0000","last_fullsized":"2023-08-28T02:30:40.609931+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:08:19.874396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":152,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.431616+0000","last_change":"2023-08-28T02:28:25.348233+0000","last_active":"2023-08-28T02:30:32.431616+0000","last_peered":"2023-08-28T02:30:32.431616+0000","last_clean":"2023-08-28T02:30:32.431616+0000","last_became_active":"2023-08-28T02:28:25.347896+0000","last_became_peered":"2023-08-28T02:28:25.347896+0000","last_unstale":"2023-08-28T02:30:32.431616+0000","last_undegraded":"2023-08-28T02:30:32.431616+0000","last_fullsized":"2023-08-28T02:30:32.431616+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:50:17.674564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.13","version":"257'7203","reported_seq":7311,"reported_epoch":257,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:30:42.029667+0000","last_change":"2023-08-28T02:28:26.468919+0000","last_active":"2023-08-28T02:30:42.029667+0000","last_peered":"2023-08-28T02:30:42.029667+0000","last_clean":"2023-08-28T02:27:16.315150+0000","last_became_active":"2023-08-28T02:28:26.463939+0000","last_became_peered":"2023-08-28T02:28:26.463939+0000","last_unstale":"2023-08-28T02:30:42.029667+0000","last_undegraded":"2023-08-28T02:28:26.355850+0000","last_fullsized":"2023-08-28T02:28:26.347877+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":7203,"log_dups_size":0,"ondisk_log_size":7203,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1703,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5315,"num_write_kb":170048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":-1703}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":259,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.452364+0000","last_change":"2023-08-28T02:27:31.328499+0000","last_active":"2023-08-28T02:30:32.452364+0000","last_peered":"2023-08-28T02:30:32.452364+0000","last_clean":"2023-08-28T02:30:32.452364+0000","last_became_active":"2023-08-28T02:27:31.328164+0000","last_became_peered":"2023-08-28T02:27:31.328164+0000","last_unstale":"2023-08-28T02:30:32.452364+0000","last_undegraded":"2023-08-28T02:30:32.452364+0000","last_fullsized":"2023-08-28T02:30:32.452364+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":134,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.431982+0000","last_change":"2023-08-28T02:28:25.348760+0000","last_active":"2023-08-28T02:30:32.431982+0000","last_peered":"2023-08-28T02:30:32.431982+0000","last_clean":"2023-08-28T02:30:32.431982+0000","last_became_active":"2023-08-28T02:28:25.348465+0000","last_became_peered":"2023-08-28T02:28:25.348465+0000","last_unstale":"2023-08-28T02:30:32.431982+0000","last_undegraded":"2023-08-28T02:30:32.431982+0000","last_fullsized":"2023-08-28T02:30:32.431982+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:43:05.659999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.10","version":"257'6961","reported_seq":7052,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:42.649037+0000","last_change":"2023-08-28T02:30:17.869105+0000","last_active":"2023-08-28T02:30:42.649037+0000","last_peered":"2023-08-28T02:30:42.649037+0000","last_clean":"2023-08-28T02:30:42.649037+0000","last_became_active":"2023-08-28T02:27:58.494108+0000","last_became_peered":"2023-08-28T02:27:58.494108+0000","last_unstale":"2023-08-28T02:30:42.649037+0000","last_undegraded":"2023-08-28T02:30:42.649037+0000","last_fullsized":"2023-08-28T02:30:42.649037+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6961,"log_dups_size":0,"ondisk_log_size":6961,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":786432,"num_objects":12,"num_object_clones":0,"num_object_copies":36,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5124,"num_write_kb":164352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":142,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.455087+0000","last_change":"2023-08-28T02:23:59.316561+0000","last_active":"2023-08-28T02:30:32.455087+0000","last_peered":"2023-08-28T02:30:32.455087+0000","last_clean":"2023-08-28T02:30:32.455087+0000","last_became_active":"2023-08-28T02:23:59.227433+0000","last_became_peered":"2023-08-28T02:23:59.227433+0000","last_unstale":"2023-08-28T02:30:32.455087+0000","last_undegraded":"2023-08-28T02:30:32.455087+0000","last_fullsized":"2023-08-28T02:30:32.455087+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":74,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.437693+0000","last_change":"2023-08-28T02:28:25.347771+0000","last_active":"2023-08-28T02:30:32.437693+0000","last_peered":"2023-08-28T02:30:32.437693+0000","last_clean":"2023-08-28T02:30:32.437693+0000","last_became_active":"2023-08-28T02:28:25.347320+0000","last_became_peered":"2023-08-28T02:28:25.347320+0000","last_unstale":"2023-08-28T02:30:32.437693+0000","last_undegraded":"2023-08-28T02:30:32.437693+0000","last_fullsized":"2023-08-28T02:30:32.437693+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:20:27.350131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.11","version":"257'7157","reported_seq":14076,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:42.658483+0000","last_change":"2023-08-28T02:30:11.322180+0000","last_active":"2023-08-28T02:30:42.658483+0000","last_peered":"2023-08-28T02:30:42.658483+0000","last_clean":"2023-08-28T02:30:42.658483+0000","last_became_active":"2023-08-28T02:29:58.475435+0000","last_became_peered":"2023-08-28T02:29:58.475435+0000","last_unstale":"2023-08-28T02:30:42.658483+0000","last_undegraded":"2023-08-28T02:30:42.658483+0000","last_fullsized":"2023-08-28T02:30:42.658483+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":7157,"log_dups_size":0,"ondisk_log_size":7157,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":655360,"num_objects":10,"num_object_clones":0,"num_object_copies":30,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5282,"num_write_kb":169344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2439,"num_bytes_recovered":159678464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"86'22","reported_seq":307,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.430747+0000","last_change":"2023-08-28T02:28:25.361783+0000","last_active":"2023-08-28T02:30:32.430747+0000","last_peered":"2023-08-28T02:30:32.430747+0000","last_clean":"2023-08-28T02:30:32.430747+0000","last_became_active":"2023-08-28T02:28:25.351418+0000","last_became_peered":"2023-08-28T02:28:25.351418+0000","last_unstale":"2023-08-28T02:30:32.430747+0000","last_undegraded":"2023-08-28T02:30:32.430747+0000","last_fullsized":"2023-08-28T02:30:32.430747+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":273,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.430519+0000","last_change":"2023-08-28T02:28:25.352395+0000","last_active":"2023-08-28T02:30:32.430519+0000","last_peered":"2023-08-28T02:30:32.430519+0000","last_clean":"2023-08-28T02:30:32.430519+0000","last_became_active":"2023-08-28T02:28:25.352189+0000","last_became_peered":"2023-08-28T02:28:25.352189+0000","last_unstale":"2023-08-28T02:30:32.430519+0000","last_undegraded":"2023-08-28T02:30:32.430519+0000","last_fullsized":"2023-08-28T02:30:32.430519+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.16","version":"257'7057","reported_seq":7147,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:42.644876+0000","last_change":"2023-08-28T02:27:58.492359+0000","last_active":"2023-08-28T02:30:42.644876+0000","last_peered":"2023-08-28T02:30:42.644876+0000","last_clean":"2023-08-28T02:30:42.644876+0000","last_became_active":"2023-08-28T02:27:58.479842+0000","last_became_peered":"2023-08-28T02:27:58.479842+0000","last_unstale":"2023-08-28T02:30:42.644876+0000","last_undegraded":"2023-08-28T02:30:42.644876+0000","last_fullsized":"2023-08-28T02:30:42.644876+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":7057,"log_dups_size":0,"ondisk_log_size":7057,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5172,"num_write_kb":165344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":127,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.453815+0000","last_change":"2023-08-28T02:23:38.983634+0000","last_active":"2023-08-28T02:30:32.453815+0000","last_peered":"2023-08-28T02:30:32.453815+0000","last_clean":"2023-08-28T02:30:32.453815+0000","last_became_active":"2023-08-28T02:23:38.344566+0000","last_became_peered":"2023-08-28T02:23:38.344566+0000","last_unstale":"2023-08-28T02:30:32.453815+0000","last_undegraded":"2023-08-28T02:30:32.453815+0000","last_fullsized":"2023-08-28T02:30:32.453815+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":134,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.432336+0000","last_change":"2023-08-28T02:27:31.344974+0000","last_active":"2023-08-28T02:30:32.432336+0000","last_peered":"2023-08-28T02:30:32.432336+0000","last_clean":"2023-08-28T02:30:32.432336+0000","last_became_active":"2023-08-28T02:27:31.344281+0000","last_became_peered":"2023-08-28T02:27:31.344281+0000","last_unstale":"2023-08-28T02:30:32.432336+0000","last_undegraded":"2023-08-28T02:30:32.432336+0000","last_fullsized":"2023-08-28T02:30:32.432336+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:04:02.821752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.17","version":"257'6990","reported_seq":7077,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:41.902836+0000","last_change":"2023-08-28T02:27:58.474771+0000","last_active":"2023-08-28T02:30:41.902836+0000","last_peered":"2023-08-28T02:30:41.902836+0000","last_clean":"2023-08-28T02:30:41.902836+0000","last_became_active":"2023-08-28T02:27:58.472976+0000","last_became_peered":"2023-08-28T02:27:58.472976+0000","last_unstale":"2023-08-28T02:30:41.902836+0000","last_undegraded":"2023-08-28T02:30:41.902836+0000","last_fullsized":"2023-08-28T02:30:41.902836+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6990,"log_dups_size":0,"ondisk_log_size":6990,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1310720,"num_objects":20,"num_object_clones":0,"num_object_copies":60,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":20,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5134,"num_write_kb":164928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":243,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.448160+0000","last_change":"2023-08-28T02:23:59.223066+0000","last_active":"2023-08-28T02:30:32.448160+0000","last_peered":"2023-08-28T02:30:32.448160+0000","last_clean":"2023-08-28T02:30:32.448160+0000","last_became_active":"2023-08-28T02:23:59.222731+0000","last_became_peered":"2023-08-28T02:23:59.222731+0000","last_unstale":"2023-08-28T02:30:32.448160+0000","last_undegraded":"2023-08-28T02:30:32.448160+0000","last_fullsized":"2023-08-28T02:30:32.448160+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:48:37.558266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":283,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.452357+0000","last_change":"2023-08-28T02:23:59.210469+0000","last_active":"2023-08-28T02:30:32.452357+0000","last_peered":"2023-08-28T02:30:32.452357+0000","last_clean":"2023-08-28T02:30:32.452357+0000","last_became_active":"2023-08-28T02:23:59.210031+0000","last_became_peered":"2023-08-28T02:23:59.210031+0000","last_unstale":"2023-08-28T02:30:32.452357+0000","last_undegraded":"2023-08-28T02:30:32.452357+0000","last_fullsized":"2023-08-28T02:30:32.452357+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.14","version":"257'7189","reported_seq":12709,"reported_epoch":257,"state":"active+clean+scrubbing","last_fresh":"2023-08-28T02:30:42.573018+0000","last_change":"2023-08-28T02:30:44.449330+0000","last_active":"2023-08-28T02:30:42.573018+0000","last_peered":"2023-08-28T02:30:42.573018+0000","last_clean":"2023-08-28T02:30:42.573018+0000","last_became_active":"2023-08-28T02:28:00.020960+0000","last_became_peered":"2023-08-28T02:28:00.020960+0000","last_unstale":"2023-08-28T02:30:42.573018+0000","last_undegraded":"2023-08-28T02:30:42.573018+0000","last_fullsized":"2023-08-28T02:30:42.573018+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":7189,"log_dups_size":0,"ondisk_log_size":7189,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5357,"num_write_kb":170848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1961,"num_bytes_recovered":128483328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":139,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.452486+0000","last_change":"2023-08-28T02:28:25.354526+0000","last_active":"2023-08-28T02:30:32.452486+0000","last_peered":"2023-08-28T02:30:32.452486+0000","last_clean":"2023-08-28T02:30:32.452486+0000","last_became_active":"2023-08-28T02:28:25.350991+0000","last_became_peered":"2023-08-28T02:28:25.350991+0000","last_unstale":"2023-08-28T02:30:32.452486+0000","last_undegraded":"2023-08-28T02:30:32.452486+0000","last_fullsized":"2023-08-28T02:30:32.452486+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":244,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:40.609633+0000","last_change":"2023-08-28T02:23:59.220932+0000","last_active":"2023-08-28T02:30:40.609633+0000","last_peered":"2023-08-28T02:30:40.609633+0000","last_clean":"2023-08-28T02:30:40.609633+0000","last_became_active":"2023-08-28T02:23:59.220488+0000","last_became_peered":"2023-08-28T02:23:59.220488+0000","last_unstale":"2023-08-28T02:30:40.609633+0000","last_undegraded":"2023-08-28T02:30:40.609633+0000","last_fullsized":"2023-08-28T02:30:40.609633+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:53:09.216340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.15","version":"257'7151","reported_seq":8430,"reported_epoch":257,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:30:42.162617+0000","last_change":"2023-08-28T02:28:25.456367+0000","last_active":"2023-08-28T02:30:42.162617+0000","last_peered":"2023-08-28T02:30:42.162617+0000","last_clean":"2023-08-28T02:27:16.315756+0000","last_became_active":"2023-08-28T02:28:25.449323+0000","last_became_peered":"2023-08-28T02:28:25.449323+0000","last_unstale":"2023-08-28T02:30:42.162617+0000","last_undegraded":"2023-08-28T02:28:25.349175+0000","last_fullsized":"2023-08-28T02:28:25.327058+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":7151,"log_dups_size":0,"ondisk_log_size":7151,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1690,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5267,"num_write_kb":168192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":294,"num_bytes_recovered":19267584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"3,7","objects":-1690}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":56,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.452509+0000","last_change":"2023-08-28T02:28:25.703333+0000","last_active":"2023-08-28T02:30:32.452509+0000","last_peered":"2023-08-28T02:30:32.452509+0000","last_clean":"2023-08-28T02:30:32.452509+0000","last_became_active":"2023-08-28T02:28:25.343515+0000","last_became_peered":"2023-08-28T02:28:25.343515+0000","last_unstale":"2023-08-28T02:30:32.452509+0000","last_undegraded":"2023-08-28T02:30:32.452509+0000","last_fullsized":"2023-08-28T02:30:32.452509+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":134,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.432378+0000","last_change":"2023-08-28T02:27:31.330059+0000","last_active":"2023-08-28T02:30:32.432378+0000","last_peered":"2023-08-28T02:30:32.432378+0000","last_clean":"2023-08-28T02:30:32.432378+0000","last_became_active":"2023-08-28T02:27:31.329769+0000","last_became_peered":"2023-08-28T02:27:31.329769+0000","last_unstale":"2023-08-28T02:30:32.432378+0000","last_undegraded":"2023-08-28T02:30:32.432378+0000","last_fullsized":"2023-08-28T02:30:32.432378+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:14:54.349909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"257'10514","reported_seq":13728,"reported_epoch":257,"state":"active+remapped+backfill_wait","last_fresh":"2023-08-28T02:30:42.665364+0000","last_change":"2023-08-28T02:29:15.939050+0000","last_active":"2023-08-28T02:30:42.665364+0000","last_peered":"2023-08-28T02:30:42.665364+0000","last_clean":"2023-08-28T02:28:24.280158+0000","last_became_active":"2023-08-28T02:28:26.465041+0000","last_became_peered":"2023-08-28T02:28:26.465041+0000","last_unstale":"2023-08-28T02:30:42.665364+0000","last_undegraded":"2023-08-28T02:30:42.665364+0000","last_fullsized":"2023-08-28T02:30:42.665364+0000","mapping_epoch":221,"log_start":"200'1900","ondisk_log_start":"200'1900","created":187,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":8614,"log_dups_size":0,"ondisk_log_size":8614,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:30:34.822957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10514,"num_write_kb":336448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1547,"num_bytes_recovered":101384192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[0,5,2],"avail_no_missing":["0","2","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":125,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.452285+0000","last_change":"2023-08-28T02:23:38.657185+0000","last_active":"2023-08-28T02:30:32.452285+0000","last_peered":"2023-08-28T02:30:32.452285+0000","last_clean":"2023-08-28T02:30:32.452285+0000","last_became_active":"2023-08-28T02:23:38.339016+0000","last_became_peered":"2023-08-28T02:23:38.339016+0000","last_unstale":"2023-08-28T02:30:32.452285+0000","last_undegraded":"2023-08-28T02:30:32.452285+0000","last_fullsized":"2023-08-28T02:30:32.452285+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":243,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.452014+0000","last_change":"2023-08-28T02:28:25.348106+0000","last_active":"2023-08-28T02:30:32.452014+0000","last_peered":"2023-08-28T02:30:32.452014+0000","last_clean":"2023-08-28T02:30:32.452014+0000","last_became_active":"2023-08-28T02:28:25.347786+0000","last_became_peered":"2023-08-28T02:28:25.347786+0000","last_unstale":"2023-08-28T02:30:32.452014+0000","last_undegraded":"2023-08-28T02:30:32.452014+0000","last_fullsized":"2023-08-28T02:30:32.452014+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"257'10494","reported_seq":15429,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:42.907437+0000","last_change":"2023-08-28T02:29:38.520992+0000","last_active":"2023-08-28T02:30:42.907437+0000","last_peered":"2023-08-28T02:30:42.907437+0000","last_clean":"2023-08-28T02:30:42.907437+0000","last_became_active":"2023-08-28T02:29:38.501413+0000","last_became_peered":"2023-08-28T02:29:38.501413+0000","last_unstale":"2023-08-28T02:30:42.907437+0000","last_undegraded":"2023-08-28T02:30:42.907437+0000","last_fullsized":"2023-08-28T02:30:42.907437+0000","mapping_epoch":242,"log_start":"217'4200","ondisk_log_start":"217'4200","created":187,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6294,"log_dups_size":0,"ondisk_log_size":6294,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:45:22.978036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10494,"num_write_kb":335808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2408,"num_bytes_recovered":149291008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":151,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.452459+0000","last_change":"2023-08-28T02:27:31.395328+0000","last_active":"2023-08-28T02:30:32.452459+0000","last_peered":"2023-08-28T02:30:32.452459+0000","last_clean":"2023-08-28T02:30:32.452459+0000","last_became_active":"2023-08-28T02:27:31.328228+0000","last_became_peered":"2023-08-28T02:27:31.328228+0000","last_unstale":"2023-08-28T02:30:32.452459+0000","last_undegraded":"2023-08-28T02:30:32.452459+0000","last_fullsized":"2023-08-28T02:30:32.452459+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":292,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.430788+0000","last_change":"2023-08-28T02:27:31.344915+0000","last_active":"2023-08-28T02:30:32.430788+0000","last_peered":"2023-08-28T02:30:32.430788+0000","last_clean":"2023-08-28T02:30:32.430788+0000","last_became_active":"2023-08-28T02:27:31.344577+0000","last_became_peered":"2023-08-28T02:27:31.344577+0000","last_unstale":"2023-08-28T02:30:32.430788+0000","last_undegraded":"2023-08-28T02:30:32.430788+0000","last_fullsized":"2023-08-28T02:30:32.430788+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"257'7156","reported_seq":7244,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:42.111757+0000","last_change":"2023-08-28T02:27:58.457913+0000","last_active":"2023-08-28T02:30:42.111757+0000","last_peered":"2023-08-28T02:30:42.111757+0000","last_clean":"2023-08-28T02:30:42.111757+0000","last_became_active":"2023-08-28T02:27:58.451333+0000","last_became_peered":"2023-08-28T02:27:58.451333+0000","last_unstale":"2023-08-28T02:30:42.111757+0000","last_undegraded":"2023-08-28T02:30:42.111757+0000","last_fullsized":"2023-08-28T02:30:42.111757+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":7156,"log_dups_size":0,"ondisk_log_size":7156,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5279,"num_write_kb":167328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":151,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.455337+0000","last_change":"2023-08-28T02:28:25.749249+0000","last_active":"2023-08-28T02:30:32.455337+0000","last_peered":"2023-08-28T02:30:32.455337+0000","last_clean":"2023-08-28T02:30:32.455337+0000","last_became_active":"2023-08-28T02:28:25.397684+0000","last_became_peered":"2023-08-28T02:28:25.397684+0000","last_unstale":"2023-08-28T02:30:32.455337+0000","last_undegraded":"2023-08-28T02:30:32.455337+0000","last_fullsized":"2023-08-28T02:30:32.455337+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":46,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.455249+0000","last_change":"2023-08-28T02:28:25.350802+0000","last_active":"2023-08-28T02:30:32.455249+0000","last_peered":"2023-08-28T02:30:32.455249+0000","last_clean":"2023-08-28T02:30:32.455249+0000","last_became_active":"2023-08-28T02:28:25.350474+0000","last_became_peered":"2023-08-28T02:28:25.350474+0000","last_unstale":"2023-08-28T02:30:32.455249+0000","last_undegraded":"2023-08-28T02:30:32.455249+0000","last_fullsized":"2023-08-28T02:30:32.455249+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"257'7014","reported_seq":7108,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:42.611218+0000","last_change":"2023-08-28T02:30:41.307401+0000","last_active":"2023-08-28T02:30:42.611218+0000","last_peered":"2023-08-28T02:30:42.611218+0000","last_clean":"2023-08-28T02:30:42.611218+0000","last_became_active":"2023-08-28T02:27:58.433161+0000","last_became_peered":"2023-08-28T02:27:58.433161+0000","last_unstale":"2023-08-28T02:30:42.611218+0000","last_undegraded":"2023-08-28T02:30:42.611218+0000","last_fullsized":"2023-08-28T02:30:42.611218+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":7014,"log_dups_size":0,"ondisk_log_size":7014,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":819200,"num_objects":12,"num_object_clones":0,"num_object_copies":36,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5144,"num_write_kb":165024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":259,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.432462+0000","last_change":"2023-08-28T02:28:25.445982+0000","last_active":"2023-08-28T02:30:32.432462+0000","last_peered":"2023-08-28T02:30:32.432462+0000","last_clean":"2023-08-28T02:30:32.432462+0000","last_became_active":"2023-08-28T02:28:25.443577+0000","last_became_peered":"2023-08-28T02:28:25.443577+0000","last_unstale":"2023-08-28T02:30:32.432462+0000","last_undegraded":"2023-08-28T02:30:32.432462+0000","last_fullsized":"2023-08-28T02:30:32.432462+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:24:03.381751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":237,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.449252+0000","last_change":"2023-08-28T02:27:31.319783+0000","last_active":"2023-08-28T02:30:32.449252+0000","last_peered":"2023-08-28T02:30:32.449252+0000","last_clean":"2023-08-28T02:30:32.449252+0000","last_became_active":"2023-08-28T02:27:31.319451+0000","last_became_peered":"2023-08-28T02:27:31.319451+0000","last_unstale":"2023-08-28T02:30:32.449252+0000","last_undegraded":"2023-08-28T02:30:32.449252+0000","last_fullsized":"2023-08-28T02:30:32.449252+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:20:37.097889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"257'10314","reported_seq":15237,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:41.883388+0000","last_change":"2023-08-28T02:29:17.383106+0000","last_active":"2023-08-28T02:30:41.883388+0000","last_peered":"2023-08-28T02:30:41.883388+0000","last_clean":"2023-08-28T02:30:41.883388+0000","last_became_active":"2023-08-28T02:29:17.367223+0000","last_became_peered":"2023-08-28T02:29:17.367223+0000","last_unstale":"2023-08-28T02:30:41.883388+0000","last_undegraded":"2023-08-28T02:30:41.883388+0000","last_fullsized":"2023-08-28T02:30:41.883388+0000","mapping_epoch":234,"log_start":"202'2400","ondisk_log_start":"202'2400","created":187,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":7914,"log_dups_size":0,"ondisk_log_size":7914,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1441792,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10314,"num_write_kb":330752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2409,"num_bytes_recovered":127533056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":121,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:40.598832+0000","last_change":"2023-08-28T02:23:59.230059+0000","last_active":"2023-08-28T02:30:40.598832+0000","last_peered":"2023-08-28T02:30:40.598832+0000","last_clean":"2023-08-28T02:30:40.598832+0000","last_became_active":"2023-08-28T02:23:59.217783+0000","last_became_peered":"2023-08-28T02:23:59.217783+0000","last_unstale":"2023-08-28T02:30:40.598832+0000","last_undegraded":"2023-08-28T02:30:40.598832+0000","last_fullsized":"2023-08-28T02:30:40.598832+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:00:12.145395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":277,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.430720+0000","last_change":"2023-08-28T02:28:25.351698+0000","last_active":"2023-08-28T02:30:32.430720+0000","last_peered":"2023-08-28T02:30:32.430720+0000","last_clean":"2023-08-28T02:30:32.430720+0000","last_became_active":"2023-08-28T02:28:25.351349+0000","last_became_peered":"2023-08-28T02:28:25.351349+0000","last_unstale":"2023-08-28T02:30:32.430720+0000","last_undegraded":"2023-08-28T02:30:32.430720+0000","last_fullsized":"2023-08-28T02:30:32.430720+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"257'10506","reported_seq":10586,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:42.647734+0000","last_change":"2023-08-28T02:26:57.390417+0000","last_active":"2023-08-28T02:30:42.647734+0000","last_peered":"2023-08-28T02:30:42.647734+0000","last_clean":"2023-08-28T02:30:42.647734+0000","last_became_active":"2023-08-28T02:26:57.390124+0000","last_became_peered":"2023-08-28T02:26:57.390124+0000","last_unstale":"2023-08-28T02:30:42.647734+0000","last_undegraded":"2023-08-28T02:30:42.647734+0000","last_fullsized":"2023-08-28T02:30:42.647734+0000","mapping_epoch":187,"log_start":"200'2100","ondisk_log_start":"200'2100","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":8406,"log_dups_size":0,"ondisk_log_size":8406,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:38:48.957967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10506,"num_write_kb":336192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":46,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.455378+0000","last_change":"2023-08-28T02:28:25.345274+0000","last_active":"2023-08-28T02:30:32.455378+0000","last_peered":"2023-08-28T02:30:32.455378+0000","last_clean":"2023-08-28T02:30:32.455378+0000","last_became_active":"2023-08-28T02:28:25.344885+0000","last_became_peered":"2023-08-28T02:28:25.344885+0000","last_unstale":"2023-08-28T02:30:32.455378+0000","last_undegraded":"2023-08-28T02:30:32.455378+0000","last_fullsized":"2023-08-28T02:30:32.455378+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":303,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.430871+0000","last_change":"2023-08-28T02:27:31.358436+0000","last_active":"2023-08-28T02:30:32.430871+0000","last_peered":"2023-08-28T02:30:32.430871+0000","last_clean":"2023-08-28T02:30:32.430871+0000","last_became_active":"2023-08-28T02:27:31.358185+0000","last_became_peered":"2023-08-28T02:27:31.358185+0000","last_unstale":"2023-08-28T02:30:32.430871+0000","last_undegraded":"2023-08-28T02:30:32.430871+0000","last_fullsized":"2023-08-28T02:30:32.430871+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"257'10638","reported_seq":13895,"reported_epoch":257,"state":"active+remapped+backfill_wait","last_fresh":"2023-08-28T02:30:42.910339+0000","last_change":"2023-08-28T02:28:26.455411+0000","last_active":"2023-08-28T02:30:42.910339+0000","last_peered":"2023-08-28T02:30:42.910339+0000","last_clean":"2023-08-28T02:28:24.332252+0000","last_became_active":"2023-08-28T02:28:26.447621+0000","last_became_peered":"2023-08-28T02:28:26.447621+0000","last_unstale":"2023-08-28T02:30:42.910339+0000","last_undegraded":"2023-08-28T02:30:42.910339+0000","last_fullsized":"2023-08-28T02:30:42.910339+0000","mapping_epoch":221,"log_start":"215'4300","ondisk_log_start":"215'4300","created":187,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6338,"log_dups_size":0,"ondisk_log_size":6338,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:28:52.578588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10638,"num_write_kb":340416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1574,"num_bytes_recovered":103153664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,5,7],"avail_no_missing":["3","5","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":166,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.430646+0000","last_change":"2023-08-28T02:23:38.347043+0000","last_active":"2023-08-28T02:30:32.430646+0000","last_peered":"2023-08-28T02:30:32.430646+0000","last_clean":"2023-08-28T02:30:32.430646+0000","last_became_active":"2023-08-28T02:23:38.346680+0000","last_became_peered":"2023-08-28T02:23:38.346680+0000","last_unstale":"2023-08-28T02:30:32.430646+0000","last_undegraded":"2023-08-28T02:30:32.430646+0000","last_fullsized":"2023-08-28T02:30:32.430646+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":223,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.448118+0000","last_change":"2023-08-28T02:28:25.347494+0000","last_active":"2023-08-28T02:30:32.448118+0000","last_peered":"2023-08-28T02:30:32.448118+0000","last_clean":"2023-08-28T02:30:32.448118+0000","last_became_active":"2023-08-28T02:28:25.347142+0000","last_became_peered":"2023-08-28T02:28:25.347142+0000","last_unstale":"2023-08-28T02:30:32.448118+0000","last_undegraded":"2023-08-28T02:30:32.448118+0000","last_fullsized":"2023-08-28T02:30:32.448118+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:37:14.384884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.d","version":"257'10120","reported_seq":14851,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:41.913641+0000","last_change":"2023-08-28T02:29:35.551357+0000","last_active":"2023-08-28T02:30:41.913641+0000","last_peered":"2023-08-28T02:30:41.913641+0000","last_clean":"2023-08-28T02:30:41.913641+0000","last_became_active":"2023-08-28T02:29:35.530821+0000","last_became_peered":"2023-08-28T02:29:35.530821+0000","last_unstale":"2023-08-28T02:30:41.913641+0000","last_undegraded":"2023-08-28T02:30:41.913641+0000","last_fullsized":"2023-08-28T02:30:41.913641+0000","mapping_epoch":240,"log_start":"194'100","ondisk_log_start":"194'100","created":187,"last_epoch_clean":241,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":10020,"log_dups_size":0,"ondisk_log_size":10020,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2293784,"num_objects":36,"num_object_clones":0,"num_object_copies":108,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":36,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10120,"num_write_kb":324929,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2312,"num_bytes_recovered":47775744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":135,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.431756+0000","last_change":"2023-08-28T02:27:31.325260+0000","last_active":"2023-08-28T02:30:32.431756+0000","last_peered":"2023-08-28T02:30:32.431756+0000","last_clean":"2023-08-28T02:30:32.431756+0000","last_became_active":"2023-08-28T02:27:31.324950+0000","last_became_peered":"2023-08-28T02:27:31.324950+0000","last_unstale":"2023-08-28T02:30:32.431756+0000","last_undegraded":"2023-08-28T02:30:32.431756+0000","last_fullsized":"2023-08-28T02:30:32.431756+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:21:33.833261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":65,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.434411+0000","last_change":"2023-08-28T02:27:31.322118+0000","last_active":"2023-08-28T02:30:32.434411+0000","last_peered":"2023-08-28T02:30:32.434411+0000","last_clean":"2023-08-28T02:30:32.434411+0000","last_became_active":"2023-08-28T02:27:31.321338+0000","last_became_peered":"2023-08-28T02:27:31.321338+0000","last_unstale":"2023-08-28T02:30:32.434411+0000","last_undegraded":"2023-08-28T02:30:32.434411+0000","last_fullsized":"2023-08-28T02:30:32.434411+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:56:14.210571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"257'6959","reported_seq":7055,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:42.649129+0000","last_change":"2023-08-28T02:30:42.438934+0000","last_active":"2023-08-28T02:30:42.649129+0000","last_peered":"2023-08-28T02:30:42.649129+0000","last_clean":"2023-08-28T02:30:42.649129+0000","last_became_active":"2023-08-28T02:27:58.437499+0000","last_became_peered":"2023-08-28T02:27:58.437499+0000","last_unstale":"2023-08-28T02:30:42.649129+0000","last_undegraded":"2023-08-28T02:30:42.649129+0000","last_fullsized":"2023-08-28T02:30:42.649129+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6959,"log_dups_size":0,"ondisk_log_size":6959,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5122,"num_write_kb":163520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"78'126","reported_seq":627,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:34.567047+0000","last_change":"2023-08-28T02:28:27.383137+0000","last_active":"2023-08-28T02:30:34.567047+0000","last_peered":"2023-08-28T02:30:34.567047+0000","last_clean":"2023-08-28T02:30:34.567047+0000","last_became_active":"2023-08-28T02:28:27.382488+0000","last_became_peered":"2023-08-28T02:28:27.382488+0000","last_unstale":"2023-08-28T02:30:34.567047+0000","last_undegraded":"2023-08-28T02:30:34.567047+0000","last_fullsized":"2023-08-28T02:30:34.567047+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:14:48.414426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":743,"num_read_kb":1331,"num_write":125,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":256,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.452279+0000","last_change":"2023-08-28T02:23:59.207148+0000","last_active":"2023-08-28T02:30:32.452279+0000","last_peered":"2023-08-28T02:30:32.452279+0000","last_clean":"2023-08-28T02:30:32.452279+0000","last_became_active":"2023-08-28T02:23:59.206802+0000","last_became_peered":"2023-08-28T02:23:59.206802+0000","last_unstale":"2023-08-28T02:30:32.452279+0000","last_undegraded":"2023-08-28T02:30:32.452279+0000","last_fullsized":"2023-08-28T02:30:32.452279+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":65,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.437616+0000","last_change":"2023-08-28T02:27:31.328027+0000","last_active":"2023-08-28T02:30:32.437616+0000","last_peered":"2023-08-28T02:30:32.437616+0000","last_clean":"2023-08-28T02:30:32.437616+0000","last_became_active":"2023-08-28T02:27:31.326529+0000","last_became_peered":"2023-08-28T02:27:31.326529+0000","last_unstale":"2023-08-28T02:30:32.437616+0000","last_undegraded":"2023-08-28T02:30:32.437616+0000","last_fullsized":"2023-08-28T02:30:32.437616+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:56:57.331355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"257'7154","reported_seq":7244,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:42.627242+0000","last_change":"2023-08-28T02:27:58.462974+0000","last_active":"2023-08-28T02:30:42.627242+0000","last_peered":"2023-08-28T02:30:42.627242+0000","last_clean":"2023-08-28T02:30:42.627242+0000","last_became_active":"2023-08-28T02:27:58.460293+0000","last_became_peered":"2023-08-28T02:27:58.460293+0000","last_unstale":"2023-08-28T02:30:42.627242+0000","last_undegraded":"2023-08-28T02:30:42.627242+0000","last_fullsized":"2023-08-28T02:30:42.627242+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":7154,"log_dups_size":0,"ondisk_log_size":7154,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294912,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5268,"num_write_kb":168736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"78'204","reported_seq":1342,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.437399+0000","last_change":"2023-08-28T02:28:59.480561+0000","last_active":"2023-08-28T02:30:32.437399+0000","last_peered":"2023-08-28T02:30:32.437399+0000","last_clean":"2023-08-28T02:30:32.437399+0000","last_became_active":"2023-08-28T02:28:59.479868+0000","last_became_peered":"2023-08-28T02:28:59.479868+0000","last_unstale":"2023-08-28T02:30:32.437399+0000","last_undegraded":"2023-08-28T02:30:32.437399+0000","last_fullsized":"2023-08-28T02:30:32.437399+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":204,"log_dups_size":0,"ondisk_log_size":204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:39:50.444629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1521,"num_read_kb":2511,"num_write":178,"num_write_kb":1175,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":251,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.430649+0000","last_change":"2023-08-28T02:28:25.352417+0000","last_active":"2023-08-28T02:30:32.430649+0000","last_peered":"2023-08-28T02:30:32.430649+0000","last_clean":"2023-08-28T02:30:32.430649+0000","last_became_active":"2023-08-28T02:28:25.352216+0000","last_became_peered":"2023-08-28T02:28:25.352216+0000","last_unstale":"2023-08-28T02:30:32.430649+0000","last_undegraded":"2023-08-28T02:30:32.430649+0000","last_fullsized":"2023-08-28T02:30:32.430649+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":65,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.438210+0000","last_change":"2023-08-28T02:27:31.376755+0000","last_active":"2023-08-28T02:30:32.438210+0000","last_peered":"2023-08-28T02:30:32.438210+0000","last_clean":"2023-08-28T02:30:32.438210+0000","last_became_active":"2023-08-28T02:27:31.373279+0000","last_became_peered":"2023-08-28T02:27:31.373279+0000","last_unstale":"2023-08-28T02:30:32.438210+0000","last_undegraded":"2023-08-28T02:30:32.438210+0000","last_fullsized":"2023-08-28T02:30:32.438210+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:26:14.628941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"257'7077","reported_seq":9921,"reported_epoch":257,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-08-28T02:30:42.164532+0000","last_change":"2023-08-28T02:30:08.028893+0000","last_active":"2023-08-28T02:30:42.164532+0000","last_peered":"2023-08-28T02:30:42.164532+0000","last_clean":"2023-08-28T02:27:16.315756+0000","last_became_active":"2023-08-28T02:28:25.400914+0000","last_became_peered":"2023-08-28T02:28:25.400914+0000","last_unstale":"2023-08-28T02:30:42.164532+0000","last_undegraded":"2023-08-28T02:28:25.339881+0000","last_fullsized":"2023-08-28T02:28:25.329514+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":7077,"log_dups_size":0,"ondisk_log_size":7077,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1769472,"num_objects":27,"num_object_clones":0,"num_object_copies":81,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":888,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":27,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5193,"num_write_kb":167040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1076,"num_bytes_recovered":60096512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"3,7","objects":-861}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"78'47","reported_seq":474,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:40.599736+0000","last_change":"2023-08-28T02:24:05.799647+0000","last_active":"2023-08-28T02:30:40.599736+0000","last_peered":"2023-08-28T02:30:40.599736+0000","last_clean":"2023-08-28T02:30:40.599736+0000","last_became_active":"2023-08-28T02:23:59.210481+0000","last_became_peered":"2023-08-28T02:23:59.210481+0000","last_unstale":"2023-08-28T02:30:40.599736+0000","last_undegraded":"2023-08-28T02:30:40.599736+0000","last_fullsized":"2023-08-28T02:30:40.599736+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":47,"log_dups_size":0,"ondisk_log_size":47,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:40:18.542375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":162,"num_read_kb":1281,"num_write":48,"num_write_kb":1153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":74,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.430822+0000","last_change":"2023-08-28T02:28:25.343406+0000","last_active":"2023-08-28T02:30:32.430822+0000","last_peered":"2023-08-28T02:30:32.430822+0000","last_clean":"2023-08-28T02:30:32.430822+0000","last_became_active":"2023-08-28T02:28:25.343065+0000","last_became_peered":"2023-08-28T02:28:25.343065+0000","last_unstale":"2023-08-28T02:30:32.430822+0000","last_undegraded":"2023-08-28T02:30:32.430822+0000","last_fullsized":"2023-08-28T02:30:32.430822+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:20:31.893609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":65,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.434476+0000","last_change":"2023-08-28T02:27:31.322967+0000","last_active":"2023-08-28T02:30:32.434476+0000","last_peered":"2023-08-28T02:30:32.434476+0000","last_clean":"2023-08-28T02:30:32.434476+0000","last_became_active":"2023-08-28T02:27:31.322655+0000","last_became_peered":"2023-08-28T02:27:31.322655+0000","last_unstale":"2023-08-28T02:30:32.434476+0000","last_undegraded":"2023-08-28T02:30:32.434476+0000","last_fullsized":"2023-08-28T02:30:32.434476+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:05:15.868115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"257'7108","reported_seq":7210,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:41.934451+0000","last_change":"2023-08-28T02:30:41.214961+0000","last_active":"2023-08-28T02:30:41.934451+0000","last_peered":"2023-08-28T02:30:41.934451+0000","last_clean":"2023-08-28T02:30:41.934451+0000","last_became_active":"2023-08-28T02:27:58.452334+0000","last_became_peered":"2023-08-28T02:27:58.452334+0000","last_unstale":"2023-08-28T02:30:41.934451+0000","last_undegraded":"2023-08-28T02:30:41.934451+0000","last_fullsized":"2023-08-28T02:30:41.934451+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":7108,"log_dups_size":0,"ondisk_log_size":7108,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393216,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5252,"num_write_kb":168256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"78'114","reported_seq":620,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.452511+0000","last_change":"2023-08-28T02:24:01.210739+0000","last_active":"2023-08-28T02:30:32.452511+0000","last_peered":"2023-08-28T02:30:32.452511+0000","last_clean":"2023-08-28T02:30:32.452511+0000","last_became_active":"2023-08-28T02:24:01.210386+0000","last_became_peered":"2023-08-28T02:24:01.210386+0000","last_unstale":"2023-08-28T02:30:32.452511+0000","last_undegraded":"2023-08-28T02:30:32.452511+0000","last_fullsized":"2023-08-28T02:30:32.452511+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":114,"log_dups_size":0,"ondisk_log_size":114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":358,"num_read_kb":1352,"num_write":102,"num_write_kb":1104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"86'139","reported_seq":467,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.452032+0000","last_change":"2023-08-28T02:29:58.484566+0000","last_active":"2023-08-28T02:30:32.452032+0000","last_peered":"2023-08-28T02:30:32.452032+0000","last_clean":"2023-08-28T02:30:32.452032+0000","last_became_active":"2023-08-28T02:29:58.484046+0000","last_became_peered":"2023-08-28T02:29:58.484046+0000","last_unstale":"2023-08-28T02:30:32.452032+0000","last_undegraded":"2023-08-28T02:30:32.452032+0000","last_fullsized":"2023-08-28T02:30:32.452032+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_clean_scrub_stamp":"2023-08-28T02:08:02.043718+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":135,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.432267+0000","last_change":"2023-08-28T02:28:25.347600+0000","last_active":"2023-08-28T02:30:32.432267+0000","last_peered":"2023-08-28T02:30:32.432267+0000","last_clean":"2023-08-28T02:30:32.432267+0000","last_became_active":"2023-08-28T02:28:25.347237+0000","last_became_peered":"2023-08-28T02:28:25.347237+0000","last_unstale":"2023-08-28T02:30:32.432267+0000","last_undegraded":"2023-08-28T02:30:32.432267+0000","last_fullsized":"2023-08-28T02:30:32.432267+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:29:26.767559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":46,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.458347+0000","last_change":"2023-08-28T02:28:25.398594+0000","last_active":"2023-08-28T02:30:32.458347+0000","last_peered":"2023-08-28T02:30:32.458347+0000","last_clean":"2023-08-28T02:30:32.458347+0000","last_became_active":"2023-08-28T02:28:25.397058+0000","last_became_peered":"2023-08-28T02:28:25.397058+0000","last_unstale":"2023-08-28T02:30:32.458347+0000","last_undegraded":"2023-08-28T02:30:32.458347+0000","last_fullsized":"2023-08-28T02:30:32.458347+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"257'7018","reported_seq":12434,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:42.556918+0000","last_change":"2023-08-28T02:30:42.438833+0000","last_active":"2023-08-28T02:30:42.556918+0000","last_peered":"2023-08-28T02:30:42.556918+0000","last_clean":"2023-08-28T02:30:42.556918+0000","last_became_active":"2023-08-28T02:28:00.019510+0000","last_became_peered":"2023-08-28T02:28:00.019510+0000","last_unstale":"2023-08-28T02:30:42.556918+0000","last_undegraded":"2023-08-28T02:30:42.556918+0000","last_fullsized":"2023-08-28T02:30:42.556918+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":7018,"log_dups_size":0,"ondisk_log_size":7018,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1146880,"num_objects":17,"num_object_clones":0,"num_object_copies":51,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":17,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5185,"num_write_kb":166496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1910,"num_bytes_recovered":125206528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'148","reported_seq":787,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.452121+0000","last_change":"2023-08-28T02:29:39.524927+0000","last_active":"2023-08-28T02:30:32.452121+0000","last_peered":"2023-08-28T02:30:32.452121+0000","last_clean":"2023-08-28T02:30:32.452121+0000","last_became_active":"2023-08-28T02:29:39.524297+0000","last_became_peered":"2023-08-28T02:29:39.524297+0000","last_unstale":"2023-08-28T02:30:32.452121+0000","last_undegraded":"2023-08-28T02:30:32.452121+0000","last_fullsized":"2023-08-28T02:30:32.452121+0000","mapping_epoch":243,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":244,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1092,"num_read_kb":1955,"num_write":127,"num_write_kb":5177,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":225,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.447559+0000","last_change":"2023-08-28T02:23:59.226986+0000","last_active":"2023-08-28T02:30:32.447559+0000","last_peered":"2023-08-28T02:30:32.447559+0000","last_clean":"2023-08-28T02:30:32.447559+0000","last_became_active":"2023-08-28T02:23:59.226529+0000","last_became_peered":"2023-08-28T02:23:59.226529+0000","last_unstale":"2023-08-28T02:30:32.447559+0000","last_undegraded":"2023-08-28T02:30:32.447559+0000","last_fullsized":"2023-08-28T02:30:32.447559+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:22:09.193619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":270,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.447340+0000","last_change":"2023-08-28T02:27:31.361298+0000","last_active":"2023-08-28T02:30:32.447340+0000","last_peered":"2023-08-28T02:30:32.447340+0000","last_clean":"2023-08-28T02:30:32.447340+0000","last_became_active":"2023-08-28T02:27:31.360939+0000","last_became_peered":"2023-08-28T02:27:31.360939+0000","last_unstale":"2023-08-28T02:30:32.447340+0000","last_undegraded":"2023-08-28T02:30:32.447340+0000","last_fullsized":"2023-08-28T02:30:32.447340+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:48:42.024920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"257'7209","reported_seq":14248,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:42.619256+0000","last_change":"2023-08-28T02:30:12.290655+0000","last_active":"2023-08-28T02:30:42.619256+0000","last_peered":"2023-08-28T02:30:42.619256+0000","last_clean":"2023-08-28T02:30:42.619256+0000","last_became_active":"2023-08-28T02:30:09.573625+0000","last_became_peered":"2023-08-28T02:30:09.573625+0000","last_unstale":"2023-08-28T02:30:42.619256+0000","last_undegraded":"2023-08-28T02:30:42.619256+0000","last_fullsized":"2023-08-28T02:30:42.619256+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":7209,"log_dups_size":0,"ondisk_log_size":7209,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5334,"num_write_kb":170368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2494,"num_bytes_recovered":144736256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"78'103","reported_seq":632,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.455242+0000","last_change":"2023-08-28T02:24:07.212128+0000","last_active":"2023-08-28T02:30:32.455242+0000","last_peered":"2023-08-28T02:30:32.455242+0000","last_clean":"2023-08-28T02:30:32.455242+0000","last_became_active":"2023-08-28T02:24:07.211562+0000","last_became_peered":"2023-08-28T02:24:07.211562+0000","last_unstale":"2023-08-28T02:30:32.455242+0000","last_undegraded":"2023-08-28T02:30:32.455242+0000","last_fullsized":"2023-08-28T02:30:32.455242+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":803,"num_read_kb":2080,"num_write":73,"num_write_kb":1244,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":65,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.438565+0000","last_change":"2023-08-28T02:27:31.377042+0000","last_active":"2023-08-28T02:30:32.438565+0000","last_peered":"2023-08-28T02:30:32.438565+0000","last_clean":"2023-08-28T02:30:32.438565+0000","last_became_active":"2023-08-28T02:27:31.373709+0000","last_became_peered":"2023-08-28T02:27:31.373709+0000","last_unstale":"2023-08-28T02:30:32.438565+0000","last_undegraded":"2023-08-28T02:30:32.438565+0000","last_fullsized":"2023-08-28T02:30:32.438565+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:12:38.286332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":211,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.449423+0000","last_change":"2023-08-28T02:23:38.329667+0000","last_active":"2023-08-28T02:30:32.449423+0000","last_peered":"2023-08-28T02:30:32.449423+0000","last_clean":"2023-08-28T02:30:32.449423+0000","last_became_active":"2023-08-28T02:23:38.329175+0000","last_became_peered":"2023-08-28T02:23:38.329175+0000","last_unstale":"2023-08-28T02:30:32.449423+0000","last_undegraded":"2023-08-28T02:30:32.449423+0000","last_fullsized":"2023-08-28T02:30:32.449423+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:21:34.815067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"257'7128","reported_seq":7257,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:41.831549+0000","last_change":"2023-08-28T02:30:41.642739+0000","last_active":"2023-08-28T02:30:41.831549+0000","last_peered":"2023-08-28T02:30:41.831549+0000","last_clean":"2023-08-28T02:30:41.831549+0000","last_became_active":"2023-08-28T02:27:58.451860+0000","last_became_peered":"2023-08-28T02:27:58.451860+0000","last_unstale":"2023-08-28T02:30:41.831549+0000","last_undegraded":"2023-08-28T02:30:41.831549+0000","last_fullsized":"2023-08-28T02:30:41.831549+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":7128,"log_dups_size":0,"ondisk_log_size":7128,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5263,"num_write_kb":168000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":121,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.432473+0000","last_change":"2023-08-28T02:23:59.232099+0000","last_active":"2023-08-28T02:30:32.432473+0000","last_peered":"2023-08-28T02:30:32.432473+0000","last_clean":"2023-08-28T02:30:32.432473+0000","last_became_active":"2023-08-28T02:23:59.219318+0000","last_became_peered":"2023-08-28T02:23:59.219318+0000","last_unstale":"2023-08-28T02:30:32.432473+0000","last_undegraded":"2023-08-28T02:30:32.432473+0000","last_fullsized":"2023-08-28T02:30:32.432473+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:59:30.093289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'112","reported_seq":411,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:39.585026+0000","last_change":"2023-08-28T02:28:59.477917+0000","last_active":"2023-08-28T02:30:39.585026+0000","last_peered":"2023-08-28T02:30:39.585026+0000","last_clean":"2023-08-28T02:30:39.585026+0000","last_became_active":"2023-08-28T02:28:59.477383+0000","last_became_peered":"2023-08-28T02:28:59.477383+0000","last_unstale":"2023-08-28T02:30:39.585026+0000","last_undegraded":"2023-08-28T02:30:39.585026+0000","last_fullsized":"2023-08-28T02:30:39.585026+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:28:45.045742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":444,"num_read_kb":473,"num_write":94,"num_write_kb":127,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":117,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.431696+0000","last_change":"2023-08-28T02:23:59.221034+0000","last_active":"2023-08-28T02:30:32.431696+0000","last_peered":"2023-08-28T02:30:32.431696+0000","last_clean":"2023-08-28T02:30:32.431696+0000","last_became_active":"2023-08-28T02:23:59.220730+0000","last_became_peered":"2023-08-28T02:23:59.220730+0000","last_unstale":"2023-08-28T02:30:32.431696+0000","last_undegraded":"2023-08-28T02:30:32.431696+0000","last_fullsized":"2023-08-28T02:30:32.431696+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:25:52.364370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"257'7148","reported_seq":7256,"reported_epoch":257,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:30:42.165800+0000","last_change":"2023-08-28T02:28:26.460507+0000","last_active":"2023-08-28T02:30:42.165800+0000","last_peered":"2023-08-28T02:30:42.165800+0000","last_clean":"2023-08-28T02:27:16.315150+0000","last_became_active":"2023-08-28T02:28:26.432234+0000","last_became_peered":"2023-08-28T02:28:26.432234+0000","last_unstale":"2023-08-28T02:30:42.165800+0000","last_undegraded":"2023-08-28T02:28:26.346974+0000","last_fullsized":"2023-08-28T02:28:26.332755+0000","mapping_epoch":221,"log_start":"0'0","ondisk_log_start":"0'0","created":187,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":7148,"log_dups_size":0,"ondisk_log_size":7148,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1048576,"num_objects":16,"num_object_clones":0,"num_object_copies":48,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1660,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":16,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5260,"num_write_kb":168832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":-1644}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"78'122","reported_seq":564,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.457481+0000","last_change":"2023-08-28T02:28:28.383306+0000","last_active":"2023-08-28T02:30:32.457481+0000","last_peered":"2023-08-28T02:30:32.457481+0000","last_clean":"2023-08-28T02:30:32.457481+0000","last_became_active":"2023-08-28T02:28:28.382672+0000","last_became_peered":"2023-08-28T02:28:28.382672+0000","last_unstale":"2023-08-28T02:30:32.457481+0000","last_undegraded":"2023-08-28T02:30:32.457481+0000","last_fullsized":"2023-08-28T02:30:32.457481+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":857,"num_read_kb":2224,"num_write":99,"num_write_kb":2312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":237,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.447427+0000","last_change":"2023-08-28T02:27:31.424373+0000","last_active":"2023-08-28T02:30:32.447427+0000","last_peered":"2023-08-28T02:30:32.447427+0000","last_clean":"2023-08-28T02:30:32.447427+0000","last_became_active":"2023-08-28T02:27:31.324505+0000","last_became_peered":"2023-08-28T02:27:31.324505+0000","last_unstale":"2023-08-28T02:30:32.447427+0000","last_undegraded":"2023-08-28T02:30:32.447427+0000","last_fullsized":"2023-08-28T02:30:32.447427+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:27:57.837524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":226,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.430446+0000","last_change":"2023-08-28T02:28:25.351840+0000","last_active":"2023-08-28T02:30:32.430446+0000","last_peered":"2023-08-28T02:30:32.430446+0000","last_clean":"2023-08-28T02:30:32.430446+0000","last_became_active":"2023-08-28T02:28:25.351399+0000","last_became_peered":"2023-08-28T02:28:25.351399+0000","last_unstale":"2023-08-28T02:30:32.430446+0000","last_undegraded":"2023-08-28T02:30:32.430446+0000","last_fullsized":"2023-08-28T02:30:32.430446+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":213,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.449397+0000","last_change":"2023-08-28T02:23:38.329474+0000","last_active":"2023-08-28T02:30:32.449397+0000","last_peered":"2023-08-28T02:30:32.449397+0000","last_clean":"2023-08-28T02:30:32.449397+0000","last_became_active":"2023-08-28T02:23:38.328505+0000","last_became_peered":"2023-08-28T02:23:38.328505+0000","last_unstale":"2023-08-28T02:30:32.449397+0000","last_undegraded":"2023-08-28T02:30:32.449397+0000","last_fullsized":"2023-08-28T02:30:32.449397+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:12:14.835809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":298,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.443547+0000","last_change":"2023-08-28T02:27:31.384467+0000","last_active":"2023-08-28T02:30:32.443547+0000","last_peered":"2023-08-28T02:30:32.443547+0000","last_clean":"2023-08-28T02:30:32.443547+0000","last_became_active":"2023-08-28T02:27:31.384183+0000","last_became_peered":"2023-08-28T02:27:31.384183+0000","last_unstale":"2023-08-28T02:30:32.443547+0000","last_undegraded":"2023-08-28T02:30:32.443547+0000","last_fullsized":"2023-08-28T02:30:32.443547+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:59:47.551892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.19","version":"257'7123","reported_seq":7216,"reported_epoch":257,"state":"active+clean+scrubbing","last_fresh":"2023-08-28T02:30:42.641704+0000","last_change":"2023-08-28T02:30:43.166001+0000","last_active":"2023-08-28T02:30:42.641704+0000","last_peered":"2023-08-28T02:30:42.641704+0000","last_clean":"2023-08-28T02:30:42.641704+0000","last_became_active":"2023-08-28T02:27:58.490323+0000","last_became_peered":"2023-08-28T02:27:58.490323+0000","last_unstale":"2023-08-28T02:30:42.641704+0000","last_undegraded":"2023-08-28T02:30:42.641704+0000","last_fullsized":"2023-08-28T02:30:42.641704+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":7123,"log_dups_size":0,"ondisk_log_size":7123,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5254,"num_write_kb":167712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":63,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.430622+0000","last_change":"2023-08-28T02:28:11.333622+0000","last_active":"2023-08-28T02:30:32.430622+0000","last_peered":"2023-08-28T02:30:32.430622+0000","last_clean":"2023-08-28T02:30:32.430622+0000","last_became_active":"2023-08-28T02:28:11.333321+0000","last_became_peered":"2023-08-28T02:28:11.333321+0000","last_unstale":"2023-08-28T02:30:32.430622+0000","last_undegraded":"2023-08-28T02:30:32.430622+0000","last_fullsized":"2023-08-28T02:30:32.430622+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":274,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.430748+0000","last_change":"2023-08-28T02:27:31.343538+0000","last_active":"2023-08-28T02:30:32.430748+0000","last_peered":"2023-08-28T02:30:32.430748+0000","last_clean":"2023-08-28T02:30:32.430748+0000","last_became_active":"2023-08-28T02:27:31.343324+0000","last_became_peered":"2023-08-28T02:27:31.343324+0000","last_unstale":"2023-08-28T02:30:32.430748+0000","last_undegraded":"2023-08-28T02:30:32.430748+0000","last_fullsized":"2023-08-28T02:30:32.430748+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.18","version":"257'6967","reported_seq":7054,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:42.172841+0000","last_change":"2023-08-28T02:27:58.496324+0000","last_active":"2023-08-28T02:30:42.172841+0000","last_peered":"2023-08-28T02:30:42.172841+0000","last_clean":"2023-08-28T02:30:42.172841+0000","last_became_active":"2023-08-28T02:27:58.494507+0000","last_became_peered":"2023-08-28T02:27:58.494507+0000","last_unstale":"2023-08-28T02:30:42.172841+0000","last_undegraded":"2023-08-28T02:30:42.172841+0000","last_fullsized":"2023-08-28T02:30:42.172841+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":209,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:26:56.367648+0000","last_clean_scrub_stamp":"2023-08-28T02:26:56.367648+0000","objects_scrubbed":0,"log_size":6967,"log_dups_size":0,"ondisk_log_size":6967,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4489216,"num_objects":69,"num_object_clones":0,"num_object_copies":207,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":69,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5091,"num_write_kb":165088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":76,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.430922+0000","last_change":"2023-08-28T02:28:25.705331+0000","last_active":"2023-08-28T02:30:32.430922+0000","last_peered":"2023-08-28T02:30:32.430922+0000","last_clean":"2023-08-28T02:30:32.430922+0000","last_became_active":"2023-08-28T02:28:25.397480+0000","last_became_peered":"2023-08-28T02:28:25.397480+0000","last_unstale":"2023-08-28T02:30:32.430922+0000","last_undegraded":"2023-08-28T02:30:32.430922+0000","last_fullsized":"2023-08-28T02:30:32.430922+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":256,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.432366+0000","last_change":"2023-08-28T02:23:59.207929+0000","last_active":"2023-08-28T02:30:32.432366+0000","last_peered":"2023-08-28T02:30:32.432366+0000","last_clean":"2023-08-28T02:30:32.432366+0000","last_became_active":"2023-08-28T02:23:59.207589+0000","last_became_peered":"2023-08-28T02:23:59.207589+0000","last_unstale":"2023-08-28T02:30:32.432366+0000","last_undegraded":"2023-08-28T02:30:32.432366+0000","last_fullsized":"2023-08-28T02:30:32.432366+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":144,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.432862+0000","last_change":"2023-08-28T02:28:25.347988+0000","last_active":"2023-08-28T02:30:32.432862+0000","last_peered":"2023-08-28T02:30:32.432862+0000","last_clean":"2023-08-28T02:30:32.432862+0000","last_became_active":"2023-08-28T02:28:25.347632+0000","last_became_peered":"2023-08-28T02:28:25.347632+0000","last_unstale":"2023-08-28T02:30:32.432862+0000","last_undegraded":"2023-08-28T02:30:32.432862+0000","last_fullsized":"2023-08-28T02:30:32.432862+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:56:20.202662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":116,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.432040+0000","last_change":"2023-08-28T02:23:59.217794+0000","last_active":"2023-08-28T02:30:32.432040+0000","last_peered":"2023-08-28T02:30:32.432040+0000","last_clean":"2023-08-28T02:30:32.432040+0000","last_became_active":"2023-08-28T02:23:59.217231+0000","last_became_peered":"2023-08-28T02:23:59.217231+0000","last_unstale":"2023-08-28T02:30:32.432040+0000","last_undegraded":"2023-08-28T02:30:32.432040+0000","last_fullsized":"2023-08-28T02:30:32.432040+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:36:26.297410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":211,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.447724+0000","last_change":"2023-08-28T02:20:19.850252+0000","last_active":"2023-08-28T02:30:32.447724+0000","last_peered":"2023-08-28T02:30:32.447724+0000","last_clean":"2023-08-28T02:30:32.447724+0000","last_became_active":"2023-08-28T02:20:19.849913+0000","last_became_peered":"2023-08-28T02:20:19.849913+0000","last_unstale":"2023-08-28T02:30:32.447724+0000","last_undegraded":"2023-08-28T02:30:32.447724+0000","last_fullsized":"2023-08-28T02:30:32.447724+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:48:47.713465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":288,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.430405+0000","last_change":"2023-08-28T02:28:25.347845+0000","last_active":"2023-08-28T02:30:32.430405+0000","last_peered":"2023-08-28T02:30:32.430405+0000","last_clean":"2023-08-28T02:30:32.430405+0000","last_became_active":"2023-08-28T02:28:25.347502+0000","last_became_peered":"2023-08-28T02:28:25.347502+0000","last_unstale":"2023-08-28T02:30:32.430405+0000","last_undegraded":"2023-08-28T02:30:32.430405+0000","last_fullsized":"2023-08-28T02:30:32.430405+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":26,"stat_sum":{"num_bytes":19660824,"num_objects":300,"num_object_clones":0,"num_object_copies":900,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5941,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":300,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":167065,"num_write_kb":5351425,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20427,"num_bytes_recovered":1166802944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":945295360,"data_stored":945291288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":189403,"ondisk_log_size":189403,"up":78,"acting":74,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":19382,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26810,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":137,"num_bytes_recovered":12582931,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430461,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":649588,"kb_used_data":348204,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93095564,"statfs":{"total":95995035648,"available":95329857536,"internally_reserved":0,"allocated":356560896,"data_stored":355845850,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:30:03 2023","interfaces":[{"interface":"back","average":{"1min":2.194,"5min":2.3999999999999999,"15min":2.0859999999999999},"min":{"1min":0.25600000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":24.379999999999999,"5min":59.753999999999998,"15min":116.755},"last":1.0700000000000001},{"interface":"front","average":{"1min":2.7120000000000002,"5min":2.7559999999999998,"15min":2.3359999999999999},"min":{"1min":0.26500000000000001,"5min":0.245,"15min":0.23999999999999999},"max":{"1min":24.213999999999999,"5min":59.765999999999998,"15min":116.971},"last":0.252}]},{"osd":1,"last update":"Mon Aug 28 02:30:11 2023","interfaces":[{"interface":"back","average":{"1min":2.496,"5min":2.2509999999999999,"15min":1.2370000000000001},"min":{"1min":0.27900000000000003,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":24.681000000000001,"5min":44.287999999999997,"15min":44.287999999999997},"last":0.50800000000000001},{"interface":"front","average":{"1min":2.7570000000000001,"5min":3.109,"15min":1.667},"min":{"1min":0.28399999999999997,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":24.189,"5min":44.393999999999998,"15min":44.393999999999998},"last":0.44700000000000001}]},{"osd":2,"last update":"Mon Aug 28 02:30:35 2023","interfaces":[{"interface":"back","average":{"1min":2.1779999999999999,"5min":3.004,"15min":2.4830000000000001},"min":{"1min":0.32400000000000001,"5min":0.28000000000000003,"15min":0.27600000000000002},"max":{"1min":7.5300000000000002,"5min":101.616,"15min":119.571},"last":0.54800000000000004},{"interface":"front","average":{"1min":1.9119999999999999,"5min":3.7210000000000001,"15min":2.875},"min":{"1min":0.32900000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.5380000000000003,"5min":101.605,"15min":119.554},"last":0.39500000000000002}]},{"osd":3,"last update":"Mon Aug 28 02:30:11 2023","interfaces":[{"interface":"back","average":{"1min":2.0110000000000001,"5min":4.0620000000000003,"15min":2.2480000000000002},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30199999999999999},"max":{"1min":10.686999999999999,"5min":99.459000000000003,"15min":107.916},"last":0.58399999999999996},{"interface":"front","average":{"1min":1.794,"5min":3.6789999999999998,"15min":2.3239999999999998},"min":{"1min":0.34300000000000003,"5min":0.311,"15min":0.27700000000000002},"max":{"1min":10.140000000000001,"5min":98.084000000000003,"15min":112.00700000000001},"last":0.33500000000000002}]},{"osd":4,"last update":"Mon Aug 28 02:30:35 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":1.26,"15min":1.254},"min":{"1min":0.28899999999999998,"5min":0.26500000000000001,"15min":0.22900000000000001},"max":{"1min":2.3929999999999998,"5min":11.696,"15min":63.048000000000002},"last":0.28999999999999998},{"interface":"front","average":{"1min":1.3660000000000001,"5min":1.347,"15min":1.24},"min":{"1min":0.316,"5min":0.26700000000000002,"15min":0.247},"max":{"1min":8.2140000000000004,"5min":10.840999999999999,"15min":62.912999999999997},"last":0.31900000000000001}]},{"osd":5,"last update":"Mon Aug 28 02:30:35 2023","interfaces":[{"interface":"back","average":{"1min":1.2,"5min":1.1859999999999999,"15min":1.075},"min":{"1min":0.32800000000000001,"5min":0.27800000000000002,"15min":0.26800000000000002},"max":{"1min":7.5110000000000001,"5min":13.788,"15min":63.021000000000001},"last":0.61799999999999999},{"interface":"front","average":{"1min":1.2949999999999999,"5min":1.4079999999999999,"15min":1.377},"min":{"1min":0.38200000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":6.7110000000000003,"5min":9.6929999999999996,"15min":63.082999999999998},"last":0.42499999999999999}]},{"osd":6,"last update":"Mon Aug 28 02:30:35 2023","interfaces":[{"interface":"back","average":{"1min":1.302,"5min":1.2529999999999999,"15min":1.3280000000000001},"min":{"1min":0.36899999999999999,"5min":0.318,"15min":0.29699999999999999},"max":{"1min":7.5369999999999999,"5min":9.7070000000000007,"15min":101.61},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.78500000000000003,"5min":1.29,"15min":1.427},"min":{"1min":0.40999999999999998,"5min":0.308,"15min":0.308},"max":{"1min":2.4780000000000002,"5min":14.423999999999999,"15min":101.621},"last":0.47099999999999997}]}]},{"osd":6,"up_from":42,"seq":180388626688,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":386792,"kb_used_data":85408,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93358360,"statfs":{"total":95995035648,"available":95598960640,"internally_reserved":0,"allocated":87457792,"data_stored":86754592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:30:11 2023","interfaces":[{"interface":"back","average":{"1min":3.246,"5min":3.911,"15min":2.7629999999999999},"min":{"1min":0.22700000000000001,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":44.512999999999998,"5min":60.625999999999998,"15min":121.247},"last":7.1980000000000004},{"interface":"front","average":{"1min":3.3300000000000001,"5min":3.7080000000000002,"15min":2.722},"min":{"1min":0.26000000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":46.637999999999998,"5min":54.273000000000003,"15min":118.977},"last":0.48399999999999999}]},{"osd":1,"last update":"Mon Aug 28 02:30:11 2023","interfaces":[{"interface":"back","average":{"1min":1.78,"5min":2.524,"15min":1.669},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23699999999999999},"max":{"1min":7.2270000000000003,"5min":99.998999999999995,"15min":99.998999999999995},"last":7.2560000000000002},{"interface":"front","average":{"1min":1.9550000000000001,"5min":2.8010000000000002,"15min":1.8100000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":8.6029999999999998,"5min":95.040000000000006,"15min":95.040000000000006},"last":0.59699999999999998}]},{"osd":2,"last update":"Mon Aug 28 02:29:53 2023","interfaces":[{"interface":"back","average":{"1min":1.4650000000000001,"5min":4.258,"15min":2.2250000000000001},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":4.3259999999999996,"5min":79.787000000000006,"15min":79.787000000000006},"last":0.43099999999999999},{"interface":"front","average":{"1min":1.3959999999999999,"5min":4.4470000000000001,"15min":2.2200000000000002},"min":{"1min":0.28899999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":7.8490000000000002,"5min":79.935000000000002,"15min":79.935000000000002},"last":3.79}]},{"osd":3,"last update":"Mon Aug 28 02:30:11 2023","interfaces":[{"interface":"back","average":{"1min":1.1419999999999999,"5min":2.7850000000000001,"15min":1.6719999999999999},"min":{"1min":0.32000000000000001,"5min":0.317,"15min":0.317},"max":{"1min":3.968,"5min":39.587000000000003,"15min":39.587000000000003},"last":21.177},{"interface":"front","average":{"1min":1.0069999999999999,"5min":2.8940000000000001,"15min":1.7589999999999999},"min":{"1min":0.35299999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":3.9780000000000002,"5min":39.593000000000004,"15min":39.593000000000004},"last":0.63}]},{"osd":4,"last update":"Mon Aug 28 02:30:18 2023","interfaces":[{"interface":"back","average":{"1min":1.0069999999999999,"5min":1.224,"15min":1.04},"min":{"1min":0.27100000000000002,"5min":0.251,"15min":0.251},"max":{"1min":3.6499999999999999,"5min":18.308,"15min":18.308},"last":7.3090000000000002},{"interface":"front","average":{"1min":0.77400000000000002,"5min":1.3899999999999999,"15min":1.131},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.26700000000000002},"max":{"1min":3.5070000000000001,"5min":18.263000000000002,"15min":18.263000000000002},"last":0.46100000000000002}]},{"osd":5,"last update":"Mon Aug 28 02:30:18 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":1.216,"15min":1.1080000000000001},"min":{"1min":0.29699999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":2.8359999999999999,"5min":9.9809999999999999,"15min":20.759},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.98999999999999999,"5min":1.113,"15min":0.95599999999999996},"min":{"1min":0.314,"5min":0.314,"15min":0.28799999999999998},"max":{"1min":3.1549999999999998,"5min":5.9500000000000002,"15min":7.3780000000000001},"last":7.2370000000000001}]},{"osd":7,"last update":"Mon Aug 28 02:29:40 2023","interfaces":[{"interface":"back","average":{"1min":0.80600000000000005,"5min":1.161,"15min":0.92700000000000005},"min":{"1min":0.34799999999999998,"5min":0.34200000000000003,"15min":0.314},"max":{"1min":3.8490000000000002,"5min":9.0830000000000002,"15min":9.0830000000000002},"last":10.605},{"interface":"front","average":{"1min":0.68400000000000005,"5min":1.3320000000000001,"15min":1.0389999999999999},"min":{"1min":0.35599999999999998,"5min":0.33500000000000002,"15min":0.29799999999999999},"max":{"1min":3.0329999999999999,"5min":9.7110000000000003,"15min":9.7110000000000003},"last":0.505}]}]},{"osd":1,"up_from":98,"seq":420906795141,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":428580,"kb_used_data":165980,"kb_used_omap":7,"kb_used_meta":262584,"kb_avail":93316572,"statfs":{"total":95995035648,"available":95556169728,"internally_reserved":0,"allocated":169963520,"data_stored":168693954,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":268886347},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:30:09 2023","interfaces":[{"interface":"back","average":{"1min":1.534,"5min":4.548,"15min":2.1379999999999999},"min":{"1min":0.20799999999999999,"5min":0.191,"15min":0.17499999999999999},"max":{"1min":8.3230000000000004,"5min":128.60599999999999,"15min":128.60599999999999},"last":0.27100000000000002},{"interface":"front","average":{"1min":1.1930000000000001,"5min":4.3760000000000003,"15min":2.1480000000000001},"min":{"1min":0.221,"5min":0.188,"15min":0.188},"max":{"1min":3.8220000000000001,"5min":128.685,"15min":128.685},"last":0.30399999999999999}]},{"osd":2,"last update":"Mon Aug 28 02:30:34 2023","interfaces":[{"interface":"back","average":{"1min":2.5640000000000001,"5min":6.2939999999999996,"15min":2.9630000000000001},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":8.1720000000000006,"5min":135.25999999999999,"15min":135.25999999999999},"last":0.24199999999999999},{"interface":"front","average":{"1min":2.1339999999999999,"5min":6.1920000000000002,"15min":2.7730000000000001},"min":{"1min":0.26700000000000002,"5min":0.23499999999999999,"15min":0.221},"max":{"1min":6.7089999999999996,"5min":135.10300000000001,"15min":135.10300000000001},"last":1.0269999999999999}]},{"osd":3,"last update":"Mon Aug 28 02:30:09 2023","interfaces":[{"interface":"back","average":{"1min":1.6899999999999999,"5min":3.7010000000000001,"15min":1.843},"min":{"1min":0.33600000000000002,"5min":0.22,"15min":0.22},"max":{"1min":3.867,"5min":123.741,"15min":123.741},"last":0.22500000000000001},{"interface":"front","average":{"1min":3.9060000000000001,"5min":4.4829999999999997,"15min":2.1459999999999999},"min":{"1min":0.501,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":13.340999999999999,"5min":123.642,"15min":123.642},"last":0.38500000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:30:09 2023","interfaces":[{"interface":"back","average":{"1min":1.6930000000000001,"5min":1.7789999999999999,"15min":1.264},"min":{"1min":0.29999999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.2939999999999996,"5min":17.600000000000001,"15min":17.600000000000001},"last":0.54900000000000004},{"interface":"front","average":{"1min":1.2170000000000001,"5min":1.6220000000000001,"15min":1.137},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.25600000000000001},"max":{"1min":3.5920000000000001,"5min":19.010000000000002,"15min":19.010000000000002},"last":0.51700000000000002}]},{"osd":5,"last update":"Mon Aug 28 02:30:09 2023","interfaces":[{"interface":"back","average":{"1min":2.242,"5min":1.6910000000000001,"15min":1.371},"min":{"1min":0.32100000000000001,"5min":0.312,"15min":0.29399999999999998},"max":{"1min":11.167999999999999,"5min":11.167999999999999,"15min":15.544},"last":0.55900000000000005},{"interface":"front","average":{"1min":2.2440000000000002,"5min":1.847,"15min":1.2549999999999999},"min":{"1min":0.34599999999999997,"5min":0.33600000000000002,"15min":0.33200000000000002},"max":{"1min":10.084,"5min":17.646999999999998,"15min":17.646999999999998},"last":0.57499999999999996}]},{"osd":6,"last update":"Mon Aug 28 02:30:09 2023","interfaces":[{"interface":"back","average":{"1min":1.7170000000000001,"5min":1.7090000000000001,"15min":1.4630000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":3.923,"5min":9.2989999999999995,"15min":15.512},"last":0.47099999999999997},{"interface":"front","average":{"1min":1.8160000000000001,"5min":1.823,"15min":1.3899999999999999},"min":{"1min":0.39600000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":6.3140000000000001,"5min":17.673999999999999,"15min":17.673999999999999},"last":0.503}]},{"osd":7,"last update":"Mon Aug 28 02:30:09 2023","interfaces":[{"interface":"back","average":{"1min":1.4059999999999999,"5min":1.7909999999999999,"15min":1.252},"min":{"1min":0.38500000000000001,"5min":0.32500000000000001,"15min":0.30199999999999999},"max":{"1min":4.0460000000000003,"5min":19.061,"15min":19.061},"last":49.575000000000003},{"interface":"front","average":{"1min":1.7889999999999999,"5min":1.905,"15min":1.4450000000000001},"min":{"1min":0.437,"5min":0.40500000000000003,"15min":0.33100000000000002},"max":{"1min":5.8890000000000002,"5min":9.3239999999999998,"15min":15.476000000000001},"last":49.319000000000003}]}]},{"osd":0,"up_from":90,"seq":386547056787,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":389108,"kb_used_data":64108,"kb_used_omap":8,"kb_used_meta":324983,"kb_avail":93356044,"statfs":{"total":95995035648,"available":95596589056,"internally_reserved":0,"allocated":65646592,"data_stored":64478369,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":332782953},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Aug 28 02:30:17 2023","interfaces":[{"interface":"back","average":{"1min":5.1619999999999999,"5min":3.004,"15min":1.8919999999999999},"min":{"1min":0.20000000000000001,"5min":0.184,"15min":0.184},"max":{"1min":56.576999999999998,"5min":56.576999999999998,"15min":56.576999999999998},"last":1.8480000000000001},{"interface":"front","average":{"1min":6.1689999999999996,"5min":3.524,"15min":1.931},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.17000000000000001},"max":{"1min":56.552999999999997,"5min":56.552999999999997,"15min":56.552999999999997},"last":6.7190000000000003}]},{"osd":2,"last update":"Mon Aug 28 02:30:34 2023","interfaces":[{"interface":"back","average":{"1min":3.234,"5min":4.3090000000000002,"15min":2.202},"min":{"1min":0.22,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":10.565,"5min":100.718,"15min":100.718},"last":2.8380000000000001},{"interface":"front","average":{"1min":1.333,"5min":4.3730000000000002,"15min":2.4159999999999999},"min":{"1min":0.24299999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":4.9989999999999997,"5min":100.962,"15min":100.962},"last":2.0920000000000001}]},{"osd":3,"last update":"Mon Aug 28 02:30:17 2023","interfaces":[{"interface":"back","average":{"1min":2.6459999999999999,"5min":6.9390000000000001,"15min":2.9430000000000001},"min":{"1min":0.24299999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":19.327000000000002,"5min":237.43899999999999,"15min":237.43899999999999},"last":3.2909999999999999},{"interface":"front","average":{"1min":1.7390000000000001,"5min":6.6399999999999997,"15min":2.859},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":6.0010000000000003,"5min":237.48099999999999,"15min":237.48099999999999},"last":2.0169999999999999}]},{"osd":4,"last update":"Mon Aug 28 02:30:12 2023","interfaces":[{"interface":"back","average":{"1min":1.7709999999999999,"5min":1.6859999999999999,"15min":1.2849999999999999},"min":{"1min":0.29699999999999999,"5min":0.29599999999999999,"15min":0.26800000000000002},"max":{"1min":9.3680000000000003,"5min":9.3680000000000003,"15min":9.6649999999999991},"last":5.0419999999999998},{"interface":"front","average":{"1min":1.536,"5min":2.0870000000000002,"15min":1.4650000000000001},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":8.7880000000000003,"5min":27.271999999999998,"15min":27.271999999999998},"last":5.0179999999999998}]},{"osd":5,"last update":"Mon Aug 28 02:30:12 2023","interfaces":[{"interface":"back","average":{"1min":1.153,"5min":2.0099999999999998,"15min":1.5609999999999999},"min":{"1min":0.34000000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":3.0230000000000001,"5min":27.367000000000001,"15min":44.000999999999998},"last":3.8290000000000002},{"interface":"front","average":{"1min":1.671,"5min":1.9079999999999999,"15min":1.488},"min":{"1min":0.34300000000000003,"5min":0.33000000000000002,"15min":0.30599999999999999},"max":{"1min":9.4489999999999998,"5min":12.334,"15min":43.953000000000003},"last":2.081}]},{"osd":6,"last update":"Mon Aug 28 02:30:12 2023","interfaces":[{"interface":"back","average":{"1min":1.276,"5min":2.0819999999999999,"15min":1.585},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":3.0510000000000002,"5min":27.331,"15min":32.417999999999999},"last":7.6340000000000003},{"interface":"front","average":{"1min":1.6930000000000001,"5min":2.173,"15min":1.5620000000000001},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.30399999999999999},"max":{"1min":9.2789999999999999,"5min":27.247,"15min":32.130000000000003},"last":2.181}]},{"osd":7,"last update":"Mon Aug 28 02:30:12 2023","interfaces":[{"interface":"back","average":{"1min":1.802,"5min":2.4769999999999999,"15min":1.583},"min":{"1min":0.36799999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":9.4209999999999994,"5min":27.385000000000002,"15min":27.385000000000002},"last":5.0830000000000002},{"interface":"front","average":{"1min":1.5069999999999999,"5min":2.339,"15min":1.599},"min":{"1min":0.38900000000000001,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":5.7300000000000004,"5min":27.213999999999999,"15min":27.213999999999999},"last":3.5990000000000002}]}]},{"osd":2,"up_from":104,"seq":446676598911,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":585284,"kb_used_data":273084,"kb_used_omap":10,"kb_used_meta":312181,"kb_avail":93159868,"statfs":{"total":95995035648,"available":95395704832,"internally_reserved":0,"allocated":279638016,"data_stored":278510747,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10792,"internal_metadata":319673816},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:30:37 2023","interfaces":[{"interface":"back","average":{"1min":2.3050000000000002,"5min":4.21,"15min":2.2599999999999998},"min":{"1min":0.20300000000000001,"5min":0.17699999999999999,"15min":0.17599999999999999},"max":{"1min":16.48,"5min":113.89100000000001,"15min":113.89100000000001},"last":0.24199999999999999},{"interface":"front","average":{"1min":2.5499999999999998,"5min":7.5940000000000003,"15min":3.472},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.19900000000000001},"max":{"1min":11.166,"5min":147.286,"15min":147.286},"last":0.23000000000000001}]},{"osd":1,"last update":"Mon Aug 28 02:30:37 2023","interfaces":[{"interface":"back","average":{"1min":2.6560000000000001,"5min":2.431,"15min":1.738},"min":{"1min":0.23799999999999999,"5min":0.21099999999999999,"15min":0.20999999999999999},"max":{"1min":11.061,"5min":11.627000000000001,"15min":30.888000000000002},"last":0.255},{"interface":"front","average":{"1min":3.351,"5min":2.8029999999999999,"15min":1.6890000000000001},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.20999999999999999},"max":{"1min":11.962999999999999,"5min":20.259,"15min":31.648},"last":0.33900000000000002}]},{"osd":3,"last update":"Mon Aug 28 02:30:09 2023","interfaces":[{"interface":"back","average":{"1min":1.6799999999999999,"5min":5.1100000000000003,"15min":2.286},"min":{"1min":0.314,"5min":0.25,"15min":0.22700000000000001},"max":{"1min":9.7629999999999999,"5min":128.63900000000001,"15min":128.63900000000001},"last":0.27200000000000002},{"interface":"front","average":{"1min":2.1070000000000002,"5min":4.9269999999999996,"15min":2.3010000000000002},"min":{"1min":0.375,"5min":0.223,"15min":0.223},"max":{"1min":8.7029999999999994,"5min":128.75700000000001,"15min":128.75700000000001},"last":0.30199999999999999}]},{"osd":4,"last update":"Mon Aug 28 02:30:37 2023","interfaces":[{"interface":"back","average":{"1min":12.464,"5min":5.617,"15min":2.661},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":207.34100000000001,"5min":207.34100000000001,"15min":207.34100000000001},"last":0.40999999999999998},{"interface":"front","average":{"1min":12.086,"5min":5.2610000000000001,"15min":2.4129999999999998},"min":{"1min":0.30299999999999999,"5min":0.28999999999999998,"15min":0.27400000000000002},"max":{"1min":207.352,"5min":207.352,"15min":207.352},"last":0.439}]},{"osd":5,"last update":"Mon Aug 28 02:30:37 2023","interfaces":[{"interface":"back","average":{"1min":1.998,"5min":5.4800000000000004,"15min":2.5379999999999998},"min":{"1min":0.33800000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":5.8120000000000003,"5min":128.696,"15min":128.696},"last":0.42599999999999999},{"interface":"front","average":{"1min":2.0249999999999999,"5min":5.1769999999999996,"15min":2.5939999999999999},"min":{"1min":0.32300000000000001,"5min":0.32100000000000001,"15min":0.30299999999999999},"max":{"1min":8.0709999999999997,"5min":128.84100000000001,"15min":128.84100000000001},"last":0.379}]},{"osd":6,"last update":"Mon Aug 28 02:30:37 2023","interfaces":[{"interface":"back","average":{"1min":1.9370000000000001,"5min":1.79,"15min":1.3500000000000001},"min":{"1min":0.32800000000000001,"5min":0.307,"15min":0.29299999999999998},"max":{"1min":11.997,"5min":11.997,"15min":11.997},"last":0.41799999999999998},{"interface":"front","average":{"1min":1.8939999999999999,"5min":2.1299999999999999,"15min":1.494},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":5.9530000000000003,"5min":11.055999999999999,"15min":11.055999999999999},"last":0.40000000000000002}]},{"osd":7,"last update":"Mon Aug 28 02:30:37 2023","interfaces":[{"interface":"back","average":{"1min":1.827,"5min":5.3289999999999997,"15min":2.5129999999999999},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.32100000000000001},"max":{"1min":6.29,"5min":128.911,"15min":128.911},"last":0.38900000000000001},{"interface":"front","average":{"1min":1.7410000000000001,"5min":5.859,"15min":2.7120000000000002},"min":{"1min":0.35599999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":4.8819999999999997,"5min":128.72399999999999,"15min":128.72399999999999},"last":0.33100000000000002}]}]},{"osd":3,"up_from":112,"seq":481036337273,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":255616,"kb_used_data":2936,"kb_used_omap":3,"kb_used_meta":252668,"kb_avail":93489536,"statfs":{"total":95995035648,"available":95733284864,"internally_reserved":0,"allocated":3006464,"data_stored":1880490,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":258732962},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:30:05 2023","interfaces":[{"interface":"back","average":{"1min":7.0190000000000001,"5min":3.1560000000000001,"15min":3.0089999999999999},"min":{"1min":0.20000000000000001,"5min":0.193,"15min":0.193},"max":{"1min":79.793000000000006,"5min":79.793000000000006,"15min":164.25299999999999},"last":0.41399999999999998},{"interface":"front","average":{"1min":6.7839999999999998,"5min":2.8380000000000001,"15min":2.9620000000000002},"min":{"1min":0.27200000000000002,"5min":0.192,"15min":0.192},"max":{"1min":78.555000000000007,"5min":78.555000000000007,"15min":164.55000000000001},"last":0.35599999999999998}]},{"osd":1,"last update":"Mon Aug 28 02:30:05 2023","interfaces":[{"interface":"back","average":{"1min":1.669,"5min":1.907,"15min":1.8680000000000001},"min":{"1min":0.245,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":6.1500000000000004,"5min":13.936,"15min":117.34699999999999},"last":0.80200000000000005},{"interface":"front","average":{"1min":2.5310000000000001,"5min":2.1499999999999999,"15min":1.968},"min":{"1min":0.27300000000000002,"5min":0.245,"15min":0.224},"max":{"1min":10.125,"5min":14.300000000000001,"15min":118.012},"last":0.77700000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:30:05 2023","interfaces":[{"interface":"back","average":{"1min":2.4199999999999999,"5min":3.157,"15min":2.8140000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.19900000000000001},"max":{"1min":10.163,"5min":65.308999999999997,"15min":120.43300000000001},"last":0.45600000000000002},{"interface":"front","average":{"1min":2.3999999999999999,"5min":2.891,"15min":2.7599999999999998},"min":{"1min":0.33500000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":7.3620000000000001,"5min":65.120999999999995,"15min":120.44499999999999},"last":0.50600000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:30:05 2023","interfaces":[{"interface":"back","average":{"1min":1.403,"5min":2.4980000000000002,"15min":1.4610000000000001},"min":{"1min":0.32800000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":4.7240000000000002,"5min":58.067999999999998,"15min":58.067999999999998},"last":0.436},{"interface":"front","average":{"1min":1.6279999999999999,"5min":2.7589999999999999,"15min":1.5269999999999999},"min":{"1min":0.40400000000000003,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":4.6280000000000001,"5min":58.088000000000001,"15min":58.088000000000001},"last":0.52100000000000002}]},{"osd":5,"last update":"Mon Aug 28 02:30:05 2023","interfaces":[{"interface":"back","average":{"1min":1.359,"5min":3.2109999999999999,"15min":1.6579999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.29599999999999999},"max":{"1min":2.923,"5min":102.904,"15min":102.904},"last":0.47299999999999998},{"interface":"front","average":{"1min":1.556,"5min":3.2509999999999999,"15min":1.6619999999999999},"min":{"1min":0.36699999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":7.0039999999999996,"5min":103.283,"15min":103.283},"last":0.63400000000000001}]},{"osd":6,"last update":"Mon Aug 28 02:30:05 2023","interfaces":[{"interface":"back","average":{"1min":1.5880000000000001,"5min":1.877,"15min":1.4330000000000001},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.314},"max":{"1min":7.0300000000000002,"5min":9.4529999999999994,"15min":72.116},"last":0.66900000000000004},{"interface":"front","average":{"1min":1.7929999999999999,"5min":1.538,"15min":1.3560000000000001},"min":{"1min":0.35599999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":7.2990000000000004,"5min":7.2990000000000004,"15min":72.030000000000001},"last":0.39200000000000002}]},{"osd":7,"last update":"Mon Aug 28 02:30:05 2023","interfaces":[{"interface":"back","average":{"1min":1.6879999999999999,"5min":2.9660000000000002,"15min":1.631},"min":{"1min":0.38300000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":4.7640000000000002,"5min":54.427999999999997,"15min":54.427999999999997},"last":27.111999999999998},{"interface":"front","average":{"1min":1.855,"5min":3.3399999999999999,"15min":1.76},"min":{"1min":0.433,"5min":0.36399999999999999,"15min":0.34799999999999998},"max":{"1min":3.8100000000000001,"5min":54.445,"15min":54.445},"last":27.149999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019146,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":325288,"kb_used_data":15776,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93419864,"statfs":{"total":95995035648,"available":95661940736,"internally_reserved":0,"allocated":16154624,"data_stored":15451230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:30:10 2023","interfaces":[{"interface":"back","average":{"1min":1.528,"5min":2.8479999999999999,"15min":2.706},"min":{"1min":0.222,"5min":0.219,"15min":0.219},"max":{"1min":13.492000000000001,"5min":43.548000000000002,"15min":138.67400000000001},"last":0.48899999999999999},{"interface":"front","average":{"1min":1.4850000000000001,"5min":2.7149999999999999,"15min":2.637},"min":{"1min":0.251,"5min":0.248,"15min":0.248},"max":{"1min":8.0060000000000002,"5min":43.515000000000001,"15min":134.14599999999999},"last":0.39000000000000001}]},{"osd":1,"last update":"Mon Aug 28 02:30:10 2023","interfaces":[{"interface":"back","average":{"1min":1.5629999999999999,"5min":2.1640000000000001,"15min":1.3919999999999999},"min":{"1min":0.23599999999999999,"5min":0.221,"15min":0.221},"max":{"1min":9.75,"5min":34.945999999999998,"15min":53.622999999999998},"last":0.52100000000000002},{"interface":"front","average":{"1min":1.4039999999999999,"5min":2.073,"15min":1.4079999999999999},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":5.2919999999999998,"5min":34.869999999999997,"15min":53.774000000000001},"last":0.36299999999999999}]},{"osd":2,"last update":"Mon Aug 28 02:30:37 2023","interfaces":[{"interface":"back","average":{"1min":4.867,"5min":4.3040000000000003,"15min":2.8250000000000002},"min":{"1min":0.33100000000000002,"5min":0.27400000000000002,"15min":0.25600000000000001},"max":{"1min":56.948999999999998,"5min":64.063000000000002,"15min":74.099000000000004},"last":0.54300000000000004},{"interface":"front","average":{"1min":5.117,"5min":4.1769999999999996,"15min":2.7330000000000001},"min":{"1min":0.377,"5min":0.28999999999999998,"15min":0.25},"max":{"1min":56.979999999999997,"5min":64.073999999999998,"15min":74.081999999999994},"last":0.443}]},{"osd":3,"last update":"Mon Aug 28 02:30:10 2023","interfaces":[{"interface":"back","average":{"1min":11.220000000000001,"5min":6.0819999999999999,"15min":2.5099999999999998},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":159.10400000000001,"5min":159.10400000000001,"15min":159.10400000000001},"last":10.050000000000001},{"interface":"front","average":{"1min":11.119999999999999,"5min":5.9939999999999998,"15min":2.5979999999999999},"min":{"1min":0.33800000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":159.232,"5min":159.232,"15min":159.232},"last":4.2149999999999999}]},{"osd":5,"last update":"Mon Aug 28 02:29:50 2023","interfaces":[{"interface":"back","average":{"1min":0.78100000000000003,"5min":2.8140000000000001,"15min":1.397},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":4.3090000000000002,"5min":124.718,"15min":124.718},"last":1.234},{"interface":"front","average":{"1min":0.73599999999999999,"5min":3.1949999999999998,"15min":1.554},"min":{"1min":0.27800000000000002,"5min":0.247,"15min":0.247},"max":{"1min":4.665,"5min":124.694,"15min":124.694},"last":10.24}]},{"osd":6,"last update":"Mon Aug 28 02:30:10 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":1.264,"15min":0.94699999999999995},"min":{"1min":0.28499999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":2.6659999999999999,"5min":11.170999999999999,"15min":11.170999999999999},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.91500000000000004,"5min":1.1180000000000001,"15min":0.83299999999999996},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":5.2679999999999998,"5min":7.3170000000000002,"15min":7.3170000000000002},"last":0.41699999999999998}]},{"osd":7,"last update":"Mon Aug 28 02:30:27 2023","interfaces":[{"interface":"back","average":{"1min":1.651,"5min":1.6990000000000001,"15min":1.1180000000000001},"min":{"1min":0.312,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":8.3490000000000002,"5min":16.584,"15min":16.584},"last":10.151999999999999},{"interface":"front","average":{"1min":1.706,"5min":1.3919999999999999,"15min":1.137},"min":{"1min":0.378,"5min":0.311,"15min":0.311},"max":{"1min":7.4619999999999997,"5min":10.826000000000001,"15min":10.826000000000001},"last":4.1710000000000003}]}]},{"osd":5,"up_from":36,"seq":154618822918,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":351644,"kb_used_data":50196,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93393508,"statfs":{"total":95995035648,"available":95634952192,"internally_reserved":0,"allocated":51400704,"data_stored":50704777,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:30:02 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":2.052,"15min":2.6469999999999998},"min":{"1min":0.214,"5min":0.214,"15min":0.20300000000000001},"max":{"1min":4.548,"5min":42.229999999999997,"15min":131.06100000000001},"last":0.252},{"interface":"front","average":{"1min":1.4710000000000001,"5min":2.371,"15min":2.8460000000000001},"min":{"1min":0.27500000000000002,"5min":0.24199999999999999,"15min":0.22900000000000001},"max":{"1min":5.8129999999999997,"5min":42.213999999999999,"15min":131.262},"last":0.47299999999999998}]},{"osd":1,"last update":"Mon Aug 28 02:30:12 2023","interfaces":[{"interface":"back","average":{"1min":1.8819999999999999,"5min":1.7869999999999999,"15min":2.387},"min":{"1min":0.27400000000000002,"5min":0.216,"15min":0.216},"max":{"1min":7.7809999999999997,"5min":30.593,"15min":183.97900000000001},"last":1.8340000000000001},{"interface":"front","average":{"1min":1.1220000000000001,"5min":2.0070000000000001,"15min":2.5510000000000002},"min":{"1min":0.23499999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":5.8810000000000002,"5min":34.616999999999997,"15min":184.33699999999999},"last":3.484}]},{"osd":2,"last update":"Mon Aug 28 02:30:41 2023","interfaces":[{"interface":"back","average":{"1min":2.7469999999999999,"5min":6.117,"15min":3.9740000000000002},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.252},"max":{"1min":11.875,"5min":176.69499999999999,"15min":176.69499999999999},"last":0.38100000000000001},{"interface":"front","average":{"1min":3.7160000000000002,"5min":6.5620000000000003,"15min":4.21},"min":{"1min":0.35299999999999998,"5min":0.34000000000000002,"15min":0.28299999999999997},"max":{"1min":11.887,"5min":176.71199999999999,"15min":176.71199999999999},"last":3.3820000000000001}]},{"osd":3,"last update":"Mon Aug 28 02:30:12 2023","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":2.8730000000000002,"15min":1.6180000000000001},"min":{"1min":0.28799999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.1070000000000002,"5min":102.64400000000001,"15min":102.64400000000001},"last":1.266},{"interface":"front","average":{"1min":1.8919999999999999,"5min":3.6509999999999998,"15min":1.956},"min":{"1min":0.28699999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":8.1660000000000004,"5min":103.771,"15min":103.771},"last":8.3089999999999993}]},{"osd":4,"last update":"Mon Aug 28 02:29:50 2023","interfaces":[{"interface":"back","average":{"1min":1.8300000000000001,"5min":1.157,"15min":0.93100000000000005},"min":{"1min":0.26700000000000002,"5min":0.246,"15min":0.246},"max":{"1min":25.306999999999999,"5min":25.306999999999999,"15min":25.306999999999999},"last":6.141},{"interface":"front","average":{"1min":1.7250000000000001,"5min":1.2430000000000001,"15min":0.95899999999999996},"min":{"1min":0.29399999999999998,"5min":0.28999999999999998,"15min":0.28000000000000003},"max":{"1min":25.347000000000001,"5min":25.347000000000001,"15min":25.347000000000001},"last":1.2909999999999999}]},{"osd":6,"last update":"Mon Aug 28 02:30:12 2023","interfaces":[{"interface":"back","average":{"1min":1.0429999999999999,"5min":1.099,"15min":1.002},"min":{"1min":0.30399999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.6210000000000004,"5min":5.9870000000000001,"15min":14.803000000000001},"last":1.2110000000000001},{"interface":"front","average":{"1min":0.95499999999999996,"5min":1.04,"15min":0.91600000000000004},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":4.9509999999999996,"5min":8.4779999999999998,"15min":8.6479999999999997},"last":1.3129999999999999}]},{"osd":7,"last update":"Mon Aug 28 02:30:29 2023","interfaces":[{"interface":"back","average":{"1min":1.1419999999999999,"5min":1.5109999999999999,"15min":1.127},"min":{"1min":0.34300000000000003,"5min":0.33300000000000002,"15min":0.31},"max":{"1min":4.0060000000000002,"5min":25.379000000000001,"15min":25.379000000000001},"last":0.41399999999999998},{"interface":"front","average":{"1min":1.3160000000000001,"5min":1.2769999999999999,"15min":1.0569999999999999},"min":{"1min":0.38300000000000001,"5min":0.33700000000000002,"15min":0.32100000000000001},"max":{"1min":5.931,"5min":25.437000000000001,"15min":25.437000000000001},"last":3.4319999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57802752,"data_stored":57802752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":162594816,"data_stored":162594816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":274661376,"data_stored":274661376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":8716288,"data_stored":8716288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":34344960,"data_stored":34340888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":76808192,"data_stored":76808192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":330366976,"data_stored":330366976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-08-28T02:30:46.974 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:47.024 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:30:47.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:47.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:30:47.128 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:30:47.154 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:47.274 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:30:47.298 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:30:47.406 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:30:47.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:47.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:30:47.452 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.5 dump_blocked_ops 2023-08-28T02:30:47.509 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:30:47.542 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:47.587 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:30:47.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:47.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:30:47.659 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is unset 2023-08-28T02:30:47.690 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:30:47.720 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:47.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:47 smithi169 ceph-mon[159415]: 6.7 scrub starts 2023-08-28T02:30:47.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:47 smithi169 ceph-mon[159415]: 6.0 scrub starts 2023-08-28T02:30:47.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:47 smithi169 ceph-mon[159415]: pgmap v627: 99 pgs: 2 active+clean+scrubbing, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 16 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 532 op/s; 5829/624 objects degraded (934.135%); 0 B/s, 22 objects/s recovering 2023-08-28T02:30:47.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:47 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/4170968319' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:30:47.812 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:30:47.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:47 smithi143 ceph-mon[183059]: 6.7 scrub starts 2023-08-28T02:30:47.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:47 smithi143 ceph-mon[183059]: 6.0 scrub starts 2023-08-28T02:30:47.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:47 smithi143 ceph-mon[183059]: pgmap v627: 99 pgs: 2 active+clean+scrubbing, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 16 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 532 op/s; 5829/624 objects degraded (934.135%); 0 B/s, 22 objects/s recovering 2023-08-28T02:30:47.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:47 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/4170968319' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:30:47.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:47 smithi143 ceph-mon[189871]: 6.7 scrub starts 2023-08-28T02:30:47.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:47 smithi143 ceph-mon[189871]: 6.0 scrub starts 2023-08-28T02:30:47.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:47 smithi143 ceph-mon[189871]: pgmap v627: 99 pgs: 2 active+clean+scrubbing, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 16 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 532 op/s; 5829/624 objects degraded (934.135%); 0 B/s, 22 objects/s recovering 2023-08-28T02:30:47.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:47 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/4170968319' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:30:48.269 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:30:48.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:48.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:30:48.372 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:30:48.402 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:48.496 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:30:48.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:48.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:30:48.601 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:30:48.627 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:48.650 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:48 smithi169 ceph-mon[159415]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:30:48.650 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:48 smithi169 ceph-mon[159415]: failed to encode map e258 with expected crc 2023-08-28T02:30:48.650 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:48 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/4170968319' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:30:48.650 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:48 smithi169 ceph-mon[159415]: osdmap e258: 8 total, 8 up, 8 in 2023-08-28T02:30:48.650 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:48 smithi169 ceph-mon[159415]: failed to encode map e258 with expected crc 2023-08-28T02:30:48.650 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:48 smithi169 ceph-mon[159415]: failed to encode map e258 with expected crc 2023-08-28T02:30:48.650 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:48 smithi169 ceph-mon[159415]: 6.2 scrub starts 2023-08-28T02:30:48.674 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:30:48.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:48.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:30:48.777 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:30:48.803 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:48.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:48 smithi143 ceph-mon[183059]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:30:48.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:48 smithi143 ceph-mon[183059]: failed to encode map e258 with expected crc 2023-08-28T02:30:48.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:48 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/4170968319' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:30:48.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:48 smithi143 ceph-mon[183059]: osdmap e258: 8 total, 8 up, 8 in 2023-08-28T02:30:48.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:48 smithi143 ceph-mon[183059]: failed to encode map e258 with expected crc 2023-08-28T02:30:48.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:48 smithi143 ceph-mon[183059]: failed to encode map e258 with expected crc 2023-08-28T02:30:48.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:48 smithi143 ceph-mon[183059]: 6.2 scrub starts 2023-08-28T02:30:48.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:48 smithi143 ceph-mon[189871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:30:48.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:48 smithi143 ceph-mon[189871]: failed to encode map e258 with expected crc 2023-08-28T02:30:48.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:48 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/4170968319' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:30:48.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:48 smithi143 ceph-mon[189871]: osdmap e258: 8 total, 8 up, 8 in 2023-08-28T02:30:48.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:48 smithi143 ceph-mon[189871]: failed to encode map e258 with expected crc 2023-08-28T02:30:48.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:48 smithi143 ceph-mon[189871]: failed to encode map e258 with expected crc 2023-08-28T02:30:48.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:48 smithi143 ceph-mon[189871]: 6.2 scrub starts 2023-08-28T02:30:49.000 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:30:49.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:49.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:30:49.103 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:30:49.134 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:49.188 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:30:49.233 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:30:49.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:49.237 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:30:49.338 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:30:49.366 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:49.418 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:30:49.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:49.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:30:49.504 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:30:49.504 INFO:teuthology.orchestra.run.smithi169.stdout: "ops": [], 2023-08-28T02:30:49.504 INFO:teuthology.orchestra.run.smithi169.stdout: "complaint_time": 30, 2023-08-28T02:30:49.504 INFO:teuthology.orchestra.run.smithi169.stdout: "num_blocked_ops": 0 2023-08-28T02:30:49.504 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:30:49.522 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:30:49.550 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:49.595 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:30:49.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:49.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:30:49.698 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:30:49.706 INFO:teuthology.orchestra.run.smithi143.stderr:pool 'unique_pool_1' removed 2023-08-28T02:30:49.783 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:49.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:49 smithi143 ceph-mon[183059]: failed to encode map e258 with expected crc 2023-08-28T02:30:49.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:49 smithi143 ceph-mon[183059]: pgmap v629: 99 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 16 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 362 op/s; 5829/624 objects degraded (934.135%); 0 B/s, 25 objects/s recovering 2023-08-28T02:30:49.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:49 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3868663500' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-08-28T02:30:49.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:49 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-08-28T02:30:49.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:49 smithi143 ceph-mon[189871]: failed to encode map e258 with expected crc 2023-08-28T02:30:49.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:49 smithi143 ceph-mon[189871]: pgmap v629: 99 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 16 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 362 op/s; 5829/624 objects degraded (934.135%); 0 B/s, 25 objects/s recovering 2023-08-28T02:30:49.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:49 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3868663500' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-08-28T02:30:49.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:49 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-08-28T02:30:49.971 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph pg dump --format=json 2023-08-28T02:30:49.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:49 smithi169 ceph-mon[159415]: failed to encode map e258 with expected crc 2023-08-28T02:30:49.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:49 smithi169 ceph-mon[159415]: pgmap v629: 99 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 16 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 362 op/s; 5829/624 objects degraded (934.135%); 0 B/s, 25 objects/s recovering 2023-08-28T02:30:49.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:49 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3868663500' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-08-28T02:30:49.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:49 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-08-28T02:30:50.287 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.5 dump_historic_ops 2023-08-28T02:30:50.634 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:30:50.817 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set noscrub 2023-08-28T02:30:50.839 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:30:50.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:50.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:30:50.854 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-08-28T02:30:50.855 INFO:tasks.radosbench:Beginning radosbench... 2023-08-28T02:30:50.855 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2023-08-28T02:30:50.855 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd pool create unique_pool_2 16 2023-08-28T02:30:50.921 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:50 smithi143 ceph-mon[189871]: 6.7 deep-scrub starts 2023-08-28T02:30:50.921 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:50 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-08-28T02:30:50.921 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:50 smithi143 ceph-mon[189871]: osdmap e259: 8 total, 8 up, 8 in 2023-08-28T02:30:50.921 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:50 smithi143 ceph-mon[189871]: failed to encode map e259 with expected crc 2023-08-28T02:30:50.921 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:50 smithi143 ceph-mon[189871]: failed to encode map e259 with expected crc 2023-08-28T02:30:50.921 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:50 smithi143 ceph-mon[189871]: failed to encode map e259 with expected crc 2023-08-28T02:30:50.921 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:50 smithi143 ceph-mon[183059]: 6.7 deep-scrub starts 2023-08-28T02:30:50.922 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:50 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-08-28T02:30:50.922 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:50 smithi143 ceph-mon[183059]: osdmap e259: 8 total, 8 up, 8 in 2023-08-28T02:30:50.922 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:50 smithi143 ceph-mon[183059]: failed to encode map e259 with expected crc 2023-08-28T02:30:50.922 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:50 smithi143 ceph-mon[183059]: failed to encode map e259 with expected crc 2023-08-28T02:30:50.922 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:50 smithi143 ceph-mon[183059]: failed to encode map e259 with expected crc 2023-08-28T02:30:50.928 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:30:50.929 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:50 smithi169 ceph-mon[159415]: 6.7 deep-scrub starts 2023-08-28T02:30:50.929 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:50 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-08-28T02:30:50.929 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:50 smithi169 ceph-mon[159415]: osdmap e259: 8 total, 8 up, 8 in 2023-08-28T02:30:50.930 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:50 smithi169 ceph-mon[159415]: failed to encode map e259 with expected crc 2023-08-28T02:30:50.930 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:50 smithi169 ceph-mon[159415]: failed to encode map e259 with expected crc 2023-08-28T02:30:50.930 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:50 smithi169 ceph-mon[159415]: failed to encode map e259 with expected crc 2023-08-28T02:30:50.944 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:30:50.972 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:51.218 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:30:51.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:51.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:30:51.263 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:30:51.285 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:30:51.313 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:30:51.321 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:30:51.334 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:30:51.348 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:51.352 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:30:51.370 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:30:51.389 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:30:51.392 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:30:51.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:51.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:30:51.414 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:30:51.415 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:30:51.416 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:30:51.416 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:30:51.416 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:30:51.416 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:30:51.416 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:30:51.416 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:30:51.416 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:30:51.496 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:30:51.522 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:51.726 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:30:51.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:51.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:30:51.830 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:30:51.855 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:51 smithi169 ceph-mon[159415]: failed to encode map e259 with expected crc 2023-08-28T02:30:51.855 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:51 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1094517872' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:30:51.856 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:51 smithi169 ceph-mon[159415]: pgmap v631: 73 pgs: 73 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-08-28T02:30:51.856 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:51 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 5829/624 objects degraded (934.135%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:30:51.858 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:51.927 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:51 smithi143 ceph-mon[189871]: failed to encode map e259 with expected crc 2023-08-28T02:30:51.927 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:51 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1094517872' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:30:51.927 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:51 smithi143 ceph-mon[189871]: pgmap v631: 73 pgs: 73 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-08-28T02:30:51.927 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:51 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 5829/624 objects degraded (934.135%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:30:51.928 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:51 smithi143 ceph-mon[183059]: failed to encode map e259 with expected crc 2023-08-28T02:30:51.928 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:51 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1094517872' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:30:51.928 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:51 smithi143 ceph-mon[183059]: pgmap v631: 73 pgs: 73 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-08-28T02:30:51.928 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:51 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 5829/624 objects degraded (934.135%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:30:51.928 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:30:51.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:51.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:30:52.032 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:30:52.061 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:52.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:30:51 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:30:51] "GET /metrics HTTP/1.1" 200 36484 "" "Prometheus/2.33.4" 2023-08-28T02:30:52.206 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:30:52.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:52.209 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:30:52.285 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:30:52.285 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:30:52.309 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:30:52.337 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:52.453 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:30:52.584 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:30:52.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:52.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:30:52.598 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:30:52.688 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:30:52.719 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:52.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:52 smithi169 ceph-mon[159415]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5829/624 objects degraded (934.135%), 4 pgs degraded, 4 pgs undersized) 2023-08-28T02:30:52.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:52 smithi169 ceph-mon[159415]: Cluster is now healthy 2023-08-28T02:30:52.817 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:30:52.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:52.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:30:52.922 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:30:52.946 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:52 smithi143 ceph-mon[183059]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5829/624 objects degraded (934.135%), 4 pgs degraded, 4 pgs undersized) 2023-08-28T02:30:52.946 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:52 smithi143 ceph-mon[183059]: Cluster is now healthy 2023-08-28T02:30:52.946 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:52 smithi143 ceph-mon[189871]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5829/624 objects degraded (934.135%), 4 pgs degraded, 4 pgs undersized) 2023-08-28T02:30:52.947 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:52 smithi143 ceph-mon[189871]: Cluster is now healthy 2023-08-28T02:30:52.949 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:52.969 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.6 dump_ops_in_flight 2023-08-28T02:30:53.279 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:30:53.280 INFO:teuthology.orchestra.run.smithi143.stderr:dumped all 2023-08-28T02:30:53.315 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:30:53.890 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:53 smithi143 ceph-mon[183059]: from='client.35325 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:30:53.890 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:53 smithi143 ceph-mon[183059]: pgmap v632: 73 pgs: 73 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-08-28T02:30:53.890 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:53 smithi143 ceph-mon[189871]: from='client.35325 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:30:53.890 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:53 smithi143 ceph-mon[189871]: pgmap v632: 73 pgs: 73 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-08-28T02:30:53.900 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:30:53.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:53.903 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:30:53.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:53 smithi169 ceph-mon[159415]: from='client.35325 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:30:53.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:53 smithi169 ceph-mon[159415]: pgmap v632: 73 pgs: 73 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-08-28T02:30:54.004 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:30:54.033 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:54.075 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:30:54.076 INFO:teuthology.orchestra.run.smithi143.stdout:{"pg_ready":true,"pg_map":{"version":631,"stamp":"2023-08-28T02:30:51.321941+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6274,"num_read_kb":13707,"num_write":1106,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":189,"num_bytes_recovered":14963785,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1170,"ondisk_log_size":1170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":309,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":3288808,"kb_used_data":914664,"kb_used_omap":29,"kb_used_meta":2374050,"kb_avail":746672408,"statfs":{"total":767960285184,"available":764592545792,"internally_reserved":0,"allocated":936615936,"data_stored":929095781,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30674,"internal_metadata":2431027246},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-109805592,"num_objects":-1677,"num_object_clones":0,"num_object_copies":-5031,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-5941,"num_objects_misplaced":-346,"num_objects_unfound":0,"num_objects_dirty":-1677,"num_whiteouts":0,"num_read":-1,"num_read_kb":-1,"num_write":-165633,"num_write_kb":-5351425,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-20427,"num_bytes_recovered":-1166802944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002505"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":258,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.432476+0000","last_change":"2023-08-28T02:27:31.367469+0000","last_active":"2023-08-28T02:30:32.432476+0000","last_peered":"2023-08-28T02:30:32.432476+0000","last_clean":"2023-08-28T02:30:32.432476+0000","last_became_active":"2023-08-28T02:27:31.366946+0000","last_became_peered":"2023-08-28T02:27:31.366946+0000","last_unstale":"2023-08-28T02:30:32.432476+0000","last_undegraded":"2023-08-28T02:30:32.432476+0000","last_fullsized":"2023-08-28T02:30:32.432476+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":144,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:40.598099+0000","last_change":"2023-08-28T02:27:31.324452+0000","last_active":"2023-08-28T02:30:40.598099+0000","last_peered":"2023-08-28T02:30:40.598099+0000","last_clean":"2023-08-28T02:30:40.598099+0000","last_became_active":"2023-08-28T02:27:31.323749+0000","last_became_peered":"2023-08-28T02:27:31.323749+0000","last_unstale":"2023-08-28T02:30:40.598099+0000","last_undegraded":"2023-08-28T02:30:40.598099+0000","last_fullsized":"2023-08-28T02:30:40.598099+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:18:10.904127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":65,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:40.597745+0000","last_change":"2023-08-28T02:27:31.337338+0000","last_active":"2023-08-28T02:30:40.597745+0000","last_peered":"2023-08-28T02:30:40.597745+0000","last_clean":"2023-08-28T02:30:40.597745+0000","last_became_active":"2023-08-28T02:27:31.337018+0000","last_became_peered":"2023-08-28T02:27:31.337018+0000","last_unstale":"2023-08-28T02:30:40.597745+0000","last_undegraded":"2023-08-28T02:30:40.597745+0000","last_fullsized":"2023-08-28T02:30:40.597745+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:46:34.845663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":246,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:40.609562+0000","last_change":"2023-08-28T02:23:59.223634+0000","last_active":"2023-08-28T02:30:40.609562+0000","last_peered":"2023-08-28T02:30:40.609562+0000","last_clean":"2023-08-28T02:30:40.609562+0000","last_became_active":"2023-08-28T02:23:59.222704+0000","last_became_peered":"2023-08-28T02:23:59.222704+0000","last_unstale":"2023-08-28T02:30:40.609562+0000","last_undegraded":"2023-08-28T02:30:40.609562+0000","last_fullsized":"2023-08-28T02:30:40.609562+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:51:52.752361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":135,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:40.598899+0000","last_change":"2023-08-28T02:27:31.318153+0000","last_active":"2023-08-28T02:30:40.598899+0000","last_peered":"2023-08-28T02:30:40.598899+0000","last_clean":"2023-08-28T02:30:40.598899+0000","last_became_active":"2023-08-28T02:27:31.317732+0000","last_became_peered":"2023-08-28T02:27:31.317732+0000","last_unstale":"2023-08-28T02:30:40.598899+0000","last_undegraded":"2023-08-28T02:30:40.598899+0000","last_fullsized":"2023-08-28T02:30:40.598899+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:06:13.421674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":46,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.457565+0000","last_change":"2023-08-28T02:28:25.348941+0000","last_active":"2023-08-28T02:30:32.457565+0000","last_peered":"2023-08-28T02:30:32.457565+0000","last_clean":"2023-08-28T02:30:32.457565+0000","last_became_active":"2023-08-28T02:28:25.348650+0000","last_became_peered":"2023-08-28T02:28:25.348650+0000","last_unstale":"2023-08-28T02:30:32.457565+0000","last_undegraded":"2023-08-28T02:30:32.457565+0000","last_fullsized":"2023-08-28T02:30:32.457565+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":226,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:43.925506+0000","last_change":"2023-08-28T02:23:59.227592+0000","last_active":"2023-08-28T02:30:43.925506+0000","last_peered":"2023-08-28T02:30:43.925506+0000","last_clean":"2023-08-28T02:30:43.925506+0000","last_became_active":"2023-08-28T02:23:59.227342+0000","last_became_peered":"2023-08-28T02:23:59.227342+0000","last_unstale":"2023-08-28T02:30:43.925506+0000","last_undegraded":"2023-08-28T02:30:43.925506+0000","last_fullsized":"2023-08-28T02:30:43.925506+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:08:19.874396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":153,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:40.599974+0000","last_change":"2023-08-28T02:28:25.348233+0000","last_active":"2023-08-28T02:30:40.599974+0000","last_peered":"2023-08-28T02:30:40.599974+0000","last_clean":"2023-08-28T02:30:40.599974+0000","last_became_active":"2023-08-28T02:28:25.347896+0000","last_became_peered":"2023-08-28T02:28:25.347896+0000","last_unstale":"2023-08-28T02:30:40.599974+0000","last_undegraded":"2023-08-28T02:30:40.599974+0000","last_fullsized":"2023-08-28T02:30:40.599974+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:50:17.674564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":259,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.452364+0000","last_change":"2023-08-28T02:27:31.328499+0000","last_active":"2023-08-28T02:30:32.452364+0000","last_peered":"2023-08-28T02:30:32.452364+0000","last_clean":"2023-08-28T02:30:32.452364+0000","last_became_active":"2023-08-28T02:27:31.328164+0000","last_became_peered":"2023-08-28T02:27:31.328164+0000","last_unstale":"2023-08-28T02:30:32.452364+0000","last_undegraded":"2023-08-28T02:30:32.452364+0000","last_fullsized":"2023-08-28T02:30:32.452364+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":135,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:40.598211+0000","last_change":"2023-08-28T02:28:25.348760+0000","last_active":"2023-08-28T02:30:40.598211+0000","last_peered":"2023-08-28T02:30:40.598211+0000","last_clean":"2023-08-28T02:30:40.598211+0000","last_became_active":"2023-08-28T02:28:25.348465+0000","last_became_peered":"2023-08-28T02:28:25.348465+0000","last_unstale":"2023-08-28T02:30:40.598211+0000","last_undegraded":"2023-08-28T02:30:40.598211+0000","last_fullsized":"2023-08-28T02:30:40.598211+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:43:05.659999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":142,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.455087+0000","last_change":"2023-08-28T02:23:59.316561+0000","last_active":"2023-08-28T02:30:32.455087+0000","last_peered":"2023-08-28T02:30:32.455087+0000","last_clean":"2023-08-28T02:30:32.455087+0000","last_became_active":"2023-08-28T02:23:59.227433+0000","last_became_peered":"2023-08-28T02:23:59.227433+0000","last_unstale":"2023-08-28T02:30:32.455087+0000","last_undegraded":"2023-08-28T02:30:32.455087+0000","last_fullsized":"2023-08-28T02:30:32.455087+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":75,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:40.580786+0000","last_change":"2023-08-28T02:28:25.347771+0000","last_active":"2023-08-28T02:30:40.580786+0000","last_peered":"2023-08-28T02:30:40.580786+0000","last_clean":"2023-08-28T02:30:40.580786+0000","last_became_active":"2023-08-28T02:28:25.347320+0000","last_became_peered":"2023-08-28T02:28:25.347320+0000","last_unstale":"2023-08-28T02:30:40.580786+0000","last_undegraded":"2023-08-28T02:30:40.580786+0000","last_fullsized":"2023-08-28T02:30:40.580786+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:20:27.350131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"86'22","reported_seq":307,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.430747+0000","last_change":"2023-08-28T02:28:25.361783+0000","last_active":"2023-08-28T02:30:32.430747+0000","last_peered":"2023-08-28T02:30:32.430747+0000","last_clean":"2023-08-28T02:30:32.430747+0000","last_became_active":"2023-08-28T02:28:25.351418+0000","last_became_peered":"2023-08-28T02:28:25.351418+0000","last_unstale":"2023-08-28T02:30:32.430747+0000","last_undegraded":"2023-08-28T02:30:32.430747+0000","last_fullsized":"2023-08-28T02:30:32.430747+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":273,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.430519+0000","last_change":"2023-08-28T02:28:25.352395+0000","last_active":"2023-08-28T02:30:32.430519+0000","last_peered":"2023-08-28T02:30:32.430519+0000","last_clean":"2023-08-28T02:30:32.430519+0000","last_became_active":"2023-08-28T02:28:25.352189+0000","last_became_peered":"2023-08-28T02:28:25.352189+0000","last_unstale":"2023-08-28T02:30:32.430519+0000","last_undegraded":"2023-08-28T02:30:32.430519+0000","last_fullsized":"2023-08-28T02:30:32.430519+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":127,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.453815+0000","last_change":"2023-08-28T02:23:38.983634+0000","last_active":"2023-08-28T02:30:32.453815+0000","last_peered":"2023-08-28T02:30:32.453815+0000","last_clean":"2023-08-28T02:30:32.453815+0000","last_became_active":"2023-08-28T02:23:38.344566+0000","last_became_peered":"2023-08-28T02:23:38.344566+0000","last_unstale":"2023-08-28T02:30:32.453815+0000","last_undegraded":"2023-08-28T02:30:32.453815+0000","last_fullsized":"2023-08-28T02:30:32.453815+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":135,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:40.604320+0000","last_change":"2023-08-28T02:27:31.344974+0000","last_active":"2023-08-28T02:30:40.604320+0000","last_peered":"2023-08-28T02:30:40.604320+0000","last_clean":"2023-08-28T02:30:40.604320+0000","last_became_active":"2023-08-28T02:27:31.344281+0000","last_became_peered":"2023-08-28T02:27:31.344281+0000","last_unstale":"2023-08-28T02:30:40.604320+0000","last_undegraded":"2023-08-28T02:30:40.604320+0000","last_fullsized":"2023-08-28T02:30:40.604320+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:04:02.821752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":244,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:40.609233+0000","last_change":"2023-08-28T02:23:59.223066+0000","last_active":"2023-08-28T02:30:40.609233+0000","last_peered":"2023-08-28T02:30:40.609233+0000","last_clean":"2023-08-28T02:30:40.609233+0000","last_became_active":"2023-08-28T02:23:59.222731+0000","last_became_peered":"2023-08-28T02:23:59.222731+0000","last_unstale":"2023-08-28T02:30:40.609233+0000","last_undegraded":"2023-08-28T02:30:40.609233+0000","last_fullsized":"2023-08-28T02:30:40.609233+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:48:37.558266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":283,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.452357+0000","last_change":"2023-08-28T02:23:59.210469+0000","last_active":"2023-08-28T02:30:32.452357+0000","last_peered":"2023-08-28T02:30:32.452357+0000","last_clean":"2023-08-28T02:30:32.452357+0000","last_became_active":"2023-08-28T02:23:59.210031+0000","last_became_peered":"2023-08-28T02:23:59.210031+0000","last_unstale":"2023-08-28T02:30:32.452357+0000","last_undegraded":"2023-08-28T02:30:32.452357+0000","last_fullsized":"2023-08-28T02:30:32.452357+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":139,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.452486+0000","last_change":"2023-08-28T02:28:25.354526+0000","last_active":"2023-08-28T02:30:32.452486+0000","last_peered":"2023-08-28T02:30:32.452486+0000","last_clean":"2023-08-28T02:30:32.452486+0000","last_became_active":"2023-08-28T02:28:25.350991+0000","last_became_peered":"2023-08-28T02:28:25.350991+0000","last_unstale":"2023-08-28T02:30:32.452486+0000","last_undegraded":"2023-08-28T02:30:32.452486+0000","last_fullsized":"2023-08-28T02:30:32.452486+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":245,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:43.925476+0000","last_change":"2023-08-28T02:23:59.220932+0000","last_active":"2023-08-28T02:30:43.925476+0000","last_peered":"2023-08-28T02:30:43.925476+0000","last_clean":"2023-08-28T02:30:43.925476+0000","last_became_active":"2023-08-28T02:23:59.220488+0000","last_became_peered":"2023-08-28T02:23:59.220488+0000","last_unstale":"2023-08-28T02:30:43.925476+0000","last_undegraded":"2023-08-28T02:30:43.925476+0000","last_fullsized":"2023-08-28T02:30:43.925476+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:53:09.216340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":56,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.452509+0000","last_change":"2023-08-28T02:28:25.703333+0000","last_active":"2023-08-28T02:30:32.452509+0000","last_peered":"2023-08-28T02:30:32.452509+0000","last_clean":"2023-08-28T02:30:32.452509+0000","last_became_active":"2023-08-28T02:28:25.343515+0000","last_became_peered":"2023-08-28T02:28:25.343515+0000","last_unstale":"2023-08-28T02:30:32.452509+0000","last_undegraded":"2023-08-28T02:30:32.452509+0000","last_fullsized":"2023-08-28T02:30:32.452509+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":135,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:40.598587+0000","last_change":"2023-08-28T02:27:31.330059+0000","last_active":"2023-08-28T02:30:40.598587+0000","last_peered":"2023-08-28T02:30:40.598587+0000","last_clean":"2023-08-28T02:30:40.598587+0000","last_became_active":"2023-08-28T02:27:31.329769+0000","last_became_peered":"2023-08-28T02:27:31.329769+0000","last_unstale":"2023-08-28T02:30:40.598587+0000","last_undegraded":"2023-08-28T02:30:40.598587+0000","last_fullsized":"2023-08-28T02:30:40.598587+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:14:54.349909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":125,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.452285+0000","last_change":"2023-08-28T02:23:38.657185+0000","last_active":"2023-08-28T02:30:32.452285+0000","last_peered":"2023-08-28T02:30:32.452285+0000","last_clean":"2023-08-28T02:30:32.452285+0000","last_became_active":"2023-08-28T02:23:38.339016+0000","last_became_peered":"2023-08-28T02:23:38.339016+0000","last_unstale":"2023-08-28T02:30:32.452285+0000","last_undegraded":"2023-08-28T02:30:32.452285+0000","last_fullsized":"2023-08-28T02:30:32.452285+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":243,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.452014+0000","last_change":"2023-08-28T02:28:25.348106+0000","last_active":"2023-08-28T02:30:32.452014+0000","last_peered":"2023-08-28T02:30:32.452014+0000","last_clean":"2023-08-28T02:30:32.452014+0000","last_became_active":"2023-08-28T02:28:25.347786+0000","last_became_peered":"2023-08-28T02:28:25.347786+0000","last_unstale":"2023-08-28T02:30:32.452014+0000","last_undegraded":"2023-08-28T02:30:32.452014+0000","last_fullsized":"2023-08-28T02:30:32.452014+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":151,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.452459+0000","last_change":"2023-08-28T02:27:31.395328+0000","last_active":"2023-08-28T02:30:32.452459+0000","last_peered":"2023-08-28T02:30:32.452459+0000","last_clean":"2023-08-28T02:30:32.452459+0000","last_became_active":"2023-08-28T02:27:31.328228+0000","last_became_peered":"2023-08-28T02:27:31.328228+0000","last_unstale":"2023-08-28T02:30:32.452459+0000","last_undegraded":"2023-08-28T02:30:32.452459+0000","last_fullsized":"2023-08-28T02:30:32.452459+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":292,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.430788+0000","last_change":"2023-08-28T02:27:31.344915+0000","last_active":"2023-08-28T02:30:32.430788+0000","last_peered":"2023-08-28T02:30:32.430788+0000","last_clean":"2023-08-28T02:30:32.430788+0000","last_became_active":"2023-08-28T02:27:31.344577+0000","last_became_peered":"2023-08-28T02:27:31.344577+0000","last_unstale":"2023-08-28T02:30:32.430788+0000","last_undegraded":"2023-08-28T02:30:32.430788+0000","last_fullsized":"2023-08-28T02:30:32.430788+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":151,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.455337+0000","last_change":"2023-08-28T02:28:25.749249+0000","last_active":"2023-08-28T02:30:32.455337+0000","last_peered":"2023-08-28T02:30:32.455337+0000","last_clean":"2023-08-28T02:30:32.455337+0000","last_became_active":"2023-08-28T02:28:25.397684+0000","last_became_peered":"2023-08-28T02:28:25.397684+0000","last_unstale":"2023-08-28T02:30:32.455337+0000","last_undegraded":"2023-08-28T02:30:32.455337+0000","last_fullsized":"2023-08-28T02:30:32.455337+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":46,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.455249+0000","last_change":"2023-08-28T02:28:25.350802+0000","last_active":"2023-08-28T02:30:32.455249+0000","last_peered":"2023-08-28T02:30:32.455249+0000","last_clean":"2023-08-28T02:30:32.455249+0000","last_became_active":"2023-08-28T02:28:25.350474+0000","last_became_peered":"2023-08-28T02:28:25.350474+0000","last_unstale":"2023-08-28T02:30:32.455249+0000","last_undegraded":"2023-08-28T02:30:32.455249+0000","last_fullsized":"2023-08-28T02:30:32.455249+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":260,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:40.604483+0000","last_change":"2023-08-28T02:28:25.445982+0000","last_active":"2023-08-28T02:30:40.604483+0000","last_peered":"2023-08-28T02:30:40.604483+0000","last_clean":"2023-08-28T02:30:40.604483+0000","last_became_active":"2023-08-28T02:28:25.443577+0000","last_became_peered":"2023-08-28T02:28:25.443577+0000","last_unstale":"2023-08-28T02:30:40.604483+0000","last_undegraded":"2023-08-28T02:30:40.604483+0000","last_fullsized":"2023-08-28T02:30:40.604483+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:24:03.381751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":238,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:40.609321+0000","last_change":"2023-08-28T02:27:31.319783+0000","last_active":"2023-08-28T02:30:40.609321+0000","last_peered":"2023-08-28T02:30:40.609321+0000","last_clean":"2023-08-28T02:30:40.609321+0000","last_became_active":"2023-08-28T02:27:31.319451+0000","last_became_peered":"2023-08-28T02:27:31.319451+0000","last_unstale":"2023-08-28T02:30:40.609321+0000","last_undegraded":"2023-08-28T02:30:40.609321+0000","last_fullsized":"2023-08-28T02:30:40.609321+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:20:37.097889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":122,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:44.247054+0000","last_change":"2023-08-28T02:23:59.230059+0000","last_active":"2023-08-28T02:30:44.247054+0000","last_peered":"2023-08-28T02:30:44.247054+0000","last_clean":"2023-08-28T02:30:44.247054+0000","last_became_active":"2023-08-28T02:23:59.217783+0000","last_became_peered":"2023-08-28T02:23:59.217783+0000","last_unstale":"2023-08-28T02:30:44.247054+0000","last_undegraded":"2023-08-28T02:30:44.247054+0000","last_fullsized":"2023-08-28T02:30:44.247054+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:00:12.145395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":277,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.430720+0000","last_change":"2023-08-28T02:28:25.351698+0000","last_active":"2023-08-28T02:30:32.430720+0000","last_peered":"2023-08-28T02:30:32.430720+0000","last_clean":"2023-08-28T02:30:32.430720+0000","last_became_active":"2023-08-28T02:28:25.351349+0000","last_became_peered":"2023-08-28T02:28:25.351349+0000","last_unstale":"2023-08-28T02:30:32.430720+0000","last_undegraded":"2023-08-28T02:30:32.430720+0000","last_fullsized":"2023-08-28T02:30:32.430720+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":46,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.455378+0000","last_change":"2023-08-28T02:28:25.345274+0000","last_active":"2023-08-28T02:30:32.455378+0000","last_peered":"2023-08-28T02:30:32.455378+0000","last_clean":"2023-08-28T02:30:32.455378+0000","last_became_active":"2023-08-28T02:28:25.344885+0000","last_became_peered":"2023-08-28T02:28:25.344885+0000","last_unstale":"2023-08-28T02:30:32.455378+0000","last_undegraded":"2023-08-28T02:30:32.455378+0000","last_fullsized":"2023-08-28T02:30:32.455378+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":303,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.430871+0000","last_change":"2023-08-28T02:27:31.358436+0000","last_active":"2023-08-28T02:30:32.430871+0000","last_peered":"2023-08-28T02:30:32.430871+0000","last_clean":"2023-08-28T02:30:32.430871+0000","last_became_active":"2023-08-28T02:27:31.358185+0000","last_became_peered":"2023-08-28T02:27:31.358185+0000","last_unstale":"2023-08-28T02:30:32.430871+0000","last_undegraded":"2023-08-28T02:30:32.430871+0000","last_fullsized":"2023-08-28T02:30:32.430871+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":166,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.430646+0000","last_change":"2023-08-28T02:23:38.347043+0000","last_active":"2023-08-28T02:30:32.430646+0000","last_peered":"2023-08-28T02:30:32.430646+0000","last_clean":"2023-08-28T02:30:32.430646+0000","last_became_active":"2023-08-28T02:23:38.346680+0000","last_became_peered":"2023-08-28T02:23:38.346680+0000","last_unstale":"2023-08-28T02:30:32.430646+0000","last_undegraded":"2023-08-28T02:30:32.430646+0000","last_fullsized":"2023-08-28T02:30:32.430646+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":224,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:40.609700+0000","last_change":"2023-08-28T02:28:25.347494+0000","last_active":"2023-08-28T02:30:40.609700+0000","last_peered":"2023-08-28T02:30:40.609700+0000","last_clean":"2023-08-28T02:30:40.609700+0000","last_became_active":"2023-08-28T02:28:25.347142+0000","last_became_peered":"2023-08-28T02:28:25.347142+0000","last_unstale":"2023-08-28T02:30:40.609700+0000","last_undegraded":"2023-08-28T02:30:40.609700+0000","last_fullsized":"2023-08-28T02:30:40.609700+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:37:14.384884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":136,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:40.601160+0000","last_change":"2023-08-28T02:27:31.325260+0000","last_active":"2023-08-28T02:30:40.601160+0000","last_peered":"2023-08-28T02:30:40.601160+0000","last_clean":"2023-08-28T02:30:40.601160+0000","last_became_active":"2023-08-28T02:27:31.324950+0000","last_became_peered":"2023-08-28T02:27:31.324950+0000","last_unstale":"2023-08-28T02:30:40.601160+0000","last_undegraded":"2023-08-28T02:30:40.601160+0000","last_fullsized":"2023-08-28T02:30:40.601160+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:21:33.833261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":66,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:40.580457+0000","last_change":"2023-08-28T02:27:31.322118+0000","last_active":"2023-08-28T02:30:40.580457+0000","last_peered":"2023-08-28T02:30:40.580457+0000","last_clean":"2023-08-28T02:30:40.580457+0000","last_became_active":"2023-08-28T02:27:31.321338+0000","last_became_peered":"2023-08-28T02:27:31.321338+0000","last_unstale":"2023-08-28T02:30:40.580457+0000","last_undegraded":"2023-08-28T02:30:40.580457+0000","last_fullsized":"2023-08-28T02:30:40.580457+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:56:14.210571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"78'126","reported_seq":628,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:40.581245+0000","last_change":"2023-08-28T02:28:27.383137+0000","last_active":"2023-08-28T02:30:40.581245+0000","last_peered":"2023-08-28T02:30:40.581245+0000","last_clean":"2023-08-28T02:30:40.581245+0000","last_became_active":"2023-08-28T02:28:27.382488+0000","last_became_peered":"2023-08-28T02:28:27.382488+0000","last_unstale":"2023-08-28T02:30:40.581245+0000","last_undegraded":"2023-08-28T02:30:40.581245+0000","last_fullsized":"2023-08-28T02:30:40.581245+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:14:48.414426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":743,"num_read_kb":1331,"num_write":125,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":256,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.452279+0000","last_change":"2023-08-28T02:23:59.207148+0000","last_active":"2023-08-28T02:30:32.452279+0000","last_peered":"2023-08-28T02:30:32.452279+0000","last_clean":"2023-08-28T02:30:32.452279+0000","last_became_active":"2023-08-28T02:23:59.206802+0000","last_became_peered":"2023-08-28T02:23:59.206802+0000","last_unstale":"2023-08-28T02:30:32.452279+0000","last_undegraded":"2023-08-28T02:30:32.452279+0000","last_fullsized":"2023-08-28T02:30:32.452279+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":66,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:40.580719+0000","last_change":"2023-08-28T02:27:31.328027+0000","last_active":"2023-08-28T02:30:40.580719+0000","last_peered":"2023-08-28T02:30:40.580719+0000","last_clean":"2023-08-28T02:30:40.580719+0000","last_became_active":"2023-08-28T02:27:31.326529+0000","last_became_peered":"2023-08-28T02:27:31.326529+0000","last_unstale":"2023-08-28T02:30:40.580719+0000","last_undegraded":"2023-08-28T02:30:40.580719+0000","last_fullsized":"2023-08-28T02:30:40.580719+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:56:57.331355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"78'204","reported_seq":1343,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:40.580078+0000","last_change":"2023-08-28T02:28:59.480561+0000","last_active":"2023-08-28T02:30:40.580078+0000","last_peered":"2023-08-28T02:30:40.580078+0000","last_clean":"2023-08-28T02:30:40.580078+0000","last_became_active":"2023-08-28T02:28:59.479868+0000","last_became_peered":"2023-08-28T02:28:59.479868+0000","last_unstale":"2023-08-28T02:30:40.580078+0000","last_undegraded":"2023-08-28T02:30:40.580078+0000","last_fullsized":"2023-08-28T02:30:40.580078+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":204,"log_dups_size":0,"ondisk_log_size":204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:39:50.444629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1521,"num_read_kb":2511,"num_write":178,"num_write_kb":1175,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":251,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.430649+0000","last_change":"2023-08-28T02:28:25.352417+0000","last_active":"2023-08-28T02:30:32.430649+0000","last_peered":"2023-08-28T02:30:32.430649+0000","last_clean":"2023-08-28T02:30:32.430649+0000","last_became_active":"2023-08-28T02:28:25.352216+0000","last_became_peered":"2023-08-28T02:28:25.352216+0000","last_unstale":"2023-08-28T02:30:32.430649+0000","last_undegraded":"2023-08-28T02:30:32.430649+0000","last_fullsized":"2023-08-28T02:30:32.430649+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":66,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:40.581096+0000","last_change":"2023-08-28T02:27:31.376755+0000","last_active":"2023-08-28T02:30:40.581096+0000","last_peered":"2023-08-28T02:30:40.581096+0000","last_clean":"2023-08-28T02:30:40.581096+0000","last_became_active":"2023-08-28T02:27:31.373279+0000","last_became_peered":"2023-08-28T02:27:31.373279+0000","last_unstale":"2023-08-28T02:30:40.581096+0000","last_undegraded":"2023-08-28T02:30:40.581096+0000","last_fullsized":"2023-08-28T02:30:40.581096+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:26:14.628941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"78'47","reported_seq":475,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:44.140481+0000","last_change":"2023-08-28T02:24:05.799647+0000","last_active":"2023-08-28T02:30:44.140481+0000","last_peered":"2023-08-28T02:30:44.140481+0000","last_clean":"2023-08-28T02:30:44.140481+0000","last_became_active":"2023-08-28T02:23:59.210481+0000","last_became_peered":"2023-08-28T02:23:59.210481+0000","last_unstale":"2023-08-28T02:30:44.140481+0000","last_undegraded":"2023-08-28T02:30:44.140481+0000","last_fullsized":"2023-08-28T02:30:44.140481+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":47,"log_dups_size":0,"ondisk_log_size":47,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:40:18.542375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":162,"num_read_kb":1281,"num_write":48,"num_write_kb":1153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":75,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:40.581263+0000","last_change":"2023-08-28T02:28:25.343406+0000","last_active":"2023-08-28T02:30:40.581263+0000","last_peered":"2023-08-28T02:30:40.581263+0000","last_clean":"2023-08-28T02:30:40.581263+0000","last_became_active":"2023-08-28T02:28:25.343065+0000","last_became_peered":"2023-08-28T02:28:25.343065+0000","last_unstale":"2023-08-28T02:30:40.581263+0000","last_undegraded":"2023-08-28T02:30:40.581263+0000","last_fullsized":"2023-08-28T02:30:40.581263+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:20:31.893609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":66,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:40.580524+0000","last_change":"2023-08-28T02:27:31.322967+0000","last_active":"2023-08-28T02:30:40.580524+0000","last_peered":"2023-08-28T02:30:40.580524+0000","last_clean":"2023-08-28T02:30:40.580524+0000","last_became_active":"2023-08-28T02:27:31.322655+0000","last_became_peered":"2023-08-28T02:27:31.322655+0000","last_unstale":"2023-08-28T02:30:40.580524+0000","last_undegraded":"2023-08-28T02:30:40.580524+0000","last_fullsized":"2023-08-28T02:30:40.580524+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:05:15.868115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"78'114","reported_seq":620,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.452511+0000","last_change":"2023-08-28T02:24:01.210739+0000","last_active":"2023-08-28T02:30:32.452511+0000","last_peered":"2023-08-28T02:30:32.452511+0000","last_clean":"2023-08-28T02:30:32.452511+0000","last_became_active":"2023-08-28T02:24:01.210386+0000","last_became_peered":"2023-08-28T02:24:01.210386+0000","last_unstale":"2023-08-28T02:30:32.452511+0000","last_undegraded":"2023-08-28T02:30:32.452511+0000","last_fullsized":"2023-08-28T02:30:32.452511+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":114,"log_dups_size":0,"ondisk_log_size":114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":358,"num_read_kb":1352,"num_write":102,"num_write_kb":1104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"86'139","reported_seq":467,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.452032+0000","last_change":"2023-08-28T02:29:58.484566+0000","last_active":"2023-08-28T02:30:32.452032+0000","last_peered":"2023-08-28T02:30:32.452032+0000","last_clean":"2023-08-28T02:30:32.452032+0000","last_became_active":"2023-08-28T02:29:58.484046+0000","last_became_peered":"2023-08-28T02:29:58.484046+0000","last_unstale":"2023-08-28T02:30:32.452032+0000","last_undegraded":"2023-08-28T02:30:32.452032+0000","last_fullsized":"2023-08-28T02:30:32.452032+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_clean_scrub_stamp":"2023-08-28T02:08:02.043718+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":136,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:40.598438+0000","last_change":"2023-08-28T02:28:25.347600+0000","last_active":"2023-08-28T02:30:40.598438+0000","last_peered":"2023-08-28T02:30:40.598438+0000","last_clean":"2023-08-28T02:30:40.598438+0000","last_became_active":"2023-08-28T02:28:25.347237+0000","last_became_peered":"2023-08-28T02:28:25.347237+0000","last_unstale":"2023-08-28T02:30:40.598438+0000","last_undegraded":"2023-08-28T02:30:40.598438+0000","last_fullsized":"2023-08-28T02:30:40.598438+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:29:26.767559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":46,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.458347+0000","last_change":"2023-08-28T02:28:25.398594+0000","last_active":"2023-08-28T02:30:32.458347+0000","last_peered":"2023-08-28T02:30:32.458347+0000","last_clean":"2023-08-28T02:30:32.458347+0000","last_became_active":"2023-08-28T02:28:25.397058+0000","last_became_peered":"2023-08-28T02:28:25.397058+0000","last_unstale":"2023-08-28T02:30:32.458347+0000","last_undegraded":"2023-08-28T02:30:32.458347+0000","last_fullsized":"2023-08-28T02:30:32.458347+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"78'148","reported_seq":787,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.452121+0000","last_change":"2023-08-28T02:29:39.524927+0000","last_active":"2023-08-28T02:30:32.452121+0000","last_peered":"2023-08-28T02:30:32.452121+0000","last_clean":"2023-08-28T02:30:32.452121+0000","last_became_active":"2023-08-28T02:29:39.524297+0000","last_became_peered":"2023-08-28T02:29:39.524297+0000","last_unstale":"2023-08-28T02:30:32.452121+0000","last_undegraded":"2023-08-28T02:30:32.452121+0000","last_fullsized":"2023-08-28T02:30:32.452121+0000","mapping_epoch":243,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":244,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1092,"num_read_kb":1955,"num_write":127,"num_write_kb":5177,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":226,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:40.609344+0000","last_change":"2023-08-28T02:23:59.226986+0000","last_active":"2023-08-28T02:30:40.609344+0000","last_peered":"2023-08-28T02:30:40.609344+0000","last_clean":"2023-08-28T02:30:40.609344+0000","last_became_active":"2023-08-28T02:23:59.226529+0000","last_became_peered":"2023-08-28T02:23:59.226529+0000","last_unstale":"2023-08-28T02:30:40.609344+0000","last_undegraded":"2023-08-28T02:30:40.609344+0000","last_fullsized":"2023-08-28T02:30:40.609344+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:22:09.193619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":271,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:40.609385+0000","last_change":"2023-08-28T02:27:31.361298+0000","last_active":"2023-08-28T02:30:40.609385+0000","last_peered":"2023-08-28T02:30:40.609385+0000","last_clean":"2023-08-28T02:30:40.609385+0000","last_became_active":"2023-08-28T02:27:31.360939+0000","last_became_peered":"2023-08-28T02:27:31.360939+0000","last_unstale":"2023-08-28T02:30:40.609385+0000","last_undegraded":"2023-08-28T02:30:40.609385+0000","last_fullsized":"2023-08-28T02:30:40.609385+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:48:42.024920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"78'103","reported_seq":632,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.455242+0000","last_change":"2023-08-28T02:24:07.212128+0000","last_active":"2023-08-28T02:30:32.455242+0000","last_peered":"2023-08-28T02:30:32.455242+0000","last_clean":"2023-08-28T02:30:32.455242+0000","last_became_active":"2023-08-28T02:24:07.211562+0000","last_became_peered":"2023-08-28T02:24:07.211562+0000","last_unstale":"2023-08-28T02:30:32.455242+0000","last_undegraded":"2023-08-28T02:30:32.455242+0000","last_fullsized":"2023-08-28T02:30:32.455242+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":803,"num_read_kb":2080,"num_write":73,"num_write_kb":1244,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":66,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:40.581368+0000","last_change":"2023-08-28T02:27:31.377042+0000","last_active":"2023-08-28T02:30:40.581368+0000","last_peered":"2023-08-28T02:30:40.581368+0000","last_clean":"2023-08-28T02:30:40.581368+0000","last_became_active":"2023-08-28T02:27:31.373709+0000","last_became_peered":"2023-08-28T02:27:31.373709+0000","last_unstale":"2023-08-28T02:30:40.581368+0000","last_undegraded":"2023-08-28T02:30:40.581368+0000","last_fullsized":"2023-08-28T02:30:40.581368+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:12:38.286332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":212,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:40.609526+0000","last_change":"2023-08-28T02:23:38.329667+0000","last_active":"2023-08-28T02:30:40.609526+0000","last_peered":"2023-08-28T02:30:40.609526+0000","last_clean":"2023-08-28T02:30:40.609526+0000","last_became_active":"2023-08-28T02:23:38.329175+0000","last_became_peered":"2023-08-28T02:23:38.329175+0000","last_unstale":"2023-08-28T02:30:40.609526+0000","last_undegraded":"2023-08-28T02:30:40.609526+0000","last_fullsized":"2023-08-28T02:30:40.609526+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:21:34.815067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":122,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:40.598667+0000","last_change":"2023-08-28T02:23:59.232099+0000","last_active":"2023-08-28T02:30:40.598667+0000","last_peered":"2023-08-28T02:30:40.598667+0000","last_clean":"2023-08-28T02:30:40.598667+0000","last_became_active":"2023-08-28T02:23:59.219318+0000","last_became_peered":"2023-08-28T02:23:59.219318+0000","last_unstale":"2023-08-28T02:30:40.598667+0000","last_undegraded":"2023-08-28T02:30:40.598667+0000","last_fullsized":"2023-08-28T02:30:40.598667+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:59:30.093289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'112","reported_seq":412,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:40.580831+0000","last_change":"2023-08-28T02:28:59.477917+0000","last_active":"2023-08-28T02:30:40.580831+0000","last_peered":"2023-08-28T02:30:40.580831+0000","last_clean":"2023-08-28T02:30:40.580831+0000","last_became_active":"2023-08-28T02:28:59.477383+0000","last_became_peered":"2023-08-28T02:28:59.477383+0000","last_unstale":"2023-08-28T02:30:40.580831+0000","last_undegraded":"2023-08-28T02:30:40.580831+0000","last_fullsized":"2023-08-28T02:30:40.580831+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:28:45.045742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":444,"num_read_kb":473,"num_write":94,"num_write_kb":127,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":118,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:40.600073+0000","last_change":"2023-08-28T02:23:59.221034+0000","last_active":"2023-08-28T02:30:40.600073+0000","last_peered":"2023-08-28T02:30:40.600073+0000","last_clean":"2023-08-28T02:30:40.600073+0000","last_became_active":"2023-08-28T02:23:59.220730+0000","last_became_peered":"2023-08-28T02:23:59.220730+0000","last_unstale":"2023-08-28T02:30:40.600073+0000","last_undegraded":"2023-08-28T02:30:40.600073+0000","last_fullsized":"2023-08-28T02:30:40.600073+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:25:52.364370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"78'122","reported_seq":564,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.457481+0000","last_change":"2023-08-28T02:28:28.383306+0000","last_active":"2023-08-28T02:30:32.457481+0000","last_peered":"2023-08-28T02:30:32.457481+0000","last_clean":"2023-08-28T02:30:32.457481+0000","last_became_active":"2023-08-28T02:28:28.382672+0000","last_became_peered":"2023-08-28T02:28:28.382672+0000","last_unstale":"2023-08-28T02:30:32.457481+0000","last_undegraded":"2023-08-28T02:30:32.457481+0000","last_fullsized":"2023-08-28T02:30:32.457481+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":857,"num_read_kb":2224,"num_write":99,"num_write_kb":2312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":238,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:40.609478+0000","last_change":"2023-08-28T02:27:31.424373+0000","last_active":"2023-08-28T02:30:40.609478+0000","last_peered":"2023-08-28T02:30:40.609478+0000","last_clean":"2023-08-28T02:30:40.609478+0000","last_became_active":"2023-08-28T02:27:31.324505+0000","last_became_peered":"2023-08-28T02:27:31.324505+0000","last_unstale":"2023-08-28T02:30:40.609478+0000","last_undegraded":"2023-08-28T02:30:40.609478+0000","last_fullsized":"2023-08-28T02:30:40.609478+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:27:57.837524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":226,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.430446+0000","last_change":"2023-08-28T02:28:25.351840+0000","last_active":"2023-08-28T02:30:32.430446+0000","last_peered":"2023-08-28T02:30:32.430446+0000","last_clean":"2023-08-28T02:30:32.430446+0000","last_became_active":"2023-08-28T02:28:25.351399+0000","last_became_peered":"2023-08-28T02:28:25.351399+0000","last_unstale":"2023-08-28T02:30:32.430446+0000","last_undegraded":"2023-08-28T02:30:32.430446+0000","last_fullsized":"2023-08-28T02:30:32.430446+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":214,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:40.609461+0000","last_change":"2023-08-28T02:23:38.329474+0000","last_active":"2023-08-28T02:30:40.609461+0000","last_peered":"2023-08-28T02:30:40.609461+0000","last_clean":"2023-08-28T02:30:40.609461+0000","last_became_active":"2023-08-28T02:23:38.328505+0000","last_became_peered":"2023-08-28T02:23:38.328505+0000","last_unstale":"2023-08-28T02:30:40.609461+0000","last_undegraded":"2023-08-28T02:30:40.609461+0000","last_fullsized":"2023-08-28T02:30:40.609461+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:12:14.835809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":299,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:40.598987+0000","last_change":"2023-08-28T02:27:31.384467+0000","last_active":"2023-08-28T02:30:40.598987+0000","last_peered":"2023-08-28T02:30:40.598987+0000","last_clean":"2023-08-28T02:30:40.598987+0000","last_became_active":"2023-08-28T02:27:31.384183+0000","last_became_peered":"2023-08-28T02:27:31.384183+0000","last_unstale":"2023-08-28T02:30:40.598987+0000","last_undegraded":"2023-08-28T02:30:40.598987+0000","last_fullsized":"2023-08-28T02:30:40.598987+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:59:47.551892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":63,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.430622+0000","last_change":"2023-08-28T02:28:11.333622+0000","last_active":"2023-08-28T02:30:32.430622+0000","last_peered":"2023-08-28T02:30:32.430622+0000","last_clean":"2023-08-28T02:30:32.430622+0000","last_became_active":"2023-08-28T02:28:11.333321+0000","last_became_peered":"2023-08-28T02:28:11.333321+0000","last_unstale":"2023-08-28T02:30:32.430622+0000","last_undegraded":"2023-08-28T02:30:32.430622+0000","last_fullsized":"2023-08-28T02:30:32.430622+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":274,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.430748+0000","last_change":"2023-08-28T02:27:31.343538+0000","last_active":"2023-08-28T02:30:32.430748+0000","last_peered":"2023-08-28T02:30:32.430748+0000","last_clean":"2023-08-28T02:30:32.430748+0000","last_became_active":"2023-08-28T02:27:31.343324+0000","last_became_peered":"2023-08-28T02:27:31.343324+0000","last_unstale":"2023-08-28T02:30:32.430748+0000","last_undegraded":"2023-08-28T02:30:32.430748+0000","last_fullsized":"2023-08-28T02:30:32.430748+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":76,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.430922+0000","last_change":"2023-08-28T02:28:25.705331+0000","last_active":"2023-08-28T02:30:32.430922+0000","last_peered":"2023-08-28T02:30:32.430922+0000","last_clean":"2023-08-28T02:30:32.430922+0000","last_became_active":"2023-08-28T02:28:25.397480+0000","last_became_peered":"2023-08-28T02:28:25.397480+0000","last_unstale":"2023-08-28T02:30:32.430922+0000","last_undegraded":"2023-08-28T02:30:32.430922+0000","last_fullsized":"2023-08-28T02:30:32.430922+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":256,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.432366+0000","last_change":"2023-08-28T02:23:59.207929+0000","last_active":"2023-08-28T02:30:32.432366+0000","last_peered":"2023-08-28T02:30:32.432366+0000","last_clean":"2023-08-28T02:30:32.432366+0000","last_became_active":"2023-08-28T02:23:59.207589+0000","last_became_peered":"2023-08-28T02:23:59.207589+0000","last_unstale":"2023-08-28T02:30:32.432366+0000","last_undegraded":"2023-08-28T02:30:32.432366+0000","last_fullsized":"2023-08-28T02:30:32.432366+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":145,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:40.598418+0000","last_change":"2023-08-28T02:28:25.347988+0000","last_active":"2023-08-28T02:30:40.598418+0000","last_peered":"2023-08-28T02:30:40.598418+0000","last_clean":"2023-08-28T02:30:40.598418+0000","last_became_active":"2023-08-28T02:28:25.347632+0000","last_became_peered":"2023-08-28T02:28:25.347632+0000","last_unstale":"2023-08-28T02:30:40.598418+0000","last_undegraded":"2023-08-28T02:30:40.598418+0000","last_fullsized":"2023-08-28T02:30:40.598418+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:56:20.202662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":117,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:40.598287+0000","last_change":"2023-08-28T02:23:59.217794+0000","last_active":"2023-08-28T02:30:40.598287+0000","last_peered":"2023-08-28T02:30:40.598287+0000","last_clean":"2023-08-28T02:30:40.598287+0000","last_became_active":"2023-08-28T02:23:59.217231+0000","last_became_peered":"2023-08-28T02:23:59.217231+0000","last_unstale":"2023-08-28T02:30:40.598287+0000","last_undegraded":"2023-08-28T02:30:40.598287+0000","last_fullsized":"2023-08-28T02:30:40.598287+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:36:26.297410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":212,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-08-28T02:30:40.609848+0000","last_change":"2023-08-28T02:20:19.850252+0000","last_active":"2023-08-28T02:30:40.609848+0000","last_peered":"2023-08-28T02:30:40.609848+0000","last_clean":"2023-08-28T02:30:40.609848+0000","last_became_active":"2023-08-28T02:20:19.849913+0000","last_became_peered":"2023-08-28T02:20:19.849913+0000","last_unstale":"2023-08-28T02:30:40.609848+0000","last_undegraded":"2023-08-28T02:30:40.609848+0000","last_fullsized":"2023-08-28T02:30:40.609848+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:48:47.713465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":288,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-08-28T02:30:32.430405+0000","last_change":"2023-08-28T02:28:25.347845+0000","last_active":"2023-08-28T02:30:32.430405+0000","last_peered":"2023-08-28T02:30:32.430405+0000","last_clean":"2023-08-28T02:30:32.430405+0000","last_became_active":"2023-08-28T02:28:25.347502+0000","last_became_peered":"2023-08-28T02:28:25.347502+0000","last_unstale":"2023-08-28T02:30:32.430405+0000","last_undegraded":"2023-08-28T02:30:32.430405+0000","last_fullsized":"2023-08-28T02:30:32.430405+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":19382,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26810,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":137,"num_bytes_recovered":12582931,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430462,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":628340,"kb_used_data":322924,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93116812,"statfs":{"total":95995035648,"available":95351615488,"internally_reserved":0,"allocated":330674176,"data_stored":329959130,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:30:03 2023","interfaces":[{"interface":"back","average":{"1min":2.194,"5min":2.3999999999999999,"15min":2.0859999999999999},"min":{"1min":0.25600000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":24.379999999999999,"5min":59.753999999999998,"15min":116.755},"last":0.433},{"interface":"front","average":{"1min":2.7120000000000002,"5min":2.7559999999999998,"15min":2.3359999999999999},"min":{"1min":0.26500000000000001,"5min":0.245,"15min":0.23999999999999999},"max":{"1min":24.213999999999999,"5min":59.765999999999998,"15min":116.971},"last":0.32900000000000001}]},{"osd":1,"last update":"Mon Aug 28 02:30:11 2023","interfaces":[{"interface":"back","average":{"1min":2.496,"5min":2.2509999999999999,"15min":1.2370000000000001},"min":{"1min":0.27900000000000003,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":24.681000000000001,"5min":44.287999999999997,"15min":44.287999999999997},"last":0.38300000000000001},{"interface":"front","average":{"1min":2.7570000000000001,"5min":3.109,"15min":1.667},"min":{"1min":0.28399999999999997,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":24.189,"5min":44.393999999999998,"15min":44.393999999999998},"last":0.42399999999999999}]},{"osd":2,"last update":"Mon Aug 28 02:30:35 2023","interfaces":[{"interface":"back","average":{"1min":2.1779999999999999,"5min":3.004,"15min":2.4830000000000001},"min":{"1min":0.32400000000000001,"5min":0.28000000000000003,"15min":0.27600000000000002},"max":{"1min":7.5300000000000002,"5min":101.616,"15min":119.571},"last":0.41699999999999998},{"interface":"front","average":{"1min":1.9119999999999999,"5min":3.7210000000000001,"15min":2.875},"min":{"1min":0.32900000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.5380000000000003,"5min":101.605,"15min":119.554},"last":0.38900000000000001}]},{"osd":3,"last update":"Mon Aug 28 02:30:11 2023","interfaces":[{"interface":"back","average":{"1min":2.0110000000000001,"5min":4.0620000000000003,"15min":2.2480000000000002},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30199999999999999},"max":{"1min":10.686999999999999,"5min":99.459000000000003,"15min":107.916},"last":0.39700000000000002},{"interface":"front","average":{"1min":1.794,"5min":3.6789999999999998,"15min":2.3239999999999998},"min":{"1min":0.34300000000000003,"5min":0.311,"15min":0.27700000000000002},"max":{"1min":10.140000000000001,"5min":98.084000000000003,"15min":112.00700000000001},"last":0.35099999999999998}]},{"osd":4,"last update":"Mon Aug 28 02:30:35 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":1.26,"15min":1.254},"min":{"1min":0.28899999999999998,"5min":0.26500000000000001,"15min":0.22900000000000001},"max":{"1min":2.3929999999999998,"5min":11.696,"15min":63.048000000000002},"last":0.28599999999999998},{"interface":"front","average":{"1min":1.3660000000000001,"5min":1.347,"15min":1.24},"min":{"1min":0.316,"5min":0.26700000000000002,"15min":0.247},"max":{"1min":8.2140000000000004,"5min":10.840999999999999,"15min":62.912999999999997},"last":0.29899999999999999}]},{"osd":5,"last update":"Mon Aug 28 02:30:35 2023","interfaces":[{"interface":"back","average":{"1min":1.2,"5min":1.1859999999999999,"15min":1.075},"min":{"1min":0.32800000000000001,"5min":0.27800000000000002,"15min":0.26800000000000002},"max":{"1min":7.5110000000000001,"5min":13.788,"15min":63.021000000000001},"last":0.35799999999999998},{"interface":"front","average":{"1min":1.2949999999999999,"5min":1.4079999999999999,"15min":1.377},"min":{"1min":0.38200000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":6.7110000000000003,"5min":9.6929999999999996,"15min":63.082999999999998},"last":0.40400000000000003}]},{"osd":6,"last update":"Mon Aug 28 02:30:35 2023","interfaces":[{"interface":"back","average":{"1min":1.302,"5min":1.2529999999999999,"15min":1.3280000000000001},"min":{"1min":0.36899999999999999,"5min":0.318,"15min":0.29699999999999999},"max":{"1min":7.5369999999999999,"5min":9.7070000000000007,"15min":101.61},"last":0.316},{"interface":"front","average":{"1min":0.78500000000000003,"5min":1.29,"15min":1.427},"min":{"1min":0.40999999999999998,"5min":0.308,"15min":0.308},"max":{"1min":2.4780000000000002,"5min":14.423999999999999,"15min":101.621},"last":0.36799999999999999}]}]},{"osd":6,"up_from":42,"seq":180388626689,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":375592,"kb_used_data":74272,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93369560,"statfs":{"total":95995035648,"available":95610429440,"internally_reserved":0,"allocated":76054528,"data_stored":75351328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:30:11 2023","interfaces":[{"interface":"back","average":{"1min":3.246,"5min":3.911,"15min":2.7629999999999999},"min":{"1min":0.22700000000000001,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":44.512999999999998,"5min":60.625999999999998,"15min":121.247},"last":0.36399999999999999},{"interface":"front","average":{"1min":3.3300000000000001,"5min":3.7080000000000002,"15min":2.722},"min":{"1min":0.26000000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":46.637999999999998,"5min":54.273000000000003,"15min":118.977},"last":0.40300000000000002}]},{"osd":1,"last update":"Mon Aug 28 02:30:11 2023","interfaces":[{"interface":"back","average":{"1min":1.78,"5min":2.524,"15min":1.669},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23699999999999999},"max":{"1min":7.2270000000000003,"5min":99.998999999999995,"15min":99.998999999999995},"last":0.39100000000000001},{"interface":"front","average":{"1min":1.9550000000000001,"5min":2.8010000000000002,"15min":1.8100000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":8.6029999999999998,"5min":95.040000000000006,"15min":95.040000000000006},"last":0.35699999999999998}]},{"osd":2,"last update":"Mon Aug 28 02:29:53 2023","interfaces":[{"interface":"back","average":{"1min":1.4650000000000001,"5min":4.258,"15min":2.2250000000000001},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":4.3259999999999996,"5min":79.787000000000006,"15min":79.787000000000006},"last":0.38200000000000001},{"interface":"front","average":{"1min":1.3959999999999999,"5min":4.4470000000000001,"15min":2.2200000000000002},"min":{"1min":0.28899999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":7.8490000000000002,"5min":79.935000000000002,"15min":79.935000000000002},"last":0.373}]},{"osd":3,"last update":"Mon Aug 28 02:30:11 2023","interfaces":[{"interface":"back","average":{"1min":1.1419999999999999,"5min":2.7850000000000001,"15min":1.6719999999999999},"min":{"1min":0.32000000000000001,"5min":0.317,"15min":0.317},"max":{"1min":3.968,"5min":39.587000000000003,"15min":39.587000000000003},"last":0.41899999999999998},{"interface":"front","average":{"1min":1.0069999999999999,"5min":2.8940000000000001,"15min":1.7589999999999999},"min":{"1min":0.35299999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":3.9780000000000002,"5min":39.593000000000004,"15min":39.593000000000004},"last":0.45400000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:30:18 2023","interfaces":[{"interface":"back","average":{"1min":1.0069999999999999,"5min":1.224,"15min":1.04},"min":{"1min":0.27100000000000002,"5min":0.251,"15min":0.251},"max":{"1min":3.6499999999999999,"5min":18.308,"15min":18.308},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.77400000000000002,"5min":1.3899999999999999,"15min":1.131},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.26700000000000002},"max":{"1min":3.5070000000000001,"5min":18.263000000000002,"15min":18.263000000000002},"last":0.316}]},{"osd":5,"last update":"Mon Aug 28 02:30:18 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":1.216,"15min":1.1080000000000001},"min":{"1min":0.29699999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":2.8359999999999999,"5min":9.9809999999999999,"15min":20.759},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.98999999999999999,"5min":1.113,"15min":0.95599999999999996},"min":{"1min":0.314,"5min":0.314,"15min":0.28799999999999998},"max":{"1min":3.1549999999999998,"5min":5.9500000000000002,"15min":7.3780000000000001},"last":0.44400000000000001}]},{"osd":7,"last update":"Mon Aug 28 02:30:44 2023","interfaces":[{"interface":"back","average":{"1min":1.744,"5min":1.151,"15min":1.0089999999999999},"min":{"1min":0.36699999999999999,"5min":0.34200000000000003,"15min":0.314},"max":{"1min":10.605,"5min":10.605,"15min":10.605},"last":0.33200000000000002},{"interface":"front","average":{"1min":1.03,"5min":1.1499999999999999,"15min":1.0720000000000001},"min":{"1min":0.38700000000000001,"5min":0.34599999999999997,"15min":0.29799999999999999},"max":{"1min":3.4529999999999998,"5min":9.7110000000000003,"15min":9.7110000000000003},"last":0.42699999999999999}]}]},{"osd":1,"up_from":98,"seq":420906795142,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":428592,"kb_used_data":165992,"kb_used_omap":7,"kb_used_meta":262584,"kb_avail":93316560,"statfs":{"total":95995035648,"available":95556157440,"internally_reserved":0,"allocated":169975808,"data_stored":168702335,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":268886347},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:30:09 2023","interfaces":[{"interface":"back","average":{"1min":1.534,"5min":4.548,"15min":2.1379999999999999},"min":{"1min":0.20799999999999999,"5min":0.191,"15min":0.17499999999999999},"max":{"1min":8.3230000000000004,"5min":128.60599999999999,"15min":128.60599999999999},"last":0.26100000000000001},{"interface":"front","average":{"1min":1.1930000000000001,"5min":4.3760000000000003,"15min":2.1480000000000001},"min":{"1min":0.221,"5min":0.188,"15min":0.188},"max":{"1min":3.8220000000000001,"5min":128.685,"15min":128.685},"last":0.46000000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:30:34 2023","interfaces":[{"interface":"back","average":{"1min":2.5640000000000001,"5min":6.2939999999999996,"15min":2.9630000000000001},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":8.1720000000000006,"5min":135.25999999999999,"15min":135.25999999999999},"last":0.36799999999999999},{"interface":"front","average":{"1min":2.1339999999999999,"5min":6.1920000000000002,"15min":2.7730000000000001},"min":{"1min":0.26700000000000002,"5min":0.23499999999999999,"15min":0.221},"max":{"1min":6.7089999999999996,"5min":135.10300000000001,"15min":135.10300000000001},"last":0.29799999999999999}]},{"osd":3,"last update":"Mon Aug 28 02:30:09 2023","interfaces":[{"interface":"back","average":{"1min":1.6899999999999999,"5min":3.7010000000000001,"15min":1.843},"min":{"1min":0.33600000000000002,"5min":0.22,"15min":0.22},"max":{"1min":3.867,"5min":123.741,"15min":123.741},"last":0.34999999999999998},{"interface":"front","average":{"1min":3.9060000000000001,"5min":4.4829999999999997,"15min":2.1459999999999999},"min":{"1min":0.501,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":13.340999999999999,"5min":123.642,"15min":123.642},"last":0.33600000000000002}]},{"osd":4,"last update":"Mon Aug 28 02:30:09 2023","interfaces":[{"interface":"back","average":{"1min":1.6930000000000001,"5min":1.7789999999999999,"15min":1.264},"min":{"1min":0.29999999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.2939999999999996,"5min":17.600000000000001,"15min":17.600000000000001},"last":0.41399999999999998},{"interface":"front","average":{"1min":1.2170000000000001,"5min":1.6220000000000001,"15min":1.137},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.25600000000000001},"max":{"1min":3.5920000000000001,"5min":19.010000000000002,"15min":19.010000000000002},"last":0.42199999999999999}]},{"osd":5,"last update":"Mon Aug 28 02:30:09 2023","interfaces":[{"interface":"back","average":{"1min":2.242,"5min":1.6910000000000001,"15min":1.371},"min":{"1min":0.32100000000000001,"5min":0.312,"15min":0.29399999999999998},"max":{"1min":11.167999999999999,"5min":11.167999999999999,"15min":15.544},"last":0.40200000000000002},{"interface":"front","average":{"1min":2.2440000000000002,"5min":1.847,"15min":1.2549999999999999},"min":{"1min":0.34599999999999997,"5min":0.33600000000000002,"15min":0.33200000000000002},"max":{"1min":10.084,"5min":17.646999999999998,"15min":17.646999999999998},"last":0.45000000000000001}]},{"osd":6,"last update":"Mon Aug 28 02:30:09 2023","interfaces":[{"interface":"back","average":{"1min":1.7170000000000001,"5min":1.7090000000000001,"15min":1.4630000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":3.923,"5min":9.2989999999999995,"15min":15.512},"last":0.59399999999999997},{"interface":"front","average":{"1min":1.8160000000000001,"5min":1.823,"15min":1.3899999999999999},"min":{"1min":0.39600000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":6.3140000000000001,"5min":17.673999999999999,"15min":17.673999999999999},"last":0.435}]},{"osd":7,"last update":"Mon Aug 28 02:30:09 2023","interfaces":[{"interface":"back","average":{"1min":1.4059999999999999,"5min":1.7909999999999999,"15min":1.252},"min":{"1min":0.38500000000000001,"5min":0.32500000000000001,"15min":0.30199999999999999},"max":{"1min":4.0460000000000003,"5min":19.061,"15min":19.061},"last":0.39200000000000002},{"interface":"front","average":{"1min":1.7889999999999999,"5min":1.905,"15min":1.4450000000000001},"min":{"1min":0.437,"5min":0.40500000000000003,"15min":0.33100000000000002},"max":{"1min":5.8890000000000002,"5min":9.3239999999999998,"15min":15.476000000000001},"last":0.57499999999999996}]}]},{"osd":0,"up_from":90,"seq":386547056788,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":389120,"kb_used_data":64120,"kb_used_omap":8,"kb_used_meta":324983,"kb_avail":93356032,"statfs":{"total":95995035648,"available":95596576768,"internally_reserved":0,"allocated":65658880,"data_stored":64486750,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":332782953},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Aug 28 02:30:17 2023","interfaces":[{"interface":"back","average":{"1min":5.1619999999999999,"5min":3.004,"15min":1.8919999999999999},"min":{"1min":0.20000000000000001,"5min":0.184,"15min":0.184},"max":{"1min":56.576999999999998,"5min":56.576999999999998,"15min":56.576999999999998},"last":0.375},{"interface":"front","average":{"1min":6.1689999999999996,"5min":3.524,"15min":1.931},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.17000000000000001},"max":{"1min":56.552999999999997,"5min":56.552999999999997,"15min":56.552999999999997},"last":0.42599999999999999}]},{"osd":2,"last update":"Mon Aug 28 02:30:34 2023","interfaces":[{"interface":"back","average":{"1min":3.234,"5min":4.3090000000000002,"15min":2.202},"min":{"1min":0.22,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":10.565,"5min":100.718,"15min":100.718},"last":0.41199999999999998},{"interface":"front","average":{"1min":1.333,"5min":4.3730000000000002,"15min":2.4159999999999999},"min":{"1min":0.24299999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":4.9989999999999997,"5min":100.962,"15min":100.962},"last":0.47899999999999998}]},{"osd":3,"last update":"Mon Aug 28 02:30:17 2023","interfaces":[{"interface":"back","average":{"1min":2.6459999999999999,"5min":6.9390000000000001,"15min":2.9430000000000001},"min":{"1min":0.24299999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":19.327000000000002,"5min":237.43899999999999,"15min":237.43899999999999},"last":0.35599999999999998},{"interface":"front","average":{"1min":1.7390000000000001,"5min":6.6399999999999997,"15min":2.859},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":6.0010000000000003,"5min":237.48099999999999,"15min":237.48099999999999},"last":0.496}]},{"osd":4,"last update":"Mon Aug 28 02:30:12 2023","interfaces":[{"interface":"back","average":{"1min":1.7709999999999999,"5min":1.6859999999999999,"15min":1.2849999999999999},"min":{"1min":0.29699999999999999,"5min":0.29599999999999999,"15min":0.26800000000000002},"max":{"1min":9.3680000000000003,"5min":9.3680000000000003,"15min":9.6649999999999991},"last":0.59199999999999997},{"interface":"front","average":{"1min":1.536,"5min":2.0870000000000002,"15min":1.4650000000000001},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":8.7880000000000003,"5min":27.271999999999998,"15min":27.271999999999998},"last":0.53200000000000003}]},{"osd":5,"last update":"Mon Aug 28 02:30:12 2023","interfaces":[{"interface":"back","average":{"1min":1.153,"5min":2.0099999999999998,"15min":1.5609999999999999},"min":{"1min":0.34000000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":3.0230000000000001,"5min":27.367000000000001,"15min":44.000999999999998},"last":0.57399999999999995},{"interface":"front","average":{"1min":1.671,"5min":1.9079999999999999,"15min":1.488},"min":{"1min":0.34300000000000003,"5min":0.33000000000000002,"15min":0.30599999999999999},"max":{"1min":9.4489999999999998,"5min":12.334,"15min":43.953000000000003},"last":0.63700000000000001}]},{"osd":6,"last update":"Mon Aug 28 02:30:12 2023","interfaces":[{"interface":"back","average":{"1min":1.276,"5min":2.0819999999999999,"15min":1.585},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":3.0510000000000002,"5min":27.331,"15min":32.417999999999999},"last":0.61599999999999999},{"interface":"front","average":{"1min":1.6930000000000001,"5min":2.173,"15min":1.5620000000000001},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.30399999999999999},"max":{"1min":9.2789999999999999,"5min":27.247,"15min":32.130000000000003},"last":0.65900000000000003}]},{"osd":7,"last update":"Mon Aug 28 02:30:12 2023","interfaces":[{"interface":"back","average":{"1min":1.802,"5min":2.4769999999999999,"15min":1.583},"min":{"1min":0.36799999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":9.4209999999999994,"5min":27.385000000000002,"15min":27.385000000000002},"last":0.67700000000000005},{"interface":"front","average":{"1min":1.5069999999999999,"5min":2.339,"15min":1.599},"min":{"1min":0.38900000000000001,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":5.7300000000000004,"5min":27.213999999999999,"15min":27.213999999999999},"last":0.66700000000000004}]}]},{"osd":2,"up_from":104,"seq":446676598912,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":575696,"kb_used_data":263496,"kb_used_omap":10,"kb_used_meta":312181,"kb_avail":93169456,"statfs":{"total":95995035648,"available":95405522944,"internally_reserved":0,"allocated":269819904,"data_stored":268688728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10792,"internal_metadata":319673816},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:30:37 2023","interfaces":[{"interface":"back","average":{"1min":2.3050000000000002,"5min":4.21,"15min":2.2599999999999998},"min":{"1min":0.20300000000000001,"5min":0.17699999999999999,"15min":0.17599999999999999},"max":{"1min":16.48,"5min":113.89100000000001,"15min":113.89100000000001},"last":0.29899999999999999},{"interface":"front","average":{"1min":2.5499999999999998,"5min":7.5940000000000003,"15min":3.472},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.19900000000000001},"max":{"1min":11.166,"5min":147.286,"15min":147.286},"last":0.316}]},{"osd":1,"last update":"Mon Aug 28 02:30:37 2023","interfaces":[{"interface":"back","average":{"1min":2.6560000000000001,"5min":2.431,"15min":1.738},"min":{"1min":0.23799999999999999,"5min":0.21099999999999999,"15min":0.20999999999999999},"max":{"1min":11.061,"5min":11.627000000000001,"15min":30.888000000000002},"last":0.28100000000000003},{"interface":"front","average":{"1min":3.351,"5min":2.8029999999999999,"15min":1.6890000000000001},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.20999999999999999},"max":{"1min":11.962999999999999,"5min":20.259,"15min":31.648},"last":0.33000000000000002}]},{"osd":3,"last update":"Mon Aug 28 02:30:09 2023","interfaces":[{"interface":"back","average":{"1min":1.6799999999999999,"5min":5.1100000000000003,"15min":2.286},"min":{"1min":0.314,"5min":0.25,"15min":0.22700000000000001},"max":{"1min":9.7629999999999999,"5min":128.63900000000001,"15min":128.63900000000001},"last":0.34599999999999997},{"interface":"front","average":{"1min":2.1070000000000002,"5min":4.9269999999999996,"15min":2.3010000000000002},"min":{"1min":0.375,"5min":0.223,"15min":0.223},"max":{"1min":8.7029999999999994,"5min":128.75700000000001,"15min":128.75700000000001},"last":0.39700000000000002}]},{"osd":4,"last update":"Mon Aug 28 02:30:37 2023","interfaces":[{"interface":"back","average":{"1min":12.464,"5min":5.617,"15min":2.661},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":207.34100000000001,"5min":207.34100000000001,"15min":207.34100000000001},"last":0.45300000000000001},{"interface":"front","average":{"1min":12.086,"5min":5.2610000000000001,"15min":2.4129999999999998},"min":{"1min":0.30299999999999999,"5min":0.28999999999999998,"15min":0.27400000000000002},"max":{"1min":207.352,"5min":207.352,"15min":207.352},"last":0.38800000000000001}]},{"osd":5,"last update":"Mon Aug 28 02:30:37 2023","interfaces":[{"interface":"back","average":{"1min":1.998,"5min":5.4800000000000004,"15min":2.5379999999999998},"min":{"1min":0.33800000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":5.8120000000000003,"5min":128.696,"15min":128.696},"last":0.40799999999999997},{"interface":"front","average":{"1min":2.0249999999999999,"5min":5.1769999999999996,"15min":2.5939999999999999},"min":{"1min":0.32300000000000001,"5min":0.32100000000000001,"15min":0.30299999999999999},"max":{"1min":8.0709999999999997,"5min":128.84100000000001,"15min":128.84100000000001},"last":0.432}]},{"osd":6,"last update":"Mon Aug 28 02:30:37 2023","interfaces":[{"interface":"back","average":{"1min":1.9370000000000001,"5min":1.79,"15min":1.3500000000000001},"min":{"1min":0.32800000000000001,"5min":0.307,"15min":0.29299999999999998},"max":{"1min":11.997,"5min":11.997,"15min":11.997},"last":0.443},{"interface":"front","average":{"1min":1.8939999999999999,"5min":2.1299999999999999,"15min":1.494},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":5.9530000000000003,"5min":11.055999999999999,"15min":11.055999999999999},"last":0.38}]},{"osd":7,"last update":"Mon Aug 28 02:30:37 2023","interfaces":[{"interface":"back","average":{"1min":1.827,"5min":5.3289999999999997,"15min":2.5129999999999999},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.32100000000000001},"max":{"1min":6.29,"5min":128.911,"15min":128.911},"last":0.47299999999999998},{"interface":"front","average":{"1min":1.7410000000000001,"5min":5.859,"15min":2.7120000000000002},"min":{"1min":0.35599999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":4.8819999999999997,"5min":128.72399999999999,"15min":128.72399999999999},"last":0.42699999999999999}]}]},{"osd":3,"up_from":112,"seq":481036337274,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":255628,"kb_used_data":2948,"kb_used_omap":3,"kb_used_meta":252668,"kb_avail":93489524,"statfs":{"total":95995035648,"available":95733272576,"internally_reserved":0,"allocated":3018752,"data_stored":1888871,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":258732962},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:30:05 2023","interfaces":[{"interface":"back","average":{"1min":7.0190000000000001,"5min":3.1560000000000001,"15min":3.0089999999999999},"min":{"1min":0.20000000000000001,"5min":0.193,"15min":0.193},"max":{"1min":79.793000000000006,"5min":79.793000000000006,"15min":164.25299999999999},"last":0.33800000000000002},{"interface":"front","average":{"1min":6.7839999999999998,"5min":2.8380000000000001,"15min":2.9620000000000002},"min":{"1min":0.27200000000000002,"5min":0.192,"15min":0.192},"max":{"1min":78.555000000000007,"5min":78.555000000000007,"15min":164.55000000000001},"last":0.26600000000000001}]},{"osd":1,"last update":"Mon Aug 28 02:30:05 2023","interfaces":[{"interface":"back","average":{"1min":1.669,"5min":1.907,"15min":1.8680000000000001},"min":{"1min":0.245,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":6.1500000000000004,"5min":13.936,"15min":117.34699999999999},"last":0.36799999999999999},{"interface":"front","average":{"1min":2.5310000000000001,"5min":2.1499999999999999,"15min":1.968},"min":{"1min":0.27300000000000002,"5min":0.245,"15min":0.224},"max":{"1min":10.125,"5min":14.300000000000001,"15min":118.012},"last":0.38100000000000001}]},{"osd":2,"last update":"Mon Aug 28 02:30:05 2023","interfaces":[{"interface":"back","average":{"1min":2.4199999999999999,"5min":3.157,"15min":2.8140000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.19900000000000001},"max":{"1min":10.163,"5min":65.308999999999997,"15min":120.43300000000001},"last":0.28599999999999998},{"interface":"front","average":{"1min":2.3999999999999999,"5min":2.891,"15min":2.7599999999999998},"min":{"1min":0.33500000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":7.3620000000000001,"5min":65.120999999999995,"15min":120.44499999999999},"last":0.32000000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:30:05 2023","interfaces":[{"interface":"back","average":{"1min":1.403,"5min":2.4980000000000002,"15min":1.4610000000000001},"min":{"1min":0.32800000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":4.7240000000000002,"5min":58.067999999999998,"15min":58.067999999999998},"last":0.35699999999999998},{"interface":"front","average":{"1min":1.6279999999999999,"5min":2.7589999999999999,"15min":1.5269999999999999},"min":{"1min":0.40400000000000003,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":4.6280000000000001,"5min":58.088000000000001,"15min":58.088000000000001},"last":0.52700000000000002}]},{"osd":5,"last update":"Mon Aug 28 02:30:05 2023","interfaces":[{"interface":"back","average":{"1min":1.359,"5min":3.2109999999999999,"15min":1.6579999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.29599999999999999},"max":{"1min":2.923,"5min":102.904,"15min":102.904},"last":0.56200000000000006},{"interface":"front","average":{"1min":1.556,"5min":3.2509999999999999,"15min":1.6619999999999999},"min":{"1min":0.36699999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":7.0039999999999996,"5min":103.283,"15min":103.283},"last":0.42799999999999999}]},{"osd":6,"last update":"Mon Aug 28 02:30:05 2023","interfaces":[{"interface":"back","average":{"1min":1.5880000000000001,"5min":1.877,"15min":1.4330000000000001},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.314},"max":{"1min":7.0300000000000002,"5min":9.4529999999999994,"15min":72.116},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.7929999999999999,"5min":1.538,"15min":1.3560000000000001},"min":{"1min":0.35599999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":7.2990000000000004,"5min":7.2990000000000004,"15min":72.030000000000001},"last":0.436}]},{"osd":7,"last update":"Mon Aug 28 02:30:05 2023","interfaces":[{"interface":"back","average":{"1min":1.6879999999999999,"5min":2.9660000000000002,"15min":1.631},"min":{"1min":0.38300000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":4.7640000000000002,"5min":54.427999999999997,"15min":54.427999999999997},"last":0.39300000000000002},{"interface":"front","average":{"1min":1.855,"5min":3.3399999999999999,"15min":1.76},"min":{"1min":0.433,"5min":0.36399999999999999,"15min":0.34799999999999998},"max":{"1min":3.8100000000000001,"5min":54.445,"15min":54.445},"last":0.59499999999999997}]}]},{"osd":4,"up_from":30,"seq":128849019147,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":316072,"kb_used_data":2464,"kb_used_omap":0,"kb_used_meta":313600,"kb_avail":93429080,"statfs":{"total":95995035648,"available":95671377920,"internally_reserved":0,"allocated":2523136,"data_stored":1819742,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321126400},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:30:10 2023","interfaces":[{"interface":"back","average":{"1min":1.528,"5min":2.8479999999999999,"15min":2.706},"min":{"1min":0.222,"5min":0.219,"15min":0.219},"max":{"1min":13.492000000000001,"5min":43.548000000000002,"15min":138.67400000000001},"last":0.42399999999999999},{"interface":"front","average":{"1min":1.4850000000000001,"5min":2.7149999999999999,"15min":2.637},"min":{"1min":0.251,"5min":0.248,"15min":0.248},"max":{"1min":8.0060000000000002,"5min":43.515000000000001,"15min":134.14599999999999},"last":0.56299999999999994}]},{"osd":1,"last update":"Mon Aug 28 02:30:10 2023","interfaces":[{"interface":"back","average":{"1min":1.5629999999999999,"5min":2.1640000000000001,"15min":1.3919999999999999},"min":{"1min":0.23599999999999999,"5min":0.221,"15min":0.221},"max":{"1min":9.75,"5min":34.945999999999998,"15min":53.622999999999998},"last":0.44800000000000001},{"interface":"front","average":{"1min":1.4039999999999999,"5min":2.073,"15min":1.4079999999999999},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":5.2919999999999998,"5min":34.869999999999997,"15min":53.774000000000001},"last":0.54200000000000004}]},{"osd":2,"last update":"Mon Aug 28 02:30:37 2023","interfaces":[{"interface":"back","average":{"1min":4.867,"5min":4.3040000000000003,"15min":2.8250000000000002},"min":{"1min":0.33100000000000002,"5min":0.27400000000000002,"15min":0.25600000000000001},"max":{"1min":56.948999999999998,"5min":64.063000000000002,"15min":74.099000000000004},"last":0.46600000000000003},{"interface":"front","average":{"1min":5.117,"5min":4.1769999999999996,"15min":2.7330000000000001},"min":{"1min":0.377,"5min":0.28999999999999998,"15min":0.25},"max":{"1min":56.979999999999997,"5min":64.073999999999998,"15min":74.081999999999994},"last":0.59199999999999997}]},{"osd":3,"last update":"Mon Aug 28 02:30:10 2023","interfaces":[{"interface":"back","average":{"1min":11.220000000000001,"5min":6.0819999999999999,"15min":2.5099999999999998},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":159.10400000000001,"5min":159.10400000000001,"15min":159.10400000000001},"last":0.65800000000000003},{"interface":"front","average":{"1min":11.119999999999999,"5min":5.9939999999999998,"15min":2.5979999999999999},"min":{"1min":0.33800000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":159.232,"5min":159.232,"15min":159.232},"last":0.52000000000000002}]},{"osd":5,"last update":"Mon Aug 28 02:29:50 2023","interfaces":[{"interface":"back","average":{"1min":0.78100000000000003,"5min":2.8140000000000001,"15min":1.397},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":4.3090000000000002,"5min":124.718,"15min":124.718},"last":0.375},{"interface":"front","average":{"1min":0.73599999999999999,"5min":3.1949999999999998,"15min":1.554},"min":{"1min":0.27800000000000002,"5min":0.247,"15min":0.247},"max":{"1min":4.665,"5min":124.694,"15min":124.694},"last":0.64000000000000001}]},{"osd":6,"last update":"Mon Aug 28 02:30:10 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":1.264,"15min":0.94699999999999995},"min":{"1min":0.28499999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":2.6659999999999999,"5min":11.170999999999999,"15min":11.170999999999999},"last":0.505},{"interface":"front","average":{"1min":0.91500000000000004,"5min":1.1180000000000001,"15min":0.83299999999999996},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":5.2679999999999998,"5min":7.3170000000000002,"15min":7.3170000000000002},"last":0.57799999999999996}]},{"osd":7,"last update":"Mon Aug 28 02:30:27 2023","interfaces":[{"interface":"back","average":{"1min":1.651,"5min":1.6990000000000001,"15min":1.1180000000000001},"min":{"1min":0.312,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":8.3490000000000002,"5min":16.584,"15min":16.584},"last":0.61499999999999999},{"interface":"front","average":{"1min":1.706,"5min":1.3919999999999999,"15min":1.137},"min":{"1min":0.378,"5min":0.311,"15min":0.311},"max":{"1min":7.4619999999999997,"5min":10.826000000000001,"15min":10.826000000000001},"last":0.48599999999999999}]}]},{"osd":5,"up_from":36,"seq":154618822919,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":319768,"kb_used_data":18448,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93425384,"statfs":{"total":95995035648,"available":95667593216,"internally_reserved":0,"allocated":18890752,"data_stored":18198897,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:30:02 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":2.052,"15min":2.6469999999999998},"min":{"1min":0.214,"5min":0.214,"15min":0.20300000000000001},"max":{"1min":4.548,"5min":42.229999999999997,"15min":131.06100000000001},"last":0.41599999999999998},{"interface":"front","average":{"1min":1.4710000000000001,"5min":2.371,"15min":2.8460000000000001},"min":{"1min":0.27500000000000002,"5min":0.24199999999999999,"15min":0.22900000000000001},"max":{"1min":5.8129999999999997,"5min":42.213999999999999,"15min":131.262},"last":0.36199999999999999}]},{"osd":1,"last update":"Mon Aug 28 02:30:12 2023","interfaces":[{"interface":"back","average":{"1min":1.8819999999999999,"5min":1.7869999999999999,"15min":2.387},"min":{"1min":0.27400000000000002,"5min":0.216,"15min":0.216},"max":{"1min":7.7809999999999997,"5min":30.593,"15min":183.97900000000001},"last":0.312},{"interface":"front","average":{"1min":1.1220000000000001,"5min":2.0070000000000001,"15min":2.5510000000000002},"min":{"1min":0.23499999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":5.8810000000000002,"5min":34.616999999999997,"15min":184.33699999999999},"last":0.40000000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:30:41 2023","interfaces":[{"interface":"back","average":{"1min":2.7469999999999999,"5min":6.117,"15min":3.9740000000000002},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.252},"max":{"1min":11.875,"5min":176.69499999999999,"15min":176.69499999999999},"last":0.61599999999999999},{"interface":"front","average":{"1min":3.7160000000000002,"5min":6.5620000000000003,"15min":4.21},"min":{"1min":0.35299999999999998,"5min":0.34000000000000002,"15min":0.28299999999999997},"max":{"1min":11.887,"5min":176.71199999999999,"15min":176.71199999999999},"last":0.625}]},{"osd":3,"last update":"Mon Aug 28 02:30:12 2023","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":2.8730000000000002,"15min":1.6180000000000001},"min":{"1min":0.28799999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.1070000000000002,"5min":102.64400000000001,"15min":102.64400000000001},"last":0.501},{"interface":"front","average":{"1min":1.8919999999999999,"5min":3.6509999999999998,"15min":1.956},"min":{"1min":0.28699999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":8.1660000000000004,"5min":103.771,"15min":103.771},"last":0.42699999999999999}]},{"osd":4,"last update":"Mon Aug 28 02:29:50 2023","interfaces":[{"interface":"back","average":{"1min":1.8300000000000001,"5min":1.157,"15min":0.93100000000000005},"min":{"1min":0.26700000000000002,"5min":0.246,"15min":0.246},"max":{"1min":25.306999999999999,"5min":25.306999999999999,"15min":25.306999999999999},"last":0.34499999999999997},{"interface":"front","average":{"1min":1.7250000000000001,"5min":1.2430000000000001,"15min":0.95899999999999996},"min":{"1min":0.29399999999999998,"5min":0.28999999999999998,"15min":0.28000000000000003},"max":{"1min":25.347000000000001,"5min":25.347000000000001,"15min":25.347000000000001},"last":0.47799999999999998}]},{"osd":6,"last update":"Mon Aug 28 02:30:12 2023","interfaces":[{"interface":"back","average":{"1min":1.0429999999999999,"5min":1.099,"15min":1.002},"min":{"1min":0.30399999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.6210000000000004,"5min":5.9870000000000001,"15min":14.803000000000001},"last":0.377},{"interface":"front","average":{"1min":0.95499999999999996,"5min":1.04,"15min":0.91600000000000004},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":4.9509999999999996,"5min":8.4779999999999998,"15min":8.6479999999999997},"last":0.45400000000000001}]},{"osd":7,"last update":"Mon Aug 28 02:30:29 2023","interfaces":[{"interface":"back","average":{"1min":1.1419999999999999,"5min":1.5109999999999999,"15min":1.127},"min":{"1min":0.34300000000000003,"5min":0.33300000000000002,"15min":0.31},"max":{"1min":4.0060000000000002,"5min":25.379000000000001,"15min":25.379000000000001},"last":0.439},{"interface":"front","average":{"1min":1.3160000000000001,"5min":1.2769999999999999,"15min":1.0569999999999999},"min":{"1min":0.38300000000000001,"5min":0.33700000000000002,"15min":0.32100000000000001},"max":{"1min":5.931,"5min":25.437000000000001,"15min":25.437000000000001},"last":0.46500000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-08-28T02:30:54.108 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-08-28T02:30:54.109 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-08-28T02:30:54.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:54.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:30:54.210 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:30:54.236 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:54.415 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:30:54.435 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:30:54.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:54.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:30:54.538 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:30:54.566 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:54.609 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:30:54.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:54.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:30:54.712 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:30:54.740 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:54.883 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:30:54.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:54.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:30:54.914 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:30:54.914 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:30:54.987 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:30:55.014 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:55.038 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is set 2023-08-28T02:30:55.124 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:55 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3832195453' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:30:55.124 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:55 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3832195453' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:30:55.209 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:30:55.210 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:55 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3832195453' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:30:55.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:55.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:30:55.312 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:30:55.338 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:55.380 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:30:55.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:55.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:30:55.483 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:30:55.514 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:55.702 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.6 dump_blocked_ops 2023-08-28T02:30:55.706 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:30:55.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:55.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:30:55.743 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:30:55.810 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:30:55.838 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:55.931 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:30:55.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:55.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:30:56.035 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:30:56.053 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:30:56.059 INFO:teuthology.orchestra.run.smithi143.stderr:pool 'unique_pool_2' created 2023-08-28T02:30:56.068 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:56.112 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:30:56.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:56.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:30:56.216 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:30:56.243 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:56.289 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:30:56.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:56.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:30:56.306 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:56 smithi169 ceph-mon[159415]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:30:56.306 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:56 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3832195453' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:30:56.306 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:56 smithi169 ceph-mon[159415]: osdmap e260: 8 total, 8 up, 8 in 2023-08-28T02:30:56.307 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:56 smithi169 ceph-mon[159415]: failed to encode map e260 with expected crc 2023-08-28T02:30:56.307 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:56 smithi169 ceph-mon[159415]: failed to encode map e260 with expected crc 2023-08-28T02:30:56.307 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:56 smithi169 ceph-mon[159415]: pgmap v634: 73 pgs: 73 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-08-28T02:30:56.307 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:56 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/4163892760' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-08-28T02:30:56.307 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:56 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-08-28T02:30:56.307 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:56 smithi169 ceph-mon[159415]: failed to encode map e260 with expected crc 2023-08-28T02:30:56.307 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:56 smithi169 ceph-mon[159415]: failed to encode map e260 with expected crc 2023-08-28T02:30:56.307 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:56 smithi169 ceph-mon[159415]: failed to encode map e260 with expected crc 2023-08-28T02:30:56.307 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:56 smithi169 ceph-mon[159415]: failed to encode map e260 with expected crc 2023-08-28T02:30:56.393 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:30:56.419 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:56.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:56 smithi143 ceph-mon[189871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:30:56.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:56 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3832195453' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:30:56.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:56 smithi143 ceph-mon[189871]: osdmap e260: 8 total, 8 up, 8 in 2023-08-28T02:30:56.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:56 smithi143 ceph-mon[189871]: failed to encode map e260 with expected crc 2023-08-28T02:30:56.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:56 smithi143 ceph-mon[189871]: failed to encode map e260 with expected crc 2023-08-28T02:30:56.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:56 smithi143 ceph-mon[189871]: pgmap v634: 73 pgs: 73 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-08-28T02:30:56.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:56 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/4163892760' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-08-28T02:30:56.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:56 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-08-28T02:30:56.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:56 smithi143 ceph-mon[189871]: failed to encode map e260 with expected crc 2023-08-28T02:30:56.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:56 smithi143 ceph-mon[189871]: failed to encode map e260 with expected crc 2023-08-28T02:30:56.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:56 smithi143 ceph-mon[189871]: failed to encode map e260 with expected crc 2023-08-28T02:30:56.453 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:56 smithi143 ceph-mon[189871]: failed to encode map e260 with expected crc 2023-08-28T02:30:56.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:56 smithi143 ceph-mon[183059]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:30:56.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:56 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3832195453' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:30:56.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:56 smithi143 ceph-mon[183059]: osdmap e260: 8 total, 8 up, 8 in 2023-08-28T02:30:56.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:56 smithi143 ceph-mon[183059]: failed to encode map e260 with expected crc 2023-08-28T02:30:56.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:56 smithi143 ceph-mon[183059]: failed to encode map e260 with expected crc 2023-08-28T02:30:56.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:56 smithi143 ceph-mon[183059]: pgmap v634: 73 pgs: 73 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-08-28T02:30:56.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:56 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/4163892760' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-08-28T02:30:56.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:56 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-08-28T02:30:56.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:56 smithi143 ceph-mon[183059]: failed to encode map e260 with expected crc 2023-08-28T02:30:56.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:56 smithi143 ceph-mon[183059]: failed to encode map e260 with expected crc 2023-08-28T02:30:56.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:56 smithi143 ceph-mon[183059]: failed to encode map e260 with expected crc 2023-08-28T02:30:56.455 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:56 smithi143 ceph-mon[183059]: failed to encode map e260 with expected crc 2023-08-28T02:30:56.513 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:30:56.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:56.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:30:56.617 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:30:56.643 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:56.743 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:30:56.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:56.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:30:56.846 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:30:56.873 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:56.917 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:30:56.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:56.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:30:57.021 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:30:57.048 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:57.146 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:30:57.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:57.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:30:57.166 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2023-08-28T02:30:57.229 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:57 smithi169 ceph-mon[159415]: failed to encode map e261 with expected crc 2023-08-28T02:30:57.230 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:57 smithi169 ceph-mon[159415]: failed to encode map e261 with expected crc 2023-08-28T02:30:57.230 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:57 smithi169 ceph-mon[159415]: failed to encode map e261 with expected crc 2023-08-28T02:30:57.230 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:57 smithi169 ceph-mon[159415]: failed to encode map e261 with expected crc 2023-08-28T02:30:57.230 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:57 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-08-28T02:30:57.230 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:57 smithi169 ceph-mon[159415]: osdmap e261: 8 total, 8 up, 8 in 2023-08-28T02:30:57.230 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:57 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:30:57.230 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:57 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:30:57.230 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:57 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:30:57.252 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:30:57.280 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:57.362 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:30:57.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:57.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:30:57.434 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:57 smithi143 ceph-mon[183059]: failed to encode map e261 with expected crc 2023-08-28T02:30:57.434 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:57 smithi143 ceph-mon[183059]: failed to encode map e261 with expected crc 2023-08-28T02:30:57.434 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:57 smithi143 ceph-mon[183059]: failed to encode map e261 with expected crc 2023-08-28T02:30:57.434 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:57 smithi143 ceph-mon[183059]: failed to encode map e261 with expected crc 2023-08-28T02:30:57.434 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:57 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-08-28T02:30:57.435 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:57 smithi143 ceph-mon[183059]: osdmap e261: 8 total, 8 up, 8 in 2023-08-28T02:30:57.435 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:57 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:30:57.435 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:57 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:30:57.435 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:57 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:30:57.435 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:57 smithi143 ceph-mon[189871]: failed to encode map e261 with expected crc 2023-08-28T02:30:57.435 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:57 smithi143 ceph-mon[189871]: failed to encode map e261 with expected crc 2023-08-28T02:30:57.435 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:57 smithi143 ceph-mon[189871]: failed to encode map e261 with expected crc 2023-08-28T02:30:57.435 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:57 smithi143 ceph-mon[189871]: failed to encode map e261 with expected crc 2023-08-28T02:30:57.435 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:57 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-08-28T02:30:57.435 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:57 smithi143 ceph-mon[189871]: osdmap e261: 8 total, 8 up, 8 in 2023-08-28T02:30:57.435 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:57 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:30:57.436 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:57 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:30:57.436 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:57 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:30:57.466 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:30:57.494 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:57.591 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:30:57.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:57.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:30:57.640 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:30:57.659 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:30:57.669 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:30:57.672 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:30:57.672 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:30:57.678 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:30:57.696 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:30:57.698 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:30:57.716 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:30:57.723 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:57.734 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:30:57.752 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:30:57.770 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set nodeep-scrub 2023-08-28T02:30:57.772 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:30:57.773 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:30:57.773 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:30:57.773 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:30:57.773 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:30:57.773 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:30:57.773 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:30:57.773 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:30:57.773 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:30:57.921 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:30:57.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:57.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:30:58.025 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:30:58.054 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:58.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:58 smithi169 ceph-mon[159415]: osdmap e262: 8 total, 8 up, 8 in 2023-08-28T02:30:58.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:58 smithi169 ceph-mon[159415]: failed to encode map e262 with expected crc 2023-08-28T02:30:58.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:58 smithi169 ceph-mon[159415]: failed to encode map e262 with expected crc 2023-08-28T02:30:58.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:58 smithi169 ceph-mon[159415]: failed to encode map e262 with expected crc 2023-08-28T02:30:58.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:58 smithi169 ceph-mon[159415]: failed to encode map e262 with expected crc 2023-08-28T02:30:58.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:58 smithi169 ceph-mon[159415]: pgmap v637: 89 pgs: 2 creating+peering, 10 unknown, 77 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-08-28T02:30:58.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:58 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3902532590' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:30:58.377 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.6 dump_historic_ops 2023-08-28T02:30:58.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:58 smithi143 ceph-mon[183059]: osdmap e262: 8 total, 8 up, 8 in 2023-08-28T02:30:58.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:58 smithi143 ceph-mon[183059]: failed to encode map e262 with expected crc 2023-08-28T02:30:58.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:58 smithi143 ceph-mon[183059]: failed to encode map e262 with expected crc 2023-08-28T02:30:58.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:58 smithi143 ceph-mon[183059]: failed to encode map e262 with expected crc 2023-08-28T02:30:58.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:58 smithi143 ceph-mon[183059]: failed to encode map e262 with expected crc 2023-08-28T02:30:58.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:58 smithi143 ceph-mon[183059]: pgmap v637: 89 pgs: 2 creating+peering, 10 unknown, 77 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-08-28T02:30:58.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:58 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3902532590' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:30:58.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:58 smithi143 ceph-mon[189871]: osdmap e262: 8 total, 8 up, 8 in 2023-08-28T02:30:58.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:58 smithi143 ceph-mon[189871]: failed to encode map e262 with expected crc 2023-08-28T02:30:58.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:58 smithi143 ceph-mon[189871]: failed to encode map e262 with expected crc 2023-08-28T02:30:58.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:58 smithi143 ceph-mon[189871]: failed to encode map e262 with expected crc 2023-08-28T02:30:58.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:58 smithi143 ceph-mon[189871]: failed to encode map e262 with expected crc 2023-08-28T02:30:58.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:58 smithi143 ceph-mon[189871]: pgmap v637: 89 pgs: 2 creating+peering, 10 unknown, 77 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-08-28T02:30:58.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:58 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3902532590' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:30:58.725 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:30:59.105 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:30:59.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:59.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:30:59.153 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:30:59.212 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:30:59.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:59 smithi169 ceph-mon[159415]: osdmap e263: 8 total, 8 up, 8 in 2023-08-28T02:30:59.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:59 smithi169 ceph-mon[159415]: failed to encode map e263 with expected crc 2023-08-28T02:30:59.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:59 smithi169 ceph-mon[159415]: failed to encode map e263 with expected crc 2023-08-28T02:30:59.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:59 smithi169 ceph-mon[159415]: failed to encode map e263 with expected crc 2023-08-28T02:30:59.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:30:59 smithi169 ceph-mon[159415]: failed to encode map e263 with expected crc 2023-08-28T02:30:59.241 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:59.299 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:59 smithi143 ceph-mon[183059]: osdmap e263: 8 total, 8 up, 8 in 2023-08-28T02:30:59.299 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:59 smithi143 ceph-mon[183059]: failed to encode map e263 with expected crc 2023-08-28T02:30:59.300 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:59 smithi143 ceph-mon[183059]: failed to encode map e263 with expected crc 2023-08-28T02:30:59.300 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:59 smithi143 ceph-mon[183059]: failed to encode map e263 with expected crc 2023-08-28T02:30:59.300 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:30:59 smithi143 ceph-mon[183059]: failed to encode map e263 with expected crc 2023-08-28T02:30:59.300 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:59 smithi143 ceph-mon[189871]: osdmap e263: 8 total, 8 up, 8 in 2023-08-28T02:30:59.300 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:59 smithi143 ceph-mon[189871]: failed to encode map e263 with expected crc 2023-08-28T02:30:59.300 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:59 smithi143 ceph-mon[189871]: failed to encode map e263 with expected crc 2023-08-28T02:30:59.300 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:59 smithi143 ceph-mon[189871]: failed to encode map e263 with expected crc 2023-08-28T02:30:59.300 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:30:59 smithi143 ceph-mon[189871]: failed to encode map e263 with expected crc 2023-08-28T02:30:59.435 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:30:59.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:59.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:30:59.539 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:30:59.569 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:59.614 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:30:59.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:59.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:30:59.718 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:30:59.747 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:30:59.845 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:30:59.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:30:59.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:30:59.948 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:30:59.977 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:00.068 INFO:teuthology.orchestra.run.smithi143.stderr:enabled application 'rados' on pool 'unique_pool_2' 2023-08-28T02:31:00.093 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:00 smithi143 ceph-mon[183059]: pgmap v639: 89 pgs: 2 creating+peering, 1 unknown, 86 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-08-28T02:31:00.093 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:00 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/4151844632' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-28T02:31:00.093 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:00 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-28T02:31:00.093 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:00 smithi143 ceph-mon[189871]: pgmap v639: 89 pgs: 2 creating+peering, 1 unknown, 86 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-08-28T02:31:00.094 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:00 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/4151844632' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-28T02:31:00.094 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:00 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-28T02:31:00.331 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:31:00.331 INFO:teuthology.orchestra.run.smithi169.stdout: "size": 20, 2023-08-28T02:31:00.331 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 600, 2023-08-28T02:31:00.331 INFO:teuthology.orchestra.run.smithi169.stdout: "ops": [ 2023-08-28T02:31:00.331 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.331 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:122090 5.9 5:9a6395dd:::benchmark_data_smithi143_236200_object11947:head [delete] snapc 0=[] ondisk+write+known_if_redirected e152)", 2023-08-28T02:31:00.331 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:56.359008+0000", 2023-08-28T02:31:00.331 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 423.97051964899998, 2023-08-28T02:31:00.332 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.190802156, 2023-08-28T02:31:00.332 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:31:00.332 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:00.332 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:31:00.332 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:31:00.332 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:31:00.332 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 122090 2023-08-28T02:31:00.332 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.332 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:31:00.332 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.332 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:31:00.332 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.359008+0000", 2023-08-28T02:31:00.332 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:31:00.333 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.333 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.333 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:31:00.333 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.359008+0000", 2023-08-28T02:31:00.333 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:31:00.333 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.333 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.333 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:31:00.333 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.359006+0000", 2023-08-28T02:31:00.333 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.2130000000000008e-06 2023-08-28T02:31:00.333 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.333 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.333 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:31:00.333 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.359015+0000", 2023-08-28T02:31:00.334 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.3277e-05 2023-08-28T02:31:00.334 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.334 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.334 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:31:00.334 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.359028+0000", 2023-08-28T02:31:00.334 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.0299999999999996e-06 2023-08-28T02:31:00.334 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.334 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.334 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:31:00.334 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.359035+0000", 2023-08-28T02:31:00.334 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.6771000000000003e-05 2023-08-28T02:31:00.334 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.334 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.335 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:31:00.335 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.359082+0000", 2023-08-28T02:31:00.335 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000141734 2023-08-28T02:31:00.335 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.335 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.335 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:31:00.335 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.359223+0000", 2023-08-28T02:31:00.335 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000157331 2023-08-28T02:31:00.335 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.335 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.335 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 4,7", 2023-08-28T02:31:00.335 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.359381+0000", 2023-08-28T02:31:00.335 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0036132569999999999 2023-08-28T02:31:00.336 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.336 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.336 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:31:00.336 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.362994+0000", 2023-08-28T02:31:00.336 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.01001399 2023-08-28T02:31:00.336 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.336 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.336 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:00.336 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.373008+0000", 2023-08-28T02:31:00.336 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.17651315000000001 2023-08-28T02:31:00.336 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.336 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.337 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:00.337 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.549521+0000", 2023-08-28T02:31:00.337 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.1438000000000004e-05 2023-08-28T02:31:00.337 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.337 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.337 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:31:00.337 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.549583+0000", 2023-08-28T02:31:00.337 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00022760799999999999 2023-08-28T02:31:00.337 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.337 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.337 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:31:00.337 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.549810+0000", 2023-08-28T02:31:00.337 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.190802156 2023-08-28T02:31:00.337 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:00.338 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:31:00.338 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:00.338 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.338 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.338 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:122093 5.9 5:90145365:::benchmark_data_smithi143_236200_object11950:head [delete] snapc 0=[] ondisk+write+known_if_redirected e152)", 2023-08-28T02:31:00.338 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:56.359053+0000", 2023-08-28T02:31:00.338 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 423.97047453900001, 2023-08-28T02:31:00.338 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.19099266200000001, 2023-08-28T02:31:00.338 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:31:00.338 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:00.338 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:31:00.338 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:31:00.339 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:31:00.339 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 122093 2023-08-28T02:31:00.339 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.339 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:31:00.339 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.339 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:31:00.339 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.359053+0000", 2023-08-28T02:31:00.339 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:31:00.339 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.339 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.339 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:31:00.339 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.359053+0000", 2023-08-28T02:31:00.339 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999995 2023-08-28T02:31:00.339 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.340 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.340 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:31:00.340 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.359053+0000", 2023-08-28T02:31:00.340 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.146e-06 2023-08-28T02:31:00.340 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.340 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.340 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:31:00.340 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.359056+0000", 2023-08-28T02:31:00.340 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.5230000000000001e-06 2023-08-28T02:31:00.340 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.340 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.340 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:31:00.341 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.359062+0000", 2023-08-28T02:31:00.341 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.7600000000000002e-06 2023-08-28T02:31:00.341 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.341 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.341 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:31:00.341 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.359067+0000", 2023-08-28T02:31:00.341 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0067806539999999997 2023-08-28T02:31:00.341 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.341 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.341 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:31:00.341 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.365848+0000", 2023-08-28T02:31:00.341 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00013536200000000001 2023-08-28T02:31:00.341 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.342 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.342 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:31:00.342 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.365983+0000", 2023-08-28T02:31:00.342 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00015116000000000001 2023-08-28T02:31:00.342 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.342 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.342 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 4,7", 2023-08-28T02:31:00.342 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.366134+0000", 2023-08-28T02:31:00.342 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.002991437 2023-08-28T02:31:00.342 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.342 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.342 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:31:00.342 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.369126+0000", 2023-08-28T02:31:00.343 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0040113329999999997 2023-08-28T02:31:00.343 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.343 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.343 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:00.343 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.373137+0000", 2023-08-28T02:31:00.343 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.17671240499999999 2023-08-28T02:31:00.343 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.343 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.343 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:00.343 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.549850+0000", 2023-08-28T02:31:00.343 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.4039000000000003e-05 2023-08-28T02:31:00.343 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.343 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.344 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:31:00.344 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.549894+0000", 2023-08-28T02:31:00.344 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000152342 2023-08-28T02:31:00.344 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.344 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.344 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:31:00.344 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.550046+0000", 2023-08-28T02:31:00.344 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.19099266200000001 2023-08-28T02:31:00.344 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:00.344 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:31:00.344 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:00.344 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.344 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.345 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:122108 5.9 5:9c0cafde:::benchmark_data_smithi143_236200_object11965:head [delete] snapc 0=[] ondisk+write+known_if_redirected e152)", 2023-08-28T02:31:00.345 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:23:56.376442+0000", 2023-08-28T02:31:00.345 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 423.953085268, 2023-08-28T02:31:00.345 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.17433146099999999, 2023-08-28T02:31:00.345 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:31:00.345 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:00.345 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:31:00.345 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:31:00.345 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:31:00.345 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 122108 2023-08-28T02:31:00.345 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.345 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:31:00.346 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.346 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:31:00.346 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.376442+0000", 2023-08-28T02:31:00.346 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:31:00.346 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.346 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.346 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:31:00.346 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.376442+0000", 2023-08-28T02:31:00.346 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:31:00.346 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.346 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.346 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:31:00.347 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.376441+0000", 2023-08-28T02:31:00.347 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.6969999999999998e-06 2023-08-28T02:31:00.347 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.347 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.347 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:31:00.347 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.376447+0000", 2023-08-28T02:31:00.347 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.4599999999999992e-06 2023-08-28T02:31:00.347 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.347 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.347 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:31:00.347 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.376456+0000", 2023-08-28T02:31:00.347 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.8930000000000004e-06 2023-08-28T02:31:00.348 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.348 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.348 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:31:00.348 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.376461+0000", 2023-08-28T02:31:00.348 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.3560999999999999e-05 2023-08-28T02:31:00.348 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.348 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.348 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:31:00.348 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.376495+0000", 2023-08-28T02:31:00.348 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.3605999999999998e-05 2023-08-28T02:31:00.348 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.348 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.348 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:31:00.349 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.376588+0000", 2023-08-28T02:31:00.349 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00010327600000000001 2023-08-28T02:31:00.349 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.349 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.349 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 4,7", 2023-08-28T02:31:00.349 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.376692+0000", 2023-08-28T02:31:00.349 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0025670110000000001 2023-08-28T02:31:00.349 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.349 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.349 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:31:00.349 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.379259+0000", 2023-08-28T02:31:00.349 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.170830232 2023-08-28T02:31:00.349 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.350 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.350 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:00.350 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.550089+0000", 2023-08-28T02:31:00.350 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000455723 2023-08-28T02:31:00.350 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.350 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.350 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:00.350 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.550544+0000", 2023-08-28T02:31:00.350 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.8425999999999999e-05 2023-08-28T02:31:00.350 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.350 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.350 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:31:00.350 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.550603+0000", 2023-08-28T02:31:00.351 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00017104699999999999 2023-08-28T02:31:00.351 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.351 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.351 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:31:00.351 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:23:56.550774+0000", 2023-08-28T02:31:00.351 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.17433146099999999 2023-08-28T02:31:00.351 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:00.351 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:31:00.351 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:00.351 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.351 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.351 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:124092 5.9 5:9a708815:::benchmark_data_smithi143_236200_object13949:head [delete] snapc 0=[] ondisk+write+known_if_redirected e155)", 2023-08-28T02:31:00.351 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:24:01.189195+0000", 2023-08-28T02:31:00.352 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 419.14033256099998, 2023-08-28T02:31:00.352 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.22172314300000001, 2023-08-28T02:31:00.352 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:31:00.352 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:00.352 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:31:00.352 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:31:00.352 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:31:00.352 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 124092 2023-08-28T02:31:00.352 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.352 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:31:00.352 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.352 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:31:00.352 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:01.189195+0000", 2023-08-28T02:31:00.353 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:31:00.353 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.353 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.353 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:31:00.353 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:01.189195+0000", 2023-08-28T02:31:00.353 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:31:00.353 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.353 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.353 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:31:00.353 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:01.189194+0000", 2023-08-28T02:31:00.353 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.8390000000000001e-06 2023-08-28T02:31:00.353 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.353 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.354 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:31:00.354 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:01.189200+0000", 2023-08-28T02:31:00.354 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.4344e-05 2023-08-28T02:31:00.354 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.354 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.354 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:31:00.354 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:01.189215+0000", 2023-08-28T02:31:00.354 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.1570000000000003e-06 2023-08-28T02:31:00.354 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.354 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.354 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:31:00.354 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:01.189222+0000", 2023-08-28T02:31:00.354 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.0998000000000001e-05 2023-08-28T02:31:00.355 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.355 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.355 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:31:00.355 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:01.189283+0000", 2023-08-28T02:31:00.355 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00014775500000000001 2023-08-28T02:31:00.355 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.355 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.355 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:31:00.355 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:01.189431+0000", 2023-08-28T02:31:00.355 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00016321799999999999 2023-08-28T02:31:00.355 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.355 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.355 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 4,7", 2023-08-28T02:31:00.356 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:01.189594+0000", 2023-08-28T02:31:00.356 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0030394419999999998 2023-08-28T02:31:00.356 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.356 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.356 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:31:00.356 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:01.192633+0000", 2023-08-28T02:31:00.356 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000372424 2023-08-28T02:31:00.356 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.356 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.356 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:00.356 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:01.193006+0000", 2023-08-28T02:31:00.356 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.217667476 2023-08-28T02:31:00.356 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.357 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.357 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:00.357 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:01.410673+0000", 2023-08-28T02:31:00.357 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.6119000000000003e-05 2023-08-28T02:31:00.357 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.357 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.357 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:31:00.357 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:01.410729+0000", 2023-08-28T02:31:00.357 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00018896399999999999 2023-08-28T02:31:00.357 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.357 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.357 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:31:00.357 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:01.410918+0000", 2023-08-28T02:31:00.357 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.22172314300000001 2023-08-28T02:31:00.358 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:00.358 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:31:00.358 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:00.358 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.358 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.358 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:132241 5.2 5:456a1d66:::benchmark_data_smithi143_236200_object22098:head [delete] snapc 0=[] ondisk+write+known_if_redirected e163)", 2023-08-28T02:31:00.358 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:24:15.744623+0000", 2023-08-28T02:31:00.358 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 404.58490447999998, 2023-08-28T02:31:00.358 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.16631565800000001, 2023-08-28T02:31:00.358 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:31:00.358 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:00.358 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:31:00.358 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:31:00.359 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:31:00.359 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 132241 2023-08-28T02:31:00.359 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.359 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:31:00.359 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.359 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:31:00.359 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:15.744623+0000", 2023-08-28T02:31:00.359 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:31:00.359 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.359 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.359 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:31:00.359 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:15.744623+0000", 2023-08-28T02:31:00.359 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:31:00.360 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.360 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.360 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:31:00.360 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:15.744621+0000", 2023-08-28T02:31:00.360 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.9389999999999997e-06 2023-08-28T02:31:00.360 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.360 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.360 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:31:00.360 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:15.744629+0000", 2023-08-28T02:31:00.360 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.4891e-05 2023-08-28T02:31:00.360 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.360 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.361 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:31:00.361 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:15.744644+0000", 2023-08-28T02:31:00.361 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.5270000000000004e-06 2023-08-28T02:31:00.361 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.361 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.361 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:31:00.361 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:15.744651+0000", 2023-08-28T02:31:00.361 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.6891999999999998e-05 2023-08-28T02:31:00.361 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.361 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.361 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:31:00.361 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:15.744698+0000", 2023-08-28T02:31:00.361 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000118609 2023-08-28T02:31:00.362 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.362 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.362 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:31:00.362 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:15.744816+0000", 2023-08-28T02:31:00.362 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012755700000000001 2023-08-28T02:31:00.362 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.362 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.362 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:31:00.362 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:15.744944+0000", 2023-08-28T02:31:00.362 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0038987599999999998 2023-08-28T02:31:00.362 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.362 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.362 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:31:00.363 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:15.748843+0000", 2023-08-28T02:31:00.363 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0084125650000000003 2023-08-28T02:31:00.363 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.363 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.363 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:00.363 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:15.757255+0000", 2023-08-28T02:31:00.363 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.15358860899999999 2023-08-28T02:31:00.363 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.363 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.363 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:00.363 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:15.910844+0000", 2023-08-28T02:31:00.363 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.6038000000000001e-05 2023-08-28T02:31:00.363 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.364 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.364 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:31:00.364 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:15.910900+0000", 2023-08-28T02:31:00.364 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.9128000000000002e-05 2023-08-28T02:31:00.364 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.364 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.364 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:31:00.364 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:15.910939+0000", 2023-08-28T02:31:00.364 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.16631565800000001 2023-08-28T02:31:00.364 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:00.364 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:31:00.364 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:00.364 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.365 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.365 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:132249 5.2 5:4a33e4c9:::benchmark_data_smithi143_236200_object22106:head [delete] snapc 0=[] ondisk+write+known_if_redirected e163)", 2023-08-28T02:31:00.365 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:24:15.757388+0000", 2023-08-28T02:31:00.365 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 404.57213944300003, 2023-08-28T02:31:00.365 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.16790682100000001, 2023-08-28T02:31:00.365 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:31:00.365 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:00.365 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:31:00.365 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:31:00.365 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:31:00.365 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 132249 2023-08-28T02:31:00.365 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.365 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:31:00.366 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.366 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:31:00.366 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:15.757388+0000", 2023-08-28T02:31:00.366 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:31:00.366 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.366 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.366 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:31:00.366 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:15.757388+0000", 2023-08-28T02:31:00.366 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:31:00.366 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.366 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.366 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:31:00.367 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:15.757387+0000", 2023-08-28T02:31:00.367 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.1749999999999999e-06 2023-08-28T02:31:00.367 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.367 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.367 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:31:00.367 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:15.757394+0000", 2023-08-28T02:31:00.367 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1212999999999999e-05 2023-08-28T02:31:00.367 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.367 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.367 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:31:00.367 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:15.757405+0000", 2023-08-28T02:31:00.367 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.3600000000000001e-06 2023-08-28T02:31:00.367 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.368 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.368 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:31:00.368 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:15.757412+0000", 2023-08-28T02:31:00.368 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.5754999999999998e-05 2023-08-28T02:31:00.368 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.368 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.368 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:31:00.368 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:15.757457+0000", 2023-08-28T02:31:00.368 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000110158 2023-08-28T02:31:00.368 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.368 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.368 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:31:00.369 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:15.757567+0000", 2023-08-28T02:31:00.369 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012417799999999999 2023-08-28T02:31:00.369 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.369 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.369 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:31:00.369 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:15.757692+0000", 2023-08-28T02:31:00.369 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0076177850000000002 2023-08-28T02:31:00.369 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.369 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.369 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:00.369 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:15.765309+0000", 2023-08-28T02:31:00.369 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.4727000000000001e-05 2023-08-28T02:31:00.370 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.370 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.370 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:31:00.370 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:15.765344+0000", 2023-08-28T02:31:00.370 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.15986233999999999 2023-08-28T02:31:00.370 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.370 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.370 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:00.370 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:15.925207+0000", 2023-08-28T02:31:00.370 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.8498000000000002e-05 2023-08-28T02:31:00.370 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.370 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.370 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:31:00.371 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:15.925255+0000", 2023-08-28T02:31:00.371 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.0123999999999997e-05 2023-08-28T02:31:00.371 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.371 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.371 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:31:00.371 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:15.925295+0000", 2023-08-28T02:31:00.371 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.16790682100000001 2023-08-28T02:31:00.371 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:00.371 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:31:00.371 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:00.371 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.371 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.371 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:132251 5.2 5:4e4c0952:::benchmark_data_smithi143_236200_object22108:head [delete] snapc 0=[] ondisk+write+known_if_redirected e163)", 2023-08-28T02:31:00.372 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:24:15.758623+0000", 2023-08-28T02:31:00.372 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 404.570905111, 2023-08-28T02:31:00.372 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.16680318099999999, 2023-08-28T02:31:00.372 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:31:00.372 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:00.372 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:31:00.372 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:31:00.372 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:31:00.372 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 132251 2023-08-28T02:31:00.372 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.372 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:31:00.372 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.373 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:31:00.373 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:15.758623+0000", 2023-08-28T02:31:00.373 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:31:00.373 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.373 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.373 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:31:00.373 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:15.758623+0000", 2023-08-28T02:31:00.373 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:31:00.373 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.373 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.373 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:31:00.373 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:15.758622+0000", 2023-08-28T02:31:00.374 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.63e-06 2023-08-28T02:31:00.374 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.374 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.374 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:31:00.374 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:15.758628+0000", 2023-08-28T02:31:00.374 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1182000000000001e-05 2023-08-28T02:31:00.374 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.374 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.374 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:31:00.374 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:15.758639+0000", 2023-08-28T02:31:00.374 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.3910000000000003e-06 2023-08-28T02:31:00.374 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.374 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.375 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:31:00.375 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:15.758646+0000", 2023-08-28T02:31:00.375 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0028465169999999998 2023-08-28T02:31:00.375 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.375 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.375 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:31:00.375 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:15.761492+0000", 2023-08-28T02:31:00.375 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00011824000000000001 2023-08-28T02:31:00.375 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.375 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.375 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:31:00.375 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:15.761611+0000", 2023-08-28T02:31:00.376 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012858999999999999 2023-08-28T02:31:00.376 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.376 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.376 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:31:00.376 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:15.761739+0000", 2023-08-28T02:31:00.376 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.011066833999999999 2023-08-28T02:31:00.376 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.376 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.376 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:31:00.376 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:15.772806+0000", 2023-08-28T02:31:00.376 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.8292999999999999e-05 2023-08-28T02:31:00.376 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.376 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.377 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:00.377 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:15.772864+0000", 2023-08-28T02:31:00.377 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.152471522 2023-08-28T02:31:00.377 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.377 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.377 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:00.377 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:15.925336+0000", 2023-08-28T02:31:00.377 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.5430000000000003e-05 2023-08-28T02:31:00.377 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.377 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.377 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:31:00.377 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:15.925391+0000", 2023-08-28T02:31:00.377 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.4569000000000001e-05 2023-08-28T02:31:00.378 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.378 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.378 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:31:00.378 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:15.925426+0000", 2023-08-28T02:31:00.378 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.16680318099999999 2023-08-28T02:31:00.378 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:00.378 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:31:00.378 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:00.378 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.378 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.378 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:140322 5.2 5:42801a01:::benchmark_data_smithi143_236200_object30179:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-08-28T02:31:00.378 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:24:28.048027+0000", 2023-08-28T02:31:00.379 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 392.28150091800001, 2023-08-28T02:31:00.379 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.192917692, 2023-08-28T02:31:00.379 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:31:00.379 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:00.379 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:31:00.379 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:31:00.379 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:31:00.379 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 140322 2023-08-28T02:31:00.379 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.379 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:31:00.379 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.379 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:31:00.379 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:28.048027+0000", 2023-08-28T02:31:00.380 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:31:00.380 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.380 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.380 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:31:00.380 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:28.048027+0000", 2023-08-28T02:31:00.380 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:31:00.380 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.380 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.380 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:31:00.380 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:28.048026+0000", 2023-08-28T02:31:00.380 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.7319999999999998e-06 2023-08-28T02:31:00.380 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.381 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.381 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:31:00.381 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:28.048030+0000", 2023-08-28T02:31:00.381 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1386000000000001e-05 2023-08-28T02:31:00.381 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.381 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.381 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:31:00.381 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:28.048042+0000", 2023-08-28T02:31:00.381 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.835e-06 2023-08-28T02:31:00.381 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.381 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.381 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:31:00.381 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:28.048048+0000", 2023-08-28T02:31:00.382 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.3040000000000001e-05 2023-08-28T02:31:00.382 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.382 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.382 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:31:00.382 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:28.048091+0000", 2023-08-28T02:31:00.382 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000121556 2023-08-28T02:31:00.382 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.382 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.382 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:31:00.382 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:28.048212+0000", 2023-08-28T02:31:00.382 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000133318 2023-08-28T02:31:00.382 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.382 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.383 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:31:00.383 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:28.048346+0000", 2023-08-28T02:31:00.383 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0041446859999999999 2023-08-28T02:31:00.383 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.383 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.383 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:31:00.383 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:28.052490+0000", 2023-08-28T02:31:00.383 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.012687311 2023-08-28T02:31:00.383 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.383 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.383 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:00.383 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:28.065178+0000", 2023-08-28T02:31:00.383 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.175701526 2023-08-28T02:31:00.384 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.384 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.384 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:00.384 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:28.240879+0000", 2023-08-28T02:31:00.384 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.6655000000000001e-05 2023-08-28T02:31:00.384 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.384 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.384 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:31:00.384 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:28.240916+0000", 2023-08-28T02:31:00.384 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.8761000000000001e-05 2023-08-28T02:31:00.384 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.384 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.384 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:31:00.384 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:28.240945+0000", 2023-08-28T02:31:00.385 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.192917692 2023-08-28T02:31:00.385 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:00.385 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:31:00.385 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:00.385 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.385 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.385 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:140342 5.2 5:43de568a:::benchmark_data_smithi143_236200_object30199:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-08-28T02:31:00.385 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:24:28.071734+0000", 2023-08-28T02:31:00.385 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 392.25779333899999, 2023-08-28T02:31:00.385 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.176868108, 2023-08-28T02:31:00.385 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:31:00.385 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:00.385 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:31:00.386 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:31:00.386 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:31:00.386 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 140342 2023-08-28T02:31:00.386 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.386 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:31:00.386 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.386 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:31:00.386 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:28.071734+0000", 2023-08-28T02:31:00.386 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:31:00.386 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.386 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.386 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:31:00.386 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:28.071734+0000", 2023-08-28T02:31:00.387 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:31:00.387 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.387 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.387 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:31:00.387 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:28.071733+0000", 2023-08-28T02:31:00.387 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.968e-06 2023-08-28T02:31:00.387 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.387 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.387 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:31:00.387 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:28.071740+0000", 2023-08-28T02:31:00.387 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.3481e-05 2023-08-28T02:31:00.387 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.387 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.388 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:31:00.388 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:28.071754+0000", 2023-08-28T02:31:00.388 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.6790000000000002e-06 2023-08-28T02:31:00.388 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.388 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.388 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:31:00.388 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:28.071759+0000", 2023-08-28T02:31:00.388 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.3640000000000002e-05 2023-08-28T02:31:00.388 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.388 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.388 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:31:00.388 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:28.071803+0000", 2023-08-28T02:31:00.388 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000136286 2023-08-28T02:31:00.389 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.389 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.389 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:31:00.389 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:28.071939+0000", 2023-08-28T02:31:00.389 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00015311200000000001 2023-08-28T02:31:00.389 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.389 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.389 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:31:00.389 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:28.072092+0000", 2023-08-28T02:31:00.389 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0040989240000000003 2023-08-28T02:31:00.389 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.389 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.389 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:31:00.390 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:28.076191+0000", 2023-08-28T02:31:00.390 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00016370100000000001 2023-08-28T02:31:00.390 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.390 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.390 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:00.390 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:28.076355+0000", 2023-08-28T02:31:00.390 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.17214728500000001 2023-08-28T02:31:00.390 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.390 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.390 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:00.390 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:28.248502+0000", 2023-08-28T02:31:00.390 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.8174999999999998e-05 2023-08-28T02:31:00.391 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.391 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.391 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:31:00.391 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:28.248560+0000", 2023-08-28T02:31:00.391 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.2141000000000002e-05 2023-08-28T02:31:00.391 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.391 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.391 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:31:00.391 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:28.248603+0000", 2023-08-28T02:31:00.391 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.176868108 2023-08-28T02:31:00.391 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:00.391 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:31:00.391 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:00.392 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.392 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.392 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:154032 5.2 5:49372087:::benchmark_data_smithi143_236200_object43889:head [delete] snapc 0=[] ondisk+write+known_if_redirected e169)", 2023-08-28T02:31:00.392 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:24:50.327519+0000", 2023-08-28T02:31:00.392 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 370.00200856499998, 2023-08-28T02:31:00.392 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.212800404, 2023-08-28T02:31:00.392 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:31:00.392 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:00.392 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:31:00.392 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:31:00.392 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:31:00.392 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 154032 2023-08-28T02:31:00.393 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.393 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:31:00.393 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.393 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:31:00.393 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:50.327519+0000", 2023-08-28T02:31:00.393 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:31:00.393 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.393 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.393 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:31:00.393 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:50.327519+0000", 2023-08-28T02:31:00.393 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:31:00.393 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.393 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.393 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:31:00.394 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:50.327517+0000", 2023-08-28T02:31:00.394 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.8079999999999994e-06 2023-08-28T02:31:00.394 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.394 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.394 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:31:00.394 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:50.327526+0000", 2023-08-28T02:31:00.394 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2852e-05 2023-08-28T02:31:00.394 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.394 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.394 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:31:00.394 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:50.327539+0000", 2023-08-28T02:31:00.394 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.1619999999999999e-06 2023-08-28T02:31:00.394 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.395 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.395 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:31:00.395 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:50.327544+0000", 2023-08-28T02:31:00.395 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.0416e-05 2023-08-28T02:31:00.395 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.395 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.395 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:31:00.395 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:50.327595+0000", 2023-08-28T02:31:00.395 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.2856999999999996e-05 2023-08-28T02:31:00.395 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.395 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.395 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:31:00.395 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:50.327677+0000", 2023-08-28T02:31:00.396 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.2812999999999995e-05 2023-08-28T02:31:00.396 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.396 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.396 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:31:00.396 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:50.327760+0000", 2023-08-28T02:31:00.396 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0056217189999999998 2023-08-28T02:31:00.396 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.396 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.396 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:31:00.396 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:50.333382+0000", 2023-08-28T02:31:00.396 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.017033096000000001 2023-08-28T02:31:00.396 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.396 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.397 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:00.397 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:50.350415+0000", 2023-08-28T02:31:00.397 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.18983824699999999 2023-08-28T02:31:00.397 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.397 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.397 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:00.397 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:50.540253+0000", 2023-08-28T02:31:00.397 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.6167000000000001e-05 2023-08-28T02:31:00.397 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.397 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.397 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:31:00.397 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:50.540290+0000", 2023-08-28T02:31:00.398 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.0091e-05 2023-08-28T02:31:00.398 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.398 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.398 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:31:00.398 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:50.540320+0000", 2023-08-28T02:31:00.398 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.212800404 2023-08-28T02:31:00.398 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:00.398 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:31:00.398 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:00.398 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.398 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.398 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:154036 5.2 5:47409524:::benchmark_data_smithi143_236200_object43893:head [delete] snapc 0=[] ondisk+write+known_if_redirected e169)", 2023-08-28T02:31:00.398 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:24:50.333442+0000", 2023-08-28T02:31:00.398 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 369.996085456, 2023-08-28T02:31:00.399 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.210855193, 2023-08-28T02:31:00.399 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:31:00.399 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:00.399 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:31:00.399 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:31:00.399 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:31:00.399 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 154036 2023-08-28T02:31:00.399 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.399 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:31:00.399 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.399 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:31:00.399 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:50.333442+0000", 2023-08-28T02:31:00.399 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:31:00.400 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.400 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.400 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:31:00.400 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:50.333442+0000", 2023-08-28T02:31:00.400 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:31:00.400 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.400 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.400 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:31:00.400 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:50.333441+0000", 2023-08-28T02:31:00.400 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.5419999999999996e-06 2023-08-28T02:31:00.400 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.400 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.400 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:31:00.401 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:50.333447+0000", 2023-08-28T02:31:00.401 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1358e-05 2023-08-28T02:31:00.401 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.401 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.401 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:31:00.401 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:50.333458+0000", 2023-08-28T02:31:00.401 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.8129999999999998e-06 2023-08-28T02:31:00.401 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.401 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.401 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:31:00.401 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:50.333465+0000", 2023-08-28T02:31:00.401 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.8767000000000002e-05 2023-08-28T02:31:00.401 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.401 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.402 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:31:00.402 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:50.333524+0000", 2023-08-28T02:31:00.402 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000130106 2023-08-28T02:31:00.402 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.402 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.402 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:31:00.402 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:50.333654+0000", 2023-08-28T02:31:00.402 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00013222 2023-08-28T02:31:00.402 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.402 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.402 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:31:00.402 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:50.333786+0000", 2023-08-28T02:31:00.403 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.015405075000000001 2023-08-28T02:31:00.403 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.403 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.403 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:31:00.403 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:50.349191+0000", 2023-08-28T02:31:00.403 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0012526670000000001 2023-08-28T02:31:00.403 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.403 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.403 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:00.403 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:50.350444+0000", 2023-08-28T02:31:00.403 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.19379410699999999 2023-08-28T02:31:00.403 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.403 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.404 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:00.404 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:50.544238+0000", 2023-08-28T02:31:00.404 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.4601999999999998e-05 2023-08-28T02:31:00.404 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.404 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.404 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:31:00.404 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:50.544272+0000", 2023-08-28T02:31:00.404 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.5157e-05 2023-08-28T02:31:00.404 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.404 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.404 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:31:00.404 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:24:50.544297+0000", 2023-08-28T02:31:00.404 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.210855193 2023-08-28T02:31:00.404 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:00.405 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:31:00.405 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:00.405 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.405 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.405 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:185297 5.2 5:48132d4d:::benchmark_data_smithi143_236200_object75154:head [delete] snapc 0=[] ondisk+write+known_if_redirected e176)", 2023-08-28T02:31:00.405 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:25:41.089397+0000", 2023-08-28T02:31:00.405 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 319.24013110099997, 2023-08-28T02:31:00.405 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.19570041199999999, 2023-08-28T02:31:00.405 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:31:00.405 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:00.405 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:31:00.405 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:31:00.405 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:31:00.406 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 185297 2023-08-28T02:31:00.406 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.406 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:31:00.406 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.406 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:31:00.406 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:41.089397+0000", 2023-08-28T02:31:00.406 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:31:00.406 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.406 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.406 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:31:00.406 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:41.089397+0000", 2023-08-28T02:31:00.406 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999976 2023-08-28T02:31:00.406 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.407 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.407 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:31:00.407 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:41.089394+0000", 2023-08-28T02:31:00.407 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0586e-05 2023-08-28T02:31:00.407 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.407 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.407 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:31:00.407 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:41.089405+0000", 2023-08-28T02:31:00.407 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.3519e-05 2023-08-28T02:31:00.407 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.407 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.407 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:31:00.407 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:41.089429+0000", 2023-08-28T02:31:00.407 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.1829999999999994e-06 2023-08-28T02:31:00.408 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.408 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.408 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:31:00.408 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:41.089437+0000", 2023-08-28T02:31:00.408 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.1070999999999999e-05 2023-08-28T02:31:00.408 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.408 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.408 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:31:00.408 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:41.089498+0000", 2023-08-28T02:31:00.408 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00013938500000000001 2023-08-28T02:31:00.408 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.408 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.408 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:31:00.409 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:41.089637+0000", 2023-08-28T02:31:00.409 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00015226500000000001 2023-08-28T02:31:00.409 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.409 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.409 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:31:00.409 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:41.089790+0000", 2023-08-28T02:31:00.409 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0050154489999999999 2023-08-28T02:31:00.409 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.409 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.409 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:31:00.409 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:41.094805+0000", 2023-08-28T02:31:00.409 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0018365149999999999 2023-08-28T02:31:00.409 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.409 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.410 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:00.410 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:41.096641+0000", 2023-08-28T02:31:00.410 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.18838258699999999 2023-08-28T02:31:00.410 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.410 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.410 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:00.410 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:41.285024+0000", 2023-08-28T02:31:00.410 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.0708999999999999e-05 2023-08-28T02:31:00.410 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.410 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.410 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:31:00.410 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:41.285065+0000", 2023-08-28T02:31:00.410 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.2301999999999997e-05 2023-08-28T02:31:00.411 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.411 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.411 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:31:00.411 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:41.285097+0000", 2023-08-28T02:31:00.411 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.19570041199999999 2023-08-28T02:31:00.411 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:00.411 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:31:00.411 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:00.411 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.411 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.411 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:186442 5.2 5:4bb621cb:::benchmark_data_smithi143_236200_object76299:head [delete] snapc 0=[] ondisk+write+known_if_redirected e177)", 2023-08-28T02:31:00.411 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:25:43.085512+0000", 2023-08-28T02:31:00.411 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 317.24401613200001, 2023-08-28T02:31:00.412 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.17261642299999999, 2023-08-28T02:31:00.412 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:31:00.412 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:00.412 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:31:00.412 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:31:00.412 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:31:00.412 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 186442 2023-08-28T02:31:00.412 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.412 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:31:00.412 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.412 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:31:00.412 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:43.085512+0000", 2023-08-28T02:31:00.413 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:31:00.413 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.413 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.413 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:31:00.413 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:43.085512+0000", 2023-08-28T02:31:00.413 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:31:00.413 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.413 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.413 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:31:00.413 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:43.085510+0000", 2023-08-28T02:31:00.413 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.49e-06 2023-08-28T02:31:00.413 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.413 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.414 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:31:00.414 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:43.085516+0000", 2023-08-28T02:31:00.414 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.9579999999999996e-06 2023-08-28T02:31:00.414 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.414 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.414 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:31:00.414 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:43.085524+0000", 2023-08-28T02:31:00.414 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.9740000000000001e-06 2023-08-28T02:31:00.414 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.414 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.414 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:31:00.414 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:43.085529+0000", 2023-08-28T02:31:00.414 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.1567999999999998e-05 2023-08-28T02:31:00.415 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.415 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.415 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:31:00.415 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:43.085581+0000", 2023-08-28T02:31:00.415 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00013925100000000001 2023-08-28T02:31:00.415 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.415 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.415 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:31:00.415 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:43.085720+0000", 2023-08-28T02:31:00.415 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00015138400000000001 2023-08-28T02:31:00.415 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.415 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.415 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:31:00.415 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:43.085872+0000", 2023-08-28T02:31:00.416 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0049987210000000002 2023-08-28T02:31:00.416 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.416 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.416 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:31:00.416 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:43.090870+0000", 2023-08-28T02:31:00.416 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0017789850000000001 2023-08-28T02:31:00.416 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.416 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.416 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:00.416 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:43.092649+0000", 2023-08-28T02:31:00.416 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.16538924199999999 2023-08-28T02:31:00.416 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.417 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.417 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:00.417 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:43.258039+0000", 2023-08-28T02:31:00.417 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.9842999999999997e-05 2023-08-28T02:31:00.417 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.417 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.417 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:31:00.417 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:43.258088+0000", 2023-08-28T02:31:00.417 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.9607000000000002e-05 2023-08-28T02:31:00.417 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.417 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.417 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:31:00.418 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:43.258128+0000", 2023-08-28T02:31:00.418 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.17261642299999999 2023-08-28T02:31:00.418 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:00.418 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:31:00.418 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:00.418 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.418 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.418 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:196497 5.2 5:44cacbbc:::benchmark_data_smithi143_236200_object86354:head [delete] snapc 0=[] ondisk+write+known_if_redirected e179)", 2023-08-28T02:31:00.418 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:25:58.988571+0000", 2023-08-28T02:31:00.418 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 301.34095660000003, 2023-08-28T02:31:00.418 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.20486649800000001, 2023-08-28T02:31:00.418 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:31:00.418 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:00.419 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:31:00.419 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:31:00.419 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:31:00.419 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 196497 2023-08-28T02:31:00.419 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.419 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:31:00.419 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.419 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:31:00.419 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:58.988571+0000", 2023-08-28T02:31:00.419 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:31:00.419 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.419 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.420 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:31:00.420 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:58.988571+0000", 2023-08-28T02:31:00.420 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:31:00.420 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.420 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.420 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:31:00.420 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:58.988569+0000", 2023-08-28T02:31:00.420 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.5660000000000007e-06 2023-08-28T02:31:00.420 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.420 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.420 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:31:00.420 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:58.988578+0000", 2023-08-28T02:31:00.420 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.4008e-05 2023-08-28T02:31:00.421 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.421 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.421 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:31:00.421 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:58.988592+0000", 2023-08-28T02:31:00.421 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.4139999999999997e-06 2023-08-28T02:31:00.421 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.421 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.421 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:31:00.421 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:58.988599+0000", 2023-08-28T02:31:00.421 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.4236e-05 2023-08-28T02:31:00.421 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.421 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.421 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:31:00.422 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:58.988653+0000", 2023-08-28T02:31:00.422 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012342000000000001 2023-08-28T02:31:00.422 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.422 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.422 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:31:00.422 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:58.988777+0000", 2023-08-28T02:31:00.422 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00014519000000000001 2023-08-28T02:31:00.422 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.422 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.422 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:31:00.422 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:58.988922+0000", 2023-08-28T02:31:00.422 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0047343890000000003 2023-08-28T02:31:00.422 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.423 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.423 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:31:00.423 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:58.993656+0000", 2023-08-28T02:31:00.423 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.001728172 2023-08-28T02:31:00.423 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.423 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.423 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:00.423 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:58.995385+0000", 2023-08-28T02:31:00.423 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.19794052200000001 2023-08-28T02:31:00.423 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.423 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.423 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:00.424 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:59.193325+0000", 2023-08-28T02:31:00.424 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.4077000000000003e-05 2023-08-28T02:31:00.424 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.424 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.424 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:31:00.424 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:59.193389+0000", 2023-08-28T02:31:00.424 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.8399000000000002e-05 2023-08-28T02:31:00.424 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.424 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.424 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:31:00.424 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:59.193438+0000", 2023-08-28T02:31:00.424 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.20486649800000001 2023-08-28T02:31:00.424 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:00.425 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:31:00.425 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:00.425 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.425 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.425 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:196505 5.2 5:4d9fb076:::benchmark_data_smithi143_236200_object86362:head [delete] snapc 0=[] ondisk+write+known_if_redirected e179)", 2023-08-28T02:31:00.425 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:25:59.000625+0000", 2023-08-28T02:31:00.425 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 301.32890282800003, 2023-08-28T02:31:00.425 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.20895017799999999, 2023-08-28T02:31:00.425 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:31:00.425 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:00.425 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:31:00.425 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:31:00.426 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:31:00.426 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 196505 2023-08-28T02:31:00.426 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.426 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:31:00.426 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.426 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:31:00.426 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:59.000625+0000", 2023-08-28T02:31:00.426 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:31:00.426 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.426 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.426 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:31:00.426 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:59.000625+0000", 2023-08-28T02:31:00.426 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999976 2023-08-28T02:31:00.427 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.427 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.427 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:31:00.427 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:59.000623+0000", 2023-08-28T02:31:00.427 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.3212e-05 2023-08-28T02:31:00.427 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.427 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.427 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:31:00.427 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:59.000636+0000", 2023-08-28T02:31:00.427 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.3873999999999999e-05 2023-08-28T02:31:00.427 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.427 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.428 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:31:00.428 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:59.000660+0000", 2023-08-28T02:31:00.428 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.8859999999999993e-06 2023-08-28T02:31:00.428 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.428 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.428 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:31:00.428 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:59.000669+0000", 2023-08-28T02:31:00.428 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.3915e-05 2023-08-28T02:31:00.428 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.428 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.428 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:31:00.428 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:59.000732+0000", 2023-08-28T02:31:00.429 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00015400799999999999 2023-08-28T02:31:00.429 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.429 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.429 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:31:00.429 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:59.000886+0000", 2023-08-28T02:31:00.429 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000165215 2023-08-28T02:31:00.429 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.429 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.429 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:31:00.429 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:59.001052+0000", 2023-08-28T02:31:00.429 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0052043979999999998 2023-08-28T02:31:00.429 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.429 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.430 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:31:00.430 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:59.006256+0000", 2023-08-28T02:31:00.430 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0061733990000000004 2023-08-28T02:31:00.430 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.430 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.430 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:00.430 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:59.012430+0000", 2023-08-28T02:31:00.430 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.197043986 2023-08-28T02:31:00.430 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.430 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.430 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:00.430 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:59.209473+0000", 2023-08-28T02:31:00.430 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.0241999999999997e-05 2023-08-28T02:31:00.431 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.431 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.431 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:31:00.431 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:59.209534+0000", 2023-08-28T02:31:00.431 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.1372999999999998e-05 2023-08-28T02:31:00.431 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.431 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.431 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:31:00.431 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:59.209575+0000", 2023-08-28T02:31:00.431 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.20895017799999999 2023-08-28T02:31:00.431 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:00.431 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:31:00.431 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:00.431 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.432 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.432 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:196896 5.2 5:4bd3324b:::benchmark_data_smithi143_236200_object86753:head [delete] snapc 0=[] ondisk+write+known_if_redirected e179)", 2023-08-28T02:31:00.432 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:25:59.793465+0000", 2023-08-28T02:31:00.432 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 300.53606228199999, 2023-08-28T02:31:00.432 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.18284690200000001, 2023-08-28T02:31:00.432 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:31:00.432 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:00.432 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:31:00.432 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:31:00.432 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:31:00.432 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 196896 2023-08-28T02:31:00.432 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.432 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:31:00.433 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.433 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:31:00.433 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:59.793465+0000", 2023-08-28T02:31:00.433 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:31:00.433 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.433 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.433 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:31:00.433 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:59.793465+0000", 2023-08-28T02:31:00.433 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999976 2023-08-28T02:31:00.433 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.433 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.433 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:31:00.433 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:59.793463+0000", 2023-08-28T02:31:00.434 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0786e-05 2023-08-28T02:31:00.434 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.434 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.434 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:31:00.434 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:59.793474+0000", 2023-08-28T02:31:00.434 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.9519000000000001e-05 2023-08-28T02:31:00.434 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.434 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.434 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:31:00.434 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:59.793493+0000", 2023-08-28T02:31:00.434 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.1210000000000007e-06 2023-08-28T02:31:00.434 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.434 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.435 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:31:00.435 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:59.793502+0000", 2023-08-28T02:31:00.435 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.6749000000000002e-05 2023-08-28T02:31:00.435 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.435 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.435 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:31:00.435 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:59.793558+0000", 2023-08-28T02:31:00.435 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00014300700000000001 2023-08-28T02:31:00.435 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.435 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.435 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:31:00.435 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:59.793701+0000", 2023-08-28T02:31:00.436 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000160457 2023-08-28T02:31:00.436 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.436 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.436 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:31:00.436 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:59.793862+0000", 2023-08-28T02:31:00.436 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0038900430000000001 2023-08-28T02:31:00.436 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.436 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.436 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:31:00.436 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:59.797752+0000", 2023-08-28T02:31:00.436 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0090678059999999994 2023-08-28T02:31:00.436 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.436 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.437 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:00.437 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:59.806820+0000", 2023-08-28T02:31:00.437 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.169396978 2023-08-28T02:31:00.437 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.437 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.437 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:00.437 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:59.976217+0000", 2023-08-28T02:31:00.437 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.5300999999999998e-05 2023-08-28T02:31:00.437 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.437 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.437 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:31:00.437 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:59.976272+0000", 2023-08-28T02:31:00.437 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.0491000000000002e-05 2023-08-28T02:31:00.438 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.438 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.438 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:31:00.438 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:25:59.976312+0000", 2023-08-28T02:31:00.438 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.18284690200000001 2023-08-28T02:31:00.438 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:00.438 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:31:00.438 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:00.438 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.438 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.438 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.44594.0:198273 5.2 5:4338f4ac:::benchmark_data_smithi143_236200_object88130:head [delete] snapc 0=[] ondisk+write+known_if_redirected e180)", 2023-08-28T02:31:00.438 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:26:02.080483+0000", 2023-08-28T02:31:00.439 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 298.24904523100003, 2023-08-28T02:31:00.439 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.16844342000000001, 2023-08-28T02:31:00.439 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:31:00.439 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:00.439 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:31:00.439 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.44594", 2023-08-28T02:31:00.439 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:31:00.439 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 198273 2023-08-28T02:31:00.439 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.439 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:31:00.439 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.439 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:31:00.439 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:02.080483+0000", 2023-08-28T02:31:00.440 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:31:00.440 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.440 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.440 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:31:00.440 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:02.080483+0000", 2023-08-28T02:31:00.440 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:31:00.440 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.440 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.440 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:31:00.440 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:02.080481+0000", 2023-08-28T02:31:00.440 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.6400000000000003e-06 2023-08-28T02:31:00.440 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.440 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.441 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:31:00.441 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:02.080489+0000", 2023-08-28T02:31:00.441 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.8048999999999999e-05 2023-08-28T02:31:00.441 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.441 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.441 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:31:00.441 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:02.080508+0000", 2023-08-28T02:31:00.441 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.6599999999999995e-06 2023-08-28T02:31:00.441 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.441 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.441 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:31:00.441 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:02.080515+0000", 2023-08-28T02:31:00.441 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.8171000000000003e-05 2023-08-28T02:31:00.442 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.442 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.442 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:31:00.442 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:02.080573+0000", 2023-08-28T02:31:00.442 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00014208499999999999 2023-08-28T02:31:00.442 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.442 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.442 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:31:00.442 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:02.080715+0000", 2023-08-28T02:31:00.442 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00015959700000000001 2023-08-28T02:31:00.442 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.442 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.442 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:31:00.443 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:02.080875+0000", 2023-08-28T02:31:00.443 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0049824020000000004 2023-08-28T02:31:00.443 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.443 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.443 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:31:00.443 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:02.085857+0000", 2023-08-28T02:31:00.443 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0031281360000000001 2023-08-28T02:31:00.443 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.443 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.443 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:00.443 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:02.088986+0000", 2023-08-28T02:31:00.443 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.15985400299999999 2023-08-28T02:31:00.443 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.444 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.444 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:00.444 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:02.248840+0000", 2023-08-28T02:31:00.444 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.2586000000000002e-05 2023-08-28T02:31:00.444 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.444 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.444 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:31:00.444 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:02.248882+0000", 2023-08-28T02:31:00.444 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.3770999999999998e-05 2023-08-28T02:31:00.444 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.444 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.444 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:31:00.444 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:26:02.248926+0000", 2023-08-28T02:31:00.445 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.16844342000000001 2023-08-28T02:31:00.445 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:00.445 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:31:00.445 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:00.445 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.445 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.445 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:78148 6.a 6.cb50d08a (undecoded) ondisk+retry+write+known_if_redirected e220)", 2023-08-28T02:31:00.445 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:28:24.333420+0000", 2023-08-28T02:31:00.445 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 155.996107881, 2023-08-28T02:31:00.445 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0005001710000001, 2023-08-28T02:31:00.445 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:31:00.445 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "reached pg", 2023-08-28T02:31:00.445 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:31:00.446 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:31:00.446 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:31:00.446 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 78148 2023-08-28T02:31:00.446 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.446 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:31:00.446 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.446 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:31:00.446 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.333420+0000", 2023-08-28T02:31:00.446 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:31:00.446 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.446 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.446 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:31:00.446 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.333420+0000", 2023-08-28T02:31:00.447 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:31:00.447 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.447 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.447 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:31:00.447 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.333418+0000", 2023-08-28T02:31:00.447 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00024257 2023-08-28T02:31:00.447 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.447 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.447 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:31:00.447 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.333661+0000", 2023-08-28T02:31:00.447 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2506e-05 2023-08-28T02:31:00.447 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.447 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.448 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:31:00.448 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.333673+0000", 2023-08-28T02:31:00.448 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.4730000000000003e-06 2023-08-28T02:31:00.448 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.448 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.448 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:31:00.448 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.333681+0000", 2023-08-28T02:31:00.448 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.0689999999999997e-05 2023-08-28T02:31:00.448 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.448 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.448 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:31:00.448 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.333732+0000", 2023-08-28T02:31:00.448 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.7439999999999996e-06 2023-08-28T02:31:00.449 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.449 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.449 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:31:00.449 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.333740+0000", 2023-08-28T02:31:00.449 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0001706100000001 2023-08-28T02:31:00.449 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.449 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.449 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:31:00.449 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:25.333911+0000", 2023-08-28T02:31:00.449 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.1880000000000008e-06 2023-08-28T02:31:00.449 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.449 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.449 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:31:00.450 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:25.333920+0000", 2023-08-28T02:31:00.450 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0005001710000001 2023-08-28T02:31:00.450 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:00.450 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:31:00.450 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:00.450 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.450 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.450 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:78161 6.a 6.3abad5fa (undecoded) ondisk+write+known_if_redirected e220)", 2023-08-28T02:31:00.450 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:28:24.353303+0000", 2023-08-28T02:31:00.450 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 155.97622447800001, 2023-08-28T02:31:00.450 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.98063798199999996, 2023-08-28T02:31:00.450 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:31:00.450 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "reached pg", 2023-08-28T02:31:00.451 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:31:00.451 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:31:00.451 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:31:00.451 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 78161 2023-08-28T02:31:00.451 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.451 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:31:00.451 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.451 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:31:00.451 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.353303+0000", 2023-08-28T02:31:00.451 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:31:00.451 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.451 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.451 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:31:00.452 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.353303+0000", 2023-08-28T02:31:00.452 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:31:00.452 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.452 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.452 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:31:00.452 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.353302+0000", 2023-08-28T02:31:00.452 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.9099e-05 2023-08-28T02:31:00.452 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.452 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.452 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:31:00.452 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.353331+0000", 2023-08-28T02:31:00.452 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.3447e-05 2023-08-28T02:31:00.453 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.453 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.453 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:31:00.453 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.353345+0000", 2023-08-28T02:31:00.453 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.5980000000000002e-06 2023-08-28T02:31:00.453 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.453 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.453 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:31:00.453 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.353351+0000", 2023-08-28T02:31:00.453 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00030897800000000001 2023-08-28T02:31:00.453 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.453 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.453 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:31:00.454 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.353660+0000", 2023-08-28T02:31:00.454 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.3219999999999999e-06 2023-08-28T02:31:00.454 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.454 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.454 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:31:00.454 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.353670+0000", 2023-08-28T02:31:00.454 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.98026650500000001 2023-08-28T02:31:00.454 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.454 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.454 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:31:00.454 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:25.333936+0000", 2023-08-28T02:31:00.454 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.2039999999999996e-06 2023-08-28T02:31:00.454 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.454 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.455 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:31:00.455 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:25.333941+0000", 2023-08-28T02:31:00.455 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.98063798199999996 2023-08-28T02:31:00.455 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:00.455 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:31:00.455 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:00.455 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.455 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.455 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:78184 6.a 6.225625ba (undecoded) ondisk+write+known_if_redirected e220)", 2023-08-28T02:31:00.455 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:28:24.405003+0000", 2023-08-28T02:31:00.455 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 155.92452435800001, 2023-08-28T02:31:00.455 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.92895428300000005, 2023-08-28T02:31:00.455 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:31:00.456 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "reached pg", 2023-08-28T02:31:00.456 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:31:00.456 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:31:00.456 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:31:00.456 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 78184 2023-08-28T02:31:00.456 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.456 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:31:00.456 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.456 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:31:00.456 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.405003+0000", 2023-08-28T02:31:00.456 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:31:00.456 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.456 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.457 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:31:00.457 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.405003+0000", 2023-08-28T02:31:00.457 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999976 2023-08-28T02:31:00.457 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.457 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.457 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:31:00.457 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.405001+0000", 2023-08-28T02:31:00.457 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.2438999999999999e-05 2023-08-28T02:31:00.457 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.457 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.457 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:31:00.457 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.405033+0000", 2023-08-28T02:31:00.457 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.1441e-05 2023-08-28T02:31:00.458 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.458 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.458 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:31:00.458 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.405055+0000", 2023-08-28T02:31:00.458 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.2890000000000003e-06 2023-08-28T02:31:00.458 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.458 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.458 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:31:00.458 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.405064+0000", 2023-08-28T02:31:00.458 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.5662999999999998e-05 2023-08-28T02:31:00.458 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.458 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.458 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:31:00.459 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.405130+0000", 2023-08-28T02:31:00.459 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1328999999999999e-05 2023-08-28T02:31:00.459 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.459 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.459 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:31:00.459 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.405141+0000", 2023-08-28T02:31:00.459 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.92881176600000004 2023-08-28T02:31:00.459 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.459 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.459 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:31:00.459 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:25.333953+0000", 2023-08-28T02:31:00.459 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.9389999999999996e-06 2023-08-28T02:31:00.459 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:00.460 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:00.460 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:31:00.460 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:25.333958+0000", 2023-08-28T02:31:00.460 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.92895428300000005 2023-08-28T02:31:00.460 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:00.460 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:31:00.460 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:00.460 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:00.460 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:31:00.460 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:31:00.461 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:00 smithi169 ceph-mon[159415]: pgmap v639: 89 pgs: 2 creating+peering, 1 unknown, 86 active+clean; 4.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2023-08-28T02:31:00.461 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:00 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/4151844632' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-28T02:31:00.461 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:00 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-28T02:31:00.692 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:31:00.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:00.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:31:00.797 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:31:00.825 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:00.920 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:31:00.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:00.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:31:01.024 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:31:01.046 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.7 dump_ops_in_flight 2023-08-28T02:31:01.055 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:01.055 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:31:01.082 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:01 smithi143 ceph-mon[183059]: failed to encode map e264 with expected crc 2023-08-28T02:31:01.082 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:01 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-08-28T02:31:01.082 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:01 smithi143 ceph-mon[183059]: osdmap e264: 8 total, 8 up, 8 in 2023-08-28T02:31:01.082 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:01 smithi143 ceph-mon[189871]: failed to encode map e264 with expected crc 2023-08-28T02:31:01.082 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:01 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-08-28T02:31:01.082 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:01 smithi143 ceph-mon[189871]: osdmap e264: 8 total, 8 up, 8 in 2023-08-28T02:31:01.100 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:31:01.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:01.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:31:01.208 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:31:01.233 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:01.283 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:31:01.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:01.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:31:01.387 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:31:01.403 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:31:01.415 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:01.421 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:31:01.461 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:31:01.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:01.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:31:01.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:01 smithi169 ceph-mon[159415]: failed to encode map e264 with expected crc 2023-08-28T02:31:01.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:01 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-08-28T02:31:01.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:01 smithi169 ceph-mon[159415]: osdmap e264: 8 total, 8 up, 8 in 2023-08-28T02:31:01.565 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:31:01.590 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:01.637 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:31:01.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:01.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:31:01.742 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:31:01.754 DEBUG:teuthology.orchestra.run.smithi143:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2023-08-28T02:31:01.755 INFO:tasks.thrashosds.thrasher:Growing pool device_health_metrics 2023-08-28T02:31:01.755 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph pg dump --format=json 2023-08-28T02:31:01.757 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-08-28T02:31:01.757 INFO:teuthology.orchestra.run:waiting for 3000 2023-08-28T02:31:01.769 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:01.806 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:hints = 1 2023-08-28T02:31:01.806 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-08-28T02:31:01.807 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Object prefix: benchmark_data_smithi143_360738 2023-08-28T02:31:01.807 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-08-28T02:31:01.807 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 0 0 0 0 0 0 - 0 2023-08-28T02:31:01.922 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:31:01.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:01.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:31:02.027 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:31:02.065 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:02.070 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:31:01 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:31:01] "GET /metrics HTTP/1.1" 200 36411 "" "Prometheus/2.33.4" 2023-08-28T02:31:02.070 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:02 smithi143 ceph-mon[189871]: pgmap v641: 89 pgs: 2 creating+peering, 87 active+clean; 4.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-08-28T02:31:02.070 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:02 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1961204680' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:31:02.071 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:02 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:31:02.071 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:02 smithi143 ceph-mon[189871]: failed to encode map e264 with expected crc 2023-08-28T02:31:02.071 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:02 smithi143 ceph-mon[189871]: failed to encode map e264 with expected crc 2023-08-28T02:31:02.081 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is set 2023-08-28T02:31:02.109 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:31:02.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:02.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:31:02.212 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:31:02.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:02 smithi169 ceph-mon[159415]: pgmap v641: 89 pgs: 2 creating+peering, 87 active+clean; 4.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-08-28T02:31:02.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:02 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1961204680' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:31:02.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:02 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:31:02.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:02 smithi169 ceph-mon[159415]: failed to encode map e264 with expected crc 2023-08-28T02:31:02.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:02 smithi169 ceph-mon[159415]: failed to encode map e264 with expected crc 2023-08-28T02:31:02.239 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:02.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:02 smithi143 ceph-mon[183059]: pgmap v641: 89 pgs: 2 creating+peering, 87 active+clean; 4.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-08-28T02:31:02.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:02 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1961204680' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:31:02.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:02 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:31:02.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:02 smithi143 ceph-mon[183059]: failed to encode map e264 with expected crc 2023-08-28T02:31:02.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:02 smithi143 ceph-mon[183059]: failed to encode map e264 with expected crc 2023-08-28T02:31:02.549 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:31:02.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:02.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:31:02.659 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:31:02.712 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:02.808 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 1 15 2458 2443 152.486 152.688 0.00306478 0.00654032 2023-08-28T02:31:03.176 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:31:03.176 INFO:teuthology.orchestra.run.smithi169.stdout: "ops": [ 2023-08-28T02:31:03.176 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.177 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:3390 7.5 7:a4090d24:::benchmark_data_smithi143_360738_object3389:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e265)", 2023-08-28T02:31:03.177 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:31:03.174450+0000", 2023-08-28T02:31:03.177 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 0.0016708280000000001, 2023-08-28T02:31:03.177 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0017147460000000001, 2023-08-28T02:31:03.177 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:31:03.177 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "waiting for sub ops", 2023-08-28T02:31:03.177 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:31:03.177 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:31:03.177 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:31:03.177 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 3390 2023-08-28T02:31:03.177 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.177 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:31:03.178 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.178 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:31:03.178 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.174450+0000", 2023-08-28T02:31:03.178 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:31:03.178 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.178 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.178 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:31:03.178 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.174450+0000", 2023-08-28T02:31:03.178 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:31:03.178 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.178 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.178 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:31:03.179 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.174448+0000", 2023-08-28T02:31:03.179 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.7917000000000006e-05 2023-08-28T02:31:03.179 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.179 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.179 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:31:03.179 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.174526+0000", 2023-08-28T02:31:03.179 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.7404999999999999e-05 2023-08-28T02:31:03.179 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.179 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.179 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:31:03.179 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.174553+0000", 2023-08-28T02:31:03.180 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.9039999999999997e-06 2023-08-28T02:31:03.180 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.180 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.180 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:31:03.180 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.174559+0000", 2023-08-28T02:31:03.180 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.9788000000000003e-05 2023-08-28T02:31:03.180 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.180 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.180 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:31:03.180 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.174629+0000", 2023-08-28T02:31:03.180 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00021291800000000001 2023-08-28T02:31:03.180 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.181 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.181 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:31:03.181 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.174842+0000", 2023-08-28T02:31:03.181 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00013115300000000001 2023-08-28T02:31:03.181 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.181 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.181 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,2", 2023-08-28T02:31:03.181 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.174973+0000", 2023-08-28T02:31:03.181 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00052333999999999996 2023-08-28T02:31:03.181 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:03.181 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:31:03.181 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:03.182 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.182 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.182 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:3391 7.5 7:ae64ad70:::benchmark_data_smithi143_360738_object3390:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e265)", 2023-08-28T02:31:03.182 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:31:03.174587+0000", 2023-08-28T02:31:03.182 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 0.0015339780000000001, 2023-08-28T02:31:03.182 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.001673089, 2023-08-28T02:31:03.182 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:31:03.182 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "waiting for sub ops", 2023-08-28T02:31:03.182 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:31:03.182 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:31:03.182 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:31:03.182 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 3391 2023-08-28T02:31:03.183 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.183 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:31:03.183 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.183 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:31:03.183 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.174587+0000", 2023-08-28T02:31:03.183 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:31:03.183 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.183 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.183 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:31:03.183 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.174587+0000", 2023-08-28T02:31:03.183 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999995 2023-08-28T02:31:03.183 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.184 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.184 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:31:03.184 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.174586+0000", 2023-08-28T02:31:03.184 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.1638999999999999e-05 2023-08-28T02:31:03.184 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.184 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.184 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:31:03.184 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.174628+0000", 2023-08-28T02:31:03.184 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2458e-05 2023-08-28T02:31:03.184 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.184 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.184 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:31:03.184 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.174640+0000", 2023-08-28T02:31:03.185 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.5389999999999998e-06 2023-08-28T02:31:03.185 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.185 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.185 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:31:03.185 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.174646+0000", 2023-08-28T02:31:03.185 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00074360099999999998 2023-08-28T02:31:03.185 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.185 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.185 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:31:03.185 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.175389+0000", 2023-08-28T02:31:03.185 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00019656200000000001 2023-08-28T02:31:03.185 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.186 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.186 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:31:03.186 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.175586+0000", 2023-08-28T02:31:03.186 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00021993699999999999 2023-08-28T02:31:03.186 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.186 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.186 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,2", 2023-08-28T02:31:03.186 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.175806+0000", 2023-08-28T02:31:03.186 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.001219166 2023-08-28T02:31:03.186 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:03.187 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:31:03.187 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:03.187 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.187 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.187 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:3392 7.5 7.2ac60ae5 (undecoded) ondisk+write+known_if_redirected e265)", 2023-08-28T02:31:03.187 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:31:03.174653+0000", 2023-08-28T02:31:03.187 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 0.0014678860000000001, 2023-08-28T02:31:03.187 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.001689074, 2023-08-28T02:31:03.187 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:31:03.187 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "reached pg", 2023-08-28T02:31:03.187 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:31:03.188 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:31:03.188 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:31:03.188 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 3392 2023-08-28T02:31:03.188 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.188 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:31:03.188 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.188 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:31:03.188 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.174653+0000", 2023-08-28T02:31:03.188 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:31:03.188 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.188 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.188 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:31:03.189 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.174653+0000", 2023-08-28T02:31:03.189 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999995 2023-08-28T02:31:03.189 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.189 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.189 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:31:03.189 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.174652+0000", 2023-08-28T02:31:03.189 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.3178999999999998e-05 2023-08-28T02:31:03.189 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.189 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.189 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:31:03.189 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.174725+0000", 2023-08-28T02:31:03.189 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.2436e-05 2023-08-28T02:31:03.189 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.190 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.190 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:31:03.190 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.174748+0000", 2023-08-28T02:31:03.190 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.835e-06 2023-08-28T02:31:03.190 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.190 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.190 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:31:03.190 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.174753+0000", 2023-08-28T02:31:03.190 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0015835420000000001 2023-08-28T02:31:03.190 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.190 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.190 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:31:03.190 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.176337+0000", 2023-08-28T02:31:03.190 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.001684285 2023-08-28T02:31:03.191 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:03.191 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:31:03.191 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:03.191 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.191 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.191 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_repop(client.55270.0:3396 7.c e265/261 7:3fa412b5:::benchmark_data_smithi143_360738_object3395:head v 265'218, mlcod=265'216)", 2023-08-28T02:31:03.191 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:31:03.175748+0000", 2023-08-28T02:31:03.191 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 0.00037222699999999999, 2023-08-28T02:31:03.191 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00066380400000000004, 2023-08-28T02:31:03.191 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:31:03.191 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "started", 2023-08-28T02:31:03.191 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:31:03.192 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.192 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:31:03.192 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.175748+0000", 2023-08-28T02:31:03.192 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:31:03.192 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.192 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.192 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:31:03.192 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.175748+0000", 2023-08-28T02:31:03.192 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:31:03.192 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.192 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.192 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:31:03.193 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.175747+0000", 2023-08-28T02:31:03.193 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.6958000000000001e-05 2023-08-28T02:31:03.193 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.193 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.193 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:31:03.193 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.175784+0000", 2023-08-28T02:31:03.193 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.5670000000000001e-06 2023-08-28T02:31:03.193 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.193 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.193 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:31:03.193 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.175794+0000", 2023-08-28T02:31:03.193 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.5839999999999996e-06 2023-08-28T02:31:03.193 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.194 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.194 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:31:03.194 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.175798+0000", 2023-08-28T02:31:03.194 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.4323999999999997e-05 2023-08-28T02:31:03.194 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.194 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.194 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:31:03.194 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.175853+0000", 2023-08-28T02:31:03.194 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.9283000000000002e-05 2023-08-28T02:31:03.194 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.194 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.194 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:31:03.194 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.175872+0000", 2023-08-28T02:31:03.195 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012369400000000001 2023-08-28T02:31:03.195 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:03.195 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:31:03.195 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:03.195 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.195 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.195 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_repop(client.55270.0:3393 7.6 e265/261 7:6e152bef:::benchmark_data_smithi143_360738_object3392:head v 265'209, mlcod=265'207)", 2023-08-28T02:31:03.195 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:31:03.175770+0000", 2023-08-28T02:31:03.195 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 0.00035085500000000001, 2023-08-28T02:31:03.195 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00070145099999999996, 2023-08-28T02:31:03.195 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:31:03.195 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "started", 2023-08-28T02:31:03.196 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:31:03.196 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.196 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:31:03.196 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.175770+0000", 2023-08-28T02:31:03.196 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:31:03.196 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.196 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.196 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:31:03.196 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.175770+0000", 2023-08-28T02:31:03.196 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:31:03.196 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.196 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.196 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:31:03.197 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.175769+0000", 2023-08-28T02:31:03.197 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.8902000000000001e-05 2023-08-28T02:31:03.197 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.197 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.197 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:31:03.197 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.175808+0000", 2023-08-28T02:31:03.197 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0937999999999999e-05 2023-08-28T02:31:03.197 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.197 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.197 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:31:03.197 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.175819+0000", 2023-08-28T02:31:03.197 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.2089999999999999e-06 2023-08-28T02:31:03.197 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.198 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.198 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:31:03.198 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.175826+0000", 2023-08-28T02:31:03.198 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012716800000000001 2023-08-28T02:31:03.198 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.198 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.198 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:31:03.198 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.175953+0000", 2023-08-28T02:31:03.198 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.917e-05 2023-08-28T02:31:03.198 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.198 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.198 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:31:03.199 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.175982+0000", 2023-08-28T02:31:03.199 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00021263200000000001 2023-08-28T02:31:03.199 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:03.199 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:31:03.199 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:03.199 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.199 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.199 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_repop(client.55270.0:3395 7.6 e265/261 7:6abd120d:::benchmark_data_smithi143_360738_object3394:head v 265'210, mlcod=265'207)", 2023-08-28T02:31:03.199 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:31:03.175997+0000", 2023-08-28T02:31:03.199 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 0.000123256, 2023-08-28T02:31:03.199 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00052576399999999996, 2023-08-28T02:31:03.199 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:31:03.199 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "started", 2023-08-28T02:31:03.200 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:31:03.200 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.200 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:31:03.200 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.175997+0000", 2023-08-28T02:31:03.200 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:31:03.200 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.200 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.200 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:31:03.200 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.175997+0000", 2023-08-28T02:31:03.200 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:31:03.200 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.200 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.201 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:31:03.201 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.175997+0000", 2023-08-28T02:31:03.201 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000152089 2023-08-28T02:31:03.201 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.201 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.201 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:31:03.201 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.176149+0000", 2023-08-28T02:31:03.201 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.1634000000000002e-05 2023-08-28T02:31:03.201 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.201 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.201 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:31:03.201 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.176180+0000", 2023-08-28T02:31:03.202 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.8710000000000002e-06 2023-08-28T02:31:03.202 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.202 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.202 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:31:03.202 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.176186+0000", 2023-08-28T02:31:03.202 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000180519 2023-08-28T02:31:03.202 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.202 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.202 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:31:03.202 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.176367+0000", 2023-08-28T02:31:03.202 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.0275000000000002e-05 2023-08-28T02:31:03.202 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.202 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.203 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:31:03.203 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.176387+0000", 2023-08-28T02:31:03.203 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00038965000000000001 2023-08-28T02:31:03.203 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:03.203 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:31:03.203 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:03.203 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.203 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.203 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_repop_reply(client.55270.0:3390 7.5 e265/261)", 2023-08-28T02:31:03.203 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:31:03.176275+0000", 2023-08-28T02:31:03.203 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 4294967295.9998455, 2023-08-28T02:31:03.203 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00032661000000000002, 2023-08-28T02:31:03.204 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:31:03.204 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "queued for pg", 2023-08-28T02:31:03.204 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:31:03.204 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.204 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:31:03.204 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.176275+0000", 2023-08-28T02:31:03.204 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:31:03.204 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.204 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.204 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:31:03.204 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.176275+0000", 2023-08-28T02:31:03.204 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999995 2023-08-28T02:31:03.205 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.205 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.205 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:31:03.205 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.176274+0000", 2023-08-28T02:31:03.205 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.3979999999999998e-06 2023-08-28T02:31:03.205 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.205 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.205 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:31:03.205 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.176280+0000", 2023-08-28T02:31:03.205 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.2730000000000008e-06 2023-08-28T02:31:03.205 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.205 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.205 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:31:03.206 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.176289+0000", 2023-08-28T02:31:03.206 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.8100000000000003e-06 2023-08-28T02:31:03.206 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:31:03.206 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:31:03.206 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:31:03.206 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:03.176295+0000", 2023-08-28T02:31:03.206 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.9789e-05 2023-08-28T02:31:03.206 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:03.206 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:31:03.206 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:03.206 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:31:03.206 INFO:teuthology.orchestra.run.smithi169.stdout: ], 2023-08-28T02:31:03.206 INFO:teuthology.orchestra.run.smithi169.stdout: "num_ops": 7 2023-08-28T02:31:03.207 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:31:03.207 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:03 smithi169 ceph-mon[159415]: failed to encode map e264 with expected crc 2023-08-28T02:31:03.207 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:03 smithi169 ceph-mon[159415]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:31:03.207 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:03 smithi169 ceph-mon[159415]: failed to encode map e265 with expected crc 2023-08-28T02:31:03.207 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:03 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:31:03.207 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:03 smithi169 ceph-mon[159415]: osdmap e265: 8 total, 8 up, 8 in 2023-08-28T02:31:03.207 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:03 smithi169 ceph-mon[159415]: failed to encode map e265 with expected crc 2023-08-28T02:31:03.208 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:03 smithi169 ceph-mon[159415]: failed to encode map e265 with expected crc 2023-08-28T02:31:03.208 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:03 smithi169 ceph-mon[159415]: failed to encode map e265 with expected crc 2023-08-28T02:31:03.208 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:03 smithi169 ceph-mon[159415]: failed to encode map e265 with expected crc 2023-08-28T02:31:03.208 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:03 smithi169 ceph-mon[159415]: failed to encode map e265 with expected crc 2023-08-28T02:31:03.208 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:03 smithi169 ceph-mon[159415]: failed to encode map e265 with expected crc 2023-08-28T02:31:03.364 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:31:03.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:03.370 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:31:03.371 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:03 smithi143 ceph-mon[183059]: failed to encode map e264 with expected crc 2023-08-28T02:31:03.371 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:03 smithi143 ceph-mon[183059]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:31:03.371 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:03 smithi143 ceph-mon[183059]: failed to encode map e265 with expected crc 2023-08-28T02:31:03.371 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:03 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:31:03.371 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:03 smithi143 ceph-mon[183059]: osdmap e265: 8 total, 8 up, 8 in 2023-08-28T02:31:03.371 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:03 smithi143 ceph-mon[183059]: failed to encode map e265 with expected crc 2023-08-28T02:31:03.371 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:03 smithi143 ceph-mon[183059]: failed to encode map e265 with expected crc 2023-08-28T02:31:03.371 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:03 smithi143 ceph-mon[183059]: failed to encode map e265 with expected crc 2023-08-28T02:31:03.371 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:03 smithi143 ceph-mon[183059]: failed to encode map e265 with expected crc 2023-08-28T02:31:03.372 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:03 smithi143 ceph-mon[183059]: failed to encode map e265 with expected crc 2023-08-28T02:31:03.372 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:03 smithi143 ceph-mon[183059]: failed to encode map e265 with expected crc 2023-08-28T02:31:03.372 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:03 smithi143 ceph-mon[189871]: failed to encode map e264 with expected crc 2023-08-28T02:31:03.372 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:03 smithi143 ceph-mon[189871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:31:03.372 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:03 smithi143 ceph-mon[189871]: failed to encode map e265 with expected crc 2023-08-28T02:31:03.372 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:03 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:31:03.372 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:03 smithi143 ceph-mon[189871]: osdmap e265: 8 total, 8 up, 8 in 2023-08-28T02:31:03.372 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:03 smithi143 ceph-mon[189871]: failed to encode map e265 with expected crc 2023-08-28T02:31:03.372 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:03 smithi143 ceph-mon[189871]: failed to encode map e265 with expected crc 2023-08-28T02:31:03.372 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:03 smithi143 ceph-mon[189871]: failed to encode map e265 with expected crc 2023-08-28T02:31:03.372 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:03 smithi143 ceph-mon[189871]: failed to encode map e265 with expected crc 2023-08-28T02:31:03.373 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:03 smithi143 ceph-mon[189871]: failed to encode map e265 with expected crc 2023-08-28T02:31:03.373 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:03 smithi143 ceph-mon[189871]: failed to encode map e265 with expected crc 2023-08-28T02:31:03.470 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:31:03.500 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:03.553 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:31:03.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:03.557 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:31:03.658 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:31:03.719 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:03.812 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 2 16 4860 4844 150.97 150.062 0.00558414 0.00660993 2023-08-28T02:31:03.847 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:31:03.879 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:31:03.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:03.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:31:03.985 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:31:04.022 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:04.288 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:31:04.291 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:04 smithi169 ceph-mon[159415]: pgmap v643: 89 pgs: 89 active+clean; 17 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2.1 MiB/s wr, 34 op/s 2023-08-28T02:31:04.292 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.7 dump_blocked_ops 2023-08-28T02:31:04.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:04.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:31:04.353 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:04 smithi143 ceph-mon[183059]: pgmap v643: 89 pgs: 89 active+clean; 17 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2.1 MiB/s wr, 34 op/s 2023-08-28T02:31:04.353 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:04 smithi143 ceph-mon[189871]: pgmap v643: 89 pgs: 89 active+clean; 17 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2.1 MiB/s wr, 34 op/s 2023-08-28T02:31:04.395 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:31:04.437 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:31:04.469 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:04.477 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:31:04.515 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:31:04.554 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:31:04.598 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:31:04.623 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:31:04.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:04.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:31:04.637 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:31:04.678 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:31:04.714 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:31:04.722 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:31:04.722 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:31:04.722 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:31:04.722 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:31:04.722 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:31:04.722 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:31:04.722 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:31:04.722 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:31:04.728 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:31:04.770 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:04.812 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 3 16 7501 7485 155.655 165.062 0.0106095 0.00640911 2023-08-28T02:31:04.836 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:31:04.893 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:31:04.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:04.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:31:05.002 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:31:05.045 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:05.134 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:31:05.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:05.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:31:05.237 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:31:05.296 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:05.303 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:05 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/521115475' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:31:05.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:05 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/521115475' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:31:05.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:05 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/521115475' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:31:05.753 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset noscrub 2023-08-28T02:31:05.812 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 4 16 10444 10428 162.713 183.938 0.00700081 0.00613815 2023-08-28T02:31:06.010 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:31:06.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:06.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:31:06.119 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:31:06.168 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:06.231 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:31:06.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:06.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:31:06.336 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:31:06.368 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:06 smithi143 ceph-mon[183059]: pgmap v644: 89 pgs: 89 active+clean; 244 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 33 MiB/s wr, 527 op/s 2023-08-28T02:31:06.369 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:06 smithi143 ceph-mon[189871]: pgmap v644: 89 pgs: 89 active+clean; 244 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 33 MiB/s wr, 527 op/s 2023-08-28T02:31:06.371 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:06.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:06 smithi169 ceph-mon[159415]: pgmap v644: 89 pgs: 89 active+clean; 244 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 33 MiB/s wr, 527 op/s 2023-08-28T02:31:06.813 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 5 16 13075 13059 163.042 164.438 0.00290547 0.0061266 2023-08-28T02:31:07.365 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:31:07.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:07.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:31:07.387 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:31:07.388 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:31:07.471 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:31:07.510 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:07.552 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:31:07.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:07.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:31:07.655 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:31:07.682 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:07 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:31:07.686 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:07.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:07 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:31:07.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:07 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:31:07.732 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:31:07.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:07.734 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:31:07.814 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 6 16 14997 14981 155.865 120.125 0.00576207 0.00640991 2023-08-28T02:31:07.836 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:31:07.882 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:07.953 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:31:07.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:07.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:31:08.058 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:31:08.084 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:08.192 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:31:08.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:08.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:31:08.283 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:31:08.299 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:31:08.368 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:08.465 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:31:08.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:08.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:31:08.568 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.7 dump_historic_ops 2023-08-28T02:31:08.574 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:31:08.642 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:08 smithi143 ceph-mon[189871]: pgmap v645: 89 pgs: 89 active+clean; 546 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-08-28T02:31:08.644 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:08 smithi143 ceph-mon[183059]: pgmap v645: 89 pgs: 89 active+clean; 546 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-08-28T02:31:08.649 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:08.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:08 smithi169 ceph-mon[159415]: pgmap v645: 89 pgs: 89 active+clean; 546 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-08-28T02:31:08.759 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:31:08.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:08.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:31:08.815 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 7 16 16685 16669 148.649 105.5 0.0134064 0.0067202 2023-08-28T02:31:08.867 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:31:08.925 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:09.021 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:31:09.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:09.025 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:31:09.056 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:31:09.126 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:31:09.157 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:31:09.190 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:09.190 INFO:teuthology.orchestra.run.smithi143.stderr:dumped all 2023-08-28T02:31:09.274 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:31:09.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:09.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:31:09.380 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:31:09.409 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:09.817 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 8 16 18607 18591 145.051 120.125 0.00703624 0.00688789 2023-08-28T02:31:10.188 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:31:10.696 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:10 smithi169 ceph-mon[159415]: from='client.45362 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:31:10.696 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:10 smithi169 ceph-mon[159415]: pgmap v646: 89 pgs: 89 active+clean; 959 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-08-28T02:31:10.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:10 smithi143 ceph-mon[183059]: from='client.45362 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:31:10.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:10 smithi143 ceph-mon[183059]: pgmap v646: 89 pgs: 89 active+clean; 959 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-08-28T02:31:10.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:10 smithi143 ceph-mon[189871]: from='client.45362 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:31:10.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:10 smithi143 ceph-mon[189871]: pgmap v646: 89 pgs: 89 active+clean; 959 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-08-28T02:31:10.819 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 9 16 19819 19803 137.334 75.75 0.00353299 0.00727528 2023-08-28T02:31:11.162 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:31:11.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:11.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:31:11.212 INFO:teuthology.orchestra.run.smithi143.stdout:{"pg_ready":true,"pg_map":{"version":645,"stamp":"2023-08-28T02:31:07.325476+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":572004275,"num_objects":8683,"num_object_clones":0,"num_object_copies":26049,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8683,"num_whiteouts":0,"num_read":6274,"num_read_kb":13707,"num_write":9761,"num_write_kb":569061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":189,"num_bytes_recovered":14963785,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":9825,"ondisk_log_size":9825,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":276,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":4588556,"kb_used_data":2152012,"kb_used_omap":29,"kb_used_meta":2436386,"kb_avail":745372660,"statfs":{"total":767960285184,"available":763261603840,"internally_reserved":0,"allocated":2203660288,"data_stored":2195995268,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30674,"internal_metadata":2494859310},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[6,5,5,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":567214080,"num_objects":8655,"num_object_clones":0,"num_object_copies":25965,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8655,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8655,"num_write_kb":553920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001928"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":266,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:01.456689+0000","last_change":"2023-08-28T02:27:31.367469+0000","last_active":"2023-08-28T02:31:01.456689+0000","last_peered":"2023-08-28T02:31:01.456689+0000","last_clean":"2023-08-28T02:31:01.456689+0000","last_became_active":"2023-08-28T02:27:31.366946+0000","last_became_peered":"2023-08-28T02:27:31.366946+0000","last_unstale":"2023-08-28T02:31:01.456689+0000","last_undegraded":"2023-08-28T02:31:01.456689+0000","last_fullsized":"2023-08-28T02:31:01.456689+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":151,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:00.252723+0000","last_change":"2023-08-28T02:27:31.324452+0000","last_active":"2023-08-28T02:31:00.252723+0000","last_peered":"2023-08-28T02:31:00.252723+0000","last_clean":"2023-08-28T02:31:00.252723+0000","last_became_active":"2023-08-28T02:27:31.323749+0000","last_became_peered":"2023-08-28T02:27:31.323749+0000","last_unstale":"2023-08-28T02:31:00.252723+0000","last_undegraded":"2023-08-28T02:31:00.252723+0000","last_fullsized":"2023-08-28T02:31:00.252723+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:18:10.904127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":72,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:00.064855+0000","last_change":"2023-08-28T02:27:31.337338+0000","last_active":"2023-08-28T02:31:00.064855+0000","last_peered":"2023-08-28T02:31:00.064855+0000","last_clean":"2023-08-28T02:31:00.064855+0000","last_became_active":"2023-08-28T02:27:31.337018+0000","last_became_peered":"2023-08-28T02:27:31.337018+0000","last_unstale":"2023-08-28T02:31:00.064855+0000","last_undegraded":"2023-08-28T02:31:00.064855+0000","last_fullsized":"2023-08-28T02:31:00.064855+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:46:34.845663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":253,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:00.064164+0000","last_change":"2023-08-28T02:23:59.223634+0000","last_active":"2023-08-28T02:31:00.064164+0000","last_peered":"2023-08-28T02:31:00.064164+0000","last_clean":"2023-08-28T02:31:00.064164+0000","last_became_active":"2023-08-28T02:23:59.222704+0000","last_became_peered":"2023-08-28T02:23:59.222704+0000","last_unstale":"2023-08-28T02:31:00.064164+0000","last_undegraded":"2023-08-28T02:31:00.064164+0000","last_fullsized":"2023-08-28T02:31:00.064164+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:51:52.752361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":142,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:00.252319+0000","last_change":"2023-08-28T02:27:31.318153+0000","last_active":"2023-08-28T02:31:00.252319+0000","last_peered":"2023-08-28T02:31:00.252319+0000","last_clean":"2023-08-28T02:31:00.252319+0000","last_became_active":"2023-08-28T02:27:31.317732+0000","last_became_peered":"2023-08-28T02:27:31.317732+0000","last_unstale":"2023-08-28T02:31:00.252319+0000","last_undegraded":"2023-08-28T02:31:00.252319+0000","last_fullsized":"2023-08-28T02:31:00.252319+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:06:13.421674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":54,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:01.457141+0000","last_change":"2023-08-28T02:28:25.348941+0000","last_active":"2023-08-28T02:31:01.457141+0000","last_peered":"2023-08-28T02:31:01.457141+0000","last_clean":"2023-08-28T02:31:01.457141+0000","last_became_active":"2023-08-28T02:28:25.348650+0000","last_became_peered":"2023-08-28T02:28:25.348650+0000","last_unstale":"2023-08-28T02:31:01.457141+0000","last_undegraded":"2023-08-28T02:31:01.457141+0000","last_fullsized":"2023-08-28T02:31:01.457141+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":233,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:00.064195+0000","last_change":"2023-08-28T02:23:59.227592+0000","last_active":"2023-08-28T02:31:00.064195+0000","last_peered":"2023-08-28T02:31:00.064195+0000","last_clean":"2023-08-28T02:31:00.064195+0000","last_became_active":"2023-08-28T02:23:59.227342+0000","last_became_peered":"2023-08-28T02:23:59.227342+0000","last_unstale":"2023-08-28T02:31:00.064195+0000","last_undegraded":"2023-08-28T02:31:00.064195+0000","last_fullsized":"2023-08-28T02:31:00.064195+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:08:19.874396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":161,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:00.252376+0000","last_change":"2023-08-28T02:28:25.348233+0000","last_active":"2023-08-28T02:31:00.252376+0000","last_peered":"2023-08-28T02:31:00.252376+0000","last_clean":"2023-08-28T02:31:00.252376+0000","last_became_active":"2023-08-28T02:28:25.347896+0000","last_became_peered":"2023-08-28T02:28:25.347896+0000","last_unstale":"2023-08-28T02:31:00.252376+0000","last_undegraded":"2023-08-28T02:31:00.252376+0000","last_fullsized":"2023-08-28T02:31:00.252376+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:50:17.674564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":267,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:00.067613+0000","last_change":"2023-08-28T02:27:31.328499+0000","last_active":"2023-08-28T02:31:00.067613+0000","last_peered":"2023-08-28T02:31:00.067613+0000","last_clean":"2023-08-28T02:31:00.067613+0000","last_became_active":"2023-08-28T02:27:31.328164+0000","last_became_peered":"2023-08-28T02:27:31.328164+0000","last_unstale":"2023-08-28T02:31:00.067613+0000","last_undegraded":"2023-08-28T02:31:00.067613+0000","last_fullsized":"2023-08-28T02:31:00.067613+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":143,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:00.252538+0000","last_change":"2023-08-28T02:28:25.348760+0000","last_active":"2023-08-28T02:31:00.252538+0000","last_peered":"2023-08-28T02:31:00.252538+0000","last_clean":"2023-08-28T02:31:00.252538+0000","last_became_active":"2023-08-28T02:28:25.348465+0000","last_became_peered":"2023-08-28T02:28:25.348465+0000","last_unstale":"2023-08-28T02:31:00.252538+0000","last_undegraded":"2023-08-28T02:31:00.252538+0000","last_fullsized":"2023-08-28T02:31:00.252538+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:43:05.659999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":150,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:01.456607+0000","last_change":"2023-08-28T02:23:59.316561+0000","last_active":"2023-08-28T02:31:01.456607+0000","last_peered":"2023-08-28T02:31:01.456607+0000","last_clean":"2023-08-28T02:31:01.456607+0000","last_became_active":"2023-08-28T02:23:59.227433+0000","last_became_peered":"2023-08-28T02:23:59.227433+0000","last_unstale":"2023-08-28T02:31:01.456607+0000","last_undegraded":"2023-08-28T02:31:01.456607+0000","last_fullsized":"2023-08-28T02:31:01.456607+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":82,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:00.065316+0000","last_change":"2023-08-28T02:28:25.347771+0000","last_active":"2023-08-28T02:31:00.065316+0000","last_peered":"2023-08-28T02:31:00.065316+0000","last_clean":"2023-08-28T02:31:00.065316+0000","last_became_active":"2023-08-28T02:28:25.347320+0000","last_became_peered":"2023-08-28T02:28:25.347320+0000","last_unstale":"2023-08-28T02:31:00.065316+0000","last_undegraded":"2023-08-28T02:31:00.065316+0000","last_fullsized":"2023-08-28T02:31:00.065316+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:20:27.350131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"86'22","reported_seq":315,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:01.457490+0000","last_change":"2023-08-28T02:28:25.361783+0000","last_active":"2023-08-28T02:31:01.457490+0000","last_peered":"2023-08-28T02:31:01.457490+0000","last_clean":"2023-08-28T02:31:01.457490+0000","last_became_active":"2023-08-28T02:28:25.351418+0000","last_became_peered":"2023-08-28T02:28:25.351418+0000","last_unstale":"2023-08-28T02:31:01.457490+0000","last_undegraded":"2023-08-28T02:31:01.457490+0000","last_fullsized":"2023-08-28T02:31:01.457490+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":281,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:01.456612+0000","last_change":"2023-08-28T02:28:25.352395+0000","last_active":"2023-08-28T02:31:01.456612+0000","last_peered":"2023-08-28T02:31:01.456612+0000","last_clean":"2023-08-28T02:31:01.456612+0000","last_became_active":"2023-08-28T02:28:25.352189+0000","last_became_peered":"2023-08-28T02:28:25.352189+0000","last_unstale":"2023-08-28T02:31:01.456612+0000","last_undegraded":"2023-08-28T02:31:01.456612+0000","last_fullsized":"2023-08-28T02:31:01.456612+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":135,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:01.456771+0000","last_change":"2023-08-28T02:23:38.983634+0000","last_active":"2023-08-28T02:31:01.456771+0000","last_peered":"2023-08-28T02:31:01.456771+0000","last_clean":"2023-08-28T02:31:01.456771+0000","last_became_active":"2023-08-28T02:23:38.344566+0000","last_became_peered":"2023-08-28T02:23:38.344566+0000","last_unstale":"2023-08-28T02:31:01.456771+0000","last_undegraded":"2023-08-28T02:31:01.456771+0000","last_fullsized":"2023-08-28T02:31:01.456771+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":143,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:00.252603+0000","last_change":"2023-08-28T02:27:31.344974+0000","last_active":"2023-08-28T02:31:00.252603+0000","last_peered":"2023-08-28T02:31:00.252603+0000","last_clean":"2023-08-28T02:31:00.252603+0000","last_became_active":"2023-08-28T02:27:31.344281+0000","last_became_peered":"2023-08-28T02:27:31.344281+0000","last_unstale":"2023-08-28T02:31:00.252603+0000","last_undegraded":"2023-08-28T02:31:00.252603+0000","last_fullsized":"2023-08-28T02:31:00.252603+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:04:02.821752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":251,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:00.063669+0000","last_change":"2023-08-28T02:23:59.223066+0000","last_active":"2023-08-28T02:31:00.063669+0000","last_peered":"2023-08-28T02:31:00.063669+0000","last_clean":"2023-08-28T02:31:00.063669+0000","last_became_active":"2023-08-28T02:23:59.222731+0000","last_became_peered":"2023-08-28T02:23:59.222731+0000","last_unstale":"2023-08-28T02:31:00.063669+0000","last_undegraded":"2023-08-28T02:31:00.063669+0000","last_fullsized":"2023-08-28T02:31:00.063669+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:48:37.558266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":291,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:00.067589+0000","last_change":"2023-08-28T02:23:59.210469+0000","last_active":"2023-08-28T02:31:00.067589+0000","last_peered":"2023-08-28T02:31:00.067589+0000","last_clean":"2023-08-28T02:31:00.067589+0000","last_became_active":"2023-08-28T02:23:59.210031+0000","last_became_peered":"2023-08-28T02:23:59.210031+0000","last_unstale":"2023-08-28T02:31:00.067589+0000","last_undegraded":"2023-08-28T02:31:00.067589+0000","last_fullsized":"2023-08-28T02:31:00.067589+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":147,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:01.456695+0000","last_change":"2023-08-28T02:28:25.354526+0000","last_active":"2023-08-28T02:31:01.456695+0000","last_peered":"2023-08-28T02:31:01.456695+0000","last_clean":"2023-08-28T02:31:01.456695+0000","last_became_active":"2023-08-28T02:28:25.350991+0000","last_became_peered":"2023-08-28T02:28:25.350991+0000","last_unstale":"2023-08-28T02:31:01.456695+0000","last_undegraded":"2023-08-28T02:31:01.456695+0000","last_fullsized":"2023-08-28T02:31:01.456695+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":252,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:00.064312+0000","last_change":"2023-08-28T02:23:59.220932+0000","last_active":"2023-08-28T02:31:00.064312+0000","last_peered":"2023-08-28T02:31:00.064312+0000","last_clean":"2023-08-28T02:31:00.064312+0000","last_became_active":"2023-08-28T02:23:59.220488+0000","last_became_peered":"2023-08-28T02:23:59.220488+0000","last_unstale":"2023-08-28T02:31:00.064312+0000","last_undegraded":"2023-08-28T02:31:00.064312+0000","last_fullsized":"2023-08-28T02:31:00.064312+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:53:09.216340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":64,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:01.456450+0000","last_change":"2023-08-28T02:28:25.703333+0000","last_active":"2023-08-28T02:31:01.456450+0000","last_peered":"2023-08-28T02:31:01.456450+0000","last_clean":"2023-08-28T02:31:01.456450+0000","last_became_active":"2023-08-28T02:28:25.343515+0000","last_became_peered":"2023-08-28T02:28:25.343515+0000","last_unstale":"2023-08-28T02:31:01.456450+0000","last_undegraded":"2023-08-28T02:31:01.456450+0000","last_fullsized":"2023-08-28T02:31:01.456450+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":142,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:00.252824+0000","last_change":"2023-08-28T02:27:31.330059+0000","last_active":"2023-08-28T02:31:00.252824+0000","last_peered":"2023-08-28T02:31:00.252824+0000","last_clean":"2023-08-28T02:31:00.252824+0000","last_became_active":"2023-08-28T02:27:31.329769+0000","last_became_peered":"2023-08-28T02:27:31.329769+0000","last_unstale":"2023-08-28T02:31:00.252824+0000","last_undegraded":"2023-08-28T02:31:00.252824+0000","last_fullsized":"2023-08-28T02:31:00.252824+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:14:54.349909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"265'406","reported_seq":419,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-08-28T02:31:04.571676+0000","last_change":"2023-08-28T02:30:57.068862+0000","last_active":"2023-08-28T02:31:04.571676+0000","last_peered":"2023-08-28T02:31:04.571676+0000","last_clean":"2023-08-28T02:31:04.571676+0000","last_became_active":"2023-08-28T02:30:57.068563+0000","last_became_peered":"2023-08-28T02:30:57.068563+0000","last_unstale":"2023-08-28T02:31:04.571676+0000","last_undegraded":"2023-08-28T02:31:04.571676+0000","last_fullsized":"2023-08-28T02:31:04.571676+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":406,"log_dups_size":0,"ondisk_log_size":406,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:39:58.505326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26607616,"num_objects":406,"num_object_clones":0,"num_object_copies":1218,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":406,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":406,"num_write_kb":25984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":133,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:01.456522+0000","last_change":"2023-08-28T02:23:38.657185+0000","last_active":"2023-08-28T02:31:01.456522+0000","last_peered":"2023-08-28T02:31:01.456522+0000","last_clean":"2023-08-28T02:31:01.456522+0000","last_became_active":"2023-08-28T02:23:38.339016+0000","last_became_peered":"2023-08-28T02:23:38.339016+0000","last_unstale":"2023-08-28T02:31:01.456522+0000","last_undegraded":"2023-08-28T02:31:01.456522+0000","last_fullsized":"2023-08-28T02:31:01.456522+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":251,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:00.067313+0000","last_change":"2023-08-28T02:28:25.348106+0000","last_active":"2023-08-28T02:31:00.067313+0000","last_peered":"2023-08-28T02:31:00.067313+0000","last_clean":"2023-08-28T02:31:00.067313+0000","last_became_active":"2023-08-28T02:28:25.347786+0000","last_became_peered":"2023-08-28T02:28:25.347786+0000","last_unstale":"2023-08-28T02:31:00.067313+0000","last_undegraded":"2023-08-28T02:31:00.067313+0000","last_fullsized":"2023-08-28T02:31:00.067313+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"265'382","reported_seq":395,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-08-28T02:31:04.240876+0000","last_change":"2023-08-28T02:30:57.067718+0000","last_active":"2023-08-28T02:31:04.240876+0000","last_peered":"2023-08-28T02:31:04.240876+0000","last_clean":"2023-08-28T02:31:04.240876+0000","last_became_active":"2023-08-28T02:30:57.067407+0000","last_became_peered":"2023-08-28T02:30:57.067407+0000","last_unstale":"2023-08-28T02:31:04.240876+0000","last_undegraded":"2023-08-28T02:31:04.240876+0000","last_fullsized":"2023-08-28T02:31:04.240876+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":382,"log_dups_size":0,"ondisk_log_size":382,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:49:39.360134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25034752,"num_objects":382,"num_object_clones":0,"num_object_copies":1146,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":382,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":382,"num_write_kb":24448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":159,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:01.456659+0000","last_change":"2023-08-28T02:27:31.395328+0000","last_active":"2023-08-28T02:31:01.456659+0000","last_peered":"2023-08-28T02:31:01.456659+0000","last_clean":"2023-08-28T02:31:01.456659+0000","last_became_active":"2023-08-28T02:27:31.328228+0000","last_became_peered":"2023-08-28T02:27:31.328228+0000","last_unstale":"2023-08-28T02:31:01.456659+0000","last_undegraded":"2023-08-28T02:31:01.456659+0000","last_fullsized":"2023-08-28T02:31:01.456659+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":300,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:01.456321+0000","last_change":"2023-08-28T02:27:31.344915+0000","last_active":"2023-08-28T02:31:01.456321+0000","last_peered":"2023-08-28T02:31:01.456321+0000","last_clean":"2023-08-28T02:31:01.456321+0000","last_became_active":"2023-08-28T02:27:31.344577+0000","last_became_peered":"2023-08-28T02:27:31.344577+0000","last_unstale":"2023-08-28T02:31:01.456321+0000","last_undegraded":"2023-08-28T02:31:01.456321+0000","last_fullsized":"2023-08-28T02:31:01.456321+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"265'342","reported_seq":355,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-08-28T02:31:03.923076+0000","last_change":"2023-08-28T02:30:57.066684+0000","last_active":"2023-08-28T02:31:03.923076+0000","last_peered":"2023-08-28T02:31:03.923076+0000","last_clean":"2023-08-28T02:31:03.923076+0000","last_became_active":"2023-08-28T02:30:57.066406+0000","last_became_peered":"2023-08-28T02:30:57.066406+0000","last_unstale":"2023-08-28T02:31:03.923076+0000","last_undegraded":"2023-08-28T02:31:03.923076+0000","last_fullsized":"2023-08-28T02:31:03.923076+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":342,"log_dups_size":0,"ondisk_log_size":342,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:35:33.176438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22413312,"num_objects":342,"num_object_clones":0,"num_object_copies":1026,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":342,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":342,"num_write_kb":21888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":159,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:01.456673+0000","last_change":"2023-08-28T02:28:25.749249+0000","last_active":"2023-08-28T02:31:01.456673+0000","last_peered":"2023-08-28T02:31:01.456673+0000","last_clean":"2023-08-28T02:31:01.456673+0000","last_became_active":"2023-08-28T02:28:25.397684+0000","last_became_peered":"2023-08-28T02:28:25.397684+0000","last_unstale":"2023-08-28T02:31:01.456673+0000","last_undegraded":"2023-08-28T02:31:01.456673+0000","last_fullsized":"2023-08-28T02:31:01.456673+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":54,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:01.457235+0000","last_change":"2023-08-28T02:28:25.350802+0000","last_active":"2023-08-28T02:31:01.457235+0000","last_peered":"2023-08-28T02:31:01.457235+0000","last_clean":"2023-08-28T02:31:01.457235+0000","last_became_active":"2023-08-28T02:28:25.350474+0000","last_became_peered":"2023-08-28T02:28:25.350474+0000","last_unstale":"2023-08-28T02:31:01.457235+0000","last_undegraded":"2023-08-28T02:31:01.457235+0000","last_fullsized":"2023-08-28T02:31:01.457235+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"265'396","reported_seq":409,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-08-28T02:31:04.138082+0000","last_change":"2023-08-28T02:30:57.071212+0000","last_active":"2023-08-28T02:31:04.138082+0000","last_peered":"2023-08-28T02:31:04.138082+0000","last_clean":"2023-08-28T02:31:04.138082+0000","last_became_active":"2023-08-28T02:30:57.070876+0000","last_became_peered":"2023-08-28T02:30:57.070876+0000","last_unstale":"2023-08-28T02:31:04.138082+0000","last_undegraded":"2023-08-28T02:31:04.138082+0000","last_fullsized":"2023-08-28T02:31:04.138082+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":396,"log_dups_size":0,"ondisk_log_size":396,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:19:29.209842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25952256,"num_objects":396,"num_object_clones":0,"num_object_copies":1188,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":396,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":396,"num_write_kb":25344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":267,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:00.252689+0000","last_change":"2023-08-28T02:28:25.445982+0000","last_active":"2023-08-28T02:31:00.252689+0000","last_peered":"2023-08-28T02:31:00.252689+0000","last_clean":"2023-08-28T02:31:00.252689+0000","last_became_active":"2023-08-28T02:28:25.443577+0000","last_became_peered":"2023-08-28T02:28:25.443577+0000","last_unstale":"2023-08-28T02:31:00.252689+0000","last_undegraded":"2023-08-28T02:31:00.252689+0000","last_fullsized":"2023-08-28T02:31:00.252689+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:24:03.381751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":245,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:00.064049+0000","last_change":"2023-08-28T02:27:31.319783+0000","last_active":"2023-08-28T02:31:00.064049+0000","last_peered":"2023-08-28T02:31:00.064049+0000","last_clean":"2023-08-28T02:31:00.064049+0000","last_became_active":"2023-08-28T02:27:31.319451+0000","last_became_peered":"2023-08-28T02:27:31.319451+0000","last_unstale":"2023-08-28T02:31:00.064049+0000","last_undegraded":"2023-08-28T02:31:00.064049+0000","last_fullsized":"2023-08-28T02:31:00.064049+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:20:37.097889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"265'342","reported_seq":354,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-08-28T02:31:03.920988+0000","last_change":"2023-08-28T02:30:57.071262+0000","last_active":"2023-08-28T02:31:03.920988+0000","last_peered":"2023-08-28T02:31:03.920988+0000","last_clean":"2023-08-28T02:31:03.920988+0000","last_became_active":"2023-08-28T02:30:57.070974+0000","last_became_peered":"2023-08-28T02:30:57.070974+0000","last_unstale":"2023-08-28T02:31:03.920988+0000","last_undegraded":"2023-08-28T02:31:03.920988+0000","last_fullsized":"2023-08-28T02:31:03.920988+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":342,"log_dups_size":0,"ondisk_log_size":342,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:40:20.990559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22413312,"num_objects":342,"num_object_clones":0,"num_object_copies":1026,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":342,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":342,"num_write_kb":21888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":129,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:00.252528+0000","last_change":"2023-08-28T02:23:59.230059+0000","last_active":"2023-08-28T02:31:00.252528+0000","last_peered":"2023-08-28T02:31:00.252528+0000","last_clean":"2023-08-28T02:31:00.252528+0000","last_became_active":"2023-08-28T02:23:59.217783+0000","last_became_peered":"2023-08-28T02:23:59.217783+0000","last_unstale":"2023-08-28T02:31:00.252528+0000","last_undegraded":"2023-08-28T02:31:00.252528+0000","last_fullsized":"2023-08-28T02:31:00.252528+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:00:12.145395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":285,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:01.457439+0000","last_change":"2023-08-28T02:28:25.351698+0000","last_active":"2023-08-28T02:31:01.457439+0000","last_peered":"2023-08-28T02:31:01.457439+0000","last_clean":"2023-08-28T02:31:01.457439+0000","last_became_active":"2023-08-28T02:28:25.351349+0000","last_became_peered":"2023-08-28T02:28:25.351349+0000","last_unstale":"2023-08-28T02:31:01.457439+0000","last_undegraded":"2023-08-28T02:31:01.457439+0000","last_fullsized":"2023-08-28T02:31:01.457439+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"265'354","reported_seq":367,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-08-28T02:31:04.137017+0000","last_change":"2023-08-28T02:30:57.069579+0000","last_active":"2023-08-28T02:31:04.137017+0000","last_peered":"2023-08-28T02:31:04.137017+0000","last_clean":"2023-08-28T02:31:04.137017+0000","last_became_active":"2023-08-28T02:30:57.069365+0000","last_became_peered":"2023-08-28T02:30:57.069365+0000","last_unstale":"2023-08-28T02:31:04.137017+0000","last_undegraded":"2023-08-28T02:31:04.137017+0000","last_fullsized":"2023-08-28T02:31:04.137017+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":354,"log_dups_size":0,"ondisk_log_size":354,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:02:07.549000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23199744,"num_objects":354,"num_object_clones":0,"num_object_copies":1062,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":354,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":354,"num_write_kb":22656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":54,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:01.457567+0000","last_change":"2023-08-28T02:28:25.345274+0000","last_active":"2023-08-28T02:31:01.457567+0000","last_peered":"2023-08-28T02:31:01.457567+0000","last_clean":"2023-08-28T02:31:01.457567+0000","last_became_active":"2023-08-28T02:28:25.344885+0000","last_became_peered":"2023-08-28T02:28:25.344885+0000","last_unstale":"2023-08-28T02:31:01.457567+0000","last_undegraded":"2023-08-28T02:31:01.457567+0000","last_fullsized":"2023-08-28T02:31:01.457567+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":311,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:01.456183+0000","last_change":"2023-08-28T02:27:31.358436+0000","last_active":"2023-08-28T02:31:01.456183+0000","last_peered":"2023-08-28T02:31:01.456183+0000","last_clean":"2023-08-28T02:31:01.456183+0000","last_became_active":"2023-08-28T02:27:31.358185+0000","last_became_peered":"2023-08-28T02:27:31.358185+0000","last_unstale":"2023-08-28T02:31:01.456183+0000","last_undegraded":"2023-08-28T02:31:01.456183+0000","last_fullsized":"2023-08-28T02:31:01.456183+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"265'839","reported_seq":851,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-08-28T02:31:07.119821+0000","last_change":"2023-08-28T02:30:57.067391+0000","last_active":"2023-08-28T02:31:07.119821+0000","last_peered":"2023-08-28T02:31:07.119821+0000","last_clean":"2023-08-28T02:31:07.119821+0000","last_became_active":"2023-08-28T02:30:57.067103+0000","last_became_peered":"2023-08-28T02:30:57.067103+0000","last_unstale":"2023-08-28T02:31:07.119821+0000","last_undegraded":"2023-08-28T02:31:07.119821+0000","last_fullsized":"2023-08-28T02:31:07.119821+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":839,"log_dups_size":0,"ondisk_log_size":839,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54984704,"num_objects":839,"num_object_clones":0,"num_object_copies":2517,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":839,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":839,"num_write_kb":53696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":174,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:01.456462+0000","last_change":"2023-08-28T02:23:38.347043+0000","last_active":"2023-08-28T02:31:01.456462+0000","last_peered":"2023-08-28T02:31:01.456462+0000","last_clean":"2023-08-28T02:31:01.456462+0000","last_became_active":"2023-08-28T02:23:38.346680+0000","last_became_peered":"2023-08-28T02:23:38.346680+0000","last_unstale":"2023-08-28T02:31:01.456462+0000","last_undegraded":"2023-08-28T02:31:01.456462+0000","last_fullsized":"2023-08-28T02:31:01.456462+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":232,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:00.064414+0000","last_change":"2023-08-28T02:28:25.347494+0000","last_active":"2023-08-28T02:31:00.064414+0000","last_peered":"2023-08-28T02:31:00.064414+0000","last_clean":"2023-08-28T02:31:00.064414+0000","last_became_active":"2023-08-28T02:28:25.347142+0000","last_became_peered":"2023-08-28T02:28:25.347142+0000","last_unstale":"2023-08-28T02:31:00.064414+0000","last_undegraded":"2023-08-28T02:31:00.064414+0000","last_fullsized":"2023-08-28T02:31:00.064414+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:37:14.384884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.c","version":"265'878","reported_seq":890,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-08-28T02:31:07.111896+0000","last_change":"2023-08-28T02:30:57.071876+0000","last_active":"2023-08-28T02:31:07.111896+0000","last_peered":"2023-08-28T02:31:07.111896+0000","last_clean":"2023-08-28T02:31:07.111896+0000","last_became_active":"2023-08-28T02:30:57.071027+0000","last_became_peered":"2023-08-28T02:30:57.071027+0000","last_unstale":"2023-08-28T02:31:07.111896+0000","last_undegraded":"2023-08-28T02:31:07.111896+0000","last_fullsized":"2023-08-28T02:31:07.111896+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":878,"log_dups_size":0,"ondisk_log_size":878,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57540608,"num_objects":878,"num_object_clones":0,"num_object_copies":2634,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":878,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":878,"num_write_kb":56192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":143,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:00.252474+0000","last_change":"2023-08-28T02:27:31.325260+0000","last_active":"2023-08-28T02:31:00.252474+0000","last_peered":"2023-08-28T02:31:00.252474+0000","last_clean":"2023-08-28T02:31:00.252474+0000","last_became_active":"2023-08-28T02:27:31.324950+0000","last_became_peered":"2023-08-28T02:27:31.324950+0000","last_unstale":"2023-08-28T02:31:00.252474+0000","last_undegraded":"2023-08-28T02:31:00.252474+0000","last_fullsized":"2023-08-28T02:31:00.252474+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:21:33.833261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":73,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:00.065530+0000","last_change":"2023-08-28T02:27:31.322118+0000","last_active":"2023-08-28T02:31:00.065530+0000","last_peered":"2023-08-28T02:31:00.065530+0000","last_clean":"2023-08-28T02:31:00.065530+0000","last_became_active":"2023-08-28T02:27:31.321338+0000","last_became_peered":"2023-08-28T02:27:31.321338+0000","last_unstale":"2023-08-28T02:31:00.065530+0000","last_undegraded":"2023-08-28T02:31:00.065530+0000","last_fullsized":"2023-08-28T02:31:00.065530+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:56:14.210571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"265'798","reported_seq":810,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-08-28T02:31:06.928708+0000","last_change":"2023-08-28T02:30:57.067740+0000","last_active":"2023-08-28T02:31:06.928708+0000","last_peered":"2023-08-28T02:31:06.928708+0000","last_clean":"2023-08-28T02:31:06.928708+0000","last_became_active":"2023-08-28T02:30:57.067464+0000","last_became_peered":"2023-08-28T02:30:57.067464+0000","last_unstale":"2023-08-28T02:31:06.928708+0000","last_undegraded":"2023-08-28T02:31:06.928708+0000","last_fullsized":"2023-08-28T02:31:06.928708+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":798,"log_dups_size":0,"ondisk_log_size":798,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52297728,"num_objects":798,"num_object_clones":0,"num_object_copies":2394,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":798,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":798,"num_write_kb":51072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"78'126","reported_seq":635,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:00.065196+0000","last_change":"2023-08-28T02:28:27.383137+0000","last_active":"2023-08-28T02:31:00.065196+0000","last_peered":"2023-08-28T02:31:00.065196+0000","last_clean":"2023-08-28T02:31:00.065196+0000","last_became_active":"2023-08-28T02:28:27.382488+0000","last_became_peered":"2023-08-28T02:28:27.382488+0000","last_unstale":"2023-08-28T02:31:00.065196+0000","last_undegraded":"2023-08-28T02:31:00.065196+0000","last_fullsized":"2023-08-28T02:31:00.065196+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:14:48.414426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":743,"num_read_kb":1331,"num_write":125,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":264,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:00.067537+0000","last_change":"2023-08-28T02:23:59.207148+0000","last_active":"2023-08-28T02:31:00.067537+0000","last_peered":"2023-08-28T02:31:00.067537+0000","last_clean":"2023-08-28T02:31:00.067537+0000","last_became_active":"2023-08-28T02:23:59.206802+0000","last_became_peered":"2023-08-28T02:23:59.206802+0000","last_unstale":"2023-08-28T02:31:00.067537+0000","last_undegraded":"2023-08-28T02:31:00.067537+0000","last_fullsized":"2023-08-28T02:31:00.067537+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":73,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:00.065216+0000","last_change":"2023-08-28T02:27:31.328027+0000","last_active":"2023-08-28T02:31:00.065216+0000","last_peered":"2023-08-28T02:31:00.065216+0000","last_clean":"2023-08-28T02:31:00.065216+0000","last_became_active":"2023-08-28T02:27:31.326529+0000","last_became_peered":"2023-08-28T02:27:31.326529+0000","last_unstale":"2023-08-28T02:31:00.065216+0000","last_undegraded":"2023-08-28T02:31:00.065216+0000","last_fullsized":"2023-08-28T02:31:00.065216+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:56:57.331355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"265'847","reported_seq":859,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-08-28T02:31:07.118937+0000","last_change":"2023-08-28T02:30:57.067534+0000","last_active":"2023-08-28T02:31:07.118937+0000","last_peered":"2023-08-28T02:31:07.118937+0000","last_clean":"2023-08-28T02:31:07.118937+0000","last_became_active":"2023-08-28T02:30:57.067256+0000","last_became_peered":"2023-08-28T02:30:57.067256+0000","last_unstale":"2023-08-28T02:31:07.118937+0000","last_undegraded":"2023-08-28T02:31:07.118937+0000","last_fullsized":"2023-08-28T02:31:07.118937+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":847,"log_dups_size":0,"ondisk_log_size":847,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55508992,"num_objects":847,"num_object_clones":0,"num_object_copies":2541,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":847,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":847,"num_write_kb":54208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"78'204","reported_seq":1350,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:00.065095+0000","last_change":"2023-08-28T02:28:59.480561+0000","last_active":"2023-08-28T02:31:00.065095+0000","last_peered":"2023-08-28T02:31:00.065095+0000","last_clean":"2023-08-28T02:31:00.065095+0000","last_became_active":"2023-08-28T02:28:59.479868+0000","last_became_peered":"2023-08-28T02:28:59.479868+0000","last_unstale":"2023-08-28T02:31:00.065095+0000","last_undegraded":"2023-08-28T02:31:00.065095+0000","last_fullsized":"2023-08-28T02:31:00.065095+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":204,"log_dups_size":0,"ondisk_log_size":204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:39:50.444629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1521,"num_read_kb":2511,"num_write":178,"num_write_kb":1175,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":259,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:01.456755+0000","last_change":"2023-08-28T02:28:25.352417+0000","last_active":"2023-08-28T02:31:01.456755+0000","last_peered":"2023-08-28T02:31:01.456755+0000","last_clean":"2023-08-28T02:31:01.456755+0000","last_became_active":"2023-08-28T02:28:25.352216+0000","last_became_peered":"2023-08-28T02:28:25.352216+0000","last_unstale":"2023-08-28T02:31:01.456755+0000","last_undegraded":"2023-08-28T02:31:01.456755+0000","last_fullsized":"2023-08-28T02:31:01.456755+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":73,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:00.065119+0000","last_change":"2023-08-28T02:27:31.376755+0000","last_active":"2023-08-28T02:31:00.065119+0000","last_peered":"2023-08-28T02:31:00.065119+0000","last_clean":"2023-08-28T02:31:00.065119+0000","last_became_active":"2023-08-28T02:27:31.373279+0000","last_became_peered":"2023-08-28T02:27:31.373279+0000","last_unstale":"2023-08-28T02:31:00.065119+0000","last_undegraded":"2023-08-28T02:31:00.065119+0000","last_fullsized":"2023-08-28T02:31:00.065119+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:26:14.628941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"265'354","reported_seq":365,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-08-28T02:31:03.917238+0000","last_change":"2023-08-28T02:30:57.062219+0000","last_active":"2023-08-28T02:31:03.917238+0000","last_peered":"2023-08-28T02:31:03.917238+0000","last_clean":"2023-08-28T02:31:03.917238+0000","last_became_active":"2023-08-28T02:30:57.061931+0000","last_became_peered":"2023-08-28T02:30:57.061931+0000","last_unstale":"2023-08-28T02:31:03.917238+0000","last_undegraded":"2023-08-28T02:31:03.917238+0000","last_fullsized":"2023-08-28T02:31:03.917238+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":354,"log_dups_size":0,"ondisk_log_size":354,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:03:40.377848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23199744,"num_objects":354,"num_object_clones":0,"num_object_copies":1062,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":354,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":354,"num_write_kb":22656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"78'47","reported_seq":482,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:00.489068+0000","last_change":"2023-08-28T02:24:05.799647+0000","last_active":"2023-08-28T02:31:00.489068+0000","last_peered":"2023-08-28T02:31:00.489068+0000","last_clean":"2023-08-28T02:31:00.489068+0000","last_became_active":"2023-08-28T02:23:59.210481+0000","last_became_peered":"2023-08-28T02:23:59.210481+0000","last_unstale":"2023-08-28T02:31:00.489068+0000","last_undegraded":"2023-08-28T02:31:00.489068+0000","last_fullsized":"2023-08-28T02:31:00.489068+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":47,"log_dups_size":0,"ondisk_log_size":47,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:40:18.542375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":162,"num_read_kb":1281,"num_write":48,"num_write_kb":1153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":82,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:00.064885+0000","last_change":"2023-08-28T02:28:25.343406+0000","last_active":"2023-08-28T02:31:00.064885+0000","last_peered":"2023-08-28T02:31:00.064885+0000","last_clean":"2023-08-28T02:31:00.064885+0000","last_became_active":"2023-08-28T02:28:25.343065+0000","last_became_peered":"2023-08-28T02:28:25.343065+0000","last_unstale":"2023-08-28T02:31:00.064885+0000","last_undegraded":"2023-08-28T02:31:00.064885+0000","last_fullsized":"2023-08-28T02:31:00.064885+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:20:31.893609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":73,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:00.065595+0000","last_change":"2023-08-28T02:27:31.322967+0000","last_active":"2023-08-28T02:31:00.065595+0000","last_peered":"2023-08-28T02:31:00.065595+0000","last_clean":"2023-08-28T02:31:00.065595+0000","last_became_active":"2023-08-28T02:27:31.322655+0000","last_became_peered":"2023-08-28T02:27:31.322655+0000","last_unstale":"2023-08-28T02:31:00.065595+0000","last_undegraded":"2023-08-28T02:31:00.065595+0000","last_fullsized":"2023-08-28T02:31:00.065595+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:05:15.868115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"265'312","reported_seq":325,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-08-28T02:31:03.923845+0000","last_change":"2023-08-28T02:30:57.071289+0000","last_active":"2023-08-28T02:31:03.923845+0000","last_peered":"2023-08-28T02:31:03.923845+0000","last_clean":"2023-08-28T02:31:03.923845+0000","last_became_active":"2023-08-28T02:30:57.070991+0000","last_became_peered":"2023-08-28T02:30:57.070991+0000","last_unstale":"2023-08-28T02:31:03.923845+0000","last_undegraded":"2023-08-28T02:31:03.923845+0000","last_fullsized":"2023-08-28T02:31:03.923845+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":312,"log_dups_size":0,"ondisk_log_size":312,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:15:02.370637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20447232,"num_objects":312,"num_object_clones":0,"num_object_copies":936,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":312,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":312,"num_write_kb":19968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"78'114","reported_seq":628,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:00.067518+0000","last_change":"2023-08-28T02:24:01.210739+0000","last_active":"2023-08-28T02:31:00.067518+0000","last_peered":"2023-08-28T02:31:00.067518+0000","last_clean":"2023-08-28T02:31:00.067518+0000","last_became_active":"2023-08-28T02:24:01.210386+0000","last_became_peered":"2023-08-28T02:24:01.210386+0000","last_unstale":"2023-08-28T02:31:00.067518+0000","last_undegraded":"2023-08-28T02:31:00.067518+0000","last_fullsized":"2023-08-28T02:31:00.067518+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":114,"log_dups_size":0,"ondisk_log_size":114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":358,"num_read_kb":1352,"num_write":102,"num_write_kb":1104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"86'139","reported_seq":475,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:01.456542+0000","last_change":"2023-08-28T02:29:58.484566+0000","last_active":"2023-08-28T02:31:01.456542+0000","last_peered":"2023-08-28T02:31:01.456542+0000","last_clean":"2023-08-28T02:31:01.456542+0000","last_became_active":"2023-08-28T02:29:58.484046+0000","last_became_peered":"2023-08-28T02:29:58.484046+0000","last_unstale":"2023-08-28T02:31:01.456542+0000","last_undegraded":"2023-08-28T02:31:01.456542+0000","last_fullsized":"2023-08-28T02:31:01.456542+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_clean_scrub_stamp":"2023-08-28T02:08:02.043718+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":143,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:00.252453+0000","last_change":"2023-08-28T02:28:25.347600+0000","last_active":"2023-08-28T02:31:00.252453+0000","last_peered":"2023-08-28T02:31:00.252453+0000","last_clean":"2023-08-28T02:31:00.252453+0000","last_became_active":"2023-08-28T02:28:25.347237+0000","last_became_peered":"2023-08-28T02:28:25.347237+0000","last_unstale":"2023-08-28T02:31:00.252453+0000","last_undegraded":"2023-08-28T02:31:00.252453+0000","last_fullsized":"2023-08-28T02:31:00.252453+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:29:26.767559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":54,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:01.456994+0000","last_change":"2023-08-28T02:28:25.398594+0000","last_active":"2023-08-28T02:31:01.456994+0000","last_peered":"2023-08-28T02:31:01.456994+0000","last_clean":"2023-08-28T02:31:01.456994+0000","last_became_active":"2023-08-28T02:28:25.397058+0000","last_became_peered":"2023-08-28T02:28:25.397058+0000","last_unstale":"2023-08-28T02:31:01.456994+0000","last_undegraded":"2023-08-28T02:31:01.456994+0000","last_fullsized":"2023-08-28T02:31:01.456994+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"265'835","reported_seq":847,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-08-28T02:31:07.156994+0000","last_change":"2023-08-28T02:30:57.071523+0000","last_active":"2023-08-28T02:31:07.156994+0000","last_peered":"2023-08-28T02:31:07.156994+0000","last_clean":"2023-08-28T02:31:07.156994+0000","last_became_active":"2023-08-28T02:30:57.071114+0000","last_became_peered":"2023-08-28T02:30:57.071114+0000","last_unstale":"2023-08-28T02:31:07.156994+0000","last_undegraded":"2023-08-28T02:31:07.156994+0000","last_fullsized":"2023-08-28T02:31:07.156994+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":835,"log_dups_size":0,"ondisk_log_size":835,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54722560,"num_objects":835,"num_object_clones":0,"num_object_copies":2505,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":835,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":835,"num_write_kb":53440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"78'148","reported_seq":795,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:01.456386+0000","last_change":"2023-08-28T02:29:39.524927+0000","last_active":"2023-08-28T02:31:01.456386+0000","last_peered":"2023-08-28T02:31:01.456386+0000","last_clean":"2023-08-28T02:31:01.456386+0000","last_became_active":"2023-08-28T02:29:39.524297+0000","last_became_peered":"2023-08-28T02:29:39.524297+0000","last_unstale":"2023-08-28T02:31:01.456386+0000","last_undegraded":"2023-08-28T02:31:01.456386+0000","last_fullsized":"2023-08-28T02:31:01.456386+0000","mapping_epoch":243,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":244,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1092,"num_read_kb":1955,"num_write":127,"num_write_kb":5177,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":233,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:00.063846+0000","last_change":"2023-08-28T02:23:59.226986+0000","last_active":"2023-08-28T02:31:00.063846+0000","last_peered":"2023-08-28T02:31:00.063846+0000","last_clean":"2023-08-28T02:31:00.063846+0000","last_became_active":"2023-08-28T02:23:59.226529+0000","last_became_peered":"2023-08-28T02:23:59.226529+0000","last_unstale":"2023-08-28T02:31:00.063846+0000","last_undegraded":"2023-08-28T02:31:00.063846+0000","last_fullsized":"2023-08-28T02:31:00.063846+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:22:09.193619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":278,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:00.063928+0000","last_change":"2023-08-28T02:27:31.361298+0000","last_active":"2023-08-28T02:31:00.063928+0000","last_peered":"2023-08-28T02:31:00.063928+0000","last_clean":"2023-08-28T02:31:00.063928+0000","last_became_active":"2023-08-28T02:27:31.360939+0000","last_became_peered":"2023-08-28T02:27:31.360939+0000","last_unstale":"2023-08-28T02:31:00.063928+0000","last_undegraded":"2023-08-28T02:31:00.063928+0000","last_fullsized":"2023-08-28T02:31:00.063928+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:48:42.024920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"265'832","reported_seq":844,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-08-28T02:31:06.931202+0000","last_change":"2023-08-28T02:30:57.071302+0000","last_active":"2023-08-28T02:31:06.931202+0000","last_peered":"2023-08-28T02:31:06.931202+0000","last_clean":"2023-08-28T02:31:06.931202+0000","last_became_active":"2023-08-28T02:30:57.070731+0000","last_became_peered":"2023-08-28T02:30:57.070731+0000","last_unstale":"2023-08-28T02:31:06.931202+0000","last_undegraded":"2023-08-28T02:31:06.931202+0000","last_fullsized":"2023-08-28T02:31:06.931202+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":832,"log_dups_size":0,"ondisk_log_size":832,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54525952,"num_objects":832,"num_object_clones":0,"num_object_copies":2496,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":832,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":832,"num_write_kb":53248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"78'103","reported_seq":640,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:01.456572+0000","last_change":"2023-08-28T02:24:07.212128+0000","last_active":"2023-08-28T02:31:01.456572+0000","last_peered":"2023-08-28T02:31:01.456572+0000","last_clean":"2023-08-28T02:31:01.456572+0000","last_became_active":"2023-08-28T02:24:07.211562+0000","last_became_peered":"2023-08-28T02:24:07.211562+0000","last_unstale":"2023-08-28T02:31:01.456572+0000","last_undegraded":"2023-08-28T02:31:01.456572+0000","last_fullsized":"2023-08-28T02:31:01.456572+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":803,"num_read_kb":2080,"num_write":73,"num_write_kb":1244,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":73,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:00.065365+0000","last_change":"2023-08-28T02:27:31.377042+0000","last_active":"2023-08-28T02:31:00.065365+0000","last_peered":"2023-08-28T02:31:00.065365+0000","last_clean":"2023-08-28T02:31:00.065365+0000","last_became_active":"2023-08-28T02:27:31.373709+0000","last_became_peered":"2023-08-28T02:27:31.373709+0000","last_unstale":"2023-08-28T02:31:00.065365+0000","last_undegraded":"2023-08-28T02:31:00.065365+0000","last_fullsized":"2023-08-28T02:31:00.065365+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:12:38.286332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":219,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:00.064269+0000","last_change":"2023-08-28T02:23:38.329667+0000","last_active":"2023-08-28T02:31:00.064269+0000","last_peered":"2023-08-28T02:31:00.064269+0000","last_clean":"2023-08-28T02:31:00.064269+0000","last_became_active":"2023-08-28T02:23:38.329175+0000","last_became_peered":"2023-08-28T02:23:38.329175+0000","last_unstale":"2023-08-28T02:31:00.064269+0000","last_undegraded":"2023-08-28T02:31:00.064269+0000","last_fullsized":"2023-08-28T02:31:00.064269+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:21:34.815067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"265'367","reported_seq":380,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-08-28T02:31:04.242201+0000","last_change":"2023-08-28T02:30:57.061826+0000","last_active":"2023-08-28T02:31:04.242201+0000","last_peered":"2023-08-28T02:31:04.242201+0000","last_clean":"2023-08-28T02:31:04.242201+0000","last_became_active":"2023-08-28T02:30:57.061555+0000","last_became_peered":"2023-08-28T02:30:57.061555+0000","last_unstale":"2023-08-28T02:31:04.242201+0000","last_undegraded":"2023-08-28T02:31:04.242201+0000","last_fullsized":"2023-08-28T02:31:04.242201+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":367,"log_dups_size":0,"ondisk_log_size":367,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:15:31.288849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24051712,"num_objects":367,"num_object_clones":0,"num_object_copies":1101,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":367,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":367,"num_write_kb":23488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":129,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:00.252868+0000","last_change":"2023-08-28T02:23:59.232099+0000","last_active":"2023-08-28T02:31:00.252868+0000","last_peered":"2023-08-28T02:31:00.252868+0000","last_clean":"2023-08-28T02:31:00.252868+0000","last_became_active":"2023-08-28T02:23:59.219318+0000","last_became_peered":"2023-08-28T02:23:59.219318+0000","last_unstale":"2023-08-28T02:31:00.252868+0000","last_undegraded":"2023-08-28T02:31:00.252868+0000","last_fullsized":"2023-08-28T02:31:00.252868+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:59:30.093289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'112","reported_seq":419,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:00.066670+0000","last_change":"2023-08-28T02:28:59.477917+0000","last_active":"2023-08-28T02:31:00.066670+0000","last_peered":"2023-08-28T02:31:00.066670+0000","last_clean":"2023-08-28T02:31:00.066670+0000","last_became_active":"2023-08-28T02:28:59.477383+0000","last_became_peered":"2023-08-28T02:28:59.477383+0000","last_unstale":"2023-08-28T02:31:00.066670+0000","last_undegraded":"2023-08-28T02:31:00.066670+0000","last_fullsized":"2023-08-28T02:31:00.066670+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:28:45.045742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":444,"num_read_kb":473,"num_write":94,"num_write_kb":127,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":125,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:00.252417+0000","last_change":"2023-08-28T02:23:59.221034+0000","last_active":"2023-08-28T02:31:00.252417+0000","last_peered":"2023-08-28T02:31:00.252417+0000","last_clean":"2023-08-28T02:31:00.252417+0000","last_became_active":"2023-08-28T02:23:59.220730+0000","last_became_peered":"2023-08-28T02:23:59.220730+0000","last_unstale":"2023-08-28T02:31:00.252417+0000","last_undegraded":"2023-08-28T02:31:00.252417+0000","last_fullsized":"2023-08-28T02:31:00.252417+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:25:52.364370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"265'371","reported_seq":384,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-08-28T02:31:04.249130+0000","last_change":"2023-08-28T02:30:57.060097+0000","last_active":"2023-08-28T02:31:04.249130+0000","last_peered":"2023-08-28T02:31:04.249130+0000","last_clean":"2023-08-28T02:31:04.249130+0000","last_became_active":"2023-08-28T02:30:57.059712+0000","last_became_peered":"2023-08-28T02:30:57.059712+0000","last_unstale":"2023-08-28T02:31:04.249130+0000","last_undegraded":"2023-08-28T02:31:04.249130+0000","last_fullsized":"2023-08-28T02:31:04.249130+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":371,"log_dups_size":0,"ondisk_log_size":371,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:54:20.362640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24313856,"num_objects":371,"num_object_clones":0,"num_object_copies":1113,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":371,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":371,"num_write_kb":23744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"78'122","reported_seq":572,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:01.457079+0000","last_change":"2023-08-28T02:28:28.383306+0000","last_active":"2023-08-28T02:31:01.457079+0000","last_peered":"2023-08-28T02:31:01.457079+0000","last_clean":"2023-08-28T02:31:01.457079+0000","last_became_active":"2023-08-28T02:28:28.382672+0000","last_became_peered":"2023-08-28T02:28:28.382672+0000","last_unstale":"2023-08-28T02:31:01.457079+0000","last_undegraded":"2023-08-28T02:31:01.457079+0000","last_fullsized":"2023-08-28T02:31:01.457079+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":857,"num_read_kb":2224,"num_write":99,"num_write_kb":2312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":245,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:00.064020+0000","last_change":"2023-08-28T02:27:31.424373+0000","last_active":"2023-08-28T02:31:00.064020+0000","last_peered":"2023-08-28T02:31:00.064020+0000","last_clean":"2023-08-28T02:31:00.064020+0000","last_became_active":"2023-08-28T02:27:31.324505+0000","last_became_peered":"2023-08-28T02:27:31.324505+0000","last_unstale":"2023-08-28T02:31:00.064020+0000","last_undegraded":"2023-08-28T02:31:00.064020+0000","last_fullsized":"2023-08-28T02:31:00.064020+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:27:57.837524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":234,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:01.456123+0000","last_change":"2023-08-28T02:28:25.351840+0000","last_active":"2023-08-28T02:31:01.456123+0000","last_peered":"2023-08-28T02:31:01.456123+0000","last_clean":"2023-08-28T02:31:01.456123+0000","last_became_active":"2023-08-28T02:28:25.351399+0000","last_became_peered":"2023-08-28T02:28:25.351399+0000","last_unstale":"2023-08-28T02:31:01.456123+0000","last_undegraded":"2023-08-28T02:31:01.456123+0000","last_fullsized":"2023-08-28T02:31:01.456123+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":221,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:00.064198+0000","last_change":"2023-08-28T02:23:38.329474+0000","last_active":"2023-08-28T02:31:00.064198+0000","last_peered":"2023-08-28T02:31:00.064198+0000","last_clean":"2023-08-28T02:31:00.064198+0000","last_became_active":"2023-08-28T02:23:38.328505+0000","last_became_peered":"2023-08-28T02:23:38.328505+0000","last_unstale":"2023-08-28T02:31:00.064198+0000","last_undegraded":"2023-08-28T02:31:00.064198+0000","last_fullsized":"2023-08-28T02:31:00.064198+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:12:14.835809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":306,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:00.489307+0000","last_change":"2023-08-28T02:27:31.384467+0000","last_active":"2023-08-28T02:31:00.489307+0000","last_peered":"2023-08-28T02:31:00.489307+0000","last_clean":"2023-08-28T02:31:00.489307+0000","last_became_active":"2023-08-28T02:27:31.384183+0000","last_became_peered":"2023-08-28T02:27:31.384183+0000","last_unstale":"2023-08-28T02:31:00.489307+0000","last_undegraded":"2023-08-28T02:31:00.489307+0000","last_fullsized":"2023-08-28T02:31:00.489307+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:59:47.551892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":71,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:01.457393+0000","last_change":"2023-08-28T02:28:11.333622+0000","last_active":"2023-08-28T02:31:01.457393+0000","last_peered":"2023-08-28T02:31:01.457393+0000","last_clean":"2023-08-28T02:31:01.457393+0000","last_became_active":"2023-08-28T02:28:11.333321+0000","last_became_peered":"2023-08-28T02:28:11.333321+0000","last_unstale":"2023-08-28T02:31:01.457393+0000","last_undegraded":"2023-08-28T02:31:01.457393+0000","last_fullsized":"2023-08-28T02:31:01.457393+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":282,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:01.456902+0000","last_change":"2023-08-28T02:27:31.343538+0000","last_active":"2023-08-28T02:31:01.456902+0000","last_peered":"2023-08-28T02:31:01.456902+0000","last_clean":"2023-08-28T02:31:01.456902+0000","last_became_active":"2023-08-28T02:27:31.343324+0000","last_became_peered":"2023-08-28T02:27:31.343324+0000","last_unstale":"2023-08-28T02:31:01.456902+0000","last_undegraded":"2023-08-28T02:31:01.456902+0000","last_fullsized":"2023-08-28T02:31:01.456902+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":84,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:01.456278+0000","last_change":"2023-08-28T02:28:25.705331+0000","last_active":"2023-08-28T02:31:01.456278+0000","last_peered":"2023-08-28T02:31:01.456278+0000","last_clean":"2023-08-28T02:31:01.456278+0000","last_became_active":"2023-08-28T02:28:25.397480+0000","last_became_peered":"2023-08-28T02:28:25.397480+0000","last_unstale":"2023-08-28T02:31:01.456278+0000","last_undegraded":"2023-08-28T02:31:01.456278+0000","last_fullsized":"2023-08-28T02:31:01.456278+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":264,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:01.456546+0000","last_change":"2023-08-28T02:23:59.207929+0000","last_active":"2023-08-28T02:31:01.456546+0000","last_peered":"2023-08-28T02:31:01.456546+0000","last_clean":"2023-08-28T02:31:01.456546+0000","last_became_active":"2023-08-28T02:23:59.207589+0000","last_became_peered":"2023-08-28T02:23:59.207589+0000","last_unstale":"2023-08-28T02:31:01.456546+0000","last_undegraded":"2023-08-28T02:31:01.456546+0000","last_fullsized":"2023-08-28T02:31:01.456546+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":152,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:00.252713+0000","last_change":"2023-08-28T02:28:25.347988+0000","last_active":"2023-08-28T02:31:00.252713+0000","last_peered":"2023-08-28T02:31:00.252713+0000","last_clean":"2023-08-28T02:31:00.252713+0000","last_became_active":"2023-08-28T02:28:25.347632+0000","last_became_peered":"2023-08-28T02:28:25.347632+0000","last_unstale":"2023-08-28T02:31:00.252713+0000","last_undegraded":"2023-08-28T02:31:00.252713+0000","last_fullsized":"2023-08-28T02:31:00.252713+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:56:20.202662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":124,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:00.252561+0000","last_change":"2023-08-28T02:23:59.217794+0000","last_active":"2023-08-28T02:31:00.252561+0000","last_peered":"2023-08-28T02:31:00.252561+0000","last_clean":"2023-08-28T02:31:00.252561+0000","last_became_active":"2023-08-28T02:23:59.217231+0000","last_became_peered":"2023-08-28T02:23:59.217231+0000","last_unstale":"2023-08-28T02:31:00.252561+0000","last_undegraded":"2023-08-28T02:31:00.252561+0000","last_fullsized":"2023-08-28T02:31:00.252561+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:36:26.297410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":219,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:00.064280+0000","last_change":"2023-08-28T02:20:19.850252+0000","last_active":"2023-08-28T02:31:00.064280+0000","last_peered":"2023-08-28T02:31:00.064280+0000","last_clean":"2023-08-28T02:31:00.064280+0000","last_became_active":"2023-08-28T02:20:19.849913+0000","last_became_peered":"2023-08-28T02:20:19.849913+0000","last_unstale":"2023-08-28T02:31:00.064280+0000","last_undegraded":"2023-08-28T02:31:00.064280+0000","last_fullsized":"2023-08-28T02:31:00.064280+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:48:47.713465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":296,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-08-28T02:31:01.455962+0000","last_change":"2023-08-28T02:28:25.347845+0000","last_active":"2023-08-28T02:31:01.455962+0000","last_peered":"2023-08-28T02:31:01.455962+0000","last_clean":"2023-08-28T02:31:01.455962+0000","last_became_active":"2023-08-28T02:28:25.347502+0000","last_became_peered":"2023-08-28T02:28:25.347502+0000","last_unstale":"2023-08-28T02:31:01.455962+0000","last_undegraded":"2023-08-28T02:31:01.455962+0000","last_fullsized":"2023-08-28T02:31:01.455962+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":567214080,"num_objects":8655,"num_object_clones":0,"num_object_copies":25965,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8655,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8655,"num_write_kb":553920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1768882176,"data_stored":1768882176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":8655,"ondisk_log_size":8655,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":19382,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26810,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":137,"num_bytes_recovered":12582931,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430466,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":840568,"kb_used_data":535088,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":92904584,"statfs":{"total":95995035648,"available":95134294016,"internally_reserved":0,"allocated":547930112,"data_stored":547208176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:31:04 2023","interfaces":[{"interface":"back","average":{"1min":1.26,"5min":2.145,"15min":2.1339999999999999},"min":{"1min":0.23000000000000001,"5min":0.22800000000000001,"15min":0.21299999999999999},"max":{"1min":7.4240000000000004,"5min":59.753999999999998,"15min":116.755},"last":2.7010000000000001},{"interface":"front","average":{"1min":1.4550000000000001,"5min":2.4199999999999999,"15min":2.3919999999999999},"min":{"1min":0.252,"5min":0.245,"15min":0.23999999999999999},"max":{"1min":6.4690000000000003,"5min":59.765999999999998,"15min":116.971},"last":1.246}]},{"osd":1,"last update":"Mon Aug 28 02:30:11 2023","interfaces":[{"interface":"back","average":{"1min":2.496,"5min":2.2509999999999999,"15min":1.2370000000000001},"min":{"1min":0.27900000000000003,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":24.681000000000001,"5min":44.287999999999997,"15min":44.287999999999997},"last":0.53700000000000003},{"interface":"front","average":{"1min":2.7570000000000001,"5min":3.109,"15min":1.667},"min":{"1min":0.28399999999999997,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":24.189,"5min":44.393999999999998,"15min":44.393999999999998},"last":1.288}]},{"osd":2,"last update":"Mon Aug 28 02:30:35 2023","interfaces":[{"interface":"back","average":{"1min":2.1779999999999999,"5min":3.004,"15min":2.4830000000000001},"min":{"1min":0.32400000000000001,"5min":0.28000000000000003,"15min":0.27600000000000002},"max":{"1min":7.5300000000000002,"5min":101.616,"15min":119.571},"last":1.6990000000000001},{"interface":"front","average":{"1min":1.9119999999999999,"5min":3.7210000000000001,"15min":2.875},"min":{"1min":0.32900000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.5380000000000003,"5min":101.605,"15min":119.554},"last":1.738}]},{"osd":3,"last update":"Mon Aug 28 02:30:11 2023","interfaces":[{"interface":"back","average":{"1min":2.0110000000000001,"5min":4.0620000000000003,"15min":2.2480000000000002},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30199999999999999},"max":{"1min":10.686999999999999,"5min":99.459000000000003,"15min":107.916},"last":0.56499999999999995},{"interface":"front","average":{"1min":1.794,"5min":3.6789999999999998,"15min":2.3239999999999998},"min":{"1min":0.34300000000000003,"5min":0.311,"15min":0.27700000000000002},"max":{"1min":10.140000000000001,"5min":98.084000000000003,"15min":112.00700000000001},"last":0.45300000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:30:35 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":1.26,"15min":1.254},"min":{"1min":0.28899999999999998,"5min":0.26500000000000001,"15min":0.22900000000000001},"max":{"1min":2.3929999999999998,"5min":11.696,"15min":63.048000000000002},"last":0.35299999999999998},{"interface":"front","average":{"1min":1.3660000000000001,"5min":1.347,"15min":1.24},"min":{"1min":0.316,"5min":0.26700000000000002,"15min":0.247},"max":{"1min":8.2140000000000004,"5min":10.840999999999999,"15min":62.912999999999997},"last":0.40899999999999997}]},{"osd":5,"last update":"Mon Aug 28 02:30:35 2023","interfaces":[{"interface":"back","average":{"1min":1.2,"5min":1.1859999999999999,"15min":1.075},"min":{"1min":0.32800000000000001,"5min":0.27800000000000002,"15min":0.26800000000000002},"max":{"1min":7.5110000000000001,"5min":13.788,"15min":63.021000000000001},"last":1.093},{"interface":"front","average":{"1min":1.2949999999999999,"5min":1.4079999999999999,"15min":1.377},"min":{"1min":0.38200000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":6.7110000000000003,"5min":9.6929999999999996,"15min":63.082999999999998},"last":0.39200000000000002}]},{"osd":6,"last update":"Mon Aug 28 02:30:35 2023","interfaces":[{"interface":"back","average":{"1min":1.302,"5min":1.2529999999999999,"15min":1.3280000000000001},"min":{"1min":0.36899999999999999,"5min":0.318,"15min":0.29699999999999999},"max":{"1min":7.5369999999999999,"5min":9.7070000000000007,"15min":101.61},"last":0.77100000000000002},{"interface":"front","average":{"1min":0.78500000000000003,"5min":1.29,"15min":1.427},"min":{"1min":0.40999999999999998,"5min":0.308,"15min":0.308},"max":{"1min":2.4780000000000002,"5min":14.423999999999999,"15min":101.621},"last":0.42399999999999999}]}]},{"osd":6,"up_from":42,"seq":180388626693,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":564268,"kb_used_data":262948,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93180884,"statfs":{"total":95995035648,"available":95417225216,"internally_reserved":0,"allocated":269258752,"data_stored":268548662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:30:11 2023","interfaces":[{"interface":"back","average":{"1min":3.246,"5min":3.911,"15min":2.7629999999999999},"min":{"1min":0.22700000000000001,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":44.512999999999998,"5min":60.625999999999998,"15min":121.247},"last":1.4830000000000001},{"interface":"front","average":{"1min":3.3300000000000001,"5min":3.7080000000000002,"15min":2.722},"min":{"1min":0.26000000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":46.637999999999998,"5min":54.273000000000003,"15min":118.977},"last":1.2769999999999999}]},{"osd":1,"last update":"Mon Aug 28 02:30:11 2023","interfaces":[{"interface":"back","average":{"1min":1.78,"5min":2.524,"15min":1.669},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23699999999999999},"max":{"1min":7.2270000000000003,"5min":99.998999999999995,"15min":99.998999999999995},"last":0.54400000000000004},{"interface":"front","average":{"1min":1.9550000000000001,"5min":2.8010000000000002,"15min":1.8100000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":8.6029999999999998,"5min":95.040000000000006,"15min":95.040000000000006},"last":1.1000000000000001}]},{"osd":2,"last update":"Mon Aug 28 02:30:54 2023","interfaces":[{"interface":"back","average":{"1min":4.6020000000000003,"5min":4.5670000000000002,"15min":2.4910000000000001},"min":{"1min":0.372,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":58.758000000000003,"5min":79.787000000000006,"15min":79.787000000000006},"last":0.51900000000000002},{"interface":"front","average":{"1min":4.6900000000000004,"5min":4.7910000000000004,"15min":2.4940000000000002},"min":{"1min":0.33700000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":58.887,"5min":79.935000000000002,"15min":79.935000000000002},"last":0.92900000000000005}]},{"osd":3,"last update":"Mon Aug 28 02:30:11 2023","interfaces":[{"interface":"back","average":{"1min":1.1419999999999999,"5min":2.7850000000000001,"15min":1.6719999999999999},"min":{"1min":0.32000000000000001,"5min":0.317,"15min":0.317},"max":{"1min":3.968,"5min":39.587000000000003,"15min":39.587000000000003},"last":1.786},{"interface":"front","average":{"1min":1.0069999999999999,"5min":2.8940000000000001,"15min":1.7589999999999999},"min":{"1min":0.35299999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":3.9780000000000002,"5min":39.593000000000004,"15min":39.593000000000004},"last":1.796}]},{"osd":4,"last update":"Mon Aug 28 02:30:18 2023","interfaces":[{"interface":"back","average":{"1min":1.0069999999999999,"5min":1.224,"15min":1.04},"min":{"1min":0.27100000000000002,"5min":0.251,"15min":0.251},"max":{"1min":3.6499999999999999,"5min":18.308,"15min":18.308},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.77400000000000002,"5min":1.3899999999999999,"15min":1.131},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.26700000000000002},"max":{"1min":3.5070000000000001,"5min":18.263000000000002,"15min":18.263000000000002},"last":0.55700000000000005}]},{"osd":5,"last update":"Mon Aug 28 02:30:18 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":1.216,"15min":1.1080000000000001},"min":{"1min":0.29699999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":2.8359999999999999,"5min":9.9809999999999999,"15min":20.759},"last":0.89800000000000002},{"interface":"front","average":{"1min":0.98999999999999999,"5min":1.113,"15min":0.95599999999999996},"min":{"1min":0.314,"5min":0.314,"15min":0.28799999999999998},"max":{"1min":3.1549999999999998,"5min":5.9500000000000002,"15min":7.3780000000000001},"last":0.69799999999999995}]},{"osd":7,"last update":"Mon Aug 28 02:30:44 2023","interfaces":[{"interface":"back","average":{"1min":1.744,"5min":1.151,"15min":1.0089999999999999},"min":{"1min":0.36699999999999999,"5min":0.34200000000000003,"15min":0.314},"max":{"1min":10.605,"5min":10.605,"15min":10.605},"last":0.72699999999999998},{"interface":"front","average":{"1min":1.03,"5min":1.1499999999999999,"15min":1.0720000000000001},"min":{"1min":0.38700000000000001,"5min":0.34599999999999997,"15min":0.29799999999999999},"max":{"1min":3.4529999999999998,"5min":9.7110000000000003,"15min":9.7110000000000003},"last":0.48199999999999998}]}]},{"osd":1,"up_from":98,"seq":420906795145,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":543624,"kb_used_data":268736,"kb_used_omap":7,"kb_used_meta":274872,"kb_avail":93201528,"statfs":{"total":95995035648,"available":95438364672,"internally_reserved":0,"allocated":275185664,"data_stored":273880305,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":281469259},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:30:09 2023","interfaces":[{"interface":"back","average":{"1min":1.534,"5min":4.548,"15min":2.1379999999999999},"min":{"1min":0.20799999999999999,"5min":0.191,"15min":0.17499999999999999},"max":{"1min":8.3230000000000004,"5min":128.60599999999999,"15min":128.60599999999999},"last":1.387},{"interface":"front","average":{"1min":1.1930000000000001,"5min":4.3760000000000003,"15min":2.1480000000000001},"min":{"1min":0.221,"5min":0.188,"15min":0.188},"max":{"1min":3.8220000000000001,"5min":128.685,"15min":128.685},"last":0.91700000000000004}]},{"osd":2,"last update":"Mon Aug 28 02:30:34 2023","interfaces":[{"interface":"back","average":{"1min":2.5640000000000001,"5min":6.2939999999999996,"15min":2.9630000000000001},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":8.1720000000000006,"5min":135.25999999999999,"15min":135.25999999999999},"last":1.7230000000000001},{"interface":"front","average":{"1min":2.1339999999999999,"5min":6.1920000000000002,"15min":2.7730000000000001},"min":{"1min":0.26700000000000002,"5min":0.23499999999999999,"15min":0.221},"max":{"1min":6.7089999999999996,"5min":135.10300000000001,"15min":135.10300000000001},"last":0.34200000000000003}]},{"osd":3,"last update":"Mon Aug 28 02:30:09 2023","interfaces":[{"interface":"back","average":{"1min":1.6899999999999999,"5min":3.7010000000000001,"15min":1.843},"min":{"1min":0.33600000000000002,"5min":0.22,"15min":0.22},"max":{"1min":3.867,"5min":123.741,"15min":123.741},"last":0.97499999999999998},{"interface":"front","average":{"1min":3.9060000000000001,"5min":4.4829999999999997,"15min":2.1459999999999999},"min":{"1min":0.501,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":13.340999999999999,"5min":123.642,"15min":123.642},"last":1.5940000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:30:09 2023","interfaces":[{"interface":"back","average":{"1min":1.6930000000000001,"5min":1.7789999999999999,"15min":1.264},"min":{"1min":0.29999999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.2939999999999996,"5min":17.600000000000001,"15min":17.600000000000001},"last":1.696},{"interface":"front","average":{"1min":1.2170000000000001,"5min":1.6220000000000001,"15min":1.137},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.25600000000000001},"max":{"1min":3.5920000000000001,"5min":19.010000000000002,"15min":19.010000000000002},"last":0.46400000000000002}]},{"osd":5,"last update":"Mon Aug 28 02:30:09 2023","interfaces":[{"interface":"back","average":{"1min":2.242,"5min":1.6910000000000001,"15min":1.371},"min":{"1min":0.32100000000000001,"5min":0.312,"15min":0.29399999999999998},"max":{"1min":11.167999999999999,"5min":11.167999999999999,"15min":15.544},"last":0.94699999999999995},{"interface":"front","average":{"1min":2.2440000000000002,"5min":1.847,"15min":1.2549999999999999},"min":{"1min":0.34599999999999997,"5min":0.33600000000000002,"15min":0.33200000000000002},"max":{"1min":10.084,"5min":17.646999999999998,"15min":17.646999999999998},"last":1.6419999999999999}]},{"osd":6,"last update":"Mon Aug 28 02:30:09 2023","interfaces":[{"interface":"back","average":{"1min":1.7170000000000001,"5min":1.7090000000000001,"15min":1.4630000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":3.923,"5min":9.2989999999999995,"15min":15.512},"last":1.085},{"interface":"front","average":{"1min":1.8160000000000001,"5min":1.823,"15min":1.3899999999999999},"min":{"1min":0.39600000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":6.3140000000000001,"5min":17.673999999999999,"15min":17.673999999999999},"last":1.671}]},{"osd":7,"last update":"Mon Aug 28 02:30:09 2023","interfaces":[{"interface":"back","average":{"1min":1.4059999999999999,"5min":1.7909999999999999,"15min":1.252},"min":{"1min":0.38500000000000001,"5min":0.32500000000000001,"15min":0.30199999999999999},"max":{"1min":4.0460000000000003,"5min":19.061,"15min":19.061},"last":0.57099999999999995},{"interface":"front","average":{"1min":1.7889999999999999,"5min":1.905,"15min":1.4450000000000001},"min":{"1min":0.437,"5min":0.40500000000000003,"15min":0.33100000000000002},"max":{"1min":5.8890000000000002,"5min":9.3239999999999998,"15min":15.476000000000001},"last":0.999}]}]},{"osd":0,"up_from":90,"seq":386547056791,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":468948,"kb_used_data":121804,"kb_used_omap":8,"kb_used_meta":347127,"kb_avail":93276204,"statfs":{"total":95995035648,"available":95514832896,"internally_reserved":0,"allocated":124727296,"data_stored":123526693,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":355458409},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,1,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Aug 28 02:30:17 2023","interfaces":[{"interface":"back","average":{"1min":5.1619999999999999,"5min":3.004,"15min":1.8919999999999999},"min":{"1min":0.20000000000000001,"5min":0.184,"15min":0.184},"max":{"1min":56.576999999999998,"5min":56.576999999999998,"15min":56.576999999999998},"last":1.877},{"interface":"front","average":{"1min":6.1689999999999996,"5min":3.524,"15min":1.931},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.17000000000000001},"max":{"1min":56.552999999999997,"5min":56.552999999999997,"15min":56.552999999999997},"last":1.423}]},{"osd":2,"last update":"Mon Aug 28 02:30:34 2023","interfaces":[{"interface":"back","average":{"1min":3.234,"5min":4.3090000000000002,"15min":2.202},"min":{"1min":0.22,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":10.565,"5min":100.718,"15min":100.718},"last":0.25900000000000001},{"interface":"front","average":{"1min":1.333,"5min":4.3730000000000002,"15min":2.4159999999999999},"min":{"1min":0.24299999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":4.9989999999999997,"5min":100.962,"15min":100.962},"last":0.84399999999999997}]},{"osd":3,"last update":"Mon Aug 28 02:30:17 2023","interfaces":[{"interface":"back","average":{"1min":2.6459999999999999,"5min":6.9390000000000001,"15min":2.9430000000000001},"min":{"1min":0.24299999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":19.327000000000002,"5min":237.43899999999999,"15min":237.43899999999999},"last":1.5669999999999999},{"interface":"front","average":{"1min":1.7390000000000001,"5min":6.6399999999999997,"15min":2.859},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":6.0010000000000003,"5min":237.48099999999999,"15min":237.48099999999999},"last":1.5860000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:30:12 2023","interfaces":[{"interface":"back","average":{"1min":1.7709999999999999,"5min":1.6859999999999999,"15min":1.2849999999999999},"min":{"1min":0.29699999999999999,"5min":0.29599999999999999,"15min":0.26800000000000002},"max":{"1min":9.3680000000000003,"5min":9.3680000000000003,"15min":9.6649999999999991},"last":0.38500000000000001},{"interface":"front","average":{"1min":1.536,"5min":2.0870000000000002,"15min":1.4650000000000001},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":8.7880000000000003,"5min":27.271999999999998,"15min":27.271999999999998},"last":1.4730000000000001}]},{"osd":5,"last update":"Mon Aug 28 02:30:12 2023","interfaces":[{"interface":"back","average":{"1min":1.153,"5min":2.0099999999999998,"15min":1.5609999999999999},"min":{"1min":0.34000000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":3.0230000000000001,"5min":27.367000000000001,"15min":44.000999999999998},"last":1.5369999999999999},{"interface":"front","average":{"1min":1.671,"5min":1.9079999999999999,"15min":1.488},"min":{"1min":0.34300000000000003,"5min":0.33000000000000002,"15min":0.30599999999999999},"max":{"1min":9.4489999999999998,"5min":12.334,"15min":43.953000000000003},"last":0.438}]},{"osd":6,"last update":"Mon Aug 28 02:30:12 2023","interfaces":[{"interface":"back","average":{"1min":1.276,"5min":2.0819999999999999,"15min":1.585},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":3.0510000000000002,"5min":27.331,"15min":32.417999999999999},"last":1.508},{"interface":"front","average":{"1min":1.6930000000000001,"5min":2.173,"15min":1.5620000000000001},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.30399999999999999},"max":{"1min":9.2789999999999999,"5min":27.247,"15min":32.130000000000003},"last":0.41299999999999998}]},{"osd":7,"last update":"Mon Aug 28 02:30:12 2023","interfaces":[{"interface":"back","average":{"1min":1.802,"5min":2.4769999999999999,"15min":1.583},"min":{"1min":0.36799999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":9.4209999999999994,"5min":27.385000000000002,"15min":27.385000000000002},"last":0.47499999999999998},{"interface":"front","average":{"1min":1.5069999999999999,"5min":2.339,"15min":1.599},"min":{"1min":0.38900000000000001,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":5.7300000000000004,"5min":27.213999999999999,"15min":27.213999999999999},"last":0.81000000000000005}]}]},{"osd":2,"up_from":104,"seq":446676598915,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":633956,"kb_used_data":299612,"kb_used_omap":10,"kb_used_meta":334325,"kb_avail":93111196,"statfs":{"total":95995035648,"available":95345864704,"internally_reserved":0,"allocated":306802688,"data_stored":305643039,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10792,"internal_metadata":342349272},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:30:37 2023","interfaces":[{"interface":"back","average":{"1min":2.3050000000000002,"5min":4.21,"15min":2.2599999999999998},"min":{"1min":0.20300000000000001,"5min":0.17699999999999999,"15min":0.17599999999999999},"max":{"1min":16.48,"5min":113.89100000000001,"15min":113.89100000000001},"last":0.40500000000000003},{"interface":"front","average":{"1min":2.5499999999999998,"5min":7.5940000000000003,"15min":3.472},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.19900000000000001},"max":{"1min":11.166,"5min":147.286,"15min":147.286},"last":0.26200000000000001}]},{"osd":1,"last update":"Mon Aug 28 02:30:37 2023","interfaces":[{"interface":"back","average":{"1min":2.6560000000000001,"5min":2.431,"15min":1.738},"min":{"1min":0.23799999999999999,"5min":0.21099999999999999,"15min":0.20999999999999999},"max":{"1min":11.061,"5min":11.627000000000001,"15min":30.888000000000002},"last":1.2849999999999999},{"interface":"front","average":{"1min":3.351,"5min":2.8029999999999999,"15min":1.6890000000000001},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.20999999999999999},"max":{"1min":11.962999999999999,"5min":20.259,"15min":31.648},"last":1.087}]},{"osd":3,"last update":"Mon Aug 28 02:30:09 2023","interfaces":[{"interface":"back","average":{"1min":1.6799999999999999,"5min":5.1100000000000003,"15min":2.286},"min":{"1min":0.314,"5min":0.25,"15min":0.22700000000000001},"max":{"1min":9.7629999999999999,"5min":128.63900000000001,"15min":128.63900000000001},"last":1.204},{"interface":"front","average":{"1min":2.1070000000000002,"5min":4.9269999999999996,"15min":2.3010000000000002},"min":{"1min":0.375,"5min":0.223,"15min":0.223},"max":{"1min":8.7029999999999994,"5min":128.75700000000001,"15min":128.75700000000001},"last":1.1839999999999999}]},{"osd":4,"last update":"Mon Aug 28 02:30:37 2023","interfaces":[{"interface":"back","average":{"1min":12.464,"5min":5.617,"15min":2.661},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":207.34100000000001,"5min":207.34100000000001,"15min":207.34100000000001},"last":0.45200000000000001},{"interface":"front","average":{"1min":12.086,"5min":5.2610000000000001,"15min":2.4129999999999998},"min":{"1min":0.30299999999999999,"5min":0.28999999999999998,"15min":0.27400000000000002},"max":{"1min":207.352,"5min":207.352,"15min":207.352},"last":0.51300000000000001}]},{"osd":5,"last update":"Mon Aug 28 02:30:37 2023","interfaces":[{"interface":"back","average":{"1min":1.998,"5min":5.4800000000000004,"15min":2.5379999999999998},"min":{"1min":0.33800000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":5.8120000000000003,"5min":128.696,"15min":128.696},"last":0.439},{"interface":"front","average":{"1min":2.0249999999999999,"5min":5.1769999999999996,"15min":2.5939999999999999},"min":{"1min":0.32300000000000001,"5min":0.32100000000000001,"15min":0.30299999999999999},"max":{"1min":8.0709999999999997,"5min":128.84100000000001,"15min":128.84100000000001},"last":0.53100000000000003}]},{"osd":6,"last update":"Mon Aug 28 02:30:37 2023","interfaces":[{"interface":"back","average":{"1min":1.9370000000000001,"5min":1.79,"15min":1.3500000000000001},"min":{"1min":0.32800000000000001,"5min":0.307,"15min":0.29299999999999998},"max":{"1min":11.997,"5min":11.997,"15min":11.997},"last":0.55800000000000005},{"interface":"front","average":{"1min":1.8939999999999999,"5min":2.1299999999999999,"15min":1.494},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":5.9530000000000003,"5min":11.055999999999999,"15min":11.055999999999999},"last":0.48899999999999999}]},{"osd":7,"last update":"Mon Aug 28 02:30:37 2023","interfaces":[{"interface":"back","average":{"1min":1.827,"5min":5.3289999999999997,"15min":2.5129999999999999},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.32100000000000001},"max":{"1min":6.29,"5min":128.911,"15min":128.911},"last":0.58699999999999997},{"interface":"front","average":{"1min":1.7410000000000001,"5min":5.859,"15min":2.7120000000000002},"min":{"1min":0.35599999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":4.8819999999999997,"5min":128.72399999999999,"15min":128.72399999999999},"last":0.34899999999999998}]}]},{"osd":3,"up_from":112,"seq":481036337277,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":480224,"kb_used_data":213592,"kb_used_omap":3,"kb_used_meta":266620,"kb_avail":93264928,"statfs":{"total":95995035648,"available":95503286272,"internally_reserved":0,"allocated":218718208,"data_stored":217559854,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":273019810},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:30:05 2023","interfaces":[{"interface":"back","average":{"1min":7.0190000000000001,"5min":3.1560000000000001,"15min":3.0089999999999999},"min":{"1min":0.20000000000000001,"5min":0.193,"15min":0.193},"max":{"1min":79.793000000000006,"5min":79.793000000000006,"15min":164.25299999999999},"last":0.90000000000000002},{"interface":"front","average":{"1min":6.7839999999999998,"5min":2.8380000000000001,"15min":2.9620000000000002},"min":{"1min":0.27200000000000002,"5min":0.192,"15min":0.192},"max":{"1min":78.555000000000007,"5min":78.555000000000007,"15min":164.55000000000001},"last":1.5580000000000001}]},{"osd":1,"last update":"Mon Aug 28 02:30:05 2023","interfaces":[{"interface":"back","average":{"1min":1.669,"5min":1.907,"15min":1.8680000000000001},"min":{"1min":0.245,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":6.1500000000000004,"5min":13.936,"15min":117.34699999999999},"last":0.83299999999999996},{"interface":"front","average":{"1min":2.5310000000000001,"5min":2.1499999999999999,"15min":1.968},"min":{"1min":0.27300000000000002,"5min":0.245,"15min":0.224},"max":{"1min":10.125,"5min":14.300000000000001,"15min":118.012},"last":0.54800000000000004}]},{"osd":2,"last update":"Mon Aug 28 02:30:05 2023","interfaces":[{"interface":"back","average":{"1min":2.4199999999999999,"5min":3.157,"15min":2.8140000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.19900000000000001},"max":{"1min":10.163,"5min":65.308999999999997,"15min":120.43300000000001},"last":0.51000000000000001},{"interface":"front","average":{"1min":2.3999999999999999,"5min":2.891,"15min":2.7599999999999998},"min":{"1min":0.33500000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":7.3620000000000001,"5min":65.120999999999995,"15min":120.44499999999999},"last":1.256}]},{"osd":4,"last update":"Mon Aug 28 02:30:05 2023","interfaces":[{"interface":"back","average":{"1min":1.403,"5min":2.4980000000000002,"15min":1.4610000000000001},"min":{"1min":0.32800000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":4.7240000000000002,"5min":58.067999999999998,"15min":58.067999999999998},"last":0.94899999999999995},{"interface":"front","average":{"1min":1.6279999999999999,"5min":2.7589999999999999,"15min":1.5269999999999999},"min":{"1min":0.40400000000000003,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":4.6280000000000001,"5min":58.088000000000001,"15min":58.088000000000001},"last":0.76500000000000001}]},{"osd":5,"last update":"Mon Aug 28 02:30:05 2023","interfaces":[{"interface":"back","average":{"1min":1.359,"5min":3.2109999999999999,"15min":1.6579999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.29599999999999999},"max":{"1min":2.923,"5min":102.904,"15min":102.904},"last":0.79700000000000004},{"interface":"front","average":{"1min":1.556,"5min":3.2509999999999999,"15min":1.6619999999999999},"min":{"1min":0.36699999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":7.0039999999999996,"5min":103.283,"15min":103.283},"last":0.59899999999999998}]},{"osd":6,"last update":"Mon Aug 28 02:30:05 2023","interfaces":[{"interface":"back","average":{"1min":1.5880000000000001,"5min":1.877,"15min":1.4330000000000001},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.314},"max":{"1min":7.0300000000000002,"5min":9.4529999999999994,"15min":72.116},"last":0.57399999999999995},{"interface":"front","average":{"1min":1.7929999999999999,"5min":1.538,"15min":1.3560000000000001},"min":{"1min":0.35599999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":7.2990000000000004,"5min":7.2990000000000004,"15min":72.030000000000001},"last":1.29}]},{"osd":7,"last update":"Mon Aug 28 02:30:05 2023","interfaces":[{"interface":"back","average":{"1min":1.6879999999999999,"5min":2.9660000000000002,"15min":1.631},"min":{"1min":0.38300000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":4.7640000000000002,"5min":54.427999999999997,"15min":54.427999999999997},"last":1.0820000000000001},{"interface":"front","average":{"1min":1.855,"5min":3.3399999999999999,"15min":1.76},"min":{"1min":0.433,"5min":0.36399999999999999,"15min":0.34799999999999998},"max":{"1min":3.8100000000000001,"5min":54.445,"15min":54.445},"last":1.002}]}]},{"osd":4,"up_from":30,"seq":128849019151,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":578220,"kb_used_data":268708,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93166932,"statfs":{"total":95995035648,"available":95402938368,"internally_reserved":0,"allocated":275156992,"data_stored":274446708,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:30:10 2023","interfaces":[{"interface":"back","average":{"1min":1.528,"5min":2.8479999999999999,"15min":2.706},"min":{"1min":0.222,"5min":0.219,"15min":0.219},"max":{"1min":13.492000000000001,"5min":43.548000000000002,"15min":138.67400000000001},"last":0.79200000000000004},{"interface":"front","average":{"1min":1.4850000000000001,"5min":2.7149999999999999,"15min":2.637},"min":{"1min":0.251,"5min":0.248,"15min":0.248},"max":{"1min":8.0060000000000002,"5min":43.515000000000001,"15min":134.14599999999999},"last":1.119}]},{"osd":1,"last update":"Mon Aug 28 02:30:10 2023","interfaces":[{"interface":"back","average":{"1min":1.5629999999999999,"5min":2.1640000000000001,"15min":1.3919999999999999},"min":{"1min":0.23599999999999999,"5min":0.221,"15min":0.221},"max":{"1min":9.75,"5min":34.945999999999998,"15min":53.622999999999998},"last":0.59899999999999998},{"interface":"front","average":{"1min":1.4039999999999999,"5min":2.073,"15min":1.4079999999999999},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":5.2919999999999998,"5min":34.869999999999997,"15min":53.774000000000001},"last":0.89100000000000001}]},{"osd":2,"last update":"Mon Aug 28 02:30:37 2023","interfaces":[{"interface":"back","average":{"1min":4.867,"5min":4.3040000000000003,"15min":2.8250000000000002},"min":{"1min":0.33100000000000002,"5min":0.27400000000000002,"15min":0.25600000000000001},"max":{"1min":56.948999999999998,"5min":64.063000000000002,"15min":74.099000000000004},"last":0.84599999999999997},{"interface":"front","average":{"1min":5.117,"5min":4.1769999999999996,"15min":2.7330000000000001},"min":{"1min":0.377,"5min":0.28999999999999998,"15min":0.25},"max":{"1min":56.979999999999997,"5min":64.073999999999998,"15min":74.081999999999994},"last":0.63800000000000001}]},{"osd":3,"last update":"Mon Aug 28 02:30:10 2023","interfaces":[{"interface":"back","average":{"1min":11.220000000000001,"5min":6.0819999999999999,"15min":2.5099999999999998},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":159.10400000000001,"5min":159.10400000000001,"15min":159.10400000000001},"last":2.2730000000000001},{"interface":"front","average":{"1min":11.119999999999999,"5min":5.9939999999999998,"15min":2.5979999999999999},"min":{"1min":0.33800000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":159.232,"5min":159.232,"15min":159.232},"last":0.57099999999999995}]},{"osd":5,"last update":"Mon Aug 28 02:30:51 2023","interfaces":[{"interface":"back","average":{"1min":1.325,"5min":2.8929999999999998,"15min":1.454},"min":{"1min":0.28299999999999997,"5min":0.246,"15min":0.246},"max":{"1min":5.2309999999999999,"5min":124.718,"15min":124.718},"last":0.36099999999999999},{"interface":"front","average":{"1min":1.7809999999999999,"5min":3.0910000000000002,"15min":1.641},"min":{"1min":0.34699999999999998,"5min":0.247,"15min":0.247},"max":{"1min":10.24,"5min":124.694,"15min":124.694},"last":0.68400000000000005}]},{"osd":6,"last update":"Mon Aug 28 02:30:10 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":1.264,"15min":0.94699999999999995},"min":{"1min":0.28499999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":2.6659999999999999,"5min":11.170999999999999,"15min":11.170999999999999},"last":0.498},{"interface":"front","average":{"1min":0.91500000000000004,"5min":1.1180000000000001,"15min":0.83299999999999996},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":5.2679999999999998,"5min":7.3170000000000002,"15min":7.3170000000000002},"last":0.44}]},{"osd":7,"last update":"Mon Aug 28 02:30:27 2023","interfaces":[{"interface":"back","average":{"1min":1.651,"5min":1.6990000000000001,"15min":1.1180000000000001},"min":{"1min":0.312,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":8.3490000000000002,"5min":16.584,"15min":16.584},"last":0.33800000000000002},{"interface":"front","average":{"1min":1.706,"5min":1.3919999999999999,"15min":1.137},"min":{"1min":0.378,"5min":0.311,"15min":0.311},"max":{"1min":7.4619999999999997,"5min":10.826000000000001,"15min":10.826000000000001},"last":0.53600000000000003}]}]},{"osd":5,"up_from":36,"seq":154618822923,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":478748,"kb_used_data":181524,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":93266404,"statfs":{"total":95995035648,"available":95504797696,"internally_reserved":0,"allocated":185880576,"data_stored":185181831,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:30:02 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":2.052,"15min":2.6469999999999998},"min":{"1min":0.214,"5min":0.214,"15min":0.20300000000000001},"max":{"1min":4.548,"5min":42.229999999999997,"15min":131.06100000000001},"last":0.502},{"interface":"front","average":{"1min":1.4710000000000001,"5min":2.371,"15min":2.8460000000000001},"min":{"1min":0.27500000000000002,"5min":0.24199999999999999,"15min":0.22900000000000001},"max":{"1min":5.8129999999999997,"5min":42.213999999999999,"15min":131.262},"last":0.60999999999999999}]},{"osd":1,"last update":"Mon Aug 28 02:30:12 2023","interfaces":[{"interface":"back","average":{"1min":1.8819999999999999,"5min":1.7869999999999999,"15min":2.387},"min":{"1min":0.27400000000000002,"5min":0.216,"15min":0.216},"max":{"1min":7.7809999999999997,"5min":30.593,"15min":183.97900000000001},"last":0.92900000000000005},{"interface":"front","average":{"1min":1.1220000000000001,"5min":2.0070000000000001,"15min":2.5510000000000002},"min":{"1min":0.23499999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":5.8810000000000002,"5min":34.616999999999997,"15min":184.33699999999999},"last":0.54200000000000004}]},{"osd":2,"last update":"Mon Aug 28 02:30:41 2023","interfaces":[{"interface":"back","average":{"1min":2.7469999999999999,"5min":6.117,"15min":3.9740000000000002},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.252},"max":{"1min":11.875,"5min":176.69499999999999,"15min":176.69499999999999},"last":0.58999999999999997},{"interface":"front","average":{"1min":3.7160000000000002,"5min":6.5620000000000003,"15min":4.21},"min":{"1min":0.35299999999999998,"5min":0.34000000000000002,"15min":0.28299999999999997},"max":{"1min":11.887,"5min":176.71199999999999,"15min":176.71199999999999},"last":0.66500000000000004}]},{"osd":3,"last update":"Mon Aug 28 02:30:12 2023","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":2.8730000000000002,"15min":1.6180000000000001},"min":{"1min":0.28799999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.1070000000000002,"5min":102.64400000000001,"15min":102.64400000000001},"last":0.38300000000000001},{"interface":"front","average":{"1min":1.8919999999999999,"5min":3.6509999999999998,"15min":1.956},"min":{"1min":0.28699999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":8.1660000000000004,"5min":103.771,"15min":103.771},"last":0.57199999999999995}]},{"osd":4,"last update":"Mon Aug 28 02:30:52 2023","interfaces":[{"interface":"back","average":{"1min":6.1349999999999998,"5min":2.0990000000000002,"15min":1.3100000000000001},"min":{"1min":0.33300000000000002,"5min":0.26700000000000002,"15min":0.246},"max":{"1min":85.769999999999996,"5min":85.769999999999996,"15min":85.769999999999996},"last":0.99199999999999999},{"interface":"front","average":{"1min":5.7480000000000002,"5min":1.9630000000000001,"15min":1.3120000000000001},"min":{"1min":0.31,"5min":0.28999999999999998,"15min":0.28000000000000003},"max":{"1min":85.825000000000003,"5min":85.825000000000003,"15min":85.825000000000003},"last":0.41799999999999998}]},{"osd":6,"last update":"Mon Aug 28 02:30:12 2023","interfaces":[{"interface":"back","average":{"1min":1.0429999999999999,"5min":1.099,"15min":1.002},"min":{"1min":0.30399999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.6210000000000004,"5min":5.9870000000000001,"15min":14.803000000000001},"last":0.96399999999999997},{"interface":"front","average":{"1min":0.95499999999999996,"5min":1.04,"15min":0.91600000000000004},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":4.9509999999999996,"5min":8.4779999999999998,"15min":8.6479999999999997},"last":0.46300000000000002}]},{"osd":7,"last update":"Mon Aug 28 02:30:29 2023","interfaces":[{"interface":"back","average":{"1min":1.1419999999999999,"5min":1.5109999999999999,"15min":1.127},"min":{"1min":0.34300000000000003,"5min":0.33300000000000002,"15min":0.31},"max":{"1min":4.0060000000000002,"5min":25.379000000000001,"15min":25.379000000000001},"last":1.3540000000000001},{"interface":"front","average":{"1min":1.3160000000000001,"5min":1.2769999999999999,"15min":1.0569999999999999},"min":{"1min":0.38300000000000001,"5min":0.33700000000000002,"15min":0.32100000000000001},"max":{"1min":5.931,"5min":25.437000000000001,"15min":25.437000000000001},"last":0.73599999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":170655744,"data_stored":170655744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":172032000,"data_stored":172032000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":141361152,"data_stored":141361152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":221773824,"data_stored":221773824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":278396928,"data_stored":278396928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":216072192,"data_stored":216072192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":278462464,"data_stored":278462464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":290127872,"data_stored":290127872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-08-28T02:31:11.218 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-08-28T02:31:11.218 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd pool set device_health_metrics pg_num 11 2023-08-28T02:31:11.242 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:31:11.242 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:31:11.274 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:31:11.429 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:11.681 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:31:11.690 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:11 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:31:11.691 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:11 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:31:11.691 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:11 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:31:11.691 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:11 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:31:11.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:11.692 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:31:11.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:11 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:31:11.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:11 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:31:11.793 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:31:11.819 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 10 16 21372 21356 133.31 97.0625 0.00798887 0.00749467 2023-08-28T02:31:11.828 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:11.868 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:31:11.952 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:31:11 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:31:11] "GET /metrics HTTP/1.1" 200 36457 "" "Prometheus/2.33.4" 2023-08-28T02:31:12.231 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:31:12.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:12.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:31:12.335 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:31:12.365 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:31:12.366 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.0 dump_ops_in_flight 2023-08-28T02:31:12.388 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:12.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:12 smithi169 ceph-mon[159415]: pgmap v647: 89 pgs: 89 active+clean; 1000 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s 2023-08-28T02:31:12.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:12 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1465363901' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:31:12.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:12 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:31:12.508 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:31:12.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:12.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:31:12.524 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is unset 2023-08-28T02:31:12.617 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:31:12.658 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:12.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:12 smithi143 ceph-mon[189871]: pgmap v647: 89 pgs: 89 active+clean; 1000 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s 2023-08-28T02:31:12.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:12 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1465363901' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:31:12.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:12 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:31:12.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:12 smithi143 ceph-mon[183059]: pgmap v647: 89 pgs: 89 active+clean; 1000 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s 2023-08-28T02:31:12.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:12 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1465363901' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:31:12.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:12 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:31:12.776 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:31:12.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:12.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:31:12.821 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 11 16 22999 22983 130.413 101.688 0.00543326 0.00766044 2023-08-28T02:31:12.880 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:31:12.947 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:13.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:13 smithi169 ceph-mon[159415]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:31:13.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:13 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:31:13.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:13 smithi169 ceph-mon[159415]: osdmap e266: 8 total, 8 up, 8 in 2023-08-28T02:31:13.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:13 smithi169 ceph-mon[159415]: failed to encode map e266 with expected crc 2023-08-28T02:31:13.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:13 smithi169 ceph-mon[159415]: failed to encode map e266 with expected crc 2023-08-28T02:31:13.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:13 smithi169 ceph-mon[159415]: failed to encode map e266 with expected crc 2023-08-28T02:31:13.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:13 smithi169 ceph-mon[159415]: failed to encode map e266 with expected crc 2023-08-28T02:31:13.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:13 smithi169 ceph-mon[159415]: failed to encode map e266 with expected crc 2023-08-28T02:31:13.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:13 smithi169 ceph-mon[159415]: failed to encode map e266 with expected crc 2023-08-28T02:31:13.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:13 smithi169 ceph-mon[159415]: failed to encode map e266 with expected crc 2023-08-28T02:31:13.822 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 12 16 24983 24967 129.878 124 0.00775189 0.00769475 2023-08-28T02:31:13.888 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:13 smithi143 ceph-mon[183059]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:31:13.888 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:13 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:31:13.888 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:13 smithi143 ceph-mon[183059]: osdmap e266: 8 total, 8 up, 8 in 2023-08-28T02:31:13.888 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:13 smithi143 ceph-mon[183059]: failed to encode map e266 with expected crc 2023-08-28T02:31:13.888 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:13 smithi143 ceph-mon[183059]: failed to encode map e266 with expected crc 2023-08-28T02:31:13.888 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:13 smithi143 ceph-mon[183059]: failed to encode map e266 with expected crc 2023-08-28T02:31:13.889 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:13 smithi143 ceph-mon[183059]: failed to encode map e266 with expected crc 2023-08-28T02:31:13.889 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:13 smithi143 ceph-mon[183059]: failed to encode map e266 with expected crc 2023-08-28T02:31:13.889 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:13 smithi143 ceph-mon[183059]: failed to encode map e266 with expected crc 2023-08-28T02:31:13.889 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:13 smithi143 ceph-mon[183059]: failed to encode map e266 with expected crc 2023-08-28T02:31:13.889 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:13 smithi143 ceph-mon[189871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:31:13.889 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:13 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:31:13.889 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:13 smithi143 ceph-mon[189871]: osdmap e266: 8 total, 8 up, 8 in 2023-08-28T02:31:13.889 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:13 smithi143 ceph-mon[189871]: failed to encode map e266 with expected crc 2023-08-28T02:31:13.889 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:13 smithi143 ceph-mon[189871]: failed to encode map e266 with expected crc 2023-08-28T02:31:13.890 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:13 smithi143 ceph-mon[189871]: failed to encode map e266 with expected crc 2023-08-28T02:31:13.890 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:13 smithi143 ceph-mon[189871]: failed to encode map e266 with expected crc 2023-08-28T02:31:13.890 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:13 smithi143 ceph-mon[189871]: failed to encode map e266 with expected crc 2023-08-28T02:31:13.890 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:13 smithi143 ceph-mon[189871]: failed to encode map e266 with expected crc 2023-08-28T02:31:13.890 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:13 smithi143 ceph-mon[189871]: failed to encode map e266 with expected crc 2023-08-28T02:31:14.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:14 smithi169 ceph-mon[159415]: pgmap v649: 89 pgs: 89 active+clean; 1.2 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2023-08-28T02:31:14.754 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:14 smithi143 ceph-mon[189871]: pgmap v649: 89 pgs: 89 active+clean; 1.2 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2023-08-28T02:31:14.755 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:14 smithi143 ceph-mon[183059]: pgmap v649: 89 pgs: 89 active+clean; 1.2 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2023-08-28T02:31:14.799 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:31:14.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:14.806 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:31:14.821 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 13 16 26353 26337 126.477 85.625 0.00765929 0.00789974 2023-08-28T02:31:14.907 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:31:14.973 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:15.132 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:31:15.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:15.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:31:15.241 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:31:15.320 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:15.822 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 14 16 28009 27993 124.837 103.5 0.0102848 0.00800307 2023-08-28T02:31:15.968 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:31:15.968 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:31:15.968 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:31:15.968 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:31:15.968 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:31:15.988 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:31:16.015 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:31:16.020 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:31:16.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:16.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:31:16.047 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:31:16.047 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:31:16.048 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:31:16.048 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:31:16.048 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:31:16.048 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:31:16.048 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:31:16.048 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:31:16.048 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:31:16.126 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:31:16.154 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:16.206 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:31:16.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:16.217 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:31:16.317 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:31:16.373 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:16.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:16 smithi143 ceph-mon[183059]: pgmap v650: 89 pgs: 89 active+clean; 1.5 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 126 MiB/s wr, 2.02k op/s 2023-08-28T02:31:16.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:16 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2264067259' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:31:16.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:16 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:31:16.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:16 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:31:16.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:16 smithi143 ceph-mon[189871]: pgmap v650: 89 pgs: 89 active+clean; 1.5 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 126 MiB/s wr, 2.02k op/s 2023-08-28T02:31:16.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:16 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2264067259' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:31:16.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:16 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:31:16.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:16 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:31:16.755 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset nodeep-scrub 2023-08-28T02:31:16.784 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:31:16.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:16.801 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:31:16.823 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 15 16 29362 29346 122.141 84.5625 0.0133554 0.00818053 2023-08-28T02:31:16.902 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:31:16.936 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:16 smithi169 ceph-mon[159415]: pgmap v650: 89 pgs: 89 active+clean; 1.5 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 126 MiB/s wr, 2.02k op/s 2023-08-28T02:31:16.936 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:16 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2264067259' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:31:16.936 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:16 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:31:16.936 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:16 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:31:16.943 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:17.005 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:31:17.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:17.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:31:17.113 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:31:17.143 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:17.564 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:31:17.565 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:31:17.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:17.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:31:17.680 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:31:17.738 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:17.824 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 16 16 30786 30770 120.066 89 0.00985783 0.00832006 2023-08-28T02:31:17.996 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:31:18.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:18.003 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:31:18.103 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:31:18.185 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:18.267 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:31:18.360 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:31:18.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:18.367 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:31:18.468 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:31:18.533 INFO:teuthology.orchestra.run.smithi143.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-08-28T02:31:18.534 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:18.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:18 smithi143 ceph-mon[183059]: pgmap v651: 89 pgs: 89 active+clean; 1.6 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-08-28T02:31:18.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:18 smithi143 ceph-mon[189871]: pgmap v651: 89 pgs: 89 active+clean; 1.6 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-08-28T02:31:18.824 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 17 16 32221 32205 118.28 89.6875 0.0039839 0.00844658 2023-08-28T02:31:18.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:18 smithi169 ceph-mon[159415]: pgmap v651: 89 pgs: 89 active+clean; 1.6 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-08-28T02:31:19.589 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:31:19.824 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 18 16 33386 33370 115.755 72.8125 0.00481057 0.0086337 2023-08-28T02:31:19.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:19 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3592104573' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-08-28T02:31:19.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:19 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-08-28T02:31:19.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:19 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3592104573' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-08-28T02:31:19.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:19 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-08-28T02:31:19.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:19 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3592104573' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-08-28T02:31:19.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:19 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-08-28T02:31:20.107 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:31:20.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:20.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:31:20.213 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:31:20.310 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:20.343 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-08-28T02:31:20.417 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:31:20.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:20.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:31:20.536 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:31:20.572 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:20.638 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:31:20.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:20.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:31:20.687 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:20 smithi143 ceph-mon[183059]: pgmap v652: 89 pgs: 89 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-08-28T02:31:20.688 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:20 smithi143 ceph-mon[189871]: pgmap v652: 89 pgs: 89 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-08-28T02:31:20.744 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:31:20.777 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:20 smithi169 ceph-mon[159415]: pgmap v652: 89 pgs: 89 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-08-28T02:31:20.780 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:20.826 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 19 16 34918 34902 114.694 95.75 0.0178352 0.00871009 2023-08-28T02:31:20.834 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:31:20.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:20.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:31:20.939 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:31:20.974 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:21.031 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:31:21.031 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [ 2023-08-28T02:31:21.031 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.031 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.55270.0:35180 7.6 7:6a64f208:::benchmark_data_smithi143_360738_object35179:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e266)", 2023-08-28T02:31:21.032 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:31:21.017535+0000", 2023-08-28T02:31:21.032 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.0049448560000000001, 2023-08-28T02:31:21.032 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0049802249999999996, 2023-08-28T02:31:21.032 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:31:21.032 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "waiting for sub ops", 2023-08-28T02:31:21.032 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:31:21.032 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.55270", 2023-08-28T02:31:21.033 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:31:21.033 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 35180 2023-08-28T02:31:21.033 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.033 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:31:21.033 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.033 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:31:21.033 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.017535+0000", 2023-08-28T02:31:21.034 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:21.034 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.034 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.034 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:31:21.034 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.017535+0000", 2023-08-28T02:31:21.034 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:21.034 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.034 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.035 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:31:21.035 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.017534+0000", 2023-08-28T02:31:21.035 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:31:21.035 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.035 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.035 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:31:21.035 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.017556+0000", 2023-08-28T02:31:21.036 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.1793999999999998e-05 2023-08-28T02:31:21.036 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.036 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.036 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:31:21.036 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.017565+0000", 2023-08-28T02:31:21.036 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.5729999999999997e-06 2023-08-28T02:31:21.036 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.036 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.037 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:31:21.037 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.017572+0000", 2023-08-28T02:31:21.037 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.8449999999999997e-06 2023-08-28T02:31:21.037 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.037 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.037 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:21.037 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.017905+0000", 2023-08-28T02:31:21.037 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00033320700000000002 2023-08-28T02:31:21.038 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.038 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.038 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:31:21.038 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.018052+0000", 2023-08-28T02:31:21.038 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00014728199999999999 2023-08-28T02:31:21.038 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.038 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.038 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 3,7", 2023-08-28T02:31:21.039 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.018195+0000", 2023-08-28T02:31:21.039 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000142433 2023-08-28T02:31:21.039 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:21.039 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:21.039 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:21.039 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.039 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.040 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop_reply(client.55270.0:35175 7.6 e266/261)", 2023-08-28T02:31:21.040 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:31:21.019512+0000", 2023-08-28T02:31:21.040 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.002968203, 2023-08-28T02:31:21.040 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0030854340000000002, 2023-08-28T02:31:21.040 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:31:21.041 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "queued for pg", 2023-08-28T02:31:21.041 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:31:21.041 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.041 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:31:21.041 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.019512+0000", 2023-08-28T02:31:21.042 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:21.042 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.042 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.042 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:31:21.042 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.019512+0000", 2023-08-28T02:31:21.042 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:21.043 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.043 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.043 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:31:21.043 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.019511+0000", 2023-08-28T02:31:21.043 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:31:21.043 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.044 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.044 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:31:21.044 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.019515+0000", 2023-08-28T02:31:21.044 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.2520000000000001e-06 2023-08-28T02:31:21.044 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.045 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.045 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:31:21.045 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.019530+0000", 2023-08-28T02:31:21.045 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.505e-05 2023-08-28T02:31:21.045 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.045 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.045 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:31:21.045 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.019538+0000", 2023-08-28T02:31:21.045 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.7419999999999997e-06 2023-08-28T02:31:21.045 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:21.046 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:21.046 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:21.046 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.046 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.046 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.55270.0:35182 7.4 7:2733d838:::benchmark_data_smithi143_360738_object35181:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e266)", 2023-08-28T02:31:21.046 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:31:21.019567+0000", 2023-08-28T02:31:21.046 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.0029129249999999998, 2023-08-28T02:31:21.046 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0030862289999999998, 2023-08-28T02:31:21.046 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:31:21.046 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "waiting for sub ops", 2023-08-28T02:31:21.047 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:31:21.047 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.55270", 2023-08-28T02:31:21.047 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:31:21.047 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 35182 2023-08-28T02:31:21.047 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.047 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:31:21.047 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.047 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:31:21.047 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.019567+0000", 2023-08-28T02:31:21.047 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:21.047 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.048 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.048 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:31:21.048 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.019567+0000", 2023-08-28T02:31:21.048 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:21.048 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.048 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.048 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:31:21.048 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.019566+0000", 2023-08-28T02:31:21.048 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:31:21.048 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.048 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.048 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:31:21.048 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.019590+0000", 2023-08-28T02:31:21.049 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.4229000000000001e-05 2023-08-28T02:31:21.049 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.049 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.049 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:31:21.049 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.019605+0000", 2023-08-28T02:31:21.049 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.4418e-05 2023-08-28T02:31:21.049 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.049 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.049 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:31:21.049 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.019612+0000", 2023-08-28T02:31:21.049 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.4529999999999996e-06 2023-08-28T02:31:21.050 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.050 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.050 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:21.050 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.019757+0000", 2023-08-28T02:31:21.050 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00014433499999999999 2023-08-28T02:31:21.050 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.050 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.050 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:31:21.050 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.019892+0000", 2023-08-28T02:31:21.050 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000135339 2023-08-28T02:31:21.050 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.050 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.050 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,4", 2023-08-28T02:31:21.051 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.020039+0000", 2023-08-28T02:31:21.051 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00014657400000000001 2023-08-28T02:31:21.051 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.051 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.051 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:31:21.051 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.021982+0000", 2023-08-28T02:31:21.051 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00194341 2023-08-28T02:31:21.051 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:21.051 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:21.051 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:21.051 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.052 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.052 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.55270.0:35181 7.e e266/261)", 2023-08-28T02:31:21.052 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:31:21.019569+0000", 2023-08-28T02:31:21.052 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.0029109800000000001, 2023-08-28T02:31:21.052 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0031643069999999999, 2023-08-28T02:31:21.052 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:31:21.052 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "queued for pg", 2023-08-28T02:31:21.052 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:31:21.052 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.053 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:31:21.053 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.019569+0000", 2023-08-28T02:31:21.053 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:21.053 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.054 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.054 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:31:21.054 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.019569+0000", 2023-08-28T02:31:21.054 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:21.054 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.054 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.054 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:31:21.054 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.019568+0000", 2023-08-28T02:31:21.055 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:31:21.055 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.055 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.055 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:31:21.055 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.019604+0000", 2023-08-28T02:31:21.055 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.5345000000000001e-05 2023-08-28T02:31:21.055 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.056 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.056 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:31:21.056 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.019619+0000", 2023-08-28T02:31:21.056 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.5546e-05 2023-08-28T02:31:21.056 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.056 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.056 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:31:21.056 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.019630+0000", 2023-08-28T02:31:21.057 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0552e-05 2023-08-28T02:31:21.057 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:21.057 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:21.057 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:21.057 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.057 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.057 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.55270.0:35185 7.6 7.e9ee2816 (undecoded) ondisk+write+known_if_redirected e266)", 2023-08-28T02:31:21.058 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:31:21.019626+0000", 2023-08-28T02:31:21.058 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.0028542429999999998, 2023-08-28T02:31:21.058 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0031594850000000001, 2023-08-28T02:31:21.058 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:31:21.058 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "queued for pg", 2023-08-28T02:31:21.058 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:31:21.058 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.55270", 2023-08-28T02:31:21.058 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:31:21.059 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 35185 2023-08-28T02:31:21.059 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.059 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:31:21.059 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.059 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:31:21.059 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.019626+0000", 2023-08-28T02:31:21.059 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:21.060 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.060 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.060 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:31:21.060 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.019626+0000", 2023-08-28T02:31:21.060 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:21.060 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.060 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.060 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:31:21.061 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.019625+0000", 2023-08-28T02:31:21.061 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:31:21.061 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.061 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.061 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:31:21.061 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.019648+0000", 2023-08-28T02:31:21.061 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.3105999999999999e-05 2023-08-28T02:31:21.061 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.062 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.062 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:31:21.062 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.019656+0000", 2023-08-28T02:31:21.062 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.0760000000000003e-06 2023-08-28T02:31:21.062 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.062 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.062 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:31:21.062 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.019662+0000", 2023-08-28T02:31:21.063 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.8529999999999997e-06 2023-08-28T02:31:21.063 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:21.063 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:21.063 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:21.063 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.063 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.063 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.55270.0:35184 7.e e266/261)", 2023-08-28T02:31:21.063 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:31:21.019906+0000", 2023-08-28T02:31:21.064 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.002573479, 2023-08-28T02:31:21.064 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.002933382, 2023-08-28T02:31:21.064 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:31:21.064 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "queued for pg", 2023-08-28T02:31:21.064 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:31:21.064 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.064 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:31:21.065 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.019906+0000", 2023-08-28T02:31:21.065 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:21.065 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.065 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.065 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:31:21.065 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.019906+0000", 2023-08-28T02:31:21.065 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:21.065 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.066 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.066 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:31:21.066 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.019906+0000", 2023-08-28T02:31:21.066 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:31:21.066 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.066 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.066 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:31:21.066 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.019927+0000", 2023-08-28T02:31:21.067 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.1339e-05 2023-08-28T02:31:21.067 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.067 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.067 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:31:21.067 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.019937+0000", 2023-08-28T02:31:21.067 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.9229999999999997e-06 2023-08-28T02:31:21.067 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.067 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.068 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:31:21.068 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.019944+0000", 2023-08-28T02:31:21.068 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.4760000000000003e-06 2023-08-28T02:31:21.068 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:21.068 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:21.068 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:21.068 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.068 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.069 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop_reply(client.55270.0:35180 7.6 e266/261)", 2023-08-28T02:31:21.069 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:31:21.020051+0000", 2023-08-28T02:31:21.069 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.0024287530000000001, 2023-08-28T02:31:21.069 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0028360640000000001, 2023-08-28T02:31:21.069 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:31:21.069 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "queued for pg", 2023-08-28T02:31:21.069 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:31:21.069 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.070 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:31:21.070 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.020051+0000", 2023-08-28T02:31:21.070 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:21.070 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.070 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.070 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:31:21.070 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.020051+0000", 2023-08-28T02:31:21.070 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:21.071 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.071 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.071 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:31:21.071 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.020050+0000", 2023-08-28T02:31:21.071 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:31:21.071 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.072 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.072 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:31:21.072 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.020056+0000", 2023-08-28T02:31:21.072 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.4090000000000002e-06 2023-08-28T02:31:21.072 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.072 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.072 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:31:21.072 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.020066+0000", 2023-08-28T02:31:21.072 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0351e-05 2023-08-28T02:31:21.073 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:21.073 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:21.073 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:31:21.073 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.020073+0000", 2023-08-28T02:31:21.073 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.6640000000000001e-06 2023-08-28T02:31:21.073 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:21.073 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:21.073 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:21.073 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:21.073 INFO:teuthology.orchestra.run.smithi143.stdout: ], 2023-08-28T02:31:21.073 INFO:teuthology.orchestra.run.smithi143.stdout: "num_ops": 7 2023-08-28T02:31:21.073 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:31:21.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:21.075 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:31:21.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:31:21.176 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:31:21.212 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:21.274 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:31:21.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:21.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:31:21.382 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:31:21.425 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:21.492 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:31:21.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:21.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:31:21.599 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:31:21.628 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:21.826 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:2023-08-28T02:31:21.827764+0000 min lat: 0.00114956 max lat: 0.255681 avg lat: 0.0087778 2023-08-28T02:31:21.826 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-08-28T02:31:21.826 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 20 16 36181 36165 112.907 78.9375 0.00241714 0.0087778 2023-08-28T02:31:21.929 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:31:21 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:31:21] "GET /metrics HTTP/1.1" 200 36457 "" "Prometheus/2.33.4" 2023-08-28T02:31:22.517 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:31:22.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:22.523 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:31:22.624 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:31:22.665 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:22 smithi143 ceph-mon[183059]: pgmap v653: 89 pgs: 89 active+clean; 1.9 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-08-28T02:31:22.666 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:22 smithi143 ceph-mon[189871]: pgmap v653: 89 pgs: 89 active+clean; 1.9 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-08-28T02:31:22.709 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:22.722 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:31:22.724 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.0 dump_blocked_ops 2023-08-28T02:31:22.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:22 smithi169 ceph-mon[159415]: pgmap v653: 89 pgs: 89 active+clean; 1.9 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-08-28T02:31:22.814 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:31:22.826 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 21 16 37389 37373 111.127 75.5 0.0282202 0.0089889 2023-08-28T02:31:22.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:22.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:31:22.933 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:31:22.967 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:23.024 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:31:23.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:23.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:31:23.132 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:31:23.167 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:23.220 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:31:23.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:23.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:31:23.326 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:31:23.457 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:23.826 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 22 16 38704 38688 109.812 82.1875 0.01032 0.00910031 2023-08-28T02:31:24.428 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is unset 2023-08-28T02:31:24.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:24 smithi143 ceph-mon[183059]: pgmap v654: 89 pgs: 89 active+clean; 2.1 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-08-28T02:31:24.781 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:24 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/604959542' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:31:24.781 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:24 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:31:24.781 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:24 smithi169 ceph-mon[159415]: pgmap v654: 89 pgs: 89 active+clean; 2.1 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-08-28T02:31:24.782 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:24 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/604959542' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:31:24.782 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:24 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:31:24.782 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:24 smithi143 ceph-mon[189871]: pgmap v654: 89 pgs: 89 active+clean; 2.1 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-08-28T02:31:24.782 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:24 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/604959542' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:31:24.782 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:24 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:31:24.827 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 23 16 39942 39926 108.403 77.375 0.0128195 0.00922014 2023-08-28T02:31:24.897 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:31:24.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:24.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:31:25.005 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:31:25.043 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:25.115 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:31:25.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:25.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:31:25.222 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:31:25.304 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:25.344 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 7, 3, 1, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-08-28T02:31:25.344 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-08-28T02:31:25.344 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_2 2023-08-28T02:31:25.345 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph pg dump --format=json 2023-08-28T02:31:25.540 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:31:25.553 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:25 smithi143 ceph-mon[183059]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:31:25.553 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:25 smithi143 ceph-mon[183059]: Cluster is now healthy 2023-08-28T02:31:25.553 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:25 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:31:25.553 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:25 smithi143 ceph-mon[183059]: osdmap e267: 8 total, 8 up, 8 in 2023-08-28T02:31:25.553 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:25 smithi143 ceph-mon[183059]: failed to encode map e267 with expected crc 2023-08-28T02:31:25.553 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:25 smithi143 ceph-mon[183059]: failed to encode map e267 with expected crc 2023-08-28T02:31:25.553 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:25 smithi143 ceph-mon[183059]: failed to encode map e267 with expected crc 2023-08-28T02:31:25.553 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:25 smithi143 ceph-mon[183059]: failed to encode map e267 with expected crc 2023-08-28T02:31:25.553 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:25 smithi143 ceph-mon[183059]: failed to encode map e267 with expected crc 2023-08-28T02:31:25.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:25.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:31:25.555 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:25 smithi143 ceph-mon[189871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:31:25.556 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:25 smithi143 ceph-mon[189871]: Cluster is now healthy 2023-08-28T02:31:25.556 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:25 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:31:25.556 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:25 smithi143 ceph-mon[189871]: osdmap e267: 8 total, 8 up, 8 in 2023-08-28T02:31:25.556 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:25 smithi143 ceph-mon[189871]: failed to encode map e267 with expected crc 2023-08-28T02:31:25.556 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:25 smithi143 ceph-mon[189871]: failed to encode map e267 with expected crc 2023-08-28T02:31:25.556 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:25 smithi143 ceph-mon[189871]: failed to encode map e267 with expected crc 2023-08-28T02:31:25.556 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:25 smithi143 ceph-mon[189871]: failed to encode map e267 with expected crc 2023-08-28T02:31:25.556 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:25 smithi143 ceph-mon[189871]: failed to encode map e267 with expected crc 2023-08-28T02:31:25.655 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:31:25.684 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:31:25.685 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:25.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:25 smithi169 ceph-mon[159415]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:31:25.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:25 smithi169 ceph-mon[159415]: Cluster is now healthy 2023-08-28T02:31:25.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:25 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:31:25.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:25 smithi169 ceph-mon[159415]: osdmap e267: 8 total, 8 up, 8 in 2023-08-28T02:31:25.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:25 smithi169 ceph-mon[159415]: failed to encode map e267 with expected crc 2023-08-28T02:31:25.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:25 smithi169 ceph-mon[159415]: failed to encode map e267 with expected crc 2023-08-28T02:31:25.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:25 smithi169 ceph-mon[159415]: failed to encode map e267 with expected crc 2023-08-28T02:31:25.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:25 smithi169 ceph-mon[159415]: failed to encode map e267 with expected crc 2023-08-28T02:31:25.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:25 smithi169 ceph-mon[159415]: failed to encode map e267 with expected crc 2023-08-28T02:31:25.744 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:31:25.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:25.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:31:25.827 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 24 16 41303 41287 107.429 85.0625 0.014688 0.00930243 2023-08-28T02:31:25.851 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:31:25.881 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:25.941 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:31:25.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:25.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:31:26.049 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:31:26.127 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:26.588 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:31:26.591 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:26 smithi143 ceph-mon[183059]: failed to encode map e267 with expected crc 2023-08-28T02:31:26.591 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:26 smithi143 ceph-mon[183059]: pgmap v656: 89 pgs: 89 active+clean; 2.3 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-08-28T02:31:26.591 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:26 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:31:26.591 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:26 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:31:26.592 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:26 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/820158732' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:31:26.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:26.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:31:26.593 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:26 smithi143 ceph-mon[189871]: failed to encode map e267 with expected crc 2023-08-28T02:31:26.593 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:26 smithi143 ceph-mon[189871]: pgmap v656: 89 pgs: 89 active+clean; 2.3 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-08-28T02:31:26.593 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:26 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:31:26.593 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:26 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:31:26.593 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:26 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/820158732' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:31:26.694 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:31:26.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:26 smithi169 ceph-mon[159415]: failed to encode map e267 with expected crc 2023-08-28T02:31:26.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:26 smithi169 ceph-mon[159415]: pgmap v656: 89 pgs: 89 active+clean; 2.3 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-08-28T02:31:26.727 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:26 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:31:26.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:26 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:31:26.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:26 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/820158732' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:31:26.730 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:26.798 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:31:26.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:26.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:31:26.827 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 25 16 42663 42647 106.532 85 0.00789479 0.00938121 2023-08-28T02:31:26.907 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:31:26.909 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:31:26.909 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:31:26.909 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:31:26.910 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:31:26.910 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:31:26.910 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:31:26.910 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:31:26.910 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:31:26.910 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:31:26.910 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:31:26.910 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:31:26.910 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:31:26.910 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:31:26.911 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:31:26.911 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:31:26.911 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:31:26.941 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:27.005 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:31:27.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:27.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:31:27.110 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:31:27.189 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:27.541 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set noscrub 2023-08-28T02:31:27.827 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 26 16 43834 43818 105.251 73.1875 0.0113792 0.00949407 2023-08-28T02:31:27.973 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:31:27.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:27.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:31:28.081 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:31:28.106 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:28.446 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:31:28.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:28.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:31:28.553 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:31:28.617 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:28 smithi143 ceph-mon[183059]: pgmap v657: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2023-08-28T02:31:28.617 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:28 smithi143 ceph-mon[189871]: pgmap v657: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2023-08-28T02:31:28.622 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:28.631 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:31:28.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:28 smithi169 ceph-mon[159415]: pgmap v657: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2023-08-28T02:31:28.824 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:31:28.827 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 27 16 44883 44867 103.781 65.5625 0.0110517 0.00962847 2023-08-28T02:31:28.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:28.831 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:31:28.932 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:31:29.010 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:29.241 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:31:29.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:29.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:31:29.289 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:31:29.326 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:31:29.326 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:31:29.353 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:31:29.384 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:29.437 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:31:29.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:29.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:31:29.543 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:31:29.606 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:29.827 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 28 16 46098 46082 102.787 75.9375 0.0181575 0.00972506 2023-08-28T02:31:29.960 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:31:29.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:29.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:31:30.063 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:31:30.088 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:30.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:30 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:31:30.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:30 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:31:30.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:30 smithi143 ceph-mon[183059]: pgmap v658: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-08-28T02:31:30.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:30 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:31:30.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:30 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:31:30.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:30 smithi143 ceph-mon[189871]: pgmap v658: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-08-28T02:31:30.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:30 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:31:30.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:30 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:31:30.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:30 smithi169 ceph-mon[159415]: pgmap v658: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-08-28T02:31:30.571 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:31:30.827 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 29 16 47052 47036 101.3 59.625 0.00226231 0.00984523 2023-08-28T02:31:30.900 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:31:30.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:30.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:31:31.009 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:31:31.080 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:31.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:31 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:31:31.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:31 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:31:31.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:31 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:31:31.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:31 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:31:31.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:31 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:31:31.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:31 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:31:31.391 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.0 dump_historic_ops 2023-08-28T02:31:31.416 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:31:31.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:31.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:31:31.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:31 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:31:31.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:31 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:31:31.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:31 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:31:31.526 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:31:31.638 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:31.833 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 30 16 48215 48199 100.334 72.6875 0.0188024 0.00996081 2023-08-28T02:31:32.020 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:31:32.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:32.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:31:32.128 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:31:32.190 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:31:31 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:31:31] "GET /metrics HTTP/1.1" 200 36465 "" "Prometheus/2.33.4" 2023-08-28T02:31:32.196 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:32.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:32 smithi143 ceph-mon[189871]: pgmap v659: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-08-28T02:31:32.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:32 smithi143 ceph-mon[183059]: pgmap v659: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-08-28T02:31:32.710 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:31:32.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:32.714 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:31:32.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:32 smithi169 ceph-mon[159415]: pgmap v659: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-08-28T02:31:32.814 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:31:32.831 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 31 16 49552 49536 99.793 83.5625 0.0117813 0.0100152 2023-08-28T02:31:32.853 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:32.914 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:31:32.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:32.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:31:32.997 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:31:33.020 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:31:33.056 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:33.112 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:31:33.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:33.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:31:33.217 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:31:33.301 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:33.372 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:31:33.449 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:31:33.471 INFO:teuthology.orchestra.run.smithi143.stderr:dumped all 2023-08-28T02:31:33.509 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:31:33.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:33.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:31:33.614 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:31:33.648 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:33.707 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:31:33.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:33.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:31:33.812 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:31:33.834 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 32 16 50628 50612 98.7669 67.25 0.0167795 0.0101191 2023-08-28T02:31:33.868 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:34.485 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is set 2023-08-28T02:31:34.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:34 smithi143 ceph-mon[183059]: pgmap v660: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-08-28T02:31:34.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:34 smithi143 ceph-mon[183059]: from='client.45386 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:31:34.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:34 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1629256951' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:31:34.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:34 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:31:34.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:34 smithi143 ceph-mon[189871]: pgmap v660: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-08-28T02:31:34.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:34 smithi143 ceph-mon[189871]: from='client.45386 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:31:34.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:34 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1629256951' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:31:34.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:34 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:31:34.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:34 smithi169 ceph-mon[159415]: pgmap v660: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-08-28T02:31:34.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:34 smithi169 ceph-mon[159415]: from='client.45386 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:31:34.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:34 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1629256951' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:31:34.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:34 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:31:34.834 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 33 16 51768 51752 97.9335 71.25 0.0122053 0.010206 2023-08-28T02:31:35.121 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:31:35.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:35.134 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:31:35.218 INFO:teuthology.orchestra.run.smithi143.stdout:{"pg_ready":true,"pg_map":{"version":660,"stamp":"2023-08-28T02:31:33.334611+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3053131699,"num_objects":46542,"num_object_clones":0,"num_object_copies":139626,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":46542,"num_whiteouts":0,"num_read":6274,"num_read_kb":13707,"num_write":47620,"num_write_kb":2992037,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":189,"num_bytes_recovered":14963785,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":47684,"ondisk_log_size":47684,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":273,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":11656428,"kb_used_data":9101036,"kb_used_omap":29,"kb_used_meta":2555298,"kb_avail":738304788,"statfs":{"total":767960285184,"available":756024102912,"internally_reserved":0,"allocated":9319460864,"data_stored":9311760540,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30674,"internal_metadata":2616625198},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":825819136,"num_objects":12601,"num_object_clones":0,"num_object_copies":37803,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12601,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12601,"num_write_kb":806464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005084"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":268,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.504962+0000","last_change":"2023-08-28T02:27:31.367469+0000","last_active":"2023-08-28T02:31:12.504962+0000","last_peered":"2023-08-28T02:31:12.504962+0000","last_clean":"2023-08-28T02:31:12.504962+0000","last_became_active":"2023-08-28T02:27:31.366946+0000","last_became_peered":"2023-08-28T02:27:31.366946+0000","last_unstale":"2023-08-28T02:31:12.504962+0000","last_undegraded":"2023-08-28T02:31:12.504962+0000","last_fullsized":"2023-08-28T02:31:12.504962+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":154,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-08-28T02:31:24.422681+0000","last_change":"2023-08-28T02:27:31.324452+0000","last_active":"2023-08-28T02:31:24.422681+0000","last_peered":"2023-08-28T02:31:24.422681+0000","last_clean":"2023-08-28T02:31:24.422681+0000","last_became_active":"2023-08-28T02:27:31.323749+0000","last_became_peered":"2023-08-28T02:27:31.323749+0000","last_unstale":"2023-08-28T02:31:24.422681+0000","last_undegraded":"2023-08-28T02:31:24.422681+0000","last_fullsized":"2023-08-28T02:31:24.422681+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:18:10.904127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":74,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.489786+0000","last_change":"2023-08-28T02:27:31.337338+0000","last_active":"2023-08-28T02:31:12.489786+0000","last_peered":"2023-08-28T02:31:12.489786+0000","last_clean":"2023-08-28T02:31:12.489786+0000","last_became_active":"2023-08-28T02:27:31.337018+0000","last_became_peered":"2023-08-28T02:27:31.337018+0000","last_unstale":"2023-08-28T02:31:12.489786+0000","last_undegraded":"2023-08-28T02:31:12.489786+0000","last_fullsized":"2023-08-28T02:31:12.489786+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:46:34.845663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":255,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.491559+0000","last_change":"2023-08-28T02:23:59.223634+0000","last_active":"2023-08-28T02:31:12.491559+0000","last_peered":"2023-08-28T02:31:12.491559+0000","last_clean":"2023-08-28T02:31:12.491559+0000","last_became_active":"2023-08-28T02:23:59.222704+0000","last_became_peered":"2023-08-28T02:23:59.222704+0000","last_unstale":"2023-08-28T02:31:12.491559+0000","last_undegraded":"2023-08-28T02:31:12.491559+0000","last_fullsized":"2023-08-28T02:31:12.491559+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:51:52.752361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":145,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:14.251432+0000","last_change":"2023-08-28T02:27:31.318153+0000","last_active":"2023-08-28T02:31:14.251432+0000","last_peered":"2023-08-28T02:31:14.251432+0000","last_clean":"2023-08-28T02:31:14.251432+0000","last_became_active":"2023-08-28T02:27:31.317732+0000","last_became_peered":"2023-08-28T02:27:31.317732+0000","last_unstale":"2023-08-28T02:31:14.251432+0000","last_undegraded":"2023-08-28T02:31:14.251432+0000","last_fullsized":"2023-08-28T02:31:14.251432+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:06:13.421674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":56,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.504274+0000","last_change":"2023-08-28T02:28:25.348941+0000","last_active":"2023-08-28T02:31:12.504274+0000","last_peered":"2023-08-28T02:31:12.504274+0000","last_clean":"2023-08-28T02:31:12.504274+0000","last_became_active":"2023-08-28T02:28:25.348650+0000","last_became_peered":"2023-08-28T02:28:25.348650+0000","last_unstale":"2023-08-28T02:31:12.504274+0000","last_undegraded":"2023-08-28T02:31:12.504274+0000","last_fullsized":"2023-08-28T02:31:12.504274+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":235,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.491674+0000","last_change":"2023-08-28T02:23:59.227592+0000","last_active":"2023-08-28T02:31:12.491674+0000","last_peered":"2023-08-28T02:31:12.491674+0000","last_clean":"2023-08-28T02:31:12.491674+0000","last_became_active":"2023-08-28T02:23:59.227342+0000","last_became_peered":"2023-08-28T02:23:59.227342+0000","last_unstale":"2023-08-28T02:31:12.491674+0000","last_undegraded":"2023-08-28T02:31:12.491674+0000","last_fullsized":"2023-08-28T02:31:12.491674+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:08:19.874396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":163,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.495549+0000","last_change":"2023-08-28T02:28:25.348233+0000","last_active":"2023-08-28T02:31:12.495549+0000","last_peered":"2023-08-28T02:31:12.495549+0000","last_clean":"2023-08-28T02:31:12.495549+0000","last_became_active":"2023-08-28T02:28:25.347896+0000","last_became_peered":"2023-08-28T02:28:25.347896+0000","last_unstale":"2023-08-28T02:31:12.495549+0000","last_undegraded":"2023-08-28T02:31:12.495549+0000","last_fullsized":"2023-08-28T02:31:12.495549+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:50:17.674564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":269,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.500463+0000","last_change":"2023-08-28T02:27:31.328499+0000","last_active":"2023-08-28T02:31:12.500463+0000","last_peered":"2023-08-28T02:31:12.500463+0000","last_clean":"2023-08-28T02:31:12.500463+0000","last_became_active":"2023-08-28T02:27:31.328164+0000","last_became_peered":"2023-08-28T02:27:31.328164+0000","last_unstale":"2023-08-28T02:31:12.500463+0000","last_undegraded":"2023-08-28T02:31:12.500463+0000","last_fullsized":"2023-08-28T02:31:12.500463+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":145,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.497749+0000","last_change":"2023-08-28T02:28:25.348760+0000","last_active":"2023-08-28T02:31:12.497749+0000","last_peered":"2023-08-28T02:31:12.497749+0000","last_clean":"2023-08-28T02:31:12.497749+0000","last_became_active":"2023-08-28T02:28:25.348465+0000","last_became_peered":"2023-08-28T02:28:25.348465+0000","last_unstale":"2023-08-28T02:31:12.497749+0000","last_undegraded":"2023-08-28T02:31:12.497749+0000","last_fullsized":"2023-08-28T02:31:12.497749+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:43:05.659999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":152,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.494457+0000","last_change":"2023-08-28T02:23:59.316561+0000","last_active":"2023-08-28T02:31:12.494457+0000","last_peered":"2023-08-28T02:31:12.494457+0000","last_clean":"2023-08-28T02:31:12.494457+0000","last_became_active":"2023-08-28T02:23:59.227433+0000","last_became_peered":"2023-08-28T02:23:59.227433+0000","last_unstale":"2023-08-28T02:31:12.494457+0000","last_undegraded":"2023-08-28T02:31:12.494457+0000","last_fullsized":"2023-08-28T02:31:12.494457+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":84,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.489623+0000","last_change":"2023-08-28T02:28:25.347771+0000","last_active":"2023-08-28T02:31:12.489623+0000","last_peered":"2023-08-28T02:31:12.489623+0000","last_clean":"2023-08-28T02:31:12.489623+0000","last_became_active":"2023-08-28T02:28:25.347320+0000","last_became_peered":"2023-08-28T02:28:25.347320+0000","last_unstale":"2023-08-28T02:31:12.489623+0000","last_undegraded":"2023-08-28T02:31:12.489623+0000","last_fullsized":"2023-08-28T02:31:12.489623+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:20:27.350131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"86'22","reported_seq":317,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.506310+0000","last_change":"2023-08-28T02:28:25.361783+0000","last_active":"2023-08-28T02:31:12.506310+0000","last_peered":"2023-08-28T02:31:12.506310+0000","last_clean":"2023-08-28T02:31:12.506310+0000","last_became_active":"2023-08-28T02:28:25.351418+0000","last_became_peered":"2023-08-28T02:28:25.351418+0000","last_unstale":"2023-08-28T02:31:12.506310+0000","last_undegraded":"2023-08-28T02:31:12.506310+0000","last_fullsized":"2023-08-28T02:31:12.506310+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18936,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":283,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.505946+0000","last_change":"2023-08-28T02:28:25.352395+0000","last_active":"2023-08-28T02:31:12.505946+0000","last_peered":"2023-08-28T02:31:12.505946+0000","last_clean":"2023-08-28T02:31:12.505946+0000","last_became_active":"2023-08-28T02:28:25.352189+0000","last_became_peered":"2023-08-28T02:28:25.352189+0000","last_unstale":"2023-08-28T02:31:12.505946+0000","last_undegraded":"2023-08-28T02:31:12.505946+0000","last_fullsized":"2023-08-28T02:31:12.505946+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":137,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.494447+0000","last_change":"2023-08-28T02:23:38.983634+0000","last_active":"2023-08-28T02:31:12.494447+0000","last_peered":"2023-08-28T02:31:12.494447+0000","last_clean":"2023-08-28T02:31:12.494447+0000","last_became_active":"2023-08-28T02:23:38.344566+0000","last_became_peered":"2023-08-28T02:23:38.344566+0000","last_unstale":"2023-08-28T02:31:12.494447+0000","last_undegraded":"2023-08-28T02:31:12.494447+0000","last_fullsized":"2023-08-28T02:31:12.494447+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":145,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.495909+0000","last_change":"2023-08-28T02:27:31.344974+0000","last_active":"2023-08-28T02:31:12.495909+0000","last_peered":"2023-08-28T02:31:12.495909+0000","last_clean":"2023-08-28T02:31:12.495909+0000","last_became_active":"2023-08-28T02:27:31.344281+0000","last_became_peered":"2023-08-28T02:27:31.344281+0000","last_unstale":"2023-08-28T02:31:12.495909+0000","last_undegraded":"2023-08-28T02:31:12.495909+0000","last_fullsized":"2023-08-28T02:31:12.495909+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:04:02.821752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":253,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.491112+0000","last_change":"2023-08-28T02:23:59.223066+0000","last_active":"2023-08-28T02:31:12.491112+0000","last_peered":"2023-08-28T02:31:12.491112+0000","last_clean":"2023-08-28T02:31:12.491112+0000","last_became_active":"2023-08-28T02:23:59.222731+0000","last_became_peered":"2023-08-28T02:23:59.222731+0000","last_unstale":"2023-08-28T02:31:12.491112+0000","last_undegraded":"2023-08-28T02:31:12.491112+0000","last_fullsized":"2023-08-28T02:31:12.491112+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:48:37.558266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":293,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.500184+0000","last_change":"2023-08-28T02:23:59.210469+0000","last_active":"2023-08-28T02:31:12.500184+0000","last_peered":"2023-08-28T02:31:12.500184+0000","last_clean":"2023-08-28T02:31:12.500184+0000","last_became_active":"2023-08-28T02:23:59.210031+0000","last_became_peered":"2023-08-28T02:23:59.210031+0000","last_unstale":"2023-08-28T02:31:12.500184+0000","last_undegraded":"2023-08-28T02:31:12.500184+0000","last_fullsized":"2023-08-28T02:31:12.500184+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":149,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.495224+0000","last_change":"2023-08-28T02:28:25.354526+0000","last_active":"2023-08-28T02:31:12.495224+0000","last_peered":"2023-08-28T02:31:12.495224+0000","last_clean":"2023-08-28T02:31:12.495224+0000","last_became_active":"2023-08-28T02:28:25.350991+0000","last_became_peered":"2023-08-28T02:28:25.350991+0000","last_unstale":"2023-08-28T02:31:12.495224+0000","last_undegraded":"2023-08-28T02:31:12.495224+0000","last_fullsized":"2023-08-28T02:31:12.495224+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":254,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.491646+0000","last_change":"2023-08-28T02:23:59.220932+0000","last_active":"2023-08-28T02:31:12.491646+0000","last_peered":"2023-08-28T02:31:12.491646+0000","last_clean":"2023-08-28T02:31:12.491646+0000","last_became_active":"2023-08-28T02:23:59.220488+0000","last_became_peered":"2023-08-28T02:23:59.220488+0000","last_unstale":"2023-08-28T02:31:12.491646+0000","last_undegraded":"2023-08-28T02:31:12.491646+0000","last_fullsized":"2023-08-28T02:31:12.491646+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:53:09.216340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":66,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.503030+0000","last_change":"2023-08-28T02:28:25.703333+0000","last_active":"2023-08-28T02:31:12.503030+0000","last_peered":"2023-08-28T02:31:12.503030+0000","last_clean":"2023-08-28T02:31:12.503030+0000","last_became_active":"2023-08-28T02:28:25.343515+0000","last_became_peered":"2023-08-28T02:28:25.343515+0000","last_unstale":"2023-08-28T02:31:12.503030+0000","last_undegraded":"2023-08-28T02:31:12.503030+0000","last_fullsized":"2023-08-28T02:31:12.503030+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":145,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:14.251548+0000","last_change":"2023-08-28T02:27:31.330059+0000","last_active":"2023-08-28T02:31:14.251548+0000","last_peered":"2023-08-28T02:31:14.251548+0000","last_clean":"2023-08-28T02:31:14.251548+0000","last_became_active":"2023-08-28T02:27:31.329769+0000","last_became_peered":"2023-08-28T02:27:31.329769+0000","last_unstale":"2023-08-28T02:31:14.251548+0000","last_undegraded":"2023-08-28T02:31:14.251548+0000","last_fullsized":"2023-08-28T02:31:14.251548+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:14:54.349909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"267'2825","reported_seq":2837,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-08-28T02:31:29.582919+0000","last_change":"2023-08-28T02:30:57.068862+0000","last_active":"2023-08-28T02:31:29.582919+0000","last_peered":"2023-08-28T02:31:29.582919+0000","last_clean":"2023-08-28T02:31:29.582919+0000","last_became_active":"2023-08-28T02:30:57.068563+0000","last_became_peered":"2023-08-28T02:30:57.068563+0000","last_unstale":"2023-08-28T02:31:29.582919+0000","last_undegraded":"2023-08-28T02:31:29.582919+0000","last_fullsized":"2023-08-28T02:31:29.582919+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":2825,"log_dups_size":0,"ondisk_log_size":2825,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:39:58.505326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185139200,"num_objects":2825,"num_object_clones":0,"num_object_copies":8475,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2825,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2825,"num_write_kb":180800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":135,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.494705+0000","last_change":"2023-08-28T02:23:38.657185+0000","last_active":"2023-08-28T02:31:12.494705+0000","last_peered":"2023-08-28T02:31:12.494705+0000","last_clean":"2023-08-28T02:31:12.494705+0000","last_became_active":"2023-08-28T02:23:38.339016+0000","last_became_peered":"2023-08-28T02:23:38.339016+0000","last_unstale":"2023-08-28T02:31:12.494705+0000","last_undegraded":"2023-08-28T02:31:12.494705+0000","last_fullsized":"2023-08-28T02:31:12.494705+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":253,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.500609+0000","last_change":"2023-08-28T02:28:25.348106+0000","last_active":"2023-08-28T02:31:12.500609+0000","last_peered":"2023-08-28T02:31:12.500609+0000","last_clean":"2023-08-28T02:31:12.500609+0000","last_became_active":"2023-08-28T02:28:25.347786+0000","last_became_peered":"2023-08-28T02:28:25.347786+0000","last_unstale":"2023-08-28T02:31:12.500609+0000","last_undegraded":"2023-08-28T02:31:12.500609+0000","last_fullsized":"2023-08-28T02:31:12.500609+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"267'2936","reported_seq":2950,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-08-28T02:31:29.387127+0000","last_change":"2023-08-28T02:30:57.067718+0000","last_active":"2023-08-28T02:31:29.387127+0000","last_peered":"2023-08-28T02:31:29.387127+0000","last_clean":"2023-08-28T02:31:29.387127+0000","last_became_active":"2023-08-28T02:30:57.067407+0000","last_became_peered":"2023-08-28T02:30:57.067407+0000","last_unstale":"2023-08-28T02:31:29.387127+0000","last_undegraded":"2023-08-28T02:31:29.387127+0000","last_fullsized":"2023-08-28T02:31:29.387127+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":2936,"log_dups_size":0,"ondisk_log_size":2936,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:49:39.360134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192413696,"num_objects":2936,"num_object_clones":0,"num_object_copies":8808,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2936,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2936,"num_write_kb":187904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":161,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.494611+0000","last_change":"2023-08-28T02:27:31.395328+0000","last_active":"2023-08-28T02:31:12.494611+0000","last_peered":"2023-08-28T02:31:12.494611+0000","last_clean":"2023-08-28T02:31:12.494611+0000","last_became_active":"2023-08-28T02:27:31.328228+0000","last_became_peered":"2023-08-28T02:27:31.328228+0000","last_unstale":"2023-08-28T02:31:12.494611+0000","last_undegraded":"2023-08-28T02:31:12.494611+0000","last_fullsized":"2023-08-28T02:31:12.494611+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":302,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.504981+0000","last_change":"2023-08-28T02:27:31.344915+0000","last_active":"2023-08-28T02:31:12.504981+0000","last_peered":"2023-08-28T02:31:12.504981+0000","last_clean":"2023-08-28T02:31:12.504981+0000","last_became_active":"2023-08-28T02:27:31.344577+0000","last_became_peered":"2023-08-28T02:27:31.344577+0000","last_unstale":"2023-08-28T02:31:12.504981+0000","last_undegraded":"2023-08-28T02:31:12.504981+0000","last_fullsized":"2023-08-28T02:31:12.504981+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"267'2866","reported_seq":2881,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-08-28T02:31:28.909798+0000","last_change":"2023-08-28T02:30:57.066684+0000","last_active":"2023-08-28T02:31:28.909798+0000","last_peered":"2023-08-28T02:31:28.909798+0000","last_clean":"2023-08-28T02:31:28.909798+0000","last_became_active":"2023-08-28T02:30:57.066406+0000","last_became_peered":"2023-08-28T02:30:57.066406+0000","last_unstale":"2023-08-28T02:31:28.909798+0000","last_undegraded":"2023-08-28T02:31:28.909798+0000","last_fullsized":"2023-08-28T02:31:28.909798+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":2866,"log_dups_size":0,"ondisk_log_size":2866,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:35:33.176438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187826176,"num_objects":2866,"num_object_clones":0,"num_object_copies":8598,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2866,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2866,"num_write_kb":183424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":161,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.494661+0000","last_change":"2023-08-28T02:28:25.749249+0000","last_active":"2023-08-28T02:31:12.494661+0000","last_peered":"2023-08-28T02:31:12.494661+0000","last_clean":"2023-08-28T02:31:12.494661+0000","last_became_active":"2023-08-28T02:28:25.397684+0000","last_became_peered":"2023-08-28T02:28:25.397684+0000","last_unstale":"2023-08-28T02:31:12.494661+0000","last_undegraded":"2023-08-28T02:31:12.494661+0000","last_fullsized":"2023-08-28T02:31:12.494661+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":56,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.503224+0000","last_change":"2023-08-28T02:28:25.350802+0000","last_active":"2023-08-28T02:31:12.503224+0000","last_peered":"2023-08-28T02:31:12.503224+0000","last_clean":"2023-08-28T02:31:12.503224+0000","last_became_active":"2023-08-28T02:28:25.350474+0000","last_became_peered":"2023-08-28T02:28:25.350474+0000","last_unstale":"2023-08-28T02:31:12.503224+0000","last_undegraded":"2023-08-28T02:31:12.503224+0000","last_fullsized":"2023-08-28T02:31:12.503224+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"267'2945","reported_seq":2960,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-08-28T02:31:29.126143+0000","last_change":"2023-08-28T02:30:57.071212+0000","last_active":"2023-08-28T02:31:29.126143+0000","last_peered":"2023-08-28T02:31:29.126143+0000","last_clean":"2023-08-28T02:31:29.126143+0000","last_became_active":"2023-08-28T02:30:57.070876+0000","last_became_peered":"2023-08-28T02:30:57.070876+0000","last_unstale":"2023-08-28T02:31:29.126143+0000","last_undegraded":"2023-08-28T02:31:29.126143+0000","last_fullsized":"2023-08-28T02:31:29.126143+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":2945,"log_dups_size":0,"ondisk_log_size":2945,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:19:29.209842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193003520,"num_objects":2945,"num_object_clones":0,"num_object_copies":8835,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2945,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2945,"num_write_kb":188480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":269,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.496066+0000","last_change":"2023-08-28T02:28:25.445982+0000","last_active":"2023-08-28T02:31:12.496066+0000","last_peered":"2023-08-28T02:31:12.496066+0000","last_clean":"2023-08-28T02:31:12.496066+0000","last_became_active":"2023-08-28T02:28:25.443577+0000","last_became_peered":"2023-08-28T02:28:25.443577+0000","last_unstale":"2023-08-28T02:31:12.496066+0000","last_undegraded":"2023-08-28T02:31:12.496066+0000","last_fullsized":"2023-08-28T02:31:12.496066+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:24:03.381751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":247,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.491259+0000","last_change":"2023-08-28T02:27:31.319783+0000","last_active":"2023-08-28T02:31:12.491259+0000","last_peered":"2023-08-28T02:31:12.491259+0000","last_clean":"2023-08-28T02:31:12.491259+0000","last_became_active":"2023-08-28T02:27:31.319451+0000","last_became_peered":"2023-08-28T02:27:31.319451+0000","last_unstale":"2023-08-28T02:31:12.491259+0000","last_undegraded":"2023-08-28T02:31:12.491259+0000","last_fullsized":"2023-08-28T02:31:12.491259+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:20:37.097889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"267'2849","reported_seq":2864,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-08-28T02:31:28.931328+0000","last_change":"2023-08-28T02:30:57.071262+0000","last_active":"2023-08-28T02:31:28.931328+0000","last_peered":"2023-08-28T02:31:28.931328+0000","last_clean":"2023-08-28T02:31:28.931328+0000","last_became_active":"2023-08-28T02:30:57.070974+0000","last_became_peered":"2023-08-28T02:30:57.070974+0000","last_unstale":"2023-08-28T02:31:28.931328+0000","last_undegraded":"2023-08-28T02:31:28.931328+0000","last_fullsized":"2023-08-28T02:31:28.931328+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":2849,"log_dups_size":0,"ondisk_log_size":2849,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:40:20.990559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186712064,"num_objects":2849,"num_object_clones":0,"num_object_copies":8547,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2849,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2849,"num_write_kb":182336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":131,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.495825+0000","last_change":"2023-08-28T02:23:59.230059+0000","last_active":"2023-08-28T02:31:12.495825+0000","last_peered":"2023-08-28T02:31:12.495825+0000","last_clean":"2023-08-28T02:31:12.495825+0000","last_became_active":"2023-08-28T02:23:59.217783+0000","last_became_peered":"2023-08-28T02:23:59.217783+0000","last_unstale":"2023-08-28T02:31:12.495825+0000","last_undegraded":"2023-08-28T02:31:12.495825+0000","last_fullsized":"2023-08-28T02:31:12.495825+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:00:12.145395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":287,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.506236+0000","last_change":"2023-08-28T02:28:25.351698+0000","last_active":"2023-08-28T02:31:12.506236+0000","last_peered":"2023-08-28T02:31:12.506236+0000","last_clean":"2023-08-28T02:31:12.506236+0000","last_became_active":"2023-08-28T02:28:25.351349+0000","last_became_peered":"2023-08-28T02:28:25.351349+0000","last_unstale":"2023-08-28T02:31:12.506236+0000","last_undegraded":"2023-08-28T02:31:12.506236+0000","last_fullsized":"2023-08-28T02:31:12.506236+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"267'2869","reported_seq":2884,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-08-28T02:31:29.134518+0000","last_change":"2023-08-28T02:30:57.069579+0000","last_active":"2023-08-28T02:31:29.134518+0000","last_peered":"2023-08-28T02:31:29.134518+0000","last_clean":"2023-08-28T02:31:29.134518+0000","last_became_active":"2023-08-28T02:30:57.069365+0000","last_became_peered":"2023-08-28T02:30:57.069365+0000","last_unstale":"2023-08-28T02:31:29.134518+0000","last_undegraded":"2023-08-28T02:31:29.134518+0000","last_fullsized":"2023-08-28T02:31:29.134518+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":2869,"log_dups_size":0,"ondisk_log_size":2869,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:02:07.549000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188022784,"num_objects":2869,"num_object_clones":0,"num_object_copies":8607,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2869,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2869,"num_write_kb":183616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":56,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.503307+0000","last_change":"2023-08-28T02:28:25.345274+0000","last_active":"2023-08-28T02:31:12.503307+0000","last_peered":"2023-08-28T02:31:12.503307+0000","last_clean":"2023-08-28T02:31:12.503307+0000","last_became_active":"2023-08-28T02:28:25.344885+0000","last_became_peered":"2023-08-28T02:28:25.344885+0000","last_unstale":"2023-08-28T02:31:12.503307+0000","last_undegraded":"2023-08-28T02:31:12.503307+0000","last_fullsized":"2023-08-28T02:31:12.503307+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":313,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.505576+0000","last_change":"2023-08-28T02:27:31.358436+0000","last_active":"2023-08-28T02:31:12.505576+0000","last_peered":"2023-08-28T02:31:12.505576+0000","last_clean":"2023-08-28T02:31:12.505576+0000","last_became_active":"2023-08-28T02:27:31.358185+0000","last_became_peered":"2023-08-28T02:27:31.358185+0000","last_unstale":"2023-08-28T02:31:12.505576+0000","last_undegraded":"2023-08-28T02:31:12.505576+0000","last_fullsized":"2023-08-28T02:31:12.505576+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"267'2946","reported_seq":2960,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-08-28T02:31:32.059588+0000","last_change":"2023-08-28T02:30:57.067391+0000","last_active":"2023-08-28T02:31:32.059588+0000","last_peered":"2023-08-28T02:31:32.059588+0000","last_clean":"2023-08-28T02:31:32.059588+0000","last_became_active":"2023-08-28T02:30:57.067103+0000","last_became_peered":"2023-08-28T02:30:57.067103+0000","last_unstale":"2023-08-28T02:31:32.059588+0000","last_undegraded":"2023-08-28T02:31:32.059588+0000","last_fullsized":"2023-08-28T02:31:32.059588+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":2946,"log_dups_size":0,"ondisk_log_size":2946,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193069056,"num_objects":2946,"num_object_clones":0,"num_object_copies":8838,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2946,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2946,"num_write_kb":188544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":176,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.505115+0000","last_change":"2023-08-28T02:23:38.347043+0000","last_active":"2023-08-28T02:31:12.505115+0000","last_peered":"2023-08-28T02:31:12.505115+0000","last_clean":"2023-08-28T02:31:12.505115+0000","last_became_active":"2023-08-28T02:23:38.346680+0000","last_became_peered":"2023-08-28T02:23:38.346680+0000","last_unstale":"2023-08-28T02:31:12.505115+0000","last_undegraded":"2023-08-28T02:31:12.505115+0000","last_fullsized":"2023-08-28T02:31:12.505115+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":234,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.491692+0000","last_change":"2023-08-28T02:28:25.347494+0000","last_active":"2023-08-28T02:31:12.491692+0000","last_peered":"2023-08-28T02:31:12.491692+0000","last_clean":"2023-08-28T02:31:12.491692+0000","last_became_active":"2023-08-28T02:28:25.347142+0000","last_became_peered":"2023-08-28T02:28:25.347142+0000","last_unstale":"2023-08-28T02:31:12.491692+0000","last_undegraded":"2023-08-28T02:31:12.491692+0000","last_fullsized":"2023-08-28T02:31:12.491692+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:37:14.384884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.c","version":"267'3029","reported_seq":3040,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-08-28T02:31:32.118915+0000","last_change":"2023-08-28T02:30:57.071876+0000","last_active":"2023-08-28T02:31:32.118915+0000","last_peered":"2023-08-28T02:31:32.118915+0000","last_clean":"2023-08-28T02:31:32.118915+0000","last_became_active":"2023-08-28T02:30:57.071027+0000","last_became_peered":"2023-08-28T02:30:57.071027+0000","last_unstale":"2023-08-28T02:31:32.118915+0000","last_undegraded":"2023-08-28T02:31:32.118915+0000","last_fullsized":"2023-08-28T02:31:32.118915+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":3029,"log_dups_size":0,"ondisk_log_size":3029,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198508544,"num_objects":3029,"num_object_clones":0,"num_object_copies":9087,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3029,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3029,"num_write_kb":193856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":145,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.496349+0000","last_change":"2023-08-28T02:27:31.325260+0000","last_active":"2023-08-28T02:31:12.496349+0000","last_peered":"2023-08-28T02:31:12.496349+0000","last_clean":"2023-08-28T02:31:12.496349+0000","last_became_active":"2023-08-28T02:27:31.324950+0000","last_became_peered":"2023-08-28T02:27:31.324950+0000","last_unstale":"2023-08-28T02:31:12.496349+0000","last_undegraded":"2023-08-28T02:31:12.496349+0000","last_fullsized":"2023-08-28T02:31:12.496349+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:21:33.833261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":75,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.492339+0000","last_change":"2023-08-28T02:27:31.322118+0000","last_active":"2023-08-28T02:31:12.492339+0000","last_peered":"2023-08-28T02:31:12.492339+0000","last_clean":"2023-08-28T02:31:12.492339+0000","last_became_active":"2023-08-28T02:27:31.321338+0000","last_became_peered":"2023-08-28T02:27:31.321338+0000","last_unstale":"2023-08-28T02:31:12.492339+0000","last_undegraded":"2023-08-28T02:31:12.492339+0000","last_fullsized":"2023-08-28T02:31:12.492339+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:56:14.210571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"267'3006","reported_seq":3020,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-08-28T02:31:31.932720+0000","last_change":"2023-08-28T02:30:57.067740+0000","last_active":"2023-08-28T02:31:31.932720+0000","last_peered":"2023-08-28T02:31:31.932720+0000","last_clean":"2023-08-28T02:31:31.932720+0000","last_became_active":"2023-08-28T02:30:57.067464+0000","last_became_peered":"2023-08-28T02:30:57.067464+0000","last_unstale":"2023-08-28T02:31:31.932720+0000","last_undegraded":"2023-08-28T02:31:31.932720+0000","last_fullsized":"2023-08-28T02:31:31.932720+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":3006,"log_dups_size":0,"ondisk_log_size":3006,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197001216,"num_objects":3006,"num_object_clones":0,"num_object_copies":9018,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3006,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3006,"num_write_kb":192384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"78'126","reported_seq":637,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.489904+0000","last_change":"2023-08-28T02:28:27.383137+0000","last_active":"2023-08-28T02:31:12.489904+0000","last_peered":"2023-08-28T02:31:12.489904+0000","last_clean":"2023-08-28T02:31:12.489904+0000","last_became_active":"2023-08-28T02:28:27.382488+0000","last_became_peered":"2023-08-28T02:28:27.382488+0000","last_unstale":"2023-08-28T02:31:12.489904+0000","last_undegraded":"2023-08-28T02:31:12.489904+0000","last_fullsized":"2023-08-28T02:31:12.489904+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:14:48.414426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":743,"num_read_kb":1331,"num_write":125,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":266,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.500096+0000","last_change":"2023-08-28T02:23:59.207148+0000","last_active":"2023-08-28T02:31:12.500096+0000","last_peered":"2023-08-28T02:31:12.500096+0000","last_clean":"2023-08-28T02:31:12.500096+0000","last_became_active":"2023-08-28T02:23:59.206802+0000","last_became_peered":"2023-08-28T02:23:59.206802+0000","last_unstale":"2023-08-28T02:31:12.500096+0000","last_undegraded":"2023-08-28T02:31:12.500096+0000","last_fullsized":"2023-08-28T02:31:12.500096+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":75,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.489497+0000","last_change":"2023-08-28T02:27:31.328027+0000","last_active":"2023-08-28T02:31:12.489497+0000","last_peered":"2023-08-28T02:31:12.489497+0000","last_clean":"2023-08-28T02:31:12.489497+0000","last_became_active":"2023-08-28T02:27:31.326529+0000","last_became_peered":"2023-08-28T02:27:31.326529+0000","last_unstale":"2023-08-28T02:31:12.489497+0000","last_undegraded":"2023-08-28T02:31:12.489497+0000","last_fullsized":"2023-08-28T02:31:12.489497+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:56:57.331355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"267'2968","reported_seq":2982,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-08-28T02:31:32.092285+0000","last_change":"2023-08-28T02:30:57.067534+0000","last_active":"2023-08-28T02:31:32.092285+0000","last_peered":"2023-08-28T02:31:32.092285+0000","last_clean":"2023-08-28T02:31:32.092285+0000","last_became_active":"2023-08-28T02:30:57.067256+0000","last_became_peered":"2023-08-28T02:30:57.067256+0000","last_unstale":"2023-08-28T02:31:32.092285+0000","last_undegraded":"2023-08-28T02:31:32.092285+0000","last_fullsized":"2023-08-28T02:31:32.092285+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":2968,"log_dups_size":0,"ondisk_log_size":2968,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194510848,"num_objects":2968,"num_object_clones":0,"num_object_copies":8904,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2968,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2968,"num_write_kb":189952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"78'204","reported_seq":1352,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.489319+0000","last_change":"2023-08-28T02:28:59.480561+0000","last_active":"2023-08-28T02:31:12.489319+0000","last_peered":"2023-08-28T02:31:12.489319+0000","last_clean":"2023-08-28T02:31:12.489319+0000","last_became_active":"2023-08-28T02:28:59.479868+0000","last_became_peered":"2023-08-28T02:28:59.479868+0000","last_unstale":"2023-08-28T02:31:12.489319+0000","last_undegraded":"2023-08-28T02:31:12.489319+0000","last_fullsized":"2023-08-28T02:31:12.489319+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":204,"log_dups_size":0,"ondisk_log_size":204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:39:50.444629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1521,"num_read_kb":2511,"num_write":178,"num_write_kb":1175,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":261,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.506094+0000","last_change":"2023-08-28T02:28:25.352417+0000","last_active":"2023-08-28T02:31:12.506094+0000","last_peered":"2023-08-28T02:31:12.506094+0000","last_clean":"2023-08-28T02:31:12.506094+0000","last_became_active":"2023-08-28T02:28:25.352216+0000","last_became_peered":"2023-08-28T02:28:25.352216+0000","last_unstale":"2023-08-28T02:31:12.506094+0000","last_undegraded":"2023-08-28T02:31:12.506094+0000","last_fullsized":"2023-08-28T02:31:12.506094+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":75,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.489883+0000","last_change":"2023-08-28T02:27:31.376755+0000","last_active":"2023-08-28T02:31:12.489883+0000","last_peered":"2023-08-28T02:31:12.489883+0000","last_clean":"2023-08-28T02:31:12.489883+0000","last_became_active":"2023-08-28T02:27:31.373279+0000","last_became_peered":"2023-08-28T02:27:31.373279+0000","last_unstale":"2023-08-28T02:31:12.489883+0000","last_undegraded":"2023-08-28T02:31:12.489883+0000","last_fullsized":"2023-08-28T02:31:12.489883+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:26:14.628941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"267'2867","reported_seq":2882,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-08-28T02:31:28.900770+0000","last_change":"2023-08-28T02:30:57.062219+0000","last_active":"2023-08-28T02:31:28.900770+0000","last_peered":"2023-08-28T02:31:28.900770+0000","last_clean":"2023-08-28T02:31:28.900770+0000","last_became_active":"2023-08-28T02:30:57.061931+0000","last_became_peered":"2023-08-28T02:30:57.061931+0000","last_unstale":"2023-08-28T02:31:28.900770+0000","last_undegraded":"2023-08-28T02:31:28.900770+0000","last_fullsized":"2023-08-28T02:31:28.900770+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":2867,"log_dups_size":0,"ondisk_log_size":2867,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:03:40.377848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187891712,"num_objects":2867,"num_object_clones":0,"num_object_copies":8601,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2867,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2867,"num_write_kb":183488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"78'47","reported_seq":484,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.495836+0000","last_change":"2023-08-28T02:24:05.799647+0000","last_active":"2023-08-28T02:31:12.495836+0000","last_peered":"2023-08-28T02:31:12.495836+0000","last_clean":"2023-08-28T02:31:12.495836+0000","last_became_active":"2023-08-28T02:23:59.210481+0000","last_became_peered":"2023-08-28T02:23:59.210481+0000","last_unstale":"2023-08-28T02:31:12.495836+0000","last_undegraded":"2023-08-28T02:31:12.495836+0000","last_fullsized":"2023-08-28T02:31:12.495836+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":47,"log_dups_size":0,"ondisk_log_size":47,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:40:18.542375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":162,"num_read_kb":1281,"num_write":48,"num_write_kb":1153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":84,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.489099+0000","last_change":"2023-08-28T02:28:25.343406+0000","last_active":"2023-08-28T02:31:12.489099+0000","last_peered":"2023-08-28T02:31:12.489099+0000","last_clean":"2023-08-28T02:31:12.489099+0000","last_became_active":"2023-08-28T02:28:25.343065+0000","last_became_peered":"2023-08-28T02:28:25.343065+0000","last_unstale":"2023-08-28T02:31:12.489099+0000","last_undegraded":"2023-08-28T02:31:12.489099+0000","last_fullsized":"2023-08-28T02:31:12.489099+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:20:31.893609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":75,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.492416+0000","last_change":"2023-08-28T02:27:31.322967+0000","last_active":"2023-08-28T02:31:12.492416+0000","last_peered":"2023-08-28T02:31:12.492416+0000","last_clean":"2023-08-28T02:31:12.492416+0000","last_became_active":"2023-08-28T02:27:31.322655+0000","last_became_peered":"2023-08-28T02:27:31.322655+0000","last_unstale":"2023-08-28T02:31:12.492416+0000","last_undegraded":"2023-08-28T02:31:12.492416+0000","last_fullsized":"2023-08-28T02:31:12.492416+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:05:15.868115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"267'2777","reported_seq":2792,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-08-28T02:31:28.917503+0000","last_change":"2023-08-28T02:30:57.071289+0000","last_active":"2023-08-28T02:31:28.917503+0000","last_peered":"2023-08-28T02:31:28.917503+0000","last_clean":"2023-08-28T02:31:28.917503+0000","last_became_active":"2023-08-28T02:30:57.070991+0000","last_became_peered":"2023-08-28T02:30:57.070991+0000","last_unstale":"2023-08-28T02:31:28.917503+0000","last_undegraded":"2023-08-28T02:31:28.917503+0000","last_fullsized":"2023-08-28T02:31:28.917503+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":2777,"log_dups_size":0,"ondisk_log_size":2777,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:15:02.370637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181993472,"num_objects":2777,"num_object_clones":0,"num_object_copies":8331,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2777,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2777,"num_write_kb":177728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"78'114","reported_seq":630,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.500344+0000","last_change":"2023-08-28T02:24:01.210739+0000","last_active":"2023-08-28T02:31:12.500344+0000","last_peered":"2023-08-28T02:31:12.500344+0000","last_clean":"2023-08-28T02:31:12.500344+0000","last_became_active":"2023-08-28T02:24:01.210386+0000","last_became_peered":"2023-08-28T02:24:01.210386+0000","last_unstale":"2023-08-28T02:31:12.500344+0000","last_undegraded":"2023-08-28T02:31:12.500344+0000","last_fullsized":"2023-08-28T02:31:12.500344+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":114,"log_dups_size":0,"ondisk_log_size":114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":358,"num_read_kb":1352,"num_write":102,"num_write_kb":1104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"86'139","reported_seq":477,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.495227+0000","last_change":"2023-08-28T02:29:58.484566+0000","last_active":"2023-08-28T02:31:12.495227+0000","last_peered":"2023-08-28T02:31:12.495227+0000","last_clean":"2023-08-28T02:31:12.495227+0000","last_became_active":"2023-08-28T02:29:58.484046+0000","last_became_peered":"2023-08-28T02:29:58.484046+0000","last_unstale":"2023-08-28T02:31:12.495227+0000","last_undegraded":"2023-08-28T02:31:12.495227+0000","last_fullsized":"2023-08-28T02:31:12.495227+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_clean_scrub_stamp":"2023-08-28T02:08:02.043718+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":145,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.497891+0000","last_change":"2023-08-28T02:28:25.347600+0000","last_active":"2023-08-28T02:31:12.497891+0000","last_peered":"2023-08-28T02:31:12.497891+0000","last_clean":"2023-08-28T02:31:12.497891+0000","last_became_active":"2023-08-28T02:28:25.347237+0000","last_became_peered":"2023-08-28T02:28:25.347237+0000","last_unstale":"2023-08-28T02:31:12.497891+0000","last_undegraded":"2023-08-28T02:31:12.497891+0000","last_fullsized":"2023-08-28T02:31:12.497891+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:29:26.767559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":56,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.503841+0000","last_change":"2023-08-28T02:28:25.398594+0000","last_active":"2023-08-28T02:31:12.503841+0000","last_peered":"2023-08-28T02:31:12.503841+0000","last_clean":"2023-08-28T02:31:12.503841+0000","last_became_active":"2023-08-28T02:28:25.397058+0000","last_became_peered":"2023-08-28T02:28:25.397058+0000","last_unstale":"2023-08-28T02:31:12.503841+0000","last_undegraded":"2023-08-28T02:31:12.503841+0000","last_fullsized":"2023-08-28T02:31:12.503841+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"267'2914","reported_seq":2928,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-08-28T02:31:32.166575+0000","last_change":"2023-08-28T02:30:57.071523+0000","last_active":"2023-08-28T02:31:32.166575+0000","last_peered":"2023-08-28T02:31:32.166575+0000","last_clean":"2023-08-28T02:31:32.166575+0000","last_became_active":"2023-08-28T02:30:57.071114+0000","last_became_peered":"2023-08-28T02:30:57.071114+0000","last_unstale":"2023-08-28T02:31:32.166575+0000","last_undegraded":"2023-08-28T02:31:32.166575+0000","last_fullsized":"2023-08-28T02:31:32.166575+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":2914,"log_dups_size":0,"ondisk_log_size":2914,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190971904,"num_objects":2914,"num_object_clones":0,"num_object_copies":8742,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2914,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2914,"num_write_kb":186496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"78'148","reported_seq":797,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.495320+0000","last_change":"2023-08-28T02:29:39.524927+0000","last_active":"2023-08-28T02:31:12.495320+0000","last_peered":"2023-08-28T02:31:12.495320+0000","last_clean":"2023-08-28T02:31:12.495320+0000","last_became_active":"2023-08-28T02:29:39.524297+0000","last_became_peered":"2023-08-28T02:29:39.524297+0000","last_unstale":"2023-08-28T02:31:12.495320+0000","last_undegraded":"2023-08-28T02:31:12.495320+0000","last_fullsized":"2023-08-28T02:31:12.495320+0000","mapping_epoch":243,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":244,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1092,"num_read_kb":1955,"num_write":127,"num_write_kb":5177,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":236,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.491160+0000","last_change":"2023-08-28T02:23:59.226986+0000","last_active":"2023-08-28T02:31:12.491160+0000","last_peered":"2023-08-28T02:31:12.491160+0000","last_clean":"2023-08-28T02:31:12.491160+0000","last_became_active":"2023-08-28T02:23:59.226529+0000","last_became_peered":"2023-08-28T02:23:59.226529+0000","last_unstale":"2023-08-28T02:31:12.491160+0000","last_undegraded":"2023-08-28T02:31:12.491160+0000","last_fullsized":"2023-08-28T02:31:12.491160+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:22:09.193619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":280,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.491370+0000","last_change":"2023-08-28T02:27:31.361298+0000","last_active":"2023-08-28T02:31:12.491370+0000","last_peered":"2023-08-28T02:31:12.491370+0000","last_clean":"2023-08-28T02:31:12.491370+0000","last_became_active":"2023-08-28T02:27:31.360939+0000","last_became_peered":"2023-08-28T02:27:31.360939+0000","last_unstale":"2023-08-28T02:31:12.491370+0000","last_undegraded":"2023-08-28T02:31:12.491370+0000","last_fullsized":"2023-08-28T02:31:12.491370+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:48:42.024920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"267'3016","reported_seq":3030,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-08-28T02:31:31.936486+0000","last_change":"2023-08-28T02:30:57.071302+0000","last_active":"2023-08-28T02:31:31.936486+0000","last_peered":"2023-08-28T02:31:31.936486+0000","last_clean":"2023-08-28T02:31:31.936486+0000","last_became_active":"2023-08-28T02:30:57.070731+0000","last_became_peered":"2023-08-28T02:30:57.070731+0000","last_unstale":"2023-08-28T02:31:31.936486+0000","last_undegraded":"2023-08-28T02:31:31.936486+0000","last_fullsized":"2023-08-28T02:31:31.936486+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":3016,"log_dups_size":0,"ondisk_log_size":3016,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197656576,"num_objects":3016,"num_object_clones":0,"num_object_copies":9048,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3016,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3016,"num_write_kb":193024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"78'103","reported_seq":642,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.494547+0000","last_change":"2023-08-28T02:24:07.212128+0000","last_active":"2023-08-28T02:31:12.494547+0000","last_peered":"2023-08-28T02:31:12.494547+0000","last_clean":"2023-08-28T02:31:12.494547+0000","last_became_active":"2023-08-28T02:24:07.211562+0000","last_became_peered":"2023-08-28T02:24:07.211562+0000","last_unstale":"2023-08-28T02:31:12.494547+0000","last_undegraded":"2023-08-28T02:31:12.494547+0000","last_fullsized":"2023-08-28T02:31:12.494547+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":803,"num_read_kb":2080,"num_write":73,"num_write_kb":1244,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":75,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.489997+0000","last_change":"2023-08-28T02:27:31.377042+0000","last_active":"2023-08-28T02:31:12.489997+0000","last_peered":"2023-08-28T02:31:12.489997+0000","last_clean":"2023-08-28T02:31:12.489997+0000","last_became_active":"2023-08-28T02:27:31.373709+0000","last_became_peered":"2023-08-28T02:27:31.373709+0000","last_unstale":"2023-08-28T02:31:12.489997+0000","last_undegraded":"2023-08-28T02:31:12.489997+0000","last_fullsized":"2023-08-28T02:31:12.489997+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:12:38.286332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":222,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.491341+0000","last_change":"2023-08-28T02:23:38.329667+0000","last_active":"2023-08-28T02:31:12.491341+0000","last_peered":"2023-08-28T02:31:12.491341+0000","last_clean":"2023-08-28T02:31:12.491341+0000","last_became_active":"2023-08-28T02:23:38.329175+0000","last_became_peered":"2023-08-28T02:23:38.329175+0000","last_unstale":"2023-08-28T02:31:12.491341+0000","last_undegraded":"2023-08-28T02:31:12.491341+0000","last_fullsized":"2023-08-28T02:31:12.491341+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:21:34.815067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"267'2876","reported_seq":2891,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-08-28T02:31:29.387074+0000","last_change":"2023-08-28T02:30:57.061826+0000","last_active":"2023-08-28T02:31:29.387074+0000","last_peered":"2023-08-28T02:31:29.387074+0000","last_clean":"2023-08-28T02:31:29.387074+0000","last_became_active":"2023-08-28T02:30:57.061555+0000","last_became_peered":"2023-08-28T02:30:57.061555+0000","last_unstale":"2023-08-28T02:31:29.387074+0000","last_undegraded":"2023-08-28T02:31:29.387074+0000","last_fullsized":"2023-08-28T02:31:29.387074+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":2876,"log_dups_size":0,"ondisk_log_size":2876,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:15:31.288849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188481536,"num_objects":2876,"num_object_clones":0,"num_object_copies":8628,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2876,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2876,"num_write_kb":184064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":131,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.496227+0000","last_change":"2023-08-28T02:23:59.232099+0000","last_active":"2023-08-28T02:31:12.496227+0000","last_peered":"2023-08-28T02:31:12.496227+0000","last_clean":"2023-08-28T02:31:12.496227+0000","last_became_active":"2023-08-28T02:23:59.219318+0000","last_became_peered":"2023-08-28T02:23:59.219318+0000","last_unstale":"2023-08-28T02:31:12.496227+0000","last_undegraded":"2023-08-28T02:31:12.496227+0000","last_fullsized":"2023-08-28T02:31:12.496227+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:59:30.093289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'112","reported_seq":421,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.492646+0000","last_change":"2023-08-28T02:28:59.477917+0000","last_active":"2023-08-28T02:31:12.492646+0000","last_peered":"2023-08-28T02:31:12.492646+0000","last_clean":"2023-08-28T02:31:12.492646+0000","last_became_active":"2023-08-28T02:28:59.477383+0000","last_became_peered":"2023-08-28T02:28:59.477383+0000","last_unstale":"2023-08-28T02:31:12.492646+0000","last_undegraded":"2023-08-28T02:31:12.492646+0000","last_fullsized":"2023-08-28T02:31:12.492646+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:28:45.045742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":444,"num_read_kb":473,"num_write":94,"num_write_kb":127,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":127,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.495620+0000","last_change":"2023-08-28T02:23:59.221034+0000","last_active":"2023-08-28T02:31:12.495620+0000","last_peered":"2023-08-28T02:31:12.495620+0000","last_clean":"2023-08-28T02:31:12.495620+0000","last_became_active":"2023-08-28T02:23:59.220730+0000","last_became_peered":"2023-08-28T02:23:59.220730+0000","last_unstale":"2023-08-28T02:31:12.495620+0000","last_undegraded":"2023-08-28T02:31:12.495620+0000","last_fullsized":"2023-08-28T02:31:12.495620+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:25:52.364370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"267'2825","reported_seq":2840,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-08-28T02:31:29.378240+0000","last_change":"2023-08-28T02:30:57.060097+0000","last_active":"2023-08-28T02:31:29.378240+0000","last_peered":"2023-08-28T02:31:29.378240+0000","last_clean":"2023-08-28T02:31:29.378240+0000","last_became_active":"2023-08-28T02:30:57.059712+0000","last_became_peered":"2023-08-28T02:30:57.059712+0000","last_unstale":"2023-08-28T02:31:29.378240+0000","last_undegraded":"2023-08-28T02:31:29.378240+0000","last_fullsized":"2023-08-28T02:31:29.378240+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":2825,"log_dups_size":0,"ondisk_log_size":2825,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:54:20.362640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185139200,"num_objects":2825,"num_object_clones":0,"num_object_copies":8475,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2825,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2825,"num_write_kb":180800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"78'122","reported_seq":574,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.504197+0000","last_change":"2023-08-28T02:28:28.383306+0000","last_active":"2023-08-28T02:31:12.504197+0000","last_peered":"2023-08-28T02:31:12.504197+0000","last_clean":"2023-08-28T02:31:12.504197+0000","last_became_active":"2023-08-28T02:28:28.382672+0000","last_became_peered":"2023-08-28T02:28:28.382672+0000","last_unstale":"2023-08-28T02:31:12.504197+0000","last_undegraded":"2023-08-28T02:31:12.504197+0000","last_fullsized":"2023-08-28T02:31:12.504197+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":857,"num_read_kb":2224,"num_write":99,"num_write_kb":2312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":247,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.491464+0000","last_change":"2023-08-28T02:27:31.424373+0000","last_active":"2023-08-28T02:31:12.491464+0000","last_peered":"2023-08-28T02:31:12.491464+0000","last_clean":"2023-08-28T02:31:12.491464+0000","last_became_active":"2023-08-28T02:27:31.324505+0000","last_became_peered":"2023-08-28T02:27:31.324505+0000","last_unstale":"2023-08-28T02:31:12.491464+0000","last_undegraded":"2023-08-28T02:31:12.491464+0000","last_fullsized":"2023-08-28T02:31:12.491464+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:27:57.837524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":236,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.504793+0000","last_change":"2023-08-28T02:28:25.351840+0000","last_active":"2023-08-28T02:31:12.504793+0000","last_peered":"2023-08-28T02:31:12.504793+0000","last_clean":"2023-08-28T02:31:12.504793+0000","last_became_active":"2023-08-28T02:28:25.351399+0000","last_became_peered":"2023-08-28T02:28:25.351399+0000","last_unstale":"2023-08-28T02:31:12.504793+0000","last_undegraded":"2023-08-28T02:31:12.504793+0000","last_fullsized":"2023-08-28T02:31:12.504793+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":224,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.491345+0000","last_change":"2023-08-28T02:23:38.329474+0000","last_active":"2023-08-28T02:31:12.491345+0000","last_peered":"2023-08-28T02:31:12.491345+0000","last_clean":"2023-08-28T02:31:12.491345+0000","last_became_active":"2023-08-28T02:23:38.328505+0000","last_became_peered":"2023-08-28T02:23:38.328505+0000","last_unstale":"2023-08-28T02:31:12.491345+0000","last_undegraded":"2023-08-28T02:31:12.491345+0000","last_fullsized":"2023-08-28T02:31:12.491345+0000","mapping_epoch":144,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:12:14.835809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":308,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.495911+0000","last_change":"2023-08-28T02:27:31.384467+0000","last_active":"2023-08-28T02:31:12.495911+0000","last_peered":"2023-08-28T02:31:12.495911+0000","last_clean":"2023-08-28T02:31:12.495911+0000","last_became_active":"2023-08-28T02:27:31.384183+0000","last_became_peered":"2023-08-28T02:27:31.384183+0000","last_unstale":"2023-08-28T02:31:12.495911+0000","last_undegraded":"2023-08-28T02:31:12.495911+0000","last_fullsized":"2023-08-28T02:31:12.495911+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:59:47.551892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":73,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.506178+0000","last_change":"2023-08-28T02:28:11.333622+0000","last_active":"2023-08-28T02:31:12.506178+0000","last_peered":"2023-08-28T02:31:12.506178+0000","last_clean":"2023-08-28T02:31:12.506178+0000","last_became_active":"2023-08-28T02:28:11.333321+0000","last_became_peered":"2023-08-28T02:28:11.333321+0000","last_unstale":"2023-08-28T02:31:12.506178+0000","last_undegraded":"2023-08-28T02:31:12.506178+0000","last_fullsized":"2023-08-28T02:31:12.506178+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":284,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.506230+0000","last_change":"2023-08-28T02:27:31.343538+0000","last_active":"2023-08-28T02:31:12.506230+0000","last_peered":"2023-08-28T02:31:12.506230+0000","last_clean":"2023-08-28T02:31:12.506230+0000","last_became_active":"2023-08-28T02:27:31.343324+0000","last_became_peered":"2023-08-28T02:27:31.343324+0000","last_unstale":"2023-08-28T02:31:12.506230+0000","last_undegraded":"2023-08-28T02:31:12.506230+0000","last_fullsized":"2023-08-28T02:31:12.506230+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":86,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.505663+0000","last_change":"2023-08-28T02:28:25.705331+0000","last_active":"2023-08-28T02:31:12.505663+0000","last_peered":"2023-08-28T02:31:12.505663+0000","last_clean":"2023-08-28T02:31:12.505663+0000","last_became_active":"2023-08-28T02:28:25.397480+0000","last_became_peered":"2023-08-28T02:28:25.397480+0000","last_unstale":"2023-08-28T02:31:12.505663+0000","last_undegraded":"2023-08-28T02:31:12.505663+0000","last_fullsized":"2023-08-28T02:31:12.505663+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":266,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.504822+0000","last_change":"2023-08-28T02:23:59.207929+0000","last_active":"2023-08-28T02:31:12.504822+0000","last_peered":"2023-08-28T02:31:12.504822+0000","last_clean":"2023-08-28T02:31:12.504822+0000","last_became_active":"2023-08-28T02:23:59.207589+0000","last_became_peered":"2023-08-28T02:23:59.207589+0000","last_unstale":"2023-08-28T02:31:12.504822+0000","last_undegraded":"2023-08-28T02:31:12.504822+0000","last_fullsized":"2023-08-28T02:31:12.504822+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":154,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.496106+0000","last_change":"2023-08-28T02:28:25.347988+0000","last_active":"2023-08-28T02:31:12.496106+0000","last_peered":"2023-08-28T02:31:12.496106+0000","last_clean":"2023-08-28T02:31:12.496106+0000","last_became_active":"2023-08-28T02:28:25.347632+0000","last_became_peered":"2023-08-28T02:28:25.347632+0000","last_unstale":"2023-08-28T02:31:12.496106+0000","last_undegraded":"2023-08-28T02:31:12.496106+0000","last_fullsized":"2023-08-28T02:31:12.496106+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:56:20.202662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":127,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:14.251474+0000","last_change":"2023-08-28T02:23:59.217794+0000","last_active":"2023-08-28T02:31:14.251474+0000","last_peered":"2023-08-28T02:31:14.251474+0000","last_clean":"2023-08-28T02:31:14.251474+0000","last_became_active":"2023-08-28T02:23:59.217231+0000","last_became_peered":"2023-08-28T02:23:59.217231+0000","last_unstale":"2023-08-28T02:31:14.251474+0000","last_undegraded":"2023-08-28T02:31:14.251474+0000","last_fullsized":"2023-08-28T02:31:14.251474+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:36:26.297410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":222,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.491761+0000","last_change":"2023-08-28T02:20:19.850252+0000","last_active":"2023-08-28T02:31:12.491761+0000","last_peered":"2023-08-28T02:31:12.491761+0000","last_clean":"2023-08-28T02:31:12.491761+0000","last_became_active":"2023-08-28T02:20:19.849913+0000","last_became_peered":"2023-08-28T02:20:19.849913+0000","last_unstale":"2023-08-28T02:31:12.491761+0000","last_undegraded":"2023-08-28T02:31:12.491761+0000","last_fullsized":"2023-08-28T02:31:12.491761+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:48:47.713465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":298,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-08-28T02:31:12.505667+0000","last_change":"2023-08-28T02:28:25.347845+0000","last_active":"2023-08-28T02:31:12.505667+0000","last_peered":"2023-08-28T02:31:12.505667+0000","last_clean":"2023-08-28T02:31:12.505667+0000","last_became_active":"2023-08-28T02:28:25.347502+0000","last_became_peered":"2023-08-28T02:28:25.347502+0000","last_unstale":"2023-08-28T02:31:12.505667+0000","last_undegraded":"2023-08-28T02:31:12.505667+0000","last_fullsized":"2023-08-28T02:31:12.505667+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":3048341504,"num_objects":46514,"num_object_clones":0,"num_object_copies":139542,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":46514,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":46514,"num_write_kb":2976896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9171042304,"data_stored":9171042304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":46514,"ondisk_log_size":46514,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":19382,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26810,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":137,"num_bytes_recovered":12582931,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430471,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1424584,"kb_used_data":1110976,"kb_used_omap":0,"kb_used_meta":313600,"kb_avail":92320568,"statfs":{"total":95995035648,"available":94536261632,"internally_reserved":0,"allocated":1137639424,"data_stored":1136916840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321126400},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:31:04 2023","interfaces":[{"interface":"back","average":{"1min":1.26,"5min":2.145,"15min":2.1339999999999999},"min":{"1min":0.23000000000000001,"5min":0.22800000000000001,"15min":0.21299999999999999},"max":{"1min":7.4240000000000004,"5min":59.753999999999998,"15min":116.755},"last":0.35799999999999998},{"interface":"front","average":{"1min":1.4550000000000001,"5min":2.4199999999999999,"15min":2.3919999999999999},"min":{"1min":0.252,"5min":0.245,"15min":0.23999999999999999},"max":{"1min":6.4690000000000003,"5min":59.765999999999998,"15min":116.971},"last":0.34999999999999998}]},{"osd":1,"last update":"Mon Aug 28 02:31:16 2023","interfaces":[{"interface":"back","average":{"1min":1.371,"5min":2.2090000000000001,"15min":1.2929999999999999},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":7.3769999999999998,"5min":44.287999999999997,"15min":44.287999999999997},"last":0.32300000000000001},{"interface":"front","average":{"1min":1.512,"5min":2.8079999999999998,"15min":1.728},"min":{"1min":0.29999999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":6.5490000000000004,"5min":44.393999999999998,"15min":44.393999999999998},"last":0.316}]},{"osd":2,"last update":"Mon Aug 28 02:30:35 2023","interfaces":[{"interface":"back","average":{"1min":2.1779999999999999,"5min":3.004,"15min":2.4830000000000001},"min":{"1min":0.32400000000000001,"5min":0.28000000000000003,"15min":0.27600000000000002},"max":{"1min":7.5300000000000002,"5min":101.616,"15min":119.571},"last":22.146000000000001},{"interface":"front","average":{"1min":1.9119999999999999,"5min":3.7210000000000001,"15min":2.875},"min":{"1min":0.32900000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.5380000000000003,"5min":101.605,"15min":119.554},"last":22.196999999999999}]},{"osd":3,"last update":"Mon Aug 28 02:31:16 2023","interfaces":[{"interface":"back","average":{"1min":1.746,"5min":3.9660000000000002,"15min":2.3199999999999998},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":11.429,"5min":99.459000000000003,"15min":107.916},"last":44.777999999999999},{"interface":"front","average":{"1min":1.845,"5min":3.5390000000000001,"15min":2.4020000000000001},"min":{"1min":0.32200000000000001,"5min":0.311,"15min":0.27700000000000002},"max":{"1min":8.3230000000000004,"5min":98.084000000000003,"15min":112.00700000000001},"last":44.414000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:30:35 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":1.26,"15min":1.254},"min":{"1min":0.28899999999999998,"5min":0.26500000000000001,"15min":0.22900000000000001},"max":{"1min":2.3929999999999998,"5min":11.696,"15min":63.048000000000002},"last":0.26900000000000002},{"interface":"front","average":{"1min":1.3660000000000001,"5min":1.347,"15min":1.24},"min":{"1min":0.316,"5min":0.26700000000000002,"15min":0.247},"max":{"1min":8.2140000000000004,"5min":10.840999999999999,"15min":62.912999999999997},"last":0.30299999999999999}]},{"osd":5,"last update":"Mon Aug 28 02:30:35 2023","interfaces":[{"interface":"back","average":{"1min":1.2,"5min":1.1859999999999999,"15min":1.075},"min":{"1min":0.32800000000000001,"5min":0.27800000000000002,"15min":0.26800000000000002},"max":{"1min":7.5110000000000001,"5min":13.788,"15min":63.021000000000001},"last":0.28899999999999998},{"interface":"front","average":{"1min":1.2949999999999999,"5min":1.4079999999999999,"15min":1.377},"min":{"1min":0.38200000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":6.7110000000000003,"5min":9.6929999999999996,"15min":63.082999999999998},"last":0.377}]},{"osd":6,"last update":"Mon Aug 28 02:30:35 2023","interfaces":[{"interface":"back","average":{"1min":1.302,"5min":1.2529999999999999,"15min":1.3280000000000001},"min":{"1min":0.36899999999999999,"5min":0.318,"15min":0.29699999999999999},"max":{"1min":7.5369999999999999,"5min":9.7070000000000007,"15min":101.61},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.78500000000000003,"5min":1.29,"15min":1.427},"min":{"1min":0.40999999999999998,"5min":0.308,"15min":0.308},"max":{"1min":2.4780000000000002,"5min":14.423999999999999,"15min":101.621},"last":0.41599999999999998}]}]},{"osd":6,"up_from":42,"seq":180388626698,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1251580,"kb_used_data":942068,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92493572,"statfs":{"total":95995035648,"available":94713417728,"internally_reserved":0,"allocated":964677632,"data_stored":963966894,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:31:14 2023","interfaces":[{"interface":"back","average":{"1min":1.5249999999999999,"5min":3.52,"15min":2.8260000000000001},"min":{"1min":0.317,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":7.1980000000000004,"5min":60.625999999999998,"15min":121.247},"last":0.25800000000000001},{"interface":"front","average":{"1min":0.90400000000000003,"5min":2.9980000000000002,"15min":2.742},"min":{"1min":0.245,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":3.4780000000000002,"5min":54.273000000000003,"15min":118.977},"last":3.726}]},{"osd":1,"last update":"Mon Aug 28 02:31:14 2023","interfaces":[{"interface":"back","average":{"1min":12.211,"5min":4.548,"15min":2.448},"min":{"1min":0.27400000000000002,"5min":0.23999999999999999,"15min":0.23699999999999999},"max":{"1min":141.87299999999999,"5min":141.87299999999999,"15min":141.87299999999999},"last":0.28699999999999998},{"interface":"front","average":{"1min":11.532999999999999,"5min":4.5510000000000002,"15min":2.54},"min":{"1min":0.27600000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":141.904,"5min":141.904,"15min":141.904},"last":0.38500000000000001}]},{"osd":2,"last update":"Mon Aug 28 02:30:54 2023","interfaces":[{"interface":"back","average":{"1min":4.6020000000000003,"5min":4.5670000000000002,"15min":2.4910000000000001},"min":{"1min":0.372,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":58.758000000000003,"5min":79.787000000000006,"15min":79.787000000000006},"last":0.42299999999999999},{"interface":"front","average":{"1min":4.6900000000000004,"5min":4.7910000000000004,"15min":2.4940000000000002},"min":{"1min":0.33700000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":58.887,"5min":79.935000000000002,"15min":79.935000000000002},"last":0.32600000000000001}]},{"osd":3,"last update":"Mon Aug 28 02:31:14 2023","interfaces":[{"interface":"back","average":{"1min":10.509,"5min":4.4299999999999997,"15min":2.3279999999999998},"min":{"1min":0.33600000000000002,"5min":0.32000000000000001,"15min":0.317},"max":{"1min":157.625,"5min":157.625,"15min":157.625},"last":0.35699999999999998},{"interface":"front","average":{"1min":9.4019999999999992,"5min":4.2720000000000002,"15min":2.3399999999999999},"min":{"1min":0.439,"5min":0.307,"15min":0.30099999999999999},"max":{"1min":157.637,"5min":157.637,"15min":157.637},"last":0.47099999999999997}]},{"osd":4,"last update":"Mon Aug 28 02:31:19 2023","interfaces":[{"interface":"back","average":{"1min":1.141,"5min":1.135,"15min":1.0880000000000001},"min":{"1min":0.27600000000000002,"5min":0.27100000000000002,"15min":0.251},"max":{"1min":7.3090000000000002,"5min":18.308,"15min":18.308},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.86499999999999999,"5min":1.147,"15min":1.1579999999999999},"min":{"1min":0.316,"5min":0.29699999999999999,"15min":0.26700000000000002},"max":{"1min":4.7690000000000001,"5min":18.263000000000002,"15min":18.263000000000002},"last":1.77}]},{"osd":5,"last update":"Mon Aug 28 02:31:19 2023","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":1.01,"15min":1.1439999999999999},"min":{"1min":0.29999999999999999,"5min":0.28899999999999998,"15min":0.26300000000000001},"max":{"1min":6.0049999999999999,"5min":8.0109999999999992,"15min":20.759},"last":0.439},{"interface":"front","average":{"1min":1.1579999999999999,"5min":0.98099999999999998,"15min":1.0009999999999999},"min":{"1min":0.33700000000000002,"5min":0.314,"15min":0.28799999999999998},"max":{"1min":7.2370000000000001,"5min":7.2370000000000001,"15min":7.3780000000000001},"last":0.44600000000000001}]},{"osd":7,"last update":"Mon Aug 28 02:30:44 2023","interfaces":[{"interface":"back","average":{"1min":1.744,"5min":1.151,"15min":1.0089999999999999},"min":{"1min":0.36699999999999999,"5min":0.34200000000000003,"15min":0.314},"max":{"1min":10.605,"5min":10.605,"15min":10.605},"last":1.609},{"interface":"front","average":{"1min":1.03,"5min":1.1499999999999999,"15min":1.0720000000000001},"min":{"1min":0.38700000000000001,"5min":0.34599999999999997,"15min":0.29799999999999999},"max":{"1min":3.4529999999999998,"5min":9.7110000000000003,"15min":9.7110000000000003},"last":0.55700000000000005}]}]},{"osd":1,"up_from":98,"seq":420906795150,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1394080,"kb_used_data":1087192,"kb_used_omap":7,"kb_used_meta":306872,"kb_avail":92351072,"statfs":{"total":95995035648,"available":94567497728,"internally_reserved":0,"allocated":1113284608,"data_stored":1111971071,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":314237259},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:31:09 2023","interfaces":[{"interface":"back","average":{"1min":2.6800000000000002,"5min":2.1230000000000002,"15min":2.2949999999999999},"min":{"1min":0.19600000000000001,"5min":0.191,"15min":0.17499999999999999},"max":{"1min":31.591999999999999,"5min":34.588999999999999,"15min":128.60599999999999},"last":0.31900000000000001},{"interface":"front","average":{"1min":3.2919999999999998,"5min":2.2290000000000001,"15min":2.3439999999999999},"min":{"1min":0.23400000000000001,"5min":0.188,"15min":0.188},"max":{"1min":31.260000000000002,"5min":34.628999999999998,"15min":128.685},"last":3.4969999999999999}]},{"osd":2,"last update":"Mon Aug 28 02:30:34 2023","interfaces":[{"interface":"back","average":{"1min":2.5640000000000001,"5min":6.2939999999999996,"15min":2.9630000000000001},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":8.1720000000000006,"5min":135.25999999999999,"15min":135.25999999999999},"last":8.4529999999999994},{"interface":"front","average":{"1min":2.1339999999999999,"5min":6.1920000000000002,"15min":2.7730000000000001},"min":{"1min":0.26700000000000002,"5min":0.23499999999999999,"15min":0.221},"max":{"1min":6.7089999999999996,"5min":135.10300000000001,"15min":135.10300000000001},"last":1.8680000000000001}]},{"osd":3,"last update":"Mon Aug 28 02:31:09 2023","interfaces":[{"interface":"back","average":{"1min":2.6709999999999998,"5min":3.8109999999999999,"15min":1.996},"min":{"1min":0.22500000000000001,"5min":0.22,"15min":0.22},"max":{"1min":11.263999999999999,"5min":123.741,"15min":123.741},"last":5.1470000000000002},{"interface":"front","average":{"1min":1.929,"5min":4.1239999999999997,"15min":2.2440000000000002},"min":{"1min":0.30099999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":11.372,"5min":123.642,"15min":123.642},"last":5.9820000000000002}]},{"osd":4,"last update":"Mon Aug 28 02:31:09 2023","interfaces":[{"interface":"back","average":{"1min":1.4470000000000001,"5min":1.397,"15min":1.319},"min":{"1min":0.39600000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":8.1989999999999998,"5min":8.1989999999999998,"15min":17.600000000000001},"last":1.921},{"interface":"front","average":{"1min":1.181,"5min":1.349,"15min":1.175},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.25600000000000001},"max":{"1min":4.7839999999999998,"5min":8.0419999999999998,"15min":19.010000000000002},"last":1.8360000000000001}]},{"osd":5,"last update":"Mon Aug 28 02:31:09 2023","interfaces":[{"interface":"back","average":{"1min":1.3660000000000001,"5min":1.6140000000000001,"15min":1.4199999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.29399999999999998},"max":{"1min":7.9029999999999996,"5min":11.167999999999999,"15min":15.544},"last":3.4489999999999998},{"interface":"front","average":{"1min":1.47,"5min":1.45,"15min":1.3100000000000001},"min":{"1min":0.35999999999999999,"5min":0.33600000000000002,"15min":0.33200000000000002},"max":{"1min":8.1010000000000009,"5min":10.084,"15min":17.646999999999998},"last":1.954}]},{"osd":6,"last update":"Mon Aug 28 02:31:09 2023","interfaces":[{"interface":"back","average":{"1min":1.5609999999999999,"5min":1.623,"15min":1.526},"min":{"1min":0.378,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":7.8719999999999999,"5min":9.2989999999999995,"15min":15.512},"last":3.4079999999999999},{"interface":"front","average":{"1min":1.5169999999999999,"5min":1.4570000000000001,"15min":1.4510000000000001},"min":{"1min":0.41899999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":8.1419999999999995,"5min":8.1419999999999995,"15min":17.673999999999999},"last":1.976}]},{"osd":7,"last update":"Mon Aug 28 02:31:09 2023","interfaces":[{"interface":"back","average":{"1min":3.8290000000000002,"5min":1.919,"15min":1.4650000000000001},"min":{"1min":0.39200000000000002,"5min":0.32500000000000001,"15min":0.30199999999999999},"max":{"1min":49.575000000000003,"5min":49.575000000000003,"15min":49.575000000000003},"last":1.891},{"interface":"front","average":{"1min":3.8279999999999998,"5min":2.2010000000000001,"15min":1.6579999999999999},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.33100000000000002},"max":{"1min":49.319000000000003,"5min":49.319000000000003,"15min":49.319000000000003},"last":3.4740000000000002}]}]},{"osd":0,"up_from":90,"seq":386547056796,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1797996,"kb_used_data":1424612,"kb_used_omap":8,"kb_used_meta":373367,"kb_avail":91947156,"statfs":{"total":95995035648,"available":94153887744,"internally_reserved":0,"allocated":1458802688,"data_stored":1457593907,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":382328169},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Aug 28 02:31:19 2023","interfaces":[{"interface":"back","average":{"1min":1.3120000000000001,"5min":2.7629999999999999,"15min":1.956},"min":{"1min":0.25,"5min":0.19400000000000001,"15min":0.184},"max":{"1min":6.9370000000000003,"5min":56.576999999999998,"15min":56.576999999999998},"last":0.191},{"interface":"front","average":{"1min":2.2160000000000002,"5min":3.2000000000000002,"15min":2.0539999999999998},"min":{"1min":0.27200000000000002,"5min":0.19600000000000001,"15min":0.17000000000000001},"max":{"1min":9.9160000000000004,"5min":56.552999999999997,"15min":56.552999999999997},"last":4.6970000000000001}]},{"osd":2,"last update":"Mon Aug 28 02:30:34 2023","interfaces":[{"interface":"back","average":{"1min":3.234,"5min":4.3090000000000002,"15min":2.202},"min":{"1min":0.22,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":10.565,"5min":100.718,"15min":100.718},"last":0.497},{"interface":"front","average":{"1min":1.333,"5min":4.3730000000000002,"15min":2.4159999999999999},"min":{"1min":0.24299999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":4.9989999999999997,"5min":100.962,"15min":100.962},"last":6.9169999999999998}]},{"osd":3,"last update":"Mon Aug 28 02:31:19 2023","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":6.3899999999999997,"15min":2.9870000000000001},"min":{"1min":0.28299999999999997,"5min":0.23499999999999999,"15min":0.23300000000000001},"max":{"1min":3.4500000000000002,"5min":237.43899999999999,"15min":237.43899999999999},"last":3.7290000000000001},{"interface":"front","average":{"1min":1.5149999999999999,"5min":6.2770000000000001,"15min":2.9340000000000002},"min":{"1min":0.34999999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":9.3719999999999999,"5min":237.48099999999999,"15min":237.48099999999999},"last":2.6680000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:31:16 2023","interfaces":[{"interface":"back","average":{"1min":1.869,"5min":1.5860000000000001,"15min":1.365},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":8.1199999999999992,"5min":9.3680000000000003,"15min":9.6649999999999991},"last":0.29099999999999998},{"interface":"front","average":{"1min":1.371,"5min":1.8080000000000001,"15min":1.5089999999999999},"min":{"1min":0.432,"5min":0.309,"15min":0.309},"max":{"1min":6.484,"5min":27.271999999999998,"15min":27.271999999999998},"last":2.621}]},{"osd":5,"last update":"Mon Aug 28 02:31:16 2023","interfaces":[{"interface":"back","average":{"1min":1.1930000000000001,"5min":1.6379999999999999,"15min":1.5960000000000001},"min":{"1min":0.41099999999999998,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":4.9029999999999996,"5min":27.367000000000001,"15min":44.000999999999998},"last":2.6850000000000001},{"interface":"front","average":{"1min":1.5820000000000001,"5min":1.589,"15min":1.55},"min":{"1min":0.308,"5min":0.308,"15min":0.30599999999999999},"max":{"1min":8.0809999999999995,"5min":9.4489999999999998,"15min":43.953000000000003},"last":0.36199999999999999}]},{"osd":6,"last update":"Mon Aug 28 02:31:16 2023","interfaces":[{"interface":"back","average":{"1min":1.7989999999999999,"5min":1.802,"15min":1.661},"min":{"1min":0.38500000000000001,"5min":0.312,"15min":0.312},"max":{"1min":7.6340000000000003,"5min":27.331,"15min":32.417999999999999},"last":2.9529999999999998},{"interface":"front","average":{"1min":1.409,"5min":1.845,"15min":1.609},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.30399999999999999},"max":{"1min":5.9290000000000003,"5min":27.247,"15min":32.130000000000003},"last":0.39400000000000002}]},{"osd":7,"last update":"Mon Aug 28 02:31:16 2023","interfaces":[{"interface":"back","average":{"1min":1.73,"5min":2.169,"15min":1.6519999999999999},"min":{"1min":0.35099999999999998,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":6.0449999999999999,"5min":27.385000000000002,"15min":27.385000000000002},"last":0.46300000000000002},{"interface":"front","average":{"1min":1.3029999999999999,"5min":1.869,"15min":1.639},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":4.3680000000000003,"5min":27.213999999999999,"15min":27.213999999999999},"last":3.069}]}]},{"osd":2,"up_from":104,"seq":446676598920,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1533180,"kb_used_data":1180788,"kb_used_omap":10,"kb_used_meta":352373,"kb_avail":92211972,"statfs":{"total":95995035648,"available":94425059328,"internally_reserved":0,"allocated":1209126912,"data_stored":1207959085,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10792,"internal_metadata":360830424},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:30:37 2023","interfaces":[{"interface":"back","average":{"1min":2.3050000000000002,"5min":4.21,"15min":2.2599999999999998},"min":{"1min":0.20300000000000001,"5min":0.17699999999999999,"15min":0.17599999999999999},"max":{"1min":16.48,"5min":113.89100000000001,"15min":113.89100000000001},"last":7.593},{"interface":"front","average":{"1min":2.5499999999999998,"5min":7.5940000000000003,"15min":3.472},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.19900000000000001},"max":{"1min":11.166,"5min":147.286,"15min":147.286},"last":6.4829999999999997}]},{"osd":1,"last update":"Mon Aug 28 02:30:37 2023","interfaces":[{"interface":"back","average":{"1min":2.6560000000000001,"5min":2.431,"15min":1.738},"min":{"1min":0.23799999999999999,"5min":0.21099999999999999,"15min":0.20999999999999999},"max":{"1min":11.061,"5min":11.627000000000001,"15min":30.888000000000002},"last":7.0730000000000004},{"interface":"front","average":{"1min":3.351,"5min":2.8029999999999999,"15min":1.6890000000000001},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.20999999999999999},"max":{"1min":11.962999999999999,"5min":20.259,"15min":31.648},"last":0.97699999999999998}]},{"osd":3,"last update":"Mon Aug 28 02:31:09 2023","interfaces":[{"interface":"back","average":{"1min":4.8330000000000002,"5min":4.0049999999999999,"15min":2.581},"min":{"1min":0.27200000000000002,"5min":0.25,"15min":0.22700000000000001},"max":{"1min":60.341000000000001,"5min":71.983999999999995,"15min":128.63900000000001},"last":6.6909999999999998},{"interface":"front","average":{"1min":4.2080000000000002,"5min":3.6629999999999998,"15min":2.556},"min":{"1min":0.29899999999999999,"5min":0.223,"15min":0.223},"max":{"1min":60.445,"5min":71.944999999999993,"15min":128.75700000000001},"last":1.004}]},{"osd":4,"last update":"Mon Aug 28 02:30:37 2023","interfaces":[{"interface":"back","average":{"1min":12.464,"5min":5.617,"15min":2.661},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":207.34100000000001,"5min":207.34100000000001,"15min":207.34100000000001},"last":0.94199999999999995},{"interface":"front","average":{"1min":12.086,"5min":5.2610000000000001,"15min":2.4129999999999998},"min":{"1min":0.30299999999999999,"5min":0.28999999999999998,"15min":0.27400000000000002},"max":{"1min":207.352,"5min":207.352,"15min":207.352},"last":2.903}]},{"osd":5,"last update":"Mon Aug 28 02:30:37 2023","interfaces":[{"interface":"back","average":{"1min":1.998,"5min":5.4800000000000004,"15min":2.5379999999999998},"min":{"1min":0.33800000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":5.8120000000000003,"5min":128.696,"15min":128.696},"last":6.6539999999999999},{"interface":"front","average":{"1min":2.0249999999999999,"5min":5.1769999999999996,"15min":2.5939999999999999},"min":{"1min":0.32300000000000001,"5min":0.32100000000000001,"15min":0.30299999999999999},"max":{"1min":8.0709999999999997,"5min":128.84100000000001,"15min":128.84100000000001},"last":2.2450000000000001}]},{"osd":6,"last update":"Mon Aug 28 02:30:37 2023","interfaces":[{"interface":"back","average":{"1min":1.9370000000000001,"5min":1.79,"15min":1.3500000000000001},"min":{"1min":0.32800000000000001,"5min":0.307,"15min":0.29299999999999998},"max":{"1min":11.997,"5min":11.997,"15min":11.997},"last":3.3679999999999999},{"interface":"front","average":{"1min":1.8939999999999999,"5min":2.1299999999999999,"15min":1.494},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":5.9530000000000003,"5min":11.055999999999999,"15min":11.055999999999999},"last":6.593}]},{"osd":7,"last update":"Mon Aug 28 02:30:37 2023","interfaces":[{"interface":"back","average":{"1min":1.827,"5min":5.3289999999999997,"15min":2.5129999999999999},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.32100000000000001},"max":{"1min":6.29,"5min":128.911,"15min":128.911},"last":2.9460000000000002},{"interface":"front","average":{"1min":1.7410000000000001,"5min":5.859,"15min":2.7120000000000002},"min":{"1min":0.35599999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":4.8819999999999997,"5min":128.72399999999999,"15min":128.72399999999999},"last":6.7160000000000002}]}]},{"osd":3,"up_from":112,"seq":481036337282,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1893240,"kb_used_data":1600368,"kb_used_omap":3,"kb_used_meta":292860,"kb_avail":91851912,"statfs":{"total":95995035648,"available":94056357888,"internally_reserved":0,"allocated":1638776832,"data_stored":1637610300,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":299889570},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:31:06 2023","interfaces":[{"interface":"back","average":{"1min":1.9330000000000001,"5min":3.1509999999999998,"15min":3.1139999999999999},"min":{"1min":0.22500000000000001,"5min":0.193,"15min":0.193},"max":{"1min":9.3670000000000009,"5min":79.793000000000006,"15min":164.25299999999999},"last":0.23300000000000001},{"interface":"front","average":{"1min":1.7909999999999999,"5min":2.7789999999999999,"15min":3.0590000000000002},"min":{"1min":0.23000000000000001,"5min":0.192,"15min":0.192},"max":{"1min":7.8159999999999998,"5min":78.555000000000007,"15min":164.55000000000001},"last":5.3040000000000003}]},{"osd":1,"last update":"Mon Aug 28 02:31:06 2023","interfaces":[{"interface":"back","average":{"1min":1.0580000000000001,"5min":1.4690000000000001,"15min":1.9159999999999999},"min":{"1min":0.26800000000000002,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":5.923,"5min":8.109,"15min":117.34699999999999},"last":0.33500000000000002},{"interface":"front","average":{"1min":1.74,"5min":2.1360000000000001,"15min":2.0590000000000002},"min":{"1min":0.27900000000000003,"5min":0.245,"15min":0.224},"max":{"1min":9.2989999999999995,"5min":14.300000000000001,"15min":118.012},"last":0.32100000000000001}]},{"osd":2,"last update":"Mon Aug 28 02:31:06 2023","interfaces":[{"interface":"back","average":{"1min":1.883,"5min":2.7970000000000002,"15min":2.915},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.19900000000000001},"max":{"1min":7.1189999999999998,"5min":65.308999999999997,"15min":120.43300000000001},"last":0.53200000000000003},{"interface":"front","average":{"1min":1.2310000000000001,"5min":2.4780000000000002,"15min":2.8170000000000002},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":7.8540000000000001,"5min":65.120999999999995,"15min":120.44499999999999},"last":5.2729999999999997}]},{"osd":4,"last update":"Mon Aug 28 02:31:06 2023","interfaces":[{"interface":"back","average":{"1min":1.4610000000000001,"5min":2.2389999999999999,"15min":1.516},"min":{"1min":0.26000000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":14.042,"5min":58.067999999999998,"15min":58.067999999999998},"last":3.7189999999999999},{"interface":"front","average":{"1min":1.173,"5min":2.3849999999999998,"15min":1.5629999999999999},"min":{"1min":0.374,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":5.8620000000000001,"5min":58.088000000000001,"15min":58.088000000000001},"last":0.44900000000000001}]},{"osd":5,"last update":"Mon Aug 28 02:31:06 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":2.9159999999999999,"15min":1.6799999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.29599999999999999},"max":{"1min":5.835,"5min":102.904,"15min":102.904},"last":0.375},{"interface":"front","average":{"1min":1.24,"5min":3.145,"15min":1.702},"min":{"1min":0.38700000000000001,"5min":0.36699999999999999,"15min":0.32800000000000001},"max":{"1min":5.0069999999999997,"5min":103.283,"15min":103.283},"last":0.56699999999999995}]},{"osd":6,"last update":"Mon Aug 28 02:31:06 2023","interfaces":[{"interface":"back","average":{"1min":1.4730000000000001,"5min":1.6859999999999999,"15min":1.4890000000000001},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.314},"max":{"1min":5.0490000000000004,"5min":9.157,"15min":72.116},"last":0.60799999999999998},{"interface":"front","average":{"1min":1.5940000000000001,"5min":1.4470000000000001,"15min":1.4219999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":7.8899999999999997,"5min":7.8899999999999997,"15min":72.030000000000001},"last":5.1760000000000002}]},{"osd":7,"last update":"Mon Aug 28 02:31:06 2023","interfaces":[{"interface":"back","average":{"1min":2.8370000000000002,"5min":2.9470000000000001,"15min":1.7769999999999999},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.32500000000000001},"max":{"1min":27.111999999999998,"5min":54.427999999999997,"15min":54.427999999999997},"last":5.2320000000000002},{"interface":"front","average":{"1min":2.4129999999999998,"5min":3.0819999999999999,"15min":1.8779999999999999},"min":{"1min":0.40600000000000003,"5min":0.36399999999999999,"15min":0.34799999999999998},"max":{"1min":27.149999999999999,"5min":54.445,"15min":54.445},"last":0.47799999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019156,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1288892,"kb_used_data":983476,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":92456260,"statfs":{"total":95995035648,"available":94675210240,"internally_reserved":0,"allocated":1007079424,"data_stored":1006368492,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:31:11 2023","interfaces":[{"interface":"back","average":{"1min":3.125,"5min":2.2149999999999999,"15min":2.8730000000000002},"min":{"1min":0.247,"5min":0.219,"15min":0.219},"max":{"1min":12.685,"5min":34.387,"15min":138.67400000000001},"last":2.5779999999999998},{"interface":"front","average":{"1min":1.012,"5min":1.742,"15min":2.665},"min":{"1min":0.28100000000000003,"5min":0.248,"15min":0.248},"max":{"1min":2.9089999999999998,"5min":33.539000000000001,"15min":134.14599999999999},"last":7.181}]},{"osd":1,"last update":"Mon Aug 28 02:31:11 2023","interfaces":[{"interface":"back","average":{"1min":2.911,"5min":2.2160000000000002,"15min":1.5509999999999999},"min":{"1min":0.30199999999999999,"5min":0.221,"15min":0.221},"max":{"1min":9.2520000000000007,"5min":34.945999999999998,"15min":53.622999999999998},"last":0.41999999999999998},{"interface":"front","average":{"1min":1.0129999999999999,"5min":1.849,"15min":1.4379999999999999},"min":{"1min":0.28699999999999998,"5min":0.25,"15min":0.25},"max":{"1min":4.4400000000000004,"5min":34.869999999999997,"15min":53.774000000000001},"last":2.5270000000000001}]},{"osd":2,"last update":"Mon Aug 28 02:30:37 2023","interfaces":[{"interface":"back","average":{"1min":4.867,"5min":4.3040000000000003,"15min":2.8250000000000002},"min":{"1min":0.33100000000000002,"5min":0.27400000000000002,"15min":0.25600000000000001},"max":{"1min":56.948999999999998,"5min":64.063000000000002,"15min":74.099000000000004},"last":2.0739999999999998},{"interface":"front","average":{"1min":5.117,"5min":4.1769999999999996,"15min":2.7330000000000001},"min":{"1min":0.377,"5min":0.28999999999999998,"15min":0.25},"max":{"1min":56.979999999999997,"5min":64.073999999999998,"15min":74.081999999999994},"last":1.234}]},{"osd":3,"last update":"Mon Aug 28 02:31:11 2023","interfaces":[{"interface":"back","average":{"1min":3.4209999999999998,"5min":6.0990000000000002,"15min":2.7050000000000001},"min":{"1min":0.37,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":10.050000000000001,"5min":159.10400000000001,"15min":159.10400000000001},"last":0.84399999999999997},{"interface":"front","average":{"1min":2.605,"5min":5.9260000000000002,"15min":2.7320000000000002},"min":{"1min":0.35999999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":9.3699999999999992,"5min":159.232,"15min":159.232},"last":2.9239999999999999}]},{"osd":5,"last update":"Mon Aug 28 02:30:51 2023","interfaces":[{"interface":"back","average":{"1min":1.325,"5min":2.8929999999999998,"15min":1.454},"min":{"1min":0.28299999999999997,"5min":0.246,"15min":0.246},"max":{"1min":5.2309999999999999,"5min":124.718,"15min":124.718},"last":0.311},{"interface":"front","average":{"1min":1.7809999999999999,"5min":3.0910000000000002,"15min":1.641},"min":{"1min":0.34699999999999998,"5min":0.247,"15min":0.247},"max":{"1min":10.24,"5min":124.694,"15min":124.694},"last":0.96899999999999997}]},{"osd":6,"last update":"Mon Aug 28 02:31:11 2023","interfaces":[{"interface":"back","average":{"1min":2.718,"5min":1.3859999999999999,"15min":1.0960000000000001},"min":{"1min":0.34599999999999997,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":14.907999999999999,"5min":14.907999999999999,"15min":14.907999999999999},"last":0.46500000000000002},{"interface":"front","average":{"1min":1.296,"5min":1.097,"15min":0.88400000000000001},"min":{"1min":0.40999999999999998,"5min":0.307,"15min":0.307},"max":{"1min":5.1609999999999996,"5min":5.6900000000000004,"15min":7.3170000000000002},"last":0.377}]},{"osd":7,"last update":"Mon Aug 28 02:31:29 2023","interfaces":[{"interface":"back","average":{"1min":1.5129999999999999,"5min":1.145,"15min":1.1850000000000001},"min":{"1min":0.30499999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":10.151999999999999,"5min":10.151999999999999,"15min":16.584},"last":1.0009999999999999},{"interface":"front","average":{"1min":2.0779999999999998,"5min":1.3400000000000001,"15min":1.2410000000000001},"min":{"1min":0.34799999999999998,"5min":0.311,"15min":0.311},"max":{"1min":9.3469999999999995,"5min":10.826000000000001,"15min":10.826000000000001},"last":0.44700000000000001}]}]},{"osd":5,"up_from":36,"seq":154618822928,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1072876,"kb_used_data":771556,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92672276,"statfs":{"total":95995035648,"available":94896410624,"internally_reserved":0,"allocated":790073344,"data_stored":789373951,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:31:08 2023","interfaces":[{"interface":"back","average":{"1min":6.0430000000000001,"5min":2.556,"15min":3.008},"min":{"1min":0.23400000000000001,"5min":0.214,"15min":0.20300000000000001},"max":{"1min":104.13200000000001,"5min":104.13200000000001,"15min":131.06100000000001},"last":0.94299999999999995},{"interface":"front","average":{"1min":6.2149999999999999,"5min":2.8620000000000001,"15min":3.2189999999999999},"min":{"1min":0.28199999999999997,"5min":0.27100000000000002,"15min":0.22900000000000001},"max":{"1min":104.142,"5min":104.142,"15min":131.262},"last":1.714}]},{"osd":1,"last update":"Mon Aug 28 02:31:14 2023","interfaces":[{"interface":"back","average":{"1min":1.347,"5min":1.7969999999999999,"15min":2.4420000000000002},"min":{"1min":0.29699999999999999,"5min":0.24399999999999999,"15min":0.216},"max":{"1min":4.3899999999999997,"5min":30.593,"15min":183.97900000000001},"last":1.9710000000000001},{"interface":"front","average":{"1min":1.3660000000000001,"5min":1.7809999999999999,"15min":2.6059999999999999},"min":{"1min":0.30599999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":3.7250000000000001,"5min":34.616999999999997,"15min":184.33699999999999},"last":2.637}]},{"osd":2,"last update":"Mon Aug 28 02:30:41 2023","interfaces":[{"interface":"back","average":{"1min":2.7469999999999999,"5min":6.117,"15min":3.9740000000000002},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.252},"max":{"1min":11.875,"5min":176.69499999999999,"15min":176.69499999999999},"last":0.91400000000000003},{"interface":"front","average":{"1min":3.7160000000000002,"5min":6.5620000000000003,"15min":4.21},"min":{"1min":0.35299999999999998,"5min":0.34000000000000002,"15min":0.28299999999999997},"max":{"1min":11.887,"5min":176.71199999999999,"15min":176.71199999999999},"last":1.931}]},{"osd":3,"last update":"Mon Aug 28 02:31:14 2023","interfaces":[{"interface":"back","average":{"1min":1.9059999999999999,"5min":2.8530000000000002,"15min":1.7050000000000001},"min":{"1min":0.32100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":10.94,"5min":102.64400000000001,"15min":102.64400000000001},"last":1.1990000000000001},{"interface":"front","average":{"1min":1.617,"5min":3.2639999999999998,"15min":2.0219999999999998},"min":{"1min":0.35099999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":8.3089999999999993,"5min":103.771,"15min":103.771},"last":1.8280000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:30:52 2023","interfaces":[{"interface":"back","average":{"1min":6.1349999999999998,"5min":2.0990000000000002,"15min":1.3100000000000001},"min":{"1min":0.33300000000000002,"5min":0.26700000000000002,"15min":0.246},"max":{"1min":85.769999999999996,"5min":85.769999999999996,"15min":85.769999999999996},"last":1.6619999999999999},{"interface":"front","average":{"1min":5.7480000000000002,"5min":1.9630000000000001,"15min":1.3120000000000001},"min":{"1min":0.31,"5min":0.28999999999999998,"15min":0.28000000000000003},"max":{"1min":85.825000000000003,"5min":85.825000000000003,"15min":85.825000000000003},"last":0.84599999999999997}]},{"osd":6,"last update":"Mon Aug 28 02:31:14 2023","interfaces":[{"interface":"back","average":{"1min":1.056,"5min":0.91600000000000004,"15min":1.0409999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":4.1449999999999996,"5min":5.7089999999999996,"15min":14.803000000000001},"last":1.7929999999999999},{"interface":"front","average":{"1min":0.81399999999999995,"5min":0.87,"15min":0.93799999999999994},"min":{"1min":0.29899999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":3.976,"5min":6.601,"15min":8.6479999999999997},"last":0.88200000000000001}]},{"osd":7,"last update":"Mon Aug 28 02:30:29 2023","interfaces":[{"interface":"back","average":{"1min":1.1419999999999999,"5min":1.5109999999999999,"15min":1.127},"min":{"1min":0.34300000000000003,"5min":0.33300000000000002,"15min":0.31},"max":{"1min":4.0060000000000002,"5min":25.379000000000001,"15min":25.379000000000001},"last":1.6140000000000001},{"interface":"front","average":{"1min":1.3160000000000001,"5min":1.2769999999999999,"15min":1.0569999999999999},"min":{"1min":0.38300000000000001,"5min":0.33700000000000002,"15min":0.32100000000000001},"max":{"1min":5.931,"5min":25.437000000000001,"15min":25.437000000000001},"last":1.778}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1469448192,"data_stored":1469448192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1123090432,"data_stored":1123090432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1122762752,"data_stored":1122762752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1674903552,"data_stored":1674903552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1007812608,"data_stored":1007812608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":799014912,"data_stored":799014912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":980221952,"data_stored":980221952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":993787904,"data_stored":993787904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-08-28T02:31:35.225 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-08-28T02:31:35.225 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd pool set unique_pool_2 pg_num 26 2023-08-28T02:31:35.234 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:31:35.308 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:35.374 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:31:35.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:35.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:31:35.478 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:31:35.511 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:35.551 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:31:35.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:35.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:31:35.657 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:35 smithi143 ceph-mon[183059]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:31:35.657 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:35 smithi143 ceph-mon[183059]: failed to encode map e268 with expected crc 2023-08-28T02:31:35.657 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:35 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:31:35.657 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:35 smithi143 ceph-mon[183059]: osdmap e268: 8 total, 8 up, 8 in 2023-08-28T02:31:35.657 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:35 smithi143 ceph-mon[183059]: failed to encode map e268 with expected crc 2023-08-28T02:31:35.657 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:35 smithi143 ceph-mon[183059]: failed to encode map e268 with expected crc 2023-08-28T02:31:35.658 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:35 smithi143 ceph-mon[183059]: failed to encode map e268 with expected crc 2023-08-28T02:31:35.658 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:31:35.659 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:35 smithi143 ceph-mon[189871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:31:35.659 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:35 smithi143 ceph-mon[189871]: failed to encode map e268 with expected crc 2023-08-28T02:31:35.659 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:35 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:31:35.659 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:35 smithi143 ceph-mon[189871]: osdmap e268: 8 total, 8 up, 8 in 2023-08-28T02:31:35.659 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:35 smithi143 ceph-mon[189871]: failed to encode map e268 with expected crc 2023-08-28T02:31:35.659 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:35 smithi143 ceph-mon[189871]: failed to encode map e268 with expected crc 2023-08-28T02:31:35.659 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:35 smithi143 ceph-mon[189871]: failed to encode map e268 with expected crc 2023-08-28T02:31:35.724 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:35.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:35 smithi169 ceph-mon[159415]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:31:35.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:35 smithi169 ceph-mon[159415]: failed to encode map e268 with expected crc 2023-08-28T02:31:35.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:35 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:31:35.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:35 smithi169 ceph-mon[159415]: osdmap e268: 8 total, 8 up, 8 in 2023-08-28T02:31:35.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:35 smithi169 ceph-mon[159415]: failed to encode map e268 with expected crc 2023-08-28T02:31:35.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:35 smithi169 ceph-mon[159415]: failed to encode map e268 with expected crc 2023-08-28T02:31:35.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:35 smithi169 ceph-mon[159415]: failed to encode map e268 with expected crc 2023-08-28T02:31:35.834 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 34 16 52787 52771 96.9268 63.6875 0.00392703 0.0103098 2023-08-28T02:31:36.343 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:31:36.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:36.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:31:36.453 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:31:36.515 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:36.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:36 smithi143 ceph-mon[183059]: pgmap v662: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2023-08-28T02:31:36.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:36 smithi143 ceph-mon[189871]: pgmap v662: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2023-08-28T02:31:36.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:36 smithi169 ceph-mon[159415]: pgmap v662: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2023-08-28T02:31:36.837 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 35 16 53636 53620 95.6674 53.0625 0.014882 0.0104469 2023-08-28T02:31:36.967 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:31:36.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:36.976 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:31:37.077 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:31:37.109 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:37.165 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:31:37.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:37.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:31:37.271 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:31:37.309 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:37.372 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:31:37.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:37.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:31:37.463 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:31:37.481 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:31:37.515 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:37.576 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:31:37.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:37.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:31:37.659 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:31:37.681 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:31:37.697 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:31:37.709 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:37.724 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:31:37.765 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:31:37.772 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:31:37.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:37.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:31:37.808 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:31:37.837 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 36 16 54765 54749 94.9704 70.5625 0.0191152 0.0105219 2023-08-28T02:31:37.841 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:31:37.878 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:31:37.884 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:31:37.909 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:37.931 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:31:37.939 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:31:37.939 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:31:37.939 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:31:37.940 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:31:37.940 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:31:37.940 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:31:37.940 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:31:37.940 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:31:37.964 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:31:37.966 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:31:37.966 INFO:teuthology.orchestra.run.smithi143.stdout: "size": 20, 2023-08-28T02:31:37.966 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 600, 2023-08-28T02:31:37.967 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [ 2023-08-28T02:31:37.967 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.967 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:123521 5.0 5:0f371aae:::benchmark_data_smithi143_236200_object13378:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e154)", 2023-08-28T02:31:37.967 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:59.196550+0000", 2023-08-28T02:31:37.967 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 458.768269858, 2023-08-28T02:31:37.967 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.128653905, 2023-08-28T02:31:37.967 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:31:37.967 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:37.967 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:31:37.967 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:31:37.967 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:31:37.968 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 123521 2023-08-28T02:31:37.968 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.968 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:31:37.968 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.968 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:31:37.968 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196550+0000", 2023-08-28T02:31:37.968 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:37.968 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.968 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.968 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:31:37.968 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196550+0000", 2023-08-28T02:31:37.968 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:37.968 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.969 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.969 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:31:37.969 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196548+0000", 2023-08-28T02:31:37.969 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:31:37.969 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.969 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.969 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:31:37.969 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196553+0000", 2023-08-28T02:31:37.969 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.6439999999999999e-06 2023-08-28T02:31:37.969 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.969 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.969 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:31:37.969 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196564+0000", 2023-08-28T02:31:37.970 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0550999999999999e-05 2023-08-28T02:31:37.970 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.970 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.970 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:31:37.970 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196573+0000", 2023-08-28T02:31:37.970 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.8619999999999998e-06 2023-08-28T02:31:37.970 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.970 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.970 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:37.970 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196799+0000", 2023-08-28T02:31:37.970 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000225913 2023-08-28T02:31:37.970 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.971 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.971 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op must wait for map", 2023-08-28T02:31:37.971 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196806+0000", 2023-08-28T02:31:37.971 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.0720000000000002e-06 2023-08-28T02:31:37.971 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.971 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.971 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:37.971 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.208800+0000", 2023-08-28T02:31:37.971 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.011993996999999999 2023-08-28T02:31:37.971 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.971 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.971 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:31:37.971 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.208811+0000", 2023-08-28T02:31:37.972 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.048e-05 2023-08-28T02:31:37.972 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.972 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.972 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:37.972 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.305571+0000", 2023-08-28T02:31:37.972 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.096759813 2023-08-28T02:31:37.972 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.972 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.972 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:31:37.972 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.307358+0000", 2023-08-28T02:31:37.972 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0017876599999999999 2023-08-28T02:31:37.972 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.973 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.973 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,3,4", 2023-08-28T02:31:37.973 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.307473+0000", 2023-08-28T02:31:37.973 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000114498 2023-08-28T02:31:37.973 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.973 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.973 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:37.973 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.317779+0000", 2023-08-28T02:31:37.973 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.010306031 2023-08-28T02:31:37.973 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.973 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.973 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:37.973 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.317817+0000", 2023-08-28T02:31:37.974 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.7766999999999999e-05 2023-08-28T02:31:37.974 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.974 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.974 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:37.974 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.324897+0000", 2023-08-28T02:31:37.974 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0070804420000000002 2023-08-28T02:31:37.974 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.974 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.974 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:31:37.974 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325027+0000", 2023-08-28T02:31:37.974 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00012982799999999999 2023-08-28T02:31:37.974 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.974 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.975 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:31:37.975 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325088+0000", 2023-08-28T02:31:37.975 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.1532999999999995e-05 2023-08-28T02:31:37.975 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.975 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.975 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:31:37.975 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325204+0000", 2023-08-28T02:31:37.975 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00011503299999999999 2023-08-28T02:31:37.975 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:37.975 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:37.975 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:37.975 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.976 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.976 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:123522 5.0 5:0bc2203e:::benchmark_data_smithi143_236200_object13379:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e154)", 2023-08-28T02:31:37.976 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:59.196585+0000", 2023-08-28T02:31:37.976 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 458.76823480600001, 2023-08-28T02:31:37.976 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1287665410000001, 2023-08-28T02:31:37.976 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:31:37.976 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:37.976 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:31:37.976 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:31:37.976 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:31:37.976 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 123522 2023-08-28T02:31:37.976 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.976 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:31:37.977 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.977 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:31:37.977 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196585+0000", 2023-08-28T02:31:37.977 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:37.977 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.977 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.977 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:31:37.977 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196585+0000", 2023-08-28T02:31:37.977 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:37.977 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.977 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.977 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:31:37.977 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196584+0000", 2023-08-28T02:31:37.978 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:31:37.978 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.978 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.978 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:31:37.978 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196587+0000", 2023-08-28T02:31:37.978 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.238e-06 2023-08-28T02:31:37.978 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.978 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.978 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:31:37.978 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196593+0000", 2023-08-28T02:31:37.978 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.9710000000000001e-06 2023-08-28T02:31:37.978 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.978 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.979 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:31:37.979 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196598+0000", 2023-08-28T02:31:37.979 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.1120000000000004e-06 2023-08-28T02:31:37.979 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.979 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.979 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:37.979 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196858+0000", 2023-08-28T02:31:37.979 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00026082200000000002 2023-08-28T02:31:37.979 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.979 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.979 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting_for_map not empty", 2023-08-28T02:31:37.979 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196863+0000", 2023-08-28T02:31:37.979 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.8999999999999997e-06 2023-08-28T02:31:37.980 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.980 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.980 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:37.980 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.209793+0000", 2023-08-28T02:31:37.980 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.012929248000000001 2023-08-28T02:31:37.980 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.980 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.980 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:31:37.980 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.209798+0000", 2023-08-28T02:31:37.980 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.7250000000000002e-06 2023-08-28T02:31:37.980 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.980 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.980 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:37.981 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.310606+0000", 2023-08-28T02:31:37.981 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.10080757 2023-08-28T02:31:37.981 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.981 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.981 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:31:37.981 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.310743+0000", 2023-08-28T02:31:37.981 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00013726600000000001 2023-08-28T02:31:37.981 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.981 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.981 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,3,4", 2023-08-28T02:31:37.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:37.983 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.310871+0000", 2023-08-28T02:31:37.983 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00012777599999999999 2023-08-28T02:31:37.983 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.983 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.983 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:37.983 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.317856+0000", 2023-08-28T02:31:37.983 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0069848460000000003 2023-08-28T02:31:37.984 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.984 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.984 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:37.984 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.317896+0000", 2023-08-28T02:31:37.984 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.9743999999999998e-05 2023-08-28T02:31:37.984 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.984 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.984 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:37.984 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.324952+0000", 2023-08-28T02:31:37.984 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0070567060000000003 2023-08-28T02:31:37.984 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.984 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.984 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:31:37.985 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325214+0000", 2023-08-28T02:31:37.985 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00026183400000000001 2023-08-28T02:31:37.985 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.985 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.985 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:31:37.985 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325264+0000", 2023-08-28T02:31:37.985 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.9496999999999997e-05 2023-08-28T02:31:37.985 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.985 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.985 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:31:37.985 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325351+0000", 2023-08-28T02:31:37.985 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.763e-05 2023-08-28T02:31:37.986 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:37.986 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:37.986 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:37.986 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.987 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.987 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:123553 5.0 5:03eb21c5:::benchmark_data_smithi143_236200_object13410:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e154)", 2023-08-28T02:31:37.987 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:59.196602+0000", 2023-08-28T02:31:37.987 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 458.76821765099999, 2023-08-28T02:31:37.987 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.128974272, 2023-08-28T02:31:37.987 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:31:37.987 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:37.987 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:31:37.987 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:31:37.987 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:31:37.987 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 123553 2023-08-28T02:31:37.987 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.988 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:31:37.988 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.988 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:31:37.988 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196602+0000", 2023-08-28T02:31:37.988 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:37.988 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.988 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.988 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:31:37.988 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196602+0000", 2023-08-28T02:31:37.988 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:37.988 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.988 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.988 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:31:37.989 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196602+0000", 2023-08-28T02:31:37.989 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:31:37.989 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.989 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.989 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:31:37.989 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196604+0000", 2023-08-28T02:31:37.989 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.0990000000000001e-06 2023-08-28T02:31:37.989 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.989 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.989 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:31:37.989 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196609+0000", 2023-08-28T02:31:37.989 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.3920000000000002e-06 2023-08-28T02:31:37.989 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.990 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.990 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:31:37.990 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196613+0000", 2023-08-28T02:31:37.990 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.1679999999999997e-06 2023-08-28T02:31:37.990 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.990 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.990 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:37.990 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196883+0000", 2023-08-28T02:31:37.990 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00026973699999999998 2023-08-28T02:31:37.990 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.990 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.990 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting_for_map not empty", 2023-08-28T02:31:37.990 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196888+0000", 2023-08-28T02:31:37.991 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.3399999999999997e-06 2023-08-28T02:31:37.991 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.991 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.991 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:37.991 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.209815+0000", 2023-08-28T02:31:37.991 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.012926936 2023-08-28T02:31:37.991 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.991 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.991 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:31:37.991 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.209821+0000", 2023-08-28T02:31:37.991 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.049e-06 2023-08-28T02:31:37.991 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.991 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.992 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:37.992 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.314325+0000", 2023-08-28T02:31:37.992 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.10450329 2023-08-28T02:31:37.992 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.992 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.992 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:31:37.992 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.314416+0000", 2023-08-28T02:31:37.992 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.1209e-05 2023-08-28T02:31:37.992 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.992 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.992 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,3,4", 2023-08-28T02:31:37.992 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.314497+0000", 2023-08-28T02:31:37.992 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.0943000000000001e-05 2023-08-28T02:31:37.992 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.993 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.993 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:37.993 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.317937+0000", 2023-08-28T02:31:37.993 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.003440088 2023-08-28T02:31:37.993 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.993 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.993 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:31:37.993 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325361+0000", 2023-08-28T02:31:37.993 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0074246319999999996 2023-08-28T02:31:37.993 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.993 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.993 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:37.994 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325398+0000", 2023-08-28T02:31:37.994 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.6105000000000001e-05 2023-08-28T02:31:37.994 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.994 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.994 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:37.994 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325471+0000", 2023-08-28T02:31:37.994 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.3779e-05 2023-08-28T02:31:37.994 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.994 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.994 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:31:37.994 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325499+0000", 2023-08-28T02:31:37.994 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.7585000000000001e-05 2023-08-28T02:31:37.994 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.994 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.995 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:31:37.995 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325576+0000", 2023-08-28T02:31:37.995 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.7211000000000003e-05 2023-08-28T02:31:37.995 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:37.995 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:37.995 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:37.995 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.995 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.995 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:123563 5.0 5:0dabf8cb:::benchmark_data_smithi143_236200_object13420:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e154)", 2023-08-28T02:31:37.995 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:59.196617+0000", 2023-08-28T02:31:37.995 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 458.76820276000001, 2023-08-28T02:31:37.995 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1291040370000001, 2023-08-28T02:31:37.996 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:31:37.996 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:37.996 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:31:37.996 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:31:37.996 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:31:37.996 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 123563 2023-08-28T02:31:37.996 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.996 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:31:37.996 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.996 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:31:37.996 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196617+0000", 2023-08-28T02:31:37.996 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:37.996 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.996 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.997 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:31:37.997 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196617+0000", 2023-08-28T02:31:37.997 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:37.997 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.997 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.997 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:31:37.997 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196616+0000", 2023-08-28T02:31:37.997 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:31:37.997 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.997 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.997 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:31:37.997 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196618+0000", 2023-08-28T02:31:37.998 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.9970000000000001e-06 2023-08-28T02:31:37.998 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.998 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.998 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:31:37.998 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196623+0000", 2023-08-28T02:31:37.998 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.8509999999999998e-06 2023-08-28T02:31:37.998 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.998 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.998 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:31:37.998 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196628+0000", 2023-08-28T02:31:37.998 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.6149999999999999e-06 2023-08-28T02:31:37.998 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.998 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.999 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:37.999 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196908+0000", 2023-08-28T02:31:37.999 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000279788 2023-08-28T02:31:37.999 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.999 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.999 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting_for_map not empty", 2023-08-28T02:31:37.999 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196912+0000", 2023-08-28T02:31:37.999 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.544e-06 2023-08-28T02:31:37.999 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:37.999 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:37.999 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:37.999 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.209839+0000", 2023-08-28T02:31:37.999 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.012926524999999999 2023-08-28T02:31:38.000 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.000 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.000 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:31:38.000 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.209845+0000", 2023-08-28T02:31:38.000 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.579e-06 2023-08-28T02:31:38.000 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.000 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.000 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:38.000 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.317968+0000", 2023-08-28T02:31:38.000 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.108122914 2023-08-28T02:31:38.000 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.000 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.001 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:31:38.001 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.318146+0000", 2023-08-28T02:31:38.001 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00017759300000000001 2023-08-28T02:31:38.001 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.001 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.001 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,3,4", 2023-08-28T02:31:38.001 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.318278+0000", 2023-08-28T02:31:38.001 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00013184000000000001 2023-08-28T02:31:38.001 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.001 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.001 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:38.001 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325014+0000", 2023-08-28T02:31:38.001 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0067361549999999997 2023-08-28T02:31:38.002 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.002 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.002 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:31:38.002 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325368+0000", 2023-08-28T02:31:38.002 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00035389200000000002 2023-08-28T02:31:38.002 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.002 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.002 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:38.002 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325421+0000", 2023-08-28T02:31:38.002 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.2902000000000003e-05 2023-08-28T02:31:38.002 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.002 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.002 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:38.003 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325612+0000", 2023-08-28T02:31:38.003 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00019104500000000001 2023-08-28T02:31:38.003 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.003 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.003 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:31:38.003 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325640+0000", 2023-08-28T02:31:38.003 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.8425999999999999e-05 2023-08-28T02:31:38.003 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.003 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.003 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:31:38.003 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325721+0000", 2023-08-28T02:31:38.003 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.0642000000000005e-05 2023-08-28T02:31:38.003 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:38.004 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:38.004 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:38.004 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.004 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.004 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:123565 5.0 5:0ec9f19d:::benchmark_data_smithi143_236200_object13422:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e154)", 2023-08-28T02:31:38.004 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:59.196632+0000", 2023-08-28T02:31:38.004 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 458.768187705, 2023-08-28T02:31:38.004 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.129315396, 2023-08-28T02:31:38.004 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:31:38.004 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:38.004 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:31:38.004 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:31:38.004 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:31:38.005 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 123565 2023-08-28T02:31:38.005 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.005 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:31:38.005 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.005 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:31:38.005 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196632+0000", 2023-08-28T02:31:38.005 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:38.005 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.005 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.005 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:31:38.005 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196632+0000", 2023-08-28T02:31:38.005 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:38.006 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.006 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.006 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:31:38.006 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196631+0000", 2023-08-28T02:31:38.006 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:31:38.006 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.006 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.006 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:31:38.006 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196634+0000", 2023-08-28T02:31:38.006 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.0650000000000001e-06 2023-08-28T02:31:38.006 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.006 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.006 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:31:38.006 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196638+0000", 2023-08-28T02:31:38.007 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.6700000000000002e-06 2023-08-28T02:31:38.007 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.007 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.007 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:31:38.007 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196643+0000", 2023-08-28T02:31:38.007 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.7859999999999996e-06 2023-08-28T02:31:38.007 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.007 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.007 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:38.007 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196931+0000", 2023-08-28T02:31:38.007 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000288044 2023-08-28T02:31:38.007 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.007 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.008 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting_for_map not empty", 2023-08-28T02:31:38.008 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196935+0000", 2023-08-28T02:31:38.008 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.3930000000000001e-06 2023-08-28T02:31:38.008 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.008 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.008 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:38.008 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.209868+0000", 2023-08-28T02:31:38.008 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.012932073000000001 2023-08-28T02:31:38.008 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.008 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.008 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:31:38.008 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.209874+0000", 2023-08-28T02:31:38.008 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.6989999999999997e-06 2023-08-28T02:31:38.009 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.009 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.009 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:38.009 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.321422+0000", 2023-08-28T02:31:38.009 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.111547675 2023-08-28T02:31:38.009 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.009 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.009 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:31:38.009 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.321549+0000", 2023-08-28T02:31:38.009 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000127028 2023-08-28T02:31:38.009 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.009 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.009 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,3,4", 2023-08-28T02:31:38.010 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.321630+0000", 2023-08-28T02:31:38.010 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.0882000000000003e-05 2023-08-28T02:31:38.010 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.010 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.010 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:31:38.010 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325428+0000", 2023-08-28T02:31:38.010 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0037977200000000001 2023-08-28T02:31:38.010 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.010 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.010 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:38.010 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325447+0000", 2023-08-28T02:31:38.010 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.906e-05 2023-08-28T02:31:38.010 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.011 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.011 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:38.011 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325752+0000", 2023-08-28T02:31:38.011 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00030524300000000002 2023-08-28T02:31:38.011 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.011 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.011 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:38.011 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325801+0000", 2023-08-28T02:31:38.011 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.9429999999999999e-05 2023-08-28T02:31:38.011 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.011 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.011 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:31:38.011 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325845+0000", 2023-08-28T02:31:38.012 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.3900000000000003e-05 2023-08-28T02:31:38.012 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.012 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.012 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:31:38.012 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325947+0000", 2023-08-28T02:31:38.012 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000102007 2023-08-28T02:31:38.012 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:38.012 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:38.012 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:38.012 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.012 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.012 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:123578 5.0 5:0d4223f2:::benchmark_data_smithi143_236200_object13435:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e154)", 2023-08-28T02:31:38.012 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:59.196647+0000", 2023-08-28T02:31:38.013 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 458.76817261000002, 2023-08-28T02:31:38.013 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.14071996, 2023-08-28T02:31:38.013 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:31:38.013 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:38.013 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:31:38.013 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:31:38.013 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:31:38.013 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 123578 2023-08-28T02:31:38.013 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.013 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:31:38.013 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.013 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:31:38.013 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196647+0000", 2023-08-28T02:31:38.014 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:38.014 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.014 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.014 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:31:38.014 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196647+0000", 2023-08-28T02:31:38.014 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:38.014 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.014 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.014 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:31:38.014 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196647+0000", 2023-08-28T02:31:38.014 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:31:38.015 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.015 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.015 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:31:38.015 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196649+0000", 2023-08-28T02:31:38.015 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.4449999999999999e-06 2023-08-28T02:31:38.015 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.015 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.015 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:31:38.015 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196654+0000", 2023-08-28T02:31:38.015 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.1139999999999997e-06 2023-08-28T02:31:38.015 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.015 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.016 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:31:38.016 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196658+0000", 2023-08-28T02:31:38.016 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.2420000000000002e-06 2023-08-28T02:31:38.016 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.016 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.016 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:38.016 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196950+0000", 2023-08-28T02:31:38.016 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00029176700000000001 2023-08-28T02:31:38.016 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.016 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.016 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting_for_map not empty", 2023-08-28T02:31:38.016 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196953+0000", 2023-08-28T02:31:38.016 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.5359999999999999e-06 2023-08-28T02:31:38.016 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.017 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.017 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:38.017 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.209894+0000", 2023-08-28T02:31:38.017 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.012941413000000001 2023-08-28T02:31:38.017 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.017 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.017 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:31:38.017 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.209898+0000", 2023-08-28T02:31:38.017 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.484e-06 2023-08-28T02:31:38.017 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.017 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.017 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:38.017 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.325997+0000", 2023-08-28T02:31:38.018 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.116099921 2023-08-28T02:31:38.018 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.018 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.018 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:31:38.018 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.326137+0000", 2023-08-28T02:31:38.018 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000139696 2023-08-28T02:31:38.018 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.018 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.018 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,3,4", 2023-08-28T02:31:38.018 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.326268+0000", 2023-08-28T02:31:38.018 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00013096399999999999 2023-08-28T02:31:38.018 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.018 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.018 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:38.019 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.335576+0000", 2023-08-28T02:31:38.019 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0093078999999999992 2023-08-28T02:31:38.019 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.019 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.019 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:38.019 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.335620+0000", 2023-08-28T02:31:38.019 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.3868000000000001e-05 2023-08-28T02:31:38.019 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.019 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.019 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:31:38.019 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.335661+0000", 2023-08-28T02:31:38.019 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.0908999999999997e-05 2023-08-28T02:31:38.020 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.020 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.020 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:38.020 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.337193+0000", 2023-08-28T02:31:38.020 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0015320399999999999 2023-08-28T02:31:38.020 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.020 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.020 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:31:38.020 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.337245+0000", 2023-08-28T02:31:38.020 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.2092999999999997e-05 2023-08-28T02:31:38.020 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.020 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.020 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:31:38.020 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.337367+0000", 2023-08-28T02:31:38.021 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000121902 2023-08-28T02:31:38.021 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:38.021 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:38.021 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:38.021 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.021 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.021 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:123581 5.0 5:06f44fcd:::benchmark_data_smithi143_236200_object13438:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e154)", 2023-08-28T02:31:38.021 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:59.196662+0000", 2023-08-28T02:31:38.021 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 458.76815703400001, 2023-08-28T02:31:38.021 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.14107384, 2023-08-28T02:31:38.021 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:31:38.021 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:38.021 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:31:38.022 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:31:38.022 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:31:38.022 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 123581 2023-08-28T02:31:38.022 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.022 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:31:38.022 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.022 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:31:38.022 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196662+0000", 2023-08-28T02:31:38.022 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:38.022 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.022 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.022 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:31:38.022 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196662+0000", 2023-08-28T02:31:38.023 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:38.023 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.023 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.023 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:31:38.023 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196662+0000", 2023-08-28T02:31:38.023 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:31:38.023 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.023 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.023 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:31:38.023 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196664+0000", 2023-08-28T02:31:38.023 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.6810000000000001e-06 2023-08-28T02:31:38.023 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.023 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.024 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:31:38.024 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196669+0000", 2023-08-28T02:31:38.024 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.0610000000000003e-06 2023-08-28T02:31:38.024 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.024 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.024 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:31:38.024 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196673+0000", 2023-08-28T02:31:38.024 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.2080000000000002e-06 2023-08-28T02:31:38.024 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.024 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.024 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:38.024 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196963+0000", 2023-08-28T02:31:38.024 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00029004399999999999 2023-08-28T02:31:38.025 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.025 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.025 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting_for_map not empty", 2023-08-28T02:31:38.025 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.196966+0000", 2023-08-28T02:31:38.025 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.362e-06 2023-08-28T02:31:38.025 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.025 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.025 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:38.025 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.209987+0000", 2023-08-28T02:31:38.025 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.013021624000000001 2023-08-28T02:31:38.025 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.025 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.025 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:31:38.026 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.209994+0000", 2023-08-28T02:31:38.026 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.6440000000000003e-06 2023-08-28T02:31:38.026 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.026 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.026 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:38.026 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.332359+0000", 2023-08-28T02:31:38.026 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1223654620000001 2023-08-28T02:31:38.026 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.026 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.026 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:31:38.026 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.332502+0000", 2023-08-28T02:31:38.026 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00014240999999999999 2023-08-28T02:31:38.027 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.027 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.027 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,3,4", 2023-08-28T02:31:38.027 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.332652+0000", 2023-08-28T02:31:38.027 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000149853 2023-08-28T02:31:38.027 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.027 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.027 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:38.027 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.335652+0000", 2023-08-28T02:31:38.027 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0030004179999999999 2023-08-28T02:31:38.027 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.027 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.027 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:31:38.028 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.335671+0000", 2023-08-28T02:31:38.028 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.9049e-05 2023-08-28T02:31:38.028 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.028 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.028 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:38.028 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.336202+0000", 2023-08-28T02:31:38.028 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00053052899999999996 2023-08-28T02:31:38.028 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.028 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.028 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:38.028 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.337555+0000", 2023-08-28T02:31:38.028 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0013538479999999999 2023-08-28T02:31:38.029 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.029 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.029 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:31:38.029 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.337610+0000", 2023-08-28T02:31:38.029 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.4595000000000002e-05 2023-08-28T02:31:38.029 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.029 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.029 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:31:38.029 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.337736+0000", 2023-08-28T02:31:38.029 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00012634199999999999 2023-08-28T02:31:38.029 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:38.029 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:38.029 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:38.030 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.030 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.030 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:124089 5.0 5:06d6dacb:::benchmark_data_smithi143_236200_object13946:head [delete] snapc 0=[] ondisk+write+known_if_redirected e155)", 2023-08-28T02:31:38.030 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:24:01.180357+0000", 2023-08-28T02:31:38.030 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 456.78446238800001, 2023-08-28T02:31:38.030 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.22593939499999999, 2023-08-28T02:31:38.030 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:31:38.030 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:38.030 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:31:38.030 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:31:38.030 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:31:38.030 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 124089 2023-08-28T02:31:38.030 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.031 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:31:38.031 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.031 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:31:38.031 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.180357+0000", 2023-08-28T02:31:38.031 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:38.031 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.031 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.031 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:31:38.031 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.180357+0000", 2023-08-28T02:31:38.031 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:38.031 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.031 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.031 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:31:38.032 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.180356+0000", 2023-08-28T02:31:38.032 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:31:38.032 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.032 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.032 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:31:38.032 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.180363+0000", 2023-08-28T02:31:38.032 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.0899999999999999e-06 2023-08-28T02:31:38.032 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.032 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.032 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:31:38.032 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.180381+0000", 2023-08-28T02:31:38.032 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.8519e-05 2023-08-28T02:31:38.033 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.033 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.033 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:31:38.033 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.180391+0000", 2023-08-28T02:31:38.033 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0192e-05 2023-08-28T02:31:38.033 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.033 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.033 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:38.033 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.180456+0000", 2023-08-28T02:31:38.033 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.4084999999999994e-05 2023-08-28T02:31:38.033 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.033 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.033 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:31:38.034 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.180607+0000", 2023-08-28T02:31:38.034 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000151232 2023-08-28T02:31:38.034 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.034 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.034 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,3,4", 2023-08-28T02:31:38.034 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.181058+0000", 2023-08-28T02:31:38.034 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00045131499999999998 2023-08-28T02:31:38.034 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.034 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.034 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:31:38.035 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.187145+0000", 2023-08-28T02:31:38.035 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0060865600000000004 2023-08-28T02:31:38.035 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.035 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.035 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:38.035 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.187725+0000", 2023-08-28T02:31:38.035 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00058020299999999995 2023-08-28T02:31:38.035 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.035 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.035 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:38.035 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.189158+0000", 2023-08-28T02:31:38.035 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.001433134 2023-08-28T02:31:38.036 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.036 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.036 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:38.036 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.406130+0000", 2023-08-28T02:31:38.036 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.216971527 2023-08-28T02:31:38.036 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.036 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.036 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:31:38.036 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.406181+0000", 2023-08-28T02:31:38.036 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.1264999999999998e-05 2023-08-28T02:31:38.036 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.036 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.037 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:31:38.037 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:01.406296+0000", 2023-08-28T02:31:38.037 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000115686 2023-08-28T02:31:38.037 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:38.037 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:38.037 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:38.037 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.037 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.037 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:130455 5.3 5:c62e95e0:::benchmark_data_smithi143_236200_object20312:head [delete] snapc 0=[] ondisk+write+known_if_redirected e163)", 2023-08-28T02:31:38.037 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:24:13.094164+0000", 2023-08-28T02:31:38.037 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 444.87065594199998, 2023-08-28T02:31:38.037 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.23304668100000001, 2023-08-28T02:31:38.037 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:31:38.038 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:38.038 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:31:38.038 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:31:38.038 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:31:38.038 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 130455 2023-08-28T02:31:38.038 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.038 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:31:38.038 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.038 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:31:38.038 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.094164+0000", 2023-08-28T02:31:38.038 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:38.038 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.038 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.039 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:31:38.039 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.094164+0000", 2023-08-28T02:31:38.039 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:38.039 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.039 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.039 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:31:38.039 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.094162+0000", 2023-08-28T02:31:38.039 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:31:38.039 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.039 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.039 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:31:38.039 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.094168+0000", 2023-08-28T02:31:38.040 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.5480000000000001e-06 2023-08-28T02:31:38.040 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.040 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.040 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:31:38.040 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.200430+0000", 2023-08-28T02:31:38.040 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.106262407 2023-08-28T02:31:38.040 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.040 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.040 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:31:38.040 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.200446+0000", 2023-08-28T02:31:38.040 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.5396e-05 2023-08-28T02:31:38.040 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.040 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.041 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:38.041 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.208625+0000", 2023-08-28T02:31:38.041 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0081796609999999995 2023-08-28T02:31:38.041 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.041 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.041 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:31:38.041 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.208750+0000", 2023-08-28T02:31:38.041 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000124692 2023-08-28T02:31:38.041 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.041 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.041 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 5,6", 2023-08-28T02:31:38.041 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.208854+0000", 2023-08-28T02:31:38.041 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000103959 2023-08-28T02:31:38.041 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.042 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.042 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:31:38.042 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.212292+0000", 2023-08-28T02:31:38.042 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0034382129999999999 2023-08-28T02:31:38.042 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.042 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.042 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:38.042 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.327100+0000", 2023-08-28T02:31:38.042 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.114808038 2023-08-28T02:31:38.042 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.042 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.042 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:38.043 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.327129+0000", 2023-08-28T02:31:38.043 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.8972e-05 2023-08-28T02:31:38.043 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.043 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.043 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:31:38.043 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.327172+0000", 2023-08-28T02:31:38.043 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.3093000000000002e-05 2023-08-28T02:31:38.043 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.043 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.043 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:31:38.043 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:13.327210+0000", 2023-08-28T02:31:38.043 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.7783000000000001e-05 2023-08-28T02:31:38.043 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:38.044 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:38.044 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:38.044 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.044 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.044 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:140334 5.0 5:0466772d:::benchmark_data_smithi143_236200_object30191:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-08-28T02:31:38.044 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:24:28.055937+0000", 2023-08-28T02:31:38.044 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 429.90888263599999, 2023-08-28T02:31:38.044 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.25254626600000002, 2023-08-28T02:31:38.044 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:31:38.044 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:38.044 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:31:38.044 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:31:38.045 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:31:38.045 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 140334 2023-08-28T02:31:38.045 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.045 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:31:38.045 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.045 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:31:38.045 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.055937+0000", 2023-08-28T02:31:38.045 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:38.045 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.045 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.045 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:31:38.045 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.055937+0000", 2023-08-28T02:31:38.045 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:38.046 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.046 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.046 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:31:38.046 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.055936+0000", 2023-08-28T02:31:38.046 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:31:38.046 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.046 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.046 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:31:38.046 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.055942+0000", 2023-08-28T02:31:38.046 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.9519999999999999e-06 2023-08-28T02:31:38.046 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.046 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.046 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:31:38.047 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.240512+0000", 2023-08-28T02:31:38.047 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.18457072699999999 2023-08-28T02:31:38.047 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.047 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.047 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:31:38.047 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.240525+0000", 2023-08-28T02:31:38.047 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.2515e-05 2023-08-28T02:31:38.047 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.047 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.047 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:38.047 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.240582+0000", 2023-08-28T02:31:38.047 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.6839000000000002e-05 2023-08-28T02:31:38.048 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.048 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.048 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:31:38.048 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.240726+0000", 2023-08-28T02:31:38.048 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0001448 2023-08-28T02:31:38.048 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.048 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.048 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,3,4", 2023-08-28T02:31:38.048 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.240865+0000", 2023-08-28T02:31:38.048 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000138601 2023-08-28T02:31:38.048 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.048 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.048 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:31:38.049 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.245256+0000", 2023-08-28T02:31:38.049 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0043909229999999997 2023-08-28T02:31:38.049 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.049 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.049 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:38.049 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.247591+0000", 2023-08-28T02:31:38.049 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0023347799999999998 2023-08-28T02:31:38.049 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.049 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.049 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:38.049 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.256080+0000", 2023-08-28T02:31:38.049 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0084893469999999995 2023-08-28T02:31:38.049 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.050 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.050 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:38.050 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.308312+0000", 2023-08-28T02:31:38.050 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.052232369000000001 2023-08-28T02:31:38.050 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.050 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.050 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:31:38.050 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.308369+0000", 2023-08-28T02:31:38.050 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.6746000000000002e-05 2023-08-28T02:31:38.050 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.050 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.050 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:31:38.051 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:28.308483+0000", 2023-08-28T02:31:38.051 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00011390700000000001 2023-08-28T02:31:38.051 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:38.051 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:38.051 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:38.051 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.051 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.051 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:196479 5.0 5:09cd934e:::benchmark_data_smithi143_236200_object86336:head [delete] snapc 0=[] ondisk+write+known_if_redirected e179)", 2023-08-28T02:31:38.051 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:25:58.967224+0000", 2023-08-28T02:31:38.051 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 338.99759508900001, 2023-08-28T02:31:38.051 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.23017068800000001, 2023-08-28T02:31:38.051 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:31:38.051 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:38.052 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:31:38.052 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:31:38.052 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:31:38.052 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 196479 2023-08-28T02:31:38.052 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.052 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:31:38.052 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.052 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:31:38.052 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.967224+0000", 2023-08-28T02:31:38.052 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:38.052 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.052 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.052 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:31:38.053 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.967224+0000", 2023-08-28T02:31:38.053 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:38.053 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.053 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.053 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:31:38.053 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.967223+0000", 2023-08-28T02:31:38.053 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:31:38.053 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.053 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.053 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:31:38.053 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.967231+0000", 2023-08-28T02:31:38.053 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.5250000000000005e-06 2023-08-28T02:31:38.053 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.054 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.054 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:31:38.054 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.967245+0000", 2023-08-28T02:31:38.054 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3628000000000001e-05 2023-08-28T02:31:38.054 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.054 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.054 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:31:38.054 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.967254+0000", 2023-08-28T02:31:38.054 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.5920000000000001e-06 2023-08-28T02:31:38.054 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.054 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.054 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:38.054 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.967321+0000", 2023-08-28T02:31:38.055 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.7597999999999999e-05 2023-08-28T02:31:38.055 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.055 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.055 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:31:38.055 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.967453+0000", 2023-08-28T02:31:38.055 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00013172800000000001 2023-08-28T02:31:38.055 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.055 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.055 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,3,4", 2023-08-28T02:31:38.055 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.967581+0000", 2023-08-28T02:31:38.055 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00012768499999999999 2023-08-28T02:31:38.055 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.056 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.056 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:31:38.056 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.976035+0000", 2023-08-28T02:31:38.056 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0084540080000000007 2023-08-28T02:31:38.056 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.056 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.056 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:38.056 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.990429+0000", 2023-08-28T02:31:38.056 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.014394239 2023-08-28T02:31:38.056 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.056 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.056 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:38.056 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:59.193123+0000", 2023-08-28T02:31:38.057 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.202694387 2023-08-28T02:31:38.057 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.057 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.057 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:38.057 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:59.197240+0000", 2023-08-28T02:31:38.057 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0041169079999999999 2023-08-28T02:31:38.057 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.057 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.057 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:31:38.057 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:59.197286+0000", 2023-08-28T02:31:38.057 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.5537000000000001e-05 2023-08-28T02:31:38.057 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.057 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.058 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:31:38.058 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:59.197395+0000", 2023-08-28T02:31:38.058 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00010937 2023-08-28T02:31:38.058 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:38.058 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:38.058 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:38.058 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.058 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.058 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:196481 5.0 5:02ad625a:::benchmark_data_smithi143_236200_object86338:head [delete] snapc 0=[] ondisk+write+known_if_redirected e179)", 2023-08-28T02:31:38.058 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:25:58.967272+0000", 2023-08-28T02:31:38.058 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 338.99754729099999, 2023-08-28T02:31:38.058 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.23031311300000001, 2023-08-28T02:31:38.059 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:31:38.059 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:38.059 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:31:38.059 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:31:38.059 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:31:38.059 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 196481 2023-08-28T02:31:38.059 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.059 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:31:38.059 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.059 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:31:38.059 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.967272+0000", 2023-08-28T02:31:38.059 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:38.062 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.062 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.062 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:31:38.062 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.967272+0000", 2023-08-28T02:31:38.062 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:38.062 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.062 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.062 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:31:38.062 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.967272+0000", 2023-08-28T02:31:38.062 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:31:38.062 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.062 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.063 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:31:38.063 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.967275+0000", 2023-08-28T02:31:38.063 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.7020000000000001e-06 2023-08-28T02:31:38.063 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.063 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.063 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:31:38.063 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.967280+0000", 2023-08-28T02:31:38.063 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.9220000000000002e-06 2023-08-28T02:31:38.063 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.063 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.063 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:31:38.063 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.967285+0000", 2023-08-28T02:31:38.063 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.4689999999999999e-06 2023-08-28T02:31:38.064 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.064 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.064 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:38.064 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.971611+0000", 2023-08-28T02:31:38.064 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0043258719999999997 2023-08-28T02:31:38.064 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.064 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.064 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:31:38.064 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.971747+0000", 2023-08-28T02:31:38.064 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00013653999999999999 2023-08-28T02:31:38.064 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.064 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.064 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,3,4", 2023-08-28T02:31:38.065 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.971883+0000", 2023-08-28T02:31:38.065 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00013597699999999999 2023-08-28T02:31:38.065 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.065 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.065 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:31:38.065 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:58.976324+0000", 2023-08-28T02:31:38.065 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0044409469999999998 2023-08-28T02:31:38.065 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.065 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.065 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:38.065 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:59.193077+0000", 2023-08-28T02:31:38.065 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.216752376 2023-08-28T02:31:38.065 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.066 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.066 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:38.066 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:59.193216+0000", 2023-08-28T02:31:38.066 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00013949899999999999 2023-08-28T02:31:38.066 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.066 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.066 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:38.066 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:59.197441+0000", 2023-08-28T02:31:38.066 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0042246360000000004 2023-08-28T02:31:38.066 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.066 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.066 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:31:38.066 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:59.197483+0000", 2023-08-28T02:31:38.067 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.1931999999999998e-05 2023-08-28T02:31:38.067 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.067 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.067 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:31:38.067 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:25:59.197585+0000", 2023-08-28T02:31:38.067 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000102631 2023-08-28T02:31:38.067 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:38.067 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:38.067 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:38.068 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.068 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.068 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:78148 6.a 6:510b0ad3:::benchmark_data_smithi143_305742_object78147:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e221)", 2023-08-28T02:31:38.068 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:28:25.332701+0000", 2023-08-28T02:31:38.068 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 192.63211870999999, 2023-08-28T02:31:38.068 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.148737852, 2023-08-28T02:31:38.068 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:31:38.068 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "reached pg", 2023-08-28T02:31:38.068 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:31:38.068 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:31:38.068 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:31:38.068 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 78148 2023-08-28T02:31:38.069 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.069 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:31:38.069 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.069 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:31:38.069 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.332701+0000", 2023-08-28T02:31:38.069 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:38.069 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.069 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.069 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:31:38.069 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.332701+0000", 2023-08-28T02:31:38.069 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:38.069 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.069 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.070 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:31:38.070 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.332699+0000", 2023-08-28T02:31:38.070 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:31:38.070 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.070 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.070 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:31:38.070 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.332728+0000", 2023-08-28T02:31:38.070 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.9193999999999999e-05 2023-08-28T02:31:38.070 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.070 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.070 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:31:38.070 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.332742+0000", 2023-08-28T02:31:38.071 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3518e-05 2023-08-28T02:31:38.071 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.071 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.071 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:31:38.071 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.332752+0000", 2023-08-28T02:31:38.071 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0026e-05 2023-08-28T02:31:38.071 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.071 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.071 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:38.071 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.334218+0000", 2023-08-28T02:31:38.071 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0014661159999999999 2023-08-28T02:31:38.072 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.072 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.072 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:31:38.072 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.334231+0000", 2023-08-28T02:31:38.072 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.358e-05 2023-08-28T02:31:38.072 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.072 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.072 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:38.072 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.470078+0000", 2023-08-28T02:31:38.072 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.135846471 2023-08-28T02:31:38.072 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.072 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.072 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for degraded object", 2023-08-28T02:31:38.073 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.471483+0000", 2023-08-28T02:31:38.073 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.001405585 2023-08-28T02:31:38.073 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.073 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.073 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:38.073 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.481350+0000", 2023-08-28T02:31:38.073 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0098665560000000003 2023-08-28T02:31:38.073 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.073 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.073 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:31:38.073 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.481439+0000", 2023-08-28T02:31:38.073 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.8771999999999996e-05 2023-08-28T02:31:38.074 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:38.074 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:38.074 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:38.074 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.074 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.074 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:78161 6.a 6:5fab5d5c:::benchmark_data_smithi143_305742_object78160:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e221)", 2023-08-28T02:31:38.074 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:28:25.332788+0000", 2023-08-28T02:31:38.074 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 192.632031635, 2023-08-28T02:31:38.074 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1547578279999999, 2023-08-28T02:31:38.074 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:31:38.074 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:38.075 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:31:38.075 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:31:38.075 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:31:38.075 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 78161 2023-08-28T02:31:38.075 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.075 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:31:38.075 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.075 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:31:38.075 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.332788+0000", 2023-08-28T02:31:38.075 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:38.075 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.075 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.076 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:31:38.076 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.332788+0000", 2023-08-28T02:31:38.076 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:38.076 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.076 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.076 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:31:38.076 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.332787+0000", 2023-08-28T02:31:38.076 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:31:38.076 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.076 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.076 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:31:38.076 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.332814+0000", 2023-08-28T02:31:38.077 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.6265999999999999e-05 2023-08-28T02:31:38.077 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.077 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.077 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:31:38.077 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.332821+0000", 2023-08-28T02:31:38.077 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.6000000000000001e-06 2023-08-28T02:31:38.077 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.077 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.077 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:31:38.077 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.332827+0000", 2023-08-28T02:31:38.077 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.6969999999999998e-06 2023-08-28T02:31:38.077 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.078 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.078 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:38.078 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.334262+0000", 2023-08-28T02:31:38.078 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0014349409999999999 2023-08-28T02:31:38.078 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.078 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.078 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:31:38.078 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.334271+0000", 2023-08-28T02:31:38.078 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.6850000000000007e-06 2023-08-28T02:31:38.078 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.078 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.078 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:38.079 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.471538+0000", 2023-08-28T02:31:38.079 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1372679139999999 2023-08-28T02:31:38.079 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.079 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.079 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:31:38.079 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.471689+0000", 2023-08-28T02:31:38.079 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00015092799999999999 2023-08-28T02:31:38.079 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.079 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.079 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 2,5,6", 2023-08-28T02:31:38.079 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.471864+0000", 2023-08-28T02:31:38.079 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00017473599999999999 2023-08-28T02:31:38.080 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.080 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.080 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:38.080 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.481479+0000", 2023-08-28T02:31:38.080 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0096152310000000001 2023-08-28T02:31:38.080 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.080 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.080 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:38.080 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.481550+0000", 2023-08-28T02:31:38.080 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.0742999999999997e-05 2023-08-28T02:31:38.080 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.080 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.080 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:31:38.081 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.481731+0000", 2023-08-28T02:31:38.081 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000180562 2023-08-28T02:31:38.081 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.081 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.081 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:38.081 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.487363+0000", 2023-08-28T02:31:38.081 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0056318879999999998 2023-08-28T02:31:38.081 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.081 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.081 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:31:38.081 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.487424+0000", 2023-08-28T02:31:38.081 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.1050000000000007e-05 2023-08-28T02:31:38.082 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.082 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.082 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:31:38.082 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.487546+0000", 2023-08-28T02:31:38.082 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00012208100000000001 2023-08-28T02:31:38.082 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:38.082 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:38.082 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:38.082 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.082 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.082 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:78184 6.a 6:5da46a44:::benchmark_data_smithi143_305742_object78183:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e221)", 2023-08-28T02:31:38.082 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:28:25.332832+0000", 2023-08-28T02:31:38.083 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 192.63198735899999, 2023-08-28T02:31:38.083 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1550407680000001, 2023-08-28T02:31:38.083 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:31:38.083 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:38.083 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:31:38.083 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:31:38.083 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:31:38.083 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 78184 2023-08-28T02:31:38.083 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.083 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:31:38.083 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.083 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:31:38.083 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.332832+0000", 2023-08-28T02:31:38.084 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:38.084 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.084 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.084 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:31:38.084 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.332832+0000", 2023-08-28T02:31:38.084 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:38.084 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.084 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.084 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:31:38.084 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.332832+0000", 2023-08-28T02:31:38.084 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:31:38.084 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.085 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.085 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:31:38.085 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.332852+0000", 2023-08-28T02:31:38.085 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.0690999999999999e-05 2023-08-28T02:31:38.085 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.085 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.085 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:31:38.085 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.332859+0000", 2023-08-28T02:31:38.085 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.5520000000000001e-06 2023-08-28T02:31:38.085 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.085 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.085 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:31:38.086 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.332864+0000", 2023-08-28T02:31:38.086 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.8899999999999998e-06 2023-08-28T02:31:38.086 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.086 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.086 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:38.086 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.334297+0000", 2023-08-28T02:31:38.086 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0014326619999999999 2023-08-28T02:31:38.086 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.086 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.086 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:31:38.086 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.334307+0000", 2023-08-28T02:31:38.087 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.083e-05 2023-08-28T02:31:38.087 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.087 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.087 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:38.087 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.474666+0000", 2023-08-28T02:31:38.087 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.140358242 2023-08-28T02:31:38.087 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.087 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.087 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:31:38.087 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.474849+0000", 2023-08-28T02:31:38.087 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00018383599999999999 2023-08-28T02:31:38.087 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.088 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.088 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 2,5,6", 2023-08-28T02:31:38.088 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.475057+0000", 2023-08-28T02:31:38.088 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00020736300000000001 2023-08-28T02:31:38.088 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.088 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.088 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:38.088 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.481516+0000", 2023-08-28T02:31:38.088 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0064590189999999999 2023-08-28T02:31:38.088 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.088 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.088 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:38.088 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.481586+0000", 2023-08-28T02:31:38.089 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.9698999999999998e-05 2023-08-28T02:31:38.089 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.089 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.089 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:31:38.089 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.481737+0000", 2023-08-28T02:31:38.089 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00015137500000000001 2023-08-28T02:31:38.089 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.089 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.089 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:38.089 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.487703+0000", 2023-08-28T02:31:38.089 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0059662990000000004 2023-08-28T02:31:38.089 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.090 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.090 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:31:38.090 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.487758+0000", 2023-08-28T02:31:38.090 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.4861999999999998e-05 2023-08-28T02:31:38.090 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.090 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.090 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:31:38.090 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.487873+0000", 2023-08-28T02:31:38.090 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000114827 2023-08-28T02:31:38.090 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:38.091 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:38.091 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:38.091 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.091 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.091 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:78190 6.a 6:5f0de539:::benchmark_data_smithi143_305742_object78189:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e221)", 2023-08-28T02:31:38.091 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:28:25.484849+0000", 2023-08-28T02:31:38.091 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 192.47997067099999, 2023-08-28T02:31:38.091 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.003235442, 2023-08-28T02:31:38.091 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:31:38.091 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:38.091 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:31:38.092 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:31:38.092 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:31:38.092 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 78190 2023-08-28T02:31:38.092 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.092 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:31:38.092 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.092 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:31:38.092 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.484849+0000", 2023-08-28T02:31:38.092 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:38.092 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.092 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.092 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:31:38.093 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.484849+0000", 2023-08-28T02:31:38.093 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:38.093 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.093 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.093 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:31:38.093 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.484847+0000", 2023-08-28T02:31:38.093 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:31:38.093 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.093 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.093 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:31:38.093 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.484876+0000", 2023-08-28T02:31:38.093 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.8646e-05 2023-08-28T02:31:38.093 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.094 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.094 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:31:38.094 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.484897+0000", 2023-08-28T02:31:38.094 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.1569e-05 2023-08-28T02:31:38.094 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.094 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.094 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:31:38.094 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.484909+0000", 2023-08-28T02:31:38.094 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1476e-05 2023-08-28T02:31:38.094 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.094 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.094 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:38.095 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.484978+0000", 2023-08-28T02:31:38.095 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.8848000000000002e-05 2023-08-28T02:31:38.095 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.095 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.095 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:31:38.095 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.484992+0000", 2023-08-28T02:31:38.095 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.4099e-05 2023-08-28T02:31:38.095 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.095 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.095 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:38.095 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.477959+0000", 2023-08-28T02:31:38.095 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.99296681499999995 2023-08-28T02:31:38.096 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.096 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.096 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:31:38.096 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.478115+0000", 2023-08-28T02:31:38.096 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000156362 2023-08-28T02:31:38.096 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.096 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.096 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 2,5,6", 2023-08-28T02:31:38.096 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.478294+0000", 2023-08-28T02:31:38.096 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000178896 2023-08-28T02:31:38.096 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.096 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.097 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:38.097 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.482309+0000", 2023-08-28T02:31:38.097 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0040148800000000002 2023-08-28T02:31:38.097 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.097 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.097 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:31:38.097 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.482672+0000", 2023-08-28T02:31:38.097 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00036328400000000001 2023-08-28T02:31:38.097 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.097 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.097 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:38.097 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.486362+0000", 2023-08-28T02:31:38.098 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0036900869999999999 2023-08-28T02:31:38.098 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.098 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.098 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:38.098 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.487919+0000", 2023-08-28T02:31:38.098 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0015570429999999999 2023-08-28T02:31:38.098 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.098 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.098 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:31:38.098 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.487973+0000", 2023-08-28T02:31:38.098 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.3764999999999998e-05 2023-08-28T02:31:38.098 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.099 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.099 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:31:38.099 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.488084+0000", 2023-08-28T02:31:38.099 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00011128699999999999 2023-08-28T02:31:38.099 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:38.099 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:38.099 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:38.099 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.099 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.099 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:94788 6.a 6:532abc0c:::benchmark_data_smithi143_305742_object11169:head [delete] snapc 0=[] ondisk+write+known_if_redirected e228)", 2023-08-28T02:31:38.099 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:28:52.336833+0000", 2023-08-28T02:31:38.099 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 165.62798608200001, 2023-08-28T02:31:38.100 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.22330499300000001, 2023-08-28T02:31:38.100 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:31:38.100 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:38.100 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:31:38.100 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:31:38.100 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:31:38.100 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 94788 2023-08-28T02:31:38.100 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.100 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:31:38.100 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.100 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:31:38.101 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:52.336833+0000", 2023-08-28T02:31:38.101 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:38.101 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.101 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.101 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:31:38.101 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:52.336833+0000", 2023-08-28T02:31:38.101 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:38.101 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.101 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.101 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:31:38.101 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:52.336831+0000", 2023-08-28T02:31:38.101 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999976 2023-08-28T02:31:38.101 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.102 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.102 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:31:38.102 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:52.336841+0000", 2023-08-28T02:31:38.102 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.3889999999999994e-06 2023-08-28T02:31:38.102 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.102 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.102 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:31:38.102 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:52.336859+0000", 2023-08-28T02:31:38.102 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.844e-05 2023-08-28T02:31:38.102 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.102 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.102 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:31:38.103 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:52.336878+0000", 2023-08-28T02:31:38.103 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.8607999999999999e-05 2023-08-28T02:31:38.103 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.103 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.103 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:38.103 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:52.340253+0000", 2023-08-28T02:31:38.103 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0033756419999999999 2023-08-28T02:31:38.103 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.103 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.103 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:31:38.103 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:52.340394+0000", 2023-08-28T02:31:38.104 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00014031300000000001 2023-08-28T02:31:38.104 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.104 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.104 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 2,5,6", 2023-08-28T02:31:38.104 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:52.340535+0000", 2023-08-28T02:31:38.104 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00014093600000000001 2023-08-28T02:31:38.104 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.104 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.104 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:38.104 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:52.347288+0000", 2023-08-28T02:31:38.104 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0067535989999999999 2023-08-28T02:31:38.104 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.104 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.104 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:38.105 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:52.351932+0000", 2023-08-28T02:31:38.105 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0046440559999999997 2023-08-28T02:31:38.105 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.105 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.105 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:31:38.105 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:52.353636+0000", 2023-08-28T02:31:38.105 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.001704295 2023-08-28T02:31:38.105 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.105 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.105 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:38.105 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:52.560018+0000", 2023-08-28T02:31:38.105 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.206381654 2023-08-28T02:31:38.106 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.106 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.106 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:31:38.106 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:52.560063+0000", 2023-08-28T02:31:38.106 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.5068000000000003e-05 2023-08-28T02:31:38.106 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.106 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.106 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:31:38.106 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:52.560138+0000", 2023-08-28T02:31:38.106 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.5198999999999996e-05 2023-08-28T02:31:38.106 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:38.106 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:38.107 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:38.107 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.107 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.107 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.55270.0:36164 7.f 7:f9cab79b:::benchmark_data_smithi143_360738_object36163:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e266)", 2023-08-28T02:31:38.107 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:31:21.653753+0000", 2023-08-28T02:31:38.107 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 16.311066097000001, 2023-08-28T02:31:38.107 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.24250607499999999, 2023-08-28T02:31:38.107 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:31:38.107 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:38.107 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:31:38.107 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.55270", 2023-08-28T02:31:38.107 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:31:38.107 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 36164 2023-08-28T02:31:38.108 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.108 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:31:38.108 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.108 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:31:38.108 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.653753+0000", 2023-08-28T02:31:38.108 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:38.108 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.108 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.108 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:31:38.108 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.653753+0000", 2023-08-28T02:31:38.108 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:38.108 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.108 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.109 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:31:38.109 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.653753+0000", 2023-08-28T02:31:38.109 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:31:38.109 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.109 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.109 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:31:38.109 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.653777+0000", 2023-08-28T02:31:38.109 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.4426e-05 2023-08-28T02:31:38.109 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.109 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.109 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:31:38.109 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.653785+0000", 2023-08-28T02:31:38.109 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.1709999999999996e-06 2023-08-28T02:31:38.110 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.110 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.110 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:31:38.110 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.653790+0000", 2023-08-28T02:31:38.110 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.4090000000000002e-06 2023-08-28T02:31:38.110 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.110 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.110 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:38.110 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.653890+0000", 2023-08-28T02:31:38.110 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.9896999999999996e-05 2023-08-28T02:31:38.110 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.110 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.110 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:31:38.111 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.653987+0000", 2023-08-28T02:31:38.111 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.7416000000000004e-05 2023-08-28T02:31:38.111 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.111 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.111 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 3,7", 2023-08-28T02:31:38.111 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.654131+0000", 2023-08-28T02:31:38.111 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000144117 2023-08-28T02:31:38.111 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.111 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.111 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:38.111 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.656091+0000", 2023-08-28T02:31:38.112 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0019596430000000001 2023-08-28T02:31:38.112 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.112 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.112 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:31:38.112 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.656437+0000", 2023-08-28T02:31:38.112 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00034572999999999998 2023-08-28T02:31:38.112 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.112 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.112 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:38.112 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.896163+0000", 2023-08-28T02:31:38.112 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.239725827 2023-08-28T02:31:38.112 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.112 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.113 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:31:38.113 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.896217+0000", 2023-08-28T02:31:38.113 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.4666000000000001e-05 2023-08-28T02:31:38.113 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.113 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.113 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:31:38.113 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.896259+0000", 2023-08-28T02:31:38.113 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.2239e-05 2023-08-28T02:31:38.113 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:38.113 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:38.113 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:38.113 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.113 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.114 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.55270.0:36171 7.6 7:6f8b801e:::benchmark_data_smithi143_360738_object36170:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e266)", 2023-08-28T02:31:38.114 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:31:21.657144+0000", 2023-08-28T02:31:38.114 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 16.307675037999999, 2023-08-28T02:31:38.114 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.24323956099999999, 2023-08-28T02:31:38.114 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:31:38.114 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:38.114 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:31:38.114 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.55270", 2023-08-28T02:31:38.114 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:31:38.114 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 36171 2023-08-28T02:31:38.114 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.114 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:31:38.115 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.115 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:31:38.115 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.657144+0000", 2023-08-28T02:31:38.115 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:38.115 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.115 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.115 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:31:38.115 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.657144+0000", 2023-08-28T02:31:38.115 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:38.115 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.115 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.115 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:31:38.116 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.657144+0000", 2023-08-28T02:31:38.116 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:31:38.116 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.116 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.116 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:31:38.116 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.657163+0000", 2023-08-28T02:31:38.116 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.8940000000000002e-05 2023-08-28T02:31:38.116 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.116 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.116 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:31:38.116 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.657169+0000", 2023-08-28T02:31:38.117 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.4080000000000003e-06 2023-08-28T02:31:38.117 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.117 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.117 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:31:38.117 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.657174+0000", 2023-08-28T02:31:38.117 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.9649999999999999e-06 2023-08-28T02:31:38.117 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.117 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.117 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:38.117 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.657212+0000", 2023-08-28T02:31:38.117 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.8039e-05 2023-08-28T02:31:38.118 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.118 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.118 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:31:38.118 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.657334+0000", 2023-08-28T02:31:38.118 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000121686 2023-08-28T02:31:38.118 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.118 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.118 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 3,7", 2023-08-28T02:31:38.118 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.657466+0000", 2023-08-28T02:31:38.118 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00013238200000000001 2023-08-28T02:31:38.118 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.118 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.119 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:31:38.119 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.659233+0000", 2023-08-28T02:31:38.119 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0017669389999999999 2023-08-28T02:31:38.119 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.119 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.119 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:38.119 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.659746+0000", 2023-08-28T02:31:38.119 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00051231699999999998 2023-08-28T02:31:38.119 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.119 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.119 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:38.119 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.900296+0000", 2023-08-28T02:31:38.120 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.240550394 2023-08-28T02:31:38.120 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.120 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.120 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:31:38.120 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.900346+0000", 2023-08-28T02:31:38.120 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.9872e-05 2023-08-28T02:31:38.120 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.120 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.120 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:31:38.120 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:21.900384+0000", 2023-08-28T02:31:38.120 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.8059999999999998e-05 2023-08-28T02:31:38.120 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:38.120 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:38.121 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:38.121 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.121 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.121 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.55270.0:39323 7.9 7:9bb52038:::benchmark_data_smithi143_360738_object39322:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e266)", 2023-08-28T02:31:38.121 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:31:24.233988+0000", 2023-08-28T02:31:38.121 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 13.730831617, 2023-08-28T02:31:38.121 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.22236424599999999, 2023-08-28T02:31:38.121 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:31:38.121 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:38.121 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:31:38.121 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.55270", 2023-08-28T02:31:38.121 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:31:38.121 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 39323 2023-08-28T02:31:38.122 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.122 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:31:38.122 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.122 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:31:38.122 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:24.233988+0000", 2023-08-28T02:31:38.122 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:38.122 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.122 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.122 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:31:38.122 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:24.233988+0000", 2023-08-28T02:31:38.122 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:38.122 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.123 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.123 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:31:38.123 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:24.233987+0000", 2023-08-28T02:31:38.123 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:31:38.123 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.123 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.123 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:31:38.123 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:24.234017+0000", 2023-08-28T02:31:38.123 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.9835000000000001e-05 2023-08-28T02:31:38.123 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.123 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.123 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:31:38.124 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:24.234027+0000", 2023-08-28T02:31:38.124 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0247e-05 2023-08-28T02:31:38.124 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.124 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.124 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:31:38.124 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:24.234034+0000", 2023-08-28T02:31:38.124 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.7009999999999999e-06 2023-08-28T02:31:38.124 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.124 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.124 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:38.124 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:24.234143+0000", 2023-08-28T02:31:38.124 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000109262 2023-08-28T02:31:38.124 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.125 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.125 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:31:38.125 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:24.234281+0000", 2023-08-28T02:31:38.125 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00013783900000000001 2023-08-28T02:31:38.125 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.125 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.125 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,4", 2023-08-28T02:31:38.125 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:24.447773+0000", 2023-08-28T02:31:38.125 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.21349274800000001 2023-08-28T02:31:38.125 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.125 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.125 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:31:38.126 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:24.451304+0000", 2023-08-28T02:31:38.126 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0035310060000000002 2023-08-28T02:31:38.126 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.126 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.126 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:38.126 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:24.451878+0000", 2023-08-28T02:31:38.126 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00057320400000000001 2023-08-28T02:31:38.126 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.126 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.126 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:38.126 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:24.456264+0000", 2023-08-28T02:31:38.126 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0043868249999999996 2023-08-28T02:31:38.127 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.127 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.127 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:31:38.127 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:24.456312+0000", 2023-08-28T02:31:38.127 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.7552999999999997e-05 2023-08-28T02:31:38.127 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:38.127 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:38.127 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:31:38.127 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:24.456352+0000", 2023-08-28T02:31:38.127 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.0166999999999997e-05 2023-08-28T02:31:38.127 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:38.127 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:38.127 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:38.128 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:38.128 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:38.128 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:31:38.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:31:38.229 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:31:38.310 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:38.386 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:31:38.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:38.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:31:38.489 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:31:38.552 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:38 smithi143 ceph-mon[183059]: pgmap v663: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-08-28T02:31:38.552 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:38 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1745952203' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:31:38.552 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:38 smithi143 ceph-mon[189871]: pgmap v663: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-08-28T02:31:38.552 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:38 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1745952203' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:31:38.558 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:38.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:38 smithi169 ceph-mon[159415]: pgmap v663: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-08-28T02:31:38.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:38 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1745952203' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:31:38.837 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 37 16 55786 55770 94.1287 63.8125 0.00634102 0.0106158 2023-08-28T02:31:39.008 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set nodeep-scrub 2023-08-28T02:31:39.616 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:31:39.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:39.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:31:39.662 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.1 dump_ops_in_flight 2023-08-28T02:31:39.725 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:31:39.756 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:39.806 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:31:39.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:39.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:31:39.840 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 38 16 56612 56596 93.0109 51.625 0.00634176 0.0107447 2023-08-28T02:31:39.918 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:31:39.994 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:40.316 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:31:40.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:40.322 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:31:40.422 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:31:40.523 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:40.624 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:31:40.645 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:31:40.652 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:40 smithi143 ceph-mon[189871]: pgmap v664: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-08-28T02:31:40.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:40.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:31:40.653 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:40 smithi143 ceph-mon[183059]: pgmap v664: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-08-28T02:31:40.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:40 smithi169 ceph-mon[159415]: pgmap v664: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-08-28T02:31:40.754 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:31:40.759 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:31:40.788 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:40.837 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 39 16 57691 57675 92.3554 67.4375 0.0244328 0.0108238 2023-08-28T02:31:40.851 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:31:40.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:40.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:31:40.960 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:31:41.002 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:41.065 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:31:41.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:41.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:31:41.170 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:31:41.260 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:41.369 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:31:41.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:41.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:31:41.480 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:31:41.513 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:41.572 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:31:41.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:41.579 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:31:41.680 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:31:41.751 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:41.838 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:2023-08-28T02:31:41.839198+0000 min lat: 0.00114956 max lat: 0.255681 avg lat: 0.0109028 2023-08-28T02:31:41.838 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-08-28T02:31:41.838 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 40 16 58731 58715 91.6719 65 0.0146604 0.0109028 2023-08-28T02:31:41.906 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:31:41.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:41.912 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:31:42.012 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:31:42.069 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:31:41 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:31:41] "GET /metrics HTTP/1.1" 200 36462 "" "Prometheus/2.33.4" 2023-08-28T02:31:42.070 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:42 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:31:42.070 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:42 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:31:42.070 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:42 smithi143 ceph-mon[189871]: pgmap v665: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-08-28T02:31:42.070 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:42 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1793918277' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]: dispatch 2023-08-28T02:31:42.079 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:42.186 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:31:42.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:42.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:31:42.227 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:31:42.293 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:31:42.316 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:42 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:31:42.316 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:42 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:31:42.316 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:42 smithi169 ceph-mon[159415]: pgmap v665: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-08-28T02:31:42.317 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:42 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1793918277' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]: dispatch 2023-08-28T02:31:42.318 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:42.360 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:31:42.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:42.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:31:42.420 INFO:teuthology.orchestra.run.smithi143.stderr:set pool 7 pg_num to 26 2023-08-28T02:31:42.443 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:42 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:31:42.443 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:42 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:31:42.443 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:42 smithi143 ceph-mon[183059]: pgmap v665: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-08-28T02:31:42.443 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:42 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1793918277' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]: dispatch 2023-08-28T02:31:42.466 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:31:42.491 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:42.838 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 41 16 59397 59381 90.4519 41.625 0.00251133 0.011037 2023-08-28T02:31:43.105 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:31:43.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:43.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:31:43.218 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:31:43.253 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:43.303 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:31:43.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:43.309 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:31:43.410 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:31:43.469 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:43.565 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:31:43.569 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:43 smithi169 ceph-mon[159415]: failed to encode map e269 with expected crc 2023-08-28T02:31:43.569 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:43 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1793918277' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]': finished 2023-08-28T02:31:43.569 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:43 smithi169 ceph-mon[159415]: osdmap e269: 8 total, 8 up, 8 in 2023-08-28T02:31:43.569 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:43 smithi169 ceph-mon[159415]: failed to encode map e269 with expected crc 2023-08-28T02:31:43.569 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:43 smithi169 ceph-mon[159415]: failed to encode map e269 with expected crc 2023-08-28T02:31:43.569 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:43 smithi169 ceph-mon[159415]: failed to encode map e269 with expected crc 2023-08-28T02:31:43.570 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:43 smithi169 ceph-mon[159415]: failed to encode map e269 with expected crc 2023-08-28T02:31:43.570 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:43 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2023-08-28T02:31:43.570 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:43 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2023-08-28T02:31:43.570 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:43 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2023-08-28T02:31:43.570 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:43 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-08-28T02:31:43.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:43.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:31:43.672 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:31:43.698 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:43 smithi143 ceph-mon[189871]: failed to encode map e269 with expected crc 2023-08-28T02:31:43.698 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:43 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1793918277' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]': finished 2023-08-28T02:31:43.698 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:43 smithi143 ceph-mon[189871]: osdmap e269: 8 total, 8 up, 8 in 2023-08-28T02:31:43.699 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:43 smithi143 ceph-mon[189871]: failed to encode map e269 with expected crc 2023-08-28T02:31:43.699 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:43 smithi143 ceph-mon[189871]: failed to encode map e269 with expected crc 2023-08-28T02:31:43.699 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:43 smithi143 ceph-mon[189871]: failed to encode map e269 with expected crc 2023-08-28T02:31:43.699 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:43 smithi143 ceph-mon[189871]: failed to encode map e269 with expected crc 2023-08-28T02:31:43.699 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:43 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2023-08-28T02:31:43.699 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:43 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2023-08-28T02:31:43.699 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:43 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2023-08-28T02:31:43.699 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:43 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-08-28T02:31:43.699 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:43 smithi143 ceph-mon[183059]: failed to encode map e269 with expected crc 2023-08-28T02:31:43.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:43 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1793918277' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]': finished 2023-08-28T02:31:43.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:43 smithi143 ceph-mon[183059]: osdmap e269: 8 total, 8 up, 8 in 2023-08-28T02:31:43.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:43 smithi143 ceph-mon[183059]: failed to encode map e269 with expected crc 2023-08-28T02:31:43.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:43 smithi143 ceph-mon[183059]: failed to encode map e269 with expected crc 2023-08-28T02:31:43.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:43 smithi143 ceph-mon[183059]: failed to encode map e269 with expected crc 2023-08-28T02:31:43.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:43 smithi143 ceph-mon[183059]: failed to encode map e269 with expected crc 2023-08-28T02:31:43.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:43 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2023-08-28T02:31:43.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:43 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2023-08-28T02:31:43.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:43 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2023-08-28T02:31:43.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:43 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-08-28T02:31:43.702 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:43.839 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 42 16 59993 59977 89.1824 37.25 0.0920282 0.011103 2023-08-28T02:31:43.890 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:31:43.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:43.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:31:43.958 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:31:43.994 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:31:44.023 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:44.074 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:31:44.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:44.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:31:44.178 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:31:44.208 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:44.236 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:31:44.236 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [], 2023-08-28T02:31:44.236 INFO:teuthology.orchestra.run.smithi143.stdout: "num_ops": 0 2023-08-28T02:31:44.236 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:31:44.577 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:31:44.580 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:44 smithi143 ceph-mon[183059]: failed to encode map e269 with expected crc 2023-08-28T02:31:44.581 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:44 smithi143 ceph-mon[183059]: failed to encode map e269 with expected crc 2023-08-28T02:31:44.581 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:44 smithi143 ceph-mon[183059]: failed to encode map e269 with expected crc 2023-08-28T02:31:44.581 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:44 smithi143 ceph-mon[183059]: pgmap v667: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 52 MiB/s wr, 829 op/s 2023-08-28T02:31:44.581 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:44 smithi143 ceph-mon[183059]: failed to encode map e270 with expected crc 2023-08-28T02:31:44.581 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:44 smithi143 ceph-mon[183059]: failed to encode map e270 with expected crc 2023-08-28T02:31:44.581 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:44 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2023-08-28T02:31:44.581 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:44 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2023-08-28T02:31:44.581 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:44 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2023-08-28T02:31:44.581 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:44 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]': finished 2023-08-28T02:31:44.581 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:44 smithi143 ceph-mon[183059]: osdmap e270: 8 total, 8 up, 8 in 2023-08-28T02:31:44.582 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:44 smithi143 ceph-mon[189871]: failed to encode map e269 with expected crc 2023-08-28T02:31:44.582 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:44 smithi143 ceph-mon[189871]: failed to encode map e269 with expected crc 2023-08-28T02:31:44.582 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:44 smithi143 ceph-mon[189871]: failed to encode map e269 with expected crc 2023-08-28T02:31:44.582 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:44 smithi143 ceph-mon[189871]: pgmap v667: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 52 MiB/s wr, 829 op/s 2023-08-28T02:31:44.582 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:44 smithi143 ceph-mon[189871]: failed to encode map e270 with expected crc 2023-08-28T02:31:44.582 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:44 smithi143 ceph-mon[189871]: failed to encode map e270 with expected crc 2023-08-28T02:31:44.582 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:44 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2023-08-28T02:31:44.582 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:44 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2023-08-28T02:31:44.582 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:44 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2023-08-28T02:31:44.582 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:44 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]': finished 2023-08-28T02:31:44.582 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:44 smithi143 ceph-mon[189871]: osdmap e270: 8 total, 8 up, 8 in 2023-08-28T02:31:44.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:44.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:31:44.684 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:31:44.711 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:44.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:44 smithi169 ceph-mon[159415]: failed to encode map e269 with expected crc 2023-08-28T02:31:44.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:44 smithi169 ceph-mon[159415]: failed to encode map e269 with expected crc 2023-08-28T02:31:44.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:44 smithi169 ceph-mon[159415]: failed to encode map e269 with expected crc 2023-08-28T02:31:44.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:44 smithi169 ceph-mon[159415]: pgmap v667: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 52 MiB/s wr, 829 op/s 2023-08-28T02:31:44.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:44 smithi169 ceph-mon[159415]: failed to encode map e270 with expected crc 2023-08-28T02:31:44.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:44 smithi169 ceph-mon[159415]: failed to encode map e270 with expected crc 2023-08-28T02:31:44.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:44 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2023-08-28T02:31:44.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:44 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2023-08-28T02:31:44.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:44 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2023-08-28T02:31:44.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:44 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]': finished 2023-08-28T02:31:44.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:44 smithi169 ceph-mon[159415]: osdmap e270: 8 total, 8 up, 8 in 2023-08-28T02:31:44.839 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 43 16 60365 60349 87.6501 23.25 0.00305599 0.011403 2023-08-28T02:31:45.454 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:31:45.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:45.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:31:45.462 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.1 dump_blocked_ops 2023-08-28T02:31:45.463 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is set 2023-08-28T02:31:45.563 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:31:45.589 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:45.630 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:31:45.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:45.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:31:45.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:45 smithi143 ceph-mon[183059]: osdmap e271: 8 total, 8 up, 8 in 2023-08-28T02:31:45.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:45 smithi143 ceph-mon[183059]: failed to encode map e271 with expected crc 2023-08-28T02:31:45.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:45 smithi143 ceph-mon[183059]: failed to encode map e271 with expected crc 2023-08-28T02:31:45.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:45 smithi143 ceph-mon[183059]: failed to encode map e271 with expected crc 2023-08-28T02:31:45.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:45 smithi143 ceph-mon[183059]: failed to encode map e271 with expected crc 2023-08-28T02:31:45.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:45 smithi143 ceph-mon[183059]: failed to encode map e271 with expected crc 2023-08-28T02:31:45.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:45 smithi143 ceph-mon[183059]: failed to encode map e271 with expected crc 2023-08-28T02:31:45.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:45 smithi143 ceph-mon[183059]: failed to encode map e271 with expected crc 2023-08-28T02:31:45.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:45 smithi143 ceph-mon[183059]: failed to encode map e271 with expected crc 2023-08-28T02:31:45.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:45 smithi143 ceph-mon[183059]: failed to encode map e271 with expected crc 2023-08-28T02:31:45.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:45 smithi143 ceph-mon[183059]: failed to encode map e271 with expected crc 2023-08-28T02:31:45.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:45 smithi143 ceph-mon[183059]: failed to encode map e271 with expected crc 2023-08-28T02:31:45.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:45 smithi143 ceph-mon[183059]: failed to encode map e271 with expected crc 2023-08-28T02:31:45.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:45 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3930677666' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:31:45.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:45 smithi143 ceph-mon[189871]: osdmap e271: 8 total, 8 up, 8 in 2023-08-28T02:31:45.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:45 smithi143 ceph-mon[189871]: failed to encode map e271 with expected crc 2023-08-28T02:31:45.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:45 smithi143 ceph-mon[189871]: failed to encode map e271 with expected crc 2023-08-28T02:31:45.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:45 smithi143 ceph-mon[189871]: failed to encode map e271 with expected crc 2023-08-28T02:31:45.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:45 smithi143 ceph-mon[189871]: failed to encode map e271 with expected crc 2023-08-28T02:31:45.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:45 smithi143 ceph-mon[189871]: failed to encode map e271 with expected crc 2023-08-28T02:31:45.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:45 smithi143 ceph-mon[189871]: failed to encode map e271 with expected crc 2023-08-28T02:31:45.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:45 smithi143 ceph-mon[189871]: failed to encode map e271 with expected crc 2023-08-28T02:31:45.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:45 smithi143 ceph-mon[189871]: failed to encode map e271 with expected crc 2023-08-28T02:31:45.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:45 smithi143 ceph-mon[189871]: failed to encode map e271 with expected crc 2023-08-28T02:31:45.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:45 smithi143 ceph-mon[189871]: failed to encode map e271 with expected crc 2023-08-28T02:31:45.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:45 smithi143 ceph-mon[189871]: failed to encode map e271 with expected crc 2023-08-28T02:31:45.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:45 smithi143 ceph-mon[189871]: failed to encode map e271 with expected crc 2023-08-28T02:31:45.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:45 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3930677666' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:31:45.736 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:31:45.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:45 smithi169 ceph-mon[159415]: osdmap e271: 8 total, 8 up, 8 in 2023-08-28T02:31:45.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:45 smithi169 ceph-mon[159415]: failed to encode map e271 with expected crc 2023-08-28T02:31:45.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:45 smithi169 ceph-mon[159415]: failed to encode map e271 with expected crc 2023-08-28T02:31:45.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:45 smithi169 ceph-mon[159415]: failed to encode map e271 with expected crc 2023-08-28T02:31:45.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:45 smithi169 ceph-mon[159415]: failed to encode map e271 with expected crc 2023-08-28T02:31:45.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:45 smithi169 ceph-mon[159415]: failed to encode map e271 with expected crc 2023-08-28T02:31:45.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:45 smithi169 ceph-mon[159415]: failed to encode map e271 with expected crc 2023-08-28T02:31:45.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:45 smithi169 ceph-mon[159415]: failed to encode map e271 with expected crc 2023-08-28T02:31:45.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:45 smithi169 ceph-mon[159415]: failed to encode map e271 with expected crc 2023-08-28T02:31:45.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:45 smithi169 ceph-mon[159415]: failed to encode map e271 with expected crc 2023-08-28T02:31:45.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:45 smithi169 ceph-mon[159415]: failed to encode map e271 with expected crc 2023-08-28T02:31:45.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:45 smithi169 ceph-mon[159415]: failed to encode map e271 with expected crc 2023-08-28T02:31:45.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:45 smithi169 ceph-mon[159415]: failed to encode map e271 with expected crc 2023-08-28T02:31:45.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:45 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3930677666' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:31:45.779 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:45.839 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 44 16 61502 61486 87.2732 71.0625 0.0115066 0.0114536 2023-08-28T02:31:45.843 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:31:45.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:45.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:31:45.953 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:31:45.995 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:46.069 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:31:46.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:46.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:31:46.174 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:31:46.276 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:46.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:46 smithi143 ceph-mon[189871]: pgmap v670: 99 pgs: 7 peering, 10 unknown, 82 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 58 MiB/s wr, 923 op/s 2023-08-28T02:31:46.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:46 smithi143 ceph-mon[189871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:31:46.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:46 smithi143 ceph-mon[189871]: failed to encode map e272 with expected crc 2023-08-28T02:31:46.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:46 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3930677666' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:31:46.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:46 smithi143 ceph-mon[189871]: osdmap e272: 8 total, 8 up, 8 in 2023-08-28T02:31:46.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:46 smithi143 ceph-mon[189871]: failed to encode map e272 with expected crc 2023-08-28T02:31:46.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:46 smithi143 ceph-mon[189871]: failed to encode map e272 with expected crc 2023-08-28T02:31:46.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:46 smithi143 ceph-mon[189871]: failed to encode map e272 with expected crc 2023-08-28T02:31:46.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:46 smithi143 ceph-mon[189871]: failed to encode map e272 with expected crc 2023-08-28T02:31:46.706 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:46 smithi143 ceph-mon[183059]: pgmap v670: 99 pgs: 7 peering, 10 unknown, 82 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 58 MiB/s wr, 923 op/s 2023-08-28T02:31:46.706 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:46 smithi143 ceph-mon[183059]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:31:46.706 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:46 smithi143 ceph-mon[183059]: failed to encode map e272 with expected crc 2023-08-28T02:31:46.706 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:46 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3930677666' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:31:46.707 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:46 smithi143 ceph-mon[183059]: osdmap e272: 8 total, 8 up, 8 in 2023-08-28T02:31:46.707 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:46 smithi143 ceph-mon[183059]: failed to encode map e272 with expected crc 2023-08-28T02:31:46.707 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:46 smithi143 ceph-mon[183059]: failed to encode map e272 with expected crc 2023-08-28T02:31:46.707 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:46 smithi143 ceph-mon[183059]: failed to encode map e272 with expected crc 2023-08-28T02:31:46.707 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:46 smithi143 ceph-mon[183059]: failed to encode map e272 with expected crc 2023-08-28T02:31:46.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:46 smithi169 ceph-mon[159415]: pgmap v670: 99 pgs: 7 peering, 10 unknown, 82 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 58 MiB/s wr, 923 op/s 2023-08-28T02:31:46.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:46 smithi169 ceph-mon[159415]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:31:46.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:46 smithi169 ceph-mon[159415]: failed to encode map e272 with expected crc 2023-08-28T02:31:46.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:46 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3930677666' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:31:46.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:46 smithi169 ceph-mon[159415]: osdmap e272: 8 total, 8 up, 8 in 2023-08-28T02:31:46.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:46 smithi169 ceph-mon[159415]: failed to encode map e272 with expected crc 2023-08-28T02:31:46.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:46 smithi169 ceph-mon[159415]: failed to encode map e272 with expected crc 2023-08-28T02:31:46.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:46 smithi169 ceph-mon[159415]: failed to encode map e272 with expected crc 2023-08-28T02:31:46.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:46 smithi169 ceph-mon[159415]: failed to encode map e272 with expected crc 2023-08-28T02:31:46.806 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:31:46.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:46.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:31:46.839 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 45 16 62833 62817 87.1824 83.1875 0.00874358 0.0114654 2023-08-28T02:31:46.915 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:31:46.997 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:47.090 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:31:47.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:47.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:31:47.198 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:31:47.228 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:31:47.288 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:47.417 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:31:47.455 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:31:47.507 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:31:47.509 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:31:47.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:47.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:31:47.546 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:31:47.580 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:31:47.612 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:31:47.622 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:31:47.652 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:31:47.681 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:47.685 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:31:47.702 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:31:47.702 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:31:47.702 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:31:47.703 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:31:47.703 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:31:47.703 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:31:47.703 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:31:47.703 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:31:47.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:47 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2507108062' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:31:47.737 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:47 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2507108062' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:31:47.738 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:47 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2507108062' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:31:47.840 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 46 16 64139 64123 87.0615 81.625 0.00970952 0.0114818 2023-08-28T02:31:48.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:48 smithi169 ceph-mon[159415]: pgmap v672: 99 pgs: 7 peering, 6 unknown, 86 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 50 MiB/s wr, 807 op/s 2023-08-28T02:31:48.840 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 47 16 65377 65361 86.8554 77.375 0.0885199 0.0114964 2023-08-28T02:31:48.882 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 7, 3, 1, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-08-28T02:31:48.883 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-08-28T02:31:48.883 INFO:tasks.thrashosds.thrasher:Removing osd 7, in_osds are: [0, 2, 4, 5, 7, 3, 1, 6] 2023-08-28T02:31:48.883 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd out 7 2023-08-28T02:31:48.928 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset noscrub 2023-08-28T02:31:48.947 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:48 smithi143 ceph-mon[183059]: pgmap v672: 99 pgs: 7 peering, 6 unknown, 86 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 50 MiB/s wr, 807 op/s 2023-08-28T02:31:48.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:48 smithi143 ceph-mon[189871]: pgmap v672: 99 pgs: 7 peering, 6 unknown, 86 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 50 MiB/s wr, 807 op/s 2023-08-28T02:31:49.605 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:31:49.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:49.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:31:49.711 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:31:49.792 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:49.808 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:31:49.840 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 48 16 66474 66458 86.4744 68.5625 0.0156257 0.0115595 2023-08-28T02:31:49.890 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:31:49.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:49.896 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:31:49.997 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:31:50.037 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:50.100 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:31:50.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:50.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:31:50.143 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:31:50.207 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:31:50.284 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:50.311 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:31:50.311 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:31:50.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:50 smithi143 ceph-mon[183059]: pgmap v673: 99 pgs: 2 peering, 2 unknown, 95 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-08-28T02:31:50.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:50 smithi143 ceph-mon[189871]: pgmap v673: 99 pgs: 2 peering, 2 unknown, 95 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-08-28T02:31:50.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:50 smithi169 ceph-mon[159415]: pgmap v673: 99 pgs: 2 peering, 2 unknown, 95 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-08-28T02:31:50.840 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 49 16 67871 67855 86.4913 87.3125 0.00905746 0.0115574 2023-08-28T02:31:51.619 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:31:51.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:51 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-08-28T02:31:51.750 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:51 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-08-28T02:31:51.750 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:51 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-08-28T02:31:51.840 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 50 16 68796 68780 85.9179 57.8125 0.0016882 0.0116001 2023-08-28T02:31:52.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:31:51 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:31:51] "GET /metrics HTTP/1.1" 200 36462 "" "Prometheus/2.33.4" 2023-08-28T02:31:52.206 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:31:52.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:52.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:31:52.306 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.1 dump_historic_ops 2023-08-28T02:31:52.310 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:31:52.378 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:52.645 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:31:52.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:52.648 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:31:52.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:52 smithi169 ceph-mon[159415]: pgmap v674: 99 pgs: 99 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s 2023-08-28T02:31:52.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:52 smithi169 ceph-mon[159415]: failed to encode map e273 with expected crc 2023-08-28T02:31:52.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:52 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "17"}]': finished 2023-08-28T02:31:52.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:52 smithi169 ceph-mon[159415]: osdmap e273: 8 total, 8 up, 8 in 2023-08-28T02:31:52.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:52 smithi169 ceph-mon[159415]: failed to encode map e273 with expected crc 2023-08-28T02:31:52.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:52 smithi169 ceph-mon[159415]: failed to encode map e273 with expected crc 2023-08-28T02:31:52.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:52 smithi169 ceph-mon[159415]: failed to encode map e273 with expected crc 2023-08-28T02:31:52.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:52 smithi169 ceph-mon[159415]: failed to encode map e273 with expected crc 2023-08-28T02:31:52.748 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:31:52.773 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:52 smithi143 ceph-mon[189871]: pgmap v674: 99 pgs: 99 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s 2023-08-28T02:31:52.773 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:52 smithi143 ceph-mon[189871]: failed to encode map e273 with expected crc 2023-08-28T02:31:52.773 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:52 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "17"}]': finished 2023-08-28T02:31:52.773 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:52 smithi143 ceph-mon[189871]: osdmap e273: 8 total, 8 up, 8 in 2023-08-28T02:31:52.773 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:52 smithi143 ceph-mon[189871]: failed to encode map e273 with expected crc 2023-08-28T02:31:52.773 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:52 smithi143 ceph-mon[189871]: failed to encode map e273 with expected crc 2023-08-28T02:31:52.774 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:52 smithi143 ceph-mon[189871]: failed to encode map e273 with expected crc 2023-08-28T02:31:52.774 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:52 smithi143 ceph-mon[189871]: failed to encode map e273 with expected crc 2023-08-28T02:31:52.774 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:52 smithi143 ceph-mon[183059]: pgmap v674: 99 pgs: 99 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s 2023-08-28T02:31:52.774 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:52 smithi143 ceph-mon[183059]: failed to encode map e273 with expected crc 2023-08-28T02:31:52.774 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:52 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "17"}]': finished 2023-08-28T02:31:52.774 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:52 smithi143 ceph-mon[183059]: osdmap e273: 8 total, 8 up, 8 in 2023-08-28T02:31:52.774 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:52 smithi143 ceph-mon[183059]: failed to encode map e273 with expected crc 2023-08-28T02:31:52.774 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:52 smithi143 ceph-mon[183059]: failed to encode map e273 with expected crc 2023-08-28T02:31:52.774 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:52 smithi143 ceph-mon[183059]: failed to encode map e273 with expected crc 2023-08-28T02:31:52.774 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:52 smithi143 ceph-mon[183059]: failed to encode map e273 with expected crc 2023-08-28T02:31:52.776 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:52.840 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 51 16 68796 68780 84.2342 0 - 0.0116001 2023-08-28T02:31:52.992 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:31:52.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:52.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:31:53.096 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:31:53.126 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:53.268 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:31:53.276 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:31:53.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:53.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:31:53.381 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:31:53.411 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:53.661 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:31:53.665 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:31:53.668 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:53 smithi143 ceph-mon[183059]: osdmap e274: 8 total, 8 up, 8 in 2023-08-28T02:31:53.668 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:53 smithi143 ceph-mon[183059]: failed to encode map e274 with expected crc 2023-08-28T02:31:53.668 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:53 smithi143 ceph-mon[183059]: failed to encode map e274 with expected crc 2023-08-28T02:31:53.668 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:53 smithi143 ceph-mon[183059]: failed to encode map e274 with expected crc 2023-08-28T02:31:53.668 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:53 smithi143 ceph-mon[183059]: failed to encode map e274 with expected crc 2023-08-28T02:31:53.668 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:53 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-08-28T02:31:53.669 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:53 smithi143 ceph-mon[189871]: osdmap e274: 8 total, 8 up, 8 in 2023-08-28T02:31:53.669 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:53 smithi143 ceph-mon[189871]: failed to encode map e274 with expected crc 2023-08-28T02:31:53.669 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:53 smithi143 ceph-mon[189871]: failed to encode map e274 with expected crc 2023-08-28T02:31:53.669 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:53 smithi143 ceph-mon[189871]: failed to encode map e274 with expected crc 2023-08-28T02:31:53.669 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:53 smithi143 ceph-mon[189871]: failed to encode map e274 with expected crc 2023-08-28T02:31:53.669 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:53 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-08-28T02:31:53.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:53.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:31:53.770 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:31:53.799 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:53 smithi169 ceph-mon[159415]: osdmap e274: 8 total, 8 up, 8 in 2023-08-28T02:31:53.799 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:53 smithi169 ceph-mon[159415]: failed to encode map e274 with expected crc 2023-08-28T02:31:53.800 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:53 smithi169 ceph-mon[159415]: failed to encode map e274 with expected crc 2023-08-28T02:31:53.800 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:53 smithi169 ceph-mon[159415]: failed to encode map e274 with expected crc 2023-08-28T02:31:53.800 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:53 smithi169 ceph-mon[159415]: failed to encode map e274 with expected crc 2023-08-28T02:31:53.800 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:53 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-08-28T02:31:53.802 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:53.840 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 52 16 68796 68780 82.6152 0 - 0.0116001 2023-08-28T02:31:53.850 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:31:53.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:53.856 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:31:53.957 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:31:53.995 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:54.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:53 smithi143 conmon[183035]: 2023-08-28T02:31:53.969+0000 7fadc3887700 -1 mon.a@0(leader).osd e275 definitely_dead 0 2023-08-28T02:31:54.519 INFO:teuthology.orchestra.run.smithi143.stderr:marked out osd.7. 2023-08-28T02:31:54.519 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is unset 2023-08-28T02:31:54.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:54 smithi143 ceph-mon[183059]: pgmap v677: 99 pgs: 99 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 57 MiB/s wr, 908 op/s 2023-08-28T02:31:54.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:54 smithi143 ceph-mon[183059]: failed to encode map e275 with expected crc 2023-08-28T02:31:54.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:54 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "18"}]': finished 2023-08-28T02:31:54.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:54 smithi143 ceph-mon[183059]: osdmap e275: 8 total, 8 up, 8 in 2023-08-28T02:31:54.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:54 smithi143 ceph-mon[183059]: failed to encode map e275 with expected crc 2023-08-28T02:31:54.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:54 smithi143 ceph-mon[183059]: failed to encode map e275 with expected crc 2023-08-28T02:31:54.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:54 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2045078637' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:31:54.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:54 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:31:54.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:54 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3878387298' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-08-28T02:31:54.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:54 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-08-28T02:31:54.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:54 smithi143 ceph-mon[183059]: Client client.admin marked osd.7 out, while it was still marked up 2023-08-28T02:31:54.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:54 smithi143 ceph-mon[189871]: pgmap v677: 99 pgs: 99 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 57 MiB/s wr, 908 op/s 2023-08-28T02:31:54.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:54 smithi143 ceph-mon[189871]: failed to encode map e275 with expected crc 2023-08-28T02:31:54.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:54 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "18"}]': finished 2023-08-28T02:31:54.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:54 smithi143 ceph-mon[189871]: osdmap e275: 8 total, 8 up, 8 in 2023-08-28T02:31:54.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:54 smithi143 ceph-mon[189871]: failed to encode map e275 with expected crc 2023-08-28T02:31:54.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:54 smithi143 ceph-mon[189871]: failed to encode map e275 with expected crc 2023-08-28T02:31:54.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:54 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2045078637' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:31:54.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:54 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:31:54.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:54 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3878387298' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-08-28T02:31:54.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:54 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-08-28T02:31:54.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:54 smithi143 ceph-mon[189871]: Client client.admin marked osd.7 out, while it was still marked up 2023-08-28T02:31:54.807 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:31:54.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:54.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:31:54.840 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 53 16 69208 69192 81.5428 8.58333 0.00224352 0.0121148 2023-08-28T02:31:54.913 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:31:54.943 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:54.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:54 smithi169 ceph-mon[159415]: pgmap v677: 99 pgs: 99 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 57 MiB/s wr, 908 op/s 2023-08-28T02:31:54.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:54 smithi169 ceph-mon[159415]: failed to encode map e275 with expected crc 2023-08-28T02:31:54.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:54 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num_actual", "val": "18"}]': finished 2023-08-28T02:31:54.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:54 smithi169 ceph-mon[159415]: osdmap e275: 8 total, 8 up, 8 in 2023-08-28T02:31:54.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:54 smithi169 ceph-mon[159415]: failed to encode map e275 with expected crc 2023-08-28T02:31:54.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:54 smithi169 ceph-mon[159415]: failed to encode map e275 with expected crc 2023-08-28T02:31:54.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:54 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2045078637' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:31:54.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:54 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:31:54.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:54 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3878387298' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-08-28T02:31:54.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:54 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-08-28T02:31:54.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:54 smithi169 ceph-mon[159415]: Client client.admin marked osd.7 out, while it was still marked up 2023-08-28T02:31:55.840 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 54 16 69208 69192 80.0336 0 - 0.0121148 2023-08-28T02:31:55.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:55 smithi143 ceph-mon[189871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:31:55.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:55 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:31:55.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:55 smithi143 ceph-mon[189871]: failed to encode map e276 with expected crc 2023-08-28T02:31:55.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:55 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-08-28T02:31:55.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:55 smithi143 ceph-mon[189871]: osdmap e276: 8 total, 8 up, 7 in 2023-08-28T02:31:55.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:55 smithi143 ceph-mon[189871]: failed to encode map e276 with expected crc 2023-08-28T02:31:55.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:55 smithi143 ceph-mon[189871]: failed to encode map e276 with expected crc 2023-08-28T02:31:55.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:55 smithi143 ceph-mon[189871]: failed to encode map e276 with expected crc 2023-08-28T02:31:55.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:55 smithi143 ceph-mon[189871]: failed to encode map e276 with expected crc 2023-08-28T02:31:55.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:55 smithi143 ceph-mon[189871]: failed to encode map e276 with expected crc 2023-08-28T02:31:55.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:55 smithi143 ceph-mon[189871]: failed to encode map e276 with expected crc 2023-08-28T02:31:55.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:55 smithi143 ceph-mon[189871]: failed to encode map e276 with expected crc 2023-08-28T02:31:55.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:55 smithi143 ceph-mon[189871]: failed to encode map e276 with expected crc 2023-08-28T02:31:55.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:55 smithi143 ceph-mon[189871]: osdmap e277: 8 total, 8 up, 7 in 2023-08-28T02:31:55.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:55 smithi143 ceph-mon[183059]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:31:55.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:55 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:31:55.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:55 smithi143 ceph-mon[183059]: failed to encode map e276 with expected crc 2023-08-28T02:31:55.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:55 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-08-28T02:31:55.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:55 smithi143 ceph-mon[183059]: osdmap e276: 8 total, 8 up, 7 in 2023-08-28T02:31:55.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:55 smithi143 ceph-mon[183059]: failed to encode map e276 with expected crc 2023-08-28T02:31:55.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:55 smithi143 ceph-mon[183059]: failed to encode map e276 with expected crc 2023-08-28T02:31:55.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:55 smithi143 ceph-mon[183059]: failed to encode map e276 with expected crc 2023-08-28T02:31:55.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:55 smithi143 ceph-mon[183059]: failed to encode map e276 with expected crc 2023-08-28T02:31:55.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:55 smithi143 ceph-mon[183059]: failed to encode map e276 with expected crc 2023-08-28T02:31:55.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:55 smithi143 ceph-mon[183059]: failed to encode map e276 with expected crc 2023-08-28T02:31:55.953 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:55 smithi143 ceph-mon[183059]: failed to encode map e276 with expected crc 2023-08-28T02:31:55.953 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:55 smithi143 ceph-mon[183059]: failed to encode map e276 with expected crc 2023-08-28T02:31:55.953 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:55 smithi143 ceph-mon[183059]: osdmap e277: 8 total, 8 up, 7 in 2023-08-28T02:31:55.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:55 smithi169 ceph-mon[159415]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:31:55.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:55 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:31:55.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:55 smithi169 ceph-mon[159415]: failed to encode map e276 with expected crc 2023-08-28T02:31:55.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:55 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-08-28T02:31:55.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:55 smithi169 ceph-mon[159415]: osdmap e276: 8 total, 8 up, 7 in 2023-08-28T02:31:55.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:55 smithi169 ceph-mon[159415]: failed to encode map e276 with expected crc 2023-08-28T02:31:55.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:55 smithi169 ceph-mon[159415]: failed to encode map e276 with expected crc 2023-08-28T02:31:55.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:55 smithi169 ceph-mon[159415]: failed to encode map e276 with expected crc 2023-08-28T02:31:55.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:55 smithi169 ceph-mon[159415]: failed to encode map e276 with expected crc 2023-08-28T02:31:55.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:55 smithi169 ceph-mon[159415]: failed to encode map e276 with expected crc 2023-08-28T02:31:55.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:55 smithi169 ceph-mon[159415]: failed to encode map e276 with expected crc 2023-08-28T02:31:55.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:55 smithi169 ceph-mon[159415]: failed to encode map e276 with expected crc 2023-08-28T02:31:55.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:55 smithi169 ceph-mon[159415]: failed to encode map e276 with expected crc 2023-08-28T02:31:55.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:55 smithi169 ceph-mon[159415]: osdmap e277: 8 total, 8 up, 7 in 2023-08-28T02:31:56.841 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 55 16 69366 69350 78.7572 4.9375 0.0287797 0.012691 2023-08-28T02:31:56.953 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:56 smithi143 ceph-mon[183059]: pgmap v680: 99 pgs: 1 peering, 1 unknown, 1 active+recovering+undersized+remapped, 96 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s; 2083/206898 objects misplaced (1.007%); 0 B/s, 6 objects/s recovering 2023-08-28T02:31:56.953 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:56 smithi143 ceph-mon[183059]: failed to encode map e277 with expected crc 2023-08-28T02:31:56.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:56 smithi143 ceph-mon[183059]: failed to encode map e277 with expected crc 2023-08-28T02:31:56.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:56 smithi143 ceph-mon[183059]: failed to encode map e277 with expected crc 2023-08-28T02:31:56.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:56 smithi143 ceph-mon[183059]: failed to encode map e277 with expected crc 2023-08-28T02:31:56.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:56 smithi143 ceph-mon[183059]: failed to encode map e277 with expected crc 2023-08-28T02:31:56.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:56 smithi143 ceph-mon[183059]: failed to encode map e277 with expected crc 2023-08-28T02:31:56.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:56 smithi143 ceph-mon[183059]: failed to encode map e277 with expected crc 2023-08-28T02:31:56.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:56 smithi143 ceph-mon[183059]: Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-08-28T02:31:56.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:56 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:31:56.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:56 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:31:56.955 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:56 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:31:56.955 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:56 smithi143 ceph-mon[183059]: osdmap e278: 8 total, 8 up, 7 in 2023-08-28T02:31:56.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:56 smithi143 ceph-mon[189871]: pgmap v680: 99 pgs: 1 peering, 1 unknown, 1 active+recovering+undersized+remapped, 96 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s; 2083/206898 objects misplaced (1.007%); 0 B/s, 6 objects/s recovering 2023-08-28T02:31:56.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:56 smithi143 ceph-mon[189871]: failed to encode map e277 with expected crc 2023-08-28T02:31:56.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:56 smithi143 ceph-mon[189871]: failed to encode map e277 with expected crc 2023-08-28T02:31:56.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:56 smithi143 ceph-mon[189871]: failed to encode map e277 with expected crc 2023-08-28T02:31:56.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:56 smithi143 ceph-mon[189871]: failed to encode map e277 with expected crc 2023-08-28T02:31:56.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:56 smithi143 ceph-mon[189871]: failed to encode map e277 with expected crc 2023-08-28T02:31:56.956 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:56 smithi143 ceph-mon[189871]: failed to encode map e277 with expected crc 2023-08-28T02:31:56.956 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:56 smithi143 ceph-mon[189871]: failed to encode map e277 with expected crc 2023-08-28T02:31:56.956 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:56 smithi143 ceph-mon[189871]: Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-08-28T02:31:56.956 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:56 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:31:56.956 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:56 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:31:56.956 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:56 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:31:56.956 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:56 smithi143 ceph-mon[189871]: osdmap e278: 8 total, 8 up, 7 in 2023-08-28T02:31:56.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:56 smithi169 ceph-mon[159415]: pgmap v680: 99 pgs: 1 peering, 1 unknown, 1 active+recovering+undersized+remapped, 96 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s; 2083/206898 objects misplaced (1.007%); 0 B/s, 6 objects/s recovering 2023-08-28T02:31:56.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:56 smithi169 ceph-mon[159415]: failed to encode map e277 with expected crc 2023-08-28T02:31:56.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:56 smithi169 ceph-mon[159415]: failed to encode map e277 with expected crc 2023-08-28T02:31:56.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:56 smithi169 ceph-mon[159415]: failed to encode map e277 with expected crc 2023-08-28T02:31:56.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:56 smithi169 ceph-mon[159415]: failed to encode map e277 with expected crc 2023-08-28T02:31:56.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:56 smithi169 ceph-mon[159415]: failed to encode map e277 with expected crc 2023-08-28T02:31:56.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:56 smithi169 ceph-mon[159415]: failed to encode map e277 with expected crc 2023-08-28T02:31:56.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:56 smithi169 ceph-mon[159415]: failed to encode map e277 with expected crc 2023-08-28T02:31:56.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:56 smithi169 ceph-mon[159415]: Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-08-28T02:31:56.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:56 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:31:56.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:56 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:31:56.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:56 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:31:56.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:56 smithi169 ceph-mon[159415]: osdmap e278: 8 total, 8 up, 7 in 2023-08-28T02:31:57.116 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:31:57.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:57.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:31:57.231 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:31:57.275 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:57.706 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:57 smithi143 ceph-mon[189871]: 7.1 scrub starts 2023-08-28T02:31:57.706 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:57 smithi143 ceph-mon[189871]: 7.4 scrub starts 2023-08-28T02:31:57.706 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:57 smithi143 ceph-mon[189871]: failed to encode map e278 with expected crc 2023-08-28T02:31:57.707 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:57 smithi143 ceph-mon[189871]: failed to encode map e278 with expected crc 2023-08-28T02:31:57.707 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:57 smithi143 ceph-mon[189871]: failed to encode map e278 with expected crc 2023-08-28T02:31:57.707 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:57 smithi143 ceph-mon[189871]: failed to encode map e278 with expected crc 2023-08-28T02:31:57.707 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:57 smithi143 ceph-mon[189871]: failed to encode map e278 with expected crc 2023-08-28T02:31:57.707 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:57 smithi143 ceph-mon[189871]: pgmap v683: 99 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 unknown, 1 active+recovering+undersized+remapped, 92 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 37 MiB/s wr, 587 op/s; 6467/207525 objects degraded (3.116%); 2083/207525 objects misplaced (1.004%); 0 B/s, 2 keys/s, 7 objects/s recovering 2023-08-28T02:31:57.709 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:57 smithi143 ceph-mon[183059]: 7.1 scrub starts 2023-08-28T02:31:57.709 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:57 smithi143 ceph-mon[183059]: 7.4 scrub starts 2023-08-28T02:31:57.709 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:57 smithi143 ceph-mon[183059]: failed to encode map e278 with expected crc 2023-08-28T02:31:57.709 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:57 smithi143 ceph-mon[183059]: failed to encode map e278 with expected crc 2023-08-28T02:31:57.709 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:57 smithi143 ceph-mon[183059]: failed to encode map e278 with expected crc 2023-08-28T02:31:57.709 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:57 smithi143 ceph-mon[183059]: failed to encode map e278 with expected crc 2023-08-28T02:31:57.709 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:57 smithi143 ceph-mon[183059]: failed to encode map e278 with expected crc 2023-08-28T02:31:57.710 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:57 smithi143 ceph-mon[183059]: pgmap v683: 99 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 unknown, 1 active+recovering+undersized+remapped, 92 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 37 MiB/s wr, 587 op/s; 6467/207525 objects degraded (3.116%); 2083/207525 objects misplaced (1.004%); 0 B/s, 2 keys/s, 7 objects/s recovering 2023-08-28T02:31:57.714 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset nodeep-scrub 2023-08-28T02:31:57.842 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 56 16 70087 70071 78.1558 45.0625 0.0363411 0.0127888 2023-08-28T02:31:57.864 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:31:57.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:57.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:31:57.982 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:31:57.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:57 smithi169 ceph-mon[159415]: 7.1 scrub starts 2023-08-28T02:31:57.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:57 smithi169 ceph-mon[159415]: 7.4 scrub starts 2023-08-28T02:31:57.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:57 smithi169 ceph-mon[159415]: failed to encode map e278 with expected crc 2023-08-28T02:31:57.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:57 smithi169 ceph-mon[159415]: failed to encode map e278 with expected crc 2023-08-28T02:31:57.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:57 smithi169 ceph-mon[159415]: failed to encode map e278 with expected crc 2023-08-28T02:31:57.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:57 smithi169 ceph-mon[159415]: failed to encode map e278 with expected crc 2023-08-28T02:31:57.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:57 smithi169 ceph-mon[159415]: failed to encode map e278 with expected crc 2023-08-28T02:31:57.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:57 smithi169 ceph-mon[159415]: pgmap v683: 99 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 unknown, 1 active+recovering+undersized+remapped, 92 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 37 MiB/s wr, 587 op/s; 6467/207525 objects degraded (3.116%); 2083/207525 objects misplaced (1.004%); 0 B/s, 2 keys/s, 7 objects/s recovering 2023-08-28T02:31:58.022 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:58.072 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:31:58.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:58.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:31:58.179 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:31:58.259 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:58.342 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:31:58.386 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:31:58.431 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:31:58.486 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:31:58.488 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:31:58.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:58.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:31:58.532 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:31:58.567 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:31:58.602 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:31:58.609 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:31:58.630 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:58.644 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:31:58.648 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:31:58.648 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:31:58.648 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:31:58.649 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:31:58.649 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:31:58.649 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:31:58.649 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:31:58.649 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:31:58.679 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:31:58.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:58.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:31:58.699 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:58 smithi143 ceph-mon[183059]: Health check failed: Degraded data redundancy: 6467/207525 objects degraded (3.116%), 4 pgs degraded (PG_DEGRADED) 2023-08-28T02:31:58.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:58 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2248790497' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:31:58.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:58 smithi143 ceph-mon[189871]: Health check failed: Degraded data redundancy: 6467/207525 objects degraded (3.116%), 4 pgs degraded (PG_DEGRADED) 2023-08-28T02:31:58.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:58 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2248790497' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:31:58.785 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:31:58.810 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:58 smithi169 ceph-mon[159415]: Health check failed: Degraded data redundancy: 6467/207525 objects degraded (3.116%), 4 pgs degraded (PG_DEGRADED) 2023-08-28T02:31:58.811 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:58 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2248790497' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:31:58.813 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:58.820 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:31:58.821 INFO:teuthology.orchestra.run.smithi143.stdout: "size": 20, 2023-08-28T02:31:58.821 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 600, 2023-08-28T02:31:58.821 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [ 2023-08-28T02:31:58.821 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.821 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:59794 6.11 6.12cfc191 (undecoded) ondisk+write+known_if_redirected e210)", 2023-08-28T02:31:58.821 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:27:58.587391+0000", 2023-08-28T02:31:58.821 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 240.224258077, 2023-08-28T02:31:58.821 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.82055777799999996, 2023-08-28T02:31:58.821 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:31:58.821 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "reached pg", 2023-08-28T02:31:58.821 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:31:58.822 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:31:58.822 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:31:58.822 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 59794 2023-08-28T02:31:58.822 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.822 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:31:58.822 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.822 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:31:58.822 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:58.587391+0000", 2023-08-28T02:31:58.822 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:58.822 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.822 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.822 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:31:58.823 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:58.587391+0000", 2023-08-28T02:31:58.823 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:58.823 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.823 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.823 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:31:58.823 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:58.587389+0000", 2023-08-28T02:31:58.823 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:31:58.823 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.823 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.823 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:31:58.823 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:58.587424+0000", 2023-08-28T02:31:58.823 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.5256999999999998e-05 2023-08-28T02:31:58.824 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.824 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.824 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:31:58.824 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:58.587444+0000", 2023-08-28T02:31:58.824 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.9881000000000002e-05 2023-08-28T02:31:58.824 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.824 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.824 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:31:58.824 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:58.587455+0000", 2023-08-28T02:31:58.824 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.025e-05 2023-08-28T02:31:58.824 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.824 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.825 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.825 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:58.587881+0000", 2023-08-28T02:31:58.825 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00042679399999999998 2023-08-28T02:31:58.825 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.825 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.825 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:31:58.825 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:58.587893+0000", 2023-08-28T02:31:58.825 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.2148e-05 2023-08-28T02:31:58.825 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.825 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.825 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.825 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:59.407942+0000", 2023-08-28T02:31:58.825 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.82004887100000001 2023-08-28T02:31:58.826 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.826 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.826 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:31:58.826 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:59.407949+0000", 2023-08-28T02:31:58.826 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.5749999999999997e-06 2023-08-28T02:31:58.826 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:58.826 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:58.826 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:58.826 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.826 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.826 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:59798 6.4 6:20730b1e:::benchmark_data_smithi143_305742_object59797:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e210)", 2023-08-28T02:31:58.826 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:27:58.595559+0000", 2023-08-28T02:31:58.827 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 240.21608997999999, 2023-08-28T02:31:58.827 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.776363484, 2023-08-28T02:31:58.827 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:31:58.827 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:58.827 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:31:58.827 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:31:58.827 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:31:58.827 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 59798 2023-08-28T02:31:58.827 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.827 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:31:58.827 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.827 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:31:58.828 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:58.595559+0000", 2023-08-28T02:31:58.828 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:58.828 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.828 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.828 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:31:58.828 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:58.595559+0000", 2023-08-28T02:31:58.828 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:58.828 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.828 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.828 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:31:58.828 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:58.595557+0000", 2023-08-28T02:31:58.828 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999976 2023-08-28T02:31:58.829 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.829 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.829 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:31:58.829 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:58.595590+0000", 2023-08-28T02:31:58.829 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.2710999999999999e-05 2023-08-28T02:31:58.829 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.829 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.829 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:31:58.829 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:58.595607+0000", 2023-08-28T02:31:58.829 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.7452000000000002e-05 2023-08-28T02:31:58.829 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.829 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.830 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:31:58.830 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:58.595617+0000", 2023-08-28T02:31:58.830 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.5570000000000001e-06 2023-08-28T02:31:58.830 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.830 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.830 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.830 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:58.595683+0000", 2023-08-28T02:31:58.830 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.6198999999999994e-05 2023-08-28T02:31:58.830 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.830 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.830 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:31:58.830 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:58.595698+0000", 2023-08-28T02:31:58.831 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.5211000000000001e-05 2023-08-28T02:31:58.831 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.831 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.831 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.831 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:00.353359+0000", 2023-08-28T02:31:58.831 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.7576612439999999 2023-08-28T02:31:58.831 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.831 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.831 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:31:58.831 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:00.355419+0000", 2023-08-28T02:31:58.831 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0020594070000000001 2023-08-28T02:31:58.831 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.832 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.832 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 3,5", 2023-08-28T02:31:58.832 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:00.359064+0000", 2023-08-28T02:31:58.832 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0036453509999999998 2023-08-28T02:31:58.832 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.832 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.832 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:31:58.832 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:00.366765+0000", 2023-08-28T02:31:58.832 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0077012749999999996 2023-08-28T02:31:58.832 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.832 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.832 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:58.833 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:00.367213+0000", 2023-08-28T02:31:58.833 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000447591 2023-08-28T02:31:58.833 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.833 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.833 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:58.833 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:00.367748+0000", 2023-08-28T02:31:58.833 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00053465700000000001 2023-08-28T02:31:58.833 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.833 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.833 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:31:58.833 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:00.368633+0000", 2023-08-28T02:31:58.834 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000885424 2023-08-28T02:31:58.834 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.834 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.834 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:31:58.834 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:00.371923+0000", 2023-08-28T02:31:58.834 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0032896589999999999 2023-08-28T02:31:58.834 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:58.834 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:58.834 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:58.834 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.834 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.834 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:59801 6.4 6:21d6ad0f:::benchmark_data_smithi143_305742_object59800:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e210)", 2023-08-28T02:31:58.835 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:27:58.604758+0000", 2023-08-28T02:31:58.835 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 240.206891192, 2023-08-28T02:31:58.835 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.783077459, 2023-08-28T02:31:58.835 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:31:58.835 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:58.835 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:31:58.835 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:31:58.835 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:31:58.835 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 59801 2023-08-28T02:31:58.835 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.835 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:31:58.835 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.836 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:31:58.836 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:58.604758+0000", 2023-08-28T02:31:58.836 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:58.836 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.836 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.836 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:31:58.836 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:58.604758+0000", 2023-08-28T02:31:58.836 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:58.836 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.836 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.836 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:31:58.836 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:58.604756+0000", 2023-08-28T02:31:58.836 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:31:58.837 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.837 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.837 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:31:58.837 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:58.604789+0000", 2023-08-28T02:31:58.837 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.2969000000000002e-05 2023-08-28T02:31:58.837 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.837 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.837 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:31:58.837 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:58.604806+0000", 2023-08-28T02:31:58.837 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.6288000000000001e-05 2023-08-28T02:31:58.837 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.837 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.838 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:31:58.838 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:58.604814+0000", 2023-08-28T02:31:58.838 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.0800000000000006e-06 2023-08-28T02:31:58.838 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.838 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.838 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.838 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:58.604891+0000", 2023-08-28T02:31:58.838 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.7350999999999999e-05 2023-08-28T02:31:58.838 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.838 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.838 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:31:58.838 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:58.604906+0000", 2023-08-28T02:31:58.839 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.4635e-05 2023-08-28T02:31:58.839 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.839 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.839 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.839 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:00.372162+0000", 2023-08-28T02:31:58.839 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.767256355 2023-08-28T02:31:58.839 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.839 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.839 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:31:58.839 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:00.374231+0000", 2023-08-28T02:31:58.839 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0020687769999999999 2023-08-28T02:31:58.839 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.839 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.840 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 3,5", 2023-08-28T02:31:58.840 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:00.377447+0000", 2023-08-28T02:31:58.840 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0032162359999999999 2023-08-28T02:31:58.840 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.840 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.840 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:58.840 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:00.382765+0000", 2023-08-28T02:31:58.840 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0053183049999999997 2023-08-28T02:31:58.840 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.840 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.840 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:31:58.840 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:00.382928+0000", 2023-08-28T02:31:58.840 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00016233099999999999 2023-08-28T02:31:58.841 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.841 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.841 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:58.841 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:00.383536+0000", 2023-08-28T02:31:58.841 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00060783600000000005 2023-08-28T02:31:58.841 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.841 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.841 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:31:58.841 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:00.384474+0000", 2023-08-28T02:31:58.841 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000938377 2023-08-28T02:31:58.841 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.841 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.841 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:31:58.842 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:00.387836+0000", 2023-08-28T02:31:58.842 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0033614980000000001 2023-08-28T02:31:58.842 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:58.842 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:58.842 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:58.842 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.842 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.842 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:59811 6.14 6:2832eb50:::benchmark_data_smithi143_305742_object59810:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e211)", 2023-08-28T02:31:58.842 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:27:59.483310+0000", 2023-08-28T02:31:58.842 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 239.328339586, 2023-08-28T02:31:58.842 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.91538435500000004, 2023-08-28T02:31:58.842 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:31:58.843 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:58.843 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:31:58.843 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:31:58.843 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:31:58.843 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 59811 2023-08-28T02:31:58.843 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.843 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:31:58.843 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.843 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:31:58.843 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:59.483310+0000", 2023-08-28T02:31:58.843 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:58.843 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.843 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.844 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:31:58.844 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:59.483310+0000", 2023-08-28T02:31:58.844 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:58.844 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.844 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.844 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:31:58.844 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:59.483308+0000", 2023-08-28T02:31:58.844 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:31:58.844 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.844 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.844 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:31:58.844 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:59.483330+0000", 2023-08-28T02:31:58.844 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.2470999999999999e-05 2023-08-28T02:31:58.845 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.845 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.845 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:31:58.845 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:59.483343+0000", 2023-08-28T02:31:58.845 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.2731999999999999e-05 2023-08-28T02:31:58.845 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.845 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.845 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:31:58.845 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:59.483351+0000", 2023-08-28T02:31:58.845 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.8769999999999993e-06 2023-08-28T02:31:58.845 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.845 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.846 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.846 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:00.372549+0000", 2023-08-28T02:31:58.846 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.88919825600000002 2023-08-28T02:31:58.846 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.846 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.846 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:31:58.846 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:00.375296+0000", 2023-08-28T02:31:58.846 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0027471309999999999 2023-08-28T02:31:58.846 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.846 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.846 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 3,5", 2023-08-28T02:31:58.846 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:00.379265+0000", 2023-08-28T02:31:58.846 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0039687799999999999 2023-08-28T02:31:58.847 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.847 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.847 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:58.847 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:00.392450+0000", 2023-08-28T02:31:58.847 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.013185429 2023-08-28T02:31:58.847 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.847 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.847 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:31:58.847 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:00.392670+0000", 2023-08-28T02:31:58.847 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00021946300000000001 2023-08-28T02:31:58.847 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.847 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.848 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:58.848 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:00.393361+0000", 2023-08-28T02:31:58.848 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00069133200000000001 2023-08-28T02:31:58.848 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.848 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.848 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:31:58.848 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:00.394478+0000", 2023-08-28T02:31:58.848 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0011164429999999999 2023-08-28T02:31:58.848 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.848 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.848 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:31:58.848 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:00.398694+0000", 2023-08-28T02:31:58.848 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0042163000000000001 2023-08-28T02:31:58.848 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:58.849 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:58.849 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:58.849 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.849 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.849 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:108933 6.11 6:8925519c:::benchmark_data_smithi143_305742_object25314:head [delete] snapc 0=[] ondisk+write+known_if_redirected e233)", 2023-08-28T02:31:58.849 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:29:13.641649+0000", 2023-08-28T02:31:58.849 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 165.169999894, 2023-08-28T02:31:58.849 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.68182749899999995, 2023-08-28T02:31:58.849 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:31:58.849 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:58.849 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:31:58.849 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:31:58.850 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:31:58.850 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 108933 2023-08-28T02:31:58.850 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.850 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:31:58.850 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.850 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:31:58.850 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:13.641649+0000", 2023-08-28T02:31:58.850 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:58.850 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.850 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.850 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:31:58.850 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:13.641649+0000", 2023-08-28T02:31:58.850 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:58.851 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.851 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.851 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:31:58.851 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:13.641648+0000", 2023-08-28T02:31:58.851 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:31:58.851 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.851 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.851 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:31:58.851 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:13.641654+0000", 2023-08-28T02:31:58.851 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.8980000000000001e-06 2023-08-28T02:31:58.851 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.851 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.852 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:31:58.852 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:13.641668+0000", 2023-08-28T02:31:58.852 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.4375999999999999e-05 2023-08-28T02:31:58.852 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.852 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.852 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:31:58.852 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:13.641676+0000", 2023-08-28T02:31:58.852 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.7610000000000007e-06 2023-08-28T02:31:58.852 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.852 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.852 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.852 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:13.711440+0000", 2023-08-28T02:31:58.853 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.069764324000000003 2023-08-28T02:31:58.853 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.853 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.853 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for missing object", 2023-08-28T02:31:58.853 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:13.714279+0000", 2023-08-28T02:31:58.853 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.002838513 2023-08-28T02:31:58.853 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.853 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.853 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.853 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:13.735069+0000", 2023-08-28T02:31:58.853 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.020790283999999999 2023-08-28T02:31:58.853 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.853 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.854 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for rw locks", 2023-08-28T02:31:58.854 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:13.737676+0000", 2023-08-28T02:31:58.854 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0026070030000000001 2023-08-28T02:31:58.854 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.854 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.854 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.854 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:14.260851+0000", 2023-08-28T02:31:58.854 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.52317517899999999 2023-08-28T02:31:58.854 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.854 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.854 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:31:58.854 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:14.262794+0000", 2023-08-28T02:31:58.854 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.001942879 2023-08-28T02:31:58.855 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.855 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.855 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 2,6", 2023-08-28T02:31:58.855 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:14.266198+0000", 2023-08-28T02:31:58.855 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0034041589999999999 2023-08-28T02:31:58.855 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.855 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.855 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:31:58.855 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:14.277731+0000", 2023-08-28T02:31:58.855 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.011532061999999999 2023-08-28T02:31:58.855 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.855 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.856 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:58.856 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:14.278579+0000", 2023-08-28T02:31:58.856 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00084854400000000001 2023-08-28T02:31:58.856 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.856 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.856 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:58.856 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:14.279405+0000", 2023-08-28T02:31:58.856 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00082615 2023-08-28T02:31:58.856 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.856 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.856 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:31:58.856 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:14.280765+0000", 2023-08-28T02:31:58.856 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.001359891 2023-08-28T02:31:58.857 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.857 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.857 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:31:58.857 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:14.323477+0000", 2023-08-28T02:31:58.857 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.042711712999999998 2023-08-28T02:31:58.857 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:58.857 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:58.857 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:58.857 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.857 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.857 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:109005 6.1 6:85e3e84e:::benchmark_data_smithi143_305742_object25386:head [delete] snapc 0=[] ondisk+write+known_if_redirected e233)", 2023-08-28T02:31:58.857 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:29:13.810931+0000", 2023-08-28T02:31:58.858 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 165.00071869499999, 2023-08-28T02:31:58.858 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.68854191200000003, 2023-08-28T02:31:58.858 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:31:58.858 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:58.858 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:31:58.858 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:31:58.858 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:31:58.858 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 109005 2023-08-28T02:31:58.858 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.858 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:31:58.858 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.858 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:31:58.858 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:13.810931+0000", 2023-08-28T02:31:58.859 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:58.859 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.859 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.859 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:31:58.859 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:13.810931+0000", 2023-08-28T02:31:58.859 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:58.859 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.859 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.859 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:31:58.859 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:13.810929+0000", 2023-08-28T02:31:58.859 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:31:58.859 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.860 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.860 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:31:58.860 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:13.810936+0000", 2023-08-28T02:31:58.860 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.9480000000000002e-06 2023-08-28T02:31:58.860 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.860 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.860 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:31:58.860 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:13.810951+0000", 2023-08-28T02:31:58.860 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.5505000000000001e-05 2023-08-28T02:31:58.860 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.860 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.860 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:31:58.860 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:13.810960+0000", 2023-08-28T02:31:58.861 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.3329999999999997e-06 2023-08-28T02:31:58.861 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.861 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.861 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.861 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:13.835437+0000", 2023-08-28T02:31:58.861 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.024477720000000001 2023-08-28T02:31:58.861 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.861 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.861 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for missing object", 2023-08-28T02:31:58.861 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:13.838074+0000", 2023-08-28T02:31:58.861 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.002637038 2023-08-28T02:31:58.861 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.862 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.862 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.862 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:13.851601+0000", 2023-08-28T02:31:58.862 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.013526817999999999 2023-08-28T02:31:58.862 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.862 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.862 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for rw locks", 2023-08-28T02:31:58.862 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:13.854146+0000", 2023-08-28T02:31:58.862 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0025447569999999999 2023-08-28T02:31:58.862 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.862 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.862 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.863 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:14.477717+0000", 2023-08-28T02:31:58.863 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.62357142200000004 2023-08-28T02:31:58.863 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.863 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.863 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:31:58.863 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:14.479777+0000", 2023-08-28T02:31:58.863 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0020596659999999999 2023-08-28T02:31:58.863 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.863 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.863 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 2,6", 2023-08-28T02:31:58.863 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:14.483485+0000", 2023-08-28T02:31:58.863 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.003707751 2023-08-28T02:31:58.864 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.864 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.864 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:58.864 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:14.492024+0000", 2023-08-28T02:31:58.864 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0085387020000000004 2023-08-28T02:31:58.864 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.864 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.864 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:58.864 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:14.492864+0000", 2023-08-28T02:31:58.864 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00084024800000000004 2023-08-28T02:31:58.864 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.864 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.864 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:31:58.865 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:14.493164+0000", 2023-08-28T02:31:58.865 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00030014199999999999 2023-08-28T02:31:58.865 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.865 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.865 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:31:58.865 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:14.494627+0000", 2023-08-28T02:31:58.865 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.001463413 2023-08-28T02:31:58.865 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.865 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.865 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:31:58.865 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:14.499472+0000", 2023-08-28T02:31:58.865 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.004845055 2023-08-28T02:31:58.865 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:58.866 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:58.866 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:58.866 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.866 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.866 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:114493 6.1 6:805da47c:::benchmark_data_smithi143_305742_object30874:head [delete] snapc 0=[] ondisk+write+known_if_redirected e237)", 2023-08-28T02:31:58.866 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:29:28.439157+0000", 2023-08-28T02:31:58.866 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 150.37249219399999, 2023-08-28T02:31:58.866 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.68878075000000005, 2023-08-28T02:31:58.866 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:31:58.866 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:58.866 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:31:58.866 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:31:58.867 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:31:58.867 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 114493 2023-08-28T02:31:58.867 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.867 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:31:58.867 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.867 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:31:58.867 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:28.439157+0000", 2023-08-28T02:31:58.867 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:58.867 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.867 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.867 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:31:58.868 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:28.439157+0000", 2023-08-28T02:31:58.868 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:58.868 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.868 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.868 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:31:58.868 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:28.439155+0000", 2023-08-28T02:31:58.868 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:31:58.868 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.868 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.868 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:31:58.868 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:28.439164+0000", 2023-08-28T02:31:58.868 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.9139999999999997e-06 2023-08-28T02:31:58.869 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.869 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.869 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:31:58.869 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:28.439189+0000", 2023-08-28T02:31:58.869 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.493e-05 2023-08-28T02:31:58.869 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.869 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.869 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:31:58.869 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:28.439198+0000", 2023-08-28T02:31:58.869 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.8370000000000003e-06 2023-08-28T02:31:58.869 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.869 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.869 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.870 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:28.487961+0000", 2023-08-28T02:31:58.870 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.048762953999999997 2023-08-28T02:31:58.870 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.870 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.870 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for missing object", 2023-08-28T02:31:58.870 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:28.491347+0000", 2023-08-28T02:31:58.870 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.003385767 2023-08-28T02:31:58.870 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.870 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.870 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.870 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:28.510368+0000", 2023-08-28T02:31:58.870 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.019021132999999999 2023-08-28T02:31:58.871 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.871 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.871 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for rw locks", 2023-08-28T02:31:58.871 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:28.513610+0000", 2023-08-28T02:31:58.871 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0032422309999999999 2023-08-28T02:31:58.871 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.871 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.871 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.871 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.105335+0000", 2023-08-28T02:31:58.871 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.59172481099999996 2023-08-28T02:31:58.871 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.871 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.872 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:31:58.872 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.107324+0000", 2023-08-28T02:31:58.872 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0019894439999999999 2023-08-28T02:31:58.872 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.872 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.872 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 2,6", 2023-08-28T02:31:58.872 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.110824+0000", 2023-08-28T02:31:58.872 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0034992560000000001 2023-08-28T02:31:58.872 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.872 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.872 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:31:58.872 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.121166+0000", 2023-08-28T02:31:58.872 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.010342136 2023-08-28T02:31:58.873 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.873 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.873 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:58.873 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.121949+0000", 2023-08-28T02:31:58.873 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00078284299999999997 2023-08-28T02:31:58.873 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.873 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.873 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:58.873 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.122600+0000", 2023-08-28T02:31:58.873 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00065097199999999999 2023-08-28T02:31:58.873 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.874 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.874 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:31:58.874 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.123564+0000", 2023-08-28T02:31:58.874 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00096427600000000002 2023-08-28T02:31:58.874 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.874 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.874 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:31:58.874 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.127938+0000", 2023-08-28T02:31:58.874 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0043739870000000002 2023-08-28T02:31:58.874 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:58.874 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:58.874 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:58.875 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.875 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.875 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:114640 6.1 6:87549d17:::benchmark_data_smithi143_305742_object31021:head [delete] snapc 0=[] ondisk+write+known_if_redirected e237)", 2023-08-28T02:31:58.875 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:29:28.691796+0000", 2023-08-28T02:31:58.875 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 150.119853199, 2023-08-28T02:31:58.875 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.681987079, 2023-08-28T02:31:58.875 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:31:58.875 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:58.875 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:31:58.875 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:31:58.875 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:31:58.875 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 114640 2023-08-28T02:31:58.876 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.876 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:31:58.876 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.876 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:31:58.876 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:28.691796+0000", 2023-08-28T02:31:58.876 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:58.876 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.876 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.876 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:31:58.876 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:28.691796+0000", 2023-08-28T02:31:58.876 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:58.876 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.877 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.877 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:31:58.877 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:28.691795+0000", 2023-08-28T02:31:58.877 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:31:58.877 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.877 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.877 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:31:58.877 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:28.691801+0000", 2023-08-28T02:31:58.877 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.993e-06 2023-08-28T02:31:58.877 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.877 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.877 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:31:58.877 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:28.691817+0000", 2023-08-28T02:31:58.878 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.6328e-05 2023-08-28T02:31:58.878 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.878 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.878 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:31:58.878 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:28.691825+0000", 2023-08-28T02:31:58.878 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.1179999999999992e-06 2023-08-28T02:31:58.878 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.878 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.878 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.878 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:28.744937+0000", 2023-08-28T02:31:58.878 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.053111683999999999 2023-08-28T02:31:58.878 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.878 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.879 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for missing object", 2023-08-28T02:31:58.879 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:28.747513+0000", 2023-08-28T02:31:58.879 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0025760689999999998 2023-08-28T02:31:58.879 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.879 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.879 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.879 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:28.762547+0000", 2023-08-28T02:31:58.879 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.015033944 2023-08-28T02:31:58.879 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.879 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.879 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for rw locks", 2023-08-28T02:31:58.879 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:28.764858+0000", 2023-08-28T02:31:58.880 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0023109279999999999 2023-08-28T02:31:58.880 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.880 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.880 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.880 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.346919+0000", 2023-08-28T02:31:58.880 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.58206165099999996 2023-08-28T02:31:58.880 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.880 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.880 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:31:58.880 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.349089+0000", 2023-08-28T02:31:58.880 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0021696110000000001 2023-08-28T02:31:58.880 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.880 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.881 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 2,6", 2023-08-28T02:31:58.881 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.352186+0000", 2023-08-28T02:31:58.881 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.003097411 2023-08-28T02:31:58.881 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.881 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.881 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:58.881 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.367309+0000", 2023-08-28T02:31:58.881 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.015122505 2023-08-28T02:31:58.881 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.881 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.881 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:31:58.881 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.367488+0000", 2023-08-28T02:31:58.882 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00017900699999999999 2023-08-28T02:31:58.882 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.882 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.882 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:58.882 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.368655+0000", 2023-08-28T02:31:58.882 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.001167434 2023-08-28T02:31:58.882 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.882 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.882 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:31:58.882 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.369626+0000", 2023-08-28T02:31:58.882 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00097038500000000004 2023-08-28T02:31:58.882 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.882 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.883 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:31:58.883 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.373783+0000", 2023-08-28T02:31:58.883 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0041574639999999996 2023-08-28T02:31:58.883 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:58.883 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:58.883 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:58.883 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.883 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.883 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:114641 6.1 6:85d5db48:::benchmark_data_smithi143_305742_object31022:head [delete] snapc 0=[] ondisk+write+known_if_redirected e237)", 2023-08-28T02:31:58.883 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:29:28.694545+0000", 2023-08-28T02:31:58.883 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 150.117104657, 2023-08-28T02:31:58.883 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.75260789100000003, 2023-08-28T02:31:58.884 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:31:58.884 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:58.884 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:31:58.884 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:31:58.884 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:31:58.884 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 114641 2023-08-28T02:31:58.884 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.884 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:31:58.884 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.884 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:31:58.884 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:28.694545+0000", 2023-08-28T02:31:58.884 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:58.884 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.885 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.885 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:31:58.885 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:28.694545+0000", 2023-08-28T02:31:58.885 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:58.885 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.885 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.885 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:31:58.885 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:28.694544+0000", 2023-08-28T02:31:58.885 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:31:58.885 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.885 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.885 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:31:58.886 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:28.694549+0000", 2023-08-28T02:31:58.886 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.8459999999999999e-06 2023-08-28T02:31:58.886 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.886 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.886 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:31:58.886 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:28.694561+0000", 2023-08-28T02:31:58.886 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.2119e-05 2023-08-28T02:31:58.886 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.886 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.886 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:31:58.886 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:28.694566+0000", 2023-08-28T02:31:58.886 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.2660000000000001e-06 2023-08-28T02:31:58.886 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.887 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.887 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.887 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:28.747789+0000", 2023-08-28T02:31:58.887 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.053223146999999998 2023-08-28T02:31:58.887 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.887 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.887 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for missing object", 2023-08-28T02:31:58.887 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:28.750547+0000", 2023-08-28T02:31:58.887 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0027581459999999999 2023-08-28T02:31:58.887 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.887 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.887 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.888 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:28.793670+0000", 2023-08-28T02:31:58.888 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.043123119000000001 2023-08-28T02:31:58.888 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.888 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.888 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for rw locks", 2023-08-28T02:31:58.888 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:28.795726+0000", 2023-08-28T02:31:58.888 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0020551620000000001 2023-08-28T02:31:58.888 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.888 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.888 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.888 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.389805+0000", 2023-08-28T02:31:58.888 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.59407932299999999 2023-08-28T02:31:58.889 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.889 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.889 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:31:58.889 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.391648+0000", 2023-08-28T02:31:58.889 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.001843438 2023-08-28T02:31:58.889 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.889 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.889 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 2,6", 2023-08-28T02:31:58.889 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.394882+0000", 2023-08-28T02:31:58.889 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0032337109999999998 2023-08-28T02:31:58.889 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.889 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.889 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:58.890 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.418644+0000", 2023-08-28T02:31:58.890 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.023762122 2023-08-28T02:31:58.890 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.890 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.890 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:31:58.890 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.418904+0000", 2023-08-28T02:31:58.890 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00026031999999999997 2023-08-28T02:31:58.890 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.890 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.890 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:58.890 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.440765+0000", 2023-08-28T02:31:58.890 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.021860706000000001 2023-08-28T02:31:58.891 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.891 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.891 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:31:58.891 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.442023+0000", 2023-08-28T02:31:58.891 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.001257853 2023-08-28T02:31:58.891 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.891 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.891 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:31:58.891 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.447152+0000", 2023-08-28T02:31:58.891 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0051294920000000003 2023-08-28T02:31:58.891 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:58.891 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:58.891 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:58.892 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.892 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.892 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:114686 6.1 6:834e77f0:::benchmark_data_smithi143_305742_object31067:head [delete] snapc 0=[] ondisk+write+known_if_redirected e237)", 2023-08-28T02:31:58.892 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:29:28.842771+0000", 2023-08-28T02:31:58.892 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 149.96887843499999, 2023-08-28T02:31:58.892 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.71471194599999999, 2023-08-28T02:31:58.892 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:31:58.892 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:58.892 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:31:58.892 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:31:58.892 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:31:58.892 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 114686 2023-08-28T02:31:58.893 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.893 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:31:58.893 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.893 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:31:58.893 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:28.842771+0000", 2023-08-28T02:31:58.893 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:58.893 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.893 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.893 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:31:58.893 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:28.842771+0000", 2023-08-28T02:31:58.893 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:58.893 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.893 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.894 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:31:58.894 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:28.842769+0000", 2023-08-28T02:31:58.894 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:31:58.894 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.894 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.894 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:31:58.894 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:28.842776+0000", 2023-08-28T02:31:58.894 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.2089999999999999e-06 2023-08-28T02:31:58.894 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.894 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.894 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:31:58.894 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:28.842788+0000", 2023-08-28T02:31:58.894 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1702000000000001e-05 2023-08-28T02:31:58.895 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.895 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.895 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:31:58.895 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:28.842795+0000", 2023-08-28T02:31:58.895 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.2699999999999999e-06 2023-08-28T02:31:58.895 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.895 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.895 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.895 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.027468+0000", 2023-08-28T02:31:58.895 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.184673107 2023-08-28T02:31:58.895 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.896 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.896 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for missing object", 2023-08-28T02:31:58.896 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.030249+0000", 2023-08-28T02:31:58.896 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0027805289999999999 2023-08-28T02:31:58.896 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.896 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.896 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.896 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.046433+0000", 2023-08-28T02:31:58.896 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.016184166 2023-08-28T02:31:58.896 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.896 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.896 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for rw locks", 2023-08-28T02:31:58.897 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.048630+0000", 2023-08-28T02:31:58.897 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0021966830000000001 2023-08-28T02:31:58.897 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.897 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.897 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.897 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.530709+0000", 2023-08-28T02:31:58.897 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.48207926899999998 2023-08-28T02:31:58.897 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.897 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.897 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:31:58.897 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.532830+0000", 2023-08-28T02:31:58.897 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0021209599999999999 2023-08-28T02:31:58.897 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.898 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.898 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 2,6", 2023-08-28T02:31:58.898 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.536359+0000", 2023-08-28T02:31:58.898 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0035289509999999998 2023-08-28T02:31:58.898 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.898 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.898 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:58.898 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.541096+0000", 2023-08-28T02:31:58.898 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.004737542 2023-08-28T02:31:58.898 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.898 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.899 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:31:58.899 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.541817+0000", 2023-08-28T02:31:58.899 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00072034499999999995 2023-08-28T02:31:58.899 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.899 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.899 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:58.899 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.546449+0000", 2023-08-28T02:31:58.899 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0046318940000000001 2023-08-28T02:31:58.899 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.899 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.899 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:31:58.899 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.552401+0000", 2023-08-28T02:31:58.900 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0059528300000000001 2023-08-28T02:31:58.900 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.900 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.900 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:31:58.900 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.557483+0000", 2023-08-28T02:31:58.900 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0050814129999999999 2023-08-28T02:31:58.900 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:58.900 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:58.900 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:58.900 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.900 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.900 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:114864 6.11 6:8b9a9268:::benchmark_data_smithi143_305742_object31245:head [delete] snapc 0=[] ondisk+write+known_if_redirected e237)", 2023-08-28T02:31:58.900 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:29:29.349414+0000", 2023-08-28T02:31:58.901 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 149.462234878, 2023-08-28T02:31:58.901 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.73168314099999998, 2023-08-28T02:31:58.901 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:31:58.901 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:58.901 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:31:58.901 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:31:58.901 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:31:58.901 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 114864 2023-08-28T02:31:58.901 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.901 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:31:58.901 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.902 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:31:58.902 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.349414+0000", 2023-08-28T02:31:58.902 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:58.902 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.902 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.902 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:31:58.902 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.349414+0000", 2023-08-28T02:31:58.902 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:58.902 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.902 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.902 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:31:58.902 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.349413+0000", 2023-08-28T02:31:58.903 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:31:58.903 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.903 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.903 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:31:58.903 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.349418+0000", 2023-08-28T02:31:58.903 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.8550000000000001e-06 2023-08-28T02:31:58.903 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.903 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.903 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:31:58.903 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.349428+0000", 2023-08-28T02:31:58.903 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.6700000000000006e-06 2023-08-28T02:31:58.903 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.903 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.904 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:31:58.904 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.349433+0000", 2023-08-28T02:31:58.904 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.5999999999999997e-06 2023-08-28T02:31:58.904 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.904 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.904 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.904 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.368544+0000", 2023-08-28T02:31:58.904 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.019110939 2023-08-28T02:31:58.904 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.904 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.904 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for missing object", 2023-08-28T02:31:58.904 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.370592+0000", 2023-08-28T02:31:58.905 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0020479790000000001 2023-08-28T02:31:58.905 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.905 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.905 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.905 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.464456+0000", 2023-08-28T02:31:58.905 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.093863536999999997 2023-08-28T02:31:58.905 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.905 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.905 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for rw locks", 2023-08-28T02:31:58.905 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.467205+0000", 2023-08-28T02:31:58.905 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0027496740000000001 2023-08-28T02:31:58.905 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.905 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.906 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.906 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:30.048605+0000", 2023-08-28T02:31:58.906 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.58140008200000004 2023-08-28T02:31:58.906 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.906 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.906 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:31:58.906 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:30.050327+0000", 2023-08-28T02:31:58.906 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0017214579999999999 2023-08-28T02:31:58.906 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.906 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.906 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 2,6", 2023-08-28T02:31:58.906 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:30.053541+0000", 2023-08-28T02:31:58.906 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.003214429 2023-08-28T02:31:58.907 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.907 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.907 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:58.907 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:30.074559+0000", 2023-08-28T02:31:58.907 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.021018085999999998 2023-08-28T02:31:58.907 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.907 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.907 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:31:58.907 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:30.074758+0000", 2023-08-28T02:31:58.907 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00019832500000000001 2023-08-28T02:31:58.907 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.907 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.908 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:58.908 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:30.075302+0000", 2023-08-28T02:31:58.908 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00054404099999999999 2023-08-28T02:31:58.908 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.908 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.908 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:31:58.908 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:30.076200+0000", 2023-08-28T02:31:58.908 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00089836199999999999 2023-08-28T02:31:58.908 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.908 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.908 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:31:58.908 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:30.081098+0000", 2023-08-28T02:31:58.909 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0048974889999999997 2023-08-28T02:31:58.909 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:58.909 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:58.909 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:58.909 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.909 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.909 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:114877 6.11 6:8fa50007:::benchmark_data_smithi143_305742_object31258:head [delete] snapc 0=[] ondisk+write+known_if_redirected e237)", 2023-08-28T02:31:58.909 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:29:29.386923+0000", 2023-08-28T02:31:58.909 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 149.424725921, 2023-08-28T02:31:58.909 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.701399628, 2023-08-28T02:31:58.909 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:31:58.910 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:58.910 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:31:58.910 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:31:58.910 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:31:58.910 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 114877 2023-08-28T02:31:58.910 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.910 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:31:58.910 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.910 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:31:58.910 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.386923+0000", 2023-08-28T02:31:58.910 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:58.910 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.910 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.911 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:31:58.911 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.386923+0000", 2023-08-28T02:31:58.911 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:58.911 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.911 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.911 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:31:58.911 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.386921+0000", 2023-08-28T02:31:58.911 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:31:58.911 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.911 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.911 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:31:58.911 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.386930+0000", 2023-08-28T02:31:58.912 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.5369999999999997e-06 2023-08-28T02:31:58.912 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.912 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.912 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:31:58.912 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.386941+0000", 2023-08-28T02:31:58.912 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1629e-05 2023-08-28T02:31:58.912 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.912 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.912 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:31:58.912 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.386948+0000", 2023-08-28T02:31:58.912 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.2729999999999999e-06 2023-08-28T02:31:58.912 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.913 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.913 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.913 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.467473+0000", 2023-08-28T02:31:58.913 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.080525726000000006 2023-08-28T02:31:58.913 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.913 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.913 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for missing object", 2023-08-28T02:31:58.913 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.470085+0000", 2023-08-28T02:31:58.913 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.002611417 2023-08-28T02:31:58.913 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.913 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.913 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.914 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.492059+0000", 2023-08-28T02:31:58.914 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.021973784 2023-08-28T02:31:58.914 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.914 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.914 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for rw locks", 2023-08-28T02:31:58.914 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.494623+0000", 2023-08-28T02:31:58.914 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0025647410000000002 2023-08-28T02:31:58.914 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.914 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.914 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.914 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:30.066465+0000", 2023-08-28T02:31:58.914 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.57184174700000001 2023-08-28T02:31:58.914 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.915 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.915 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:31:58.915 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:30.068031+0000", 2023-08-28T02:31:58.915 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.001565503 2023-08-28T02:31:58.915 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.915 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.915 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 2,6", 2023-08-28T02:31:58.915 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:30.070824+0000", 2023-08-28T02:31:58.915 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0027935740000000001 2023-08-28T02:31:58.915 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.915 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.916 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:58.916 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:30.081729+0000", 2023-08-28T02:31:58.916 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.010904422 2023-08-28T02:31:58.916 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.916 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.916 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:31:58.916 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:30.081911+0000", 2023-08-28T02:31:58.916 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000182726 2023-08-28T02:31:58.916 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.916 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.916 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:58.916 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:30.082657+0000", 2023-08-28T02:31:58.917 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00074565500000000002 2023-08-28T02:31:58.917 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.917 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.917 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:31:58.917 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:30.083658+0000", 2023-08-28T02:31:58.917 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0010005420000000001 2023-08-28T02:31:58.917 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.917 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.917 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:31:58.917 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:30.088323+0000", 2023-08-28T02:31:58.917 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0046653550000000004 2023-08-28T02:31:58.917 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:58.917 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:58.918 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:58.918 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.918 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.918 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:114880 6.1 6:83610f67:::benchmark_data_smithi143_305742_object31261:head [delete] snapc 0=[] ondisk+write+known_if_redirected e237)", 2023-08-28T02:31:58.918 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:29:29.397356+0000", 2023-08-28T02:31:58.918 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 149.414293512, 2023-08-28T02:31:58.918 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.755545091, 2023-08-28T02:31:58.918 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:31:58.918 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:58.918 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:31:58.918 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:31:58.918 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:31:58.919 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 114880 2023-08-28T02:31:58.919 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.919 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:31:58.919 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.919 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:31:58.919 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.397356+0000", 2023-08-28T02:31:58.919 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:58.919 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.919 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.919 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:31:58.919 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.397356+0000", 2023-08-28T02:31:58.919 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:58.920 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.920 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.920 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:31:58.920 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.397354+0000", 2023-08-28T02:31:58.920 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:31:58.920 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.920 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.920 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:31:58.920 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.397361+0000", 2023-08-28T02:31:58.920 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.5610000000000004e-06 2023-08-28T02:31:58.920 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.920 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.921 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:31:58.921 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.397375+0000", 2023-08-28T02:31:58.921 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3994e-05 2023-08-28T02:31:58.921 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.921 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.921 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:31:58.921 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.397385+0000", 2023-08-28T02:31:58.921 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0094e-05 2023-08-28T02:31:58.921 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.921 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.921 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.921 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.472248+0000", 2023-08-28T02:31:58.922 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.074862924999999997 2023-08-28T02:31:58.922 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.922 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.922 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for missing object", 2023-08-28T02:31:58.922 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.474876+0000", 2023-08-28T02:31:58.922 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0026280280000000001 2023-08-28T02:31:58.922 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.922 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.922 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.922 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.488519+0000", 2023-08-28T02:31:58.922 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.013642771999999999 2023-08-28T02:31:58.922 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.923 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.923 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for rw locks", 2023-08-28T02:31:58.923 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.491415+0000", 2023-08-28T02:31:58.923 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.002896287 2023-08-28T02:31:58.923 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.923 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.923 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.923 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:30.101295+0000", 2023-08-28T02:31:58.923 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.609879807 2023-08-28T02:31:58.923 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.923 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.923 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:31:58.924 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:30.103069+0000", 2023-08-28T02:31:58.924 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0017743450000000001 2023-08-28T02:31:58.924 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.924 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.924 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 2,6", 2023-08-28T02:31:58.924 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:30.106340+0000", 2023-08-28T02:31:58.924 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0032705490000000002 2023-08-28T02:31:58.924 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.924 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.924 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:58.924 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:30.127983+0000", 2023-08-28T02:31:58.924 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.02164276 2023-08-28T02:31:58.924 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.925 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.925 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:31:58.925 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:30.128195+0000", 2023-08-28T02:31:58.925 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00021271199999999999 2023-08-28T02:31:58.925 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.925 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.925 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:58.925 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:30.146802+0000", 2023-08-28T02:31:58.925 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.018607202 2023-08-28T02:31:58.925 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.925 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.925 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:31:58.926 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:30.148041+0000", 2023-08-28T02:31:58.926 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0012387699999999999 2023-08-28T02:31:58.926 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.926 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.926 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:31:58.926 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:30.152901+0000", 2023-08-28T02:31:58.926 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0048596070000000002 2023-08-28T02:31:58.926 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:58.926 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:58.926 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:58.926 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.926 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.927 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:114882 6.1 6:855bae5b:::benchmark_data_smithi143_305742_object31263:head [delete] snapc 0=[] ondisk+write+known_if_redirected e237)", 2023-08-28T02:31:58.927 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:29:29.447952+0000", 2023-08-28T02:31:58.927 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 149.363697282, 2023-08-28T02:31:58.927 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.71292710000000004, 2023-08-28T02:31:58.927 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:31:58.927 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:58.927 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:31:58.927 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:31:58.927 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:31:58.927 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 114882 2023-08-28T02:31:58.927 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.927 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:31:58.928 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.928 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:31:58.928 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.447952+0000", 2023-08-28T02:31:58.928 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:58.928 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.928 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.928 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:31:58.928 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.447952+0000", 2023-08-28T02:31:58.928 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:58.928 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.928 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.928 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:31:58.929 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.447950+0000", 2023-08-28T02:31:58.929 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:31:58.929 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.929 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.929 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:31:58.929 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.447958+0000", 2023-08-28T02:31:58.929 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.0520000000000002e-06 2023-08-28T02:31:58.929 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.929 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.929 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:31:58.929 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.447973+0000", 2023-08-28T02:31:58.929 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.4429e-05 2023-08-28T02:31:58.930 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.930 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.930 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:31:58.930 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.447982+0000", 2023-08-28T02:31:58.930 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.9579999999999996e-06 2023-08-28T02:31:58.930 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.930 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.930 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.930 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.475127+0000", 2023-08-28T02:31:58.930 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.027145181000000001 2023-08-28T02:31:58.930 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.930 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.931 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for missing object", 2023-08-28T02:31:58.931 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.477766+0000", 2023-08-28T02:31:58.931 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0026387559999999999 2023-08-28T02:31:58.931 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.931 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.931 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.931 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.501232+0000", 2023-08-28T02:31:58.931 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.023466272 2023-08-28T02:31:58.931 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.931 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.931 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for rw locks", 2023-08-28T02:31:58.931 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.503836+0000", 2023-08-28T02:31:58.932 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0026044359999999999 2023-08-28T02:31:58.932 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.932 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.932 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.932 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:30.118671+0000", 2023-08-28T02:31:58.932 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.614835036 2023-08-28T02:31:58.932 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.932 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.932 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:31:58.932 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:30.120380+0000", 2023-08-28T02:31:58.932 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.001708336 2023-08-28T02:31:58.932 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.933 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.933 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 2,6", 2023-08-28T02:31:58.933 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:30.123906+0000", 2023-08-28T02:31:58.933 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.003526628 2023-08-28T02:31:58.933 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.933 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.933 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:31:58.933 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:30.145404+0000", 2023-08-28T02:31:58.933 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.021497115000000001 2023-08-28T02:31:58.933 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.933 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.933 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:58.934 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:30.145963+0000", 2023-08-28T02:31:58.934 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00055964899999999995 2023-08-28T02:31:58.934 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.934 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.934 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:58.934 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:30.154442+0000", 2023-08-28T02:31:58.934 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0084789410000000003 2023-08-28T02:31:58.934 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.934 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.934 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:31:58.934 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:30.155713+0000", 2023-08-28T02:31:58.935 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.001270478 2023-08-28T02:31:58.935 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.935 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.935 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:31:58.935 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:30.160879+0000", 2023-08-28T02:31:58.935 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.005166477 2023-08-28T02:31:58.935 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:58.935 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:58.935 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:58.935 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.935 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.936 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:114937 6.11 6:8a8ef5ff:::benchmark_data_smithi143_305742_object31318:head [delete] snapc 0=[] ondisk+write+known_if_redirected e238)", 2023-08-28T02:31:58.936 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:29:29.616518+0000", 2023-08-28T02:31:58.936 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 149.19513151000001, 2023-08-28T02:31:58.936 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.69828699699999996, 2023-08-28T02:31:58.936 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:31:58.936 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:58.936 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:31:58.936 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:31:58.936 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:31:58.936 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 114937 2023-08-28T02:31:58.936 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.936 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:31:58.936 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.937 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:31:58.937 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.616518+0000", 2023-08-28T02:31:58.937 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:58.937 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.937 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.937 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:31:58.937 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.616518+0000", 2023-08-28T02:31:58.937 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:58.937 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.937 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.937 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:31:58.937 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.616516+0000", 2023-08-28T02:31:58.938 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:31:58.938 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.938 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.938 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:31:58.938 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.616523+0000", 2023-08-28T02:31:58.938 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.4159999999999998e-06 2023-08-28T02:31:58.938 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.938 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.938 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:31:58.938 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.616536+0000", 2023-08-28T02:31:58.938 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.2938e-05 2023-08-28T02:31:58.938 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.938 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.939 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:31:58.939 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.616544+0000", 2023-08-28T02:31:58.939 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.9300000000000003e-06 2023-08-28T02:31:58.939 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.939 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.939 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.939 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.681978+0000", 2023-08-28T02:31:58.939 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.065433742000000003 2023-08-28T02:31:58.939 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.939 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.939 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for missing object", 2023-08-28T02:31:58.939 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.685071+0000", 2023-08-28T02:31:58.940 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0030933800000000002 2023-08-28T02:31:58.940 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.940 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.940 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.940 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.735068+0000", 2023-08-28T02:31:58.940 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.049996772000000002 2023-08-28T02:31:58.940 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.940 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.940 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for rw locks", 2023-08-28T02:31:58.940 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:29.737828+0000", 2023-08-28T02:31:58.940 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0027594659999999999 2023-08-28T02:31:58.940 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.940 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.941 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.941 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:30.298510+0000", 2023-08-28T02:31:58.941 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.56068251700000005 2023-08-28T02:31:58.941 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.941 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.941 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:31:58.941 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:30.300619+0000", 2023-08-28T02:31:58.941 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0021087749999999998 2023-08-28T02:31:58.941 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.941 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.941 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 2,6", 2023-08-28T02:31:58.941 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:30.304226+0000", 2023-08-28T02:31:58.941 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.003607051 2023-08-28T02:31:58.942 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.942 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.942 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:58.942 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:30.308151+0000", 2023-08-28T02:31:58.942 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0039248490000000002 2023-08-28T02:31:58.942 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.942 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.942 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:31:58.942 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:30.308340+0000", 2023-08-28T02:31:58.942 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00018918099999999999 2023-08-28T02:31:58.942 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.942 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.942 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:58.943 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:30.308915+0000", 2023-08-28T02:31:58.943 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00057476200000000004 2023-08-28T02:31:58.943 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.943 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.943 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:31:58.943 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:30.309939+0000", 2023-08-28T02:31:58.943 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0010247100000000001 2023-08-28T02:31:58.943 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.943 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.943 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:31:58.943 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:30.314805+0000", 2023-08-28T02:31:58.943 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0048653639999999996 2023-08-28T02:31:58.944 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:58.944 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:58.944 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:58.944 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.944 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.944 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:121177 6.11 6:892f3543:::benchmark_data_smithi143_305742_object37558:head [delete] snapc 0=[] ondisk+write+known_if_redirected e245)", 2023-08-28T02:31:58.944 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:29:43.710332+0000", 2023-08-28T02:31:58.944 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 135.101317526, 2023-08-28T02:31:58.944 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.68894212700000002, 2023-08-28T02:31:58.944 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:31:58.944 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:58.944 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:31:58.944 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:31:58.945 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:31:58.945 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 121177 2023-08-28T02:31:58.945 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.945 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:31:58.945 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.945 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:31:58.945 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:43.710332+0000", 2023-08-28T02:31:58.945 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:58.945 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.945 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.945 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:31:58.945 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:43.710332+0000", 2023-08-28T02:31:58.946 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:58.946 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.946 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.946 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:31:58.946 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:43.710330+0000", 2023-08-28T02:31:58.946 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:31:58.946 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.946 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.946 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:31:58.946 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:43.710337+0000", 2023-08-28T02:31:58.946 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.7859999999999999e-06 2023-08-28T02:31:58.946 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.946 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.947 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:31:58.947 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:43.710349+0000", 2023-08-28T02:31:58.947 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1888e-05 2023-08-28T02:31:58.947 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.947 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.947 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:31:58.947 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:43.710357+0000", 2023-08-28T02:31:58.947 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.49e-06 2023-08-28T02:31:58.947 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.947 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.947 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.948 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:43.732445+0000", 2023-08-28T02:31:58.948 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.022087861 2023-08-28T02:31:58.948 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.948 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.948 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for missing object", 2023-08-28T02:31:58.948 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:43.734275+0000", 2023-08-28T02:31:58.948 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0018296359999999999 2023-08-28T02:31:58.948 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.948 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.948 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.948 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:43.748020+0000", 2023-08-28T02:31:58.948 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.013745603 2023-08-28T02:31:58.948 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.949 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.949 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for rw locks", 2023-08-28T02:31:58.949 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:43.749908+0000", 2023-08-28T02:31:58.949 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0018877869999999999 2023-08-28T02:31:58.949 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.949 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.949 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.949 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:44.383640+0000", 2023-08-28T02:31:58.949 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.63373167699999999 2023-08-28T02:31:58.949 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.949 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.949 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:31:58.949 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:44.385533+0000", 2023-08-28T02:31:58.950 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0018928840000000001 2023-08-28T02:31:58.950 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.950 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.950 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 2,6", 2023-08-28T02:31:58.950 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:44.388785+0000", 2023-08-28T02:31:58.950 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0032524350000000001 2023-08-28T02:31:58.950 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.950 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.950 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:58.950 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:44.393014+0000", 2023-08-28T02:31:58.950 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0042285170000000002 2023-08-28T02:31:58.950 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.951 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.951 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:58.951 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:44.393651+0000", 2023-08-28T02:31:58.951 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00063736200000000004 2023-08-28T02:31:58.951 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.951 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.951 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:31:58.951 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:44.393855+0000", 2023-08-28T02:31:58.951 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000203644 2023-08-28T02:31:58.951 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.951 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.951 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:31:58.952 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:44.394900+0000", 2023-08-28T02:31:58.952 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.001045549 2023-08-28T02:31:58.952 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.952 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.952 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:31:58.952 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:29:44.399274+0000", 2023-08-28T02:31:58.952 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0043735939999999997 2023-08-28T02:31:58.952 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:58.952 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:58.952 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:58.952 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.952 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.952 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:140686 6.1 6:82335874:::benchmark_data_smithi143_305742_object57067:head [delete] snapc 0=[] ondisk+write+known_if_redirected e251)", 2023-08-28T02:31:58.953 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:30:08.646555+0000", 2023-08-28T02:31:58.953 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 110.165093978, 2023-08-28T02:31:58.953 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.94579028099999995, 2023-08-28T02:31:58.953 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:31:58.953 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:58.953 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:31:58.953 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:31:58.953 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:31:58.953 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 140686 2023-08-28T02:31:58.953 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.953 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:31:58.953 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.953 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:31:58.954 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:08.646555+0000", 2023-08-28T02:31:58.954 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:58.954 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.954 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.954 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:31:58.954 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:08.646555+0000", 2023-08-28T02:31:58.954 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:58.954 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.954 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.954 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:31:58.954 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:08.646554+0000", 2023-08-28T02:31:58.954 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:31:58.955 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.955 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.955 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:31:58.955 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:08.646568+0000", 2023-08-28T02:31:58.955 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.4079e-05 2023-08-28T02:31:58.955 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.955 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.955 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:31:58.955 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:08.646589+0000", 2023-08-28T02:31:58.955 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.1427999999999999e-05 2023-08-28T02:31:58.955 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.955 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.956 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:31:58.956 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:08.646598+0000", 2023-08-28T02:31:58.956 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.7180000000000002e-06 2023-08-28T02:31:58.956 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.956 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.956 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.956 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:08.646683+0000", 2023-08-28T02:31:58.956 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.5470999999999995e-05 2023-08-28T02:31:58.956 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.956 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.956 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:31:58.956 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:08.646695+0000", 2023-08-28T02:31:58.957 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1567e-05 2023-08-28T02:31:58.957 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.957 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.957 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.957 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.588927+0000", 2023-08-28T02:31:58.957 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.94223240500000005 2023-08-28T02:31:58.957 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.957 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.957 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:31:58.957 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.589108+0000", 2023-08-28T02:31:58.957 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00018099300000000001 2023-08-28T02:31:58.957 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.958 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.958 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 2,6", 2023-08-28T02:31:58.958 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.589272+0000", 2023-08-28T02:31:58.958 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00016356899999999999 2023-08-28T02:31:58.958 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.958 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.958 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:58.958 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.592211+0000", 2023-08-28T02:31:58.958 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00293925 2023-08-28T02:31:58.958 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.958 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.958 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:58.958 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.592265+0000", 2023-08-28T02:31:58.959 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.3588000000000003e-05 2023-08-28T02:31:58.959 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.959 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.959 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:31:58.959 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.592271+0000", 2023-08-28T02:31:58.959 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.5610000000000004e-06 2023-08-28T02:31:58.959 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.959 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.959 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:31:58.959 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.592304+0000", 2023-08-28T02:31:58.959 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.2444000000000003e-05 2023-08-28T02:31:58.959 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.959 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.960 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:31:58.960 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.592346+0000", 2023-08-28T02:31:58.960 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.1872000000000002e-05 2023-08-28T02:31:58.960 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:58.960 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:58.960 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:58.960 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.960 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.960 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:140746 6.1 6:81379348:::benchmark_data_smithi143_305742_object57127:head [delete] snapc 0=[] ondisk+write+known_if_redirected e251)", 2023-08-28T02:31:58.960 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:30:08.711674+0000", 2023-08-28T02:31:58.960 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 110.099974786, 2023-08-28T02:31:58.960 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.887529821, 2023-08-28T02:31:58.961 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:31:58.961 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:58.961 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:31:58.961 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:31:58.961 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:31:58.961 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 140746 2023-08-28T02:31:58.961 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.961 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:31:58.961 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.961 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:31:58.961 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:08.711674+0000", 2023-08-28T02:31:58.961 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:58.961 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.962 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.962 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:31:58.962 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:08.711674+0000", 2023-08-28T02:31:58.962 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:58.962 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.962 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.962 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:31:58.962 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:08.711673+0000", 2023-08-28T02:31:58.962 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:31:58.962 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.962 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.962 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:31:58.963 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:08.711679+0000", 2023-08-28T02:31:58.963 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.3659999999999997e-06 2023-08-28T02:31:58.963 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.963 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.963 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:31:58.963 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:08.711690+0000", 2023-08-28T02:31:58.963 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0151e-05 2023-08-28T02:31:58.963 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.963 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.963 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:31:58.963 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:08.711695+0000", 2023-08-28T02:31:58.963 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.8780000000000003e-06 2023-08-28T02:31:58.963 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.964 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.964 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.964 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:08.711748+0000", 2023-08-28T02:31:58.964 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.2645000000000002e-05 2023-08-28T02:31:58.964 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.964 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.964 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:31:58.964 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:08.711758+0000", 2023-08-28T02:31:58.964 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.7650000000000005e-06 2023-08-28T02:31:58.964 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.964 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.964 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.965 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.592398+0000", 2023-08-28T02:31:58.965 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.88063988100000001 2023-08-28T02:31:58.965 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.965 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.965 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:31:58.965 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.592528+0000", 2023-08-28T02:31:58.965 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000129944 2023-08-28T02:31:58.965 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.965 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.965 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 2,6", 2023-08-28T02:31:58.965 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.592656+0000", 2023-08-28T02:31:58.965 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00012866599999999999 2023-08-28T02:31:58.966 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.966 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.966 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:58.966 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.599085+0000", 2023-08-28T02:31:58.966 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0064289849999999999 2023-08-28T02:31:58.966 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.966 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.966 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:31:58.966 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.599097+0000", 2023-08-28T02:31:58.966 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.2159e-05 2023-08-28T02:31:58.966 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.966 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.967 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:58.967 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.599126+0000", 2023-08-28T02:31:58.967 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.8538e-05 2023-08-28T02:31:58.967 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.967 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.967 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:31:58.967 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.599166+0000", 2023-08-28T02:31:58.967 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.9839999999999998e-05 2023-08-28T02:31:58.967 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.967 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.967 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:31:58.967 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.599204+0000", 2023-08-28T02:31:58.968 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.8439999999999998e-05 2023-08-28T02:31:58.968 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:58.968 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:58.968 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:58.968 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.968 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.968 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:140811 6.1 6:86b0a199:::benchmark_data_smithi143_305742_object57192:head [delete] snapc 0=[] ondisk+write+known_if_redirected e251)", 2023-08-28T02:31:58.968 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:30:08.777943+0000", 2023-08-28T02:31:58.968 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 110.033706528, 2023-08-28T02:31:58.968 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.82139114099999999, 2023-08-28T02:31:58.968 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:31:58.968 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:58.969 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:31:58.969 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:31:58.969 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:31:58.969 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 140811 2023-08-28T02:31:58.969 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.969 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:31:58.969 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.969 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:31:58.969 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:08.777943+0000", 2023-08-28T02:31:58.969 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:58.969 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.969 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.970 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:31:58.970 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:08.777943+0000", 2023-08-28T02:31:58.970 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:58.970 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.970 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.970 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:31:58.970 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:08.777941+0000", 2023-08-28T02:31:58.970 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:31:58.970 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.970 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.970 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:31:58.970 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:08.777948+0000", 2023-08-28T02:31:58.970 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.3649999999999998e-06 2023-08-28T02:31:58.971 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.971 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.971 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:31:58.971 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:08.777964+0000", 2023-08-28T02:31:58.971 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.5818999999999999e-05 2023-08-28T02:31:58.971 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.971 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.971 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:31:58.971 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:08.777973+0000", 2023-08-28T02:31:58.971 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.6300000000000004e-06 2023-08-28T02:31:58.971 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.971 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.972 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.972 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:08.778071+0000", 2023-08-28T02:31:58.972 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.8478000000000001e-05 2023-08-28T02:31:58.972 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.972 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.972 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:31:58.972 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:08.778085+0000", 2023-08-28T02:31:58.972 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3551e-05 2023-08-28T02:31:58.972 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.972 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.972 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.972 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.594667+0000", 2023-08-28T02:31:58.972 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.81658207400000005 2023-08-28T02:31:58.973 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.973 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.973 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:31:58.973 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.594788+0000", 2023-08-28T02:31:58.973 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00012129999999999999 2023-08-28T02:31:58.973 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.973 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.973 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 2,6", 2023-08-28T02:31:58.973 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.594908+0000", 2023-08-28T02:31:58.973 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000120375 2023-08-28T02:31:58.973 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.973 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.973 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:58.974 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.599244+0000", 2023-08-28T02:31:58.974 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0043352920000000001 2023-08-28T02:31:58.974 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.974 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.974 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:31:58.974 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.599252+0000", 2023-08-28T02:31:58.974 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.225e-06 2023-08-28T02:31:58.974 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.974 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.974 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:58.974 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.599275+0000", 2023-08-28T02:31:58.974 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.2597999999999999e-05 2023-08-28T02:31:58.974 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.975 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.975 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:31:58.975 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.599300+0000", 2023-08-28T02:31:58.975 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.5462000000000001e-05 2023-08-28T02:31:58.975 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.975 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.975 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:31:58.975 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.599334+0000", 2023-08-28T02:31:58.975 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.3797999999999997e-05 2023-08-28T02:31:58.975 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:58.975 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:58.975 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:58.976 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.976 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.976 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:140862 6.1 6:816c4e36:::benchmark_data_smithi143_305742_object57243:head [delete] snapc 0=[] ondisk+write+known_if_redirected e251)", 2023-08-28T02:31:58.976 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:30:08.828237+0000", 2023-08-28T02:31:58.976 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 109.98341236, 2023-08-28T02:31:58.976 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.77755967100000001, 2023-08-28T02:31:58.976 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:31:58.976 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:31:58.976 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:31:58.976 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:31:58.976 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:31:58.976 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 140862 2023-08-28T02:31:58.977 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.977 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:31:58.977 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.977 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:31:58.977 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:08.828237+0000", 2023-08-28T02:31:58.977 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:58.977 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.977 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.977 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:31:58.977 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:08.828237+0000", 2023-08-28T02:31:58.977 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:31:58.977 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.977 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.978 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:31:58.978 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:08.828235+0000", 2023-08-28T02:31:58.978 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:31:58.978 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.978 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.978 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:31:58.978 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:08.828243+0000", 2023-08-28T02:31:58.978 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.9889999999999992e-06 2023-08-28T02:31:58.978 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.978 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.978 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:31:58.978 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:08.828257+0000", 2023-08-28T02:31:58.979 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.384e-05 2023-08-28T02:31:58.979 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.979 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.979 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:31:58.979 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:08.828265+0000", 2023-08-28T02:31:58.979 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.9030000000000004e-06 2023-08-28T02:31:58.979 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.979 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.979 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.979 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:08.828349+0000", 2023-08-28T02:31:58.979 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.3950999999999996e-05 2023-08-28T02:31:58.979 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.980 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.980 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:31:58.980 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:08.828362+0000", 2023-08-28T02:31:58.980 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3220999999999999e-05 2023-08-28T02:31:58.980 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.980 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.980 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:31:58.980 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.596758+0000", 2023-08-28T02:31:58.980 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.76839603899999998 2023-08-28T02:31:58.980 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.980 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.980 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:31:58.981 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.596872+0000", 2023-08-28T02:31:58.981 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000114014 2023-08-28T02:31:58.981 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.981 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.981 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 2,6", 2023-08-28T02:31:58.981 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.596980+0000", 2023-08-28T02:31:58.981 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000107655 2023-08-28T02:31:58.981 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.981 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.981 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:58.981 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.599376+0000", 2023-08-28T02:31:58.981 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0023960959999999999 2023-08-28T02:31:58.981 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.982 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.982 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:31:58.982 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.599386+0000", 2023-08-28T02:31:58.982 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.5319999999999995e-06 2023-08-28T02:31:58.982 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.982 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.982 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:31:58.982 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.605702+0000", 2023-08-28T02:31:58.982 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0063164390000000001 2023-08-28T02:31:58.982 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.982 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.982 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:31:58.983 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.605754+0000", 2023-08-28T02:31:58.983 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.2451000000000003e-05 2023-08-28T02:31:58.983 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:31:58.983 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:31:58.983 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:31:58.983 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.605797+0000", 2023-08-28T02:31:58.983 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.2138000000000002e-05 2023-08-28T02:31:58.983 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:58.983 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:58.983 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:58.983 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:31:58.983 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:31:58.984 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:31:58.984 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 57 16 70811 70795 77.5771 45.25 0.016311 0.0128856 2023-08-28T02:31:58.985 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:31:58.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:58.985 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:31:58.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:31:59.086 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:31:59.117 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:59.172 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:31:59.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:59.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:31:59.279 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:31:59.312 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:59.368 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:31:59.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:31:59.373 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:31:59.474 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:31:59.520 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:31:59.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:31:59 smithi169 ceph-mon[159415]: pgmap v684: 99 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 unknown, 2 active+recovering+undersized+remapped, 83 active+clean; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 35 MiB/s wr, 563 op/s; 21424/210378 objects degraded (10.184%); 4242/210378 objects misplaced (2.016%); 459 KiB/s, 1 keys/s, 19 objects/s recovering 2023-08-28T02:31:59.845 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 58 16 71506 71490 76.9865 43.4375 0.00389253 0.0129704 2023-08-28T02:31:59.956 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:31:59 smithi143 ceph-mon[183059]: pgmap v684: 99 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 unknown, 2 active+recovering+undersized+remapped, 83 active+clean; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 35 MiB/s wr, 563 op/s; 21424/210378 objects degraded (10.184%); 4242/210378 objects misplaced (2.016%); 459 KiB/s, 1 keys/s, 19 objects/s recovering 2023-08-28T02:31:59.957 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:31:59 smithi143 ceph-mon[189871]: pgmap v684: 99 pgs: 12 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 unknown, 2 active+recovering+undersized+remapped, 83 active+clean; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 35 MiB/s wr, 563 op/s; 21424/210378 objects degraded (10.184%); 4242/210378 objects misplaced (2.016%); 459 KiB/s, 1 keys/s, 19 objects/s recovering 2023-08-28T02:32:00.047 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:32:00.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:00.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:32:00.167 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:32:00.219 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:00.758 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:32:00.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:00.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:32:00.786 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.2 dump_ops_in_flight 2023-08-28T02:32:00.845 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 59 16 72306 72290 76.5288 50 0.0261246 0.0130606 2023-08-28T02:32:00.867 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:32:00.900 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:00.951 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:32:00.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:00.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:32:01.058 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:32:01.087 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:01.142 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:32:01.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:01.149 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:32:01.250 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:32:01.339 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:01.467 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:01 smithi169 ceph-mon[159415]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-08-28T02:32:01.507 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:32:01.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:01.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:32:01.616 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:32:01.648 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:01.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:01 smithi143 ceph-mon[183059]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-08-28T02:32:01.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:01 smithi143 ceph-mon[189871]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-08-28T02:32:01.703 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:32:01.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:01.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:32:01.816 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:32:01.845 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:2023-08-28T02:32:01.846829+0000 min lat: 0.00114956 max lat: 2.65098 avg lat: 0.013126 2023-08-28T02:32:01.845 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-08-28T02:32:01.845 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 60 16 73171 73155 76.1545 54.0625 0.0280799 0.013126 2023-08-28T02:32:01.879 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:02.201 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:32:01 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:32:01] "GET /metrics HTTP/1.1" 200 36550 "" "Prometheus/2.33.4" 2023-08-28T02:32:02.393 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:32:02.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:02.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:32:02.497 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:32:02.527 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:02.578 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:32:02.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:02.584 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:32:02.685 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:32:02.690 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:32:02.704 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:02 smithi143 ceph-mon[183059]: pgmap v685: 99 pgs: 14 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 83 active+clean; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 18 MiB/s wr, 283 op/s; 21446/212226 objects degraded (10.105%); 3686/212226 objects misplaced (1.737%); 5.1 MiB/s, 1 keys/s, 92 objects/s recovering 2023-08-28T02:32:02.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:02 smithi143 ceph-mon[189871]: pgmap v685: 99 pgs: 14 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 83 active+clean; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 18 MiB/s wr, 283 op/s; 21446/212226 objects degraded (10.105%); 3686/212226 objects misplaced (1.737%); 5.1 MiB/s, 1 keys/s, 92 objects/s recovering 2023-08-28T02:32:02.759 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:32:02.765 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:02.845 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 61 16 73818 73802 75.5692 40.4375 0.0301455 0.0132252 2023-08-28T02:32:02.964 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:32:02.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:02.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:32:02.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:02 smithi169 ceph-mon[159415]: pgmap v685: 99 pgs: 14 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 83 active+clean; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 18 MiB/s wr, 283 op/s; 21446/212226 objects degraded (10.105%); 3686/212226 objects misplaced (1.737%); 5.1 MiB/s, 1 keys/s, 92 objects/s recovering 2023-08-28T02:32:03.005 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 1, 6] out_osds: [7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-08-28T02:32:03.006 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-08-28T02:32:03.006 INFO:tasks.thrashosds.thrasher:Removing osd 1, in_osds are: [0, 2, 4, 5, 3, 1, 6] 2023-08-28T02:32:03.006 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd out 1 2023-08-28T02:32:03.086 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:32:03.240 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:03.426 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:32:03.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:03.461 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:32:03.562 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:32:03.633 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:03.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:03 smithi143 ceph-mon[183059]: pgmap v686: 99 pgs: 14 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 83 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 22 MiB/s wr, 344 op/s; 21446/215178 objects degraded (9.967%); 3686/215178 objects misplaced (1.713%); 4.9 MiB/s, 1 keys/s, 80 objects/s recovering 2023-08-28T02:32:03.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:03 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 21446/215178 objects degraded (9.967%), 14 pgs degraded (PG_DEGRADED) 2023-08-28T02:32:03.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:03 smithi143 ceph-mon[189871]: pgmap v686: 99 pgs: 14 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 83 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 22 MiB/s wr, 344 op/s; 21446/215178 objects degraded (9.967%); 3686/215178 objects misplaced (1.713%); 4.9 MiB/s, 1 keys/s, 80 objects/s recovering 2023-08-28T02:32:03.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:03 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 21446/215178 objects degraded (9.967%), 14 pgs degraded (PG_DEGRADED) 2023-08-28T02:32:03.845 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 62 16 74511 74495 75.0491 43.3125 0.0304657 0.0133191 2023-08-28T02:32:03.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:03 smithi169 ceph-mon[159415]: pgmap v686: 99 pgs: 14 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 83 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 22 MiB/s wr, 344 op/s; 21446/215178 objects degraded (9.967%); 3686/215178 objects misplaced (1.713%); 4.9 MiB/s, 1 keys/s, 80 objects/s recovering 2023-08-28T02:32:03.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:03 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 21446/215178 objects degraded (9.967%), 14 pgs degraded (PG_DEGRADED) 2023-08-28T02:32:04.578 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is unset 2023-08-28T02:32:04.664 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:32:04.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:04.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:32:04.775 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:32:04.842 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:04 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2622048647' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:32:04.842 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:04 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:32:04.842 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:04 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2622048647' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:32:04.843 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:04 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:32:04.845 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 63 16 75109 75093 74.4514 37.375 0.0260733 0.0134252 2023-08-28T02:32:04.849 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:04.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:04 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2622048647' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:32:04.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:04 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:32:05.549 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:32:05.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:05.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:32:05.658 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:32:05.688 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:05.693 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:05 smithi143 ceph-mon[183059]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:32:05.693 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:05 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:32:05.693 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:05 smithi143 ceph-mon[183059]: osdmap e279: 8 total, 8 up, 7 in 2023-08-28T02:32:05.693 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:05 smithi143 ceph-mon[183059]: failed to encode map e279 with expected crc 2023-08-28T02:32:05.693 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:05 smithi143 ceph-mon[183059]: failed to encode map e279 with expected crc 2023-08-28T02:32:05.693 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:05 smithi143 ceph-mon[183059]: failed to encode map e279 with expected crc 2023-08-28T02:32:05.693 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:05 smithi143 ceph-mon[183059]: failed to encode map e279 with expected crc 2023-08-28T02:32:05.694 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:05 smithi143 ceph-mon[183059]: failed to encode map e279 with expected crc 2023-08-28T02:32:05.694 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:05 smithi143 ceph-mon[183059]: pgmap v688: 99 pgs: 14 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 83 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 38 MiB/s wr, 602 op/s; 21446/222897 objects degraded (9.621%); 3316/222897 objects misplaced (1.488%); 7.0 MiB/s, 1 keys/s, 114 objects/s recovering 2023-08-28T02:32:05.698 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:05 smithi143 ceph-mon[189871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:32:05.698 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:05 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:32:05.699 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:05 smithi143 ceph-mon[189871]: osdmap e279: 8 total, 8 up, 7 in 2023-08-28T02:32:05.699 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:05 smithi143 ceph-mon[189871]: failed to encode map e279 with expected crc 2023-08-28T02:32:05.699 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:05 smithi143 ceph-mon[189871]: failed to encode map e279 with expected crc 2023-08-28T02:32:05.699 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:05 smithi143 ceph-mon[189871]: failed to encode map e279 with expected crc 2023-08-28T02:32:05.699 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:05 smithi143 ceph-mon[189871]: failed to encode map e279 with expected crc 2023-08-28T02:32:05.699 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:05 smithi143 ceph-mon[189871]: failed to encode map e279 with expected crc 2023-08-28T02:32:05.699 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:05 smithi143 ceph-mon[189871]: pgmap v688: 99 pgs: 14 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 83 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 38 MiB/s wr, 602 op/s; 21446/222897 objects degraded (9.621%); 3316/222897 objects misplaced (1.488%); 7.0 MiB/s, 1 keys/s, 114 objects/s recovering 2023-08-28T02:32:05.741 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:32:05.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:05.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:32:05.804 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:32:05.850 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:32:05.851 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 64 16 75862 75846 74.0191 47.0625 0.0108648 0.0135047 2023-08-28T02:32:05.967 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:05.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:05 smithi169 ceph-mon[159415]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:32:05.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:05 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:32:05.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:05 smithi169 ceph-mon[159415]: osdmap e279: 8 total, 8 up, 7 in 2023-08-28T02:32:05.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:05 smithi169 ceph-mon[159415]: failed to encode map e279 with expected crc 2023-08-28T02:32:05.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:05 smithi169 ceph-mon[159415]: failed to encode map e279 with expected crc 2023-08-28T02:32:05.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:05 smithi169 ceph-mon[159415]: failed to encode map e279 with expected crc 2023-08-28T02:32:05.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:05 smithi169 ceph-mon[159415]: failed to encode map e279 with expected crc 2023-08-28T02:32:05.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:05 smithi169 ceph-mon[159415]: failed to encode map e279 with expected crc 2023-08-28T02:32:05.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:05 smithi169 ceph-mon[159415]: pgmap v688: 99 pgs: 14 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 83 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 38 MiB/s wr, 602 op/s; 21446/222897 objects degraded (9.621%); 3316/222897 objects misplaced (1.488%); 7.0 MiB/s, 1 keys/s, 114 objects/s recovering 2023-08-28T02:32:06.092 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:32:06.122 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:32:06.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:06.126 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:32:06.227 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:32:06.243 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:32:06.243 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [ 2023-08-28T02:32:06.243 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:06.243 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.55270.0:76134 7.e 7:7a780df1:::benchmark_data_smithi143_360738_object76133:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e279)", 2023-08-28T02:32:06.243 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:32:06.237103+0000", 2023-08-28T02:32:06.244 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.0060045109999999997, 2023-08-28T02:32:06.244 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.006043405, 2023-08-28T02:32:06.244 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:06.244 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "waiting for sub ops", 2023-08-28T02:32:06.244 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:32:06.244 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.55270", 2023-08-28T02:32:06.244 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:32:06.244 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 76134 2023-08-28T02:32:06.244 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:06.244 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:06.244 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:06.244 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:06.245 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:06.237103+0000", 2023-08-28T02:32:06.245 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:06.245 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:06.245 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:06.245 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:06.245 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:06.237103+0000", 2023-08-28T02:32:06.245 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:06.245 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:06.245 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:06.245 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:06.245 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:06.237102+0000", 2023-08-28T02:32:06.245 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:32:06.245 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:06.246 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:06.246 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:06.246 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:06.237147+0000", 2023-08-28T02:32:06.246 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.5850999999999999e-05 2023-08-28T02:32:06.246 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:06.246 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:06.246 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:06.246 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:06.237159+0000", 2023-08-28T02:32:06.246 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1993e-05 2023-08-28T02:32:06.246 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:06.246 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:06.246 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:06.246 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:06.237168+0000", 2023-08-28T02:32:06.247 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.354e-06 2023-08-28T02:32:06.247 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:06.247 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:06.247 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:06.247 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:06.237452+0000", 2023-08-28T02:32:06.247 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00028373999999999999 2023-08-28T02:32:06.247 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:06.247 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:06.247 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:32:06.247 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:06.237628+0000", 2023-08-28T02:32:06.247 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000175947 2023-08-28T02:32:06.247 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:06.247 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:06.248 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:32:06.248 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:06.237811+0000", 2023-08-28T02:32:06.248 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00018372899999999999 2023-08-28T02:32:06.248 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:06.248 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:06.248 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:06.248 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:06.248 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:06.248 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.55270.0:76140 7.e 7:7dbd465f:::benchmark_data_smithi143_360738_object76139:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e279)", 2023-08-28T02:32:06.248 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:32:06.237179+0000", 2023-08-28T02:32:06.248 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.0059287109999999997, 2023-08-28T02:32:06.248 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0060579529999999996, 2023-08-28T02:32:06.248 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:06.249 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "waiting for sub ops", 2023-08-28T02:32:06.249 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:32:06.249 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.55270", 2023-08-28T02:32:06.249 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:32:06.249 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 76140 2023-08-28T02:32:06.249 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:06.249 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:06.249 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:06.249 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:06.249 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:06.237179+0000", 2023-08-28T02:32:06.249 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:06.249 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:06.250 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:06.250 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:06.250 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:06.237179+0000", 2023-08-28T02:32:06.250 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:06.250 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:06.250 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:06.250 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:06.250 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:06.237179+0000", 2023-08-28T02:32:06.250 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:32:06.250 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:06.250 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:06.250 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:06.250 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:06.237202+0000", 2023-08-28T02:32:06.251 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.3212000000000001e-05 2023-08-28T02:32:06.251 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:06.251 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:06.251 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:06.251 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:06.237210+0000", 2023-08-28T02:32:06.251 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.0900000000000005e-06 2023-08-28T02:32:06.251 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:06.251 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:06.251 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:06.251 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:06.237216+0000", 2023-08-28T02:32:06.251 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.3029999999999996e-06 2023-08-28T02:32:06.251 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:06.251 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:06.252 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:06.252 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:06.241246+0000", 2023-08-28T02:32:06.252 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0040292100000000001 2023-08-28T02:32:06.252 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:06.252 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:06.252 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:32:06.252 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:06.241394+0000", 2023-08-28T02:32:06.252 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000148868 2023-08-28T02:32:06.252 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:06.252 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:06.252 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:32:06.252 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:06.241543+0000", 2023-08-28T02:32:06.252 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000148462 2023-08-28T02:32:06.253 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:06.253 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:06.253 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:06.253 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:06.253 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:06.253 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.55270.0:76143 7.c e279/277 7:33887b70:::benchmark_data_smithi143_360738_object76142:head v 279'4690, mlcod=0'0)", 2023-08-28T02:32:06.253 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:32:06.242833+0000", 2023-08-28T02:32:06.253 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.000275171, 2023-08-28T02:32:06.253 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00047917499999999998, 2023-08-28T02:32:06.253 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:06.253 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "started", 2023-08-28T02:32:06.253 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:06.253 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:06.254 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:06.254 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:06.242833+0000", 2023-08-28T02:32:06.254 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:06.254 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:06.254 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:06.254 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:06.254 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:06.242833+0000", 2023-08-28T02:32:06.254 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:06.254 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:06.254 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:06.254 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:06.254 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:06.242832+0000", 2023-08-28T02:32:06.254 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:32:06.255 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:06.255 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:06.255 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:06.255 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:06.242875+0000", 2023-08-28T02:32:06.255 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.2905999999999999e-05 2023-08-28T02:32:06.255 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:06.255 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:06.255 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:06.255 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:06.242887+0000", 2023-08-28T02:32:06.255 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.2869e-05 2023-08-28T02:32:06.255 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:06.255 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:06.256 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:06.256 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:06.242896+0000", 2023-08-28T02:32:06.256 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.901e-06 2023-08-28T02:32:06.256 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:06.256 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:06.256 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:06.256 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:06.242955+0000", 2023-08-28T02:32:06.256 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.8217000000000002e-05 2023-08-28T02:32:06.256 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:06.256 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:06.256 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:32:06.257 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:06.242986+0000", 2023-08-28T02:32:06.257 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.1621000000000001e-05 2023-08-28T02:32:06.258 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:06.258 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:06.258 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:06.258 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:06.258 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:06.258 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.55270.0:76132 7.e 7:779df3b6:::benchmark_data_smithi143_360738_object76131:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e279)", 2023-08-28T02:32:06.258 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:32:06.223901+0000", 2023-08-28T02:32:06.258 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.019206876000000001, 2023-08-28T02:32:06.258 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.019475401, 2023-08-28T02:32:06.258 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:06.258 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "waiting for sub ops", 2023-08-28T02:32:06.258 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:32:06.258 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.55270", 2023-08-28T02:32:06.259 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:32:06.259 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 76132 2023-08-28T02:32:06.259 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:06.259 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:06.259 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:06.259 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:06.259 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:06.223901+0000", 2023-08-28T02:32:06.259 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:06.259 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:06.259 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:06.259 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:06.259 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:06.223901+0000", 2023-08-28T02:32:06.259 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:06.260 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:06.260 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:06.260 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:06.260 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:06.223900+0000", 2023-08-28T02:32:06.260 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:32:06.260 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:06.260 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:06.260 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:06.260 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:06.223927+0000", 2023-08-28T02:32:06.260 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.6965999999999999e-05 2023-08-28T02:32:06.260 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:06.260 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:06.260 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:06.261 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:06.223943+0000", 2023-08-28T02:32:06.261 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.5933000000000001e-05 2023-08-28T02:32:06.261 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:06.261 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:06.261 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:06.261 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:06.223950+0000", 2023-08-28T02:32:06.261 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.0369999999999997e-06 2023-08-28T02:32:06.261 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:06.261 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:06.261 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:06.261 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:06.224031+0000", 2023-08-28T02:32:06.261 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.1172000000000001e-05 2023-08-28T02:32:06.261 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:06.262 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:06.262 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:32:06.262 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:06.224177+0000", 2023-08-28T02:32:06.262 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00014564499999999999 2023-08-28T02:32:06.262 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:06.262 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:06.262 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:32:06.262 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:06.224315+0000", 2023-08-28T02:32:06.262 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000138873 2023-08-28T02:32:06.262 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:06.262 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:06.262 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:06.262 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:06.227373+0000", 2023-08-28T02:32:06.263 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.003057822 2023-08-28T02:32:06.263 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:06.263 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:06.263 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:32:06.263 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:06.236142+0000", 2023-08-28T02:32:06.263 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0087683139999999993 2023-08-28T02:32:06.263 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:06.263 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:06.263 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:06.263 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:06.263 INFO:teuthology.orchestra.run.smithi143.stdout: ], 2023-08-28T02:32:06.263 INFO:teuthology.orchestra.run.smithi143.stdout: "num_ops": 4 2023-08-28T02:32:06.263 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:32:06.264 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:06.307 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:32:06.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:06.312 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:32:06.413 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:32:06.470 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:06.849 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 65 16 76661 76645 73.6488 49.9375 0.00702565 0.0135739 2023-08-28T02:32:07.135 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:32:07.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:07.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:32:07.243 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:32:07.315 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:07.716 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set noscrub 2023-08-28T02:32:07.853 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 66 16 77291 77275 73.1263 39.375 0.0334753 0.0136692 2023-08-28T02:32:08.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:08 smithi143 ceph-mon[183059]: pgmap v689: 99 pgs: 14 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 83 active+clean; 4.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 38 MiB/s wr, 615 op/s; 21446/226014 objects degraded (9.489%); 3316/226014 objects misplaced (1.467%); 6.2 MiB/s, 100 objects/s recovering 2023-08-28T02:32:08.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:08 smithi143 ceph-mon[189871]: pgmap v689: 99 pgs: 14 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 83 active+clean; 4.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 38 MiB/s wr, 615 op/s; 21446/226014 objects degraded (9.489%); 3316/226014 objects misplaced (1.467%); 6.2 MiB/s, 100 objects/s recovering 2023-08-28T02:32:08.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:08 smithi169 ceph-mon[159415]: pgmap v689: 99 pgs: 14 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 83 active+clean; 4.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 38 MiB/s wr, 615 op/s; 21446/226014 objects degraded (9.489%); 3316/226014 objects misplaced (1.467%); 6.2 MiB/s, 100 objects/s recovering 2023-08-28T02:32:08.854 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 67 16 78053 78037 72.7458 47.625 0.0110231 0.0137411 2023-08-28T02:32:08.983 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.2 dump_blocked_ops 2023-08-28T02:32:09.378 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:32:09.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:09.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:32:09.481 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:32:09.555 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:09.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:09 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 21446/228759 objects degraded (9.375%), 14 pgs degraded (PG_DEGRADED) 2023-08-28T02:32:09.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:09 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 21446/228759 objects degraded (9.375%), 14 pgs degraded (PG_DEGRADED) 2023-08-28T02:32:09.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:09 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 21446/228759 objects degraded (9.375%), 14 pgs degraded (PG_DEGRADED) 2023-08-28T02:32:09.812 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:32:09.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:09.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:32:09.853 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 68 16 78826 78810 72.3867 48.3125 0.00356403 0.0138092 2023-08-28T02:32:09.922 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:32:10.000 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:10.145 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:32:10.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:10.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:32:10.252 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:32:10.334 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:10.334 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:32:10.390 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:32:10.485 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:32:10.491 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:32:10.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:10.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:32:10.538 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:32:10.597 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:32:10.598 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:32:10.625 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:10 smithi169 ceph-mon[159415]: pgmap v690: 99 pgs: 14 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 83 active+clean; 4.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 38 MiB/s wr, 612 op/s; 21446/228759 objects degraded (9.375%); 3316/228759 objects misplaced (1.450%); 5.8 MiB/s, 92 objects/s recovering 2023-08-28T02:32:10.625 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:10 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2350574836' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:32:10.625 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:10.645 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:32:10.674 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:32:10.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:10.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:32:10.684 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:32:10.722 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:32:10.764 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:32:10.781 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:32:10.794 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:32:10.814 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:10.832 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:32:10.833 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:32:10.833 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:32:10.833 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:32:10.833 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:32:10.833 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:32:10.833 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:32:10.833 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:32:10.853 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 69 16 79512 79496 71.959 42.875 0.00931655 0.0138913 2023-08-28T02:32:10.861 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:32:10.861 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:10 smithi143 ceph-mon[183059]: pgmap v690: 99 pgs: 14 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 83 active+clean; 4.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 38 MiB/s wr, 612 op/s; 21446/228759 objects degraded (9.375%); 3316/228759 objects misplaced (1.450%); 5.8 MiB/s, 92 objects/s recovering 2023-08-28T02:32:10.862 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:10 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2350574836' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:32:10.862 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:10 smithi143 ceph-mon[189871]: pgmap v690: 99 pgs: 14 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 83 active+clean; 4.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 38 MiB/s wr, 612 op/s; 21446/228759 objects degraded (9.375%); 3316/228759 objects misplaced (1.450%); 5.8 MiB/s, 92 objects/s recovering 2023-08-28T02:32:10.862 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:10 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2350574836' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:32:10.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:10.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:32:10.967 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:32:11.101 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:11.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:11 smithi143 conmon[183035]: 2023-08-28T02:32:11.027+0000 7fadc3887700 -1 mon.a@0(leader).osd e279 definitely_dead 0 2023-08-28T02:32:11.576 INFO:teuthology.orchestra.run.smithi143.stderr:marked out osd.1. 2023-08-28T02:32:11.854 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 70 16 80093 80077 71.45 36.3125 0.00529722 0.0139368 2023-08-28T02:32:11.927 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:11 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3264180744' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-08-28T02:32:11.927 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:11 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-08-28T02:32:11.927 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:11 smithi143 ceph-mon[183059]: Client client.admin marked osd.1 out, while it was still marked up 2023-08-28T02:32:11.927 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:11 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:32:11.928 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:11 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:32:11.928 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:11 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3264180744' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-08-28T02:32:11.928 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:11 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-08-28T02:32:11.928 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:11 smithi143 ceph-mon[189871]: Client client.admin marked osd.1 out, while it was still marked up 2023-08-28T02:32:11.928 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:11 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:32:11.928 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:11 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:32:11.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:11 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3264180744' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-08-28T02:32:11.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:11 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-08-28T02:32:11.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:11 smithi169 ceph-mon[159415]: Client client.admin marked osd.1 out, while it was still marked up 2023-08-28T02:32:11.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:11 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:32:11.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:11 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:32:12.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:32:11 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:32:11] "GET /metrics HTTP/1.1" 200 36581 "" "Prometheus/2.33.4" 2023-08-28T02:32:12.854 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 71 16 80099 80083 70.4492 0.375 0.00335022 0.0139773 2023-08-28T02:32:12.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:12 smithi143 ceph-mon[189871]: pgmap v691: 99 pgs: 14 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 83 active+clean; 4.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 45 MiB/s wr, 719 op/s; 21446/233835 objects degraded (9.171%); 2814/233835 objects misplaced (1.203%); 5.4 MiB/s, 87 objects/s recovering 2023-08-28T02:32:12.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:12 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-08-28T02:32:12.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:12 smithi143 ceph-mon[189871]: osdmap e280: 8 total, 8 up, 6 in 2023-08-28T02:32:12.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:12 smithi143 ceph-mon[189871]: failed to encode map e280 with expected crc 2023-08-28T02:32:12.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:12 smithi143 ceph-mon[189871]: failed to encode map e280 with expected crc 2023-08-28T02:32:12.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:12 smithi143 ceph-mon[189871]: failed to encode map e280 with expected crc 2023-08-28T02:32:12.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:12 smithi143 ceph-mon[183059]: pgmap v691: 99 pgs: 14 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 83 active+clean; 4.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 45 MiB/s wr, 719 op/s; 21446/233835 objects degraded (9.171%); 2814/233835 objects misplaced (1.203%); 5.4 MiB/s, 87 objects/s recovering 2023-08-28T02:32:12.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:12 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-08-28T02:32:12.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:12 smithi143 ceph-mon[183059]: osdmap e280: 8 total, 8 up, 6 in 2023-08-28T02:32:12.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:12 smithi143 ceph-mon[183059]: failed to encode map e280 with expected crc 2023-08-28T02:32:12.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:12 smithi143 ceph-mon[183059]: failed to encode map e280 with expected crc 2023-08-28T02:32:12.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:12 smithi143 ceph-mon[183059]: failed to encode map e280 with expected crc 2023-08-28T02:32:12.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:12 smithi169 ceph-mon[159415]: pgmap v691: 99 pgs: 14 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 83 active+clean; 4.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 45 MiB/s wr, 719 op/s; 21446/233835 objects degraded (9.171%); 2814/233835 objects misplaced (1.203%); 5.4 MiB/s, 87 objects/s recovering 2023-08-28T02:32:12.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:12 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-08-28T02:32:12.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:12 smithi169 ceph-mon[159415]: osdmap e280: 8 total, 8 up, 6 in 2023-08-28T02:32:12.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:12 smithi169 ceph-mon[159415]: failed to encode map e280 with expected crc 2023-08-28T02:32:12.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:12 smithi169 ceph-mon[159415]: failed to encode map e280 with expected crc 2023-08-28T02:32:12.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:12 smithi169 ceph-mon[159415]: failed to encode map e280 with expected crc 2023-08-28T02:32:13.577 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:32:13.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:13.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:32:13.685 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:32:13.761 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:13.854 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 72 16 80182 80166 69.5433 5.1875 0.008615 0.0143733 2023-08-28T02:32:13.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:13 smithi143 ceph-mon[183059]: failed to encode map e281 with expected crc 2023-08-28T02:32:13.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:13 smithi143 ceph-mon[183059]: osdmap e281: 8 total, 8 up, 6 in 2023-08-28T02:32:13.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:13 smithi143 ceph-mon[183059]: failed to encode map e281 with expected crc 2023-08-28T02:32:13.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:13 smithi143 ceph-mon[183059]: failed to encode map e281 with expected crc 2023-08-28T02:32:13.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:13 smithi143 ceph-mon[183059]: failed to encode map e281 with expected crc 2023-08-28T02:32:13.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:13 smithi143 ceph-mon[183059]: failed to encode map e281 with expected crc 2023-08-28T02:32:13.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:13 smithi143 ceph-mon[183059]: failed to encode map e281 with expected crc 2023-08-28T02:32:13.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:13 smithi143 ceph-mon[183059]: failed to encode map e281 with expected crc 2023-08-28T02:32:13.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:13 smithi143 ceph-mon[183059]: failed to encode map e281 with expected crc 2023-08-28T02:32:13.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:13 smithi143 ceph-mon[189871]: failed to encode map e281 with expected crc 2023-08-28T02:32:13.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:13 smithi143 ceph-mon[189871]: osdmap e281: 8 total, 8 up, 6 in 2023-08-28T02:32:13.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:13 smithi143 ceph-mon[189871]: failed to encode map e281 with expected crc 2023-08-28T02:32:13.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:13 smithi143 ceph-mon[189871]: failed to encode map e281 with expected crc 2023-08-28T02:32:13.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:13 smithi143 ceph-mon[189871]: failed to encode map e281 with expected crc 2023-08-28T02:32:13.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:13 smithi143 ceph-mon[189871]: failed to encode map e281 with expected crc 2023-08-28T02:32:13.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:13 smithi143 ceph-mon[189871]: failed to encode map e281 with expected crc 2023-08-28T02:32:13.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:13 smithi143 ceph-mon[189871]: failed to encode map e281 with expected crc 2023-08-28T02:32:13.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:13 smithi143 ceph-mon[189871]: failed to encode map e281 with expected crc 2023-08-28T02:32:13.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:13 smithi169 ceph-mon[159415]: failed to encode map e281 with expected crc 2023-08-28T02:32:13.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:13 smithi169 ceph-mon[159415]: osdmap e281: 8 total, 8 up, 6 in 2023-08-28T02:32:13.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:13 smithi169 ceph-mon[159415]: failed to encode map e281 with expected crc 2023-08-28T02:32:13.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:13 smithi169 ceph-mon[159415]: failed to encode map e281 with expected crc 2023-08-28T02:32:13.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:13 smithi169 ceph-mon[159415]: failed to encode map e281 with expected crc 2023-08-28T02:32:13.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:13 smithi169 ceph-mon[159415]: failed to encode map e281 with expected crc 2023-08-28T02:32:13.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:13 smithi169 ceph-mon[159415]: failed to encode map e281 with expected crc 2023-08-28T02:32:13.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:13 smithi169 ceph-mon[159415]: failed to encode map e281 with expected crc 2023-08-28T02:32:13.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:13 smithi169 ceph-mon[159415]: failed to encode map e281 with expected crc 2023-08-28T02:32:14.854 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 73 16 80860 80844 69.1712 42.375 0.0153849 0.0144505 2023-08-28T02:32:14.955 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:14 smithi143 ceph-mon[183059]: pgmap v694: 99 pgs: 1 remapped+peering, 21 peering, 13 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 62 active+clean; 4.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 36 MiB/s wr, 578 op/s; 19269/236802 objects degraded (8.137%); 2814/236802 objects misplaced (1.188%); 3.9 MiB/s, 59 objects/s recovering 2023-08-28T02:32:14.956 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:14 smithi143 ceph-mon[183059]: Health check failed: Reduced data availability: 5 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-08-28T02:32:14.956 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:14 smithi143 ceph-mon[183059]: failed to encode map e282 with expected crc 2023-08-28T02:32:14.956 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:14 smithi143 ceph-mon[183059]: failed to encode map e282 with expected crc 2023-08-28T02:32:14.956 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:14 smithi143 ceph-mon[183059]: osdmap e282: 8 total, 8 up, 6 in 2023-08-28T02:32:14.956 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:14 smithi143 ceph-mon[183059]: failed to encode map e282 with expected crc 2023-08-28T02:32:14.956 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:14 smithi143 ceph-mon[183059]: failed to encode map e282 with expected crc 2023-08-28T02:32:14.956 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:14 smithi143 ceph-mon[183059]: failed to encode map e282 with expected crc 2023-08-28T02:32:14.956 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:14 smithi143 ceph-mon[189871]: pgmap v694: 99 pgs: 1 remapped+peering, 21 peering, 13 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 62 active+clean; 4.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 36 MiB/s wr, 578 op/s; 19269/236802 objects degraded (8.137%); 2814/236802 objects misplaced (1.188%); 3.9 MiB/s, 59 objects/s recovering 2023-08-28T02:32:14.956 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:14 smithi143 ceph-mon[189871]: Health check failed: Reduced data availability: 5 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-08-28T02:32:14.956 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:14 smithi143 ceph-mon[189871]: failed to encode map e282 with expected crc 2023-08-28T02:32:14.957 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:14 smithi143 ceph-mon[189871]: failed to encode map e282 with expected crc 2023-08-28T02:32:14.957 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:14 smithi143 ceph-mon[189871]: osdmap e282: 8 total, 8 up, 6 in 2023-08-28T02:32:14.957 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:14 smithi143 ceph-mon[189871]: failed to encode map e282 with expected crc 2023-08-28T02:32:14.957 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:14 smithi143 ceph-mon[189871]: failed to encode map e282 with expected crc 2023-08-28T02:32:14.957 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:14 smithi143 ceph-mon[189871]: failed to encode map e282 with expected crc 2023-08-28T02:32:14.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:14 smithi169 ceph-mon[159415]: pgmap v694: 99 pgs: 1 remapped+peering, 21 peering, 13 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 62 active+clean; 4.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 36 MiB/s wr, 578 op/s; 19269/236802 objects degraded (8.137%); 2814/236802 objects misplaced (1.188%); 3.9 MiB/s, 59 objects/s recovering 2023-08-28T02:32:14.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:14 smithi169 ceph-mon[159415]: Health check failed: Reduced data availability: 5 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-08-28T02:32:14.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:14 smithi169 ceph-mon[159415]: failed to encode map e282 with expected crc 2023-08-28T02:32:14.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:14 smithi169 ceph-mon[159415]: failed to encode map e282 with expected crc 2023-08-28T02:32:14.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:14 smithi169 ceph-mon[159415]: osdmap e282: 8 total, 8 up, 6 in 2023-08-28T02:32:14.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:14 smithi169 ceph-mon[159415]: failed to encode map e282 with expected crc 2023-08-28T02:32:14.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:14 smithi169 ceph-mon[159415]: failed to encode map e282 with expected crc 2023-08-28T02:32:14.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:14 smithi169 ceph-mon[159415]: failed to encode map e282 with expected crc 2023-08-28T02:32:15.001 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:32:15.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:15.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:32:15.107 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:32:15.139 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:15.212 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:32:15.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:15.217 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:32:15.318 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:32:15.350 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:15.414 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:32:15.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:15.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:32:15.528 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:32:15.618 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:15 smithi143 ceph-mon[189871]: osdmap e283: 8 total, 8 up, 6 in 2023-08-28T02:32:15.618 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:15 smithi143 ceph-mon[189871]: failed to encode map e283 with expected crc 2023-08-28T02:32:15.619 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:15 smithi143 ceph-mon[189871]: failed to encode map e283 with expected crc 2023-08-28T02:32:15.619 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:15 smithi143 ceph-mon[189871]: failed to encode map e283 with expected crc 2023-08-28T02:32:15.619 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:15 smithi143 ceph-mon[189871]: failed to encode map e283 with expected crc 2023-08-28T02:32:15.619 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:15 smithi143 ceph-mon[189871]: 7.1 scrub starts 2023-08-28T02:32:15.620 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:15 smithi143 ceph-mon[183059]: osdmap e283: 8 total, 8 up, 6 in 2023-08-28T02:32:15.620 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:15 smithi143 ceph-mon[183059]: failed to encode map e283 with expected crc 2023-08-28T02:32:15.620 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:15 smithi143 ceph-mon[183059]: failed to encode map e283 with expected crc 2023-08-28T02:32:15.620 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:15 smithi143 ceph-mon[183059]: failed to encode map e283 with expected crc 2023-08-28T02:32:15.620 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:15 smithi143 ceph-mon[183059]: failed to encode map e283 with expected crc 2023-08-28T02:32:15.620 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:15 smithi143 ceph-mon[183059]: 7.1 scrub starts 2023-08-28T02:32:15.624 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:15.684 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:32:15.684 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:32:15.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:15 smithi169 ceph-mon[159415]: osdmap e283: 8 total, 8 up, 6 in 2023-08-28T02:32:15.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:15 smithi169 ceph-mon[159415]: failed to encode map e283 with expected crc 2023-08-28T02:32:15.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:15 smithi169 ceph-mon[159415]: failed to encode map e283 with expected crc 2023-08-28T02:32:15.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:15 smithi169 ceph-mon[159415]: failed to encode map e283 with expected crc 2023-08-28T02:32:15.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:15 smithi169 ceph-mon[159415]: failed to encode map e283 with expected crc 2023-08-28T02:32:15.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:15 smithi169 ceph-mon[159415]: 7.1 scrub starts 2023-08-28T02:32:15.767 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:32:15.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:15.780 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:32:15.854 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 74 16 81605 81589 68.8658 46.5625 0.00756749 0.0145146 2023-08-28T02:32:15.880 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:32:15.962 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:16.045 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:32:16.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:16.058 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:32:16.159 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:32:16.195 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:16.266 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:32:16.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:16.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:32:16.374 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:32:16.417 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:16.484 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:32:16.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:16.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:32:16.589 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:32:16.592 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is set 2023-08-28T02:32:16.618 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:16.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:16 smithi169 ceph-mon[159415]: pgmap v697: 99 pgs: 1 remapped+peering, 21 peering, 20 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 56 active+clean; 4.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 38 MiB/s wr, 600 op/s; 37983/241050 objects degraded (15.757%); 1909/241050 objects misplaced (0.792%); 7.9 MiB/s, 134 objects/s recovering 2023-08-28T02:32:16.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:16 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 37983/241050 objects degraded (15.757%), 20 pgs degraded (PG_DEGRADED) 2023-08-28T02:32:16.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:16 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/951772910' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:32:16.739 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:32:16.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:16.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:32:16.851 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:32:16.859 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 75 16 82346 82330 68.5603 46.3125 0.0327625 0.0145807 2023-08-28T02:32:16.909 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:16 smithi143 ceph-mon[183059]: pgmap v697: 99 pgs: 1 remapped+peering, 21 peering, 20 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 56 active+clean; 4.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 38 MiB/s wr, 600 op/s; 37983/241050 objects degraded (15.757%); 1909/241050 objects misplaced (0.792%); 7.9 MiB/s, 134 objects/s recovering 2023-08-28T02:32:16.909 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:16 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 37983/241050 objects degraded (15.757%), 20 pgs degraded (PG_DEGRADED) 2023-08-28T02:32:16.909 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:16 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/951772910' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:32:16.910 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:16 smithi143 ceph-mon[189871]: pgmap v697: 99 pgs: 1 remapped+peering, 21 peering, 20 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 56 active+clean; 4.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 38 MiB/s wr, 600 op/s; 37983/241050 objects degraded (15.757%); 1909/241050 objects misplaced (0.792%); 7.9 MiB/s, 134 objects/s recovering 2023-08-28T02:32:16.910 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:16 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 37983/241050 objects degraded (15.757%), 20 pgs degraded (PG_DEGRADED) 2023-08-28T02:32:16.911 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:16 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/951772910' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:32:16.913 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:16.999 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:32:17.175 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:32:17.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:17.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:32:17.283 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:32:17.319 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:17.397 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:32:17.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:17.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:32:17.503 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:32:17.532 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:17.567 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:17 smithi169 ceph-mon[159415]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:32:17.568 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:17 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/951772910' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:32:17.568 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:17 smithi169 ceph-mon[159415]: osdmap e284: 8 total, 8 up, 6 in 2023-08-28T02:32:17.568 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:17 smithi169 ceph-mon[159415]: failed to encode map e284 with expected crc 2023-08-28T02:32:17.568 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:17 smithi169 ceph-mon[159415]: failed to encode map e284 with expected crc 2023-08-28T02:32:17.568 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:17 smithi169 ceph-mon[159415]: failed to encode map e284 with expected crc 2023-08-28T02:32:17.568 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:17 smithi169 ceph-mon[159415]: failed to encode map e284 with expected crc 2023-08-28T02:32:17.568 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:17 smithi169 ceph-mon[159415]: pgmap v699: 99 pgs: 25 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 73 active+clean; 5.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 29 MiB/s wr, 463 op/s; 52898/243537 objects degraded (21.721%); 1909/243537 objects misplaced (0.784%); 6.6 MiB/s, 118 objects/s recovering 2023-08-28T02:32:17.577 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:32:17.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:17.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:32:17.682 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:32:17.726 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:17.780 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.2 dump_historic_ops 2023-08-28T02:32:17.789 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:32:17.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:17.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:32:17.841 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:17 smithi143 ceph-mon[183059]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:32:17.841 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:17 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/951772910' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:32:17.841 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:17 smithi143 ceph-mon[183059]: osdmap e284: 8 total, 8 up, 6 in 2023-08-28T02:32:17.841 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:17 smithi143 ceph-mon[183059]: failed to encode map e284 with expected crc 2023-08-28T02:32:17.841 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:17 smithi143 ceph-mon[183059]: failed to encode map e284 with expected crc 2023-08-28T02:32:17.842 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:17 smithi143 ceph-mon[183059]: failed to encode map e284 with expected crc 2023-08-28T02:32:17.842 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:17 smithi143 ceph-mon[183059]: failed to encode map e284 with expected crc 2023-08-28T02:32:17.842 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:17 smithi143 ceph-mon[183059]: pgmap v699: 99 pgs: 25 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 73 active+clean; 5.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 29 MiB/s wr, 463 op/s; 52898/243537 objects degraded (21.721%); 1909/243537 objects misplaced (0.784%); 6.6 MiB/s, 118 objects/s recovering 2023-08-28T02:32:17.845 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:17 smithi143 ceph-mon[189871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:32:17.845 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:17 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/951772910' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:32:17.846 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:17 smithi143 ceph-mon[189871]: osdmap e284: 8 total, 8 up, 6 in 2023-08-28T02:32:17.846 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:17 smithi143 ceph-mon[189871]: failed to encode map e284 with expected crc 2023-08-28T02:32:17.846 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:17 smithi143 ceph-mon[189871]: failed to encode map e284 with expected crc 2023-08-28T02:32:17.846 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:17 smithi143 ceph-mon[189871]: failed to encode map e284 with expected crc 2023-08-28T02:32:17.846 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:17 smithi143 ceph-mon[189871]: failed to encode map e284 with expected crc 2023-08-28T02:32:17.846 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:17 smithi143 ceph-mon[189871]: pgmap v699: 99 pgs: 25 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 73 active+clean; 5.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 29 MiB/s wr, 463 op/s; 52898/243537 objects degraded (21.721%); 1909/243537 objects misplaced (0.784%); 6.6 MiB/s, 118 objects/s recovering 2023-08-28T02:32:17.861 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 76 15 83054 83039 68.2414 44.3125 0.0255713 0.0146486 2023-08-28T02:32:17.895 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:32:17.944 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:18.004 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:32:18.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:18.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:32:18.114 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:32:18.210 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:18.274 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:32:18.330 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:32:18.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:18.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:32:18.440 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:32:18.479 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:18.551 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:32:18.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:18.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:32:18.657 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:32:18.691 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:18.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:18 smithi169 ceph-mon[159415]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 5 pgs peering) 2023-08-28T02:32:18.753 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:32:18.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:18.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:32:18.835 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:32:18.860 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 77 16 83810 83794 67.9681 47.1875 0.012621 0.0147061 2023-08-28T02:32:18.864 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:32:18.944 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:18 smithi143 ceph-mon[189871]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 5 pgs peering) 2023-08-28T02:32:18.945 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:18 smithi143 ceph-mon[183059]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 5 pgs peering) 2023-08-28T02:32:18.951 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:19.054 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:32:19.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:19.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:32:19.162 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:32:19.237 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:19.863 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 78 16 84485 84469 67.6345 42.1875 0.0298443 0.0147794 2023-08-28T02:32:19.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:19 smithi143 ceph-mon[183059]: pgmap v700: 99 pgs: 1 active+recovering+undersized+remapped, 25 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 33 MiB/s wr, 525 op/s; 52898/246273 objects degraded (21.479%); 1554/246273 objects misplaced (0.631%); 9.0 MiB/s, 154 objects/s recovering 2023-08-28T02:32:19.951 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:32:19.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:19 smithi143 ceph-mon[189871]: pgmap v700: 99 pgs: 1 active+recovering+undersized+remapped, 25 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 33 MiB/s wr, 525 op/s; 52898/246273 objects degraded (21.479%); 1554/246273 objects misplaced (0.631%); 9.0 MiB/s, 154 objects/s recovering 2023-08-28T02:32:19.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:19.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:32:19.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:19 smithi169 ceph-mon[159415]: pgmap v700: 99 pgs: 1 active+recovering+undersized+remapped, 25 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 33 MiB/s wr, 525 op/s; 52898/246273 objects degraded (21.479%); 1554/246273 objects misplaced (0.631%); 9.0 MiB/s, 154 objects/s recovering 2023-08-28T02:32:20.059 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:32:20.100 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 6] out_osds: [7, 1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-08-28T02:32:20.100 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-08-28T02:32:20.101 INFO:tasks.thrashosds.thrasher:Growing pool device_health_metrics 2023-08-28T02:32:20.101 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph pg dump --format=json 2023-08-28T02:32:20.102 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set nodeep-scrub 2023-08-28T02:32:20.124 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:20.305 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:32:20.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:20.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:32:20.416 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:32:20.450 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:20.510 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:32:20.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:20.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:32:20.617 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:32:20.721 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:20.846 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:32:20.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:20.852 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:32:20.863 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:32:20.863 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 79 15 85130 85115 67.2896 40.375 0.0436799 0.0148551 2023-08-28T02:32:20.907 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:32:20.942 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:32:20.953 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:32:20.996 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:32:21.024 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:21.032 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:32:21.066 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:32:21.123 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:32:21.153 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:20 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1497100445' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:32:21.153 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:20 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1497100445' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:32:21.164 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:32:21.169 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:32:21.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:21.178 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:32:21.191 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:32:21.191 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:32:21.191 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:32:21.191 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:32:21.191 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:32:21.191 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:32:21.192 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:32:21.192 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:32:21.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:20 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1497100445' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:32:21.280 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:32:21.356 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:21.473 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:32:21.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:21.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:32:21.584 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:32:21.677 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:21.864 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:2023-08-28T02:32:21.865209+0000 min lat: 0.00114956 max lat: 2.65098 avg lat: 0.014903 2023-08-28T02:32:21.864 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-08-28T02:32:21.864 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 80 16 85938 85922 67.079 50.4375 0.0242375 0.014903 2023-08-28T02:32:22.037 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:32:22.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:22.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:32:22.146 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:32:22.169 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:21 smithi169 ceph-mon[159415]: pgmap v701: 99 pgs: 1 active+recovering+undersized+remapped, 25 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 37 MiB/s wr, 592 op/s; 52898/250740 objects degraded (21.097%); 1554/250740 objects misplaced (0.620%); 6.9 MiB/s, 118 objects/s recovering 2023-08-28T02:32:22.169 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:21 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 52898/246273 objects degraded (21.479%), 25 pgs degraded (PG_DEGRADED) 2023-08-28T02:32:22.171 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:22.202 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:32:21 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:32:21] "GET /metrics HTTP/1.1" 200 36581 "" "Prometheus/2.33.4" 2023-08-28T02:32:22.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:21 smithi143 ceph-mon[189871]: pgmap v701: 99 pgs: 1 active+recovering+undersized+remapped, 25 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 37 MiB/s wr, 592 op/s; 52898/250740 objects degraded (21.097%); 1554/250740 objects misplaced (0.620%); 6.9 MiB/s, 118 objects/s recovering 2023-08-28T02:32:22.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:21 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 52898/246273 objects degraded (21.479%), 25 pgs degraded (PG_DEGRADED) 2023-08-28T02:32:22.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:21 smithi143 ceph-mon[183059]: pgmap v701: 99 pgs: 1 active+recovering+undersized+remapped, 25 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 37 MiB/s wr, 592 op/s; 52898/250740 objects degraded (21.097%); 1554/250740 objects misplaced (0.620%); 6.9 MiB/s, 118 objects/s recovering 2023-08-28T02:32:22.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:21 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 52898/246273 objects degraded (21.479%), 25 pgs degraded (PG_DEGRADED) 2023-08-28T02:32:22.240 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:32:22.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:22.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:32:22.347 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:32:22.385 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:22.452 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:32:22.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:22.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:32:22.561 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:32:22.594 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:22.653 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:32:22.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:22.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:32:22.759 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:32:22.848 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:22.864 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 81 16 86694 86678 66.8337 47.25 0.0370963 0.0149569 2023-08-28T02:32:23.008 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:32:23.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:23.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:32:23.025 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:32:23.030 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:32:23.119 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:32:23.140 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:32:23.140 INFO:teuthology.orchestra.run.smithi143.stdout: "size": 20, 2023-08-28T02:32:23.140 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 600, 2023-08-28T02:32:23.140 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [ 2023-08-28T02:32:23.140 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.140 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:123503 5.a 5:555b8910:::benchmark_data_smithi143_236200_object13360:head [delete] snapc 0=[] ondisk+write+known_if_redirected e153)", 2023-08-28T02:32:23.141 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:58.222782+0000", 2023-08-28T02:32:23.141 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 504.91114520999997, 2023-08-28T02:32:23.141 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.2403772000000002, 2023-08-28T02:32:23.141 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:23.141 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:32:23.141 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:32:23.141 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:32:23.141 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:32:23.141 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 123503 2023-08-28T02:32:23.141 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.141 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:23.141 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.142 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:23.142 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.222782+0000", 2023-08-28T02:32:23.142 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:23.142 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.142 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.142 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:23.142 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.222782+0000", 2023-08-28T02:32:23.142 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:23.142 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.142 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.142 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:23.142 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.222781+0000", 2023-08-28T02:32:23.143 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:32:23.143 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.143 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.143 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:23.143 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.222787+0000", 2023-08-28T02:32:23.143 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.7150000000000003e-06 2023-08-28T02:32:23.143 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.143 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.143 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:23.143 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.222797+0000", 2023-08-28T02:32:23.143 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0451999999999999e-05 2023-08-28T02:32:23.143 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.144 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.144 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:23.144 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.222803+0000", 2023-08-28T02:32:23.144 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.4310000000000002e-06 2023-08-28T02:32:23.144 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.144 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.144 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.144 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.222849+0000", 2023-08-28T02:32:23.144 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.6094999999999998e-05 2023-08-28T02:32:23.144 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.144 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.145 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:23.145 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.222858+0000", 2023-08-28T02:32:23.145 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.5659999999999996e-06 2023-08-28T02:32:23.145 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.145 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.145 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.145 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.200084+0000", 2023-08-28T02:32:23.145 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.97722584499999998 2023-08-28T02:32:23.145 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.145 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.145 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:23.146 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.200097+0000", 2023-08-28T02:32:23.146 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3039e-05 2023-08-28T02:32:23.146 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.146 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.146 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.146 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.442271+0000", 2023-08-28T02:32:23.146 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.242173792 2023-08-28T02:32:23.146 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.146 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.146 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:32:23.146 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.442456+0000", 2023-08-28T02:32:23.146 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00018502300000000001 2023-08-28T02:32:23.147 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.147 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.147 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 3,4", 2023-08-28T02:32:23.147 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.442620+0000", 2023-08-28T02:32:23.147 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00016423800000000001 2023-08-28T02:32:23.147 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.147 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.147 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:23.147 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.452029+0000", 2023-08-28T02:32:23.147 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0094086980000000001 2023-08-28T02:32:23.147 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.147 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.148 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:23.148 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.452057+0000", 2023-08-28T02:32:23.148 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.8232e-05 2023-08-28T02:32:23.148 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.148 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.148 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:32:23.148 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.458060+0000", 2023-08-28T02:32:23.148 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0060031529999999998 2023-08-28T02:32:23.148 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.148 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.148 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:32:23.148 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.458115+0000", 2023-08-28T02:32:23.149 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.5056999999999998e-05 2023-08-28T02:32:23.149 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.149 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.149 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:32:23.149 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.463159+0000", 2023-08-28T02:32:23.149 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0050438940000000002 2023-08-28T02:32:23.149 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:23.149 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:23.149 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:23.149 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.150 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.150 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:123513 5.a 5:5ff717e5:::benchmark_data_smithi143_236200_object13370:head [delete] snapc 0=[] ondisk+write+known_if_redirected e153)", 2023-08-28T02:32:23.150 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:58.297142+0000", 2023-08-28T02:32:23.150 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 504.836785378, 2023-08-28T02:32:23.150 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.2035575340000002, 2023-08-28T02:32:23.150 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:23.150 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:32:23.150 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:32:23.150 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:32:23.150 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:32:23.151 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 123513 2023-08-28T02:32:23.151 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.151 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:23.151 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.151 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:23.151 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.297142+0000", 2023-08-28T02:32:23.151 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:23.151 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.151 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.151 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:23.151 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.297142+0000", 2023-08-28T02:32:23.151 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:23.151 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.152 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.152 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:23.152 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.297140+0000", 2023-08-28T02:32:23.152 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:32:23.152 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.152 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.152 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:23.152 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.297151+0000", 2023-08-28T02:32:23.152 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0607e-05 2023-08-28T02:32:23.152 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.152 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.152 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:23.153 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.297168+0000", 2023-08-28T02:32:23.153 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.7388e-05 2023-08-28T02:32:23.153 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.153 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.153 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:23.153 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.297178+0000", 2023-08-28T02:32:23.153 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.1970000000000002e-06 2023-08-28T02:32:23.153 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.153 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.153 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.153 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.297251+0000", 2023-08-28T02:32:23.153 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.3744000000000004e-05 2023-08-28T02:32:23.154 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.154 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.154 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:23.154 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.297275+0000", 2023-08-28T02:32:23.154 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.3640999999999999e-05 2023-08-28T02:32:23.154 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.154 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.154 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.154 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.200183+0000", 2023-08-28T02:32:23.154 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.90290833199999998 2023-08-28T02:32:23.154 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.154 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.154 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:23.155 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.200188+0000", 2023-08-28T02:32:23.155 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.0710000000000002e-06 2023-08-28T02:32:23.155 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.155 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.155 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.155 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.452078+0000", 2023-08-28T02:32:23.155 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.2518892269999999 2023-08-28T02:32:23.155 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.155 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.155 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:32:23.155 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.452234+0000", 2023-08-28T02:32:23.155 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00015632499999999999 2023-08-28T02:32:23.155 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.156 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.156 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 3,4", 2023-08-28T02:32:23.156 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.452396+0000", 2023-08-28T02:32:23.156 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00016166299999999999 2023-08-28T02:32:23.156 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.156 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.156 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:23.156 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.467375+0000", 2023-08-28T02:32:23.156 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.014979276 2023-08-28T02:32:23.156 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.156 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.156 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:23.156 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.467419+0000", 2023-08-28T02:32:23.157 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.4060999999999997e-05 2023-08-28T02:32:23.157 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.157 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.157 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:32:23.157 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.495946+0000", 2023-08-28T02:32:23.157 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.028527211 2023-08-28T02:32:23.157 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.157 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.157 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:32:23.157 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.495999+0000", 2023-08-28T02:32:23.157 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.2587999999999999e-05 2023-08-28T02:32:23.157 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.158 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.158 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:32:23.158 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.500700+0000", 2023-08-28T02:32:23.158 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0047008629999999996 2023-08-28T02:32:23.158 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:23.158 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:23.158 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:23.158 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.158 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.158 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:123523 5.a 5:56155c30:::benchmark_data_smithi143_236200_object13380:head [delete] snapc 0=[] ondisk+write+known_if_redirected e153)", 2023-08-28T02:32:23.158 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:58.306350+0000", 2023-08-28T02:32:23.158 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 504.82757782700003, 2023-08-28T02:32:23.158 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.1987806069999998, 2023-08-28T02:32:23.159 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:23.159 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:32:23.159 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:32:23.159 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:32:23.159 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:32:23.159 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 123523 2023-08-28T02:32:23.159 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.159 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:23.159 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.159 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:23.159 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.306350+0000", 2023-08-28T02:32:23.159 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:23.159 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.160 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.160 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:23.160 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.306350+0000", 2023-08-28T02:32:23.160 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:23.160 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.160 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.160 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:23.160 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.306348+0000", 2023-08-28T02:32:23.160 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:32:23.160 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.160 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.160 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:23.160 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.306353+0000", 2023-08-28T02:32:23.161 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.2650000000000004e-06 2023-08-28T02:32:23.161 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.161 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.161 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:23.161 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.306364+0000", 2023-08-28T02:32:23.161 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0898e-05 2023-08-28T02:32:23.161 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.161 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.161 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:23.161 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.306369+0000", 2023-08-28T02:32:23.161 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.0390000000000004e-06 2023-08-28T02:32:23.161 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.161 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.162 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.162 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.306418+0000", 2023-08-28T02:32:23.162 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.8380000000000001e-05 2023-08-28T02:32:23.162 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.162 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.162 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:23.162 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.306427+0000", 2023-08-28T02:32:23.162 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.7559999999999994e-06 2023-08-28T02:32:23.162 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.162 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.162 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.162 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.200212+0000", 2023-08-28T02:32:23.162 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.89378428399999998 2023-08-28T02:32:23.163 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.163 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.163 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:23.163 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.200219+0000", 2023-08-28T02:32:23.163 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.8689999999999998e-06 2023-08-28T02:32:23.163 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.163 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.163 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.163 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.463202+0000", 2023-08-28T02:32:23.163 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.262982901 2023-08-28T02:32:23.163 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.164 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.164 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:32:23.164 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.463371+0000", 2023-08-28T02:32:23.164 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00016935499999999999 2023-08-28T02:32:23.164 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.164 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.164 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 3,4", 2023-08-28T02:32:23.164 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.463535+0000", 2023-08-28T02:32:23.164 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00016377199999999999 2023-08-28T02:32:23.164 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.164 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.164 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:23.165 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.500774+0000", 2023-08-28T02:32:23.165 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.037239743999999998 2023-08-28T02:32:23.165 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.165 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.165 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:32:23.165 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.500849+0000", 2023-08-28T02:32:23.165 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.5003e-05 2023-08-28T02:32:23.165 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.165 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.165 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:23.165 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.500925+0000", 2023-08-28T02:32:23.165 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.5839000000000004e-05 2023-08-28T02:32:23.166 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.166 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.166 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:32:23.166 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.500977+0000", 2023-08-28T02:32:23.166 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.2008000000000001e-05 2023-08-28T02:32:23.166 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.166 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.166 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:32:23.166 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.505130+0000", 2023-08-28T02:32:23.166 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00415293 2023-08-28T02:32:23.166 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:23.166 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:23.167 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:23.167 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.167 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.167 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:123537 5.a 5:5265cd7c:::benchmark_data_smithi143_236200_object13394:head [delete] snapc 0=[] ondisk+write+known_if_redirected e153)", 2023-08-28T02:32:23.167 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:58.323752+0000", 2023-08-28T02:32:23.167 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 504.810174894, 2023-08-28T02:32:23.167 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.1859482130000001, 2023-08-28T02:32:23.167 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:23.167 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:32:23.167 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:32:23.167 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:32:23.167 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:32:23.168 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 123537 2023-08-28T02:32:23.168 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.168 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:23.168 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.168 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:23.168 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.323752+0000", 2023-08-28T02:32:23.168 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:23.168 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.168 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.168 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:23.168 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.323752+0000", 2023-08-28T02:32:23.168 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:23.168 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.169 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.169 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:23.169 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.323752+0000", 2023-08-28T02:32:23.169 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:32:23.169 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.169 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.169 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:23.169 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.323757+0000", 2023-08-28T02:32:23.169 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.028e-06 2023-08-28T02:32:23.169 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.169 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.169 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:23.170 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.323765+0000", 2023-08-28T02:32:23.170 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.5109999999999997e-06 2023-08-28T02:32:23.170 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.170 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.170 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:23.170 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.323770+0000", 2023-08-28T02:32:23.170 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.8699999999999997e-06 2023-08-28T02:32:23.170 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.170 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.171 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.171 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.323831+0000", 2023-08-28T02:32:23.171 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.0418999999999999e-05 2023-08-28T02:32:23.171 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.171 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.171 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:23.171 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.323845+0000", 2023-08-28T02:32:23.171 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3713999999999999e-05 2023-08-28T02:32:23.171 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.171 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.171 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.171 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.200234+0000", 2023-08-28T02:32:23.171 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.87638942399999997 2023-08-28T02:32:23.172 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.172 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.172 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:23.172 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.200238+0000", 2023-08-28T02:32:23.172 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.9779999999999999e-06 2023-08-28T02:32:23.172 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.172 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.172 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.172 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.467441+0000", 2023-08-28T02:32:23.172 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.2672027729999999 2023-08-28T02:32:23.172 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.172 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.173 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:32:23.173 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.467585+0000", 2023-08-28T02:32:23.173 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00014433600000000001 2023-08-28T02:32:23.173 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.173 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.173 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 3,4", 2023-08-28T02:32:23.173 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.467730+0000", 2023-08-28T02:32:23.173 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00014534699999999999 2023-08-28T02:32:23.173 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.173 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.173 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:23.173 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.500836+0000", 2023-08-28T02:32:23.174 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.033105694999999997 2023-08-28T02:32:23.174 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.174 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.174 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:32:23.174 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.500860+0000", 2023-08-28T02:32:23.174 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.3791999999999999e-05 2023-08-28T02:32:23.174 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.174 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.174 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:23.174 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.505235+0000", 2023-08-28T02:32:23.174 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0043751559999999998 2023-08-28T02:32:23.174 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.174 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.175 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:32:23.175 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.505299+0000", 2023-08-28T02:32:23.175 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.3999999999999997e-05 2023-08-28T02:32:23.175 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.175 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.175 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:32:23.175 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.509701+0000", 2023-08-28T02:32:23.175 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0044016899999999998 2023-08-28T02:32:23.175 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:23.175 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:23.175 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:23.175 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.176 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.176 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:123554 5.a 5:54a525aa:::benchmark_data_smithi143_236200_object13411:head [delete] snapc 0=[] ondisk+write+known_if_redirected e153)", 2023-08-28T02:32:23.176 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:58.345326+0000", 2023-08-28T02:32:23.176 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 504.788601787, 2023-08-28T02:32:23.176 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.1684292410000001, 2023-08-28T02:32:23.176 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:23.176 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:32:23.176 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:32:23.176 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:32:23.176 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:32:23.176 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 123554 2023-08-28T02:32:23.176 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.177 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:23.177 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.177 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:23.177 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.345326+0000", 2023-08-28T02:32:23.177 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:23.177 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.177 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.177 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:23.177 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.345326+0000", 2023-08-28T02:32:23.177 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:23.177 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.177 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.177 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:23.178 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.345323+0000", 2023-08-28T02:32:23.178 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999976 2023-08-28T02:32:23.178 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.178 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.178 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:23.178 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.345332+0000", 2023-08-28T02:32:23.178 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.6880000000000005e-06 2023-08-28T02:32:23.178 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.178 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.178 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:23.178 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.345345+0000", 2023-08-28T02:32:23.178 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.2381999999999999e-05 2023-08-28T02:32:23.179 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.179 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.179 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:23.179 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.345353+0000", 2023-08-28T02:32:23.179 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.337e-06 2023-08-28T02:32:23.179 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.179 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.179 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.179 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.345429+0000", 2023-08-28T02:32:23.179 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.6173000000000004e-05 2023-08-28T02:32:23.179 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.179 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.179 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:23.180 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.345443+0000", 2023-08-28T02:32:23.180 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3913999999999999e-05 2023-08-28T02:32:23.180 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.180 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.180 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.180 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.200250+0000", 2023-08-28T02:32:23.180 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.85480715399999996 2023-08-28T02:32:23.180 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.180 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.180 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:23.180 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.200255+0000", 2023-08-28T02:32:23.180 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.4309999999999996e-06 2023-08-28T02:32:23.180 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.181 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.181 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.181 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.471833+0000", 2023-08-28T02:32:23.181 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.271578461 2023-08-28T02:32:23.181 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.181 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.181 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:32:23.181 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.471999+0000", 2023-08-28T02:32:23.181 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000166484 2023-08-28T02:32:23.181 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.181 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.181 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 3,4", 2023-08-28T02:32:23.181 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.472168+0000", 2023-08-28T02:32:23.182 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00016872799999999999 2023-08-28T02:32:23.182 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.182 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.182 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:32:23.182 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.500870+0000", 2023-08-28T02:32:23.182 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.028701918 2023-08-28T02:32:23.182 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.182 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.182 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:23.182 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.505189+0000", 2023-08-28T02:32:23.182 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0043191790000000002 2023-08-28T02:32:23.182 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.182 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.183 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:23.183 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.509765+0000", 2023-08-28T02:32:23.183 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0045760050000000002 2023-08-28T02:32:23.183 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.183 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.183 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:32:23.183 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.509828+0000", 2023-08-28T02:32:23.183 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.3187000000000003e-05 2023-08-28T02:32:23.183 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.183 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.183 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:32:23.183 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.513755+0000", 2023-08-28T02:32:23.184 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0039263520000000001 2023-08-28T02:32:23.184 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:23.184 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:23.184 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:23.184 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.184 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.184 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:123574 5.a 5:5ee9c70d:::benchmark_data_smithi143_236200_object13431:head [delete] snapc 0=[] ondisk+write+known_if_redirected e153)", 2023-08-28T02:32:23.184 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:58.374516+0000", 2023-08-28T02:32:23.184 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 504.75941106200003, 2023-08-28T02:32:23.184 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.172229722, 2023-08-28T02:32:23.184 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:23.184 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:32:23.184 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:32:23.185 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:32:23.185 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:32:23.185 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 123574 2023-08-28T02:32:23.185 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.185 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:23.185 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.185 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:23.185 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.374516+0000", 2023-08-28T02:32:23.185 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:23.185 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.185 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.185 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:23.186 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.374516+0000", 2023-08-28T02:32:23.186 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:23.186 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.186 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.186 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:23.186 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.374515+0000", 2023-08-28T02:32:23.186 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:32:23.186 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.186 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.186 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:23.186 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.374522+0000", 2023-08-28T02:32:23.186 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.9489999999999999e-06 2023-08-28T02:32:23.187 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.187 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.187 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:23.187 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.374535+0000", 2023-08-28T02:32:23.187 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3182e-05 2023-08-28T02:32:23.187 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.187 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.187 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:23.187 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.374543+0000", 2023-08-28T02:32:23.187 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.571e-06 2023-08-28T02:32:23.187 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.187 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.188 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.188 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.374589+0000", 2023-08-28T02:32:23.188 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.6687000000000002e-05 2023-08-28T02:32:23.188 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.188 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.188 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:23.188 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.374599+0000", 2023-08-28T02:32:23.188 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.1020000000000003e-06 2023-08-28T02:32:23.188 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.188 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.188 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.188 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.200353+0000", 2023-08-28T02:32:23.188 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.82575429 2023-08-28T02:32:23.189 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.189 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.189 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:23.189 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.200362+0000", 2023-08-28T02:32:23.189 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.1980000000000007e-06 2023-08-28T02:32:23.189 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.189 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.189 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.189 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.513795+0000", 2023-08-28T02:32:23.189 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3134333 2023-08-28T02:32:23.189 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.189 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.190 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:32:23.190 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.513903+0000", 2023-08-28T02:32:23.190 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000107829 2023-08-28T02:32:23.190 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.190 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.190 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 3,4", 2023-08-28T02:32:23.190 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.513996+0000", 2023-08-28T02:32:23.190 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.2631000000000001e-05 2023-08-28T02:32:23.190 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.190 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.190 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:23.190 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.525312+0000", 2023-08-28T02:32:23.190 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.011315777000000001 2023-08-28T02:32:23.191 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.191 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.191 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:32:23.191 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.525317+0000", 2023-08-28T02:32:23.191 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.7130000000000002e-06 2023-08-28T02:32:23.191 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.191 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.191 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:23.191 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.541898+0000", 2023-08-28T02:32:23.191 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.016580305 2023-08-28T02:32:23.191 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.191 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.192 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:32:23.192 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.541950+0000", 2023-08-28T02:32:23.192 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.2343999999999999e-05 2023-08-28T02:32:23.192 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.192 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.192 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:32:23.192 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:00.546746+0000", 2023-08-28T02:32:23.192 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0047960629999999997 2023-08-28T02:32:23.192 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:23.192 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:23.192 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:23.192 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.193 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.193 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:132203 5.a 5:5c355ea7:::benchmark_data_smithi143_236200_object22060:head [delete] snapc 0=[] ondisk+write+known_if_redirected e163)", 2023-08-28T02:32:23.193 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:24:15.698205+0000", 2023-08-28T02:32:23.193 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 487.43572240700001, 2023-08-28T02:32:23.193 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.31137872999999999, 2023-08-28T02:32:23.193 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:23.193 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:32:23.193 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:32:23.193 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:32:23.193 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:32:23.193 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 132203 2023-08-28T02:32:23.193 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.193 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:23.194 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.194 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:23.194 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:15.698205+0000", 2023-08-28T02:32:23.194 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:23.194 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.194 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.194 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:23.194 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:15.698205+0000", 2023-08-28T02:32:23.194 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:23.194 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.194 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.194 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:23.195 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:15.698205+0000", 2023-08-28T02:32:23.195 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:32:23.195 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.195 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.195 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:23.195 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:15.698207+0000", 2023-08-28T02:32:23.195 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.5770000000000001e-06 2023-08-28T02:32:23.195 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.195 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.195 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:23.195 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:15.698212+0000", 2023-08-28T02:32:23.195 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.0849999999999996e-06 2023-08-28T02:32:23.196 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.196 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.196 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:23.196 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:15.698217+0000", 2023-08-28T02:32:23.196 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.1139999999999997e-06 2023-08-28T02:32:23.196 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.196 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.196 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.196 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:15.724602+0000", 2023-08-28T02:32:23.196 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.026384642999999999 2023-08-28T02:32:23.196 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.196 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.196 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for rw locks", 2023-08-28T02:32:23.197 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:15.724716+0000", 2023-08-28T02:32:23.197 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000113571 2023-08-28T02:32:23.197 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.197 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.197 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.197 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:15.762330+0000", 2023-08-28T02:32:23.197 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.037613949000000001 2023-08-28T02:32:23.197 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.197 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.197 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:32:23.197 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:15.762423+0000", 2023-08-28T02:32:23.197 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.3737000000000001e-05 2023-08-28T02:32:23.197 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.198 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.198 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 3,4", 2023-08-28T02:32:23.198 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:15.762525+0000", 2023-08-28T02:32:23.198 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000101537 2023-08-28T02:32:23.198 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.198 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.198 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:32:23.198 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:16.001139+0000", 2023-08-28T02:32:23.198 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.23861400199999999 2023-08-28T02:32:23.198 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.198 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.198 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:23.199 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:16.001191+0000", 2023-08-28T02:32:23.199 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.2435000000000002e-05 2023-08-28T02:32:23.199 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.199 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.199 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:23.199 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:16.005872+0000", 2023-08-28T02:32:23.199 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0046805869999999999 2023-08-28T02:32:23.199 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.199 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.199 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:32:23.199 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:16.005936+0000", 2023-08-28T02:32:23.199 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.3793000000000004e-05 2023-08-28T02:32:23.199 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.200 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.200 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:32:23.200 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:16.009584+0000", 2023-08-28T02:32:23.200 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00364811 2023-08-28T02:32:23.200 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:23.200 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:23.200 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:23.200 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.200 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.200 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:143161 5.a 5:565d73ce:::benchmark_data_smithi143_236200_object33018:head [delete] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-08-28T02:32:23.200 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:24:32.420910+0000", 2023-08-28T02:32:23.200 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 470.71301750600003, 2023-08-28T02:32:23.200 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.31516173400000003, 2023-08-28T02:32:23.201 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:23.201 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:32:23.201 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:32:23.201 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:32:23.201 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:32:23.201 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 143161 2023-08-28T02:32:23.201 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.201 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:23.201 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.201 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:23.202 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:32.420910+0000", 2023-08-28T02:32:23.202 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:23.202 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.202 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.202 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:23.202 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:32.420910+0000", 2023-08-28T02:32:23.202 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:23.202 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.202 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.202 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:23.202 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:32.420909+0000", 2023-08-28T02:32:23.202 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:32:23.203 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.203 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.203 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:23.203 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:32.420912+0000", 2023-08-28T02:32:23.203 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.6290000000000002e-06 2023-08-28T02:32:23.203 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.203 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.203 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:23.203 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:32.420918+0000", 2023-08-28T02:32:23.203 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.7119999999999997e-06 2023-08-28T02:32:23.203 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.203 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.204 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:23.204 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:32.420923+0000", 2023-08-28T02:32:23.204 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.0270000000000003e-06 2023-08-28T02:32:23.204 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.204 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.204 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.204 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:32.453891+0000", 2023-08-28T02:32:23.204 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.032968277999999997 2023-08-28T02:32:23.204 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.204 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.204 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:32:23.204 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:32.454012+0000", 2023-08-28T02:32:23.205 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000120451 2023-08-28T02:32:23.205 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.205 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.205 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 3,4", 2023-08-28T02:32:23.205 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:32.454155+0000", 2023-08-28T02:32:23.205 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00014329699999999999 2023-08-28T02:32:23.205 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.205 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.205 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:32:23.205 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:32.461338+0000", 2023-08-28T02:32:23.205 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0071832700000000003 2023-08-28T02:32:23.205 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.205 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.206 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:23.206 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:32.463403+0000", 2023-08-28T02:32:23.206 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0020643940000000002 2023-08-28T02:32:23.206 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.206 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.206 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:23.206 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:32.731678+0000", 2023-08-28T02:32:23.206 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.26827508900000002 2023-08-28T02:32:23.206 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.206 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.206 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:32:23.206 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:32.731733+0000", 2023-08-28T02:32:23.207 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.5476999999999998e-05 2023-08-28T02:32:23.207 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.207 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.207 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:32:23.207 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:24:32.736072+0000", 2023-08-28T02:32:23.207 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0043385009999999998 2023-08-28T02:32:23.207 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:23.207 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:23.207 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:23.207 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.207 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.207 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:24545 6.1 6:819240ee:::benchmark_data_smithi143_305742_object24544:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e196)", 2023-08-28T02:32:23.207 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:27:16.333386+0000", 2023-08-28T02:32:23.208 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 306.80054156199998, 2023-08-28T02:32:23.208 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.107399708, 2023-08-28T02:32:23.208 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:23.208 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:32:23.208 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:32:23.208 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:32:23.208 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:32:23.208 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 24545 2023-08-28T02:32:23.208 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.208 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:23.208 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.208 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:23.209 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.333386+0000", 2023-08-28T02:32:23.209 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:23.209 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.209 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.209 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:23.209 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.333386+0000", 2023-08-28T02:32:23.209 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:23.209 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.209 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.209 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:23.209 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.333385+0000", 2023-08-28T02:32:23.209 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:32:23.209 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.210 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.210 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:23.210 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.333405+0000", 2023-08-28T02:32:23.210 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.0767999999999998e-05 2023-08-28T02:32:23.210 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.210 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.210 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:23.210 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.333418+0000", 2023-08-28T02:32:23.210 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3084e-05 2023-08-28T02:32:23.210 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.210 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.210 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:23.210 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.333427+0000", 2023-08-28T02:32:23.211 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.5189999999999992e-06 2023-08-28T02:32:23.211 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.211 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.211 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.211 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.333474+0000", 2023-08-28T02:32:23.211 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.6675000000000003e-05 2023-08-28T02:32:23.211 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.211 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.211 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:23.211 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.333481+0000", 2023-08-28T02:32:23.211 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.8290000000000008e-06 2023-08-28T02:32:23.211 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.211 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.212 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.212 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:17.309618+0000", 2023-08-28T02:32:23.212 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.97613674299999997 2023-08-28T02:32:23.212 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.212 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.212 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:23.212 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:17.309628+0000", 2023-08-28T02:32:23.212 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.7170000000000003e-06 2023-08-28T02:32:23.212 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.212 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.212 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.212 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.428119+0000", 2023-08-28T02:32:23.213 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.118491452 2023-08-28T02:32:23.213 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.213 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.213 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:32:23.213 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.428936+0000", 2023-08-28T02:32:23.213 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00081677699999999998 2023-08-28T02:32:23.213 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.213 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.213 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,7", 2023-08-28T02:32:23.213 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.429153+0000", 2023-08-28T02:32:23.213 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00021667799999999999 2023-08-28T02:32:23.213 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.213 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.214 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:32:23.214 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.432892+0000", 2023-08-28T02:32:23.214 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.003739118 2023-08-28T02:32:23.214 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.214 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.214 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:23.214 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.436505+0000", 2023-08-28T02:32:23.214 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0036134600000000002 2023-08-28T02:32:23.214 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.214 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.214 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:23.214 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.439273+0000", 2023-08-28T02:32:23.214 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.002767944 2023-08-28T02:32:23.215 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.215 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.215 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:32:23.215 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.439344+0000", 2023-08-28T02:32:23.215 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.0192999999999997e-05 2023-08-28T02:32:23.215 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.215 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.215 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:32:23.215 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.440786+0000", 2023-08-28T02:32:23.215 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.001441969 2023-08-28T02:32:23.215 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:23.215 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:23.215 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:23.216 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.216 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.216 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:24548 6.1 6:8685cf96:::benchmark_data_smithi143_305742_object24547:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e196)", 2023-08-28T02:32:23.216 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:27:16.341156+0000", 2023-08-28T02:32:23.216 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 306.79277188100002, 2023-08-28T02:32:23.216 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.1066428840000002, 2023-08-28T02:32:23.216 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:23.216 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:32:23.216 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:32:23.216 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:32:23.216 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:32:23.216 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 24548 2023-08-28T02:32:23.217 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.217 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:23.217 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.217 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:23.217 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.341156+0000", 2023-08-28T02:32:23.217 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:23.217 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.217 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.217 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:23.217 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.341156+0000", 2023-08-28T02:32:23.217 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:23.217 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.217 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.218 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:23.218 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.341154+0000", 2023-08-28T02:32:23.218 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:32:23.218 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.218 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.218 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:23.218 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.341184+0000", 2023-08-28T02:32:23.218 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.0436000000000001e-05 2023-08-28T02:32:23.218 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.218 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.218 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:23.218 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.341202+0000", 2023-08-28T02:32:23.219 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.7795000000000002e-05 2023-08-28T02:32:23.219 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.219 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.219 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:23.219 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.341212+0000", 2023-08-28T02:32:23.219 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.5780000000000005e-06 2023-08-28T02:32:23.219 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.219 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.219 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.219 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.341270+0000", 2023-08-28T02:32:23.219 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.8084000000000002e-05 2023-08-28T02:32:23.219 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.219 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.220 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:23.220 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.341282+0000", 2023-08-28T02:32:23.220 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.2583e-05 2023-08-28T02:32:23.220 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.220 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.220 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.220 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:17.309642+0000", 2023-08-28T02:32:23.220 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.96835962600000003 2023-08-28T02:32:23.220 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.220 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.220 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:23.220 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:17.309646+0000", 2023-08-28T02:32:23.221 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.2570000000000001e-06 2023-08-28T02:32:23.221 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.221 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.221 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.221 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.431296+0000", 2023-08-28T02:32:23.221 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.121649441 2023-08-28T02:32:23.221 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.221 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.221 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:32:23.221 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.431444+0000", 2023-08-28T02:32:23.221 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00014787599999999999 2023-08-28T02:32:23.221 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.222 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.222 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,7", 2023-08-28T02:32:23.222 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.431632+0000", 2023-08-28T02:32:23.222 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000188853 2023-08-28T02:32:23.222 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.222 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.222 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:23.222 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.440843+0000", 2023-08-28T02:32:23.222 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0092102920000000001 2023-08-28T02:32:23.222 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.222 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.222 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:32:23.222 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.441048+0000", 2023-08-28T02:32:23.223 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00020495800000000001 2023-08-28T02:32:23.223 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.223 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.223 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:23.223 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.446692+0000", 2023-08-28T02:32:23.223 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0056445159999999996 2023-08-28T02:32:23.223 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.223 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.223 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:32:23.223 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.446748+0000", 2023-08-28T02:32:23.223 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.5485000000000002e-05 2023-08-28T02:32:23.223 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.223 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.224 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:32:23.224 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.447798+0000", 2023-08-28T02:32:23.224 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0010507699999999999 2023-08-28T02:32:23.224 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:23.224 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:23.224 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:23.224 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.224 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.224 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:59698 6.11 6:8e3fbc99:::benchmark_data_smithi143_305742_object59697:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e211)", 2023-08-28T02:32:23.224 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:27:59.408193+0000", 2023-08-28T02:32:23.224 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 263.725734252, 2023-08-28T02:32:23.224 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0331296480000001, 2023-08-28T02:32:23.224 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:23.225 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "reached pg", 2023-08-28T02:32:23.225 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:32:23.225 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:32:23.225 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:32:23.225 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 59698 2023-08-28T02:32:23.225 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.225 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:23.225 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.225 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:23.225 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:59.408193+0000", 2023-08-28T02:32:23.225 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:23.225 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.225 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.226 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:23.226 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:59.408193+0000", 2023-08-28T02:32:23.226 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:23.226 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.226 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.226 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:23.226 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:59.408192+0000", 2023-08-28T02:32:23.226 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:32:23.226 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.226 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.226 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:23.226 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:59.408227+0000", 2023-08-28T02:32:23.227 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.4910999999999998e-05 2023-08-28T02:32:23.227 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.227 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.227 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:23.227 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:59.408242+0000", 2023-08-28T02:32:23.227 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.5937e-05 2023-08-28T02:32:23.227 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.227 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.227 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:23.227 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:59.408251+0000", 2023-08-28T02:32:23.227 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.191e-06 2023-08-28T02:32:23.227 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.227 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.228 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.228 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:59.408303+0000", 2023-08-28T02:32:23.228 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.2648999999999997e-05 2023-08-28T02:32:23.228 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.228 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.228 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op must wait for map", 2023-08-28T02:32:23.228 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:59.408311+0000", 2023-08-28T02:32:23.228 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.6680000000000001e-06 2023-08-28T02:32:23.228 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.228 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.228 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.228 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:59.414888+0000", 2023-08-28T02:32:23.228 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0065772419999999996 2023-08-28T02:32:23.229 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.229 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.229 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:23.229 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:59.414895+0000", 2023-08-28T02:32:23.230 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.0639999999999996e-06 2023-08-28T02:32:23.230 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.230 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.231 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.231 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:00.441246+0000", 2023-08-28T02:32:23.231 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0263504489999999 2023-08-28T02:32:23.232 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.232 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.232 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:32:23.232 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:00.441323+0000", 2023-08-28T02:32:23.233 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.7078999999999998e-05 2023-08-28T02:32:23.233 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:23.233 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:23.233 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:23.234 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.234 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.234 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:59732 6.11 6:8f81ff89:::benchmark_data_smithi143_305742_object59731:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e211)", 2023-08-28T02:32:23.235 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:27:59.408262+0000", 2023-08-28T02:32:23.235 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 263.72566531500001, 2023-08-28T02:32:23.235 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0469317220000001, 2023-08-28T02:32:23.235 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:23.236 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:32:23.236 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:32:23.236 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:32:23.236 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:32:23.237 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 59732 2023-08-28T02:32:23.237 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.237 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:23.237 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.237 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:23.238 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:59.408262+0000", 2023-08-28T02:32:23.238 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:23.239 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.239 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.239 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:23.240 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:59.408262+0000", 2023-08-28T02:32:23.240 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:23.240 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.240 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.240 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:23.240 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:59.408262+0000", 2023-08-28T02:32:23.240 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:32:23.240 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.240 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.240 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:23.241 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:59.408291+0000", 2023-08-28T02:32:23.241 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.9251000000000002e-05 2023-08-28T02:32:23.241 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.241 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.241 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:23.241 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:59.408299+0000", 2023-08-28T02:32:23.241 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.1040000000000007e-06 2023-08-28T02:32:23.241 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.241 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.241 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:23.241 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:59.408304+0000", 2023-08-28T02:32:23.241 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.2079999999999999e-06 2023-08-28T02:32:23.242 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.242 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.242 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.242 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:59.408376+0000", 2023-08-28T02:32:23.242 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.1569999999999994e-05 2023-08-28T02:32:23.242 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.242 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.242 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting_for_map not empty", 2023-08-28T02:32:23.242 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:59.408382+0000", 2023-08-28T02:32:23.242 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.789e-06 2023-08-28T02:32:23.242 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.242 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.243 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.243 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:59.414906+0000", 2023-08-28T02:32:23.243 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0065248809999999997 2023-08-28T02:32:23.243 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.243 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.243 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:23.243 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:59.414910+0000", 2023-08-28T02:32:23.243 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.6279999999999998e-06 2023-08-28T02:32:23.243 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.243 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.243 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.244 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:00.441345+0000", 2023-08-28T02:32:23.244 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0264347700000001 2023-08-28T02:32:23.244 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.244 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.244 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:32:23.244 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:00.441476+0000", 2023-08-28T02:32:23.244 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00013084699999999999 2023-08-28T02:32:23.244 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.244 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.244 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,7", 2023-08-28T02:32:23.244 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:00.441627+0000", 2023-08-28T02:32:23.244 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000151061 2023-08-28T02:32:23.244 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.245 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.245 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:23.245 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:00.445542+0000", 2023-08-28T02:32:23.245 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0039150629999999999 2023-08-28T02:32:23.245 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.245 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.245 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:32:23.245 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:00.449882+0000", 2023-08-28T02:32:23.245 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0043398040000000001 2023-08-28T02:32:23.245 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.245 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.245 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:23.246 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:00.454458+0000", 2023-08-28T02:32:23.246 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0045768010000000001 2023-08-28T02:32:23.246 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.246 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.246 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:32:23.246 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:00.454534+0000", 2023-08-28T02:32:23.246 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.5281e-05 2023-08-28T02:32:23.246 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.246 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.246 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:32:23.246 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:00.455194+0000", 2023-08-28T02:32:23.246 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00066015099999999999 2023-08-28T02:32:23.246 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:23.247 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:23.247 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:23.247 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.247 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.247 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:59794 6.11 6:8983f348:::benchmark_data_smithi143_305742_object59793:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e211)", 2023-08-28T02:32:23.247 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:27:59.408312+0000", 2023-08-28T02:32:23.247 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 263.725615839, 2023-08-28T02:32:23.247 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0473480749999999, 2023-08-28T02:32:23.247 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:23.247 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:32:23.247 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:32:23.247 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:32:23.248 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:32:23.248 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 59794 2023-08-28T02:32:23.248 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.248 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:23.248 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.248 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:23.248 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:59.408312+0000", 2023-08-28T02:32:23.248 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:23.248 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.248 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.248 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:23.248 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:59.408312+0000", 2023-08-28T02:32:23.249 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:23.249 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.249 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.249 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:23.249 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:59.408311+0000", 2023-08-28T02:32:23.249 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:32:23.249 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.249 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.249 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:23.249 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:59.408331+0000", 2023-08-28T02:32:23.249 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.0120999999999999e-05 2023-08-28T02:32:23.249 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.249 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.250 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:23.250 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:59.408340+0000", 2023-08-28T02:32:23.250 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.2380000000000003e-06 2023-08-28T02:32:23.250 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.250 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.250 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:23.250 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:59.408346+0000", 2023-08-28T02:32:23.250 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.1290000000000004e-06 2023-08-28T02:32:23.250 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.250 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.250 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.250 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:59.408412+0000", 2023-08-28T02:32:23.251 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.6514999999999996e-05 2023-08-28T02:32:23.251 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.251 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.251 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting_for_map not empty", 2023-08-28T02:32:23.251 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:59.408418+0000", 2023-08-28T02:32:23.251 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.0560000000000001e-06 2023-08-28T02:32:23.251 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.251 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.251 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.251 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:59.414921+0000", 2023-08-28T02:32:23.251 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0065029099999999998 2023-08-28T02:32:23.251 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.251 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.252 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:23.252 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:59.414924+0000", 2023-08-28T02:32:23.252 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.3560000000000001e-06 2023-08-28T02:32:23.252 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.252 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.252 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.252 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:00.442692+0000", 2023-08-28T02:32:23.252 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0277672229999999 2023-08-28T02:32:23.252 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.252 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.252 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:32:23.252 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:00.442836+0000", 2023-08-28T02:32:23.253 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00014442799999999999 2023-08-28T02:32:23.253 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.253 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.253 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,7", 2023-08-28T02:32:23.253 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:00.442996+0000", 2023-08-28T02:32:23.253 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00015949000000000001 2023-08-28T02:32:23.253 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.253 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.253 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:23.253 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:00.448430+0000", 2023-08-28T02:32:23.253 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0054341019999999997 2023-08-28T02:32:23.253 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.254 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.254 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:32:23.254 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:00.449889+0000", 2023-08-28T02:32:23.254 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0014592229999999999 2023-08-28T02:32:23.254 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.254 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.254 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:23.254 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:00.455239+0000", 2023-08-28T02:32:23.254 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0053500830000000003 2023-08-28T02:32:23.254 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.254 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.254 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:32:23.255 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:00.455286+0000", 2023-08-28T02:32:23.255 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.7431e-05 2023-08-28T02:32:23.255 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.255 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.255 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:32:23.255 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:00.455660+0000", 2023-08-28T02:32:23.255 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00037324899999999998 2023-08-28T02:32:23.255 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:23.255 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:23.255 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:23.255 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.255 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.255 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:78149 6.1 6:82fffdc4:::benchmark_data_smithi143_305742_object78148:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e219)", 2023-08-28T02:32:23.256 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:28:24.326076+0000", 2023-08-28T02:32:23.256 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 238.80785088900001, 2023-08-28T02:32:23.256 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.016205689, 2023-08-28T02:32:23.256 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:23.256 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "reached pg", 2023-08-28T02:32:23.256 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:32:23.256 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:32:23.256 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:32:23.256 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 78149 2023-08-28T02:32:23.256 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.256 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:23.257 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.257 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:23.257 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.326076+0000", 2023-08-28T02:32:23.257 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:23.257 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.257 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.257 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:23.257 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.326076+0000", 2023-08-28T02:32:23.257 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:23.257 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.257 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.257 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:23.258 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.326075+0000", 2023-08-28T02:32:23.258 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:32:23.258 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.258 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.258 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:23.258 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.326103+0000", 2023-08-28T02:32:23.258 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.8589e-05 2023-08-28T02:32:23.258 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.258 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.258 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:23.258 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.326129+0000", 2023-08-28T02:32:23.258 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.5792999999999999e-05 2023-08-28T02:32:23.259 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.259 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.259 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:23.259 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.326141+0000", 2023-08-28T02:32:23.259 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1888999999999999e-05 2023-08-28T02:32:23.259 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.259 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.259 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.259 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.326191+0000", 2023-08-28T02:32:23.259 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.9629999999999997e-05 2023-08-28T02:32:23.259 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.259 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.259 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:32:23.260 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.326358+0000", 2023-08-28T02:32:23.260 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00016697200000000001 2023-08-28T02:32:23.260 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.260 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.260 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,7", 2023-08-28T02:32:23.260 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.326521+0000", 2023-08-28T02:32:23.260 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00016292300000000001 2023-08-28T02:32:23.260 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.260 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.260 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:32:23.260 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.329628+0000", 2023-08-28T02:32:23.260 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0031076260000000001 2023-08-28T02:32:23.261 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.261 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.261 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:23.261 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.336462+0000", 2023-08-28T02:32:23.261 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0068336389999999999 2023-08-28T02:32:23.261 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.261 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.261 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.261 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.351313+0000", 2023-08-28T02:32:23.261 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.014851358 2023-08-28T02:32:23.261 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.261 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.262 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:23.262 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.351331+0000", 2023-08-28T02:32:23.262 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.753e-05 2023-08-28T02:32:23.262 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.262 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.262 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.262 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.342270+0000", 2023-08-28T02:32:23.262 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.99093923500000003 2023-08-28T02:32:23.262 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.262 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.262 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:32:23.262 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.342282+0000", 2023-08-28T02:32:23.263 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.2288e-05 2023-08-28T02:32:23.263 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:23.263 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:23.263 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:23.263 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.263 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.263 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:78160 6.1 6.6deeb01 (undecoded) ondisk+write+known_if_redirected e220)", 2023-08-28T02:32:23.263 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:28:24.352694+0000", 2023-08-28T02:32:23.263 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 238.781232899, 2023-08-28T02:32:23.263 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.98961628400000001, 2023-08-28T02:32:23.263 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:23.263 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "reached pg", 2023-08-28T02:32:23.263 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:32:23.264 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:32:23.264 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:32:23.264 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 78160 2023-08-28T02:32:23.264 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.264 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:23.264 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.264 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:23.264 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.352694+0000", 2023-08-28T02:32:23.264 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:23.264 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.264 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.264 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:23.265 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.352694+0000", 2023-08-28T02:32:23.265 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:23.265 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.265 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.265 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:23.265 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.352692+0000", 2023-08-28T02:32:23.265 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:32:23.265 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.265 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.265 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:23.265 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.352721+0000", 2023-08-28T02:32:23.265 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.8733000000000001e-05 2023-08-28T02:32:23.266 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.266 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.266 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:23.266 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.352734+0000", 2023-08-28T02:32:23.266 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.2747e-05 2023-08-28T02:32:23.266 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.266 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.266 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:23.266 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.352743+0000", 2023-08-28T02:32:23.266 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.935e-06 2023-08-28T02:32:23.266 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.266 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.267 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.267 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.354881+0000", 2023-08-28T02:32:23.267 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0021386310000000002 2023-08-28T02:32:23.267 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.267 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.267 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:23.267 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.354895+0000", 2023-08-28T02:32:23.267 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3149000000000001e-05 2023-08-28T02:32:23.267 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.267 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.267 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.267 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.342302+0000", 2023-08-28T02:32:23.267 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.98740746599999996 2023-08-28T02:32:23.268 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.268 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.268 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:32:23.268 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.342311+0000", 2023-08-28T02:32:23.268 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.6880000000000005e-06 2023-08-28T02:32:23.268 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:23.268 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:23.268 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:23.268 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.268 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.268 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:78173 6.11 6.ea3b2151 (undecoded) ondisk+write+known_if_redirected e220)", 2023-08-28T02:32:23.268 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:28:24.381161+0000", 2023-08-28T02:32:23.269 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 238.75276591799999, 2023-08-28T02:32:23.269 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.96107617899999997, 2023-08-28T02:32:23.269 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:23.269 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "reached pg", 2023-08-28T02:32:23.269 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:32:23.269 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:32:23.269 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:32:23.269 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 78173 2023-08-28T02:32:23.269 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.269 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:23.269 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.269 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:23.269 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.381161+0000", 2023-08-28T02:32:23.270 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:23.270 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.270 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.270 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:23.270 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.381161+0000", 2023-08-28T02:32:23.270 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:23.270 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.270 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.270 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:23.270 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.381161+0000", 2023-08-28T02:32:23.270 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:32:23.270 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.271 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.271 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:23.271 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.381183+0000", 2023-08-28T02:32:23.271 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.1946e-05 2023-08-28T02:32:23.271 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.271 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.271 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:23.271 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.381190+0000", 2023-08-28T02:32:23.271 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.6839999999999999e-06 2023-08-28T02:32:23.271 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.271 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.271 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:23.272 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.381195+0000", 2023-08-28T02:32:23.272 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.0599999999999998e-06 2023-08-28T02:32:23.272 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.272 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.273 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.273 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.384135+0000", 2023-08-28T02:32:23.273 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0029400030000000001 2023-08-28T02:32:23.273 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.273 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.273 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:23.273 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.384147+0000", 2023-08-28T02:32:23.273 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.2187e-05 2023-08-28T02:32:23.273 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.273 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.273 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.274 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.342226+0000", 2023-08-28T02:32:23.274 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.95807960999999997 2023-08-28T02:32:23.274 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.274 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.274 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:32:23.274 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.342238+0000", 2023-08-28T02:32:23.274 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1212e-05 2023-08-28T02:32:23.274 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:23.274 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:23.274 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:23.274 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.274 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.275 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:78174 6.1 6.b8248b01 (undecoded) ondisk+write+known_if_redirected e220)", 2023-08-28T02:32:23.275 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:28:24.381206+0000", 2023-08-28T02:32:23.275 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 238.75272157500001, 2023-08-28T02:32:23.275 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.96113224600000002, 2023-08-28T02:32:23.275 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:23.275 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "reached pg", 2023-08-28T02:32:23.275 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:32:23.275 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:32:23.275 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:32:23.275 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 78174 2023-08-28T02:32:23.275 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.276 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:23.276 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.276 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:23.276 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.381206+0000", 2023-08-28T02:32:23.276 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:23.276 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.276 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.276 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:23.276 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.381206+0000", 2023-08-28T02:32:23.276 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:23.276 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.276 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.277 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:23.277 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.381205+0000", 2023-08-28T02:32:23.277 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:32:23.277 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.277 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.277 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:23.277 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.381227+0000", 2023-08-28T02:32:23.277 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.209e-05 2023-08-28T02:32:23.277 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.277 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.277 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:23.278 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.381234+0000", 2023-08-28T02:32:23.278 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.438e-06 2023-08-28T02:32:23.278 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.278 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.278 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:23.278 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.381239+0000", 2023-08-28T02:32:23.278 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.3650000000000003e-06 2023-08-28T02:32:23.278 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.278 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.278 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.278 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.384173+0000", 2023-08-28T02:32:23.279 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.002933902 2023-08-28T02:32:23.279 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.279 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.279 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:23.279 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.384180+0000", 2023-08-28T02:32:23.279 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.9879999999999998e-06 2023-08-28T02:32:23.279 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.279 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.279 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.279 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.342329+0000", 2023-08-28T02:32:23.279 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.95814922300000005 2023-08-28T02:32:23.279 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.280 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.280 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:32:23.280 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.342338+0000", 2023-08-28T02:32:23.280 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.6540000000000005e-06 2023-08-28T02:32:23.280 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:23.280 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:23.280 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:23.280 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.280 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.280 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:78179 6.1 6.22a5a1a1 (undecoded) ondisk+write+known_if_redirected e220)", 2023-08-28T02:32:23.280 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:28:24.391267+0000", 2023-08-28T02:32:23.280 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 238.74265991999999, 2023-08-28T02:32:23.281 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.95109916100000003, 2023-08-28T02:32:23.281 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:23.281 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "reached pg", 2023-08-28T02:32:23.281 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:32:23.281 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:32:23.281 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:32:23.281 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 78179 2023-08-28T02:32:23.281 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.281 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:23.281 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.281 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:23.281 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.391267+0000", 2023-08-28T02:32:23.282 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:23.282 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.282 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.282 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:23.282 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.391267+0000", 2023-08-28T02:32:23.282 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:23.282 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.282 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.282 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:23.282 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.391266+0000", 2023-08-28T02:32:23.282 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:32:23.282 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.283 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.283 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:23.283 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.391291+0000", 2023-08-28T02:32:23.283 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.5092999999999998e-05 2023-08-28T02:32:23.283 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.283 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.283 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:23.283 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.391310+0000", 2023-08-28T02:32:23.283 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.9589000000000002e-05 2023-08-28T02:32:23.283 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.283 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.283 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:23.284 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.391319+0000", 2023-08-28T02:32:23.284 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.3790000000000006e-06 2023-08-28T02:32:23.284 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.284 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.284 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.284 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.391374+0000", 2023-08-28T02:32:23.284 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.5108000000000002e-05 2023-08-28T02:32:23.284 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.284 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.284 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:23.284 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.391384+0000", 2023-08-28T02:32:23.284 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0173000000000001e-05 2023-08-28T02:32:23.284 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.285 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.285 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.285 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.342357+0000", 2023-08-28T02:32:23.285 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.95097256900000005 2023-08-28T02:32:23.285 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.285 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.285 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:32:23.285 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.342367+0000", 2023-08-28T02:32:23.285 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.9229999999999997e-06 2023-08-28T02:32:23.285 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:23.285 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:23.285 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:23.286 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.286 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.286 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:141082 6.f 6:f0935753:::benchmark_data_smithi143_305742_object57463:head [delete] snapc 0=[] ondisk+write+known_if_redirected e251)", 2023-08-28T02:32:23.286 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:30:09.252855+0000", 2023-08-28T02:32:23.286 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 133.88107266, 2023-08-28T02:32:23.286 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.39694396100000001, 2023-08-28T02:32:23.286 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:23.286 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:32:23.286 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:32:23.286 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:32:23.286 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:32:23.286 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 141082 2023-08-28T02:32:23.287 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.287 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:23.287 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.287 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:23.287 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.252855+0000", 2023-08-28T02:32:23.287 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:23.287 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.287 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.287 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:23.287 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.252855+0000", 2023-08-28T02:32:23.287 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:23.287 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.288 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.288 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:23.288 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.252853+0000", 2023-08-28T02:32:23.288 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:32:23.288 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.288 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.288 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:23.288 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.252860+0000", 2023-08-28T02:32:23.288 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.17e-06 2023-08-28T02:32:23.288 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.288 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.288 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:23.288 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.252875+0000", 2023-08-28T02:32:23.289 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.4548e-05 2023-08-28T02:32:23.289 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.289 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.289 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:23.289 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.252882+0000", 2023-08-28T02:32:23.289 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.3610000000000003e-06 2023-08-28T02:32:23.289 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.289 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.289 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.289 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.252933+0000", 2023-08-28T02:32:23.289 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.0788999999999998e-05 2023-08-28T02:32:23.289 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.290 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.290 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:32:23.290 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.253075+0000", 2023-08-28T02:32:23.290 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00014222799999999999 2023-08-28T02:32:23.290 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.290 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.290 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 3,4", 2023-08-28T02:32:23.290 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.253196+0000", 2023-08-28T02:32:23.290 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00012029899999999999 2023-08-28T02:32:23.290 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.290 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.290 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:23.291 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.259516+0000", 2023-08-28T02:32:23.291 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0063205070000000004 2023-08-28T02:32:23.291 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.291 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.291 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:32:23.291 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.259529+0000", 2023-08-28T02:32:23.291 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.2566e-05 2023-08-28T02:32:23.291 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.291 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.291 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:23.291 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.649708+0000", 2023-08-28T02:32:23.291 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.39017906499999999 2023-08-28T02:32:23.292 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.292 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.292 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:32:23.292 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.649760+0000", 2023-08-28T02:32:23.292 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.2079000000000001e-05 2023-08-28T02:32:23.292 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.292 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.292 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:32:23.292 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.649799+0000", 2023-08-28T02:32:23.292 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.8862000000000002e-05 2023-08-28T02:32:23.292 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:23.292 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:23.292 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:23.293 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.293 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.293 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:141083 6.f 6:f8234a08:::benchmark_data_smithi143_305742_object57464:head [delete] snapc 0=[] ondisk+write+known_if_redirected e251)", 2023-08-28T02:32:23.293 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:30:09.253719+0000", 2023-08-28T02:32:23.293 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 133.88020888400001, 2023-08-28T02:32:23.293 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.39619021700000001, 2023-08-28T02:32:23.293 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:23.293 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:32:23.293 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:32:23.293 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:32:23.293 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:32:23.293 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 141083 2023-08-28T02:32:23.294 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.294 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:23.294 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.294 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:23.294 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.253719+0000", 2023-08-28T02:32:23.294 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:23.294 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.294 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.294 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:23.294 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.253719+0000", 2023-08-28T02:32:23.294 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:23.294 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.294 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.295 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:23.295 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.253718+0000", 2023-08-28T02:32:23.295 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:32:23.295 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.295 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.295 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:23.295 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.253722+0000", 2023-08-28T02:32:23.295 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.9700000000000001e-06 2023-08-28T02:32:23.295 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.295 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.295 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:23.295 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.253732+0000", 2023-08-28T02:32:23.296 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0169999999999999e-05 2023-08-28T02:32:23.296 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.296 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.296 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:23.296 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.253737+0000", 2023-08-28T02:32:23.296 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.9450000000000001e-06 2023-08-28T02:32:23.296 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.296 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.296 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:23.296 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.256486+0000", 2023-08-28T02:32:23.296 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0027491339999999999 2023-08-28T02:32:23.296 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.297 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.297 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:32:23.297 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.256573+0000", 2023-08-28T02:32:23.297 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.6601999999999995e-05 2023-08-28T02:32:23.297 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.297 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.297 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 3,4", 2023-08-28T02:32:23.297 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.256644+0000", 2023-08-28T02:32:23.297 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.1495999999999994e-05 2023-08-28T02:32:23.297 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.297 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.297 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:32:23.298 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.259671+0000", 2023-08-28T02:32:23.298 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.003027105 2023-08-28T02:32:23.298 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.298 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.298 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:23.298 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.259805+0000", 2023-08-28T02:32:23.298 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00013322599999999999 2023-08-28T02:32:23.298 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.298 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.298 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:23.298 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.649836+0000", 2023-08-28T02:32:23.298 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.39003163600000001 2023-08-28T02:32:23.298 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.299 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.299 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:32:23.299 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.649877+0000", 2023-08-28T02:32:23.299 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.0500999999999997e-05 2023-08-28T02:32:23.299 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:23.299 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:23.299 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:32:23.299 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:30:09.649909+0000", 2023-08-28T02:32:23.299 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.2058999999999999e-05 2023-08-28T02:32:23.299 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:23.299 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:23.299 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:23.300 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:23.300 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:23.300 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:32:23.300 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:23.419 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:32:23.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:23.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:32:23.537 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:32:23.621 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:23.865 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 82 16 87427 87411 66.5775 45.8125 0.0265056 0.0150083 2023-08-28T02:32:24.498 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:32:24.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:24.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:32:24.541 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.3 dump_ops_in_flight 2023-08-28T02:32:24.598 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:32:24.605 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:32:24.634 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:24.686 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:24 smithi143 ceph-mon[183059]: pgmap v702: 99 pgs: 1 active+recovering+undersized+remapped, 25 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 32 MiB/s wr, 507 op/s; 52898/254502 objects degraded (20.785%); 1554/254502 objects misplaced (0.611%); 2.5 MiB/s, 41 objects/s recovering 2023-08-28T02:32:24.686 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:24 smithi143 ceph-mon[189871]: pgmap v702: 99 pgs: 1 active+recovering+undersized+remapped, 25 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 32 MiB/s wr, 507 op/s; 52898/254502 objects degraded (20.785%); 1554/254502 objects misplaced (0.611%); 2.5 MiB/s, 41 objects/s recovering 2023-08-28T02:32:24.689 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:32:24.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:24.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:32:24.697 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:24 smithi169 ceph-mon[159415]: pgmap v702: 99 pgs: 1 active+recovering+undersized+remapped, 25 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 32 MiB/s wr, 507 op/s; 52898/254502 objects degraded (20.785%); 1554/254502 objects misplaced (0.611%); 2.5 MiB/s, 41 objects/s recovering 2023-08-28T02:32:24.796 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:32:24.865 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 83 16 88073 88057 66.2619 40.375 0.0298897 0.0150843 2023-08-28T02:32:24.920 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:25.865 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 84 16 88816 88800 66.026 46.4375 0.00914491 0.0151366 2023-08-28T02:32:26.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:26 smithi143 ceph-mon[183059]: pgmap v703: 99 pgs: 1 active+recovering+undersized+remapped, 25 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 43 MiB/s wr, 685 op/s; 52898/261627 objects degraded (20.219%); 1178/261627 objects misplaced (0.450%); 4.6 MiB/s, 74 objects/s recovering 2023-08-28T02:32:26.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:26 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:32:26.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:26 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:32:26.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:26 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 52898/261627 objects degraded (20.219%), 25 pgs degraded (PG_DEGRADED) 2023-08-28T02:32:26.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:26 smithi143 ceph-mon[189871]: pgmap v703: 99 pgs: 1 active+recovering+undersized+remapped, 25 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 43 MiB/s wr, 685 op/s; 52898/261627 objects degraded (20.219%); 1178/261627 objects misplaced (0.450%); 4.6 MiB/s, 74 objects/s recovering 2023-08-28T02:32:26.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:26 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:32:26.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:26 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:32:26.481 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:32:26.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:26.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:32:26.586 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:32:26.662 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:26.707 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:26 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 52898/261627 objects degraded (20.219%), 25 pgs degraded (PG_DEGRADED) 2023-08-28T02:32:26.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:26 smithi169 ceph-mon[159415]: pgmap v703: 99 pgs: 1 active+recovering+undersized+remapped, 25 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 43 MiB/s wr, 685 op/s; 52898/261627 objects degraded (20.219%); 1178/261627 objects misplaced (0.450%); 4.6 MiB/s, 74 objects/s recovering 2023-08-28T02:32:26.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:26 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:32:26.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:26 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:32:26.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:26 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 52898/261627 objects degraded (20.219%), 25 pgs degraded (PG_DEGRADED) 2023-08-28T02:32:26.819 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:32:26.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:26.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:32:26.865 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 85 16 89516 89500 65.764 43.75 0.0255377 0.0152001 2023-08-28T02:32:26.932 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:32:27.021 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:27.300 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:32:27.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:27.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:32:27.351 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:32:27.418 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:32:27.447 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:27.503 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:32:27.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:27.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:32:27.566 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:32:27.612 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:32:27.704 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:27.869 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 86 16 90176 90160 65.4759 41.25 0.0189243 0.0152677 2023-08-28T02:32:27.958 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:32:27.964 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:32:27.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:27.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:32:27.978 INFO:teuthology.orchestra.run.smithi143.stderr:dumped all 2023-08-28T02:32:28.069 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:32:28.102 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:28.161 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:32:28.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:28.167 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:32:28.268 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:32:28.354 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:28.460 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:28 smithi169 ceph-mon[159415]: pgmap v704: 99 pgs: 1 active+recovering+undersized+remapped, 25 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 47 MiB/s wr, 747 op/s; 52898/265263 objects degraded (19.942%); 1178/265263 objects misplaced (0.444%); 4.2 MiB/s, 68 objects/s recovering 2023-08-28T02:32:28.460 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:28 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3813176952' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:32:28.460 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:28 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:32:28.526 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is set 2023-08-28T02:32:28.569 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:28 smithi143 ceph-mon[183059]: pgmap v704: 99 pgs: 1 active+recovering+undersized+remapped, 25 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 47 MiB/s wr, 747 op/s; 52898/265263 objects degraded (19.942%); 1178/265263 objects misplaced (0.444%); 4.2 MiB/s, 68 objects/s recovering 2023-08-28T02:32:28.569 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:28 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3813176952' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:32:28.570 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:28 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:32:28.570 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:28 smithi143 ceph-mon[189871]: pgmap v704: 99 pgs: 1 active+recovering+undersized+remapped, 25 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 47 MiB/s wr, 747 op/s; 52898/265263 objects degraded (19.942%); 1178/265263 objects misplaced (0.444%); 4.2 MiB/s, 68 objects/s recovering 2023-08-28T02:32:28.570 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:28 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3813176952' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:32:28.570 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:28 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:32:28.869 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 87 16 90954 90938 65.2823 48.625 0.0167631 0.0153124 2023-08-28T02:32:29.591 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:32:29.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:29.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:32:29.709 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:32:29.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:29 smithi169 ceph-mon[159415]: from='client.55366 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:32:29.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:29 smithi169 ceph-mon[159415]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:32:29.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:29 smithi169 ceph-mon[159415]: failed to encode map e285 with expected crc 2023-08-28T02:32:29.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:29 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:32:29.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:29 smithi169 ceph-mon[159415]: osdmap e285: 8 total, 8 up, 6 in 2023-08-28T02:32:29.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:29 smithi169 ceph-mon[159415]: failed to encode map e285 with expected crc 2023-08-28T02:32:29.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:29 smithi169 ceph-mon[159415]: failed to encode map e285 with expected crc 2023-08-28T02:32:29.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:29 smithi169 ceph-mon[159415]: failed to encode map e285 with expected crc 2023-08-28T02:32:29.753 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:29.809 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:32:29.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:29.813 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:32:29.869 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 88 16 91524 91508 64.9454 35.625 0.0129462 0.015379 2023-08-28T02:32:29.914 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:32:29.943 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:29.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:29 smithi143 ceph-mon[183059]: from='client.55366 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:32:29.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:29 smithi143 ceph-mon[183059]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:32:29.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:29 smithi143 ceph-mon[183059]: failed to encode map e285 with expected crc 2023-08-28T02:32:29.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:29 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:32:29.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:29 smithi143 ceph-mon[183059]: osdmap e285: 8 total, 8 up, 6 in 2023-08-28T02:32:29.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:29 smithi143 ceph-mon[183059]: failed to encode map e285 with expected crc 2023-08-28T02:32:29.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:29 smithi143 ceph-mon[183059]: failed to encode map e285 with expected crc 2023-08-28T02:32:29.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:29 smithi143 ceph-mon[183059]: failed to encode map e285 with expected crc 2023-08-28T02:32:29.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:29 smithi143 ceph-mon[189871]: from='client.55366 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:32:29.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:29 smithi143 ceph-mon[189871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:32:29.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:29 smithi143 ceph-mon[189871]: failed to encode map e285 with expected crc 2023-08-28T02:32:29.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:29 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:32:29.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:29 smithi143 ceph-mon[189871]: osdmap e285: 8 total, 8 up, 6 in 2023-08-28T02:32:29.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:29 smithi143 ceph-mon[189871]: failed to encode map e285 with expected crc 2023-08-28T02:32:29.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:29 smithi143 ceph-mon[189871]: failed to encode map e285 with expected crc 2023-08-28T02:32:29.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:29 smithi143 ceph-mon[189871]: failed to encode map e285 with expected crc 2023-08-28T02:32:30.011 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:32:30.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:30.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:32:30.119 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:32:30.206 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:30.872 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 89 16 92133 92117 64.6413 38.0625 0.0541742 0.0154608 2023-08-28T02:32:30.934 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:30 smithi143 ceph-mon[183059]: pgmap v706: 99 pgs: 1 active+recovering+undersized+remapped, 25 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 45 MiB/s wr, 724 op/s; 52898/268020 objects degraded (19.737%); 743/268020 objects misplaced (0.277%); 5.1 MiB/s, 81 objects/s recovering 2023-08-28T02:32:30.935 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:30 smithi143 ceph-mon[189871]: pgmap v706: 99 pgs: 1 active+recovering+undersized+remapped, 25 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 45 MiB/s wr, 724 op/s; 52898/268020 objects degraded (19.737%); 743/268020 objects misplaced (0.277%); 5.1 MiB/s, 81 objects/s recovering 2023-08-28T02:32:30.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:30 smithi169 ceph-mon[159415]: pgmap v706: 99 pgs: 1 active+recovering+undersized+remapped, 25 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 45 MiB/s wr, 724 op/s; 52898/268020 objects degraded (19.737%); 743/268020 objects misplaced (0.277%); 5.1 MiB/s, 81 objects/s recovering 2023-08-28T02:32:31.190 INFO:teuthology.orchestra.run.smithi143.stdout:{"pg_ready":true,"pg_map":{"version":704,"stamp":"2023-08-28T02:32:27.367674+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5797713843,"num_objects":88421,"num_object_clones":0,"num_object_copies":265263,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":52898,"num_objects_misplaced":1178,"num_objects_unfound":0,"num_objects_dirty":88421,"num_whiteouts":0,"num_read":6274,"num_read_kb":13707,"num_write":89499,"num_write_kb":5672293,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3011,"num_bytes_recovered":192569980,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":126404,"ondisk_log_size":126404,"up":297,"acting":271,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":301,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":15950844,"kb_used_data":13785420,"kb_used_omap":22,"kb_used_meta":2165353,"kb_avail":546520068,"statfs":{"total":575970213888,"available":559636549632,"internally_reserved":0,"allocated":14116270080,"data_stored":14110240363,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22813,"internal_metadata":2217322211},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,6,9,4,1,0,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":528941056,"num_objects":8071,"num_object_clones":0,"num_object_copies":24213,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14915,"num_objects_misplaced":-731,"num_objects_unfound":0,"num_objects_dirty":8071,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8071,"num_write_kb":516544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":741,"num_bytes_recovered":47972352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.793828"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":294,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.543702+0000","last_change":"2023-08-28T02:32:12.539921+0000","last_active":"2023-08-28T02:32:14.543702+0000","last_peered":"2023-08-28T02:32:14.543702+0000","last_clean":"2023-08-28T02:32:14.543702+0000","last_became_active":"2023-08-28T02:32:12.539525+0000","last_became_peered":"2023-08-28T02:32:12.539525+0000","last_unstale":"2023-08-28T02:32:14.543702+0000","last_undegraded":"2023-08-28T02:32:14.543702+0000","last_fullsized":"2023-08-28T02:32:14.543702+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":180,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.622382+0000","last_change":"2023-08-28T02:32:12.548921+0000","last_active":"2023-08-28T02:32:14.622382+0000","last_peered":"2023-08-28T02:32:14.622382+0000","last_clean":"2023-08-28T02:32:14.622382+0000","last_became_active":"2023-08-28T02:32:12.548637+0000","last_became_peered":"2023-08-28T02:32:12.548637+0000","last_unstale":"2023-08-28T02:32:14.622382+0000","last_undegraded":"2023-08-28T02:32:14.622382+0000","last_fullsized":"2023-08-28T02:32:14.622382+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:29:19.998560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":147,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.619934+0000","last_change":"2023-08-28T02:32:12.556957+0000","last_active":"2023-08-28T02:32:14.619934+0000","last_peered":"2023-08-28T02:32:14.619934+0000","last_clean":"2023-08-28T02:32:14.619934+0000","last_became_active":"2023-08-28T02:32:12.556654+0000","last_became_peered":"2023-08-28T02:32:12.556654+0000","last_unstale":"2023-08-28T02:32:14.619934+0000","last_undegraded":"2023-08-28T02:32:14.619934+0000","last_fullsized":"2023-08-28T02:32:14.619934+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:02:42.648965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":290,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.621180+0000","last_change":"2023-08-28T02:32:12.561131+0000","last_active":"2023-08-28T02:32:14.621180+0000","last_peered":"2023-08-28T02:32:14.621180+0000","last_clean":"2023-08-28T02:32:14.621180+0000","last_became_active":"2023-08-28T02:32:12.560745+0000","last_became_peered":"2023-08-28T02:32:12.560745+0000","last_unstale":"2023-08-28T02:32:14.621180+0000","last_undegraded":"2023-08-28T02:32:14.621180+0000","last_fullsized":"2023-08-28T02:32:14.621180+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:35:44.081233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":171,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.622959+0000","last_change":"2023-08-28T02:32:12.566622+0000","last_active":"2023-08-28T02:32:14.622959+0000","last_peered":"2023-08-28T02:32:14.622959+0000","last_clean":"2023-08-28T02:32:14.622959+0000","last_became_active":"2023-08-28T02:32:12.566301+0000","last_became_peered":"2023-08-28T02:32:12.566301+0000","last_unstale":"2023-08-28T02:32:14.622959+0000","last_undegraded":"2023-08-28T02:32:14.622959+0000","last_fullsized":"2023-08-28T02:32:14.622959+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:38:55.871441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":81,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.546019+0000","last_change":"2023-08-28T02:32:12.545437+0000","last_active":"2023-08-28T02:32:14.546019+0000","last_peered":"2023-08-28T02:32:14.546019+0000","last_clean":"2023-08-28T02:32:14.546019+0000","last_became_active":"2023-08-28T02:32:12.545052+0000","last_became_peered":"2023-08-28T02:32:12.545052+0000","last_unstale":"2023-08-28T02:32:14.546019+0000","last_undegraded":"2023-08-28T02:32:14.546019+0000","last_fullsized":"2023-08-28T02:32:14.546019+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.13","version":"284'4635","reported_seq":4677,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-08-28T02:32:24.636715+0000","last_change":"2023-08-28T02:31:44.484177+0000","last_active":"2023-08-28T02:32:24.636715+0000","last_peered":"2023-08-28T02:32:24.636715+0000","last_clean":"2023-08-28T02:32:24.636715+0000","last_became_active":"2023-08-28T02:31:44.482443+0000","last_became_peered":"2023-08-28T02:31:44.482443+0000","last_unstale":"2023-08-28T02:32:24.636715+0000","last_undegraded":"2023-08-28T02:32:24.636715+0000","last_fullsized":"2023-08-28T02:32:24.636715+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":4635,"log_dups_size":0,"ondisk_log_size":4635,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181207040,"num_objects":2765,"num_object_clones":0,"num_object_copies":8295,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2765,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2765,"num_write_kb":176960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":252,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.629198+0000","last_change":"2023-08-28T02:23:59.227592+0000","last_active":"2023-08-28T02:32:14.629198+0000","last_peered":"2023-08-28T02:32:14.629198+0000","last_clean":"2023-08-28T02:32:14.629198+0000","last_became_active":"2023-08-28T02:23:59.227342+0000","last_became_peered":"2023-08-28T02:23:59.227342+0000","last_unstale":"2023-08-28T02:32:14.629198+0000","last_undegraded":"2023-08-28T02:32:14.629198+0000","last_fullsized":"2023-08-28T02:32:14.629198+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:08:19.874396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":189,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.622984+0000","last_change":"2023-08-28T02:32:12.540139+0000","last_active":"2023-08-28T02:32:14.622984+0000","last_peered":"2023-08-28T02:32:14.622984+0000","last_clean":"2023-08-28T02:32:14.622984+0000","last_became_active":"2023-08-28T02:32:12.538934+0000","last_became_peered":"2023-08-28T02:32:12.538934+0000","last_unstale":"2023-08-28T02:32:14.622984+0000","last_undegraded":"2023-08-28T02:32:14.622984+0000","last_fullsized":"2023-08-28T02:32:14.622984+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:37:56.848843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.12","version":"284'4695","reported_seq":4745,"reported_epoch":284,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:32:24.634712+0000","last_change":"2023-08-28T02:32:13.713549+0000","last_active":"2023-08-28T02:32:24.634712+0000","last_peered":"2023-08-28T02:32:24.634712+0000","last_clean":"2023-08-28T02:32:11.498129+0000","last_became_active":"2023-08-28T02:32:13.708666+0000","last_became_peered":"2023-08-28T02:32:13.708666+0000","last_unstale":"2023-08-28T02:32:24.634712+0000","last_undegraded":"2023-08-28T02:32:13.549267+0000","last_fullsized":"2023-08-28T02:32:13.523984+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":4695,"log_dups_size":0,"ondisk_log_size":4695,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182845440,"num_objects":2790,"num_object_clones":0,"num_object_copies":8370,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2542,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2790,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2790,"num_write_kb":178560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2],"avail_no_missing":["3","1","2"],"object_location_counts":[{"shards":"1,2,3","objects":2542},{"shards":"2,3","objects":248}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":304,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.545308+0000","last_change":"2023-08-28T02:32:12.559057+0000","last_active":"2023-08-28T02:32:14.545308+0000","last_peered":"2023-08-28T02:32:14.545308+0000","last_clean":"2023-08-28T02:32:14.545308+0000","last_became_active":"2023-08-28T02:32:12.558620+0000","last_became_peered":"2023-08-28T02:32:12.558620+0000","last_unstale":"2023-08-28T02:32:14.545308+0000","last_undegraded":"2023-08-28T02:32:14.545308+0000","last_fullsized":"2023-08-28T02:32:14.545308+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":171,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.620098+0000","last_change":"2023-08-28T02:31:55.525178+0000","last_active":"2023-08-28T02:32:14.620098+0000","last_peered":"2023-08-28T02:32:14.620098+0000","last_clean":"2023-08-28T02:32:14.620098+0000","last_became_active":"2023-08-28T02:31:55.524801+0000","last_became_peered":"2023-08-28T02:31:55.524801+0000","last_unstale":"2023-08-28T02:32:14.620098+0000","last_undegraded":"2023-08-28T02:32:14.620098+0000","last_fullsized":"2023-08-28T02:32:14.620098+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:47:30.078960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.11","version":"284'4725","reported_seq":4774,"reported_epoch":284,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:32:27.027463+0000","last_change":"2023-08-28T02:32:12.658439+0000","last_active":"2023-08-28T02:32:27.027463+0000","last_peered":"2023-08-28T02:32:27.027463+0000","last_clean":"2023-08-28T02:31:51.768660+0000","last_became_active":"2023-08-28T02:32:12.654076+0000","last_became_peered":"2023-08-28T02:32:12.654076+0000","last_unstale":"2023-08-28T02:32:27.027463+0000","last_undegraded":"2023-08-28T02:32:12.552682+0000","last_fullsized":"2023-08-28T02:32:12.528254+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":4725,"log_dups_size":0,"ondisk_log_size":4725,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186155008,"num_objects":2841,"num_object_clones":0,"num_object_copies":8523,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2554,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2841,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2841,"num_write_kb":181792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3],"avail_no_missing":["6","3","4"],"object_location_counts":[{"shards":"1,3,6","objects":377},{"shards":"3,4,6","objects":2177},{"shards":"3,6","objects":287}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":50,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.622618+0000","last_change":"2023-08-28T02:31:55.532306+0000","last_active":"2023-08-28T02:32:14.622618+0000","last_peered":"2023-08-28T02:32:14.622618+0000","last_clean":"2023-08-28T02:32:14.622618+0000","last_became_active":"2023-08-28T02:31:55.524276+0000","last_became_peered":"2023-08-28T02:31:55.524276+0000","last_unstale":"2023-08-28T02:32:14.622618+0000","last_undegraded":"2023-08-28T02:32:14.622618+0000","last_fullsized":"2023-08-28T02:32:14.622618+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:27:06.850784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":43,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.544892+0000","last_change":"2023-08-28T02:32:12.545376+0000","last_active":"2023-08-28T02:32:14.544892+0000","last_peered":"2023-08-28T02:32:14.544892+0000","last_clean":"2023-08-28T02:32:14.544892+0000","last_became_active":"2023-08-28T02:32:12.544509+0000","last_became_peered":"2023-08-28T02:32:12.544509+0000","last_unstale":"2023-08-28T02:32:14.544892+0000","last_undegraded":"2023-08-28T02:32:14.544892+0000","last_fullsized":"2023-08-28T02:32:14.544892+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.10","version":"284'4572","reported_seq":6926,"reported_epoch":284,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:32:24.638919+0000","last_change":"2023-08-28T02:32:13.720731+0000","last_active":"2023-08-28T02:32:24.638919+0000","last_peered":"2023-08-28T02:32:24.638919+0000","last_clean":"2023-08-28T02:31:51.241963+0000","last_became_active":"2023-08-28T02:32:13.712970+0000","last_became_peered":"2023-08-28T02:32:13.712970+0000","last_unstale":"2023-08-28T02:32:24.638919+0000","last_undegraded":"2023-08-28T02:32:13.660204+0000","last_fullsized":"2023-08-28T02:32:13.525855+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":4572,"log_dups_size":0,"ondisk_log_size":4572,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178421760,"num_objects":2723,"num_object_clones":0,"num_object_copies":8169,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3486,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2723,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2723,"num_write_kb":174240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1148,"num_bytes_recovered":75235328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,2,3","objects":391},{"shards":"1,2,3,5","objects":1149},{"shards":"1,3,5","objects":973},{"shards":"2,3","objects":210}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"86'22","reported_seq":353,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.545540+0000","last_change":"2023-08-28T02:31:55.538001+0000","last_active":"2023-08-28T02:32:14.545540+0000","last_peered":"2023-08-28T02:32:14.545540+0000","last_clean":"2023-08-28T02:32:14.545540+0000","last_became_active":"2023-08-28T02:31:55.518604+0000","last_became_peered":"2023-08-28T02:31:55.518604+0000","last_unstale":"2023-08-28T02:32:14.545540+0000","last_undegraded":"2023-08-28T02:32:14.545540+0000","last_fullsized":"2023-08-28T02:32:14.545540+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":318,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.543974+0000","last_change":"2023-08-28T02:32:12.532691+0000","last_active":"2023-08-28T02:32:14.543974+0000","last_peered":"2023-08-28T02:32:14.543974+0000","last_clean":"2023-08-28T02:32:14.543974+0000","last_became_active":"2023-08-28T02:32:12.532358+0000","last_became_peered":"2023-08-28T02:32:12.532358+0000","last_unstale":"2023-08-28T02:32:14.543974+0000","last_undegraded":"2023-08-28T02:32:14.543974+0000","last_fullsized":"2023-08-28T02:32:14.543974+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.17","version":"284'4644","reported_seq":4682,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-08-28T02:32:27.079919+0000","last_change":"2023-08-28T02:31:44.475304+0000","last_active":"2023-08-28T02:32:27.079919+0000","last_peered":"2023-08-28T02:32:27.079919+0000","last_clean":"2023-08-28T02:32:27.079919+0000","last_became_active":"2023-08-28T02:31:44.473853+0000","last_became_peered":"2023-08-28T02:31:44.473853+0000","last_unstale":"2023-08-28T02:32:27.079919+0000","last_undegraded":"2023-08-28T02:32:27.079919+0000","last_fullsized":"2023-08-28T02:32:27.079919+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":4644,"log_dups_size":0,"ondisk_log_size":4644,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182190080,"num_objects":2780,"num_object_clones":0,"num_object_copies":8340,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2780,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2780,"num_write_kb":177920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":57,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.544066+0000","last_change":"2023-08-28T02:31:55.528552+0000","last_active":"2023-08-28T02:32:14.544066+0000","last_peered":"2023-08-28T02:32:14.544066+0000","last_clean":"2023-08-28T02:32:14.544066+0000","last_became_active":"2023-08-28T02:31:55.518079+0000","last_became_peered":"2023-08-28T02:31:55.518079+0000","last_unstale":"2023-08-28T02:32:14.544066+0000","last_undegraded":"2023-08-28T02:32:14.544066+0000","last_fullsized":"2023-08-28T02:32:14.544066+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":180,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.621432+0000","last_change":"2023-08-28T02:32:12.554706+0000","last_active":"2023-08-28T02:32:14.621432+0000","last_peered":"2023-08-28T02:32:14.621432+0000","last_clean":"2023-08-28T02:32:14.621432+0000","last_became_active":"2023-08-28T02:32:12.554420+0000","last_became_peered":"2023-08-28T02:32:12.554420+0000","last_unstale":"2023-08-28T02:32:14.621432+0000","last_undegraded":"2023-08-28T02:32:14.621432+0000","last_fullsized":"2023-08-28T02:32:14.621432+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:26:56.387675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.16","version":"284'4470","reported_seq":6384,"reported_epoch":284,"state":"active+recovering+undersized+remapped","last_fresh":"2023-08-28T02:32:23.976287+0000","last_change":"2023-08-28T02:32:12.628513+0000","last_active":"2023-08-28T02:32:23.976287+0000","last_peered":"2023-08-28T02:32:23.976287+0000","last_clean":"2023-08-28T02:31:54.342331+0000","last_became_active":"2023-08-28T02:31:56.608729+0000","last_became_peered":"2023-08-28T02:31:56.608729+0000","last_unstale":"2023-08-28T02:32:23.976287+0000","last_undegraded":"2023-08-28T02:32:23.976287+0000","last_fullsized":"2023-08-28T02:31:56.507801+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":4470,"log_dups_size":0,"ondisk_log_size":4470,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174555136,"num_objects":2664,"num_object_clones":0,"num_object_copies":7992,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1178,"num_objects_unfound":0,"num_objects_dirty":2664,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2664,"num_write_kb":170464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":933,"num_bytes_recovered":61145088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":1486},{"shards":"0,3,7","objects":1178}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":288,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.619066+0000","last_change":"2023-08-28T02:32:12.541306+0000","last_active":"2023-08-28T02:32:14.619066+0000","last_peered":"2023-08-28T02:32:14.619066+0000","last_clean":"2023-08-28T02:32:14.619066+0000","last_became_active":"2023-08-28T02:32:12.540974+0000","last_became_peered":"2023-08-28T02:32:12.540974+0000","last_unstale":"2023-08-28T02:32:14.619066+0000","last_undegraded":"2023-08-28T02:32:14.619066+0000","last_fullsized":"2023-08-28T02:32:14.619066+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:13:23.174912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":319,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.543931+0000","last_change":"2023-08-28T02:31:55.521470+0000","last_active":"2023-08-28T02:32:14.543931+0000","last_peered":"2023-08-28T02:32:14.543931+0000","last_clean":"2023-08-28T02:32:14.543931+0000","last_became_active":"2023-08-28T02:31:55.520398+0000","last_became_peered":"2023-08-28T02:31:55.520398+0000","last_unstale":"2023-08-28T02:32:14.543931+0000","last_undegraded":"2023-08-28T02:32:14.543931+0000","last_fullsized":"2023-08-28T02:32:14.543931+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.15","version":"284'4555","reported_seq":4609,"reported_epoch":284,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:32:24.546414+0000","last_change":"2023-08-28T02:32:12.688339+0000","last_active":"2023-08-28T02:32:24.546414+0000","last_peered":"2023-08-28T02:32:24.546414+0000","last_clean":"2023-08-28T02:31:54.454430+0000","last_became_active":"2023-08-28T02:32:12.680524+0000","last_became_peered":"2023-08-28T02:32:12.680524+0000","last_unstale":"2023-08-28T02:32:24.546414+0000","last_undegraded":"2023-08-28T02:32:12.595476+0000","last_fullsized":"2023-08-28T02:32:12.567587+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":4555,"log_dups_size":0,"ondisk_log_size":4555,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179077120,"num_objects":2733,"num_object_clones":0,"num_object_copies":8199,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2501,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2733,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2733,"num_write_kb":174880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0],"avail_no_missing":["2","0","7"],"object_location_counts":[{"shards":"0,1,2","objects":354},{"shards":"0,2","objects":232},{"shards":"0,2,7","objects":2147}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":113,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.545043+0000","last_change":"2023-08-28T02:31:55.517851+0000","last_active":"2023-08-28T02:32:14.545043+0000","last_peered":"2023-08-28T02:32:14.545043+0000","last_clean":"2023-08-28T02:32:14.545043+0000","last_became_active":"2023-08-28T02:31:55.517320+0000","last_became_peered":"2023-08-28T02:31:55.517320+0000","last_unstale":"2023-08-28T02:32:14.545043+0000","last_undegraded":"2023-08-28T02:32:14.545043+0000","last_fullsized":"2023-08-28T02:32:14.545043+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":280,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.621965+0000","last_change":"2023-08-28T02:31:55.528250+0000","last_active":"2023-08-28T02:32:14.621965+0000","last_peered":"2023-08-28T02:32:14.621965+0000","last_clean":"2023-08-28T02:32:14.621965+0000","last_became_active":"2023-08-28T02:31:55.527950+0000","last_became_peered":"2023-08-28T02:31:55.527950+0000","last_unstale":"2023-08-28T02:32:14.621965+0000","last_undegraded":"2023-08-28T02:32:14.621965+0000","last_fullsized":"2023-08-28T02:32:14.621965+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:39:46.120437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.14","version":"284'4700","reported_seq":4750,"reported_epoch":284,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:32:23.761412+0000","last_change":"2023-08-28T02:32:13.751679+0000","last_active":"2023-08-28T02:32:23.761412+0000","last_peered":"2023-08-28T02:32:23.761412+0000","last_clean":"2023-08-28T02:32:11.458469+0000","last_became_active":"2023-08-28T02:32:13.745927+0000","last_became_peered":"2023-08-28T02:32:13.745927+0000","last_unstale":"2023-08-28T02:32:23.761412+0000","last_undegraded":"2023-08-28T02:32:13.557474+0000","last_fullsized":"2023-08-28T02:32:13.527835+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":4700,"log_dups_size":0,"ondisk_log_size":4700,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181633024,"num_objects":2772,"num_object_clones":0,"num_object_copies":8316,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2477,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2772,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2772,"num_write_kb":177376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":2477},{"shards":"0,4","objects":295}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":82,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.543712+0000","last_change":"2023-08-28T02:28:25.703333+0000","last_active":"2023-08-28T02:32:14.543712+0000","last_peered":"2023-08-28T02:32:14.543712+0000","last_clean":"2023-08-28T02:32:14.543712+0000","last_became_active":"2023-08-28T02:28:25.343515+0000","last_became_peered":"2023-08-28T02:28:25.343515+0000","last_unstale":"2023-08-28T02:32:14.543712+0000","last_undegraded":"2023-08-28T02:32:14.543712+0000","last_fullsized":"2023-08-28T02:32:14.543712+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":180,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.621920+0000","last_change":"2023-08-28T02:32:12.557826+0000","last_active":"2023-08-28T02:32:14.621920+0000","last_peered":"2023-08-28T02:32:14.621920+0000","last_clean":"2023-08-28T02:32:14.621920+0000","last_became_active":"2023-08-28T02:32:12.557519+0000","last_became_peered":"2023-08-28T02:32:12.557519+0000","last_unstale":"2023-08-28T02:32:14.621920+0000","last_undegraded":"2023-08-28T02:32:14.621920+0000","last_fullsized":"2023-08-28T02:32:14.621920+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:03:26.544225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"284'5577","reported_seq":5616,"reported_epoch":284,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:32:26.896746+0000","last_change":"2023-08-28T02:32:13.749012+0000","last_active":"2023-08-28T02:32:26.896746+0000","last_peered":"2023-08-28T02:32:26.896746+0000","last_clean":"2023-08-28T02:32:11.456138+0000","last_became_active":"2023-08-28T02:32:13.738316+0000","last_became_peered":"2023-08-28T02:32:13.738316+0000","last_unstale":"2023-08-28T02:32:26.896746+0000","last_undegraded":"2023-08-28T02:32:13.561921+0000","last_fullsized":"2023-08-28T02:32:13.526057+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5577,"log_dups_size":0,"ondisk_log_size":5577,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365494272,"num_objects":5577,"num_object_clones":0,"num_object_copies":16731,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4923,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5577,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5577,"num_write_kb":356928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6],"avail_no_missing":["5","1","6"],"object_location_counts":[{"shards":"1,5,6","objects":4923},{"shards":"5,6","objects":654}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":132,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.625291+0000","last_change":"2023-08-28T02:32:12.592204+0000","last_active":"2023-08-28T02:32:14.625291+0000","last_peered":"2023-08-28T02:32:14.625291+0000","last_clean":"2023-08-28T02:32:14.625291+0000","last_became_active":"2023-08-28T02:32:12.552412+0000","last_became_peered":"2023-08-28T02:32:12.552412+0000","last_unstale":"2023-08-28T02:32:14.625291+0000","last_undegraded":"2023-08-28T02:32:14.625291+0000","last_fullsized":"2023-08-28T02:32:14.625291+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:11:10.774576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":279,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.545669+0000","last_change":"2023-08-28T02:31:55.514646+0000","last_active":"2023-08-28T02:32:14.545669+0000","last_peered":"2023-08-28T02:32:14.545669+0000","last_clean":"2023-08-28T02:32:14.545669+0000","last_became_active":"2023-08-28T02:31:55.514306+0000","last_became_peered":"2023-08-28T02:31:55.514306+0000","last_unstale":"2023-08-28T02:32:14.545669+0000","last_undegraded":"2023-08-28T02:32:14.545669+0000","last_fullsized":"2023-08-28T02:32:14.545669+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"284'5599","reported_seq":5631,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-08-28T02:32:24.619213+0000","last_change":"2023-08-28T02:30:57.067718+0000","last_active":"2023-08-28T02:32:24.619213+0000","last_peered":"2023-08-28T02:32:24.619213+0000","last_clean":"2023-08-28T02:32:24.619213+0000","last_became_active":"2023-08-28T02:30:57.067407+0000","last_became_peered":"2023-08-28T02:30:57.067407+0000","last_unstale":"2023-08-28T02:32:24.619213+0000","last_undegraded":"2023-08-28T02:32:24.619213+0000","last_fullsized":"2023-08-28T02:32:24.619213+0000","mapping_epoch":261,"log_start":"265'665","ondisk_log_start":"265'665","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":4934,"log_dups_size":0,"ondisk_log_size":4934,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:49:39.360134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":366936064,"num_objects":5599,"num_object_clones":0,"num_object_copies":16797,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5599,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5599,"num_write_kb":358336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":66,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.545178+0000","last_change":"2023-08-28T02:32:12.553341+0000","last_active":"2023-08-28T02:32:14.545178+0000","last_peered":"2023-08-28T02:32:14.545178+0000","last_clean":"2023-08-28T02:32:14.545178+0000","last_became_active":"2023-08-28T02:32:12.548020+0000","last_became_peered":"2023-08-28T02:32:12.548020+0000","last_unstale":"2023-08-28T02:32:14.545178+0000","last_undegraded":"2023-08-28T02:32:14.545178+0000","last_fullsized":"2023-08-28T02:32:14.545178+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":328,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.544960+0000","last_change":"2023-08-28T02:32:12.555667+0000","last_active":"2023-08-28T02:32:14.544960+0000","last_peered":"2023-08-28T02:32:14.544960+0000","last_clean":"2023-08-28T02:32:14.544960+0000","last_became_active":"2023-08-28T02:32:12.555381+0000","last_became_peered":"2023-08-28T02:32:12.555381+0000","last_unstale":"2023-08-28T02:32:14.544960+0000","last_undegraded":"2023-08-28T02:32:14.544960+0000","last_fullsized":"2023-08-28T02:32:14.544960+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"284'4652","reported_seq":4702,"reported_epoch":284,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:32:23.828221+0000","last_change":"2023-08-28T02:32:13.761239+0000","last_active":"2023-08-28T02:32:23.828221+0000","last_peered":"2023-08-28T02:32:23.828221+0000","last_clean":"2023-08-28T02:32:11.446297+0000","last_became_active":"2023-08-28T02:32:13.756862+0000","last_became_peered":"2023-08-28T02:32:13.756862+0000","last_unstale":"2023-08-28T02:32:23.828221+0000","last_undegraded":"2023-08-28T02:32:13.559831+0000","last_fullsized":"2023-08-28T02:32:13.528960+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":4652,"log_dups_size":0,"ondisk_log_size":4652,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180060160,"num_objects":2747,"num_object_clones":0,"num_object_copies":8241,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2538,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2747,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2747,"num_write_kb":175840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":2538},{"shards":"0,4","objects":209}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":53,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.544004+0000","last_change":"2023-08-28T02:31:55.531216+0000","last_active":"2023-08-28T02:32:14.544004+0000","last_peered":"2023-08-28T02:32:14.544004+0000","last_clean":"2023-08-28T02:32:14.544004+0000","last_became_active":"2023-08-28T02:31:55.521010+0000","last_became_peered":"2023-08-28T02:31:55.521010+0000","last_unstale":"2023-08-28T02:32:14.544004+0000","last_undegraded":"2023-08-28T02:32:14.544004+0000","last_fullsized":"2023-08-28T02:32:14.544004+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":90,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.544770+0000","last_change":"2023-08-28T02:32:12.545719+0000","last_active":"2023-08-28T02:32:14.544770+0000","last_peered":"2023-08-28T02:32:14.544770+0000","last_clean":"2023-08-28T02:32:14.544770+0000","last_became_active":"2023-08-28T02:32:12.545389+0000","last_became_peered":"2023-08-28T02:32:12.545389+0000","last_unstale":"2023-08-28T02:32:14.544770+0000","last_undegraded":"2023-08-28T02:32:14.544770+0000","last_fullsized":"2023-08-28T02:32:14.544770+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"284'4684","reported_seq":4733,"reported_epoch":284,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:32:24.571034+0000","last_change":"2023-08-28T02:31:56.709104+0000","last_active":"2023-08-28T02:32:24.571034+0000","last_peered":"2023-08-28T02:32:24.571034+0000","last_clean":"2023-08-28T02:31:54.438195+0000","last_became_active":"2023-08-28T02:31:56.704078+0000","last_became_peered":"2023-08-28T02:31:56.704078+0000","last_unstale":"2023-08-28T02:32:24.571034+0000","last_undegraded":"2023-08-28T02:31:56.543040+0000","last_fullsized":"2023-08-28T02:31:56.504447+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":4684,"log_dups_size":0,"ondisk_log_size":4684,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181272576,"num_objects":2766,"num_object_clones":0,"num_object_copies":8298,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2171,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2766,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2766,"num_write_kb":177024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4],"avail_no_missing":["2","4","7"],"object_location_counts":[{"shards":"2,4","objects":595},{"shards":"2,4,7","objects":2171}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":286,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.621538+0000","last_change":"2023-08-28T02:28:25.445982+0000","last_active":"2023-08-28T02:32:14.621538+0000","last_peered":"2023-08-28T02:32:14.621538+0000","last_clean":"2023-08-28T02:32:14.621538+0000","last_became_active":"2023-08-28T02:28:25.443577+0000","last_became_peered":"2023-08-28T02:28:25.443577+0000","last_unstale":"2023-08-28T02:32:14.621538+0000","last_undegraded":"2023-08-28T02:32:14.621538+0000","last_fullsized":"2023-08-28T02:32:14.621538+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:24:03.381751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":274,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.546313+0000","last_change":"2023-08-28T02:32:12.537433+0000","last_active":"2023-08-28T02:32:14.546313+0000","last_peered":"2023-08-28T02:32:14.546313+0000","last_clean":"2023-08-28T02:32:14.546313+0000","last_became_active":"2023-08-28T02:32:12.537116+0000","last_became_peered":"2023-08-28T02:32:12.537116+0000","last_unstale":"2023-08-28T02:32:14.546313+0000","last_undegraded":"2023-08-28T02:32:14.546313+0000","last_fullsized":"2023-08-28T02:32:14.546313+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:39:29.641441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"284'5538","reported_seq":5578,"reported_epoch":284,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:32:23.800270+0000","last_change":"2023-08-28T02:31:56.699340+0000","last_active":"2023-08-28T02:32:23.800270+0000","last_peered":"2023-08-28T02:32:23.800270+0000","last_clean":"2023-08-28T02:31:54.434903+0000","last_became_active":"2023-08-28T02:31:56.674791+0000","last_became_peered":"2023-08-28T02:31:56.674791+0000","last_unstale":"2023-08-28T02:32:23.800270+0000","last_undegraded":"2023-08-28T02:31:56.545322+0000","last_fullsized":"2023-08-28T02:31:56.508287+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5538,"log_dups_size":0,"ondisk_log_size":5538,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:26:02.602584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362938368,"num_objects":5538,"num_object_clones":0,"num_object_copies":16614,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4325,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5538,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5538,"num_write_kb":354432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":1213},{"shards":"0,3,7","objects":4325}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":148,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.621016+0000","last_change":"2023-08-28T02:23:59.230059+0000","last_active":"2023-08-28T02:32:14.621016+0000","last_peered":"2023-08-28T02:32:14.621016+0000","last_clean":"2023-08-28T02:32:14.621016+0000","last_became_active":"2023-08-28T02:23:59.217783+0000","last_became_peered":"2023-08-28T02:23:59.217783+0000","last_unstale":"2023-08-28T02:32:14.621016+0000","last_undegraded":"2023-08-28T02:32:14.621016+0000","last_fullsized":"2023-08-28T02:32:14.621016+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:00:12.145395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":304,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.545460+0000","last_change":"2023-08-28T02:28:25.351698+0000","last_active":"2023-08-28T02:32:14.545460+0000","last_peered":"2023-08-28T02:32:14.545460+0000","last_clean":"2023-08-28T02:32:14.545460+0000","last_became_active":"2023-08-28T02:28:25.351349+0000","last_became_peered":"2023-08-28T02:28:25.351349+0000","last_unstale":"2023-08-28T02:32:14.545460+0000","last_undegraded":"2023-08-28T02:32:14.545460+0000","last_fullsized":"2023-08-28T02:32:14.545460+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"284'5574","reported_seq":5606,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-08-28T02:32:24.521337+0000","last_change":"2023-08-28T02:30:57.069579+0000","last_active":"2023-08-28T02:32:24.521337+0000","last_peered":"2023-08-28T02:32:24.521337+0000","last_clean":"2023-08-28T02:32:24.521337+0000","last_became_active":"2023-08-28T02:30:57.069365+0000","last_became_peered":"2023-08-28T02:30:57.069365+0000","last_unstale":"2023-08-28T02:32:24.521337+0000","last_undegraded":"2023-08-28T02:32:24.521337+0000","last_fullsized":"2023-08-28T02:32:24.521337+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5574,"log_dups_size":0,"ondisk_log_size":5574,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:02:07.549000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365297664,"num_objects":5574,"num_object_clones":0,"num_object_copies":16722,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5574,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5574,"num_write_kb":356736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":81,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.544507+0000","last_change":"2023-08-28T02:32:12.542737+0000","last_active":"2023-08-28T02:32:14.544507+0000","last_peered":"2023-08-28T02:32:14.544507+0000","last_clean":"2023-08-28T02:32:14.544507+0000","last_became_active":"2023-08-28T02:32:12.542358+0000","last_became_peered":"2023-08-28T02:32:12.542358+0000","last_unstale":"2023-08-28T02:32:14.544507+0000","last_undegraded":"2023-08-28T02:32:14.544507+0000","last_fullsized":"2023-08-28T02:32:14.544507+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":339,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.544834+0000","last_change":"2023-08-28T02:32:12.597747+0000","last_active":"2023-08-28T02:32:14.544834+0000","last_peered":"2023-08-28T02:32:14.544834+0000","last_clean":"2023-08-28T02:32:14.544834+0000","last_became_active":"2023-08-28T02:32:12.597325+0000","last_became_peered":"2023-08-28T02:32:12.597325+0000","last_unstale":"2023-08-28T02:32:14.544834+0000","last_undegraded":"2023-08-28T02:32:14.544834+0000","last_fullsized":"2023-08-28T02:32:14.544834+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"284'5468","reported_seq":5506,"reported_epoch":284,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:32:27.116714+0000","last_change":"2023-08-28T02:32:13.768348+0000","last_active":"2023-08-28T02:32:27.116714+0000","last_peered":"2023-08-28T02:32:27.116714+0000","last_clean":"2023-08-28T02:32:11.510497+0000","last_became_active":"2023-08-28T02:32:13.753715+0000","last_became_peered":"2023-08-28T02:32:13.753715+0000","last_unstale":"2023-08-28T02:32:27.116714+0000","last_undegraded":"2023-08-28T02:32:13.563550+0000","last_fullsized":"2023-08-28T02:32:13.528957+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5468,"log_dups_size":0,"ondisk_log_size":5468,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358350848,"num_objects":5468,"num_object_clones":0,"num_object_copies":16404,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4880,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5468,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5468,"num_write_kb":349952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0],"avail_no_missing":["6","0","1"],"object_location_counts":[{"shards":"0,1,6","objects":4880},{"shards":"0,6","objects":588}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":211,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.545124+0000","last_change":"2023-08-28T02:32:12.540152+0000","last_active":"2023-08-28T02:32:14.545124+0000","last_peered":"2023-08-28T02:32:14.545124+0000","last_clean":"2023-08-28T02:32:14.545124+0000","last_became_active":"2023-08-28T02:32:12.539745+0000","last_became_peered":"2023-08-28T02:32:12.539745+0000","last_unstale":"2023-08-28T02:32:14.545124+0000","last_undegraded":"2023-08-28T02:32:14.545124+0000","last_fullsized":"2023-08-28T02:32:14.545124+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":260,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.622048+0000","last_change":"2023-08-28T02:32:12.563231+0000","last_active":"2023-08-28T02:32:14.622048+0000","last_peered":"2023-08-28T02:32:14.622048+0000","last_clean":"2023-08-28T02:32:14.622048+0000","last_became_active":"2023-08-28T02:32:12.562912+0000","last_became_peered":"2023-08-28T02:32:12.562912+0000","last_unstale":"2023-08-28T02:32:14.622048+0000","last_undegraded":"2023-08-28T02:32:14.622048+0000","last_fullsized":"2023-08-28T02:32:14.622048+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:19:20.087638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.c","version":"284'5511","reported_seq":5549,"reported_epoch":284,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:32:27.127624+0000","last_change":"2023-08-28T02:31:56.737061+0000","last_active":"2023-08-28T02:32:27.127624+0000","last_peered":"2023-08-28T02:32:27.127624+0000","last_clean":"2023-08-28T02:31:54.457709+0000","last_became_active":"2023-08-28T02:31:56.726804+0000","last_became_peered":"2023-08-28T02:31:56.726804+0000","last_unstale":"2023-08-28T02:32:27.127624+0000","last_undegraded":"2023-08-28T02:31:56.538425+0000","last_fullsized":"2023-08-28T02:31:56.509601+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5511,"log_dups_size":0,"ondisk_log_size":5511,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361168896,"num_objects":5511,"num_object_clones":0,"num_object_copies":16533,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4274,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5511,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5511,"num_write_kb":352704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6","objects":1237},{"shards":"3,6,7","objects":4274}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":180,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.623399+0000","last_change":"2023-08-28T02:32:12.533289+0000","last_active":"2023-08-28T02:32:14.623399+0000","last_peered":"2023-08-28T02:32:14.623399+0000","last_clean":"2023-08-28T02:32:14.623399+0000","last_became_active":"2023-08-28T02:32:12.531239+0000","last_became_peered":"2023-08-28T02:32:12.531239+0000","last_unstale":"2023-08-28T02:32:14.623399+0000","last_undegraded":"2023-08-28T02:32:14.623399+0000","last_fullsized":"2023-08-28T02:32:14.623399+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:00:42.379037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":157,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.622063+0000","last_change":"2023-08-28T02:32:12.538989+0000","last_active":"2023-08-28T02:32:14.622063+0000","last_peered":"2023-08-28T02:32:14.622063+0000","last_clean":"2023-08-28T02:32:14.622063+0000","last_became_active":"2023-08-28T02:32:12.538665+0000","last_became_peered":"2023-08-28T02:32:12.538665+0000","last_unstale":"2023-08-28T02:32:14.622063+0000","last_undegraded":"2023-08-28T02:32:14.622063+0000","last_fullsized":"2023-08-28T02:32:14.622063+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:10:39.664111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.1","version":"284'4682","reported_seq":4726,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-08-28T02:32:26.853994+0000","last_change":"2023-08-28T02:32:14.983506+0000","last_active":"2023-08-28T02:32:26.853994+0000","last_peered":"2023-08-28T02:32:26.853994+0000","last_clean":"2023-08-28T02:32:26.853994+0000","last_became_active":"2023-08-28T02:31:44.479295+0000","last_became_peered":"2023-08-28T02:31:44.479295+0000","last_unstale":"2023-08-28T02:32:26.853994+0000","last_undegraded":"2023-08-28T02:32:26.853994+0000","last_fullsized":"2023-08-28T02:32:26.853994+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":4682,"log_dups_size":0,"ondisk_log_size":4682,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183336960,"num_objects":2797,"num_object_clones":0,"num_object_copies":8391,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2797,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2797,"num_write_kb":179040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"78'126","reported_seq":638,"reported_epoch":283,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:32:14.627878+0000","last_change":"2023-08-28T02:32:13.751483+0000","last_active":"2023-08-28T02:32:14.627878+0000","last_peered":"2023-08-28T02:32:14.627878+0000","last_clean":"2023-08-28T02:27:29.502868+0000","last_became_active":"2023-08-28T02:32:13.742983+0000","last_became_peered":"2023-08-28T02:32:13.742983+0000","last_unstale":"2023-08-28T02:32:14.627878+0000","last_undegraded":"2023-08-28T02:32:13.559413+0000","last_fullsized":"2023-08-28T02:32:13.558668+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:27:33.187037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":22,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":743,"num_read_kb":1331,"num_write":125,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":283,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.543867+0000","last_change":"2023-08-28T02:23:59.207148+0000","last_active":"2023-08-28T02:32:14.543867+0000","last_peered":"2023-08-28T02:32:14.543867+0000","last_clean":"2023-08-28T02:32:14.543867+0000","last_became_active":"2023-08-28T02:23:59.206802+0000","last_became_peered":"2023-08-28T02:23:59.206802+0000","last_unstale":"2023-08-28T02:32:14.543867+0000","last_undegraded":"2023-08-28T02:32:14.543867+0000","last_fullsized":"2023-08-28T02:32:14.543867+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":66,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.544659+0000","last_change":"2023-08-28T02:32:12.536058+0000","last_active":"2023-08-28T02:32:14.544659+0000","last_peered":"2023-08-28T02:32:14.544659+0000","last_clean":"2023-08-28T02:32:14.544659+0000","last_became_active":"2023-08-28T02:32:12.535709+0000","last_became_peered":"2023-08-28T02:32:12.535709+0000","last_unstale":"2023-08-28T02:32:14.544659+0000","last_undegraded":"2023-08-28T02:32:14.544659+0000","last_fullsized":"2023-08-28T02:32:14.544659+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.7","version":"284'4624","reported_seq":4663,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-08-28T02:32:27.062240+0000","last_change":"2023-08-28T02:31:44.472455+0000","last_active":"2023-08-28T02:32:27.062240+0000","last_peered":"2023-08-28T02:32:27.062240+0000","last_clean":"2023-08-28T02:32:27.062240+0000","last_became_active":"2023-08-28T02:31:44.470716+0000","last_became_peered":"2023-08-28T02:31:44.470716+0000","last_unstale":"2023-08-28T02:32:27.062240+0000","last_undegraded":"2023-08-28T02:32:27.062240+0000","last_fullsized":"2023-08-28T02:32:27.062240+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":4624,"log_dups_size":0,"ondisk_log_size":4624,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180879360,"num_objects":2760,"num_object_clones":0,"num_object_copies":8280,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2760,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2760,"num_write_kb":176640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"78'204","reported_seq":1254,"reported_epoch":283,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:32:14.544617+0000","last_change":"2023-08-28T02:32:13.675829+0000","last_active":"2023-08-28T02:32:14.544617+0000","last_peered":"2023-08-28T02:32:14.544617+0000","last_clean":"2023-08-28T02:26:59.547026+0000","last_became_active":"2023-08-28T02:32:13.675281+0000","last_became_peered":"2023-08-28T02:32:13.675281+0000","last_unstale":"2023-08-28T02:32:14.544617+0000","last_undegraded":"2023-08-28T02:32:13.529799+0000","last_fullsized":"2023-08-28T02:32:13.528913+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":204,"log_dups_size":0,"ondisk_log_size":204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1521,"num_read_kb":2511,"num_write":178,"num_write_kb":1175,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5],"avail_no_missing":["6","1","5"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":278,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.544151+0000","last_change":"2023-08-28T02:28:25.352417+0000","last_active":"2023-08-28T02:32:14.544151+0000","last_peered":"2023-08-28T02:32:14.544151+0000","last_clean":"2023-08-28T02:32:14.544151+0000","last_became_active":"2023-08-28T02:28:25.352216+0000","last_became_peered":"2023-08-28T02:28:25.352216+0000","last_unstale":"2023-08-28T02:32:14.544151+0000","last_undegraded":"2023-08-28T02:32:14.544151+0000","last_fullsized":"2023-08-28T02:32:14.544151+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":147,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.623756+0000","last_change":"2023-08-28T02:32:12.546599+0000","last_active":"2023-08-28T02:32:14.623756+0000","last_peered":"2023-08-28T02:32:14.623756+0000","last_clean":"2023-08-28T02:32:14.623756+0000","last_became_active":"2023-08-28T02:32:12.546272+0000","last_became_peered":"2023-08-28T02:32:12.546272+0000","last_unstale":"2023-08-28T02:32:14.623756+0000","last_undegraded":"2023-08-28T02:32:14.623756+0000","last_fullsized":"2023-08-28T02:32:14.623756+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T14:00:12.174693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.4","version":"284'4710","reported_seq":4761,"reported_epoch":284,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:32:23.768789+0000","last_change":"2023-08-28T02:32:13.761045+0000","last_active":"2023-08-28T02:32:23.768789+0000","last_peered":"2023-08-28T02:32:23.768789+0000","last_clean":"2023-08-28T02:32:11.480338+0000","last_became_active":"2023-08-28T02:32:13.754435+0000","last_became_peered":"2023-08-28T02:32:13.754435+0000","last_unstale":"2023-08-28T02:32:23.768789+0000","last_undegraded":"2023-08-28T02:32:13.560253+0000","last_fullsized":"2023-08-28T02:32:13.529038+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":4710,"log_dups_size":0,"ondisk_log_size":4710,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182288384,"num_objects":2781,"num_object_clones":0,"num_object_copies":8343,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2612,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2781,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2781,"num_write_kb":178016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":2612},{"shards":"0,4","objects":169}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"78'47","reported_seq":501,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.551272+0000","last_change":"2023-08-28T02:24:05.799647+0000","last_active":"2023-08-28T02:32:14.551272+0000","last_peered":"2023-08-28T02:32:14.551272+0000","last_clean":"2023-08-28T02:32:14.551272+0000","last_became_active":"2023-08-28T02:23:59.210481+0000","last_became_peered":"2023-08-28T02:23:59.210481+0000","last_unstale":"2023-08-28T02:32:14.551272+0000","last_undegraded":"2023-08-28T02:32:14.551272+0000","last_fullsized":"2023-08-28T02:32:14.551272+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":47,"log_dups_size":0,"ondisk_log_size":47,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:40:18.542375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":162,"num_read_kb":1281,"num_write":48,"num_write_kb":1153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":79,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.551241+0000","last_change":"2023-08-28T02:32:12.559636+0000","last_active":"2023-08-28T02:32:14.551241+0000","last_peered":"2023-08-28T02:32:14.551241+0000","last_clean":"2023-08-28T02:32:14.551241+0000","last_became_active":"2023-08-28T02:32:12.559312+0000","last_became_peered":"2023-08-28T02:32:12.559312+0000","last_unstale":"2023-08-28T02:32:14.551241+0000","last_undegraded":"2023-08-28T02:32:14.551241+0000","last_fullsized":"2023-08-28T02:32:14.551241+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:52:49.389204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":96,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.544375+0000","last_change":"2023-08-28T02:32:12.544145+0000","last_active":"2023-08-28T02:32:14.544375+0000","last_peered":"2023-08-28T02:32:14.544375+0000","last_clean":"2023-08-28T02:32:14.544375+0000","last_became_active":"2023-08-28T02:32:12.543857+0000","last_became_peered":"2023-08-28T02:32:12.543857+0000","last_unstale":"2023-08-28T02:32:14.544375+0000","last_undegraded":"2023-08-28T02:32:14.544375+0000","last_fullsized":"2023-08-28T02:32:14.544375+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.6","version":"284'4459","reported_seq":4514,"reported_epoch":284,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:32:23.712801+0000","last_change":"2023-08-28T02:32:12.589239+0000","last_active":"2023-08-28T02:32:23.712801+0000","last_peered":"2023-08-28T02:32:23.712801+0000","last_clean":"2023-08-28T02:31:54.462738+0000","last_became_active":"2023-08-28T02:31:56.607923+0000","last_became_peered":"2023-08-28T02:31:56.607923+0000","last_unstale":"2023-08-28T02:32:23.712801+0000","last_undegraded":"2023-08-28T02:32:12.587852+0000","last_fullsized":"2023-08-28T02:31:56.514083+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":4459,"log_dups_size":0,"ondisk_log_size":4459,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173834240,"num_objects":2652,"num_object_clones":0,"num_object_copies":7956,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2060,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2652,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2652,"num_write_kb":169760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":592},{"shards":"0,3,7","objects":2060}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"78'114","reported_seq":680,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.626878+0000","last_change":"2023-08-28T02:32:14.626614+0000","last_active":"2023-08-28T02:32:14.626878+0000","last_peered":"2023-08-28T02:32:14.626878+0000","last_clean":"2023-08-28T02:32:14.626878+0000","last_became_active":"2023-08-28T02:32:14.626181+0000","last_became_peered":"2023-08-28T02:32:14.626181+0000","last_unstale":"2023-08-28T02:32:14.626878+0000","last_undegraded":"2023-08-28T02:32:14.626878+0000","last_fullsized":"2023-08-28T02:32:14.626878+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":114,"log_dups_size":0,"ondisk_log_size":114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":358,"num_read_kb":1352,"num_write":102,"num_write_kb":1104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"86'139","reported_seq":350,"reported_epoch":283,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:32:14.545987+0000","last_change":"2023-08-28T02:32:12.576777+0000","last_active":"2023-08-28T02:32:14.545987+0000","last_peered":"2023-08-28T02:32:14.545987+0000","last_clean":"2023-08-28T02:23:32.292225+0000","last_became_active":"2023-08-28T02:32:12.576343+0000","last_became_peered":"2023-08-28T02:32:12.576343+0000","last_unstale":"2023-08-28T02:32:14.545987+0000","last_undegraded":"2023-08-28T02:32:12.519532+0000","last_fullsized":"2023-08-28T02:32:12.519002+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_clean_scrub_stamp":"2023-08-28T02:08:02.043718+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:23:13.976013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":163,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-08-28T02:32:16.638728+0000","last_change":"2023-08-28T02:28:25.347600+0000","last_active":"2023-08-28T02:32:16.638728+0000","last_peered":"2023-08-28T02:32:16.638728+0000","last_clean":"2023-08-28T02:32:16.638728+0000","last_became_active":"2023-08-28T02:28:25.347237+0000","last_became_peered":"2023-08-28T02:28:25.347237+0000","last_unstale":"2023-08-28T02:32:16.638728+0000","last_undegraded":"2023-08-28T02:32:16.638728+0000","last_fullsized":"2023-08-28T02:32:16.638728+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:29:26.767559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":81,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.544352+0000","last_change":"2023-08-28T02:31:55.517732+0000","last_active":"2023-08-28T02:32:14.544352+0000","last_peered":"2023-08-28T02:32:14.544352+0000","last_clean":"2023-08-28T02:32:14.544352+0000","last_became_active":"2023-08-28T02:31:55.517347+0000","last_became_peered":"2023-08-28T02:31:55.517347+0000","last_unstale":"2023-08-28T02:32:14.544352+0000","last_undegraded":"2023-08-28T02:32:14.544352+0000","last_fullsized":"2023-08-28T02:32:14.544352+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"284'4519","reported_seq":4574,"reported_epoch":284,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:32:24.523702+0000","last_change":"2023-08-28T02:32:12.689748+0000","last_active":"2023-08-28T02:32:24.523702+0000","last_peered":"2023-08-28T02:32:24.523702+0000","last_clean":"2023-08-28T02:31:54.356655+0000","last_became_active":"2023-08-28T02:32:12.681925+0000","last_became_peered":"2023-08-28T02:32:12.681925+0000","last_unstale":"2023-08-28T02:32:24.523702+0000","last_undegraded":"2023-08-28T02:32:12.566090+0000","last_fullsized":"2023-08-28T02:32:12.537464+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":4519,"log_dups_size":0,"ondisk_log_size":4519,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176717824,"num_objects":2696,"num_object_clones":0,"num_object_copies":8088,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2442,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2696,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2696,"num_write_kb":172576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0],"avail_no_missing":["2","0","7"],"object_location_counts":[{"shards":"0,1,2","objects":311},{"shards":"0,2","objects":254},{"shards":"0,2,7","objects":2131}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"78'148","reported_seq":644,"reported_epoch":283,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:32:14.546268+0000","last_change":"2023-08-28T02:32:13.715622+0000","last_active":"2023-08-28T02:32:14.546268+0000","last_peered":"2023-08-28T02:32:14.546268+0000","last_clean":"2023-08-28T02:12:46.181557+0000","last_became_active":"2023-08-28T02:32:13.715183+0000","last_became_peered":"2023-08-28T02:32:13.715183+0000","last_unstale":"2023-08-28T02:32:14.546268+0000","last_undegraded":"2023-08-28T02:32:13.524642+0000","last_fullsized":"2023-08-28T02:32:13.523857+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":244,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:53:37.409215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":22,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1092,"num_read_kb":1955,"num_write":127,"num_write_kb":5177,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":253,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.626110+0000","last_change":"2023-08-28T02:23:59.226986+0000","last_active":"2023-08-28T02:32:14.626110+0000","last_peered":"2023-08-28T02:32:14.626110+0000","last_clean":"2023-08-28T02:32:14.626110+0000","last_became_active":"2023-08-28T02:23:59.226529+0000","last_became_peered":"2023-08-28T02:23:59.226529+0000","last_unstale":"2023-08-28T02:32:14.626110+0000","last_undegraded":"2023-08-28T02:32:14.626110+0000","last_fullsized":"2023-08-28T02:32:14.626110+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:22:09.193619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":307,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.620969+0000","last_change":"2023-08-28T02:32:12.542910+0000","last_active":"2023-08-28T02:32:14.620969+0000","last_peered":"2023-08-28T02:32:14.620969+0000","last_clean":"2023-08-28T02:32:14.620969+0000","last_became_active":"2023-08-28T02:32:12.542589+0000","last_became_peered":"2023-08-28T02:32:12.542589+0000","last_unstale":"2023-08-28T02:32:14.620969+0000","last_undegraded":"2023-08-28T02:32:14.620969+0000","last_fullsized":"2023-08-28T02:32:14.620969+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:31:25.114805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"284'4693","reported_seq":4741,"reported_epoch":284,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:32:26.927736+0000","last_change":"2023-08-28T02:32:13.728675+0000","last_active":"2023-08-28T02:32:26.927736+0000","last_peered":"2023-08-28T02:32:26.927736+0000","last_clean":"2023-08-28T02:32:11.496413+0000","last_became_active":"2023-08-28T02:32:13.724348+0000","last_became_peered":"2023-08-28T02:32:13.724348+0000","last_unstale":"2023-08-28T02:32:26.927736+0000","last_undegraded":"2023-08-28T02:32:13.618777+0000","last_fullsized":"2023-08-28T02:32:13.527296+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":4693,"log_dups_size":0,"ondisk_log_size":4693,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186351616,"num_objects":2843,"num_object_clones":0,"num_object_copies":8529,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2540,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2843,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2843,"num_write_kb":181984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3],"avail_no_missing":["5","1","3"],"object_location_counts":[{"shards":"1,3,5","objects":2540},{"shards":"3,5","objects":303}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"78'103","reported_seq":584,"reported_epoch":283,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:32:14.624497+0000","last_change":"2023-08-28T02:32:12.544878+0000","last_active":"2023-08-28T02:32:14.624497+0000","last_peered":"2023-08-28T02:32:14.624497+0000","last_clean":"2023-08-28T02:23:32.292540+0000","last_became_active":"2023-08-28T02:32:12.544529+0000","last_became_peered":"2023-08-28T02:32:12.544529+0000","last_unstale":"2023-08-28T02:32:14.624497+0000","last_undegraded":"2023-08-28T02:32:12.522079+0000","last_fullsized":"2023-08-28T02:32:12.521598+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:47:48.424599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":803,"num_read_kb":2080,"num_write":73,"num_write_kb":1244,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":70,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.623827+0000","last_change":"2023-08-28T02:32:12.545020+0000","last_active":"2023-08-28T02:32:14.623827+0000","last_peered":"2023-08-28T02:32:14.623827+0000","last_clean":"2023-08-28T02:32:14.623827+0000","last_became_active":"2023-08-28T02:32:12.544325+0000","last_became_peered":"2023-08-28T02:32:12.544325+0000","last_unstale":"2023-08-28T02:32:14.623827+0000","last_undegraded":"2023-08-28T02:32:14.623827+0000","last_fullsized":"2023-08-28T02:32:14.623827+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:51:15.229274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":257,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.619101+0000","last_change":"2023-08-28T02:32:12.541959+0000","last_active":"2023-08-28T02:32:14.619101+0000","last_peered":"2023-08-28T02:32:14.619101+0000","last_clean":"2023-08-28T02:32:14.619101+0000","last_became_active":"2023-08-28T02:32:12.541430+0000","last_became_peered":"2023-08-28T02:32:12.541430+0000","last_unstale":"2023-08-28T02:32:14.619101+0000","last_undegraded":"2023-08-28T02:32:14.619101+0000","last_fullsized":"2023-08-28T02:32:14.619101+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:31:10.203644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"284'4560","reported_seq":4601,"reported_epoch":284,"state":"active+clean","last_fresh":"2023-08-28T02:32:24.606522+0000","last_change":"2023-08-28T02:31:44.467379+0000","last_active":"2023-08-28T02:32:24.606522+0000","last_peered":"2023-08-28T02:32:24.606522+0000","last_clean":"2023-08-28T02:32:24.606522+0000","last_became_active":"2023-08-28T02:31:44.444605+0000","last_became_peered":"2023-08-28T02:31:44.444605+0000","last_unstale":"2023-08-28T02:32:24.606522+0000","last_undegraded":"2023-08-28T02:32:24.606522+0000","last_fullsized":"2023-08-28T02:32:24.606522+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":4560,"log_dups_size":0,"ondisk_log_size":4560,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176291840,"num_objects":2690,"num_object_clones":0,"num_object_copies":8070,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2690,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2690,"num_write_kb":172160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":161,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.621993+0000","last_change":"2023-08-28T02:31:55.535099+0000","last_active":"2023-08-28T02:32:14.621993+0000","last_peered":"2023-08-28T02:32:14.621993+0000","last_clean":"2023-08-28T02:32:14.621993+0000","last_became_active":"2023-08-28T02:31:55.527768+0000","last_became_peered":"2023-08-28T02:31:55.527768+0000","last_unstale":"2023-08-28T02:32:14.621993+0000","last_undegraded":"2023-08-28T02:32:14.621993+0000","last_fullsized":"2023-08-28T02:32:14.621993+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:43:30.804793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'112","reported_seq":343,"reported_epoch":283,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:32:14.543889+0000","last_change":"2023-08-28T02:32:13.543899+0000","last_active":"2023-08-28T02:32:14.543889+0000","last_peered":"2023-08-28T02:32:14.543889+0000","last_clean":"2023-08-28T02:12:45.636147+0000","last_became_active":"2023-08-28T02:32:13.543459+0000","last_became_peered":"2023-08-28T02:32:13.543459+0000","last_unstale":"2023-08-28T02:32:14.543889+0000","last_undegraded":"2023-08-28T02:32:13.529496+0000","last_fullsized":"2023-08-28T02:32:13.528938+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":444,"num_read_kb":473,"num_write":94,"num_write_kb":127,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4],"avail_no_missing":["6","1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":163,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.623042+0000","last_change":"2023-08-28T02:32:12.536765+0000","last_active":"2023-08-28T02:32:14.623042+0000","last_peered":"2023-08-28T02:32:14.623042+0000","last_clean":"2023-08-28T02:32:14.623042+0000","last_became_active":"2023-08-28T02:32:12.536414+0000","last_became_peered":"2023-08-28T02:32:12.536414+0000","last_unstale":"2023-08-28T02:32:14.623042+0000","last_undegraded":"2023-08-28T02:32:14.623042+0000","last_fullsized":"2023-08-28T02:32:14.623042+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:44:57.468677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"284'4693","reported_seq":4742,"reported_epoch":284,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:32:24.606468+0000","last_change":"2023-08-28T02:32:13.718088+0000","last_active":"2023-08-28T02:32:24.606468+0000","last_peered":"2023-08-28T02:32:24.606468+0000","last_clean":"2023-08-28T02:32:11.480934+0000","last_became_active":"2023-08-28T02:32:13.712864+0000","last_became_peered":"2023-08-28T02:32:13.712864+0000","last_unstale":"2023-08-28T02:32:24.606468+0000","last_undegraded":"2023-08-28T02:32:13.546449+0000","last_fullsized":"2023-08-28T02:32:13.529059+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":4693,"log_dups_size":0,"ondisk_log_size":4693,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182714368,"num_objects":2788,"num_object_clones":0,"num_object_copies":8364,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2533,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2788,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2788,"num_write_kb":178432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2],"avail_no_missing":["3","1","2"],"object_location_counts":[{"shards":"1,2,3","objects":2533},{"shards":"2,3","objects":255}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"78'122","reported_seq":598,"reported_epoch":283,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:32:14.545638+0000","last_change":"2023-08-28T02:31:56.541881+0000","last_active":"2023-08-28T02:32:14.545638+0000","last_peered":"2023-08-28T02:32:14.545638+0000","last_clean":"2023-08-28T02:31:53.497132+0000","last_became_active":"2023-08-28T02:31:56.541504+0000","last_became_peered":"2023-08-28T02:31:56.541504+0000","last_unstale":"2023-08-28T02:32:14.545638+0000","last_undegraded":"2023-08-28T02:31:56.510521+0000","last_fullsized":"2023-08-28T02:31:56.509717+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":857,"num_read_kb":2224,"num_write":99,"num_write_kb":2312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2],"avail_no_missing":["6","2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":278,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.621067+0000","last_change":"2023-08-28T02:32:12.561743+0000","last_active":"2023-08-28T02:32:14.621067+0000","last_peered":"2023-08-28T02:32:14.621067+0000","last_clean":"2023-08-28T02:32:14.621067+0000","last_became_active":"2023-08-28T02:32:12.546533+0000","last_became_peered":"2023-08-28T02:32:12.546533+0000","last_unstale":"2023-08-28T02:32:14.621067+0000","last_undegraded":"2023-08-28T02:32:14.621067+0000","last_fullsized":"2023-08-28T02:32:14.621067+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:55:54.134853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":253,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.544688+0000","last_change":"2023-08-28T02:28:25.351840+0000","last_active":"2023-08-28T02:32:14.544688+0000","last_peered":"2023-08-28T02:32:14.544688+0000","last_clean":"2023-08-28T02:32:14.544688+0000","last_became_active":"2023-08-28T02:28:25.351399+0000","last_became_peered":"2023-08-28T02:28:25.351399+0000","last_unstale":"2023-08-28T02:32:14.544688+0000","last_undegraded":"2023-08-28T02:32:14.544688+0000","last_fullsized":"2023-08-28T02:32:14.544688+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":259,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.623073+0000","last_change":"2023-08-28T02:32:12.541774+0000","last_active":"2023-08-28T02:32:14.623073+0000","last_peered":"2023-08-28T02:32:14.623073+0000","last_clean":"2023-08-28T02:32:14.623073+0000","last_became_active":"2023-08-28T02:32:12.540966+0000","last_became_peered":"2023-08-28T02:32:12.540966+0000","last_unstale":"2023-08-28T02:32:14.623073+0000","last_undegraded":"2023-08-28T02:32:14.623073+0000","last_fullsized":"2023-08-28T02:32:14.623073+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:41:00.704680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":343,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.623896+0000","last_change":"2023-08-28T02:32:12.558843+0000","last_active":"2023-08-28T02:32:14.623896+0000","last_peered":"2023-08-28T02:32:14.623896+0000","last_clean":"2023-08-28T02:32:14.623896+0000","last_became_active":"2023-08-28T02:32:12.558475+0000","last_became_peered":"2023-08-28T02:32:12.558475+0000","last_unstale":"2023-08-28T02:32:14.623896+0000","last_undegraded":"2023-08-28T02:32:14.623896+0000","last_fullsized":"2023-08-28T02:32:14.623896+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:54:39.164281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.18","version":"284'4714","reported_seq":6311,"reported_epoch":284,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:32:24.569238+0000","last_change":"2023-08-28T02:32:12.557727+0000","last_active":"2023-08-28T02:32:24.569238+0000","last_peered":"2023-08-28T02:32:24.569238+0000","last_clean":"2023-08-28T02:31:54.412195+0000","last_became_active":"2023-08-28T02:31:56.623609+0000","last_became_peered":"2023-08-28T02:31:56.623609+0000","last_unstale":"2023-08-28T02:32:24.569238+0000","last_undegraded":"2023-08-28T02:32:12.554864+0000","last_fullsized":"2023-08-28T02:31:56.505024+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":4714,"log_dups_size":0,"ondisk_log_size":4714,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183238656,"num_objects":2796,"num_object_clones":0,"num_object_copies":8388,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1472,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2796,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2796,"num_write_kb":178944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":772,"num_bytes_recovered":50593792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4],"avail_no_missing":["2","4","7"],"object_location_counts":[{"shards":"2,4","objects":1324},{"shards":"2,4,7","objects":1472}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":103,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.545386+0000","last_change":"2023-08-28T02:32:12.550657+0000","last_active":"2023-08-28T02:32:14.545386+0000","last_peered":"2023-08-28T02:32:14.545386+0000","last_clean":"2023-08-28T02:32:14.545386+0000","last_became_active":"2023-08-28T02:32:12.547856+0000","last_became_peered":"2023-08-28T02:32:12.547856+0000","last_unstale":"2023-08-28T02:32:14.545386+0000","last_undegraded":"2023-08-28T02:32:14.545386+0000","last_fullsized":"2023-08-28T02:32:14.545386+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":310,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.544303+0000","last_change":"2023-08-28T02:32:12.543426+0000","last_active":"2023-08-28T02:32:14.544303+0000","last_peered":"2023-08-28T02:32:14.544303+0000","last_clean":"2023-08-28T02:32:14.544303+0000","last_became_active":"2023-08-28T02:32:12.543021+0000","last_became_peered":"2023-08-28T02:32:12.543021+0000","last_unstale":"2023-08-28T02:32:14.544303+0000","last_undegraded":"2023-08-28T02:32:14.544303+0000","last_fullsized":"2023-08-28T02:32:14.544303+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.19","version":"284'4646","reported_seq":4696,"reported_epoch":284,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:32:23.909240+0000","last_change":"2023-08-28T02:32:13.754657+0000","last_active":"2023-08-28T02:32:23.909240+0000","last_peered":"2023-08-28T02:32:23.909240+0000","last_clean":"2023-08-28T02:32:11.482841+0000","last_became_active":"2023-08-28T02:32:13.710521+0000","last_became_peered":"2023-08-28T02:32:13.710521+0000","last_unstale":"2023-08-28T02:32:23.909240+0000","last_undegraded":"2023-08-28T02:32:13.546474+0000","last_fullsized":"2023-08-28T02:32:13.523884+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":4646,"log_dups_size":0,"ondisk_log_size":4646,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179666944,"num_objects":2742,"num_object_clones":0,"num_object_copies":8226,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2480,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2742,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2742,"num_write_kb":175456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":2480},{"shards":"0,4","objects":262}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":103,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.544918+0000","last_change":"2023-08-28T02:28:25.705331+0000","last_active":"2023-08-28T02:32:14.544918+0000","last_peered":"2023-08-28T02:32:14.544918+0000","last_clean":"2023-08-28T02:32:14.544918+0000","last_became_active":"2023-08-28T02:28:25.397480+0000","last_became_peered":"2023-08-28T02:28:25.397480+0000","last_unstale":"2023-08-28T02:32:14.544918+0000","last_undegraded":"2023-08-28T02:32:14.544918+0000","last_fullsized":"2023-08-28T02:32:14.544918+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":283,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.543544+0000","last_change":"2023-08-28T02:23:59.207929+0000","last_active":"2023-08-28T02:32:14.543544+0000","last_peered":"2023-08-28T02:32:14.543544+0000","last_clean":"2023-08-28T02:32:14.543544+0000","last_became_active":"2023-08-28T02:23:59.207589+0000","last_became_peered":"2023-08-28T02:23:59.207589+0000","last_unstale":"2023-08-28T02:32:14.543544+0000","last_undegraded":"2023-08-28T02:32:14.543544+0000","last_fullsized":"2023-08-28T02:32:14.543544+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":172,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.620646+0000","last_change":"2023-08-28T02:28:25.347988+0000","last_active":"2023-08-28T02:32:14.620646+0000","last_peered":"2023-08-28T02:32:14.620646+0000","last_clean":"2023-08-28T02:32:14.620646+0000","last_became_active":"2023-08-28T02:28:25.347632+0000","last_became_peered":"2023-08-28T02:28:25.347632+0000","last_unstale":"2023-08-28T02:32:14.620646+0000","last_undegraded":"2023-08-28T02:32:14.620646+0000","last_fullsized":"2023-08-28T02:32:14.620646+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:56:20.202662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":144,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.620160+0000","last_change":"2023-08-28T02:23:59.217794+0000","last_active":"2023-08-28T02:32:14.620160+0000","last_peered":"2023-08-28T02:32:14.620160+0000","last_clean":"2023-08-28T02:32:14.620160+0000","last_became_active":"2023-08-28T02:23:59.217231+0000","last_became_peered":"2023-08-28T02:23:59.217231+0000","last_unstale":"2023-08-28T02:32:14.620160+0000","last_undegraded":"2023-08-28T02:32:14.620160+0000","last_fullsized":"2023-08-28T02:32:14.620160+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:36:26.297410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":239,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.629275+0000","last_change":"2023-08-28T02:20:19.850252+0000","last_active":"2023-08-28T02:32:14.629275+0000","last_peered":"2023-08-28T02:32:14.629275+0000","last_clean":"2023-08-28T02:32:14.629275+0000","last_became_active":"2023-08-28T02:20:19.849913+0000","last_became_peered":"2023-08-28T02:20:19.849913+0000","last_unstale":"2023-08-28T02:32:14.629275+0000","last_undegraded":"2023-08-28T02:32:14.629275+0000","last_fullsized":"2023-08-28T02:32:14.629275+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:48:47.713465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":333,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-08-28T02:32:14.545425+0000","last_change":"2023-08-28T02:32:12.538328+0000","last_active":"2023-08-28T02:32:14.545425+0000","last_peered":"2023-08-28T02:32:14.545425+0000","last_clean":"2023-08-28T02:32:14.545425+0000","last_became_active":"2023-08-28T02:32:12.537953+0000","last_became_peered":"2023-08-28T02:32:12.537953+0000","last_unstale":"2023-08-28T02:32:14.545425+0000","last_undegraded":"2023-08-28T02:32:14.545425+0000","last_fullsized":"2023-08-28T02:32:14.545425+0000","mapping_epoch":280,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":281,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":26,"stat_sum":{"num_bytes":5792923648,"num_objects":88393,"num_object_clones":0,"num_object_copies":265179,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":52810,"num_objects_misplaced":1178,"num_objects_unfound":0,"num_objects_dirty":88393,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":88393,"num_write_kb":5657152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2856,"num_bytes_recovered":187170816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17613651968,"data_stored":17613651968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125234,"ondisk_log_size":125234,"up":78,"acting":59,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":24124,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":21484,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26810,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":86,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":97,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":18,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430482,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626709,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2223016,"kb_used_data":1921696,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":91522136,"statfs":{"total":95995035648,"available":93718667264,"internally_reserved":0,"allocated":1967816704,"data_stored":1967069523,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2,0,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:32:14 2023","interfaces":[{"interface":"back","average":{"1min":5.0129999999999999,"5min":2.4260000000000002,"15min":3.1219999999999999},"min":{"1min":0.255,"5min":0.22700000000000001,"15min":0.20799999999999999},"max":{"1min":52.191000000000003,"5min":52.191000000000003,"15min":121.247},"last":0.92200000000000004},{"interface":"front","average":{"1min":5.5629999999999997,"5min":2.5920000000000001,"15min":3.0720000000000001},"min":{"1min":0.38800000000000001,"5min":0.245,"15min":0.23200000000000001},"max":{"1min":52.323999999999998,"5min":52.323999999999998,"15min":118.977},"last":2.8199999999999998}]},{"osd":1,"last update":"Mon Aug 28 02:32:14 2023","interfaces":[{"interface":"back","average":{"1min":0.94199999999999995,"5min":3.5569999999999999,"15min":2.4750000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23699999999999999},"max":{"1min":3.1749999999999998,"5min":141.87299999999999,"15min":141.87299999999999},"last":1.9390000000000001},{"interface":"front","average":{"1min":1.006,"5min":3.4980000000000002,"15min":2.569},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":5.899,"5min":141.904,"15min":141.904},"last":0.70499999999999996}]},{"osd":2,"last update":"Mon Aug 28 02:31:55 2023","interfaces":[{"interface":"back","average":{"1min":1.4370000000000001,"5min":2.9460000000000002,"15min":2.5459999999999998},"min":{"1min":0.309,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":9.8580000000000005,"5min":58.758000000000003,"15min":79.787000000000006},"last":1.486},{"interface":"front","average":{"1min":0.95899999999999996,"5min":2.9279999999999999,"15min":2.52},"min":{"1min":0.32600000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.0830000000000002,"5min":58.887,"15min":79.935000000000002},"last":2.298}]},{"osd":3,"last update":"Mon Aug 28 02:32:14 2023","interfaces":[{"interface":"back","average":{"1min":1.4219999999999999,"5min":4.0259999999999998,"15min":2.3780000000000001},"min":{"1min":0.33900000000000002,"5min":0.32000000000000001,"15min":0.317},"max":{"1min":7.165,"5min":157.625,"15min":157.625},"last":2.7280000000000002},{"interface":"front","average":{"1min":1.343,"5min":3.8519999999999999,"15min":2.3839999999999999},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":7.202,"5min":157.637,"15min":157.637},"last":2.8530000000000002}]},{"osd":4,"last update":"Mon Aug 28 02:32:21 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":1.012,"15min":1.0940000000000001},"min":{"1min":0.27300000000000002,"5min":0.27100000000000002,"15min":0.251},"max":{"1min":1.6899999999999999,"5min":18.308,"15min":18.308},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.752,"5min":1.083,"15min":1.175},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"max":{"1min":3.7639999999999998,"5min":18.263000000000002,"15min":18.263000000000002},"last":0.76600000000000001}]},{"osd":5,"last update":"Mon Aug 28 02:32:21 2023","interfaces":[{"interface":"back","average":{"1min":0.72599999999999998,"5min":0.85999999999999999,"15min":1.161},"min":{"1min":0.317,"5min":0.29699999999999999,"15min":0.26300000000000001},"max":{"1min":4.2800000000000002,"5min":6.0049999999999999,"15min":20.759},"last":0.498},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.871,"15min":1.004},"min":{"1min":0.34000000000000002,"5min":0.314,"15min":0.28799999999999998},"max":{"1min":1.724,"5min":7.2370000000000001,"15min":7.3780000000000001},"last":0.63200000000000001}]},{"osd":7,"last update":"Mon Aug 28 02:31:45 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.92900000000000005,"15min":1.018},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.314},"max":{"1min":1.778,"5min":10.605,"15min":10.605},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.88700000000000001,"15min":1.0760000000000001},"min":{"1min":0.38400000000000001,"5min":0.34599999999999997,"15min":0.29799999999999999},"max":{"1min":1.625,"5min":5.1900000000000004,"15min":9.7110000000000003},"last":0.73599999999999999}]}]},{"osd":1,"up_from":0,"seq":420906795161,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":90,"seq":386547056807,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3216596,"kb_used_data":2793164,"kb_used_omap":8,"kb_used_meta":423415,"kb_avail":90528556,"statfs":{"total":95995035648,"available":92701241344,"internally_reserved":0,"allocated":2860199936,"data_stored":2858903282,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":433577321},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,0,0,0,0,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Aug 28 02:32:21 2023","interfaces":[{"interface":"back","average":{"1min":4.4160000000000004,"5min":3.46,"15min":2.2269999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.184},"max":{"1min":61.932000000000002,"5min":61.932000000000002,"15min":61.932000000000002},"last":0.216},{"interface":"front","average":{"1min":5.8650000000000002,"5min":4.2249999999999996,"15min":2.4199999999999999},"min":{"1min":0.28999999999999998,"5min":0.19600000000000001,"15min":0.17000000000000001},"max":{"1min":62.061999999999998,"5min":62.061999999999998,"15min":62.061999999999998},"last":0.28899999999999998}]},{"osd":2,"last update":"Mon Aug 28 02:31:37 2023","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":3.7909999999999999,"15min":2.2389999999999999},"min":{"1min":0.20599999999999999,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":3.2229999999999999,"5min":100.718,"15min":100.718},"last":0.32400000000000001},{"interface":"front","average":{"1min":2.1299999999999999,"5min":3.5830000000000002,"15min":2.532},"min":{"1min":0.246,"5min":0.23400000000000001,"15min":0.21199999999999999},"max":{"1min":9.1880000000000006,"5min":100.962,"15min":100.962},"last":0.20000000000000001}]},{"osd":3,"last update":"Mon Aug 28 02:32:21 2023","interfaces":[{"interface":"back","average":{"1min":8.1999999999999993,"5min":7.5330000000000004,"15min":3.5089999999999999},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23300000000000001},"max":{"1min":126.896,"5min":237.43899999999999,"15min":237.43899999999999},"last":4.9080000000000004},{"interface":"front","average":{"1min":7.8310000000000004,"5min":7.3810000000000002,"15min":3.4300000000000002},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":127.01600000000001,"5min":237.48099999999999,"15min":237.48099999999999},"last":0.65200000000000002}]},{"osd":4,"last update":"Mon Aug 28 02:32:16 2023","interfaces":[{"interface":"back","average":{"1min":1.3959999999999999,"5min":1.637,"15min":1.4139999999999999},"min":{"1min":0.252,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":8.9559999999999995,"5min":9.3680000000000003,"15min":9.6649999999999991},"last":0.26300000000000001},{"interface":"front","average":{"1min":1.3660000000000001,"5min":1.6180000000000001,"15min":1.5529999999999999},"min":{"1min":0.31,"5min":0.309,"15min":0.309},"max":{"1min":5.4470000000000001,"5min":27.271999999999998,"15min":27.271999999999998},"last":0.36699999999999999}]},{"osd":5,"last update":"Mon Aug 28 02:32:16 2023","interfaces":[{"interface":"back","average":{"1min":1.208,"5min":1.46,"15min":1.633},"min":{"1min":0.33200000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":3.871,"5min":27.367000000000001,"15min":44.000999999999998},"last":0.33900000000000002},{"interface":"front","average":{"1min":1.4330000000000001,"5min":1.635,"15min":1.6020000000000001},"min":{"1min":0.311,"5min":0.308,"15min":0.30599999999999999},"max":{"1min":9.0310000000000006,"5min":9.4489999999999998,"15min":43.953000000000003},"last":0.38700000000000001}]},{"osd":6,"last update":"Mon Aug 28 02:32:16 2023","interfaces":[{"interface":"back","average":{"1min":1.6459999999999999,"5min":1.696,"15min":1.7270000000000001},"min":{"1min":0.375,"5min":0.312,"15min":0.312},"max":{"1min":4.1269999999999998,"5min":27.331,"15min":32.417999999999999},"last":0.311},{"interface":"front","average":{"1min":1.4330000000000001,"5min":1.8380000000000001,"15min":1.657},"min":{"1min":0.33900000000000002,"5min":0.32700000000000001,"15min":0.30399999999999999},"max":{"1min":9.0009999999999994,"5min":27.247,"15min":32.130000000000003},"last":0.35699999999999998}]},{"osd":7,"last update":"Mon Aug 28 02:32:16 2023","interfaces":[{"interface":"back","average":{"1min":1.6060000000000001,"5min":2.25,"15min":1.712},"min":{"1min":0.379,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":10.775,"5min":27.385000000000002,"15min":27.385000000000002},"last":0.40600000000000003},{"interface":"front","average":{"1min":2.6110000000000002,"5min":2.0979999999999999,"15min":1.766},"min":{"1min":0.41799999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":7.5350000000000001,"5min":27.213999999999999,"15min":27.213999999999999},"last":0.378}]}]},{"osd":2,"up_from":104,"seq":446676598931,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2770600,"kb_used_data":2351776,"kb_used_omap":10,"kb_used_meta":418805,"kb_avail":90974552,"statfs":{"total":95995035648,"available":93157941248,"internally_reserved":0,"allocated":2408218624,"data_stored":2406962698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10792,"internal_metadata":428856792},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,5,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:31:40 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":2.7090000000000001,"15min":2.3149999999999999},"min":{"1min":0.23499999999999999,"5min":0.185,"15min":0.17599999999999999},"max":{"1min":7.593,"5min":113.89100000000001,"15min":113.89100000000001},"last":1.712},{"interface":"front","average":{"1min":1.2689999999999999,"5min":3.335,"15min":3.532},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.19900000000000001},"max":{"1min":6.4829999999999997,"5min":114.291,"15min":147.286},"last":7.1189999999999998}]},{"osd":1,"last update":"Mon Aug 28 02:31:40 2023","interfaces":[{"interface":"back","average":{"1min":1.1479999999999999,"5min":1.77,"15min":1.7909999999999999},"min":{"1min":0.255,"5min":0.21099999999999999,"15min":0.20999999999999999},"max":{"1min":7.0730000000000004,"5min":11.627000000000001,"15min":30.888000000000002},"last":3.6179999999999999},{"interface":"front","average":{"1min":0.78000000000000003,"5min":2.1709999999999998,"15min":1.7190000000000001},"min":{"1min":0.255,"5min":0.22900000000000001,"15min":0.20999999999999999},"max":{"1min":2.363,"5min":11.962999999999999,"15min":31.648},"last":3.4820000000000002}]},{"osd":3,"last update":"Mon Aug 28 02:32:12 2023","interfaces":[{"interface":"back","average":{"1min":5.0350000000000001,"5min":4.2489999999999997,"15min":2.8900000000000001},"min":{"1min":0.33100000000000002,"5min":0.25,"15min":0.22700000000000001},"max":{"1min":56.168999999999997,"5min":71.983999999999995,"15min":128.63900000000001},"last":3.855},{"interface":"front","average":{"1min":4.7439999999999998,"5min":3.9049999999999998,"15min":2.8460000000000001},"min":{"1min":0.29799999999999999,"5min":0.223,"15min":0.223},"max":{"1min":56.018000000000001,"5min":71.944999999999993,"15min":128.75700000000001},"last":0.32600000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:31:40 2023","interfaces":[{"interface":"back","average":{"1min":0.74099999999999999,"5min":5.0279999999999996,"15min":2.6709999999999998},"min":{"1min":0.32500000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.3999999999999999,"5min":207.34100000000001,"15min":207.34100000000001},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.73399999999999999,"5min":4.7370000000000001,"15min":2.419},"min":{"1min":0.34599999999999997,"5min":0.30299999999999999,"15min":0.27400000000000002},"max":{"1min":2.903,"5min":207.352,"15min":207.352},"last":1.6739999999999999}]},{"osd":5,"last update":"Mon Aug 28 02:31:40 2023","interfaces":[{"interface":"back","average":{"1min":1.3149999999999999,"5min":3.6120000000000001,"15min":2.585},"min":{"1min":0.40799999999999997,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":6.6539999999999999,"5min":94.614000000000004,"15min":128.696},"last":3.5790000000000002},{"interface":"front","average":{"1min":0.79500000000000004,"5min":3.452,"15min":2.6099999999999999},"min":{"1min":0.34799999999999998,"5min":0.32300000000000001,"15min":0.30299999999999999},"max":{"1min":2.4340000000000002,"5min":94.534000000000006,"15min":128.84100000000001},"last":3.5070000000000001}]},{"osd":6,"last update":"Mon Aug 28 02:31:40 2023","interfaces":[{"interface":"back","average":{"1min":0.92000000000000004,"5min":1.425,"15min":1.371},"min":{"1min":0.41799999999999998,"5min":0.32800000000000001,"15min":0.29299999999999998},"max":{"1min":3.3679999999999999,"5min":11.997,"15min":11.997},"last":3.173},{"interface":"front","average":{"1min":1.3340000000000001,"5min":1.647,"15min":1.5409999999999999},"min":{"1min":0.38,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.593,"5min":10.278,"15min":11.055999999999999},"last":3.6819999999999999}]},{"osd":7,"last update":"Mon Aug 28 02:31:40 2023","interfaces":[{"interface":"back","average":{"1min":0.86799999999999999,"5min":3.532,"15min":2.5310000000000001},"min":{"1min":0.38900000000000001,"5min":0.33500000000000002,"15min":0.32100000000000001},"max":{"1min":3.1869999999999998,"5min":108.904,"15min":128.911},"last":2.585},{"interface":"front","average":{"1min":1.3300000000000001,"5min":3.8580000000000001,"15min":2.7610000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":6.7160000000000002,"5min":108.217,"15min":128.72399999999999},"last":3.657}]}]},{"osd":3,"up_from":112,"seq":481036337293,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3730204,"kb_used_data":3319188,"kb_used_omap":3,"kb_used_meta":411004,"kb_avail":90014948,"statfs":{"total":95995035648,"available":92175306752,"internally_reserved":0,"allocated":3398848512,"data_stored":3397597717,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":420869026},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,4,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:32:07 2023","interfaces":[{"interface":"back","average":{"1min":3.0910000000000002,"5min":3.4129999999999998,"15min":3.2970000000000002},"min":{"1min":0.23300000000000001,"5min":0.20000000000000001,"15min":0.193},"max":{"1min":11.305,"5min":79.793000000000006,"15min":164.25299999999999},"last":3.234},{"interface":"front","average":{"1min":2.444,"5min":3.0110000000000001,"15min":3.2000000000000002},"min":{"1min":0.26400000000000001,"5min":0.192,"15min":0.192},"max":{"1min":11.571,"5min":78.555000000000007,"15min":164.55000000000001},"last":2.371}]},{"osd":1,"last update":"Mon Aug 28 02:32:07 2023","interfaces":[{"interface":"back","average":{"1min":2.5379999999999998,"5min":1.79,"15min":2.0619999999999998},"min":{"1min":0.27300000000000002,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":10.907999999999999,"5min":10.907999999999999,"15min":117.34699999999999},"last":0.54000000000000004},{"interface":"front","average":{"1min":2.4249999999999998,"5min":2.238,"15min":2.1949999999999998},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.224},"max":{"1min":10.079000000000001,"5min":10.125,"15min":118.012},"last":6.0549999999999997}]},{"osd":2,"last update":"Mon Aug 28 02:32:07 2023","interfaces":[{"interface":"back","average":{"1min":2.8330000000000002,"5min":2.9409999999999998,"15min":3.0790000000000002},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.19900000000000001},"max":{"1min":7.6959999999999997,"5min":65.308999999999997,"15min":120.43300000000001},"last":6.4569999999999999},{"interface":"front","average":{"1min":2.5289999999999999,"5min":2.613,"15min":2.96},"min":{"1min":0.249,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":9.7309999999999999,"5min":65.120999999999995,"15min":120.44499999999999},"last":0.51100000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:32:07 2023","interfaces":[{"interface":"back","average":{"1min":1.4390000000000001,"5min":2.2730000000000001,"15min":1.5700000000000001},"min":{"1min":0.34499999999999997,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":4.2030000000000003,"5min":58.067999999999998,"15min":58.067999999999998},"last":0.45900000000000002},{"interface":"front","average":{"1min":1.3799999999999999,"5min":2.2959999999999998,"15min":1.613},"min":{"1min":0.36199999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":6.218,"5min":58.088000000000001,"15min":58.088000000000001},"last":0.39900000000000002}]},{"osd":5,"last update":"Mon Aug 28 02:32:07 2023","interfaces":[{"interface":"back","average":{"1min":1.4670000000000001,"5min":1.712,"15min":1.7350000000000001},"min":{"1min":0.31,"5min":0.31,"15min":0.29599999999999999},"max":{"1min":6.2599999999999998,"5min":34.060000000000002,"15min":102.904},"last":0.42799999999999999},{"interface":"front","average":{"1min":1.4359999999999999,"5min":1.952,"15min":1.756},"min":{"1min":0.38700000000000001,"5min":0.36699999999999999,"15min":0.32800000000000001},"max":{"1min":4.7590000000000003,"5min":34.079999999999998,"15min":103.283},"last":3.3260000000000001}]},{"osd":6,"last update":"Mon Aug 28 02:32:07 2023","interfaces":[{"interface":"back","average":{"1min":1.5609999999999999,"5min":1.7390000000000001,"15min":1.55},"min":{"1min":0.34300000000000003,"5min":0.33500000000000002,"15min":0.314},"max":{"1min":5.1079999999999997,"5min":9.157,"15min":72.116},"last":3.3660000000000001},{"interface":"front","average":{"1min":1.5840000000000001,"5min":1.645,"15min":1.488},"min":{"1min":0.41999999999999998,"5min":0.314,"15min":0.314},"max":{"1min":5.1760000000000002,"5min":7.8899999999999997,"15min":72.030000000000001},"last":0.48899999999999999}]},{"osd":7,"last update":"Mon Aug 28 02:32:07 2023","interfaces":[{"interface":"back","average":{"1min":1.6859999999999999,"5min":3.0920000000000001,"15min":1.845},"min":{"1min":0.39800000000000002,"5min":0.33800000000000002,"15min":0.32500000000000001},"max":{"1min":5.2320000000000002,"5min":54.427999999999997,"15min":54.427999999999997},"last":2.2389999999999999},{"interface":"front","average":{"1min":1.5429999999999999,"5min":3.052,"15min":1.9379999999999999},"min":{"1min":0.443,"5min":0.36399999999999999,"15min":0.34799999999999998},"max":{"1min":6.3019999999999996,"5min":54.445,"15min":54.445},"last":2.3050000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019167,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2162340,"kb_used_data":1852828,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":91582812,"statfs":{"total":95995035648,"available":93780799488,"internally_reserved":0,"allocated":1897295872,"data_stored":1896552559,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:32:14 2023","interfaces":[{"interface":"back","average":{"1min":5.9870000000000001,"5min":2.8799999999999999,"15min":3.2309999999999999},"min":{"1min":0.25600000000000001,"5min":0.219,"15min":0.219},"max":{"1min":80.655000000000001,"5min":80.655000000000001,"15min":138.67400000000001},"last":7.9119999999999999},{"interface":"front","average":{"1min":6.5659999999999998,"5min":2.6150000000000002,"15min":3.0630000000000002},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":80.427000000000007,"5min":80.427000000000007,"15min":134.14599999999999},"last":0.39700000000000002}]},{"osd":1,"last update":"Mon Aug 28 02:32:14 2023","interfaces":[{"interface":"back","average":{"1min":1.3540000000000001,"5min":2.0819999999999999,"15min":1.6060000000000001},"min":{"1min":0.27200000000000002,"5min":0.221,"15min":0.221},"max":{"1min":6.7460000000000004,"5min":34.945999999999998,"15min":53.622999999999998},"last":0.42099999999999999},{"interface":"front","average":{"1min":1.353,"5min":1.7350000000000001,"15min":1.49},"min":{"1min":0.29699999999999999,"5min":0.25,"15min":0.25},"max":{"1min":6.2439999999999998,"5min":34.869999999999997,"15min":53.774000000000001},"last":0.58199999999999996}]},{"osd":2,"last update":"Mon Aug 28 02:31:37 2023","interfaces":[{"interface":"back","average":{"1min":1.353,"5min":3.3559999999999999,"15min":2.879},"min":{"1min":0.30199999999999999,"5min":0.27400000000000002,"15min":0.25600000000000001},"max":{"1min":9.2949999999999999,"5min":64.063000000000002,"15min":74.099000000000004},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.89100000000000001,"5min":3.4159999999999999,"15min":2.7559999999999998},"min":{"1min":0.33500000000000002,"5min":0.28999999999999998,"15min":0.25},"max":{"1min":2.7000000000000002,"5min":64.073999999999998,"15min":74.081999999999994},"last":3.8999999999999999}]},{"osd":3,"last update":"Mon Aug 28 02:32:14 2023","interfaces":[{"interface":"back","average":{"1min":14.24,"5min":8.4719999999999995,"15min":3.621},"min":{"1min":0.34200000000000003,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":203.583,"5min":203.583,"15min":203.583},"last":1.208},{"interface":"front","average":{"1min":15.619999999999999,"5min":8.5960000000000001,"15min":3.7349999999999999},"min":{"1min":0.45100000000000001,"5min":0.33800000000000002,"15min":0.32500000000000001},"max":{"1min":203.41399999999999,"5min":203.41399999999999,"15min":203.41399999999999},"last":0.73799999999999999}]},{"osd":5,"last update":"Mon Aug 28 02:31:54 2023","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":2.4790000000000001,"15min":1.4550000000000001},"min":{"1min":0.29499999999999998,"5min":0.246,"15min":0.246},"max":{"1min":1.3480000000000001,"5min":124.718,"15min":124.718},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.64200000000000002,"5min":2.577,"15min":1.6519999999999999},"min":{"1min":0.28499999999999998,"5min":0.247,"15min":0.247},"max":{"1min":1.522,"5min":124.694,"15min":124.694},"last":0.40899999999999997}]},{"osd":6,"last update":"Mon Aug 28 02:32:14 2023","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":1.087,"15min":1.109},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":3.109,"5min":14.907999999999999,"15min":14.907999999999999},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.70499999999999996,"5min":0.89100000000000001,"15min":0.89500000000000002},"min":{"1min":0.32800000000000001,"5min":0.307,"15min":0.307},"max":{"1min":3.1219999999999999,"5min":5.2679999999999998,"15min":7.3170000000000002},"last":0.64500000000000002}]},{"osd":7,"last update":"Mon Aug 28 02:31:29 2023","interfaces":[{"interface":"back","average":{"1min":1.5129999999999999,"5min":1.145,"15min":1.1850000000000001},"min":{"1min":0.30499999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":10.151999999999999,"5min":10.151999999999999,"15min":16.584},"last":0.45700000000000002},{"interface":"front","average":{"1min":2.0779999999999998,"5min":1.3400000000000001,"15min":1.2410000000000001},"min":{"1min":0.34799999999999998,"5min":0.311,"15min":0.311},"max":{"1min":9.3469999999999995,"5min":10.826000000000001,"15min":10.826000000000001},"last":0.44800000000000001}]}]},{"osd":5,"up_from":36,"seq":154618822939,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1848088,"kb_used_data":1546768,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":91897064,"statfs":{"total":95995035648,"available":94102593536,"internally_reserved":0,"allocated":1583890432,"data_stored":1583154584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:32:09 2023","interfaces":[{"interface":"back","average":{"1min":5.5720000000000001,"5min":3.371,"15min":3.3380000000000001},"min":{"1min":0.23400000000000001,"5min":0.214,"15min":0.20300000000000001},"max":{"1min":57.753999999999998,"5min":104.13200000000001,"15min":131.06100000000001},"last":2.7069999999999999},{"interface":"front","average":{"1min":5.5380000000000003,"5min":3.5659999999999998,"15min":3.5459999999999998},"min":{"1min":0.27300000000000002,"5min":0.27100000000000002,"15min":0.22900000000000001},"max":{"1min":57.762999999999998,"5min":104.142,"15min":131.262},"last":2.6930000000000001}]},{"osd":1,"last update":"Mon Aug 28 02:32:15 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":1.5509999999999999,"15min":2.4569999999999999},"min":{"1min":0.27300000000000002,"5min":0.25,"15min":0.216},"max":{"1min":2.4550000000000001,"5min":30.593,"15min":183.97900000000001},"last":4.5359999999999996},{"interface":"front","average":{"1min":0.81899999999999995,"5min":1.5449999999999999,"15min":2.6240000000000001},"min":{"1min":0.308,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":2.637,"5min":34.616999999999997,"15min":184.33699999999999},"last":2.3849999999999998}]},{"osd":2,"last update":"Mon Aug 28 02:31:44 2023","interfaces":[{"interface":"back","average":{"1min":1.3140000000000001,"5min":3.2309999999999999,"15min":4.0229999999999997},"min":{"1min":0.317,"5min":0.25800000000000001,"15min":0.252},"max":{"1min":11.24,"5min":100.56699999999999,"15min":176.69499999999999},"last":1.024},{"interface":"front","average":{"1min":1.391,"5min":3.585,"15min":4.2640000000000002},"min":{"1min":0.35399999999999998,"5min":0.34000000000000002,"15min":0.28299999999999997},"max":{"1min":11.262,"5min":100.53700000000001,"15min":176.71199999999999},"last":4.7830000000000004}]},{"osd":3,"last update":"Mon Aug 28 02:32:15 2023","interfaces":[{"interface":"back","average":{"1min":12.782999999999999,"5min":5.0609999999999999,"15min":2.516},"min":{"1min":0.29699999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":128.571,"5min":128.571,"15min":128.571},"last":0.94999999999999996},{"interface":"front","average":{"1min":13.773999999999999,"5min":5.4390000000000001,"15min":2.899},"min":{"1min":0.36399999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":128.684,"5min":128.684,"15min":128.684},"last":1.5840000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:31:55 2023","interfaces":[{"interface":"back","average":{"1min":0.71299999999999997,"5min":1.966,"15min":1.3260000000000001},"min":{"1min":0.28100000000000003,"5min":0.26700000000000002,"15min":0.246},"max":{"1min":2.3799999999999999,"5min":85.769999999999996,"15min":85.769999999999996},"last":1.496},{"interface":"front","average":{"1min":0.55100000000000005,"5min":1.859,"15min":1.3149999999999999},"min":{"1min":0.313,"5min":0.28999999999999998,"15min":0.28000000000000003},"max":{"1min":2.1509999999999998,"5min":85.825000000000003,"15min":85.825000000000003},"last":0.91600000000000004}]},{"osd":6,"last update":"Mon Aug 28 02:32:15 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.80600000000000005,"15min":1.0409999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":1.7929999999999999,"5min":4.6210000000000004,"15min":14.803000000000001},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.755,"5min":0.84199999999999997,"15min":0.95299999999999996},"min":{"1min":0.30199999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":2.4470000000000001,"5min":4.9509999999999996,"15min":8.6479999999999997},"last":1.1830000000000001}]},{"osd":7,"last update":"Mon Aug 28 02:31:33 2023","interfaces":[{"interface":"back","average":{"1min":9.2910000000000004,"5min":2.806,"15min":1.7130000000000001},"min":{"1min":0.371,"5min":0.33300000000000002,"15min":0.31},"max":{"1min":164.13800000000001,"5min":164.13800000000001,"15min":164.13800000000001},"last":0.96899999999999997},{"interface":"front","average":{"1min":9.3249999999999993,"5min":2.8340000000000001,"15min":1.6439999999999999},"min":{"1min":0.39100000000000001,"5min":0.33700000000000002,"15min":0.32100000000000001},"max":{"1min":161.786,"5min":161.786,"15min":161.786},"last":1.5129999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2856189952,"data_stored":2856189952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2036924416,"data_stored":2036924416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2431713280,"data_stored":2431713280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3397779456,"data_stored":3397779456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1910439936,"data_stored":1910439936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1596063744,"data_stored":1596063744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1977810944,"data_stored":1977810944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1406730240,"data_stored":1406730240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-08-28T02:32:31.197 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-08-28T02:32:31.197 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd pool set device_health_metrics pg_num 21 2023-08-28T02:32:31.198 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:32:31.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:31.212 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:32:31.313 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:32:31.457 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:31.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:31 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:32:31.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:31 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 52898/268020 objects degraded (19.737%), 25 pgs degraded (PG_DEGRADED) 2023-08-28T02:32:31.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:31 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:32:31.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:31 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 52898/268020 objects degraded (19.737%), 25 pgs degraded (PG_DEGRADED) 2023-08-28T02:32:31.725 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:32:31.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:31.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:32:31.838 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:32:31.873 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Total time run: 90.0166 2023-08-28T02:32:31.873 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Total writes made: 92767 2023-08-28T02:32:31.873 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Write size: 65536 2023-08-28T02:32:31.873 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Object size: 65536 2023-08-28T02:32:31.873 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Bandwidth (MB/sec): 64.4097 2023-08-28T02:32:31.873 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Stddev Bandwidth: 36.253 2023-08-28T02:32:31.873 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Max bandwidth (MB/sec): 183.938 2023-08-28T02:32:31.874 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Min bandwidth (MB/sec): 0 2023-08-28T02:32:31.874 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Average IOPS: 1030 2023-08-28T02:32:31.874 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Stddev IOPS: 580.053 2023-08-28T02:32:31.874 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Max IOPS: 2943 2023-08-28T02:32:31.874 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Min IOPS: 0 2023-08-28T02:32:31.874 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Average Latency(s): 0.0155213 2023-08-28T02:32:31.874 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Stddev Latency(s): 0.0565423 2023-08-28T02:32:31.874 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Max latency(s): 2.65098 2023-08-28T02:32:31.874 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Min latency(s): 0.00114956 2023-08-28T02:32:31.874 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:31.891 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Cleaning up (deleting benchmark objects) 2023-08-28T02:32:31.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:31 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:32:31.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:31 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 52898/268020 objects degraded (19.737%), 25 pgs degraded (PG_DEGRADED) 2023-08-28T02:32:31.992 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:32:32.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:32.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:32:32.008 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:32:31 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:32:31] "GET /metrics HTTP/1.1" 200 36579 "" "Prometheus/2.33.4" 2023-08-28T02:32:32.011 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:32:32.051 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:32:32.051 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [ 2023-08-28T02:32:32.051 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.051 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.55270.0:92849 7.13 7:c8c68c8b:::benchmark_data_smithi143_360738_object79:head [delete] snapc 0=[] ondisk+write+known_if_redirected e285)", 2023-08-28T02:32:32.051 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:32:32.025179+0000", 2023-08-28T02:32:32.052 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.015952167, 2023-08-28T02:32:32.052 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.015988080000000002, 2023-08-28T02:32:32.052 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:32.052 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "waiting for sub ops", 2023-08-28T02:32:32.052 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:32:32.052 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.55270", 2023-08-28T02:32:32.052 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:32:32.052 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 92849 2023-08-28T02:32:32.052 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.052 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:32.052 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.052 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:32.053 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.025179+0000", 2023-08-28T02:32:32.053 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:32.053 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.053 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.053 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:32.053 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.025179+0000", 2023-08-28T02:32:32.053 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:32.053 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.053 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.053 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:32.053 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.025177+0000", 2023-08-28T02:32:32.053 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:32:32.054 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.054 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.054 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:32.054 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.025185+0000", 2023-08-28T02:32:32.054 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.9990000000000008e-06 2023-08-28T02:32:32.054 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.054 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.054 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:32.054 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.025202+0000", 2023-08-28T02:32:32.055 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.7067000000000001e-05 2023-08-28T02:32:32.055 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.055 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.055 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:32.055 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.025212+0000", 2023-08-28T02:32:32.055 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.6479999999999998e-06 2023-08-28T02:32:32.055 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.055 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.055 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:32.055 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.025274+0000", 2023-08-28T02:32:32.055 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.2071999999999998e-05 2023-08-28T02:32:32.056 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.056 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.056 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:32:32.056 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.025432+0000", 2023-08-28T02:32:32.056 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000158255 2023-08-28T02:32:32.056 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.056 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.056 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 2,4", 2023-08-28T02:32:32.056 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.025561+0000", 2023-08-28T02:32:32.056 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00012907500000000001 2023-08-28T02:32:32.056 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.057 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.057 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:32.057 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.034103+0000", 2023-08-28T02:32:32.057 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0085419469999999994 2023-08-28T02:32:32.057 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.057 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.057 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:32:32.057 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.034113+0000", 2023-08-28T02:32:32.057 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.4369999999999996e-06 2023-08-28T02:32:32.058 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:32.058 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:32.058 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:32.058 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.058 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.058 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.55270.0:92859 7.c e285/277 7:3aa8d520:::benchmark_data_smithi143_360738_object89:head v 285'5701, mlcod=0'0)", 2023-08-28T02:32:32.058 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:32:32.036508+0000", 2023-08-28T02:32:32.058 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.0046238360000000001, 2023-08-28T02:32:32.058 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0047578509999999996, 2023-08-28T02:32:32.058 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:32.058 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "started", 2023-08-28T02:32:32.059 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:32.059 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.059 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:32.059 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.036508+0000", 2023-08-28T02:32:32.059 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:32.059 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.059 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.060 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:32.060 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.036508+0000", 2023-08-28T02:32:32.060 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:32.060 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.060 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.060 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:32.060 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.036506+0000", 2023-08-28T02:32:32.060 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:32:32.060 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.060 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.061 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:32.061 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.036519+0000", 2023-08-28T02:32:32.061 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.2303000000000001e-05 2023-08-28T02:32:32.061 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.061 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.061 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:32.061 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.036544+0000", 2023-08-28T02:32:32.061 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.5395999999999999e-05 2023-08-28T02:32:32.061 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.061 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.062 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:32.062 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.036556+0000", 2023-08-28T02:32:32.062 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.145e-05 2023-08-28T02:32:32.062 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.062 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.062 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:32.062 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.039344+0000", 2023-08-28T02:32:32.062 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0027880270000000002 2023-08-28T02:32:32.062 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.063 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.063 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:32:32.063 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.039368+0000", 2023-08-28T02:32:32.063 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.3981e-05 2023-08-28T02:32:32.063 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:32.063 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:32.063 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:32.063 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.063 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.064 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.55270.0:92851 7.8 e285/277 7:12e69d75:::benchmark_data_smithi143_360738_object81:head v 285'4859, mlcod=0'0)", 2023-08-28T02:32:32.064 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:32:32.037122+0000", 2023-08-28T02:32:32.064 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.0040100120000000003, 2023-08-28T02:32:32.064 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0041976790000000002, 2023-08-28T02:32:32.064 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:32.064 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "started", 2023-08-28T02:32:32.064 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:32.064 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.064 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:32.065 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.037122+0000", 2023-08-28T02:32:32.065 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:32.065 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.065 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.065 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:32.065 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.037122+0000", 2023-08-28T02:32:32.065 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:32.065 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.066 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.066 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:32.066 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.037121+0000", 2023-08-28T02:32:32.066 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:32:32.066 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.066 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.066 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:32.066 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.037127+0000", 2023-08-28T02:32:32.066 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.1120000000000001e-06 2023-08-28T02:32:32.067 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.067 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.067 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:32.067 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.037135+0000", 2023-08-28T02:32:32.067 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.3359999999999997e-06 2023-08-28T02:32:32.067 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.067 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.067 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:32.067 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.037140+0000", 2023-08-28T02:32:32.067 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.3020000000000002e-06 2023-08-28T02:32:32.068 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.068 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.068 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:32.068 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.038819+0000", 2023-08-28T02:32:32.068 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.001678618 2023-08-28T02:32:32.068 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.068 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.068 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:32:32.068 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.038849+0000", 2023-08-28T02:32:32.068 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.0621999999999999e-05 2023-08-28T02:32:32.068 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:32.069 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:32.069 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:32.069 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.069 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.069 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.55270.0:92854 7.10 7:0eca47ab:::benchmark_data_smithi143_360738_object84:head [delete] snapc 0=[] ondisk+write+known_if_redirected e285)", 2023-08-28T02:32:32.069 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:32:32.037171+0000", 2023-08-28T02:32:32.069 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.0039603709999999999, 2023-08-28T02:32:32.069 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0041992169999999999, 2023-08-28T02:32:32.069 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:32.069 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "reached pg", 2023-08-28T02:32:32.069 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:32:32.069 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.55270", 2023-08-28T02:32:32.069 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:32:32.070 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 92854 2023-08-28T02:32:32.070 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.070 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:32.070 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.070 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:32.070 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.037171+0000", 2023-08-28T02:32:32.070 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:32.070 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.070 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.070 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:32.070 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.037171+0000", 2023-08-28T02:32:32.071 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:32.071 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.071 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.071 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:32.071 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.037170+0000", 2023-08-28T02:32:32.071 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:32:32.071 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.071 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.071 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:32.071 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.037175+0000", 2023-08-28T02:32:32.071 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.4809999999999997e-06 2023-08-28T02:32:32.072 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.072 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.072 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:32.072 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.037186+0000", 2023-08-28T02:32:32.072 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1328000000000001e-05 2023-08-28T02:32:32.072 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.072 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.072 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:32.072 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.037193+0000", 2023-08-28T02:32:32.072 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.8240000000000002e-06 2023-08-28T02:32:32.072 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.072 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.073 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:32.073 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.041248+0000", 2023-08-28T02:32:32.073 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.004054584 2023-08-28T02:32:32.073 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:32.073 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:32.073 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:32.073 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.073 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.073 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.55270.0:92860 7.2 7:47c06e52:::benchmark_data_smithi143_360738_object90:head [delete] snapc 0=[] ondisk+write+known_if_redirected e285)", 2023-08-28T02:32:32.073 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:32:32.037200+0000", 2023-08-28T02:32:32.073 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.0039318599999999997, 2023-08-28T02:32:32.073 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0042252330000000001, 2023-08-28T02:32:32.074 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:32.074 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "waiting for sub ops", 2023-08-28T02:32:32.074 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:32:32.074 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.55270", 2023-08-28T02:32:32.074 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:32:32.074 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 92860 2023-08-28T02:32:32.074 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.074 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:32.074 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.074 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:32.074 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.037200+0000", 2023-08-28T02:32:32.074 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:32.074 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.075 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.075 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:32.075 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.037200+0000", 2023-08-28T02:32:32.075 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:32.075 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.075 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.075 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:32.075 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.037199+0000", 2023-08-28T02:32:32.075 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:32:32.075 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.075 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.075 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:32.075 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.037202+0000", 2023-08-28T02:32:32.076 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.089e-06 2023-08-28T02:32:32.076 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.076 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.076 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:32.076 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.037209+0000", 2023-08-28T02:32:32.076 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.4489999999999996e-06 2023-08-28T02:32:32.076 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.076 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.076 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:32.076 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.037214+0000", 2023-08-28T02:32:32.076 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.7019999999999998e-06 2023-08-28T02:32:32.076 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.077 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.077 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:32.077 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.037617+0000", 2023-08-28T02:32:32.077 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00040283000000000002 2023-08-28T02:32:32.077 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.077 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.077 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:32:32.077 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.037804+0000", 2023-08-28T02:32:32.077 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00018696000000000001 2023-08-28T02:32:32.077 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.077 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.077 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 2,6", 2023-08-28T02:32:32.078 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.037987+0000", 2023-08-28T02:32:32.078 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000183086 2023-08-28T02:32:32.078 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:32.078 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:32.078 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:32.078 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.078 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.078 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.55270.0:92857 7.1 e285/270 7:838caa44:::benchmark_data_smithi143_360738_object87:head v 285'4784, mlcod=285'4782)", 2023-08-28T02:32:32.078 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:32:32.039414+0000", 2023-08-28T02:32:32.078 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.001717459, 2023-08-28T02:32:32.078 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0020727229999999998, 2023-08-28T02:32:32.078 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:32.079 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "started", 2023-08-28T02:32:32.079 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:32.079 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.079 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:32.079 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.039414+0000", 2023-08-28T02:32:32.079 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:32.079 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.079 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.079 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:32.079 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.039414+0000", 2023-08-28T02:32:32.079 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:32.079 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.080 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.080 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:32.080 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.039413+0000", 2023-08-28T02:32:32.080 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:32:32.080 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.080 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.080 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:32.080 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.039423+0000", 2023-08-28T02:32:32.080 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.8810000000000008e-06 2023-08-28T02:32:32.080 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.080 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.080 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:32.080 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.039435+0000", 2023-08-28T02:32:32.081 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.2063e-05 2023-08-28T02:32:32.081 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.081 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.081 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:32.081 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.039444+0000", 2023-08-28T02:32:32.081 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.9039999999999998e-06 2023-08-28T02:32:32.081 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.081 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.081 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:32.081 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.040229+0000", 2023-08-28T02:32:32.081 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00078502599999999997 2023-08-28T02:32:32.081 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.081 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.082 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:32:32.082 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.040252+0000", 2023-08-28T02:32:32.082 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.3176e-05 2023-08-28T02:32:32.082 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:32.082 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:32.082 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:32.082 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.082 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.082 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.55270.0:92855 7.f e285/277 7:f73c933f:::benchmark_data_smithi143_360738_object85:head v 285'5874, mlcod=0'0)", 2023-08-28T02:32:32.082 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:32:32.040159+0000", 2023-08-28T02:32:32.082 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.00097224500000000005, 2023-08-28T02:32:32.082 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.001382917, 2023-08-28T02:32:32.083 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:32.083 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "started", 2023-08-28T02:32:32.083 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:32.083 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.083 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:32.083 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.040159+0000", 2023-08-28T02:32:32.083 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:32.083 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.083 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.083 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:32.083 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.040159+0000", 2023-08-28T02:32:32.083 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:32.083 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.084 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.084 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:32.084 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.040159+0000", 2023-08-28T02:32:32.084 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:32:32.084 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.084 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.084 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:32.084 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.040166+0000", 2023-08-28T02:32:32.084 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.7910000000000005e-06 2023-08-28T02:32:32.084 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.084 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.084 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:32.084 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.040179+0000", 2023-08-28T02:32:32.085 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.2303999999999999e-05 2023-08-28T02:32:32.085 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.085 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.085 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:32.085 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.040186+0000", 2023-08-28T02:32:32.085 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.7149999999999998e-06 2023-08-28T02:32:32.085 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.085 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.085 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:32.085 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.040273+0000", 2023-08-28T02:32:32.085 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.6408000000000003e-05 2023-08-28T02:32:32.085 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:32.086 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:32.086 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:32:32.086 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:32.040299+0000", 2023-08-28T02:32:32.086 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.6449e-05 2023-08-28T02:32:32.086 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:32.086 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:32.086 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:32.086 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:32.086 INFO:teuthology.orchestra.run.smithi143.stdout: ], 2023-08-28T02:32:32.086 INFO:teuthology.orchestra.run.smithi143.stdout: "num_ops": 7 2023-08-28T02:32:32.086 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:32:32.087 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:32:32.104 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:32:32.104 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:32:32.109 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:32:32.140 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:32.140 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:32:32.187 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:32:32.193 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:32:32.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:32.196 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:32:32.219 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:32:32.261 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:32:32.297 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:32:32.308 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:32:32.314 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:32:32.314 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:32:32.314 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:32:32.314 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:32:32.314 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:32:32.314 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:32:32.315 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:32:32.315 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:32:32.340 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:32.587 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:32:32.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:32.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:32:32.680 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset noscrub 2023-08-28T02:32:32.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:32 smithi143 ceph-mon[189871]: pgmap v707: 99 pgs: 1 active+recovering+undersized+remapped, 25 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.5 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 45 MiB/s wr, 716 op/s; 52898/272250 objects degraded (19.430%); 743/272250 objects misplaced (0.273%); 5.1 MiB/s, 81 objects/s recovering 2023-08-28T02:32:32.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:32 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2911693600' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:32:32.705 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:32 smithi143 ceph-mon[183059]: pgmap v707: 99 pgs: 1 active+recovering+undersized+remapped, 25 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.5 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 45 MiB/s wr, 716 op/s; 52898/272250 objects degraded (19.430%); 743/272250 objects misplaced (0.273%); 5.1 MiB/s, 81 objects/s recovering 2023-08-28T02:32:32.706 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:32 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2911693600' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:32:32.707 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:32:32.789 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:32.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:32 smithi169 ceph-mon[159415]: pgmap v707: 99 pgs: 1 active+recovering+undersized+remapped, 25 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.5 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 45 MiB/s wr, 716 op/s; 52898/272250 objects degraded (19.430%); 743/272250 objects misplaced (0.273%); 5.1 MiB/s, 81 objects/s recovering 2023-08-28T02:32:32.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:32 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2911693600' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:32:34.135 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:32:34.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:34.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:32:34.242 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:32:34.250 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.3 dump_blocked_ops 2023-08-28T02:32:34.342 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:34.493 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:32:34.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:34.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:32:34.600 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:32:34.699 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:34 smithi143 ceph-mon[189871]: pgmap v708: 99 pgs: 1 active+recovering+undersized+remapped, 25 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.6 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 102 B/s rd, 43 MiB/s wr, 697 op/s; 52898/275238 objects degraded (19.219%); 743/275238 objects misplaced (0.270%); 5.1 MiB/s, 81 objects/s recovering 2023-08-28T02:32:34.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:34 smithi143 ceph-mon[183059]: pgmap v708: 99 pgs: 1 active+recovering+undersized+remapped, 25 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.6 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 102 B/s rd, 43 MiB/s wr, 697 op/s; 52898/275238 objects degraded (19.219%); 743/275238 objects misplaced (0.270%); 5.1 MiB/s, 81 objects/s recovering 2023-08-28T02:32:34.719 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:34.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:34 smithi169 ceph-mon[159415]: pgmap v708: 99 pgs: 1 active+recovering+undersized+remapped, 25 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.6 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 102 B/s rd, 43 MiB/s wr, 697 op/s; 52898/275238 objects degraded (19.219%); 743/275238 objects misplaced (0.270%); 5.1 MiB/s, 81 objects/s recovering 2023-08-28T02:32:35.423 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:32:35.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:35.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:32:35.464 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:32:35.534 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:32:35.607 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:35.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:35 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:32:35.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:35 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:32:35.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:35 smithi143 ceph-mon[183059]: pgmap v709: 99 pgs: 1 active+recovering+undersized+remapped, 25 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.6 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 102 B/s rd, 35 MiB/s wr, 672 op/s; 52898/274950 objects degraded (19.239%); 276/274950 objects misplaced (0.100%); 5.6 MiB/s, 90 objects/s recovering 2023-08-28T02:32:35.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:35 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:32:35.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:35 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:32:35.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:35 smithi143 ceph-mon[189871]: pgmap v709: 99 pgs: 1 active+recovering+undersized+remapped, 25 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.6 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 102 B/s rd, 35 MiB/s wr, 672 op/s; 52898/274950 objects degraded (19.239%); 276/274950 objects misplaced (0.100%); 5.6 MiB/s, 90 objects/s recovering 2023-08-28T02:32:35.797 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:32:35.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:35.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:32:35.913 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:32:35.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:35 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:32:35.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:35 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:32:35.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:35 smithi169 ceph-mon[159415]: pgmap v709: 99 pgs: 1 active+recovering+undersized+remapped, 25 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.6 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 102 B/s rd, 35 MiB/s wr, 672 op/s; 52898/274950 objects degraded (19.239%); 276/274950 objects misplaced (0.100%); 5.6 MiB/s, 90 objects/s recovering 2023-08-28T02:32:36.010 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:36.035 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:32:36.396 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:32:36.408 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:32:36.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:36.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:32:36.511 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:32:36.604 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:36.686 INFO:teuthology.orchestra.run.smithi143.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-08-28T02:32:36.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:36 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 52898/274950 objects degraded (19.239%), 25 pgs degraded (PG_DEGRADED) 2023-08-28T02:32:36.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:36 smithi143 ceph-mon[183059]: osdmap e286: 8 total, 8 up, 6 in 2023-08-28T02:32:36.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:36 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 52898/274950 objects degraded (19.239%), 25 pgs degraded (PG_DEGRADED) 2023-08-28T02:32:36.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:36 smithi143 ceph-mon[189871]: osdmap e286: 8 total, 8 up, 6 in 2023-08-28T02:32:36.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:36 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 52898/274950 objects degraded (19.239%), 25 pgs degraded (PG_DEGRADED) 2023-08-28T02:32:36.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:36 smithi169 ceph-mon[159415]: osdmap e286: 8 total, 8 up, 6 in 2023-08-28T02:32:37.688 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:32:37.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:37.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:32:37.795 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:32:37.831 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:37.897 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:32:37.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:37.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:32:37.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:37 smithi143 ceph-mon[189871]: failed to encode map e286 with expected crc 2023-08-28T02:32:37.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:37 smithi143 ceph-mon[189871]: failed to encode map e286 with expected crc 2023-08-28T02:32:37.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:37 smithi143 ceph-mon[189871]: failed to encode map e286 with expected crc 2023-08-28T02:32:37.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:37 smithi143 ceph-mon[189871]: failed to encode map e286 with expected crc 2023-08-28T02:32:37.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:37 smithi143 ceph-mon[189871]: failed to encode map e286 with expected crc 2023-08-28T02:32:37.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:37 smithi143 ceph-mon[189871]: failed to encode map e286 with expected crc 2023-08-28T02:32:37.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:37 smithi143 ceph-mon[189871]: failed to encode map e286 with expected crc 2023-08-28T02:32:37.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:37 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/883713382' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-08-28T02:32:37.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:37 smithi143 ceph-mon[189871]: pgmap v711: 99 pgs: 1 active+recovering+undersized+remapped, 25 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.5 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 115 B/s rd, 31 MiB/s wr, 746 op/s; 52898/271620 objects degraded (19.475%); 276/271620 objects misplaced (0.102%); 6.3 MiB/s, 101 objects/s recovering 2023-08-28T02:32:37.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:37 smithi143 ceph-mon[189871]: osdmap e287: 8 total, 8 up, 6 in 2023-08-28T02:32:37.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:37 smithi143 ceph-mon[183059]: failed to encode map e286 with expected crc 2023-08-28T02:32:37.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:37 smithi143 ceph-mon[183059]: failed to encode map e286 with expected crc 2023-08-28T02:32:37.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:37 smithi143 ceph-mon[183059]: failed to encode map e286 with expected crc 2023-08-28T02:32:37.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:37 smithi143 ceph-mon[183059]: failed to encode map e286 with expected crc 2023-08-28T02:32:37.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:37 smithi143 ceph-mon[183059]: failed to encode map e286 with expected crc 2023-08-28T02:32:37.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:37 smithi143 ceph-mon[183059]: failed to encode map e286 with expected crc 2023-08-28T02:32:37.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:37 smithi143 ceph-mon[183059]: failed to encode map e286 with expected crc 2023-08-28T02:32:37.955 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:37 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/883713382' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-08-28T02:32:37.955 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:37 smithi143 ceph-mon[183059]: pgmap v711: 99 pgs: 1 active+recovering+undersized+remapped, 25 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.5 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 115 B/s rd, 31 MiB/s wr, 746 op/s; 52898/271620 objects degraded (19.475%); 276/271620 objects misplaced (0.102%); 6.3 MiB/s, 101 objects/s recovering 2023-08-28T02:32:37.955 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:37 smithi143 ceph-mon[183059]: osdmap e287: 8 total, 8 up, 6 in 2023-08-28T02:32:37.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:37 smithi169 ceph-mon[159415]: failed to encode map e286 with expected crc 2023-08-28T02:32:37.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:37 smithi169 ceph-mon[159415]: failed to encode map e286 with expected crc 2023-08-28T02:32:37.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:37 smithi169 ceph-mon[159415]: failed to encode map e286 with expected crc 2023-08-28T02:32:37.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:37 smithi169 ceph-mon[159415]: failed to encode map e286 with expected crc 2023-08-28T02:32:37.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:37 smithi169 ceph-mon[159415]: failed to encode map e286 with expected crc 2023-08-28T02:32:37.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:37 smithi169 ceph-mon[159415]: failed to encode map e286 with expected crc 2023-08-28T02:32:37.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:37 smithi169 ceph-mon[159415]: failed to encode map e286 with expected crc 2023-08-28T02:32:37.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:37 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/883713382' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-08-28T02:32:37.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:37 smithi169 ceph-mon[159415]: pgmap v711: 99 pgs: 1 active+recovering+undersized+remapped, 25 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 5.5 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 115 B/s rd, 31 MiB/s wr, 746 op/s; 52898/271620 objects degraded (19.475%); 276/271620 objects misplaced (0.102%); 6.3 MiB/s, 101 objects/s recovering 2023-08-28T02:32:37.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:37 smithi169 ceph-mon[159415]: osdmap e287: 8 total, 8 up, 6 in 2023-08-28T02:32:38.005 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:32:38.043 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:38.101 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:32:38.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:38.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:32:38.175 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-08-28T02:32:38.208 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:32:38.344 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:38.505 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:32:38.632 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:32:38.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:38.648 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:32:38.748 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:32:38.756 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:38 smithi143 ceph-mon[183059]: failed to encode map e287 with expected crc 2023-08-28T02:32:38.756 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:38 smithi143 ceph-mon[183059]: failed to encode map e287 with expected crc 2023-08-28T02:32:38.756 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:38 smithi143 ceph-mon[183059]: failed to encode map e287 with expected crc 2023-08-28T02:32:38.756 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:38 smithi143 ceph-mon[183059]: failed to encode map e287 with expected crc 2023-08-28T02:32:38.756 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:38 smithi143 ceph-mon[189871]: failed to encode map e287 with expected crc 2023-08-28T02:32:38.756 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:38 smithi143 ceph-mon[189871]: failed to encode map e287 with expected crc 2023-08-28T02:32:38.756 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:38 smithi143 ceph-mon[189871]: failed to encode map e287 with expected crc 2023-08-28T02:32:38.756 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:38 smithi143 ceph-mon[189871]: failed to encode map e287 with expected crc 2023-08-28T02:32:38.783 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:38.846 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:32:38.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:38.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:32:38.951 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:32:38.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:38 smithi169 ceph-mon[159415]: failed to encode map e287 with expected crc 2023-08-28T02:32:38.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:38 smithi169 ceph-mon[159415]: failed to encode map e287 with expected crc 2023-08-28T02:32:38.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:38 smithi169 ceph-mon[159415]: failed to encode map e287 with expected crc 2023-08-28T02:32:38.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:38 smithi169 ceph-mon[159415]: failed to encode map e287 with expected crc 2023-08-28T02:32:39.002 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:39.064 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:32:39.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:39.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:32:39.173 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:32:39.211 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:39.254 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:32:39.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:39.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:32:39.359 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:32:39.421 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:39.749 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is unset 2023-08-28T02:32:39.890 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:32:39.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:39.902 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:32:39.968 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:39 smithi143 ceph-mon[183059]: pgmap v713: 99 pgs: 25 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 5.5 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 127 B/s rd, 16 MiB/s wr, 644 op/s; 52898/269052 objects degraded (19.661%); 5.8 MiB/s, 92 objects/s recovering 2023-08-28T02:32:39.968 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:39 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3025552789' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:32:39.968 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:39 smithi143 ceph-mon[189871]: pgmap v713: 99 pgs: 25 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 5.5 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 127 B/s rd, 16 MiB/s wr, 644 op/s; 52898/269052 objects degraded (19.661%); 5.8 MiB/s, 92 objects/s recovering 2023-08-28T02:32:39.968 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:39 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3025552789' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:32:39.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:39 smithi169 ceph-mon[159415]: pgmap v713: 99 pgs: 25 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 5.5 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 127 B/s rd, 16 MiB/s wr, 644 op/s; 52898/269052 objects degraded (19.661%); 5.8 MiB/s, 92 objects/s recovering 2023-08-28T02:32:39.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:39 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3025552789' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:32:40.003 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:32:40.095 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:40.880 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:32:40.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:40.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:32:40.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:40 smithi169 ceph-mon[159415]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:32:40.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:40 smithi169 ceph-mon[159415]: failed to encode map e288 with expected crc 2023-08-28T02:32:40.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:40 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3025552789' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:32:40.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:40 smithi169 ceph-mon[159415]: osdmap e288: 8 total, 8 up, 6 in 2023-08-28T02:32:40.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:40 smithi169 ceph-mon[159415]: failed to encode map e288 with expected crc 2023-08-28T02:32:40.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:40 smithi169 ceph-mon[159415]: failed to encode map e288 with expected crc 2023-08-28T02:32:40.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:40 smithi169 ceph-mon[159415]: failed to encode map e288 with expected crc 2023-08-28T02:32:40.994 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:32:40.997 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:40 smithi143 ceph-mon[183059]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:32:40.997 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:40 smithi143 ceph-mon[183059]: failed to encode map e288 with expected crc 2023-08-28T02:32:40.997 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:40 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3025552789' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:32:40.997 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:40 smithi143 ceph-mon[183059]: osdmap e288: 8 total, 8 up, 6 in 2023-08-28T02:32:40.998 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:40 smithi143 ceph-mon[183059]: failed to encode map e288 with expected crc 2023-08-28T02:32:40.998 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:40 smithi143 ceph-mon[183059]: failed to encode map e288 with expected crc 2023-08-28T02:32:40.998 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:40 smithi143 ceph-mon[183059]: failed to encode map e288 with expected crc 2023-08-28T02:32:40.998 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:40 smithi143 ceph-mon[189871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:32:40.998 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:40 smithi143 ceph-mon[189871]: failed to encode map e288 with expected crc 2023-08-28T02:32:40.998 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:40 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3025552789' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:32:40.999 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:40 smithi143 ceph-mon[189871]: osdmap e288: 8 total, 8 up, 6 in 2023-08-28T02:32:40.999 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:40 smithi143 ceph-mon[189871]: failed to encode map e288 with expected crc 2023-08-28T02:32:40.999 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:40 smithi143 ceph-mon[189871]: failed to encode map e288 with expected crc 2023-08-28T02:32:40.999 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:40 smithi143 ceph-mon[189871]: failed to encode map e288 with expected crc 2023-08-28T02:32:41.039 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:41.101 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:32:41.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:41.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:32:41.206 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:32:41.234 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:41.456 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:32:41.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:41.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:32:41.565 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:32:41.638 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:41.825 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:32:41.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:41.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:32:41.951 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:32:41.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:41 smithi169 ceph-mon[159415]: failed to encode map e288 with expected crc 2023-08-28T02:32:41.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:41 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:32:41.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:41 smithi169 ceph-mon[159415]: pgmap v715: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 24 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 5.4 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 542 op/s; 50835/265176 objects degraded (19.170%); 1881/265176 objects misplaced (0.709%); 4.8 MiB/s, 76 objects/s recovering 2023-08-28T02:32:41.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:41 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 52898/269052 objects degraded (19.661%), 25 pgs degraded (PG_DEGRADED) 2023-08-28T02:32:41.990 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:42.029 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:32:41 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:32:41] "GET /metrics HTTP/1.1" 200 36585 "" "Prometheus/2.33.4" 2023-08-28T02:32:42.029 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:41 smithi143 ceph-mon[189871]: failed to encode map e288 with expected crc 2023-08-28T02:32:42.029 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:41 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:32:42.029 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:41 smithi143 ceph-mon[189871]: pgmap v715: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 24 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 5.4 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 542 op/s; 50835/265176 objects degraded (19.170%); 1881/265176 objects misplaced (0.709%); 4.8 MiB/s, 76 objects/s recovering 2023-08-28T02:32:42.029 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:41 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 52898/269052 objects degraded (19.661%), 25 pgs degraded (PG_DEGRADED) 2023-08-28T02:32:42.030 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:41 smithi143 ceph-mon[183059]: failed to encode map e288 with expected crc 2023-08-28T02:32:42.030 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:41 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:32:42.030 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:41 smithi143 ceph-mon[183059]: pgmap v715: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 24 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 5.4 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 542 op/s; 50835/265176 objects degraded (19.170%); 1881/265176 objects misplaced (0.709%); 4.8 MiB/s, 76 objects/s recovering 2023-08-28T02:32:42.030 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:41 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 52898/269052 objects degraded (19.661%), 25 pgs degraded (PG_DEGRADED) 2023-08-28T02:32:42.044 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:32:42.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:42.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:32:42.150 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:32:42.179 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:42.244 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:32:42.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:42.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:32:42.310 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:32:42.310 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:32:42.352 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:32:42.391 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:42.446 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:32:42.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:42.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:32:42.555 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:32:42.585 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:43.176 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 6] out_osds: [7, 1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-08-28T02:32:43.176 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-08-28T02:32:43.176 INFO:tasks.thrashosds.thrasher:Killing osd 7, live_osds are [0, 1, 2, 3, 4, 5, 6, 7] 2023-08-28T02:32:43.176 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2023-08-28T02:32:43.177 DEBUG:teuthology.orchestra.run.smithi169:> sudo systemctl stop ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@osd.7 2023-08-28T02:32:43.491 INFO:journalctl@ceph.osd.7.smithi169.stdout:Aug 28 02:32:43 smithi169 systemd[1]: Stopping Ceph osd.7 for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:32:43.500 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:32:43.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:43.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:32:43.552 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.3 dump_historic_ops 2023-08-28T02:32:43.628 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:32:43.804 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:43.833 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset nodeep-scrub 2023-08-28T02:32:43.896 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:32:43.970 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:32:43.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:43.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:32:43.982 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:32:44.045 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:32:44.075 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:32:44.087 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:32:44.109 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:44.125 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:32:44.161 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:32:44.190 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:32:44.228 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:32:44.254 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:32:44.254 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:32:44.254 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:32:44.254 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:32:44.254 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:32:44.254 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:32:44.254 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:32:44.254 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:32:44.263 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:32:44.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:44.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:32:44.370 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:32:44.466 INFO:journalctl@ceph.osd.7.smithi169.stdout:Aug 28 02:32:44 smithi169 conmon[124434]: debug 2023-08-28T02:32:44.255+0000 7fa0e6a66700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-08-28T02:32:44.466 INFO:journalctl@ceph.osd.7.smithi169.stdout:Aug 28 02:32:44 smithi169 conmon[124434]: debug 2023-08-28T02:32:44.255+0000 7fa0e6a66700 -1 osd.7 288 *** Got signal Terminated *** 2023-08-28T02:32:44.466 INFO:journalctl@ceph.osd.7.smithi169.stdout:Aug 28 02:32:44 smithi169 conmon[124434]: debug 2023-08-28T02:32:44.255+0000 7fa0e6a66700 -1 osd.7 288 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-08-28T02:32:44.486 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:44.705 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:44 smithi143 ceph-mon[183059]: pgmap v716: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 24 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 5.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 629 op/s; 50835/261891 objects degraded (19.411%); 1881/261891 objects misplaced (0.718%); 4.1 MiB/s, 66 objects/s recovering 2023-08-28T02:32:44.705 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:44 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1152286198' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:32:44.705 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:44 smithi143 ceph-mon[183059]: osd.7 marked itself down and dead 2023-08-28T02:32:44.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:44 smithi143 ceph-mon[189871]: pgmap v716: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 24 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 5.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 629 op/s; 50835/261891 objects degraded (19.411%); 1881/261891 objects misplaced (0.718%); 4.1 MiB/s, 66 objects/s recovering 2023-08-28T02:32:44.705 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:44 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1152286198' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:32:44.706 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:44 smithi143 ceph-mon[189871]: osd.7 marked itself down and dead 2023-08-28T02:32:44.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:44 smithi169 ceph-mon[159415]: pgmap v716: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 24 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 5.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 629 op/s; 50835/261891 objects degraded (19.411%); 1881/261891 objects misplaced (0.718%); 4.1 MiB/s, 66 objects/s recovering 2023-08-28T02:32:44.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:44 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1152286198' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:32:44.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:44 smithi169 ceph-mon[159415]: osd.7 marked itself down and dead 2023-08-28T02:32:45.238 INFO:journalctl@ceph.osd.7.smithi169.stdout:Aug 28 02:32:44 smithi169 bash[239427]: 64fa1422c85ab69cd88d41bb8b6861ce69f08f926340506ca9df04bfc46b102e 2023-08-28T02:32:45.264 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:32:45.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:45.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:32:45.352 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:32:45.375 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:32:45.469 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:45.687 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:32:45.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:45 smithi143 ceph-mon[183059]: failed to encode map e289 with expected crc 2023-08-28T02:32:45.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:45 smithi143 ceph-mon[183059]: osdmap e289: 8 total, 7 up, 6 in 2023-08-28T02:32:45.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:45 smithi143 ceph-mon[183059]: failed to encode map e289 with expected crc 2023-08-28T02:32:45.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:45 smithi143 ceph-mon[183059]: failed to encode map e289 with expected crc 2023-08-28T02:32:45.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:45 smithi143 ceph-mon[189871]: failed to encode map e289 with expected crc 2023-08-28T02:32:45.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:45 smithi143 ceph-mon[189871]: osdmap e289: 8 total, 7 up, 6 in 2023-08-28T02:32:45.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:45 smithi143 ceph-mon[189871]: failed to encode map e289 with expected crc 2023-08-28T02:32:45.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:45 smithi143 ceph-mon[189871]: failed to encode map e289 with expected crc 2023-08-28T02:32:45.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:45.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:32:45.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:45 smithi169 ceph-mon[159415]: failed to encode map e289 with expected crc 2023-08-28T02:32:45.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:45 smithi169 ceph-mon[159415]: osdmap e289: 8 total, 7 up, 6 in 2023-08-28T02:32:45.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:45 smithi169 ceph-mon[159415]: failed to encode map e289 with expected crc 2023-08-28T02:32:45.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:45 smithi169 ceph-mon[159415]: failed to encode map e289 with expected crc 2023-08-28T02:32:45.805 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:32:45.855 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:45.939 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:32:45.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:45.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:32:46.045 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:32:46.069 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:32:46.167 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:46.294 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:32:46.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:46.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:32:46.412 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:32:46.460 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:46.537 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:32:46.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:46.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:32:46.644 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:32:46.691 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:46.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:46 smithi169 ceph-mon[159415]: pgmap v718: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 24 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 5.2 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 698 op/s; 52492/255030 objects degraded (20.583%); 5.4 MiB/s, 86 objects/s recovering 2023-08-28T02:32:46.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:46 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:32:46.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:46 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:32:46.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:46 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 52492/255030 objects degraded (20.583%), 25 pgs degraded (PG_DEGRADED) 2023-08-28T02:32:46.865 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:32:46.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:46.870 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:32:46.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:46 smithi143 ceph-mon[189871]: pgmap v718: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 24 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 5.2 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 698 op/s; 52492/255030 objects degraded (20.583%); 5.4 MiB/s, 86 objects/s recovering 2023-08-28T02:32:46.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:46 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:32:46.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:46 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:32:46.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:46 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 52492/255030 objects degraded (20.583%), 25 pgs degraded (PG_DEGRADED) 2023-08-28T02:32:46.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:46 smithi143 ceph-mon[183059]: pgmap v718: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 24 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 5.2 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 698 op/s; 52492/255030 objects degraded (20.583%); 5.4 MiB/s, 86 objects/s recovering 2023-08-28T02:32:46.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:46 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:32:46.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:46 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:32:46.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:46 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 52492/255030 objects degraded (20.583%), 25 pgs degraded (PG_DEGRADED) 2023-08-28T02:32:46.971 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:32:47.011 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:47.146 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:32:47.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:47.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:32:47.254 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:32:47.349 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:47.395 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:32:47.451 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:32:47.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:47.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:32:47.557 DEBUG:teuthology.orchestra.run.smithi169:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@osd.7.service' 2023-08-28T02:32:47.561 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:47 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:32:47.562 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:47 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:32:47.562 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:47 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:32:47.562 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:47 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:32:47.562 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:47 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:32:47.562 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:47 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:32:47.562 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:32:47.623 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-08-28T02:32:47.623 INFO:tasks.cephadm.osd.7:Stopped osd.7 2023-08-28T02:32:47.649 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:47.720 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:32:47.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:47.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:32:47.826 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:32:47.886 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:47.967 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:32:47.967 INFO:teuthology.orchestra.run.smithi143.stdout: "size": 20, 2023-08-28T02:32:47.967 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 600, 2023-08-28T02:32:47.968 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [ 2023-08-28T02:32:47.968 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.968 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:123521 5.0 5.7558ecf0 (undecoded) ondisk+write+known_if_redirected e153)", 2023-08-28T02:32:47.968 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:58.301375+0000", 2023-08-28T02:32:47.968 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 529.66222108099998, 2023-08-28T02:32:47.968 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.89313394000000002, 2023-08-28T02:32:47.968 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:47.968 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "reached pg", 2023-08-28T02:32:47.968 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:32:47.968 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:32:47.968 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:32:47.968 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 123521 2023-08-28T02:32:47.968 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.969 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:47.969 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.969 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:47.969 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.301375+0000", 2023-08-28T02:32:47.969 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:47.969 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.969 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.969 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:47.969 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.301375+0000", 2023-08-28T02:32:47.969 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:47.969 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.969 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.969 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:47.970 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.301374+0000", 2023-08-28T02:32:47.970 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:32:47.970 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.970 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.970 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:47.970 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.301380+0000", 2023-08-28T02:32:47.970 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.2729999999999999e-06 2023-08-28T02:32:47.970 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.970 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.970 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:47.970 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.301390+0000", 2023-08-28T02:32:47.970 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0088000000000001e-05 2023-08-28T02:32:47.971 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.971 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.971 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:47.971 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.301402+0000", 2023-08-28T02:32:47.971 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.2335e-05 2023-08-28T02:32:47.971 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.971 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.971 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:47.971 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.301463+0000", 2023-08-28T02:32:47.971 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.0585999999999999e-05 2023-08-28T02:32:47.971 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.971 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.971 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:47.972 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.301478+0000", 2023-08-28T02:32:47.972 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.4983e-05 2023-08-28T02:32:47.972 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.972 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.972 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:47.972 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.194498+0000", 2023-08-28T02:32:47.972 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.89302048199999995 2023-08-28T02:32:47.972 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.972 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.972 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:32:47.972 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.194509+0000", 2023-08-28T02:32:47.972 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0518e-05 2023-08-28T02:32:47.972 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:47.973 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:47.973 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:47.973 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.973 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.973 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:123522 5.0 5.7c0443d0 (undecoded) ondisk+write+known_if_redirected e153)", 2023-08-28T02:32:47.973 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:58.306232+0000", 2023-08-28T02:32:47.973 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 529.65736399699995, 2023-08-28T02:32:47.973 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.88830339700000005, 2023-08-28T02:32:47.973 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:47.973 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "reached pg", 2023-08-28T02:32:47.973 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:32:47.973 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:32:47.973 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:32:47.974 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 123522 2023-08-28T02:32:47.974 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.974 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:47.974 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.974 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:47.974 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.306232+0000", 2023-08-28T02:32:47.974 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:47.974 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.974 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.974 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:47.974 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.306232+0000", 2023-08-28T02:32:47.974 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:47.974 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.975 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.975 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:47.975 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.306230+0000", 2023-08-28T02:32:47.975 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999976 2023-08-28T02:32:47.975 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.975 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.975 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:47.975 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.306243+0000", 2023-08-28T02:32:47.975 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3753e-05 2023-08-28T02:32:47.975 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.975 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.975 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:47.976 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.306285+0000", 2023-08-28T02:32:47.976 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.1990999999999998e-05 2023-08-28T02:32:47.976 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.976 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.976 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:47.976 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.306295+0000", 2023-08-28T02:32:47.976 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.6299999999999993e-06 2023-08-28T02:32:47.976 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.976 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.976 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:47.976 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.306347+0000", 2023-08-28T02:32:47.976 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.2488999999999998e-05 2023-08-28T02:32:47.976 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.977 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.977 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:47.977 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.306356+0000", 2023-08-28T02:32:47.977 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.1270000000000003e-06 2023-08-28T02:32:47.977 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.977 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.977 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:47.977 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.194528+0000", 2023-08-28T02:32:47.977 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.88817234 2023-08-28T02:32:47.977 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.977 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.977 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:32:47.977 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.194535+0000", 2023-08-28T02:32:47.977 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.537e-06 2023-08-28T02:32:47.978 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:47.978 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:47.978 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:47.978 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.978 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.978 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:123553 5.0 5.a384d7c0 (undecoded) ondisk+write+known_if_redirected e153)", 2023-08-28T02:32:47.978 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:58.342227+0000", 2023-08-28T02:32:47.978 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 529.62136899999996, 2023-08-28T02:32:47.978 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.852330637, 2023-08-28T02:32:47.978 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:47.978 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "reached pg", 2023-08-28T02:32:47.978 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:32:47.978 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:32:47.979 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:32:47.979 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 123553 2023-08-28T02:32:47.979 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.979 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:47.979 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.979 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:47.979 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.342227+0000", 2023-08-28T02:32:47.979 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:47.979 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.979 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.979 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:47.979 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.342227+0000", 2023-08-28T02:32:47.979 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:47.980 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.980 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.980 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:47.980 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.342225+0000", 2023-08-28T02:32:47.980 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:32:47.980 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.980 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.980 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:47.980 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.342236+0000", 2023-08-28T02:32:47.980 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0883999999999999e-05 2023-08-28T02:32:47.980 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.980 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.980 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:47.981 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.342261+0000", 2023-08-28T02:32:47.981 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.5318000000000001e-05 2023-08-28T02:32:47.981 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.981 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.981 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:47.981 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.342282+0000", 2023-08-28T02:32:47.981 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.0397000000000001e-05 2023-08-28T02:32:47.981 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.981 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.981 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:47.981 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.342338+0000", 2023-08-28T02:32:47.981 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.6208000000000001e-05 2023-08-28T02:32:47.981 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.982 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.982 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:47.982 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.342349+0000", 2023-08-28T02:32:47.982 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0902e-05 2023-08-28T02:32:47.982 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.982 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.982 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:47.982 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.194551+0000", 2023-08-28T02:32:47.982 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.85220246499999996 2023-08-28T02:32:47.982 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.982 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.982 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:32:47.982 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.194558+0000", 2023-08-28T02:32:47.983 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.533e-06 2023-08-28T02:32:47.983 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:47.983 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:47.983 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:47.983 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.983 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.983 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:123563 5.0 5.d31fd5b0 (undecoded) ondisk+write+known_if_redirected e153)", 2023-08-28T02:32:47.983 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:58.354660+0000", 2023-08-28T02:32:47.983 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 529.60893652599998, 2023-08-28T02:32:47.983 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.83992108399999998, 2023-08-28T02:32:47.983 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:47.983 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "reached pg", 2023-08-28T02:32:47.983 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:32:47.984 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:32:47.984 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:32:47.984 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 123563 2023-08-28T02:32:47.984 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.984 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:47.984 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.984 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:47.984 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.354660+0000", 2023-08-28T02:32:47.984 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:47.984 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.984 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.984 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:47.984 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.354660+0000", 2023-08-28T02:32:47.985 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:47.985 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.985 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.985 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:47.985 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.354657+0000", 2023-08-28T02:32:47.985 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999976 2023-08-28T02:32:47.985 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.985 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.985 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:47.985 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.354667+0000", 2023-08-28T02:32:47.985 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.008e-05 2023-08-28T02:32:47.985 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.986 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.986 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:47.986 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.354691+0000", 2023-08-28T02:32:47.986 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.3669999999999999e-05 2023-08-28T02:32:47.986 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.986 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.986 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:47.986 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.354702+0000", 2023-08-28T02:32:47.986 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.135e-05 2023-08-28T02:32:47.986 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.986 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.986 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:47.986 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.354771+0000", 2023-08-28T02:32:47.987 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.8449999999999997e-05 2023-08-28T02:32:47.987 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.987 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.987 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:47.987 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.354784+0000", 2023-08-28T02:32:47.987 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3247999999999999e-05 2023-08-28T02:32:47.987 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.987 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.987 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:47.987 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.194573+0000", 2023-08-28T02:32:47.987 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.83978885700000006 2023-08-28T02:32:47.987 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.987 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.988 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:32:47.988 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.194581+0000", 2023-08-28T02:32:47.988 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.8059999999999995e-06 2023-08-28T02:32:47.988 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:47.988 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:47.988 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:47.988 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.988 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.988 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:123565 5.0 5.b98f9370 (undecoded) ondisk+write+known_if_redirected e153)", 2023-08-28T02:32:47.988 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:58.361101+0000", 2023-08-28T02:32:47.988 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 529.602494783, 2023-08-28T02:32:47.988 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.83350137999999996, 2023-08-28T02:32:47.988 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:47.989 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "reached pg", 2023-08-28T02:32:47.989 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:32:47.989 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:32:47.989 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:32:47.989 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 123565 2023-08-28T02:32:47.989 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.989 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:47.989 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.989 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:47.989 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.361101+0000", 2023-08-28T02:32:47.989 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:47.989 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.989 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.990 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:47.990 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.361101+0000", 2023-08-28T02:32:47.990 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:47.990 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.990 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.990 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:47.990 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.361099+0000", 2023-08-28T02:32:47.990 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:32:47.990 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.990 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.990 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:47.990 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.361108+0000", 2023-08-28T02:32:47.991 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.7749999999999998e-06 2023-08-28T02:32:47.991 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.991 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.991 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:47.991 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.361129+0000", 2023-08-28T02:32:47.991 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.1199000000000002e-05 2023-08-28T02:32:47.991 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.991 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.991 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:47.991 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.361139+0000", 2023-08-28T02:32:47.991 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.9960000000000005e-06 2023-08-28T02:32:47.991 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.991 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.992 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:47.992 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.361195+0000", 2023-08-28T02:32:47.992 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.5661000000000001e-05 2023-08-28T02:32:47.992 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.992 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.992 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:47.992 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.361207+0000", 2023-08-28T02:32:47.992 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1593999999999999e-05 2023-08-28T02:32:47.992 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.992 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.992 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:47.992 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.194596+0000", 2023-08-28T02:32:47.992 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.83338983600000005 2023-08-28T02:32:47.993 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.993 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.993 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:32:47.993 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.194603+0000", 2023-08-28T02:32:47.993 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.319e-06 2023-08-28T02:32:47.993 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:47.993 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:47.993 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:47.993 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.993 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.993 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.44594.0:123578 5.0 5.4fc442b0 (undecoded) ondisk+write+known_if_redirected e153)", 2023-08-28T02:32:47.993 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:23:58.384080+0000", 2023-08-28T02:32:47.993 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 529.57951643000001, 2023-08-28T02:32:47.993 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.81054503600000005, 2023-08-28T02:32:47.994 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:47.994 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "reached pg", 2023-08-28T02:32:47.994 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:32:47.994 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.44594", 2023-08-28T02:32:47.994 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/1673791452", 2023-08-28T02:32:47.994 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 123578 2023-08-28T02:32:47.994 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.994 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:47.994 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.994 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:47.994 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.384080+0000", 2023-08-28T02:32:47.994 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:47.994 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.995 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.995 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:47.995 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.384080+0000", 2023-08-28T02:32:47.995 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:47.995 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.995 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.995 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:47.995 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.384078+0000", 2023-08-28T02:32:47.995 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:32:47.995 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.995 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.995 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:47.995 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.384102+0000", 2023-08-28T02:32:47.996 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.4003e-05 2023-08-28T02:32:47.996 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.996 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.996 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:47.996 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.384126+0000", 2023-08-28T02:32:47.996 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.4209000000000001e-05 2023-08-28T02:32:47.996 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.996 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.996 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:47.996 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.384137+0000", 2023-08-28T02:32:47.996 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1518000000000001e-05 2023-08-28T02:32:47.996 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.996 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.997 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:47.997 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.384202+0000", 2023-08-28T02:32:47.997 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.478e-05 2023-08-28T02:32:47.997 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.997 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.997 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:47.997 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:58.384214+0000", 2023-08-28T02:32:47.997 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1683e-05 2023-08-28T02:32:47.997 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.997 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.997 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:47.997 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.194618+0000", 2023-08-28T02:32:47.997 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.81040443399999995 2023-08-28T02:32:47.997 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.998 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.998 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:32:47.998 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:23:59.194625+0000", 2023-08-28T02:32:47.998 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.5309999999999998e-06 2023-08-28T02:32:47.998 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:47.998 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:47.998 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:47.998 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.998 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.998 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:24531 6.b 6:df250847:::benchmark_data_smithi143_305742_object24530:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e195)", 2023-08-28T02:32:47.998 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:27:16.320361+0000", 2023-08-28T02:32:47.998 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 331.64323475999998, 2023-08-28T02:32:47.999 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.1070281099999999, 2023-08-28T02:32:47.999 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:47.999 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "reached pg", 2023-08-28T02:32:47.999 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:32:47.999 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:32:47.999 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:32:47.999 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 24531 2023-08-28T02:32:47.999 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:47.999 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:47.999 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:47.999 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:47.999 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.320361+0000", 2023-08-28T02:32:47.999 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:48.000 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.000 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.000 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:48.000 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.320361+0000", 2023-08-28T02:32:48.000 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:48.000 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.000 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.000 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:48.000 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.320360+0000", 2023-08-28T02:32:48.000 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:32:48.000 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.000 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.001 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:48.001 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.320379+0000", 2023-08-28T02:32:48.001 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.9511999999999999e-05 2023-08-28T02:32:48.001 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.001 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.001 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:48.001 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.320390+0000", 2023-08-28T02:32:48.001 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.045e-05 2023-08-28T02:32:48.001 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.001 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.001 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:48.001 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.320398+0000", 2023-08-28T02:32:48.002 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.1170000000000004e-06 2023-08-28T02:32:48.002 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.002 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.002 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:48.002 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.320457+0000", 2023-08-28T02:32:48.002 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.8777999999999999e-05 2023-08-28T02:32:48.002 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.002 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.002 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:32:48.002 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.320637+0000", 2023-08-28T02:32:48.002 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000180027 2023-08-28T02:32:48.002 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.002 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.003 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 6,7", 2023-08-28T02:32:48.003 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.320816+0000", 2023-08-28T02:32:48.003 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000178896 2023-08-28T02:32:48.003 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.003 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.003 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:48.003 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.326752+0000", 2023-08-28T02:32:48.003 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0059362620000000003 2023-08-28T02:32:48.003 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.003 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.003 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:48.003 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.326764+0000", 2023-08-28T02:32:48.004 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.2121e-05 2023-08-28T02:32:48.004 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.004 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.004 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:48.004 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:17.317026+0000", 2023-08-28T02:32:48.004 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.99026151500000004 2023-08-28T02:32:48.004 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.004 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.004 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:48.004 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:17.317041+0000", 2023-08-28T02:32:48.004 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.5868000000000001e-05 2023-08-28T02:32:48.004 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.004 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.005 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:48.005 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.411484+0000", 2023-08-28T02:32:48.005 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.094442685 2023-08-28T02:32:48.005 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.005 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.005 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for degraded object", 2023-08-28T02:32:48.005 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.412525+0000", 2023-08-28T02:32:48.005 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0010407999999999999 2023-08-28T02:32:48.005 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.005 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.005 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:48.005 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.427277+0000", 2023-08-28T02:32:48.005 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.014751874999999999 2023-08-28T02:32:48.006 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.006 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.006 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:32:48.006 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.427389+0000", 2023-08-28T02:32:48.006 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00011259700000000001 2023-08-28T02:32:48.006 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:48.006 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:48.006 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:48.006 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.006 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.006 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:24535 6.b 6:d57aa623:::benchmark_data_smithi143_305742_object24534:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e195)", 2023-08-28T02:32:48.006 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:27:16.320888+0000", 2023-08-28T02:32:48.007 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 331.64270825099999, 2023-08-28T02:32:48.007 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.1085812970000002, 2023-08-28T02:32:48.007 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:48.007 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "reached pg", 2023-08-28T02:32:48.007 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:32:48.007 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:32:48.007 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:32:48.007 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 24535 2023-08-28T02:32:48.007 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.007 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:48.007 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.007 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:48.007 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.320888+0000", 2023-08-28T02:32:48.008 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:48.008 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.008 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.008 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:48.008 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.320888+0000", 2023-08-28T02:32:48.008 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:48.008 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.008 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.008 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:48.008 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.320887+0000", 2023-08-28T02:32:48.008 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:32:48.008 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.008 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.009 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:48.009 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.320908+0000", 2023-08-28T02:32:48.009 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.124e-05 2023-08-28T02:32:48.009 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.009 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.009 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:48.009 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.320921+0000", 2023-08-28T02:32:48.009 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.259e-05 2023-08-28T02:32:48.009 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.009 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.009 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:48.009 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.320929+0000", 2023-08-28T02:32:48.009 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.0539999999999995e-06 2023-08-28T02:32:48.010 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.010 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.010 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:48.010 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.321877+0000", 2023-08-28T02:32:48.010 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00094830700000000001 2023-08-28T02:32:48.010 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.010 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.010 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:32:48.010 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.322041+0000", 2023-08-28T02:32:48.010 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00016411900000000001 2023-08-28T02:32:48.010 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.010 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.010 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 6,7", 2023-08-28T02:32:48.011 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.322212+0000", 2023-08-28T02:32:48.011 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00017074500000000001 2023-08-28T02:32:48.011 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.011 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.011 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:48.011 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.326837+0000", 2023-08-28T02:32:48.011 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0046252649999999999 2023-08-28T02:32:48.011 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.011 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.011 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:48.011 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.326845+0000", 2023-08-28T02:32:48.011 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.1589999999999993e-06 2023-08-28T02:32:48.012 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.012 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.012 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:48.012 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:17.318726+0000", 2023-08-28T02:32:48.012 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.99188029 2023-08-28T02:32:48.012 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.012 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.012 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:48.012 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:17.318739+0000", 2023-08-28T02:32:48.012 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3410999999999999e-05 2023-08-28T02:32:48.012 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.012 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.012 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:48.013 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.413991+0000", 2023-08-28T02:32:48.013 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.095252347 2023-08-28T02:32:48.013 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.013 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.013 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for degraded object", 2023-08-28T02:32:48.013 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.414851+0000", 2023-08-28T02:32:48.013 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00085915000000000004 2023-08-28T02:32:48.013 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.013 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.013 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:48.013 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.429365+0000", 2023-08-28T02:32:48.013 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.01451469 2023-08-28T02:32:48.013 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.014 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.014 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:32:48.014 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.429469+0000", 2023-08-28T02:32:48.014 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00010385100000000001 2023-08-28T02:32:48.014 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:48.014 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:48.014 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:48.014 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.014 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.014 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:24539 6.c 6:37ee2adc:::benchmark_data_smithi143_305742_object24538:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e196)", 2023-08-28T02:32:48.014 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:27:16.328855+0000", 2023-08-28T02:32:48.014 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 331.63474155900002, 2023-08-28T02:32:48.015 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.0684156310000001, 2023-08-28T02:32:48.015 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:48.015 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:32:48.015 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:32:48.015 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:32:48.015 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:32:48.015 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 24539 2023-08-28T02:32:48.015 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.015 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:48.015 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.015 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:48.015 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.328855+0000", 2023-08-28T02:32:48.015 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:48.016 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.016 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.016 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:48.016 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.328855+0000", 2023-08-28T02:32:48.016 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:48.016 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.016 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.016 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:48.016 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.328853+0000", 2023-08-28T02:32:48.016 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:32:48.016 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.016 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.016 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:48.017 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.328886+0000", 2023-08-28T02:32:48.017 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.3365000000000003e-05 2023-08-28T02:32:48.017 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.017 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.017 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:48.017 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.328899+0000", 2023-08-28T02:32:48.017 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.2480000000000001e-05 2023-08-28T02:32:48.017 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.017 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.017 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:48.017 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.328907+0000", 2023-08-28T02:32:48.017 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.4090000000000003e-06 2023-08-28T02:32:48.018 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.018 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.018 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:48.018 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.328959+0000", 2023-08-28T02:32:48.018 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.2089999999999998e-05 2023-08-28T02:32:48.018 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.018 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.018 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:48.018 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.328970+0000", 2023-08-28T02:32:48.018 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0574000000000001e-05 2023-08-28T02:32:48.018 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.018 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.018 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:48.019 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:17.318763+0000", 2023-08-28T02:32:48.019 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.98979299099999996 2023-08-28T02:32:48.019 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.019 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.019 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:48.019 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:17.318773+0000", 2023-08-28T02:32:48.019 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.9199999999999999e-06 2023-08-28T02:32:48.019 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.019 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.019 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:48.019 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.389871+0000", 2023-08-28T02:32:48.019 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0710983110000001 2023-08-28T02:32:48.019 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.020 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.020 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:32:48.020 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.390037+0000", 2023-08-28T02:32:48.020 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000166016 2023-08-28T02:32:48.020 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.020 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.020 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 5,7", 2023-08-28T02:32:48.020 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.390215+0000", 2023-08-28T02:32:48.020 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00017822000000000001 2023-08-28T02:32:48.020 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.020 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.020 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:48.020 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.392419+0000", 2023-08-28T02:32:48.021 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0022036859999999998 2023-08-28T02:32:48.021 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.021 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.021 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:48.021 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.395920+0000", 2023-08-28T02:32:48.021 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0035014930000000001 2023-08-28T02:32:48.021 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.021 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.021 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:32:48.021 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.396023+0000", 2023-08-28T02:32:48.021 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00010273899999999999 2023-08-28T02:32:48.021 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.021 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.022 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:32:48.022 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.396082+0000", 2023-08-28T02:32:48.022 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.8977999999999997e-05 2023-08-28T02:32:48.022 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.022 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.022 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:32:48.022 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.397270+0000", 2023-08-28T02:32:48.022 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0011880510000000001 2023-08-28T02:32:48.022 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:48.022 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:48.022 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:48.022 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.023 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.023 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:24543 6.c 6:3872c264:::benchmark_data_smithi143_305742_object24542:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e196)", 2023-08-28T02:32:48.023 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:27:16.333172+0000", 2023-08-28T02:32:48.023 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 331.63042394199999, 2023-08-28T02:32:48.023 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.0692718540000001, 2023-08-28T02:32:48.023 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:48.023 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:32:48.023 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:32:48.023 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:32:48.023 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:32:48.023 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 24543 2023-08-28T02:32:48.023 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.023 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:48.024 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.024 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:48.024 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.333172+0000", 2023-08-28T02:32:48.024 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:48.024 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.024 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.024 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:48.024 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.333172+0000", 2023-08-28T02:32:48.024 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:48.024 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.024 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.024 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:48.024 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.333170+0000", 2023-08-28T02:32:48.025 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:32:48.025 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.025 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.025 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:48.025 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.333197+0000", 2023-08-28T02:32:48.025 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.6242e-05 2023-08-28T02:32:48.025 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.025 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.025 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:48.025 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.333212+0000", 2023-08-28T02:32:48.025 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.5457000000000001e-05 2023-08-28T02:32:48.025 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.025 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.026 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:48.026 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.333221+0000", 2023-08-28T02:32:48.026 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.0110000000000006e-06 2023-08-28T02:32:48.026 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.026 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.026 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:48.026 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.333266+0000", 2023-08-28T02:32:48.026 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.5185000000000002e-05 2023-08-28T02:32:48.026 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.026 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.026 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:48.026 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.333277+0000", 2023-08-28T02:32:48.026 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1113e-05 2023-08-28T02:32:48.027 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.027 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.027 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:48.027 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:17.318796+0000", 2023-08-28T02:32:48.027 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.98551865400000005 2023-08-28T02:32:48.027 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.027 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.027 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:48.027 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:17.318803+0000", 2023-08-28T02:32:48.027 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.8669999999999996e-06 2023-08-28T02:32:48.027 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.027 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.027 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:48.028 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.392449+0000", 2023-08-28T02:32:48.028 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0736462950000001 2023-08-28T02:32:48.028 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.028 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.028 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:32:48.028 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.392620+0000", 2023-08-28T02:32:48.028 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00017026000000000001 2023-08-28T02:32:48.028 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.028 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.028 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 5,7", 2023-08-28T02:32:48.028 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.392829+0000", 2023-08-28T02:32:48.028 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000209471 2023-08-28T02:32:48.029 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.029 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.029 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:32:48.029 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.397281+0000", 2023-08-28T02:32:48.029 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0044518149999999996 2023-08-28T02:32:48.029 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.029 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.029 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:48.029 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.397661+0000", 2023-08-28T02:32:48.029 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00038041300000000002 2023-08-28T02:32:48.029 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.029 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.029 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:48.030 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.401154+0000", 2023-08-28T02:32:48.030 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0034931839999999999 2023-08-28T02:32:48.030 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.030 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.030 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:32:48.030 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.401223+0000", 2023-08-28T02:32:48.030 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.8558000000000003e-05 2023-08-28T02:32:48.030 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.030 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.030 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:32:48.030 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.402444+0000", 2023-08-28T02:32:48.030 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.001220989 2023-08-28T02:32:48.030 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:48.031 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:48.031 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:48.031 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.031 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.031 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:24547 6.b 6:d3c1f3d7:::benchmark_data_smithi143_305742_object24546:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e196)", 2023-08-28T02:32:48.031 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:27:16.334218+0000", 2023-08-28T02:32:48.031 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 331.62937780999999, 2023-08-28T02:32:48.031 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.0963319720000002, 2023-08-28T02:32:48.031 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:48.031 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:32:48.031 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:32:48.031 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:32:48.032 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:32:48.032 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 24547 2023-08-28T02:32:48.032 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.032 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:48.032 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.032 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:48.032 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.334218+0000", 2023-08-28T02:32:48.032 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:48.032 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.032 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.032 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:48.032 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.334218+0000", 2023-08-28T02:32:48.032 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:48.033 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.033 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.033 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:48.033 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.334218+0000", 2023-08-28T02:32:48.033 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:32:48.033 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.033 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.033 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:48.033 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.334239+0000", 2023-08-28T02:32:48.033 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.1067999999999999e-05 2023-08-28T02:32:48.033 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.033 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.033 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:48.033 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.334246+0000", 2023-08-28T02:32:48.034 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.99e-06 2023-08-28T02:32:48.034 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.034 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.034 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:48.034 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.334251+0000", 2023-08-28T02:32:48.034 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.57e-06 2023-08-28T02:32:48.034 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.034 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.034 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:48.034 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.334304+0000", 2023-08-28T02:32:48.034 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.2512999999999997e-05 2023-08-28T02:32:48.034 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.035 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.035 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:48.035 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.334311+0000", 2023-08-28T02:32:48.035 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.9480000000000002e-06 2023-08-28T02:32:48.035 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.035 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.035 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:48.035 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:17.318776+0000", 2023-08-28T02:32:48.035 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.98446481100000005 2023-08-28T02:32:48.035 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.035 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.035 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:48.035 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:17.318784+0000", 2023-08-28T02:32:48.036 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.5399999999999996e-06 2023-08-28T02:32:48.036 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.036 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.036 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:48.036 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.414890+0000", 2023-08-28T02:32:48.036 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0961061059999999 2023-08-28T02:32:48.036 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.036 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.036 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:32:48.036 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.415073+0000", 2023-08-28T02:32:48.036 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000182981 2023-08-28T02:32:48.036 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.036 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.037 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 4,7", 2023-08-28T02:32:48.037 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.415266+0000", 2023-08-28T02:32:48.037 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00019221099999999999 2023-08-28T02:32:48.037 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.037 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.037 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:32:48.037 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.419976+0000", 2023-08-28T02:32:48.037 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0047105030000000004 2023-08-28T02:32:48.037 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.037 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.037 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:48.037 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.429512+0000", 2023-08-28T02:32:48.037 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0095358749999999992 2023-08-28T02:32:48.037 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.038 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.038 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:48.038 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.429591+0000", 2023-08-28T02:32:48.038 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.9344000000000005e-05 2023-08-28T02:32:48.038 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.038 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.038 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:32:48.038 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.429643+0000", 2023-08-28T02:32:48.038 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.1643e-05 2023-08-28T02:32:48.038 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.038 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.038 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:32:48.038 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.430550+0000", 2023-08-28T02:32:48.039 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00090736799999999998 2023-08-28T02:32:48.039 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:48.039 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:48.039 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:48.039 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.039 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.039 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:24550 6.c 6:3ac8366e:::benchmark_data_smithi143_305742_object24549:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e196)", 2023-08-28T02:32:48.039 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:27:16.341304+0000", 2023-08-28T02:32:48.039 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 331.62229193500002, 2023-08-28T02:32:48.039 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.0622437910000002, 2023-08-28T02:32:48.039 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:48.039 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:32:48.039 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:32:48.040 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:32:48.040 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:32:48.040 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 24550 2023-08-28T02:32:48.040 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.040 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:48.040 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.040 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:48.040 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.341304+0000", 2023-08-28T02:32:48.040 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:48.040 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.040 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.040 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:48.040 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.341304+0000", 2023-08-28T02:32:48.041 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:48.041 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.041 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.041 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:48.041 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.341302+0000", 2023-08-28T02:32:48.041 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:32:48.041 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.041 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.041 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:48.041 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.341327+0000", 2023-08-28T02:32:48.041 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.4898000000000001e-05 2023-08-28T02:32:48.041 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.042 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.042 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:48.042 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.341344+0000", 2023-08-28T02:32:48.042 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.649e-05 2023-08-28T02:32:48.042 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.042 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.042 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:48.042 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.341353+0000", 2023-08-28T02:32:48.042 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.8079999999999994e-06 2023-08-28T02:32:48.042 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.042 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.042 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:48.042 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.341409+0000", 2023-08-28T02:32:48.043 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.6702e-05 2023-08-28T02:32:48.043 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.043 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.043 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:48.043 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.341418+0000", 2023-08-28T02:32:48.043 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.3480000000000004e-06 2023-08-28T02:32:48.043 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.043 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.043 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:48.043 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:17.318825+0000", 2023-08-28T02:32:48.043 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.977407523 2023-08-28T02:32:48.043 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.043 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.044 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:48.044 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:17.318833+0000", 2023-08-28T02:32:48.044 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.2609999999999996e-06 2023-08-28T02:32:48.044 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.044 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.044 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:48.044 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.394062+0000", 2023-08-28T02:32:48.044 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.075229827 2023-08-28T02:32:48.044 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.044 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.044 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:32:48.044 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.394241+0000", 2023-08-28T02:32:48.045 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00017895699999999999 2023-08-28T02:32:48.045 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.045 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.045 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 5,7", 2023-08-28T02:32:48.045 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.394429+0000", 2023-08-28T02:32:48.045 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00018816500000000001 2023-08-28T02:32:48.045 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.045 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.045 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:32:48.045 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.397287+0000", 2023-08-28T02:32:48.045 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00285746 2023-08-28T02:32:48.045 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.045 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.046 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:48.046 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.397696+0000", 2023-08-28T02:32:48.046 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00040932400000000003 2023-08-28T02:32:48.046 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.046 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.046 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:48.046 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.402502+0000", 2023-08-28T02:32:48.046 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0048053749999999998 2023-08-28T02:32:48.046 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.046 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.046 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:32:48.046 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.402561+0000", 2023-08-28T02:32:48.046 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.8909000000000001e-05 2023-08-28T02:32:48.047 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.047 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.047 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:32:48.047 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.403548+0000", 2023-08-28T02:32:48.047 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00098740299999999998 2023-08-28T02:32:48.047 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:48.047 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:48.047 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:48.047 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.047 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.047 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:24552 6.b 6:df693f16:::benchmark_data_smithi143_305742_object24551:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e196)", 2023-08-28T02:32:48.047 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:27:16.342014+0000", 2023-08-28T02:32:48.048 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 331.62158221999999, 2023-08-28T02:32:48.048 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.089540714, 2023-08-28T02:32:48.048 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:48.048 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:32:48.048 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:32:48.048 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:32:48.048 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:32:48.048 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 24552 2023-08-28T02:32:48.048 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.048 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:48.048 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.048 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:48.048 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.342014+0000", 2023-08-28T02:32:48.049 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:48.049 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.049 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.049 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:48.049 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.342014+0000", 2023-08-28T02:32:48.049 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:48.049 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.049 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.049 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:48.049 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.342013+0000", 2023-08-28T02:32:48.049 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:32:48.049 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.049 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.050 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:48.050 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.342033+0000", 2023-08-28T02:32:48.050 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.9669000000000001e-05 2023-08-28T02:32:48.050 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.050 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.050 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:48.050 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.342053+0000", 2023-08-28T02:32:48.050 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.9826999999999998e-05 2023-08-28T02:32:48.050 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.050 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.050 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:48.050 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.342065+0000", 2023-08-28T02:32:48.051 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.2617e-05 2023-08-28T02:32:48.051 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.051 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.051 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:48.051 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.342125+0000", 2023-08-28T02:32:48.051 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.9490000000000001e-05 2023-08-28T02:32:48.051 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.051 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.051 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:48.051 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.342133+0000", 2023-08-28T02:32:48.051 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.6810000000000004e-06 2023-08-28T02:32:48.051 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.051 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.052 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:48.052 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:17.318812+0000", 2023-08-28T02:32:48.052 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.97667820100000002 2023-08-28T02:32:48.052 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.052 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.052 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:48.052 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:17.318820+0000", 2023-08-28T02:32:48.052 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.2409999999999995e-06 2023-08-28T02:32:48.052 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.052 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.052 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:48.052 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.416769+0000", 2023-08-28T02:32:48.052 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.097949235 2023-08-28T02:32:48.053 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.053 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.053 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:32:48.053 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.416955+0000", 2023-08-28T02:32:48.053 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00018632300000000001 2023-08-28T02:32:48.053 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.053 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.053 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 4,7", 2023-08-28T02:32:48.053 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.417157+0000", 2023-08-28T02:32:48.053 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00020180999999999999 2023-08-28T02:32:48.053 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.053 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.053 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:32:48.054 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.419984+0000", 2023-08-28T02:32:48.054 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.002826781 2023-08-28T02:32:48.054 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.054 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.054 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:48.054 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.429553+0000", 2023-08-28T02:32:48.054 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0095693670000000005 2023-08-28T02:32:48.054 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.054 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.054 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:48.054 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.430600+0000", 2023-08-28T02:32:48.054 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.001047217 2023-08-28T02:32:48.055 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.055 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.055 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:32:48.055 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.430650+0000", 2023-08-28T02:32:48.055 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.9903e-05 2023-08-28T02:32:48.055 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.055 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.055 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:32:48.055 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.431555+0000", 2023-08-28T02:32:48.055 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00090419100000000004 2023-08-28T02:32:48.055 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:48.055 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:48.055 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:48.056 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.056 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.056 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:24559 6.b 6:d824c343:::benchmark_data_smithi143_305742_object24558:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e196)", 2023-08-28T02:32:48.056 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:27:16.354268+0000", 2023-08-28T02:32:48.056 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 331.60932855700003, 2023-08-28T02:32:48.056 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.0782777179999998, 2023-08-28T02:32:48.056 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:48.056 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:32:48.056 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:32:48.056 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:32:48.056 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:32:48.056 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 24559 2023-08-28T02:32:48.056 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.057 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:48.057 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.057 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:48.057 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.354268+0000", 2023-08-28T02:32:48.057 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:48.057 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.057 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.057 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:48.057 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.354268+0000", 2023-08-28T02:32:48.057 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:48.057 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.057 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.058 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:48.058 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.354267+0000", 2023-08-28T02:32:48.058 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:32:48.058 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.058 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.058 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:48.058 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.354286+0000", 2023-08-28T02:32:48.058 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.9307e-05 2023-08-28T02:32:48.058 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.058 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.058 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:48.058 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.354297+0000", 2023-08-28T02:32:48.058 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1544e-05 2023-08-28T02:32:48.059 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.059 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.059 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:48.059 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.354303+0000", 2023-08-28T02:32:48.059 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.6860000000000003e-06 2023-08-28T02:32:48.059 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.059 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.059 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:48.059 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.354363+0000", 2023-08-28T02:32:48.059 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.0337999999999997e-05 2023-08-28T02:32:48.059 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.059 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.060 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:48.060 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:16.354376+0000", 2023-08-28T02:32:48.060 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.2218999999999999e-05 2023-08-28T02:32:48.060 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.060 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.060 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:48.060 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:17.318866+0000", 2023-08-28T02:32:48.060 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.96449035000000005 2023-08-28T02:32:48.060 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.060 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.060 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:48.060 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:17.318874+0000", 2023-08-28T02:32:48.060 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.4039999999999995e-06 2023-08-28T02:32:48.061 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.061 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.061 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:48.061 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.418477+0000", 2023-08-28T02:32:48.061 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.099602873 2023-08-28T02:32:48.061 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.061 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.061 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:32:48.061 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.418650+0000", 2023-08-28T02:32:48.061 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000173134 2023-08-28T02:32:48.061 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.061 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.061 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 4,7", 2023-08-28T02:32:48.062 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.418804+0000", 2023-08-28T02:32:48.062 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00015363400000000001 2023-08-28T02:32:48.062 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.062 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.062 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:32:48.062 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.420430+0000", 2023-08-28T02:32:48.062 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0016257139999999999 2023-08-28T02:32:48.062 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.062 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.062 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:48.062 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.431611+0000", 2023-08-28T02:32:48.062 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.011181590999999999 2023-08-28T02:32:48.062 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.063 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.063 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:48.063 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.431651+0000", 2023-08-28T02:32:48.063 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.9746000000000002e-05 2023-08-28T02:32:48.063 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.063 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.063 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:32:48.063 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.431700+0000", 2023-08-28T02:32:48.063 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.9177e-05 2023-08-28T02:32:48.063 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.063 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.063 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:32:48.064 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:27:18.432545+0000", 2023-08-28T02:32:48.064 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00084502299999999998 2023-08-28T02:32:48.064 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:48.064 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:48.064 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:48.064 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.064 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.064 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:78158 6.c 6:3c2b9257:::benchmark_data_smithi143_305742_object78157:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e220)", 2023-08-28T02:32:48.064 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:28:24.351475+0000", 2023-08-28T02:32:48.064 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 263.61212084099998, 2023-08-28T02:32:48.064 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.115024359, 2023-08-28T02:32:48.064 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:48.064 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:32:48.065 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:32:48.065 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:32:48.065 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:32:48.065 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 78158 2023-08-28T02:32:48.065 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.065 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:48.065 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.065 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:48.065 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.351475+0000", 2023-08-28T02:32:48.065 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:48.065 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.065 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.065 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:48.066 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.351475+0000", 2023-08-28T02:32:48.066 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:48.066 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.066 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.066 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:48.066 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.351473+0000", 2023-08-28T02:32:48.066 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999976 2023-08-28T02:32:48.066 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.066 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.066 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:48.066 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.351505+0000", 2023-08-28T02:32:48.066 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.2289999999999997e-05 2023-08-28T02:32:48.067 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.067 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.067 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:48.067 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.351530+0000", 2023-08-28T02:32:48.067 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.4946000000000001e-05 2023-08-28T02:32:48.067 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.067 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.067 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:48.067 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.351544+0000", 2023-08-28T02:32:48.067 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3298e-05 2023-08-28T02:32:48.067 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.067 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.067 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:48.068 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.352280+0000", 2023-08-28T02:32:48.068 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00073618100000000003 2023-08-28T02:32:48.068 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.068 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.068 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:48.068 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.352291+0000", 2023-08-28T02:32:48.068 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0964e-05 2023-08-28T02:32:48.068 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.068 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.068 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:48.068 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.331636+0000", 2023-08-28T02:32:48.068 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.97934522000000002 2023-08-28T02:32:48.068 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.068 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.069 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:48.069 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.331647+0000", 2023-08-28T02:32:48.069 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1392e-05 2023-08-28T02:32:48.069 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.069 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.069 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:48.069 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.452208+0000", 2023-08-28T02:32:48.069 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.120560625 2023-08-28T02:32:48.069 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.069 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.069 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:32:48.069 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.452384+0000", 2023-08-28T02:32:48.069 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00017561200000000001 2023-08-28T02:32:48.070 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.070 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.070 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 5,6,7", 2023-08-28T02:32:48.070 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.452555+0000", 2023-08-28T02:32:48.070 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00017118799999999999 2023-08-28T02:32:48.070 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.070 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.070 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:48.070 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.455618+0000", 2023-08-28T02:32:48.070 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0030633460000000002 2023-08-28T02:32:48.070 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.070 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.071 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:32:48.071 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.455688+0000", 2023-08-28T02:32:48.071 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.9611999999999997e-05 2023-08-28T02:32:48.071 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.071 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.071 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:48.071 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.458168+0000", 2023-08-28T02:32:48.071 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0024801239999999998 2023-08-28T02:32:48.071 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.071 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.071 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:48.071 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.466293+0000", 2023-08-28T02:32:48.071 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0081255110000000002 2023-08-28T02:32:48.071 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.072 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.072 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:32:48.072 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.466366+0000", 2023-08-28T02:32:48.072 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.2871000000000001e-05 2023-08-28T02:32:48.072 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.072 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.072 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:32:48.072 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.466500+0000", 2023-08-28T02:32:48.072 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00013335899999999999 2023-08-28T02:32:48.072 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:48.072 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:48.072 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:48.072 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.073 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.073 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:78162 6.c 6:318d1535:::benchmark_data_smithi143_305742_object78161:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e220)", 2023-08-28T02:32:48.073 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:28:24.353558+0000", 2023-08-28T02:32:48.073 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 263.61003843200001, 2023-08-28T02:32:48.073 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.1131857620000001, 2023-08-28T02:32:48.073 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:48.073 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:32:48.073 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:32:48.073 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:32:48.073 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:32:48.073 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 78162 2023-08-28T02:32:48.073 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.073 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:48.074 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.074 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:48.074 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.353558+0000", 2023-08-28T02:32:48.074 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:48.074 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.074 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.074 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:48.074 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.353558+0000", 2023-08-28T02:32:48.074 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:48.074 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.074 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.074 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:48.074 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.353556+0000", 2023-08-28T02:32:48.075 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:32:48.075 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.075 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.075 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:48.075 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.353594+0000", 2023-08-28T02:32:48.075 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.7595999999999997e-05 2023-08-28T02:32:48.075 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.075 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.075 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:48.075 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.353608+0000", 2023-08-28T02:32:48.075 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.4416000000000001e-05 2023-08-28T02:32:48.075 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.075 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.076 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:48.076 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.353617+0000", 2023-08-28T02:32:48.076 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.9069999999999996e-06 2023-08-28T02:32:48.076 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.076 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.076 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:48.076 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.353671+0000", 2023-08-28T02:32:48.076 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.3871e-05 2023-08-28T02:32:48.076 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.076 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.076 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:48.076 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.353681+0000", 2023-08-28T02:32:48.077 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0713999999999999e-05 2023-08-28T02:32:48.077 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.077 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.077 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:48.077 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.331885+0000", 2023-08-28T02:32:48.077 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.97820372 2023-08-28T02:32:48.077 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.077 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.077 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:48.077 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.331895+0000", 2023-08-28T02:32:48.077 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.9920000000000002e-06 2023-08-28T02:32:48.077 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.077 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.077 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:48.078 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.455726+0000", 2023-08-28T02:32:48.078 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1238307970000001 2023-08-28T02:32:48.078 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.078 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.078 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:32:48.078 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.455870+0000", 2023-08-28T02:32:48.078 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000143814 2023-08-28T02:32:48.078 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.078 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.078 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 5,6,7", 2023-08-28T02:32:48.078 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.456053+0000", 2023-08-28T02:32:48.078 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000183234 2023-08-28T02:32:48.078 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.079 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.079 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:32:48.079 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.461802+0000", 2023-08-28T02:32:48.079 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0057491449999999998 2023-08-28T02:32:48.079 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.079 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.079 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:48.079 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.464367+0000", 2023-08-28T02:32:48.079 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0025651480000000002 2023-08-28T02:32:48.079 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.079 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.079 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:48.079 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.464470+0000", 2023-08-28T02:32:48.080 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00010294 2023-08-28T02:32:48.080 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.080 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.080 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:48.080 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.466571+0000", 2023-08-28T02:32:48.080 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0021002669999999998 2023-08-28T02:32:48.080 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.080 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.080 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:32:48.080 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.466630+0000", 2023-08-28T02:32:48.080 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.9827000000000002e-05 2023-08-28T02:32:48.080 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.080 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.081 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:32:48.081 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.466743+0000", 2023-08-28T02:32:48.081 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000113047 2023-08-28T02:32:48.081 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:48.081 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:48.081 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:48.081 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.081 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.081 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:78191 6.c 6:388e80c8:::benchmark_data_smithi143_305742_object78190:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e221)", 2023-08-28T02:32:48.081 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:28:25.485179+0000", 2023-08-28T02:32:48.081 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 262.47841727299999, 2023-08-28T02:32:48.081 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.98181897100000004, 2023-08-28T02:32:48.081 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:48.082 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:32:48.082 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:32:48.082 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:32:48.082 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:32:48.082 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 78191 2023-08-28T02:32:48.082 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.082 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:48.082 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.082 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:48.082 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.485179+0000", 2023-08-28T02:32:48.082 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:48.082 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.082 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.083 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:48.083 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.485179+0000", 2023-08-28T02:32:48.083 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:48.083 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.083 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.083 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:48.083 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.485177+0000", 2023-08-28T02:32:48.083 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:32:48.083 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.083 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.083 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:48.083 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.485203+0000", 2023-08-28T02:32:48.083 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.6021e-05 2023-08-28T02:32:48.084 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.084 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.084 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:48.084 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.485220+0000", 2023-08-28T02:32:48.084 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.7220000000000001e-05 2023-08-28T02:32:48.084 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.084 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.084 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:48.084 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.485230+0000", 2023-08-28T02:32:48.084 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.3940000000000002e-06 2023-08-28T02:32:48.084 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.084 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.084 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:48.085 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.485298+0000", 2023-08-28T02:32:48.085 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.8418999999999997e-05 2023-08-28T02:32:48.085 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.085 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.085 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:48.085 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.485312+0000", 2023-08-28T02:32:48.085 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.4382000000000001e-05 2023-08-28T02:32:48.085 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.085 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.085 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:48.085 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.458189+0000", 2023-08-28T02:32:48.085 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.97287682399999997 2023-08-28T02:32:48.085 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.085 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.086 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:32:48.086 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.458284+0000", 2023-08-28T02:32:48.086 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.4900000000000003e-05 2023-08-28T02:32:48.086 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.086 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.086 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 5,6,7", 2023-08-28T02:32:48.086 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.458384+0000", 2023-08-28T02:32:48.086 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.9763e-05 2023-08-28T02:32:48.086 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.086 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.086 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:32:48.086 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.461806+0000", 2023-08-28T02:32:48.086 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0034222649999999999 2023-08-28T02:32:48.087 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.087 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.087 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:48.087 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.464511+0000", 2023-08-28T02:32:48.087 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0027045929999999999 2023-08-28T02:32:48.087 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.087 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.087 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:48.087 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.464548+0000", 2023-08-28T02:32:48.087 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.7626999999999997e-05 2023-08-28T02:32:48.087 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.087 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.087 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:48.088 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.466803+0000", 2023-08-28T02:32:48.088 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0022543279999999999 2023-08-28T02:32:48.088 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.088 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.088 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:32:48.088 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.466866+0000", 2023-08-28T02:32:48.088 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.3870999999999999e-05 2023-08-28T02:32:48.088 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.088 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.088 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:32:48.088 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.466998+0000", 2023-08-28T02:32:48.088 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00013130699999999999 2023-08-28T02:32:48.088 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:48.089 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:48.089 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:48.089 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.089 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.089 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:78192 6.c 6:36cee94b:::benchmark_data_smithi143_305742_object78191:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e221)", 2023-08-28T02:32:48.089 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:28:25.494860+0000", 2023-08-28T02:32:48.089 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 262.46873601999999, 2023-08-28T02:32:48.089 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.97873924599999995, 2023-08-28T02:32:48.089 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:48.089 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:32:48.089 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:32:48.089 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:32:48.089 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:32:48.090 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 78192 2023-08-28T02:32:48.090 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.090 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:48.090 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.090 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:48.090 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.494860+0000", 2023-08-28T02:32:48.090 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:48.090 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.090 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.090 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:48.090 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.494860+0000", 2023-08-28T02:32:48.090 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:48.090 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.091 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.091 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:48.091 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.494859+0000", 2023-08-28T02:32:48.091 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:32:48.091 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.091 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.091 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:48.091 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.494878+0000", 2023-08-28T02:32:48.091 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.8918000000000001e-05 2023-08-28T02:32:48.091 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.091 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.091 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:48.092 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.494891+0000", 2023-08-28T02:32:48.092 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3122000000000001e-05 2023-08-28T02:32:48.092 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.092 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.092 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:48.092 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.494897+0000", 2023-08-28T02:32:48.092 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.1929999999999998e-06 2023-08-28T02:32:48.092 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.092 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.092 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:48.092 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.494945+0000", 2023-08-28T02:32:48.092 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.8053999999999998e-05 2023-08-28T02:32:48.093 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.093 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.093 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:48.093 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.494956+0000", 2023-08-28T02:32:48.093 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0399e-05 2023-08-28T02:32:48.093 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.093 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.093 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:48.093 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.459931+0000", 2023-08-28T02:32:48.093 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.96497512699999999 2023-08-28T02:32:48.093 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.093 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.093 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:32:48.094 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.460075+0000", 2023-08-28T02:32:48.094 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000144571 2023-08-28T02:32:48.094 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.094 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.094 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 5,6,7", 2023-08-28T02:32:48.094 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.460181+0000", 2023-08-28T02:32:48.094 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000105993 2023-08-28T02:32:48.094 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.094 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.094 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:32:48.094 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.461924+0000", 2023-08-28T02:32:48.094 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.001743047 2023-08-28T02:32:48.094 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.095 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.095 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:48.095 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.464585+0000", 2023-08-28T02:32:48.095 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0026605359999999998 2023-08-28T02:32:48.095 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.095 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.095 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:48.095 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.464630+0000", 2023-08-28T02:32:48.095 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.4801e-05 2023-08-28T02:32:48.095 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.095 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.095 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:48.095 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.473424+0000", 2023-08-28T02:32:48.096 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0087939469999999999 2023-08-28T02:32:48.096 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.096 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.096 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:32:48.096 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.473484+0000", 2023-08-28T02:32:48.096 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.0270999999999999e-05 2023-08-28T02:32:48.096 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.096 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.096 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:32:48.096 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.473599+0000", 2023-08-28T02:32:48.096 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000115362 2023-08-28T02:32:48.096 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:48.096 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:48.097 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:48.097 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.097 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.097 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.55270.0:59980 7.3 7:c246cc2e:::benchmark_data_smithi143_360738_object59979:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e270)", 2023-08-28T02:32:48.097 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:31:43.482132+0000", 2023-08-28T02:32:48.097 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 64.481463667, 2023-08-28T02:32:48.097 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.981863826, 2023-08-28T02:32:48.097 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:48.097 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "reached pg", 2023-08-28T02:32:48.097 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:32:48.097 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.55270", 2023-08-28T02:32:48.097 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:32:48.098 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 59980 2023-08-28T02:32:48.098 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.098 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:48.098 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.098 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:48.098 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.482132+0000", 2023-08-28T02:32:48.098 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:48.098 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.098 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.098 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:48.098 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.482132+0000", 2023-08-28T02:32:48.098 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:48.098 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.099 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.099 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:48.099 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.482131+0000", 2023-08-28T02:32:48.099 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:32:48.099 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.099 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.099 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:48.099 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.482158+0000", 2023-08-28T02:32:48.099 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.7708999999999998e-05 2023-08-28T02:32:48.099 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.099 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.099 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:48.100 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.482178+0000", 2023-08-28T02:32:48.100 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.9840999999999999e-05 2023-08-28T02:32:48.100 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.100 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.100 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:48.100 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.482188+0000", 2023-08-28T02:32:48.100 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.7610000000000002e-06 2023-08-28T02:32:48.100 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.100 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.100 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:48.100 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.483028+0000", 2023-08-28T02:32:48.100 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00084029899999999997 2023-08-28T02:32:48.100 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.100 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.101 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op must wait for map", 2023-08-28T02:32:48.101 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.483034+0000", 2023-08-28T02:32:48.101 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.3290000000000001e-06 2023-08-28T02:32:48.101 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.101 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.101 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:48.102 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:47 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:32:48.102 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:47 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:32:48.102 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:47 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:32:48.102 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.641234+0000", 2023-08-28T02:32:48.102 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.15820019499999999 2023-08-28T02:32:48.102 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.102 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.102 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:48.102 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.641247+0000", 2023-08-28T02:32:48.102 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3149999999999999e-05 2023-08-28T02:32:48.103 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.103 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.103 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:48.103 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:44.463896+0000", 2023-08-28T02:32:48.103 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.82264939100000001 2023-08-28T02:32:48.103 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.103 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.103 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:32:48.103 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:44.463996+0000", 2023-08-28T02:32:48.103 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.9920000000000006e-05 2023-08-28T02:32:48.103 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:48.103 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:48.103 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:48.104 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.104 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.104 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.55270.0:59989 7.3 7:c10c4578:::benchmark_data_smithi143_360738_object59988:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e270)", 2023-08-28T02:32:48.104 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:31:43.497113+0000", 2023-08-28T02:32:48.104 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 64.466482627000005, 2023-08-28T02:32:48.104 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.98866365899999997, 2023-08-28T02:32:48.104 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:32:48.104 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:32:48.104 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:32:48.104 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.55270", 2023-08-28T02:32:48.104 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:32:48.104 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 59989 2023-08-28T02:32:48.104 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.105 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:32:48.105 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.105 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:32:48.105 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.497113+0000", 2023-08-28T02:32:48.105 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:48.105 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.105 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.105 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:32:48.105 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.497113+0000", 2023-08-28T02:32:48.105 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:32:48.105 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.105 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.105 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:32:48.106 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.497111+0000", 2023-08-28T02:32:48.106 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999971 2023-08-28T02:32:48.106 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.106 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.106 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:32:48.106 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.497159+0000", 2023-08-28T02:32:48.106 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.8668000000000002e-05 2023-08-28T02:32:48.106 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.106 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.106 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:32:48.106 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.497184+0000", 2023-08-28T02:32:48.106 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.4264999999999999e-05 2023-08-28T02:32:48.106 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.106 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.107 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:32:48.107 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.497198+0000", 2023-08-28T02:32:48.107 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3845999999999999e-05 2023-08-28T02:32:48.107 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.107 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.107 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:48.107 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.641768+0000", 2023-08-28T02:32:48.107 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.144570584 2023-08-28T02:32:48.107 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.107 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.107 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:32:48.107 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.641775+0000", 2023-08-28T02:32:48.107 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.7329999999999997e-06 2023-08-28T02:32:48.108 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.108 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.108 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:32:48.108 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:44.467475+0000", 2023-08-28T02:32:48.108 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.82569998700000002 2023-08-28T02:32:48.108 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.108 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.108 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:32:48.108 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:44.467619+0000", 2023-08-28T02:32:48.108 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000143819 2023-08-28T02:32:48.108 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.108 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.108 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 2,4", 2023-08-28T02:32:48.109 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:44.467768+0000", 2023-08-28T02:32:48.109 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000149087 2023-08-28T02:32:48.109 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.109 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.109 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:32:48.109 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:44.473197+0000", 2023-08-28T02:32:48.109 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0054293579999999996 2023-08-28T02:32:48.109 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.109 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.109 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:48.109 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:44.476440+0000", 2023-08-28T02:32:48.109 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0032426199999999999 2023-08-28T02:32:48.109 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.110 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.110 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:32:48.110 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:44.485699+0000", 2023-08-28T02:32:48.110 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0092596469999999993 2023-08-28T02:32:48.110 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.110 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.110 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:32:48.110 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:44.485744+0000", 2023-08-28T02:32:48.110 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.4656000000000001e-05 2023-08-28T02:32:48.110 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:32:48.110 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:32:48.110 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:32:48.110 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:44.485777+0000", 2023-08-28T02:32:48.111 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.3037000000000002e-05 2023-08-28T02:32:48.111 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:48.111 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:48.111 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:48.111 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:32:48.111 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:32:48.111 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:32:48.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:48.112 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:32:48.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:32:48.213 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:32:48.261 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:48.330 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:32:48.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:48.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:32:48.436 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:32:48.487 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:48.552 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:32:48.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:48.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:32:48.658 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:32:48.685 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:48.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:48 smithi143 ceph-mon[183059]: pgmap v719: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 24 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 5.1 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 721 op/s; 52492/251724 objects degraded (20.853%); 3.2 MiB/s, 50 objects/s recovering 2023-08-28T02:32:48.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:48 smithi143 ceph-mon[189871]: pgmap v719: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 24 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 5.1 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 721 op/s; 52492/251724 objects degraded (20.853%); 3.2 MiB/s, 50 objects/s recovering 2023-08-28T02:32:48.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:48 smithi169 ceph-mon[159415]: pgmap v719: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 24 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 5.1 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 721 op/s; 52492/251724 objects degraded (20.853%); 3.2 MiB/s, 50 objects/s recovering 2023-08-28T02:32:49.610 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:32:49.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:49.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:32:49.716 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:32:49.741 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.4 dump_ops_in_flight 2023-08-28T02:32:49.825 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:50.010 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:32:50.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:50.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:32:50.121 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:32:50.166 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:50.193 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:32:50.237 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:32:50.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:50.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:32:50.248 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:32:50.348 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:32:50.438 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:50.540 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:32:50.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:50.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:32:50.647 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:32:50.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:50 smithi143 ceph-mon[189871]: pgmap v720: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 24 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 5.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 552 op/s; 52492/249168 objects degraded (21.067%); 1.4 MiB/s, 23 objects/s recovering 2023-08-28T02:32:50.708 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:50 smithi143 ceph-mon[183059]: pgmap v720: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 24 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 5.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 552 op/s; 52492/249168 objects degraded (21.067%); 1.4 MiB/s, 23 objects/s recovering 2023-08-28T02:32:50.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:50 smithi169 ceph-mon[159415]: pgmap v720: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 24 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 5.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 552 op/s; 52492/249168 objects degraded (21.067%); 1.4 MiB/s, 23 objects/s recovering 2023-08-28T02:32:50.753 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:50.981 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:32:50.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:50.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:32:51.090 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:32:51.169 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:51.274 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:32:51.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:51.282 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:32:51.383 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:32:51.427 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:51.536 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is unset 2023-08-28T02:32:51.717 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:51 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1192070204' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:32:51.717 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:51 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:32:51.717 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:51 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 52492/249168 objects degraded (21.067%), 25 pgs degraded (PG_DEGRADED) 2023-08-28T02:32:51.730 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:32:51.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:51.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:32:51.843 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:32:51.924 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:51 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1192070204' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:32:51.924 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:51 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:32:51.924 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:51 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 52492/249168 objects degraded (21.067%), 25 pgs degraded (PG_DEGRADED) 2023-08-28T02:32:51.924 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:51 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1192070204' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:32:51.925 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:51 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:32:51.925 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:51 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 52492/249168 objects degraded (21.067%), 25 pgs degraded (PG_DEGRADED) 2023-08-28T02:32:51.927 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:52.169 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:32:52.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:52.179 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:32:52.182 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:32:51 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:32:51] "GET /metrics HTTP/1.1" 200 36585 "" "Prometheus/2.33.4" 2023-08-28T02:32:52.280 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:32:52.320 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:32:52.320 INFO:teuthology.orchestra.run.smithi169.stdout: "ops": [], 2023-08-28T02:32:52.320 INFO:teuthology.orchestra.run.smithi169.stdout: "num_ops": 0 2023-08-28T02:32:52.320 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:32:52.346 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:52.559 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:32:52.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:52.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:32:52.624 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 6] out_osds: [7, 1] dead_osds: [7] live_osds: [0, 1, 2, 3, 4, 5, 6] 2023-08-28T02:32:52.624 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-08-28T02:32:52.624 INFO:tasks.thrashosds.thrasher:Adding osd 1 2023-08-28T02:32:52.624 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd in 1 2023-08-28T02:32:52.666 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:32:52.690 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:52 smithi169 ceph-mon[159415]: pgmap v721: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 24 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 5.0 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 676 op/s; 52212/244863 objects degraded (21.323%); 3.1 MiB/s, 50 objects/s recovering 2023-08-28T02:32:52.690 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:52 smithi169 ceph-mon[159415]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:32:52.690 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:52 smithi169 ceph-mon[159415]: failed to encode map e290 with expected crc 2023-08-28T02:32:52.690 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:52 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:32:52.690 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:52 smithi169 ceph-mon[159415]: osdmap e290: 8 total, 7 up, 6 in 2023-08-28T02:32:52.690 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:52 smithi169 ceph-mon[159415]: failed to encode map e290 with expected crc 2023-08-28T02:32:52.692 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:52.734 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:32:52.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:52.737 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:32:52.838 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:32:52.853 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:52 smithi143 ceph-mon[189871]: pgmap v721: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 24 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 5.0 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 676 op/s; 52212/244863 objects degraded (21.323%); 3.1 MiB/s, 50 objects/s recovering 2023-08-28T02:32:52.853 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:52 smithi143 ceph-mon[189871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:32:52.853 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:52 smithi143 ceph-mon[189871]: failed to encode map e290 with expected crc 2023-08-28T02:32:52.853 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:52 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:32:52.853 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:52 smithi143 ceph-mon[189871]: osdmap e290: 8 total, 7 up, 6 in 2023-08-28T02:32:52.853 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:52 smithi143 ceph-mon[189871]: failed to encode map e290 with expected crc 2023-08-28T02:32:52.857 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:52 smithi143 ceph-mon[183059]: pgmap v721: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 24 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 5.0 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 676 op/s; 52212/244863 objects degraded (21.323%); 3.1 MiB/s, 50 objects/s recovering 2023-08-28T02:32:52.857 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:52 smithi143 ceph-mon[183059]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:32:52.857 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:52 smithi143 ceph-mon[183059]: failed to encode map e290 with expected crc 2023-08-28T02:32:52.857 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:52 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:32:52.857 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:52 smithi143 ceph-mon[183059]: osdmap e290: 8 total, 7 up, 6 in 2023-08-28T02:32:52.857 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:52 smithi143 ceph-mon[183059]: failed to encode map e290 with expected crc 2023-08-28T02:32:52.946 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:53.063 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:32:53.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:53.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:32:53.170 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:32:53.206 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.4 dump_blocked_ops 2023-08-28T02:32:53.260 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:53.377 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:32:53.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:53.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:32:53.400 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:32:53.447 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:32:53.481 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:32:53.482 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:32:53.491 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:32:53.528 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:53.541 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:32:53.542 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:53 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/4120583921' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:32:53.542 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:53 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/4120583921' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:32:53.581 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:32:53.617 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:32:53.619 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:32:53.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:53.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:32:53.654 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:32:53.657 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:32:53.657 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:32:53.657 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:32:53.657 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:32:53.657 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:32:53.657 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:32:53.657 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:32:53.658 INFO:teuthology.orchestra.run.smithi143.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-08-28T02:32:53.704 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:32:53.727 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:32:53.768 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:53.864 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:32:53.867 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:53 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/4120583921' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:32:53.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:53.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:32:53.969 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:32:54.056 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:54.887 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set noscrub 2023-08-28T02:32:54.937 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:54 smithi143 ceph-mon[189871]: pgmap v723: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 24 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.9 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 669 op/s; 52212/241194 objects degraded (21.647%); 3.5 MiB/s, 56 objects/s recovering 2023-08-28T02:32:54.937 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:54 smithi143 ceph-mon[183059]: pgmap v723: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 24 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.9 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 669 op/s; 52212/241194 objects degraded (21.647%); 3.5 MiB/s, 56 objects/s recovering 2023-08-28T02:32:54.969 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:32:54.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:54.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:32:54.985 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:54 smithi169 ceph-mon[159415]: pgmap v723: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 24 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.9 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 669 op/s; 52212/241194 objects degraded (21.647%); 3.5 MiB/s, 56 objects/s recovering 2023-08-28T02:32:55.083 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:32:55.161 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:55.476 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:32:55.476 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:32:55.543 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-08-28T02:32:55.544 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-08-28T02:32:55.654 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:32:55.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:55.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:32:55.762 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:32:55.864 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:55.958 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:32:56.051 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:32:56.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:56.064 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:32:56.165 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:32:56.243 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:56.270 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.4 dump_historic_ops 2023-08-28T02:32:56.424 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:32:56.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:56.426 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:32:56.527 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:32:56.574 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:56 smithi169 ceph-mon[159415]: pgmap v724: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 24 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 687 op/s; 51892/234384 objects degraded (22.140%); 3.7 MiB/s, 59 objects/s recovering 2023-08-28T02:32:56.574 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:56 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:32:56.574 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:56 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:32:56.575 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:56 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 51892/234384 objects degraded (22.140%), 25 pgs degraded (PG_DEGRADED) 2023-08-28T02:32:56.578 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:56.650 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:32:56.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:56.658 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:32:56.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:56 smithi143 ceph-mon[189871]: pgmap v724: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 24 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 687 op/s; 51892/234384 objects degraded (22.140%); 3.7 MiB/s, 59 objects/s recovering 2023-08-28T02:32:56.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:56 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:32:56.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:56 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:32:56.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:56 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 51892/234384 objects degraded (22.140%), 25 pgs degraded (PG_DEGRADED) 2023-08-28T02:32:56.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:56 smithi143 ceph-mon[183059]: pgmap v724: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 24 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 687 op/s; 51892/234384 objects degraded (22.140%); 3.7 MiB/s, 59 objects/s recovering 2023-08-28T02:32:56.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:56 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:32:56.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:56 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:32:56.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:56 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 51892/234384 objects degraded (22.140%), 25 pgs degraded (PG_DEGRADED) 2023-08-28T02:32:56.758 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:32:56.793 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:32:56.836 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:56.953 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:32:56.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:56.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:32:57.060 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:32:57.114 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:56 smithi143 conmon[183035]: 2023-08-28T02:32:56.773+0000 7fadc3887700 -1 mon.a@0(leader).osd e290 definitely_dead 0 2023-08-28T02:32:57.119 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:57.221 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:32:57.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:57.227 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:32:57.328 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:32:57.392 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:57.513 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:32:57.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:57.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:32:57.544 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:32:57.616 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:32:57.642 INFO:teuthology.orchestra.run.smithi143.stderr:marked in osd.1. 2023-08-28T02:32:57.735 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:57.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:57 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2621229324' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-08-28T02:32:57.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:57 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-08-28T02:32:57.845 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:32:57.847 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:57 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2621229324' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-08-28T02:32:57.847 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:57 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-08-28T02:32:57.847 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:57 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2621229324' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-08-28T02:32:57.847 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:57 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-08-28T02:32:57.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:57.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:32:57.949 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:32:57.976 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:58.220 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:32:58.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:58.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:32:58.324 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:32:58.352 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:58.444 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:32:58.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:58.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:32:58.547 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:32:58.578 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:58.751 INFO:tasks.thrashosds.thrasher:Added osd 1 2023-08-28T02:32:58.835 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:32:58.836 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:32:58.852 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:32:58.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:58.860 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:32:58.868 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:58 smithi169 ceph-mon[159415]: pgmap v725: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 24 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 694 op/s; 51892/230877 objects degraded (22.476%); 3.7 MiB/s, 59 objects/s recovering 2023-08-28T02:32:58.868 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:58 smithi169 ceph-mon[159415]: failed to encode map e291 with expected crc 2023-08-28T02:32:58.868 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:58 smithi169 ceph-mon[159415]: failed to encode map e291 with expected crc 2023-08-28T02:32:58.868 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:58 smithi169 ceph-mon[159415]: failed to encode map e291 with expected crc 2023-08-28T02:32:58.868 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:58 smithi169 ceph-mon[159415]: failed to encode map e291 with expected crc 2023-08-28T02:32:58.868 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:58 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-08-28T02:32:58.868 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:58 smithi169 ceph-mon[159415]: osdmap e291: 8 total, 7 up, 7 in 2023-08-28T02:32:58.868 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:58 smithi169 ceph-mon[159415]: failed to encode map e291 with expected crc 2023-08-28T02:32:58.868 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:58 smithi169 ceph-mon[159415]: failed to encode map e291 with expected crc 2023-08-28T02:32:58.879 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:58 smithi143 ceph-mon[189871]: pgmap v725: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 24 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 694 op/s; 51892/230877 objects degraded (22.476%); 3.7 MiB/s, 59 objects/s recovering 2023-08-28T02:32:58.880 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:58 smithi143 ceph-mon[189871]: failed to encode map e291 with expected crc 2023-08-28T02:32:58.880 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:58 smithi143 ceph-mon[189871]: failed to encode map e291 with expected crc 2023-08-28T02:32:58.880 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:58 smithi143 ceph-mon[189871]: failed to encode map e291 with expected crc 2023-08-28T02:32:58.880 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:58 smithi143 ceph-mon[189871]: failed to encode map e291 with expected crc 2023-08-28T02:32:58.880 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:58 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-08-28T02:32:58.880 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:58 smithi143 ceph-mon[189871]: osdmap e291: 8 total, 7 up, 7 in 2023-08-28T02:32:58.880 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:58 smithi143 ceph-mon[189871]: failed to encode map e291 with expected crc 2023-08-28T02:32:58.880 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:58 smithi143 ceph-mon[189871]: failed to encode map e291 with expected crc 2023-08-28T02:32:58.881 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:58 smithi143 ceph-mon[183059]: pgmap v725: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 24 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 694 op/s; 51892/230877 objects degraded (22.476%); 3.7 MiB/s, 59 objects/s recovering 2023-08-28T02:32:58.881 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:58 smithi143 ceph-mon[183059]: failed to encode map e291 with expected crc 2023-08-28T02:32:58.882 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:58 smithi143 ceph-mon[183059]: failed to encode map e291 with expected crc 2023-08-28T02:32:58.882 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:58 smithi143 ceph-mon[183059]: failed to encode map e291 with expected crc 2023-08-28T02:32:58.882 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:58 smithi143 ceph-mon[183059]: failed to encode map e291 with expected crc 2023-08-28T02:32:58.882 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:58 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-08-28T02:32:58.882 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:58 smithi143 ceph-mon[183059]: osdmap e291: 8 total, 7 up, 7 in 2023-08-28T02:32:58.882 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:58 smithi143 ceph-mon[183059]: failed to encode map e291 with expected crc 2023-08-28T02:32:58.882 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:58 smithi143 ceph-mon[183059]: failed to encode map e291 with expected crc 2023-08-28T02:32:58.961 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:32:59.113 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:32:59.262 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:59.328 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:32:59.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:59.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:32:59.432 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:32:59.468 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:59.679 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is set 2023-08-28T02:32:59.709 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:32:59.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:32:59.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:32:59.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:59 smithi169 ceph-mon[159415]: failed to encode map e292 with expected crc 2023-08-28T02:32:59.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:59 smithi169 ceph-mon[159415]: failed to encode map e292 with expected crc 2023-08-28T02:32:59.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:59 smithi169 ceph-mon[159415]: failed to encode map e292 with expected crc 2023-08-28T02:32:59.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:59 smithi169 ceph-mon[159415]: osdmap e292: 8 total, 7 up, 7 in 2023-08-28T02:32:59.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:59 smithi169 ceph-mon[159415]: failed to encode map e292 with expected crc 2023-08-28T02:32:59.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:59 smithi169 ceph-mon[159415]: failed to encode map e292 with expected crc 2023-08-28T02:32:59.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:59 smithi169 ceph-mon[159415]: failed to encode map e292 with expected crc 2023-08-28T02:32:59.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:59 smithi169 ceph-mon[159415]: failed to encode map e292 with expected crc 2023-08-28T02:32:59.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:59 smithi169 ceph-mon[159415]: pgmap v728: 99 pgs: 2 peering, 2 active+recovering+undersized+degraded+remapped, 21 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 679 op/s; 45697/228843 objects degraded (19.969%); 2.7 MiB/s, 43 objects/s recovering 2023-08-28T02:32:59.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:59 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3316065170' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:32:59.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:32:59 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:32:59.765 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.5 dump_ops_in_flight 2023-08-28T02:32:59.876 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:32:59.924 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:32:59.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:59 smithi143 ceph-mon[189871]: failed to encode map e292 with expected crc 2023-08-28T02:32:59.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:59 smithi143 ceph-mon[189871]: failed to encode map e292 with expected crc 2023-08-28T02:32:59.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:59 smithi143 ceph-mon[189871]: failed to encode map e292 with expected crc 2023-08-28T02:32:59.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:59 smithi143 ceph-mon[189871]: osdmap e292: 8 total, 7 up, 7 in 2023-08-28T02:32:59.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:59 smithi143 ceph-mon[189871]: failed to encode map e292 with expected crc 2023-08-28T02:32:59.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:59 smithi143 ceph-mon[189871]: failed to encode map e292 with expected crc 2023-08-28T02:32:59.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:59 smithi143 ceph-mon[189871]: failed to encode map e292 with expected crc 2023-08-28T02:32:59.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:59 smithi143 ceph-mon[189871]: failed to encode map e292 with expected crc 2023-08-28T02:32:59.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:59 smithi143 ceph-mon[189871]: pgmap v728: 99 pgs: 2 peering, 2 active+recovering+undersized+degraded+remapped, 21 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 679 op/s; 45697/228843 objects degraded (19.969%); 2.7 MiB/s, 43 objects/s recovering 2023-08-28T02:32:59.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:59 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3316065170' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:32:59.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:32:59 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:32:59.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:59 smithi143 ceph-mon[183059]: failed to encode map e292 with expected crc 2023-08-28T02:32:59.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:59 smithi143 ceph-mon[183059]: failed to encode map e292 with expected crc 2023-08-28T02:32:59.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:59 smithi143 ceph-mon[183059]: failed to encode map e292 with expected crc 2023-08-28T02:32:59.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:59 smithi143 ceph-mon[183059]: osdmap e292: 8 total, 7 up, 7 in 2023-08-28T02:32:59.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:59 smithi143 ceph-mon[183059]: failed to encode map e292 with expected crc 2023-08-28T02:32:59.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:59 smithi143 ceph-mon[183059]: failed to encode map e292 with expected crc 2023-08-28T02:32:59.953 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:59 smithi143 ceph-mon[183059]: failed to encode map e292 with expected crc 2023-08-28T02:32:59.953 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:59 smithi143 ceph-mon[183059]: failed to encode map e292 with expected crc 2023-08-28T02:32:59.953 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:59 smithi143 ceph-mon[183059]: pgmap v728: 99 pgs: 2 peering, 2 active+recovering+undersized+degraded+remapped, 21 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 679 op/s; 45697/228843 objects degraded (19.969%); 2.7 MiB/s, 43 objects/s recovering 2023-08-28T02:32:59.953 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:59 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3316065170' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:32:59.953 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:32:59 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:32:59.995 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:33:00.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:00.003 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:33:00.104 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:33:00.145 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:00.210 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:33:00.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:00.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:33:00.315 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:33:00.362 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:33:00.415 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:00.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:00 smithi143 ceph-mon[183059]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:33:00.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:00 smithi143 ceph-mon[183059]: failed to encode map e293 with expected crc 2023-08-28T02:33:00.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:00 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:33:00.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:00 smithi143 ceph-mon[183059]: osdmap e293: 8 total, 7 up, 7 in 2023-08-28T02:33:00.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:00 smithi143 ceph-mon[183059]: failed to encode map e293 with expected crc 2023-08-28T02:33:00.703 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:00 smithi143 ceph-mon[183059]: failed to encode map e293 with expected crc 2023-08-28T02:33:00.704 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:00 smithi143 ceph-mon[183059]: failed to encode map e293 with expected crc 2023-08-28T02:33:00.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:00 smithi143 ceph-mon[189871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:33:00.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:00 smithi143 ceph-mon[189871]: failed to encode map e293 with expected crc 2023-08-28T02:33:00.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:00 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:33:00.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:00 smithi143 ceph-mon[189871]: osdmap e293: 8 total, 7 up, 7 in 2023-08-28T02:33:00.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:00 smithi143 ceph-mon[189871]: failed to encode map e293 with expected crc 2023-08-28T02:33:00.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:00 smithi143 ceph-mon[189871]: failed to encode map e293 with expected crc 2023-08-28T02:33:00.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:00 smithi143 ceph-mon[189871]: failed to encode map e293 with expected crc 2023-08-28T02:33:00.983 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:00 smithi169 ceph-mon[159415]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:33:00.984 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:00 smithi169 ceph-mon[159415]: failed to encode map e293 with expected crc 2023-08-28T02:33:00.984 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:00 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:33:00.984 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:00 smithi169 ceph-mon[159415]: osdmap e293: 8 total, 7 up, 7 in 2023-08-28T02:33:00.984 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:00 smithi169 ceph-mon[159415]: failed to encode map e293 with expected crc 2023-08-28T02:33:00.984 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:00 smithi169 ceph-mon[159415]: failed to encode map e293 with expected crc 2023-08-28T02:33:00.984 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:00 smithi169 ceph-mon[159415]: failed to encode map e293 with expected crc 2023-08-28T02:33:01.050 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:33:01.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:01.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:33:01.153 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:33:01.182 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:01.224 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:33:01.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:01.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:33:01.327 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:33:01.438 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:01.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:01 smithi143 ceph-mon[183059]: osdmap e294: 8 total, 7 up, 7 in 2023-08-28T02:33:01.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:01 smithi143 ceph-mon[183059]: failed to encode map e294 with expected crc 2023-08-28T02:33:01.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:01 smithi143 ceph-mon[183059]: failed to encode map e294 with expected crc 2023-08-28T02:33:01.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:01 smithi143 ceph-mon[183059]: failed to encode map e294 with expected crc 2023-08-28T02:33:01.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:01 smithi143 ceph-mon[183059]: failed to encode map e294 with expected crc 2023-08-28T02:33:01.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:01 smithi143 ceph-mon[183059]: pgmap v731: 99 pgs: 6 peering, 1 active+recovering+undersized+degraded+remapped, 18 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 443 op/s; 40609/225552 objects degraded (18.004%); 6.7 MiB/s, 117 objects/s recovering 2023-08-28T02:33:01.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:01 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 45697/228843 objects degraded (19.969%), 23 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-08-28T02:33:01.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:01 smithi143 ceph-mon[189871]: osdmap e294: 8 total, 7 up, 7 in 2023-08-28T02:33:01.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:01 smithi143 ceph-mon[189871]: failed to encode map e294 with expected crc 2023-08-28T02:33:01.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:01 smithi143 ceph-mon[189871]: failed to encode map e294 with expected crc 2023-08-28T02:33:01.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:01 smithi143 ceph-mon[189871]: failed to encode map e294 with expected crc 2023-08-28T02:33:01.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:01 smithi143 ceph-mon[189871]: failed to encode map e294 with expected crc 2023-08-28T02:33:01.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:01 smithi143 ceph-mon[189871]: pgmap v731: 99 pgs: 6 peering, 1 active+recovering+undersized+degraded+remapped, 18 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 443 op/s; 40609/225552 objects degraded (18.004%); 6.7 MiB/s, 117 objects/s recovering 2023-08-28T02:33:01.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:01 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 45697/228843 objects degraded (19.969%), 23 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-08-28T02:33:01.925 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:33:01.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:01.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:33:01.945 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:01 smithi169 ceph-mon[159415]: osdmap e294: 8 total, 7 up, 7 in 2023-08-28T02:33:01.946 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:01 smithi169 ceph-mon[159415]: failed to encode map e294 with expected crc 2023-08-28T02:33:01.946 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:01 smithi169 ceph-mon[159415]: failed to encode map e294 with expected crc 2023-08-28T02:33:01.946 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:01 smithi169 ceph-mon[159415]: failed to encode map e294 with expected crc 2023-08-28T02:33:01.946 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:01 smithi169 ceph-mon[159415]: failed to encode map e294 with expected crc 2023-08-28T02:33:01.946 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:01 smithi169 ceph-mon[159415]: pgmap v731: 99 pgs: 6 peering, 1 active+recovering+undersized+degraded+remapped, 18 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 443 op/s; 40609/225552 objects degraded (18.004%); 6.7 MiB/s, 117 objects/s recovering 2023-08-28T02:33:01.946 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:01 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 45697/228843 objects degraded (19.969%), 23 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-08-28T02:33:02.029 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:33:02.074 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:02.136 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:33:02.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:02.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:33:02.198 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:33:02.199 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:33:02.205 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:33:01 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:33:01] "GET /metrics HTTP/1.1" 200 36583 "" "Prometheus/2.33.4" 2023-08-28T02:33:02.241 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:33:02.345 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:02.506 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:33:02.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:02.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:33:02.610 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:33:02.639 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:02.829 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:33:02.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:02.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:33:02.891 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:02 smithi143 ceph-mon[183059]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-08-28T02:33:02.892 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:02 smithi143 ceph-mon[189871]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-08-28T02:33:02.938 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:33:02.969 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:02 smithi169 ceph-mon[159415]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-08-28T02:33:02.974 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:03.016 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.5 dump_blocked_ops 2023-08-28T02:33:03.029 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:33:03.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:03.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:33:03.139 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:33:03.181 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:03.202 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:33:03.238 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:33:03.241 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:33:03.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:03.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:33:03.284 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:33:03.331 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:33:03.348 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:33:03.361 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:33:03.390 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:03.412 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:33:03.446 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:33:03.448 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:33:03.448 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:33:03.448 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:33:03.448 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:33:03.449 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:33:03.449 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:33:03.449 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:33:03.449 INFO:teuthology.orchestra.run.smithi143.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-08-28T02:33:03.455 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:33:03.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:03.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:33:03.500 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:33:03.562 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:33:03.664 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:03.709 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:03 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1013447546' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:33:03.709 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:03 smithi169 ceph-mon[159415]: pgmap v732: 99 pgs: 6 peering, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 454 op/s; 32893/222972 objects degraded (14.752%); 4.6 MiB/s, 83 objects/s recovering 2023-08-28T02:33:03.752 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 6, 1] out_osds: [7] dead_osds: [7] live_osds: [0, 1, 2, 3, 4, 5, 6] 2023-08-28T02:33:03.752 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-08-28T02:33:03.753 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-08-28T02:33:03.753 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph pg dump --format=json 2023-08-28T02:33:03.794 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:33:03.811 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:03 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1013447546' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:33:03.811 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:03 smithi143 ceph-mon[189871]: pgmap v732: 99 pgs: 6 peering, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 454 op/s; 32893/222972 objects degraded (14.752%); 4.6 MiB/s, 83 objects/s recovering 2023-08-28T02:33:03.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:03.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:33:03.813 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:03 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1013447546' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:33:03.813 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:03 smithi143 ceph-mon[183059]: pgmap v732: 99 pgs: 6 peering, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 454 op/s; 32893/222972 objects degraded (14.752%); 4.6 MiB/s, 83 objects/s recovering 2023-08-28T02:33:03.913 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:33:03.983 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:04.096 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:33:04.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:04.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:33:04.212 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:33:04.238 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:04.282 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:33:04.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:04.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:33:04.385 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:33:04.453 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:04.563 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:33:04.564 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set nodeep-scrub 2023-08-28T02:33:04.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:04.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:33:04.672 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:33:04.802 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:05.163 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:33:05.171 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-08-28T02:33:05.172 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-08-28T02:33:05.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:05.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:33:05.249 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:33:05.249 INFO:teuthology.orchestra.run.smithi169.stdout: "ops": [], 2023-08-28T02:33:05.249 INFO:teuthology.orchestra.run.smithi169.stdout: "complaint_time": 30, 2023-08-28T02:33:05.249 INFO:teuthology.orchestra.run.smithi169.stdout: "num_blocked_ops": 0 2023-08-28T02:33:05.249 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:33:05.274 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:33:05.314 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:05.471 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:33:05.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:05.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:33:05.574 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:33:05.644 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:05.696 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:33:05.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:05 smithi169 ceph-mon[159415]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-08-28T02:33:05.741 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:33:05.836 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:33:05.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:05.852 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:33:05.852 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:05 smithi143 ceph-mon[183059]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-08-28T02:33:05.852 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:05 smithi143 ceph-mon[189871]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-08-28T02:33:05.953 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:33:06.045 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:06.071 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.5 dump_historic_ops 2023-08-28T02:33:06.302 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:33:06.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:06.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:33:06.418 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:33:06.505 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:33:06.615 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:06.778 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:06 smithi169 ceph-mon[159415]: pgmap v733: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 730 op/s; 32739/216006 objects degraded (15.157%); 5.6 MiB/s, 98 objects/s recovering 2023-08-28T02:33:06.779 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:06 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 32739/216006 objects degraded (15.157%), 22 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-08-28T02:33:06.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:06 smithi143 ceph-mon[183059]: pgmap v733: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 730 op/s; 32739/216006 objects degraded (15.157%); 5.6 MiB/s, 98 objects/s recovering 2023-08-28T02:33:06.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:06 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 32739/216006 objects degraded (15.157%), 22 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-08-28T02:33:06.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:06 smithi143 ceph-mon[189871]: pgmap v733: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 730 op/s; 32739/216006 objects degraded (15.157%); 5.6 MiB/s, 98 objects/s recovering 2023-08-28T02:33:06.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:06 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 32739/216006 objects degraded (15.157%), 22 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-08-28T02:33:07.173 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:33:07.736 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:33:07.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:07.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:33:07.846 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:33:07.945 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:08.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:08.140 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:33:08.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:33:08.241 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:33:08.271 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:33:08.271 INFO:teuthology.orchestra.run.smithi169.stdout: "size": 20, 2023-08-28T02:33:08.271 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 600, 2023-08-28T02:33:08.271 INFO:teuthology.orchestra.run.smithi169.stdout: "ops": [ 2023-08-28T02:33:08.271 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.272 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:24529 6.a 6:5d60f11f:::benchmark_data_smithi143_305742_object24528:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e195)", 2023-08-28T02:33:08.272 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:27:16.316338+0000", 2023-08-28T02:33:08.272 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 351.95297846400001, 2023-08-28T02:33:08.272 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.1223833000000001, 2023-08-28T02:33:08.272 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:33:08.272 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "reached pg", 2023-08-28T02:33:08.272 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:33:08.272 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:33:08.272 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:33:08.272 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 24529 2023-08-28T02:33:08.272 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.272 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:33:08.273 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.273 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:33:08.273 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.316338+0000", 2023-08-28T02:33:08.273 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:33:08.273 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.273 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.273 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:33:08.273 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.316338+0000", 2023-08-28T02:33:08.273 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999995 2023-08-28T02:33:08.273 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.273 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.273 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:33:08.274 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.316338+0000", 2023-08-28T02:33:08.274 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.0701000000000003e-05 2023-08-28T02:33:08.274 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.274 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.274 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:33:08.274 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.316368+0000", 2023-08-28T02:33:08.274 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.1279999999999996e-06 2023-08-28T02:33:08.274 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.274 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.274 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:33:08.274 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.316374+0000", 2023-08-28T02:33:08.275 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.1609999999999996e-06 2023-08-28T02:33:08.275 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.275 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.275 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:33:08.275 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.316379+0000", 2023-08-28T02:33:08.275 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.7463000000000002e-05 2023-08-28T02:33:08.275 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.275 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.275 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:33:08.275 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.316436+0000", 2023-08-28T02:33:08.275 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000108937 2023-08-28T02:33:08.275 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.276 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.276 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:33:08.276 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.316545+0000", 2023-08-28T02:33:08.276 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00015960400000000001 2023-08-28T02:33:08.276 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.276 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.276 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,6", 2023-08-28T02:33:08.276 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.316705+0000", 2023-08-28T02:33:08.276 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.002000862 2023-08-28T02:33:08.276 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.276 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.277 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:33:08.277 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.318705+0000", 2023-08-28T02:33:08.277 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00033078399999999998 2023-08-28T02:33:08.277 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.277 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.277 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:33:08.277 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.319036+0000", 2023-08-28T02:33:08.277 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0036924560000000002 2023-08-28T02:33:08.277 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.277 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.278 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:33:08.278 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.322729+0000", 2023-08-28T02:33:08.278 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0696999999999999e-05 2023-08-28T02:33:08.278 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.278 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.278 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:33:08.278 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.322739+0000", 2023-08-28T02:33:08.278 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.997422806 2023-08-28T02:33:08.278 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.278 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.278 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:33:08.279 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:17.320162+0000", 2023-08-28T02:33:08.279 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1532000000000001e-05 2023-08-28T02:33:08.279 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.279 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.279 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:33:08.279 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:17.320174+0000", 2023-08-28T02:33:08.279 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1184618099999999 2023-08-28T02:33:08.279 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.279 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.279 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:33:08.280 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.438635+0000", 2023-08-28T02:33:08.280 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.6031000000000004e-05 2023-08-28T02:33:08.280 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.280 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.280 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:33:08.280 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.438722+0000", 2023-08-28T02:33:08.280 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.1223833000000001 2023-08-28T02:33:08.280 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:33:08.280 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:33:08.280 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:33:08.281 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.281 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.281 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:24537 6.4 6:28e59a94:::benchmark_data_smithi143_305742_object24536:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e195)", 2023-08-28T02:33:08.281 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:27:16.321295+0000", 2023-08-28T02:33:08.281 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 351.94802125400003, 2023-08-28T02:33:08.281 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.0777337359999999, 2023-08-28T02:33:08.281 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:33:08.281 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:33:08.281 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:33:08.281 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:33:08.282 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:33:08.282 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 24537 2023-08-28T02:33:08.282 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.282 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:33:08.282 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.282 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:33:08.282 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.321295+0000", 2023-08-28T02:33:08.282 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:33:08.282 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.282 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.283 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:33:08.283 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.321295+0000", 2023-08-28T02:33:08.283 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999995 2023-08-28T02:33:08.283 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.283 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.283 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:33:08.283 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.321295+0000", 2023-08-28T02:33:08.283 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.9675999999999998e-05 2023-08-28T02:33:08.283 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.283 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.283 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:33:08.284 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.321325+0000", 2023-08-28T02:33:08.284 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.5349999999999999e-06 2023-08-28T02:33:08.284 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.284 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.284 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:33:08.284 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.321332+0000", 2023-08-28T02:33:08.284 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.3789999999999999e-06 2023-08-28T02:33:08.284 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.284 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.284 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:33:08.285 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.321336+0000", 2023-08-28T02:33:08.285 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0038322999999999999 2023-08-28T02:33:08.285 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.285 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.285 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:33:08.285 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.325169+0000", 2023-08-28T02:33:08.285 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.5019999999999998e-06 2023-08-28T02:33:08.285 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.285 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.286 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:33:08.286 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.325178+0000", 2023-08-28T02:33:08.286 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.99164512999999999 2023-08-28T02:33:08.286 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.286 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.286 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:33:08.286 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:17.316823+0000", 2023-08-28T02:33:08.286 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2853999999999999e-05 2023-08-28T02:33:08.286 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.287 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.287 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:33:08.287 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:17.316836+0000", 2023-08-28T02:33:08.287 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0684663839999999 2023-08-28T02:33:08.287 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.287 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.287 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:33:08.287 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.385303+0000", 2023-08-28T02:33:08.287 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00014184199999999999 2023-08-28T02:33:08.287 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.288 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.288 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:33:08.288 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.385444+0000", 2023-08-28T02:33:08.288 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00014312299999999999 2023-08-28T02:33:08.288 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.288 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.288 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 3,4", 2023-08-28T02:33:08.288 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.385588+0000", 2023-08-28T02:33:08.288 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0019724999999999999 2023-08-28T02:33:08.288 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.289 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.289 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:33:08.289 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.387560+0000", 2023-08-28T02:33:08.289 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0033467089999999998 2023-08-28T02:33:08.289 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.289 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.289 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:33:08.289 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.390907+0000", 2023-08-28T02:33:08.289 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0067727239999999999 2023-08-28T02:33:08.290 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.290 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.290 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:33:08.290 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.397680+0000", 2023-08-28T02:33:08.290 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.6778000000000002e-05 2023-08-28T02:33:08.290 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.290 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.290 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:33:08.290 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.397766+0000", 2023-08-28T02:33:08.290 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.001262846 2023-08-28T02:33:08.290 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.291 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.291 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:33:08.291 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.399029+0000", 2023-08-28T02:33:08.291 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.0777337359999999 2023-08-28T02:33:08.291 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:33:08.291 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:33:08.291 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:33:08.291 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.291 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.291 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:24538 6.4 6:2aa5ab67:::benchmark_data_smithi143_305742_object24537:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e195)", 2023-08-28T02:33:08.291 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:27:16.332135+0000", 2023-08-28T02:33:08.291 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 351.93718123500003, 2023-08-28T02:33:08.291 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.06786067, 2023-08-28T02:33:08.292 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:33:08.292 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:33:08.292 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:33:08.292 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:33:08.292 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:33:08.292 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 24538 2023-08-28T02:33:08.292 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.292 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:33:08.292 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.292 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:33:08.292 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.332135+0000", 2023-08-28T02:33:08.292 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:33:08.292 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.292 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.293 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:33:08.293 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.332135+0000", 2023-08-28T02:33:08.293 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:33:08.293 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.293 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.293 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:33:08.293 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.332134+0000", 2023-08-28T02:33:08.293 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.9920999999999999e-05 2023-08-28T02:33:08.293 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.293 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.293 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:33:08.293 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.332184+0000", 2023-08-28T02:33:08.294 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.4516999999999999e-05 2023-08-28T02:33:08.294 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.294 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.294 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:33:08.294 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.332208+0000", 2023-08-28T02:33:08.294 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.9049999999999997e-06 2023-08-28T02:33:08.294 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.294 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.294 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:33:08.294 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.332216+0000", 2023-08-28T02:33:08.294 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0016499900000000001 2023-08-28T02:33:08.294 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.294 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.294 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:33:08.295 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.333866+0000", 2023-08-28T02:33:08.295 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.4530000000000008e-06 2023-08-28T02:33:08.295 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.295 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.295 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:33:08.295 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.333876+0000", 2023-08-28T02:33:08.295 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.98298540099999998 2023-08-28T02:33:08.295 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.295 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.295 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:33:08.295 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:17.316861+0000", 2023-08-28T02:33:08.295 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.3659999999999992e-06 2023-08-28T02:33:08.295 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.296 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.296 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:33:08.296 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:17.316869+0000", 2023-08-28T02:33:08.296 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0707238489999999 2023-08-28T02:33:08.296 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.296 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.296 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:33:08.296 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.387593+0000", 2023-08-28T02:33:08.296 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00013506600000000001 2023-08-28T02:33:08.296 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.296 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.296 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:33:08.297 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.387728+0000", 2023-08-28T02:33:08.297 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000163043 2023-08-28T02:33:08.297 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.297 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.297 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 3,4", 2023-08-28T02:33:08.297 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.387891+0000", 2023-08-28T02:33:08.297 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.001260326 2023-08-28T02:33:08.297 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.297 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.297 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:33:08.297 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.389152+0000", 2023-08-28T02:33:08.297 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0028075629999999999 2023-08-28T02:33:08.297 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.298 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.298 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:33:08.298 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.391959+0000", 2023-08-28T02:33:08.298 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0071406500000000001 2023-08-28T02:33:08.298 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.298 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.298 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:33:08.298 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.399100+0000", 2023-08-28T02:33:08.298 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.2137000000000004e-05 2023-08-28T02:33:08.298 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.298 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.298 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:33:08.299 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.399172+0000", 2023-08-28T02:33:08.299 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00082417600000000003 2023-08-28T02:33:08.299 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.299 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.299 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:33:08.299 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.399996+0000", 2023-08-28T02:33:08.299 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.06786067 2023-08-28T02:33:08.299 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:33:08.299 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:33:08.299 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:33:08.299 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.299 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.299 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:24542 6.a 6:57630bd8:::benchmark_data_smithi143_305742_object24541:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e196)", 2023-08-28T02:33:08.300 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:27:16.332962+0000", 2023-08-28T02:33:08.300 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 351.93635459199999, 2023-08-28T02:33:08.300 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.1171907980000002, 2023-08-28T02:33:08.300 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:33:08.300 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:33:08.300 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:33:08.300 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:33:08.300 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:33:08.300 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 24542 2023-08-28T02:33:08.300 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.300 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:33:08.300 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.301 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:33:08.301 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.332962+0000", 2023-08-28T02:33:08.301 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:33:08.301 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.301 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.301 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:33:08.301 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.332962+0000", 2023-08-28T02:33:08.301 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:33:08.301 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.301 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.301 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:33:08.301 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.332961+0000", 2023-08-28T02:33:08.302 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00094618000000000002 2023-08-28T02:33:08.302 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.302 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.302 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:33:08.302 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.333907+0000", 2023-08-28T02:33:08.302 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.7800000000000006e-06 2023-08-28T02:33:08.302 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.302 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.302 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:33:08.302 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.333916+0000", 2023-08-28T02:33:08.302 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.9120000000000003e-06 2023-08-28T02:33:08.302 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.303 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.303 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:33:08.303 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.333922+0000", 2023-08-28T02:33:08.303 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00015135300000000001 2023-08-28T02:33:08.303 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.303 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.303 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:33:08.303 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.334073+0000", 2023-08-28T02:33:08.303 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.8440000000000004e-06 2023-08-28T02:33:08.303 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.303 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.303 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:33:08.303 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:16.334082+0000", 2023-08-28T02:33:08.303 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.98611508999999997 2023-08-28T02:33:08.304 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.304 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.304 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:33:08.304 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:17.320197+0000", 2023-08-28T02:33:08.304 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.3210000000000005e-06 2023-08-28T02:33:08.304 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.304 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.304 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:33:08.304 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:17.320206+0000", 2023-08-28T02:33:08.304 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.119032764 2023-08-28T02:33:08.304 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.305 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.305 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:33:08.305 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.439238+0000", 2023-08-28T02:33:08.305 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00014970000000000001 2023-08-28T02:33:08.305 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.305 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.305 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:33:08.305 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.439388+0000", 2023-08-28T02:33:08.305 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00021345199999999999 2023-08-28T02:33:08.305 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.305 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.305 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,2", 2023-08-28T02:33:08.306 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.439602+0000", 2023-08-28T02:33:08.306 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0019913299999999999 2023-08-28T02:33:08.306 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.306 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.306 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:33:08.306 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.441593+0000", 2023-08-28T02:33:08.306 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0018547699999999999 2023-08-28T02:33:08.306 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.306 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.306 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:33:08.306 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.443448+0000", 2023-08-28T02:33:08.306 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0050988140000000001 2023-08-28T02:33:08.306 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.307 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.307 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:33:08.307 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.448547+0000", 2023-08-28T02:33:08.307 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.4356000000000003e-05 2023-08-28T02:33:08.307 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.307 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.307 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:33:08.307 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.448641+0000", 2023-08-28T02:33:08.307 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.001511992 2023-08-28T02:33:08.307 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.307 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.307 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:33:08.308 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:27:18.450153+0000", 2023-08-28T02:33:08.308 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.1171907980000002 2023-08-28T02:33:08.308 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:33:08.308 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:33:08.308 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:33:08.308 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.308 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.308 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:65345 6.a 6:5cafef95:::benchmark_data_smithi143_305742_object65344:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-08-28T02:33:08.308 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:28:07.393170+0000", 2023-08-28T02:33:08.308 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 300.87614652299999, 2023-08-28T02:33:08.308 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.97206194499999998, 2023-08-28T02:33:08.308 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:33:08.309 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:33:08.309 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:33:08.309 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:33:08.309 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:33:08.309 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 65345 2023-08-28T02:33:08.309 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.309 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:33:08.309 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.309 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:33:08.309 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.393170+0000", 2023-08-28T02:33:08.309 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:33:08.309 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.309 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.310 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:33:08.310 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.393170+0000", 2023-08-28T02:33:08.310 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999976 2023-08-28T02:33:08.310 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.310 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.310 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:33:08.310 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.393168+0000", 2023-08-28T02:33:08.310 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.392e-05 2023-08-28T02:33:08.310 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.310 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.310 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:33:08.311 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.393212+0000", 2023-08-28T02:33:08.311 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1738e-05 2023-08-28T02:33:08.311 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.311 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.311 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:33:08.311 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.393224+0000", 2023-08-28T02:33:08.311 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.5649999999999998e-06 2023-08-28T02:33:08.311 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.311 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.311 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:33:08.311 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.393230+0000", 2023-08-28T02:33:08.312 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.4586000000000002e-05 2023-08-28T02:33:08.312 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.312 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.312 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:33:08.312 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.393285+0000", 2023-08-28T02:33:08.312 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.8400000000000007e-06 2023-08-28T02:33:08.312 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.312 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.312 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:33:08.312 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.393295+0000", 2023-08-28T02:33:08.313 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.96419019500000003 2023-08-28T02:33:08.313 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.313 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.313 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:33:08.313 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.357485+0000", 2023-08-28T02:33:08.313 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00013377699999999999 2023-08-28T02:33:08.313 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.313 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.313 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:33:08.313 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.357619+0000", 2023-08-28T02:33:08.313 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012924900000000001 2023-08-28T02:33:08.314 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.314 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.314 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,2", 2023-08-28T02:33:08.314 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.357748+0000", 2023-08-28T02:33:08.314 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.006065857 2023-08-28T02:33:08.314 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.314 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.314 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:33:08.314 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.363814+0000", 2023-08-28T02:33:08.315 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0012912659999999999 2023-08-28T02:33:08.315 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.315 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.315 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:33:08.315 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.365105+0000", 2023-08-28T02:33:08.315 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.3149000000000003e-05 2023-08-28T02:33:08.315 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.315 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.315 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:33:08.315 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.365148+0000", 2023-08-28T02:33:08.315 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.8458999999999997e-05 2023-08-28T02:33:08.316 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.316 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.316 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:33:08.316 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.365207+0000", 2023-08-28T02:33:08.316 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.5528e-05 2023-08-28T02:33:08.316 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.316 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.316 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:33:08.316 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.365232+0000", 2023-08-28T02:33:08.316 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.97206194499999998 2023-08-28T02:33:08.316 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:33:08.317 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:33:08.317 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:33:08.317 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.317 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.317 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:65370 6.a 6:5b0efaa9:::benchmark_data_smithi143_305742_object65369:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-08-28T02:33:08.317 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:28:07.421258+0000", 2023-08-28T02:33:08.317 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 300.84805855100001, 2023-08-28T02:33:08.317 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.95615907099999997, 2023-08-28T02:33:08.317 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:33:08.318 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:33:08.318 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:33:08.318 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:33:08.318 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:33:08.318 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 65370 2023-08-28T02:33:08.318 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.318 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:33:08.318 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.318 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:33:08.318 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.421258+0000", 2023-08-28T02:33:08.318 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:33:08.319 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.319 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.319 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:33:08.319 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.421258+0000", 2023-08-28T02:33:08.319 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:33:08.319 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.319 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.319 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:33:08.319 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.421257+0000", 2023-08-28T02:33:08.319 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.1618000000000004e-05 2023-08-28T02:33:08.320 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.320 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.320 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:33:08.320 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.421338+0000", 2023-08-28T02:33:08.320 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.7480000000000005e-06 2023-08-28T02:33:08.320 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.320 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.320 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:33:08.320 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.421348+0000", 2023-08-28T02:33:08.320 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.7060000000000003e-06 2023-08-28T02:33:08.320 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.321 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.321 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:33:08.321 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.421353+0000", 2023-08-28T02:33:08.321 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.9262000000000003e-05 2023-08-28T02:33:08.321 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.321 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.321 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:33:08.321 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.421412+0000", 2023-08-28T02:33:08.321 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0037000000000001e-05 2023-08-28T02:33:08.322 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.322 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.322 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:33:08.322 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.421422+0000", 2023-08-28T02:33:08.322 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.93841324500000001 2023-08-28T02:33:08.322 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.322 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.322 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:33:08.322 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.359835+0000", 2023-08-28T02:33:08.322 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00010686199999999999 2023-08-28T02:33:08.323 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.323 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.323 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:33:08.323 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.359942+0000", 2023-08-28T02:33:08.323 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000114066 2023-08-28T02:33:08.323 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.323 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.323 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,2", 2023-08-28T02:33:08.323 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.360056+0000", 2023-08-28T02:33:08.323 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0051866639999999997 2023-08-28T02:33:08.324 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.324 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.324 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:33:08.324 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.365243+0000", 2023-08-28T02:33:08.324 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0065147149999999999 2023-08-28T02:33:08.324 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.324 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.324 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:33:08.324 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.371758+0000", 2023-08-28T02:33:08.324 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0055873479999999998 2023-08-28T02:33:08.324 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.325 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.325 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:33:08.325 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377345+0000", 2023-08-28T02:33:08.325 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.3902000000000001e-05 2023-08-28T02:33:08.325 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.325 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.325 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:33:08.325 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377389+0000", 2023-08-28T02:33:08.325 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.8276000000000002e-05 2023-08-28T02:33:08.325 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.325 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.325 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:33:08.325 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377417+0000", 2023-08-28T02:33:08.326 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.95615907099999997 2023-08-28T02:33:08.326 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:33:08.326 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:33:08.326 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:33:08.326 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.326 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.326 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:65382 6.a 6:591f19d2:::benchmark_data_smithi143_305742_object65381:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-08-28T02:33:08.326 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:28:07.430975+0000", 2023-08-28T02:33:08.326 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 300.83834138999998, 2023-08-28T02:33:08.326 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.94658987900000002, 2023-08-28T02:33:08.326 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:33:08.326 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:33:08.327 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:33:08.327 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:33:08.327 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:33:08.327 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 65382 2023-08-28T02:33:08.327 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.327 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:33:08.327 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.327 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:33:08.327 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.430975+0000", 2023-08-28T02:33:08.327 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:33:08.327 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.327 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.328 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:33:08.328 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.430975+0000", 2023-08-28T02:33:08.328 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:33:08.328 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.328 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.328 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:33:08.328 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.430974+0000", 2023-08-28T02:33:08.328 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.9980000000000004e-05 2023-08-28T02:33:08.328 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.328 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.328 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:33:08.328 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.431044+0000", 2023-08-28T02:33:08.328 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.115e-06 2023-08-28T02:33:08.329 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.329 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.329 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:33:08.329 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.431053+0000", 2023-08-28T02:33:08.329 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.6990000000000002e-06 2023-08-28T02:33:08.329 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.329 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.329 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:33:08.329 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.431058+0000", 2023-08-28T02:33:08.329 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.4104999999999994e-05 2023-08-28T02:33:08.329 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.330 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.330 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:33:08.330 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.431152+0000", 2023-08-28T02:33:08.330 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.5532999999999998e-05 2023-08-28T02:33:08.330 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.330 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.330 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:33:08.330 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.431168+0000", 2023-08-28T02:33:08.330 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.92994798499999998 2023-08-28T02:33:08.330 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.330 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.330 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:33:08.331 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.361116+0000", 2023-08-28T02:33:08.331 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000128291 2023-08-28T02:33:08.331 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.331 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.331 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:33:08.331 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.361244+0000", 2023-08-28T02:33:08.331 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00016667299999999999 2023-08-28T02:33:08.331 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.331 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.331 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,2", 2023-08-28T02:33:08.331 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.361411+0000", 2023-08-28T02:33:08.331 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0038359240000000001 2023-08-28T02:33:08.331 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.332 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.332 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:33:08.332 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.365247+0000", 2023-08-28T02:33:08.332 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0093736270000000007 2023-08-28T02:33:08.332 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.332 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.332 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:33:08.332 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.374620+0000", 2023-08-28T02:33:08.332 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0028494010000000001 2023-08-28T02:33:08.332 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.332 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.332 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:33:08.333 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377470+0000", 2023-08-28T02:33:08.333 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.486e-05 2023-08-28T02:33:08.333 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.333 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.333 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:33:08.333 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377525+0000", 2023-08-28T02:33:08.333 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.0658999999999998e-05 2023-08-28T02:33:08.333 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.333 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.333 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:33:08.333 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377565+0000", 2023-08-28T02:33:08.333 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.94658987900000002 2023-08-28T02:33:08.334 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:33:08.334 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:33:08.334 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:33:08.334 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.334 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.334 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:65411 6.a 6:59a52dc1:::benchmark_data_smithi143_305742_object65410:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-08-28T02:33:08.334 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:28:07.493832+0000", 2023-08-28T02:33:08.334 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 300.77548476099997, 2023-08-28T02:33:08.334 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.88382154300000004, 2023-08-28T02:33:08.334 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:33:08.334 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:33:08.334 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:33:08.334 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:33:08.335 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:33:08.335 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 65411 2023-08-28T02:33:08.335 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.335 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:33:08.335 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.335 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:33:08.335 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493832+0000", 2023-08-28T02:33:08.335 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:33:08.335 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.335 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.335 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:33:08.335 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493832+0000", 2023-08-28T02:33:08.335 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999995 2023-08-28T02:33:08.336 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.336 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.336 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:33:08.336 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493831+0000", 2023-08-28T02:33:08.336 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.7842999999999997e-05 2023-08-28T02:33:08.336 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.336 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.336 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:33:08.336 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493869+0000", 2023-08-28T02:33:08.336 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.6300000000000003e-06 2023-08-28T02:33:08.336 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.336 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.337 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:33:08.337 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493875+0000", 2023-08-28T02:33:08.337 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.6119999999999999e-06 2023-08-28T02:33:08.337 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.337 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.337 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:33:08.337 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493879+0000", 2023-08-28T02:33:08.337 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.4725999999999997e-05 2023-08-28T02:33:08.337 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.337 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.337 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:33:08.337 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493933+0000", 2023-08-28T02:33:08.338 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.5399999999999996e-06 2023-08-28T02:33:08.338 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.338 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.338 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:33:08.338 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493942+0000", 2023-08-28T02:33:08.338 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.868725466 2023-08-28T02:33:08.338 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.338 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.338 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:33:08.338 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.362667+0000", 2023-08-28T02:33:08.338 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012190500000000001 2023-08-28T02:33:08.338 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.339 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.339 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:33:08.339 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.362789+0000", 2023-08-28T02:33:08.339 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000101784 2023-08-28T02:33:08.339 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.339 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.339 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,2", 2023-08-28T02:33:08.339 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.362891+0000", 2023-08-28T02:33:08.339 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0023590709999999999 2023-08-28T02:33:08.339 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.339 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.339 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:33:08.339 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.365250+0000", 2023-08-28T02:33:08.340 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0094366910000000005 2023-08-28T02:33:08.340 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.340 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.340 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:33:08.340 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.374687+0000", 2023-08-28T02:33:08.340 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0029174880000000002 2023-08-28T02:33:08.340 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.340 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.340 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:33:08.340 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377604+0000", 2023-08-28T02:33:08.340 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.8928000000000002e-05 2023-08-28T02:33:08.340 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.341 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.341 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:33:08.341 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377633+0000", 2023-08-28T02:33:08.341 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.0352000000000001e-05 2023-08-28T02:33:08.341 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.341 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.341 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:33:08.341 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377653+0000", 2023-08-28T02:33:08.341 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.88382154300000004 2023-08-28T02:33:08.341 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:33:08.341 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:33:08.341 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:33:08.341 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.342 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.342 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:65413 6.a 6:51617f2c:::benchmark_data_smithi143_305742_object65412:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-08-28T02:33:08.342 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:28:07.493890+0000", 2023-08-28T02:33:08.342 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 300.77542669500002, 2023-08-28T02:33:08.342 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.88389938400000001, 2023-08-28T02:33:08.342 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:33:08.342 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:33:08.342 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:33:08.342 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:33:08.342 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:33:08.342 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 65413 2023-08-28T02:33:08.343 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.343 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:33:08.343 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.343 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:33:08.343 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493890+0000", 2023-08-28T02:33:08.343 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:33:08.343 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.343 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.343 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:33:08.343 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493890+0000", 2023-08-28T02:33:08.343 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999995 2023-08-28T02:33:08.343 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.343 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.344 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:33:08.344 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493889+0000", 2023-08-28T02:33:08.344 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.5259999999999998e-05 2023-08-28T02:33:08.344 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.344 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.344 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:33:08.344 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493925+0000", 2023-08-28T02:33:08.344 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.6899999999999997e-06 2023-08-28T02:33:08.344 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.344 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.344 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:33:08.344 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493930+0000", 2023-08-28T02:33:08.345 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.9160000000000003e-06 2023-08-28T02:33:08.345 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.345 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.345 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:33:08.345 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493934+0000", 2023-08-28T02:33:08.345 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.5824e-05 2023-08-28T02:33:08.345 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.345 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.345 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:33:08.345 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493980+0000", 2023-08-28T02:33:08.345 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0889e-05 2023-08-28T02:33:08.345 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.346 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.346 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:33:08.346 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493991+0000", 2023-08-28T02:33:08.346 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.86984277300000001 2023-08-28T02:33:08.346 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.346 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.346 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:33:08.346 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.363834+0000", 2023-08-28T02:33:08.346 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.9104999999999996e-05 2023-08-28T02:33:08.346 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.346 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.346 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:33:08.347 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.363913+0000", 2023-08-28T02:33:08.347 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.9914999999999995e-05 2023-08-28T02:33:08.347 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.347 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.347 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,2", 2023-08-28T02:33:08.347 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.364013+0000", 2023-08-28T02:33:08.347 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.003811933 2023-08-28T02:33:08.347 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.347 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.347 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:33:08.348 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.367825+0000", 2023-08-28T02:33:08.348 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0069041399999999996 2023-08-28T02:33:08.348 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.348 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.348 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:33:08.348 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.374729+0000", 2023-08-28T02:33:08.348 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0029736039999999999 2023-08-28T02:33:08.348 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.348 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.348 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:33:08.348 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377703+0000", 2023-08-28T02:33:08.348 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.8238999999999997e-05 2023-08-28T02:33:08.349 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.349 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.349 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:33:08.349 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377751+0000", 2023-08-28T02:33:08.349 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.8593000000000001e-05 2023-08-28T02:33:08.349 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.349 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.349 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:33:08.349 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377789+0000", 2023-08-28T02:33:08.350 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.88389938400000001 2023-08-28T02:33:08.350 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:33:08.350 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:33:08.350 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:33:08.350 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.350 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.350 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:65445 6.a 6:554d4ed8:::benchmark_data_smithi143_305742_object65444:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-08-28T02:33:08.350 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:28:07.533277+0000", 2023-08-28T02:33:08.350 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 300.73603962499999, 2023-08-28T02:33:08.351 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.84459700800000004, 2023-08-28T02:33:08.351 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:33:08.351 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:33:08.351 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:33:08.351 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:33:08.351 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:33:08.351 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 65445 2023-08-28T02:33:08.351 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.351 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:33:08.351 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.351 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:33:08.352 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533277+0000", 2023-08-28T02:33:08.352 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:33:08.352 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.352 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.352 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:33:08.352 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533277+0000", 2023-08-28T02:33:08.352 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:33:08.352 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.352 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.352 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:33:08.353 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533276+0000", 2023-08-28T02:33:08.353 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.8158999999999996e-05 2023-08-28T02:33:08.353 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.353 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.353 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:33:08.353 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533344+0000", 2023-08-28T02:33:08.353 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.1166e-05 2023-08-28T02:33:08.353 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.353 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.353 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:33:08.354 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533365+0000", 2023-08-28T02:33:08.354 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.2080000000000002e-06 2023-08-28T02:33:08.354 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.354 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.354 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:33:08.354 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533369+0000", 2023-08-28T02:33:08.354 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.5699999999999998e-05 2023-08-28T02:33:08.354 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.354 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.354 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:33:08.354 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533435+0000", 2023-08-28T02:33:08.355 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0674e-05 2023-08-28T02:33:08.355 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.355 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.355 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:33:08.355 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533446+0000", 2023-08-28T02:33:08.355 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.83182787300000005 2023-08-28T02:33:08.355 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.355 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.355 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:33:08.355 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.365274+0000", 2023-08-28T02:33:08.356 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000119156 2023-08-28T02:33:08.356 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.356 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.356 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:33:08.356 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.365393+0000", 2023-08-28T02:33:08.356 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000164434 2023-08-28T02:33:08.356 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.356 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.356 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,2", 2023-08-28T02:33:08.356 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.365557+0000", 2023-08-28T02:33:08.356 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0062156579999999998 2023-08-28T02:33:08.357 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.357 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.357 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:33:08.357 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.371773+0000", 2023-08-28T02:33:08.357 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0055333500000000002 2023-08-28T02:33:08.357 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.357 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.357 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:33:08.357 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377306+0000", 2023-08-28T02:33:08.358 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00051990900000000004 2023-08-28T02:33:08.358 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.358 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.358 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:33:08.358 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377826+0000", 2023-08-28T02:33:08.358 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.7752000000000001e-05 2023-08-28T02:33:08.358 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.358 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.358 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:33:08.358 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377854+0000", 2023-08-28T02:33:08.358 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.012e-05 2023-08-28T02:33:08.359 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.359 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.359 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:33:08.359 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377874+0000", 2023-08-28T02:33:08.359 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.84459700800000004 2023-08-28T02:33:08.359 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:33:08.359 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:33:08.359 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:33:08.359 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.359 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.360 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:65447 6.a 6:5c3c3d34:::benchmark_data_smithi143_305742_object65446:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-08-28T02:33:08.360 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:28:07.533387+0000", 2023-08-28T02:33:08.360 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 300.73592971199997, 2023-08-28T02:33:08.360 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.858966224, 2023-08-28T02:33:08.360 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:33:08.360 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:33:08.360 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:33:08.360 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:33:08.360 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:33:08.360 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 65447 2023-08-28T02:33:08.361 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.361 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:33:08.361 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.361 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:33:08.361 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533387+0000", 2023-08-28T02:33:08.361 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:33:08.361 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.361 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.361 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:33:08.361 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533387+0000", 2023-08-28T02:33:08.362 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:33:08.362 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.362 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.362 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:33:08.362 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533386+0000", 2023-08-28T02:33:08.362 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.5514999999999999e-05 2023-08-28T02:33:08.362 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.362 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.362 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:33:08.362 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533452+0000", 2023-08-28T02:33:08.362 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.9199999999999998e-06 2023-08-28T02:33:08.363 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.363 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.363 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:33:08.363 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533459+0000", 2023-08-28T02:33:08.363 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.9409999999999998e-06 2023-08-28T02:33:08.363 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.363 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.363 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:33:08.363 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533464+0000", 2023-08-28T02:33:08.363 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.6830999999999999e-05 2023-08-28T02:33:08.363 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.363 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.363 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:33:08.364 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533510+0000", 2023-08-28T02:33:08.364 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.3380999999999999e-05 2023-08-28T02:33:08.364 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.364 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.364 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:33:08.364 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533524+0000", 2023-08-28T02:33:08.364 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.83310036300000001 2023-08-28T02:33:08.364 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.364 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.364 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:33:08.364 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.366624+0000", 2023-08-28T02:33:08.365 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012930600000000001 2023-08-28T02:33:08.365 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.365 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.365 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:33:08.365 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.366753+0000", 2023-08-28T02:33:08.365 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000102413 2023-08-28T02:33:08.365 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.365 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.365 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,2", 2023-08-28T02:33:08.365 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.366856+0000", 2023-08-28T02:33:08.365 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0077729440000000004 2023-08-28T02:33:08.365 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.365 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.366 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:33:08.366 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.374629+0000", 2023-08-28T02:33:08.366 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0032932349999999998 2023-08-28T02:33:08.366 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.366 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.366 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:33:08.366 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377922+0000", 2023-08-28T02:33:08.366 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.01433544 2023-08-28T02:33:08.366 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.366 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.366 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:33:08.366 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.392257+0000", 2023-08-28T02:33:08.366 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.2425e-05 2023-08-28T02:33:08.367 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.367 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.367 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:33:08.367 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.392310+0000", 2023-08-28T02:33:08.367 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.3294999999999998e-05 2023-08-28T02:33:08.367 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.367 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.367 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:33:08.367 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.392353+0000", 2023-08-28T02:33:08.367 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.858966224 2023-08-28T02:33:08.367 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:33:08.367 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:33:08.367 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:33:08.368 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.368 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.368 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:65449 6.a 6:53253814:::benchmark_data_smithi143_305742_object65448:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-08-28T02:33:08.368 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:28:07.533542+0000", 2023-08-28T02:33:08.368 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 300.73577428200002, 2023-08-28T02:33:08.368 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.85898421800000002, 2023-08-28T02:33:08.368 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:33:08.368 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:33:08.368 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:33:08.368 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:33:08.368 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:33:08.368 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 65449 2023-08-28T02:33:08.368 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.369 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:33:08.369 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.369 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:33:08.369 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533542+0000", 2023-08-28T02:33:08.369 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:33:08.369 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.369 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.369 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:33:08.369 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533542+0000", 2023-08-28T02:33:08.369 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999995 2023-08-28T02:33:08.369 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.369 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.369 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:33:08.369 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533542+0000", 2023-08-28T02:33:08.370 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.8541999999999999e-05 2023-08-28T02:33:08.370 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.370 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.370 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:33:08.370 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533570+0000", 2023-08-28T02:33:08.370 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.267e-06 2023-08-28T02:33:08.370 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.370 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.370 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:33:08.370 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533578+0000", 2023-08-28T02:33:08.370 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.6210000000000003e-06 2023-08-28T02:33:08.370 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.371 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.371 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:33:08.371 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533582+0000", 2023-08-28T02:33:08.371 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.1579000000000002e-05 2023-08-28T02:33:08.371 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.371 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.371 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:33:08.371 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533634+0000", 2023-08-28T02:33:08.371 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.8409999999999995e-06 2023-08-28T02:33:08.371 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.371 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.371 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:33:08.371 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533644+0000", 2023-08-28T02:33:08.372 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.83422247699999996 2023-08-28T02:33:08.372 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.372 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.372 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:33:08.372 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.367866+0000", 2023-08-28T02:33:08.372 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00013198600000000001 2023-08-28T02:33:08.372 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.372 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.372 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:33:08.372 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.367998+0000", 2023-08-28T02:33:08.372 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00016983100000000001 2023-08-28T02:33:08.372 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.373 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.373 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,2", 2023-08-28T02:33:08.373 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.368168+0000", 2023-08-28T02:33:08.373 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0064664470000000002 2023-08-28T02:33:08.373 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.373 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.373 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:33:08.373 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.374634+0000", 2023-08-28T02:33:08.373 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0074225960000000001 2023-08-28T02:33:08.373 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.373 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.373 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:33:08.373 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.382057+0000", 2023-08-28T02:33:08.374 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.010359019000000001 2023-08-28T02:33:08.374 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.374 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.374 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:33:08.374 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.392416+0000", 2023-08-28T02:33:08.374 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.0843e-05 2023-08-28T02:33:08.374 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.374 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.374 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:33:08.374 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.392477+0000", 2023-08-28T02:33:08.374 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.9842999999999997e-05 2023-08-28T02:33:08.374 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.375 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.375 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:33:08.375 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.392527+0000", 2023-08-28T02:33:08.375 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.85898421800000002 2023-08-28T02:33:08.375 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:33:08.375 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:33:08.375 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:33:08.375 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.375 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.375 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:65517 6.a 6:57043958:::benchmark_data_smithi143_305742_object65516:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-08-28T02:33:08.375 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:28:07.651846+0000", 2023-08-28T02:33:08.375 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 300.61747034899997, 2023-08-28T02:33:08.375 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.74079570299999997, 2023-08-28T02:33:08.376 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:33:08.376 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:33:08.376 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:33:08.376 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:33:08.376 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:33:08.376 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 65517 2023-08-28T02:33:08.376 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.376 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:33:08.376 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.376 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:33:08.376 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.651846+0000", 2023-08-28T02:33:08.376 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:33:08.376 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.377 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.377 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:33:08.377 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.651846+0000", 2023-08-28T02:33:08.377 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:33:08.377 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.377 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.377 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:33:08.377 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.651845+0000", 2023-08-28T02:33:08.377 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.3328000000000002e-05 2023-08-28T02:33:08.377 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.377 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.377 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:33:08.377 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.651899+0000", 2023-08-28T02:33:08.378 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1822e-05 2023-08-28T02:33:08.378 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.378 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.378 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:33:08.378 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.651910+0000", 2023-08-28T02:33:08.378 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.1270000000000002e-06 2023-08-28T02:33:08.378 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.378 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.378 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:33:08.378 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.651915+0000", 2023-08-28T02:33:08.378 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.2360000000000005e-05 2023-08-28T02:33:08.379 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.379 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.379 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:33:08.379 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.651988+0000", 2023-08-28T02:33:08.379 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.1109999999999997e-06 2023-08-28T02:33:08.379 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.379 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.379 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:33:08.379 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.651997+0000", 2023-08-28T02:33:08.379 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.71717974100000004 2023-08-28T02:33:08.379 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.380 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.380 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:33:08.380 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.369177+0000", 2023-08-28T02:33:08.380 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012856099999999999 2023-08-28T02:33:08.380 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.380 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.380 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:33:08.380 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.369305+0000", 2023-08-28T02:33:08.380 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00010799200000000001 2023-08-28T02:33:08.380 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.380 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.380 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,2", 2023-08-28T02:33:08.381 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.369413+0000", 2023-08-28T02:33:08.381 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.005225861 2023-08-28T02:33:08.381 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.381 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.381 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:33:08.381 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.374639+0000", 2023-08-28T02:33:08.381 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.013565384999999999 2023-08-28T02:33:08.381 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.381 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.381 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:33:08.381 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.388205+0000", 2023-08-28T02:33:08.381 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0043731919999999997 2023-08-28T02:33:08.381 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.382 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.382 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:33:08.382 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.392578+0000", 2023-08-28T02:33:08.382 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.9850000000000001e-05 2023-08-28T02:33:08.382 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.382 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.382 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:33:08.382 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.392608+0000", 2023-08-28T02:33:08.382 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.4477999999999998e-05 2023-08-28T02:33:08.382 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.382 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.383 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:33:08.383 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.392642+0000", 2023-08-28T02:33:08.383 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.74079570299999997 2023-08-28T02:33:08.383 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:33:08.383 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:33:08.383 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:33:08.383 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.383 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.383 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:65528 6.a 6:584fd7a4:::benchmark_data_smithi143_305742_object65527:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-08-28T02:33:08.384 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:28:07.678223+0000", 2023-08-28T02:33:08.384 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 300.59109333800001, 2023-08-28T02:33:08.384 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.71583466399999995, 2023-08-28T02:33:08.384 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:33:08.384 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:33:08.384 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:33:08.384 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:33:08.384 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:33:08.384 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 65528 2023-08-28T02:33:08.384 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.384 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:33:08.384 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.385 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:33:08.385 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.678223+0000", 2023-08-28T02:33:08.385 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:33:08.385 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.385 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.385 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:33:08.385 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.678223+0000", 2023-08-28T02:33:08.385 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:33:08.385 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.386 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.386 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:33:08.386 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.678222+0000", 2023-08-28T02:33:08.386 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.5943999999999999e-05 2023-08-28T02:33:08.386 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.386 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.386 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:33:08.386 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.678268+0000", 2023-08-28T02:33:08.386 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0148e-05 2023-08-28T02:33:08.386 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.387 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.387 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:33:08.387 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.678278+0000", 2023-08-28T02:33:08.387 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.0350000000000001e-06 2023-08-28T02:33:08.387 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.387 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.387 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:33:08.387 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.678286+0000", 2023-08-28T02:33:08.387 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.7000000000000003e-05 2023-08-28T02:33:08.387 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.388 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.388 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:33:08.388 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.678343+0000", 2023-08-28T02:33:08.388 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.8419999999999994e-06 2023-08-28T02:33:08.388 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.388 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.388 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:33:08.388 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.678352+0000", 2023-08-28T02:33:08.388 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.69213458500000002 2023-08-28T02:33:08.388 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.389 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.389 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:33:08.389 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.370487+0000", 2023-08-28T02:33:08.389 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012174400000000001 2023-08-28T02:33:08.389 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.389 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.389 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:33:08.389 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.370608+0000", 2023-08-28T02:33:08.389 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00016430399999999999 2023-08-28T02:33:08.389 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.390 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.390 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,2", 2023-08-28T02:33:08.390 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.370773+0000", 2023-08-28T02:33:08.390 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0039615850000000001 2023-08-28T02:33:08.390 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.390 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.390 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:33:08.390 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.374734+0000", 2023-08-28T02:33:08.390 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.017946527 2023-08-28T02:33:08.391 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.391 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.391 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:33:08.391 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.392681+0000", 2023-08-28T02:33:08.391 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.001296632 2023-08-28T02:33:08.391 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.391 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.391 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:33:08.391 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.393977+0000", 2023-08-28T02:33:08.392 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.4098999999999999e-05 2023-08-28T02:33:08.392 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.392 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.392 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:33:08.392 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.394022+0000", 2023-08-28T02:33:08.392 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.6463e-05 2023-08-28T02:33:08.392 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.392 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.392 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:33:08.392 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.394058+0000", 2023-08-28T02:33:08.393 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.71583466399999995 2023-08-28T02:33:08.393 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:33:08.393 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:33:08.393 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:33:08.393 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.393 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.393 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:65590 6.a 6:52c121f0:::benchmark_data_smithi143_305742_object65589:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-08-28T02:33:08.393 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:28:07.793373+0000", 2023-08-28T02:33:08.393 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 300.47594381900001, 2023-08-28T02:33:08.394 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.60446993100000002, 2023-08-28T02:33:08.394 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:33:08.394 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:33:08.394 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:33:08.394 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:33:08.394 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:33:08.394 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 65590 2023-08-28T02:33:08.394 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.394 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:33:08.394 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.395 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:33:08.395 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.793373+0000", 2023-08-28T02:33:08.395 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:33:08.395 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.395 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.395 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:33:08.395 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.793373+0000", 2023-08-28T02:33:08.395 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:33:08.395 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.396 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.396 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:33:08.396 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.793372+0000", 2023-08-28T02:33:08.396 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.9734000000000002e-05 2023-08-28T02:33:08.396 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.396 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.396 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:33:08.396 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.793421+0000", 2023-08-28T02:33:08.396 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2872e-05 2023-08-28T02:33:08.396 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.397 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.397 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:33:08.397 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.793434+0000", 2023-08-28T02:33:08.397 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.4409999999999998e-06 2023-08-28T02:33:08.397 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.397 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.397 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:33:08.397 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.793441+0000", 2023-08-28T02:33:08.397 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.2967000000000003e-05 2023-08-28T02:33:08.397 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.397 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.397 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:33:08.397 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.793494+0000", 2023-08-28T02:33:08.398 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.2439999999999996e-06 2023-08-28T02:33:08.398 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.398 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.398 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:33:08.398 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.793501+0000", 2023-08-28T02:33:08.398 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.57831116699999996 2023-08-28T02:33:08.398 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.398 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.398 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:33:08.398 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.371812+0000", 2023-08-28T02:33:08.398 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000123433 2023-08-28T02:33:08.398 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.398 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.399 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:33:08.399 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.371936+0000", 2023-08-28T02:33:08.399 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00016441700000000001 2023-08-28T02:33:08.399 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.399 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.399 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,2", 2023-08-28T02:33:08.399 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.372100+0000", 2023-08-28T02:33:08.399 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0026377420000000002 2023-08-28T02:33:08.399 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.399 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.399 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:33:08.399 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.374738+0000", 2023-08-28T02:33:08.399 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.019368895000000001 2023-08-28T02:33:08.400 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.400 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.400 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:33:08.400 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.394107+0000", 2023-08-28T02:33:08.400 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.003625634 2023-08-28T02:33:08.400 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.400 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.400 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:33:08.400 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.397732+0000", 2023-08-28T02:33:08.400 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.1490999999999998e-05 2023-08-28T02:33:08.400 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.400 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.401 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:33:08.401 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.397794+0000", 2023-08-28T02:33:08.401 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.9030000000000003e-05 2023-08-28T02:33:08.401 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.401 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.401 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:33:08.401 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.397843+0000", 2023-08-28T02:33:08.401 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.60446993100000002 2023-08-28T02:33:08.401 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:33:08.401 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:33:08.401 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:33:08.401 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.401 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.402 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:65630 6.a 6:54d1eb7a:::benchmark_data_smithi143_305742_object65629:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-08-28T02:33:08.402 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:28:07.879482+0000", 2023-08-28T02:33:08.402 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 300.38983512099998, 2023-08-28T02:33:08.402 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.51854038599999996, 2023-08-28T02:33:08.402 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:33:08.402 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:33:08.402 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:33:08.402 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:33:08.402 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:33:08.402 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 65630 2023-08-28T02:33:08.402 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.402 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:33:08.403 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.403 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:33:08.403 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.879482+0000", 2023-08-28T02:33:08.403 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:33:08.403 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.403 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.403 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:33:08.403 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.879482+0000", 2023-08-28T02:33:08.403 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999976 2023-08-28T02:33:08.403 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.403 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.403 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:33:08.404 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.879479+0000", 2023-08-28T02:33:08.404 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.5662999999999999e-05 2023-08-28T02:33:08.404 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.404 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.404 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:33:08.404 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.879535+0000", 2023-08-28T02:33:08.404 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.6812000000000001e-05 2023-08-28T02:33:08.404 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.404 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.404 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:33:08.404 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.879552+0000", 2023-08-28T02:33:08.404 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.8879999999999997e-06 2023-08-28T02:33:08.404 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.405 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.405 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:33:08.405 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.879559+0000", 2023-08-28T02:33:08.405 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.8687000000000002e-05 2023-08-28T02:33:08.405 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.405 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.405 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:33:08.405 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.879628+0000", 2023-08-28T02:33:08.405 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2177999999999999e-05 2023-08-28T02:33:08.405 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.405 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.405 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:33:08.406 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.879640+0000", 2023-08-28T02:33:08.406 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.49364018300000001 2023-08-28T02:33:08.406 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.406 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.406 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:33:08.406 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.373281+0000", 2023-08-28T02:33:08.406 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000123545 2023-08-28T02:33:08.406 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.406 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.406 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:33:08.406 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.373404+0000", 2023-08-28T02:33:08.406 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00016205899999999999 2023-08-28T02:33:08.406 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.407 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.407 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,2", 2023-08-28T02:33:08.407 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.373566+0000", 2023-08-28T02:33:08.407 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.003690292 2023-08-28T02:33:08.407 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.407 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.407 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:33:08.407 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377256+0000", 2023-08-28T02:33:08.407 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.016889149999999999 2023-08-28T02:33:08.407 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.407 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.407 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:33:08.408 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.394146+0000", 2023-08-28T02:33:08.408 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0037648989999999999 2023-08-28T02:33:08.408 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.408 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.408 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:33:08.408 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.397910+0000", 2023-08-28T02:33:08.408 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.1008999999999998e-05 2023-08-28T02:33:08.408 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.408 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.408 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:33:08.408 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.397971+0000", 2023-08-28T02:33:08.408 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.0457000000000002e-05 2023-08-28T02:33:08.408 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.409 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.409 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:33:08.409 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.398022+0000", 2023-08-28T02:33:08.409 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.51854038599999996 2023-08-28T02:33:08.409 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:33:08.409 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:33:08.409 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:33:08.409 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.409 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.409 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:59974 7.0 7:055944e3:::benchmark_data_smithi143_360738_object59973:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e270)", 2023-08-28T02:33:08.409 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:31:43.477791+0000", 2023-08-28T02:33:08.409 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 84.791525931999999, 2023-08-28T02:33:08.409 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.99199723799999995, 2023-08-28T02:33:08.410 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:33:08.410 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "reached pg", 2023-08-28T02:33:08.410 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:33:08.410 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:33:08.410 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:33:08.410 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 59974 2023-08-28T02:33:08.410 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.410 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:33:08.410 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.410 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:33:08.410 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.477791+0000", 2023-08-28T02:33:08.410 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:33:08.410 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.411 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.411 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:33:08.411 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.477791+0000", 2023-08-28T02:33:08.411 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999967 2023-08-28T02:33:08.411 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.411 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.411 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:33:08.411 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.477787+0000", 2023-08-28T02:33:08.411 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.3393999999999995e-05 2023-08-28T02:33:08.411 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.411 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.411 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:33:08.411 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.477871+0000", 2023-08-28T02:33:08.412 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.5343000000000003e-05 2023-08-28T02:33:08.412 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.412 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.412 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:33:08.412 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.477916+0000", 2023-08-28T02:33:08.412 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2707999999999999e-05 2023-08-28T02:33:08.412 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.412 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.412 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:33:08.412 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.477929+0000", 2023-08-28T02:33:08.412 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.011155768999999999 2023-08-28T02:33:08.412 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.412 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.413 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:33:08.413 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.489085+0000", 2023-08-28T02:33:08.413 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.8269999999999993e-06 2023-08-28T02:33:08.413 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.413 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.413 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:33:08.413 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.489094+0000", 2023-08-28T02:33:08.413 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.98061759000000004 2023-08-28T02:33:08.413 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.413 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.413 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:33:08.413 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.469712+0000", 2023-08-28T02:33:08.413 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.6050000000000005e-05 2023-08-28T02:33:08.414 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.414 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.414 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:33:08.414 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.469788+0000", 2023-08-28T02:33:08.414 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.99199723799999995 2023-08-28T02:33:08.414 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:33:08.414 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:33:08.414 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:33:08.414 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.414 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.414 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:59977 7.0 7:00ce683a:::benchmark_data_smithi143_360738_object59976:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e270)", 2023-08-28T02:33:08.414 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:31:43.477944+0000", 2023-08-28T02:33:08.414 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 84.791372385000003, 2023-08-28T02:33:08.414 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.99213284300000004, 2023-08-28T02:33:08.415 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:33:08.415 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "reached pg", 2023-08-28T02:33:08.415 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:33:08.415 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:33:08.415 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:33:08.415 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 59977 2023-08-28T02:33:08.415 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.415 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:33:08.415 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.415 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:33:08.415 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.477944+0000", 2023-08-28T02:33:08.415 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:33:08.416 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.416 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.416 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:33:08.416 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.477944+0000", 2023-08-28T02:33:08.416 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999995 2023-08-28T02:33:08.416 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.416 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.416 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:33:08.416 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.477944+0000", 2023-08-28T02:33:08.416 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.2744000000000002e-05 2023-08-28T02:33:08.416 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.416 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.417 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:33:08.417 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.477996+0000", 2023-08-28T02:33:08.417 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.0423e-05 2023-08-28T02:33:08.417 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.417 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.417 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:33:08.417 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.478027+0000", 2023-08-28T02:33:08.417 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.2459999999999997e-06 2023-08-28T02:33:08.417 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.417 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.418 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:33:08.418 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.478034+0000", 2023-08-28T02:33:08.418 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.011097246 2023-08-28T02:33:08.418 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.418 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.418 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:33:08.418 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.489131+0000", 2023-08-28T02:33:08.418 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.455e-06 2023-08-28T02:33:08.418 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.418 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.418 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:33:08.418 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.489138+0000", 2023-08-28T02:33:08.419 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.98088636200000001 2023-08-28T02:33:08.419 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.419 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.419 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:33:08.419 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.470024+0000", 2023-08-28T02:33:08.419 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.2932000000000001e-05 2023-08-28T02:33:08.419 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.419 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.419 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:33:08.419 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.470077+0000", 2023-08-28T02:33:08.419 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.99213284300000004 2023-08-28T02:33:08.419 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:33:08.419 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:33:08.420 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:33:08.420 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.420 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.420 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:59978 7.10 7:0e447851:::benchmark_data_smithi143_360738_object59977:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e270)", 2023-08-28T02:33:08.420 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:31:43.478040+0000", 2023-08-28T02:33:08.420 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 84.791277003000005, 2023-08-28T02:33:08.420 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0064679560000001, 2023-08-28T02:33:08.420 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:33:08.420 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "reached pg", 2023-08-28T02:33:08.420 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:33:08.420 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:33:08.420 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:33:08.420 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 59978 2023-08-28T02:33:08.421 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.421 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:33:08.421 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.421 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:33:08.421 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.478040+0000", 2023-08-28T02:33:08.421 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:33:08.421 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.421 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.421 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:33:08.421 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.478040+0000", 2023-08-28T02:33:08.421 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999995 2023-08-28T02:33:08.421 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.421 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.422 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:33:08.422 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.478039+0000", 2023-08-28T02:33:08.422 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.7616e-05 2023-08-28T02:33:08.422 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.422 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.422 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:33:08.422 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.478077+0000", 2023-08-28T02:33:08.422 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.9510000000000006e-06 2023-08-28T02:33:08.422 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.422 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.422 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:33:08.422 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.478085+0000", 2023-08-28T02:33:08.423 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.6410000000000001e-06 2023-08-28T02:33:08.423 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.423 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.423 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:33:08.423 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.478089+0000", 2023-08-28T02:33:08.423 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.011218619000000001 2023-08-28T02:33:08.423 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.423 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.423 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:33:08.424 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.489308+0000", 2023-08-28T02:33:08.424 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.6300000000000003e-06 2023-08-28T02:33:08.424 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.424 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.424 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:33:08.424 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.489313+0000", 2023-08-28T02:33:08.424 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.99510897799999998 2023-08-28T02:33:08.424 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.424 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.424 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:33:08.425 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.484422+0000", 2023-08-28T02:33:08.425 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.5159000000000002e-05 2023-08-28T02:33:08.425 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.425 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.425 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:33:08.425 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.484508+0000", 2023-08-28T02:33:08.425 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0064679560000001 2023-08-28T02:33:08.425 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:33:08.425 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:33:08.425 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:33:08.425 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.426 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.426 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:59984 7.0 7:01f850c8:::benchmark_data_smithi143_360738_object59983:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e270)", 2023-08-28T02:33:08.426 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:31:43.478095+0000", 2023-08-28T02:33:08.426 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 84.791222073, 2023-08-28T02:33:08.426 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.992082256, 2023-08-28T02:33:08.426 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:33:08.426 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "reached pg", 2023-08-28T02:33:08.426 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:33:08.426 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:33:08.426 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:33:08.426 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 59984 2023-08-28T02:33:08.427 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.427 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:33:08.427 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.427 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:33:08.427 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.478095+0000", 2023-08-28T02:33:08.427 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:33:08.427 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.427 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.427 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:33:08.427 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.478095+0000", 2023-08-28T02:33:08.428 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999995 2023-08-28T02:33:08.428 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.428 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.428 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:33:08.428 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.478094+0000", 2023-08-28T02:33:08.428 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.0431999999999999e-05 2023-08-28T02:33:08.428 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.428 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.428 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:33:08.428 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.478125+0000", 2023-08-28T02:33:08.429 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.8859999999999998e-06 2023-08-28T02:33:08.429 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.429 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.429 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:33:08.429 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.478131+0000", 2023-08-28T02:33:08.429 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.4510000000000002e-06 2023-08-28T02:33:08.429 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.429 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.429 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:33:08.429 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.478136+0000", 2023-08-28T02:33:08.429 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.011034641 2023-08-28T02:33:08.430 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.430 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.430 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:33:08.430 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.489170+0000", 2023-08-28T02:33:08.430 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.1199999999999996e-06 2023-08-28T02:33:08.430 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.430 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.430 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:33:08.430 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.489178+0000", 2023-08-28T02:33:08.430 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.98092388699999999 2023-08-28T02:33:08.431 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.431 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.431 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:33:08.431 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.470101+0000", 2023-08-28T02:33:08.431 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.5358999999999995e-05 2023-08-28T02:33:08.431 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:08.431 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:08.431 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:33:08.431 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.470177+0000", 2023-08-28T02:33:08.431 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.992082256 2023-08-28T02:33:08.432 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:33:08.432 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:33:08.432 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:33:08.432 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:33:08.432 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:33:08.432 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:33:08.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:08.434 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:08.434 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:33:08.434 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:33:08.535 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:33:08.568 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:08.615 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:33:08.617 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:08 smithi169 ceph-mon[159415]: pgmap v734: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.3 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 678 op/s; 32739/212556 objects degraded (15.403%); 4.6 MiB/s, 81 objects/s recovering 2023-08-28T02:33:08.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:08.618 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:33:08.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:08 smithi143 ceph-mon[183059]: pgmap v734: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.3 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 678 op/s; 32739/212556 objects degraded (15.403%); 4.6 MiB/s, 81 objects/s recovering 2023-08-28T02:33:08.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:08 smithi143 ceph-mon[189871]: pgmap v734: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.3 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 678 op/s; 32739/212556 objects degraded (15.403%); 4.6 MiB/s, 81 objects/s recovering 2023-08-28T02:33:08.719 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:33:08.868 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:09.007 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:33:09.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:09.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:33:09.111 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:33:09.136 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:09.155 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.6 dump_ops_in_flight 2023-08-28T02:33:09.178 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:33:09.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:09.180 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:33:09.281 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:33:09.375 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:09.571 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:33:09.632 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:33:09.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:09.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:33:09.757 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:33:09.855 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:09.960 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:33:09.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:09.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:33:10.068 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:33:10.102 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:10.212 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:33:10.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:10.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:33:10.316 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:33:10.343 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:10.385 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:33:10.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:10.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:33:10.441 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:33:10.488 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:33:10.518 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:10.599 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:33:10.614 INFO:teuthology.orchestra.run.smithi143.stderr:dumped all 2023-08-28T02:33:10.730 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:33:10.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:10.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:33:10.736 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:10 smithi169 ceph-mon[159415]: pgmap v735: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 637 op/s; 32553/210183 objects degraded (15.488%); 5.0 MiB/s, 81 objects/s recovering 2023-08-28T02:33:10.836 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:33:10.899 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:10 smithi143 ceph-mon[183059]: pgmap v735: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 637 op/s; 32553/210183 objects degraded (15.488%); 5.0 MiB/s, 81 objects/s recovering 2023-08-28T02:33:10.899 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:10 smithi143 ceph-mon[189871]: pgmap v735: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 637 op/s; 32553/210183 objects degraded (15.488%); 5.0 MiB/s, 81 objects/s recovering 2023-08-28T02:33:11.117 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:11.303 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:33:11.303 INFO:teuthology.orchestra.run.smithi169.stdout: "ops": [ 2023-08-28T02:33:11.303 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:11.303 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:118480 7.7 7:e7584714:::benchmark_data_smithi143_360738_object25710:head [delete] snapc 0=[] ondisk+write+known_if_redirected e294)", 2023-08-28T02:33:11.304 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:33:11.279193+0000", 2023-08-28T02:33:11.304 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 0.024773087999999999, 2023-08-28T02:33:11.304 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.024795103999999998, 2023-08-28T02:33:11.304 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:33:11.304 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "waiting for sub ops", 2023-08-28T02:33:11.304 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:33:11.304 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:33:11.304 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:33:11.304 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 118480 2023-08-28T02:33:11.304 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:11.304 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:33:11.305 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:11.305 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:33:11.305 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:11.279193+0000", 2023-08-28T02:33:11.305 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:33:11.305 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:11.305 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:11.305 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:33:11.305 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:11.279193+0000", 2023-08-28T02:33:11.305 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:33:11.305 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:11.305 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:11.305 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:33:11.305 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:11.279192+0000", 2023-08-28T02:33:11.306 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.5489999999999997e-06 2023-08-28T02:33:11.306 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:11.306 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:11.306 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:33:11.306 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:11.279197+0000", 2023-08-28T02:33:11.306 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1291000000000001e-05 2023-08-28T02:33:11.306 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:11.306 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:11.306 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:33:11.306 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:11.279208+0000", 2023-08-28T02:33:11.306 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.9640000000000002e-06 2023-08-28T02:33:11.306 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:11.306 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:11.307 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:33:11.307 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:11.279213+0000", 2023-08-28T02:33:11.307 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.9790000000000003e-05 2023-08-28T02:33:11.307 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:11.307 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:11.307 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:33:11.307 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:11.279263+0000", 2023-08-28T02:33:11.307 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000105219 2023-08-28T02:33:11.307 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:11.307 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:11.307 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:33:11.307 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:11.279368+0000", 2023-08-28T02:33:11.308 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.6508000000000005e-05 2023-08-28T02:33:11.308 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:11.308 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:11.308 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,3", 2023-08-28T02:33:11.308 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:11.279465+0000", 2023-08-28T02:33:11.308 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0022313670000000002 2023-08-28T02:33:11.308 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:11.308 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:11.308 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:33:11.308 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:11.281696+0000", 2023-08-28T02:33:11.308 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0078533800000000001 2023-08-28T02:33:11.308 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:11.309 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:11.309 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:33:11.309 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:11.289550+0000", 2023-08-28T02:33:11.309 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.010356850000000001 2023-08-28T02:33:11.309 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:33:11.309 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:33:11.309 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:33:11.309 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:11.309 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:11.309 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:118498 7.11 7:8af3feb7:::benchmark_data_smithi143_360738_object25728:head [delete] snapc 0=[] ondisk+write+known_if_redirected e294)", 2023-08-28T02:33:11.309 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:33:11.303257+0000", 2023-08-28T02:33:11.309 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 0.00070871200000000001, 2023-08-28T02:33:11.309 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000790991, 2023-08-28T02:33:11.310 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:33:11.310 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "waiting for sub ops", 2023-08-28T02:33:11.310 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:33:11.310 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:33:11.310 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:33:11.310 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 118498 2023-08-28T02:33:11.310 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:11.310 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:33:11.310 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:11.310 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:33:11.310 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:11.303257+0000", 2023-08-28T02:33:11.310 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:33:11.311 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:11.311 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:11.311 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:33:11.311 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:11.303257+0000", 2023-08-28T02:33:11.311 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:33:11.311 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:11.311 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:11.311 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:33:11.311 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:11.303256+0000", 2023-08-28T02:33:11.311 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.9079999999999998e-06 2023-08-28T02:33:11.311 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:11.311 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:11.311 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:33:11.312 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:11.303263+0000", 2023-08-28T02:33:11.312 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.5072000000000001e-05 2023-08-28T02:33:11.312 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:11.312 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:11.312 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:33:11.312 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:11.303278+0000", 2023-08-28T02:33:11.312 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.1359999999999996e-06 2023-08-28T02:33:11.312 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:11.312 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:11.312 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:33:11.312 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:11.303283+0000", 2023-08-28T02:33:11.312 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.787e-05 2023-08-28T02:33:11.313 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:11.313 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:11.313 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:33:11.313 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:11.303341+0000", 2023-08-28T02:33:11.313 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000191373 2023-08-28T02:33:11.313 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:11.313 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:11.313 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:33:11.313 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:11.303532+0000", 2023-08-28T02:33:11.313 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000151989 2023-08-28T02:33:11.313 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:33:11.313 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:33:11.314 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 1,3", 2023-08-28T02:33:11.314 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:11.303684+0000", 2023-08-28T02:33:11.314 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00042699900000000001 2023-08-28T02:33:11.314 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:33:11.314 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:33:11.314 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:33:11.314 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:33:11.314 INFO:teuthology.orchestra.run.smithi169.stdout: ], 2023-08-28T02:33:11.314 INFO:teuthology.orchestra.run.smithi169.stdout: "num_ops": 2 2023-08-28T02:33:11.314 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:33:11.557 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is set 2023-08-28T02:33:11.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:11 smithi169 ceph-mon[159415]: from='client.35472 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:33:11.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:11 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/839265580' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:33:11.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:11 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:33:11.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:11 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:33:11.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:11 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:33:11.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:11 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 32553/210183 objects degraded (15.488%), 22 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-08-28T02:33:11.932 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:11 smithi143 ceph-mon[183059]: from='client.35472 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:33:11.932 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:11 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/839265580' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:33:11.932 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:11 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:33:11.932 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:11 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:33:11.932 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:11 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:33:11.932 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:11 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 32553/210183 objects degraded (15.488%), 22 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-08-28T02:33:11.933 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:33:11 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:33:11] "GET /metrics HTTP/1.1" 200 36583 "" "Prometheus/2.33.4" 2023-08-28T02:33:11.933 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:11 smithi143 ceph-mon[189871]: from='client.35472 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:33:11.933 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:11 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/839265580' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:33:11.933 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:11 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:33:11.933 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:11 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:33:11.933 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:11 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:33:11.934 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:11 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 32553/210183 objects degraded (15.488%), 22 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-08-28T02:33:12.067 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.6 dump_blocked_ops 2023-08-28T02:33:12.303 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:33:12.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:12.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:33:12.411 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:33:12.491 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:12.505 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:33:12.818 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:12 smithi169 ceph-mon[159415]: pgmap v736: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 607 op/s; 32553/205974 objects degraded (15.804%); 2.2 MiB/s, 35 objects/s recovering 2023-08-28T02:33:12.818 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:12 smithi169 ceph-mon[159415]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:33:12.818 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:12 smithi169 ceph-mon[159415]: failed to encode map e295 with expected crc 2023-08-28T02:33:12.818 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:12 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:33:12.818 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:12 smithi169 ceph-mon[159415]: osdmap e295: 8 total, 7 up, 7 in 2023-08-28T02:33:12.818 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:12 smithi169 ceph-mon[159415]: failed to encode map e295 with expected crc 2023-08-28T02:33:12.818 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:12 smithi169 ceph-mon[159415]: failed to encode map e295 with expected crc 2023-08-28T02:33:12.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:12 smithi143 ceph-mon[183059]: pgmap v736: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 607 op/s; 32553/205974 objects degraded (15.804%); 2.2 MiB/s, 35 objects/s recovering 2023-08-28T02:33:12.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:12 smithi143 ceph-mon[183059]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:33:12.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:12 smithi143 ceph-mon[183059]: failed to encode map e295 with expected crc 2023-08-28T02:33:12.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:12 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:33:12.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:12 smithi143 ceph-mon[183059]: osdmap e295: 8 total, 7 up, 7 in 2023-08-28T02:33:12.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:12 smithi143 ceph-mon[183059]: failed to encode map e295 with expected crc 2023-08-28T02:33:12.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:12 smithi143 ceph-mon[183059]: failed to encode map e295 with expected crc 2023-08-28T02:33:12.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:12 smithi143 ceph-mon[189871]: pgmap v736: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 607 op/s; 32553/205974 objects degraded (15.804%); 2.2 MiB/s, 35 objects/s recovering 2023-08-28T02:33:12.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:12 smithi143 ceph-mon[189871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:33:12.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:12 smithi143 ceph-mon[189871]: failed to encode map e295 with expected crc 2023-08-28T02:33:12.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:12 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:33:12.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:12 smithi143 ceph-mon[189871]: osdmap e295: 8 total, 7 up, 7 in 2023-08-28T02:33:12.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:12 smithi143 ceph-mon[189871]: failed to encode map e295 with expected crc 2023-08-28T02:33:12.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:12 smithi143 ceph-mon[189871]: failed to encode map e295 with expected crc 2023-08-28T02:33:13.332 INFO:teuthology.orchestra.run.smithi143.stdout:{"pg_ready":true,"pg_map":{"version":735,"stamp":"2023-08-28T02:33:09.391379+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4594407371,"num_objects":70061,"num_object_clones":0,"num_object_copies":210183,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":32553,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":70061,"num_whiteouts":0,"num_read":6275,"num_read_kb":13708,"num_write":116609,"num_write_kb":5952230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6051,"num_bytes_recovered":387998392,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":147314,"ondisk_log_size":147314,"up":297,"acting":279,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":343,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":16059000,"kb_used_data":13357248,"kb_used_omap":29,"kb_used_meta":2701282,"kb_avail":640157064,"statfs":{"total":671965249536,"available":655520833536,"internally_reserved":0,"allocated":13677821952,"data_stored":13669970914,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30674,"internal_metadata":2766112814},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,1,5,10,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-407633920,"num_objects":-6220,"num_object_clones":0,"num_object_copies":-18660,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-13075,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-6220,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6220,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":794,"num_bytes_recovered":50987030,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.762378"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":313,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.638412+0000","last_change":"2023-08-28T02:32:58.740384+0000","last_active":"2023-08-28T02:32:59.638412+0000","last_peered":"2023-08-28T02:32:59.638412+0000","last_clean":"2023-08-28T02:32:59.638412+0000","last_became_active":"2023-08-28T02:32:58.739959+0000","last_became_peered":"2023-08-28T02:32:58.739959+0000","last_unstale":"2023-08-28T02:32:59.638412+0000","last_undegraded":"2023-08-28T02:32:59.638412+0000","last_fullsized":"2023-08-28T02:32:59.638412+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":199,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.638348+0000","last_change":"2023-08-28T02:32:58.634375+0000","last_active":"2023-08-28T02:32:59.638348+0000","last_peered":"2023-08-28T02:32:59.638348+0000","last_clean":"2023-08-28T02:32:59.638348+0000","last_became_active":"2023-08-28T02:32:58.632726+0000","last_became_peered":"2023-08-28T02:32:58.632726+0000","last_unstale":"2023-08-28T02:32:59.638348+0000","last_undegraded":"2023-08-28T02:32:59.638348+0000","last_fullsized":"2023-08-28T02:32:59.638348+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:20:27.730725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":13,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.638241+0000","last_change":"2023-08-28T02:32:58.628387+0000","last_active":"2023-08-28T02:32:59.638241+0000","last_peered":"2023-08-28T02:32:59.638241+0000","last_clean":"2023-08-28T02:32:59.638241+0000","last_became_active":"2023-08-28T02:32:58.628071+0000","last_became_peered":"2023-08-28T02:32:58.628071+0000","last_unstale":"2023-08-28T02:32:59.638241+0000","last_undegraded":"2023-08-28T02:32:59.638241+0000","last_fullsized":"2023-08-28T02:32:59.638241+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:25:55.264433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":310,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.652764+0000","last_change":"2023-08-28T02:32:58.772285+0000","last_active":"2023-08-28T02:32:59.652764+0000","last_peered":"2023-08-28T02:32:59.652764+0000","last_clean":"2023-08-28T02:32:59.652764+0000","last_became_active":"2023-08-28T02:32:58.771439+0000","last_became_peered":"2023-08-28T02:32:58.771439+0000","last_unstale":"2023-08-28T02:32:59.652764+0000","last_undegraded":"2023-08-28T02:32:59.652764+0000","last_fullsized":"2023-08-28T02:32:59.652764+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:10:48.541785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":190,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.647706+0000","last_change":"2023-08-28T02:32:58.779602+0000","last_active":"2023-08-28T02:32:59.647706+0000","last_peered":"2023-08-28T02:32:59.647706+0000","last_clean":"2023-08-28T02:32:59.647706+0000","last_became_active":"2023-08-28T02:32:58.778945+0000","last_became_peered":"2023-08-28T02:32:58.778945+0000","last_unstale":"2023-08-28T02:32:59.647706+0000","last_undegraded":"2023-08-28T02:32:59.647706+0000","last_fullsized":"2023-08-28T02:32:59.647706+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:35:55.832928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":100,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.639937+0000","last_change":"2023-08-28T02:32:58.626766+0000","last_active":"2023-08-28T02:32:59.639937+0000","last_peered":"2023-08-28T02:32:59.639937+0000","last_clean":"2023-08-28T02:32:59.639937+0000","last_became_active":"2023-08-28T02:32:58.626415+0000","last_became_peered":"2023-08-28T02:32:58.626415+0000","last_unstale":"2023-08-28T02:32:59.639937+0000","last_undegraded":"2023-08-28T02:32:59.639937+0000","last_fullsized":"2023-08-28T02:32:59.639937+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.13","version":"294'5426","reported_seq":5478,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-08-28T02:33:04.684156+0000","last_change":"2023-08-28T02:31:44.484177+0000","last_active":"2023-08-28T02:33:04.684156+0000","last_peered":"2023-08-28T02:33:04.684156+0000","last_clean":"2023-08-28T02:33:04.684156+0000","last_became_active":"2023-08-28T02:31:44.482443+0000","last_became_peered":"2023-08-28T02:31:44.482443+0000","last_unstale":"2023-08-28T02:33:04.684156+0000","last_undegraded":"2023-08-28T02:33:04.684156+0000","last_fullsized":"2023-08-28T02:33:04.684156+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5426,"log_dups_size":0,"ondisk_log_size":5426,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148897792,"num_objects":2272,"num_object_clones":0,"num_object_copies":6816,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2272,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3556,"num_write_kb":186496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":263,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-08-28T02:33:00.662441+0000","last_change":"2023-08-28T02:23:59.227592+0000","last_active":"2023-08-28T02:33:00.662441+0000","last_peered":"2023-08-28T02:33:00.662441+0000","last_clean":"2023-08-28T02:33:00.662441+0000","last_became_active":"2023-08-28T02:23:59.227342+0000","last_became_peered":"2023-08-28T02:23:59.227342+0000","last_unstale":"2023-08-28T02:33:00.662441+0000","last_undegraded":"2023-08-28T02:33:00.662441+0000","last_fullsized":"2023-08-28T02:33:00.662441+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:08:19.874396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":208,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.647750+0000","last_change":"2023-08-28T02:32:58.779435+0000","last_active":"2023-08-28T02:32:59.647750+0000","last_peered":"2023-08-28T02:32:59.647750+0000","last_clean":"2023-08-28T02:32:59.647750+0000","last_became_active":"2023-08-28T02:32:58.778629+0000","last_became_peered":"2023-08-28T02:32:58.778629+0000","last_unstale":"2023-08-28T02:32:59.647750+0000","last_undegraded":"2023-08-28T02:32:59.647750+0000","last_fullsized":"2023-08-28T02:32:59.647750+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:25:32.193898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.12","version":"294'5525","reported_seq":5593,"reported_epoch":294,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:33:04.719223+0000","last_change":"2023-08-28T02:32:58.866748+0000","last_active":"2023-08-28T02:33:04.719223+0000","last_peered":"2023-08-28T02:33:04.719223+0000","last_clean":"2023-08-28T02:32:11.498129+0000","last_became_active":"2023-08-28T02:32:58.859951+0000","last_became_peered":"2023-08-28T02:32:58.859951+0000","last_unstale":"2023-08-28T02:33:04.719223+0000","last_undegraded":"2023-08-28T02:32:58.636278+0000","last_fullsized":"2023-08-28T02:32:58.628896+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5525,"log_dups_size":0,"ondisk_log_size":5525,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145752064,"num_objects":2224,"num_object_clones":0,"num_object_copies":6672,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":970,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2224,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3620,"num_write_kb":187008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[{"shards":"2,3","objects":1831},{"shards":"2,3,6","objects":393}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":323,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.642214+0000","last_change":"2023-08-28T02:32:58.636365+0000","last_active":"2023-08-28T02:32:59.642214+0000","last_peered":"2023-08-28T02:32:59.642214+0000","last_clean":"2023-08-28T02:32:59.642214+0000","last_became_active":"2023-08-28T02:32:58.635998+0000","last_became_peered":"2023-08-28T02:32:58.635998+0000","last_unstale":"2023-08-28T02:32:59.642214+0000","last_undegraded":"2023-08-28T02:32:59.642214+0000","last_fullsized":"2023-08-28T02:32:59.642214+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":181,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.637761+0000","last_change":"2023-08-28T02:31:55.525178+0000","last_active":"2023-08-28T02:32:59.637761+0000","last_peered":"2023-08-28T02:32:59.637761+0000","last_clean":"2023-08-28T02:32:59.637761+0000","last_became_active":"2023-08-28T02:31:55.524801+0000","last_became_peered":"2023-08-28T02:31:55.524801+0000","last_unstale":"2023-08-28T02:32:59.637761+0000","last_undegraded":"2023-08-28T02:32:59.637761+0000","last_fullsized":"2023-08-28T02:32:59.637761+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:47:30.078960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.11","version":"294'5567","reported_seq":5634,"reported_epoch":294,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:33:07.141926+0000","last_change":"2023-08-28T02:32:58.859410+0000","last_active":"2023-08-28T02:33:07.141926+0000","last_peered":"2023-08-28T02:33:07.141926+0000","last_clean":"2023-08-28T02:31:51.768660+0000","last_became_active":"2023-08-28T02:32:58.852854+0000","last_became_peered":"2023-08-28T02:32:58.852854+0000","last_unstale":"2023-08-28T02:33:07.141926+0000","last_undegraded":"2023-08-28T02:32:58.629343+0000","last_fullsized":"2023-08-28T02:32:58.617323+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5567,"log_dups_size":0,"ondisk_log_size":5567,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144474112,"num_objects":2205,"num_object_clones":0,"num_object_copies":6615,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2588,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2205,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3683,"num_write_kb":188384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3],"avail_no_missing":["6","3","4"],"object_location_counts":[{"shards":"2,3,6","objects":411},{"shards":"3,4,6","objects":1438},{"shards":"3,6","objects":180}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":60,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.638539+0000","last_change":"2023-08-28T02:31:55.532306+0000","last_active":"2023-08-28T02:32:59.638539+0000","last_peered":"2023-08-28T02:32:59.638539+0000","last_clean":"2023-08-28T02:32:59.638539+0000","last_became_active":"2023-08-28T02:31:55.524276+0000","last_became_peered":"2023-08-28T02:31:55.524276+0000","last_unstale":"2023-08-28T02:32:59.638539+0000","last_undegraded":"2023-08-28T02:32:59.638539+0000","last_fullsized":"2023-08-28T02:32:59.638539+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:27:06.850784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.634436+0000","last_change":"2023-08-28T02:32:58.658343+0000","last_active":"2023-08-28T02:32:59.634436+0000","last_peered":"2023-08-28T02:32:59.634436+0000","last_clean":"2023-08-28T02:32:59.634436+0000","last_became_active":"2023-08-28T02:32:58.654410+0000","last_became_peered":"2023-08-28T02:32:58.654410+0000","last_unstale":"2023-08-28T02:32:59.634436+0000","last_undegraded":"2023-08-28T02:32:59.634436+0000","last_fullsized":"2023-08-28T02:32:59.634436+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:46:29.260724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.10","version":"294'5404","reported_seq":7778,"reported_epoch":294,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:33:04.724122+0000","last_change":"2023-08-28T02:32:58.871328+0000","last_active":"2023-08-28T02:33:04.724122+0000","last_peered":"2023-08-28T02:33:04.724122+0000","last_clean":"2023-08-28T02:31:51.241963+0000","last_became_active":"2023-08-28T02:32:58.820349+0000","last_became_peered":"2023-08-28T02:32:58.820349+0000","last_unstale":"2023-08-28T02:33:04.724122+0000","last_undegraded":"2023-08-28T02:32:58.770460+0000","last_fullsized":"2023-08-28T02:32:58.618203+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5404,"log_dups_size":0,"ondisk_log_size":5404,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144211968,"num_objects":2201,"num_object_clones":0,"num_object_copies":6603,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1913,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2201,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3555,"num_write_kb":184160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1150,"num_bytes_recovered":75235328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"1,3,5","objects":973},{"shards":"2,3","objects":842},{"shards":"2,3,4","objects":386}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"86'22","reported_seq":363,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.638069+0000","last_change":"2023-08-28T02:31:55.538001+0000","last_active":"2023-08-28T02:32:59.638069+0000","last_peered":"2023-08-28T02:32:59.638069+0000","last_clean":"2023-08-28T02:32:59.638069+0000","last_became_active":"2023-08-28T02:31:55.518604+0000","last_became_peered":"2023-08-28T02:31:55.518604+0000","last_unstale":"2023-08-28T02:32:59.638069+0000","last_undegraded":"2023-08-28T02:32:59.638069+0000","last_fullsized":"2023-08-28T02:32:59.638069+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":337,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.637934+0000","last_change":"2023-08-28T02:32:58.629451+0000","last_active":"2023-08-28T02:32:59.637934+0000","last_peered":"2023-08-28T02:32:59.637934+0000","last_clean":"2023-08-28T02:32:59.637934+0000","last_became_active":"2023-08-28T02:32:58.628767+0000","last_became_peered":"2023-08-28T02:32:58.628767+0000","last_unstale":"2023-08-28T02:32:59.637934+0000","last_undegraded":"2023-08-28T02:32:59.637934+0000","last_fullsized":"2023-08-28T02:32:59.637934+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.17","version":"294'5398","reported_seq":5446,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-08-28T02:33:07.077584+0000","last_change":"2023-08-28T02:31:44.475304+0000","last_active":"2023-08-28T02:33:07.077584+0000","last_peered":"2023-08-28T02:33:07.077584+0000","last_clean":"2023-08-28T02:33:07.077584+0000","last_became_active":"2023-08-28T02:31:44.473853+0000","last_became_peered":"2023-08-28T02:31:44.473853+0000","last_unstale":"2023-08-28T02:33:07.077584+0000","last_undegraded":"2023-08-28T02:33:07.077584+0000","last_fullsized":"2023-08-28T02:33:07.077584+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5398,"log_dups_size":0,"ondisk_log_size":5398,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143785984,"num_objects":2194,"num_object_clones":0,"num_object_copies":6582,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2194,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3534,"num_write_kb":183296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":67,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.638037+0000","last_change":"2023-08-28T02:31:55.528552+0000","last_active":"2023-08-28T02:32:59.638037+0000","last_peered":"2023-08-28T02:32:59.638037+0000","last_clean":"2023-08-28T02:32:59.638037+0000","last_became_active":"2023-08-28T02:31:55.518079+0000","last_became_peered":"2023-08-28T02:31:55.518079+0000","last_unstale":"2023-08-28T02:32:59.638037+0000","last_undegraded":"2023-08-28T02:32:59.638037+0000","last_fullsized":"2023-08-28T02:32:59.638037+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":199,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.639109+0000","last_change":"2023-08-28T02:32:58.680534+0000","last_active":"2023-08-28T02:32:59.639109+0000","last_peered":"2023-08-28T02:32:59.639109+0000","last_clean":"2023-08-28T02:32:59.639109+0000","last_became_active":"2023-08-28T02:32:58.680145+0000","last_became_peered":"2023-08-28T02:32:58.680145+0000","last_unstale":"2023-08-28T02:32:59.639109+0000","last_undegraded":"2023-08-28T02:32:59.639109+0000","last_fullsized":"2023-08-28T02:32:59.639109+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:56:40.113013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.16","version":"294'5442","reported_seq":9732,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-08-28T02:33:09.225781+0000","last_change":"2023-08-28T02:32:37.600254+0000","last_active":"2023-08-28T02:33:09.225781+0000","last_peered":"2023-08-28T02:33:09.225781+0000","last_clean":"2023-08-28T02:33:09.225781+0000","last_became_active":"2023-08-28T02:32:37.596763+0000","last_became_peered":"2023-08-28T02:32:37.596763+0000","last_unstale":"2023-08-28T02:33:09.225781+0000","last_undegraded":"2023-08-28T02:33:09.225781+0000","last_fullsized":"2023-08-28T02:33:09.225781+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":270,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5442,"log_dups_size":0,"ondisk_log_size":5442,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134840320,"num_objects":2058,"num_object_clones":0,"num_object_copies":6174,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2058,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3636,"num_write_kb":182176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2111,"num_bytes_recovered":138346496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":308,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.636146+0000","last_change":"2023-08-28T02:32:58.746929+0000","last_active":"2023-08-28T02:32:59.636146+0000","last_peered":"2023-08-28T02:32:59.636146+0000","last_clean":"2023-08-28T02:32:59.636146+0000","last_became_active":"2023-08-28T02:32:58.746590+0000","last_became_peered":"2023-08-28T02:32:58.746590+0000","last_unstale":"2023-08-28T02:32:59.636146+0000","last_undegraded":"2023-08-28T02:32:59.636146+0000","last_fullsized":"2023-08-28T02:32:59.636146+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:51:11.974154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":329,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.641941+0000","last_change":"2023-08-28T02:31:55.521470+0000","last_active":"2023-08-28T02:32:59.641941+0000","last_peered":"2023-08-28T02:32:59.641941+0000","last_clean":"2023-08-28T02:32:59.641941+0000","last_became_active":"2023-08-28T02:31:55.520398+0000","last_became_peered":"2023-08-28T02:31:55.520398+0000","last_unstale":"2023-08-28T02:32:59.641941+0000","last_undegraded":"2023-08-28T02:32:59.641941+0000","last_fullsized":"2023-08-28T02:32:59.641941+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.15","version":"294'5357","reported_seq":5429,"reported_epoch":294,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:33:04.639922+0000","last_change":"2023-08-28T02:32:58.933245+0000","last_active":"2023-08-28T02:33:04.639922+0000","last_peered":"2023-08-28T02:33:04.639922+0000","last_clean":"2023-08-28T02:31:54.454430+0000","last_became_active":"2023-08-28T02:32:58.916111+0000","last_became_peered":"2023-08-28T02:32:58.916111+0000","last_unstale":"2023-08-28T02:33:04.639922+0000","last_undegraded":"2023-08-28T02:32:58.647695+0000","last_fullsized":"2023-08-28T02:32:58.611609+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5357,"log_dups_size":0,"ondisk_log_size":5357,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145915904,"num_objects":2227,"num_object_clones":0,"num_object_copies":6681,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2539,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2227,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3535,"num_write_kb":184352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":222},{"shards":"0,2,6","objects":392}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":123,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.637981+0000","last_change":"2023-08-28T02:31:55.517851+0000","last_active":"2023-08-28T02:32:59.637981+0000","last_peered":"2023-08-28T02:32:59.637981+0000","last_clean":"2023-08-28T02:32:59.637981+0000","last_became_active":"2023-08-28T02:31:55.517320+0000","last_became_peered":"2023-08-28T02:31:55.517320+0000","last_unstale":"2023-08-28T02:32:59.637981+0000","last_undegraded":"2023-08-28T02:32:59.637981+0000","last_fullsized":"2023-08-28T02:32:59.637981+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":291,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-08-28T02:33:00.662371+0000","last_change":"2023-08-28T02:31:55.528250+0000","last_active":"2023-08-28T02:33:00.662371+0000","last_peered":"2023-08-28T02:33:00.662371+0000","last_clean":"2023-08-28T02:33:00.662371+0000","last_became_active":"2023-08-28T02:31:55.527950+0000","last_became_peered":"2023-08-28T02:31:55.527950+0000","last_unstale":"2023-08-28T02:33:00.662371+0000","last_undegraded":"2023-08-28T02:33:00.662371+0000","last_fullsized":"2023-08-28T02:33:00.662371+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:39:46.120437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.14","version":"294'5689","reported_seq":5757,"reported_epoch":294,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:33:09.258343+0000","last_change":"2023-08-28T02:32:58.849338+0000","last_active":"2023-08-28T02:33:09.258343+0000","last_peered":"2023-08-28T02:33:09.258343+0000","last_clean":"2023-08-28T02:32:11.458469+0000","last_became_active":"2023-08-28T02:32:58.840304+0000","last_became_peered":"2023-08-28T02:32:58.840304+0000","last_unstale":"2023-08-28T02:33:09.258343+0000","last_undegraded":"2023-08-28T02:32:58.655685+0000","last_fullsized":"2023-08-28T02:32:58.616249+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5689,"log_dups_size":0,"ondisk_log_size":5689,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141852672,"num_objects":2165,"num_object_clones":0,"num_object_copies":6495,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":979,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2165,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3761,"num_write_kb":189600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":1748},{"shards":"0,4,6","objects":417}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":92,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.638600+0000","last_change":"2023-08-28T02:28:25.703333+0000","last_active":"2023-08-28T02:32:59.638600+0000","last_peered":"2023-08-28T02:32:59.638600+0000","last_clean":"2023-08-28T02:32:59.638600+0000","last_became_active":"2023-08-28T02:28:25.343515+0000","last_became_peered":"2023-08-28T02:28:25.343515+0000","last_unstale":"2023-08-28T02:32:59.638600+0000","last_undegraded":"2023-08-28T02:32:59.638600+0000","last_fullsized":"2023-08-28T02:32:59.638600+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":199,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.648964+0000","last_change":"2023-08-28T02:32:58.758016+0000","last_active":"2023-08-28T02:32:59.648964+0000","last_peered":"2023-08-28T02:32:59.648964+0000","last_clean":"2023-08-28T02:32:59.648964+0000","last_became_active":"2023-08-28T02:32:58.757660+0000","last_became_peered":"2023-08-28T02:32:58.757660+0000","last_unstale":"2023-08-28T02:32:59.648964+0000","last_undegraded":"2023-08-28T02:32:59.648964+0000","last_fullsized":"2023-08-28T02:32:59.648964+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:33:06.239503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"294'7136","reported_seq":7193,"reported_epoch":294,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:33:06.962388+0000","last_change":"2023-08-28T02:32:58.916501+0000","last_active":"2023-08-28T02:33:06.962388+0000","last_peered":"2023-08-28T02:33:06.962388+0000","last_clean":"2023-08-28T02:32:11.456138+0000","last_became_active":"2023-08-28T02:32:58.901683+0000","last_became_peered":"2023-08-28T02:32:58.901683+0000","last_unstale":"2023-08-28T02:33:06.962388+0000","last_undegraded":"2023-08-28T02:32:58.633417+0000","last_fullsized":"2023-08-28T02:32:58.615180+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":7136,"log_dups_size":0,"ondisk_log_size":7136,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":287178752,"num_objects":4382,"num_object_clones":0,"num_object_copies":13146,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1873,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4382,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7136,"num_write_kb":368576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"2,5,6","objects":836},{"shards":"5,6","objects":3546}],"blocked_by":[],"up_primary":1,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":155,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.640625+0000","last_change":"2023-08-28T02:32:58.782840+0000","last_active":"2023-08-28T02:32:59.640625+0000","last_peered":"2023-08-28T02:32:59.640625+0000","last_clean":"2023-08-28T02:32:59.640625+0000","last_became_active":"2023-08-28T02:32:58.683096+0000","last_became_peered":"2023-08-28T02:32:58.683096+0000","last_unstale":"2023-08-28T02:32:59.640625+0000","last_undegraded":"2023-08-28T02:32:59.640625+0000","last_fullsized":"2023-08-28T02:32:59.640625+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:10:08.037078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":289,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.642001+0000","last_change":"2023-08-28T02:31:55.514646+0000","last_active":"2023-08-28T02:32:59.642001+0000","last_peered":"2023-08-28T02:32:59.642001+0000","last_clean":"2023-08-28T02:32:59.642001+0000","last_became_active":"2023-08-28T02:31:55.514306+0000","last_became_peered":"2023-08-28T02:31:55.514306+0000","last_unstale":"2023-08-28T02:32:59.642001+0000","last_undegraded":"2023-08-28T02:32:59.642001+0000","last_fullsized":"2023-08-28T02:32:59.642001+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"294'7225","reported_seq":7267,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-08-28T02:33:04.733604+0000","last_change":"2023-08-28T02:30:57.067718+0000","last_active":"2023-08-28T02:33:04.733604+0000","last_peered":"2023-08-28T02:33:04.733604+0000","last_clean":"2023-08-28T02:33:04.733604+0000","last_became_active":"2023-08-28T02:30:57.067407+0000","last_became_peered":"2023-08-28T02:30:57.067407+0000","last_unstale":"2023-08-28T02:33:04.733604+0000","last_undegraded":"2023-08-28T02:33:04.733604+0000","last_fullsized":"2023-08-28T02:33:04.733604+0000","mapping_epoch":261,"log_start":"266'2065","ondisk_log_start":"266'2065","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5160,"log_dups_size":0,"ondisk_log_size":5160,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:49:39.360134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297861120,"num_objects":4545,"num_object_clones":0,"num_object_copies":13635,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4545,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7225,"num_write_kb":376640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":89,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.638116+0000","last_change":"2023-08-28T02:32:58.856906+0000","last_active":"2023-08-28T02:32:59.638116+0000","last_peered":"2023-08-28T02:32:59.638116+0000","last_clean":"2023-08-28T02:32:59.638116+0000","last_became_active":"2023-08-28T02:32:58.740028+0000","last_became_peered":"2023-08-28T02:32:58.740028+0000","last_unstale":"2023-08-28T02:32:59.638116+0000","last_undegraded":"2023-08-28T02:32:59.638116+0000","last_fullsized":"2023-08-28T02:32:59.638116+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":347,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.637770+0000","last_change":"2023-08-28T02:32:58.740396+0000","last_active":"2023-08-28T02:32:59.637770+0000","last_peered":"2023-08-28T02:32:59.637770+0000","last_clean":"2023-08-28T02:32:59.637770+0000","last_became_active":"2023-08-28T02:32:58.739941+0000","last_became_peered":"2023-08-28T02:32:58.739941+0000","last_unstale":"2023-08-28T02:32:59.637770+0000","last_undegraded":"2023-08-28T02:32:59.637770+0000","last_fullsized":"2023-08-28T02:32:59.637770+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"294'5613","reported_seq":5681,"reported_epoch":294,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:33:09.345914+0000","last_change":"2023-08-28T02:32:58.880642+0000","last_active":"2023-08-28T02:33:09.345914+0000","last_peered":"2023-08-28T02:33:09.345914+0000","last_clean":"2023-08-28T02:32:11.446297+0000","last_became_active":"2023-08-28T02:32:58.851158+0000","last_became_peered":"2023-08-28T02:32:58.851158+0000","last_unstale":"2023-08-28T02:33:09.345914+0000","last_undegraded":"2023-08-28T02:32:58.644337+0000","last_fullsized":"2023-08-28T02:32:58.616553+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5613,"log_dups_size":0,"ondisk_log_size":5613,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136740864,"num_objects":2086,"num_object_clones":0,"num_object_copies":6258,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1015,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2086,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3708,"num_write_kb":185440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":1684},{"shards":"0,4,6","objects":402}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":63,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.642030+0000","last_change":"2023-08-28T02:31:55.531216+0000","last_active":"2023-08-28T02:32:59.642030+0000","last_peered":"2023-08-28T02:32:59.642030+0000","last_clean":"2023-08-28T02:32:59.642030+0000","last_became_active":"2023-08-28T02:31:55.521010+0000","last_became_peered":"2023-08-28T02:31:55.521010+0000","last_unstale":"2023-08-28T02:32:59.642030+0000","last_undegraded":"2023-08-28T02:32:59.642030+0000","last_fullsized":"2023-08-28T02:32:59.642030+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":109,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.638603+0000","last_change":"2023-08-28T02:32:58.628915+0000","last_active":"2023-08-28T02:32:59.638603+0000","last_peered":"2023-08-28T02:32:59.638603+0000","last_clean":"2023-08-28T02:32:59.638603+0000","last_became_active":"2023-08-28T02:32:58.628562+0000","last_became_peered":"2023-08-28T02:32:58.628562+0000","last_unstale":"2023-08-28T02:32:59.638603+0000","last_undegraded":"2023-08-28T02:32:59.638603+0000","last_fullsized":"2023-08-28T02:32:59.638603+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"294'5550","reported_seq":8437,"reported_epoch":294,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:33:04.652190+0000","last_change":"2023-08-28T02:32:58.683607+0000","last_active":"2023-08-28T02:33:04.652190+0000","last_peered":"2023-08-28T02:33:04.652190+0000","last_clean":"2023-08-28T02:31:54.438195+0000","last_became_active":"2023-08-28T02:31:56.704078+0000","last_became_peered":"2023-08-28T02:31:56.704078+0000","last_unstale":"2023-08-28T02:33:04.652190+0000","last_undegraded":"2023-08-28T02:31:56.543040+0000","last_fullsized":"2023-08-28T02:31:56.504447+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5550,"log_dups_size":0,"ondisk_log_size":5550,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147062784,"num_objects":2244,"num_object_clones":0,"num_object_copies":6732,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":759,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2244,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3632,"num_write_kb":188032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1412,"num_bytes_recovered":92536832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,4","objects":1485}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":297,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.639229+0000","last_change":"2023-08-28T02:28:25.445982+0000","last_active":"2023-08-28T02:32:59.639229+0000","last_peered":"2023-08-28T02:32:59.639229+0000","last_clean":"2023-08-28T02:32:59.639229+0000","last_became_active":"2023-08-28T02:28:25.443577+0000","last_became_peered":"2023-08-28T02:28:25.443577+0000","last_unstale":"2023-08-28T02:32:59.639229+0000","last_undegraded":"2023-08-28T02:32:59.639229+0000","last_fullsized":"2023-08-28T02:32:59.639229+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:24:03.381751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":293,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.640287+0000","last_change":"2023-08-28T02:32:58.887874+0000","last_active":"2023-08-28T02:32:59.640287+0000","last_peered":"2023-08-28T02:32:59.640287+0000","last_clean":"2023-08-28T02:32:59.640287+0000","last_became_active":"2023-08-28T02:32:58.887377+0000","last_became_peered":"2023-08-28T02:32:58.887377+0000","last_unstale":"2023-08-28T02:32:59.640287+0000","last_undegraded":"2023-08-28T02:32:59.640287+0000","last_fullsized":"2023-08-28T02:32:59.640287+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:08:07.234984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"294'7432","reported_seq":8267,"reported_epoch":294,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-08-28T02:33:09.357679+0000","last_change":"2023-08-28T02:32:58.846760+0000","last_active":"2023-08-28T02:33:09.357679+0000","last_peered":"2023-08-28T02:33:09.357679+0000","last_clean":"2023-08-28T02:31:54.434903+0000","last_became_active":"2023-08-28T02:31:56.674791+0000","last_became_peered":"2023-08-28T02:31:56.674791+0000","last_unstale":"2023-08-28T02:33:09.357679+0000","last_undegraded":"2023-08-28T02:31:56.545322+0000","last_fullsized":"2023-08-28T02:31:56.508287+0000","mapping_epoch":277,"log_start":"266'1700","ondisk_log_start":"266'1700","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5732,"log_dups_size":0,"ondisk_log_size":5732,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:26:02.602584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":282591232,"num_objects":4312,"num_object_clones":0,"num_object_copies":12936,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3933,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4312,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7432,"num_write_kb":375808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":392,"num_bytes_recovered":25690112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":379}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":159,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.746728+0000","last_change":"2023-08-28T02:23:59.230059+0000","last_active":"2023-08-28T02:32:59.746728+0000","last_peered":"2023-08-28T02:32:59.746728+0000","last_clean":"2023-08-28T02:32:59.746728+0000","last_became_active":"2023-08-28T02:23:59.217783+0000","last_became_peered":"2023-08-28T02:23:59.217783+0000","last_unstale":"2023-08-28T02:32:59.746728+0000","last_undegraded":"2023-08-28T02:32:59.746728+0000","last_fullsized":"2023-08-28T02:32:59.746728+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:00:12.145395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":314,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.637974+0000","last_change":"2023-08-28T02:28:25.351698+0000","last_active":"2023-08-28T02:32:59.637974+0000","last_peered":"2023-08-28T02:32:59.637974+0000","last_clean":"2023-08-28T02:32:59.637974+0000","last_became_active":"2023-08-28T02:28:25.351349+0000","last_became_peered":"2023-08-28T02:28:25.351349+0000","last_unstale":"2023-08-28T02:32:59.637974+0000","last_undegraded":"2023-08-28T02:32:59.637974+0000","last_fullsized":"2023-08-28T02:32:59.637974+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"294'7172","reported_seq":7214,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-08-28T02:33:04.683712+0000","last_change":"2023-08-28T02:30:57.069579+0000","last_active":"2023-08-28T02:33:04.683712+0000","last_peered":"2023-08-28T02:33:04.683712+0000","last_clean":"2023-08-28T02:33:04.683712+0000","last_became_active":"2023-08-28T02:30:57.069365+0000","last_became_peered":"2023-08-28T02:30:57.069365+0000","last_unstale":"2023-08-28T02:33:04.683712+0000","last_undegraded":"2023-08-28T02:33:04.683712+0000","last_fullsized":"2023-08-28T02:33:04.683712+0000","mapping_epoch":261,"log_start":"265'600","ondisk_log_start":"265'600","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":6572,"log_dups_size":0,"ondisk_log_size":6572,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:02:07.549000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298057728,"num_objects":4548,"num_object_clones":0,"num_object_copies":13644,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4548,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7172,"num_write_kb":375040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":100,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.638987+0000","last_change":"2023-08-28T02:32:58.629249+0000","last_active":"2023-08-28T02:32:59.638987+0000","last_peered":"2023-08-28T02:32:59.638987+0000","last_clean":"2023-08-28T02:32:59.638987+0000","last_became_active":"2023-08-28T02:32:58.628583+0000","last_became_peered":"2023-08-28T02:32:58.628583+0000","last_unstale":"2023-08-28T02:32:59.638987+0000","last_undegraded":"2023-08-28T02:32:59.638987+0000","last_fullsized":"2023-08-28T02:32:59.638987+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":358,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.638099+0000","last_change":"2023-08-28T02:32:58.663953+0000","last_active":"2023-08-28T02:32:59.638099+0000","last_peered":"2023-08-28T02:32:59.638099+0000","last_clean":"2023-08-28T02:32:59.638099+0000","last_became_active":"2023-08-28T02:32:58.663627+0000","last_became_peered":"2023-08-28T02:32:58.663627+0000","last_unstale":"2023-08-28T02:32:59.638099+0000","last_undegraded":"2023-08-28T02:32:59.638099+0000","last_fullsized":"2023-08-28T02:32:59.638099+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"294'7026","reported_seq":7085,"reported_epoch":294,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:33:07.182918+0000","last_change":"2023-08-28T02:32:58.874546+0000","last_active":"2023-08-28T02:33:07.182918+0000","last_peered":"2023-08-28T02:33:07.182918+0000","last_clean":"2023-08-28T02:32:11.510497+0000","last_became_active":"2023-08-28T02:32:58.863497+0000","last_became_peered":"2023-08-28T02:32:58.863497+0000","last_unstale":"2023-08-28T02:33:07.182918+0000","last_undegraded":"2023-08-28T02:32:58.633053+0000","last_fullsized":"2023-08-28T02:32:58.617566+0000","mapping_epoch":291,"log_start":"265'1200","ondisk_log_start":"265'1200","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5826,"log_dups_size":0,"ondisk_log_size":5826,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":279773208,"num_objects":4270,"num_object_clones":0,"num_object_copies":12810,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1818,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4270,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7026,"num_write_kb":361409,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,3,6","objects":768},{"shards":"0,6","objects":3502}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":230,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.637876+0000","last_change":"2023-08-28T02:32:58.740311+0000","last_active":"2023-08-28T02:32:59.637876+0000","last_peered":"2023-08-28T02:32:59.637876+0000","last_clean":"2023-08-28T02:32:59.637876+0000","last_became_active":"2023-08-28T02:32:58.739897+0000","last_became_peered":"2023-08-28T02:32:58.739897+0000","last_unstale":"2023-08-28T02:32:59.637876+0000","last_undegraded":"2023-08-28T02:32:59.637876+0000","last_fullsized":"2023-08-28T02:32:59.637876+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":279,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.654393+0000","last_change":"2023-08-28T02:32:58.766795+0000","last_active":"2023-08-28T02:32:59.654393+0000","last_peered":"2023-08-28T02:32:59.654393+0000","last_clean":"2023-08-28T02:32:59.654393+0000","last_became_active":"2023-08-28T02:32:58.766441+0000","last_became_peered":"2023-08-28T02:32:58.766441+0000","last_unstale":"2023-08-28T02:32:59.654393+0000","last_undegraded":"2023-08-28T02:32:59.654393+0000","last_fullsized":"2023-08-28T02:32:59.654393+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:58:13.963961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.c","version":"294'7161","reported_seq":7209,"reported_epoch":294,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:33:07.166767+0000","last_change":"2023-08-28T02:31:56.737061+0000","last_active":"2023-08-28T02:33:07.166767+0000","last_peered":"2023-08-28T02:33:07.166767+0000","last_clean":"2023-08-28T02:31:54.457709+0000","last_became_active":"2023-08-28T02:31:56.726804+0000","last_became_peered":"2023-08-28T02:31:56.726804+0000","last_unstale":"2023-08-28T02:33:07.166767+0000","last_undegraded":"2023-08-28T02:31:56.538425+0000","last_fullsized":"2023-08-28T02:31:56.509601+0000","mapping_epoch":277,"log_start":"265'1300","ondisk_log_start":"265'1300","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5861,"log_dups_size":0,"ondisk_log_size":5861,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":277020672,"num_objects":4227,"num_object_clones":0,"num_object_copies":12681,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4274,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4227,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7161,"num_write_kb":364416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3],"avail_no_missing":["6","3"],"object_location_counts":[{"shards":"3,6","objects":-47}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":200,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.647831+0000","last_change":"2023-08-28T02:32:58.779257+0000","last_active":"2023-08-28T02:32:59.647831+0000","last_peered":"2023-08-28T02:32:59.647831+0000","last_clean":"2023-08-28T02:32:59.647831+0000","last_became_active":"2023-08-28T02:32:58.778295+0000","last_became_peered":"2023-08-28T02:32:58.778295+0000","last_unstale":"2023-08-28T02:32:59.647831+0000","last_undegraded":"2023-08-28T02:32:59.647831+0000","last_fullsized":"2023-08-28T02:32:59.647831+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:50:49.444715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.632407+0000","last_change":"2023-08-28T02:32:58.755765+0000","last_active":"2023-08-28T02:32:59.632407+0000","last_peered":"2023-08-28T02:32:59.632407+0000","last_clean":"2023-08-28T02:32:59.632407+0000","last_became_active":"2023-08-28T02:32:58.755438+0000","last_became_peered":"2023-08-28T02:32:58.755438+0000","last_unstale":"2023-08-28T02:32:59.632407+0000","last_undegraded":"2023-08-28T02:32:59.632407+0000","last_fullsized":"2023-08-28T02:32:59.632407+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:14:11.498611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"294'5459","reported_seq":5513,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-08-28T02:33:06.875251+0000","last_change":"2023-08-28T02:32:14.983506+0000","last_active":"2023-08-28T02:33:06.875251+0000","last_peered":"2023-08-28T02:33:06.875251+0000","last_clean":"2023-08-28T02:33:06.875251+0000","last_became_active":"2023-08-28T02:31:44.479295+0000","last_became_peered":"2023-08-28T02:31:44.479295+0000","last_unstale":"2023-08-28T02:33:06.875251+0000","last_undegraded":"2023-08-28T02:33:06.875251+0000","last_fullsized":"2023-08-28T02:33:06.875251+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5459,"log_dups_size":0,"ondisk_log_size":5459,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144998400,"num_objects":2212,"num_object_clones":0,"num_object_copies":6636,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2212,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3574,"num_write_kb":185184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"78'126","reported_seq":671,"reported_epoch":293,"state":"active+recovery_wait+degraded","last_fresh":"2023-08-28T02:32:59.660815+0000","last_change":"2023-08-28T02:32:59.660815+0000","last_active":"2023-08-28T02:32:59.660815+0000","last_peered":"2023-08-28T02:32:59.660815+0000","last_clean":"2023-08-28T02:27:29.502868+0000","last_became_active":"2023-08-28T02:32:59.660388+0000","last_became_peered":"2023-08-28T02:32:59.660388+0000","last_unstale":"2023-08-28T02:32:59.660815+0000","last_undegraded":"2023-08-28T02:32:59.633238+0000","last_fullsized":"2023-08-28T02:32:59.660815+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:58:54.023571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":743,"num_read_kb":1331,"num_write":125,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":["1","0"],"object_location_counts":[{"shards":"0,1","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":293,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.641862+0000","last_change":"2023-08-28T02:23:59.207148+0000","last_active":"2023-08-28T02:32:59.641862+0000","last_peered":"2023-08-28T02:32:59.641862+0000","last_clean":"2023-08-28T02:32:59.641862+0000","last_became_active":"2023-08-28T02:23:59.206802+0000","last_became_peered":"2023-08-28T02:23:59.206802+0000","last_unstale":"2023-08-28T02:32:59.641862+0000","last_undegraded":"2023-08-28T02:32:59.641862+0000","last_fullsized":"2023-08-28T02:32:59.641862+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.633890+0000","last_change":"2023-08-28T02:32:58.626357+0000","last_active":"2023-08-28T02:32:59.633890+0000","last_peered":"2023-08-28T02:32:59.633890+0000","last_clean":"2023-08-28T02:32:59.633890+0000","last_became_active":"2023-08-28T02:32:58.622671+0000","last_became_peered":"2023-08-28T02:32:58.622671+0000","last_unstale":"2023-08-28T02:32:59.633890+0000","last_undegraded":"2023-08-28T02:32:59.633890+0000","last_fullsized":"2023-08-28T02:32:59.633890+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:21:30.262904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"294'5479","reported_seq":5528,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-08-28T02:33:07.077806+0000","last_change":"2023-08-28T02:31:44.472455+0000","last_active":"2023-08-28T02:33:07.077806+0000","last_peered":"2023-08-28T02:33:07.077806+0000","last_clean":"2023-08-28T02:33:07.077806+0000","last_became_active":"2023-08-28T02:31:44.470716+0000","last_became_peered":"2023-08-28T02:31:44.470716+0000","last_unstale":"2023-08-28T02:33:07.077806+0000","last_undegraded":"2023-08-28T02:33:07.077806+0000","last_fullsized":"2023-08-28T02:33:07.077806+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5479,"log_dups_size":0,"ondisk_log_size":5479,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138215424,"num_objects":2109,"num_object_clones":0,"num_object_copies":6327,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2109,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3615,"num_write_kb":183168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"78'204","reported_seq":1377,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.650361+0000","last_change":"2023-08-28T02:32:59.644574+0000","last_active":"2023-08-28T02:32:59.650361+0000","last_peered":"2023-08-28T02:32:59.650361+0000","last_clean":"2023-08-28T02:32:59.650361+0000","last_became_active":"2023-08-28T02:32:59.644053+0000","last_became_peered":"2023-08-28T02:32:59.644053+0000","last_unstale":"2023-08-28T02:32:59.650361+0000","last_undegraded":"2023-08-28T02:32:59.650361+0000","last_fullsized":"2023-08-28T02:32:59.650361+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":204,"log_dups_size":0,"ondisk_log_size":204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:55:51.460483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1521,"num_read_kb":2511,"num_write":178,"num_write_kb":1175,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":288,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.638122+0000","last_change":"2023-08-28T02:28:25.352417+0000","last_active":"2023-08-28T02:32:59.638122+0000","last_peered":"2023-08-28T02:32:59.638122+0000","last_clean":"2023-08-28T02:32:59.638122+0000","last_became_active":"2023-08-28T02:28:25.352216+0000","last_became_peered":"2023-08-28T02:28:25.352216+0000","last_unstale":"2023-08-28T02:32:59.638122+0000","last_undegraded":"2023-08-28T02:32:59.638122+0000","last_fullsized":"2023-08-28T02:32:59.638122+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.632859+0000","last_change":"2023-08-28T02:32:58.720576+0000","last_active":"2023-08-28T02:32:59.632859+0000","last_peered":"2023-08-28T02:32:59.632859+0000","last_clean":"2023-08-28T02:32:59.632859+0000","last_became_active":"2023-08-28T02:32:58.719924+0000","last_became_peered":"2023-08-28T02:32:58.719924+0000","last_unstale":"2023-08-28T02:32:59.632859+0000","last_undegraded":"2023-08-28T02:32:59.632859+0000","last_fullsized":"2023-08-28T02:32:59.632859+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:28:50.697438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"294'5724","reported_seq":5794,"reported_epoch":294,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:33:09.345968+0000","last_change":"2023-08-28T02:32:58.870443+0000","last_active":"2023-08-28T02:33:09.345968+0000","last_peered":"2023-08-28T02:33:09.345968+0000","last_clean":"2023-08-28T02:32:11.480338+0000","last_became_active":"2023-08-28T02:32:58.848058+0000","last_became_peered":"2023-08-28T02:32:58.848058+0000","last_unstale":"2023-08-28T02:33:09.345968+0000","last_undegraded":"2023-08-28T02:32:58.664973+0000","last_fullsized":"2023-08-28T02:32:58.616174+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5724,"log_dups_size":0,"ondisk_log_size":5724,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139689984,"num_objects":2131,"num_object_clones":0,"num_object_copies":6393,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1029,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2131,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3795,"num_write_kb":189664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":1711},{"shards":"0,4,6","objects":420}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"78'47","reported_seq":512,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.639857+0000","last_change":"2023-08-28T02:24:05.799647+0000","last_active":"2023-08-28T02:32:59.639857+0000","last_peered":"2023-08-28T02:32:59.639857+0000","last_clean":"2023-08-28T02:32:59.639857+0000","last_became_active":"2023-08-28T02:23:59.210481+0000","last_became_peered":"2023-08-28T02:23:59.210481+0000","last_unstale":"2023-08-28T02:32:59.639857+0000","last_undegraded":"2023-08-28T02:32:59.639857+0000","last_fullsized":"2023-08-28T02:32:59.639857+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":47,"log_dups_size":0,"ondisk_log_size":47,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:40:18.542375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":162,"num_read_kb":1281,"num_write":48,"num_write_kb":1153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":13,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.632174+0000","last_change":"2023-08-28T02:32:58.622079+0000","last_active":"2023-08-28T02:32:59.632174+0000","last_peered":"2023-08-28T02:32:59.632174+0000","last_clean":"2023-08-28T02:32:59.632174+0000","last_became_active":"2023-08-28T02:32:58.621819+0000","last_became_peered":"2023-08-28T02:32:58.621819+0000","last_unstale":"2023-08-28T02:32:59.632174+0000","last_undegraded":"2023-08-28T02:32:59.632174+0000","last_fullsized":"2023-08-28T02:32:59.632174+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:03:36.676500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.632328+0000","last_change":"2023-08-28T02:32:58.724424+0000","last_active":"2023-08-28T02:32:59.632328+0000","last_peered":"2023-08-28T02:32:59.632328+0000","last_clean":"2023-08-28T02:32:59.632328+0000","last_became_active":"2023-08-28T02:32:58.724127+0000","last_became_peered":"2023-08-28T02:32:58.724127+0000","last_unstale":"2023-08-28T02:32:59.632328+0000","last_undegraded":"2023-08-28T02:32:59.632328+0000","last_fullsized":"2023-08-28T02:32:59.632328+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:02:25.178712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"294'5366","reported_seq":5432,"reported_epoch":294,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:33:09.363155+0000","last_change":"2023-08-28T02:32:12.589239+0000","last_active":"2023-08-28T02:33:09.363155+0000","last_peered":"2023-08-28T02:33:09.363155+0000","last_clean":"2023-08-28T02:31:54.462738+0000","last_became_active":"2023-08-28T02:31:56.607923+0000","last_became_peered":"2023-08-28T02:31:56.607923+0000","last_unstale":"2023-08-28T02:33:09.363155+0000","last_undegraded":"2023-08-28T02:32:12.587852+0000","last_fullsized":"2023-08-28T02:31:56.514083+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5366,"log_dups_size":0,"ondisk_log_size":5366,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135888896,"num_objects":2073,"num_object_clones":0,"num_object_copies":6219,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2060,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2073,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3559,"num_write_kb":180256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":13}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"78'114","reported_seq":690,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.641806+0000","last_change":"2023-08-28T02:32:14.626614+0000","last_active":"2023-08-28T02:32:59.641806+0000","last_peered":"2023-08-28T02:32:59.641806+0000","last_clean":"2023-08-28T02:32:59.641806+0000","last_became_active":"2023-08-28T02:32:14.626181+0000","last_became_peered":"2023-08-28T02:32:14.626181+0000","last_unstale":"2023-08-28T02:32:59.641806+0000","last_undegraded":"2023-08-28T02:32:59.641806+0000","last_fullsized":"2023-08-28T02:32:59.641806+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":114,"log_dups_size":0,"ondisk_log_size":114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":358,"num_read_kb":1352,"num_write":102,"num_write_kb":1104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"86'139","reported_seq":369,"reported_epoch":293,"state":"active+recovery_wait+degraded","last_fresh":"2023-08-28T02:32:59.651360+0000","last_change":"2023-08-28T02:32:59.651360+0000","last_active":"2023-08-28T02:32:59.651360+0000","last_peered":"2023-08-28T02:32:59.651360+0000","last_clean":"2023-08-28T02:23:32.292225+0000","last_became_active":"2023-08-28T02:32:59.651089+0000","last_became_peered":"2023-08-28T02:32:59.651089+0000","last_unstale":"2023-08-28T02:32:59.651360+0000","last_undegraded":"2023-08-28T02:32:59.636553+0000","last_fullsized":"2023-08-28T02:32:59.651360+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_clean_scrub_stamp":"2023-08-28T02:08:02.043718+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:12:30.956176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":["0","6"],"object_location_counts":[{"shards":"0,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":173,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.638795+0000","last_change":"2023-08-28T02:28:25.347600+0000","last_active":"2023-08-28T02:32:59.638795+0000","last_peered":"2023-08-28T02:32:59.638795+0000","last_clean":"2023-08-28T02:32:59.638795+0000","last_became_active":"2023-08-28T02:28:25.347237+0000","last_became_peered":"2023-08-28T02:28:25.347237+0000","last_unstale":"2023-08-28T02:32:59.638795+0000","last_undegraded":"2023-08-28T02:32:59.638795+0000","last_fullsized":"2023-08-28T02:32:59.638795+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:29:26.767559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":91,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.640123+0000","last_change":"2023-08-28T02:31:55.517732+0000","last_active":"2023-08-28T02:32:59.640123+0000","last_peered":"2023-08-28T02:32:59.640123+0000","last_clean":"2023-08-28T02:32:59.640123+0000","last_became_active":"2023-08-28T02:31:55.517347+0000","last_became_peered":"2023-08-28T02:31:55.517347+0000","last_unstale":"2023-08-28T02:32:59.640123+0000","last_undegraded":"2023-08-28T02:32:59.640123+0000","last_fullsized":"2023-08-28T02:32:59.640123+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"294'5306","reported_seq":5379,"reported_epoch":294,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:33:04.620511+0000","last_change":"2023-08-28T02:32:58.875577+0000","last_active":"2023-08-28T02:33:04.620511+0000","last_peered":"2023-08-28T02:33:04.620511+0000","last_clean":"2023-08-28T02:31:54.356655+0000","last_became_active":"2023-08-28T02:32:58.865219+0000","last_became_peered":"2023-08-28T02:32:58.865219+0000","last_unstale":"2023-08-28T02:33:04.620511+0000","last_undegraded":"2023-08-28T02:32:58.625567+0000","last_fullsized":"2023-08-28T02:32:58.612068+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5306,"log_dups_size":0,"ondisk_log_size":5306,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145981440,"num_objects":2227,"num_object_clones":0,"num_object_copies":6681,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2532,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2227,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3483,"num_write_kb":182752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":200},{"shards":"0,2,6","objects":401}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"78'148","reported_seq":721,"reported_epoch":293,"state":"active+recovery_wait+degraded","last_fresh":"2023-08-28T02:32:59.661621+0000","last_change":"2023-08-28T02:32:59.661621+0000","last_active":"2023-08-28T02:32:59.661621+0000","last_peered":"2023-08-28T02:32:59.661621+0000","last_clean":"2023-08-28T02:12:46.181557+0000","last_became_active":"2023-08-28T02:32:59.661273+0000","last_became_peered":"2023-08-28T02:32:59.661273+0000","last_unstale":"2023-08-28T02:32:59.661621+0000","last_undegraded":"2023-08-28T02:32:59.632715+0000","last_fullsized":"2023-08-28T02:32:59.661621+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":244,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:58:40.796785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1092,"num_read_kb":1955,"num_write":127,"num_write_kb":5177,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":["1","0"],"object_location_counts":[{"shards":"0,1","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":263,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.638427+0000","last_change":"2023-08-28T02:23:59.226986+0000","last_active":"2023-08-28T02:32:59.638427+0000","last_peered":"2023-08-28T02:32:59.638427+0000","last_clean":"2023-08-28T02:32:59.638427+0000","last_became_active":"2023-08-28T02:23:59.226529+0000","last_became_peered":"2023-08-28T02:23:59.226529+0000","last_unstale":"2023-08-28T02:32:59.638427+0000","last_undegraded":"2023-08-28T02:32:59.638427+0000","last_fullsized":"2023-08-28T02:32:59.638427+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:22:09.193619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":326,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.648882+0000","last_change":"2023-08-28T02:32:58.744332+0000","last_active":"2023-08-28T02:32:59.648882+0000","last_peered":"2023-08-28T02:32:59.648882+0000","last_clean":"2023-08-28T02:32:59.648882+0000","last_became_active":"2023-08-28T02:32:58.744025+0000","last_became_peered":"2023-08-28T02:32:58.744025+0000","last_unstale":"2023-08-28T02:32:59.648882+0000","last_undegraded":"2023-08-28T02:32:59.648882+0000","last_fullsized":"2023-08-28T02:32:59.648882+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:41:15.957049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"294'5467","reported_seq":5533,"reported_epoch":294,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:33:06.890475+0000","last_change":"2023-08-28T02:32:58.811565+0000","last_active":"2023-08-28T02:33:06.890475+0000","last_peered":"2023-08-28T02:33:06.890475+0000","last_clean":"2023-08-28T02:32:11.496413+0000","last_became_active":"2023-08-28T02:32:58.798063+0000","last_became_peered":"2023-08-28T02:32:58.798063+0000","last_unstale":"2023-08-28T02:33:06.890475+0000","last_undegraded":"2023-08-28T02:32:58.623060+0000","last_fullsized":"2023-08-28T02:32:58.615162+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5467,"log_dups_size":0,"ondisk_log_size":5467,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148471808,"num_objects":2265,"num_object_clones":0,"num_object_copies":6795,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":911,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2265,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3617,"num_write_kb":188256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"2,3,5","objects":379},{"shards":"3,5","objects":1886}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"78'103","reported_seq":603,"reported_epoch":293,"state":"active+recovery_wait+degraded","last_fresh":"2023-08-28T02:32:59.656648+0000","last_change":"2023-08-28T02:32:59.656648+0000","last_active":"2023-08-28T02:32:59.656648+0000","last_peered":"2023-08-28T02:32:59.656648+0000","last_clean":"2023-08-28T02:23:32.292540+0000","last_became_active":"2023-08-28T02:32:59.656276+0000","last_became_peered":"2023-08-28T02:32:59.656276+0000","last_unstale":"2023-08-28T02:32:59.656648+0000","last_undegraded":"2023-08-28T02:32:59.643455+0000","last_fullsized":"2023-08-28T02:32:59.656648+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:54:56.658032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":803,"num_read_kb":2080,"num_write":73,"num_write_kb":1244,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":13,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.632786+0000","last_change":"2023-08-28T02:32:58.720401+0000","last_active":"2023-08-28T02:32:59.632786+0000","last_peered":"2023-08-28T02:32:59.632786+0000","last_clean":"2023-08-28T02:32:59.632786+0000","last_became_active":"2023-08-28T02:32:58.719518+0000","last_became_peered":"2023-08-28T02:32:58.719518+0000","last_unstale":"2023-08-28T02:32:59.632786+0000","last_undegraded":"2023-08-28T02:32:59.632786+0000","last_fullsized":"2023-08-28T02:32:59.632786+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:46:28.947812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":276,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.640523+0000","last_change":"2023-08-28T02:32:58.887691+0000","last_active":"2023-08-28T02:32:59.640523+0000","last_peered":"2023-08-28T02:32:59.640523+0000","last_clean":"2023-08-28T02:32:59.640523+0000","last_became_active":"2023-08-28T02:32:58.886906+0000","last_became_peered":"2023-08-28T02:32:58.886906+0000","last_unstale":"2023-08-28T02:32:59.640523+0000","last_undegraded":"2023-08-28T02:32:59.640523+0000","last_fullsized":"2023-08-28T02:32:59.640523+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:39:29.229171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"294'5422","reported_seq":5475,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-08-28T02:33:04.747121+0000","last_change":"2023-08-28T02:31:44.467379+0000","last_active":"2023-08-28T02:33:04.747121+0000","last_peered":"2023-08-28T02:33:04.747121+0000","last_clean":"2023-08-28T02:33:04.747121+0000","last_became_active":"2023-08-28T02:31:44.444605+0000","last_became_peered":"2023-08-28T02:31:44.444605+0000","last_unstale":"2023-08-28T02:33:04.747121+0000","last_undegraded":"2023-08-28T02:33:04.747121+0000","last_fullsized":"2023-08-28T02:33:04.747121+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5422,"log_dups_size":0,"ondisk_log_size":5422,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142475264,"num_objects":2174,"num_object_clones":0,"num_object_copies":6522,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2174,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3552,"num_write_kb":183232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":172,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.649065+0000","last_change":"2023-08-28T02:31:55.535099+0000","last_active":"2023-08-28T02:32:59.649065+0000","last_peered":"2023-08-28T02:32:59.649065+0000","last_clean":"2023-08-28T02:32:59.649065+0000","last_became_active":"2023-08-28T02:31:55.527768+0000","last_became_peered":"2023-08-28T02:31:55.527768+0000","last_unstale":"2023-08-28T02:32:59.649065+0000","last_undegraded":"2023-08-28T02:32:59.649065+0000","last_fullsized":"2023-08-28T02:32:59.649065+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:43:30.804793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'112","reported_seq":446,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.647397+0000","last_change":"2023-08-28T02:32:59.643465+0000","last_active":"2023-08-28T02:32:59.647397+0000","last_peered":"2023-08-28T02:32:59.647397+0000","last_clean":"2023-08-28T02:32:59.647397+0000","last_became_active":"2023-08-28T02:32:59.642979+0000","last_became_peered":"2023-08-28T02:32:59.642979+0000","last_unstale":"2023-08-28T02:32:59.647397+0000","last_undegraded":"2023-08-28T02:32:59.647397+0000","last_fullsized":"2023-08-28T02:32:59.647397+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:48:45.923015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":444,"num_read_kb":473,"num_write":94,"num_write_kb":127,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":182,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.647791+0000","last_change":"2023-08-28T02:32:58.833408+0000","last_active":"2023-08-28T02:32:59.647791+0000","last_peered":"2023-08-28T02:32:59.647791+0000","last_clean":"2023-08-28T02:32:59.647791+0000","last_became_active":"2023-08-28T02:32:58.821347+0000","last_became_peered":"2023-08-28T02:32:58.821347+0000","last_unstale":"2023-08-28T02:32:59.647791+0000","last_undegraded":"2023-08-28T02:32:59.647791+0000","last_fullsized":"2023-08-28T02:32:59.647791+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:41:25.683179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"294'5506","reported_seq":5574,"reported_epoch":294,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:33:04.748536+0000","last_change":"2023-08-28T02:32:58.866701+0000","last_active":"2023-08-28T02:33:04.748536+0000","last_peered":"2023-08-28T02:33:04.748536+0000","last_clean":"2023-08-28T02:32:11.480934+0000","last_became_active":"2023-08-28T02:32:58.858761+0000","last_became_peered":"2023-08-28T02:32:58.858761+0000","last_unstale":"2023-08-28T02:33:04.748536+0000","last_undegraded":"2023-08-28T02:32:58.628529+0000","last_fullsized":"2023-08-28T02:32:58.617723+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5506,"log_dups_size":0,"ondisk_log_size":5506,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148176896,"num_objects":2261,"num_object_clones":0,"num_object_copies":6783,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":937,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2261,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3601,"num_write_kb":187584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[{"shards":"2,3","objects":1876},{"shards":"2,3,6","objects":385}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"78'122","reported_seq":637,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-08-28T02:33:00.672889+0000","last_change":"2023-08-28T02:33:00.672889+0000","last_active":"2023-08-28T02:33:00.672889+0000","last_peered":"2023-08-28T02:33:00.672889+0000","last_clean":"2023-08-28T02:33:00.672889+0000","last_became_active":"2023-08-28T02:33:00.672494+0000","last_became_peered":"2023-08-28T02:33:00.672494+0000","last_unstale":"2023-08-28T02:33:00.672889+0000","last_undegraded":"2023-08-28T02:33:00.672889+0000","last_fullsized":"2023-08-28T02:33:00.672889+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":857,"num_read_kb":2224,"num_write":99,"num_write_kb":2312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":301,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.652636+0000","last_change":"2023-08-28T02:32:58.795995+0000","last_active":"2023-08-28T02:32:59.652636+0000","last_peered":"2023-08-28T02:32:59.652636+0000","last_clean":"2023-08-28T02:32:59.652636+0000","last_became_active":"2023-08-28T02:32:58.771872+0000","last_became_peered":"2023-08-28T02:32:58.771872+0000","last_unstale":"2023-08-28T02:32:59.652636+0000","last_undegraded":"2023-08-28T02:32:59.652636+0000","last_fullsized":"2023-08-28T02:32:59.652636+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:16:02.965467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":263,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.637658+0000","last_change":"2023-08-28T02:28:25.351840+0000","last_active":"2023-08-28T02:32:59.637658+0000","last_peered":"2023-08-28T02:32:59.637658+0000","last_clean":"2023-08-28T02:32:59.637658+0000","last_became_active":"2023-08-28T02:28:25.351399+0000","last_became_peered":"2023-08-28T02:28:25.351399+0000","last_unstale":"2023-08-28T02:32:59.637658+0000","last_undegraded":"2023-08-28T02:32:59.637658+0000","last_fullsized":"2023-08-28T02:32:59.637658+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":278,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.640447+0000","last_change":"2023-08-28T02:32:58.751818+0000","last_active":"2023-08-28T02:32:59.640447+0000","last_peered":"2023-08-28T02:32:59.640447+0000","last_clean":"2023-08-28T02:32:59.640447+0000","last_became_active":"2023-08-28T02:32:58.743388+0000","last_became_peered":"2023-08-28T02:32:58.743388+0000","last_unstale":"2023-08-28T02:32:59.640447+0000","last_undegraded":"2023-08-28T02:32:59.640447+0000","last_fullsized":"2023-08-28T02:32:59.640447+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:52:12.910174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":363,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.641587+0000","last_change":"2023-08-28T02:32:58.655915+0000","last_active":"2023-08-28T02:32:59.641587+0000","last_peered":"2023-08-28T02:32:59.641587+0000","last_clean":"2023-08-28T02:32:59.641587+0000","last_became_active":"2023-08-28T02:32:58.655657+0000","last_became_peered":"2023-08-28T02:32:58.655657+0000","last_unstale":"2023-08-28T02:32:59.641587+0000","last_undegraded":"2023-08-28T02:32:59.641587+0000","last_fullsized":"2023-08-28T02:32:59.641587+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:14:33.807968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.18","version":"294'5586","reported_seq":7193,"reported_epoch":294,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:33:04.692164+0000","last_change":"2023-08-28T02:32:12.557727+0000","last_active":"2023-08-28T02:33:04.692164+0000","last_peered":"2023-08-28T02:33:04.692164+0000","last_clean":"2023-08-28T02:31:54.412195+0000","last_became_active":"2023-08-28T02:31:56.623609+0000","last_became_peered":"2023-08-28T02:31:56.623609+0000","last_unstale":"2023-08-28T02:33:04.692164+0000","last_undegraded":"2023-08-28T02:32:12.554864+0000","last_fullsized":"2023-08-28T02:31:56.505024+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5586,"log_dups_size":0,"ondisk_log_size":5586,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147193856,"num_objects":2246,"num_object_clones":0,"num_object_copies":6738,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1472,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2246,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3668,"num_write_kb":189248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":772,"num_bytes_recovered":50593792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,4","objects":774}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":126,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.637946+0000","last_change":"2023-08-28T02:32:58.839847+0000","last_active":"2023-08-28T02:32:59.637946+0000","last_peered":"2023-08-28T02:32:59.637946+0000","last_clean":"2023-08-28T02:32:59.637946+0000","last_became_active":"2023-08-28T02:32:58.628159+0000","last_became_peered":"2023-08-28T02:32:58.628159+0000","last_unstale":"2023-08-28T02:32:59.637946+0000","last_undegraded":"2023-08-28T02:32:59.637946+0000","last_fullsized":"2023-08-28T02:32:59.637946+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":329,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.638266+0000","last_change":"2023-08-28T02:32:58.629596+0000","last_active":"2023-08-28T02:32:59.638266+0000","last_peered":"2023-08-28T02:32:59.638266+0000","last_clean":"2023-08-28T02:32:59.638266+0000","last_became_active":"2023-08-28T02:32:58.629136+0000","last_became_peered":"2023-08-28T02:32:58.629136+0000","last_unstale":"2023-08-28T02:32:59.638266+0000","last_undegraded":"2023-08-28T02:32:59.638266+0000","last_fullsized":"2023-08-28T02:32:59.638266+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.19","version":"294'5571","reported_seq":5639,"reported_epoch":294,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:33:09.281087+0000","last_change":"2023-08-28T02:32:58.908496+0000","last_active":"2023-08-28T02:33:09.281087+0000","last_peered":"2023-08-28T02:33:09.281087+0000","last_clean":"2023-08-28T02:32:11.482841+0000","last_became_active":"2023-08-28T02:32:58.900137+0000","last_became_peered":"2023-08-28T02:32:58.900137+0000","last_unstale":"2023-08-28T02:33:09.281087+0000","last_undegraded":"2023-08-28T02:32:58.650852+0000","last_fullsized":"2023-08-28T02:32:58.621843+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5571,"log_dups_size":0,"ondisk_log_size":5571,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142508032,"num_objects":2175,"num_object_clones":0,"num_object_copies":6525,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":919,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2175,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3667,"num_write_kb":186912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":1777},{"shards":"0,4,6","objects":398}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":113,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.638205+0000","last_change":"2023-08-28T02:28:25.705331+0000","last_active":"2023-08-28T02:32:59.638205+0000","last_peered":"2023-08-28T02:32:59.638205+0000","last_clean":"2023-08-28T02:32:59.638205+0000","last_became_active":"2023-08-28T02:28:25.397480+0000","last_became_peered":"2023-08-28T02:28:25.397480+0000","last_unstale":"2023-08-28T02:32:59.638205+0000","last_undegraded":"2023-08-28T02:32:59.638205+0000","last_fullsized":"2023-08-28T02:32:59.638205+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":293,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.637672+0000","last_change":"2023-08-28T02:23:59.207929+0000","last_active":"2023-08-28T02:32:59.637672+0000","last_peered":"2023-08-28T02:32:59.637672+0000","last_clean":"2023-08-28T02:32:59.637672+0000","last_became_active":"2023-08-28T02:23:59.207589+0000","last_became_peered":"2023-08-28T02:23:59.207589+0000","last_unstale":"2023-08-28T02:32:59.637672+0000","last_undegraded":"2023-08-28T02:32:59.637672+0000","last_fullsized":"2023-08-28T02:32:59.637672+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":182,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.648497+0000","last_change":"2023-08-28T02:28:25.347988+0000","last_active":"2023-08-28T02:32:59.648497+0000","last_peered":"2023-08-28T02:32:59.648497+0000","last_clean":"2023-08-28T02:32:59.648497+0000","last_became_active":"2023-08-28T02:28:25.347632+0000","last_became_peered":"2023-08-28T02:28:25.347632+0000","last_unstale":"2023-08-28T02:32:59.648497+0000","last_undegraded":"2023-08-28T02:32:59.648497+0000","last_fullsized":"2023-08-28T02:32:59.648497+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:56:20.202662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":154,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.637825+0000","last_change":"2023-08-28T02:23:59.217794+0000","last_active":"2023-08-28T02:32:59.637825+0000","last_peered":"2023-08-28T02:32:59.637825+0000","last_clean":"2023-08-28T02:32:59.637825+0000","last_became_active":"2023-08-28T02:23:59.217231+0000","last_became_peered":"2023-08-28T02:23:59.217231+0000","last_unstale":"2023-08-28T02:32:59.637825+0000","last_undegraded":"2023-08-28T02:32:59.637825+0000","last_fullsized":"2023-08-28T02:32:59.637825+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:36:26.297410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":249,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.642452+0000","last_change":"2023-08-28T02:20:19.850252+0000","last_active":"2023-08-28T02:32:59.642452+0000","last_peered":"2023-08-28T02:32:59.642452+0000","last_clean":"2023-08-28T02:32:59.642452+0000","last_became_active":"2023-08-28T02:20:19.849913+0000","last_became_peered":"2023-08-28T02:20:19.849913+0000","last_unstale":"2023-08-28T02:32:59.642452+0000","last_undegraded":"2023-08-28T02:32:59.642452+0000","last_fullsized":"2023-08-28T02:32:59.642452+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:48:47.713465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":352,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-08-28T02:32:59.637579+0000","last_change":"2023-08-28T02:32:58.630270+0000","last_active":"2023-08-28T02:32:59.637579+0000","last_peered":"2023-08-28T02:32:59.637579+0000","last_clean":"2023-08-28T02:32:59.637579+0000","last_became_active":"2023-08-28T02:32:58.629958+0000","last_became_peered":"2023-08-28T02:32:58.629958+0000","last_unstale":"2023-08-28T02:32:59.637579+0000","last_undegraded":"2023-08-28T02:32:59.637579+0000","last_fullsized":"2023-08-28T02:32:59.637579+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":26,"stat_sum":{"num_bytes":4589617176,"num_objects":70033,"num_object_clones":0,"num_object_copies":210099,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":32521,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":70033,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":115503,"num_write_kb":5937089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5842,"num_bytes_recovered":382599168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14976561152,"data_stored":14976548936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":146144,"ondisk_log_size":146144,"up":78,"acting":60,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":58,"num_bytes_recovered":24146,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":21462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26810,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":30,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":147,"num_bytes_recovered":4194342,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430485,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626717,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1895524,"kb_used_data":1594204,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":91849628,"statfs":{"total":95995035648,"available":94054019072,"internally_reserved":0,"allocated":1632464896,"data_stored":1631676142,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:32:14 2023","interfaces":[{"interface":"back","average":{"1min":5.0129999999999999,"5min":2.4260000000000002,"15min":3.1219999999999999},"min":{"1min":0.255,"5min":0.22700000000000001,"15min":0.20799999999999999},"max":{"1min":52.191000000000003,"5min":52.191000000000003,"15min":121.247},"last":0.66900000000000004},{"interface":"front","average":{"1min":5.5629999999999997,"5min":2.5920000000000001,"15min":3.0720000000000001},"min":{"1min":0.38800000000000001,"5min":0.245,"15min":0.23200000000000001},"max":{"1min":52.323999999999998,"5min":52.323999999999998,"15min":118.977},"last":0.26000000000000001}]},{"osd":1,"last update":"Mon Aug 28 02:32:14 2023","interfaces":[{"interface":"back","average":{"1min":0.94199999999999995,"5min":3.5569999999999999,"15min":2.4750000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23699999999999999},"max":{"1min":3.1749999999999998,"5min":141.87299999999999,"15min":141.87299999999999},"last":2.0299999999999998},{"interface":"front","average":{"1min":1.006,"5min":3.4980000000000002,"15min":2.569},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":5.899,"5min":141.904,"15min":141.904},"last":0.29599999999999999}]},{"osd":2,"last update":"Mon Aug 28 02:32:55 2023","interfaces":[{"interface":"back","average":{"1min":16.728999999999999,"5min":6.0549999999999997,"15min":3.6200000000000001},"min":{"1min":0.308,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":234.876,"5min":234.876,"15min":234.876},"last":1.903},{"interface":"front","average":{"1min":16.361000000000001,"5min":6.0460000000000003,"15min":3.5720000000000001},"min":{"1min":0.36399999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":234.482,"5min":234.482,"15min":234.482},"last":7.3319999999999999}]},{"osd":3,"last update":"Mon Aug 28 02:32:14 2023","interfaces":[{"interface":"back","average":{"1min":1.4219999999999999,"5min":4.0259999999999998,"15min":2.3780000000000001},"min":{"1min":0.33900000000000002,"5min":0.32000000000000001,"15min":0.317},"max":{"1min":7.165,"5min":157.625,"15min":157.625},"last":3.335},{"interface":"front","average":{"1min":1.343,"5min":3.8519999999999999,"15min":2.3839999999999999},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":7.202,"5min":157.637,"15min":157.637},"last":3.3660000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:32:21 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":1.012,"15min":1.0940000000000001},"min":{"1min":0.27300000000000002,"5min":0.27100000000000002,"15min":0.251},"max":{"1min":1.6899999999999999,"5min":18.308,"15min":18.308},"last":0.307},{"interface":"front","average":{"1min":0.752,"5min":1.083,"15min":1.175},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"max":{"1min":3.7639999999999998,"5min":18.263000000000002,"15min":18.263000000000002},"last":0.33200000000000002}]},{"osd":5,"last update":"Mon Aug 28 02:32:21 2023","interfaces":[{"interface":"back","average":{"1min":0.72599999999999998,"5min":0.85999999999999999,"15min":1.161},"min":{"1min":0.317,"5min":0.29699999999999999,"15min":0.26300000000000001},"max":{"1min":4.2800000000000002,"5min":6.0049999999999999,"15min":20.759},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.871,"15min":1.004},"min":{"1min":0.34000000000000002,"5min":0.314,"15min":0.28799999999999998},"max":{"1min":1.724,"5min":7.2370000000000001,"15min":7.3780000000000001},"last":0.40000000000000002}]},{"osd":7,"last update":"Mon Aug 28 02:31:45 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.92900000000000005,"15min":1.018},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.314},"max":{"1min":1.778,"5min":10.605,"15min":10.605},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.88700000000000001,"15min":1.0760000000000001},"min":{"1min":0.38400000000000001,"5min":0.34599999999999997,"15min":0.29799999999999999},"max":{"1min":1.625,"5min":5.1900000000000004,"15min":9.7110000000000003},"last":0.42799999999999999}]}]},{"osd":1,"up_from":98,"seq":420906795169,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2291184,"kb_used_data":1911976,"kb_used_omap":7,"kb_used_meta":379064,"kb_avail":91453968,"statfs":{"total":95995035648,"available":93648863232,"internally_reserved":0,"allocated":1957863424,"data_stored":1956380878,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":388161867},"hb_peers":[0,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:32:10 2023","interfaces":[{"interface":"back","average":{"1min":4.0709999999999997,"5min":2.681,"15min":2.544},"min":{"1min":0.19500000000000001,"5min":0.191,"15min":0.17499999999999999},"max":{"1min":41.838000000000001,"5min":41.838000000000001,"15min":128.60599999999999},"last":0.60699999999999998},{"interface":"front","average":{"1min":5.1390000000000002,"5min":2.8610000000000002,"15min":2.6629999999999998},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.188},"max":{"1min":41.276000000000003,"5min":41.276000000000003,"15min":128.685},"last":0.81299999999999994}]},{"osd":2,"last update":"Mon Aug 28 02:32:40 2023","interfaces":[{"interface":"back","average":{"1min":3.7370000000000001,"5min":3.9079999999999999,"15min":3.2599999999999998},"min":{"1min":0.215,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":24.245999999999999,"5min":90.177000000000007,"15min":135.25999999999999},"last":0.54800000000000004},{"interface":"front","average":{"1min":3.5539999999999998,"5min":3.5910000000000002,"15min":3.0350000000000001},"min":{"1min":0.23599999999999999,"5min":0.218,"15min":0.218},"max":{"1min":12.691000000000001,"5min":89.587999999999994,"15min":135.10300000000001},"last":0.64300000000000002}]},{"osd":3,"last update":"Mon Aug 28 02:32:10 2023","interfaces":[{"interface":"back","average":{"1min":4.298,"5min":4.4660000000000002,"15min":2.258},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":40.210999999999999,"5min":123.741,"15min":123.741},"last":0.501},{"interface":"front","average":{"1min":3.71,"5min":4.7110000000000003,"15min":2.4609999999999999},"min":{"1min":0.253,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":40.25,"5min":123.642,"15min":123.642},"last":0.59099999999999997}]},{"osd":4,"last update":"Mon Aug 28 02:32:10 2023","interfaces":[{"interface":"back","average":{"1min":1.165,"5min":1.4059999999999999,"15min":1.355},"min":{"1min":0.28499999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":4.9169999999999998,"5min":8.1989999999999998,"15min":17.600000000000001},"last":0.623},{"interface":"front","average":{"1min":1.3200000000000001,"5min":1.359,"15min":1.2210000000000001},"min":{"1min":0.307,"5min":0.29399999999999998,"15min":0.25600000000000001},"max":{"1min":10.316000000000001,"5min":10.316000000000001,"15min":19.010000000000002},"last":0.67000000000000004}]},{"osd":5,"last update":"Mon Aug 28 02:32:10 2023","interfaces":[{"interface":"back","average":{"1min":1.625,"5min":1.615,"15min":1.486},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":10.364000000000001,"5min":11.167999999999999,"15min":15.544},"last":0.42899999999999999},{"interface":"front","average":{"1min":1.2030000000000001,"5min":1.536,"15min":1.347},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":4.9980000000000002,"5min":10.084,"15min":17.646999999999998},"last":0.67700000000000005}]},{"osd":6,"last update":"Mon Aug 28 02:32:10 2023","interfaces":[{"interface":"back","average":{"1min":1.6639999999999999,"5min":1.5840000000000001,"15min":1.5960000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":10.420999999999999,"5min":10.420999999999999,"15min":15.512},"last":0.46600000000000003},{"interface":"front","average":{"1min":1.446,"5min":1.4970000000000001,"15min":1.506},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":4.968,"5min":8.1419999999999995,"15min":17.673999999999999},"last":0.65300000000000002}]},{"osd":7,"last update":"Mon Aug 28 02:32:10 2023","interfaces":[{"interface":"back","average":{"1min":1.4450000000000001,"5min":2.0299999999999998,"15min":1.5189999999999999},"min":{"1min":0.34399999999999997,"5min":0.32500000000000001,"15min":0.30199999999999999},"max":{"1min":10.353999999999999,"5min":49.575000000000003,"15min":49.575000000000003},"last":0.442},{"interface":"front","average":{"1min":1.8899999999999999,"5min":2.2469999999999999,"15min":1.7410000000000001},"min":{"1min":0.42199999999999999,"5min":0.39500000000000002,"15min":0.33100000000000002},"max":{"1min":10.449999999999999,"5min":49.319000000000003,"15min":49.319000000000003},"last":0.51500000000000001}]}]},{"osd":0,"up_from":90,"seq":386547056816,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2667832,"kb_used_data":2192688,"kb_used_omap":8,"kb_used_meta":475127,"kb_avail":91077320,"statfs":{"total":95995035648,"available":93263175680,"internally_reserved":0,"allocated":2245312512,"data_stored":2243937821,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":486530409},"hb_peers":[1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,5,4,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Aug 28 02:32:21 2023","interfaces":[{"interface":"back","average":{"1min":4.4160000000000004,"5min":3.46,"15min":2.2269999999999999},"min":{"1min":0.191,"5min":0.191,"15min":0.184},"max":{"1min":61.932000000000002,"5min":61.932000000000002,"15min":61.932000000000002},"last":1.6299999999999999},{"interface":"front","average":{"1min":5.8650000000000002,"5min":4.2249999999999996,"15min":2.4199999999999999},"min":{"1min":0.28999999999999998,"5min":0.19600000000000001,"15min":0.17000000000000001},"max":{"1min":62.061999999999998,"5min":62.061999999999998,"15min":62.061999999999998},"last":3.4900000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:32:40 2023","interfaces":[{"interface":"back","average":{"1min":4.9809999999999999,"5min":3.7869999999999999,"15min":2.5470000000000002},"min":{"1min":0.22800000000000001,"5min":0.20599999999999999,"15min":0.19500000000000001},"max":{"1min":46.927999999999997,"5min":100.718,"15min":100.718},"last":1.4470000000000001},{"interface":"front","average":{"1min":5.1260000000000003,"5min":3.5899999999999999,"15min":2.8479999999999999},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":46.942999999999998,"5min":100.962,"15min":100.962},"last":3.427}]},{"osd":3,"last update":"Mon Aug 28 02:32:21 2023","interfaces":[{"interface":"back","average":{"1min":8.1999999999999993,"5min":7.5330000000000004,"15min":3.5089999999999999},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23300000000000001},"max":{"1min":126.896,"5min":237.43899999999999,"15min":237.43899999999999},"last":9.6449999999999996},{"interface":"front","average":{"1min":7.8310000000000004,"5min":7.3810000000000002,"15min":3.4300000000000002},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":127.01600000000001,"5min":237.48099999999999,"15min":237.48099999999999},"last":3.891}]},{"osd":4,"last update":"Mon Aug 28 02:32:16 2023","interfaces":[{"interface":"back","average":{"1min":1.3959999999999999,"5min":1.637,"15min":1.4139999999999999},"min":{"1min":0.252,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":8.9559999999999995,"5min":9.3680000000000003,"15min":9.6649999999999991},"last":1.423},{"interface":"front","average":{"1min":1.3660000000000001,"5min":1.6180000000000001,"15min":1.5529999999999999},"min":{"1min":0.31,"5min":0.309,"15min":0.309},"max":{"1min":5.4470000000000001,"5min":27.271999999999998,"15min":27.271999999999998},"last":3.919}]},{"osd":5,"last update":"Mon Aug 28 02:32:16 2023","interfaces":[{"interface":"back","average":{"1min":1.208,"5min":1.46,"15min":1.633},"min":{"1min":0.33200000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":3.871,"5min":27.367000000000001,"15min":44.000999999999998},"last":3.9470000000000001},{"interface":"front","average":{"1min":1.4330000000000001,"5min":1.635,"15min":1.6020000000000001},"min":{"1min":0.311,"5min":0.308,"15min":0.30599999999999999},"max":{"1min":9.0310000000000006,"5min":9.4489999999999998,"15min":43.953000000000003},"last":1.3700000000000001}]},{"osd":6,"last update":"Mon Aug 28 02:32:16 2023","interfaces":[{"interface":"back","average":{"1min":1.6459999999999999,"5min":1.696,"15min":1.7270000000000001},"min":{"1min":0.375,"5min":0.312,"15min":0.312},"max":{"1min":4.1269999999999998,"5min":27.331,"15min":32.417999999999999},"last":3.972},{"interface":"front","average":{"1min":1.4330000000000001,"5min":1.8380000000000001,"15min":1.657},"min":{"1min":0.33900000000000002,"5min":0.32700000000000001,"15min":0.30399999999999999},"max":{"1min":9.0009999999999994,"5min":27.247,"15min":32.130000000000003},"last":1.3999999999999999}]},{"osd":7,"last update":"Mon Aug 28 02:32:16 2023","interfaces":[{"interface":"back","average":{"1min":1.6060000000000001,"5min":2.25,"15min":1.712},"min":{"1min":0.379,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":10.775,"5min":27.385000000000002,"15min":27.385000000000002},"last":4.3529999999999998},{"interface":"front","average":{"1min":2.6110000000000002,"5min":2.0979999999999999,"15min":1.766},"min":{"1min":0.41799999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":7.5350000000000001,"5min":27.213999999999999,"15min":27.213999999999999},"last":2.4550000000000001}]}]},{"osd":2,"up_from":104,"seq":446676598939,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2485700,"kb_used_data":2015356,"kb_used_omap":10,"kb_used_meta":470325,"kb_avail":91259452,"statfs":{"total":95995035648,"available":93449678848,"internally_reserved":0,"allocated":2063724544,"data_stored":2062398418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10792,"internal_metadata":481613272},"hb_peers":[0,1,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:32:43 2023","interfaces":[{"interface":"back","average":{"1min":22.899000000000001,"5min":5.8789999999999996,"15min":3.8210000000000002},"min":{"1min":0.193,"5min":0.185,"15min":0.17599999999999999},"max":{"1min":185.93799999999999,"5min":185.93799999999999,"15min":185.93799999999999},"last":7.8479999999999999},{"interface":"front","average":{"1min":22.541,"5min":6.1950000000000003,"15min":5.0110000000000001},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.19900000000000001},"max":{"1min":187.90700000000001,"5min":187.90700000000001,"15min":187.90700000000001},"last":0.23400000000000001}]},{"osd":1,"last update":"Mon Aug 28 02:32:42 2023","interfaces":[{"interface":"back","average":{"1min":2.2650000000000001,"5min":1.998,"15min":1.9179999999999999},"min":{"1min":0.22800000000000001,"5min":0.21099999999999999,"15min":0.20999999999999999},"max":{"1min":13.25,"5min":13.25,"15min":30.888000000000002},"last":0.26400000000000001},{"interface":"front","average":{"1min":1.784,"5min":2.1259999999999999,"15min":1.8149999999999999},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":9.4329999999999998,"5min":11.962999999999999,"15min":31.648},"last":0.57999999999999996}]},{"osd":3,"last update":"Mon Aug 28 02:32:12 2023","interfaces":[{"interface":"back","average":{"1min":5.0350000000000001,"5min":4.2489999999999997,"15min":2.8900000000000001},"min":{"1min":0.33100000000000002,"5min":0.25,"15min":0.22700000000000001},"max":{"1min":56.168999999999997,"5min":71.983999999999995,"15min":128.63900000000001},"last":1.5569999999999999},{"interface":"front","average":{"1min":4.7439999999999998,"5min":3.9049999999999998,"15min":2.8460000000000001},"min":{"1min":0.29799999999999999,"5min":0.223,"15min":0.223},"max":{"1min":56.018000000000001,"5min":71.944999999999993,"15min":128.75700000000001},"last":1.591}]},{"osd":4,"last update":"Mon Aug 28 02:32:42 2023","interfaces":[{"interface":"back","average":{"1min":1.647,"5min":4.9749999999999996,"15min":2.7410000000000001},"min":{"1min":0.311,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":13.273,"5min":207.34100000000001,"15min":207.34100000000001},"last":0.624},{"interface":"front","average":{"1min":2.0630000000000002,"5min":4.9260000000000002,"15min":2.5129999999999999},"min":{"1min":0.36799999999999999,"5min":0.30299999999999999,"15min":0.27400000000000002},"max":{"1min":13.222,"5min":207.352,"15min":207.352},"last":6.9640000000000004}]},{"osd":5,"last update":"Mon Aug 28 02:32:42 2023","interfaces":[{"interface":"back","average":{"1min":1.8919999999999999,"5min":3.641,"15min":2.6699999999999999},"min":{"1min":0.33800000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":13.335000000000001,"5min":94.614000000000004,"15min":128.696},"last":0.41099999999999998},{"interface":"front","average":{"1min":1.8500000000000001,"5min":3.4860000000000002,"15min":2.6960000000000002},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30299999999999999},"max":{"1min":13.303000000000001,"5min":94.534000000000006,"15min":128.84100000000001},"last":0.60699999999999998}]},{"osd":6,"last update":"Mon Aug 28 02:32:42 2023","interfaces":[{"interface":"back","average":{"1min":2.1859999999999999,"5min":1.633,"15min":1.4770000000000001},"min":{"1min":0.35699999999999998,"5min":0.32800000000000001,"15min":0.29299999999999998},"max":{"1min":13.292999999999999,"5min":13.292999999999999,"15min":13.292999999999999},"last":7.0110000000000001},{"interface":"front","average":{"1min":1.8680000000000001,"5min":1.754,"15min":1.625},"min":{"1min":0.40300000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":13.311,"5min":13.311,"15min":13.311},"last":0.44900000000000001}]},{"osd":7,"last update":"Mon Aug 28 02:32:42 2023","interfaces":[{"interface":"back","average":{"1min":2.0649999999999999,"5min":3.6520000000000001,"15min":2.6280000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.32100000000000001},"max":{"1min":13.321999999999999,"5min":108.904,"15min":128.911},"last":0.502},{"interface":"front","average":{"1min":2.0739999999999998,"5min":3.8719999999999999,"15min":2.859},"min":{"1min":0.38800000000000001,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":13.353,"5min":108.217,"15min":128.72399999999999},"last":0.625}]}]},{"osd":3,"up_from":112,"seq":481036337301,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3264192,"kb_used_data":2787000,"kb_used_omap":3,"kb_used_meta":476924,"kb_avail":90480960,"statfs":{"total":95995035648,"available":92652503040,"internally_reserved":0,"allocated":2853888000,"data_stored":2852559168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":488371106},"hb_peers":[0,1,2,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:32:07 2023","interfaces":[{"interface":"back","average":{"1min":3.0910000000000002,"5min":3.4129999999999998,"15min":3.2970000000000002},"min":{"1min":0.23300000000000001,"5min":0.20000000000000001,"15min":0.193},"max":{"1min":11.305,"5min":79.793000000000006,"15min":164.25299999999999},"last":6.3470000000000004},{"interface":"front","average":{"1min":2.444,"5min":3.0110000000000001,"15min":3.2000000000000002},"min":{"1min":0.26400000000000001,"5min":0.192,"15min":0.192},"max":{"1min":11.571,"5min":78.555000000000007,"15min":164.55000000000001},"last":0.45100000000000001}]},{"osd":1,"last update":"Mon Aug 28 02:32:07 2023","interfaces":[{"interface":"back","average":{"1min":2.5379999999999998,"5min":1.79,"15min":2.0619999999999998},"min":{"1min":0.27300000000000002,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":10.907999999999999,"5min":10.907999999999999,"15min":117.34699999999999},"last":6.4420000000000002},{"interface":"front","average":{"1min":2.4249999999999998,"5min":2.238,"15min":2.1949999999999998},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.224},"max":{"1min":10.079000000000001,"5min":10.125,"15min":118.012},"last":6.319}]},{"osd":2,"last update":"Mon Aug 28 02:32:07 2023","interfaces":[{"interface":"back","average":{"1min":2.8330000000000002,"5min":2.9409999999999998,"15min":3.0790000000000002},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.19900000000000001},"max":{"1min":7.6959999999999997,"5min":65.308999999999997,"15min":120.43300000000001},"last":16.768000000000001},{"interface":"front","average":{"1min":2.5289999999999999,"5min":2.613,"15min":2.96},"min":{"1min":0.249,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":9.7309999999999999,"5min":65.120999999999995,"15min":120.44499999999999},"last":5.859}]},{"osd":4,"last update":"Mon Aug 28 02:32:07 2023","interfaces":[{"interface":"back","average":{"1min":1.4390000000000001,"5min":2.2730000000000001,"15min":1.5700000000000001},"min":{"1min":0.34499999999999997,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":4.2030000000000003,"5min":58.067999999999998,"15min":58.067999999999998},"last":0.38400000000000001},{"interface":"front","average":{"1min":1.3799999999999999,"5min":2.2959999999999998,"15min":1.613},"min":{"1min":0.36199999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":6.218,"5min":58.088000000000001,"15min":58.088000000000001},"last":6.4640000000000004}]},{"osd":5,"last update":"Mon Aug 28 02:32:07 2023","interfaces":[{"interface":"back","average":{"1min":1.4670000000000001,"5min":1.712,"15min":1.7350000000000001},"min":{"1min":0.31,"5min":0.31,"15min":0.29599999999999999},"max":{"1min":6.2599999999999998,"5min":34.060000000000002,"15min":102.904},"last":6.492},{"interface":"front","average":{"1min":1.4359999999999999,"5min":1.952,"15min":1.756},"min":{"1min":0.38700000000000001,"5min":0.36699999999999999,"15min":0.32800000000000001},"max":{"1min":4.7590000000000003,"5min":34.079999999999998,"15min":103.283},"last":6.3890000000000002}]},{"osd":6,"last update":"Mon Aug 28 02:32:07 2023","interfaces":[{"interface":"back","average":{"1min":1.5609999999999999,"5min":1.7390000000000001,"15min":1.55},"min":{"1min":0.34300000000000003,"5min":0.33500000000000002,"15min":0.314},"max":{"1min":5.1079999999999997,"5min":9.157,"15min":72.116},"last":6.4109999999999996},{"interface":"front","average":{"1min":1.5840000000000001,"5min":1.645,"15min":1.488},"min":{"1min":0.41999999999999998,"5min":0.314,"15min":0.314},"max":{"1min":5.1760000000000002,"5min":7.8899999999999997,"15min":72.030000000000001},"last":0.42699999999999999}]},{"osd":7,"last update":"Mon Aug 28 02:32:07 2023","interfaces":[{"interface":"back","average":{"1min":1.6859999999999999,"5min":3.0920000000000001,"15min":1.845},"min":{"1min":0.39800000000000002,"5min":0.33800000000000002,"15min":0.32500000000000001},"max":{"1min":5.2320000000000002,"5min":54.427999999999997,"15min":54.427999999999997},"last":7.0629999999999997},{"interface":"front","average":{"1min":1.5429999999999999,"5min":3.052,"15min":1.9379999999999999},"min":{"1min":0.443,"5min":0.36399999999999999,"15min":0.34799999999999998},"max":{"1min":6.3019999999999996,"5min":54.445,"15min":54.445},"last":10.754}]}]},{"osd":4,"up_from":30,"seq":128849019175,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1835100,"kb_used_data":1533780,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":91910052,"statfs":{"total":95995035648,"available":94115893248,"internally_reserved":0,"allocated":1570590720,"data_stored":1569809906,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:32:14 2023","interfaces":[{"interface":"back","average":{"1min":5.9870000000000001,"5min":2.8799999999999999,"15min":3.2309999999999999},"min":{"1min":0.25600000000000001,"5min":0.219,"15min":0.219},"max":{"1min":80.655000000000001,"5min":80.655000000000001,"15min":138.67400000000001},"last":3.7610000000000001},{"interface":"front","average":{"1min":6.5659999999999998,"5min":2.6150000000000002,"15min":3.0630000000000002},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":80.427000000000007,"5min":80.427000000000007,"15min":134.14599999999999},"last":2.5390000000000001}]},{"osd":1,"last update":"Mon Aug 28 02:32:14 2023","interfaces":[{"interface":"back","average":{"1min":1.3540000000000001,"5min":2.0819999999999999,"15min":1.6060000000000001},"min":{"1min":0.27200000000000002,"5min":0.221,"15min":0.221},"max":{"1min":6.7460000000000004,"5min":34.945999999999998,"15min":53.622999999999998},"last":7.508},{"interface":"front","average":{"1min":1.353,"5min":1.7350000000000001,"15min":1.49},"min":{"1min":0.29699999999999999,"5min":0.25,"15min":0.25},"max":{"1min":6.2439999999999998,"5min":34.869999999999997,"15min":53.774000000000001},"last":3.7010000000000001}]},{"osd":2,"last update":"Mon Aug 28 02:32:43 2023","interfaces":[{"interface":"back","average":{"1min":5.3719999999999999,"5min":3.2530000000000001,"15min":3.2000000000000002},"min":{"1min":0.33900000000000002,"5min":0.27400000000000002,"15min":0.25600000000000001},"max":{"1min":61.777000000000001,"5min":61.777000000000001,"15min":74.099000000000004},"last":3.8300000000000001},{"interface":"front","average":{"1min":5.3499999999999996,"5min":3.2919999999999998,"15min":3.077},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.25},"max":{"1min":61.601999999999997,"5min":61.601999999999997,"15min":74.081999999999994},"last":0.34100000000000003}]},{"osd":3,"last update":"Mon Aug 28 02:32:14 2023","interfaces":[{"interface":"back","average":{"1min":14.24,"5min":8.4719999999999995,"15min":3.621},"min":{"1min":0.34200000000000003,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":203.583,"5min":203.583,"15min":203.583},"last":0.378},{"interface":"front","average":{"1min":15.619999999999999,"5min":8.5960000000000001,"15min":3.7349999999999999},"min":{"1min":0.45100000000000001,"5min":0.33800000000000002,"15min":0.32500000000000001},"max":{"1min":203.41399999999999,"5min":203.41399999999999,"15min":203.41399999999999},"last":1.8400000000000001}]},{"osd":5,"last update":"Mon Aug 28 02:32:55 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":2.4329999999999998,"15min":1.46},"min":{"1min":0.27600000000000002,"5min":0.246,"15min":0.246},"max":{"1min":2.3149999999999999,"5min":124.718,"15min":124.718},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.53800000000000003,"5min":2.5779999999999998,"15min":1.6559999999999999},"min":{"1min":0.30199999999999999,"5min":0.247,"15min":0.247},"max":{"1min":1.9930000000000001,"5min":124.694,"15min":124.694},"last":0.32500000000000001}]},{"osd":6,"last update":"Mon Aug 28 02:32:14 2023","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":1.087,"15min":1.109},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":3.109,"5min":14.907999999999999,"15min":14.907999999999999},"last":0.94999999999999996},{"interface":"front","average":{"1min":0.70499999999999996,"5min":0.89100000000000001,"15min":0.89500000000000002},"min":{"1min":0.32800000000000001,"5min":0.307,"15min":0.307},"max":{"1min":3.1219999999999999,"5min":5.2679999999999998,"15min":7.3170000000000002},"last":0.91900000000000004}]},{"osd":7,"last update":"Mon Aug 28 02:32:34 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":1.0149999999999999,"15min":1.1819999999999999},"min":{"1min":0.32400000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.2210000000000001,"5min":10.151999999999999,"15min":16.584},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.53100000000000003,"5min":1.0900000000000001,"15min":1.2390000000000001},"min":{"1min":0.35999999999999999,"5min":0.311,"15min":0.311},"max":{"1min":1.5109999999999999,"5min":9.3469999999999995,"15min":10.826000000000001},"last":0.434}]}]},{"osd":5,"up_from":36,"seq":154618822947,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1619468,"kb_used_data":1322244,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":92125684,"statfs":{"total":95995035648,"available":94336700416,"internally_reserved":0,"allocated":1353977856,"data_stored":1353208581,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:32:09 2023","interfaces":[{"interface":"back","average":{"1min":5.5720000000000001,"5min":3.371,"15min":3.3380000000000001},"min":{"1min":0.23400000000000001,"5min":0.214,"15min":0.20300000000000001},"max":{"1min":57.753999999999998,"5min":104.13200000000001,"15min":131.06100000000001},"last":5.9850000000000003},{"interface":"front","average":{"1min":5.5380000000000003,"5min":3.5659999999999998,"15min":3.5459999999999998},"min":{"1min":0.27300000000000002,"5min":0.27100000000000002,"15min":0.22900000000000001},"max":{"1min":57.762999999999998,"5min":104.142,"15min":131.262},"last":0.71299999999999997}]},{"osd":1,"last update":"Mon Aug 28 02:32:15 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":1.5509999999999999,"15min":2.4569999999999999},"min":{"1min":0.27300000000000002,"5min":0.25,"15min":0.216},"max":{"1min":2.4550000000000001,"5min":30.593,"15min":183.97900000000001},"last":6.8639999999999999},{"interface":"front","average":{"1min":0.81899999999999995,"5min":1.5449999999999999,"15min":2.6240000000000001},"min":{"1min":0.308,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":2.637,"5min":34.616999999999997,"15min":184.33699999999999},"last":0.32600000000000001}]},{"osd":2,"last update":"Mon Aug 28 02:32:48 2023","interfaces":[{"interface":"back","average":{"1min":4.4649999999999999,"5min":3.8860000000000001,"15min":4.2809999999999997},"min":{"1min":0.318,"5min":0.25800000000000001,"15min":0.252},"max":{"1min":56.725999999999999,"5min":100.56699999999999,"15min":176.69499999999999},"last":2.0390000000000001},{"interface":"front","average":{"1min":4.6669999999999998,"5min":4.2709999999999999,"15min":4.5359999999999996},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.28299999999999997},"max":{"1min":56.750999999999998,"5min":100.53700000000001,"15min":176.71199999999999},"last":0.85799999999999998}]},{"osd":3,"last update":"Mon Aug 28 02:32:15 2023","interfaces":[{"interface":"back","average":{"1min":12.782999999999999,"5min":5.0609999999999999,"15min":2.516},"min":{"1min":0.29699999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":128.571,"5min":128.571,"15min":128.571},"last":2.0720000000000001},{"interface":"front","average":{"1min":13.773999999999999,"5min":5.4390000000000001,"15min":2.899},"min":{"1min":0.36399999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":128.684,"5min":128.684,"15min":128.684},"last":0.35699999999999998}]},{"osd":4,"last update":"Mon Aug 28 02:32:58 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":1.9330000000000001,"15min":1.3320000000000001},"min":{"1min":0.253,"5min":0.253,"15min":0.246},"max":{"1min":2.0600000000000001,"5min":85.769999999999996,"15min":85.769999999999996},"last":2.056},{"interface":"front","average":{"1min":0.48499999999999999,"5min":1.831,"15min":1.3160000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28000000000000003},"max":{"1min":1.764,"5min":85.825000000000003,"15min":85.825000000000003},"last":2.1949999999999998}]},{"osd":6,"last update":"Mon Aug 28 02:32:15 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.80600000000000005,"15min":1.0409999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":1.7929999999999999,"5min":4.6210000000000004,"15min":14.803000000000001},"last":2.657},{"interface":"front","average":{"1min":0.755,"5min":0.84199999999999997,"15min":0.95299999999999996},"min":{"1min":0.30199999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":2.4470000000000001,"5min":4.9509999999999996,"15min":8.6479999999999997},"last":2.2759999999999998}]},{"osd":7,"last update":"Mon Aug 28 02:32:33 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":2.71,"15min":1.7150000000000001},"min":{"1min":0.35099999999999998,"5min":0.33300000000000002,"15min":0.31},"max":{"1min":2.266,"5min":164.13800000000001,"15min":164.13800000000001},"last":0.872},{"interface":"front","average":{"1min":0.58499999999999996,"5min":2.7229999999999999,"15min":1.645},"min":{"1min":0.34100000000000003,"5min":0.33700000000000002,"15min":0.32100000000000001},"max":{"1min":1.9139999999999999,"5min":161.786,"15min":161.786},"last":0.57799999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2232029184,"data_stored":2232025112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1944911872,"data_stored":1944911872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2049572864,"data_stored":2049572864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2830110720,"data_stored":2830106648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1557200896,"data_stored":1557200896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1341390848,"data_stored":1341390848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1624379392,"data_stored":1624375320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1396965376,"data_stored":1396965376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-08-28T02:33:13.340 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd pool set device_health_metrics pgp_num 21 2023-08-28T02:33:13.359 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:33:13.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:13.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:33:13.463 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:33:13.496 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:13.700 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:33:13.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:13.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:33:13.805 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:33:13.852 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:13.934 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:33:13.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:13.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:33:14.036 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:33:14.063 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:14.267 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:33:14.267 INFO:teuthology.orchestra.run.smithi169.stdout: "ops": [], 2023-08-28T02:33:14.267 INFO:teuthology.orchestra.run.smithi169.stdout: "complaint_time": 30, 2023-08-28T02:33:14.268 INFO:teuthology.orchestra.run.smithi169.stdout: "num_blocked_ops": 0 2023-08-28T02:33:14.268 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:33:14.418 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:33:14.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:14.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:33:14.522 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:33:14.557 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:14.613 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:33:14.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:14.617 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:33:14.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:14 smithi143 ceph-mon[183059]: pgmap v738: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 665 op/s; 32553/203013 objects degraded (16.035%); 2.3 MiB/s, 37 objects/s recovering 2023-08-28T02:33:14.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:14 smithi143 ceph-mon[189871]: pgmap v738: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 665 op/s; 32553/203013 objects degraded (16.035%); 2.3 MiB/s, 37 objects/s recovering 2023-08-28T02:33:14.718 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:33:14.744 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:14.754 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:33:14.801 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:33:14.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:14.805 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:33:14.805 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:14 smithi169 ceph-mon[159415]: pgmap v738: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 665 op/s; 32553/203013 objects degraded (16.035%); 2.3 MiB/s, 37 objects/s recovering 2023-08-28T02:33:14.906 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:33:14.995 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:15.163 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.6 dump_historic_ops 2023-08-28T02:33:15.269 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:33:15.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:15.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:33:15.302 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:33:15.351 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:33:15.354 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset noscrub 2023-08-28T02:33:15.380 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:33:15.394 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:33:15.445 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:33:15.478 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:15.506 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:33:15.550 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:33:15.601 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:33:15.626 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:33:15.635 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:33:15.635 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:33:15.635 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:33:15.635 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:33:15.635 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:33:15.635 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:33:15.635 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:33:15.635 INFO:teuthology.orchestra.run.smithi143.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-08-28T02:33:15.833 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:15 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2142364320' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:33:15.834 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:15 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2142364320' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:33:15.865 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:15 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2142364320' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:33:16.124 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:33:16.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:16.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:33:16.238 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:33:16.335 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:16.880 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:16 smithi169 ceph-mon[159415]: pgmap v739: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 632 op/s; 32384/197028 objects degraded (16.436%); 2.2 MiB/s, 35 objects/s recovering 2023-08-28T02:33:16.880 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:16 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 32384/197028 objects degraded (16.436%), 22 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-08-28T02:33:16.946 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:16 smithi143 ceph-mon[183059]: pgmap v739: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 632 op/s; 32384/197028 objects degraded (16.436%); 2.2 MiB/s, 35 objects/s recovering 2023-08-28T02:33:16.947 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:16 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 32384/197028 objects degraded (16.436%), 22 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-08-28T02:33:16.947 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:16 smithi143 ceph-mon[189871]: pgmap v739: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 632 op/s; 32384/197028 objects degraded (16.436%); 2.2 MiB/s, 35 objects/s recovering 2023-08-28T02:33:16.947 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:16 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 32384/197028 objects degraded (16.436%), 22 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-08-28T02:33:17.386 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:33:17.386 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:33:17.558 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:33:17.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:17.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:33:17.669 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:33:17.698 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-08-28T02:33:17.699 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-08-28T02:33:17.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:17 smithi169 ceph-mon[159415]: pgmap v740: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.9 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 32384/193911 objects degraded (16.700%); 2.2 MiB/s, 35 objects/s recovering 2023-08-28T02:33:17.779 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:17.843 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:33:17.845 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:17 smithi143 ceph-mon[183059]: pgmap v740: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.9 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 32384/193911 objects degraded (16.700%); 2.2 MiB/s, 35 objects/s recovering 2023-08-28T02:33:17.846 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:17 smithi143 ceph-mon[189871]: pgmap v740: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.9 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 32384/193911 objects degraded (16.700%); 2.2 MiB/s, 35 objects/s recovering 2023-08-28T02:33:17.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:17.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:33:17.949 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:33:17.985 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:18.268 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:33:18.274 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.0 dump_ops_in_flight 2023-08-28T02:33:18.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:18.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:33:18.295 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:33:18.376 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:33:18.410 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:18.460 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:33:18.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:18.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:33:18.567 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:33:18.733 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:18.787 INFO:teuthology.orchestra.run.smithi143.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-08-28T02:33:18.905 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:18 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2189481272' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-08-28T02:33:18.905 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:18 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-08-28T02:33:18.908 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:18 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2189481272' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-08-28T02:33:18.908 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:18 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-08-28T02:33:18.973 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:33:18.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:18.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:33:19.080 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:33:19.134 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:19.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:18 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2189481272' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-08-28T02:33:19.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:18 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-08-28T02:33:19.699 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:33:20.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:19 smithi143 ceph-mon[183059]: pgmap v741: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 622 op/s; 32182/191490 objects degraded (16.806%); 2.3 MiB/s, 37 objects/s recovering 2023-08-28T02:33:20.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:19 smithi143 ceph-mon[189871]: pgmap v741: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 622 op/s; 32182/191490 objects degraded (16.806%); 2.3 MiB/s, 37 objects/s recovering 2023-08-28T02:33:20.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:19 smithi169 ceph-mon[159415]: pgmap v741: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 622 op/s; 32182/191490 objects degraded (16.806%); 2.3 MiB/s, 37 objects/s recovering 2023-08-28T02:33:21.057 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:33:21.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:21.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:33:21.169 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:33:21.199 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:21.254 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:33:21.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:21.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:33:21.311 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-08-28T02:33:21.360 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:33:21.390 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:21.437 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:33:21.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:21.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:33:21.502 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:33:21.544 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:33:21.570 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:33:21.573 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:21.621 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:33:21.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:21.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:33:21.726 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:33:21.754 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:21.779 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:21 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 32182/187659 objects degraded (17.149%), 22 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-08-28T02:33:21.779 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:21 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 32182/187659 objects degraded (17.149%), 22 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-08-28T02:33:21.807 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:33:21.813 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:21 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 32182/187659 objects degraded (17.149%), 22 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-08-28T02:33:21.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:21.814 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:33:21.915 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:33:21.943 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:21.998 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:33:22.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:22.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:33:22.105 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:33:22.129 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:33:21 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:33:21] "GET /metrics HTTP/1.1" 200 36583 "" "Prometheus/2.33.4" 2023-08-28T02:33:22.225 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:22.956 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:22 smithi143 ceph-mon[189871]: pgmap v742: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 609 op/s; 32182/187659 objects degraded (17.149%); 2.3 MiB/s, 37 objects/s recovering 2023-08-28T02:33:22.956 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:22 smithi143 ceph-mon[183059]: pgmap v742: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 609 op/s; 32182/187659 objects degraded (17.149%); 2.3 MiB/s, 37 objects/s recovering 2023-08-28T02:33:22.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:22 smithi169 ceph-mon[159415]: pgmap v742: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 609 op/s; 32182/187659 objects degraded (17.149%); 2.3 MiB/s, 37 objects/s recovering 2023-08-28T02:33:23.598 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is unset 2023-08-28T02:33:23.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:23 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3602001957' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:33:23.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:23 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:33:23.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:23 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3602001957' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:33:23.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:23 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:33:23.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:23 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3602001957' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:33:23.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:23 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:33:24.707 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:24 smithi143 ceph-mon[183059]: pgmap v743: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 606 op/s; 32182/184422 objects degraded (17.450%); 2.0 MiB/s, 31 objects/s recovering 2023-08-28T02:33:24.707 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:24 smithi143 ceph-mon[183059]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:33:24.707 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:24 smithi143 ceph-mon[183059]: failed to encode map e296 with expected crc 2023-08-28T02:33:24.707 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:24 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:33:24.707 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:24 smithi143 ceph-mon[183059]: osdmap e296: 8 total, 7 up, 7 in 2023-08-28T02:33:24.707 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:24 smithi143 ceph-mon[183059]: failed to encode map e296 with expected crc 2023-08-28T02:33:24.707 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:24 smithi143 ceph-mon[189871]: pgmap v743: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 606 op/s; 32182/184422 objects degraded (17.450%); 2.0 MiB/s, 31 objects/s recovering 2023-08-28T02:33:24.707 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:24 smithi143 ceph-mon[189871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:33:24.708 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:24 smithi143 ceph-mon[189871]: failed to encode map e296 with expected crc 2023-08-28T02:33:24.708 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:24 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:33:24.708 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:24 smithi143 ceph-mon[189871]: osdmap e296: 8 total, 7 up, 7 in 2023-08-28T02:33:24.708 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:24 smithi143 ceph-mon[189871]: failed to encode map e296 with expected crc 2023-08-28T02:33:24.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:24 smithi169 ceph-mon[159415]: pgmap v743: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 606 op/s; 32182/184422 objects degraded (17.450%); 2.0 MiB/s, 31 objects/s recovering 2023-08-28T02:33:24.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:24 smithi169 ceph-mon[159415]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:33:24.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:24 smithi169 ceph-mon[159415]: failed to encode map e296 with expected crc 2023-08-28T02:33:24.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:24 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:33:24.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:24 smithi169 ceph-mon[159415]: osdmap e296: 8 total, 7 up, 7 in 2023-08-28T02:33:24.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:24 smithi169 ceph-mon[159415]: failed to encode map e296 with expected crc 2023-08-28T02:33:25.079 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:33:25.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:25.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:33:25.202 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:33:25.311 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:25.530 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:33:25.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:25.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:33:25.640 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:33:25.675 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:25.725 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:33:25.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:25.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:33:25.833 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:33:25.863 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:25.914 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:33:25.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:25.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:33:25.933 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:33:25.934 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:33:26.023 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:33:26.051 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:26.093 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:33:26.100 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:33:26.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:26.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:33:26.138 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:33:26.173 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:33:26.209 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:33:26.252 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:33:26.278 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:33:26.312 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 6, 1] out_osds: [7] dead_osds: [7] live_osds: [0, 1, 2, 3, 4, 5, 6] 2023-08-28T02:33:26.312 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-08-28T02:33:26.313 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-08-28T02:33:26.313 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph pg dump --format=json 2023-08-28T02:33:26.327 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:33:26.329 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:26.362 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:33:26.363 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:33:26.363 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:33:26.363 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:33:26.363 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:33:26.363 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:33:26.363 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:33:26.363 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:33:26.364 INFO:teuthology.orchestra.run.smithi143.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-08-28T02:33:26.438 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:33:26.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:26.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:33:26.548 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:33:26.562 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:26 smithi143 ceph-mon[183059]: pgmap v745: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.6 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 653 op/s; 31954/177414 objects degraded (18.011%); 2.7 MiB/s, 42 objects/s recovering 2023-08-28T02:33:26.562 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:26 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:33:26.562 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:26 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1114854464' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:33:26.563 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:26 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 31954/177414 objects degraded (18.011%), 22 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-08-28T02:33:26.563 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:26 smithi143 ceph-mon[189871]: pgmap v745: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.6 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 653 op/s; 31954/177414 objects degraded (18.011%); 2.7 MiB/s, 42 objects/s recovering 2023-08-28T02:33:26.563 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:26 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:33:26.563 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:26 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1114854464' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:33:26.563 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:26 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 31954/177414 objects degraded (18.011%), 22 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-08-28T02:33:26.689 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:26.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:26 smithi169 ceph-mon[159415]: pgmap v745: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.6 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 653 op/s; 31954/177414 objects degraded (18.011%); 2.7 MiB/s, 42 objects/s recovering 2023-08-28T02:33:26.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:26 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:33:26.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:26 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1114854464' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:33:26.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:26 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 31954/177414 objects degraded (18.011%), 22 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-08-28T02:33:27.208 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset nodeep-scrub 2023-08-28T02:33:27.925 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:33:27.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:27.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:33:28.031 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:33:28.059 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:28.072 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.0 dump_blocked_ops 2023-08-28T02:33:28.110 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:33:28.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:28.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:33:28.217 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:33:28.303 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-08-28T02:33:28.303 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-08-28T02:33:28.320 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:28.473 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:33:28.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:28.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:33:28.488 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:33:28.536 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:33:28.574 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:28 smithi143 ceph-mon[189871]: pgmap v746: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 664 op/s; 31954/173976 objects degraded (18.367%); 2.7 MiB/s, 42 objects/s recovering 2023-08-28T02:33:28.590 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:33:28.618 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:28.682 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:33:28.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:28.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:33:28.788 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:33:28.849 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:28 smithi143 ceph-mon[183059]: pgmap v746: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 664 op/s; 31954/173976 objects degraded (18.367%); 2.7 MiB/s, 42 objects/s recovering 2023-08-28T02:33:28.900 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:28.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:28 smithi169 ceph-mon[159415]: pgmap v746: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 664 op/s; 31954/173976 objects degraded (18.367%); 2.7 MiB/s, 42 objects/s recovering 2023-08-28T02:33:29.238 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:33:29.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:29.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:33:29.345 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:33:29.450 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:30.304 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:33:30.589 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:30 smithi143 ceph-mon[183059]: pgmap v747: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 31954/173976 objects degraded (18.367%); 1.4 MiB/s, 22 objects/s recovering 2023-08-28T02:33:30.650 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:33:30.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:30.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:33:30.764 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:33:30.794 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:30.849 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:33:30.852 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:30 smithi169 ceph-mon[159415]: pgmap v747: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 31954/173976 objects degraded (18.367%); 1.4 MiB/s, 22 objects/s recovering 2023-08-28T02:33:30.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:30.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:33:30.954 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:33:30.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:30 smithi143 ceph-mon[189871]: pgmap v747: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 31954/173976 objects degraded (18.367%); 1.4 MiB/s, 22 objects/s recovering 2023-08-28T02:33:30.981 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:31.025 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:33:31.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:31.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:33:31.467 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:33:31.524 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:33:31.611 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:31 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 31954/173976 objects degraded (18.367%), 22 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-08-28T02:33:31.611 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:31.843 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:33:31.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:31.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:33:31.856 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:31 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 31954/173976 objects degraded (18.367%), 22 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-08-28T02:33:31.951 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:33:31.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:31 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 31954/173976 objects degraded (18.367%), 22 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-08-28T02:33:32.015 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:33:32.071 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:32.157 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:33:31 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:33:31] "GET /metrics HTTP/1.1" 200 36583 "" "Prometheus/2.33.4" 2023-08-28T02:33:32.286 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:33:32.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:32.298 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:33:32.398 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:33:32.518 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:32.673 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is unset 2023-08-28T02:33:32.711 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:33:32.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:32.729 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:32 smithi143 ceph-mon[189871]: pgmap v748: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 672 op/s; 31684/167481 objects degraded (18.918%); 3.1 MiB/s, 49 objects/s recovering 2023-08-28T02:33:32.729 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:32 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3452044152' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:33:32.729 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:32 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:33:32.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:33:32.731 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:32 smithi143 ceph-mon[183059]: pgmap v748: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 672 op/s; 31684/167481 objects degraded (18.918%); 3.1 MiB/s, 49 objects/s recovering 2023-08-28T02:33:32.731 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:32 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3452044152' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:33:32.731 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:32 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:33:32.776 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:33:32.797 INFO:teuthology.orchestra.run.smithi143.stderr:dumped all 2023-08-28T02:33:32.831 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:33:32.906 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:32.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:32 smithi169 ceph-mon[159415]: pgmap v748: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 672 op/s; 31684/167481 objects degraded (18.918%); 3.1 MiB/s, 49 objects/s recovering 2023-08-28T02:33:32.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:32 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3452044152' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:33:32.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:32 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:33:33.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:33 smithi143 ceph-mon[183059]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:33:33.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:33 smithi143 ceph-mon[183059]: failed to encode map e297 with expected crc 2023-08-28T02:33:33.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:33 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:33:33.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:33 smithi143 ceph-mon[183059]: osdmap e297: 8 total, 7 up, 7 in 2023-08-28T02:33:33.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:33 smithi143 ceph-mon[183059]: failed to encode map e297 with expected crc 2023-08-28T02:33:33.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:33 smithi143 ceph-mon[189871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:33:33.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:33 smithi143 ceph-mon[189871]: failed to encode map e297 with expected crc 2023-08-28T02:33:33.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:33 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:33:33.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:33 smithi143 ceph-mon[189871]: osdmap e297: 8 total, 7 up, 7 in 2023-08-28T02:33:33.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:33 smithi143 ceph-mon[189871]: failed to encode map e297 with expected crc 2023-08-28T02:33:33.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:33 smithi169 ceph-mon[159415]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:33:33.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:33 smithi169 ceph-mon[159415]: failed to encode map e297 with expected crc 2023-08-28T02:33:33.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:33 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:33:33.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:33 smithi169 ceph-mon[159415]: osdmap e297: 8 total, 7 up, 7 in 2023-08-28T02:33:33.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:33 smithi169 ceph-mon[159415]: failed to encode map e297 with expected crc 2023-08-28T02:33:34.318 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:33:34.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:34.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:33:34.425 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:33:34.452 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:34.515 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:33:34.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:34.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:33:34.623 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:33:34.626 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:34 smithi143 ceph-mon[183059]: from='client.45500 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:33:34.627 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:34 smithi143 ceph-mon[183059]: pgmap v750: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 683 op/s; 31684/164208 objects degraded (19.295%); 3.2 MiB/s, 50 objects/s recovering 2023-08-28T02:33:34.658 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:34.709 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:33:34.713 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:34 smithi169 ceph-mon[159415]: from='client.45500 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:33:34.713 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:34 smithi169 ceph-mon[159415]: pgmap v750: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 683 op/s; 31684/164208 objects degraded (19.295%); 3.2 MiB/s, 50 objects/s recovering 2023-08-28T02:33:34.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:34.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:33:34.816 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:33:34.847 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:34.906 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:33:34.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:34.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:33:34.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:34 smithi143 ceph-mon[189871]: from='client.45500 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:33:34.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:34 smithi143 ceph-mon[189871]: pgmap v750: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 683 op/s; 31684/164208 objects degraded (19.295%); 3.2 MiB/s, 50 objects/s recovering 2023-08-28T02:33:35.011 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:33:35.039 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:35.106 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:33:35.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:35.113 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:33:35.213 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:33:35.241 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:35.291 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:33:35.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:35.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:33:35.400 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:33:35.428 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:35.483 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:33:35.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:35.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:33:35.591 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:33:35.618 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:36.127 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:33:36.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:36.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:33:36.232 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:33:36.267 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:36.324 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:33:36.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:36.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:33:36.369 INFO:teuthology.orchestra.run.smithi143.stdout:{"pg_ready":true,"pg_map":{"version":749,"stamp":"2023-08-28T02:33:32.606234+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3590068171,"num_objects":54736,"num_object_clones":0,"num_object_copies":164208,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":31684,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":54736,"num_whiteouts":0,"num_read":6275,"num_read_kb":13708,"num_write":131934,"num_write_kb":5952230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6920,"num_bytes_recovered":444949176,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":157739,"ondisk_log_size":157739,"up":297,"acting":279,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":343,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":13386228,"kb_used_data":10670652,"kb_used_omap":29,"kb_used_meta":2715234,"kb_avail":642829836,"statfs":{"total":671965249536,"available":658257752064,"internally_reserved":0,"allocated":10926747648,"data_stored":10918823213,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30674,"internal_metadata":2780399662},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,1,8,2,5,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-441581568,"num_objects":-6738,"num_object_clones":0,"num_object_copies":-20214,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-498,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-6738,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6738,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":498,"num_bytes_recovered":32636928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.203333"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":315,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.555343+0000","last_change":"2023-08-28T02:32:58.740384+0000","last_active":"2023-08-28T02:33:11.555343+0000","last_peered":"2023-08-28T02:33:11.555343+0000","last_clean":"2023-08-28T02:33:11.555343+0000","last_became_active":"2023-08-28T02:32:58.739959+0000","last_became_peered":"2023-08-28T02:32:58.739959+0000","last_unstale":"2023-08-28T02:33:11.555343+0000","last_undegraded":"2023-08-28T02:33:11.555343+0000","last_fullsized":"2023-08-28T02:33:11.555343+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":201,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.553302+0000","last_change":"2023-08-28T02:32:58.634375+0000","last_active":"2023-08-28T02:33:11.553302+0000","last_peered":"2023-08-28T02:33:11.553302+0000","last_clean":"2023-08-28T02:33:11.553302+0000","last_became_active":"2023-08-28T02:32:58.632726+0000","last_became_peered":"2023-08-28T02:32:58.632726+0000","last_unstale":"2023-08-28T02:33:11.553302+0000","last_undegraded":"2023-08-28T02:33:11.553302+0000","last_fullsized":"2023-08-28T02:33:11.553302+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:20:27.730725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":15,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.722567+0000","last_change":"2023-08-28T02:32:58.628387+0000","last_active":"2023-08-28T02:33:11.722567+0000","last_peered":"2023-08-28T02:33:11.722567+0000","last_clean":"2023-08-28T02:33:11.722567+0000","last_became_active":"2023-08-28T02:32:58.628071+0000","last_became_peered":"2023-08-28T02:32:58.628071+0000","last_unstale":"2023-08-28T02:33:11.722567+0000","last_undegraded":"2023-08-28T02:33:11.722567+0000","last_fullsized":"2023-08-28T02:33:11.722567+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:25:55.264433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":312,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.726278+0000","last_change":"2023-08-28T02:32:58.772285+0000","last_active":"2023-08-28T02:33:11.726278+0000","last_peered":"2023-08-28T02:33:11.726278+0000","last_clean":"2023-08-28T02:33:11.726278+0000","last_became_active":"2023-08-28T02:32:58.771439+0000","last_became_peered":"2023-08-28T02:32:58.771439+0000","last_unstale":"2023-08-28T02:33:11.726278+0000","last_undegraded":"2023-08-28T02:33:11.726278+0000","last_fullsized":"2023-08-28T02:33:11.726278+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:10:48.541785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":192,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.554614+0000","last_change":"2023-08-28T02:32:58.779602+0000","last_active":"2023-08-28T02:33:11.554614+0000","last_peered":"2023-08-28T02:33:11.554614+0000","last_clean":"2023-08-28T02:33:11.554614+0000","last_became_active":"2023-08-28T02:32:58.778945+0000","last_became_peered":"2023-08-28T02:32:58.778945+0000","last_unstale":"2023-08-28T02:33:11.554614+0000","last_undegraded":"2023-08-28T02:33:11.554614+0000","last_fullsized":"2023-08-28T02:33:11.554614+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:35:55.832928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":102,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.740834+0000","last_change":"2023-08-28T02:32:58.626766+0000","last_active":"2023-08-28T02:33:11.740834+0000","last_peered":"2023-08-28T02:33:11.740834+0000","last_clean":"2023-08-28T02:33:11.740834+0000","last_became_active":"2023-08-28T02:32:58.626415+0000","last_became_peered":"2023-08-28T02:32:58.626415+0000","last_unstale":"2023-08-28T02:33:11.740834+0000","last_undegraded":"2023-08-28T02:33:11.740834+0000","last_fullsized":"2023-08-28T02:33:11.740834+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.13","version":"296'5968","reported_seq":6022,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-08-28T02:33:29.691179+0000","last_change":"2023-08-28T02:31:44.484177+0000","last_active":"2023-08-28T02:33:29.691179+0000","last_peered":"2023-08-28T02:33:29.691179+0000","last_clean":"2023-08-28T02:33:29.691179+0000","last_became_active":"2023-08-28T02:31:44.482443+0000","last_became_peered":"2023-08-28T02:31:44.482443+0000","last_unstale":"2023-08-28T02:33:29.691179+0000","last_undegraded":"2023-08-28T02:33:29.691179+0000","last_fullsized":"2023-08-28T02:33:29.691179+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5968,"log_dups_size":0,"ondisk_log_size":5968,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113377280,"num_objects":1730,"num_object_clones":0,"num_object_copies":5190,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1730,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4098,"num_write_kb":186496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":265,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.723924+0000","last_change":"2023-08-28T02:23:59.227592+0000","last_active":"2023-08-28T02:33:11.723924+0000","last_peered":"2023-08-28T02:33:11.723924+0000","last_clean":"2023-08-28T02:33:11.723924+0000","last_became_active":"2023-08-28T02:23:59.227342+0000","last_became_peered":"2023-08-28T02:23:59.227342+0000","last_unstale":"2023-08-28T02:33:11.723924+0000","last_undegraded":"2023-08-28T02:33:11.723924+0000","last_fullsized":"2023-08-28T02:33:11.723924+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:08:19.874396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":211,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:19.774071+0000","last_change":"2023-08-28T02:32:58.779435+0000","last_active":"2023-08-28T02:33:19.774071+0000","last_peered":"2023-08-28T02:33:19.774071+0000","last_clean":"2023-08-28T02:33:19.774071+0000","last_became_active":"2023-08-28T02:32:58.778629+0000","last_became_peered":"2023-08-28T02:32:58.778629+0000","last_unstale":"2023-08-28T02:33:19.774071+0000","last_undegraded":"2023-08-28T02:33:19.774071+0000","last_fullsized":"2023-08-28T02:33:19.774071+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:25:32.193898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.12","version":"296'6032","reported_seq":6102,"reported_epoch":296,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:33:29.693813+0000","last_change":"2023-08-28T02:32:58.866748+0000","last_active":"2023-08-28T02:33:29.693813+0000","last_peered":"2023-08-28T02:33:29.693813+0000","last_clean":"2023-08-28T02:32:11.498129+0000","last_became_active":"2023-08-28T02:32:58.859951+0000","last_became_peered":"2023-08-28T02:32:58.859951+0000","last_unstale":"2023-08-28T02:33:29.693813+0000","last_undegraded":"2023-08-28T02:32:58.636278+0000","last_fullsized":"2023-08-28T02:32:58.628896+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":6032,"log_dups_size":0,"ondisk_log_size":6032,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112525312,"num_objects":1717,"num_object_clones":0,"num_object_copies":5151,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":970,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1717,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4127,"num_write_kb":187008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[{"shards":"2,3","objects":1324},{"shards":"2,3,6","objects":393}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":325,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.551524+0000","last_change":"2023-08-28T02:32:58.636365+0000","last_active":"2023-08-28T02:33:11.551524+0000","last_peered":"2023-08-28T02:33:11.551524+0000","last_clean":"2023-08-28T02:33:11.551524+0000","last_became_active":"2023-08-28T02:32:58.635998+0000","last_became_peered":"2023-08-28T02:32:58.635998+0000","last_unstale":"2023-08-28T02:33:11.551524+0000","last_undegraded":"2023-08-28T02:33:11.551524+0000","last_fullsized":"2023-08-28T02:33:11.551524+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":184,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:19.774104+0000","last_change":"2023-08-28T02:31:55.525178+0000","last_active":"2023-08-28T02:33:19.774104+0000","last_peered":"2023-08-28T02:33:19.774104+0000","last_clean":"2023-08-28T02:33:19.774104+0000","last_became_active":"2023-08-28T02:31:55.524801+0000","last_became_peered":"2023-08-28T02:31:55.524801+0000","last_unstale":"2023-08-28T02:33:19.774104+0000","last_undegraded":"2023-08-28T02:33:19.774104+0000","last_fullsized":"2023-08-28T02:33:19.774104+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:47:30.078960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.11","version":"296'6021","reported_seq":6090,"reported_epoch":296,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:33:32.137371+0000","last_change":"2023-08-28T02:32:58.859410+0000","last_active":"2023-08-28T02:33:32.137371+0000","last_peered":"2023-08-28T02:33:32.137371+0000","last_clean":"2023-08-28T02:31:51.768660+0000","last_became_active":"2023-08-28T02:32:58.852854+0000","last_became_peered":"2023-08-28T02:32:58.852854+0000","last_unstale":"2023-08-28T02:33:32.137371+0000","last_undegraded":"2023-08-28T02:32:58.629343+0000","last_fullsized":"2023-08-28T02:32:58.617323+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":6021,"log_dups_size":0,"ondisk_log_size":6021,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114720768,"num_objects":1751,"num_object_clones":0,"num_object_copies":5253,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2588,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1751,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4137,"num_write_kb":188384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3],"avail_no_missing":["6","3","4"],"object_location_counts":[{"shards":"2,3,6","objects":411},{"shards":"3,4,6","objects":984},{"shards":"3,6","objects":-274}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":62,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.553462+0000","last_change":"2023-08-28T02:31:55.532306+0000","last_active":"2023-08-28T02:33:11.553462+0000","last_peered":"2023-08-28T02:33:11.553462+0000","last_clean":"2023-08-28T02:33:11.553462+0000","last_became_active":"2023-08-28T02:31:55.524276+0000","last_became_peered":"2023-08-28T02:31:55.524276+0000","last_unstale":"2023-08-28T02:33:11.553462+0000","last_undegraded":"2023-08-28T02:33:11.553462+0000","last_fullsized":"2023-08-28T02:33:11.553462+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:27:06.850784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":15,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.722934+0000","last_change":"2023-08-28T02:32:58.658343+0000","last_active":"2023-08-28T02:33:11.722934+0000","last_peered":"2023-08-28T02:33:11.722934+0000","last_clean":"2023-08-28T02:33:11.722934+0000","last_became_active":"2023-08-28T02:32:58.654410+0000","last_became_peered":"2023-08-28T02:32:58.654410+0000","last_unstale":"2023-08-28T02:33:11.722934+0000","last_undegraded":"2023-08-28T02:33:11.722934+0000","last_fullsized":"2023-08-28T02:33:11.722934+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:46:29.260724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.10","version":"296'5915","reported_seq":8371,"reported_epoch":296,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:33:29.770128+0000","last_change":"2023-08-28T02:32:58.871328+0000","last_active":"2023-08-28T02:33:29.770128+0000","last_peered":"2023-08-28T02:33:29.770128+0000","last_clean":"2023-08-28T02:31:51.241963+0000","last_became_active":"2023-08-28T02:32:58.820349+0000","last_became_peered":"2023-08-28T02:32:58.820349+0000","last_unstale":"2023-08-28T02:33:29.770128+0000","last_undegraded":"2023-08-28T02:32:58.770460+0000","last_fullsized":"2023-08-28T02:32:58.618203+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5915,"log_dups_size":0,"ondisk_log_size":5915,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110723072,"num_objects":1690,"num_object_clones":0,"num_object_copies":5070,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1873,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1690,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4066,"num_write_kb":184160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1190,"num_bytes_recovered":77856768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"1,3,5","objects":933},{"shards":"2,3","objects":371},{"shards":"2,3,4","objects":386}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"86'22","reported_seq":365,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.740327+0000","last_change":"2023-08-28T02:31:55.538001+0000","last_active":"2023-08-28T02:33:11.740327+0000","last_peered":"2023-08-28T02:33:11.740327+0000","last_clean":"2023-08-28T02:33:11.740327+0000","last_became_active":"2023-08-28T02:31:55.518604+0000","last_became_peered":"2023-08-28T02:31:55.518604+0000","last_unstale":"2023-08-28T02:33:11.740327+0000","last_undegraded":"2023-08-28T02:33:11.740327+0000","last_fullsized":"2023-08-28T02:33:11.740327+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":339,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.555322+0000","last_change":"2023-08-28T02:32:58.629451+0000","last_active":"2023-08-28T02:33:11.555322+0000","last_peered":"2023-08-28T02:33:11.555322+0000","last_clean":"2023-08-28T02:33:11.555322+0000","last_became_active":"2023-08-28T02:32:58.628767+0000","last_became_peered":"2023-08-28T02:32:58.628767+0000","last_unstale":"2023-08-28T02:33:11.555322+0000","last_undegraded":"2023-08-28T02:33:11.555322+0000","last_fullsized":"2023-08-28T02:33:11.555322+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.17","version":"296'5907","reported_seq":5957,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-08-28T02:33:32.112677+0000","last_change":"2023-08-28T02:31:44.475304+0000","last_active":"2023-08-28T02:33:32.112677+0000","last_peered":"2023-08-28T02:33:32.112677+0000","last_clean":"2023-08-28T02:33:32.112677+0000","last_became_active":"2023-08-28T02:31:44.473853+0000","last_became_peered":"2023-08-28T02:31:44.473853+0000","last_unstale":"2023-08-28T02:33:32.112677+0000","last_undegraded":"2023-08-28T02:33:32.112677+0000","last_fullsized":"2023-08-28T02:33:32.112677+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5907,"log_dups_size":0,"ondisk_log_size":5907,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110428160,"num_objects":1685,"num_object_clones":0,"num_object_copies":5055,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1685,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4043,"num_write_kb":183296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":69,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.555359+0000","last_change":"2023-08-28T02:31:55.528552+0000","last_active":"2023-08-28T02:33:11.555359+0000","last_peered":"2023-08-28T02:33:11.555359+0000","last_clean":"2023-08-28T02:33:11.555359+0000","last_became_active":"2023-08-28T02:31:55.518079+0000","last_became_peered":"2023-08-28T02:31:55.518079+0000","last_unstale":"2023-08-28T02:33:11.555359+0000","last_undegraded":"2023-08-28T02:33:11.555359+0000","last_fullsized":"2023-08-28T02:33:11.555359+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":202,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:19.774197+0000","last_change":"2023-08-28T02:32:58.680534+0000","last_active":"2023-08-28T02:33:19.774197+0000","last_peered":"2023-08-28T02:33:19.774197+0000","last_clean":"2023-08-28T02:33:19.774197+0000","last_became_active":"2023-08-28T02:32:58.680145+0000","last_became_peered":"2023-08-28T02:32:58.680145+0000","last_unstale":"2023-08-28T02:33:19.774197+0000","last_undegraded":"2023-08-28T02:33:19.774197+0000","last_fullsized":"2023-08-28T02:33:19.774197+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:56:40.113013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.16","version":"296'5874","reported_seq":10166,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-08-28T02:33:29.379826+0000","last_change":"2023-08-28T02:32:37.600254+0000","last_active":"2023-08-28T02:33:29.379826+0000","last_peered":"2023-08-28T02:33:29.379826+0000","last_clean":"2023-08-28T02:33:29.379826+0000","last_became_active":"2023-08-28T02:32:37.596763+0000","last_became_peered":"2023-08-28T02:32:37.596763+0000","last_unstale":"2023-08-28T02:33:29.379826+0000","last_undegraded":"2023-08-28T02:33:29.379826+0000","last_fullsized":"2023-08-28T02:33:29.379826+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":270,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5874,"log_dups_size":0,"ondisk_log_size":5874,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106528768,"num_objects":1626,"num_object_clones":0,"num_object_copies":4878,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1626,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4068,"num_write_kb":182176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2111,"num_bytes_recovered":138346496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":310,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.725507+0000","last_change":"2023-08-28T02:32:58.746929+0000","last_active":"2023-08-28T02:33:11.725507+0000","last_peered":"2023-08-28T02:33:11.725507+0000","last_clean":"2023-08-28T02:33:11.725507+0000","last_became_active":"2023-08-28T02:32:58.746590+0000","last_became_peered":"2023-08-28T02:32:58.746590+0000","last_unstale":"2023-08-28T02:33:11.725507+0000","last_undegraded":"2023-08-28T02:33:11.725507+0000","last_fullsized":"2023-08-28T02:33:11.725507+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:51:11.974154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":331,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.551542+0000","last_change":"2023-08-28T02:31:55.521470+0000","last_active":"2023-08-28T02:33:11.551542+0000","last_peered":"2023-08-28T02:33:11.551542+0000","last_clean":"2023-08-28T02:33:11.551542+0000","last_became_active":"2023-08-28T02:31:55.520398+0000","last_became_peered":"2023-08-28T02:31:55.520398+0000","last_unstale":"2023-08-28T02:33:11.551542+0000","last_undegraded":"2023-08-28T02:33:11.551542+0000","last_fullsized":"2023-08-28T02:33:11.551542+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.15","version":"296'5824","reported_seq":5898,"reported_epoch":296,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:33:29.636765+0000","last_change":"2023-08-28T02:32:58.933245+0000","last_active":"2023-08-28T02:33:29.636765+0000","last_peered":"2023-08-28T02:33:29.636765+0000","last_clean":"2023-08-28T02:31:54.454430+0000","last_became_active":"2023-08-28T02:32:58.916111+0000","last_became_peered":"2023-08-28T02:32:58.916111+0000","last_unstale":"2023-08-28T02:33:29.636765+0000","last_undegraded":"2023-08-28T02:32:58.647695+0000","last_fullsized":"2023-08-28T02:32:58.611609+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5824,"log_dups_size":0,"ondisk_log_size":5824,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115310592,"num_objects":1760,"num_object_clones":0,"num_object_copies":5280,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2539,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1760,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4002,"num_write_kb":184352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":-245},{"shards":"0,2,6","objects":392}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":125,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.555388+0000","last_change":"2023-08-28T02:31:55.517851+0000","last_active":"2023-08-28T02:33:11.555388+0000","last_peered":"2023-08-28T02:33:11.555388+0000","last_clean":"2023-08-28T02:33:11.555388+0000","last_became_active":"2023-08-28T02:31:55.517320+0000","last_became_peered":"2023-08-28T02:31:55.517320+0000","last_unstale":"2023-08-28T02:33:11.555388+0000","last_undegraded":"2023-08-28T02:33:11.555388+0000","last_fullsized":"2023-08-28T02:33:11.555388+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":293,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.726660+0000","last_change":"2023-08-28T02:31:55.528250+0000","last_active":"2023-08-28T02:33:11.726660+0000","last_peered":"2023-08-28T02:33:11.726660+0000","last_clean":"2023-08-28T02:33:11.726660+0000","last_became_active":"2023-08-28T02:31:55.527950+0000","last_became_peered":"2023-08-28T02:31:55.527950+0000","last_unstale":"2023-08-28T02:33:11.726660+0000","last_undegraded":"2023-08-28T02:33:11.726660+0000","last_fullsized":"2023-08-28T02:33:11.726660+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:39:46.120437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.14","version":"296'6054","reported_seq":6124,"reported_epoch":296,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:33:29.173698+0000","last_change":"2023-08-28T02:32:58.849338+0000","last_active":"2023-08-28T02:33:29.173698+0000","last_peered":"2023-08-28T02:33:29.173698+0000","last_clean":"2023-08-28T02:32:11.458469+0000","last_became_active":"2023-08-28T02:32:58.840304+0000","last_became_peered":"2023-08-28T02:32:58.840304+0000","last_unstale":"2023-08-28T02:33:29.173698+0000","last_undegraded":"2023-08-28T02:32:58.655685+0000","last_fullsized":"2023-08-28T02:32:58.616249+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":6054,"log_dups_size":0,"ondisk_log_size":6054,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117932032,"num_objects":1800,"num_object_clones":0,"num_object_copies":5400,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":979,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1800,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4126,"num_write_kb":189600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":1383},{"shards":"0,4,6","objects":417}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":94,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.727294+0000","last_change":"2023-08-28T02:28:25.703333+0000","last_active":"2023-08-28T02:33:11.727294+0000","last_peered":"2023-08-28T02:33:11.727294+0000","last_clean":"2023-08-28T02:33:11.727294+0000","last_became_active":"2023-08-28T02:28:25.343515+0000","last_became_peered":"2023-08-28T02:28:25.343515+0000","last_unstale":"2023-08-28T02:33:11.727294+0000","last_undegraded":"2023-08-28T02:33:11.727294+0000","last_fullsized":"2023-08-28T02:33:11.727294+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":201,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.553420+0000","last_change":"2023-08-28T02:32:58.758016+0000","last_active":"2023-08-28T02:33:11.553420+0000","last_peered":"2023-08-28T02:33:11.553420+0000","last_clean":"2023-08-28T02:33:11.553420+0000","last_became_active":"2023-08-28T02:32:58.757660+0000","last_became_peered":"2023-08-28T02:32:58.757660+0000","last_unstale":"2023-08-28T02:33:11.553420+0000","last_undegraded":"2023-08-28T02:33:11.553420+0000","last_fullsized":"2023-08-28T02:33:11.553420+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:33:06.239503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"296'8108","reported_seq":8167,"reported_epoch":296,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:33:31.951092+0000","last_change":"2023-08-28T02:32:58.916501+0000","last_active":"2023-08-28T02:33:31.951092+0000","last_peered":"2023-08-28T02:33:31.951092+0000","last_clean":"2023-08-28T02:32:11.456138+0000","last_became_active":"2023-08-28T02:32:58.901683+0000","last_became_peered":"2023-08-28T02:32:58.901683+0000","last_unstale":"2023-08-28T02:33:31.951092+0000","last_undegraded":"2023-08-28T02:32:58.633417+0000","last_fullsized":"2023-08-28T02:32:58.615180+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":8108,"log_dups_size":0,"ondisk_log_size":8108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":223477760,"num_objects":3410,"num_object_clones":0,"num_object_copies":10230,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1873,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3410,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8108,"num_write_kb":368576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"2,5,6","objects":836},{"shards":"5,6","objects":2574}],"blocked_by":[],"up_primary":1,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":157,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.752570+0000","last_change":"2023-08-28T02:32:58.782840+0000","last_active":"2023-08-28T02:33:11.752570+0000","last_peered":"2023-08-28T02:33:11.752570+0000","last_clean":"2023-08-28T02:33:11.752570+0000","last_became_active":"2023-08-28T02:32:58.683096+0000","last_became_peered":"2023-08-28T02:32:58.683096+0000","last_unstale":"2023-08-28T02:33:11.752570+0000","last_undegraded":"2023-08-28T02:33:11.752570+0000","last_fullsized":"2023-08-28T02:33:11.752570+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:10:08.037078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":291,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.551201+0000","last_change":"2023-08-28T02:31:55.514646+0000","last_active":"2023-08-28T02:33:11.551201+0000","last_peered":"2023-08-28T02:33:11.551201+0000","last_clean":"2023-08-28T02:33:11.551201+0000","last_became_active":"2023-08-28T02:31:55.514306+0000","last_became_peered":"2023-08-28T02:31:55.514306+0000","last_unstale":"2023-08-28T02:33:11.551201+0000","last_undegraded":"2023-08-28T02:33:11.551201+0000","last_fullsized":"2023-08-28T02:33:11.551201+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"296'8325","reported_seq":8370,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-08-28T02:33:29.769919+0000","last_change":"2023-08-28T02:30:57.067718+0000","last_active":"2023-08-28T02:33:29.769919+0000","last_peered":"2023-08-28T02:33:29.769919+0000","last_clean":"2023-08-28T02:33:29.769919+0000","last_became_active":"2023-08-28T02:30:57.067407+0000","last_became_peered":"2023-08-28T02:30:57.067407+0000","last_unstale":"2023-08-28T02:33:29.769919+0000","last_undegraded":"2023-08-28T02:33:29.769919+0000","last_fullsized":"2023-08-28T02:33:29.769919+0000","mapping_epoch":261,"log_start":"267'2965","ondisk_log_start":"267'2965","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5360,"log_dups_size":0,"ondisk_log_size":5360,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:49:39.360134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225771520,"num_objects":3445,"num_object_clones":0,"num_object_copies":10335,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3445,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8325,"num_write_kb":376640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":91,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.555452+0000","last_change":"2023-08-28T02:32:58.856906+0000","last_active":"2023-08-28T02:33:11.555452+0000","last_peered":"2023-08-28T02:33:11.555452+0000","last_clean":"2023-08-28T02:33:11.555452+0000","last_became_active":"2023-08-28T02:32:58.740028+0000","last_became_peered":"2023-08-28T02:32:58.740028+0000","last_unstale":"2023-08-28T02:33:11.555452+0000","last_undegraded":"2023-08-28T02:33:11.555452+0000","last_fullsized":"2023-08-28T02:33:11.555452+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":349,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.555374+0000","last_change":"2023-08-28T02:32:58.740396+0000","last_active":"2023-08-28T02:33:11.555374+0000","last_peered":"2023-08-28T02:33:11.555374+0000","last_clean":"2023-08-28T02:33:11.555374+0000","last_became_active":"2023-08-28T02:32:58.739941+0000","last_became_peered":"2023-08-28T02:32:58.739941+0000","last_unstale":"2023-08-28T02:33:11.555374+0000","last_undegraded":"2023-08-28T02:33:11.555374+0000","last_fullsized":"2023-08-28T02:33:11.555374+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"296'6020","reported_seq":6091,"reported_epoch":296,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:33:29.302504+0000","last_change":"2023-08-28T02:32:58.880642+0000","last_active":"2023-08-28T02:33:29.302504+0000","last_peered":"2023-08-28T02:33:29.302504+0000","last_clean":"2023-08-28T02:32:11.446297+0000","last_became_active":"2023-08-28T02:32:58.851158+0000","last_became_peered":"2023-08-28T02:32:58.851158+0000","last_unstale":"2023-08-28T02:33:29.302504+0000","last_undegraded":"2023-08-28T02:32:58.644337+0000","last_fullsized":"2023-08-28T02:32:58.616553+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":6020,"log_dups_size":0,"ondisk_log_size":6020,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110067712,"num_objects":1679,"num_object_clones":0,"num_object_copies":5037,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1015,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1679,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4115,"num_write_kb":185440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":1277},{"shards":"0,4,6","objects":402}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":65,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.551628+0000","last_change":"2023-08-28T02:31:55.531216+0000","last_active":"2023-08-28T02:33:11.551628+0000","last_peered":"2023-08-28T02:33:11.551628+0000","last_clean":"2023-08-28T02:33:11.551628+0000","last_became_active":"2023-08-28T02:31:55.521010+0000","last_became_peered":"2023-08-28T02:31:55.521010+0000","last_unstale":"2023-08-28T02:33:11.551628+0000","last_undegraded":"2023-08-28T02:33:11.551628+0000","last_fullsized":"2023-08-28T02:33:11.551628+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":111,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.727389+0000","last_change":"2023-08-28T02:32:58.628915+0000","last_active":"2023-08-28T02:33:11.727389+0000","last_peered":"2023-08-28T02:33:11.727389+0000","last_clean":"2023-08-28T02:33:11.727389+0000","last_became_active":"2023-08-28T02:32:58.628562+0000","last_became_peered":"2023-08-28T02:32:58.628562+0000","last_unstale":"2023-08-28T02:33:11.727389+0000","last_undegraded":"2023-08-28T02:33:11.727389+0000","last_fullsized":"2023-08-28T02:33:11.727389+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"296'6044","reported_seq":8934,"reported_epoch":296,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:33:29.655767+0000","last_change":"2023-08-28T02:32:58.683607+0000","last_active":"2023-08-28T02:33:29.655767+0000","last_peered":"2023-08-28T02:33:29.655767+0000","last_clean":"2023-08-28T02:31:54.438195+0000","last_became_active":"2023-08-28T02:31:56.704078+0000","last_became_peered":"2023-08-28T02:31:56.704078+0000","last_unstale":"2023-08-28T02:33:29.655767+0000","last_undegraded":"2023-08-28T02:31:56.543040+0000","last_fullsized":"2023-08-28T02:31:56.504447+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":6044,"log_dups_size":0,"ondisk_log_size":6044,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114688000,"num_objects":1750,"num_object_clones":0,"num_object_copies":5250,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":759,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1750,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4126,"num_write_kb":188032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1412,"num_bytes_recovered":92536832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,4","objects":991}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":299,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.553767+0000","last_change":"2023-08-28T02:28:25.445982+0000","last_active":"2023-08-28T02:33:11.553767+0000","last_peered":"2023-08-28T02:33:11.553767+0000","last_clean":"2023-08-28T02:33:11.553767+0000","last_became_active":"2023-08-28T02:28:25.443577+0000","last_became_peered":"2023-08-28T02:28:25.443577+0000","last_unstale":"2023-08-28T02:33:11.553767+0000","last_undegraded":"2023-08-28T02:33:11.553767+0000","last_fullsized":"2023-08-28T02:33:11.553767+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:24:03.381751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":295,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.725636+0000","last_change":"2023-08-28T02:32:58.887874+0000","last_active":"2023-08-28T02:33:11.725636+0000","last_peered":"2023-08-28T02:33:11.725636+0000","last_clean":"2023-08-28T02:33:11.725636+0000","last_became_active":"2023-08-28T02:32:58.887377+0000","last_became_peered":"2023-08-28T02:32:58.887377+0000","last_unstale":"2023-08-28T02:33:11.725636+0000","last_undegraded":"2023-08-28T02:33:11.725636+0000","last_fullsized":"2023-08-28T02:33:11.725636+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:08:07.234984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"296'8261","reported_seq":10757,"reported_epoch":296,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-08-28T02:33:29.386087+0000","last_change":"2023-08-28T02:32:58.846760+0000","last_active":"2023-08-28T02:33:29.386087+0000","last_peered":"2023-08-28T02:33:29.386087+0000","last_clean":"2023-08-28T02:31:54.434903+0000","last_became_active":"2023-08-28T02:31:56.674791+0000","last_became_peered":"2023-08-28T02:31:56.674791+0000","last_unstale":"2023-08-28T02:33:29.386087+0000","last_undegraded":"2023-08-28T02:31:56.545322+0000","last_fullsized":"2023-08-28T02:31:56.508287+0000","mapping_epoch":277,"log_start":"267'2600","ondisk_log_start":"267'2600","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5661,"log_dups_size":0,"ondisk_log_size":5661,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:26:02.602584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228261888,"num_objects":3483,"num_object_clones":0,"num_object_copies":10449,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3104,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3483,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8261,"num_write_kb":375808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1221,"num_bytes_recovered":80019456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":379}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":161,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.553098+0000","last_change":"2023-08-28T02:23:59.230059+0000","last_active":"2023-08-28T02:33:11.553098+0000","last_peered":"2023-08-28T02:33:11.553098+0000","last_clean":"2023-08-28T02:33:11.553098+0000","last_became_active":"2023-08-28T02:23:59.217783+0000","last_became_peered":"2023-08-28T02:23:59.217783+0000","last_unstale":"2023-08-28T02:33:11.553098+0000","last_undegraded":"2023-08-28T02:33:11.553098+0000","last_fullsized":"2023-08-28T02:33:11.553098+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:00:12.145395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":316,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.740206+0000","last_change":"2023-08-28T02:28:25.351698+0000","last_active":"2023-08-28T02:33:11.740206+0000","last_peered":"2023-08-28T02:33:11.740206+0000","last_clean":"2023-08-28T02:33:11.740206+0000","last_became_active":"2023-08-28T02:28:25.351349+0000","last_became_peered":"2023-08-28T02:28:25.351349+0000","last_unstale":"2023-08-28T02:33:11.740206+0000","last_undegraded":"2023-08-28T02:33:11.740206+0000","last_fullsized":"2023-08-28T02:33:11.740206+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"296'8230","reported_seq":8275,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-08-28T02:33:29.628429+0000","last_change":"2023-08-28T02:30:57.069579+0000","last_active":"2023-08-28T02:33:29.628429+0000","last_peered":"2023-08-28T02:33:29.628429+0000","last_clean":"2023-08-28T02:33:29.628429+0000","last_became_active":"2023-08-28T02:30:57.069365+0000","last_became_peered":"2023-08-28T02:30:57.069365+0000","last_unstale":"2023-08-28T02:33:29.628429+0000","last_undegraded":"2023-08-28T02:33:29.628429+0000","last_fullsized":"2023-08-28T02:33:29.628429+0000","mapping_epoch":261,"log_start":"266'1700","ondisk_log_start":"266'1700","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":6530,"log_dups_size":0,"ondisk_log_size":6530,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:02:07.549000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228720640,"num_objects":3490,"num_object_clones":0,"num_object_copies":10470,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3490,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8230,"num_write_kb":375040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":102,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.727297+0000","last_change":"2023-08-28T02:32:58.629249+0000","last_active":"2023-08-28T02:33:11.727297+0000","last_peered":"2023-08-28T02:33:11.727297+0000","last_clean":"2023-08-28T02:33:11.727297+0000","last_became_active":"2023-08-28T02:32:58.628583+0000","last_became_peered":"2023-08-28T02:32:58.628583+0000","last_unstale":"2023-08-28T02:33:11.727297+0000","last_undegraded":"2023-08-28T02:33:11.727297+0000","last_fullsized":"2023-08-28T02:33:11.727297+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":360,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.739976+0000","last_change":"2023-08-28T02:32:58.663953+0000","last_active":"2023-08-28T02:33:11.739976+0000","last_peered":"2023-08-28T02:33:11.739976+0000","last_clean":"2023-08-28T02:33:11.739976+0000","last_became_active":"2023-08-28T02:32:58.663627+0000","last_became_peered":"2023-08-28T02:32:58.663627+0000","last_unstale":"2023-08-28T02:33:11.739976+0000","last_undegraded":"2023-08-28T02:33:11.739976+0000","last_fullsized":"2023-08-28T02:33:11.739976+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"296'8005","reported_seq":8065,"reported_epoch":296,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:33:32.161483+0000","last_change":"2023-08-28T02:32:58.874546+0000","last_active":"2023-08-28T02:33:32.161483+0000","last_peered":"2023-08-28T02:33:32.161483+0000","last_clean":"2023-08-28T02:32:11.510497+0000","last_became_active":"2023-08-28T02:32:58.863497+0000","last_became_peered":"2023-08-28T02:32:58.863497+0000","last_unstale":"2023-08-28T02:33:32.161483+0000","last_undegraded":"2023-08-28T02:32:58.633053+0000","last_fullsized":"2023-08-28T02:32:58.617566+0000","mapping_epoch":291,"log_start":"266'2200","ondisk_log_start":"266'2200","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5805,"log_dups_size":0,"ondisk_log_size":5805,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":215613464,"num_objects":3291,"num_object_clones":0,"num_object_copies":9873,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1818,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3291,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8005,"num_write_kb":361409,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,3,6","objects":768},{"shards":"0,6","objects":2523}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":232,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.555488+0000","last_change":"2023-08-28T02:32:58.740311+0000","last_active":"2023-08-28T02:33:11.555488+0000","last_peered":"2023-08-28T02:33:11.555488+0000","last_clean":"2023-08-28T02:33:11.555488+0000","last_became_active":"2023-08-28T02:32:58.739897+0000","last_became_peered":"2023-08-28T02:32:58.739897+0000","last_unstale":"2023-08-28T02:33:11.555488+0000","last_undegraded":"2023-08-28T02:33:11.555488+0000","last_fullsized":"2023-08-28T02:33:11.555488+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":282,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:14.378105+0000","last_change":"2023-08-28T02:32:58.766795+0000","last_active":"2023-08-28T02:33:14.378105+0000","last_peered":"2023-08-28T02:33:14.378105+0000","last_clean":"2023-08-28T02:33:14.378105+0000","last_became_active":"2023-08-28T02:32:58.766441+0000","last_became_peered":"2023-08-28T02:32:58.766441+0000","last_unstale":"2023-08-28T02:33:14.378105+0000","last_undegraded":"2023-08-28T02:33:14.378105+0000","last_fullsized":"2023-08-28T02:33:14.378105+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:58:13.963961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.c","version":"296'8108","reported_seq":8158,"reported_epoch":296,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:33:32.196389+0000","last_change":"2023-08-28T02:31:56.737061+0000","last_active":"2023-08-28T02:33:32.196389+0000","last_peered":"2023-08-28T02:33:32.196389+0000","last_clean":"2023-08-28T02:31:54.457709+0000","last_became_active":"2023-08-28T02:31:56.726804+0000","last_became_peered":"2023-08-28T02:31:56.726804+0000","last_unstale":"2023-08-28T02:33:32.196389+0000","last_undegraded":"2023-08-28T02:31:56.538425+0000","last_fullsized":"2023-08-28T02:31:56.509601+0000","mapping_epoch":277,"log_start":"266'2300","ondisk_log_start":"266'2300","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5808,"log_dups_size":0,"ondisk_log_size":5808,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214958080,"num_objects":3280,"num_object_clones":0,"num_object_copies":9840,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4274,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3280,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8108,"num_write_kb":364416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3],"avail_no_missing":["6","3"],"object_location_counts":[{"shards":"3,6","objects":-994}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":202,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.554846+0000","last_change":"2023-08-28T02:32:58.779257+0000","last_active":"2023-08-28T02:33:11.554846+0000","last_peered":"2023-08-28T02:33:11.554846+0000","last_clean":"2023-08-28T02:33:11.554846+0000","last_became_active":"2023-08-28T02:32:58.778295+0000","last_became_peered":"2023-08-28T02:32:58.778295+0000","last_unstale":"2023-08-28T02:33:11.554846+0000","last_undegraded":"2023-08-28T02:33:11.554846+0000","last_fullsized":"2023-08-28T02:33:11.554846+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:50:49.444715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":15,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.725188+0000","last_change":"2023-08-28T02:32:58.755765+0000","last_active":"2023-08-28T02:33:11.725188+0000","last_peered":"2023-08-28T02:33:11.725188+0000","last_clean":"2023-08-28T02:33:11.725188+0000","last_became_active":"2023-08-28T02:32:58.755438+0000","last_became_peered":"2023-08-28T02:32:58.755438+0000","last_unstale":"2023-08-28T02:33:11.725188+0000","last_undegraded":"2023-08-28T02:33:11.725188+0000","last_fullsized":"2023-08-28T02:33:11.725188+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:14:11.498611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"296'5974","reported_seq":6030,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-08-28T02:33:31.874349+0000","last_change":"2023-08-28T02:32:14.983506+0000","last_active":"2023-08-28T02:33:31.874349+0000","last_peered":"2023-08-28T02:33:31.874349+0000","last_clean":"2023-08-28T02:33:31.874349+0000","last_became_active":"2023-08-28T02:31:44.479295+0000","last_became_peered":"2023-08-28T02:31:44.479295+0000","last_unstale":"2023-08-28T02:33:31.874349+0000","last_undegraded":"2023-08-28T02:33:31.874349+0000","last_fullsized":"2023-08-28T02:33:31.874349+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5974,"log_dups_size":0,"ondisk_log_size":5974,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111247360,"num_objects":1697,"num_object_clones":0,"num_object_copies":5091,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1697,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4089,"num_write_kb":185184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"78'126","reported_seq":673,"reported_epoch":295,"state":"active+recovery_wait+degraded","last_fresh":"2023-08-28T02:33:11.722371+0000","last_change":"2023-08-28T02:32:59.660815+0000","last_active":"2023-08-28T02:33:11.722371+0000","last_peered":"2023-08-28T02:33:11.722371+0000","last_clean":"2023-08-28T02:27:29.502868+0000","last_became_active":"2023-08-28T02:32:59.660388+0000","last_became_peered":"2023-08-28T02:32:59.660388+0000","last_unstale":"2023-08-28T02:33:11.722371+0000","last_undegraded":"2023-08-28T02:32:59.633238+0000","last_fullsized":"2023-08-28T02:33:11.722371+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:58:54.023571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":743,"num_read_kb":1331,"num_write":125,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":["1","0"],"object_location_counts":[{"shards":"0,1","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":295,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.551457+0000","last_change":"2023-08-28T02:23:59.207148+0000","last_active":"2023-08-28T02:33:11.551457+0000","last_peered":"2023-08-28T02:33:11.551457+0000","last_clean":"2023-08-28T02:33:11.551457+0000","last_became_active":"2023-08-28T02:23:59.206802+0000","last_became_peered":"2023-08-28T02:23:59.206802+0000","last_unstale":"2023-08-28T02:33:11.551457+0000","last_undegraded":"2023-08-28T02:33:11.551457+0000","last_fullsized":"2023-08-28T02:33:11.551457+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":15,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.722641+0000","last_change":"2023-08-28T02:32:58.626357+0000","last_active":"2023-08-28T02:33:11.722641+0000","last_peered":"2023-08-28T02:33:11.722641+0000","last_clean":"2023-08-28T02:33:11.722641+0000","last_became_active":"2023-08-28T02:32:58.622671+0000","last_became_peered":"2023-08-28T02:32:58.622671+0000","last_unstale":"2023-08-28T02:33:11.722641+0000","last_undegraded":"2023-08-28T02:33:11.722641+0000","last_fullsized":"2023-08-28T02:33:11.722641+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:21:30.262904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"296'5954","reported_seq":6005,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-08-28T02:33:32.113007+0000","last_change":"2023-08-28T02:31:44.472455+0000","last_active":"2023-08-28T02:33:32.113007+0000","last_peered":"2023-08-28T02:33:32.113007+0000","last_clean":"2023-08-28T02:33:32.113007+0000","last_became_active":"2023-08-28T02:31:44.470716+0000","last_became_peered":"2023-08-28T02:31:44.470716+0000","last_unstale":"2023-08-28T02:33:32.113007+0000","last_undegraded":"2023-08-28T02:33:32.113007+0000","last_fullsized":"2023-08-28T02:33:32.113007+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5954,"log_dups_size":0,"ondisk_log_size":5954,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107085824,"num_objects":1634,"num_object_clones":0,"num_object_copies":4902,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1634,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4090,"num_write_kb":183168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"78'204","reported_seq":1379,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.722778+0000","last_change":"2023-08-28T02:32:59.644574+0000","last_active":"2023-08-28T02:33:11.722778+0000","last_peered":"2023-08-28T02:33:11.722778+0000","last_clean":"2023-08-28T02:33:11.722778+0000","last_became_active":"2023-08-28T02:32:59.644053+0000","last_became_peered":"2023-08-28T02:32:59.644053+0000","last_unstale":"2023-08-28T02:33:11.722778+0000","last_undegraded":"2023-08-28T02:33:11.722778+0000","last_fullsized":"2023-08-28T02:33:11.722778+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":204,"log_dups_size":0,"ondisk_log_size":204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:55:51.460483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1521,"num_read_kb":2511,"num_write":178,"num_write_kb":1175,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":290,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.555411+0000","last_change":"2023-08-28T02:28:25.352417+0000","last_active":"2023-08-28T02:33:11.555411+0000","last_peered":"2023-08-28T02:33:11.555411+0000","last_clean":"2023-08-28T02:33:11.555411+0000","last_became_active":"2023-08-28T02:28:25.352216+0000","last_became_peered":"2023-08-28T02:28:25.352216+0000","last_unstale":"2023-08-28T02:33:11.555411+0000","last_undegraded":"2023-08-28T02:33:11.555411+0000","last_fullsized":"2023-08-28T02:33:11.555411+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":15,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.723235+0000","last_change":"2023-08-28T02:32:58.720576+0000","last_active":"2023-08-28T02:33:11.723235+0000","last_peered":"2023-08-28T02:33:11.723235+0000","last_clean":"2023-08-28T02:33:11.723235+0000","last_became_active":"2023-08-28T02:32:58.719924+0000","last_became_peered":"2023-08-28T02:32:58.719924+0000","last_unstale":"2023-08-28T02:33:11.723235+0000","last_undegraded":"2023-08-28T02:33:11.723235+0000","last_fullsized":"2023-08-28T02:33:11.723235+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:28:50.697438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"296'6139","reported_seq":6211,"reported_epoch":296,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:33:29.095509+0000","last_change":"2023-08-28T02:32:58.870443+0000","last_active":"2023-08-28T02:33:29.095509+0000","last_peered":"2023-08-28T02:33:29.095509+0000","last_clean":"2023-08-28T02:32:11.480338+0000","last_became_active":"2023-08-28T02:32:58.848058+0000","last_became_peered":"2023-08-28T02:32:58.848058+0000","last_unstale":"2023-08-28T02:33:29.095509+0000","last_undegraded":"2023-08-28T02:32:58.664973+0000","last_fullsized":"2023-08-28T02:32:58.616174+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":6139,"log_dups_size":0,"ondisk_log_size":6139,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112492544,"num_objects":1716,"num_object_clones":0,"num_object_copies":5148,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1029,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1716,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4210,"num_write_kb":189664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":1296},{"shards":"0,4,6","objects":420}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"78'47","reported_seq":514,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.751329+0000","last_change":"2023-08-28T02:24:05.799647+0000","last_active":"2023-08-28T02:33:11.751329+0000","last_peered":"2023-08-28T02:33:11.751329+0000","last_clean":"2023-08-28T02:33:11.751329+0000","last_became_active":"2023-08-28T02:23:59.210481+0000","last_became_peered":"2023-08-28T02:23:59.210481+0000","last_unstale":"2023-08-28T02:33:11.751329+0000","last_undegraded":"2023-08-28T02:33:11.751329+0000","last_fullsized":"2023-08-28T02:33:11.751329+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":47,"log_dups_size":0,"ondisk_log_size":47,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:40:18.542375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":162,"num_read_kb":1281,"num_write":48,"num_write_kb":1153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":15,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.722511+0000","last_change":"2023-08-28T02:32:58.622079+0000","last_active":"2023-08-28T02:33:11.722511+0000","last_peered":"2023-08-28T02:33:11.722511+0000","last_clean":"2023-08-28T02:33:11.722511+0000","last_became_active":"2023-08-28T02:32:58.621819+0000","last_became_peered":"2023-08-28T02:32:58.621819+0000","last_unstale":"2023-08-28T02:33:11.722511+0000","last_undegraded":"2023-08-28T02:33:11.722511+0000","last_fullsized":"2023-08-28T02:33:11.722511+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:03:36.676500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":15,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.723461+0000","last_change":"2023-08-28T02:32:58.724424+0000","last_active":"2023-08-28T02:33:11.723461+0000","last_peered":"2023-08-28T02:33:11.723461+0000","last_clean":"2023-08-28T02:33:11.723461+0000","last_became_active":"2023-08-28T02:32:58.724127+0000","last_became_peered":"2023-08-28T02:32:58.724127+0000","last_unstale":"2023-08-28T02:33:11.723461+0000","last_undegraded":"2023-08-28T02:33:11.723461+0000","last_fullsized":"2023-08-28T02:33:11.723461+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:02:25.178712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"296'5766","reported_seq":5834,"reported_epoch":296,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:33:29.372132+0000","last_change":"2023-08-28T02:32:12.589239+0000","last_active":"2023-08-28T02:33:29.372132+0000","last_peered":"2023-08-28T02:33:29.372132+0000","last_clean":"2023-08-28T02:31:54.462738+0000","last_became_active":"2023-08-28T02:31:56.607923+0000","last_became_peered":"2023-08-28T02:31:56.607923+0000","last_unstale":"2023-08-28T02:33:29.372132+0000","last_undegraded":"2023-08-28T02:32:12.587852+0000","last_fullsized":"2023-08-28T02:31:56.514083+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5766,"log_dups_size":0,"ondisk_log_size":5766,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109674496,"num_objects":1673,"num_object_clones":0,"num_object_copies":5019,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2060,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1673,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3959,"num_write_kb":180256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":-387}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"78'114","reported_seq":692,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.551449+0000","last_change":"2023-08-28T02:32:14.626614+0000","last_active":"2023-08-28T02:33:11.551449+0000","last_peered":"2023-08-28T02:33:11.551449+0000","last_clean":"2023-08-28T02:33:11.551449+0000","last_became_active":"2023-08-28T02:32:14.626181+0000","last_became_peered":"2023-08-28T02:32:14.626181+0000","last_unstale":"2023-08-28T02:33:11.551449+0000","last_undegraded":"2023-08-28T02:33:11.551449+0000","last_fullsized":"2023-08-28T02:33:11.551449+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":114,"log_dups_size":0,"ondisk_log_size":114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":358,"num_read_kb":1352,"num_write":102,"num_write_kb":1104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"86'139","reported_seq":371,"reported_epoch":295,"state":"active+recovery_wait+degraded","last_fresh":"2023-08-28T02:33:11.724117+0000","last_change":"2023-08-28T02:32:59.651360+0000","last_active":"2023-08-28T02:33:11.724117+0000","last_peered":"2023-08-28T02:33:11.724117+0000","last_clean":"2023-08-28T02:23:32.292225+0000","last_became_active":"2023-08-28T02:32:59.651089+0000","last_became_peered":"2023-08-28T02:32:59.651089+0000","last_unstale":"2023-08-28T02:33:11.724117+0000","last_undegraded":"2023-08-28T02:32:59.636553+0000","last_fullsized":"2023-08-28T02:33:11.724117+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_clean_scrub_stamp":"2023-08-28T02:08:02.043718+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:12:30.956176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":["0","6"],"object_location_counts":[{"shards":"0,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":175,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.555019+0000","last_change":"2023-08-28T02:28:25.347600+0000","last_active":"2023-08-28T02:33:11.555019+0000","last_peered":"2023-08-28T02:33:11.555019+0000","last_clean":"2023-08-28T02:33:11.555019+0000","last_became_active":"2023-08-28T02:28:25.347237+0000","last_became_peered":"2023-08-28T02:28:25.347237+0000","last_unstale":"2023-08-28T02:33:11.555019+0000","last_undegraded":"2023-08-28T02:33:11.555019+0000","last_fullsized":"2023-08-28T02:33:11.555019+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:29:26.767559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":93,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.740853+0000","last_change":"2023-08-28T02:31:55.517732+0000","last_active":"2023-08-28T02:33:11.740853+0000","last_peered":"2023-08-28T02:33:11.740853+0000","last_clean":"2023-08-28T02:33:11.740853+0000","last_became_active":"2023-08-28T02:31:55.517347+0000","last_became_peered":"2023-08-28T02:31:55.517347+0000","last_unstale":"2023-08-28T02:33:11.740853+0000","last_undegraded":"2023-08-28T02:33:11.740853+0000","last_fullsized":"2023-08-28T02:33:11.740853+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"296'5800","reported_seq":5875,"reported_epoch":296,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:33:29.637296+0000","last_change":"2023-08-28T02:32:58.875577+0000","last_active":"2023-08-28T02:33:29.637296+0000","last_peered":"2023-08-28T02:33:29.637296+0000","last_clean":"2023-08-28T02:31:54.356655+0000","last_became_active":"2023-08-28T02:32:58.865219+0000","last_became_peered":"2023-08-28T02:32:58.865219+0000","last_unstale":"2023-08-28T02:33:29.637296+0000","last_undegraded":"2023-08-28T02:32:58.625567+0000","last_fullsized":"2023-08-28T02:32:58.612068+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5800,"log_dups_size":0,"ondisk_log_size":5800,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113606656,"num_objects":1733,"num_object_clones":0,"num_object_copies":5199,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2532,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1733,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3977,"num_write_kb":182752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":-294},{"shards":"0,2,6","objects":401}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"78'148","reported_seq":723,"reported_epoch":295,"state":"active+recovery_wait+degraded","last_fresh":"2023-08-28T02:33:11.722434+0000","last_change":"2023-08-28T02:32:59.661621+0000","last_active":"2023-08-28T02:33:11.722434+0000","last_peered":"2023-08-28T02:33:11.722434+0000","last_clean":"2023-08-28T02:12:46.181557+0000","last_became_active":"2023-08-28T02:32:59.661273+0000","last_became_peered":"2023-08-28T02:32:59.661273+0000","last_unstale":"2023-08-28T02:33:11.722434+0000","last_undegraded":"2023-08-28T02:32:59.632715+0000","last_fullsized":"2023-08-28T02:33:11.722434+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":244,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:58:40.796785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1092,"num_read_kb":1955,"num_write":127,"num_write_kb":5177,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":["1","0"],"object_location_counts":[{"shards":"0,1","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":266,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:14.376857+0000","last_change":"2023-08-28T02:23:59.226986+0000","last_active":"2023-08-28T02:33:14.376857+0000","last_peered":"2023-08-28T02:33:14.376857+0000","last_clean":"2023-08-28T02:33:14.376857+0000","last_became_active":"2023-08-28T02:23:59.226529+0000","last_became_peered":"2023-08-28T02:23:59.226529+0000","last_unstale":"2023-08-28T02:33:14.376857+0000","last_undegraded":"2023-08-28T02:33:14.376857+0000","last_fullsized":"2023-08-28T02:33:14.376857+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:22:09.193619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":328,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.726075+0000","last_change":"2023-08-28T02:32:58.744332+0000","last_active":"2023-08-28T02:33:11.726075+0000","last_peered":"2023-08-28T02:33:11.726075+0000","last_clean":"2023-08-28T02:33:11.726075+0000","last_became_active":"2023-08-28T02:32:58.744025+0000","last_became_peered":"2023-08-28T02:32:58.744025+0000","last_unstale":"2023-08-28T02:33:11.726075+0000","last_undegraded":"2023-08-28T02:33:11.726075+0000","last_fullsized":"2023-08-28T02:33:11.726075+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:41:15.957049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"296'5958","reported_seq":6026,"reported_epoch":296,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:33:31.895352+0000","last_change":"2023-08-28T02:32:58.811565+0000","last_active":"2023-08-28T02:33:31.895352+0000","last_peered":"2023-08-28T02:33:31.895352+0000","last_clean":"2023-08-28T02:32:11.496413+0000","last_became_active":"2023-08-28T02:32:58.798063+0000","last_became_peered":"2023-08-28T02:32:58.798063+0000","last_unstale":"2023-08-28T02:33:31.895352+0000","last_undegraded":"2023-08-28T02:32:58.623060+0000","last_fullsized":"2023-08-28T02:32:58.615162+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5958,"log_dups_size":0,"ondisk_log_size":5958,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116293632,"num_objects":1774,"num_object_clones":0,"num_object_copies":5322,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":911,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1774,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4108,"num_write_kb":188256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"2,3,5","objects":379},{"shards":"3,5","objects":1395}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"78'103","reported_seq":605,"reported_epoch":295,"state":"active+recovery_wait+degraded","last_fresh":"2023-08-28T02:33:11.726654+0000","last_change":"2023-08-28T02:32:59.656648+0000","last_active":"2023-08-28T02:33:11.726654+0000","last_peered":"2023-08-28T02:33:11.726654+0000","last_clean":"2023-08-28T02:23:32.292540+0000","last_became_active":"2023-08-28T02:32:59.656276+0000","last_became_peered":"2023-08-28T02:32:59.656276+0000","last_unstale":"2023-08-28T02:33:11.726654+0000","last_undegraded":"2023-08-28T02:32:59.643455+0000","last_fullsized":"2023-08-28T02:33:11.726654+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:54:56.658032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":803,"num_read_kb":2080,"num_write":73,"num_write_kb":1244,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":15,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.723187+0000","last_change":"2023-08-28T02:32:58.720401+0000","last_active":"2023-08-28T02:33:11.723187+0000","last_peered":"2023-08-28T02:33:11.723187+0000","last_clean":"2023-08-28T02:33:11.723187+0000","last_became_active":"2023-08-28T02:32:58.719518+0000","last_became_peered":"2023-08-28T02:32:58.719518+0000","last_unstale":"2023-08-28T02:33:11.723187+0000","last_undegraded":"2023-08-28T02:33:11.723187+0000","last_fullsized":"2023-08-28T02:33:11.723187+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:46:28.947812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":279,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:14.376905+0000","last_change":"2023-08-28T02:32:58.887691+0000","last_active":"2023-08-28T02:33:14.376905+0000","last_peered":"2023-08-28T02:33:14.376905+0000","last_clean":"2023-08-28T02:33:14.376905+0000","last_became_active":"2023-08-28T02:32:58.886906+0000","last_became_peered":"2023-08-28T02:32:58.886906+0000","last_unstale":"2023-08-28T02:33:14.376905+0000","last_undegraded":"2023-08-28T02:33:14.376905+0000","last_fullsized":"2023-08-28T02:33:14.376905+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:39:29.229171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"296'5939","reported_seq":5994,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-08-28T02:33:29.699922+0000","last_change":"2023-08-28T02:31:44.467379+0000","last_active":"2023-08-28T02:33:29.699922+0000","last_peered":"2023-08-28T02:33:29.699922+0000","last_clean":"2023-08-28T02:33:29.699922+0000","last_became_active":"2023-08-28T02:31:44.444605+0000","last_became_peered":"2023-08-28T02:31:44.444605+0000","last_unstale":"2023-08-28T02:33:29.699922+0000","last_undegraded":"2023-08-28T02:33:29.699922+0000","last_fullsized":"2023-08-28T02:33:29.699922+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5939,"log_dups_size":0,"ondisk_log_size":5939,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108593152,"num_objects":1657,"num_object_clones":0,"num_object_copies":4971,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1657,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4069,"num_write_kb":183232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":174,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.553516+0000","last_change":"2023-08-28T02:31:55.535099+0000","last_active":"2023-08-28T02:33:11.553516+0000","last_peered":"2023-08-28T02:33:11.553516+0000","last_clean":"2023-08-28T02:33:11.553516+0000","last_became_active":"2023-08-28T02:31:55.527768+0000","last_became_peered":"2023-08-28T02:31:55.527768+0000","last_unstale":"2023-08-28T02:33:11.553516+0000","last_undegraded":"2023-08-28T02:33:11.553516+0000","last_fullsized":"2023-08-28T02:33:11.553516+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:43:30.804793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'112","reported_seq":448,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.724434+0000","last_change":"2023-08-28T02:32:59.643465+0000","last_active":"2023-08-28T02:33:11.724434+0000","last_peered":"2023-08-28T02:33:11.724434+0000","last_clean":"2023-08-28T02:33:11.724434+0000","last_became_active":"2023-08-28T02:32:59.642979+0000","last_became_peered":"2023-08-28T02:32:59.642979+0000","last_unstale":"2023-08-28T02:33:11.724434+0000","last_undegraded":"2023-08-28T02:33:11.724434+0000","last_fullsized":"2023-08-28T02:33:11.724434+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:48:45.923015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":444,"num_read_kb":473,"num_write":94,"num_write_kb":127,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":184,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.554767+0000","last_change":"2023-08-28T02:32:58.833408+0000","last_active":"2023-08-28T02:33:11.554767+0000","last_peered":"2023-08-28T02:33:11.554767+0000","last_clean":"2023-08-28T02:33:11.554767+0000","last_became_active":"2023-08-28T02:32:58.821347+0000","last_became_peered":"2023-08-28T02:32:58.821347+0000","last_unstale":"2023-08-28T02:33:11.554767+0000","last_undegraded":"2023-08-28T02:33:11.554767+0000","last_fullsized":"2023-08-28T02:33:11.554767+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:41:25.683179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"296'5999","reported_seq":6069,"reported_epoch":296,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:33:29.624930+0000","last_change":"2023-08-28T02:32:58.866701+0000","last_active":"2023-08-28T02:33:29.624930+0000","last_peered":"2023-08-28T02:33:29.624930+0000","last_clean":"2023-08-28T02:32:11.480934+0000","last_became_active":"2023-08-28T02:32:58.858761+0000","last_became_peered":"2023-08-28T02:32:58.858761+0000","last_unstale":"2023-08-28T02:33:29.624930+0000","last_undegraded":"2023-08-28T02:32:58.628529+0000","last_fullsized":"2023-08-28T02:32:58.617723+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5999,"log_dups_size":0,"ondisk_log_size":5999,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115867648,"num_objects":1768,"num_object_clones":0,"num_object_copies":5304,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":937,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1768,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4094,"num_write_kb":187584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[{"shards":"2,3","objects":1383},{"shards":"2,3,6","objects":385}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"78'122","reported_seq":639,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.740789+0000","last_change":"2023-08-28T02:33:00.672889+0000","last_active":"2023-08-28T02:33:11.740789+0000","last_peered":"2023-08-28T02:33:11.740789+0000","last_clean":"2023-08-28T02:33:11.740789+0000","last_became_active":"2023-08-28T02:33:00.672494+0000","last_became_peered":"2023-08-28T02:33:00.672494+0000","last_unstale":"2023-08-28T02:33:11.740789+0000","last_undegraded":"2023-08-28T02:33:11.740789+0000","last_fullsized":"2023-08-28T02:33:11.740789+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":857,"num_read_kb":2224,"num_write":99,"num_write_kb":2312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":303,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.726156+0000","last_change":"2023-08-28T02:32:58.795995+0000","last_active":"2023-08-28T02:33:11.726156+0000","last_peered":"2023-08-28T02:33:11.726156+0000","last_clean":"2023-08-28T02:33:11.726156+0000","last_became_active":"2023-08-28T02:32:58.771872+0000","last_became_peered":"2023-08-28T02:32:58.771872+0000","last_unstale":"2023-08-28T02:33:11.726156+0000","last_undegraded":"2023-08-28T02:33:11.726156+0000","last_fullsized":"2023-08-28T02:33:11.726156+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:16:02.965467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":265,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.555194+0000","last_change":"2023-08-28T02:28:25.351840+0000","last_active":"2023-08-28T02:33:11.555194+0000","last_peered":"2023-08-28T02:33:11.555194+0000","last_clean":"2023-08-28T02:33:11.555194+0000","last_became_active":"2023-08-28T02:28:25.351399+0000","last_became_peered":"2023-08-28T02:28:25.351399+0000","last_unstale":"2023-08-28T02:33:11.555194+0000","last_undegraded":"2023-08-28T02:33:11.555194+0000","last_fullsized":"2023-08-28T02:33:11.555194+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":280,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.725856+0000","last_change":"2023-08-28T02:32:58.751818+0000","last_active":"2023-08-28T02:33:11.725856+0000","last_peered":"2023-08-28T02:33:11.725856+0000","last_clean":"2023-08-28T02:33:11.725856+0000","last_became_active":"2023-08-28T02:32:58.743388+0000","last_became_peered":"2023-08-28T02:32:58.743388+0000","last_unstale":"2023-08-28T02:33:11.725856+0000","last_undegraded":"2023-08-28T02:33:11.725856+0000","last_fullsized":"2023-08-28T02:33:11.725856+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:52:12.910174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":365,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.752406+0000","last_change":"2023-08-28T02:32:58.655915+0000","last_active":"2023-08-28T02:33:11.752406+0000","last_peered":"2023-08-28T02:33:11.752406+0000","last_clean":"2023-08-28T02:33:11.752406+0000","last_became_active":"2023-08-28T02:32:58.655657+0000","last_became_peered":"2023-08-28T02:32:58.655657+0000","last_unstale":"2023-08-28T02:33:11.752406+0000","last_undegraded":"2023-08-28T02:33:11.752406+0000","last_fullsized":"2023-08-28T02:33:11.752406+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:14:33.807968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.18","version":"296'6122","reported_seq":7731,"reported_epoch":296,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:33:29.719386+0000","last_change":"2023-08-28T02:32:12.557727+0000","last_active":"2023-08-28T02:33:29.719386+0000","last_peered":"2023-08-28T02:33:29.719386+0000","last_clean":"2023-08-28T02:31:54.412195+0000","last_became_active":"2023-08-28T02:31:56.623609+0000","last_became_peered":"2023-08-28T02:31:56.623609+0000","last_unstale":"2023-08-28T02:33:29.719386+0000","last_undegraded":"2023-08-28T02:32:12.554864+0000","last_fullsized":"2023-08-28T02:31:56.505024+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":6122,"log_dups_size":0,"ondisk_log_size":6122,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112066560,"num_objects":1710,"num_object_clones":0,"num_object_copies":5130,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1472,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1710,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4204,"num_write_kb":189248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":772,"num_bytes_recovered":50593792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,4","objects":238}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":128,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.740119+0000","last_change":"2023-08-28T02:32:58.839847+0000","last_active":"2023-08-28T02:33:11.740119+0000","last_peered":"2023-08-28T02:33:11.740119+0000","last_clean":"2023-08-28T02:33:11.740119+0000","last_became_active":"2023-08-28T02:32:58.628159+0000","last_became_peered":"2023-08-28T02:32:58.628159+0000","last_unstale":"2023-08-28T02:33:11.740119+0000","last_undegraded":"2023-08-28T02:33:11.740119+0000","last_fullsized":"2023-08-28T02:33:11.740119+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":331,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.740066+0000","last_change":"2023-08-28T02:32:58.629596+0000","last_active":"2023-08-28T02:33:11.740066+0000","last_peered":"2023-08-28T02:33:11.740066+0000","last_clean":"2023-08-28T02:33:11.740066+0000","last_became_active":"2023-08-28T02:32:58.629136+0000","last_became_peered":"2023-08-28T02:32:58.629136+0000","last_unstale":"2023-08-28T02:33:11.740066+0000","last_undegraded":"2023-08-28T02:33:11.740066+0000","last_fullsized":"2023-08-28T02:33:11.740066+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.19","version":"296'5987","reported_seq":6057,"reported_epoch":296,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:33:29.370061+0000","last_change":"2023-08-28T02:32:58.908496+0000","last_active":"2023-08-28T02:33:29.370061+0000","last_peered":"2023-08-28T02:33:29.370061+0000","last_clean":"2023-08-28T02:32:11.482841+0000","last_became_active":"2023-08-28T02:32:58.900137+0000","last_became_peered":"2023-08-28T02:32:58.900137+0000","last_unstale":"2023-08-28T02:33:29.370061+0000","last_undegraded":"2023-08-28T02:32:58.650852+0000","last_fullsized":"2023-08-28T02:32:58.621843+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5987,"log_dups_size":0,"ondisk_log_size":5987,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115245056,"num_objects":1759,"num_object_clones":0,"num_object_copies":5277,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":919,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1759,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4083,"num_write_kb":186912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":1361},{"shards":"0,4,6","objects":398}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":115,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.740093+0000","last_change":"2023-08-28T02:28:25.705331+0000","last_active":"2023-08-28T02:33:11.740093+0000","last_peered":"2023-08-28T02:33:11.740093+0000","last_clean":"2023-08-28T02:33:11.740093+0000","last_became_active":"2023-08-28T02:28:25.397480+0000","last_became_peered":"2023-08-28T02:28:25.397480+0000","last_unstale":"2023-08-28T02:33:11.740093+0000","last_undegraded":"2023-08-28T02:33:11.740093+0000","last_fullsized":"2023-08-28T02:33:11.740093+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":295,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.555173+0000","last_change":"2023-08-28T02:23:59.207929+0000","last_active":"2023-08-28T02:33:11.555173+0000","last_peered":"2023-08-28T02:33:11.555173+0000","last_clean":"2023-08-28T02:33:11.555173+0000","last_became_active":"2023-08-28T02:23:59.207589+0000","last_became_peered":"2023-08-28T02:23:59.207589+0000","last_unstale":"2023-08-28T02:33:11.555173+0000","last_undegraded":"2023-08-28T02:33:11.555173+0000","last_fullsized":"2023-08-28T02:33:11.555173+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":184,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.553214+0000","last_change":"2023-08-28T02:28:25.347988+0000","last_active":"2023-08-28T02:33:11.553214+0000","last_peered":"2023-08-28T02:33:11.553214+0000","last_clean":"2023-08-28T02:33:11.553214+0000","last_became_active":"2023-08-28T02:28:25.347632+0000","last_became_peered":"2023-08-28T02:28:25.347632+0000","last_unstale":"2023-08-28T02:33:11.553214+0000","last_undegraded":"2023-08-28T02:33:11.553214+0000","last_fullsized":"2023-08-28T02:33:11.553214+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:56:20.202662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":156,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.553196+0000","last_change":"2023-08-28T02:23:59.217794+0000","last_active":"2023-08-28T02:33:11.553196+0000","last_peered":"2023-08-28T02:33:11.553196+0000","last_clean":"2023-08-28T02:33:11.553196+0000","last_became_active":"2023-08-28T02:23:59.217231+0000","last_became_peered":"2023-08-28T02:23:59.217231+0000","last_unstale":"2023-08-28T02:33:11.553196+0000","last_undegraded":"2023-08-28T02:33:11.553196+0000","last_fullsized":"2023-08-28T02:33:11.553196+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:36:26.297410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":252,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:14.378133+0000","last_change":"2023-08-28T02:20:19.850252+0000","last_active":"2023-08-28T02:33:14.378133+0000","last_peered":"2023-08-28T02:33:14.378133+0000","last_clean":"2023-08-28T02:33:14.378133+0000","last_became_active":"2023-08-28T02:20:19.849913+0000","last_became_peered":"2023-08-28T02:20:19.849913+0000","last_unstale":"2023-08-28T02:33:14.378133+0000","last_undegraded":"2023-08-28T02:33:14.378133+0000","last_fullsized":"2023-08-28T02:33:14.378133+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:48:47.713465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":354,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-08-28T02:33:11.555098+0000","last_change":"2023-08-28T02:32:58.630270+0000","last_active":"2023-08-28T02:33:11.555098+0000","last_peered":"2023-08-28T02:33:11.555098+0000","last_clean":"2023-08-28T02:33:11.555098+0000","last_became_active":"2023-08-28T02:32:58.629958+0000","last_became_peered":"2023-08-28T02:32:58.629958+0000","last_unstale":"2023-08-28T02:33:11.555098+0000","last_undegraded":"2023-08-28T02:33:11.555098+0000","last_fullsized":"2023-08-28T02:33:11.555098+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":26,"stat_sum":{"num_bytes":3585277976,"num_objects":54708,"num_object_clones":0,"num_object_copies":164124,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":31652,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":54708,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":130828,"num_write_kb":5937089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6711,"num_bytes_recovered":439549952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12207403008,"data_stored":12207390792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":156569,"ondisk_log_size":156569,"up":78,"acting":60,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":58,"num_bytes_recovered":24146,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":21462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26810,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":30,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":147,"num_bytes_recovered":4194342,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430485,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626722,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1586748,"kb_used_data":1281332,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":92158404,"statfs":{"total":95995035648,"available":94370205696,"internally_reserved":0,"allocated":1312083968,"data_stored":1311287418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:33:15 2023","interfaces":[{"interface":"back","average":{"1min":7.093,"5min":3.5960000000000001,"15min":3.5569999999999999},"min":{"1min":0.27100000000000002,"5min":0.22700000000000001,"15min":0.20799999999999999},"max":{"1min":121.78,"5min":121.78,"15min":121.78},"last":2.448},{"interface":"front","average":{"1min":7.6619999999999999,"5min":3.7519999999999998,"15min":3.5430000000000001},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23200000000000001},"max":{"1min":121.79900000000001,"5min":121.79900000000001,"15min":121.79900000000001},"last":0.246}]},{"osd":1,"last update":"Mon Aug 28 02:33:15 2023","interfaces":[{"interface":"back","average":{"1min":1.2629999999999999,"5min":3.5289999999999999,"15min":2.5230000000000001},"min":{"1min":0.247,"5min":0.23799999999999999,"15min":0.23699999999999999},"max":{"1min":4.1609999999999996,"5min":141.87299999999999,"15min":141.87299999999999},"last":1.0429999999999999},{"interface":"front","average":{"1min":1.554,"5min":3.488,"15min":2.6349999999999998},"min":{"1min":0.27900000000000003,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":8.5180000000000007,"5min":141.904,"15min":141.904},"last":1.6040000000000001}]},{"osd":2,"last update":"Mon Aug 28 02:32:55 2023","interfaces":[{"interface":"back","average":{"1min":16.728999999999999,"5min":6.0549999999999997,"15min":3.6200000000000001},"min":{"1min":0.308,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":234.876,"5min":234.876,"15min":234.876},"last":1.252},{"interface":"front","average":{"1min":16.361000000000001,"5min":6.0460000000000003,"15min":3.5720000000000001},"min":{"1min":0.36399999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":234.482,"5min":234.482,"15min":234.482},"last":1.8320000000000001}]},{"osd":3,"last update":"Mon Aug 28 02:33:15 2023","interfaces":[{"interface":"back","average":{"1min":9.0250000000000004,"5min":4.7560000000000002,"15min":2.9350000000000001},"min":{"1min":0.33800000000000002,"5min":0.32000000000000001,"15min":0.317},"max":{"1min":143.19499999999999,"5min":157.625,"15min":157.625},"last":0.437},{"interface":"front","average":{"1min":9.0060000000000002,"5min":4.4889999999999999,"15min":2.9390000000000001},"min":{"1min":0.39500000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":143.20699999999999,"5min":157.637,"15min":157.637},"last":0.35899999999999999}]},{"osd":4,"last update":"Mon Aug 28 02:33:21 2023","interfaces":[{"interface":"back","average":{"1min":0.70699999999999996,"5min":1.0049999999999999,"15min":1.1120000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.251},"max":{"1min":3.8610000000000002,"5min":18.308,"15min":18.308},"last":0.308},{"interface":"front","average":{"1min":0.82399999999999995,"5min":1.01,"15min":1.198},"min":{"1min":0.27800000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"max":{"1min":3.8450000000000002,"5min":18.263000000000002,"15min":18.263000000000002},"last":0.32100000000000001}]},{"osd":5,"last update":"Mon Aug 28 02:33:21 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.76400000000000001,"15min":1.1679999999999999},"min":{"1min":0.30499999999999999,"5min":0.29699999999999999,"15min":0.26300000000000001},"max":{"1min":1.4199999999999999,"5min":6.0049999999999999,"15min":20.759},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.69699999999999995,"5min":0.84999999999999998,"15min":1.018},"min":{"1min":0.32700000000000001,"5min":0.314,"15min":0.311},"max":{"1min":2.972,"5min":7.2370000000000001,"15min":7.3780000000000001},"last":0.47699999999999998}]},{"osd":7,"last update":"Mon Aug 28 02:31:45 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.92900000000000005,"15min":1.018},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.314},"max":{"1min":1.778,"5min":10.605,"15min":10.605},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.88700000000000001,"15min":1.0760000000000001},"min":{"1min":0.38400000000000001,"5min":0.34599999999999997,"15min":0.29799999999999999},"max":{"1min":1.625,"5min":5.1900000000000004,"15min":9.7110000000000003},"last":0.42799999999999999}]}]},{"osd":1,"up_from":98,"seq":420906795174,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1920464,"kb_used_data":1531528,"kb_used_omap":7,"kb_used_meta":388920,"kb_avail":91824688,"statfs":{"total":95995035648,"available":94028480512,"internally_reserved":0,"allocated":1568284672,"data_stored":1566786800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":398254411},"hb_peers":[0,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:33:12 2023","interfaces":[{"interface":"back","average":{"1min":7.7069999999999999,"5min":3.5390000000000001,"15min":3.0350000000000001},"min":{"1min":0.249,"5min":0.19500000000000001,"15min":0.17499999999999999},"max":{"1min":39.634999999999998,"5min":41.838000000000001,"15min":128.60599999999999},"last":4.0659999999999998},{"interface":"front","average":{"1min":6.6349999999999998,"5min":3.6829999999999998,"15min":3.0819999999999999},"min":{"1min":0.22900000000000001,"5min":0.19700000000000001,"15min":0.188},"max":{"1min":39.832000000000001,"5min":41.276000000000003,"15min":128.685},"last":13.734999999999999}]},{"osd":2,"last update":"Mon Aug 28 02:32:40 2023","interfaces":[{"interface":"back","average":{"1min":3.7370000000000001,"5min":3.9079999999999999,"15min":3.2599999999999998},"min":{"1min":0.215,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":24.245999999999999,"5min":90.177000000000007,"15min":135.25999999999999},"last":2.3610000000000002},{"interface":"front","average":{"1min":3.5539999999999998,"5min":3.5910000000000002,"15min":3.0350000000000001},"min":{"1min":0.23599999999999999,"5min":0.218,"15min":0.218},"max":{"1min":12.691000000000001,"5min":89.587999999999994,"15min":135.10300000000001},"last":5.9900000000000002}]},{"osd":3,"last update":"Mon Aug 28 02:33:12 2023","interfaces":[{"interface":"back","average":{"1min":10.009,"5min":4.1340000000000003,"15min":2.9009999999999998},"min":{"1min":0.35399999999999998,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":41.448,"5min":41.448,"15min":123.741},"last":25.867000000000001},{"interface":"front","average":{"1min":10.324999999999999,"5min":4.4710000000000001,"15min":3.1179999999999999},"min":{"1min":0.375,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":43.142000000000003,"5min":43.142000000000003,"15min":123.642},"last":2.3380000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:33:12 2023","interfaces":[{"interface":"back","average":{"1min":1.238,"5min":1.3779999999999999,"15min":1.3959999999999999},"min":{"1min":0.308,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":5.8380000000000001,"5min":8.1989999999999998,"15min":17.600000000000001},"last":2.2549999999999999},{"interface":"front","average":{"1min":2,"5min":1.4410000000000001,"15min":1.3129999999999999},"min":{"1min":0.31,"5min":0.29399999999999998,"15min":0.25600000000000001},"max":{"1min":12.66,"5min":12.66,"15min":19.010000000000002},"last":0.51500000000000001}]},{"osd":5,"last update":"Mon Aug 28 02:33:12 2023","interfaces":[{"interface":"back","average":{"1min":1.486,"5min":1.679,"15min":1.542},"min":{"1min":0.39000000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":4.117,"5min":11.167999999999999,"15min":15.544},"last":6.2919999999999998},{"interface":"front","average":{"1min":1.3819999999999999,"5min":1.5129999999999999,"15min":1.397},"min":{"1min":0.34100000000000003,"5min":0.307,"15min":0.307},"max":{"1min":5.8099999999999996,"5min":10.084,"15min":17.646999999999998},"last":2.3159999999999998}]},{"osd":6,"last update":"Mon Aug 28 02:33:12 2023","interfaces":[{"interface":"back","average":{"1min":1.4470000000000001,"5min":1.659,"15min":1.651},"min":{"1min":0.41999999999999998,"5min":0.311,"15min":0.311},"max":{"1min":4.1459999999999999,"5min":10.420999999999999,"15min":15.512},"last":6.3300000000000001},{"interface":"front","average":{"1min":1.3879999999999999,"5min":1.5129999999999999,"15min":1.5580000000000001},"min":{"1min":0.36799999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":5.8609999999999998,"5min":8.1419999999999995,"15min":17.673999999999999},"last":2.294}]},{"osd":7,"last update":"Mon Aug 28 02:32:10 2023","interfaces":[{"interface":"back","average":{"1min":1.4450000000000001,"5min":2.0299999999999998,"15min":1.5189999999999999},"min":{"1min":0.34399999999999997,"5min":0.32500000000000001,"15min":0.30199999999999999},"max":{"1min":10.353999999999999,"5min":49.575000000000003,"15min":49.575000000000003},"last":0.442},{"interface":"front","average":{"1min":1.8899999999999999,"5min":2.2469999999999999,"15min":1.7410000000000001},"min":{"1min":0.42199999999999999,"5min":0.39500000000000002,"15min":0.33100000000000002},"max":{"1min":10.449999999999999,"5min":49.319000000000003,"15min":49.319000000000003},"last":0.51500000000000001}]}]},{"osd":0,"up_from":90,"seq":386547056820,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2268628,"kb_used_data":1793484,"kb_used_omap":8,"kb_used_meta":475127,"kb_avail":91476524,"statfs":{"total":95995035648,"available":93671960576,"internally_reserved":0,"allocated":1836527616,"data_stored":1835141364,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":486530409},"hb_peers":[1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,0,1,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Aug 28 02:33:25 2023","interfaces":[{"interface":"back","average":{"1min":2.2069999999999999,"5min":3.1930000000000001,"15min":2.351},"min":{"1min":0.216,"5min":0.191,"15min":0.184},"max":{"1min":5.585,"5min":61.932000000000002,"15min":61.932000000000002},"last":8.3170000000000002},{"interface":"front","average":{"1min":3.9590000000000001,"5min":4.2130000000000001,"15min":2.6589999999999998},"min":{"1min":0.28899999999999998,"5min":0.19600000000000001,"15min":0.17000000000000001},"max":{"1min":15.068,"5min":62.061999999999998,"15min":62.061999999999998},"last":6.9160000000000004}]},{"osd":2,"last update":"Mon Aug 28 02:32:40 2023","interfaces":[{"interface":"back","average":{"1min":4.9809999999999999,"5min":3.7869999999999999,"15min":2.5470000000000002},"min":{"1min":0.22800000000000001,"5min":0.20599999999999999,"15min":0.19500000000000001},"max":{"1min":46.927999999999997,"5min":100.718,"15min":100.718},"last":16.356999999999999},{"interface":"front","average":{"1min":5.1260000000000003,"5min":3.5899999999999999,"15min":2.8479999999999999},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":46.942999999999998,"5min":100.962,"15min":100.962},"last":25.047999999999998}]},{"osd":3,"last update":"Mon Aug 28 02:33:25 2023","interfaces":[{"interface":"back","average":{"1min":9.2739999999999991,"5min":4.5679999999999996,"15min":4.1029999999999998},"min":{"1min":0.309,"5min":0.23899999999999999,"15min":0.23300000000000001},"max":{"1min":119.351,"5min":126.896,"15min":237.43899999999999},"last":9.6920000000000002},{"interface":"front","average":{"1min":9.173,"5min":4.4299999999999997,"15min":4.0149999999999997},"min":{"1min":0.35199999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":119.378,"5min":127.01600000000001,"15min":237.48099999999999},"last":10.417}]},{"osd":4,"last update":"Mon Aug 28 02:33:19 2023","interfaces":[{"interface":"back","average":{"1min":1.506,"5min":1.7869999999999999,"15min":1.47},"min":{"1min":0.26300000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":4.2590000000000003,"5min":9.3680000000000003,"15min":9.6649999999999991},"last":1.016},{"interface":"front","average":{"1min":2.484,"5min":1.6080000000000001,"15min":1.671},"min":{"1min":0.36699999999999999,"5min":0.309,"15min":0.309},"max":{"1min":5.4489999999999998,"5min":8.7880000000000003,"15min":27.271999999999998},"last":0.72599999999999998}]},{"osd":5,"last update":"Mon Aug 28 02:33:19 2023","interfaces":[{"interface":"back","average":{"1min":2.593,"5min":1.4530000000000001,"15min":1.7629999999999999},"min":{"1min":0.33900000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":5.5190000000000001,"5min":5.5190000000000001,"15min":44.000999999999998},"last":0.76000000000000001},{"interface":"front","average":{"1min":1.546,"5min":1.7849999999999999,"15min":1.661},"min":{"1min":0.307,"5min":0.307,"15min":0.30599999999999999},"max":{"1min":4.2919999999999998,"5min":9.4489999999999998,"15min":43.953000000000003},"last":1.0660000000000001}]},{"osd":6,"last update":"Mon Aug 28 02:33:19 2023","interfaces":[{"interface":"back","average":{"1min":2.5169999999999999,"5min":1.677,"15min":1.851},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":5.4889999999999999,"5min":7.6340000000000003,"15min":32.417999999999999},"last":0.78500000000000003},{"interface":"front","average":{"1min":1.573,"5min":1.716,"15min":1.7150000000000001},"min":{"1min":0.32900000000000001,"5min":0.32700000000000001,"15min":0.30399999999999999},"max":{"1min":4.327,"5min":9.2789999999999999,"15min":32.130000000000003},"last":1.095}]},{"osd":7,"last update":"Mon Aug 28 02:32:16 2023","interfaces":[{"interface":"back","average":{"1min":1.6060000000000001,"5min":2.25,"15min":1.712},"min":{"1min":0.379,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":10.775,"5min":27.385000000000002,"15min":27.385000000000002},"last":4.3529999999999998},{"interface":"front","average":{"1min":2.6110000000000002,"5min":2.0979999999999999,"15min":1.766},"min":{"1min":0.41799999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":7.5350000000000001,"5min":27.213999999999999,"15min":27.213999999999999},"last":2.4550000000000001}]}]},{"osd":2,"up_from":104,"seq":446676598944,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2055200,"kb_used_data":1592920,"kb_used_omap":10,"kb_used_meta":462133,"kb_avail":91689952,"statfs":{"total":95995035648,"available":93890510848,"internally_reserved":0,"allocated":1631150080,"data_stored":1629812393,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10792,"internal_metadata":473224664},"hb_peers":[0,1,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:32:43 2023","interfaces":[{"interface":"back","average":{"1min":22.899000000000001,"5min":5.8789999999999996,"15min":3.8210000000000002},"min":{"1min":0.193,"5min":0.185,"15min":0.17599999999999999},"max":{"1min":185.93799999999999,"5min":185.93799999999999,"15min":185.93799999999999},"last":0.28199999999999997},{"interface":"front","average":{"1min":22.541,"5min":6.1950000000000003,"15min":5.0110000000000001},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.19900000000000001},"max":{"1min":187.90700000000001,"5min":187.90700000000001,"15min":187.90700000000001},"last":0.379}]},{"osd":1,"last update":"Mon Aug 28 02:32:42 2023","interfaces":[{"interface":"back","average":{"1min":2.2650000000000001,"5min":1.998,"15min":1.9179999999999999},"min":{"1min":0.22800000000000001,"5min":0.21099999999999999,"15min":0.20999999999999999},"max":{"1min":13.25,"5min":13.25,"15min":30.888000000000002},"last":0.49399999999999999},{"interface":"front","average":{"1min":1.784,"5min":2.1259999999999999,"15min":1.8149999999999999},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":9.4329999999999998,"5min":11.962999999999999,"15min":31.648},"last":1.0740000000000001}]},{"osd":3,"last update":"Mon Aug 28 02:33:14 2023","interfaces":[{"interface":"back","average":{"1min":11.523999999999999,"5min":5.1539999999999999,"15min":3.6309999999999998},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.22700000000000001},"max":{"1min":134.07599999999999,"5min":134.07599999999999,"15min":134.07599999999999},"last":0.41599999999999998},{"interface":"front","average":{"1min":12.227,"5min":5.069,"15min":3.6349999999999998},"min":{"1min":0.29299999999999998,"5min":0.26100000000000001,"15min":0.223},"max":{"1min":134.172,"5min":134.172,"15min":134.172},"last":4.2050000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:32:42 2023","interfaces":[{"interface":"back","average":{"1min":1.647,"5min":4.9749999999999996,"15min":2.7410000000000001},"min":{"1min":0.311,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":13.273,"5min":207.34100000000001,"15min":207.34100000000001},"last":1.145},{"interface":"front","average":{"1min":2.0630000000000002,"5min":4.9260000000000002,"15min":2.5129999999999999},"min":{"1min":0.36799999999999999,"5min":0.30299999999999999,"15min":0.27400000000000002},"max":{"1min":13.222,"5min":207.352,"15min":207.352},"last":0.34999999999999998}]},{"osd":5,"last update":"Mon Aug 28 02:32:42 2023","interfaces":[{"interface":"back","average":{"1min":1.8919999999999999,"5min":3.641,"15min":2.6699999999999999},"min":{"1min":0.33800000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":13.335000000000001,"5min":94.614000000000004,"15min":128.696},"last":0.44400000000000001},{"interface":"front","average":{"1min":1.8500000000000001,"5min":3.4860000000000002,"15min":2.6960000000000002},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30299999999999999},"max":{"1min":13.303000000000001,"5min":94.534000000000006,"15min":128.84100000000001},"last":1.1200000000000001}]},{"osd":6,"last update":"Mon Aug 28 02:32:42 2023","interfaces":[{"interface":"back","average":{"1min":2.1859999999999999,"5min":1.633,"15min":1.4770000000000001},"min":{"1min":0.35699999999999998,"5min":0.32800000000000001,"15min":0.29299999999999998},"max":{"1min":13.292999999999999,"5min":13.292999999999999,"15min":13.292999999999999},"last":0.40799999999999997},{"interface":"front","average":{"1min":1.8680000000000001,"5min":1.754,"15min":1.625},"min":{"1min":0.40300000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":13.311,"5min":13.311,"15min":13.311},"last":0.46800000000000003}]},{"osd":7,"last update":"Mon Aug 28 02:32:42 2023","interfaces":[{"interface":"back","average":{"1min":2.0649999999999999,"5min":3.6520000000000001,"15min":2.6280000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.32100000000000001},"max":{"1min":13.321999999999999,"5min":108.904,"15min":128.911},"last":0.502},{"interface":"front","average":{"1min":2.0739999999999998,"5min":3.8719999999999999,"15min":2.859},"min":{"1min":0.38800000000000001,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":13.353,"5min":108.217,"15min":128.72399999999999},"last":0.625}]}]},{"osd":3,"up_from":112,"seq":481036337306,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2642780,"kb_used_data":2169812,"kb_used_omap":3,"kb_used_meta":472828,"kb_avail":91102372,"statfs":{"total":95995035648,"available":93288828928,"internally_reserved":0,"allocated":2221887488,"data_stored":2220547095,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":484176802},"hb_peers":[0,1,2,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,2,3,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:33:09 2023","interfaces":[{"interface":"back","average":{"1min":5.3890000000000002,"5min":4.1029999999999998,"15min":3.633},"min":{"1min":0.23799999999999999,"5min":0.20000000000000001,"15min":0.193},"max":{"1min":23.853000000000002,"5min":79.793000000000006,"15min":164.25299999999999},"last":0.38300000000000001},{"interface":"front","average":{"1min":2.8660000000000001,"5min":3.2639999999999998,"15min":3.3679999999999999},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.192},"max":{"1min":23.782,"5min":78.555000000000007,"15min":164.55000000000001},"last":0.252}]},{"osd":1,"last update":"Mon Aug 28 02:33:10 2023","interfaces":[{"interface":"back","average":{"1min":3.5699999999999998,"5min":2.246,"15min":2.278},"min":{"1min":0.22800000000000001,"5min":0.22700000000000001,"15min":0.20699999999999999},"max":{"1min":14.865,"5min":14.865,"15min":117.34699999999999},"last":2.621},{"interface":"front","average":{"1min":3.8239999999999998,"5min":2.6459999999999999,"15min":2.4249999999999998},"min":{"1min":0.27300000000000002,"5min":0.23599999999999999,"15min":0.224},"max":{"1min":12.624000000000001,"5min":12.624000000000001,"15min":118.012},"last":0.41299999999999998}]},{"osd":2,"last update":"Mon Aug 28 02:33:10 2023","interfaces":[{"interface":"back","average":{"1min":5.3109999999999999,"5min":3.0139999999999998,"15min":3.4079999999999999},"min":{"1min":0.251,"5min":0.20100000000000001,"15min":0.19900000000000001},"max":{"1min":16.768000000000001,"5min":16.768000000000001,"15min":120.43300000000001},"last":0.47399999999999998},{"interface":"front","average":{"1min":3.0339999999999998,"5min":2.2730000000000001,"15min":3.137},"min":{"1min":0.245,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":9.0909999999999993,"5min":9.7309999999999999,"15min":120.44499999999999},"last":0.28799999999999998}]},{"osd":4,"last update":"Mon Aug 28 02:33:10 2023","interfaces":[{"interface":"back","average":{"1min":1.2509999999999999,"5min":2.258,"15min":1.6120000000000001},"min":{"1min":0.30399999999999999,"5min":0.26000000000000001,"15min":0.25900000000000001},"max":{"1min":6.4059999999999997,"5min":58.067999999999998,"15min":58.067999999999998},"last":0.32600000000000001},{"interface":"front","average":{"1min":2.859,"5min":2.6339999999999999,"15min":1.7609999999999999},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":10.613,"5min":58.088000000000001,"15min":58.088000000000001},"last":2.5600000000000001}]},{"osd":5,"last update":"Mon Aug 28 02:33:10 2023","interfaces":[{"interface":"back","average":{"1min":3.1699999999999999,"5min":2.0960000000000001,"15min":1.9039999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":10.722,"5min":34.060000000000002,"15min":102.904},"last":2.597},{"interface":"front","average":{"1min":3.8100000000000001,"5min":2.3900000000000001,"15min":1.968},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":9.8580000000000005,"5min":34.079999999999998,"15min":103.283},"last":0.44500000000000001}]},{"osd":6,"last update":"Mon Aug 28 02:33:09 2023","interfaces":[{"interface":"back","average":{"1min":3.4319999999999999,"5min":2.0670000000000002,"15min":1.7370000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.314},"max":{"1min":9.827,"5min":9.827,"15min":72.116},"last":0.501},{"interface":"front","average":{"1min":1.28,"5min":1.5720000000000001,"15min":1.5329999999999999},"min":{"1min":0.34200000000000003,"5min":0.314,"15min":0.314},"max":{"1min":6.3760000000000003,"5min":7.8899999999999997,"15min":72.030000000000001},"last":0.48299999999999998}]},{"osd":7,"last update":"Mon Aug 28 02:32:07 2023","interfaces":[{"interface":"back","average":{"1min":1.6859999999999999,"5min":3.0920000000000001,"15min":1.845},"min":{"1min":0.39800000000000002,"5min":0.33800000000000002,"15min":0.32500000000000001},"max":{"1min":5.2320000000000002,"5min":54.427999999999997,"15min":54.427999999999997},"last":7.0629999999999997},{"interface":"front","average":{"1min":1.5429999999999999,"5min":3.052,"15min":1.9379999999999999},"min":{"1min":0.443,"5min":0.36399999999999999,"15min":0.34799999999999998},"max":{"1min":6.3019999999999996,"5min":54.445,"15min":54.445},"last":10.754}]}]},{"osd":4,"up_from":30,"seq":128849019180,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1551476,"kb_used_data":1241964,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92193676,"statfs":{"total":95995035648,"available":94406324224,"internally_reserved":0,"allocated":1271771136,"data_stored":1270982526,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:33:19 2023","interfaces":[{"interface":"back","average":{"1min":19.853000000000002,"5min":6.609,"15min":4.5140000000000002},"min":{"1min":0.26900000000000002,"5min":0.219,"15min":0.219},"max":{"1min":131.893,"5min":131.893,"15min":138.67400000000001},"last":0.25700000000000001},{"interface":"front","average":{"1min":19.280999999999999,"5min":6.2789999999999999,"15min":4.3079999999999998},"min":{"1min":0.29699999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":131.96700000000001,"5min":131.96700000000001,"15min":134.14599999999999},"last":3.2669999999999999}]},{"osd":1,"last update":"Mon Aug 28 02:33:19 2023","interfaces":[{"interface":"back","average":{"1min":1.4890000000000001,"5min":2.1320000000000001,"15min":1.671},"min":{"1min":0.29299999999999998,"5min":0.23599999999999999,"15min":0.221},"max":{"1min":7.508,"5min":34.945999999999998,"15min":53.622999999999998},"last":0.39900000000000002},{"interface":"front","average":{"1min":1.988,"5min":1.8360000000000001,"15min":1.585},"min":{"1min":0.27000000000000002,"5min":0.25,"15min":0.25},"max":{"1min":6.766,"5min":34.869999999999997,"15min":53.774000000000001},"last":0.245}]},{"osd":2,"last update":"Mon Aug 28 02:32:43 2023","interfaces":[{"interface":"back","average":{"1min":5.3719999999999999,"5min":3.2530000000000001,"15min":3.2000000000000002},"min":{"1min":0.33900000000000002,"5min":0.27400000000000002,"15min":0.25600000000000001},"max":{"1min":61.777000000000001,"5min":61.777000000000001,"15min":74.099000000000004},"last":0.31},{"interface":"front","average":{"1min":5.3499999999999996,"5min":3.2919999999999998,"15min":3.077},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.25},"max":{"1min":61.601999999999997,"5min":61.601999999999997,"15min":74.081999999999994},"last":0.42099999999999999}]},{"osd":3,"last update":"Mon Aug 28 02:33:19 2023","interfaces":[{"interface":"back","average":{"1min":14.218999999999999,"5min":9.3070000000000004,"15min":4.5350000000000001},"min":{"1min":0.34000000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":105.491,"5min":203.583,"15min":203.583},"last":0.41299999999999998},{"interface":"front","average":{"1min":15.113,"5min":9.5359999999999996,"15min":4.7030000000000003},"min":{"1min":0.38900000000000001,"5min":0.33800000000000002,"15min":0.32500000000000001},"max":{"1min":105.801,"5min":203.41399999999999,"15min":203.41399999999999},"last":0.36599999999999999}]},{"osd":5,"last update":"Mon Aug 28 02:32:55 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":2.4329999999999998,"15min":1.46},"min":{"1min":0.27600000000000002,"5min":0.246,"15min":0.246},"max":{"1min":2.3149999999999999,"5min":124.718,"15min":124.718},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.53800000000000003,"5min":2.5779999999999998,"15min":1.6559999999999999},"min":{"1min":0.30199999999999999,"5min":0.247,"15min":0.247},"max":{"1min":1.9930000000000001,"5min":124.694,"15min":124.694},"last":0.33200000000000002}]},{"osd":6,"last update":"Mon Aug 28 02:33:19 2023","interfaces":[{"interface":"back","average":{"1min":0.78600000000000003,"5min":1.089,"15min":1.127},"min":{"1min":0.314,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":2.3279999999999998,"5min":14.907999999999999,"15min":14.907999999999999},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.81499999999999995,"5min":0.89800000000000002,"15min":0.91200000000000003},"min":{"1min":0.34999999999999998,"5min":0.307,"15min":0.307},"max":{"1min":1.6890000000000001,"5min":5.2679999999999998,"15min":7.3170000000000002},"last":0.373}]},{"osd":7,"last update":"Mon Aug 28 02:32:34 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":1.0149999999999999,"15min":1.1819999999999999},"min":{"1min":0.32400000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.2210000000000001,"5min":10.151999999999999,"15min":16.584},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.53100000000000003,"5min":1.0900000000000001,"15min":1.2390000000000001},"min":{"1min":0.35999999999999999,"5min":0.311,"15min":0.311},"max":{"1min":1.5109999999999999,"5min":9.3469999999999995,"15min":10.826000000000001},"last":0.434}]}]},{"osd":5,"up_from":36,"seq":154618822952,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1360932,"kb_used_data":1059612,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92384220,"statfs":{"total":95995035648,"available":94601441280,"internally_reserved":0,"allocated":1085042688,"data_stored":1084265617,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:33:09 2023","interfaces":[{"interface":"back","average":{"1min":1.5700000000000001,"5min":3.4510000000000001,"15min":3.4009999999999998},"min":{"1min":0.245,"5min":0.214,"15min":0.20300000000000001},"max":{"1min":5.9850000000000003,"5min":104.13200000000001,"15min":131.06100000000001},"last":2.79},{"interface":"front","average":{"1min":1.5029999999999999,"5min":3.6230000000000002,"15min":3.6040000000000001},"min":{"1min":0.27400000000000002,"5min":0.27100000000000002,"15min":0.22900000000000001},"max":{"1min":5.6390000000000002,"5min":104.142,"15min":131.262},"last":2.7450000000000001}]},{"osd":1,"last update":"Mon Aug 28 02:33:18 2023","interfaces":[{"interface":"back","average":{"1min":1.8069999999999999,"5min":1.6599999999999999,"15min":2.5430000000000001},"min":{"1min":0.33400000000000002,"5min":0.25,"15min":0.216},"max":{"1min":6.8639999999999999,"5min":30.593,"15min":183.97900000000001},"last":0.55100000000000005},{"interface":"front","average":{"1min":1.52,"5min":1.671,"15min":2.6899999999999999},"min":{"1min":0.29099999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":6.6600000000000001,"5min":34.616999999999997,"15min":184.33699999999999},"last":0.32800000000000001}]},{"osd":2,"last update":"Mon Aug 28 02:32:48 2023","interfaces":[{"interface":"back","average":{"1min":4.4649999999999999,"5min":3.8860000000000001,"15min":4.2809999999999997},"min":{"1min":0.318,"5min":0.25800000000000001,"15min":0.252},"max":{"1min":56.725999999999999,"5min":100.56699999999999,"15min":176.69499999999999},"last":0.60599999999999998},{"interface":"front","average":{"1min":4.6669999999999998,"5min":4.2709999999999999,"15min":4.5359999999999996},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.28299999999999997},"max":{"1min":56.750999999999998,"5min":100.53700000000001,"15min":176.71199999999999},"last":0.27600000000000002}]},{"osd":3,"last update":"Mon Aug 28 02:33:18 2023","interfaces":[{"interface":"back","average":{"1min":12.134,"5min":7.2750000000000004,"15min":3.2839999999999998},"min":{"1min":0.56699999999999995,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":109.63,"5min":128.571,"15min":128.571},"last":0.63100000000000001},{"interface":"front","average":{"1min":10.960000000000001,"5min":7.3040000000000003,"15min":3.5880000000000001},"min":{"1min":0.34899999999999998,"5min":0.28399999999999997,"15min":0.26200000000000001},"max":{"1min":109.55800000000001,"5min":128.684,"15min":128.684},"last":2.3580000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:32:58 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":1.9330000000000001,"15min":1.3320000000000001},"min":{"1min":0.253,"5min":0.253,"15min":0.246},"max":{"1min":2.0600000000000001,"5min":85.769999999999996,"15min":85.769999999999996},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.48499999999999999,"5min":1.831,"15min":1.3160000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28000000000000003},"max":{"1min":1.764,"5min":85.825000000000003,"15min":85.825000000000003},"last":0.56899999999999995}]},{"osd":6,"last update":"Mon Aug 28 02:33:18 2023","interfaces":[{"interface":"back","average":{"1min":0.86099999999999999,"5min":0.80800000000000005,"15min":1.0649999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":3.9990000000000001,"5min":4.6210000000000004,"15min":14.803000000000001},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.77100000000000002,"5min":0.78700000000000003,"15min":0.96799999999999997},"min":{"1min":0.29999999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":2.5910000000000002,"5min":4.9509999999999996,"15min":8.6479999999999997},"last":0.65500000000000003}]},{"osd":7,"last update":"Mon Aug 28 02:32:33 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":2.71,"15min":1.7150000000000001},"min":{"1min":0.35099999999999998,"5min":0.33300000000000002,"15min":0.31},"max":{"1min":2.266,"5min":164.13800000000001,"15min":164.13800000000001},"last":0.872},{"interface":"front","average":{"1min":0.58499999999999996,"5min":2.7229999999999999,"15min":1.645},"min":{"1min":0.34100000000000003,"5min":0.33700000000000002,"15min":0.32100000000000001},"max":{"1min":1.9139999999999999,"5min":161.786,"15min":161.786},"last":0.57799999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1807355904,"data_stored":1807351832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1551302656,"data_stored":1551302656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1611333632,"data_stored":1611333632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2192838656,"data_stored":2192834584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1265500160,"data_stored":1265500160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1075904512,"data_stored":1075904512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1306202112,"data_stored":1306198040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1396965376,"data_stored":1396965376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-08-28T02:33:36.376 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd pool set device_health_metrics pgp_num 21 2023-08-28T02:33:36.432 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:33:36.467 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:36.520 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:33:36.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:36.528 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:33:36.628 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:33:36.657 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:36.709 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:33:36.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:36.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:33:36.737 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:36 smithi143 ceph-mon[183059]: pgmap v751: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.2 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 632 op/s; 31342/158439 objects degraded (19.782%); 3.8 MiB/s, 61 objects/s recovering 2023-08-28T02:33:36.738 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:36 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 31342/158439 objects degraded (19.782%), 22 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-08-28T02:33:36.738 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:36 smithi143 ceph-mon[189871]: pgmap v751: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.2 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 632 op/s; 31342/158439 objects degraded (19.782%); 3.8 MiB/s, 61 objects/s recovering 2023-08-28T02:33:36.738 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:36 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 31342/158439 objects degraded (19.782%), 22 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-08-28T02:33:36.814 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:33:36.840 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:36 smithi169 ceph-mon[159415]: pgmap v751: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.2 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 632 op/s; 31342/158439 objects degraded (19.782%); 3.8 MiB/s, 61 objects/s recovering 2023-08-28T02:33:36.840 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:36 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 31342/158439 objects degraded (19.782%), 22 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-08-28T02:33:36.842 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:36.894 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:33:36.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:36.899 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:33:37.000 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:33:37.146 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:37.194 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:33:37.194 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [], 2023-08-28T02:33:37.194 INFO:teuthology.orchestra.run.smithi143.stdout: "complaint_time": 30, 2023-08-28T02:33:37.194 INFO:teuthology.orchestra.run.smithi143.stdout: "num_blocked_ops": 0 2023-08-28T02:33:37.195 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:33:37.256 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:33:37.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:37.265 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:33:37.366 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:33:37.397 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:37.456 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:33:37.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:37.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:33:37.562 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:33:37.669 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:33:37.680 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:37.705 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:33:37.749 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:33:37.788 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:33:37.816 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:33:37.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:37.822 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:37 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2013227979' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:33:37.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:33:37.823 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:37 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2013227979' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:33:37.832 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:33:37.857 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:33:37.875 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:33:37.875 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:33:37.876 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:33:37.876 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:33:37.876 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:33:37.876 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:33:37.876 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:33:37.876 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:33:37.876 INFO:teuthology.orchestra.run.smithi143.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-08-28T02:33:37.924 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:33:37.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:37 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2013227979' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:33:38.003 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:38.111 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:33:38.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:38.118 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:33:38.218 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:33:38.284 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:38.788 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:33:38.791 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:38 smithi143 ceph-mon[183059]: pgmap v752: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 623 op/s; 31342/155274 objects degraded (20.185%); 3.8 MiB/s, 61 objects/s recovering 2023-08-28T02:33:38.791 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:38 smithi143 ceph-mon[189871]: pgmap v752: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 623 op/s; 31342/155274 objects degraded (20.185%); 3.8 MiB/s, 61 objects/s recovering 2023-08-28T02:33:38.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:38.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:33:38.889 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set noscrub 2023-08-28T02:33:38.893 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:33:38.924 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:38 smithi169 ceph-mon[159415]: pgmap v752: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 623 op/s; 31342/155274 objects degraded (20.185%); 3.8 MiB/s, 61 objects/s recovering 2023-08-28T02:33:38.926 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:38.977 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:33:38.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:38.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:33:39.085 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:33:39.111 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.0 dump_historic_ops 2023-08-28T02:33:39.113 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:39.163 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:33:39.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:39.167 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:33:39.268 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:33:39.394 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-08-28T02:33:39.395 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-08-28T02:33:39.403 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:39.409 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:33:39.540 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:33:39.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:39.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:33:39.658 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:33:39.690 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:39 smithi143 ceph-mon[183059]: pgmap v753: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 623 op/s; 31342/155274 objects degraded (20.185%); 3.8 MiB/s, 61 objects/s recovering 2023-08-28T02:33:39.726 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:39.863 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:33:39.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:39.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:33:39.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:39 smithi143 ceph-mon[189871]: pgmap v753: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 623 op/s; 31342/155274 objects degraded (20.185%); 3.8 MiB/s, 61 objects/s recovering 2023-08-28T02:33:39.972 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:33:39.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:39 smithi169 ceph-mon[159415]: pgmap v753: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 623 op/s; 31342/155274 objects degraded (20.185%); 3.8 MiB/s, 61 objects/s recovering 2023-08-28T02:33:40.067 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:33:40.078 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:40.316 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:33:40.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:40.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:33:40.435 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:33:40.464 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:40.528 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:33:40.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:40.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:33:40.633 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:33:40.659 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:40.702 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:33:40.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:40.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:33:40.809 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:33:40.861 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:41.202 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:33:41.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:41.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:33:41.301 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:33:41.312 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:33:41.340 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:41.386 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:33:41.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:41.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:33:41.395 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:33:41.491 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:33:41.519 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:41.570 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:33:41.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:41.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:33:41.676 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:33:41.770 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:42.077 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:33:41 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:33:41] "GET /metrics HTTP/1.1" 200 36580 "" "Prometheus/2.33.4" 2023-08-28T02:33:42.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:42 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:33:42.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:42 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:33:42.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:42 smithi143 ceph-mon[189871]: pgmap v754: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 610 op/s; 30974/149166 objects degraded (20.765%); 4.4 MiB/s, 70 objects/s recovering 2023-08-28T02:33:42.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:42 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 31342/155274 objects degraded (20.185%), 22 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-08-28T02:33:42.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:42 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:33:42.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:42 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:33:42.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:42 smithi143 ceph-mon[183059]: pgmap v754: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 610 op/s; 30974/149166 objects degraded (20.765%); 4.4 MiB/s, 70 objects/s recovering 2023-08-28T02:33:42.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:42 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 31342/155274 objects degraded (20.185%), 22 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-08-28T02:33:42.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:42 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:33:42.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:42 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:33:42.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:42 smithi169 ceph-mon[159415]: pgmap v754: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 610 op/s; 30974/149166 objects degraded (20.765%); 4.4 MiB/s, 70 objects/s recovering 2023-08-28T02:33:42.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:42 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 31342/155274 objects degraded (20.185%), 22 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-08-28T02:33:43.144 INFO:teuthology.orchestra.run.smithi143.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-08-28T02:33:43.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:43 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3848215878' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-08-28T02:33:43.454 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:43 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-08-28T02:33:43.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:43 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3848215878' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-08-28T02:33:43.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:43 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-08-28T02:33:43.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:43 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3848215878' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-08-28T02:33:43.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:43 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-08-28T02:33:44.303 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:33:44.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:44.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:33:44.411 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:33:44.435 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:44 smithi169 ceph-mon[159415]: pgmap v755: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 565 op/s; 30974/145875 objects degraded (21.233%); 4.1 MiB/s, 65 objects/s recovering 2023-08-28T02:33:44.437 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:44.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:44 smithi143 ceph-mon[183059]: pgmap v755: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 565 op/s; 30974/145875 objects degraded (21.233%); 4.1 MiB/s, 65 objects/s recovering 2023-08-28T02:33:44.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:44 smithi143 ceph-mon[189871]: pgmap v755: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 565 op/s; 30974/145875 objects degraded (21.233%); 4.1 MiB/s, 65 objects/s recovering 2023-08-28T02:33:44.479 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:33:44.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:44.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:33:44.584 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:33:44.665 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:44.904 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:33:44.922 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-08-28T02:33:44.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:44.922 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:33:45.023 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:33:45.149 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:45.289 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:33:45.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:45.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:33:45.399 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:33:45.407 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:33:45.506 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:33:45.507 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:33:45.549 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:45.779 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:33:45.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:45.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:33:45.887 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:33:45.971 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:46.095 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:33:46.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:46.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:33:46.214 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:33:46.247 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:46.299 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:33:46.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:46.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:33:46.406 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:33:46.435 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:46.491 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:33:46.492 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:46 smithi169 ceph-mon[159415]: pgmap v756: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.8 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 678 op/s; 30555/139758 objects degraded (21.863%); 5.9 MiB/s, 93 objects/s recovering 2023-08-28T02:33:46.493 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:46 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/4262852522' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:33:46.493 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:46 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:33:46.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:46.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:33:46.536 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is set 2023-08-28T02:33:46.599 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:33:46.630 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:46.686 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:33:46.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:46.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:33:46.792 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:33:46.826 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:46.886 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:33:46.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:46.890 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:33:46.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:46 smithi143 ceph-mon[183059]: pgmap v756: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.8 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 678 op/s; 30555/139758 objects degraded (21.863%); 5.9 MiB/s, 93 objects/s recovering 2023-08-28T02:33:46.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:46 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/4262852522' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:33:46.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:46 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:33:46.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:46 smithi143 ceph-mon[189871]: pgmap v756: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.8 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 678 op/s; 30555/139758 objects degraded (21.863%); 5.9 MiB/s, 93 objects/s recovering 2023-08-28T02:33:46.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:46 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/4262852522' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:33:46.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:46 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:33:46.992 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:33:47.084 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:47.162 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.1 dump_ops_in_flight 2023-08-28T02:33:47.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:47 smithi169 ceph-mon[159415]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:33:47.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:47 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 30555/139758 objects degraded (21.863%), 22 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-08-28T02:33:47.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:47 smithi169 ceph-mon[159415]: failed to encode map e298 with expected crc 2023-08-28T02:33:47.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:47 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:33:47.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:47 smithi169 ceph-mon[159415]: osdmap e298: 8 total, 7 up, 7 in 2023-08-28T02:33:47.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:47 smithi169 ceph-mon[159415]: failed to encode map e298 with expected crc 2023-08-28T02:33:47.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:47 smithi169 ceph-mon[159415]: failed to encode map e298 with expected crc 2023-08-28T02:33:47.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:47 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:33:47.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:47 smithi143 ceph-mon[183059]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:33:47.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:47 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 30555/139758 objects degraded (21.863%), 22 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-08-28T02:33:47.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:47 smithi143 ceph-mon[183059]: failed to encode map e298 with expected crc 2023-08-28T02:33:47.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:47 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:33:47.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:47 smithi143 ceph-mon[183059]: osdmap e298: 8 total, 7 up, 7 in 2023-08-28T02:33:47.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:47 smithi143 ceph-mon[183059]: failed to encode map e298 with expected crc 2023-08-28T02:33:47.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:47 smithi143 ceph-mon[183059]: failed to encode map e298 with expected crc 2023-08-28T02:33:47.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:47 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:33:47.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:47 smithi143 ceph-mon[189871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:33:47.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:47 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 30555/139758 objects degraded (21.863%), 22 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-08-28T02:33:47.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:47 smithi143 ceph-mon[189871]: failed to encode map e298 with expected crc 2023-08-28T02:33:47.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:47 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:33:47.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:47 smithi143 ceph-mon[189871]: osdmap e298: 8 total, 7 up, 7 in 2023-08-28T02:33:47.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:47 smithi143 ceph-mon[189871]: failed to encode map e298 with expected crc 2023-08-28T02:33:47.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:47 smithi143 ceph-mon[189871]: failed to encode map e298 with expected crc 2023-08-28T02:33:47.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:47 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:33:48.822 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:33:48.824 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:48 smithi143 ceph-mon[183059]: pgmap v758: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 630 op/s; 30555/136326 objects degraded (22.413%); 4.9 MiB/s, 78 objects/s recovering 2023-08-28T02:33:48.826 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:48 smithi143 ceph-mon[189871]: pgmap v758: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 630 op/s; 30555/136326 objects degraded (22.413%); 4.9 MiB/s, 78 objects/s recovering 2023-08-28T02:33:48.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:48.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:33:48.933 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:33:48.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:48 smithi169 ceph-mon[159415]: pgmap v758: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 630 op/s; 30555/136326 objects degraded (22.413%); 4.9 MiB/s, 78 objects/s recovering 2023-08-28T02:33:49.063 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:49.104 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:33:49.340 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:33:49.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:49.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:33:49.445 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:33:49.563 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:49.710 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:33:49.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:49.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:33:49.811 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:33:49.819 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:33:49.859 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:33:49.898 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:33:49.908 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:49.924 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 6, 1] out_osds: [7] dead_osds: [7] live_osds: [0, 1, 2, 3, 4, 5, 6] 2023-08-28T02:33:49.924 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-08-28T02:33:49.924 INFO:tasks.thrashosds.thrasher:Reweighting osd 2 to 0.7649501359118464 2023-08-28T02:33:49.924 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd reweight 2 0.7649501359118464 2023-08-28T02:33:49.941 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:33:49.976 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:33:50.020 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:33:50.058 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:33:50.063 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:33:50.063 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:33:50.064 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:33:50.064 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:33:50.064 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:33:50.064 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:33:50.064 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:33:50.064 INFO:teuthology.orchestra.run.smithi143.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-08-28T02:33:50.138 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:33:50.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:50.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:33:50.254 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:33:50.285 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:50.332 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:33:50.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:50.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:33:50.438 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:33:50.523 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:50.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:50 smithi169 ceph-mon[159415]: pgmap v759: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 630 op/s; 30555/136326 objects degraded (22.413%); 4.9 MiB/s, 78 objects/s recovering 2023-08-28T02:33:50.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:50 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3952208991' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:33:50.870 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set nodeep-scrub 2023-08-28T02:33:50.931 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:50 smithi143 ceph-mon[183059]: pgmap v759: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 630 op/s; 30555/136326 objects degraded (22.413%); 4.9 MiB/s, 78 objects/s recovering 2023-08-28T02:33:50.931 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:50 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3952208991' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:33:50.931 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:50 smithi143 ceph-mon[189871]: pgmap v759: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 630 op/s; 30555/136326 objects degraded (22.413%); 4.9 MiB/s, 78 objects/s recovering 2023-08-28T02:33:50.931 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:50 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3952208991' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:33:51.931 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:51 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:33:51.931 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:51 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:33:51.932 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:51 smithi143 ceph-mon[183059]: pgmap v760: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 632 op/s; 30155/130155 objects degraded (23.169%); 5.1 MiB/s, 81 objects/s recovering 2023-08-28T02:33:51.932 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:51 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 30555/136326 objects degraded (22.413%), 22 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-08-28T02:33:51.932 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:51 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:33:51.932 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:51 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:33:51.932 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:51 smithi143 ceph-mon[189871]: pgmap v760: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 632 op/s; 30155/130155 objects degraded (23.169%); 5.1 MiB/s, 81 objects/s recovering 2023-08-28T02:33:51.932 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:51 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 30555/136326 objects degraded (22.413%), 22 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-08-28T02:33:51.937 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:33:51 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:33:51] "GET /metrics HTTP/1.1" 200 36580 "" "Prometheus/2.33.4" 2023-08-28T02:33:52.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:51 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:33:52.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:51 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:33:52.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:51 smithi169 ceph-mon[159415]: pgmap v760: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 632 op/s; 30155/130155 objects degraded (23.169%); 5.1 MiB/s, 81 objects/s recovering 2023-08-28T02:33:52.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:51 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 30555/136326 objects degraded (22.413%), 22 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-08-28T02:33:52.716 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:33:52.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:52.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:33:52.826 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:33:52.861 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:52.870 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-08-28T02:33:52.871 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-08-28T02:33:52.920 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:33:52.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:52.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:33:53.026 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:33:53.055 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:53.107 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:33:53.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:53.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:33:53.212 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:33:53.242 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:53.256 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:33:53.263 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:33:53.292 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:33:53.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:53.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:33:53.398 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:33:53.427 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:53.435 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:33:53.435 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [ 2023-08-28T02:33:53.435 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:33:53.435 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.55270.0:144850 7.5 e298/291 7:a097cf7b:::benchmark_data_smithi143_360738_object52080:head v 298'6227, mlcod=265'1038)", 2023-08-28T02:33:53.435 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:33:53.414714+0000", 2023-08-28T02:33:53.435 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.0020179590000000002, 2023-08-28T02:33:53.435 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0020485830000000001, 2023-08-28T02:33:53.435 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:33:53.436 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "started", 2023-08-28T02:33:53.436 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:33:53.436 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:33:53.436 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:33:53.436 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:33:53.414714+0000", 2023-08-28T02:33:53.436 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:33:53.436 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:33:53.436 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:33:53.436 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:33:53.437 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:33:53.414714+0000", 2023-08-28T02:33:53.437 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:33:53.437 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:33:53.437 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:33:53.437 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:33:53.437 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:33:53.414713+0000", 2023-08-28T02:33:53.437 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:33:53.437 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:33:53.437 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:33:53.437 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:33:53.438 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:33:53.414723+0000", 2023-08-28T02:33:53.438 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.9760000000000007e-06 2023-08-28T02:33:53.438 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:33:53.438 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:33:53.438 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:33:53.438 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:33:53.414736+0000", 2023-08-28T02:33:53.438 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.327e-05 2023-08-28T02:33:53.438 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:33:53.438 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:33:53.438 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:33:53.439 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:33:53.414744+0000", 2023-08-28T02:33:53.439 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.3289999999999994e-06 2023-08-28T02:33:53.439 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:33:53.439 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:33:53.439 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:33:53.439 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:33:53.414908+0000", 2023-08-28T02:33:53.439 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000163578 2023-08-28T02:33:53.439 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:33:53.439 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:33:53.439 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:33:53.439 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:33:53.414935+0000", 2023-08-28T02:33:53.440 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.7072000000000001e-05 2023-08-28T02:33:53.440 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:33:53.440 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:33:53.440 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:33:53.440 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:33:53.440 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:33:53.440 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.55270.0:144856 7.d e298/291 7:bc28ef66:::benchmark_data_smithi143_360738_object52086:head v 298'8799, mlcod=279'4880)", 2023-08-28T02:33:53.440 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:33:53.415864+0000", 2023-08-28T02:33:53.440 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.00086729999999999999, 2023-08-28T02:33:53.440 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00094130400000000001, 2023-08-28T02:33:53.441 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:33:53.441 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "started", 2023-08-28T02:33:53.441 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:33:53.441 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:33:53.441 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:33:53.441 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:33:53.415864+0000", 2023-08-28T02:33:53.441 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:33:53.441 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:33:53.441 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:33:53.441 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:33:53.441 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:33:53.415864+0000", 2023-08-28T02:33:53.442 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:33:53.442 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:33:53.442 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:33:53.442 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:33:53.442 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:33:53.415864+0000", 2023-08-28T02:33:53.442 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:33:53.442 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:33:53.442 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:33:53.442 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:33:53.442 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:33:53.415870+0000", 2023-08-28T02:33:53.443 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.9710000000000001e-06 2023-08-28T02:33:53.443 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:33:53.443 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:33:53.443 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:33:53.443 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:33:53.415885+0000", 2023-08-28T02:33:53.443 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.4898e-05 2023-08-28T02:33:53.443 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:33:53.443 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:33:53.443 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:33:53.443 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:33:53.415890+0000", 2023-08-28T02:33:53.443 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.7679999999999997e-06 2023-08-28T02:33:53.444 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:33:53.444 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:33:53.444 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:33:53.444 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:33:53.415940+0000", 2023-08-28T02:33:53.444 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.9243000000000003e-05 2023-08-28T02:33:53.444 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:33:53.444 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:33:53.444 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:33:53.444 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:33:53.415967+0000", 2023-08-28T02:33:53.444 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.7691999999999998e-05 2023-08-28T02:33:53.444 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:33:53.445 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:33:53.445 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:33:53.445 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:33:53.445 INFO:teuthology.orchestra.run.smithi143.stdout: ], 2023-08-28T02:33:53.445 INFO:teuthology.orchestra.run.smithi143.stdout: "num_ops": 2 2023-08-28T02:33:53.445 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:33:53.479 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:33:53.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:53.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:33:53.584 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:33:53.673 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:54.744 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:33:54.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:54.761 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:54 smithi143 ceph-mon[183059]: pgmap v761: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 621 op/s; 30155/127215 objects degraded (23.704%); 5.1 MiB/s, 81 objects/s recovering 2023-08-28T02:33:54.762 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:54 smithi143 ceph-mon[189871]: pgmap v761: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 621 op/s; 30155/127215 objects degraded (23.704%); 5.1 MiB/s, 81 objects/s recovering 2023-08-28T02:33:54.762 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:33:54.863 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:33:54.872 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:33:54.983 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:54.983 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.1 dump_blocked_ops 2023-08-28T02:33:54.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:54 smithi169 ceph-mon[159415]: pgmap v761: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 621 op/s; 30155/127215 objects degraded (23.704%); 5.1 MiB/s, 81 objects/s recovering 2023-08-28T02:33:55.352 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:33:55.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:55.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:33:55.469 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:33:55.504 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:55.549 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:33:55.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:55.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:33:55.657 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:33:55.760 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:56.222 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:33:56.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:56.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:33:56.340 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:33:56.371 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:56.423 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:33:56.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:56.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:33:56.519 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:56 smithi143 ceph-mon[183059]: pgmap v762: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.5 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 29681/121134 objects degraded (24.503%); 5.5 MiB/s, 87 objects/s recovering 2023-08-28T02:33:56.519 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:56 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:33:56.520 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:56 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:33:56.520 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:56 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 29681/121134 objects degraded (24.503%), 22 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-08-28T02:33:56.530 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:33:56.632 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:56.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:56 smithi169 ceph-mon[159415]: pgmap v762: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.5 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 29681/121134 objects degraded (24.503%); 5.5 MiB/s, 87 objects/s recovering 2023-08-28T02:33:56.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:56 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:33:56.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:56 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:33:56.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:56 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 29681/121134 objects degraded (24.503%), 22 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-08-28T02:33:56.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:56 smithi143 ceph-mon[189871]: pgmap v762: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.5 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 29681/121134 objects degraded (24.503%); 5.5 MiB/s, 87 objects/s recovering 2023-08-28T02:33:56.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:56 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:33:56.956 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:56 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:33:56.956 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:56 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 29681/121134 objects degraded (24.503%), 22 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-08-28T02:33:57.191 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:33:57.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:57.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:33:57.303 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:33:57.330 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:57.377 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:33:57.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:57.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:33:57.482 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:33:57.624 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:57.861 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:33:57.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:57.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:33:57.894 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:33:57.975 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:33:57.995 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:33:58.003 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:58.052 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:33:58.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:58.058 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:33:58.159 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:33:58.274 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:58.596 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is set 2023-08-28T02:33:58.837 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:58 smithi143 ceph-mon[183059]: pgmap v763: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.4 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 666 op/s; 29681/117900 objects degraded (25.175%); 5.0 MiB/s, 79 objects/s recovering 2023-08-28T02:33:58.837 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:58 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/207112294' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:33:58.837 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:58 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:33:58.837 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:58 smithi143 ceph-mon[189871]: pgmap v763: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.4 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 666 op/s; 29681/117900 objects degraded (25.175%); 5.0 MiB/s, 79 objects/s recovering 2023-08-28T02:33:58.837 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:58 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/207112294' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:33:58.837 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:58 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:33:58.919 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:33:58.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:33:58.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:33:58.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:58 smithi169 ceph-mon[159415]: pgmap v763: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.4 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 666 op/s; 29681/117900 objects degraded (25.175%); 5.0 MiB/s, 79 objects/s recovering 2023-08-28T02:33:58.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:58 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/207112294' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:33:58.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:58 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:33:59.039 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:33:59.122 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:33:59.607 INFO:teuthology.orchestra.run.smithi143.stderr:reweighted osd.2 to 0.76495 (c3d3) 2023-08-28T02:33:59.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:59 smithi143 ceph-mon[183059]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:33:59.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:59 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:33:59.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:59 smithi143 ceph-mon[183059]: osdmap e299: 8 total, 7 up, 7 in 2023-08-28T02:33:59.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:59 smithi143 ceph-mon[183059]: failed to encode map e299 with expected crc 2023-08-28T02:33:59.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:59 smithi143 ceph-mon[183059]: failed to encode map e299 with expected crc 2023-08-28T02:33:59.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:59 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/4067189011' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.7649501359118464}]: dispatch 2023-08-28T02:33:59.702 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:33:59 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.7649501359118464}]: dispatch 2023-08-28T02:33:59.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:59 smithi143 ceph-mon[189871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:33:59.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:59 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:33:59.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:59 smithi143 ceph-mon[189871]: osdmap e299: 8 total, 7 up, 7 in 2023-08-28T02:33:59.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:59 smithi143 ceph-mon[189871]: failed to encode map e299 with expected crc 2023-08-28T02:33:59.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:59 smithi143 ceph-mon[189871]: failed to encode map e299 with expected crc 2023-08-28T02:33:59.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:59 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/4067189011' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.7649501359118464}]: dispatch 2023-08-28T02:33:59.703 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:33:59 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.7649501359118464}]: dispatch 2023-08-28T02:33:59.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:59 smithi169 ceph-mon[159415]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:33:59.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:59 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:33:59.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:59 smithi169 ceph-mon[159415]: osdmap e299: 8 total, 7 up, 7 in 2023-08-28T02:33:59.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:59 smithi169 ceph-mon[159415]: failed to encode map e299 with expected crc 2023-08-28T02:33:59.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:59 smithi169 ceph-mon[159415]: failed to encode map e299 with expected crc 2023-08-28T02:33:59.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:59 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/4067189011' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.7649501359118464}]: dispatch 2023-08-28T02:33:59.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:33:59 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.7649501359118464}]: dispatch 2023-08-28T02:34:00.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:00 smithi143 ceph-mon[183059]: pgmap v765: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.4 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 613 op/s; 29681/117900 objects degraded (25.175%); 5.5 MiB/s, 87 objects/s recovering 2023-08-28T02:34:00.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:00 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 0.7649501359118464}]': finished 2023-08-28T02:34:00.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:00 smithi143 ceph-mon[183059]: osdmap e300: 8 total, 7 up, 7 in 2023-08-28T02:34:00.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:00 smithi143 ceph-mon[183059]: failed to encode map e300 with expected crc 2023-08-28T02:34:00.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:00 smithi143 ceph-mon[183059]: failed to encode map e300 with expected crc 2023-08-28T02:34:00.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:00 smithi143 ceph-mon[183059]: failed to encode map e300 with expected crc 2023-08-28T02:34:00.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:00 smithi143 ceph-mon[183059]: failed to encode map e300 with expected crc 2023-08-28T02:34:00.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:00 smithi143 ceph-mon[183059]: failed to encode map e300 with expected crc 2023-08-28T02:34:00.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:00 smithi143 ceph-mon[183059]: failed to encode map e300 with expected crc 2023-08-28T02:34:00.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:00 smithi143 ceph-mon[183059]: failed to encode map e300 with expected crc 2023-08-28T02:34:00.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:00 smithi143 ceph-mon[183059]: failed to encode map e300 with expected crc 2023-08-28T02:34:00.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:00 smithi143 ceph-mon[183059]: failed to encode map e300 with expected crc 2023-08-28T02:34:00.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:00 smithi143 ceph-mon[183059]: failed to encode map e300 with expected crc 2023-08-28T02:34:00.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:00 smithi143 ceph-mon[189871]: pgmap v765: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.4 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 613 op/s; 29681/117900 objects degraded (25.175%); 5.5 MiB/s, 87 objects/s recovering 2023-08-28T02:34:00.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:00 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 0.7649501359118464}]': finished 2023-08-28T02:34:00.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:00 smithi143 ceph-mon[189871]: osdmap e300: 8 total, 7 up, 7 in 2023-08-28T02:34:00.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:00 smithi143 ceph-mon[189871]: failed to encode map e300 with expected crc 2023-08-28T02:34:00.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:00 smithi143 ceph-mon[189871]: failed to encode map e300 with expected crc 2023-08-28T02:34:00.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:00 smithi143 ceph-mon[189871]: failed to encode map e300 with expected crc 2023-08-28T02:34:00.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:00 smithi143 ceph-mon[189871]: failed to encode map e300 with expected crc 2023-08-28T02:34:00.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:00 smithi143 ceph-mon[189871]: failed to encode map e300 with expected crc 2023-08-28T02:34:00.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:00 smithi143 ceph-mon[189871]: failed to encode map e300 with expected crc 2023-08-28T02:34:00.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:00 smithi143 ceph-mon[189871]: failed to encode map e300 with expected crc 2023-08-28T02:34:00.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:00 smithi143 ceph-mon[189871]: failed to encode map e300 with expected crc 2023-08-28T02:34:00.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:00 smithi143 ceph-mon[189871]: failed to encode map e300 with expected crc 2023-08-28T02:34:00.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:00 smithi143 ceph-mon[189871]: failed to encode map e300 with expected crc 2023-08-28T02:34:00.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:00 smithi169 ceph-mon[159415]: pgmap v765: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.4 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 613 op/s; 29681/117900 objects degraded (25.175%); 5.5 MiB/s, 87 objects/s recovering 2023-08-28T02:34:00.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:00 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 0.7649501359118464}]': finished 2023-08-28T02:34:00.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:00 smithi169 ceph-mon[159415]: osdmap e300: 8 total, 7 up, 7 in 2023-08-28T02:34:00.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:00 smithi169 ceph-mon[159415]: failed to encode map e300 with expected crc 2023-08-28T02:34:00.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:00 smithi169 ceph-mon[159415]: failed to encode map e300 with expected crc 2023-08-28T02:34:00.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:00 smithi169 ceph-mon[159415]: failed to encode map e300 with expected crc 2023-08-28T02:34:00.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:00 smithi169 ceph-mon[159415]: failed to encode map e300 with expected crc 2023-08-28T02:34:00.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:00 smithi169 ceph-mon[159415]: failed to encode map e300 with expected crc 2023-08-28T02:34:00.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:00 smithi169 ceph-mon[159415]: failed to encode map e300 with expected crc 2023-08-28T02:34:00.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:00 smithi169 ceph-mon[159415]: failed to encode map e300 with expected crc 2023-08-28T02:34:00.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:00 smithi169 ceph-mon[159415]: failed to encode map e300 with expected crc 2023-08-28T02:34:00.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:00 smithi169 ceph-mon[159415]: failed to encode map e300 with expected crc 2023-08-28T02:34:00.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:00 smithi169 ceph-mon[159415]: failed to encode map e300 with expected crc 2023-08-28T02:34:01.338 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:34:01.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:01.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:34:01.447 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:34:01.527 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:01.636 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:01 smithi143 ceph-mon[183059]: osdmap e301: 8 total, 7 up, 7 in 2023-08-28T02:34:01.636 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:01 smithi143 ceph-mon[183059]: failed to encode map e301 with expected crc 2023-08-28T02:34:01.636 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:01 smithi143 ceph-mon[183059]: failed to encode map e301 with expected crc 2023-08-28T02:34:01.636 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:01 smithi143 ceph-mon[183059]: failed to encode map e301 with expected crc 2023-08-28T02:34:01.636 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:01 smithi143 ceph-mon[183059]: failed to encode map e301 with expected crc 2023-08-28T02:34:01.637 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:01 smithi143 ceph-mon[183059]: failed to encode map e301 with expected crc 2023-08-28T02:34:01.637 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:01 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 29681/117900 objects degraded (25.175%), 22 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-08-28T02:34:01.934 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:34:01 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:34:01] "GET /metrics HTTP/1.1" 200 36581 "" "Prometheus/2.33.4" 2023-08-28T02:34:01.935 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:01 smithi143 ceph-mon[189871]: osdmap e301: 8 total, 7 up, 7 in 2023-08-28T02:34:01.935 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:01 smithi143 ceph-mon[189871]: failed to encode map e301 with expected crc 2023-08-28T02:34:01.935 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:01 smithi143 ceph-mon[189871]: failed to encode map e301 with expected crc 2023-08-28T02:34:01.935 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:01 smithi143 ceph-mon[189871]: failed to encode map e301 with expected crc 2023-08-28T02:34:01.936 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:01 smithi143 ceph-mon[189871]: failed to encode map e301 with expected crc 2023-08-28T02:34:01.936 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:01 smithi143 ceph-mon[189871]: failed to encode map e301 with expected crc 2023-08-28T02:34:01.936 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:01 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 29681/117900 objects degraded (25.175%), 22 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-08-28T02:34:01.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:01 smithi169 ceph-mon[159415]: osdmap e301: 8 total, 7 up, 7 in 2023-08-28T02:34:01.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:01 smithi169 ceph-mon[159415]: failed to encode map e301 with expected crc 2023-08-28T02:34:01.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:01 smithi169 ceph-mon[159415]: failed to encode map e301 with expected crc 2023-08-28T02:34:01.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:01 smithi169 ceph-mon[159415]: failed to encode map e301 with expected crc 2023-08-28T02:34:01.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:01 smithi169 ceph-mon[159415]: failed to encode map e301 with expected crc 2023-08-28T02:34:01.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:01 smithi169 ceph-mon[159415]: failed to encode map e301 with expected crc 2023-08-28T02:34:01.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:01 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 29681/117900 objects degraded (25.175%), 22 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-08-28T02:34:02.547 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:34:02.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:02.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:34:02.653 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:34:02.756 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:02.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:02 smithi143 ceph-mon[183059]: pgmap v768: 99 pgs: 1 peering, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 2.3 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 512 op/s; 29226/111900 objects degraded (26.118%); 4.7 MiB/s, 75 objects/s recovering 2023-08-28T02:34:02.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:02 smithi143 ceph-mon[183059]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-08-28T02:34:02.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:02 smithi143 ceph-mon[189871]: pgmap v768: 99 pgs: 1 peering, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 2.3 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 512 op/s; 29226/111900 objects degraded (26.118%); 4.7 MiB/s, 75 objects/s recovering 2023-08-28T02:34:02.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:02 smithi143 ceph-mon[189871]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-08-28T02:34:02.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:02 smithi169 ceph-mon[159415]: pgmap v768: 99 pgs: 1 peering, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 2.3 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 512 op/s; 29226/111900 objects degraded (26.118%); 4.7 MiB/s, 75 objects/s recovering 2023-08-28T02:34:02.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:02 smithi169 ceph-mon[159415]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-08-28T02:34:03.306 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:34:03.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:03.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:34:03.411 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:34:03.418 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:34:03.418 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [], 2023-08-28T02:34:03.418 INFO:teuthology.orchestra.run.smithi143.stdout: "complaint_time": 30, 2023-08-28T02:34:03.419 INFO:teuthology.orchestra.run.smithi143.stdout: "num_blocked_ops": 0 2023-08-28T02:34:03.419 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:34:03.490 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:03.710 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset noscrub 2023-08-28T02:34:04.538 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:34:04.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:04.542 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:34:04.642 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:34:04.747 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:04.791 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:04 smithi143 ceph-mon[183059]: pgmap v769: 99 pgs: 1 peering, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 2.2 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 523 op/s; 29226/108483 objects degraded (26.941%); 4.7 MiB/s, 75 objects/s recovering 2023-08-28T02:34:04.806 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:04 smithi143 ceph-mon[189871]: pgmap v769: 99 pgs: 1 peering, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 2.2 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 523 op/s; 29226/108483 objects degraded (26.941%); 4.7 MiB/s, 75 objects/s recovering 2023-08-28T02:34:04.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:04 smithi169 ceph-mon[159415]: pgmap v769: 99 pgs: 1 peering, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 2.2 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 523 op/s; 29226/108483 objects degraded (26.941%); 4.7 MiB/s, 75 objects/s recovering 2023-08-28T02:34:05.035 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:34:05.038 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.1 dump_historic_ops 2023-08-28T02:34:05.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:05.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:34:05.144 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:34:05.233 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:05.291 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:34:05.297 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:34:05.338 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:34:05.365 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:34:05.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:05.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:34:05.401 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:34:05.466 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:34:05.484 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:34:05.511 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:34:05.562 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:34:05.585 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:34:05.586 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:34:05.586 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:34:05.587 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:34:05.587 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:34:05.587 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:34:05.587 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:34:05.587 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:34:05.587 INFO:teuthology.orchestra.run.smithi143.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-08-28T02:34:05.637 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:05.854 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:34:05.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:05.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:34:05.961 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:34:05.992 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:06.049 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:34:06.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:06.056 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:34:06.157 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:34:06.184 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:06.227 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:34:06.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:06.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:34:06.335 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:34:06.364 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:06 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:34:06.364 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:06 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:34:06.364 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:06 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/992433377' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:34:06.364 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:06 smithi169 ceph-mon[159415]: pgmap v770: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.1 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 763 op/s; 28553/102132 objects degraded (27.957%); 5.1 MiB/s, 164 objects/s recovering 2023-08-28T02:34:06.367 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:06.427 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:34:06.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:06.434 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:34:06.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:06 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:34:06.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:06 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:34:06.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:06 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/992433377' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:34:06.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:06 smithi143 ceph-mon[183059]: pgmap v770: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.1 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 763 op/s; 28553/102132 objects degraded (27.957%); 5.1 MiB/s, 164 objects/s recovering 2023-08-28T02:34:06.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:06 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:34:06.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:06 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:34:06.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:06 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/992433377' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:34:06.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:06 smithi143 ceph-mon[189871]: pgmap v770: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.1 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 763 op/s; 28553/102132 objects degraded (27.957%); 5.1 MiB/s, 164 objects/s recovering 2023-08-28T02:34:06.535 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:34:06.567 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:06.626 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:34:06.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:06.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:34:06.732 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:34:06.794 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:07.046 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:07 smithi143 ceph-mon[183059]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-08-28T02:34:07.046 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:07 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 28553/102132 objects degraded (27.957%), 22 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-08-28T02:34:07.046 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:07 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:34:07.046 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:07 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:34:07.047 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:07 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:34:07.093 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 6, 1] out_osds: [7] dead_osds: [7] live_osds: [0, 1, 2, 3, 4, 5, 6] 2023-08-28T02:34:07.093 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-08-28T02:34:07.093 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-08-28T02:34:07.093 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-08-28T02:34:07.093 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-08-28T02:34:07.094 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-08-28T02:34:07.442 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:07 smithi143 ceph-mon[189871]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-08-28T02:34:07.442 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:07 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 28553/102132 objects degraded (27.957%), 22 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-08-28T02:34:07.442 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:07 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:34:07.442 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:07 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:34:07.442 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:07 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:34:07.473 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-08-28T02:34:07.474 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-08-28T02:34:07.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:07 smithi169 ceph-mon[159415]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-08-28T02:34:07.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:07 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 28553/102132 objects degraded (27.957%), 22 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-08-28T02:34:07.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:07 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:34:07.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:07 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:34:07.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:07 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:34:07.607 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:34:07.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:07.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:34:07.714 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:34:07.748 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:07.795 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:34:07.797 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:34:07.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:07.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:34:07.902 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:34:07.905 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:34:07.935 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:07.981 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:34:07.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:07.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:34:08.086 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:34:08.117 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:08.172 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:34:08.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:08.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:34:08.278 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:34:08.364 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:08 smithi143 ceph-mon[183059]: pgmap v771: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.0 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 785 op/s; 28553/99033 objects degraded (28.832%); 4.4 MiB/s, 141 objects/s recovering 2023-08-28T02:34:08.364 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:08 smithi143 ceph-mon[189871]: pgmap v771: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.0 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 785 op/s; 28553/99033 objects degraded (28.832%); 4.4 MiB/s, 141 objects/s recovering 2023-08-28T02:34:08.382 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:08.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:08 smithi169 ceph-mon[159415]: pgmap v771: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.0 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 785 op/s; 28553/99033 objects degraded (28.832%); 4.4 MiB/s, 141 objects/s recovering 2023-08-28T02:34:08.514 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:34:08.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:08.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:34:08.632 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:34:08.703 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:09.134 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is unset 2023-08-28T02:34:09.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:09 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2525079942' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:34:09.455 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:09 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:34:09.455 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:09 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2525079942' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:34:09.455 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:09 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:34:09.475 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:34:09.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:09 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2525079942' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:34:09.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:09 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:34:09.502 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:34:09.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:09.505 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:34:09.606 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:34:09.632 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:09.674 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:34:09.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:09.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:34:09.782 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:34:09.863 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:10.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:10 smithi143 ceph-mon[183059]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:34:10.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:10 smithi143 ceph-mon[183059]: failed to encode map e302 with expected crc 2023-08-28T02:34:10.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:10 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:34:10.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:10 smithi143 ceph-mon[183059]: osdmap e302: 8 total, 7 up, 7 in 2023-08-28T02:34:10.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:10 smithi143 ceph-mon[183059]: failed to encode map e302 with expected crc 2023-08-28T02:34:10.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:10 smithi143 ceph-mon[183059]: failed to encode map e302 with expected crc 2023-08-28T02:34:10.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:10 smithi143 ceph-mon[183059]: pgmap v773: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.0 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 486 op/s; 28553/99033 objects degraded (28.832%); 769 KiB/s, 76 objects/s recovering 2023-08-28T02:34:10.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:10 smithi143 ceph-mon[189871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:34:10.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:10 smithi143 ceph-mon[189871]: failed to encode map e302 with expected crc 2023-08-28T02:34:10.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:10 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:34:10.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:10 smithi143 ceph-mon[189871]: osdmap e302: 8 total, 7 up, 7 in 2023-08-28T02:34:10.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:10 smithi143 ceph-mon[189871]: failed to encode map e302 with expected crc 2023-08-28T02:34:10.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:10 smithi143 ceph-mon[189871]: failed to encode map e302 with expected crc 2023-08-28T02:34:10.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:10 smithi143 ceph-mon[189871]: pgmap v773: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.0 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 486 op/s; 28553/99033 objects degraded (28.832%); 769 KiB/s, 76 objects/s recovering 2023-08-28T02:34:10.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:10 smithi169 ceph-mon[159415]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:34:10.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:10 smithi169 ceph-mon[159415]: failed to encode map e302 with expected crc 2023-08-28T02:34:10.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:10 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:34:10.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:10 smithi169 ceph-mon[159415]: osdmap e302: 8 total, 7 up, 7 in 2023-08-28T02:34:10.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:10 smithi169 ceph-mon[159415]: failed to encode map e302 with expected crc 2023-08-28T02:34:10.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:10 smithi169 ceph-mon[159415]: failed to encode map e302 with expected crc 2023-08-28T02:34:10.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:10 smithi169 ceph-mon[159415]: pgmap v773: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.0 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 486 op/s; 28553/99033 objects degraded (28.832%); 769 KiB/s, 76 objects/s recovering 2023-08-28T02:34:10.929 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:34:10.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:10.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:34:11.050 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:34:11.078 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:11.132 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:34:11.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:11.136 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:34:11.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:11 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:34:11.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:11 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:34:11.237 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:34:11.314 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:11.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:11 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:34:11.646 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:34:11.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:11.650 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:34:11.751 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:34:11.791 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:11.929 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:34:11 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:34:11] "GET /metrics HTTP/1.1" 200 36595 "" "Prometheus/2.33.4" 2023-08-28T02:34:11.934 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:34:11.994 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:34:11.995 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:34:11.995 INFO:teuthology.orchestra.run.smithi143.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-08-28T02:34:11.995 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:34:11.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:11.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:34:12.099 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:34:12.133 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:12.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:12 smithi143 ceph-mon[183059]: failed to encode map e303 with expected crc 2023-08-28T02:34:12.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:12 smithi143 ceph-mon[183059]: osdmap e303: 8 total, 7 up, 7 in 2023-08-28T02:34:12.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:12 smithi143 ceph-mon[183059]: failed to encode map e303 with expected crc 2023-08-28T02:34:12.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:12 smithi143 ceph-mon[183059]: pgmap v775: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.9 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 621 op/s; 27895/93558 objects degraded (29.816%); 1.4 MiB/s, 166 objects/s recovering 2023-08-28T02:34:12.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:12 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 28553/99033 objects degraded (28.832%), 22 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-08-28T02:34:12.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:12 smithi143 ceph-mon[189871]: failed to encode map e303 with expected crc 2023-08-28T02:34:12.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:12 smithi143 ceph-mon[189871]: osdmap e303: 8 total, 7 up, 7 in 2023-08-28T02:34:12.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:12 smithi143 ceph-mon[189871]: failed to encode map e303 with expected crc 2023-08-28T02:34:12.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:12 smithi143 ceph-mon[189871]: pgmap v775: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.9 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 621 op/s; 27895/93558 objects degraded (29.816%); 1.4 MiB/s, 166 objects/s recovering 2023-08-28T02:34:12.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:12 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 28553/99033 objects degraded (28.832%), 22 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-08-28T02:34:12.269 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:34:12.269 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:34:12.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:12 smithi169 ceph-mon[159415]: failed to encode map e303 with expected crc 2023-08-28T02:34:12.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:12 smithi169 ceph-mon[159415]: osdmap e303: 8 total, 7 up, 7 in 2023-08-28T02:34:12.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:12 smithi169 ceph-mon[159415]: failed to encode map e303 with expected crc 2023-08-28T02:34:12.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:12 smithi169 ceph-mon[159415]: pgmap v775: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.9 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 621 op/s; 27895/93558 objects degraded (29.816%); 1.4 MiB/s, 166 objects/s recovering 2023-08-28T02:34:12.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:12 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 28553/99033 objects degraded (28.832%), 22 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-08-28T02:34:12.972 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset nodeep-scrub 2023-08-28T02:34:13.417 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:13 smithi143 ceph-mon[189871]: failed to encode map e304 with expected crc 2023-08-28T02:34:13.417 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:13 smithi143 ceph-mon[189871]: osdmap e304: 8 total, 7 up, 7 in 2023-08-28T02:34:13.417 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:13 smithi143 ceph-mon[189871]: failed to encode map e304 with expected crc 2023-08-28T02:34:13.417 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:13 smithi143 ceph-mon[189871]: failed to encode map e304 with expected crc 2023-08-28T02:34:13.417 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:13 smithi143 ceph-mon[189871]: failed to encode map e304 with expected crc 2023-08-28T02:34:13.417 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:13 smithi143 ceph-mon[183059]: failed to encode map e304 with expected crc 2023-08-28T02:34:13.417 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:13 smithi143 ceph-mon[183059]: osdmap e304: 8 total, 7 up, 7 in 2023-08-28T02:34:13.418 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:13 smithi143 ceph-mon[183059]: failed to encode map e304 with expected crc 2023-08-28T02:34:13.418 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:13 smithi143 ceph-mon[183059]: failed to encode map e304 with expected crc 2023-08-28T02:34:13.418 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:13 smithi143 ceph-mon[183059]: failed to encode map e304 with expected crc 2023-08-28T02:34:13.453 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:34:13.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:13.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:34:13.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:13 smithi169 ceph-mon[159415]: failed to encode map e304 with expected crc 2023-08-28T02:34:13.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:13 smithi169 ceph-mon[159415]: osdmap e304: 8 total, 7 up, 7 in 2023-08-28T02:34:13.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:13 smithi169 ceph-mon[159415]: failed to encode map e304 with expected crc 2023-08-28T02:34:13.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:13 smithi169 ceph-mon[159415]: failed to encode map e304 with expected crc 2023-08-28T02:34:13.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:13 smithi169 ceph-mon[159415]: failed to encode map e304 with expected crc 2023-08-28T02:34:13.570 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:34:13.597 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:13.970 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:34:13.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:13.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:34:14.084 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:34:14.114 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:14.167 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:34:14.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:14.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:34:14.218 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.2 dump_ops_in_flight 2023-08-28T02:34:14.271 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:34:14.362 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:14.363 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:14 smithi143 ceph-mon[189871]: pgmap v777: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.9 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 453 op/s; 27895/90861 objects degraded (30.701%); 810 KiB/s, 109 objects/s recovering 2023-08-28T02:34:14.367 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:14 smithi143 ceph-mon[183059]: pgmap v777: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.9 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 453 op/s; 27895/90861 objects degraded (30.701%); 810 KiB/s, 109 objects/s recovering 2023-08-28T02:34:14.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:14 smithi169 ceph-mon[159415]: pgmap v777: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 17 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.9 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 453 op/s; 27895/90861 objects degraded (30.701%); 810 KiB/s, 109 objects/s recovering 2023-08-28T02:34:14.500 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:34:14.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:14.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:34:14.606 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:34:14.634 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:15.072 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:34:15.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:15.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:34:15.171 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:34:15.188 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:34:15.300 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:15.438 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:34:15.524 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:34:15.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:15.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:34:15.640 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:34:15.742 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:15.980 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:34:15.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:15.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:34:16.090 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:34:16.177 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:16.330 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:34:16.331 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:34:16.331 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:34:16.331 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:34:16.331 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:34:16.331 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:34:16.370 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:34:16.380 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:34:16.380 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:34:16.380 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:34:16.381 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:34:16.381 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:34:16.381 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:34:16.381 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:34:16.381 INFO:teuthology.orchestra.run.smithi143.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-08-28T02:34:16.845 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:34:16.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:16.852 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:34:16.852 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:16 smithi143 ceph-mon[189871]: pgmap v778: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 16 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.8 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 684 op/s; 27469/86043 objects degraded (31.925%); 3.4 MiB/s, 171 objects/s recovering 2023-08-28T02:34:16.852 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:16 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1181397393' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:34:16.855 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:16 smithi143 ceph-mon[183059]: pgmap v778: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 16 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.8 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 684 op/s; 27469/86043 objects degraded (31.925%); 3.4 MiB/s, 171 objects/s recovering 2023-08-28T02:34:16.855 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:16 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1181397393' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:34:16.953 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:34:16.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:16 smithi169 ceph-mon[159415]: pgmap v778: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 16 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.8 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 684 op/s; 27469/86043 objects degraded (31.925%); 3.4 MiB/s, 171 objects/s recovering 2023-08-28T02:34:16.977 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:16 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1181397393' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:34:16.980 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:17.030 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:34:17.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:17.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:34:17.135 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:34:17.163 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:17.212 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:34:17.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:17.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:34:17.318 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:34:17.343 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:17.383 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:34:17.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:17.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:34:17.488 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:34:17.579 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:17.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:17 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 27469/86043 objects degraded (31.925%), 21 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-08-28T02:34:17.957 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:17 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 27469/86043 objects degraded (31.925%), 21 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-08-28T02:34:17.957 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:17 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 27469/86043 objects degraded (31.925%), 21 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-08-28T02:34:17.993 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:34:18.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:18.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:34:18.103 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:34:18.193 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:18.529 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-08-28T02:34:18.530 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-08-28T02:34:18.766 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:34:18.773 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:18 smithi143 ceph-mon[183059]: pgmap v779: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 16 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.7 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 668 op/s; 27469/82965 objects degraded (33.109%); 2.7 MiB/s, 135 objects/s recovering 2023-08-28T02:34:18.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:18.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:34:18.778 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:18 smithi143 ceph-mon[189871]: pgmap v779: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 16 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.7 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 668 op/s; 27469/82965 objects degraded (33.109%); 2.7 MiB/s, 135 objects/s recovering 2023-08-28T02:34:18.868 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 6, 1] out_osds: [7] dead_osds: [7] live_osds: [0, 1, 2, 3, 4, 5, 6] 2023-08-28T02:34:18.868 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-08-28T02:34:18.868 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-08-28T02:34:18.868 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph pg dump --format=json 2023-08-28T02:34:18.876 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:34:18.901 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:18 smithi169 ceph-mon[159415]: pgmap v779: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 16 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.7 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 668 op/s; 27469/82965 objects degraded (33.109%); 2.7 MiB/s, 135 objects/s recovering 2023-08-28T02:34:18.904 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:18.954 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:34:18.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:18.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:34:19.059 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:34:19.172 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:19.406 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:34:19.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:19.427 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:34:19.528 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:34:19.556 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:19.603 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:34:19.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:19.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:34:19.709 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:34:19.719 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:34:19.719 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [ 2023-08-28T02:34:19.719 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.720 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.55270.0:160366 7.8 7:178a1107:::benchmark_data_smithi143_360738_object67596:head [delete] snapc 0=[] ondisk+write+known_if_redirected e304)", 2023-08-28T02:34:19.720 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:34:19.702156+0000", 2023-08-28T02:34:19.720 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.017246867999999999, 2023-08-28T02:34:19.720 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.017285997000000001, 2023-08-28T02:34:19.720 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:34:19.720 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "waiting for sub ops", 2023-08-28T02:34:19.720 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:34:19.720 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.55270", 2023-08-28T02:34:19.720 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:34:19.720 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 160366 2023-08-28T02:34:19.720 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.720 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:34:19.721 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.721 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:34:19.721 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.702156+0000", 2023-08-28T02:34:19.721 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:34:19.721 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.721 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.721 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:34:19.721 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.702156+0000", 2023-08-28T02:34:19.721 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:34:19.721 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.721 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.721 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:34:19.722 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.702155+0000", 2023-08-28T02:34:19.722 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:34:19.722 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.722 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.722 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:34:19.722 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.702164+0000", 2023-08-28T02:34:19.722 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.884e-06 2023-08-28T02:34:19.722 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.722 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.722 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:34:19.722 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.702184+0000", 2023-08-28T02:34:19.722 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.0633e-05 2023-08-28T02:34:19.723 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.723 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.723 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:34:19.723 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.702193+0000", 2023-08-28T02:34:19.723 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.5580000000000001e-06 2023-08-28T02:34:19.723 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.723 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.723 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:34:19.723 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.702233+0000", 2023-08-28T02:34:19.723 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.9878e-05 2023-08-28T02:34:19.723 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.723 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.723 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:34:19.724 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.702402+0000", 2023-08-28T02:34:19.724 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000169598 2023-08-28T02:34:19.724 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.724 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.724 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 3,4", 2023-08-28T02:34:19.724 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.702968+0000", 2023-08-28T02:34:19.724 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00056607499999999998 2023-08-28T02:34:19.724 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.724 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.724 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:34:19.724 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.706065+0000", 2023-08-28T02:34:19.724 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0030967120000000002 2023-08-28T02:34:19.725 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.725 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.725 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:34:19.725 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.711235+0000", 2023-08-28T02:34:19.725 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0051700490000000003 2023-08-28T02:34:19.725 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:34:19.725 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:34:19.725 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:34:19.725 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.725 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.725 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.55270.0:160370 7.15 7:ad6748e7:::benchmark_data_smithi143_360738_object67600:head [delete] snapc 0=[] ondisk+write+known_if_redirected e304)", 2023-08-28T02:34:19.725 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:34:19.702220+0000", 2023-08-28T02:34:19.726 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.017183542, 2023-08-28T02:34:19.726 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.017320827, 2023-08-28T02:34:19.726 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:34:19.726 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "waiting for sub ops", 2023-08-28T02:34:19.726 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:34:19.726 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.55270", 2023-08-28T02:34:19.726 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:34:19.726 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 160370 2023-08-28T02:34:19.726 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.726 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:34:19.726 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.726 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:34:19.727 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.702220+0000", 2023-08-28T02:34:19.727 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:34:19.727 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.727 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.727 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:34:19.727 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.702220+0000", 2023-08-28T02:34:19.727 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:34:19.727 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.727 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.727 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:34:19.727 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.702219+0000", 2023-08-28T02:34:19.727 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:34:19.728 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.728 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.728 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:34:19.728 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.702223+0000", 2023-08-28T02:34:19.728 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.9840000000000003e-06 2023-08-28T02:34:19.728 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.728 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.728 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:34:19.728 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.702232+0000", 2023-08-28T02:34:19.728 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.6940000000000001e-06 2023-08-28T02:34:19.728 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.728 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.728 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:34:19.729 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.702238+0000", 2023-08-28T02:34:19.729 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.7020000000000004e-06 2023-08-28T02:34:19.729 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.729 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.729 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:34:19.729 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.702297+0000", 2023-08-28T02:34:19.729 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.8278e-05 2023-08-28T02:34:19.730 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.730 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.730 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:34:19.730 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.702464+0000", 2023-08-28T02:34:19.730 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00016767400000000001 2023-08-28T02:34:19.730 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.730 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.730 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,1", 2023-08-28T02:34:19.730 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.702970+0000", 2023-08-28T02:34:19.730 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00050584899999999995 2023-08-28T02:34:19.730 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.730 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.731 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:34:19.731 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.705864+0000", 2023-08-28T02:34:19.731 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0028932469999999998 2023-08-28T02:34:19.731 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:34:19.731 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:34:19.731 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:34:19.731 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.731 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.731 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.55270.0:160376 7.8 7:15a9691a:::benchmark_data_smithi143_360738_object67606:head [delete] snapc 0=[] ondisk+write+known_if_redirected e304)", 2023-08-28T02:34:19.731 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:34:19.702252+0000", 2023-08-28T02:34:19.731 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.017150911000000001, 2023-08-28T02:34:19.731 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.017371622999999999, 2023-08-28T02:34:19.732 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:34:19.732 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "waiting for sub ops", 2023-08-28T02:34:19.732 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:34:19.732 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.55270", 2023-08-28T02:34:19.732 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:34:19.732 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 160376 2023-08-28T02:34:19.732 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.732 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:34:19.732 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.732 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:34:19.732 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.702252+0000", 2023-08-28T02:34:19.732 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:34:19.732 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.733 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.733 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:34:19.733 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.702252+0000", 2023-08-28T02:34:19.733 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:34:19.733 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.733 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.733 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:34:19.733 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.702252+0000", 2023-08-28T02:34:19.733 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:34:19.734 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.734 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.734 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:34:19.734 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.702255+0000", 2023-08-28T02:34:19.734 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.5659999999999998e-06 2023-08-28T02:34:19.735 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.735 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.735 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:34:19.735 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.702262+0000", 2023-08-28T02:34:19.735 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.0199999999999997e-06 2023-08-28T02:34:19.735 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.736 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.736 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:34:19.736 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.702267+0000", 2023-08-28T02:34:19.736 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.0080000000000002e-06 2023-08-28T02:34:19.736 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.736 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.736 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:34:19.736 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.708171+0000", 2023-08-28T02:34:19.737 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0059040589999999997 2023-08-28T02:34:19.737 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.737 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.737 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:34:19.737 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.708339+0000", 2023-08-28T02:34:19.737 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00016713200000000001 2023-08-28T02:34:19.737 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.737 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.738 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 3,4", 2023-08-28T02:34:19.738 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.708849+0000", 2023-08-28T02:34:19.738 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00051013100000000004 2023-08-28T02:34:19.738 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.738 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.738 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:34:19.738 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.711269+0000", 2023-08-28T02:34:19.738 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.002420452 2023-08-28T02:34:19.738 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:34:19.738 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:34:19.738 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:34:19.738 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.739 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.739 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.55270.0:160377 7.e 7:7e225aa7:::benchmark_data_smithi143_360738_object67607:head [delete] snapc 0=[] ondisk+write+known_if_redirected e304)", 2023-08-28T02:34:19.739 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:34:19.702275+0000", 2023-08-28T02:34:19.739 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.017127851999999999, 2023-08-28T02:34:19.739 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.017432119999999999, 2023-08-28T02:34:19.739 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:34:19.739 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "waiting for sub ops", 2023-08-28T02:34:19.739 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:34:19.739 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.55270", 2023-08-28T02:34:19.739 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:34:19.739 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 160377 2023-08-28T02:34:19.739 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.740 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:34:19.740 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.740 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:34:19.740 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.702275+0000", 2023-08-28T02:34:19.740 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:34:19.740 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.740 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.740 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:34:19.740 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.702275+0000", 2023-08-28T02:34:19.740 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:34:19.740 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.740 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.740 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:34:19.741 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.702275+0000", 2023-08-28T02:34:19.741 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:34:19.741 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.741 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.741 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:34:19.741 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.702278+0000", 2023-08-28T02:34:19.741 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.0460000000000001e-06 2023-08-28T02:34:19.741 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.741 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.741 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:34:19.741 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.702284+0000", 2023-08-28T02:34:19.741 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.2709999999999998e-06 2023-08-28T02:34:19.742 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.742 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.742 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:34:19.742 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.702289+0000", 2023-08-28T02:34:19.742 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.4490000000000001e-06 2023-08-28T02:34:19.742 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.742 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.742 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:34:19.743 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.702322+0000", 2023-08-28T02:34:19.743 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.3000000000000003e-05 2023-08-28T02:34:19.743 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.743 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.743 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:34:19.743 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.702424+0000", 2023-08-28T02:34:19.743 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000102048 2023-08-28T02:34:19.743 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.743 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.743 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:34:19.743 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.702549+0000", 2023-08-28T02:34:19.744 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00012494700000000001 2023-08-28T02:34:19.744 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.744 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.744 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:34:19.744 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.709782+0000", 2023-08-28T02:34:19.744 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0072333340000000001 2023-08-28T02:34:19.744 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.744 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.744 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:34:19.744 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.717087+0000", 2023-08-28T02:34:19.744 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0073048260000000004 2023-08-28T02:34:19.744 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:34:19.744 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:34:19.744 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:34:19.745 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.745 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.745 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.55270.0:160367 7.c e304/277 7:3bfc747a:::benchmark_data_smithi143_360738_object67597:head v 304'9881, mlcod=0'0)", 2023-08-28T02:34:19.745 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:34:19.704124+0000", 2023-08-28T02:34:19.745 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.01527937, 2023-08-28T02:34:19.745 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.015670324999999999, 2023-08-28T02:34:19.745 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:34:19.745 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "started", 2023-08-28T02:34:19.745 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:34:19.745 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.745 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:34:19.745 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.704124+0000", 2023-08-28T02:34:19.745 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:34:19.746 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.746 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.746 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:34:19.746 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.704124+0000", 2023-08-28T02:34:19.746 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:34:19.746 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.746 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.746 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:34:19.746 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.704123+0000", 2023-08-28T02:34:19.746 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:34:19.746 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.746 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.747 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:34:19.747 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.704129+0000", 2023-08-28T02:34:19.747 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.857e-06 2023-08-28T02:34:19.747 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.747 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.747 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:34:19.747 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.704140+0000", 2023-08-28T02:34:19.747 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.093e-05 2023-08-28T02:34:19.747 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.747 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.748 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:34:19.748 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.704151+0000", 2023-08-28T02:34:19.748 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1221e-05 2023-08-28T02:34:19.748 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.748 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.748 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:34:19.748 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.704248+0000", 2023-08-28T02:34:19.749 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.7072999999999997e-05 2023-08-28T02:34:19.749 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.749 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.749 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:34:19.749 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.704271+0000", 2023-08-28T02:34:19.750 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.2906000000000001e-05 2023-08-28T02:34:19.750 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:34:19.750 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:34:19.750 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:34:19.750 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.750 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.751 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.55270.0:160368 7.c e304/277)", 2023-08-28T02:34:19.751 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:34:19.711196+0000", 2023-08-28T02:34:19.751 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.0082071390000000004, 2023-08-28T02:34:19.751 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0086562960000000008, 2023-08-28T02:34:19.751 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:34:19.752 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "queued for pg", 2023-08-28T02:34:19.752 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:34:19.752 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.752 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:34:19.752 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.711196+0000", 2023-08-28T02:34:19.752 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:34:19.753 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.753 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.753 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:34:19.753 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.711196+0000", 2023-08-28T02:34:19.753 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:34:19.753 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.753 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.753 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:34:19.753 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.711196+0000", 2023-08-28T02:34:19.753 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:34:19.753 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.754 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.754 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:34:19.754 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.711200+0000", 2023-08-28T02:34:19.754 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.9339999999999999e-06 2023-08-28T02:34:19.754 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.754 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.754 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:34:19.754 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.711206+0000", 2023-08-28T02:34:19.754 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.5440000000000004e-06 2023-08-28T02:34:19.754 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:19.754 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:19.754 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:34:19.754 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:19.711211+0000", 2023-08-28T02:34:19.755 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.3250000000000001e-06 2023-08-28T02:34:19.755 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:34:19.755 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:34:19.755 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:34:19.755 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:34:19.755 INFO:teuthology.orchestra.run.smithi143.stdout: ], 2023-08-28T02:34:19.755 INFO:teuthology.orchestra.run.smithi143.stdout: "num_ops": 6 2023-08-28T02:34:19.755 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:34:19.755 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:19.780 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:34:19.789 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:34:19.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:19.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:34:19.894 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:34:19.921 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:19.972 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:34:19.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:19.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:34:20.078 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:34:20.141 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:20.531 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:34:20.566 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is unset 2023-08-28T02:34:20.827 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:34:20.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:20.840 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:34:20.841 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:20 smithi143 ceph-mon[189871]: pgmap v780: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 16 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.7 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 425 op/s; 27469/82965 objects degraded (33.109%); 2.1 MiB/s, 51 objects/s recovering 2023-08-28T02:34:20.841 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:20 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2703140338' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:34:20.842 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:20 smithi143 ceph-mon[183059]: pgmap v780: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 16 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.7 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 425 op/s; 27469/82965 objects degraded (33.109%); 2.1 MiB/s, 51 objects/s recovering 2023-08-28T02:34:20.843 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:20 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2703140338' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:34:20.941 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:34:20.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:20 smithi169 ceph-mon[159415]: pgmap v780: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 16 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.7 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 425 op/s; 27469/82965 objects degraded (33.109%); 2.1 MiB/s, 51 objects/s recovering 2023-08-28T02:34:20.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:20 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2703140338' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:34:21.019 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:21.036 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.2 dump_blocked_ops 2023-08-28T02:34:21.777 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:34:21.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:21.794 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:34:21.860 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:21 smithi143 ceph-mon[183059]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:34:21.861 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:21 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2703140338' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:34:21.861 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:21 smithi143 ceph-mon[183059]: osdmap e305: 8 total, 7 up, 7 in 2023-08-28T02:34:21.861 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:21 smithi143 ceph-mon[183059]: failed to encode map e305 with expected crc 2023-08-28T02:34:21.861 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:21 smithi143 ceph-mon[183059]: failed to encode map e305 with expected crc 2023-08-28T02:34:21.861 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:21 smithi143 ceph-mon[183059]: failed to encode map e305 with expected crc 2023-08-28T02:34:21.861 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:21 smithi143 ceph-mon[183059]: failed to encode map e305 with expected crc 2023-08-28T02:34:21.861 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:21 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 27469/82965 objects degraded (33.109%), 21 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-08-28T02:34:21.862 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:21 smithi143 ceph-mon[189871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:34:21.862 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:21 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2703140338' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:34:21.862 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:21 smithi143 ceph-mon[189871]: osdmap e305: 8 total, 7 up, 7 in 2023-08-28T02:34:21.862 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:21 smithi143 ceph-mon[189871]: failed to encode map e305 with expected crc 2023-08-28T02:34:21.862 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:21 smithi143 ceph-mon[189871]: failed to encode map e305 with expected crc 2023-08-28T02:34:21.862 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:21 smithi143 ceph-mon[189871]: failed to encode map e305 with expected crc 2023-08-28T02:34:21.862 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:21 smithi143 ceph-mon[189871]: failed to encode map e305 with expected crc 2023-08-28T02:34:21.862 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:21 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 27469/82965 objects degraded (33.109%), 21 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-08-28T02:34:21.895 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:34:21.985 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:21.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:21 smithi169 ceph-mon[159415]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:34:21.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:21 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2703140338' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:34:21.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:21 smithi169 ceph-mon[159415]: osdmap e305: 8 total, 7 up, 7 in 2023-08-28T02:34:21.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:21 smithi169 ceph-mon[159415]: failed to encode map e305 with expected crc 2023-08-28T02:34:21.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:21 smithi169 ceph-mon[159415]: failed to encode map e305 with expected crc 2023-08-28T02:34:21.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:21 smithi169 ceph-mon[159415]: failed to encode map e305 with expected crc 2023-08-28T02:34:21.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:21 smithi169 ceph-mon[159415]: failed to encode map e305 with expected crc 2023-08-28T02:34:21.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:21 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 27469/82965 objects degraded (33.109%), 21 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-08-28T02:34:22.122 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:34:22.132 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:34:21 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:34:21] "GET /metrics HTTP/1.1" 200 36595 "" "Prometheus/2.33.4" 2023-08-28T02:34:22.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:22.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:34:22.234 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:34:22.334 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:22.489 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:34:22.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:22.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:34:22.603 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:34:22.632 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:22.686 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:34:22.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:22.694 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:34:22.795 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:34:22.823 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:22.882 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:34:22.885 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:22 smithi169 ceph-mon[159415]: pgmap v782: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 16 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.6 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 27161/77055 objects degraded (35.249%); 2.4 MiB/s, 79 objects/s recovering 2023-08-28T02:34:22.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:22.886 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:34:22.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:22 smithi143 ceph-mon[183059]: pgmap v782: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 16 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.6 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 27161/77055 objects degraded (35.249%); 2.4 MiB/s, 79 objects/s recovering 2023-08-28T02:34:22.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:22 smithi143 ceph-mon[189871]: pgmap v782: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 16 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.6 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 27161/77055 objects degraded (35.249%); 2.4 MiB/s, 79 objects/s recovering 2023-08-28T02:34:22.987 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:34:23.062 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:34:23.080 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:23.123 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:34:23.217 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:34:23.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:23.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:34:23.324 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:34:23.438 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:23.819 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:34:23.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:23.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:34:23.916 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set noscrub 2023-08-28T02:34:23.926 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:34:24.088 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:24.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:24 smithi143 ceph-mon[183059]: pgmap v783: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 16 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.5 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 559 op/s; 27161/74076 objects degraded (36.666%); 2.3 MiB/s, 73 objects/s recovering 2023-08-28T02:34:24.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:24 smithi143 ceph-mon[189871]: pgmap v783: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 16 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.5 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 559 op/s; 27161/74076 objects degraded (36.666%); 2.3 MiB/s, 73 objects/s recovering 2023-08-28T02:34:24.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:24.900 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:34:24.900 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:34:24.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:24 smithi169 ceph-mon[159415]: pgmap v783: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 16 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.5 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 559 op/s; 27161/74076 objects degraded (36.666%); 2.3 MiB/s, 73 objects/s recovering 2023-08-28T02:34:25.000 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:34:25.085 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:34:25.098 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:25.118 INFO:teuthology.orchestra.run.smithi143.stderr:dumped all 2023-08-28T02:34:25.322 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:34:25.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:25.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:34:25.435 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:34:25.463 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:25.516 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:34:25.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:25.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:34:25.623 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:34:25.650 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:25.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:25 smithi169 ceph-mon[159415]: from='client.35517 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:34:25.916 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:25 smithi143 ceph-mon[183059]: from='client.35517 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:34:25.926 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:25 smithi143 ceph-mon[189871]: from='client.35517 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:34:26.609 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:34:26.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:26.624 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:34:26.654 INFO:teuthology.orchestra.run.smithi143.stdout:{"pg_ready":true,"pg_map":{"version":783,"stamp":"2023-08-28T02:34:23.445924+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1621104587,"num_objects":24692,"num_object_clones":0,"num_object_copies":74076,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":27161,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24692,"num_whiteouts":0,"num_read":6275,"num_read_kb":13708,"num_write":161978,"num_write_kb":5952230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11446,"num_bytes_recovered":641688802,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":177147,"ondisk_log_size":177147,"up":297,"acting":280,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":344,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":8281776,"kb_used_data":5514680,"kb_used_omap":29,"kb_used_meta":2766946,"kb_avail":647934288,"statfs":{"total":671965249536,"available":663484710912,"internally_reserved":0,"allocated":5647032320,"data_stored":5638765066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30674,"internal_metadata":2833352750},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,7,3,0,4,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-366673920,"num_objects":-5595,"num_object_clones":0,"num_object_copies":-16785,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-734,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-5595,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5595,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":735,"num_bytes_recovered":23724032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005825"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":324,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-08-28T02:34:12.148831+0000","last_change":"2023-08-28T02:32:58.740384+0000","last_active":"2023-08-28T02:34:12.148831+0000","last_peered":"2023-08-28T02:34:12.148831+0000","last_clean":"2023-08-28T02:34:12.148831+0000","last_became_active":"2023-08-28T02:32:58.739959+0000","last_became_peered":"2023-08-28T02:32:58.739959+0000","last_unstale":"2023-08-28T02:34:12.148831+0000","last_undegraded":"2023-08-28T02:34:12.148831+0000","last_fullsized":"2023-08-28T02:34:12.148831+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":210,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-08-28T02:34:11.145883+0000","last_change":"2023-08-28T02:32:58.634375+0000","last_active":"2023-08-28T02:34:11.145883+0000","last_peered":"2023-08-28T02:34:11.145883+0000","last_clean":"2023-08-28T02:34:11.145883+0000","last_became_active":"2023-08-28T02:32:58.632726+0000","last_became_peered":"2023-08-28T02:32:58.632726+0000","last_unstale":"2023-08-28T02:34:11.145883+0000","last_undegraded":"2023-08-28T02:34:11.145883+0000","last_fullsized":"2023-08-28T02:34:11.145883+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:20:27.730725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":23,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-08-28T02:34:11.146298+0000","last_change":"2023-08-28T02:32:58.628387+0000","last_active":"2023-08-28T02:34:11.146298+0000","last_peered":"2023-08-28T02:34:11.146298+0000","last_clean":"2023-08-28T02:34:11.146298+0000","last_became_active":"2023-08-28T02:32:58.628071+0000","last_became_peered":"2023-08-28T02:32:58.628071+0000","last_unstale":"2023-08-28T02:34:11.146298+0000","last_undegraded":"2023-08-28T02:34:11.146298+0000","last_fullsized":"2023-08-28T02:34:11.146298+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:25:55.264433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":320,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-08-28T02:34:11.152746+0000","last_change":"2023-08-28T02:32:58.772285+0000","last_active":"2023-08-28T02:34:11.152746+0000","last_peered":"2023-08-28T02:34:11.152746+0000","last_clean":"2023-08-28T02:34:11.152746+0000","last_became_active":"2023-08-28T02:32:58.771439+0000","last_became_peered":"2023-08-28T02:32:58.771439+0000","last_unstale":"2023-08-28T02:34:11.152746+0000","last_undegraded":"2023-08-28T02:34:11.152746+0000","last_fullsized":"2023-08-28T02:34:11.152746+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:10:48.541785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":201,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-08-28T02:34:11.155629+0000","last_change":"2023-08-28T02:32:58.779602+0000","last_active":"2023-08-28T02:34:11.155629+0000","last_peered":"2023-08-28T02:34:11.155629+0000","last_clean":"2023-08-28T02:34:11.155629+0000","last_became_active":"2023-08-28T02:32:58.778945+0000","last_became_peered":"2023-08-28T02:32:58.778945+0000","last_unstale":"2023-08-28T02:34:11.155629+0000","last_undegraded":"2023-08-28T02:34:11.155629+0000","last_fullsized":"2023-08-28T02:34:11.155629+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:35:55.832928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":111,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-08-28T02:34:12.235201+0000","last_change":"2023-08-28T02:32:58.626766+0000","last_active":"2023-08-28T02:34:12.235201+0000","last_peered":"2023-08-28T02:34:12.235201+0000","last_clean":"2023-08-28T02:34:12.235201+0000","last_became_active":"2023-08-28T02:32:58.626415+0000","last_became_peered":"2023-08-28T02:32:58.626415+0000","last_unstale":"2023-08-28T02:34:12.235201+0000","last_undegraded":"2023-08-28T02:34:12.235201+0000","last_fullsized":"2023-08-28T02:34:12.235201+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.13","version":"304'6856","reported_seq":6919,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-08-28T02:34:19.751538+0000","last_change":"2023-08-28T02:31:44.484177+0000","last_active":"2023-08-28T02:34:19.751538+0000","last_peered":"2023-08-28T02:34:19.751538+0000","last_clean":"2023-08-28T02:34:19.751538+0000","last_became_active":"2023-08-28T02:31:44.482443+0000","last_became_peered":"2023-08-28T02:31:44.482443+0000","last_unstale":"2023-08-28T02:34:19.751538+0000","last_undegraded":"2023-08-28T02:34:19.751538+0000","last_fullsized":"2023-08-28T02:34:19.751538+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":6856,"log_dups_size":0,"ondisk_log_size":6856,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55181312,"num_objects":842,"num_object_clones":0,"num_object_copies":2526,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":842,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4986,"num_write_kb":186496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":273,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-08-28T02:34:11.152224+0000","last_change":"2023-08-28T02:23:59.227592+0000","last_active":"2023-08-28T02:34:11.152224+0000","last_peered":"2023-08-28T02:34:11.152224+0000","last_clean":"2023-08-28T02:34:11.152224+0000","last_became_active":"2023-08-28T02:23:59.227342+0000","last_became_peered":"2023-08-28T02:23:59.227342+0000","last_unstale":"2023-08-28T02:34:11.152224+0000","last_undegraded":"2023-08-28T02:34:11.152224+0000","last_fullsized":"2023-08-28T02:34:11.152224+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:08:19.874396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":219,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-08-28T02:34:11.155692+0000","last_change":"2023-08-28T02:32:58.779435+0000","last_active":"2023-08-28T02:34:11.155692+0000","last_peered":"2023-08-28T02:34:11.155692+0000","last_clean":"2023-08-28T02:34:11.155692+0000","last_became_active":"2023-08-28T02:32:58.778629+0000","last_became_peered":"2023-08-28T02:32:58.778629+0000","last_unstale":"2023-08-28T02:34:11.155692+0000","last_undegraded":"2023-08-28T02:34:11.155692+0000","last_fullsized":"2023-08-28T02:34:11.155692+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:25:32.193898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.12","version":"304'6985","reported_seq":7064,"reported_epoch":304,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:34:19.654198+0000","last_change":"2023-08-28T02:32:58.866748+0000","last_active":"2023-08-28T02:34:19.654198+0000","last_peered":"2023-08-28T02:34:19.654198+0000","last_clean":"2023-08-28T02:32:11.498129+0000","last_became_active":"2023-08-28T02:32:58.859951+0000","last_became_peered":"2023-08-28T02:32:58.859951+0000","last_unstale":"2023-08-28T02:34:19.654198+0000","last_undegraded":"2023-08-28T02:32:58.636278+0000","last_fullsized":"2023-08-28T02:32:58.628896+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":6985,"log_dups_size":0,"ondisk_log_size":6985,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50069504,"num_objects":764,"num_object_clones":0,"num_object_copies":2292,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":970,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":764,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5080,"num_write_kb":187008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[{"shards":"2,3","objects":371},{"shards":"2,3,6","objects":393}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":334,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-08-28T02:34:12.230549+0000","last_change":"2023-08-28T02:32:58.636365+0000","last_active":"2023-08-28T02:34:12.230549+0000","last_peered":"2023-08-28T02:34:12.230549+0000","last_clean":"2023-08-28T02:34:12.230549+0000","last_became_active":"2023-08-28T02:32:58.635998+0000","last_became_peered":"2023-08-28T02:32:58.635998+0000","last_unstale":"2023-08-28T02:34:12.230549+0000","last_undegraded":"2023-08-28T02:34:12.230549+0000","last_fullsized":"2023-08-28T02:34:12.230549+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":192,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-08-28T02:34:11.144865+0000","last_change":"2023-08-28T02:31:55.525178+0000","last_active":"2023-08-28T02:34:11.144865+0000","last_peered":"2023-08-28T02:34:11.144865+0000","last_clean":"2023-08-28T02:34:11.144865+0000","last_became_active":"2023-08-28T02:31:55.524801+0000","last_became_peered":"2023-08-28T02:31:55.524801+0000","last_unstale":"2023-08-28T02:34:11.144865+0000","last_undegraded":"2023-08-28T02:34:11.144865+0000","last_fullsized":"2023-08-28T02:34:11.144865+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:47:30.078960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.11","version":"305'7017","reported_seq":7095,"reported_epoch":305,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:34:22.151661+0000","last_change":"2023-08-28T02:32:58.859410+0000","last_active":"2023-08-28T02:34:22.151661+0000","last_peered":"2023-08-28T02:34:22.151661+0000","last_clean":"2023-08-28T02:31:51.768660+0000","last_became_active":"2023-08-28T02:32:58.852854+0000","last_became_peered":"2023-08-28T02:32:58.852854+0000","last_unstale":"2023-08-28T02:34:22.151661+0000","last_undegraded":"2023-08-28T02:32:58.629343+0000","last_fullsized":"2023-08-28T02:32:58.617323+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":7017,"log_dups_size":0,"ondisk_log_size":7017,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49446912,"num_objects":755,"num_object_clones":0,"num_object_copies":2265,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2588,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":755,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5133,"num_write_kb":188384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3],"avail_no_missing":["6","3","4"],"object_location_counts":[{"shards":"2,3,6","objects":411},{"shards":"3,6","objects":-1270}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":70,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-08-28T02:34:11.150627+0000","last_change":"2023-08-28T02:31:55.532306+0000","last_active":"2023-08-28T02:34:11.150627+0000","last_peered":"2023-08-28T02:34:11.150627+0000","last_clean":"2023-08-28T02:34:11.150627+0000","last_became_active":"2023-08-28T02:31:55.524276+0000","last_became_peered":"2023-08-28T02:31:55.524276+0000","last_unstale":"2023-08-28T02:34:11.150627+0000","last_undegraded":"2023-08-28T02:34:11.150627+0000","last_fullsized":"2023-08-28T02:34:11.150627+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:27:06.850784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":23,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-08-28T02:34:11.146769+0000","last_change":"2023-08-28T02:32:58.658343+0000","last_active":"2023-08-28T02:34:11.146769+0000","last_peered":"2023-08-28T02:34:11.146769+0000","last_clean":"2023-08-28T02:34:11.146769+0000","last_became_active":"2023-08-28T02:32:58.654410+0000","last_became_peered":"2023-08-28T02:32:58.654410+0000","last_unstale":"2023-08-28T02:34:11.146769+0000","last_undegraded":"2023-08-28T02:34:11.146769+0000","last_fullsized":"2023-08-28T02:34:11.146769+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:46:29.260724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.10","version":"304'6814","reported_seq":11075,"reported_epoch":304,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:34:19.794066+0000","last_change":"2023-08-28T02:32:58.871328+0000","last_active":"2023-08-28T02:34:19.794066+0000","last_peered":"2023-08-28T02:34:19.794066+0000","last_clean":"2023-08-28T02:31:51.241963+0000","last_became_active":"2023-08-28T02:32:58.820349+0000","last_became_peered":"2023-08-28T02:32:58.820349+0000","last_unstale":"2023-08-28T02:34:19.794066+0000","last_undegraded":"2023-08-28T02:32:58.770460+0000","last_fullsized":"2023-08-28T02:32:58.618203+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":6814,"log_dups_size":0,"ondisk_log_size":6814,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51806208,"num_objects":791,"num_object_clones":0,"num_object_copies":2373,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":975,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":791,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4965,"num_write_kb":184160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2088,"num_bytes_recovered":136708096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"1,3,5","objects":35},{"shards":"2,3","objects":370},{"shards":"2,3,4","objects":386}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"86'22","reported_seq":374,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-08-28T02:34:12.149185+0000","last_change":"2023-08-28T02:31:55.538001+0000","last_active":"2023-08-28T02:34:12.149185+0000","last_peered":"2023-08-28T02:34:12.149185+0000","last_clean":"2023-08-28T02:34:12.149185+0000","last_became_active":"2023-08-28T02:31:55.518604+0000","last_became_peered":"2023-08-28T02:31:55.518604+0000","last_unstale":"2023-08-28T02:34:12.149185+0000","last_undegraded":"2023-08-28T02:34:12.149185+0000","last_fullsized":"2023-08-28T02:34:12.149185+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":348,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-08-28T02:34:12.148895+0000","last_change":"2023-08-28T02:32:58.629451+0000","last_active":"2023-08-28T02:34:12.148895+0000","last_peered":"2023-08-28T02:34:12.148895+0000","last_clean":"2023-08-28T02:34:12.148895+0000","last_became_active":"2023-08-28T02:32:58.628767+0000","last_became_peered":"2023-08-28T02:32:58.628767+0000","last_unstale":"2023-08-28T02:34:12.148895+0000","last_undegraded":"2023-08-28T02:34:12.148895+0000","last_fullsized":"2023-08-28T02:34:12.148895+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.17","version":"305'6880","reported_seq":6939,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-08-28T02:34:22.042200+0000","last_change":"2023-08-28T02:31:44.475304+0000","last_active":"2023-08-28T02:34:22.042200+0000","last_peered":"2023-08-28T02:34:22.042200+0000","last_clean":"2023-08-28T02:34:22.042200+0000","last_became_active":"2023-08-28T02:31:44.473853+0000","last_became_peered":"2023-08-28T02:31:44.473853+0000","last_unstale":"2023-08-28T02:34:22.042200+0000","last_undegraded":"2023-08-28T02:34:22.042200+0000","last_fullsized":"2023-08-28T02:34:22.042200+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":6880,"log_dups_size":0,"ondisk_log_size":6880,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46661632,"num_objects":712,"num_object_clones":0,"num_object_copies":2136,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":712,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5016,"num_write_kb":183296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":78,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-08-28T02:34:12.148989+0000","last_change":"2023-08-28T02:31:55.528552+0000","last_active":"2023-08-28T02:34:12.148989+0000","last_peered":"2023-08-28T02:34:12.148989+0000","last_clean":"2023-08-28T02:34:12.148989+0000","last_became_active":"2023-08-28T02:31:55.518079+0000","last_became_peered":"2023-08-28T02:31:55.518079+0000","last_unstale":"2023-08-28T02:34:12.148989+0000","last_undegraded":"2023-08-28T02:34:12.148989+0000","last_fullsized":"2023-08-28T02:34:12.148989+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":210,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-08-28T02:34:11.145386+0000","last_change":"2023-08-28T02:32:58.680534+0000","last_active":"2023-08-28T02:34:11.145386+0000","last_peered":"2023-08-28T02:34:11.145386+0000","last_clean":"2023-08-28T02:34:11.145386+0000","last_became_active":"2023-08-28T02:32:58.680145+0000","last_became_peered":"2023-08-28T02:32:58.680145+0000","last_unstale":"2023-08-28T02:34:11.145386+0000","last_undegraded":"2023-08-28T02:34:11.145386+0000","last_fullsized":"2023-08-28T02:34:11.145386+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:56:40.113013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.16","version":"304'6703","reported_seq":11004,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-08-28T02:34:19.455559+0000","last_change":"2023-08-28T02:32:37.600254+0000","last_active":"2023-08-28T02:34:19.455559+0000","last_peered":"2023-08-28T02:34:19.455559+0000","last_clean":"2023-08-28T02:34:19.455559+0000","last_became_active":"2023-08-28T02:32:37.596763+0000","last_became_peered":"2023-08-28T02:32:37.596763+0000","last_unstale":"2023-08-28T02:34:19.455559+0000","last_undegraded":"2023-08-28T02:34:19.455559+0000","last_fullsized":"2023-08-28T02:34:19.455559+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":270,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":6703,"log_dups_size":0,"ondisk_log_size":6703,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52199424,"num_objects":797,"num_object_clones":0,"num_object_copies":2391,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":797,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4897,"num_write_kb":182176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2111,"num_bytes_recovered":138346496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":318,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-08-28T02:34:11.148976+0000","last_change":"2023-08-28T02:32:58.746929+0000","last_active":"2023-08-28T02:34:11.148976+0000","last_peered":"2023-08-28T02:34:11.148976+0000","last_clean":"2023-08-28T02:34:11.148976+0000","last_became_active":"2023-08-28T02:32:58.746590+0000","last_became_peered":"2023-08-28T02:32:58.746590+0000","last_unstale":"2023-08-28T02:34:11.148976+0000","last_undegraded":"2023-08-28T02:34:11.148976+0000","last_fullsized":"2023-08-28T02:34:11.148976+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:51:11.974154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":340,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-08-28T02:34:12.230693+0000","last_change":"2023-08-28T02:31:55.521470+0000","last_active":"2023-08-28T02:34:12.230693+0000","last_peered":"2023-08-28T02:34:12.230693+0000","last_clean":"2023-08-28T02:34:12.230693+0000","last_became_active":"2023-08-28T02:31:55.520398+0000","last_became_peered":"2023-08-28T02:31:55.520398+0000","last_unstale":"2023-08-28T02:34:12.230693+0000","last_undegraded":"2023-08-28T02:34:12.230693+0000","last_fullsized":"2023-08-28T02:34:12.230693+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.15","version":"304'6799","reported_seq":6881,"reported_epoch":304,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:34:19.842449+0000","last_change":"2023-08-28T02:32:58.933245+0000","last_active":"2023-08-28T02:34:19.842449+0000","last_peered":"2023-08-28T02:34:19.842449+0000","last_clean":"2023-08-28T02:31:54.454430+0000","last_became_active":"2023-08-28T02:32:58.916111+0000","last_became_peered":"2023-08-28T02:32:58.916111+0000","last_unstale":"2023-08-28T02:34:19.842449+0000","last_undegraded":"2023-08-28T02:32:58.647695+0000","last_fullsized":"2023-08-28T02:32:58.611609+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":6799,"log_dups_size":0,"ondisk_log_size":6799,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51412992,"num_objects":785,"num_object_clones":0,"num_object_copies":2355,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2539,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":785,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4977,"num_write_kb":184352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":-1220},{"shards":"0,2,6","objects":392}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":134,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-08-28T02:34:12.148987+0000","last_change":"2023-08-28T02:31:55.517851+0000","last_active":"2023-08-28T02:34:12.148987+0000","last_peered":"2023-08-28T02:34:12.148987+0000","last_clean":"2023-08-28T02:34:12.148987+0000","last_became_active":"2023-08-28T02:31:55.517320+0000","last_became_peered":"2023-08-28T02:31:55.517320+0000","last_unstale":"2023-08-28T02:34:12.148987+0000","last_undegraded":"2023-08-28T02:34:12.148987+0000","last_fullsized":"2023-08-28T02:34:12.148987+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":301,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-08-28T02:34:11.153729+0000","last_change":"2023-08-28T02:31:55.528250+0000","last_active":"2023-08-28T02:34:11.153729+0000","last_peered":"2023-08-28T02:34:11.153729+0000","last_clean":"2023-08-28T02:34:11.153729+0000","last_became_active":"2023-08-28T02:31:55.527950+0000","last_became_peered":"2023-08-28T02:31:55.527950+0000","last_unstale":"2023-08-28T02:34:11.153729+0000","last_undegraded":"2023-08-28T02:34:11.153729+0000","last_fullsized":"2023-08-28T02:34:11.153729+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:39:46.120437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.14","version":"304'6976","reported_seq":7055,"reported_epoch":304,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:34:19.387237+0000","last_change":"2023-08-28T02:32:58.849338+0000","last_active":"2023-08-28T02:34:19.387237+0000","last_peered":"2023-08-28T02:34:19.387237+0000","last_clean":"2023-08-28T02:32:11.458469+0000","last_became_active":"2023-08-28T02:32:58.840304+0000","last_became_peered":"2023-08-28T02:32:58.840304+0000","last_unstale":"2023-08-28T02:34:19.387237+0000","last_undegraded":"2023-08-28T02:32:58.655685+0000","last_fullsized":"2023-08-28T02:32:58.616249+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":6976,"log_dups_size":0,"ondisk_log_size":6976,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57507840,"num_objects":878,"num_object_clones":0,"num_object_copies":2634,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":979,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":878,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5048,"num_write_kb":189600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":461},{"shards":"0,4,6","objects":417}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":103,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-08-28T02:34:12.234289+0000","last_change":"2023-08-28T02:28:25.703333+0000","last_active":"2023-08-28T02:34:12.234289+0000","last_peered":"2023-08-28T02:34:12.234289+0000","last_clean":"2023-08-28T02:34:12.234289+0000","last_became_active":"2023-08-28T02:28:25.343515+0000","last_became_peered":"2023-08-28T02:28:25.343515+0000","last_unstale":"2023-08-28T02:34:12.234289+0000","last_undegraded":"2023-08-28T02:34:12.234289+0000","last_fullsized":"2023-08-28T02:34:12.234289+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":210,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-08-28T02:34:11.152449+0000","last_change":"2023-08-28T02:32:58.758016+0000","last_active":"2023-08-28T02:34:11.152449+0000","last_peered":"2023-08-28T02:34:11.152449+0000","last_clean":"2023-08-28T02:34:11.152449+0000","last_became_active":"2023-08-28T02:32:58.757660+0000","last_became_peered":"2023-08-28T02:32:58.757660+0000","last_unstale":"2023-08-28T02:34:11.152449+0000","last_undegraded":"2023-08-28T02:34:11.152449+0000","last_fullsized":"2023-08-28T02:34:11.152449+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:33:06.239503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"305'10023","reported_seq":10091,"reported_epoch":305,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:34:21.942464+0000","last_change":"2023-08-28T02:32:58.916501+0000","last_active":"2023-08-28T02:34:21.942464+0000","last_peered":"2023-08-28T02:34:21.942464+0000","last_clean":"2023-08-28T02:32:11.456138+0000","last_became_active":"2023-08-28T02:32:58.901683+0000","last_became_peered":"2023-08-28T02:32:58.901683+0000","last_unstale":"2023-08-28T02:34:21.942464+0000","last_undegraded":"2023-08-28T02:32:58.633417+0000","last_fullsized":"2023-08-28T02:32:58.615180+0000","mapping_epoch":291,"log_start":"265'1100","ondisk_log_start":"265'1100","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":8923,"log_dups_size":0,"ondisk_log_size":8923,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97976320,"num_objects":1495,"num_object_clones":0,"num_object_copies":4485,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1873,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1495,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10023,"num_write_kb":368576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"2,5,6","objects":836},{"shards":"5,6","objects":659}],"blocked_by":[],"up_primary":1,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":166,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-08-28T02:34:12.151831+0000","last_change":"2023-08-28T02:32:58.782840+0000","last_active":"2023-08-28T02:34:12.151831+0000","last_peered":"2023-08-28T02:34:12.151831+0000","last_clean":"2023-08-28T02:34:12.151831+0000","last_became_active":"2023-08-28T02:32:58.683096+0000","last_became_peered":"2023-08-28T02:32:58.683096+0000","last_unstale":"2023-08-28T02:34:12.151831+0000","last_undegraded":"2023-08-28T02:34:12.151831+0000","last_fullsized":"2023-08-28T02:34:12.151831+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:10:08.037078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":300,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-08-28T02:34:12.230520+0000","last_change":"2023-08-28T02:31:55.514646+0000","last_active":"2023-08-28T02:34:12.230520+0000","last_peered":"2023-08-28T02:34:12.230520+0000","last_clean":"2023-08-28T02:34:12.230520+0000","last_became_active":"2023-08-28T02:31:55.514306+0000","last_became_peered":"2023-08-28T02:31:55.514306+0000","last_unstale":"2023-08-28T02:34:12.230520+0000","last_undegraded":"2023-08-28T02:34:12.230520+0000","last_fullsized":"2023-08-28T02:34:12.230520+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"304'10203","reported_seq":10256,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-08-28T02:34:19.809290+0000","last_change":"2023-08-28T02:30:57.067718+0000","last_active":"2023-08-28T02:34:19.809290+0000","last_peered":"2023-08-28T02:34:19.809290+0000","last_clean":"2023-08-28T02:34:19.809290+0000","last_became_active":"2023-08-28T02:30:57.067407+0000","last_became_peered":"2023-08-28T02:30:57.067407+0000","last_unstale":"2023-08-28T02:34:19.809290+0000","last_undegraded":"2023-08-28T02:34:19.809290+0000","last_fullsized":"2023-08-28T02:34:19.809290+0000","mapping_epoch":261,"log_start":"279'4865","ondisk_log_start":"279'4865","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5338,"log_dups_size":0,"ondisk_log_size":5338,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:49:39.360134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102694912,"num_objects":1567,"num_object_clones":0,"num_object_copies":4701,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1567,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10203,"num_write_kb":376640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":100,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-08-28T02:34:12.149003+0000","last_change":"2023-08-28T02:32:58.856906+0000","last_active":"2023-08-28T02:34:12.149003+0000","last_peered":"2023-08-28T02:34:12.149003+0000","last_clean":"2023-08-28T02:34:12.149003+0000","last_became_active":"2023-08-28T02:32:58.740028+0000","last_became_peered":"2023-08-28T02:32:58.740028+0000","last_unstale":"2023-08-28T02:34:12.149003+0000","last_undegraded":"2023-08-28T02:34:12.149003+0000","last_fullsized":"2023-08-28T02:34:12.149003+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":358,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-08-28T02:34:12.148882+0000","last_change":"2023-08-28T02:32:58.740396+0000","last_active":"2023-08-28T02:34:12.148882+0000","last_peered":"2023-08-28T02:34:12.148882+0000","last_clean":"2023-08-28T02:34:12.148882+0000","last_became_active":"2023-08-28T02:32:58.739941+0000","last_became_peered":"2023-08-28T02:32:58.739941+0000","last_unstale":"2023-08-28T02:34:12.148882+0000","last_undegraded":"2023-08-28T02:34:12.148882+0000","last_fullsized":"2023-08-28T02:34:12.148882+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"304'6980","reported_seq":7059,"reported_epoch":304,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:34:19.417175+0000","last_change":"2023-08-28T02:32:58.880642+0000","last_active":"2023-08-28T02:34:19.417175+0000","last_peered":"2023-08-28T02:34:19.417175+0000","last_clean":"2023-08-28T02:32:11.446297+0000","last_became_active":"2023-08-28T02:32:58.851158+0000","last_became_peered":"2023-08-28T02:32:58.851158+0000","last_unstale":"2023-08-28T02:34:19.417175+0000","last_undegraded":"2023-08-28T02:32:58.644337+0000","last_fullsized":"2023-08-28T02:32:58.616553+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":6980,"log_dups_size":0,"ondisk_log_size":6980,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47153152,"num_objects":719,"num_object_clones":0,"num_object_copies":2157,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1015,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":719,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5075,"num_write_kb":185440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":317},{"shards":"0,4,6","objects":402}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":74,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-08-28T02:34:12.230747+0000","last_change":"2023-08-28T02:31:55.531216+0000","last_active":"2023-08-28T02:34:12.230747+0000","last_peered":"2023-08-28T02:34:12.230747+0000","last_clean":"2023-08-28T02:34:12.230747+0000","last_became_active":"2023-08-28T02:31:55.521010+0000","last_became_peered":"2023-08-28T02:31:55.521010+0000","last_unstale":"2023-08-28T02:34:12.230747+0000","last_undegraded":"2023-08-28T02:34:12.230747+0000","last_fullsized":"2023-08-28T02:34:12.230747+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":120,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-08-28T02:34:12.234508+0000","last_change":"2023-08-28T02:32:58.628915+0000","last_active":"2023-08-28T02:34:12.234508+0000","last_peered":"2023-08-28T02:34:12.234508+0000","last_clean":"2023-08-28T02:34:12.234508+0000","last_became_active":"2023-08-28T02:32:58.628562+0000","last_became_peered":"2023-08-28T02:32:58.628562+0000","last_unstale":"2023-08-28T02:34:12.234508+0000","last_undegraded":"2023-08-28T02:34:12.234508+0000","last_fullsized":"2023-08-28T02:34:12.234508+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"304'6982","reported_seq":9880,"reported_epoch":304,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:34:19.730629+0000","last_change":"2023-08-28T02:32:58.683607+0000","last_active":"2023-08-28T02:34:19.730629+0000","last_peered":"2023-08-28T02:34:19.730629+0000","last_clean":"2023-08-28T02:31:54.438195+0000","last_became_active":"2023-08-28T02:31:56.704078+0000","last_became_peered":"2023-08-28T02:31:56.704078+0000","last_unstale":"2023-08-28T02:34:19.730629+0000","last_undegraded":"2023-08-28T02:31:56.543040+0000","last_fullsized":"2023-08-28T02:31:56.504447+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":6982,"log_dups_size":0,"ondisk_log_size":6982,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53215232,"num_objects":812,"num_object_clones":0,"num_object_copies":2436,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":759,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":812,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5064,"num_write_kb":188032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1412,"num_bytes_recovered":92536832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,4","objects":53}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":307,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-08-28T02:34:11.145550+0000","last_change":"2023-08-28T02:28:25.445982+0000","last_active":"2023-08-28T02:34:11.145550+0000","last_peered":"2023-08-28T02:34:11.145550+0000","last_clean":"2023-08-28T02:34:11.145550+0000","last_became_active":"2023-08-28T02:28:25.443577+0000","last_became_peered":"2023-08-28T02:28:25.443577+0000","last_unstale":"2023-08-28T02:34:11.145550+0000","last_undegraded":"2023-08-28T02:34:11.145550+0000","last_fullsized":"2023-08-28T02:34:11.145550+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:24:03.381751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":303,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-08-28T02:34:11.151504+0000","last_change":"2023-08-28T02:32:58.887874+0000","last_active":"2023-08-28T02:34:11.151504+0000","last_peered":"2023-08-28T02:34:11.151504+0000","last_clean":"2023-08-28T02:34:11.151504+0000","last_became_active":"2023-08-28T02:32:58.887377+0000","last_became_peered":"2023-08-28T02:32:58.887377+0000","last_unstale":"2023-08-28T02:34:11.151504+0000","last_undegraded":"2023-08-28T02:34:11.151504+0000","last_fullsized":"2023-08-28T02:34:11.151504+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:08:07.234984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"304'10086","reported_seq":18809,"reported_epoch":304,"state":"active+recovery_wait","last_fresh":"2023-08-28T02:34:19.423558+0000","last_change":"2023-08-28T02:34:12.218769+0000","last_active":"2023-08-28T02:34:19.423558+0000","last_peered":"2023-08-28T02:34:19.423558+0000","last_clean":"2023-08-28T02:31:54.434903+0000","last_became_active":"2023-08-28T02:34:12.173792+0000","last_became_peered":"2023-08-28T02:34:12.173792+0000","last_unstale":"2023-08-28T02:34:19.423558+0000","last_undegraded":"2023-08-28T02:34:19.423558+0000","last_fullsized":"2023-08-28T02:34:19.423558+0000","mapping_epoch":303,"log_start":"278'4400","ondisk_log_start":"278'4400","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5686,"log_dups_size":0,"ondisk_log_size":5686,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:01:42.988637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108658688,"num_objects":1658,"num_object_clones":0,"num_object_copies":4974,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1658,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10086,"num_write_kb":375808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4326,"num_bytes_recovered":204275712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":["0","3","4"],"object_location_counts":[{"shards":"0,3,4","objects":1658}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":169,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-08-28T02:34:11.145297+0000","last_change":"2023-08-28T02:23:59.230059+0000","last_active":"2023-08-28T02:34:11.145297+0000","last_peered":"2023-08-28T02:34:11.145297+0000","last_clean":"2023-08-28T02:34:11.145297+0000","last_became_active":"2023-08-28T02:23:59.217783+0000","last_became_peered":"2023-08-28T02:23:59.217783+0000","last_unstale":"2023-08-28T02:34:11.145297+0000","last_undegraded":"2023-08-28T02:34:11.145297+0000","last_fullsized":"2023-08-28T02:34:11.145297+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:00:12.145395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":325,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-08-28T02:34:12.149084+0000","last_change":"2023-08-28T02:28:25.351698+0000","last_active":"2023-08-28T02:34:12.149084+0000","last_peered":"2023-08-28T02:34:12.149084+0000","last_clean":"2023-08-28T02:34:12.149084+0000","last_became_active":"2023-08-28T02:28:25.351349+0000","last_became_peered":"2023-08-28T02:28:25.351349+0000","last_unstale":"2023-08-28T02:34:12.149084+0000","last_undegraded":"2023-08-28T02:34:12.149084+0000","last_fullsized":"2023-08-28T02:34:12.149084+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"304'10167","reported_seq":10220,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-08-28T02:34:19.815847+0000","last_change":"2023-08-28T02:30:57.069579+0000","last_active":"2023-08-28T02:34:19.815847+0000","last_peered":"2023-08-28T02:34:19.815847+0000","last_clean":"2023-08-28T02:34:19.815847+0000","last_became_active":"2023-08-28T02:30:57.069365+0000","last_became_peered":"2023-08-28T02:30:57.069365+0000","last_unstale":"2023-08-28T02:34:19.815847+0000","last_undegraded":"2023-08-28T02:34:19.815847+0000","last_fullsized":"2023-08-28T02:34:19.815847+0000","mapping_epoch":261,"log_start":"268'3600","ondisk_log_start":"268'3600","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":6567,"log_dups_size":0,"ondisk_log_size":6567,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:02:07.549000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101777408,"num_objects":1553,"num_object_clones":0,"num_object_copies":4659,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1553,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10167,"num_write_kb":375040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":111,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-08-28T02:34:12.234411+0000","last_change":"2023-08-28T02:32:58.629249+0000","last_active":"2023-08-28T02:34:12.234411+0000","last_peered":"2023-08-28T02:34:12.234411+0000","last_clean":"2023-08-28T02:34:12.234411+0000","last_became_active":"2023-08-28T02:32:58.628583+0000","last_became_peered":"2023-08-28T02:32:58.628583+0000","last_unstale":"2023-08-28T02:34:12.234411+0000","last_undegraded":"2023-08-28T02:34:12.234411+0000","last_fullsized":"2023-08-28T02:34:12.234411+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":369,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-08-28T02:34:12.148988+0000","last_change":"2023-08-28T02:32:58.663953+0000","last_active":"2023-08-28T02:34:12.148988+0000","last_peered":"2023-08-28T02:34:12.148988+0000","last_clean":"2023-08-28T02:34:12.148988+0000","last_became_active":"2023-08-28T02:32:58.663627+0000","last_became_peered":"2023-08-28T02:32:58.663627+0000","last_unstale":"2023-08-28T02:34:12.148988+0000","last_undegraded":"2023-08-28T02:34:12.148988+0000","last_fullsized":"2023-08-28T02:34:12.148988+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"305'9858","reported_seq":9928,"reported_epoch":305,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:34:22.180361+0000","last_change":"2023-08-28T02:32:58.874546+0000","last_active":"2023-08-28T02:34:22.180361+0000","last_peered":"2023-08-28T02:34:22.180361+0000","last_clean":"2023-08-28T02:32:11.510497+0000","last_became_active":"2023-08-28T02:32:58.863497+0000","last_became_peered":"2023-08-28T02:32:58.863497+0000","last_unstale":"2023-08-28T02:34:22.180361+0000","last_undegraded":"2023-08-28T02:32:58.633053+0000","last_fullsized":"2023-08-28T02:32:58.617566+0000","mapping_epoch":291,"log_start":"275'4194","ondisk_log_start":"275'4194","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5664,"log_dups_size":0,"ondisk_log_size":5664,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94175256,"num_objects":1438,"num_object_clones":0,"num_object_copies":4314,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1818,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1438,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9858,"num_write_kb":361409,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,3,6","objects":768},{"shards":"0,6","objects":670}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":241,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-08-28T02:34:12.149090+0000","last_change":"2023-08-28T02:32:58.740311+0000","last_active":"2023-08-28T02:34:12.149090+0000","last_peered":"2023-08-28T02:34:12.149090+0000","last_clean":"2023-08-28T02:34:12.149090+0000","last_became_active":"2023-08-28T02:32:58.739897+0000","last_became_peered":"2023-08-28T02:32:58.739897+0000","last_unstale":"2023-08-28T02:34:12.149090+0000","last_undegraded":"2023-08-28T02:34:12.149090+0000","last_fullsized":"2023-08-28T02:34:12.149090+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":290,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-08-28T02:34:11.153799+0000","last_change":"2023-08-28T02:32:58.766795+0000","last_active":"2023-08-28T02:34:11.153799+0000","last_peered":"2023-08-28T02:34:11.153799+0000","last_clean":"2023-08-28T02:34:11.153799+0000","last_became_active":"2023-08-28T02:32:58.766441+0000","last_became_peered":"2023-08-28T02:32:58.766441+0000","last_unstale":"2023-08-28T02:34:11.153799+0000","last_undegraded":"2023-08-28T02:34:11.153799+0000","last_fullsized":"2023-08-28T02:34:11.153799+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:58:13.963961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.c","version":"305'9960","reported_seq":10019,"reported_epoch":305,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:34:22.189659+0000","last_change":"2023-08-28T02:31:56.737061+0000","last_active":"2023-08-28T02:34:22.189659+0000","last_peered":"2023-08-28T02:34:22.189659+0000","last_clean":"2023-08-28T02:31:54.457709+0000","last_became_active":"2023-08-28T02:31:56.726804+0000","last_became_peered":"2023-08-28T02:31:56.726804+0000","last_unstale":"2023-08-28T02:34:22.189659+0000","last_undegraded":"2023-08-28T02:31:56.538425+0000","last_fullsized":"2023-08-28T02:31:56.509601+0000","mapping_epoch":277,"log_start":"273'4242","ondisk_log_start":"273'4242","created":261,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":5718,"log_dups_size":0,"ondisk_log_size":5718,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93585408,"num_objects":1428,"num_object_clones":0,"num_object_copies":4284,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4274,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1428,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9960,"num_write_kb":364416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3],"avail_no_missing":["6","3"],"object_location_counts":[{"shards":"3,6","objects":-2846}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":210,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-08-28T02:34:11.155845+0000","last_change":"2023-08-28T02:32:58.779257+0000","last_active":"2023-08-28T02:34:11.155845+0000","last_peered":"2023-08-28T02:34:11.155845+0000","last_clean":"2023-08-28T02:34:11.155845+0000","last_became_active":"2023-08-28T02:32:58.778295+0000","last_became_peered":"2023-08-28T02:32:58.778295+0000","last_unstale":"2023-08-28T02:34:11.155845+0000","last_undegraded":"2023-08-28T02:34:11.155845+0000","last_fullsized":"2023-08-28T02:34:11.155845+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:50:49.444715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":23,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-08-28T02:34:11.147524+0000","last_change":"2023-08-28T02:32:58.755765+0000","last_active":"2023-08-28T02:34:11.147524+0000","last_peered":"2023-08-28T02:34:11.147524+0000","last_clean":"2023-08-28T02:34:11.147524+0000","last_became_active":"2023-08-28T02:32:58.755438+0000","last_became_peered":"2023-08-28T02:32:58.755438+0000","last_unstale":"2023-08-28T02:34:11.147524+0000","last_undegraded":"2023-08-28T02:34:11.147524+0000","last_fullsized":"2023-08-28T02:34:11.147524+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:14:11.498611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"305'6920","reported_seq":6985,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-08-28T02:34:21.888226+0000","last_change":"2023-08-28T02:32:14.983506+0000","last_active":"2023-08-28T02:34:21.888226+0000","last_peered":"2023-08-28T02:34:21.888226+0000","last_clean":"2023-08-28T02:34:21.888226+0000","last_became_active":"2023-08-28T02:31:44.479295+0000","last_became_peered":"2023-08-28T02:31:44.479295+0000","last_unstale":"2023-08-28T02:34:21.888226+0000","last_undegraded":"2023-08-28T02:34:21.888226+0000","last_fullsized":"2023-08-28T02:34:21.888226+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":6920,"log_dups_size":0,"ondisk_log_size":6920,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49250304,"num_objects":751,"num_object_clones":0,"num_object_copies":2253,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":751,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5035,"num_write_kb":185184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"78'126","reported_seq":681,"reported_epoch":303,"state":"active+recovery_wait+degraded","last_fresh":"2023-08-28T02:34:11.146963+0000","last_change":"2023-08-28T02:32:59.660815+0000","last_active":"2023-08-28T02:34:11.146963+0000","last_peered":"2023-08-28T02:34:11.146963+0000","last_clean":"2023-08-28T02:27:29.502868+0000","last_became_active":"2023-08-28T02:32:59.660388+0000","last_became_peered":"2023-08-28T02:32:59.660388+0000","last_unstale":"2023-08-28T02:34:11.146963+0000","last_undegraded":"2023-08-28T02:32:59.633238+0000","last_fullsized":"2023-08-28T02:34:11.146963+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:58:54.023571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":743,"num_read_kb":1331,"num_write":125,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":["1","0"],"object_location_counts":[{"shards":"0,1","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":304,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-08-28T02:34:12.230626+0000","last_change":"2023-08-28T02:23:59.207148+0000","last_active":"2023-08-28T02:34:12.230626+0000","last_peered":"2023-08-28T02:34:12.230626+0000","last_clean":"2023-08-28T02:34:12.230626+0000","last_became_active":"2023-08-28T02:23:59.206802+0000","last_became_peered":"2023-08-28T02:23:59.206802+0000","last_unstale":"2023-08-28T02:34:12.230626+0000","last_undegraded":"2023-08-28T02:34:12.230626+0000","last_fullsized":"2023-08-28T02:34:12.230626+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":23,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-08-28T02:34:11.146496+0000","last_change":"2023-08-28T02:32:58.626357+0000","last_active":"2023-08-28T02:34:11.146496+0000","last_peered":"2023-08-28T02:34:11.146496+0000","last_clean":"2023-08-28T02:34:11.146496+0000","last_became_active":"2023-08-28T02:32:58.622671+0000","last_became_peered":"2023-08-28T02:32:58.622671+0000","last_unstale":"2023-08-28T02:34:11.146496+0000","last_undegraded":"2023-08-28T02:34:11.146496+0000","last_fullsized":"2023-08-28T02:34:11.146496+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:21:30.262904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"305'6883","reported_seq":6943,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-08-28T02:34:22.172734+0000","last_change":"2023-08-28T02:31:44.472455+0000","last_active":"2023-08-28T02:34:22.172734+0000","last_peered":"2023-08-28T02:34:22.172734+0000","last_clean":"2023-08-28T02:34:22.172734+0000","last_became_active":"2023-08-28T02:31:44.470716+0000","last_became_peered":"2023-08-28T02:31:44.470716+0000","last_unstale":"2023-08-28T02:34:22.172734+0000","last_undegraded":"2023-08-28T02:34:22.172734+0000","last_fullsized":"2023-08-28T02:34:22.172734+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":6883,"log_dups_size":0,"ondisk_log_size":6883,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46202880,"num_objects":705,"num_object_clones":0,"num_object_copies":2115,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":705,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5019,"num_write_kb":183168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"78'204","reported_seq":1387,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-08-28T02:34:11.146706+0000","last_change":"2023-08-28T02:32:59.644574+0000","last_active":"2023-08-28T02:34:11.146706+0000","last_peered":"2023-08-28T02:34:11.146706+0000","last_clean":"2023-08-28T02:34:11.146706+0000","last_became_active":"2023-08-28T02:32:59.644053+0000","last_became_peered":"2023-08-28T02:32:59.644053+0000","last_unstale":"2023-08-28T02:34:11.146706+0000","last_undegraded":"2023-08-28T02:34:11.146706+0000","last_fullsized":"2023-08-28T02:34:11.146706+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":204,"log_dups_size":0,"ondisk_log_size":204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:55:51.460483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1521,"num_read_kb":2511,"num_write":178,"num_write_kb":1175,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":299,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-08-28T02:34:12.149003+0000","last_change":"2023-08-28T02:28:25.352417+0000","last_active":"2023-08-28T02:34:12.149003+0000","last_peered":"2023-08-28T02:34:12.149003+0000","last_clean":"2023-08-28T02:34:12.149003+0000","last_became_active":"2023-08-28T02:28:25.352216+0000","last_became_peered":"2023-08-28T02:28:25.352216+0000","last_unstale":"2023-08-28T02:34:12.149003+0000","last_undegraded":"2023-08-28T02:34:12.149003+0000","last_fullsized":"2023-08-28T02:34:12.149003+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":23,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-08-28T02:34:11.146749+0000","last_change":"2023-08-28T02:32:58.720576+0000","last_active":"2023-08-28T02:34:11.146749+0000","last_peered":"2023-08-28T02:34:11.146749+0000","last_clean":"2023-08-28T02:34:11.146749+0000","last_became_active":"2023-08-28T02:32:58.719924+0000","last_became_peered":"2023-08-28T02:32:58.719924+0000","last_unstale":"2023-08-28T02:34:11.146749+0000","last_undegraded":"2023-08-28T02:34:11.146749+0000","last_fullsized":"2023-08-28T02:34:11.146749+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:28:50.697438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"304'7136","reported_seq":7216,"reported_epoch":304,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:34:19.416578+0000","last_change":"2023-08-28T02:32:58.870443+0000","last_active":"2023-08-28T02:34:19.416578+0000","last_peered":"2023-08-28T02:34:19.416578+0000","last_clean":"2023-08-28T02:32:11.480338+0000","last_became_active":"2023-08-28T02:32:58.848058+0000","last_became_peered":"2023-08-28T02:32:58.848058+0000","last_unstale":"2023-08-28T02:34:19.416578+0000","last_undegraded":"2023-08-28T02:32:58.664973+0000","last_fullsized":"2023-08-28T02:32:58.616174+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":7136,"log_dups_size":0,"ondisk_log_size":7136,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47153152,"num_objects":719,"num_object_clones":0,"num_object_copies":2157,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1029,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":719,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5207,"num_write_kb":189664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":299},{"shards":"0,4,6","objects":420}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"78'47","reported_seq":522,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-08-28T02:34:11.157555+0000","last_change":"2023-08-28T02:24:05.799647+0000","last_active":"2023-08-28T02:34:11.157555+0000","last_peered":"2023-08-28T02:34:11.157555+0000","last_clean":"2023-08-28T02:34:11.157555+0000","last_became_active":"2023-08-28T02:23:59.210481+0000","last_became_peered":"2023-08-28T02:23:59.210481+0000","last_unstale":"2023-08-28T02:34:11.157555+0000","last_undegraded":"2023-08-28T02:34:11.157555+0000","last_fullsized":"2023-08-28T02:34:11.157555+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":47,"log_dups_size":0,"ondisk_log_size":47,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:40:18.542375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":162,"num_read_kb":1281,"num_write":48,"num_write_kb":1153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":23,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-08-28T02:34:11.148322+0000","last_change":"2023-08-28T02:32:58.622079+0000","last_active":"2023-08-28T02:34:11.148322+0000","last_peered":"2023-08-28T02:34:11.148322+0000","last_clean":"2023-08-28T02:34:11.148322+0000","last_became_active":"2023-08-28T02:32:58.621819+0000","last_became_peered":"2023-08-28T02:32:58.621819+0000","last_unstale":"2023-08-28T02:34:11.148322+0000","last_undegraded":"2023-08-28T02:34:11.148322+0000","last_fullsized":"2023-08-28T02:34:11.148322+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:03:36.676500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":23,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-08-28T02:34:11.147483+0000","last_change":"2023-08-28T02:32:58.724424+0000","last_active":"2023-08-28T02:34:11.147483+0000","last_peered":"2023-08-28T02:34:11.147483+0000","last_clean":"2023-08-28T02:34:11.147483+0000","last_became_active":"2023-08-28T02:32:58.724127+0000","last_became_peered":"2023-08-28T02:32:58.724127+0000","last_unstale":"2023-08-28T02:34:11.147483+0000","last_undegraded":"2023-08-28T02:34:11.147483+0000","last_fullsized":"2023-08-28T02:34:11.147483+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:02:25.178712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"304'6624","reported_seq":6700,"reported_epoch":304,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:34:19.474599+0000","last_change":"2023-08-28T02:32:12.589239+0000","last_active":"2023-08-28T02:34:19.474599+0000","last_peered":"2023-08-28T02:34:19.474599+0000","last_clean":"2023-08-28T02:31:54.462738+0000","last_became_active":"2023-08-28T02:31:56.607923+0000","last_became_peered":"2023-08-28T02:31:56.607923+0000","last_unstale":"2023-08-28T02:34:19.474599+0000","last_undegraded":"2023-08-28T02:32:12.587852+0000","last_fullsized":"2023-08-28T02:31:56.514083+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":6624,"log_dups_size":0,"ondisk_log_size":6624,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53444608,"num_objects":815,"num_object_clones":0,"num_object_copies":2445,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2060,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":815,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4817,"num_write_kb":180256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":-1245}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"78'114","reported_seq":701,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-08-28T02:34:12.231359+0000","last_change":"2023-08-28T02:32:14.626614+0000","last_active":"2023-08-28T02:34:12.231359+0000","last_peered":"2023-08-28T02:34:12.231359+0000","last_clean":"2023-08-28T02:34:12.231359+0000","last_became_active":"2023-08-28T02:32:14.626181+0000","last_became_peered":"2023-08-28T02:32:14.626181+0000","last_unstale":"2023-08-28T02:34:12.231359+0000","last_undegraded":"2023-08-28T02:34:12.231359+0000","last_fullsized":"2023-08-28T02:34:12.231359+0000","mapping_epoch":282,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":114,"log_dups_size":0,"ondisk_log_size":114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":358,"num_read_kb":1352,"num_write":102,"num_write_kb":1104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"86'139","reported_seq":380,"reported_epoch":303,"state":"active+recovery_wait+degraded","last_fresh":"2023-08-28T02:34:11.151129+0000","last_change":"2023-08-28T02:32:59.651360+0000","last_active":"2023-08-28T02:34:11.151129+0000","last_peered":"2023-08-28T02:34:11.151129+0000","last_clean":"2023-08-28T02:23:32.292225+0000","last_became_active":"2023-08-28T02:32:59.651089+0000","last_became_peered":"2023-08-28T02:32:59.651089+0000","last_unstale":"2023-08-28T02:34:11.151129+0000","last_undegraded":"2023-08-28T02:32:59.636553+0000","last_fullsized":"2023-08-28T02:34:11.151129+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_clean_scrub_stamp":"2023-08-28T02:08:02.043718+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:12:30.956176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":["0","6"],"object_location_counts":[{"shards":"0,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":183,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-08-28T02:34:11.151092+0000","last_change":"2023-08-28T02:28:25.347600+0000","last_active":"2023-08-28T02:34:11.151092+0000","last_peered":"2023-08-28T02:34:11.151092+0000","last_clean":"2023-08-28T02:34:11.151092+0000","last_became_active":"2023-08-28T02:28:25.347237+0000","last_became_peered":"2023-08-28T02:28:25.347237+0000","last_unstale":"2023-08-28T02:34:11.151092+0000","last_undegraded":"2023-08-28T02:34:11.151092+0000","last_fullsized":"2023-08-28T02:34:11.151092+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:29:26.767559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":102,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-08-28T02:34:12.236469+0000","last_change":"2023-08-28T02:31:55.517732+0000","last_active":"2023-08-28T02:34:12.236469+0000","last_peered":"2023-08-28T02:34:12.236469+0000","last_clean":"2023-08-28T02:34:12.236469+0000","last_became_active":"2023-08-28T02:31:55.517347+0000","last_became_peered":"2023-08-28T02:31:55.517347+0000","last_unstale":"2023-08-28T02:34:12.236469+0000","last_undegraded":"2023-08-28T02:34:12.236469+0000","last_fullsized":"2023-08-28T02:34:12.236469+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"304'6760","reported_seq":6844,"reported_epoch":304,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:34:19.769057+0000","last_change":"2023-08-28T02:32:58.875577+0000","last_active":"2023-08-28T02:34:19.769057+0000","last_peered":"2023-08-28T02:34:19.769057+0000","last_clean":"2023-08-28T02:31:54.356655+0000","last_became_active":"2023-08-28T02:32:58.865219+0000","last_became_peered":"2023-08-28T02:32:58.865219+0000","last_unstale":"2023-08-28T02:34:19.769057+0000","last_undegraded":"2023-08-28T02:32:58.625567+0000","last_fullsized":"2023-08-28T02:32:58.612068+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":6760,"log_dups_size":0,"ondisk_log_size":6760,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50692096,"num_objects":773,"num_object_clones":0,"num_object_copies":2319,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2532,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":773,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4937,"num_write_kb":182752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":-1254},{"shards":"0,2,6","objects":401}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"78'148","reported_seq":732,"reported_epoch":303,"state":"active+recovery_wait+degraded","last_fresh":"2023-08-28T02:34:11.148591+0000","last_change":"2023-08-28T02:32:59.661621+0000","last_active":"2023-08-28T02:34:11.148591+0000","last_peered":"2023-08-28T02:34:11.148591+0000","last_clean":"2023-08-28T02:12:46.181557+0000","last_became_active":"2023-08-28T02:32:59.661273+0000","last_became_peered":"2023-08-28T02:32:59.661273+0000","last_unstale":"2023-08-28T02:34:11.148591+0000","last_undegraded":"2023-08-28T02:32:59.632715+0000","last_fullsized":"2023-08-28T02:34:11.148591+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":244,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:58:40.796785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1092,"num_read_kb":1955,"num_write":127,"num_write_kb":5177,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":["1","0"],"object_location_counts":[{"shards":"0,1","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":274,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-08-28T02:34:11.153250+0000","last_change":"2023-08-28T02:23:59.226986+0000","last_active":"2023-08-28T02:34:11.153250+0000","last_peered":"2023-08-28T02:34:11.153250+0000","last_clean":"2023-08-28T02:34:11.153250+0000","last_became_active":"2023-08-28T02:23:59.226529+0000","last_became_peered":"2023-08-28T02:23:59.226529+0000","last_unstale":"2023-08-28T02:34:11.153250+0000","last_undegraded":"2023-08-28T02:34:11.153250+0000","last_fullsized":"2023-08-28T02:34:11.153250+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:22:09.193619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":337,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-08-28T02:34:11.151545+0000","last_change":"2023-08-28T02:32:58.744332+0000","last_active":"2023-08-28T02:34:11.151545+0000","last_peered":"2023-08-28T02:34:11.151545+0000","last_clean":"2023-08-28T02:34:11.151545+0000","last_became_active":"2023-08-28T02:32:58.744025+0000","last_became_peered":"2023-08-28T02:32:58.744025+0000","last_unstale":"2023-08-28T02:34:11.151545+0000","last_undegraded":"2023-08-28T02:34:11.151545+0000","last_fullsized":"2023-08-28T02:34:11.151545+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:41:15.957049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"305'6938","reported_seq":7015,"reported_epoch":305,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:34:21.950129+0000","last_change":"2023-08-28T02:32:58.811565+0000","last_active":"2023-08-28T02:34:21.950129+0000","last_peered":"2023-08-28T02:34:21.950129+0000","last_clean":"2023-08-28T02:32:11.496413+0000","last_became_active":"2023-08-28T02:32:58.798063+0000","last_became_peered":"2023-08-28T02:32:58.798063+0000","last_unstale":"2023-08-28T02:34:21.950129+0000","last_undegraded":"2023-08-28T02:32:58.623060+0000","last_fullsized":"2023-08-28T02:32:58.615162+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":6938,"log_dups_size":0,"ondisk_log_size":6938,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52068352,"num_objects":794,"num_object_clones":0,"num_object_copies":2382,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":911,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":794,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5088,"num_write_kb":188256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"2,3,5","objects":379},{"shards":"3,5","objects":415}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"78'103","reported_seq":614,"reported_epoch":303,"state":"active+recovery_wait+degraded","last_fresh":"2023-08-28T02:34:11.150590+0000","last_change":"2023-08-28T02:32:59.656648+0000","last_active":"2023-08-28T02:34:11.150590+0000","last_peered":"2023-08-28T02:34:11.150590+0000","last_clean":"2023-08-28T02:23:32.292540+0000","last_became_active":"2023-08-28T02:32:59.656276+0000","last_became_peered":"2023-08-28T02:32:59.656276+0000","last_unstale":"2023-08-28T02:34:11.150590+0000","last_undegraded":"2023-08-28T02:32:59.643455+0000","last_fullsized":"2023-08-28T02:34:11.150590+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:54:56.658032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":803,"num_read_kb":2080,"num_write":73,"num_write_kb":1244,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":23,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-08-28T02:34:11.146682+0000","last_change":"2023-08-28T02:32:58.720401+0000","last_active":"2023-08-28T02:34:11.146682+0000","last_peered":"2023-08-28T02:34:11.146682+0000","last_clean":"2023-08-28T02:34:11.146682+0000","last_became_active":"2023-08-28T02:32:58.719518+0000","last_became_peered":"2023-08-28T02:32:58.719518+0000","last_unstale":"2023-08-28T02:34:11.146682+0000","last_undegraded":"2023-08-28T02:34:11.146682+0000","last_fullsized":"2023-08-28T02:34:11.146682+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:46:28.947812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":287,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-08-28T02:34:11.152491+0000","last_change":"2023-08-28T02:32:58.887691+0000","last_active":"2023-08-28T02:34:11.152491+0000","last_peered":"2023-08-28T02:34:11.152491+0000","last_clean":"2023-08-28T02:34:11.152491+0000","last_became_active":"2023-08-28T02:32:58.886906+0000","last_became_peered":"2023-08-28T02:32:58.886906+0000","last_unstale":"2023-08-28T02:34:11.152491+0000","last_undegraded":"2023-08-28T02:34:11.152491+0000","last_fullsized":"2023-08-28T02:34:11.152491+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:39:29.229171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"304'6848","reported_seq":6911,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-08-28T02:34:19.817486+0000","last_change":"2023-08-28T02:31:44.467379+0000","last_active":"2023-08-28T02:34:19.817486+0000","last_peered":"2023-08-28T02:34:19.817486+0000","last_clean":"2023-08-28T02:34:19.817486+0000","last_became_active":"2023-08-28T02:31:44.444605+0000","last_became_peered":"2023-08-28T02:31:44.444605+0000","last_unstale":"2023-08-28T02:34:19.817486+0000","last_undegraded":"2023-08-28T02:34:19.817486+0000","last_fullsized":"2023-08-28T02:34:19.817486+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":6848,"log_dups_size":0,"ondisk_log_size":6848,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49020928,"num_objects":748,"num_object_clones":0,"num_object_copies":2244,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":748,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4978,"num_write_kb":183232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":182,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-08-28T02:34:11.152540+0000","last_change":"2023-08-28T02:31:55.535099+0000","last_active":"2023-08-28T02:34:11.152540+0000","last_peered":"2023-08-28T02:34:11.152540+0000","last_clean":"2023-08-28T02:34:11.152540+0000","last_became_active":"2023-08-28T02:31:55.527768+0000","last_became_peered":"2023-08-28T02:31:55.527768+0000","last_unstale":"2023-08-28T02:34:11.152540+0000","last_undegraded":"2023-08-28T02:34:11.152540+0000","last_fullsized":"2023-08-28T02:34:11.152540+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:43:30.804793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'112","reported_seq":456,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-08-28T02:34:11.147408+0000","last_change":"2023-08-28T02:32:59.643465+0000","last_active":"2023-08-28T02:34:11.147408+0000","last_peered":"2023-08-28T02:34:11.147408+0000","last_clean":"2023-08-28T02:34:11.147408+0000","last_became_active":"2023-08-28T02:32:59.642979+0000","last_became_peered":"2023-08-28T02:32:59.642979+0000","last_unstale":"2023-08-28T02:34:11.147408+0000","last_undegraded":"2023-08-28T02:34:11.147408+0000","last_fullsized":"2023-08-28T02:34:11.147408+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:48:45.923015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":444,"num_read_kb":473,"num_write":94,"num_write_kb":127,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":193,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-08-28T02:34:11.155767+0000","last_change":"2023-08-28T02:32:58.833408+0000","last_active":"2023-08-28T02:34:11.155767+0000","last_peered":"2023-08-28T02:34:11.155767+0000","last_clean":"2023-08-28T02:34:11.155767+0000","last_became_active":"2023-08-28T02:32:58.821347+0000","last_became_peered":"2023-08-28T02:32:58.821347+0000","last_unstale":"2023-08-28T02:34:11.155767+0000","last_undegraded":"2023-08-28T02:34:11.155767+0000","last_fullsized":"2023-08-28T02:34:11.155767+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:41:25.683179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"304'6973","reported_seq":7051,"reported_epoch":304,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:34:19.698985+0000","last_change":"2023-08-28T02:32:58.866701+0000","last_active":"2023-08-28T02:34:19.698985+0000","last_peered":"2023-08-28T02:34:19.698985+0000","last_clean":"2023-08-28T02:32:11.480934+0000","last_became_active":"2023-08-28T02:32:58.858761+0000","last_became_peered":"2023-08-28T02:32:58.858761+0000","last_unstale":"2023-08-28T02:34:19.698985+0000","last_undegraded":"2023-08-28T02:32:58.628529+0000","last_fullsized":"2023-08-28T02:32:58.617723+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":261,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":6973,"log_dups_size":0,"ondisk_log_size":6973,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52035584,"num_objects":794,"num_object_clones":0,"num_object_copies":2382,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":937,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":794,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5068,"num_write_kb":187584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[{"shards":"2,3","objects":409},{"shards":"2,3,6","objects":385}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"78'122","reported_seq":648,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-08-28T02:34:12.235129+0000","last_change":"2023-08-28T02:33:00.672889+0000","last_active":"2023-08-28T02:34:12.235129+0000","last_peered":"2023-08-28T02:34:12.235129+0000","last_clean":"2023-08-28T02:34:12.235129+0000","last_became_active":"2023-08-28T02:33:00.672494+0000","last_became_peered":"2023-08-28T02:33:00.672494+0000","last_unstale":"2023-08-28T02:34:12.235129+0000","last_undegraded":"2023-08-28T02:34:12.235129+0000","last_fullsized":"2023-08-28T02:34:12.235129+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":857,"num_read_kb":2224,"num_write":99,"num_write_kb":2312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":312,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-08-28T02:34:11.151624+0000","last_change":"2023-08-28T02:32:58.795995+0000","last_active":"2023-08-28T02:34:11.151624+0000","last_peered":"2023-08-28T02:34:11.151624+0000","last_clean":"2023-08-28T02:34:11.151624+0000","last_became_active":"2023-08-28T02:32:58.771872+0000","last_became_peered":"2023-08-28T02:32:58.771872+0000","last_unstale":"2023-08-28T02:34:11.151624+0000","last_undegraded":"2023-08-28T02:34:11.151624+0000","last_fullsized":"2023-08-28T02:34:11.151624+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:16:02.965467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":274,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-08-28T02:34:12.148766+0000","last_change":"2023-08-28T02:28:25.351840+0000","last_active":"2023-08-28T02:34:12.148766+0000","last_peered":"2023-08-28T02:34:12.148766+0000","last_clean":"2023-08-28T02:34:12.148766+0000","last_became_active":"2023-08-28T02:28:25.351399+0000","last_became_peered":"2023-08-28T02:28:25.351399+0000","last_unstale":"2023-08-28T02:34:12.148766+0000","last_undegraded":"2023-08-28T02:34:12.148766+0000","last_fullsized":"2023-08-28T02:34:12.148766+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":289,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-08-28T02:34:11.151632+0000","last_change":"2023-08-28T02:32:58.751818+0000","last_active":"2023-08-28T02:34:11.151632+0000","last_peered":"2023-08-28T02:34:11.151632+0000","last_clean":"2023-08-28T02:34:11.151632+0000","last_became_active":"2023-08-28T02:32:58.743388+0000","last_became_peered":"2023-08-28T02:32:58.743388+0000","last_unstale":"2023-08-28T02:34:11.151632+0000","last_undegraded":"2023-08-28T02:34:11.151632+0000","last_fullsized":"2023-08-28T02:34:11.151632+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:52:12.910174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":367,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-08-28T02:34:11.146604+0000","last_change":"2023-08-28T02:34:00.655723+0000","last_active":"2023-08-28T02:34:11.146604+0000","last_peered":"2023-08-28T02:34:11.146604+0000","last_clean":"2023-08-28T02:34:11.146604+0000","last_became_active":"2023-08-28T02:34:00.655388+0000","last_became_peered":"2023-08-28T02:34:00.655388+0000","last_unstale":"2023-08-28T02:34:11.146604+0000","last_undegraded":"2023-08-28T02:34:11.146604+0000","last_fullsized":"2023-08-28T02:34:11.146604+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:08:38.272132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.18","version":"304'7065","reported_seq":9726,"reported_epoch":304,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-08-28T02:34:19.852464+0000","last_change":"2023-08-28T02:34:10.148063+0000","last_active":"2023-08-28T02:34:19.852464+0000","last_peered":"2023-08-28T02:34:19.852464+0000","last_clean":"2023-08-28T02:31:54.412195+0000","last_became_active":"2023-08-28T02:31:56.623609+0000","last_became_peered":"2023-08-28T02:31:56.623609+0000","last_unstale":"2023-08-28T02:34:19.852464+0000","last_undegraded":"2023-08-28T02:32:12.554864+0000","last_fullsized":"2023-08-28T02:31:56.505024+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":7065,"log_dups_size":0,"ondisk_log_size":7065,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50266112,"num_objects":767,"num_object_clones":0,"num_object_copies":2301,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":951,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":767,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5147,"num_write_kb":189248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1293,"num_bytes_recovered":64225280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,4","objects":-184}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":137,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-08-28T02:34:12.149006+0000","last_change":"2023-08-28T02:32:58.839847+0000","last_active":"2023-08-28T02:34:12.149006+0000","last_peered":"2023-08-28T02:34:12.149006+0000","last_clean":"2023-08-28T02:34:12.149006+0000","last_became_active":"2023-08-28T02:32:58.628159+0000","last_became_peered":"2023-08-28T02:32:58.628159+0000","last_unstale":"2023-08-28T02:34:12.149006+0000","last_undegraded":"2023-08-28T02:34:12.149006+0000","last_fullsized":"2023-08-28T02:34:12.149006+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":340,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-08-28T02:34:12.149091+0000","last_change":"2023-08-28T02:32:58.629596+0000","last_active":"2023-08-28T02:34:12.149091+0000","last_peered":"2023-08-28T02:34:12.149091+0000","last_clean":"2023-08-28T02:34:12.149091+0000","last_became_active":"2023-08-28T02:32:58.629136+0000","last_became_peered":"2023-08-28T02:32:58.629136+0000","last_unstale":"2023-08-28T02:34:12.149091+0000","last_undegraded":"2023-08-28T02:34:12.149091+0000","last_fullsized":"2023-08-28T02:34:12.149091+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.19","version":"304'6942","reported_seq":7021,"reported_epoch":304,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:34:19.491336+0000","last_change":"2023-08-28T02:32:58.908496+0000","last_active":"2023-08-28T02:34:19.491336+0000","last_peered":"2023-08-28T02:34:19.491336+0000","last_clean":"2023-08-28T02:32:11.482841+0000","last_became_active":"2023-08-28T02:32:58.900137+0000","last_became_peered":"2023-08-28T02:32:58.900137+0000","last_unstale":"2023-08-28T02:34:19.491336+0000","last_undegraded":"2023-08-28T02:32:58.650852+0000","last_fullsized":"2023-08-28T02:32:58.621843+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":270,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:30:56.036302+0000","last_clean_scrub_stamp":"2023-08-28T02:30:56.036302+0000","objects_scrubbed":0,"log_size":6942,"log_dups_size":0,"ondisk_log_size":6942,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52658176,"num_objects":804,"num_object_clones":0,"num_object_copies":2412,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":919,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":804,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5038,"num_write_kb":186912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":406},{"shards":"0,4,6","objects":398}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":124,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-08-28T02:34:12.149165+0000","last_change":"2023-08-28T02:28:25.705331+0000","last_active":"2023-08-28T02:34:12.149165+0000","last_peered":"2023-08-28T02:34:12.149165+0000","last_clean":"2023-08-28T02:34:12.149165+0000","last_became_active":"2023-08-28T02:28:25.397480+0000","last_became_peered":"2023-08-28T02:28:25.397480+0000","last_unstale":"2023-08-28T02:34:12.149165+0000","last_undegraded":"2023-08-28T02:34:12.149165+0000","last_fullsized":"2023-08-28T02:34:12.149165+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":304,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-08-28T02:34:12.148711+0000","last_change":"2023-08-28T02:23:59.207929+0000","last_active":"2023-08-28T02:34:12.148711+0000","last_peered":"2023-08-28T02:34:12.148711+0000","last_clean":"2023-08-28T02:34:12.148711+0000","last_became_active":"2023-08-28T02:23:59.207589+0000","last_became_peered":"2023-08-28T02:23:59.207589+0000","last_unstale":"2023-08-28T02:34:12.148711+0000","last_undegraded":"2023-08-28T02:34:12.148711+0000","last_fullsized":"2023-08-28T02:34:12.148711+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":193,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-08-28T02:34:11.150878+0000","last_change":"2023-08-28T02:28:25.347988+0000","last_active":"2023-08-28T02:34:11.150878+0000","last_peered":"2023-08-28T02:34:11.150878+0000","last_clean":"2023-08-28T02:34:11.150878+0000","last_became_active":"2023-08-28T02:28:25.347632+0000","last_became_peered":"2023-08-28T02:28:25.347632+0000","last_unstale":"2023-08-28T02:34:11.150878+0000","last_undegraded":"2023-08-28T02:34:11.150878+0000","last_fullsized":"2023-08-28T02:34:11.150878+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:56:20.202662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":165,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-08-28T02:34:11.144945+0000","last_change":"2023-08-28T02:23:59.217794+0000","last_active":"2023-08-28T02:34:11.144945+0000","last_peered":"2023-08-28T02:34:11.144945+0000","last_clean":"2023-08-28T02:34:11.144945+0000","last_became_active":"2023-08-28T02:23:59.217231+0000","last_became_peered":"2023-08-28T02:23:59.217231+0000","last_unstale":"2023-08-28T02:34:11.144945+0000","last_undegraded":"2023-08-28T02:34:11.144945+0000","last_fullsized":"2023-08-28T02:34:11.144945+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:36:26.297410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":275,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-08-28T02:34:11.152321+0000","last_change":"2023-08-28T02:34:00.680026+0000","last_active":"2023-08-28T02:34:11.152321+0000","last_peered":"2023-08-28T02:34:11.152321+0000","last_clean":"2023-08-28T02:34:11.152321+0000","last_became_active":"2023-08-28T02:34:00.646108+0000","last_became_peered":"2023-08-28T02:34:00.646108+0000","last_unstale":"2023-08-28T02:34:11.152321+0000","last_undegraded":"2023-08-28T02:34:11.152321+0000","last_fullsized":"2023-08-28T02:34:11.152321+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":301,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:36:36.638915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":363,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-08-28T02:34:12.148677+0000","last_change":"2023-08-28T02:32:58.630270+0000","last_active":"2023-08-28T02:34:12.148677+0000","last_peered":"2023-08-28T02:34:12.148677+0000","last_clean":"2023-08-28T02:34:12.148677+0000","last_became_active":"2023-08-28T02:32:58.629958+0000","last_became_peered":"2023-08-28T02:32:58.629958+0000","last_unstale":"2023-08-28T02:34:12.148677+0000","last_undegraded":"2023-08-28T02:34:12.148677+0000","last_fullsized":"2023-08-28T02:34:12.148677+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":26,"stat_sum":{"num_bytes":1616314392,"num_objects":24664,"num_object_clones":0,"num_object_copies":73992,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":27129,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24664,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":160872,"num_write_kb":5937089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11235,"num_bytes_recovered":636289024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6951874560,"data_stored":6951862344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":175977,"ondisk_log_size":175977,"up":78,"acting":61,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":24700,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":22016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26810,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":30,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":147,"num_bytes_recovered":4194342,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430485,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626732,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":986540,"kb_used_data":677028,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":92758612,"statfs":{"total":95995035648,"available":94984818688,"internally_reserved":0,"allocated":693276672,"data_stored":692441411,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,0,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:34:18 2023","interfaces":[{"interface":"back","average":{"1min":1.206,"5min":3.6160000000000001,"15min":3.5990000000000002},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.20799999999999999},"max":{"1min":3.8079999999999998,"5min":121.78,"15min":121.78},"last":0.28899999999999998},{"interface":"front","average":{"1min":1.792,"5min":3.8500000000000001,"15min":3.6219999999999999},"min":{"1min":0.246,"5min":0.23699999999999999,"15min":0.23200000000000001},"max":{"1min":5.8360000000000003,"5min":121.79900000000001,"15min":121.79900000000001},"last":0.94899999999999995}]},{"osd":1,"last update":"Mon Aug 28 02:34:18 2023","interfaces":[{"interface":"back","average":{"1min":1.2310000000000001,"5min":3.4849999999999999,"15min":2.569},"min":{"1min":0.245,"5min":0.23799999999999999,"15min":0.23699999999999999},"max":{"1min":7.5149999999999997,"5min":141.87299999999999,"15min":141.87299999999999},"last":0.66700000000000004},{"interface":"front","average":{"1min":2.0310000000000001,"5min":3.6150000000000002,"15min":2.7320000000000002},"min":{"1min":0.317,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":7.8710000000000004,"5min":141.904,"15min":141.904},"last":0.317}]},{"osd":2,"last update":"Mon Aug 28 02:33:57 2023","interfaces":[{"interface":"back","average":{"1min":2.452,"5min":5.3369999999999997,"15min":3.7429999999999999},"min":{"1min":0.35899999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":9.7550000000000008,"5min":234.876,"15min":234.876},"last":0.45600000000000002},{"interface":"front","average":{"1min":2.504,"5min":5.1820000000000004,"15min":3.7010000000000001},"min":{"1min":0.32100000000000001,"5min":0.28899999999999998,"15min":0.28499999999999998},"max":{"1min":11.260999999999999,"5min":234.482,"15min":234.482},"last":0.76200000000000001}]},{"osd":3,"last update":"Mon Aug 28 02:34:18 2023","interfaces":[{"interface":"back","average":{"1min":2.2490000000000001,"5min":4.8689999999999998,"15min":3.04},"min":{"1min":0.40500000000000003,"5min":0.32000000000000001,"15min":0.317},"max":{"1min":7.4059999999999997,"5min":157.625,"15min":157.625},"last":0.79600000000000004},{"interface":"front","average":{"1min":2.2639999999999998,"5min":4.6040000000000001,"15min":3.044},"min":{"1min":0.35899999999999999,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":7.3860000000000001,"5min":157.637,"15min":157.637},"last":0.81799999999999995}]},{"osd":4,"last update":"Mon Aug 28 02:33:21 2023","interfaces":[{"interface":"back","average":{"1min":0.70699999999999996,"5min":1.0049999999999999,"15min":1.1120000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.251},"max":{"1min":3.8610000000000002,"5min":18.308,"15min":18.308},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.82399999999999995,"5min":1.01,"15min":1.198},"min":{"1min":0.27800000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"max":{"1min":3.8450000000000002,"5min":18.263000000000002,"15min":18.263000000000002},"last":0.35199999999999998}]},{"osd":5,"last update":"Mon Aug 28 02:33:21 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.76400000000000001,"15min":1.1679999999999999},"min":{"1min":0.30499999999999999,"5min":0.29699999999999999,"15min":0.26300000000000001},"max":{"1min":1.4199999999999999,"5min":6.0049999999999999,"15min":20.759},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.69699999999999995,"5min":0.84999999999999998,"15min":1.018},"min":{"1min":0.32700000000000001,"5min":0.314,"15min":0.311},"max":{"1min":2.972,"5min":7.2370000000000001,"15min":7.3780000000000001},"last":0.46600000000000003}]},{"osd":7,"last update":"Mon Aug 28 02:31:45 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.92900000000000005,"15min":1.018},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.314},"max":{"1min":1.778,"5min":10.605,"15min":10.605},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.88700000000000001,"15min":1.0760000000000001},"min":{"1min":0.38400000000000001,"5min":0.34599999999999997,"15min":0.29799999999999999},"max":{"1min":1.625,"5min":5.1900000000000004,"15min":9.7110000000000003},"last":0.42799999999999999}]}]},{"osd":1,"up_from":98,"seq":420906795184,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1210576,"kb_used_data":807688,"kb_used_omap":7,"kb_used_meta":402872,"kb_avail":92534576,"statfs":{"total":95995035648,"available":94755405824,"internally_reserved":0,"allocated":827072512,"data_stored":825513381,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":412541259},"hb_peers":[0,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3],"upper_bound":4},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:34:15 2023","interfaces":[{"interface":"back","average":{"1min":9.7840000000000007,"5min":5.1550000000000002,"15min":3.665},"min":{"1min":0.189,"5min":0.189,"15min":0.17499999999999999},"max":{"1min":129.346,"5min":129.346,"15min":129.346},"last":0.21299999999999999},{"interface":"front","average":{"1min":9.1449999999999996,"5min":5.0800000000000001,"15min":3.6680000000000001},"min":{"1min":0.20499999999999999,"5min":0.19700000000000001,"15min":0.188},"max":{"1min":120.774,"5min":120.774,"15min":128.685},"last":3.2290000000000001}]},{"osd":2,"last update":"Mon Aug 28 02:33:40 2023","interfaces":[{"interface":"back","average":{"1min":1.946,"5min":2.206,"15min":3.367},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":7.5129999999999999,"5min":24.245999999999999,"15min":135.25999999999999},"last":5.0110000000000001},{"interface":"front","average":{"1min":2.8540000000000001,"5min":2.2280000000000002,"15min":3.2000000000000002},"min":{"1min":0.223,"5min":0.218,"15min":0.218},"max":{"1min":11.439,"5min":12.691000000000001,"15min":135.10300000000001},"last":8.0470000000000006}]},{"osd":3,"last update":"Mon Aug 28 02:34:15 2023","interfaces":[{"interface":"back","average":{"1min":13.522,"5min":6.4379999999999997,"15min":3.7789999999999999},"min":{"1min":0.25600000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":96.768000000000001,"5min":96.768000000000001,"15min":123.741},"last":6.9619999999999997},{"interface":"front","average":{"1min":11.941000000000001,"5min":6.3620000000000001,"15min":3.8839999999999999},"min":{"1min":0.29099999999999998,"5min":0.253,"15min":0.23699999999999999},"max":{"1min":96.519000000000005,"5min":96.519000000000005,"15min":123.642},"last":4.9770000000000003}]},{"osd":4,"last update":"Mon Aug 28 02:34:15 2023","interfaces":[{"interface":"back","average":{"1min":1.859,"5min":1.48,"15min":1.478},"min":{"1min":0.315,"5min":0.28499999999999998,"15min":0.27500000000000002},"max":{"1min":6.5309999999999997,"5min":8.1989999999999998,"15min":17.600000000000001},"last":1.0760000000000001},{"interface":"front","average":{"1min":1.04,"5min":1.351,"15min":1.341},"min":{"1min":0.30499999999999999,"5min":0.29399999999999998,"15min":0.25600000000000001},"max":{"1min":5.1680000000000001,"5min":12.66,"15min":19.010000000000002},"last":0.439}]},{"osd":5,"last update":"Mon Aug 28 02:34:15 2023","interfaces":[{"interface":"back","average":{"1min":1.905,"5min":1.724,"15min":1.627},"min":{"1min":0.312,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":6.6040000000000001,"5min":11.167999999999999,"15min":15.544},"last":3.294},{"interface":"front","average":{"1min":1.879,"5min":1.635,"15min":1.4790000000000001},"min":{"1min":0.35499999999999998,"5min":0.307,"15min":0.307},"max":{"1min":6.5,"5min":10.084,"15min":17.646999999999998},"last":1.0089999999999999}]},{"osd":6,"last update":"Mon Aug 28 02:34:15 2023","interfaces":[{"interface":"back","average":{"1min":1.923,"5min":1.6619999999999999,"15min":1.738},"min":{"1min":0.35199999999999998,"5min":0.311,"15min":0.311},"max":{"1min":6.641,"5min":10.420999999999999,"15min":15.512},"last":3.3239999999999998},{"interface":"front","average":{"1min":2.298,"5min":1.6930000000000001,"15min":1.6699999999999999},"min":{"1min":0.375,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":8.3109999999999999,"5min":8.3109999999999999,"15min":17.673999999999999},"last":1.0429999999999999}]},{"osd":7,"last update":"Mon Aug 28 02:32:10 2023","interfaces":[{"interface":"back","average":{"1min":1.4450000000000001,"5min":2.0299999999999998,"15min":1.5189999999999999},"min":{"1min":0.34399999999999997,"5min":0.32500000000000001,"15min":0.30199999999999999},"max":{"1min":10.353999999999999,"5min":49.575000000000003,"15min":49.575000000000003},"last":0.442},{"interface":"front","average":{"1min":1.8899999999999999,"5min":2.2469999999999999,"15min":1.7410000000000001},"min":{"1min":0.42199999999999999,"5min":0.39500000000000002,"15min":0.33100000000000002},"max":{"1min":10.449999999999999,"5min":49.319000000000003,"15min":49.319000000000003},"last":0.51500000000000001}]}]},{"osd":0,"up_from":90,"seq":386547056830,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1309324,"kb_used_data":816068,"kb_used_omap":8,"kb_used_meta":493175,"kb_avail":92435828,"statfs":{"total":95995035648,"available":94654287872,"internally_reserved":0,"allocated":835653632,"data_stored":834213622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":505011561},"hb_peers":[1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Aug 28 02:33:25 2023","interfaces":[{"interface":"back","average":{"1min":2.2069999999999999,"5min":3.1930000000000001,"15min":2.351},"min":{"1min":0.216,"5min":0.191,"15min":0.184},"max":{"1min":5.585,"5min":61.932000000000002,"15min":61.932000000000002},"last":7.7149999999999999},{"interface":"front","average":{"1min":3.9590000000000001,"5min":4.2130000000000001,"15min":2.6589999999999998},"min":{"1min":0.28899999999999998,"5min":0.19600000000000001,"15min":0.17000000000000001},"max":{"1min":15.068,"5min":62.061999999999998,"15min":62.061999999999998},"last":5.2460000000000004}]},{"osd":2,"last update":"Mon Aug 28 02:33:42 2023","interfaces":[{"interface":"back","average":{"1min":2.9409999999999998,"5min":2.8069999999999999,"15min":2.7200000000000002},"min":{"1min":0.56699999999999995,"5min":0.20599999999999999,"15min":0.19500000000000001},"max":{"1min":16.356999999999999,"5min":46.927999999999997,"15min":100.718},"last":4.7930000000000001},{"interface":"front","average":{"1min":5.0330000000000004,"5min":3.0790000000000002,"15min":3.157},"min":{"1min":0.29999999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":25.047999999999998,"5min":46.942999999999998,"15min":100.962},"last":5.9059999999999997}]},{"osd":3,"last update":"Mon Aug 28 02:33:25 2023","interfaces":[{"interface":"back","average":{"1min":9.2739999999999991,"5min":4.5679999999999996,"15min":4.1029999999999998},"min":{"1min":0.309,"5min":0.23899999999999999,"15min":0.23300000000000001},"max":{"1min":119.351,"5min":126.896,"15min":237.43899999999999},"last":1.2350000000000001},{"interface":"front","average":{"1min":9.173,"5min":4.4299999999999997,"15min":4.0149999999999997},"min":{"1min":0.35199999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":119.378,"5min":127.01600000000001,"15min":237.48099999999999},"last":1.2729999999999999}]},{"osd":4,"last update":"Mon Aug 28 02:33:19 2023","interfaces":[{"interface":"back","average":{"1min":1.506,"5min":1.7869999999999999,"15min":1.47},"min":{"1min":0.26300000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":4.2590000000000003,"5min":9.3680000000000003,"15min":9.6649999999999991},"last":0.442},{"interface":"front","average":{"1min":2.484,"5min":1.6080000000000001,"15min":1.671},"min":{"1min":0.36699999999999999,"5min":0.309,"15min":0.309},"max":{"1min":5.4489999999999998,"5min":8.7880000000000003,"15min":27.271999999999998},"last":0.54900000000000004}]},{"osd":5,"last update":"Mon Aug 28 02:33:19 2023","interfaces":[{"interface":"back","average":{"1min":2.593,"5min":1.4530000000000001,"15min":1.7629999999999999},"min":{"1min":0.33900000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":5.5190000000000001,"5min":5.5190000000000001,"15min":44.000999999999998},"last":0.49299999999999999},{"interface":"front","average":{"1min":1.546,"5min":1.7849999999999999,"15min":1.661},"min":{"1min":0.307,"5min":0.307,"15min":0.30599999999999999},"max":{"1min":4.2919999999999998,"5min":9.4489999999999998,"15min":43.953000000000003},"last":0.46899999999999997}]},{"osd":6,"last update":"Mon Aug 28 02:33:19 2023","interfaces":[{"interface":"back","average":{"1min":2.5169999999999999,"5min":1.677,"15min":1.851},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":5.4889999999999999,"5min":7.6340000000000003,"15min":32.417999999999999},"last":0.52500000000000002},{"interface":"front","average":{"1min":1.573,"5min":1.716,"15min":1.7150000000000001},"min":{"1min":0.32900000000000001,"5min":0.32700000000000001,"15min":0.30399999999999999},"max":{"1min":4.327,"5min":9.2789999999999999,"15min":32.130000000000003},"last":0.39800000000000002}]},{"osd":7,"last update":"Mon Aug 28 02:32:16 2023","interfaces":[{"interface":"back","average":{"1min":1.6060000000000001,"5min":2.25,"15min":1.712},"min":{"1min":0.379,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":10.775,"5min":27.385000000000002,"15min":27.385000000000002},"last":4.3529999999999998},{"interface":"front","average":{"1min":2.6110000000000002,"5min":2.0979999999999999,"15min":1.766},"min":{"1min":0.41799999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":7.5350000000000001,"5min":27.213999999999999,"15min":27.213999999999999},"last":2.4550000000000001}]}]},{"osd":2,"up_from":104,"seq":446676598954,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1335968,"kb_used_data":859864,"kb_used_omap":10,"kb_used_meta":476085,"kb_avail":92409184,"statfs":{"total":95995035648,"available":94627004416,"internally_reserved":0,"allocated":880500736,"data_stored":879101790,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10792,"internal_metadata":487511512},"hb_peers":[0,1,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:33:43 2023","interfaces":[{"interface":"back","average":{"1min":5.008,"5min":6.5449999999999999,"15min":4.133},"min":{"1min":0.191,"5min":0.185,"15min":0.17599999999999999},"max":{"1min":22.338999999999999,"5min":185.93799999999999,"15min":185.93799999999999},"last":5.4889999999999999},{"interface":"front","average":{"1min":2.411,"5min":6.0339999999999998,"15min":5.1470000000000002},"min":{"1min":0.22,"5min":0.22,"15min":0.19900000000000001},"max":{"1min":9.4000000000000004,"5min":187.90700000000001,"15min":187.90700000000001},"last":5.5709999999999997}]},{"osd":1,"last update":"Mon Aug 28 02:33:43 2023","interfaces":[{"interface":"back","average":{"1min":1.8149999999999999,"5min":1.8340000000000001,"15min":2.0150000000000001},"min":{"1min":0.26400000000000001,"5min":0.22800000000000001,"15min":0.20999999999999999},"max":{"1min":8.6150000000000002,"5min":13.25,"15min":30.888000000000002},"last":12.648},{"interface":"front","average":{"1min":2.677,"5min":2.0489999999999999,"15min":1.9710000000000001},"min":{"1min":0.29799999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":10.196999999999999,"5min":11.962999999999999,"15min":31.648},"last":7.7050000000000001}]},{"osd":3,"last update":"Mon Aug 28 02:34:15 2023","interfaces":[{"interface":"back","average":{"1min":3.2090000000000001,"5min":5.2560000000000002,"15min":3.8180000000000001},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.22700000000000001},"max":{"1min":10.289,"5min":134.07599999999999,"15min":134.07599999999999},"last":10.212999999999999},{"interface":"front","average":{"1min":3.3599999999999999,"5min":5.3289999999999997,"15min":3.8340000000000001},"min":{"1min":0.51300000000000001,"5min":0.29299999999999998,"15min":0.223},"max":{"1min":8.7899999999999991,"5min":134.172,"15min":134.172},"last":1.4930000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:33:43 2023","interfaces":[{"interface":"back","average":{"1min":1.839,"5min":4.9249999999999998,"15min":2.8239999999999998},"min":{"1min":0.33800000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":8.9160000000000004,"5min":207.34100000000001,"15min":207.34100000000001},"last":1.4610000000000001},{"interface":"front","average":{"1min":3.7789999999999999,"5min":5.3689999999999998,"15min":2.722},"min":{"1min":0.32000000000000001,"5min":0.30299999999999999,"15min":0.27400000000000002},"max":{"1min":22.303999999999998,"5min":207.352,"15min":207.352},"last":1.875}]},{"osd":5,"last update":"Mon Aug 28 02:33:43 2023","interfaces":[{"interface":"back","average":{"1min":1.544,"5min":3.48,"15min":2.7309999999999999},"min":{"1min":0.35099999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":8.5440000000000005,"5min":94.614000000000004,"15min":128.696},"last":5.6319999999999997},{"interface":"front","average":{"1min":1.827,"5min":3.4060000000000001,"15min":2.7799999999999998},"min":{"1min":0.375,"5min":0.30399999999999999,"15min":0.30299999999999999},"max":{"1min":8.8650000000000002,"5min":94.534000000000006,"15min":128.84100000000001},"last":1.4199999999999999}]},{"osd":6,"last update":"Mon Aug 28 02:33:43 2023","interfaces":[{"interface":"back","average":{"1min":3.8260000000000001,"5min":2.0680000000000001,"15min":1.6919999999999999},"min":{"1min":0.35499999999999998,"5min":0.32800000000000001,"15min":0.29299999999999998},"max":{"1min":22.25,"5min":22.25,"15min":22.25},"last":1.911},{"interface":"front","average":{"1min":1.605,"5min":1.6060000000000001,"15min":1.6910000000000001},"min":{"1min":0.32600000000000001,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":8.4979999999999993,"5min":13.311,"15min":13.311},"last":5.6070000000000002}]},{"osd":7,"last update":"Mon Aug 28 02:32:42 2023","interfaces":[{"interface":"back","average":{"1min":2.0649999999999999,"5min":3.6520000000000001,"15min":2.6280000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.32100000000000001},"max":{"1min":13.321999999999999,"5min":108.904,"15min":128.911},"last":0.502},{"interface":"front","average":{"1min":2.0739999999999998,"5min":3.8719999999999999,"15min":2.859},"min":{"1min":0.38800000000000001,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":13.353,"5min":108.217,"15min":128.72399999999999},"last":0.625}]}]},{"osd":3,"up_from":112,"seq":481036337316,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1549464,"kb_used_data":1066768,"kb_used_omap":3,"kb_used_meta":482684,"kb_avail":92195688,"statfs":{"total":95995035648,"available":94408384512,"internally_reserved":0,"allocated":1092370432,"data_stored":1090972545,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":494269346},"hb_peers":[0,1,2,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,1,0,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:34:12 2023","interfaces":[{"interface":"back","average":{"1min":3.3069999999999999,"5min":4.1470000000000002,"15min":3.8300000000000001},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.193},"max":{"1min":12.989000000000001,"5min":79.793000000000006,"15min":164.25299999999999},"last":7.75},{"interface":"front","average":{"1min":3.681,"5min":3.5129999999999999,"15min":3.5920000000000001},"min":{"1min":0.252,"5min":0.21299999999999999,"15min":0.192},"max":{"1min":21.48,"5min":78.555000000000007,"15min":164.55000000000001},"last":3.4590000000000001}]},{"osd":1,"last update":"Mon Aug 28 02:34:12 2023","interfaces":[{"interface":"back","average":{"1min":2.9420000000000002,"5min":2.355,"15min":2.4510000000000001},"min":{"1min":0.32200000000000001,"5min":0.22800000000000001,"15min":0.20699999999999999},"max":{"1min":13.422000000000001,"5min":14.865,"15min":117.34699999999999},"last":11.391999999999999},{"interface":"front","average":{"1min":3.4649999999999999,"5min":2.7970000000000002,"15min":2.6309999999999998},"min":{"1min":0.34999999999999998,"5min":0.23599999999999999,"15min":0.224},"max":{"1min":15.458,"5min":15.458,"15min":118.012},"last":7.7999999999999998}]},{"osd":2,"last update":"Mon Aug 28 02:34:12 2023","interfaces":[{"interface":"back","average":{"1min":4.734,"5min":3.4359999999999999,"15min":3.698},"min":{"1min":0.32300000000000001,"5min":0.20100000000000001,"15min":0.19900000000000001},"max":{"1min":17.777000000000001,"5min":17.777000000000001,"15min":120.43300000000001},"last":3.3439999999999999},{"interface":"front","average":{"1min":3.0390000000000001,"5min":2.4460000000000002,"15min":3.3149999999999999},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":11.167,"5min":11.167,"15min":120.44499999999999},"last":3.2090000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:34:12 2023","interfaces":[{"interface":"back","average":{"1min":1.421,"5min":1.395,"15min":1.665},"min":{"1min":0.32300000000000001,"5min":0.26000000000000001,"15min":0.25900000000000001},"max":{"1min":6.9370000000000003,"5min":14.042,"15min":58.067999999999998},"last":2.9460000000000002},{"interface":"front","average":{"1min":1.984,"5min":1.804,"15min":1.851},"min":{"1min":0.39500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":5.6639999999999997,"5min":10.613,"15min":58.088000000000001},"last":0.41199999999999998}]},{"osd":5,"last update":"Mon Aug 28 02:34:12 2023","interfaces":[{"interface":"back","average":{"1min":2.056,"5min":1.8049999999999999,"15min":1.9990000000000001},"min":{"1min":0.379,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.7000000000000002,"5min":10.722,"15min":102.904},"last":0.45400000000000001},{"interface":"front","average":{"1min":2.597,"5min":2.1269999999999998,"15min":2.0990000000000002},"min":{"1min":0.40400000000000003,"5min":0.313,"15min":0.313},"max":{"1min":12.925000000000001,"5min":12.925000000000001,"15min":103.283},"last":3.306}]},{"osd":6,"last update":"Mon Aug 28 02:34:12 2023","interfaces":[{"interface":"back","average":{"1min":2.6190000000000002,"5min":2.1339999999999999,"15min":1.869},"min":{"1min":0.39500000000000002,"5min":0.33200000000000002,"15min":0.314},"max":{"1min":12.965999999999999,"5min":12.965999999999999,"15min":72.116},"last":3.3290000000000002},{"interface":"front","average":{"1min":1.458,"5min":1.5409999999999999,"15min":1.591},"min":{"1min":0.35799999999999998,"5min":0.314,"15min":0.314},"max":{"1min":6.8810000000000002,"5min":7.8899999999999997,"15min":72.030000000000001},"last":2.9889999999999999}]},{"osd":7,"last update":"Mon Aug 28 02:32:07 2023","interfaces":[{"interface":"back","average":{"1min":1.6859999999999999,"5min":3.0920000000000001,"15min":1.845},"min":{"1min":0.39800000000000002,"5min":0.33800000000000002,"15min":0.32500000000000001},"max":{"1min":5.2320000000000002,"5min":54.427999999999997,"15min":54.427999999999997},"last":7.0629999999999997},{"interface":"front","average":{"1min":1.5429999999999999,"5min":3.052,"15min":1.9379999999999999},"min":{"1min":0.443,"5min":0.36399999999999999,"15min":0.34799999999999998},"max":{"1min":6.3019999999999996,"5min":54.445,"15min":54.445},"last":10.754}]}]},{"osd":4,"up_from":30,"seq":128849019190,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1010336,"kb_used_data":709016,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92734816,"statfs":{"total":95995035648,"available":94960451584,"internally_reserved":0,"allocated":726032384,"data_stored":725208605,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:33:19 2023","interfaces":[{"interface":"back","average":{"1min":19.853000000000002,"5min":6.609,"15min":4.5140000000000002},"min":{"1min":0.26900000000000002,"5min":0.219,"15min":0.219},"max":{"1min":131.893,"5min":131.893,"15min":138.67400000000001},"last":0.85499999999999998},{"interface":"front","average":{"1min":19.280999999999999,"5min":6.2789999999999999,"15min":4.3079999999999998},"min":{"1min":0.29699999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":131.96700000000001,"5min":131.96700000000001,"15min":134.14599999999999},"last":4.5439999999999996}]},{"osd":1,"last update":"Mon Aug 28 02:33:19 2023","interfaces":[{"interface":"back","average":{"1min":1.4890000000000001,"5min":2.1320000000000001,"15min":1.671},"min":{"1min":0.29299999999999998,"5min":0.23599999999999999,"15min":0.221},"max":{"1min":7.508,"5min":34.945999999999998,"15min":53.622999999999998},"last":0.28899999999999998},{"interface":"front","average":{"1min":1.988,"5min":1.8360000000000001,"15min":1.585},"min":{"1min":0.27000000000000002,"5min":0.25,"15min":0.25},"max":{"1min":6.766,"5min":34.869999999999997,"15min":53.774000000000001},"last":1.6819999999999999}]},{"osd":2,"last update":"Mon Aug 28 02:33:45 2023","interfaces":[{"interface":"back","average":{"1min":8.3070000000000004,"5min":4.6420000000000003,"15min":3.7170000000000001},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":107.786,"5min":107.786,"15min":107.786},"last":0.51600000000000001},{"interface":"front","average":{"1min":8.1820000000000004,"5min":4.6020000000000003,"15min":3.5859999999999999},"min":{"1min":0.30499999999999999,"5min":0.27300000000000002,"15min":0.25},"max":{"1min":107.599,"5min":107.599,"15min":107.599},"last":3.4289999999999998}]},{"osd":3,"last update":"Mon Aug 28 02:33:19 2023","interfaces":[{"interface":"back","average":{"1min":14.218999999999999,"5min":9.3070000000000004,"15min":4.5350000000000001},"min":{"1min":0.34000000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":105.491,"5min":203.583,"15min":203.583},"last":1.2370000000000001},{"interface":"front","average":{"1min":15.113,"5min":9.5359999999999996,"15min":4.7030000000000003},"min":{"1min":0.38900000000000001,"5min":0.33800000000000002,"15min":0.32500000000000001},"max":{"1min":105.801,"5min":203.41399999999999,"15min":203.41399999999999},"last":0.97399999999999998}]},{"osd":5,"last update":"Mon Aug 28 02:33:56 2023","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.70999999999999996,"15min":1.456},"min":{"1min":0.26300000000000001,"5min":0.246,"15min":0.246},"max":{"1min":1.73,"5min":5.2309999999999999,"15min":124.718},"last":0.26300000000000001},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.85899999999999999,"15min":1.661},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":2.444,"5min":10.24,"15min":124.694},"last":0.30499999999999999}]},{"osd":6,"last update":"Mon Aug 28 02:33:19 2023","interfaces":[{"interface":"back","average":{"1min":0.78600000000000003,"5min":1.089,"15min":1.127},"min":{"1min":0.314,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":2.3279999999999998,"5min":14.907999999999999,"15min":14.907999999999999},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.81499999999999995,"5min":0.89800000000000002,"15min":0.91200000000000003},"min":{"1min":0.34999999999999998,"5min":0.307,"15min":0.307},"max":{"1min":1.6890000000000001,"5min":5.2679999999999998,"15min":7.3170000000000002},"last":0.29699999999999999}]},{"osd":7,"last update":"Mon Aug 28 02:32:34 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":1.0149999999999999,"15min":1.1819999999999999},"min":{"1min":0.32400000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.2210000000000001,"5min":10.151999999999999,"15min":16.584},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.53100000000000003,"5min":1.0900000000000001,"15min":1.2390000000000001},"min":{"1min":0.35999999999999999,"5min":0.311,"15min":0.311},"max":{"1min":1.5109999999999999,"5min":9.3469999999999995,"15min":10.826000000000001},"last":0.434}]}]},{"osd":5,"up_from":36,"seq":154618822962,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":879568,"kb_used_data":578248,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92865584,"statfs":{"total":95995035648,"available":95094358016,"internally_reserved":0,"allocated":592125952,"data_stored":591313712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:34:12 2023","interfaces":[{"interface":"back","average":{"1min":10.289999999999999,"5min":4.8899999999999997,"15min":4.0449999999999999},"min":{"1min":0.23200000000000001,"5min":0.214,"15min":0.20300000000000001},"max":{"1min":128.72800000000001,"5min":128.72800000000001,"15min":131.06100000000001},"last":2.6200000000000001},{"interface":"front","average":{"1min":10.435,"5min":5.032,"15min":4.258},"min":{"1min":0.307,"5min":0.27300000000000002,"15min":0.22900000000000001},"max":{"1min":128.75200000000001,"5min":128.75200000000001,"15min":131.262},"last":2.6360000000000001}]},{"osd":1,"last update":"Mon Aug 28 02:34:21 2023","interfaces":[{"interface":"back","average":{"1min":1.1299999999999999,"5min":1.3839999999999999,"15min":2.5830000000000002},"min":{"1min":0.30599999999999999,"5min":0.27300000000000002,"15min":0.216},"max":{"1min":4.8739999999999997,"5min":7.7809999999999997,"15min":183.97900000000001},"last":0.36599999999999999},{"interface":"front","average":{"1min":1.8320000000000001,"5min":1.331,"15min":2.7759999999999998},"min":{"1min":0.26400000000000001,"5min":0.23499999999999999,"15min":0.23200000000000001},"max":{"1min":11.875,"5min":11.875,"15min":184.33699999999999},"last":0.496}]},{"osd":2,"last update":"Mon Aug 28 02:33:50 2023","interfaces":[{"interface":"back","average":{"1min":2.3929999999999998,"5min":2.508,"15min":4.4009999999999998},"min":{"1min":0.35799999999999998,"5min":0.25800000000000001,"15min":0.252},"max":{"1min":12.004,"5min":56.725999999999999,"15min":176.69499999999999},"last":0.47299999999999998},{"interface":"front","average":{"1min":2.3570000000000002,"5min":2.875,"15min":4.6539999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":12.025,"5min":56.750999999999998,"15min":176.71199999999999},"last":0.53000000000000003}]},{"osd":3,"last update":"Mon Aug 28 02:34:21 2023","interfaces":[{"interface":"back","average":{"1min":3.3959999999999999,"5min":6.2480000000000002,"15min":3.4700000000000002},"min":{"1min":0.32900000000000001,"5min":0.28799999999999998,"15min":0.26100000000000001},"max":{"1min":31.248999999999999,"5min":128.571,"15min":128.571},"last":1.718},{"interface":"front","average":{"1min":3.5699999999999998,"5min":6.3620000000000001,"15min":3.7839999999999998},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.26200000000000001},"max":{"1min":31.692,"5min":128.684,"15min":128.684},"last":2.0510000000000002}]},{"osd":4,"last update":"Mon Aug 28 02:34:02 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":1.96,"15min":1.3400000000000001},"min":{"1min":0.25900000000000001,"5min":0.253,"15min":0.246},"max":{"1min":3.0259999999999998,"5min":85.769999999999996,"15min":85.769999999999996},"last":0.27200000000000002},{"interface":"front","average":{"1min":0.60799999999999998,"5min":1.823,"15min":1.323},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":2.7690000000000001,"5min":85.825000000000003,"15min":85.825000000000003},"last":0.28799999999999998}]},{"osd":6,"last update":"Mon Aug 28 02:34:21 2023","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":0.76700000000000002,"15min":1.0580000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.442,"5min":4.6210000000000004,"15min":14.803000000000001},"last":0.30399999999999999},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.73999999999999999,"15min":0.96099999999999997},"min":{"1min":0.29599999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.65500000000000003,"5min":4.9509999999999996,"15min":8.6479999999999997},"last":0.32200000000000001}]},{"osd":7,"last update":"Mon Aug 28 02:32:33 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":2.71,"15min":1.7150000000000001},"min":{"1min":0.35099999999999998,"5min":0.33300000000000002,"15min":0.31},"max":{"1min":2.266,"5min":164.13800000000001,"15min":164.13800000000001},"last":0.872},{"interface":"front","average":{"1min":0.58499999999999996,"5min":2.7229999999999999,"15min":1.645},"min":{"1min":0.34100000000000003,"5min":0.33700000000000002,"15min":0.32100000000000001},"max":{"1min":1.9139999999999999,"5min":161.786,"15min":161.786},"last":0.57799999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":825757696,"data_stored":825753624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":805634048,"data_stored":805634048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":866713600,"data_stored":866713600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1080037376,"data_stored":1080033304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":709689344,"data_stored":709689344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":580255744,"data_stored":580255744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":686821376,"data_stored":686817304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1396965376,"data_stored":1396965376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-08-28T02:34:26.661 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd pool set device_health_metrics pgp_num 21 2023-08-28T02:34:26.725 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:34:26.882 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:26 smithi143 ceph-mon[183059]: pgmap v784: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 16 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.4 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 601 op/s; 26927/67992 objects degraded (39.603%); 793 KiB/s, 54 objects/s recovering 2023-08-28T02:34:26.882 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:26 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:34:26.882 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:26 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:34:26.882 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:26 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 26927/67992 objects degraded (39.603%), 21 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-08-28T02:34:26.882 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:26 smithi143 ceph-mon[189871]: pgmap v784: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 16 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.4 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 601 op/s; 26927/67992 objects degraded (39.603%); 793 KiB/s, 54 objects/s recovering 2023-08-28T02:34:26.882 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:26 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:34:26.882 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:26 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:34:26.883 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:26 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 26927/67992 objects degraded (39.603%), 21 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-08-28T02:34:26.902 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:26.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:26 smithi169 ceph-mon[159415]: pgmap v784: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 16 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.4 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 601 op/s; 26927/67992 objects degraded (39.603%); 793 KiB/s, 54 objects/s recovering 2023-08-28T02:34:26.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:26 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:34:26.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:26 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:34:26.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:26 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 26927/67992 objects degraded (39.603%), 21 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-08-28T02:34:27.121 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:34:27.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:27.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:34:27.233 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:34:27.248 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:34:27.261 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:27.310 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:34:27.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:27.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:34:27.418 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:34:27.445 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:27.447 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:34:27.486 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:34:27.493 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:34:27.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:27.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:34:27.513 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:34:27.513 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [], 2023-08-28T02:34:27.513 INFO:teuthology.orchestra.run.smithi143.stdout: "complaint_time": 30, 2023-08-28T02:34:27.513 INFO:teuthology.orchestra.run.smithi143.stdout: "num_blocked_ops": 0 2023-08-28T02:34:27.513 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:34:27.549 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:34:27.598 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:34:27.621 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:34:27.656 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:34:27.691 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:34:27.708 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:27.730 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:34:27.756 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:34:27.756 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:34:27.756 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:34:27.756 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:34:27.757 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:34:27.757 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:34:27.757 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:34:27.757 INFO:teuthology.orchestra.run.smithi143.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-08-28T02:34:27.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:27 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3545635057' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:34:27.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:27 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3545635057' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:34:27.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:27 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3545635057' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:34:28.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:28 smithi143 ceph-mon[183059]: pgmap v785: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 16 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.3 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 597 op/s; 26927/65028 objects degraded (41.408%); 793 KiB/s, 54 objects/s recovering 2023-08-28T02:34:28.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:28 smithi143 ceph-mon[189871]: pgmap v785: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 16 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.3 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 597 op/s; 26927/65028 objects degraded (41.408%); 793 KiB/s, 54 objects/s recovering 2023-08-28T02:34:28.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:28 smithi169 ceph-mon[159415]: pgmap v785: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 16 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.3 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 597 op/s; 26927/65028 objects degraded (41.408%); 793 KiB/s, 54 objects/s recovering 2023-08-28T02:34:29.135 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:34:29.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:29.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:34:29.228 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-08-28T02:34:29.228 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-08-28T02:34:29.243 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:34:29.341 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.2 dump_historic_ops 2023-08-28T02:34:29.372 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:29.515 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:34:29.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:29.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:34:29.560 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:34:29.633 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:34:29.751 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:30.495 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:34:30.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:30.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:34:30.601 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:34:30.629 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:30.677 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:34:30.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:30.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:34:30.765 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:30 smithi143 ceph-mon[183059]: pgmap v786: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 16 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.3 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 597 op/s; 26927/65028 objects degraded (41.408%); 793 KiB/s, 54 objects/s recovering 2023-08-28T02:34:30.765 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:30 smithi143 ceph-mon[189871]: pgmap v786: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 16 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.3 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 597 op/s; 26927/65028 objects degraded (41.408%); 793 KiB/s, 54 objects/s recovering 2023-08-28T02:34:30.783 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:34:30.870 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:30.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:30 smithi169 ceph-mon[159415]: pgmap v786: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 16 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.3 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 597 op/s; 26927/65028 objects degraded (41.408%); 793 KiB/s, 54 objects/s recovering 2023-08-28T02:34:31.074 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:34:31.229 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:34:31.423 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:34:31.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:31.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:34:31.537 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:34:31.567 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:31.616 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:34:31.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:31.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:34:31.720 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:34:31.749 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:31.809 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:34:31.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:31.814 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:34:31.914 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:34:31.929 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:31 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 26927/65028 objects degraded (41.408%), 21 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-08-28T02:34:31.930 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:31 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 26927/65028 objects degraded (41.408%), 21 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-08-28T02:34:31.930 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:34:31 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:34:31] "GET /metrics HTTP/1.1" 200 36592 "" "Prometheus/2.33.4" 2023-08-28T02:34:31.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:31 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 26927/65028 objects degraded (41.408%), 21 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-08-28T02:34:32.003 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:32.174 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:34:32.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:32.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:34:32.282 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:34:32.313 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:32.363 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:34:32.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:32.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:34:32.468 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:34:32.567 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:32.704 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:32 smithi143 ceph-mon[183059]: pgmap v787: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 16 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.2 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 544 op/s; 26712/59262 objects degraded (45.074%); 205 KiB/s, 41 objects/s recovering 2023-08-28T02:34:32.704 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:32 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3147459283' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:34:32.704 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:32 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:34:32.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:32 smithi143 ceph-mon[189871]: pgmap v787: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 16 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.2 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 544 op/s; 26712/59262 objects degraded (45.074%); 205 KiB/s, 41 objects/s recovering 2023-08-28T02:34:32.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:32 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3147459283' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:34:32.704 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:32 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:34:32.760 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is set 2023-08-28T02:34:32.954 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:34:32.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:32.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:34:32.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:32 smithi169 ceph-mon[159415]: pgmap v787: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 16 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.2 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 544 op/s; 26712/59262 objects degraded (45.074%); 205 KiB/s, 41 objects/s recovering 2023-08-28T02:34:32.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:32 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3147459283' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:34:32.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:32 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:34:33.064 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:34:33.141 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:33.628 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:34:33.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:33.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:34:33.793 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:34:33.875 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:33.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:33 smithi143 ceph-mon[183059]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:34:33.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:33 smithi143 ceph-mon[183059]: failed to encode map e306 with expected crc 2023-08-28T02:34:33.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:33 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:34:33.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:33 smithi143 ceph-mon[183059]: osdmap e306: 8 total, 7 up, 7 in 2023-08-28T02:34:33.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:33 smithi143 ceph-mon[183059]: failed to encode map e306 with expected crc 2023-08-28T02:34:33.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:33 smithi143 ceph-mon[183059]: failed to encode map e306 with expected crc 2023-08-28T02:34:33.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:33 smithi143 ceph-mon[183059]: failed to encode map e306 with expected crc 2023-08-28T02:34:33.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:33 smithi143 ceph-mon[183059]: failed to encode map e306 with expected crc 2023-08-28T02:34:33.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:33 smithi143 ceph-mon[183059]: failed to encode map e306 with expected crc 2023-08-28T02:34:33.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:33 smithi143 ceph-mon[183059]: pgmap v789: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 16 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.1 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 603 op/s; 26712/55959 objects degraded (47.735%); 224 KiB/s, 44 objects/s recovering 2023-08-28T02:34:33.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:33 smithi143 ceph-mon[189871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:34:33.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:33 smithi143 ceph-mon[189871]: failed to encode map e306 with expected crc 2023-08-28T02:34:33.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:33 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:34:33.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:33 smithi143 ceph-mon[189871]: osdmap e306: 8 total, 7 up, 7 in 2023-08-28T02:34:33.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:33 smithi143 ceph-mon[189871]: failed to encode map e306 with expected crc 2023-08-28T02:34:33.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:33 smithi143 ceph-mon[189871]: failed to encode map e306 with expected crc 2023-08-28T02:34:33.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:33 smithi143 ceph-mon[189871]: failed to encode map e306 with expected crc 2023-08-28T02:34:33.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:33 smithi143 ceph-mon[189871]: failed to encode map e306 with expected crc 2023-08-28T02:34:33.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:33 smithi143 ceph-mon[189871]: failed to encode map e306 with expected crc 2023-08-28T02:34:33.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:33 smithi143 ceph-mon[189871]: pgmap v789: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 16 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.1 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 603 op/s; 26712/55959 objects degraded (47.735%); 224 KiB/s, 44 objects/s recovering 2023-08-28T02:34:33.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:33 smithi169 ceph-mon[159415]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:34:33.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:33 smithi169 ceph-mon[159415]: failed to encode map e306 with expected crc 2023-08-28T02:34:33.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:33 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:34:33.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:33 smithi169 ceph-mon[159415]: osdmap e306: 8 total, 7 up, 7 in 2023-08-28T02:34:33.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:33 smithi169 ceph-mon[159415]: failed to encode map e306 with expected crc 2023-08-28T02:34:33.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:33 smithi169 ceph-mon[159415]: failed to encode map e306 with expected crc 2023-08-28T02:34:33.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:33 smithi169 ceph-mon[159415]: failed to encode map e306 with expected crc 2023-08-28T02:34:33.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:33 smithi169 ceph-mon[159415]: failed to encode map e306 with expected crc 2023-08-28T02:34:33.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:33 smithi169 ceph-mon[159415]: failed to encode map e306 with expected crc 2023-08-28T02:34:33.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:33 smithi169 ceph-mon[159415]: pgmap v789: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 16 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.1 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 603 op/s; 26712/55959 objects degraded (47.735%); 224 KiB/s, 44 objects/s recovering 2023-08-28T02:34:34.210 INFO:teuthology.orchestra.run.smithi143.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-08-28T02:34:34.606 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:34:34.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:34.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:34:34.713 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:34:34.803 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:34.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:34 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3625743769' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-08-28T02:34:34.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:34 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-08-28T02:34:34.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:34 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3625743769' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-08-28T02:34:34.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:34 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-08-28T02:34:34.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:34 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3625743769' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-08-28T02:34:34.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:34 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-08-28T02:34:35.208 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:34:35.439 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:34:35.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:35.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:34:35.566 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:34:35.587 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-08-28T02:34:35.672 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:35.803 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:34:35.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:35.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:34:35.917 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:34:35.944 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:35.994 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:34:35.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:35.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:34:36.046 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:35 smithi143 ceph-mon[183059]: pgmap v790: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 16 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.0 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 616 op/s; 26422/49482 objects degraded (53.397%); 0 B/s, 50 objects/s recovering 2023-08-28T02:34:36.046 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:35 smithi143 ceph-mon[189871]: pgmap v790: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 16 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.0 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 616 op/s; 26422/49482 objects degraded (53.397%); 0 B/s, 50 objects/s recovering 2023-08-28T02:34:36.073 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:34:36.073 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:34:36.100 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:34:36.192 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:36.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:35 smithi169 ceph-mon[159415]: pgmap v790: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 16 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.0 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 616 op/s; 26422/49482 objects degraded (53.397%); 0 B/s, 50 objects/s recovering 2023-08-28T02:34:36.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:36 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 26422/49482 objects degraded (53.397%), 21 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-08-28T02:34:36.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:36 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 26422/49482 objects degraded (53.397%), 21 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-08-28T02:34:37.013 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:34:37.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:37.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:34:37.121 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:34:37.207 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:37.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:36 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 26422/49482 objects degraded (53.397%), 21 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-08-28T02:34:37.249 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set nodeep-scrub 2023-08-28T02:34:37.630 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:34:37.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:37.636 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:34:37.736 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:34:37.763 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:37.792 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.3 dump_ops_in_flight 2023-08-28T02:34:37.816 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:34:37.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:37.820 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:34:37.921 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:34:37.948 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:37.958 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:37 smithi143 ceph-mon[183059]: pgmap v791: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 16 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 966 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 625 op/s; 26422/46227 objects degraded (57.157%); 0 B/s, 50 objects/s recovering 2023-08-28T02:34:37.967 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:37 smithi143 ceph-mon[189871]: pgmap v791: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 16 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 966 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 625 op/s; 26422/46227 objects degraded (57.157%); 0 B/s, 50 objects/s recovering 2023-08-28T02:34:37.996 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:34:38.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:38.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:34:38.105 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:34:38.131 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:37 smithi169 ceph-mon[159415]: pgmap v791: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 16 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 966 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 625 op/s; 26422/46227 objects degraded (57.157%); 0 B/s, 50 objects/s recovering 2023-08-28T02:34:38.132 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:38.180 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:34:38.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:38.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:34:38.287 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:34:38.302 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:34:38.415 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:38.459 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:34:38.520 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:34:38.562 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:34:38.603 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:34:38.650 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:34:38.686 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:34:38.687 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:34:38.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:38.701 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:34:38.720 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:34:38.736 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:34:38.736 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:34:38.736 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:34:38.736 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:34:38.736 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:34:38.736 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:34:38.736 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:34:38.736 INFO:teuthology.orchestra.run.smithi143.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-08-28T02:34:38.802 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:34:38.897 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:38.980 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:38 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1497960029' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:34:38.980 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:38 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1497960029' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:34:39.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:38 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1497960029' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:34:39.569 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:34:39.593 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:34:39.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:39.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:34:39.698 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:34:39.724 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:39.772 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:34:39.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:39.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:34:39.876 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:34:39.953 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:39 smithi143 ceph-mon[183059]: osdmap e307: 8 total, 7 up, 7 in 2023-08-28T02:34:39.953 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:39 smithi143 ceph-mon[183059]: failed to encode map e307 with expected crc 2023-08-28T02:34:39.953 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:39 smithi143 ceph-mon[183059]: failed to encode map e307 with expected crc 2023-08-28T02:34:39.953 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:39 smithi143 ceph-mon[183059]: failed to encode map e307 with expected crc 2023-08-28T02:34:39.953 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:39 smithi143 ceph-mon[183059]: failed to encode map e307 with expected crc 2023-08-28T02:34:39.953 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:39 smithi143 ceph-mon[183059]: pgmap v793: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 16 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 966 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 542 op/s; 26422/46227 objects degraded (57.157%); 0 B/s, 36 objects/s recovering 2023-08-28T02:34:39.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:39 smithi143 ceph-mon[189871]: osdmap e307: 8 total, 7 up, 7 in 2023-08-28T02:34:39.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:39 smithi143 ceph-mon[189871]: failed to encode map e307 with expected crc 2023-08-28T02:34:39.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:39 smithi143 ceph-mon[189871]: failed to encode map e307 with expected crc 2023-08-28T02:34:39.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:39 smithi143 ceph-mon[189871]: failed to encode map e307 with expected crc 2023-08-28T02:34:39.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:39 smithi143 ceph-mon[189871]: failed to encode map e307 with expected crc 2023-08-28T02:34:39.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:39 smithi143 ceph-mon[189871]: pgmap v793: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 16 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 966 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 542 op/s; 26422/46227 objects degraded (57.157%); 0 B/s, 36 objects/s recovering 2023-08-28T02:34:39.967 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:39.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:39 smithi169 ceph-mon[159415]: osdmap e307: 8 total, 7 up, 7 in 2023-08-28T02:34:39.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:39 smithi169 ceph-mon[159415]: failed to encode map e307 with expected crc 2023-08-28T02:34:39.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:39 smithi169 ceph-mon[159415]: failed to encode map e307 with expected crc 2023-08-28T02:34:39.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:39 smithi169 ceph-mon[159415]: failed to encode map e307 with expected crc 2023-08-28T02:34:39.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:39 smithi169 ceph-mon[159415]: failed to encode map e307 with expected crc 2023-08-28T02:34:39.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:39 smithi169 ceph-mon[159415]: pgmap v793: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 16 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 966 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 542 op/s; 26422/46227 objects degraded (57.157%); 0 B/s, 36 objects/s recovering 2023-08-28T02:34:40.588 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 6, 1] out_osds: [7] dead_osds: [7] live_osds: [0, 1, 2, 3, 4, 5, 6] 2023-08-28T02:34:40.589 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-08-28T02:34:40.589 INFO:tasks.thrashosds.thrasher:Reviving osd 7 2023-08-28T02:34:40.589 INFO:tasks.cephadm.osd.7:Restarting osd.7 (starting--it wasn't running)... 2023-08-28T02:34:40.589 DEBUG:teuthology.orchestra.run.smithi169:osd.7> sudo journalctl -f -n 0 -u ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@osd.7.service 2023-08-28T02:34:40.592 DEBUG:teuthology.orchestra.run.smithi169:> sudo systemctl start ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315@osd.7 2023-08-28T02:34:40.622 INFO:journalctl@ceph.osd.7.smithi169.stdout:-- Logs begin at Mon 2023-08-28 01:49:52 UTC. -- 2023-08-28T02:34:40.676 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:34:40.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:40.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:34:40.790 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:34:40.834 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:40.911 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:34:40.914 INFO:journalctl@ceph.osd.7.smithi169.stdout:Aug 28 02:34:40 smithi169 systemd[1]: Starting Ceph osd.7 for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315... 2023-08-28T02:34:40.915 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:40 smithi169 ceph-mon[159415]: failed to encode map e308 with expected crc 2023-08-28T02:34:40.915 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:40 smithi169 ceph-mon[159415]: failed to encode map e308 with expected crc 2023-08-28T02:34:40.915 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:40 smithi169 ceph-mon[159415]: osdmap e308: 8 total, 7 up, 7 in 2023-08-28T02:34:40.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:40.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:34:40.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:40 smithi143 ceph-mon[183059]: failed to encode map e308 with expected crc 2023-08-28T02:34:40.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:40 smithi143 ceph-mon[183059]: failed to encode map e308 with expected crc 2023-08-28T02:34:40.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:40 smithi143 ceph-mon[183059]: osdmap e308: 8 total, 7 up, 7 in 2023-08-28T02:34:40.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:40 smithi143 ceph-mon[189871]: failed to encode map e308 with expected crc 2023-08-28T02:34:40.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:40 smithi143 ceph-mon[189871]: failed to encode map e308 with expected crc 2023-08-28T02:34:40.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:40 smithi143 ceph-mon[189871]: osdmap e308: 8 total, 7 up, 7 in 2023-08-28T02:34:41.017 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:34:41.049 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-08-28T02:34:41.050 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-08-28T02:34:41.122 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:41.419 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:34:41.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:41.427 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:34:41.527 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:34:41.605 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:41.930 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:34:41 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:34:41] "GET /metrics HTTP/1.1" 200 36578 "" "Prometheus/2.33.4" 2023-08-28T02:34:42.059 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:34:42.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:42.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:34:42.076 INFO:journalctl@ceph.osd.7.smithi169.stdout:Aug 28 02:34:41 smithi169 bash[251126]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2023-08-28T02:34:42.076 INFO:journalctl@ceph.osd.7.smithi169.stdout:Aug 28 02:34:41 smithi169 bash[251126]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2023-08-28T02:34:42.076 INFO:journalctl@ceph.osd.7.smithi169.stdout:Aug 28 02:34:41 smithi169 bash[251126]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2023-08-28T02:34:42.076 INFO:journalctl@ceph.osd.7.smithi169.stdout:Aug 28 02:34:41 smithi169 bash[251126]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2023-08-28T02:34:42.076 INFO:journalctl@ceph.osd.7.smithi169.stdout:Aug 28 02:34:41 smithi169 bash[251126]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2023-08-28T02:34:42.076 INFO:journalctl@ceph.osd.7.smithi169.stdout:Aug 28 02:34:41 smithi169 bash[251126]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2023-08-28T02:34:42.076 INFO:journalctl@ceph.osd.7.smithi169.stdout:Aug 28 02:34:41 smithi169 bash[251126]: --> ceph-volume raw activate successful for osd ID: 7 2023-08-28T02:34:42.170 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:34:42.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:42 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:34:42.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:42 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:34:42.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:42 smithi143 ceph-mon[183059]: pgmap v795: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 850 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 26046/40680 objects degraded (64.027%); 0 B/s, 83 objects/s recovering 2023-08-28T02:34:42.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:42 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 26422/46227 objects degraded (57.157%), 21 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-08-28T02:34:42.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:42 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:34:42.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:42 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:34:42.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:42 smithi143 ceph-mon[189871]: pgmap v795: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 850 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 26046/40680 objects degraded (64.027%); 0 B/s, 83 objects/s recovering 2023-08-28T02:34:42.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:42 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 26422/46227 objects degraded (57.157%), 21 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-08-28T02:34:42.210 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:42.259 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:34:42.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:42.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:34:42.362 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:34:42.390 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:42 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:34:42.390 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:42 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:34:42.390 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:42 smithi169 ceph-mon[159415]: pgmap v795: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 850 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 26046/40680 objects degraded (64.027%); 0 B/s, 83 objects/s recovering 2023-08-28T02:34:42.390 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:42 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 26422/46227 objects degraded (57.157%), 21 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-08-28T02:34:42.395 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:42.450 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:34:42.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:42.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:34:42.553 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:34:42.581 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:42.633 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:34:42.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:42.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:34:42.736 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:34:42.765 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:42.814 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:34:42.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:42.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:34:42.918 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:34:43.006 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:43.051 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:34:43.092 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:43 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2217027315' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:34:43.092 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:43 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:34:43.110 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is set 2023-08-28T02:34:43.370 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:43 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2217027315' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:34:43.370 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:43 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:34:43.371 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:43 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2217027315' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:34:43.371 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:43 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:34:43.395 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:34:43.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:43.412 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:34:43.513 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:34:43.608 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:44.132 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:34:44.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:44.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:34:44.241 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:34:44.269 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:44.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:44 smithi143 ceph-mon[183059]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:34:44.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:44 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:34:44.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:44 smithi143 ceph-mon[183059]: osdmap e309: 8 total, 7 up, 7 in 2023-08-28T02:34:44.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:44 smithi143 ceph-mon[183059]: failed to encode map e309 with expected crc 2023-08-28T02:34:44.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:44 smithi143 ceph-mon[183059]: failed to encode map e309 with expected crc 2023-08-28T02:34:44.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:44 smithi143 ceph-mon[183059]: failed to encode map e309 with expected crc 2023-08-28T02:34:44.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:44 smithi143 ceph-mon[183059]: failed to encode map e309 with expected crc 2023-08-28T02:34:44.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:44 smithi143 ceph-mon[183059]: failed to encode map e309 with expected crc 2023-08-28T02:34:44.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:44 smithi143 ceph-mon[183059]: pgmap v797: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 791 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 465 op/s; 26046/37845 objects degraded (68.823%); 0 B/s, 62 objects/s recovering 2023-08-28T02:34:44.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:44 smithi143 ceph-mon[189871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:34:44.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:44 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:34:44.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:44 smithi143 ceph-mon[189871]: osdmap e309: 8 total, 7 up, 7 in 2023-08-28T02:34:44.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:44 smithi143 ceph-mon[189871]: failed to encode map e309 with expected crc 2023-08-28T02:34:44.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:44 smithi143 ceph-mon[189871]: failed to encode map e309 with expected crc 2023-08-28T02:34:44.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:44 smithi143 ceph-mon[189871]: failed to encode map e309 with expected crc 2023-08-28T02:34:44.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:44 smithi143 ceph-mon[189871]: failed to encode map e309 with expected crc 2023-08-28T02:34:44.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:44 smithi143 ceph-mon[189871]: failed to encode map e309 with expected crc 2023-08-28T02:34:44.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:44 smithi143 ceph-mon[189871]: pgmap v797: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 791 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 465 op/s; 26046/37845 objects degraded (68.823%); 0 B/s, 62 objects/s recovering 2023-08-28T02:34:44.473 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:34:44.475 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:44 smithi169 ceph-mon[159415]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:34:44.475 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:44 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:34:44.475 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:44 smithi169 ceph-mon[159415]: osdmap e309: 8 total, 7 up, 7 in 2023-08-28T02:34:44.475 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:44 smithi169 ceph-mon[159415]: failed to encode map e309 with expected crc 2023-08-28T02:34:44.475 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:44 smithi169 ceph-mon[159415]: failed to encode map e309 with expected crc 2023-08-28T02:34:44.476 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:44 smithi169 ceph-mon[159415]: failed to encode map e309 with expected crc 2023-08-28T02:34:44.476 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:44 smithi169 ceph-mon[159415]: failed to encode map e309 with expected crc 2023-08-28T02:34:44.476 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:44 smithi169 ceph-mon[159415]: failed to encode map e309 with expected crc 2023-08-28T02:34:44.476 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:44 smithi169 ceph-mon[159415]: pgmap v797: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 791 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 465 op/s; 26046/37845 objects degraded (68.823%); 0 B/s, 62 objects/s recovering 2023-08-28T02:34:44.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:44.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:34:44.577 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:34:44.676 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:44.724 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:34:44.861 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:34:44.861 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [ 2023-08-28T02:34:44.861 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.861 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.55270.0:175716 7.a 7:59f74609:::benchmark_data_smithi143_360738_object82946:head [delete] snapc 0=[] ondisk+write+known_if_redirected e309)", 2023-08-28T02:34:44.861 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:34:44.821593+0000", 2023-08-28T02:34:44.861 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.028500378999999999, 2023-08-28T02:34:44.862 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.028527567, 2023-08-28T02:34:44.862 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:34:44.862 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "waiting for sub ops", 2023-08-28T02:34:44.862 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:34:44.862 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.55270", 2023-08-28T02:34:44.862 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:34:44.862 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 175716 2023-08-28T02:34:44.862 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.862 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:34:44.862 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.862 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:34:44.862 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.821593+0000", 2023-08-28T02:34:44.863 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:34:44.863 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.863 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.863 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:34:44.863 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.821593+0000", 2023-08-28T02:34:44.863 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:34:44.863 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.863 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.863 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:34:44.863 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.821590+0000", 2023-08-28T02:34:44.863 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999971 2023-08-28T02:34:44.863 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.863 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.864 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:34:44.864 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.821602+0000", 2023-08-28T02:34:44.864 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1585e-05 2023-08-28T02:34:44.864 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.864 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.864 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:34:44.864 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.821626+0000", 2023-08-28T02:34:44.864 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.4167000000000001e-05 2023-08-28T02:34:44.864 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.864 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.864 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:34:44.864 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.821637+0000", 2023-08-28T02:34:44.864 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1352000000000001e-05 2023-08-28T02:34:44.864 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.865 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.865 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:34:44.865 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.830185+0000", 2023-08-28T02:34:44.865 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0085477260000000003 2023-08-28T02:34:44.865 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.865 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.865 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:34:44.865 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.834162+0000", 2023-08-28T02:34:44.865 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0039766660000000002 2023-08-28T02:34:44.865 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.865 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.865 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 2,5", 2023-08-28T02:34:44.865 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.834277+0000", 2023-08-28T02:34:44.866 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00011503299999999999 2023-08-28T02:34:44.866 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.866 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.866 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:34:44.866 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.842308+0000", 2023-08-28T02:34:44.866 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0080311199999999992 2023-08-28T02:34:44.866 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:34:44.866 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:34:44.866 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:34:44.866 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.866 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.866 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.55270.0:175720 7.2 7:40e67587:::benchmark_data_smithi143_360738_object82950:head [delete] snapc 0=[] ondisk+write+known_if_redirected e309)", 2023-08-28T02:34:44.866 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:34:44.821656+0000", 2023-08-28T02:34:44.867 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.028437114999999999, 2023-08-28T02:34:44.867 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.028542060000000001, 2023-08-28T02:34:44.867 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:34:44.867 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "waiting for sub ops", 2023-08-28T02:34:44.867 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:34:44.867 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.55270", 2023-08-28T02:34:44.867 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:34:44.867 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 175720 2023-08-28T02:34:44.867 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.867 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:34:44.867 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.867 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:34:44.867 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.821656+0000", 2023-08-28T02:34:44.868 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:34:44.868 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.868 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.868 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:34:44.868 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.821656+0000", 2023-08-28T02:34:44.868 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:34:44.868 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.868 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.868 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:34:44.868 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.821656+0000", 2023-08-28T02:34:44.868 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:34:44.868 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.868 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.869 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:34:44.869 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.821660+0000", 2023-08-28T02:34:44.869 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.6110000000000004e-06 2023-08-28T02:34:44.869 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.869 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.869 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:34:44.869 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.821668+0000", 2023-08-28T02:34:44.869 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.3629999999999996e-06 2023-08-28T02:34:44.869 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.869 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.869 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:34:44.869 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.821673+0000", 2023-08-28T02:34:44.869 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.4310000000000002e-06 2023-08-28T02:34:44.869 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.870 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.870 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:34:44.870 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.836887+0000", 2023-08-28T02:34:44.870 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.015214327999999999 2023-08-28T02:34:44.870 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.870 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.870 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:34:44.870 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.837046+0000", 2023-08-28T02:34:44.870 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000158345 2023-08-28T02:34:44.870 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.870 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.870 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,2", 2023-08-28T02:34:44.871 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.837562+0000", 2023-08-28T02:34:44.871 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00051652000000000002 2023-08-28T02:34:44.871 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:34:44.871 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:34:44.871 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:34:44.871 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.871 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.871 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.55270.0:175723 7.10 7:0a1ff0e0:::benchmark_data_smithi143_360738_object82953:head [delete] snapc 0=[] ondisk+write+known_if_redirected e309)", 2023-08-28T02:34:44.871 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:34:44.821678+0000", 2023-08-28T02:34:44.871 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.028415058999999999, 2023-08-28T02:34:44.871 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.028600355000000001, 2023-08-28T02:34:44.871 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:34:44.871 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "waiting for sub ops", 2023-08-28T02:34:44.872 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:34:44.872 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.55270", 2023-08-28T02:34:44.872 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:34:44.872 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 175723 2023-08-28T02:34:44.872 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.872 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:34:44.872 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.872 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:34:44.872 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.821678+0000", 2023-08-28T02:34:44.872 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:34:44.872 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.872 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.873 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:34:44.873 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.821678+0000", 2023-08-28T02:34:44.873 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:34:44.873 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.873 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.873 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:34:44.873 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.821678+0000", 2023-08-28T02:34:44.873 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:34:44.873 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.873 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.873 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:34:44.873 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.821680+0000", 2023-08-28T02:34:44.873 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.7030000000000002e-06 2023-08-28T02:34:44.873 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.874 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.874 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:34:44.874 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.821686+0000", 2023-08-28T02:34:44.874 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.9920000000000004e-06 2023-08-28T02:34:44.874 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.874 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.874 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:34:44.874 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.821691+0000", 2023-08-28T02:34:44.874 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.9559999999999996e-06 2023-08-28T02:34:44.874 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.874 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.874 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:34:44.875 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.821822+0000", 2023-08-28T02:34:44.875 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00013039799999999999 2023-08-28T02:34:44.875 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.875 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.875 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:34:44.875 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.821992+0000", 2023-08-28T02:34:44.875 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00017026300000000001 2023-08-28T02:34:44.875 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.875 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.875 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,2", 2023-08-28T02:34:44.875 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.822658+0000", 2023-08-28T02:34:44.875 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00066572500000000004 2023-08-28T02:34:44.875 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.876 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.876 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:34:44.876 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.843271+0000", 2023-08-28T02:34:44.876 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.020612946 2023-08-28T02:34:44.876 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.876 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.876 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:34:44.876 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.843282+0000", 2023-08-28T02:34:44.876 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1704e-05 2023-08-28T02:34:44.876 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:34:44.876 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:34:44.876 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:34:44.876 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.877 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.877 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.55270.0:175724 7.a 7:58552196:::benchmark_data_smithi143_360738_object82954:head [delete] snapc 0=[] ondisk+write+known_if_redirected e309)", 2023-08-28T02:34:44.877 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:34:44.821704+0000", 2023-08-28T02:34:44.877 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.028389414000000002, 2023-08-28T02:34:44.877 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.028660054000000001, 2023-08-28T02:34:44.877 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:34:44.877 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "waiting for sub ops", 2023-08-28T02:34:44.877 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:34:44.877 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.55270", 2023-08-28T02:34:44.877 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:34:44.877 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 175724 2023-08-28T02:34:44.877 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.877 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:34:44.878 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.878 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:34:44.878 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.821704+0000", 2023-08-28T02:34:44.878 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:34:44.878 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.878 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.878 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:34:44.878 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.821704+0000", 2023-08-28T02:34:44.878 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:34:44.878 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.878 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.878 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:34:44.878 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.821703+0000", 2023-08-28T02:34:44.879 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:34:44.879 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.879 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.879 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:34:44.879 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.821706+0000", 2023-08-28T02:34:44.879 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.5639999999999999e-06 2023-08-28T02:34:44.879 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.879 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.879 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:34:44.879 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.821712+0000", 2023-08-28T02:34:44.879 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.9769999999999997e-06 2023-08-28T02:34:44.879 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.879 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.880 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:34:44.880 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.821717+0000", 2023-08-28T02:34:44.880 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.1089999999999997e-06 2023-08-28T02:34:44.880 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.880 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.880 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:34:44.880 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.838094+0000", 2023-08-28T02:34:44.880 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.01637744 2023-08-28T02:34:44.880 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.880 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.880 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:34:44.880 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.838210+0000", 2023-08-28T02:34:44.880 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000115668 2023-08-28T02:34:44.881 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.881 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.881 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 2,5", 2023-08-28T02:34:44.881 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.838319+0000", 2023-08-28T02:34:44.881 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00010841299999999999 2023-08-28T02:34:44.881 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.881 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.881 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:34:44.881 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.842482+0000", 2023-08-28T02:34:44.881 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0041635429999999996 2023-08-28T02:34:44.881 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:34:44.881 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:34:44.881 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:34:44.882 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.882 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.882 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.55270.0:175718 7.0 e309/291 7:02167425:::benchmark_data_smithi143_360738_object82948:head v 309'7415, mlcod=279'4412)", 2023-08-28T02:34:44.882 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:34:44.823556+0000", 2023-08-28T02:34:44.882 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.026536929000000001, 2023-08-28T02:34:44.882 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.026877998, 2023-08-28T02:34:44.882 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:34:44.882 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "started", 2023-08-28T02:34:44.882 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:34:44.882 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.882 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:34:44.882 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.823556+0000", 2023-08-28T02:34:44.883 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:34:44.883 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.883 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.883 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:34:44.883 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.823556+0000", 2023-08-28T02:34:44.883 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:34:44.883 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.883 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.883 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:34:44.883 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.823555+0000", 2023-08-28T02:34:44.883 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:34:44.883 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.884 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.884 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:34:44.884 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.823566+0000", 2023-08-28T02:34:44.884 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1199000000000001e-05 2023-08-28T02:34:44.884 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.884 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.884 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:34:44.884 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.823585+0000", 2023-08-28T02:34:44.884 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.8620000000000001e-05 2023-08-28T02:34:44.884 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.884 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.884 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:34:44.884 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.823594+0000", 2023-08-28T02:34:44.884 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.7730000000000005e-06 2023-08-28T02:34:44.885 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.885 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.885 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:34:44.885 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.825752+0000", 2023-08-28T02:34:44.885 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0021588829999999999 2023-08-28T02:34:44.885 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.885 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.885 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:34:44.885 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.825779+0000", 2023-08-28T02:34:44.885 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.6347999999999999e-05 2023-08-28T02:34:44.885 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:34:44.885 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:34:44.885 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:34:44.885 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.886 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.886 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.55270.0:175717 7.11 e309/291 7:8e7ba588:::benchmark_data_smithi143_360738_object82947:head v 309'7457, mlcod=265'1072)", 2023-08-28T02:34:44.886 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:34:44.825142+0000", 2023-08-28T02:34:44.886 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.024951654, 2023-08-28T02:34:44.886 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.025345469999999998, 2023-08-28T02:34:44.886 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:34:44.886 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "started", 2023-08-28T02:34:44.886 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:34:44.886 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.886 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:34:44.886 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.825142+0000", 2023-08-28T02:34:44.886 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:34:44.886 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.886 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.887 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:34:44.887 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.825142+0000", 2023-08-28T02:34:44.887 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:34:44.887 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.887 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.887 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:34:44.887 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.825140+0000", 2023-08-28T02:34:44.887 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:34:44.887 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.887 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.887 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:34:44.887 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.825151+0000", 2023-08-28T02:34:44.887 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0496000000000001e-05 2023-08-28T02:34:44.888 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.888 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.888 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:34:44.888 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.825167+0000", 2023-08-28T02:34:44.888 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.6943999999999999e-05 2023-08-28T02:34:44.888 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.888 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.888 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:34:44.888 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.825176+0000", 2023-08-28T02:34:44.888 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.9670000000000007e-06 2023-08-28T02:34:44.888 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.888 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.888 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:34:44.889 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.825243+0000", 2023-08-28T02:34:44.889 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.7002000000000007e-05 2023-08-28T02:34:44.889 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.889 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.889 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:34:44.889 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.825269+0000", 2023-08-28T02:34:44.889 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.5718999999999999e-05 2023-08-28T02:34:44.889 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:34:44.889 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:34:44.889 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:34:44.889 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.889 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.889 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.55270.0:175722 7.c e309/277 7:3fe1c729:::benchmark_data_smithi143_360738_object82952:head v 309'10798, mlcod=0'0)", 2023-08-28T02:34:44.889 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:34:44.826697+0000", 2023-08-28T02:34:44.890 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.023396705, 2023-08-28T02:34:44.890 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.023842894, 2023-08-28T02:34:44.890 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:34:44.890 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "started", 2023-08-28T02:34:44.890 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:34:44.890 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.890 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:34:44.890 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.826697+0000", 2023-08-28T02:34:44.890 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:34:44.890 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.890 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.890 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:34:44.890 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.826697+0000", 2023-08-28T02:34:44.891 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:34:44.891 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.891 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.891 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:34:44.891 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.826695+0000", 2023-08-28T02:34:44.891 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:34:44.891 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.891 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.891 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:34:44.891 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.826703+0000", 2023-08-28T02:34:44.891 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.5660000000000001e-06 2023-08-28T02:34:44.891 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.892 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.892 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:34:44.892 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.826715+0000", 2023-08-28T02:34:44.892 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.2199e-05 2023-08-28T02:34:44.892 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.892 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.892 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:34:44.892 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.826723+0000", 2023-08-28T02:34:44.892 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.712e-06 2023-08-28T02:34:44.892 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.892 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.892 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:34:44.892 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.826782+0000", 2023-08-28T02:34:44.893 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.9438999999999998e-05 2023-08-28T02:34:44.893 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.893 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.893 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:34:44.893 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.826804+0000", 2023-08-28T02:34:44.893 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.1868000000000001e-05 2023-08-28T02:34:44.893 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:34:44.893 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:34:44.893 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:34:44.893 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.893 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.893 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.55270.0:175727 7.13 7:cefea1a8:::benchmark_data_smithi143_360738_object82957:head [delete] snapc 0=[] ondisk+write+known_if_redirected e309)", 2023-08-28T02:34:44.893 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:34:44.829349+0000", 2023-08-28T02:34:44.893 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.020744507999999998, 2023-08-28T02:34:44.894 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.021247515000000002, 2023-08-28T02:34:44.894 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:34:44.894 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "waiting for sub ops", 2023-08-28T02:34:44.894 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:34:44.894 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.55270", 2023-08-28T02:34:44.894 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:34:44.894 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 175727 2023-08-28T02:34:44.894 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.894 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:34:44.894 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.894 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:34:44.894 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.829349+0000", 2023-08-28T02:34:44.894 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:34:44.895 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.895 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.895 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:34:44.895 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.829349+0000", 2023-08-28T02:34:44.895 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:34:44.895 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.895 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.895 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:34:44.895 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.829347+0000", 2023-08-28T02:34:44.895 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:34:44.895 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.895 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.895 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:34:44.896 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.829356+0000", 2023-08-28T02:34:44.896 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.6249999999999996e-06 2023-08-28T02:34:44.896 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.896 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.896 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:34:44.896 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.829371+0000", 2023-08-28T02:34:44.896 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.5891000000000001e-05 2023-08-28T02:34:44.896 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.896 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.896 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:34:44.896 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.829381+0000", 2023-08-28T02:34:44.896 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.4029999999999996e-06 2023-08-28T02:34:44.896 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.897 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.897 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:34:44.897 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.829436+0000", 2023-08-28T02:34:44.897 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.5185000000000001e-05 2023-08-28T02:34:44.897 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.897 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.897 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:34:44.897 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.829578+0000", 2023-08-28T02:34:44.897 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00014151699999999999 2023-08-28T02:34:44.897 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.897 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.897 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 2,4", 2023-08-28T02:34:44.897 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.830115+0000", 2023-08-28T02:34:44.898 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000537066 2023-08-28T02:34:44.898 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.898 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.898 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:34:44.898 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.837272+0000", 2023-08-28T02:34:44.898 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0071570669999999996 2023-08-28T02:34:44.898 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:34:44.898 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:34:44.898 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:34:44.898 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.898 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.898 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.55270.0:175725 7.18 e309/307 7:1adffac0:::benchmark_data_smithi143_360738_object82955:head v 309'7570, mlcod=309'7568)", 2023-08-28T02:34:44.898 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:34:44.829420+0000", 2023-08-28T02:34:44.899 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.020673324, 2023-08-28T02:34:44.899 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.021245094999999999, 2023-08-28T02:34:44.899 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:34:44.899 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "started", 2023-08-28T02:34:44.899 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:34:44.899 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.899 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:34:44.899 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.829420+0000", 2023-08-28T02:34:44.899 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:34:44.899 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.899 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.899 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:34:44.900 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.829420+0000", 2023-08-28T02:34:44.900 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:34:44.900 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.900 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.900 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:34:44.900 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.829419+0000", 2023-08-28T02:34:44.900 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:34:44.900 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.900 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.900 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:34:44.900 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.829427+0000", 2023-08-28T02:34:44.900 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.371e-06 2023-08-28T02:34:44.900 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.900 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.901 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:34:44.901 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.829437+0000", 2023-08-28T02:34:44.901 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.6830000000000003e-06 2023-08-28T02:34:44.901 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.901 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.901 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:34:44.901 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.829443+0000", 2023-08-28T02:34:44.901 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.6110000000000001e-06 2023-08-28T02:34:44.901 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.901 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.901 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:34:44.901 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.843309+0000", 2023-08-28T02:34:44.902 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.013865776 2023-08-28T02:34:44.902 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.902 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.902 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:34:44.902 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.843329+0000", 2023-08-28T02:34:44.902 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.0338000000000001e-05 2023-08-28T02:34:44.902 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:34:44.902 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:34:44.902 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:34:44.902 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.902 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.902 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.55270.0:175731 7.3 7:c2fc02ed:::benchmark_data_smithi143_360738_object82961:head [delete] snapc 0=[] ondisk+write+known_if_redirected e309)", 2023-08-28T02:34:44.902 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:34:44.842525+0000", 2023-08-28T02:34:44.903 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.0075682639999999999, 2023-08-28T02:34:44.903 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0081952859999999995, 2023-08-28T02:34:44.903 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:34:44.903 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "waiting for sub ops", 2023-08-28T02:34:44.903 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:34:44.903 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.55270", 2023-08-28T02:34:44.903 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:34:44.903 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 175731 2023-08-28T02:34:44.903 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.903 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:34:44.903 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.903 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:34:44.904 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.842525+0000", 2023-08-28T02:34:44.904 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:34:44.904 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.904 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.904 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:34:44.904 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.842525+0000", 2023-08-28T02:34:44.904 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:34:44.904 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.904 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.904 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:34:44.904 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.842523+0000", 2023-08-28T02:34:44.904 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:34:44.904 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.904 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.905 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:34:44.905 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.842530+0000", 2023-08-28T02:34:44.905 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.7870000000000004e-06 2023-08-28T02:34:44.905 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.905 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.905 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:34:44.905 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.842544+0000", 2023-08-28T02:34:44.905 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3380000000000001e-05 2023-08-28T02:34:44.905 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.905 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.905 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:34:44.905 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.842552+0000", 2023-08-28T02:34:44.905 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.1289999999999996e-06 2023-08-28T02:34:44.906 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.906 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.906 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:34:44.906 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.846503+0000", 2023-08-28T02:34:44.906 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0039512330000000002 2023-08-28T02:34:44.906 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.906 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.906 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:34:44.906 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.846631+0000", 2023-08-28T02:34:44.906 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00012834700000000001 2023-08-28T02:34:44.906 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.906 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.907 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 2,4", 2023-08-28T02:34:44.907 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.847164+0000", 2023-08-28T02:34:44.907 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00053293700000000004 2023-08-28T02:34:44.907 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:34:44.907 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:34:44.907 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:34:44.907 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.907 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.907 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.55270.0:175719 7.16 e309/286 7:6c13bb9d:::benchmark_data_smithi143_360738_object82949:head v 309'7204, mlcod=309'7202)", 2023-08-28T02:34:44.907 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:34:44.843104+0000", 2023-08-28T02:34:44.907 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.0069895770000000003, 2023-08-28T02:34:44.907 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0076937139999999999, 2023-08-28T02:34:44.907 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:34:44.908 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "started", 2023-08-28T02:34:44.908 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:34:44.908 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.908 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:34:44.908 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.843104+0000", 2023-08-28T02:34:44.908 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:34:44.908 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.908 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.908 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:34:44.908 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.843104+0000", 2023-08-28T02:34:44.908 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:34:44.908 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.908 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.908 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:34:44.909 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.843103+0000", 2023-08-28T02:34:44.909 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:34:44.909 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.909 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.909 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:34:44.909 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.843110+0000", 2023-08-28T02:34:44.909 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.939e-06 2023-08-28T02:34:44.909 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.909 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.909 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:34:44.909 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.843126+0000", 2023-08-28T02:34:44.909 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.6699999999999999e-05 2023-08-28T02:34:44.909 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.910 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.910 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:34:44.910 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.843135+0000", 2023-08-28T02:34:44.910 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.5730000000000008e-06 2023-08-28T02:34:44.910 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.910 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.910 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:34:44.910 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.843181+0000", 2023-08-28T02:34:44.910 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.6171999999999998e-05 2023-08-28T02:34:44.910 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.910 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.910 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:34:44.911 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.843202+0000", 2023-08-28T02:34:44.911 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.0890000000000002e-05 2023-08-28T02:34:44.911 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:34:44.911 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:34:44.911 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:34:44.911 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.911 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.911 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop_reply(client.55270.0:175731 7.3 e309/270)", 2023-08-28T02:34:44.911 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:34:44.849823+0000", 2023-08-28T02:34:44.911 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.00027053400000000001, 2023-08-28T02:34:44.911 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0010245740000000001, 2023-08-28T02:34:44.911 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:34:44.911 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "queued for pg", 2023-08-28T02:34:44.912 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:34:44.912 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.912 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:34:44.912 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.849823+0000", 2023-08-28T02:34:44.912 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:34:44.912 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.912 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.912 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:34:44.912 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.849823+0000", 2023-08-28T02:34:44.912 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:34:44.912 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.912 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.912 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:34:44.913 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.849822+0000", 2023-08-28T02:34:44.913 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:34:44.913 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.913 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.913 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:34:44.913 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.849829+0000", 2023-08-28T02:34:44.913 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.8249999999999999e-06 2023-08-28T02:34:44.913 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.913 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.913 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:34:44.913 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.849840+0000", 2023-08-28T02:34:44.913 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1895e-05 2023-08-28T02:34:44.913 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.913 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.914 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:34:44.914 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.849849+0000", 2023-08-28T02:34:44.914 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.3399999999999998e-06 2023-08-28T02:34:44.914 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:34:44.914 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:34:44.914 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:34:44.914 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.914 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.914 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.55270.0:175734 7.17 e309/270)", 2023-08-28T02:34:44.914 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:34:44.850601+0000", 2023-08-28T02:34:44.914 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 4294967295.9994917, 2023-08-28T02:34:44.914 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00030136900000000001, 2023-08-28T02:34:44.914 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:34:44.915 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "queued for pg", 2023-08-28T02:34:44.915 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:34:44.915 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.915 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:34:44.915 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.850601+0000", 2023-08-28T02:34:44.915 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:34:44.915 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.915 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.915 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:34:44.915 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.850601+0000", 2023-08-28T02:34:44.915 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:34:44.915 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.916 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.916 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:34:44.916 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.850601+0000", 2023-08-28T02:34:44.916 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:34:44.916 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.916 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.916 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:34:44.916 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.850607+0000", 2023-08-28T02:34:44.916 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.9769999999999997e-06 2023-08-28T02:34:44.916 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.916 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.916 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:34:44.916 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.850619+0000", 2023-08-28T02:34:44.916 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.2505e-05 2023-08-28T02:34:44.917 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:34:44.917 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:34:44.917 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:34:44.917 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:44.850627+0000", 2023-08-28T02:34:44.917 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.1729999999999995e-06 2023-08-28T02:34:44.917 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:34:44.917 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:34:44.917 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:34:44.917 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:34:44.917 INFO:teuthology.orchestra.run.smithi143.stdout: ], 2023-08-28T02:34:44.917 INFO:teuthology.orchestra.run.smithi143.stdout: "num_ops": 13 2023-08-28T02:34:44.917 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:34:44.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:44.918 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:34:44.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:34:45.019 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:34:45.097 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:45.133 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.7 dump_ops_in_flight 2023-08-28T02:34:45.487 INFO:journalctl@ceph.osd.7.smithi169.stdout:Aug 28 02:34:45 smithi169 conmon[251851]: debug 2023-08-28T02:34:45.114+0000 7efc735a4700 1 -- 172.21.15.169:0/845918171 <== mon.1 v2:172.21.15.169:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x556b18460680 con 0x556b18427800 2023-08-28T02:34:45.487 INFO:journalctl@ceph.osd.7.smithi169.stdout:Aug 28 02:34:45 smithi169 bash[251812]: bc3ea7a8fd17fbd9526bb68033ff48dcd5fa3314dce15c67dc1045cc56a99290 2023-08-28T02:34:45.488 INFO:journalctl@ceph.osd.7.smithi169.stdout:Aug 28 02:34:45 smithi169 systemd[1]: Started Ceph osd.7 for 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315. 2023-08-28T02:34:45.613 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:34:45.899 INFO:journalctl@ceph.osd.7.smithi169.stdout:Aug 28 02:34:45 smithi169 conmon[251851]: debug 2023-08-28T02:34:45.636+0000 7efc7b01f200 -1 Falling back to public interface 2023-08-28T02:34:46.282 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:34:46.306 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.3 dump_blocked_ops 2023-08-28T02:34:46.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:46.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:34:46.346 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset noscrub 2023-08-28T02:34:46.416 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:34:46.452 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:46.504 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:34:46.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:46.506 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:34:46.607 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:34:46.636 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:46.692 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:34:46.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:46.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:34:46.794 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:34:46.824 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:46 smithi169 ceph-mon[159415]: pgmap v798: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 658 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 740 op/s; 25563/31455 objects degraded (81.268%); 0 B/s, 129 objects/s recovering 2023-08-28T02:34:46.829 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:46.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:46 smithi143 ceph-mon[183059]: pgmap v798: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 658 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 740 op/s; 25563/31455 objects degraded (81.268%); 0 B/s, 129 objects/s recovering 2023-08-28T02:34:46.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:46 smithi143 ceph-mon[189871]: pgmap v798: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 658 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 740 op/s; 25563/31455 objects degraded (81.268%); 0 B/s, 129 objects/s recovering 2023-08-28T02:34:46.960 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:34:46.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:46.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:34:47.063 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:34:47.092 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:47.187 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:34:47.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:47.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:34:47.292 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:34:47.336 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:47.498 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:34:47.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:47.500 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:34:47.574 INFO:teuthology.orchestra.run.smithi169.stderr:no valid command found; 10 closest matches: 2023-08-28T02:34:47.574 INFO:teuthology.orchestra.run.smithi169.stderr:0 2023-08-28T02:34:47.574 INFO:teuthology.orchestra.run.smithi169.stderr:1 2023-08-28T02:34:47.574 INFO:teuthology.orchestra.run.smithi169.stderr:2 2023-08-28T02:34:47.574 INFO:teuthology.orchestra.run.smithi169.stderr:abort 2023-08-28T02:34:47.575 INFO:teuthology.orchestra.run.smithi169.stderr:assert 2023-08-28T02:34:47.575 INFO:teuthology.orchestra.run.smithi169.stderr:config diff 2023-08-28T02:34:47.575 INFO:teuthology.orchestra.run.smithi169.stderr:config diff get 2023-08-28T02:34:47.575 INFO:teuthology.orchestra.run.smithi169.stderr:config get 2023-08-28T02:34:47.575 INFO:teuthology.orchestra.run.smithi169.stderr:config help [] 2023-08-28T02:34:47.575 INFO:teuthology.orchestra.run.smithi169.stderr:config set ... 2023-08-28T02:34:47.575 INFO:teuthology.orchestra.run.smithi169.stderr:admin_socket: invalid command 2023-08-28T02:34:47.600 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:34:47.704 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:47.766 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:47 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 25563/31455 objects degraded (81.268%), 20 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-08-28T02:34:47.786 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:34:47.851 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:34:47.863 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:47 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 25563/31455 objects degraded (81.268%), 20 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-08-28T02:34:47.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:47.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:34:47.864 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:47 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 25563/31455 objects degraded (81.268%), 20 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-08-28T02:34:47.900 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:34:47.965 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:34:47.999 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:48.262 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:34:48.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:48.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:34:48.345 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:34:48.367 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:34:48.392 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:34:48.392 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-08-28T02:34:48.393 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-7, ['dump_ops_in_flight'] 2023-08-28T02:34:48.416 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:48.439 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:34:48.472 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:34:48.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:48.477 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:34:48.500 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:34:48.540 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:34:48.576 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:34:48.578 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:34:48.612 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:48.635 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:34:48.636 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:34:48.636 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:34:48.636 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:34:48.636 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:34:48.636 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:34:48.636 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:34:48.636 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:34:48.636 INFO:teuthology.orchestra.run.smithi143.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-08-28T02:34:48.673 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:34:48.680 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:48 smithi169 ceph-mon[159415]: pgmap v799: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 589 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 753 op/s; 25563/28134 objects degraded (90.862%); 0 B/s, 107 objects/s recovering 2023-08-28T02:34:48.680 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:48 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3909538319' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:34:48.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:48.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:34:48.782 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:34:48.838 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:48 smithi143 ceph-mon[183059]: pgmap v799: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 589 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 753 op/s; 25563/28134 objects degraded (90.862%); 0 B/s, 107 objects/s recovering 2023-08-28T02:34:48.838 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:48 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3909538319' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:34:48.844 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:48.844 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:48 smithi143 ceph-mon[189871]: pgmap v799: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 589 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 753 op/s; 25563/28134 objects degraded (90.862%); 0 B/s, 107 objects/s recovering 2023-08-28T02:34:48.844 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:48 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3909538319' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:34:49.367 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:34:49.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:49.376 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:34:49.477 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:34:49.506 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:49.553 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:34:49.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:49.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:34:49.657 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:34:49.699 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:49.760 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:34:49.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:49.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:34:49.867 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:34:49.960 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:50.198 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:34:50.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:50.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:34:50.309 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:34:50.375 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:50.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:50 smithi143 ceph-mon[183059]: pgmap v800: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 589 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 549 op/s; 25563/28134 objects degraded (90.862%); 0 B/s, 75 objects/s recovering 2023-08-28T02:34:50.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:50 smithi143 ceph-mon[189871]: pgmap v800: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 589 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 549 op/s; 25563/28134 objects degraded (90.862%); 0 B/s, 75 objects/s recovering 2023-08-28T02:34:50.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:50 smithi169 ceph-mon[159415]: pgmap v800: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 589 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 549 op/s; 25563/28134 objects degraded (90.862%); 0 B/s, 75 objects/s recovering 2023-08-28T02:34:51.654 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:34:51.659 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:51 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 25563/28134 objects degraded (90.862%), 20 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-08-28T02:34:51.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:51.661 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:34:51.662 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:51 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 25563/28134 objects degraded (90.862%), 20 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-08-28T02:34:51.762 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:34:51.771 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-08-28T02:34:51.771 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-08-28T02:34:51.796 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:51.856 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:34:51.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:51.862 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:34:51.863 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:51 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 25563/28134 objects degraded (90.862%), 20 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-08-28T02:34:51.931 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:34:51 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:34:51] "GET /metrics HTTP/1.1" 200 36578 "" "Prometheus/2.33.4" 2023-08-28T02:34:51.963 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:34:52.056 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:52.360 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:34:52.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:52.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:34:52.474 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:34:52.490 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:34:52.491 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:34:52.509 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:52.568 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:34:52.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:52.573 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:34:52.674 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:34:52.704 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:52.760 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:34:52.765 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:52 smithi169 ceph-mon[159415]: pgmap v801: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 465 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 616 op/s; 25115/22173 objects degraded (113.268%); 0 B/s, 93 objects/s recovering 2023-08-28T02:34:52.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:52.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:34:52.867 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:34:52.912 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:52.937 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:52 smithi143 ceph-mon[183059]: pgmap v801: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 465 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 616 op/s; 25115/22173 objects degraded (113.268%); 0 B/s, 93 objects/s recovering 2023-08-28T02:34:52.937 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:52 smithi143 ceph-mon[189871]: pgmap v801: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 465 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 616 op/s; 25115/22173 objects degraded (113.268%); 0 B/s, 93 objects/s recovering 2023-08-28T02:34:52.973 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:34:52.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:52.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:34:53.079 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:34:53.109 INFO:journalctl@ceph.osd.7.smithi169.stdout:Aug 28 02:34:52 smithi169 conmon[251851]: debug 2023-08-28T02:34:52.933+0000 7efc7b01f200 -1 osd.7 288 log_to_monitors {default=true} 2023-08-28T02:34:53.111 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:53.160 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:34:53.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:53.167 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:34:53.267 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:34:53.298 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:53.347 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:34:53.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:53.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:34:53.394 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.7 dump_ops_in_flight 2023-08-28T02:34:53.450 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:34:53.491 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:53.554 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:34:53.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:53.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:34:53.608 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:53 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/4181871726' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:34:53.609 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:53 smithi143 ceph-mon[189871]: from='osd.7 [v2:172.21.15.169:6824/2944332380,v1:172.21.15.169:6825/2944332380]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-08-28T02:34:53.663 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:34:53.681 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is unset 2023-08-28T02:34:53.696 INFO:journalctl@ceph.osd.7.smithi169.stdout:Aug 28 02:34:53 smithi169 conmon[251851]: debug 2023-08-28T02:34:53.637+0000 7efc69d14700 -1 osd.7 288 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-08-28T02:34:53.696 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:53 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/4181871726' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:34:53.696 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:53 smithi169 ceph-mon[159415]: from='osd.7 [v2:172.21.15.169:6824/2944332380,v1:172.21.15.169:6825/2944332380]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-08-28T02:34:53.699 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:53.713 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.3 dump_historic_ops 2023-08-28T02:34:53.767 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:34:53.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:53.772 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:34:53.772 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:34:53.852 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:34:53.869 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:53 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/4181871726' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:34:53.869 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:53 smithi143 ceph-mon[183059]: from='osd.7 [v2:172.21.15.169:6824/2944332380,v1:172.21.15.169:6825/2944332380]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-08-28T02:34:53.872 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:34:54.015 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:54.118 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:34:54.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:54.122 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:34:54.222 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:34:54.252 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:54.369 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:34:54.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:54.373 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:34:54.474 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:34:54.507 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:54.614 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:34:54.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:54.617 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:34:54.717 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:34:54.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:54 smithi169 ceph-mon[159415]: pgmap v802: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 397 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 607 op/s; 25115/18930 objects degraded (132.673%); 0 B/s, 89 objects/s recovering 2023-08-28T02:34:54.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:54 smithi169 ceph-mon[159415]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:34:54.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:54 smithi169 ceph-mon[159415]: failed to encode map e310 with expected crc 2023-08-28T02:34:54.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:54 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/4181871726' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:34:54.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:54 smithi169 ceph-mon[159415]: from='osd.7 [v2:172.21.15.169:6824/2944332380,v1:172.21.15.169:6825/2944332380]' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-08-28T02:34:54.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:54 smithi169 ceph-mon[159415]: osdmap e310: 8 total, 7 up, 7 in 2023-08-28T02:34:54.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:54 smithi169 ceph-mon[159415]: from='osd.7 [v2:172.21.15.169:6824/2944332380,v1:172.21.15.169:6825/2944332380]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2023-08-28T02:34:54.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:54 smithi169 ceph-mon[159415]: failed to encode map e310 with expected crc 2023-08-28T02:34:54.814 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:54.820 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:34:54.855 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:34:54.942 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:34:54.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:54 smithi143 ceph-mon[183059]: pgmap v802: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 397 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 607 op/s; 25115/18930 objects degraded (132.673%); 0 B/s, 89 objects/s recovering 2023-08-28T02:34:54.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:54 smithi143 ceph-mon[183059]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:34:54.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:54 smithi143 ceph-mon[183059]: failed to encode map e310 with expected crc 2023-08-28T02:34:54.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:54 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/4181871726' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:34:54.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:54 smithi143 ceph-mon[183059]: from='osd.7 [v2:172.21.15.169:6824/2944332380,v1:172.21.15.169:6825/2944332380]' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-08-28T02:34:54.955 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:54 smithi143 ceph-mon[183059]: osdmap e310: 8 total, 7 up, 7 in 2023-08-28T02:34:54.955 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:54 smithi143 ceph-mon[183059]: from='osd.7 [v2:172.21.15.169:6824/2944332380,v1:172.21.15.169:6825/2944332380]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2023-08-28T02:34:54.955 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:54 smithi143 ceph-mon[183059]: failed to encode map e310 with expected crc 2023-08-28T02:34:54.958 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:54 smithi143 ceph-mon[189871]: pgmap v802: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 397 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 607 op/s; 25115/18930 objects degraded (132.673%); 0 B/s, 89 objects/s recovering 2023-08-28T02:34:54.958 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:54 smithi143 ceph-mon[189871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:34:54.958 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:54 smithi143 ceph-mon[189871]: failed to encode map e310 with expected crc 2023-08-28T02:34:54.958 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:54 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/4181871726' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:34:54.958 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:54 smithi143 ceph-mon[189871]: from='osd.7 [v2:172.21.15.169:6824/2944332380,v1:172.21.15.169:6825/2944332380]' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-08-28T02:34:54.958 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:54 smithi143 ceph-mon[189871]: osdmap e310: 8 total, 7 up, 7 in 2023-08-28T02:34:54.958 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:54 smithi143 ceph-mon[189871]: from='osd.7 [v2:172.21.15.169:6824/2944332380,v1:172.21.15.169:6825/2944332380]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi169", "root=default"]}]: dispatch 2023-08-28T02:34:54.958 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:54 smithi143 ceph-mon[189871]: failed to encode map e310 with expected crc 2023-08-28T02:34:54.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:54.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:34:55.060 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:34:55.090 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:55.140 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:34:55.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:55.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:34:55.243 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:34:55.272 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:55.330 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:34:55.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:55.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:34:55.434 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:34:55.461 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:55.675 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:34:55.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:55.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:34:55.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:55 smithi169 ceph-mon[159415]: osd.7 [v2:172.21.15.169:6824/2944332380,v1:172.21.15.169:6825/2944332380] boot 2023-08-28T02:34:55.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:55 smithi169 ceph-mon[159415]: osdmap e311: 8 total, 8 up, 7 in 2023-08-28T02:34:55.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:55 smithi169 ceph-mon[159415]: failed to encode map e311 with expected crc 2023-08-28T02:34:55.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:55 smithi169 ceph-mon[159415]: failed to encode map e311 with expected crc 2023-08-28T02:34:55.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:55 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-28T02:34:55.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:55 smithi169 ceph-mon[159415]: failed to encode map e311 with expected crc 2023-08-28T02:34:55.780 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:34:55.868 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:55.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:55 smithi143 ceph-mon[189871]: osd.7 [v2:172.21.15.169:6824/2944332380,v1:172.21.15.169:6825/2944332380] boot 2023-08-28T02:34:55.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:55 smithi143 ceph-mon[189871]: osdmap e311: 8 total, 8 up, 7 in 2023-08-28T02:34:55.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:55 smithi143 ceph-mon[189871]: failed to encode map e311 with expected crc 2023-08-28T02:34:55.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:55 smithi143 ceph-mon[189871]: failed to encode map e311 with expected crc 2023-08-28T02:34:55.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:55 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-28T02:34:55.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:55 smithi143 ceph-mon[189871]: failed to encode map e311 with expected crc 2023-08-28T02:34:55.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:55 smithi143 ceph-mon[183059]: osd.7 [v2:172.21.15.169:6824/2944332380,v1:172.21.15.169:6825/2944332380] boot 2023-08-28T02:34:55.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:55 smithi143 ceph-mon[183059]: osdmap e311: 8 total, 8 up, 7 in 2023-08-28T02:34:55.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:55 smithi143 ceph-mon[183059]: failed to encode map e311 with expected crc 2023-08-28T02:34:55.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:55 smithi143 ceph-mon[183059]: failed to encode map e311 with expected crc 2023-08-28T02:34:55.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:55 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-28T02:34:55.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:55 smithi143 ceph-mon[183059]: failed to encode map e311 with expected crc 2023-08-28T02:34:56.221 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:34:56.221 INFO:teuthology.orchestra.run.smithi169.stdout: "ops": [], 2023-08-28T02:34:56.221 INFO:teuthology.orchestra.run.smithi169.stdout: "num_ops": 0 2023-08-28T02:34:56.221 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:34:56.668 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset nodeep-scrub 2023-08-28T02:34:56.694 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:34:56.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:56.698 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:34:56.799 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:34:56.919 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:56 smithi143 ceph-mon[183059]: failed to encode map e289 with expected crc 2023-08-28T02:34:56.919 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:56 smithi143 ceph-mon[183059]: failed to encode map e311 with expected crc 2023-08-28T02:34:56.919 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:56 smithi143 ceph-mon[183059]: pgmap v805: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 273 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 631 op/s; 24513/12963 objects degraded (189.100%); 0 B/s, 131 objects/s recovering 2023-08-28T02:34:56.919 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:56 smithi143 ceph-mon[183059]: 7.1 scrub starts 2023-08-28T02:34:56.919 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:56 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:34:56.919 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:56 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 24513/12963 objects degraded (189.100%), 20 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-08-28T02:34:56.920 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:56 smithi143 ceph-mon[189871]: failed to encode map e289 with expected crc 2023-08-28T02:34:56.920 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:56 smithi143 ceph-mon[189871]: failed to encode map e311 with expected crc 2023-08-28T02:34:56.920 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:56 smithi143 ceph-mon[189871]: pgmap v805: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 273 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 631 op/s; 24513/12963 objects degraded (189.100%); 0 B/s, 131 objects/s recovering 2023-08-28T02:34:56.920 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:56 smithi143 ceph-mon[189871]: 7.1 scrub starts 2023-08-28T02:34:56.920 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:56 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:34:56.920 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:56 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 24513/12963 objects degraded (189.100%), 20 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-08-28T02:34:56.922 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:56.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:56 smithi169 ceph-mon[159415]: failed to encode map e289 with expected crc 2023-08-28T02:34:56.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:56 smithi169 ceph-mon[159415]: failed to encode map e311 with expected crc 2023-08-28T02:34:56.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:56 smithi169 ceph-mon[159415]: pgmap v805: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 273 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 631 op/s; 24513/12963 objects degraded (189.100%); 0 B/s, 131 objects/s recovering 2023-08-28T02:34:56.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:56 smithi169 ceph-mon[159415]: 7.1 scrub starts 2023-08-28T02:34:56.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:56 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:34:56.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:56 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 24513/12963 objects degraded (189.100%), 20 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-08-28T02:34:57.243 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:34:57.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:57.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:34:57.360 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:34:57.465 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:57.641 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:34:57.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:57.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:34:57.757 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:34:57.790 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:57.848 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:34:57.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:57.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:34:57.896 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:34:57.957 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:34:57.987 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:58.009 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:34:58.009 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:34:58.037 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:34:58.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:58.044 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:34:58.145 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:34:58.193 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:34:58.229 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:34:58.233 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:58.272 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:34:58.309 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:34:58.340 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:34:58.370 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:34:58.399 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:34:58.428 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:34:58.430 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:34:58.430 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:34:58.430 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:34:58.430 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:34:58.430 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:34:58.430 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:34:58.430 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:34:58.430 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:34:58.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:58 smithi143 ceph-mon[183059]: pgmap v806: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 212 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 753 op/s; 24513/10038 objects degraded (244.202%); 0 B/s, 131 objects/s recovering 2023-08-28T02:34:58.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:58 smithi143 ceph-mon[183059]: 7.1 scrub starts 2023-08-28T02:34:58.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:58 smithi143 ceph-mon[183059]: osdmap e312: 8 total, 8 up, 7 in 2023-08-28T02:34:58.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:58 smithi143 ceph-mon[183059]: failed to encode map e312 with expected crc 2023-08-28T02:34:58.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:58 smithi143 ceph-mon[183059]: failed to encode map e312 with expected crc 2023-08-28T02:34:58.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:58 smithi143 ceph-mon[183059]: failed to encode map e312 with expected crc 2023-08-28T02:34:58.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:58 smithi143 ceph-mon[183059]: failed to encode map e312 with expected crc 2023-08-28T02:34:58.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:58 smithi143 ceph-mon[183059]: failed to encode map e312 with expected crc 2023-08-28T02:34:58.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:58 smithi143 ceph-mon[183059]: failed to encode map e312 with expected crc 2023-08-28T02:34:58.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:58 smithi143 ceph-mon[183059]: failed to encode map e312 with expected crc 2023-08-28T02:34:58.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:58 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2829733418' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:34:58.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:58 smithi143 ceph-mon[189871]: pgmap v806: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 212 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 753 op/s; 24513/10038 objects degraded (244.202%); 0 B/s, 131 objects/s recovering 2023-08-28T02:34:58.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:58 smithi143 ceph-mon[189871]: 7.1 scrub starts 2023-08-28T02:34:58.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:58 smithi143 ceph-mon[189871]: osdmap e312: 8 total, 8 up, 7 in 2023-08-28T02:34:58.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:58 smithi143 ceph-mon[189871]: failed to encode map e312 with expected crc 2023-08-28T02:34:58.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:58 smithi143 ceph-mon[189871]: failed to encode map e312 with expected crc 2023-08-28T02:34:58.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:58 smithi143 ceph-mon[189871]: failed to encode map e312 with expected crc 2023-08-28T02:34:58.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:58 smithi143 ceph-mon[189871]: failed to encode map e312 with expected crc 2023-08-28T02:34:58.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:58 smithi143 ceph-mon[189871]: failed to encode map e312 with expected crc 2023-08-28T02:34:58.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:58 smithi143 ceph-mon[189871]: failed to encode map e312 with expected crc 2023-08-28T02:34:58.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:58 smithi143 ceph-mon[189871]: failed to encode map e312 with expected crc 2023-08-28T02:34:58.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:58 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2829733418' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:34:58.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:58 smithi169 ceph-mon[159415]: pgmap v806: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 212 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 753 op/s; 24513/10038 objects degraded (244.202%); 0 B/s, 131 objects/s recovering 2023-08-28T02:34:58.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:58 smithi169 ceph-mon[159415]: 7.1 scrub starts 2023-08-28T02:34:58.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:58 smithi169 ceph-mon[159415]: osdmap e312: 8 total, 8 up, 7 in 2023-08-28T02:34:58.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:58 smithi169 ceph-mon[159415]: failed to encode map e312 with expected crc 2023-08-28T02:34:58.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:58 smithi169 ceph-mon[159415]: failed to encode map e312 with expected crc 2023-08-28T02:34:58.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:58 smithi169 ceph-mon[159415]: failed to encode map e312 with expected crc 2023-08-28T02:34:58.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:58 smithi169 ceph-mon[159415]: failed to encode map e312 with expected crc 2023-08-28T02:34:58.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:58 smithi169 ceph-mon[159415]: failed to encode map e312 with expected crc 2023-08-28T02:34:58.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:58 smithi169 ceph-mon[159415]: failed to encode map e312 with expected crc 2023-08-28T02:34:58.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:58 smithi169 ceph-mon[159415]: failed to encode map e312 with expected crc 2023-08-28T02:34:58.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:58 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2829733418' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:34:59.229 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:34:59.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:59.241 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:34:59.341 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:34:59.435 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:34:59.698 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:59 smithi143 ceph-mon[189871]: osdmap e313: 8 total, 8 up, 7 in 2023-08-28T02:34:59.698 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:59 smithi143 ceph-mon[189871]: failed to encode map e313 with expected crc 2023-08-28T02:34:59.698 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:59 smithi143 ceph-mon[189871]: failed to encode map e313 with expected crc 2023-08-28T02:34:59.698 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:34:59 smithi143 ceph-mon[189871]: failed to encode map e313 with expected crc 2023-08-28T02:34:59.871 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:34:59.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:34:59.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:34:59.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:59 smithi143 ceph-mon[183059]: osdmap e313: 8 total, 8 up, 7 in 2023-08-28T02:34:59.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:59 smithi143 ceph-mon[183059]: failed to encode map e313 with expected crc 2023-08-28T02:34:59.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:59 smithi143 ceph-mon[183059]: failed to encode map e313 with expected crc 2023-08-28T02:34:59.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:34:59 smithi143 ceph-mon[183059]: failed to encode map e313 with expected crc 2023-08-28T02:34:59.980 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:34:59.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:59 smithi169 ceph-mon[159415]: osdmap e313: 8 total, 8 up, 7 in 2023-08-28T02:34:59.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:59 smithi169 ceph-mon[159415]: failed to encode map e313 with expected crc 2023-08-28T02:34:59.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:59 smithi169 ceph-mon[159415]: failed to encode map e313 with expected crc 2023-08-28T02:34:59.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:34:59 smithi169 ceph-mon[159415]: failed to encode map e313 with expected crc 2023-08-28T02:35:00.007 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:00.293 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.4 dump_ops_in_flight 2023-08-28T02:35:00.503 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:35:00.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:00.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:35:00.613 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:35:00.689 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:00.703 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:00 smithi169 ceph-mon[159415]: failed to encode map e313 with expected crc 2023-08-28T02:35:00.703 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:00 smithi169 ceph-mon[159415]: pgmap v809: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 212 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 506 op/s; 24513/10038 objects degraded (244.202%); 0 B/s, 102 objects/s recovering 2023-08-28T02:35:00.703 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:00 smithi169 ceph-mon[159415]: 7.1 scrub starts 2023-08-28T02:35:00.725 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:35:00.917 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:35:00.931 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:00 smithi143 ceph-mon[189871]: failed to encode map e313 with expected crc 2023-08-28T02:35:00.931 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:00 smithi143 ceph-mon[189871]: pgmap v809: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 212 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 506 op/s; 24513/10038 objects degraded (244.202%); 0 B/s, 102 objects/s recovering 2023-08-28T02:35:00.931 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:00 smithi143 ceph-mon[189871]: 7.1 scrub starts 2023-08-28T02:35:00.932 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:00 smithi143 ceph-mon[183059]: failed to encode map e313 with expected crc 2023-08-28T02:35:00.932 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:00 smithi143 ceph-mon[183059]: pgmap v809: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 212 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 506 op/s; 24513/10038 objects degraded (244.202%); 0 B/s, 102 objects/s recovering 2023-08-28T02:35:00.932 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:00 smithi143 ceph-mon[183059]: 7.1 scrub starts 2023-08-28T02:35:00.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:00.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:35:01.041 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:35:01.152 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:01.160 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Removed 92767 objects 2023-08-28T02:35:01.160 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Clean up completed and total clean up time :149.271 2023-08-28T02:35:01.202 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:35:01.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:01.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:35:01.306 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:35:01.348 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:01.444 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:35:01.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:01.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:35:01.548 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:35:01.575 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:01.619 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:35:01.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:01.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:35:01.720 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is unset 2023-08-28T02:35:01.722 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:35:01.748 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:01.791 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:35:01.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:01.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:35:01.894 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:35:01.923 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:01.968 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:35:01.970 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:01 smithi143 ceph-mon[183059]: 7.1 scrub starts 2023-08-28T02:35:01.970 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:01 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2893705662' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:35:01.970 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:01 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:35:01.970 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:01 smithi143 ceph-mon[183059]: pgmap v810: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 92 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 610 op/s; 24115/4272 objects degraded (564.490%); 0 B/s, 58 objects/s recovering 2023-08-28T02:35:01.970 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:01 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 24513/10038 objects degraded (244.202%), 20 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-08-28T02:35:01.971 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:01 smithi143 ceph-mon[189871]: 7.1 scrub starts 2023-08-28T02:35:01.971 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:01 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2893705662' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:35:01.971 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:01 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:35:01.971 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:01 smithi143 ceph-mon[189871]: pgmap v810: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 92 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 610 op/s; 24115/4272 objects degraded (564.490%); 0 B/s, 58 objects/s recovering 2023-08-28T02:35:01.971 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:01 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 24513/10038 objects degraded (244.202%), 20 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-08-28T02:35:01.971 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:35:01 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:35:01] "GET /metrics HTTP/1.1" 200 36451 "" "Prometheus/2.33.4" 2023-08-28T02:35:01.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:01.972 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:35:01.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:01 smithi169 ceph-mon[159415]: 7.1 scrub starts 2023-08-28T02:35:01.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:01 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2893705662' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:35:01.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:01 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:35:01.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:01 smithi169 ceph-mon[159415]: pgmap v810: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 15 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 92 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 610 op/s; 24115/4272 objects degraded (564.490%); 0 B/s, 58 objects/s recovering 2023-08-28T02:35:01.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:01 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 24513/10038 objects degraded (244.202%), 20 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-08-28T02:35:02.073 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 6, 1] out_osds: [7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-08-28T02:35:02.073 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-08-28T02:35:02.073 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-08-28T02:35:02.073 INFO:tasks.thrashosds.thrasher:Setting osd 5 primary_affinity to 0.064531 2023-08-28T02:35:02.073 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd primary-affinity 5 0.06453062395986575 2023-08-28T02:35:02.074 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:35:02.099 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:02.143 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:35:02.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:02.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:35:02.246 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:35:02.272 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:02.319 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:35:02.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:02.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:35:02.416 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:35:02.422 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:35:02.449 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:02.491 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:35:02.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:02.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:35:02.594 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:35:02.620 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:02.664 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:35:02.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:02.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:35:02.713 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:35:02.767 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:35:02.796 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:02.838 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:35:02.846 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:35:02.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:02.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:35:02.950 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:35:02.974 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:02 smithi143 ceph-mon[183059]: 7.1 scrub starts 2023-08-28T02:35:02.974 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:02 smithi143 ceph-mon[183059]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:35:02.974 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:02 smithi143 ceph-mon[183059]: failed to encode map e314 with expected crc 2023-08-28T02:35:02.974 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:02 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:35:02.974 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:02 smithi143 ceph-mon[183059]: osdmap e314: 8 total, 8 up, 7 in 2023-08-28T02:35:02.975 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:02 smithi143 ceph-mon[183059]: failed to encode map e314 with expected crc 2023-08-28T02:35:02.975 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:02 smithi143 ceph-mon[183059]: failed to encode map e314 with expected crc 2023-08-28T02:35:02.975 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:02 smithi143 ceph-mon[189871]: 7.1 scrub starts 2023-08-28T02:35:02.975 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:02 smithi143 ceph-mon[189871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:35:02.975 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:02 smithi143 ceph-mon[189871]: failed to encode map e314 with expected crc 2023-08-28T02:35:02.975 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:02 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:35:02.975 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:02 smithi143 ceph-mon[189871]: osdmap e314: 8 total, 8 up, 7 in 2023-08-28T02:35:02.975 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:02 smithi143 ceph-mon[189871]: failed to encode map e314 with expected crc 2023-08-28T02:35:02.976 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:02 smithi143 ceph-mon[189871]: failed to encode map e314 with expected crc 2023-08-28T02:35:02.977 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:02.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:02 smithi169 ceph-mon[159415]: 7.1 scrub starts 2023-08-28T02:35:02.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:02 smithi169 ceph-mon[159415]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:35:02.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:02 smithi169 ceph-mon[159415]: failed to encode map e314 with expected crc 2023-08-28T02:35:02.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:02 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:35:02.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:02 smithi169 ceph-mon[159415]: osdmap e314: 8 total, 8 up, 7 in 2023-08-28T02:35:02.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:02 smithi169 ceph-mon[159415]: failed to encode map e314 with expected crc 2023-08-28T02:35:02.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:02 smithi169 ceph-mon[159415]: failed to encode map e314 with expected crc 2023-08-28T02:35:03.306 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:35:03.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:03.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:35:03.410 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:35:03.438 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:03.714 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:35:03.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:03.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:35:03.818 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:35:03.846 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:03.890 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:35:03.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:03.896 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:35:03.974 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:03 smithi169 ceph-mon[159415]: failed to encode map e314 with expected crc 2023-08-28T02:35:03.974 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:03 smithi169 ceph-mon[159415]: failed to encode map e314 with expected crc 2023-08-28T02:35:03.974 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:03 smithi169 ceph-mon[159415]: 7.1 scrub starts 2023-08-28T02:35:03.974 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:03 smithi169 ceph-mon[159415]: pgmap v812: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 45 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 459 op/s; 23795/2025 objects degraded (1175.062%); 0 B/s, 119 objects/s recovering 2023-08-28T02:35:03.996 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:35:04.020 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:03 smithi143 ceph-mon[183059]: failed to encode map e314 with expected crc 2023-08-28T02:35:04.020 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:03 smithi143 ceph-mon[183059]: failed to encode map e314 with expected crc 2023-08-28T02:35:04.021 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:03 smithi143 ceph-mon[183059]: 7.1 scrub starts 2023-08-28T02:35:04.021 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:03 smithi143 ceph-mon[183059]: pgmap v812: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 45 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 459 op/s; 23795/2025 objects degraded (1175.062%); 0 B/s, 119 objects/s recovering 2023-08-28T02:35:04.021 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:03 smithi143 ceph-mon[189871]: failed to encode map e314 with expected crc 2023-08-28T02:35:04.021 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:03 smithi143 ceph-mon[189871]: failed to encode map e314 with expected crc 2023-08-28T02:35:04.021 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:03 smithi143 ceph-mon[189871]: 7.1 scrub starts 2023-08-28T02:35:04.021 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:03 smithi143 ceph-mon[189871]: pgmap v812: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 45 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 459 op/s; 23795/2025 objects degraded (1175.062%); 0 B/s, 119 objects/s recovering 2023-08-28T02:35:04.023 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:04.211 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:35:04.212 INFO:teuthology.orchestra.run.smithi169.stdout: "ops": [], 2023-08-28T02:35:04.212 INFO:teuthology.orchestra.run.smithi169.stdout: "num_ops": 0 2023-08-28T02:35:04.212 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:35:04.330 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:35:04.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:04.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:35:04.433 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:35:04.465 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:04.514 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:35:04.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:04.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:35:04.617 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:35:04.648 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:04.678 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set noscrub 2023-08-28T02:35:04.752 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:35:04.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:04.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:35:04.858 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:35:04.887 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:04.940 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:35:04.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:04.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:35:04.952 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:35:04.971 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:35:04.998 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:35:05.000 INFO:teuthology.orchestra.run.smithi143.stderr:set osd.5 primary-affinity to 0.0645306 (842292) 2023-08-28T02:35:05.027 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:35:05.044 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:35:05.050 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:35:05.071 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:05.073 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:35:05.092 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:35:05.111 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:35:05.112 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:35:05.112 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:35:05.112 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:35:05.112 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:35:05.113 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:35:05.113 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:35:05.113 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:35:05.113 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:35:05.118 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:35:05.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:05.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:35:05.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:04 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/636606832' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.06453062395986575}]: dispatch 2023-08-28T02:35:05.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:04 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.06453062395986575}]: dispatch 2023-08-28T02:35:05.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:04 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1716499190' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:35:05.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:04 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/636606832' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.06453062395986575}]: dispatch 2023-08-28T02:35:05.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:04 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.06453062395986575}]: dispatch 2023-08-28T02:35:05.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:04 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1716499190' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:35:05.222 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:35:05.250 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:05.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:04 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/636606832' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.06453062395986575}]: dispatch 2023-08-28T02:35:05.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:04 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.06453062395986575}]: dispatch 2023-08-28T02:35:05.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:04 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1716499190' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:35:05.806 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2023-08-28T02:35:05.806 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2023-08-28T02:35:06.002 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.4 dump_blocked_ops 2023-08-28T02:35:06.010 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:35:06.013 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:06 smithi143 ceph-mon[183059]: failed to encode map e315 with expected crc 2023-08-28T02:35:06.014 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:06 smithi143 ceph-mon[183059]: failed to encode map e315 with expected crc 2023-08-28T02:35:06.014 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:06 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.06453062395986575}]': finished 2023-08-28T02:35:06.014 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:06 smithi143 ceph-mon[183059]: osdmap e315: 8 total, 8 up, 7 in 2023-08-28T02:35:06.014 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:06 smithi143 ceph-mon[183059]: pgmap v814: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 20 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 488 op/s; 23795/843 objects degraded (2822.657%); 0 B/s, 105 objects/s recovering 2023-08-28T02:35:06.014 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:06 smithi143 ceph-mon[183059]: failed to encode map e315 with expected crc 2023-08-28T02:35:06.014 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:06 smithi143 ceph-mon[183059]: osdmap e316: 8 total, 8 up, 7 in 2023-08-28T02:35:06.014 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:06 smithi143 ceph-mon[189871]: failed to encode map e315 with expected crc 2023-08-28T02:35:06.014 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:06 smithi143 ceph-mon[189871]: failed to encode map e315 with expected crc 2023-08-28T02:35:06.014 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:06 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.06453062395986575}]': finished 2023-08-28T02:35:06.014 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:06 smithi143 ceph-mon[189871]: osdmap e315: 8 total, 8 up, 7 in 2023-08-28T02:35:06.015 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:06 smithi143 ceph-mon[189871]: pgmap v814: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 20 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 488 op/s; 23795/843 objects degraded (2822.657%); 0 B/s, 105 objects/s recovering 2023-08-28T02:35:06.015 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:06 smithi143 ceph-mon[189871]: failed to encode map e315 with expected crc 2023-08-28T02:35:06.015 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:06 smithi143 ceph-mon[189871]: osdmap e316: 8 total, 8 up, 7 in 2023-08-28T02:35:06.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:06.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:35:06.117 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:35:06.143 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:06.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:06 smithi169 ceph-mon[159415]: failed to encode map e315 with expected crc 2023-08-28T02:35:06.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:06 smithi169 ceph-mon[159415]: failed to encode map e315 with expected crc 2023-08-28T02:35:06.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:06 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.06453062395986575}]': finished 2023-08-28T02:35:06.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:06 smithi169 ceph-mon[159415]: osdmap e315: 8 total, 8 up, 7 in 2023-08-28T02:35:06.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:06 smithi169 ceph-mon[159415]: pgmap v814: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 20 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 488 op/s; 23795/843 objects degraded (2822.657%); 0 B/s, 105 objects/s recovering 2023-08-28T02:35:06.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:06 smithi169 ceph-mon[159415]: failed to encode map e315 with expected crc 2023-08-28T02:35:06.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:06 smithi169 ceph-mon[159415]: osdmap e316: 8 total, 8 up, 7 in 2023-08-28T02:35:06.244 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:35:06.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:06.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:35:06.348 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:35:06.348 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:35:06.356 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:35:06.373 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:35:06.380 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:06.481 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:35:06.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:06.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:35:06.585 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:35:06.612 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:06.861 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:35:06.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:06.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:35:06.964 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:35:06.993 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:07.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:07 smithi143 ceph-mon[189871]: failed to encode map e316 with expected crc 2023-08-28T02:35:07.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:07 smithi143 ceph-mon[189871]: failed to encode map e316 with expected crc 2023-08-28T02:35:07.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:07 smithi143 ceph-mon[189871]: failed to encode map e316 with expected crc 2023-08-28T02:35:07.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:07 smithi143 ceph-mon[189871]: failed to encode map e316 with expected crc 2023-08-28T02:35:07.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:07 smithi143 ceph-mon[189871]: failed to encode map e316 with expected crc 2023-08-28T02:35:07.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:07 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 23795/843 objects degraded (2822.657%), 19 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-08-28T02:35:07.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:07 smithi143 ceph-mon[183059]: failed to encode map e316 with expected crc 2023-08-28T02:35:07.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:07 smithi143 ceph-mon[183059]: failed to encode map e316 with expected crc 2023-08-28T02:35:07.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:07 smithi143 ceph-mon[183059]: failed to encode map e316 with expected crc 2023-08-28T02:35:07.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:07 smithi143 ceph-mon[183059]: failed to encode map e316 with expected crc 2023-08-28T02:35:07.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:07 smithi143 ceph-mon[183059]: failed to encode map e316 with expected crc 2023-08-28T02:35:07.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:07 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 23795/843 objects degraded (2822.657%), 19 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-08-28T02:35:07.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:07 smithi169 ceph-mon[159415]: failed to encode map e316 with expected crc 2023-08-28T02:35:07.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:07 smithi169 ceph-mon[159415]: failed to encode map e316 with expected crc 2023-08-28T02:35:07.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:07 smithi169 ceph-mon[159415]: failed to encode map e316 with expected crc 2023-08-28T02:35:07.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:07 smithi169 ceph-mon[159415]: failed to encode map e316 with expected crc 2023-08-28T02:35:07.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:07 smithi169 ceph-mon[159415]: failed to encode map e316 with expected crc 2023-08-28T02:35:07.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:07 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 23795/843 objects degraded (2822.657%), 19 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-08-28T02:35:07.541 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:35:07.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:07.545 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:35:07.646 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:35:07.675 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:08.252 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:35:08.279 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:08 smithi143 ceph-mon[189871]: 7.1 deep-scrub starts 2023-08-28T02:35:08.279 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:08 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:35:08.279 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:08 smithi143 ceph-mon[189871]: pgmap v816: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 20 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 222 op/s; 22883/843 objects degraded (2714.472%); 0 B/s, 205 objects/s recovering 2023-08-28T02:35:08.279 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:08 smithi143 ceph-mon[183059]: 7.1 deep-scrub starts 2023-08-28T02:35:08.279 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:08 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:35:08.279 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:08 smithi143 ceph-mon[183059]: pgmap v816: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 20 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 222 op/s; 22883/843 objects degraded (2714.472%); 0 B/s, 205 objects/s recovering 2023-08-28T02:35:08.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:08 smithi169 ceph-mon[159415]: 7.1 deep-scrub starts 2023-08-28T02:35:08.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:08 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:35:08.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:08 smithi169 ceph-mon[159415]: pgmap v816: 99 pgs: 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 14 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 20 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 222 op/s; 22883/843 objects degraded (2714.472%); 0 B/s, 205 objects/s recovering 2023-08-28T02:35:08.866 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:35:09.027 INFO:teuthology.orchestra.run.smithi143.stderr:pool 'unique_pool_2' removed 2023-08-28T02:35:09.027 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is set 2023-08-28T02:35:09.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:09 smithi143 ceph-mon[183059]: 7.1 scrub starts 2023-08-28T02:35:09.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:09 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1245481959' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-08-28T02:35:09.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:09 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-08-28T02:35:09.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:09 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3578537542' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:35:09.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:09 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:35:09.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:09 smithi143 ceph-mon[189871]: 7.1 scrub starts 2023-08-28T02:35:09.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:09 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1245481959' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-08-28T02:35:09.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:09 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-08-28T02:35:09.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:09 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3578537542' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:35:09.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:09 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:35:09.464 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:35:09.465 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:09 smithi169 ceph-mon[159415]: 7.1 scrub starts 2023-08-28T02:35:09.465 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:09 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1245481959' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-08-28T02:35:09.465 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:09 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-08-28T02:35:09.465 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:09 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3578537542' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:35:09.466 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:09 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:35:09.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:09.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:35:09.567 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:35:09.599 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:09.667 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:35:09.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:09.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:35:09.774 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:35:09.810 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:09.865 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:35:09.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:09.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:35:09.969 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:35:10.018 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:10.208 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:35:10.209 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:35:10.220 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-08-28T02:35:10.220 INFO:tasks.radosbench:Beginning radosbench... 2023-08-28T02:35:10.220 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2023-08-28T02:35:10.221 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd pool create unique_pool_3 16 2023-08-28T02:35:10.232 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:10 smithi169 ceph-mon[159415]: 7.1 scrub starts 2023-08-28T02:35:10.232 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:10 smithi169 ceph-mon[159415]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:35:10.232 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:10 smithi169 ceph-mon[159415]: failed to encode map e317 with expected crc 2023-08-28T02:35:10.232 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:10 smithi169 ceph-mon[159415]: failed to encode map e317 with expected crc 2023-08-28T02:35:10.232 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:10 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-08-28T02:35:10.232 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:10 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:35:10.232 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:10 smithi169 ceph-mon[159415]: osdmap e317: 8 total, 8 up, 7 in 2023-08-28T02:35:10.232 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:10 smithi169 ceph-mon[159415]: pgmap v818: 73 pgs: 4 active+recovery_wait+degraded, 69 active+clean; 4.6 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 32/84 objects degraded (38.095%) 2023-08-28T02:35:10.232 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:10 smithi169 ceph-mon[159415]: failed to encode map e317 with expected crc 2023-08-28T02:35:10.284 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:10 smithi143 ceph-mon[183059]: 7.1 scrub starts 2023-08-28T02:35:10.284 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:10 smithi143 ceph-mon[183059]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:35:10.284 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:10 smithi143 ceph-mon[183059]: failed to encode map e317 with expected crc 2023-08-28T02:35:10.284 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:10 smithi143 ceph-mon[183059]: failed to encode map e317 with expected crc 2023-08-28T02:35:10.284 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:10 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-08-28T02:35:10.284 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:10 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:35:10.284 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:10 smithi143 ceph-mon[183059]: osdmap e317: 8 total, 8 up, 7 in 2023-08-28T02:35:10.284 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:10 smithi143 ceph-mon[183059]: pgmap v818: 73 pgs: 4 active+recovery_wait+degraded, 69 active+clean; 4.6 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 32/84 objects degraded (38.095%) 2023-08-28T02:35:10.285 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:10 smithi143 ceph-mon[183059]: failed to encode map e317 with expected crc 2023-08-28T02:35:10.285 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:10 smithi143 ceph-mon[189871]: 7.1 scrub starts 2023-08-28T02:35:10.285 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:10 smithi143 ceph-mon[189871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:35:10.285 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:10 smithi143 ceph-mon[189871]: failed to encode map e317 with expected crc 2023-08-28T02:35:10.285 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:10 smithi143 ceph-mon[189871]: failed to encode map e317 with expected crc 2023-08-28T02:35:10.285 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:10 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-08-28T02:35:10.285 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:10 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:35:10.285 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:10 smithi143 ceph-mon[189871]: osdmap e317: 8 total, 8 up, 7 in 2023-08-28T02:35:10.285 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:10 smithi143 ceph-mon[189871]: pgmap v818: 73 pgs: 4 active+recovery_wait+degraded, 69 active+clean; 4.6 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 32/84 objects degraded (38.095%) 2023-08-28T02:35:10.285 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:10 smithi143 ceph-mon[189871]: failed to encode map e317 with expected crc 2023-08-28T02:35:10.530 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:35:10.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:10.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:35:10.633 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:35:10.663 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:11.017 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 6, 1] out_osds: [7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-08-28T02:35:11.017 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd reweight 0 1 2023-08-28T02:35:11.075 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:35:11.194 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:11 smithi143 ceph-mon[183059]: failed to encode map e317 with expected crc 2023-08-28T02:35:11.194 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:11 smithi143 ceph-mon[189871]: failed to encode map e317 with expected crc 2023-08-28T02:35:11.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:11 smithi169 ceph-mon[159415]: failed to encode map e317 with expected crc 2023-08-28T02:35:11.415 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:35:11.436 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:35:11.459 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:35:11.483 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:35:11.504 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:35:11.523 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:35:11.547 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:35:11.568 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:35:11.569 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:35:11.569 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:35:11.569 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:35:11.569 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:35:11.569 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:35:11.569 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:35:11.570 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:35:11.570 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:35:11.814 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:35:11.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:11.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:35:11.916 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.4 dump_historic_ops 2023-08-28T02:35:11.917 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:35:11.947 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:11.996 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:35:12.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:12.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:35:12.102 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:35:12.133 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:12.194 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set nodeep-scrub 2023-08-28T02:35:12.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:12 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:35:12.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:12 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1203330192' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:35:12.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:12 smithi143 ceph-mon[183059]: pgmap v819: 73 pgs: 4 active+recovery_wait+degraded, 69 active+clean; 4.6 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 32/84 objects degraded (38.095%) 2023-08-28T02:35:12.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:12 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 32/84 objects degraded (38.095%), 4 pgs degraded (PG_DEGRADED) 2023-08-28T02:35:12.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:12 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:35:12.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:12 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1203330192' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:35:12.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:12 smithi143 ceph-mon[189871]: pgmap v819: 73 pgs: 4 active+recovery_wait+degraded, 69 active+clean; 4.6 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 32/84 objects degraded (38.095%) 2023-08-28T02:35:12.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:12 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 32/84 objects degraded (38.095%), 4 pgs degraded (PG_DEGRADED) 2023-08-28T02:35:12.201 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:35:11 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:35:11] "GET /metrics HTTP/1.1" 200 34214 "" "Prometheus/2.33.4" 2023-08-28T02:35:12.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:12 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:35:12.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:12 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1203330192' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:35:12.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:12 smithi169 ceph-mon[159415]: pgmap v819: 73 pgs: 4 active+recovery_wait+degraded, 69 active+clean; 4.6 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 32/84 objects degraded (38.095%) 2023-08-28T02:35:12.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:12 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 32/84 objects degraded (38.095%), 4 pgs degraded (PG_DEGRADED) 2023-08-28T02:35:12.360 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:35:12.533 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:35:12.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:12.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:35:12.618 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:35:12.638 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:35:12.670 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:12.923 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:35:12.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:12.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:35:12.947 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:35:13.027 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:35:13.054 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:13.106 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:35:13.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:13.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:35:13.210 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:35:13.238 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:13.496 INFO:teuthology.orchestra.run.smithi143.stderr:pool 'unique_pool_3' created 2023-08-28T02:35:13.543 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:35:13.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:13.546 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:35:13.647 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:35:13.677 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:13.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:13 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/59200847' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-08-28T02:35:13.700 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:13 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/59200847' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-08-28T02:35:13.762 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:13 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/59200847' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-08-28T02:35:14.274 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:35:14.274 INFO:teuthology.orchestra.run.smithi169.stdout: "size": 20, 2023-08-28T02:35:14.275 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 600, 2023-08-28T02:35:14.275 INFO:teuthology.orchestra.run.smithi169.stdout: "ops": [ 2023-08-28T02:35:14.275 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.275 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:59975 7.11 7:8c3bcdae:::benchmark_data_smithi143_360738_object59974:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e270)", 2023-08-28T02:35:14.275 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:31:43.478067+0000", 2023-08-28T02:35:14.275 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 210.79439535399999, 2023-08-28T02:35:14.275 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0095256909999999, 2023-08-28T02:35:14.275 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:35:14.275 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "reached pg", 2023-08-28T02:35:14.275 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:35:14.275 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:35:14.275 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:35:14.276 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 59975 2023-08-28T02:35:14.276 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.276 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:35:14.276 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.276 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:35:14.276 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.478067+0000", 2023-08-28T02:35:14.276 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:35:14.276 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.276 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.276 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:35:14.276 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.478067+0000", 2023-08-28T02:35:14.276 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999971 2023-08-28T02:35:14.277 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.277 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.277 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:35:14.277 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.478064+0000", 2023-08-28T02:35:14.277 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.5537999999999997e-05 2023-08-28T02:35:14.277 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.277 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.277 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:35:14.277 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.478110+0000", 2023-08-28T02:35:14.277 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.8550000000000002e-05 2023-08-28T02:35:14.277 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.277 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.277 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:35:14.278 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.478148+0000", 2023-08-28T02:35:14.278 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.5773999999999998e-05 2023-08-28T02:35:14.278 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.278 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.278 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:35:14.278 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.478164+0000", 2023-08-28T02:35:14.278 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.037114269999999998 2023-08-28T02:35:14.278 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.278 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.278 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:35:14.278 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.515278+0000", 2023-08-28T02:35:14.278 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.524e-06 2023-08-28T02:35:14.278 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.279 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.279 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:35:14.279 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.515287+0000", 2023-08-28T02:35:14.279 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.97223011299999995 2023-08-28T02:35:14.279 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.279 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.279 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:35:14.279 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.487517+0000", 2023-08-28T02:35:14.279 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.5663999999999999e-05 2023-08-28T02:35:14.279 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.279 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.279 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:35:14.279 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.487593+0000", 2023-08-28T02:35:14.280 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0095256909999999 2023-08-28T02:35:14.280 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:14.280 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:35:14.280 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:14.280 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.280 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.280 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:143796 7.1 7:876ffb33:::benchmark_data_smithi143_360738_object51026:head [delete] snapc 0=[] ondisk+write+known_if_redirected e298)", 2023-08-28T02:35:14.280 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:33:51.657413+0000", 2023-08-28T02:35:14.280 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 82.615049079000002, 2023-08-28T02:35:14.280 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.115093801, 2023-08-28T02:35:14.280 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:35:14.280 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:35:14.280 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:35:14.281 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:35:14.281 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:35:14.281 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 143796 2023-08-28T02:35:14.281 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.281 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:35:14.281 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.281 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:35:14.281 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:51.657413+0000", 2023-08-28T02:35:14.281 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:35:14.281 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.281 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.281 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:35:14.281 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:51.657413+0000", 2023-08-28T02:35:14.282 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:35:14.282 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.282 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.282 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:35:14.282 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:51.657412+0000", 2023-08-28T02:35:14.282 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2508e-05 2023-08-28T02:35:14.282 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.282 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.282 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:35:14.282 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:51.657425+0000", 2023-08-28T02:35:14.282 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2927e-05 2023-08-28T02:35:14.282 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.282 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.283 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:35:14.283 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:51.657438+0000", 2023-08-28T02:35:14.283 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.2789999999999998e-06 2023-08-28T02:35:14.283 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.283 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.283 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:35:14.283 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:51.657443+0000", 2023-08-28T02:35:14.283 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.9660000000000003e-05 2023-08-28T02:35:14.283 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.283 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.283 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:35:14.283 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:51.657482+0000", 2023-08-28T02:35:14.283 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.1770999999999999e-05 2023-08-28T02:35:14.284 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.284 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.284 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:35:14.284 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:51.657574+0000", 2023-08-28T02:35:14.284 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.3653999999999996e-05 2023-08-28T02:35:14.284 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.284 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.284 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 3,6", 2023-08-28T02:35:14.284 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:51.657658+0000", 2023-08-28T02:35:14.284 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0019574240000000001 2023-08-28T02:35:14.284 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.284 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.284 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:35:14.285 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:51.659615+0000", 2023-08-28T02:35:14.285 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00014770199999999999 2023-08-28T02:35:14.285 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.285 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.285 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:14.285 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:51.659763+0000", 2023-08-28T02:35:14.285 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.112639683 2023-08-28T02:35:14.285 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.285 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.285 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:14.285 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:51.772403+0000", 2023-08-28T02:35:14.285 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.9156000000000001e-05 2023-08-28T02:35:14.285 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.286 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.286 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:35:14.286 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:51.772462+0000", 2023-08-28T02:35:14.286 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.5278000000000003e-05 2023-08-28T02:35:14.286 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.286 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.286 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:35:14.286 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:51.772507+0000", 2023-08-28T02:35:14.286 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.115093801 2023-08-28T02:35:14.286 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:14.286 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:35:14.286 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:14.287 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.287 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.287 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:144076 7.1 7:868cd9f6:::benchmark_data_smithi143_360738_object51306:head [delete] snapc 0=[] ondisk+write+known_if_redirected e298)", 2023-08-28T02:35:14.287 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:33:52.143372+0000", 2023-08-28T02:35:14.287 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 82.129090903000005, 2023-08-28T02:35:14.287 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.050356223999999998, 2023-08-28T02:35:14.287 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:35:14.287 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:35:14.287 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:35:14.287 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:35:14.287 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:35:14.287 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 144076 2023-08-28T02:35:14.288 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.288 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:35:14.288 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.288 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:35:14.288 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:52.143372+0000", 2023-08-28T02:35:14.288 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:35:14.288 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.288 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.288 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:35:14.288 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:52.143372+0000", 2023-08-28T02:35:14.288 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:35:14.288 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.288 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.288 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:35:14.289 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:52.143370+0000", 2023-08-28T02:35:14.289 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.6810000000000004e-06 2023-08-28T02:35:14.289 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.289 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.289 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:35:14.289 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:52.143378+0000", 2023-08-28T02:35:14.289 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.5692000000000002e-05 2023-08-28T02:35:14.289 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.289 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.289 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:35:14.289 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:52.143394+0000", 2023-08-28T02:35:14.289 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.1380000000000001e-06 2023-08-28T02:35:14.289 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.290 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.290 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:35:14.290 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:52.143401+0000", 2023-08-28T02:35:14.290 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.5191000000000002e-05 2023-08-28T02:35:14.290 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.290 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.290 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:35:14.290 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:52.143446+0000", 2023-08-28T02:35:14.290 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.0285999999999996e-05 2023-08-28T02:35:14.290 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.290 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.290 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:35:14.290 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:52.143537+0000", 2023-08-28T02:35:14.291 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.6891999999999994e-05 2023-08-28T02:35:14.291 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.291 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.291 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 3,6", 2023-08-28T02:35:14.291 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:52.143623+0000", 2023-08-28T02:35:14.291 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0020228960000000002 2023-08-28T02:35:14.291 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.291 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.291 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:35:14.291 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:52.145646+0000", 2023-08-28T02:35:14.291 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00019028599999999999 2023-08-28T02:35:14.291 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.292 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.292 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:14.292 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:52.145837+0000", 2023-08-28T02:35:14.292 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.047824405 2023-08-28T02:35:14.292 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.292 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.292 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:14.292 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:52.193661+0000", 2023-08-28T02:35:14.292 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.8859999999999997e-05 2023-08-28T02:35:14.292 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.292 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.292 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:35:14.293 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:52.193700+0000", 2023-08-28T02:35:14.293 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.783e-05 2023-08-28T02:35:14.293 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.293 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.293 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:35:14.293 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:52.193728+0000", 2023-08-28T02:35:14.293 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.050356223999999998 2023-08-28T02:35:14.293 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:14.293 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:35:14.293 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:14.293 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.293 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.293 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:146256 7.1 7:810f68a1:::benchmark_data_smithi143_360738_object53486:head [delete] snapc 0=[] ondisk+write+known_if_redirected e298)", 2023-08-28T02:35:14.294 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:33:55.490175+0000", 2023-08-28T02:35:14.294 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 78.782287503999996, 2023-08-28T02:35:14.294 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.12448308800000001, 2023-08-28T02:35:14.294 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:35:14.294 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:35:14.294 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:35:14.294 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:35:14.294 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:35:14.294 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 146256 2023-08-28T02:35:14.294 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.294 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:35:14.294 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.294 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:35:14.295 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:55.490175+0000", 2023-08-28T02:35:14.295 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:35:14.295 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.295 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.295 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:35:14.295 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:55.490175+0000", 2023-08-28T02:35:14.295 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:35:14.295 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.295 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.295 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:35:14.295 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:55.490174+0000", 2023-08-28T02:35:14.295 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.4389999999999997e-06 2023-08-28T02:35:14.295 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.296 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.296 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:35:14.296 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:55.490180+0000", 2023-08-28T02:35:14.296 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.4796e-05 2023-08-28T02:35:14.296 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.296 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.296 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:35:14.296 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:55.490195+0000", 2023-08-28T02:35:14.296 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.6680000000000001e-06 2023-08-28T02:35:14.296 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.296 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.296 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:35:14.296 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:55.490202+0000", 2023-08-28T02:35:14.297 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.6190999999999999e-05 2023-08-28T02:35:14.297 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.297 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.297 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:35:14.297 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:55.490249+0000", 2023-08-28T02:35:14.297 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000118342 2023-08-28T02:35:14.297 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.297 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.297 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:35:14.297 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:55.490367+0000", 2023-08-28T02:35:14.297 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.9988999999999994e-05 2023-08-28T02:35:14.297 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.298 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.298 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 3,6", 2023-08-28T02:35:14.298 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:55.490467+0000", 2023-08-28T02:35:14.298 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0050395489999999999 2023-08-28T02:35:14.298 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.298 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.298 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:35:14.298 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:55.495507+0000", 2023-08-28T02:35:14.298 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.003952873 2023-08-28T02:35:14.298 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.298 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.298 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:14.298 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:55.499459+0000", 2023-08-28T02:35:14.299 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.11509736499999999 2023-08-28T02:35:14.299 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.299 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.299 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:14.299 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:55.614557+0000", 2023-08-28T02:35:14.299 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.5517999999999999e-05 2023-08-28T02:35:14.299 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.299 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.299 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:35:14.299 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:55.614612+0000", 2023-08-28T02:35:14.299 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.5763000000000002e-05 2023-08-28T02:35:14.299 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.299 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.300 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:35:14.300 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:55.614658+0000", 2023-08-28T02:35:14.300 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.12448308800000001 2023-08-28T02:35:14.300 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:14.300 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:35:14.300 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:14.300 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.300 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.300 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:146259 7.1 7:841d2ee1:::benchmark_data_smithi143_360738_object53489:head [delete] snapc 0=[] ondisk+write+known_if_redirected e298)", 2023-08-28T02:35:14.300 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:33:55.490216+0000", 2023-08-28T02:35:14.300 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 78.782246673000003, 2023-08-28T02:35:14.300 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.12454114199999999, 2023-08-28T02:35:14.300 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:35:14.301 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:35:14.301 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:35:14.301 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:35:14.301 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:35:14.301 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 146259 2023-08-28T02:35:14.301 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.301 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:35:14.301 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.301 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:35:14.301 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:55.490216+0000", 2023-08-28T02:35:14.301 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:35:14.301 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.301 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.302 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:35:14.302 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:55.490216+0000", 2023-08-28T02:35:14.302 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999995 2023-08-28T02:35:14.302 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.302 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.302 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:35:14.302 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:55.490215+0000", 2023-08-28T02:35:14.302 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.7249999999999999e-06 2023-08-28T02:35:14.302 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.302 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.302 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:35:14.302 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:55.490219+0000", 2023-08-28T02:35:14.302 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.7440000000000001e-06 2023-08-28T02:35:14.303 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.303 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.303 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:35:14.303 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:55.490226+0000", 2023-08-28T02:35:14.303 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.3789999999999999e-06 2023-08-28T02:35:14.303 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.303 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.303 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:35:14.303 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:55.490230+0000", 2023-08-28T02:35:14.303 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.002549553 2023-08-28T02:35:14.303 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.303 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.304 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:35:14.304 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:55.492780+0000", 2023-08-28T02:35:14.304 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000119406 2023-08-28T02:35:14.304 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.304 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.304 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:35:14.304 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:55.492899+0000", 2023-08-28T02:35:14.304 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000114584 2023-08-28T02:35:14.304 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.304 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.304 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 3,6", 2023-08-28T02:35:14.304 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:55.493014+0000", 2023-08-28T02:35:14.305 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0026743230000000002 2023-08-28T02:35:14.305 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.305 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.305 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:35:14.305 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:55.495688+0000", 2023-08-28T02:35:14.305 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.003915684 2023-08-28T02:35:14.305 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.305 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.305 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:14.305 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:55.499604+0000", 2023-08-28T02:35:14.305 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.115092114 2023-08-28T02:35:14.305 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.305 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.306 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:14.306 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:55.614696+0000", 2023-08-28T02:35:14.306 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.3532999999999999e-05 2023-08-28T02:35:14.306 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.306 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.306 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:35:14.306 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:55.614729+0000", 2023-08-28T02:35:14.306 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.7695000000000002e-05 2023-08-28T02:35:14.306 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.306 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.306 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:35:14.306 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:55.614757+0000", 2023-08-28T02:35:14.306 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.12454114199999999 2023-08-28T02:35:14.307 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:14.307 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:35:14.307 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:14.307 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.307 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.307 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:147677 7.1 7:8362fcb8:::benchmark_data_smithi143_360738_object54907:head [delete] snapc 0=[] ondisk+write+known_if_redirected e298)", 2023-08-28T02:35:14.307 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:33:57.863041+0000", 2023-08-28T02:35:14.307 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 76.409421226999996, 2023-08-28T02:35:14.307 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.034047075000000003, 2023-08-28T02:35:14.307 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:35:14.307 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:35:14.307 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:35:14.308 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:35:14.308 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:35:14.308 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 147677 2023-08-28T02:35:14.308 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.308 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:35:14.308 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.308 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:35:14.308 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:57.863041+0000", 2023-08-28T02:35:14.308 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:35:14.308 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.308 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.308 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:35:14.308 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:57.863041+0000", 2023-08-28T02:35:14.308 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:35:14.309 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.309 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.309 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:35:14.309 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:57.863040+0000", 2023-08-28T02:35:14.309 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.4790000000000004e-06 2023-08-28T02:35:14.309 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.309 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.309 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:35:14.309 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:57.863046+0000", 2023-08-28T02:35:14.309 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1262e-05 2023-08-28T02:35:14.309 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.309 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.309 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:35:14.310 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:57.863057+0000", 2023-08-28T02:35:14.310 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.9209999999999997e-06 2023-08-28T02:35:14.310 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.310 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.310 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:35:14.310 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:57.863063+0000", 2023-08-28T02:35:14.310 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.5339e-05 2023-08-28T02:35:14.310 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.310 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.310 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:35:14.310 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:57.863118+0000", 2023-08-28T02:35:14.310 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000123499 2023-08-28T02:35:14.310 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.311 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.311 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:35:14.311 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:57.863242+0000", 2023-08-28T02:35:14.311 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00014113400000000001 2023-08-28T02:35:14.311 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.311 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.311 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 3,6", 2023-08-28T02:35:14.311 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:57.863383+0000", 2023-08-28T02:35:14.311 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0045955140000000002 2023-08-28T02:35:14.311 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.311 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.311 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:35:14.311 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:57.867978+0000", 2023-08-28T02:35:14.312 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00032195299999999999 2023-08-28T02:35:14.312 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.312 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.312 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:14.312 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:57.868300+0000", 2023-08-28T02:35:14.312 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.028724487999999999 2023-08-28T02:35:14.312 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.312 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.312 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:14.312 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:57.897025+0000", 2023-08-28T02:35:14.312 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.7326999999999997e-05 2023-08-28T02:35:14.312 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.312 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.313 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:35:14.313 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:57.897062+0000", 2023-08-28T02:35:14.313 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.6250000000000001e-05 2023-08-28T02:35:14.313 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.313 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.313 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:35:14.313 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:57.897088+0000", 2023-08-28T02:35:14.313 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.034047075000000003 2023-08-28T02:35:14.313 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:14.313 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:35:14.313 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:14.313 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.314 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.314 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:147680 7.1 7:8269f046:::benchmark_data_smithi143_360738_object54910:head [delete] snapc 0=[] ondisk+write+known_if_redirected e298)", 2023-08-28T02:35:14.314 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:33:57.863236+0000", 2023-08-28T02:35:14.314 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 76.409226661000005, 2023-08-28T02:35:14.314 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.033947175000000003, 2023-08-28T02:35:14.314 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:35:14.314 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:35:14.314 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:35:14.314 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:35:14.314 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:35:14.314 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 147680 2023-08-28T02:35:14.314 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.314 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:35:14.315 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.315 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:35:14.315 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:57.863236+0000", 2023-08-28T02:35:14.315 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:35:14.315 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.315 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.315 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:35:14.315 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:57.863236+0000", 2023-08-28T02:35:14.315 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999995 2023-08-28T02:35:14.315 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.315 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.315 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:35:14.315 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:57.863235+0000", 2023-08-28T02:35:14.316 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2656999999999999e-05 2023-08-28T02:35:14.316 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.316 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.316 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:35:14.316 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:57.863248+0000", 2023-08-28T02:35:14.316 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.0220000000000001e-06 2023-08-28T02:35:14.316 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.316 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.316 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:35:14.316 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:57.863254+0000", 2023-08-28T02:35:14.316 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.6490000000000001e-06 2023-08-28T02:35:14.316 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.316 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.317 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:35:14.317 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:57.863257+0000", 2023-08-28T02:35:14.317 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0026487949999999998 2023-08-28T02:35:14.317 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.317 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.317 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:35:14.317 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:57.865906+0000", 2023-08-28T02:35:14.317 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000114182 2023-08-28T02:35:14.317 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.317 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.317 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:35:14.317 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:57.866020+0000", 2023-08-28T02:35:14.318 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.9716999999999995e-05 2023-08-28T02:35:14.318 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.318 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.318 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 3,6", 2023-08-28T02:35:14.318 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:57.866110+0000", 2023-08-28T02:35:14.318 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.001976383 2023-08-28T02:35:14.318 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.318 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.318 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:35:14.318 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:57.868087+0000", 2023-08-28T02:35:14.318 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00032435400000000001 2023-08-28T02:35:14.318 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.318 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.318 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:14.319 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:57.868411+0000", 2023-08-28T02:35:14.319 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.028713140000000002 2023-08-28T02:35:14.319 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.319 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.319 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:14.319 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:57.897124+0000", 2023-08-28T02:35:14.319 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.8391e-05 2023-08-28T02:35:14.319 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.319 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.319 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:35:14.319 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:57.897152+0000", 2023-08-28T02:35:14.319 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.0540000000000002e-05 2023-08-28T02:35:14.320 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.320 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.320 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:35:14.320 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:57.897183+0000", 2023-08-28T02:35:14.320 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.033947175000000003 2023-08-28T02:35:14.320 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:14.320 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:35:14.320 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:14.320 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.320 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.320 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:148941 7.1 7:81503dd1:::benchmark_data_smithi143_360738_object56171:head [delete] snapc 0=[] ondisk+write+known_if_redirected e300)", 2023-08-28T02:35:14.320 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:33:59.827219+0000", 2023-08-28T02:35:14.320 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 74.445243314999999, 2023-08-28T02:35:14.321 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.090323267999999998, 2023-08-28T02:35:14.321 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:35:14.321 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:35:14.321 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:35:14.321 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:35:14.321 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:35:14.321 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 148941 2023-08-28T02:35:14.321 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.321 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:35:14.321 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.321 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:35:14.321 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:59.827219+0000", 2023-08-28T02:35:14.321 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:35:14.322 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.322 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.322 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:35:14.322 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:59.827219+0000", 2023-08-28T02:35:14.322 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:35:14.322 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.322 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.322 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:35:14.322 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:59.827218+0000", 2023-08-28T02:35:14.322 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.4900000000000003e-06 2023-08-28T02:35:14.322 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.322 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.323 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:35:14.323 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:59.827225+0000", 2023-08-28T02:35:14.323 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.3648e-05 2023-08-28T02:35:14.323 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.323 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.323 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:35:14.323 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:59.827239+0000", 2023-08-28T02:35:14.323 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.7510000000000008e-06 2023-08-28T02:35:14.323 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.323 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.323 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:35:14.323 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:59.827247+0000", 2023-08-28T02:35:14.323 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.4864000000000003e-05 2023-08-28T02:35:14.324 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.324 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.324 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:35:14.324 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:59.827292+0000", 2023-08-28T02:35:14.324 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00011667099999999999 2023-08-28T02:35:14.324 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.324 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.324 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:35:14.324 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:59.827408+0000", 2023-08-28T02:35:14.324 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012895800000000001 2023-08-28T02:35:14.324 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.324 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.324 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 3,6", 2023-08-28T02:35:14.325 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:59.827537+0000", 2023-08-28T02:35:14.325 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.002258187 2023-08-28T02:35:14.325 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.325 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.325 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:14.325 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:59.829795+0000", 2023-08-28T02:35:14.325 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.5559e-05 2023-08-28T02:35:14.325 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.325 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.325 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:35:14.325 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:59.829821+0000", 2023-08-28T02:35:14.325 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.087655644000000005 2023-08-28T02:35:14.326 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.326 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.326 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:14.326 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:59.917477+0000", 2023-08-28T02:35:14.326 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.9270000000000002e-05 2023-08-28T02:35:14.326 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.326 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.326 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:35:14.326 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:59.917516+0000", 2023-08-28T02:35:14.326 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.6531000000000001e-05 2023-08-28T02:35:14.326 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.326 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.326 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:35:14.327 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:59.917542+0000", 2023-08-28T02:35:14.327 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.090323267999999998 2023-08-28T02:35:14.327 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:14.327 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:35:14.327 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:14.327 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.327 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.327 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:148946 7.1 7:80598a9c:::benchmark_data_smithi143_360738_object56176:head [delete] snapc 0=[] ondisk+write+known_if_redirected e300)", 2023-08-28T02:35:14.327 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:33:59.831895+0000", 2023-08-28T02:35:14.327 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 74.440567399000003, 2023-08-28T02:35:14.327 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.089853637, 2023-08-28T02:35:14.327 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:35:14.327 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:35:14.328 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:35:14.328 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:35:14.328 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:35:14.328 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 148946 2023-08-28T02:35:14.328 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.328 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:35:14.328 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.328 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:35:14.328 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:59.831895+0000", 2023-08-28T02:35:14.328 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:35:14.328 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.328 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.328 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:35:14.329 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:59.831895+0000", 2023-08-28T02:35:14.329 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:35:14.329 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.329 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.329 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:35:14.329 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:59.831894+0000", 2023-08-28T02:35:14.329 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.1359999999999996e-06 2023-08-28T02:35:14.329 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.329 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.329 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:35:14.329 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:59.831899+0000", 2023-08-28T02:35:14.329 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.8570000000000001e-06 2023-08-28T02:35:14.330 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.330 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.330 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:35:14.330 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:59.831907+0000", 2023-08-28T02:35:14.330 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.006e-06 2023-08-28T02:35:14.330 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.330 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.330 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:35:14.330 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:59.831913+0000", 2023-08-28T02:35:14.330 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.7431999999999997e-05 2023-08-28T02:35:14.330 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.330 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.331 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:35:14.331 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:59.831950+0000", 2023-08-28T02:35:14.331 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012408800000000001 2023-08-28T02:35:14.331 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.331 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.331 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:35:14.331 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:59.832074+0000", 2023-08-28T02:35:14.331 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00013205399999999999 2023-08-28T02:35:14.331 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.331 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.331 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 3,6", 2023-08-28T02:35:14.331 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:59.832206+0000", 2023-08-28T02:35:14.331 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0024957429999999999 2023-08-28T02:35:14.332 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.332 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.332 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:14.332 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:59.834702+0000", 2023-08-28T02:35:14.332 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00090144900000000002 2023-08-28T02:35:14.332 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.332 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.332 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:35:14.332 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:59.835603+0000", 2023-08-28T02:35:14.332 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.086086360000000001 2023-08-28T02:35:14.332 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.332 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.332 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:14.332 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:59.921690+0000", 2023-08-28T02:35:14.333 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.4527000000000003e-05 2023-08-28T02:35:14.333 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.333 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.333 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:35:14.333 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:59.921724+0000", 2023-08-28T02:35:14.333 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.4528e-05 2023-08-28T02:35:14.333 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.333 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.333 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:35:14.333 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:59.921749+0000", 2023-08-28T02:35:14.333 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.089853637 2023-08-28T02:35:14.333 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:14.333 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:35:14.334 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:14.334 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.334 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.334 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:149205 7.1 7:8464a1f1:::benchmark_data_smithi143_360738_object56435:head [delete] snapc 0=[] ondisk+write+known_if_redirected e300)", 2023-08-28T02:35:14.334 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:34:00.247183+0000", 2023-08-28T02:35:14.334 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 74.025279608999995, 2023-08-28T02:35:14.334 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.14220756600000001, 2023-08-28T02:35:14.334 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:35:14.334 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:35:14.334 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:35:14.334 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:35:14.334 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:35:14.335 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 149205 2023-08-28T02:35:14.335 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.335 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:35:14.335 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.335 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:35:14.335 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:00.247183+0000", 2023-08-28T02:35:14.335 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:35:14.335 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.335 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.335 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:35:14.335 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:00.247183+0000", 2023-08-28T02:35:14.335 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:35:14.335 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.336 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.336 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:35:14.336 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:00.247181+0000", 2023-08-28T02:35:14.336 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.2979999999999997e-06 2023-08-28T02:35:14.336 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.336 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.336 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:35:14.336 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:00.247188+0000", 2023-08-28T02:35:14.336 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.1866e-05 2023-08-28T02:35:14.336 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.336 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.336 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:35:14.336 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:00.247210+0000", 2023-08-28T02:35:14.337 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.2250000000000003e-06 2023-08-28T02:35:14.337 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.337 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.337 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:35:14.337 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:00.247217+0000", 2023-08-28T02:35:14.337 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.4197999999999999e-05 2023-08-28T02:35:14.337 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.337 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.337 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:35:14.337 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:00.247261+0000", 2023-08-28T02:35:14.337 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.8661999999999999e-05 2023-08-28T02:35:14.337 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.338 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.338 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:35:14.338 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:00.247350+0000", 2023-08-28T02:35:14.338 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.7546000000000004e-05 2023-08-28T02:35:14.338 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.338 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.338 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 3,6", 2023-08-28T02:35:14.338 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:00.247447+0000", 2023-08-28T02:35:14.338 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0020366659999999999 2023-08-28T02:35:14.338 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.338 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.338 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:35:14.338 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:00.249484+0000", 2023-08-28T02:35:14.339 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00086826299999999998 2023-08-28T02:35:14.339 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.339 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.339 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:14.339 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:00.250352+0000", 2023-08-28T02:35:14.339 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.138973664 2023-08-28T02:35:14.339 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.339 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.339 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:14.339 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:00.389326+0000", 2023-08-28T02:35:14.339 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.7370999999999998e-05 2023-08-28T02:35:14.339 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.339 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.340 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:35:14.340 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:00.389363+0000", 2023-08-28T02:35:14.340 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.7154000000000001e-05 2023-08-28T02:35:14.340 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.340 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.340 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:35:14.340 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:00.389390+0000", 2023-08-28T02:35:14.340 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.14220756600000001 2023-08-28T02:35:14.340 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:14.340 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:35:14.340 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:14.340 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.340 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.341 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:158430 7.1 7:80cd2604:::benchmark_data_smithi143_360738_object65660:head [delete] snapc 0=[] ondisk+write+known_if_redirected e304)", 2023-08-28T02:35:14.341 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:34:16.371279+0000", 2023-08-28T02:35:14.341 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 57.901183347, 2023-08-28T02:35:14.341 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.161253328, 2023-08-28T02:35:14.341 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:35:14.341 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:35:14.341 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:35:14.341 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:35:14.341 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:35:14.341 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 158430 2023-08-28T02:35:14.341 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.341 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:35:14.341 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.342 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:35:14.342 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:16.371279+0000", 2023-08-28T02:35:14.342 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:35:14.342 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.342 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.342 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:35:14.342 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:16.371279+0000", 2023-08-28T02:35:14.342 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:35:14.342 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.342 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.342 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:35:14.342 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:16.371278+0000", 2023-08-28T02:35:14.342 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.6760000000000007e-06 2023-08-28T02:35:14.343 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.343 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.343 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:35:14.343 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:16.371285+0000", 2023-08-28T02:35:14.343 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1573000000000001e-05 2023-08-28T02:35:14.343 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.343 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.343 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:35:14.343 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:16.371297+0000", 2023-08-28T02:35:14.343 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.7949999999999998e-06 2023-08-28T02:35:14.343 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.343 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.343 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:35:14.344 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:16.371302+0000", 2023-08-28T02:35:14.344 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.2237999999999997e-05 2023-08-28T02:35:14.344 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.344 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.344 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:35:14.344 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:16.371354+0000", 2023-08-28T02:35:14.344 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000141848 2023-08-28T02:35:14.344 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.344 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.344 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:35:14.344 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:16.371496+0000", 2023-08-28T02:35:14.344 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00015076400000000001 2023-08-28T02:35:14.344 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.345 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.345 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 3,6", 2023-08-28T02:35:14.345 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:16.371647+0000", 2023-08-28T02:35:14.345 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0026210489999999999 2023-08-28T02:35:14.345 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.345 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.345 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:35:14.345 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:16.374268+0000", 2023-08-28T02:35:14.345 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.7988000000000002e-05 2023-08-28T02:35:14.345 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.345 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.345 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:14.345 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:16.374316+0000", 2023-08-28T02:35:14.345 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.15809690500000001 2023-08-28T02:35:14.346 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.346 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.346 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:14.346 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:16.532413+0000", 2023-08-28T02:35:14.346 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.7624000000000002e-05 2023-08-28T02:35:14.346 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.346 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.346 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:35:14.346 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:16.532480+0000", 2023-08-28T02:35:14.346 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.2240000000000001e-05 2023-08-28T02:35:14.346 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.346 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.347 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:35:14.347 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:16.532532+0000", 2023-08-28T02:35:14.347 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.161253328 2023-08-28T02:35:14.347 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:14.347 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:35:14.347 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:14.347 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.347 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.347 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:158440 7.1 7:838be274:::benchmark_data_smithi143_360738_object65670:head [delete] snapc 0=[] ondisk+write+known_if_redirected e304)", 2023-08-28T02:35:14.347 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:34:16.382803+0000", 2023-08-28T02:35:14.347 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 57.889659268999999, 2023-08-28T02:35:14.347 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.15928136400000001, 2023-08-28T02:35:14.347 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:35:14.348 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:35:14.348 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:35:14.348 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:35:14.348 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:35:14.348 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 158440 2023-08-28T02:35:14.348 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.348 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:35:14.348 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.348 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:35:14.348 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:16.382803+0000", 2023-08-28T02:35:14.348 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:35:14.348 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.348 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.349 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:35:14.349 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:16.382803+0000", 2023-08-28T02:35:14.349 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:35:14.349 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.349 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.349 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:35:14.349 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:16.382801+0000", 2023-08-28T02:35:14.349 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.269e-05 2023-08-28T02:35:14.349 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.349 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.349 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:35:14.349 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:16.382814+0000", 2023-08-28T02:35:14.350 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.7422999999999999e-05 2023-08-28T02:35:14.350 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.350 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.350 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:35:14.350 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:16.382831+0000", 2023-08-28T02:35:14.350 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.627e-06 2023-08-28T02:35:14.350 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.350 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.350 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:35:14.350 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:16.382839+0000", 2023-08-28T02:35:14.350 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000108966 2023-08-28T02:35:14.350 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.350 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.351 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:35:14.351 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:16.382948+0000", 2023-08-28T02:35:14.351 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00014038200000000001 2023-08-28T02:35:14.351 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.351 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.351 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:35:14.351 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:16.383088+0000", 2023-08-28T02:35:14.351 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00014513800000000001 2023-08-28T02:35:14.351 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.351 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.351 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 3,6", 2023-08-28T02:35:14.351 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:16.383234+0000", 2023-08-28T02:35:14.351 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0024725099999999998 2023-08-28T02:35:14.352 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.352 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.352 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:35:14.352 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:16.385706+0000", 2023-08-28T02:35:14.352 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.001644532 2023-08-28T02:35:14.352 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.352 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.352 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:14.352 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:16.387351+0000", 2023-08-28T02:35:14.352 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.15463265000000001 2023-08-28T02:35:14.352 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.352 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.352 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:14.353 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:16.541983+0000", 2023-08-28T02:35:14.353 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.8091999999999999e-05 2023-08-28T02:35:14.353 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.353 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.353 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:35:14.353 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:16.542041+0000", 2023-08-28T02:35:14.353 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.3166e-05 2023-08-28T02:35:14.353 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.353 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.353 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:35:14.353 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:16.542085+0000", 2023-08-28T02:35:14.353 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.15928136400000001 2023-08-28T02:35:14.353 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:14.354 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:35:14.354 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:14.354 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.354 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.354 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:163217 7.1 7:869bced0:::benchmark_data_smithi143_360738_object70447:head [delete] snapc 0=[] ondisk+write+known_if_redirected e305)", 2023-08-28T02:35:14.354 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:34:24.288341+0000", 2023-08-28T02:35:14.354 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 49.984121569000003, 2023-08-28T02:35:14.354 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.034039477999999998, 2023-08-28T02:35:14.354 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:35:14.354 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:35:14.354 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:35:14.354 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:35:14.354 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:35:14.355 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 163217 2023-08-28T02:35:14.355 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.355 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:35:14.355 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.355 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:35:14.355 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:24.288341+0000", 2023-08-28T02:35:14.355 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:35:14.355 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.355 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.355 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:35:14.355 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:24.288341+0000", 2023-08-28T02:35:14.355 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:35:14.355 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.356 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.356 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:35:14.356 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:24.288340+0000", 2023-08-28T02:35:14.356 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.0730000000000001e-06 2023-08-28T02:35:14.356 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.356 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.356 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:35:14.356 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:24.288346+0000", 2023-08-28T02:35:14.356 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.3254999999999999e-05 2023-08-28T02:35:14.356 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.356 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.356 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:35:14.356 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:24.288359+0000", 2023-08-28T02:35:14.357 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.2100000000000001e-06 2023-08-28T02:35:14.357 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.357 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.357 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:35:14.357 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:24.288364+0000", 2023-08-28T02:35:14.357 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.3310999999999999e-05 2023-08-28T02:35:14.357 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.357 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.357 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:35:14.357 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:24.288408+0000", 2023-08-28T02:35:14.357 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000104764 2023-08-28T02:35:14.357 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.357 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.358 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:35:14.358 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:24.288512+0000", 2023-08-28T02:35:14.358 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00048308100000000001 2023-08-28T02:35:14.358 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.358 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.358 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 3,6", 2023-08-28T02:35:14.358 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:24.288995+0000", 2023-08-28T02:35:14.358 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0022612690000000002 2023-08-28T02:35:14.358 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.358 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.358 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:35:14.358 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:24.291257+0000", 2023-08-28T02:35:14.359 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000772516 2023-08-28T02:35:14.359 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.359 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.359 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:14.359 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:24.292029+0000", 2023-08-28T02:35:14.359 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.030282815000000001 2023-08-28T02:35:14.359 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.359 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.359 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:14.359 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:24.322312+0000", 2023-08-28T02:35:14.359 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.8380000000000002e-05 2023-08-28T02:35:14.359 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.359 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.360 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:35:14.360 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:24.322350+0000", 2023-08-28T02:35:14.360 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.9988000000000002e-05 2023-08-28T02:35:14.360 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.360 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.360 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:35:14.360 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:24.322380+0000", 2023-08-28T02:35:14.360 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.034039477999999998 2023-08-28T02:35:14.360 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:14.360 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:35:14.360 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:14.360 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.360 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.360 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:163449 7.1 7:83e4fd0f:::benchmark_data_smithi143_360738_object70679:head [delete] snapc 0=[] ondisk+write+known_if_redirected e305)", 2023-08-28T02:35:14.361 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:34:24.634149+0000", 2023-08-28T02:35:14.361 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 49.638313338000003, 2023-08-28T02:35:14.361 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.139553234, 2023-08-28T02:35:14.361 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:35:14.361 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:35:14.361 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:35:14.361 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:35:14.361 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:35:14.361 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 163449 2023-08-28T02:35:14.361 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.361 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:35:14.361 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.361 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:35:14.362 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:24.634149+0000", 2023-08-28T02:35:14.362 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:35:14.362 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.362 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.362 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:35:14.362 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:24.634149+0000", 2023-08-28T02:35:14.362 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:35:14.362 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.362 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.362 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:35:14.362 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:24.634147+0000", 2023-08-28T02:35:14.362 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.5611999999999999e-05 2023-08-28T02:35:14.362 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.363 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.363 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:35:14.363 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:24.634163+0000", 2023-08-28T02:35:14.363 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.6013000000000001e-05 2023-08-28T02:35:14.363 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.363 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.363 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:35:14.363 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:24.634179+0000", 2023-08-28T02:35:14.363 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.2779999999999996e-06 2023-08-28T02:35:14.363 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.363 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.363 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:35:14.364 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:24.634186+0000", 2023-08-28T02:35:14.364 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.2177e-05 2023-08-28T02:35:14.364 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.364 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.364 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:35:14.364 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:24.634228+0000", 2023-08-28T02:35:14.364 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.1291000000000005e-05 2023-08-28T02:35:14.364 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.364 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.364 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:35:14.364 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:24.634310+0000", 2023-08-28T02:35:14.364 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00042957 2023-08-28T02:35:14.364 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.364 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.365 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 3,6", 2023-08-28T02:35:14.365 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:24.634739+0000", 2023-08-28T02:35:14.365 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.002547583 2023-08-28T02:35:14.365 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.365 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.365 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:35:14.365 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:24.637287+0000", 2023-08-28T02:35:14.365 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00032691900000000001 2023-08-28T02:35:14.365 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.365 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.365 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:14.365 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:24.637614+0000", 2023-08-28T02:35:14.365 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.136030445 2023-08-28T02:35:14.366 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.366 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.366 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:14.366 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:24.773644+0000", 2023-08-28T02:35:14.366 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.2370999999999999e-05 2023-08-28T02:35:14.366 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.366 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.366 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:35:14.366 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:24.773676+0000", 2023-08-28T02:35:14.366 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.5919999999999999e-05 2023-08-28T02:35:14.366 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.366 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.366 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:35:14.367 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:24.773702+0000", 2023-08-28T02:35:14.367 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.139553234 2023-08-28T02:35:14.367 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:14.367 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:35:14.367 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:14.367 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.367 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.367 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:164226 7.1 7:846afa54:::benchmark_data_smithi143_360738_object71456:head [delete] snapc 0=[] ondisk+write+known_if_redirected e305)", 2023-08-28T02:35:14.367 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:34:25.970842+0000", 2023-08-28T02:35:14.367 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 48.301620890000002, 2023-08-28T02:35:14.367 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.12171931800000001, 2023-08-28T02:35:14.367 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:35:14.368 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:35:14.368 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:35:14.368 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:35:14.368 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:35:14.368 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 164226 2023-08-28T02:35:14.368 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.368 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:35:14.368 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.368 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:35:14.368 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:25.970842+0000", 2023-08-28T02:35:14.368 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:35:14.368 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.368 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.369 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:35:14.369 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:25.970842+0000", 2023-08-28T02:35:14.369 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:35:14.369 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.369 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.369 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:35:14.369 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:25.970840+0000", 2023-08-28T02:35:14.369 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.8829999999999995e-06 2023-08-28T02:35:14.369 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.369 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.369 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:35:14.369 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:25.970849+0000", 2023-08-28T02:35:14.369 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.3522e-05 2023-08-28T02:35:14.370 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.370 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.370 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:35:14.370 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:25.970863+0000", 2023-08-28T02:35:14.370 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.5359999999999998e-06 2023-08-28T02:35:14.370 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.370 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.370 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:35:14.370 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:25.970869+0000", 2023-08-28T02:35:14.370 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.2506999999999998e-05 2023-08-28T02:35:14.370 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.370 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.370 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:35:14.371 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:25.970912+0000", 2023-08-28T02:35:14.371 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.1592e-05 2023-08-28T02:35:14.371 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.371 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.371 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:35:14.371 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:25.971003+0000", 2023-08-28T02:35:14.371 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00043353899999999998 2023-08-28T02:35:14.371 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.371 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.371 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 3,6", 2023-08-28T02:35:14.371 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:25.971437+0000", 2023-08-28T02:35:14.371 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0029681740000000001 2023-08-28T02:35:14.371 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.372 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.372 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:35:14.372 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:25.974405+0000", 2023-08-28T02:35:14.372 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00097959900000000005 2023-08-28T02:35:14.372 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.372 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.372 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:14.372 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:25.975385+0000", 2023-08-28T02:35:14.372 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.117107459 2023-08-28T02:35:14.372 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.372 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.372 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:14.372 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:26.092492+0000", 2023-08-28T02:35:14.373 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.9909e-05 2023-08-28T02:35:14.373 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.373 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.373 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:35:14.373 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:26.092532+0000", 2023-08-28T02:35:14.373 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.8986999999999999e-05 2023-08-28T02:35:14.373 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.373 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.373 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:35:14.373 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:26.092561+0000", 2023-08-28T02:35:14.373 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.12171931800000001 2023-08-28T02:35:14.373 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:14.374 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:35:14.374 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:14.374 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.374 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.374 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:164283 7.1 7:83819ea1:::benchmark_data_smithi143_360738_object71513:head [delete] snapc 0=[] ondisk+write+known_if_redirected e305)", 2023-08-28T02:35:14.374 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:34:26.142493+0000", 2023-08-28T02:35:14.374 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 48.129969938000002, 2023-08-28T02:35:14.374 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.076128350999999997, 2023-08-28T02:35:14.374 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:35:14.374 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:35:14.374 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:35:14.374 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:35:14.374 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:35:14.375 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 164283 2023-08-28T02:35:14.375 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.375 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:35:14.375 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.375 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:35:14.375 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:26.142493+0000", 2023-08-28T02:35:14.375 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:35:14.375 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.375 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.375 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:35:14.375 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:26.142493+0000", 2023-08-28T02:35:14.375 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:35:14.376 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.376 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.376 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:35:14.376 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:26.142491+0000", 2023-08-28T02:35:14.376 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.1130000000000001e-06 2023-08-28T02:35:14.376 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.376 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.376 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:35:14.376 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:26.142499+0000", 2023-08-28T02:35:14.376 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.6001000000000001e-05 2023-08-28T02:35:14.376 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.376 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.376 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:35:14.377 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:26.142515+0000", 2023-08-28T02:35:14.377 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.7469999999999999e-06 2023-08-28T02:35:14.377 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.377 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.377 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:35:14.377 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:26.142522+0000", 2023-08-28T02:35:14.377 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.0390999999999999e-05 2023-08-28T02:35:14.377 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.377 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.377 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:35:14.377 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:26.142572+0000", 2023-08-28T02:35:14.377 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000100313 2023-08-28T02:35:14.377 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.378 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.378 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:35:14.378 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:26.142672+0000", 2023-08-28T02:35:14.378 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00041908499999999998 2023-08-28T02:35:14.378 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.378 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.378 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 3,6", 2023-08-28T02:35:14.378 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:26.143091+0000", 2023-08-28T02:35:14.378 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0020980909999999998 2023-08-28T02:35:14.378 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.378 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.378 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:35:14.378 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:26.145190+0000", 2023-08-28T02:35:14.379 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000368024 2023-08-28T02:35:14.379 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.379 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.379 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:14.379 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:26.145558+0000", 2023-08-28T02:35:14.379 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.072995357999999996 2023-08-28T02:35:14.379 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.379 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.379 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:14.379 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:26.218553+0000", 2023-08-28T02:35:14.379 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.3018e-05 2023-08-28T02:35:14.379 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.379 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.380 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:35:14.380 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:26.218596+0000", 2023-08-28T02:35:14.380 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.4933e-05 2023-08-28T02:35:14.380 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.380 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.380 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:35:14.380 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:26.218621+0000", 2023-08-28T02:35:14.380 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.076128350999999997 2023-08-28T02:35:14.380 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:14.380 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:35:14.380 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:14.380 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.380 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.381 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:173440 7.1 7:84928ebc:::benchmark_data_smithi143_360738_object80670:head [delete] snapc 0=[] ondisk+write+known_if_redirected e308)", 2023-08-28T02:35:14.381 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:34:41.284914+0000", 2023-08-28T02:35:14.381 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 32.987548293000003, 2023-08-28T02:35:14.381 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.031912703000000001, 2023-08-28T02:35:14.381 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:35:14.381 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:35:14.381 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:35:14.381 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:35:14.381 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:35:14.381 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 173440 2023-08-28T02:35:14.381 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.381 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:35:14.381 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.382 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:35:14.382 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:41.284914+0000", 2023-08-28T02:35:14.382 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:35:14.382 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.382 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.382 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:35:14.382 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:41.284914+0000", 2023-08-28T02:35:14.382 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:35:14.382 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.382 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.382 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:35:14.382 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:41.284913+0000", 2023-08-28T02:35:14.382 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.4229999999999999e-06 2023-08-28T02:35:14.383 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.383 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.383 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:35:14.383 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:41.284920+0000", 2023-08-28T02:35:14.383 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.4742e-05 2023-08-28T02:35:14.383 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.383 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.383 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:35:14.383 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:41.284935+0000", 2023-08-28T02:35:14.383 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.0210000000000002e-06 2023-08-28T02:35:14.383 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.383 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.383 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:35:14.384 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:41.284942+0000", 2023-08-28T02:35:14.384 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00097831400000000009 2023-08-28T02:35:14.384 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.384 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.384 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:35:14.384 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:41.285920+0000", 2023-08-28T02:35:14.384 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000133134 2023-08-28T02:35:14.384 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.384 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.384 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:35:14.384 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:41.286053+0000", 2023-08-28T02:35:14.384 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00042114799999999999 2023-08-28T02:35:14.384 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.385 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.385 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 3,6", 2023-08-28T02:35:14.385 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:41.286475+0000", 2023-08-28T02:35:14.385 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0020959809999999998 2023-08-28T02:35:14.385 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.385 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.385 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:35:14.385 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:41.288571+0000", 2023-08-28T02:35:14.385 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.002339102 2023-08-28T02:35:14.385 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.385 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.385 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:14.385 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:41.290910+0000", 2023-08-28T02:35:14.385 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.025826344000000001 2023-08-28T02:35:14.386 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.386 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.386 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:14.386 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:41.316736+0000", 2023-08-28T02:35:14.386 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.8140000000000003e-05 2023-08-28T02:35:14.386 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.386 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.386 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:35:14.386 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:41.316784+0000", 2023-08-28T02:35:14.386 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.2740000000000001e-05 2023-08-28T02:35:14.386 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.386 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.386 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:35:14.387 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:41.316827+0000", 2023-08-28T02:35:14.387 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.031912703000000001 2023-08-28T02:35:14.387 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:14.387 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:35:14.387 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:14.387 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.387 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.387 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:176919 7.1 7:856ab0a8:::benchmark_data_smithi143_360738_object84149:head [delete] snapc 0=[] ondisk+write+known_if_redirected e309)", 2023-08-28T02:35:14.387 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:34:46.524310+0000", 2023-08-28T02:35:14.387 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 27.748152280999999, 2023-08-28T02:35:14.387 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.087456969999999995, 2023-08-28T02:35:14.388 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:35:14.388 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:35:14.388 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:35:14.388 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:35:14.388 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:35:14.388 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 176919 2023-08-28T02:35:14.388 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.388 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:35:14.388 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.388 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:35:14.388 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:46.524310+0000", 2023-08-28T02:35:14.388 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:35:14.388 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.389 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.389 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:35:14.389 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:46.524310+0000", 2023-08-28T02:35:14.389 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:35:14.389 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.389 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.389 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:35:14.389 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:46.524309+0000", 2023-08-28T02:35:14.389 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.0020000000000003e-06 2023-08-28T02:35:14.389 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.389 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.389 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:35:14.389 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:46.524315+0000", 2023-08-28T02:35:14.390 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0797e-05 2023-08-28T02:35:14.390 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.390 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.390 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:35:14.390 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:46.524326+0000", 2023-08-28T02:35:14.390 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.8930000000000004e-06 2023-08-28T02:35:14.390 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.390 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.390 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:35:14.390 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:46.524330+0000", 2023-08-28T02:35:14.390 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.4068e-05 2023-08-28T02:35:14.390 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.390 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.391 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:35:14.391 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:46.524365+0000", 2023-08-28T02:35:14.391 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.1799999999999997e-05 2023-08-28T02:35:14.391 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.391 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.391 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:35:14.391 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:46.524436+0000", 2023-08-28T02:35:14.391 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00040786499999999999 2023-08-28T02:35:14.391 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.391 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.391 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 3,6", 2023-08-28T02:35:14.391 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:46.524844+0000", 2023-08-28T02:35:14.391 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0019829180000000002 2023-08-28T02:35:14.391 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.392 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.392 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:35:14.392 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:46.526827+0000", 2023-08-28T02:35:14.392 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.079387315999999999 2023-08-28T02:35:14.392 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.392 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.392 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:14.392 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:46.606214+0000", 2023-08-28T02:35:14.392 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0054907200000000001 2023-08-28T02:35:14.392 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.392 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.392 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:14.392 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:46.611705+0000", 2023-08-28T02:35:14.393 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.7345999999999998e-05 2023-08-28T02:35:14.393 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.393 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.393 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:35:14.393 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:46.611742+0000", 2023-08-28T02:35:14.393 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.4666000000000001e-05 2023-08-28T02:35:14.393 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.393 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.393 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:35:14.393 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:46.611767+0000", 2023-08-28T02:35:14.393 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.087456969999999995 2023-08-28T02:35:14.393 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:14.393 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:35:14.394 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:14.394 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.394 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.394 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_repop(client.55270.0:176918 7.3 e309/270 7:c07d80b1:::benchmark_data_smithi143_360738_object84148:head v 309'7331, mlcod=309'7329)", 2023-08-28T02:35:14.394 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:34:46.527717+0000", 2023-08-28T02:35:14.394 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 27.744745768000001, 2023-08-28T02:35:14.394 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.080659117000000002, 2023-08-28T02:35:14.394 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:35:14.394 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:35:14.394 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:35:14.394 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.394 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:35:14.394 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:46.527717+0000", 2023-08-28T02:35:14.395 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:35:14.395 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.395 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.395 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:35:14.395 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:46.527717+0000", 2023-08-28T02:35:14.395 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999976 2023-08-28T02:35:14.395 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.395 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.395 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:35:14.395 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:46.527714+0000", 2023-08-28T02:35:14.395 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.7689999999999998e-05 2023-08-28T02:35:14.395 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.395 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.396 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:35:14.396 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:46.527732+0000", 2023-08-28T02:35:14.396 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.076170817000000002 2023-08-28T02:35:14.396 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.396 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.396 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:35:14.396 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:46.603903+0000", 2023-08-28T02:35:14.396 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.4620999999999999e-05 2023-08-28T02:35:14.396 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.396 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.396 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:35:14.396 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:46.603918+0000", 2023-08-28T02:35:14.396 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00062368700000000005 2023-08-28T02:35:14.397 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.397 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.397 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:35:14.397 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:46.604541+0000", 2023-08-28T02:35:14.397 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.1522000000000002e-05 2023-08-28T02:35:14.397 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.397 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.397 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:35:14.397 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:46.604563+0000", 2023-08-28T02:35:14.397 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0037715439999999999 2023-08-28T02:35:14.397 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.397 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.398 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:35:14.398 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:46.608334+0000", 2023-08-28T02:35:14.398 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.1473000000000001e-05 2023-08-28T02:35:14.398 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.398 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.398 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:35:14.398 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:46.608376+0000", 2023-08-28T02:35:14.398 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.080659117000000002 2023-08-28T02:35:14.398 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:14.398 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:35:14.398 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:14.398 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.398 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.399 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_repop(client.55270.0:176917 7.19 e309/291 7:9865963d:::benchmark_data_smithi143_360738_object84147:head v 309'7426, mlcod=279'4427)", 2023-08-28T02:35:14.399 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:34:46.528620+0000", 2023-08-28T02:35:14.399 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 27.743842761, 2023-08-28T02:35:14.399 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.080724378999999999, 2023-08-28T02:35:14.399 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:35:14.399 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:35:14.399 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:35:14.399 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.399 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:35:14.399 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:46.528620+0000", 2023-08-28T02:35:14.399 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:35:14.399 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.400 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.400 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:35:14.400 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:46.528620+0000", 2023-08-28T02:35:14.400 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:35:14.400 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.400 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.400 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:35:14.400 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:46.528619+0000", 2023-08-28T02:35:14.400 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.46e-06 2023-08-28T02:35:14.400 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.400 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.400 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:35:14.400 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:46.528625+0000", 2023-08-28T02:35:14.401 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.075289781 2023-08-28T02:35:14.401 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.401 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.401 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:35:14.401 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:46.603915+0000", 2023-08-28T02:35:14.401 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0147e-05 2023-08-28T02:35:14.401 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.401 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.401 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:35:14.401 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:46.603925+0000", 2023-08-28T02:35:14.401 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.002313519 2023-08-28T02:35:14.401 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.402 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.402 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:35:14.402 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:46.606239+0000", 2023-08-28T02:35:14.402 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.1715000000000001e-05 2023-08-28T02:35:14.402 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.402 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.402 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:35:14.402 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:46.606260+0000", 2023-08-28T02:35:14.402 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.003046436 2023-08-28T02:35:14.402 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.402 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.402 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:35:14.402 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:46.609307+0000", 2023-08-28T02:35:14.403 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.7224999999999997e-05 2023-08-28T02:35:14.403 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:14.403 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:14.403 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:35:14.403 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:34:46.609344+0000", 2023-08-28T02:35:14.403 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.080724378999999999 2023-08-28T02:35:14.403 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:14.403 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:35:14.403 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:14.403 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:14.403 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:35:14.403 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:35:14.497 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:14 smithi143 ceph-mon[183059]: pgmap v820: 73 pgs: 2 active+recovery_wait+degraded, 71 active+clean; 4.6 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 10/84 objects degraded (11.905%) 2023-08-28T02:35:14.497 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:14 smithi143 ceph-mon[183059]: failed to encode map e318 with expected crc 2023-08-28T02:35:14.497 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:14 smithi143 ceph-mon[183059]: failed to encode map e318 with expected crc 2023-08-28T02:35:14.497 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:14 smithi143 ceph-mon[183059]: failed to encode map e318 with expected crc 2023-08-28T02:35:14.497 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:14 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/59200847' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-08-28T02:35:14.497 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:14 smithi143 ceph-mon[183059]: osdmap e318: 8 total, 8 up, 7 in 2023-08-28T02:35:14.497 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:14 smithi143 ceph-mon[183059]: failed to encode map e318 with expected crc 2023-08-28T02:35:14.497 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:14 smithi143 ceph-mon[183059]: failed to encode map e318 with expected crc 2023-08-28T02:35:14.497 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:14 smithi143 ceph-mon[183059]: failed to encode map e318 with expected crc 2023-08-28T02:35:14.498 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:14 smithi143 ceph-mon[183059]: failed to encode map e318 with expected crc 2023-08-28T02:35:14.498 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:14 smithi143 ceph-mon[183059]: osdmap e319: 8 total, 8 up, 7 in 2023-08-28T02:35:14.542 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:35:14.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:14 smithi169 ceph-mon[159415]: pgmap v820: 73 pgs: 2 active+recovery_wait+degraded, 71 active+clean; 4.6 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 10/84 objects degraded (11.905%) 2023-08-28T02:35:14.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:14 smithi169 ceph-mon[159415]: failed to encode map e318 with expected crc 2023-08-28T02:35:14.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:14 smithi169 ceph-mon[159415]: failed to encode map e318 with expected crc 2023-08-28T02:35:14.728 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:14 smithi169 ceph-mon[159415]: failed to encode map e318 with expected crc 2023-08-28T02:35:14.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:14 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/59200847' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-08-28T02:35:14.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:14 smithi169 ceph-mon[159415]: osdmap e318: 8 total, 8 up, 7 in 2023-08-28T02:35:14.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:14 smithi169 ceph-mon[159415]: failed to encode map e318 with expected crc 2023-08-28T02:35:14.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:14 smithi169 ceph-mon[159415]: failed to encode map e318 with expected crc 2023-08-28T02:35:14.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:14 smithi169 ceph-mon[159415]: failed to encode map e318 with expected crc 2023-08-28T02:35:14.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:14 smithi169 ceph-mon[159415]: failed to encode map e318 with expected crc 2023-08-28T02:35:14.729 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:14 smithi169 ceph-mon[159415]: osdmap e319: 8 total, 8 up, 7 in 2023-08-28T02:35:14.870 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:35:14.872 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:14 smithi143 ceph-mon[189871]: pgmap v820: 73 pgs: 2 active+recovery_wait+degraded, 71 active+clean; 4.6 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 10/84 objects degraded (11.905%) 2023-08-28T02:35:14.872 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:14 smithi143 ceph-mon[189871]: failed to encode map e318 with expected crc 2023-08-28T02:35:14.872 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:14 smithi143 ceph-mon[189871]: failed to encode map e318 with expected crc 2023-08-28T02:35:14.872 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:14 smithi143 ceph-mon[189871]: failed to encode map e318 with expected crc 2023-08-28T02:35:14.872 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:14 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/59200847' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-08-28T02:35:14.872 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:14 smithi143 ceph-mon[189871]: osdmap e318: 8 total, 8 up, 7 in 2023-08-28T02:35:14.873 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:14 smithi143 ceph-mon[189871]: failed to encode map e318 with expected crc 2023-08-28T02:35:14.873 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:14 smithi143 ceph-mon[189871]: failed to encode map e318 with expected crc 2023-08-28T02:35:14.873 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:14 smithi143 ceph-mon[189871]: failed to encode map e318 with expected crc 2023-08-28T02:35:14.873 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:14 smithi143 ceph-mon[189871]: failed to encode map e318 with expected crc 2023-08-28T02:35:14.873 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:14 smithi143 ceph-mon[189871]: osdmap e319: 8 total, 8 up, 7 in 2023-08-28T02:35:14.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:14.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:35:14.974 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:35:15.004 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:15.272 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2023-08-28T02:35:15.406 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:35:15.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:15.409 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.5 dump_ops_in_flight 2023-08-28T02:35:15.410 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:35:15.512 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:35:15.541 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:15.592 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:35:15.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:15.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:35:15.628 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:35:15.699 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:35:15.729 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:15.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:15 smithi169 ceph-mon[159415]: failed to encode map e319 with expected crc 2023-08-28T02:35:15.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:15 smithi169 ceph-mon[159415]: failed to encode map e319 with expected crc 2023-08-28T02:35:15.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:15 smithi169 ceph-mon[159415]: failed to encode map e319 with expected crc 2023-08-28T02:35:15.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:15 smithi169 ceph-mon[159415]: failed to encode map e319 with expected crc 2023-08-28T02:35:15.828 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:35:15.868 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:15 smithi143 ceph-mon[183059]: failed to encode map e319 with expected crc 2023-08-28T02:35:15.868 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:15 smithi143 ceph-mon[183059]: failed to encode map e319 with expected crc 2023-08-28T02:35:15.868 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:15 smithi143 ceph-mon[183059]: failed to encode map e319 with expected crc 2023-08-28T02:35:15.868 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:15 smithi143 ceph-mon[183059]: failed to encode map e319 with expected crc 2023-08-28T02:35:15.869 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:15 smithi143 ceph-mon[189871]: failed to encode map e319 with expected crc 2023-08-28T02:35:15.869 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:15 smithi143 ceph-mon[189871]: failed to encode map e319 with expected crc 2023-08-28T02:35:15.869 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:15 smithi143 ceph-mon[189871]: failed to encode map e319 with expected crc 2023-08-28T02:35:15.869 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:15 smithi143 ceph-mon[189871]: failed to encode map e319 with expected crc 2023-08-28T02:35:15.886 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:35:15.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:15.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:35:15.990 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:35:16.018 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:16.126 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:35:16.172 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:35:16.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:16.178 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:35:16.279 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:35:16.307 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:16.556 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:35:16.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:16.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:35:16.622 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is set 2023-08-28T02:35:16.622 INFO:teuthology.orchestra.run.smithi143.stderr:reweighted osd.0 to 1 (10000) 2023-08-28T02:35:16.650 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:16 smithi143 ceph-mon[183059]: pgmap v823: 89 pgs: 8 unknown, 81 active+clean; 4.6 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 723 KiB/s, 1 objects/s recovering 2023-08-28T02:35:16.651 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:16 smithi143 ceph-mon[183059]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10/84 objects degraded (11.905%), 2 pgs degraded) 2023-08-28T02:35:16.651 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:16 smithi143 ceph-mon[183059]: failed to encode map e320 with expected crc 2023-08-28T02:35:16.651 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:16 smithi143 ceph-mon[183059]: osdmap e320: 8 total, 8 up, 7 in 2023-08-28T02:35:16.651 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:16 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:35:16.651 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:16 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:35:16.651 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:16 smithi143 ceph-mon[183059]: failed to encode map e320 with expected crc 2023-08-28T02:35:16.651 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:16 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/744294827' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-08-28T02:35:16.651 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:16 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-08-28T02:35:16.651 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:16 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1838843017' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:35:16.651 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:16 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:35:16.652 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:16 smithi143 ceph-mon[189871]: pgmap v823: 89 pgs: 8 unknown, 81 active+clean; 4.6 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 723 KiB/s, 1 objects/s recovering 2023-08-28T02:35:16.652 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:16 smithi143 ceph-mon[189871]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10/84 objects degraded (11.905%), 2 pgs degraded) 2023-08-28T02:35:16.652 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:16 smithi143 ceph-mon[189871]: failed to encode map e320 with expected crc 2023-08-28T02:35:16.652 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:16 smithi143 ceph-mon[189871]: osdmap e320: 8 total, 8 up, 7 in 2023-08-28T02:35:16.652 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:16 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:35:16.652 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:16 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:35:16.652 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:16 smithi143 ceph-mon[189871]: failed to encode map e320 with expected crc 2023-08-28T02:35:16.652 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:16 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/744294827' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-08-28T02:35:16.652 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:16 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-08-28T02:35:16.653 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:16 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1838843017' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:35:16.653 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:16 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:35:16.660 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:35:16.689 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:16.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:16 smithi169 ceph-mon[159415]: pgmap v823: 89 pgs: 8 unknown, 81 active+clean; 4.6 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 723 KiB/s, 1 objects/s recovering 2023-08-28T02:35:16.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:16 smithi169 ceph-mon[159415]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10/84 objects degraded (11.905%), 2 pgs degraded) 2023-08-28T02:35:16.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:16 smithi169 ceph-mon[159415]: failed to encode map e320 with expected crc 2023-08-28T02:35:16.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:16 smithi169 ceph-mon[159415]: osdmap e320: 8 total, 8 up, 7 in 2023-08-28T02:35:16.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:16 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:35:16.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:16 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:35:16.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:16 smithi169 ceph-mon[159415]: failed to encode map e320 with expected crc 2023-08-28T02:35:16.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:16 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/744294827' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-08-28T02:35:16.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:16 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-08-28T02:35:16.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:16 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1838843017' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:35:16.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:16 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:35:17.035 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:35:17.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:17.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:35:17.139 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:35:17.167 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:17.209 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:35:17.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:17.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:35:17.314 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:35:17.340 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:17.514 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:35:17.514 INFO:teuthology.orchestra.run.smithi169.stdout: "ops": [], 2023-08-28T02:35:17.514 INFO:teuthology.orchestra.run.smithi169.stdout: "num_ops": 0 2023-08-28T02:35:17.515 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:35:17.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:17 smithi143 ceph-mon[183059]: failed to encode map e320 with expected crc 2023-08-28T02:35:17.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:17 smithi143 ceph-mon[183059]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:35:17.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:17 smithi143 ceph-mon[183059]: failed to encode map e321 with expected crc 2023-08-28T02:35:17.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:17 smithi143 ceph-mon[183059]: failed to encode map e321 with expected crc 2023-08-28T02:35:17.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:17 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-08-28T02:35:17.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:17 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:35:17.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:17 smithi143 ceph-mon[183059]: osdmap e321: 8 total, 8 up, 7 in 2023-08-28T02:35:17.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:17 smithi143 ceph-mon[189871]: failed to encode map e320 with expected crc 2023-08-28T02:35:17.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:17 smithi143 ceph-mon[189871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:35:17.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:17 smithi143 ceph-mon[189871]: failed to encode map e321 with expected crc 2023-08-28T02:35:17.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:17 smithi143 ceph-mon[189871]: failed to encode map e321 with expected crc 2023-08-28T02:35:17.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:17 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-08-28T02:35:17.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:17 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:35:17.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:17 smithi143 ceph-mon[189871]: osdmap e321: 8 total, 8 up, 7 in 2023-08-28T02:35:17.984 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:35:17.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:17.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:35:17.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:17 smithi169 ceph-mon[159415]: failed to encode map e320 with expected crc 2023-08-28T02:35:17.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:17 smithi169 ceph-mon[159415]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:35:17.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:17 smithi169 ceph-mon[159415]: failed to encode map e321 with expected crc 2023-08-28T02:35:17.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:17 smithi169 ceph-mon[159415]: failed to encode map e321 with expected crc 2023-08-28T02:35:17.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:17 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-08-28T02:35:17.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:17 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:35:17.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:17 smithi169 ceph-mon[159415]: osdmap e321: 8 total, 8 up, 7 in 2023-08-28T02:35:18.088 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:35:18.113 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:18.277 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.5 dump_blocked_ops 2023-08-28T02:35:18.308 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:35:18.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:18.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:35:18.412 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:35:18.439 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:18.600 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:35:18.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:18.604 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:35:18.632 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:35:18.705 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:35:18.732 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:18.883 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:18 smithi169 ceph-mon[159415]: failed to encode map e321 with expected crc 2023-08-28T02:35:18.883 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:18 smithi169 ceph-mon[159415]: pgmap v826: 89 pgs: 4 unknown, 85 active+clean; 4.6 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 144 KiB/s, 2 objects/s recovering 2023-08-28T02:35:18.883 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:18 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:35:18.883 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:18 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:35:18.883 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:18 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:35:18.884 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:18 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:35:18.884 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:18 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:35:18.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:18 smithi143 ceph-mon[189871]: failed to encode map e321 with expected crc 2023-08-28T02:35:18.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:18 smithi143 ceph-mon[189871]: pgmap v826: 89 pgs: 4 unknown, 85 active+clean; 4.6 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 144 KiB/s, 2 objects/s recovering 2023-08-28T02:35:18.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:18 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:35:18.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:18 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:35:18.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:18 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:35:18.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:18 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:35:18.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:18 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:35:18.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:18 smithi143 ceph-mon[183059]: failed to encode map e321 with expected crc 2023-08-28T02:35:18.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:18 smithi143 ceph-mon[183059]: pgmap v826: 89 pgs: 4 unknown, 85 active+clean; 4.6 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 144 KiB/s, 2 objects/s recovering 2023-08-28T02:35:18.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:18 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:35:18.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:18 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:35:18.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:18 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:35:18.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:18 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:35:18.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:18 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:35:19.237 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:35:19.239 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd reweight 2 1 2023-08-28T02:35:19.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:19.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:35:19.342 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:35:19.370 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:19.400 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:35:19.419 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:35:19.438 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:35:19.460 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:35:19.479 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:35:19.498 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:35:19.520 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:35:19.521 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:35:19.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:19.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:35:19.543 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:35:19.544 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:35:19.544 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:35:19.544 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:35:19.544 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:35:19.544 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:35:19.545 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:35:19.545 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:35:19.545 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:35:19.625 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:35:19.656 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:19.708 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:35:19.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:19.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:35:19.788 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:19 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1474113051' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:35:19.788 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:19 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1474113051' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:35:19.811 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:35:19.838 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:19.881 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:35:19.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:19.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:35:19.984 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:35:19.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:19 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1474113051' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:35:20.018 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:20.117 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:35:20.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:20.119 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:35:20.220 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:35:20.240 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:35:20.240 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:35:20.248 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:20.291 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:35:20.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:20.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:35:20.368 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset noscrub 2023-08-28T02:35:20.395 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:35:20.432 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:35:20.438 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:20.485 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:35:20.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:20.489 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:35:20.590 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:35:20.617 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:20.632 INFO:teuthology.orchestra.run.smithi143.stderr:enabled application 'rados' on pool 'unique_pool_3' 2023-08-28T02:35:20.668 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:35:20.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:20.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:35:20.773 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:35:20.800 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:20.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:20 smithi143 ceph-mon[183059]: pgmap v827: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 96 KiB/s, 1 objects/s recovering 2023-08-28T02:35:20.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:20 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3035209652' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-28T02:35:20.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:20 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-28T02:35:20.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:20 smithi143 ceph-mon[189871]: pgmap v827: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 96 KiB/s, 1 objects/s recovering 2023-08-28T02:35:20.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:20 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3035209652' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-28T02:35:20.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:20 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-28T02:35:20.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:20 smithi169 ceph-mon[159415]: pgmap v827: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 96 KiB/s, 1 objects/s recovering 2023-08-28T02:35:20.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:20 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3035209652' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-28T02:35:20.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:20 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-28T02:35:21.012 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:35:21.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:21.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:35:21.075 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.5 dump_historic_ops 2023-08-28T02:35:21.115 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:35:21.145 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:21.187 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:35:21.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:21.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:35:21.294 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:35:21.321 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:21.415 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:35:21.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:21.418 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:35:21.431 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:35:21.518 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:35:21.544 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:21.638 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:35:21.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:21.641 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:35:21.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:21 smithi143 ceph-mon[183059]: failed to encode map e322 with expected crc 2023-08-28T02:35:21.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:21 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-08-28T02:35:21.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:21 smithi143 ceph-mon[183059]: osdmap e322: 8 total, 8 up, 7 in 2023-08-28T02:35:21.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:21 smithi143 ceph-mon[183059]: failed to encode map e322 with expected crc 2023-08-28T02:35:21.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:21 smithi143 ceph-mon[183059]: failed to encode map e322 with expected crc 2023-08-28T02:35:21.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:21 smithi143 ceph-mon[183059]: failed to encode map e322 with expected crc 2023-08-28T02:35:21.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:21 smithi143 ceph-mon[189871]: failed to encode map e322 with expected crc 2023-08-28T02:35:21.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:21 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-08-28T02:35:21.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:21 smithi143 ceph-mon[189871]: osdmap e322: 8 total, 8 up, 7 in 2023-08-28T02:35:21.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:21 smithi143 ceph-mon[189871]: failed to encode map e322 with expected crc 2023-08-28T02:35:21.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:21 smithi143 ceph-mon[189871]: failed to encode map e322 with expected crc 2023-08-28T02:35:21.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:21 smithi143 ceph-mon[189871]: failed to encode map e322 with expected crc 2023-08-28T02:35:21.742 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:35:21.769 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:21.914 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:35:21.916 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:21 smithi169 ceph-mon[159415]: failed to encode map e322 with expected crc 2023-08-28T02:35:21.916 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:21 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-08-28T02:35:21.916 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:21 smithi169 ceph-mon[159415]: osdmap e322: 8 total, 8 up, 7 in 2023-08-28T02:35:21.916 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:21 smithi169 ceph-mon[159415]: failed to encode map e322 with expected crc 2023-08-28T02:35:21.916 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:21 smithi169 ceph-mon[159415]: failed to encode map e322 with expected crc 2023-08-28T02:35:21.916 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:21 smithi169 ceph-mon[159415]: failed to encode map e322 with expected crc 2023-08-28T02:35:21.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:21.917 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:35:21.981 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:35:22.018 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:35:22.047 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:22.108 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:35:22.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:22.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:35:22.188 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:35:21 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:35:21] "GET /metrics HTTP/1.1" 200 34214 "" "Prometheus/2.33.4" 2023-08-28T02:35:22.211 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:35:22.239 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:22.336 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:35:22.386 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:35:22.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:22.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:35:22.489 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:35:22.518 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:22.673 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:35:22.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:22.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:35:22.777 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:35:22.802 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:22 smithi169 ceph-mon[159415]: pgmap v829: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-08-28T02:35:22.816 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:22.897 DEBUG:teuthology.orchestra.run.smithi143:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2023-08-28T02:35:22.899 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-08-28T02:35:22.899 INFO:teuthology.orchestra.run:waiting for 3000 2023-08-28T02:35:22.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:22 smithi143 ceph-mon[183059]: pgmap v829: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-08-28T02:35:22.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:22 smithi143 ceph-mon[189871]: pgmap v829: 89 pgs: 89 active+clean; 4.6 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail 2023-08-28T02:35:22.961 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:35:22.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:22.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:35:22.997 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:35:22.997 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:35:23.066 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:35:23.092 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:23.165 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:35:23.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:23.168 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:35:23.172 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:hints = 1 2023-08-28T02:35:23.172 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-08-28T02:35:23.172 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Object prefix: benchmark_data_smithi143_419289 2023-08-28T02:35:23.172 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-08-28T02:35:23.173 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 0 0 0 0 0 0 - 0 2023-08-28T02:35:23.269 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:35:23.321 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:23.351 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:35:23.423 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:35:23.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:23.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:35:23.526 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:35:23.563 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:23.685 INFO:teuthology.orchestra.run.smithi143.stderr:reweighted osd.2 to 1 (10000) 2023-08-28T02:35:23.687 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:23 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1714632147' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-08-28T02:35:23.687 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:23 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-08-28T02:35:23.687 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:23 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1714632147' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-08-28T02:35:23.687 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:23 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-08-28T02:35:23.815 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:23 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1714632147' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-08-28T02:35:23.815 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:23 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-08-28T02:35:23.833 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.6 dump_ops_in_flight 2023-08-28T02:35:23.835 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:35:23.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:23.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:35:23.940 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:35:23.970 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:24.173 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 1 16 1171 1155 72.1817 72.1875 0.00125059 0.0073492 2023-08-28T02:35:24.181 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:35:24.675 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is unset 2023-08-28T02:35:24.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:24 smithi169 ceph-mon[159415]: pgmap v830: 89 pgs: 89 active+clean; 8.0 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 441 KiB/s wr, 6 op/s 2023-08-28T02:35:24.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:24 smithi169 ceph-mon[159415]: failed to encode map e323 with expected crc 2023-08-28T02:35:24.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:24 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-08-28T02:35:24.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:24 smithi169 ceph-mon[159415]: osdmap e323: 8 total, 8 up, 7 in 2023-08-28T02:35:24.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:24 smithi169 ceph-mon[159415]: failed to encode map e323 with expected crc 2023-08-28T02:35:24.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:24 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1248219115' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:35:24.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:24 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:35:24.897 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:24 smithi143 ceph-mon[189871]: pgmap v830: 89 pgs: 89 active+clean; 8.0 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 441 KiB/s wr, 6 op/s 2023-08-28T02:35:24.898 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:24 smithi143 ceph-mon[189871]: failed to encode map e323 with expected crc 2023-08-28T02:35:24.898 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:24 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-08-28T02:35:24.898 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:24 smithi143 ceph-mon[189871]: osdmap e323: 8 total, 8 up, 7 in 2023-08-28T02:35:24.898 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:24 smithi143 ceph-mon[189871]: failed to encode map e323 with expected crc 2023-08-28T02:35:24.898 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:24 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1248219115' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:35:24.898 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:24 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:35:24.898 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:24 smithi143 ceph-mon[183059]: pgmap v830: 89 pgs: 89 active+clean; 8.0 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 441 KiB/s wr, 6 op/s 2023-08-28T02:35:24.898 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:24 smithi143 ceph-mon[183059]: failed to encode map e323 with expected crc 2023-08-28T02:35:24.898 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:24 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-08-28T02:35:24.899 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:24 smithi143 ceph-mon[183059]: osdmap e323: 8 total, 8 up, 7 in 2023-08-28T02:35:24.899 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:24 smithi143 ceph-mon[183059]: failed to encode map e323 with expected crc 2023-08-28T02:35:24.899 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:24 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1248219115' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:35:24.899 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:24 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:35:25.173 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 2 16 2212 2196 68.6182 65.0625 0.00107334 0.0133677 2023-08-28T02:35:25.536 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:35:25.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:25.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:35:25.643 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:35:25.710 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:25.908 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:25 smithi169 ceph-mon[159415]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:35:25.908 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:25 smithi169 ceph-mon[159415]: failed to encode map e324 with expected crc 2023-08-28T02:35:25.908 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:25 smithi169 ceph-mon[159415]: failed to encode map e324 with expected crc 2023-08-28T02:35:25.908 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:25 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:35:25.908 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:25 smithi169 ceph-mon[159415]: osdmap e324: 8 total, 8 up, 7 in 2023-08-28T02:35:25.909 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:25 smithi169 ceph-mon[159415]: failed to encode map e324 with expected crc 2023-08-28T02:35:25.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:25 smithi143 ceph-mon[183059]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:35:25.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:25 smithi143 ceph-mon[183059]: failed to encode map e324 with expected crc 2023-08-28T02:35:25.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:25 smithi143 ceph-mon[183059]: failed to encode map e324 with expected crc 2023-08-28T02:35:25.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:25 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:35:25.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:25 smithi143 ceph-mon[183059]: osdmap e324: 8 total, 8 up, 7 in 2023-08-28T02:35:25.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:25 smithi143 ceph-mon[183059]: failed to encode map e324 with expected crc 2023-08-28T02:35:25.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:25 smithi143 ceph-mon[189871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:35:25.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:25 smithi143 ceph-mon[189871]: failed to encode map e324 with expected crc 2023-08-28T02:35:25.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:25 smithi143 ceph-mon[189871]: failed to encode map e324 with expected crc 2023-08-28T02:35:25.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:25 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:35:25.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:25 smithi143 ceph-mon[189871]: osdmap e324: 8 total, 8 up, 7 in 2023-08-28T02:35:25.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:25 smithi143 ceph-mon[189871]: failed to encode map e324 with expected crc 2023-08-28T02:35:26.173 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 3 16 4422 4406 91.7825 138.125 0.00521815 0.0108806 2023-08-28T02:35:26.179 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:35:26.179 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:35:26.544 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:35:26.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:26.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:35:26.654 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:35:26.675 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd reweight 4 1 2023-08-28T02:35:26.687 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:26.768 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:35:26.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:26.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:35:26.851 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:26 smithi143 ceph-mon[189871]: failed to encode map e324 with expected crc 2023-08-28T02:35:26.851 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:26 smithi143 ceph-mon[189871]: pgmap v833: 89 pgs: 1 active+recovering, 1 peering, 87 active+clean; 57 MiB data, 3.3 GiB used, 622 GiB / 626 GiB avail; 8.7 MiB/s wr, 138 op/s; 25/2583 objects misplaced (0.968%); 512 KiB/s, 7 objects/s recovering 2023-08-28T02:35:26.851 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:26 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:35:26.851 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:26 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:35:26.851 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:26 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:35:26.852 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:26 smithi143 ceph-mon[183059]: failed to encode map e324 with expected crc 2023-08-28T02:35:26.852 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:26 smithi143 ceph-mon[183059]: pgmap v833: 89 pgs: 1 active+recovering, 1 peering, 87 active+clean; 57 MiB data, 3.3 GiB used, 622 GiB / 626 GiB avail; 8.7 MiB/s wr, 138 op/s; 25/2583 objects misplaced (0.968%); 512 KiB/s, 7 objects/s recovering 2023-08-28T02:35:26.852 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:26 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:35:26.852 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:26 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:35:26.852 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:26 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:35:26.872 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:35:26.914 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:26.973 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:35:26.974 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:26 smithi169 ceph-mon[159415]: failed to encode map e324 with expected crc 2023-08-28T02:35:26.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:26 smithi169 ceph-mon[159415]: pgmap v833: 89 pgs: 1 active+recovering, 1 peering, 87 active+clean; 57 MiB data, 3.3 GiB used, 622 GiB / 626 GiB avail; 8.7 MiB/s wr, 138 op/s; 25/2583 objects misplaced (0.968%); 512 KiB/s, 7 objects/s recovering 2023-08-28T02:35:26.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:26 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:35:26.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:26 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:35:26.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:26 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:35:26.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:26.976 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:35:27.077 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:35:27.110 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:27.173 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 4 16 6192 6176 96.4869 110.625 0.00871856 0.0103517 2023-08-28T02:35:27.214 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:35:27.239 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:35:27.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:27.242 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:35:27.258 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:35:27.301 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:35:27.302 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:35:27.312 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.6 dump_blocked_ops 2023-08-28T02:35:27.341 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:35:27.342 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:35:27.370 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:35:27.394 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:27.407 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:35:27.438 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:35:27.470 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:35:27.473 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:35:27.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:27.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:35:27.485 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:35:27.485 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:35:27.485 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:35:27.485 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:35:27.485 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:35:27.485 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:35:27.485 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:35:27.485 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:35:27.576 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:35:27.634 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:27.809 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:35:27.813 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:27 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/487302891' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:35:27.813 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:27 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/487302891' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:35:27.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:27 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/487302891' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:35:28.173 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 5 16 8782 8766 109.56 161.875 0.00153986 0.0090504 2023-08-28T02:35:28.313 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:35:28.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:28.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:35:28.419 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:35:28.465 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:28.531 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:35:28.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:28.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:35:28.624 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset nodeep-scrub 2023-08-28T02:35:28.634 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:35:28.715 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:28.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:28 smithi169 ceph-mon[159415]: pgmap v834: 89 pgs: 1 active+recovering, 1 peering, 87 active+clean; 150 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 21 MiB/s wr, 339 op/s; 25/7077 objects misplaced (0.353%); 448 KiB/s, 6 objects/s recovering 2023-08-28T02:35:28.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:28 smithi143 ceph-mon[189871]: pgmap v834: 89 pgs: 1 active+recovering, 1 peering, 87 active+clean; 150 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 21 MiB/s wr, 339 op/s; 25/7077 objects misplaced (0.353%); 448 KiB/s, 6 objects/s recovering 2023-08-28T02:35:28.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:28 smithi143 ceph-mon[183059]: pgmap v834: 89 pgs: 1 active+recovering, 1 peering, 87 active+clean; 150 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 21 MiB/s wr, 339 op/s; 25/7077 objects misplaced (0.353%); 448 KiB/s, 6 objects/s recovering 2023-08-28T02:35:29.173 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 6 16 10763 10747 111.934 123.812 0.00500011 0.00892274 2023-08-28T02:35:30.105 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:35:30.105 INFO:teuthology.orchestra.run.smithi169.stdout: "ops": [], 2023-08-28T02:35:30.105 INFO:teuthology.orchestra.run.smithi169.stdout: "complaint_time": 30, 2023-08-28T02:35:30.105 INFO:teuthology.orchestra.run.smithi169.stdout: "num_blocked_ops": 0 2023-08-28T02:35:30.105 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:35:30.173 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 7 16 13372 13356 119.235 163.062 0.00715069 0.00837967 2023-08-28T02:35:30.291 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:35:30.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:30.298 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:35:30.399 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:35:30.432 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:30.475 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:35:30.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:30.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:35:30.578 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:35:30.606 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:30.672 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:35:30.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:30.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:35:30.775 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:35:30.810 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:30.853 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:35:30.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:30.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:35:30.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:30 smithi143 ceph-mon[183059]: pgmap v835: 89 pgs: 1 active+recovering, 1 peering, 87 active+clean; 286 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 35 MiB/s wr, 562 op/s; 25/13587 objects misplaced (0.184%); 384 KiB/s, 5 objects/s recovering 2023-08-28T02:35:30.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:30 smithi143 ceph-mon[189871]: pgmap v835: 89 pgs: 1 active+recovering, 1 peering, 87 active+clean; 286 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 35 MiB/s wr, 562 op/s; 25/13587 objects misplaced (0.184%); 384 KiB/s, 5 objects/s recovering 2023-08-28T02:35:30.957 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:35:30.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:30 smithi169 ceph-mon[159415]: pgmap v835: 89 pgs: 1 active+recovering, 1 peering, 87 active+clean; 286 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 35 MiB/s wr, 562 op/s; 25/13587 objects misplaced (0.184%); 384 KiB/s, 5 objects/s recovering 2023-08-28T02:35:30.995 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:31.055 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:35:31.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:31.058 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:35:31.159 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:35:31.173 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 8 16 15104 15088 117.861 108.25 0.0058845 0.00847685 2023-08-28T02:35:31.191 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:31.210 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:35:31.382 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:35:31.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:31.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:35:31.398 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.6 dump_historic_ops 2023-08-28T02:35:31.487 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:35:31.535 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:31.604 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:35:31.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:31.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:35:31.712 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:35:31.753 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:31.825 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:35:31.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:31.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:35:31.933 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:35:31.947 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:35:31.993 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:35:31 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:35:31] "GET /metrics HTTP/1.1" 200 36535 "" "Prometheus/2.33.4" 2023-08-28T02:35:32.003 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:32.160 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:35:32.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:32.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:35:32.173 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 9 16 17109 17093 118.686 125.312 0.006559 0.00841929 2023-08-28T02:35:32.265 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:35:32.311 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:32.435 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:35:32.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:32.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:35:32.542 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:35:32.604 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:32.713 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:35:32.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:32.722 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:35:32.738 INFO:teuthology.orchestra.run.smithi143.stderr:reweighted osd.4 to 1 (10000) 2023-08-28T02:35:32.824 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:35:32.862 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:32.911 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:35:32.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:32.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:35:32.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:32 smithi169 ceph-mon[159415]: pgmap v836: 89 pgs: 89 active+clean; 621 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s; 584 KiB/s, 9 objects/s recovering 2023-08-28T02:35:32.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:32 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/641822817' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-08-28T02:35:33.014 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:35:33.042 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:33.074 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:35:33.085 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:35:33.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:33.088 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:35:33.099 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:32 smithi143 ceph-mon[183059]: pgmap v836: 89 pgs: 89 active+clean; 621 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s; 584 KiB/s, 9 objects/s recovering 2023-08-28T02:35:33.100 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:32 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/641822817' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-08-28T02:35:33.100 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:32 smithi143 ceph-mon[189871]: pgmap v836: 89 pgs: 89 active+clean; 621 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s; 584 KiB/s, 9 objects/s recovering 2023-08-28T02:35:33.100 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:32 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/641822817' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-08-28T02:35:33.174 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 10 16 18880 18864 117.886 110.688 0.0016281 0.00847085 2023-08-28T02:35:33.188 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:35:33.214 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:33.257 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:35:33.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:33.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:35:33.361 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:35:33.390 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:33.788 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:33 smithi169 ceph-mon[159415]: failed to encode map e325 with expected crc 2023-08-28T02:35:33.788 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:33 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/641822817' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-08-28T02:35:33.788 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:33 smithi169 ceph-mon[159415]: osdmap e325: 8 total, 8 up, 7 in 2023-08-28T02:35:33.789 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:33 smithi169 ceph-mon[159415]: failed to encode map e325 with expected crc 2023-08-28T02:35:33.789 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:33 smithi169 ceph-mon[159415]: pgmap v838: 89 pgs: 89 active+clean; 949 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 105 MiB/s wr, 1.68k op/s; 529 KiB/s, 8 objects/s recovering 2023-08-28T02:35:33.791 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:35:33.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:33.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:35:33.899 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:35:33.956 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:34.030 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:35:34.030 INFO:teuthology.orchestra.run.smithi169.stdout: "size": 20, 2023-08-28T02:35:34.031 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 600, 2023-08-28T02:35:34.031 INFO:teuthology.orchestra.run.smithi169.stdout: "ops": [ 2023-08-28T02:35:34.031 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.031 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:78148 6.a 6.cb50d08a (undecoded) ondisk+retry+write+known_if_redirected e220)", 2023-08-28T02:35:34.031 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:28:24.333420+0000", 2023-08-28T02:35:34.031 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 429.69552449700001, 2023-08-28T02:35:34.031 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0005001710000001, 2023-08-28T02:35:34.032 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:35:34.032 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "reached pg", 2023-08-28T02:35:34.032 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:35:34.032 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:35:34.032 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:35:34.032 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 78148 2023-08-28T02:35:34.032 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.032 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:35:34.033 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.033 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:35:34.033 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.333420+0000", 2023-08-28T02:35:34.033 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:35:34.033 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.033 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.033 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:35:34.033 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.333420+0000", 2023-08-28T02:35:34.034 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:35:34.034 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.034 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.034 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:35:34.034 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.333418+0000", 2023-08-28T02:35:34.034 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00024257 2023-08-28T02:35:34.034 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.034 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.035 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:35:34.035 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.333661+0000", 2023-08-28T02:35:34.035 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2506e-05 2023-08-28T02:35:34.035 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.035 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.035 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:35:34.036 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.333673+0000", 2023-08-28T02:35:34.036 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.4730000000000003e-06 2023-08-28T02:35:34.036 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.036 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.036 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:35:34.036 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.333681+0000", 2023-08-28T02:35:34.036 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.0689999999999997e-05 2023-08-28T02:35:34.037 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.037 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.037 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:35:34.037 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.333732+0000", 2023-08-28T02:35:34.037 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.7439999999999996e-06 2023-08-28T02:35:34.037 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.037 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.038 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:35:34.038 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.333740+0000", 2023-08-28T02:35:34.038 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0001706100000001 2023-08-28T02:35:34.038 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.038 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.038 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:35:34.038 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:25.333911+0000", 2023-08-28T02:35:34.039 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.1880000000000008e-06 2023-08-28T02:35:34.039 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.039 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.039 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:35:34.039 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:25.333920+0000", 2023-08-28T02:35:34.039 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0005001710000001 2023-08-28T02:35:34.039 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:34.040 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:35:34.040 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:34.040 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.040 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.040 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:78161 6.a 6.3abad5fa (undecoded) ondisk+write+known_if_redirected e220)", 2023-08-28T02:35:34.040 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:28:24.353303+0000", 2023-08-28T02:35:34.041 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 429.67564109400001, 2023-08-28T02:35:34.041 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.98063798199999996, 2023-08-28T02:35:34.041 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:35:34.041 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "reached pg", 2023-08-28T02:35:34.041 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:35:34.041 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:35:34.041 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:35:34.042 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 78161 2023-08-28T02:35:34.042 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.042 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:35:34.042 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.042 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:35:34.042 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.353303+0000", 2023-08-28T02:35:34.042 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:35:34.043 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.043 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.043 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:35:34.043 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.353303+0000", 2023-08-28T02:35:34.043 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:35:34.043 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.043 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.044 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:35:34.044 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.353302+0000", 2023-08-28T02:35:34.044 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.9099e-05 2023-08-28T02:35:34.044 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.044 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.044 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:35:34.044 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.353331+0000", 2023-08-28T02:35:34.045 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.3447e-05 2023-08-28T02:35:34.045 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.045 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.045 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:35:34.045 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.353345+0000", 2023-08-28T02:35:34.045 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.5980000000000002e-06 2023-08-28T02:35:34.045 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.046 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.046 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:35:34.046 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.353351+0000", 2023-08-28T02:35:34.046 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00030897800000000001 2023-08-28T02:35:34.046 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.046 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.046 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:35:34.047 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.353660+0000", 2023-08-28T02:35:34.047 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.3219999999999999e-06 2023-08-28T02:35:34.047 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.047 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.047 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:35:34.047 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.353670+0000", 2023-08-28T02:35:34.048 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.98026650500000001 2023-08-28T02:35:34.048 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.048 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.048 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:35:34.048 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:25.333936+0000", 2023-08-28T02:35:34.048 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.2039999999999996e-06 2023-08-28T02:35:34.049 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.049 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.049 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:35:34.049 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:25.333941+0000", 2023-08-28T02:35:34.049 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.98063798199999996 2023-08-28T02:35:34.049 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:34.050 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:35:34.050 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:34.050 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.050 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.050 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:78184 6.a 6.225625ba (undecoded) ondisk+write+known_if_redirected e220)", 2023-08-28T02:35:34.050 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:28:24.405003+0000", 2023-08-28T02:35:34.050 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 429.62394097399999, 2023-08-28T02:35:34.051 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.92895428300000005, 2023-08-28T02:35:34.051 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:35:34.051 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "reached pg", 2023-08-28T02:35:34.051 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:35:34.051 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:35:34.051 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:35:34.051 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 78184 2023-08-28T02:35:34.052 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.052 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:35:34.052 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.052 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:35:34.052 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.405003+0000", 2023-08-28T02:35:34.052 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:35:34.052 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.053 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.053 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:35:34.053 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.405003+0000", 2023-08-28T02:35:34.053 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999976 2023-08-28T02:35:34.053 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.053 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.053 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:35:34.054 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.405001+0000", 2023-08-28T02:35:34.054 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.2438999999999999e-05 2023-08-28T02:35:34.054 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.054 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.054 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:35:34.054 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.405033+0000", 2023-08-28T02:35:34.054 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.1441e-05 2023-08-28T02:35:34.055 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.055 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.055 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:35:34.055 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.405055+0000", 2023-08-28T02:35:34.055 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.2890000000000003e-06 2023-08-28T02:35:34.055 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.055 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.056 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:35:34.056 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.405064+0000", 2023-08-28T02:35:34.056 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.5662999999999998e-05 2023-08-28T02:35:34.056 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.056 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.056 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:35:34.056 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.405130+0000", 2023-08-28T02:35:34.057 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1328999999999999e-05 2023-08-28T02:35:34.057 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.057 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.057 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:35:34.057 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:24.405141+0000", 2023-08-28T02:35:34.057 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.92881176600000004 2023-08-28T02:35:34.057 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.058 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.058 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:35:34.058 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:25.333953+0000", 2023-08-28T02:35:34.058 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.9389999999999996e-06 2023-08-28T02:35:34.058 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.058 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.058 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:35:34.059 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:25.333958+0000", 2023-08-28T02:35:34.059 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.92895428300000005 2023-08-28T02:35:34.059 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:34.059 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:35:34.059 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:34.059 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.059 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.060 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:59156 7.7 7:e651d29d:::benchmark_data_smithi143_360738_object59155:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e268)", 2023-08-28T02:35:34.060 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:31:42.218748+0000", 2023-08-28T02:35:34.060 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 231.81019591800001, 2023-08-28T02:35:34.062 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.30541675200000001, 2023-08-28T02:35:34.062 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:35:34.062 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:35:34.062 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:35:34.062 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:35:34.062 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:35:34.062 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 59156 2023-08-28T02:35:34.063 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.063 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:35:34.063 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.063 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:35:34.063 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.218748+0000", 2023-08-28T02:35:34.063 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:35:34.063 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.064 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.064 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:35:34.064 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.218748+0000", 2023-08-28T02:35:34.064 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:35:34.064 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.064 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.064 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:35:34.065 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.218747+0000", 2023-08-28T02:35:34.065 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.2304000000000003e-05 2023-08-28T02:35:34.065 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.065 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.065 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:35:34.065 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.218840+0000", 2023-08-28T02:35:34.065 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.1790000000000008e-06 2023-08-28T02:35:34.066 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.066 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.066 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:35:34.066 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.218848+0000", 2023-08-28T02:35:34.066 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.1410000000000005e-06 2023-08-28T02:35:34.066 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.066 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.067 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:35:34.067 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.218856+0000", 2023-08-28T02:35:34.067 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.7073000000000001e-05 2023-08-28T02:35:34.067 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.067 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.067 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:35:34.068 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.218903+0000", 2023-08-28T02:35:34.068 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000116227 2023-08-28T02:35:34.068 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.068 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.068 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:35:34.068 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.219019+0000", 2023-08-28T02:35:34.068 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00016372199999999999 2023-08-28T02:35:34.069 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.069 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.069 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,3", 2023-08-28T02:35:34.069 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.219183+0000", 2023-08-28T02:35:34.069 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0017889679999999999 2023-08-28T02:35:34.069 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.070 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.070 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:35:34.070 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.220972+0000", 2023-08-28T02:35:34.070 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.025971195999999998 2023-08-28T02:35:34.070 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.070 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.070 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:34.071 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.246943+0000", 2023-08-28T02:35:34.071 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.27711988100000001 2023-08-28T02:35:34.071 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.071 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.071 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:34.071 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.524063+0000", 2023-08-28T02:35:34.071 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.3491000000000001e-05 2023-08-28T02:35:34.071 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.072 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.072 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:35:34.072 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.524107+0000", 2023-08-28T02:35:34.072 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.8653999999999998e-05 2023-08-28T02:35:34.072 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.072 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.072 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:35:34.073 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.524165+0000", 2023-08-28T02:35:34.073 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.30541675200000001 2023-08-28T02:35:34.073 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:34.073 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:35:34.073 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:34.073 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.073 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.074 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:59160 7.7 7:efb4b659:::benchmark_data_smithi143_360738_object59159:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e268)", 2023-08-28T02:35:34.074 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:31:42.221041+0000", 2023-08-28T02:35:34.074 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 231.807903702, 2023-08-28T02:35:34.074 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.30908384300000002, 2023-08-28T02:35:34.074 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:35:34.074 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:35:34.074 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:35:34.075 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:35:34.075 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:35:34.075 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 59160 2023-08-28T02:35:34.075 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.075 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:35:34.075 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.075 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:35:34.076 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.221041+0000", 2023-08-28T02:35:34.076 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:35:34.076 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.076 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.076 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:35:34.076 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.221041+0000", 2023-08-28T02:35:34.076 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999995 2023-08-28T02:35:34.077 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.077 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.077 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:35:34.077 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.221040+0000", 2023-08-28T02:35:34.077 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.7115e-05 2023-08-28T02:35:34.077 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.077 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.078 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:35:34.078 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.221077+0000", 2023-08-28T02:35:34.078 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.04e-06 2023-08-28T02:35:34.078 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.078 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.078 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:35:34.078 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.221082+0000", 2023-08-28T02:35:34.079 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.1319999999999998e-06 2023-08-28T02:35:34.079 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.079 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.079 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:35:34.079 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.221086+0000", 2023-08-28T02:35:34.079 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.5927000000000003e-05 2023-08-28T02:35:34.079 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.080 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.080 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:35:34.080 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.221131+0000", 2023-08-28T02:35:34.080 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012290200000000001 2023-08-28T02:35:34.080 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.080 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.081 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:35:34.081 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.221254+0000", 2023-08-28T02:35:34.081 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000191273 2023-08-28T02:35:34.081 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.081 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.081 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,3", 2023-08-28T02:35:34.081 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.221446+0000", 2023-08-28T02:35:34.082 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00321025 2023-08-28T02:35:34.082 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.082 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.082 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:35:34.082 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.224656+0000", 2023-08-28T02:35:34.082 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.022316244999999998 2023-08-28T02:35:34.082 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.083 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.083 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:34.083 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.246972+0000", 2023-08-28T02:35:34.083 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.283081268 2023-08-28T02:35:34.083 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.083 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.083 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:34.084 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.530053+0000", 2023-08-28T02:35:34.084 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.4394999999999998e-05 2023-08-28T02:35:34.084 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.084 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.084 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:35:34.084 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.530088+0000", 2023-08-28T02:35:34.084 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.6699999999999998e-05 2023-08-28T02:35:34.084 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.085 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.085 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:35:34.085 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.530125+0000", 2023-08-28T02:35:34.085 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.30908384300000002 2023-08-28T02:35:34.085 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:34.085 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:35:34.085 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:34.086 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.086 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.086 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:59162 7.7 7:eedf689e:::benchmark_data_smithi143_360738_object59161:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e268)", 2023-08-28T02:35:34.086 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:31:42.221096+0000", 2023-08-28T02:35:34.086 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 231.80784884600001, 2023-08-28T02:35:34.086 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.311517979, 2023-08-28T02:35:34.086 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:35:34.087 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:35:34.087 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:35:34.087 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:35:34.087 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:35:34.087 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 59162 2023-08-28T02:35:34.087 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.087 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:35:34.088 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.088 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:35:34.088 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.221096+0000", 2023-08-28T02:35:34.088 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:35:34.088 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.088 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.088 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:35:34.088 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.221096+0000", 2023-08-28T02:35:34.089 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999995 2023-08-28T02:35:34.089 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.089 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.089 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:35:34.089 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.221095+0000", 2023-08-28T02:35:34.089 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.6899000000000001e-05 2023-08-28T02:35:34.090 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.090 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.090 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:35:34.090 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.221122+0000", 2023-08-28T02:35:34.090 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.1060000000000002e-06 2023-08-28T02:35:34.090 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.090 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.091 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:35:34.091 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.221126+0000", 2023-08-28T02:35:34.091 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.6129999999999998e-06 2023-08-28T02:35:34.091 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.091 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.091 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:35:34.091 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.221129+0000", 2023-08-28T02:35:34.092 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.001819312 2023-08-28T02:35:34.092 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.092 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.092 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:35:34.092 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.222948+0000", 2023-08-28T02:35:34.092 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00014037199999999999 2023-08-28T02:35:34.092 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.093 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.093 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:35:34.093 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.223088+0000", 2023-08-28T02:35:34.093 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00016837199999999999 2023-08-28T02:35:34.093 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.093 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.093 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,3", 2023-08-28T02:35:34.093 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.223257+0000", 2023-08-28T02:35:34.094 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0015392699999999999 2023-08-28T02:35:34.094 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.094 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.094 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:35:34.094 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.224796+0000", 2023-08-28T02:35:34.094 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.022328396 2023-08-28T02:35:34.095 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.095 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.095 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:34.095 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.247124+0000", 2023-08-28T02:35:34.095 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.28539166999999999 2023-08-28T02:35:34.095 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.095 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.095 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:34.096 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.532516+0000", 2023-08-28T02:35:34.096 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.3111e-05 2023-08-28T02:35:34.096 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.096 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.096 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:35:34.096 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.532569+0000", 2023-08-28T02:35:34.096 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.4298999999999997e-05 2023-08-28T02:35:34.096 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.096 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.096 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:35:34.096 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.532614+0000", 2023-08-28T02:35:34.096 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.311517979 2023-08-28T02:35:34.096 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:34.097 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:35:34.097 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:34.097 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.097 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.097 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:59172 7.c 7:348f4251:::benchmark_data_smithi143_360738_object59171:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e268)", 2023-08-28T02:35:34.097 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:31:42.244405+0000", 2023-08-28T02:35:34.097 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 231.784539585, 2023-08-28T02:35:34.097 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.28357411100000002, 2023-08-28T02:35:34.097 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:35:34.097 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:35:34.097 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:35:34.098 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:35:34.098 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:35:34.098 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 59172 2023-08-28T02:35:34.098 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.098 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:35:34.098 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.098 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:35:34.098 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.244405+0000", 2023-08-28T02:35:34.098 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:35:34.098 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.098 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.098 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:35:34.098 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.244405+0000", 2023-08-28T02:35:34.099 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:35:34.099 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.099 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.099 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:35:34.099 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.244404+0000", 2023-08-28T02:35:34.099 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.3516000000000006e-05 2023-08-28T02:35:34.099 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.099 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.099 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:35:34.099 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.244468+0000", 2023-08-28T02:35:34.099 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1113e-05 2023-08-28T02:35:34.099 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.099 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.100 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:35:34.100 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.244479+0000", 2023-08-28T02:35:34.100 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.9339999999999997e-06 2023-08-28T02:35:34.100 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.100 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.100 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:35:34.100 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.244484+0000", 2023-08-28T02:35:34.100 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.7425000000000001e-05 2023-08-28T02:35:34.100 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.100 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.100 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:35:34.100 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.244531+0000", 2023-08-28T02:35:34.100 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00013291299999999999 2023-08-28T02:35:34.101 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.101 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.101 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:35:34.101 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.244664+0000", 2023-08-28T02:35:34.101 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00016822899999999999 2023-08-28T02:35:34.101 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.101 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.101 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 3,7", 2023-08-28T02:35:34.101 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.244832+0000", 2023-08-28T02:35:34.101 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0024013200000000002 2023-08-28T02:35:34.101 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.101 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.101 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:35:34.101 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.247233+0000", 2023-08-28T02:35:34.102 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.001143853 2023-08-28T02:35:34.102 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.102 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.102 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:34.102 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.248377+0000", 2023-08-28T02:35:34.102 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.27954063299999998 2023-08-28T02:35:34.102 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.102 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.102 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:34.102 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.527918+0000", 2023-08-28T02:35:34.102 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.5543000000000002e-05 2023-08-28T02:35:34.102 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.102 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.103 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:35:34.103 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.527953+0000", 2023-08-28T02:35:34.103 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.5420000000000001e-05 2023-08-28T02:35:34.103 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.103 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.103 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:35:34.103 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.527979+0000", 2023-08-28T02:35:34.103 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.28357411100000002 2023-08-28T02:35:34.103 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:34.103 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:35:34.103 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:34.103 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.104 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.104 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:59983 7.17 7:e8265a08:::benchmark_data_smithi143_360738_object59982:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e270)", 2023-08-28T02:35:34.104 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:31:43.479107+0000", 2023-08-28T02:35:34.104 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 230.549837747, 2023-08-28T02:35:34.104 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.99602258499999996, 2023-08-28T02:35:34.104 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:35:34.104 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "reached pg", 2023-08-28T02:35:34.104 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:35:34.105 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:35:34.105 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:35:34.105 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 59983 2023-08-28T02:35:34.105 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.105 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:35:34.105 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.105 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:35:34.105 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.479107+0000", 2023-08-28T02:35:34.106 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:35:34.106 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.106 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.106 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:35:34.106 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.479107+0000", 2023-08-28T02:35:34.106 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:35:34.106 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.106 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.107 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:35:34.107 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.479105+0000", 2023-08-28T02:35:34.107 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.4614999999999998e-05 2023-08-28T02:35:34.107 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.107 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.107 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:35:34.107 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.479159+0000", 2023-08-28T02:35:34.108 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.5138e-05 2023-08-28T02:35:34.108 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.108 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.108 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:35:34.108 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.479174+0000", 2023-08-28T02:35:34.108 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.9370000000000004e-06 2023-08-28T02:35:34.108 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.108 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.109 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:35:34.109 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.479182+0000", 2023-08-28T02:35:34.109 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.043864379000000002 2023-08-28T02:35:34.109 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.109 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.109 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:35:34.109 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.523047+0000", 2023-08-28T02:35:34.109 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.3399999999999998e-06 2023-08-28T02:35:34.110 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.110 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.110 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:35:34.110 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.523055+0000", 2023-08-28T02:35:34.110 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.95200016600000004 2023-08-28T02:35:34.110 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.110 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.110 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:35:34.110 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.475055+0000", 2023-08-28T02:35:34.111 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.4060000000000006e-05 2023-08-28T02:35:34.111 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.111 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.111 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:35:34.111 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.475129+0000", 2023-08-28T02:35:34.111 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.99602258499999996 2023-08-28T02:35:34.111 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:34.111 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:35:34.112 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:34.112 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.112 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.112 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:59986 7.17 7:ee7c11b0:::benchmark_data_smithi143_360738_object59985:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e270)", 2023-08-28T02:35:34.112 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:31:43.479190+0000", 2023-08-28T02:35:34.112 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 230.54975437799999, 2023-08-28T02:35:34.112 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.011258244, 2023-08-28T02:35:34.112 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:35:34.113 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:35:34.113 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:35:34.113 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:35:34.113 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:35:34.113 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 59986 2023-08-28T02:35:34.113 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.113 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:35:34.113 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.113 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:35:34.114 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.479190+0000", 2023-08-28T02:35:34.114 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:35:34.114 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.114 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.114 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:35:34.114 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.479190+0000", 2023-08-28T02:35:34.114 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999995 2023-08-28T02:35:34.114 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.115 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.115 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:35:34.115 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.479189+0000", 2023-08-28T02:35:34.115 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.6390999999999998e-05 2023-08-28T02:35:34.115 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.115 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.115 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:35:34.115 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.479226+0000", 2023-08-28T02:35:34.116 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.0899999999999999e-06 2023-08-28T02:35:34.116 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.116 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.116 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:35:34.116 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.479233+0000", 2023-08-28T02:35:34.116 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.4750000000000004e-06 2023-08-28T02:35:34.116 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.116 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.116 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:35:34.117 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.479237+0000", 2023-08-28T02:35:34.117 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.043836669000000002 2023-08-28T02:35:34.117 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.117 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.117 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:35:34.117 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.523074+0000", 2023-08-28T02:35:34.117 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.6149999999999999e-06 2023-08-28T02:35:34.117 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.118 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.118 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:35:34.118 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.523079+0000", 2023-08-28T02:35:34.118 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.95230524400000005 2023-08-28T02:35:34.118 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.118 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.118 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:35:34.118 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.475384+0000", 2023-08-28T02:35:34.119 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012201699999999999 2023-08-28T02:35:34.119 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.119 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.119 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:35:34.119 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.475506+0000", 2023-08-28T02:35:34.119 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00013047800000000001 2023-08-28T02:35:34.119 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.119 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.119 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,3", 2023-08-28T02:35:34.120 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.475636+0000", 2023-08-28T02:35:34.120 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0035618910000000002 2023-08-28T02:35:34.120 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.120 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.120 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:35:34.120 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.479198+0000", 2023-08-28T02:35:34.120 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0041008520000000003 2023-08-28T02:35:34.120 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.121 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.121 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:34.121 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.483299+0000", 2023-08-28T02:35:34.121 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0070752510000000003 2023-08-28T02:35:34.121 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.121 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.121 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:34.121 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.490374+0000", 2023-08-28T02:35:34.121 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.6100999999999999e-05 2023-08-28T02:35:34.122 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.122 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.122 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:35:34.122 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.490411+0000", 2023-08-28T02:35:34.122 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.7731000000000002e-05 2023-08-28T02:35:34.122 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.122 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.122 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:35:34.123 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.490448+0000", 2023-08-28T02:35:34.123 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.011258244 2023-08-28T02:35:34.123 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:34.123 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:35:34.123 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:34.123 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.123 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.123 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:59987 7.17 7:ea8b344b:::benchmark_data_smithi143_360738_object59986:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e270)", 2023-08-28T02:35:34.124 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:31:43.490101+0000", 2023-08-28T02:35:34.124 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 230.538843848, 2023-08-28T02:35:34.124 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0004954230000001, 2023-08-28T02:35:34.124 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:35:34.124 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:35:34.124 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:35:34.124 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:35:34.124 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:35:34.124 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 59987 2023-08-28T02:35:34.125 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.125 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:35:34.125 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.125 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:35:34.125 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.490101+0000", 2023-08-28T02:35:34.125 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:35:34.125 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.125 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.126 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:35:34.126 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.490101+0000", 2023-08-28T02:35:34.126 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:35:34.126 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.126 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.126 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:35:34.126 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.490099+0000", 2023-08-28T02:35:34.126 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.5648000000000001e-05 2023-08-28T02:35:34.127 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.127 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.127 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:35:34.127 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.490144+0000", 2023-08-28T02:35:34.127 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.7496999999999999e-05 2023-08-28T02:35:34.127 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.127 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.127 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:35:34.127 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.490162+0000", 2023-08-28T02:35:34.128 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.0639999999999996e-06 2023-08-28T02:35:34.128 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.128 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.128 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:35:34.128 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.490169+0000", 2023-08-28T02:35:34.128 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.032926438000000002 2023-08-28T02:35:34.128 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.128 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.129 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:35:34.129 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.523095+0000", 2023-08-28T02:35:34.129 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.6530000000000002e-06 2023-08-28T02:35:34.129 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.129 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.129 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:35:34.129 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.523100+0000", 2023-08-28T02:35:34.129 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.95330794900000004 2023-08-28T02:35:34.129 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.129 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.129 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:35:34.130 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.476408+0000", 2023-08-28T02:35:34.130 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012834599999999999 2023-08-28T02:35:34.130 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.130 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.130 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:35:34.130 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.476536+0000", 2023-08-28T02:35:34.130 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00072788400000000004 2023-08-28T02:35:34.130 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.130 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.130 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,3", 2023-08-28T02:35:34.130 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.477264+0000", 2023-08-28T02:35:34.130 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.001939807 2023-08-28T02:35:34.130 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.131 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.131 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:35:34.131 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.479204+0000", 2023-08-28T02:35:34.131 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0042188959999999998 2023-08-28T02:35:34.131 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.131 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.131 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:34.131 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.483423+0000", 2023-08-28T02:35:34.131 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0070798140000000002 2023-08-28T02:35:34.131 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.131 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.131 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:34.131 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.490503+0000", 2023-08-28T02:35:34.132 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.4703999999999997e-05 2023-08-28T02:35:34.132 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.132 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.132 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:35:34.132 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.490558+0000", 2023-08-28T02:35:34.132 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.8408999999999998e-05 2023-08-28T02:35:34.132 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.132 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.132 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:35:34.133 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.490596+0000", 2023-08-28T02:35:34.133 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0004954230000001 2023-08-28T02:35:34.133 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:34.133 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:35:34.133 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:34.133 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.133 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.133 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:73185 7.c 7:3902c7aa:::benchmark_data_smithi143_360738_object73184:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e278)", 2023-08-28T02:35:34.134 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:32:01.861018+0000", 2023-08-28T02:35:34.134 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 212.167926814, 2023-08-28T02:35:34.134 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.30387867000000002, 2023-08-28T02:35:34.134 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:35:34.134 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:35:34.134 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:35:34.134 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:35:34.134 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:35:34.135 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 73185 2023-08-28T02:35:34.135 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.135 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:35:34.135 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.135 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:35:34.135 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:01.861018+0000", 2023-08-28T02:35:34.135 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:35:34.135 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.135 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.136 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:35:34.136 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:01.861018+0000", 2023-08-28T02:35:34.136 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:35:34.136 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.136 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.136 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:35:34.136 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:01.861017+0000", 2023-08-28T02:35:34.136 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.9664999999999999e-05 2023-08-28T02:35:34.136 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.136 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.136 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:35:34.136 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:01.861036+0000", 2023-08-28T02:35:34.136 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.9590000000000003e-06 2023-08-28T02:35:34.137 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.137 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.137 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:35:34.137 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:01.861041+0000", 2023-08-28T02:35:34.137 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.19e-06 2023-08-28T02:35:34.137 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.137 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.137 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:35:34.137 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:01.861045+0000", 2023-08-28T02:35:34.138 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.3587999999999997e-05 2023-08-28T02:35:34.138 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.138 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.138 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:35:34.138 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:01.861088+0000", 2023-08-28T02:35:34.138 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000118669 2023-08-28T02:35:34.138 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.138 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.139 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:35:34.139 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:01.861207+0000", 2023-08-28T02:35:34.139 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000120389 2023-08-28T02:35:34.139 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.139 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.139 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 2,3", 2023-08-28T02:35:34.139 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:01.861327+0000", 2023-08-28T02:35:34.139 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.002126976 2023-08-28T02:35:34.140 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.140 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.140 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:35:34.140 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:01.863454+0000", 2023-08-28T02:35:34.140 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0044199699999999996 2023-08-28T02:35:34.140 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.140 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.140 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:34.140 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:01.867874+0000", 2023-08-28T02:35:34.140 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.29321470100000002 2023-08-28T02:35:34.140 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.141 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.141 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:34.141 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:02.161089+0000", 2023-08-28T02:35:34.141 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.0504000000000001e-05 2023-08-28T02:35:34.141 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.141 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.141 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:35:34.141 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:02.161159+0000", 2023-08-28T02:35:34.141 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0037368029999999999 2023-08-28T02:35:34.141 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.141 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.141 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:35:34.141 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:02.164896+0000", 2023-08-28T02:35:34.142 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.30387867000000002 2023-08-28T02:35:34.142 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:34.142 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:35:34.142 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:34.142 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.142 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.142 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:73196 7.c 7:3aac1676:::benchmark_data_smithi143_360738_object73195:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e278)", 2023-08-28T02:35:34.142 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:32:01.871375+0000", 2023-08-28T02:35:34.142 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 212.15756923699999, 2023-08-28T02:35:34.142 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.29715278499999997, 2023-08-28T02:35:34.142 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:35:34.142 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:35:34.142 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:35:34.143 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:35:34.143 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:35:34.143 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 73196 2023-08-28T02:35:34.143 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.143 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:35:34.143 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.143 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:35:34.143 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:01.871375+0000", 2023-08-28T02:35:34.143 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:35:34.143 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.143 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.143 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:35:34.143 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:01.871375+0000", 2023-08-28T02:35:34.143 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:35:34.144 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.144 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.144 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:35:34.144 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:01.871374+0000", 2023-08-28T02:35:34.144 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.8094000000000001e-05 2023-08-28T02:35:34.144 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.144 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.144 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:35:34.144 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:01.871462+0000", 2023-08-28T02:35:34.144 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.1780000000000003e-06 2023-08-28T02:35:34.144 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.144 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.144 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:35:34.145 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:01.871471+0000", 2023-08-28T02:35:34.145 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.9480000000000001e-06 2023-08-28T02:35:34.145 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.145 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.145 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:35:34.145 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:01.871474+0000", 2023-08-28T02:35:34.145 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.8983000000000003e-05 2023-08-28T02:35:34.145 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.145 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.145 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:35:34.145 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:01.871513+0000", 2023-08-28T02:35:34.145 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.7836000000000003e-05 2023-08-28T02:35:34.145 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.145 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.146 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:35:34.146 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:01.871611+0000", 2023-08-28T02:35:34.146 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000142748 2023-08-28T02:35:34.146 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.146 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.146 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 2,3", 2023-08-28T02:35:34.146 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:01.871754+0000", 2023-08-28T02:35:34.146 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0020802260000000001 2023-08-28T02:35:34.146 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.146 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.146 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:35:34.146 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:01.873834+0000", 2023-08-28T02:35:34.146 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0030338280000000001 2023-08-28T02:35:34.147 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.147 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.147 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:34.147 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:01.876868+0000", 2023-08-28T02:35:34.147 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.288150458 2023-08-28T02:35:34.147 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.147 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.147 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:34.147 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:02.165019+0000", 2023-08-28T02:35:34.147 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.9417000000000003e-05 2023-08-28T02:35:34.147 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.147 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.147 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:35:34.148 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:02.165088+0000", 2023-08-28T02:35:34.148 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0034399349999999999 2023-08-28T02:35:34.148 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.148 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.148 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:35:34.148 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:02.168528+0000", 2023-08-28T02:35:34.148 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.29715278499999997 2023-08-28T02:35:34.148 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:34.148 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:35:34.148 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:34.148 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.148 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.148 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:74735 7.11 7:8c3c856e:::benchmark_data_smithi143_360738_object74734:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e278)", 2023-08-28T02:35:34.148 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:32:04.074263+0000", 2023-08-28T02:35:34.149 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 209.95468118900001, 2023-08-28T02:35:34.149 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.36364874699999999, 2023-08-28T02:35:34.149 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:35:34.149 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:35:34.149 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:35:34.149 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:35:34.149 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:35:34.149 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 74735 2023-08-28T02:35:34.149 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.149 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:35:34.149 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.149 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:35:34.149 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.074263+0000", 2023-08-28T02:35:34.150 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:35:34.150 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.150 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.150 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:35:34.150 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.074263+0000", 2023-08-28T02:35:34.150 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:35:34.150 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.150 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.150 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:35:34.150 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.074262+0000", 2023-08-28T02:35:34.150 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.2809000000000003e-05 2023-08-28T02:35:34.150 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.150 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.151 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:35:34.151 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.074305+0000", 2023-08-28T02:35:34.151 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.135e-05 2023-08-28T02:35:34.151 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.151 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.151 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:35:34.151 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.074316+0000", 2023-08-28T02:35:34.151 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.7039999999999994e-06 2023-08-28T02:35:34.151 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.151 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.151 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:35:34.151 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.074324+0000", 2023-08-28T02:35:34.151 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.9311000000000002e-05 2023-08-28T02:35:34.151 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.152 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.152 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:35:34.152 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.074383+0000", 2023-08-28T02:35:34.152 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00013908 2023-08-28T02:35:34.152 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.152 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.152 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:35:34.152 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.074522+0000", 2023-08-28T02:35:34.152 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00013703000000000001 2023-08-28T02:35:34.152 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.152 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.152 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 1,3", 2023-08-28T02:35:34.152 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.074659+0000", 2023-08-28T02:35:34.153 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0013023920000000001 2023-08-28T02:35:34.153 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.153 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.153 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:35:34.153 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.075962+0000", 2023-08-28T02:35:34.153 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0023683659999999998 2023-08-28T02:35:34.153 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.153 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.153 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:34.153 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.078330+0000", 2023-08-28T02:35:34.153 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.35812925699999998 2023-08-28T02:35:34.153 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.153 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.153 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:34.154 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.436459+0000", 2023-08-28T02:35:34.154 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.4033999999999999e-05 2023-08-28T02:35:34.154 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.154 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.154 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:35:34.154 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.436503+0000", 2023-08-28T02:35:34.154 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0014087240000000001 2023-08-28T02:35:34.154 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.154 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.154 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:35:34.154 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.437912+0000", 2023-08-28T02:35:34.154 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.36364874699999999 2023-08-28T02:35:34.154 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:34.155 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:35:34.155 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:34.155 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.155 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.155 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:74737 7.c 7:30dd3236:::benchmark_data_smithi143_360738_object74736:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e278)", 2023-08-28T02:35:34.155 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:32:04.074337+0000", 2023-08-28T02:35:34.155 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 209.95460701600001, 2023-08-28T02:35:34.155 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.36470671900000001, 2023-08-28T02:35:34.155 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:35:34.155 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:35:34.155 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:35:34.155 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:35:34.155 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:35:34.156 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 74737 2023-08-28T02:35:34.156 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.156 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:35:34.156 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.156 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:35:34.156 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.074337+0000", 2023-08-28T02:35:34.156 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:35:34.156 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.156 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.156 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:35:34.156 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.074337+0000", 2023-08-28T02:35:34.156 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999995 2023-08-28T02:35:34.156 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.156 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.157 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:35:34.157 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.074337+0000", 2023-08-28T02:35:34.157 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.5392e-05 2023-08-28T02:35:34.157 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.157 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.157 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:35:34.157 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.074362+0000", 2023-08-28T02:35:34.157 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.7339999999999999e-06 2023-08-28T02:35:34.157 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.157 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.157 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:35:34.157 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.074367+0000", 2023-08-28T02:35:34.157 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.0759999999999996e-06 2023-08-28T02:35:34.157 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.158 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.158 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:35:34.158 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.074371+0000", 2023-08-28T02:35:34.158 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.5117999999999998e-05 2023-08-28T02:35:34.158 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.158 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.158 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:35:34.158 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.074406+0000", 2023-08-28T02:35:34.158 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000115262 2023-08-28T02:35:34.158 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.158 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.158 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:35:34.158 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.074521+0000", 2023-08-28T02:35:34.159 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000138424 2023-08-28T02:35:34.159 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.159 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.159 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 2,3", 2023-08-28T02:35:34.159 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.074660+0000", 2023-08-28T02:35:34.159 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0020618849999999998 2023-08-28T02:35:34.159 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.159 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.159 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:35:34.159 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.076722+0000", 2023-08-28T02:35:34.159 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.28327240100000001 2023-08-28T02:35:34.159 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.159 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.160 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:34.160 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.359994+0000", 2023-08-28T02:35:34.160 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.076505733000000006 2023-08-28T02:35:34.160 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.160 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.160 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:34.160 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.436500+0000", 2023-08-28T02:35:34.160 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.8109999999999998e-05 2023-08-28T02:35:34.160 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.160 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.160 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:35:34.160 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.436548+0000", 2023-08-28T02:35:34.160 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0024961839999999998 2023-08-28T02:35:34.161 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.161 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.161 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:35:34.161 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.439044+0000", 2023-08-28T02:35:34.161 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.36470671900000001 2023-08-28T02:35:34.161 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:34.161 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:35:34.161 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:34.161 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.161 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.161 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:74743 7.c 7:3b888767:::benchmark_data_smithi143_360738_object74742:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e278)", 2023-08-28T02:35:34.161 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:32:04.079960+0000", 2023-08-28T02:35:34.161 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 209.94898390700001, 2023-08-28T02:35:34.161 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.363101022, 2023-08-28T02:35:34.162 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:35:34.162 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:35:34.162 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:35:34.162 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:35:34.162 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:35:34.162 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 74743 2023-08-28T02:35:34.162 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.162 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:35:34.162 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.162 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:35:34.162 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.079960+0000", 2023-08-28T02:35:34.162 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:35:34.162 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.163 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.163 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:35:34.163 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.079960+0000", 2023-08-28T02:35:34.163 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:35:34.163 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.163 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.163 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:35:34.163 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.079959+0000", 2023-08-28T02:35:34.163 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.8719999999999999e-05 2023-08-28T02:35:34.163 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.163 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.163 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:35:34.163 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.079988+0000", 2023-08-28T02:35:34.164 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1620999999999999e-05 2023-08-28T02:35:34.164 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.164 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.164 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:35:34.164 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.080000+0000", 2023-08-28T02:35:34.164 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.4839999999999998e-06 2023-08-28T02:35:34.164 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.164 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.164 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:35:34.164 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.080004+0000", 2023-08-28T02:35:34.164 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.3571000000000001e-05 2023-08-28T02:35:34.164 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.164 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.164 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:35:34.165 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.080038+0000", 2023-08-28T02:35:34.165 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000134991 2023-08-28T02:35:34.165 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.165 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.165 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:35:34.165 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.080173+0000", 2023-08-28T02:35:34.165 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00014020800000000001 2023-08-28T02:35:34.165 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.165 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.165 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 2,3", 2023-08-28T02:35:34.165 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.080313+0000", 2023-08-28T02:35:34.165 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00187014 2023-08-28T02:35:34.166 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.166 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.166 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:35:34.166 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.082183+0000", 2023-08-28T02:35:34.166 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.28375494000000001 2023-08-28T02:35:34.166 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.166 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.166 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:34.167 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.365938+0000", 2023-08-28T02:35:34.167 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.075251154000000001 2023-08-28T02:35:34.167 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.167 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.167 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:34.167 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.441189+0000", 2023-08-28T02:35:34.167 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.6300000000000001e-05 2023-08-28T02:35:34.167 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.168 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.168 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:35:34.168 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.441236+0000", 2023-08-28T02:35:34.168 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0018259249999999999 2023-08-28T02:35:34.168 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.168 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.168 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:35:34.168 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.443061+0000", 2023-08-28T02:35:34.169 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.363101022 2023-08-28T02:35:34.169 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:34.169 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:35:34.169 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:34.169 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.169 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.169 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:105956 7.c 7:3e062ca2:::benchmark_data_smithi143_360738_object13186:head [delete] snapc 0=[] ondisk+write+known_if_redirected e289)", 2023-08-28T02:35:34.169 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:32:51.395239+0000", 2023-08-28T02:35:34.169 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 162.63370540400001, 2023-08-28T02:35:34.170 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.27643455700000003, 2023-08-28T02:35:34.170 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:35:34.170 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:35:34.170 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:35:34.170 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:35:34.170 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:35:34.170 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 105956 2023-08-28T02:35:34.170 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.171 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:35:34.171 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.171 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:35:34.171 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.395239+0000", 2023-08-28T02:35:34.171 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:35:34.171 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.171 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.171 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:35:34.171 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.395239+0000", 2023-08-28T02:35:34.172 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:35:34.172 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.172 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.172 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:35:34.172 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.395238+0000", 2023-08-28T02:35:34.172 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.2100000000000003e-06 2023-08-28T02:35:34.172 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.172 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.172 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:35:34.172 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.395242+0000", 2023-08-28T02:35:34.172 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.2470000000000008e-06 2023-08-28T02:35:34.172 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.172 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.173 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:35:34.173 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.395250+0000", 2023-08-28T02:35:34.173 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.3158e-05 2023-08-28T02:35:34.173 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.173 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.173 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:35:34.173 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.395263+0000", 2023-08-28T02:35:34.173 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.4830000000000001e-05 2023-08-28T02:35:34.173 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.173 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.173 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:35:34.173 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.395328+0000", 2023-08-28T02:35:34.173 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00017673999999999999 2023-08-28T02:35:34.174 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.174 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.174 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:35:34.174 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.395505+0000", 2023-08-28T02:35:34.174 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000195909 2023-08-28T02:35:34.174 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.174 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.174 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 2,3", 2023-08-28T02:35:34.174 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.395701+0000", 2023-08-28T02:35:34.174 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0030780730000000002 2023-08-28T02:35:34.174 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.174 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.174 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:35:34.174 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.398779+0000", 2023-08-28T02:35:34.175 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0088778889999999999 2023-08-28T02:35:34.175 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.175 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.175 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:34.175 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.407657+0000", 2023-08-28T02:35:34.175 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.26167327899999998 2023-08-28T02:35:34.175 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.175 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.175 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:34.175 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.669330+0000", 2023-08-28T02:35:34.175 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.9592999999999999e-05 2023-08-28T02:35:34.175 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.175 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.176 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:35:34.176 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.669370+0000", 2023-08-28T02:35:34.176 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0023037980000000001 2023-08-28T02:35:34.176 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.176 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.176 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:35:34.176 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.671674+0000", 2023-08-28T02:35:34.176 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.27643455700000003 2023-08-28T02:35:34.176 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:34.176 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:35:34.176 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:34.176 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.176 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.177 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:105964 7.c 7:36493e71:::benchmark_data_smithi143_360738_object13194:head [delete] snapc 0=[] ondisk+write+known_if_redirected e289)", 2023-08-28T02:35:34.177 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:32:51.402399+0000", 2023-08-28T02:35:34.177 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 162.626545424, 2023-08-28T02:35:34.177 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.27163401599999998, 2023-08-28T02:35:34.177 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:35:34.177 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:35:34.177 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:35:34.177 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:35:34.177 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:35:34.177 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 105964 2023-08-28T02:35:34.177 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.177 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:35:34.177 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.177 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:35:34.178 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.402399+0000", 2023-08-28T02:35:34.178 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:35:34.178 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.178 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.178 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:35:34.178 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.402399+0000", 2023-08-28T02:35:34.178 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:35:34.178 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.179 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.179 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:35:34.179 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.402398+0000", 2023-08-28T02:35:34.179 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.0360000000000003e-06 2023-08-28T02:35:34.179 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.179 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.179 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:35:34.179 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.402404+0000", 2023-08-28T02:35:34.180 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0203e-05 2023-08-28T02:35:34.180 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.180 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.180 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:35:34.180 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.402414+0000", 2023-08-28T02:35:34.180 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.0510000000000002e-06 2023-08-28T02:35:34.180 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.180 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.180 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:35:34.181 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.402420+0000", 2023-08-28T02:35:34.181 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.0081999999999999e-05 2023-08-28T02:35:34.181 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.181 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.181 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:35:34.181 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.402470+0000", 2023-08-28T02:35:34.181 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000207629 2023-08-28T02:35:34.181 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.181 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.181 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:35:34.181 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.402678+0000", 2023-08-28T02:35:34.181 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00018327 2023-08-28T02:35:34.181 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.182 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.182 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 2,3", 2023-08-28T02:35:34.182 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.402861+0000", 2023-08-28T02:35:34.182 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0048228159999999997 2023-08-28T02:35:34.182 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.182 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.182 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:34.182 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.407684+0000", 2023-08-28T02:35:34.182 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.3079999999999998e-06 2023-08-28T02:35:34.182 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.182 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.182 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:35:34.182 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.407689+0000", 2023-08-28T02:35:34.183 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.26405565800000003 2023-08-28T02:35:34.183 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.183 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.183 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:34.183 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.671745+0000", 2023-08-28T02:35:34.183 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.2169999999999995e-05 2023-08-28T02:35:34.183 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.183 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.184 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:35:34.184 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.671817+0000", 2023-08-28T02:35:34.184 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0022160819999999999 2023-08-28T02:35:34.184 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.184 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.184 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:35:34.184 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.674033+0000", 2023-08-28T02:35:34.184 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.27163401599999998 2023-08-28T02:35:34.184 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:34.185 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:35:34.185 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:34.185 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.185 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.185 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:105965 7.c 7:3fff0c9c:::benchmark_data_smithi143_360738_object13195:head [delete] snapc 0=[] ondisk+write+known_if_redirected e289)", 2023-08-28T02:35:34.185 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:32:51.404926+0000", 2023-08-28T02:35:34.185 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 162.62401807099999, 2023-08-28T02:35:34.185 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.27546642500000001, 2023-08-28T02:35:34.186 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:35:34.186 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:35:34.186 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:35:34.186 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:35:34.186 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:35:34.186 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 105965 2023-08-28T02:35:34.186 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.186 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:35:34.187 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.187 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:35:34.187 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.404926+0000", 2023-08-28T02:35:34.187 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:35:34.187 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.187 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.187 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:35:34.187 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.404926+0000", 2023-08-28T02:35:34.188 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:35:34.188 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.188 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.188 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:35:34.188 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.404925+0000", 2023-08-28T02:35:34.188 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.1909999999999997e-06 2023-08-28T02:35:34.188 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.188 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.189 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:35:34.189 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.404931+0000", 2023-08-28T02:35:34.189 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.3281e-05 2023-08-28T02:35:34.189 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.189 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.189 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:35:34.189 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.404944+0000", 2023-08-28T02:35:34.189 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.1380000000000001e-06 2023-08-28T02:35:34.190 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.190 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.190 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:35:34.190 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.404952+0000", 2023-08-28T02:35:34.190 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00029024699999999999 2023-08-28T02:35:34.190 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.190 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.190 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:35:34.190 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.405242+0000", 2023-08-28T02:35:34.191 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000117006 2023-08-28T02:35:34.191 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.191 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.191 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:35:34.191 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.405359+0000", 2023-08-28T02:35:34.191 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00013951400000000001 2023-08-28T02:35:34.191 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.191 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.191 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 2,3", 2023-08-28T02:35:34.191 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.405498+0000", 2023-08-28T02:35:34.191 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0023264449999999999 2023-08-28T02:35:34.191 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.192 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.192 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:35:34.192 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.407825+0000", 2023-08-28T02:35:34.192 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0015149759999999999 2023-08-28T02:35:34.192 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.192 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.192 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:34.192 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.409340+0000", 2023-08-28T02:35:34.192 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.26746703300000002 2023-08-28T02:35:34.192 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.192 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.192 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:34.192 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.676807+0000", 2023-08-28T02:35:34.193 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.4528e-05 2023-08-28T02:35:34.193 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.193 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.193 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:35:34.193 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.676901+0000", 2023-08-28T02:35:34.193 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0034913460000000002 2023-08-28T02:35:34.193 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.193 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.193 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:35:34.193 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.680393+0000", 2023-08-28T02:35:34.193 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.27546642500000001 2023-08-28T02:35:34.193 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:34.193 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:35:34.193 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:34.194 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.194 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.194 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:105970 7.c 7:3d49304d:::benchmark_data_smithi143_360738_object13200:head [delete] snapc 0=[] ondisk+write+known_if_redirected e289)", 2023-08-28T02:35:34.194 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:32:51.413324+0000", 2023-08-28T02:35:34.194 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 162.61562069499999, 2023-08-28T02:35:34.194 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.27555664499999999, 2023-08-28T02:35:34.194 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:35:34.194 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:35:34.194 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:35:34.194 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:35:34.194 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:35:34.194 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 105970 2023-08-28T02:35:34.194 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.195 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:35:34.195 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.195 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:35:34.195 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.413324+0000", 2023-08-28T02:35:34.195 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:35:34.195 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.195 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.195 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:35:34.195 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.413324+0000", 2023-08-28T02:35:34.195 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:35:34.195 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.195 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.196 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:35:34.196 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.413322+0000", 2023-08-28T02:35:34.196 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.9150000000000001e-06 2023-08-28T02:35:34.196 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.196 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.196 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:35:34.196 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.413328+0000", 2023-08-28T02:35:34.196 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2621e-05 2023-08-28T02:35:34.196 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.196 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.196 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:35:34.196 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.413341+0000", 2023-08-28T02:35:34.197 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.353e-06 2023-08-28T02:35:34.197 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.197 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.197 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:35:34.197 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.413347+0000", 2023-08-28T02:35:34.197 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.1079999999999999e-05 2023-08-28T02:35:34.197 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.197 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.197 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:35:34.197 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.413398+0000", 2023-08-28T02:35:34.197 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00016214199999999999 2023-08-28T02:35:34.197 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.198 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.198 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:35:34.198 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.413560+0000", 2023-08-28T02:35:34.198 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000131607 2023-08-28T02:35:34.198 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.198 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.198 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 2,3", 2023-08-28T02:35:34.198 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.413692+0000", 2023-08-28T02:35:34.198 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.003280331 2023-08-28T02:35:34.198 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.198 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.198 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:35:34.198 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.416972+0000", 2023-08-28T02:35:34.199 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000391078 2023-08-28T02:35:34.199 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.199 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.199 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:34.199 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.417363+0000", 2023-08-28T02:35:34.199 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.26862200200000003 2023-08-28T02:35:34.199 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.199 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.199 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:34.199 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.685985+0000", 2023-08-28T02:35:34.199 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.6096e-05 2023-08-28T02:35:34.199 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.199 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.200 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:35:34.200 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.686042+0000", 2023-08-28T02:35:34.200 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0028387550000000001 2023-08-28T02:35:34.200 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.200 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.200 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:35:34.200 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.688880+0000", 2023-08-28T02:35:34.200 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.27555664499999999 2023-08-28T02:35:34.200 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:34.200 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:35:34.200 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:34.200 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.200 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.201 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:143016 7.c 7:38903764:::benchmark_data_smithi143_360738_object50246:head [delete] snapc 0=[] ondisk+write+known_if_redirected e298)", 2023-08-28T02:35:34.201 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:33:50.175069+0000", 2023-08-28T02:35:34.201 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 103.853875855, 2023-08-28T02:35:34.201 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.27420446500000001, 2023-08-28T02:35:34.201 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:35:34.201 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:35:34.201 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:35:34.201 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:35:34.201 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:35:34.201 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 143016 2023-08-28T02:35:34.201 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.201 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:35:34.201 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.201 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:35:34.202 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:50.175069+0000", 2023-08-28T02:35:34.202 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:35:34.202 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.202 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.202 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:35:34.202 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:50.175069+0000", 2023-08-28T02:35:34.202 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:35:34.202 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.202 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.202 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:35:34.202 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:50.175067+0000", 2023-08-28T02:35:34.202 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.5510000000000005e-06 2023-08-28T02:35:34.202 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.203 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.203 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:35:34.203 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:50.175076+0000", 2023-08-28T02:35:34.203 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.9357000000000001e-05 2023-08-28T02:35:34.203 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.203 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.203 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:35:34.203 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:50.175096+0000", 2023-08-28T02:35:34.203 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.1040000000000007e-06 2023-08-28T02:35:34.203 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.203 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.203 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:35:34.203 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:50.175104+0000", 2023-08-28T02:35:34.204 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.2086000000000001e-05 2023-08-28T02:35:34.204 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.204 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.204 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:35:34.204 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:50.175176+0000", 2023-08-28T02:35:34.204 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00017281600000000001 2023-08-28T02:35:34.204 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.204 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.204 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:35:34.204 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:50.175349+0000", 2023-08-28T02:35:34.204 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000197777 2023-08-28T02:35:34.204 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.204 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.205 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 2,3", 2023-08-28T02:35:34.205 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:50.175546+0000", 2023-08-28T02:35:34.205 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0030292029999999998 2023-08-28T02:35:34.205 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.205 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.205 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:35:34.205 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:50.178576+0000", 2023-08-28T02:35:34.205 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0074006970000000004 2023-08-28T02:35:34.206 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.206 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.206 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:34.206 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:50.185976+0000", 2023-08-28T02:35:34.206 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.26137191700000001 2023-08-28T02:35:34.206 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.206 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.206 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:34.206 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:50.447348+0000", 2023-08-28T02:35:34.207 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.0931000000000003e-05 2023-08-28T02:35:34.207 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.207 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.207 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:35:34.207 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:50.447409+0000", 2023-08-28T02:35:34.207 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.001863881 2023-08-28T02:35:34.207 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:35:34.207 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:35:34.208 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:35:34.208 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:33:50.449273+0000", 2023-08-28T02:35:34.208 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.27420446500000001 2023-08-28T02:35:34.208 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:34.208 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:35:34.208 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:34.208 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:35:34.208 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:35:34.209 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:35:34.209 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 11 16 20192 20176 114.623 82 0.00163752 0.00856114 2023-08-28T02:35:34.210 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:33 smithi143 ceph-mon[183059]: failed to encode map e325 with expected crc 2023-08-28T02:35:34.210 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:33 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/641822817' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-08-28T02:35:34.210 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:33 smithi143 ceph-mon[183059]: osdmap e325: 8 total, 8 up, 7 in 2023-08-28T02:35:34.210 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:33 smithi143 ceph-mon[183059]: failed to encode map e325 with expected crc 2023-08-28T02:35:34.210 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:33 smithi143 ceph-mon[183059]: pgmap v838: 89 pgs: 89 active+clean; 949 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 105 MiB/s wr, 1.68k op/s; 529 KiB/s, 8 objects/s recovering 2023-08-28T02:35:34.210 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:33 smithi143 ceph-mon[189871]: failed to encode map e325 with expected crc 2023-08-28T02:35:34.211 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:33 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/641822817' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-08-28T02:35:34.211 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:33 smithi143 ceph-mon[189871]: osdmap e325: 8 total, 8 up, 7 in 2023-08-28T02:35:34.211 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:33 smithi143 ceph-mon[189871]: failed to encode map e325 with expected crc 2023-08-28T02:35:34.211 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:33 smithi143 ceph-mon[189871]: pgmap v838: 89 pgs: 89 active+clean; 949 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 105 MiB/s wr, 1.68k op/s; 529 KiB/s, 8 objects/s recovering 2023-08-28T02:35:34.796 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:35:34.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:34.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:35:34.809 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd reweight 5 1 2023-08-28T02:35:34.887 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:35:34.908 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:35:34.977 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.7 dump_ops_in_flight 2023-08-28T02:35:34.995 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:34 smithi143 ceph-mon[189871]: failed to encode map e325 with expected crc 2023-08-28T02:35:34.996 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:34 smithi143 ceph-mon[183059]: failed to encode map e325 with expected crc 2023-08-28T02:35:35.006 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:35.012 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:34 smithi169 ceph-mon[159415]: failed to encode map e325 with expected crc 2023-08-28T02:35:35.175 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 12 16 22013 21997 114.55 113.812 0.00731355 0.00872063 2023-08-28T02:35:35.352 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:35:35.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:35.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:35:35.439 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:35:35.463 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:35:35.503 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:35.581 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:35:35.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:35.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:35:35.686 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:35:35.752 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:35.761 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:35 smithi143 ceph-mon[183059]: pgmap v839: 89 pgs: 89 active+clean; 1.2 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 117 MiB/s wr, 1.88k op/s; 160 KiB/s, 2 objects/s recovering 2023-08-28T02:35:35.827 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:35:35.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:35.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:35:35.934 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:35:35.995 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:35 smithi143 ceph-mon[189871]: pgmap v839: 89 pgs: 89 active+clean; 1.2 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 117 MiB/s wr, 1.88k op/s; 160 KiB/s, 2 objects/s recovering 2023-08-28T02:35:35.996 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:36.107 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:35:36.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:36.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:35:36.174 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 13 16 23746 23730 114.069 108.312 0.0057569 0.00876095 2023-08-28T02:35:36.217 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:35:36.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:35 smithi169 ceph-mon[159415]: pgmap v839: 89 pgs: 89 active+clean; 1.2 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 117 MiB/s wr, 1.88k op/s; 160 KiB/s, 2 objects/s recovering 2023-08-28T02:35:36.258 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:36.319 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:35:36.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:36.321 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:35:36.422 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:35:36.448 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:36.489 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:35:36.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:36.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:35:36.592 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:35:36.654 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:36.759 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:35:36.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:36.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:35:36.786 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is unset 2023-08-28T02:35:36.867 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:35:36.912 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:36.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:36 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3271652996' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:35:36.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:36 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:35:36.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:36 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:35:36.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:36 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3271652996' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:35:36.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:36 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:35:36.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:36 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:35:36.983 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:35:36.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:36.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:35:37.090 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:35:37.128 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:36 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3271652996' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:35:37.128 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:36 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:35:37.128 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:36 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:35:37.132 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:37.174 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 14 16 25276 25260 112.751 95.625 0.0064998 0.00886281 2023-08-28T02:35:37.308 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:35:37.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:37.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:35:37.412 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:35:37.477 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:37.566 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:35:37.566 INFO:teuthology.orchestra.run.smithi169.stdout: "ops": [], 2023-08-28T02:35:37.566 INFO:teuthology.orchestra.run.smithi169.stdout: "num_ops": 0 2023-08-28T02:35:37.567 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:35:37.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:37 smithi169 ceph-mon[159415]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:35:37.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:37 smithi169 ceph-mon[159415]: Cluster is now healthy 2023-08-28T02:35:37.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:37 smithi169 ceph-mon[159415]: failed to encode map e326 with expected crc 2023-08-28T02:35:37.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:37 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:35:37.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:37 smithi169 ceph-mon[159415]: osdmap e326: 8 total, 8 up, 7 in 2023-08-28T02:35:37.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:37 smithi169 ceph-mon[159415]: failed to encode map e326 with expected crc 2023-08-28T02:35:37.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:37 smithi169 ceph-mon[159415]: failed to encode map e326 with expected crc 2023-08-28T02:35:37.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:37 smithi169 ceph-mon[159415]: failed to encode map e326 with expected crc 2023-08-28T02:35:37.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:37 smithi169 ceph-mon[159415]: pgmap v841: 89 pgs: 89 active+clean; 1.3 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 134 MiB/s wr, 2.14k op/s; 200 KiB/s, 3 objects/s recovering 2023-08-28T02:35:38.151 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:35:38.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:38.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:35:38.156 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:37 smithi143 ceph-mon[183059]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:35:38.156 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:37 smithi143 ceph-mon[183059]: Cluster is now healthy 2023-08-28T02:35:38.156 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:37 smithi143 ceph-mon[183059]: failed to encode map e326 with expected crc 2023-08-28T02:35:38.157 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:37 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:35:38.157 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:37 smithi143 ceph-mon[183059]: osdmap e326: 8 total, 8 up, 7 in 2023-08-28T02:35:38.157 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:37 smithi143 ceph-mon[183059]: failed to encode map e326 with expected crc 2023-08-28T02:35:38.157 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:37 smithi143 ceph-mon[183059]: failed to encode map e326 with expected crc 2023-08-28T02:35:38.157 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:37 smithi143 ceph-mon[183059]: failed to encode map e326 with expected crc 2023-08-28T02:35:38.157 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:37 smithi143 ceph-mon[183059]: pgmap v841: 89 pgs: 89 active+clean; 1.3 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 134 MiB/s wr, 2.14k op/s; 200 KiB/s, 3 objects/s recovering 2023-08-28T02:35:38.157 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:37 smithi143 ceph-mon[189871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:35:38.157 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:37 smithi143 ceph-mon[189871]: Cluster is now healthy 2023-08-28T02:35:38.157 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:37 smithi143 ceph-mon[189871]: failed to encode map e326 with expected crc 2023-08-28T02:35:38.157 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:37 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:35:38.158 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:37 smithi143 ceph-mon[189871]: osdmap e326: 8 total, 8 up, 7 in 2023-08-28T02:35:38.158 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:37 smithi143 ceph-mon[189871]: failed to encode map e326 with expected crc 2023-08-28T02:35:38.158 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:37 smithi143 ceph-mon[189871]: failed to encode map e326 with expected crc 2023-08-28T02:35:38.158 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:37 smithi143 ceph-mon[189871]: failed to encode map e326 with expected crc 2023-08-28T02:35:38.158 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:37 smithi143 ceph-mon[189871]: pgmap v841: 89 pgs: 89 active+clean; 1.3 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 134 MiB/s wr, 2.14k op/s; 200 KiB/s, 3 objects/s recovering 2023-08-28T02:35:38.176 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 15 16 26808 26792 111.604 95.75 0.00451672 0.00895379 2023-08-28T02:35:38.257 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:35:38.313 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:38.594 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:35:38.627 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.7 dump_blocked_ops 2023-08-28T02:35:38.627 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:35:38.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:38.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:35:38.736 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:35:38.774 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:35:38.796 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:38.836 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:35:38.871 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:35:38.909 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:35:38.943 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:35:38.946 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:35:38.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:38.950 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:35:38.981 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:35:38.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:38 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/481213295' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:35:39.017 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:35:39.050 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:35:39.058 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:35:39.069 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:35:39.069 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:35:39.069 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:35:39.069 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:35:39.069 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:35:39.069 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:35:39.069 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:35:39.069 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:35:39.096 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:38 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/481213295' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:35:39.096 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:38 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/481213295' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:35:39.098 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:39.140 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:35:39.172 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:35:39.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:39.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:35:39.178 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 16 16 28355 28339 110.665 96.6875 0.00966082 0.00903004 2023-08-28T02:35:39.278 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:35:39.312 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:39.433 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:35:39.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:39.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:35:39.539 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:35:39.586 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:39.855 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:35:39.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:39.858 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:35:39.958 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:35:40.014 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:40.077 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:35:40.079 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:39 smithi143 ceph-mon[183059]: pgmap v842: 89 pgs: 89 active+clean; 1.4 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-08-28T02:35:40.079 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:39 smithi143 ceph-mon[189871]: pgmap v842: 89 pgs: 89 active+clean; 1.4 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-08-28T02:35:40.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:40.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:35:40.177 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 17 16 30016 30000 110.262 103.812 0.00189498 0.00899789 2023-08-28T02:35:40.181 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:35:40.206 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:40.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:39 smithi169 ceph-mon[159415]: pgmap v842: 89 pgs: 89 active+clean; 1.4 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-08-28T02:35:40.249 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:35:40.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:40.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:35:40.283 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set noscrub 2023-08-28T02:35:40.352 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:35:40.383 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:40.726 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:35:40.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:40.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:35:40.832 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:35:40.939 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:41.178 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 18 16 30886 30870 107.157 54.375 0.00672181 0.00932485 2023-08-28T02:35:41.243 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:35:41.243 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:35:41.353 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:35:41.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:41.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:35:41.464 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:35:41.540 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:41.620 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:35:41.694 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:35:41.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:41.701 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:35:41.802 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:35:41.876 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:41.989 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:35:41.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:41.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:35:42.096 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:35:42.154 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:42 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:35:42.154 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:42 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:35:42.154 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:42 smithi143 ceph-mon[183059]: pgmap v843: 89 pgs: 89 active+clean; 1.7 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 107 MiB/s wr, 1.72k op/s 2023-08-28T02:35:42.154 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:42 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:35:42.154 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:42 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:35:42.155 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:42 smithi143 ceph-mon[189871]: pgmap v843: 89 pgs: 89 active+clean; 1.7 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 107 MiB/s wr, 1.72k op/s 2023-08-28T02:35:42.155 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:35:41 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:35:41] "GET /metrics HTTP/1.1" 200 36529 "" "Prometheus/2.33.4" 2023-08-28T02:35:42.161 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:42.178 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 19 16 32154 32138 105.688 79.25 0.0195155 0.00945361 2023-08-28T02:35:42.197 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.7 dump_historic_ops 2023-08-28T02:35:42.377 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:35:42.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:42.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:35:42.484 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:35:42.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:42 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:35:42.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:42 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:35:42.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:42 smithi169 ceph-mon[159415]: pgmap v843: 89 pgs: 89 active+clean; 1.7 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 107 MiB/s wr, 1.72k op/s 2023-08-28T02:35:42.527 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:42.605 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:35:42.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:42.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:35:42.708 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:35:42.715 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:35:42.776 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:42.835 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:35:42.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:42.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:35:42.938 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:35:42.967 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:43.012 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:35:43.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:43.014 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:35:43.073 INFO:teuthology.orchestra.run.smithi143.stderr:reweighted osd.5 to 1 (10000) 2023-08-28T02:35:43.115 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:35:43.143 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:43.178 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:2023-08-28T02:35:43.179499+0000 min lat: 0.00106302 max lat: 1.01701 avg lat: 0.00950117 2023-08-28T02:35:43.178 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-08-28T02:35:43.178 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 20 16 33250 33234 103.829 68.5 0.00218345 0.00950117 2023-08-28T02:35:43.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:43 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3526324648' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-08-28T02:35:43.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:43 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-08-28T02:35:43.301 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:35:43.372 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:43 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3526324648' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-08-28T02:35:43.372 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:43 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-08-28T02:35:43.372 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:43 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3526324648' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-08-28T02:35:43.372 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:43 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-08-28T02:35:43.392 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:35:43.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:43.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:35:43.496 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:35:43.544 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:43.812 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:35:43.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:43.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:35:43.916 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:35:43.943 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:44.069 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:35:44.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:44.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:35:44.174 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:35:44.178 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 21 16 34026 34010 101.194 48.5 0.00652174 0.0098771 2023-08-28T02:35:44.210 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:44 smithi169 ceph-mon[159415]: failed to encode map e327 with expected crc 2023-08-28T02:35:44.210 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:44 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-08-28T02:35:44.210 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:44 smithi169 ceph-mon[159415]: osdmap e327: 8 total, 8 up, 7 in 2023-08-28T02:35:44.210 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:44 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2023-08-28T02:35:44.211 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:44 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-08-28T02:35:44.211 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:44 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2023-08-28T02:35:44.211 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:44 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2023-08-28T02:35:44.211 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:44 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 0]}]: dispatch 2023-08-28T02:35:44.211 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:44 smithi169 ceph-mon[159415]: pgmap v845: 89 pgs: 89 active+clean; 1.9 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-08-28T02:35:44.218 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:44.339 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:35:44.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:44.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:35:44.443 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:35:44.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:44 smithi143 ceph-mon[183059]: failed to encode map e327 with expected crc 2023-08-28T02:35:44.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:44 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-08-28T02:35:44.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:44 smithi143 ceph-mon[183059]: osdmap e327: 8 total, 8 up, 7 in 2023-08-28T02:35:44.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:44 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2023-08-28T02:35:44.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:44 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-08-28T02:35:44.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:44 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2023-08-28T02:35:44.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:44 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2023-08-28T02:35:44.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:44 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 0]}]: dispatch 2023-08-28T02:35:44.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:44 smithi143 ceph-mon[183059]: pgmap v845: 89 pgs: 89 active+clean; 1.9 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-08-28T02:35:44.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:44 smithi143 ceph-mon[189871]: failed to encode map e327 with expected crc 2023-08-28T02:35:44.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:44 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-08-28T02:35:44.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:44 smithi143 ceph-mon[189871]: osdmap e327: 8 total, 8 up, 7 in 2023-08-28T02:35:44.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:44 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2023-08-28T02:35:44.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:44 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-08-28T02:35:44.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:44 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2023-08-28T02:35:44.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:44 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2023-08-28T02:35:44.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:44 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 0]}]: dispatch 2023-08-28T02:35:44.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:44 smithi143 ceph-mon[189871]: pgmap v845: 89 pgs: 89 active+clean; 1.9 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-08-28T02:35:44.487 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:44.550 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:35:44.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:44.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:35:44.576 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd reweight 3 1 2023-08-28T02:35:44.622 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:35:44.622 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:35:44.658 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:35:44.736 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:44.890 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:35:44.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:44.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:35:44.934 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:35:44.996 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:35:45.023 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:45.066 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:35:45.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:45.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:35:45.169 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:35:45.178 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 22 16 35275 35259 100.142 78.0625 0.00707793 0.0099818 2023-08-28T02:35:45.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:45 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2023-08-28T02:35:45.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:45 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2023-08-28T02:35:45.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:45 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2023-08-28T02:35:45.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:45 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2023-08-28T02:35:45.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:45 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 0]}]': finished 2023-08-28T02:35:45.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:45 smithi143 ceph-mon[183059]: osdmap e328: 8 total, 8 up, 7 in 2023-08-28T02:35:45.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:45 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2023-08-28T02:35:45.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:45 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2023-08-28T02:35:45.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:45 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2023-08-28T02:35:45.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:45 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2023-08-28T02:35:45.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:45 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 0]}]': finished 2023-08-28T02:35:45.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:45 smithi143 ceph-mon[189871]: osdmap e328: 8 total, 8 up, 7 in 2023-08-28T02:35:45.202 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:45.436 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:35:45.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:45.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:35:45.439 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:45 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2023-08-28T02:35:45.439 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:45 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2023-08-28T02:35:45.439 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:45 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2023-08-28T02:35:45.439 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:45 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2023-08-28T02:35:45.439 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:45 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 0]}]': finished 2023-08-28T02:35:45.439 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:45 smithi169 ceph-mon[159415]: osdmap e328: 8 total, 8 up, 7 in 2023-08-28T02:35:45.503 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.0 dump_ops_in_flight 2023-08-28T02:35:45.523 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:35:45.539 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:35:45.571 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:45.632 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:35:45.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:45.637 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:35:45.738 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:35:45.829 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:45.901 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:35:45.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:45.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:35:46.008 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:35:46.055 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:46.100 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:46 smithi143 ceph-mon[183059]: failed to encode map e328 with expected crc 2023-08-28T02:35:46.100 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:46 smithi143 ceph-mon[183059]: failed to encode map e329 with expected crc 2023-08-28T02:35:46.100 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:46 smithi143 ceph-mon[183059]: failed to encode map e329 with expected crc 2023-08-28T02:35:46.101 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:46 smithi143 ceph-mon[183059]: failed to encode map e329 with expected crc 2023-08-28T02:35:46.101 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:46 smithi143 ceph-mon[183059]: osdmap e329: 8 total, 8 up, 7 in 2023-08-28T02:35:46.101 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:46 smithi143 ceph-mon[183059]: failed to encode map e329 with expected crc 2023-08-28T02:35:46.101 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:46 smithi143 ceph-mon[183059]: failed to encode map e329 with expected crc 2023-08-28T02:35:46.101 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:46 smithi143 ceph-mon[183059]: failed to encode map e329 with expected crc 2023-08-28T02:35:46.101 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:46 smithi143 ceph-mon[183059]: failed to encode map e329 with expected crc 2023-08-28T02:35:46.102 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:46 smithi143 ceph-mon[189871]: failed to encode map e328 with expected crc 2023-08-28T02:35:46.102 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:46 smithi143 ceph-mon[189871]: failed to encode map e329 with expected crc 2023-08-28T02:35:46.102 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:46 smithi143 ceph-mon[189871]: failed to encode map e329 with expected crc 2023-08-28T02:35:46.102 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:46 smithi143 ceph-mon[189871]: failed to encode map e329 with expected crc 2023-08-28T02:35:46.102 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:46 smithi143 ceph-mon[189871]: osdmap e329: 8 total, 8 up, 7 in 2023-08-28T02:35:46.102 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:46 smithi143 ceph-mon[189871]: failed to encode map e329 with expected crc 2023-08-28T02:35:46.102 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:46 smithi143 ceph-mon[189871]: failed to encode map e329 with expected crc 2023-08-28T02:35:46.102 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:46 smithi143 ceph-mon[189871]: failed to encode map e329 with expected crc 2023-08-28T02:35:46.102 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:46 smithi143 ceph-mon[189871]: failed to encode map e329 with expected crc 2023-08-28T02:35:46.102 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:46 smithi143 ceph-mon[189871]: failed to encode map e329 with expected crc 2023-08-28T02:35:46.102 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:46 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1385523654' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:35:46.102 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:46 smithi143 ceph-mon[189871]: pgmap v848: 89 pgs: 2 peering, 87 active+clean; 2.1 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-08-28T02:35:46.120 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is set 2023-08-28T02:35:46.123 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:35:46.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:46.129 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:35:46.178 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 23 16 36732 36716 99.7474 91.0625 0.00502986 0.0100195 2023-08-28T02:35:46.230 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:35:46.301 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:46.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:46 smithi143 ceph-mon[183059]: failed to encode map e329 with expected crc 2023-08-28T02:35:46.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:46 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1385523654' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:35:46.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:46 smithi143 ceph-mon[183059]: pgmap v848: 89 pgs: 2 peering, 87 active+clean; 2.1 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-08-28T02:35:46.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:46 smithi169 ceph-mon[159415]: failed to encode map e328 with expected crc 2023-08-28T02:35:46.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:46 smithi169 ceph-mon[159415]: failed to encode map e329 with expected crc 2023-08-28T02:35:46.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:46 smithi169 ceph-mon[159415]: failed to encode map e329 with expected crc 2023-08-28T02:35:46.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:46 smithi169 ceph-mon[159415]: failed to encode map e329 with expected crc 2023-08-28T02:35:46.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:46 smithi169 ceph-mon[159415]: osdmap e329: 8 total, 8 up, 7 in 2023-08-28T02:35:46.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:46 smithi169 ceph-mon[159415]: failed to encode map e329 with expected crc 2023-08-28T02:35:46.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:46 smithi169 ceph-mon[159415]: failed to encode map e329 with expected crc 2023-08-28T02:35:46.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:46 smithi169 ceph-mon[159415]: failed to encode map e329 with expected crc 2023-08-28T02:35:46.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:46 smithi169 ceph-mon[159415]: failed to encode map e329 with expected crc 2023-08-28T02:35:46.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:46 smithi169 ceph-mon[159415]: failed to encode map e329 with expected crc 2023-08-28T02:35:46.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:46 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1385523654' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:35:46.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:46 smithi169 ceph-mon[159415]: pgmap v848: 89 pgs: 2 peering, 87 active+clean; 2.1 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-08-28T02:35:47.178 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 24 16 37588 37572 97.8205 53.5 0.00280259 0.010216 2023-08-28T02:35:47.373 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:47 smithi143 ceph-mon[183059]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:35:47.373 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:47 smithi143 ceph-mon[183059]: failed to encode map e330 with expected crc 2023-08-28T02:35:47.373 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:47 smithi143 ceph-mon[183059]: failed to encode map e330 with expected crc 2023-08-28T02:35:47.373 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:47 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1385523654' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:35:47.373 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:47 smithi143 ceph-mon[183059]: osdmap e330: 8 total, 8 up, 7 in 2023-08-28T02:35:47.374 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:47 smithi143 ceph-mon[183059]: failed to encode map e330 with expected crc 2023-08-28T02:35:47.374 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:47 smithi143 ceph-mon[189871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:35:47.374 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:47 smithi143 ceph-mon[189871]: failed to encode map e330 with expected crc 2023-08-28T02:35:47.374 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:47 smithi143 ceph-mon[189871]: failed to encode map e330 with expected crc 2023-08-28T02:35:47.374 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:47 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1385523654' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:35:47.374 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:47 smithi143 ceph-mon[189871]: osdmap e330: 8 total, 8 up, 7 in 2023-08-28T02:35:47.374 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:47 smithi143 ceph-mon[189871]: failed to encode map e330 with expected crc 2023-08-28T02:35:47.408 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:35:47.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:47.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:35:47.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:47 smithi169 ceph-mon[159415]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:35:47.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:47 smithi169 ceph-mon[159415]: failed to encode map e330 with expected crc 2023-08-28T02:35:47.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:47 smithi169 ceph-mon[159415]: failed to encode map e330 with expected crc 2023-08-28T02:35:47.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:47 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1385523654' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:35:47.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:47 smithi169 ceph-mon[159415]: osdmap e330: 8 total, 8 up, 7 in 2023-08-28T02:35:47.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:47 smithi169 ceph-mon[159415]: failed to encode map e330 with expected crc 2023-08-28T02:35:47.517 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:35:47.543 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:47.586 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:35:47.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:47.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:35:47.696 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:35:47.721 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:47.764 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:35:47.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:47.773 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:35:47.874 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:35:47.911 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:47.966 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:35:47.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:47.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:35:48.073 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:35:48.127 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:48.178 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 25 16 38763 38747 96.8451 73.4375 0.00452457 0.0103196 2023-08-28T02:35:48.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:48 smithi143 ceph-mon[183059]: pgmap v850: 89 pgs: 2 peering, 87 active+clean; 2.2 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-08-28T02:35:48.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:48 smithi143 ceph-mon[189871]: pgmap v850: 89 pgs: 2 peering, 87 active+clean; 2.2 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-08-28T02:35:48.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:48 smithi169 ceph-mon[159415]: pgmap v850: 89 pgs: 2 peering, 87 active+clean; 2.2 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-08-28T02:35:48.359 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:35:48.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:48.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:35:48.464 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:35:48.495 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:48.552 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:35:48.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:48.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:35:48.656 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:35:48.686 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:49.178 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:35:49.179 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:35:49.179 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:35:49.181 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:35:49.182 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 26 16 40159 40143 96.464 87.25 0.00993484 0.0103602 2023-08-28T02:35:49.216 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:35:49.260 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:35:49.295 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:35:49.326 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:35:49.340 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:35:49.340 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:35:49.340 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:35:49.340 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:35:49.340 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:35:49.340 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:35:49.340 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:35:49.340 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:35:49.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:49 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/109371002' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:35:49.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:49 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/109371002' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:35:49.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:49 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/109371002' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:35:50.027 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:35:50.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:50.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:35:50.133 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:35:50.181 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 27 16 41295 41279 95.5208 71 0.00682427 0.0104637 2023-08-28T02:35:50.206 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:50.258 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set nodeep-scrub 2023-08-28T02:35:50.320 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:50 smithi143 ceph-mon[189871]: pgmap v851: 89 pgs: 1 peering, 88 active+clean; 2.3 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-08-28T02:35:50.322 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:50 smithi143 ceph-mon[183059]: pgmap v851: 89 pgs: 1 peering, 88 active+clean; 2.3 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-08-28T02:35:50.365 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:35:50.370 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:35:50.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:50.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:35:50.481 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:35:50.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:50 smithi169 ceph-mon[159415]: pgmap v851: 89 pgs: 1 peering, 88 active+clean; 2.3 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-08-28T02:35:50.595 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:50.818 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:35:50.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:50.828 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:35:50.929 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:35:50.961 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:51.022 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:35:51.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:51.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:35:51.127 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:35:51.156 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:51.187 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 28 16 42524 42508 94.8355 76.8125 0.0118735 0.0105377 2023-08-28T02:35:51.213 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:35:51.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:51.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:35:51.319 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:35:51.404 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:51.416 INFO:teuthology.orchestra.run.smithi143.stderr:reweighted osd.3 to 1 (10000) 2023-08-28T02:35:51.486 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:35:51.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:51 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1240738781' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-08-28T02:35:51.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:51 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-08-28T02:35:51.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:51.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:35:51.589 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:35:51.638 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:51 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1240738781' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-08-28T02:35:51.638 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:51 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-08-28T02:35:51.644 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:51 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1240738781' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-08-28T02:35:51.644 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:51 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-08-28T02:35:51.648 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:51.755 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:35:51.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:51.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:35:51.864 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:35:51.890 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:51.932 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:35:51.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:51.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:35:52.039 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:35:52.064 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:52.187 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 29 16 43460 43444 93.5821 58.5 0.00478857 0.0106808 2023-08-28T02:35:52.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:35:51 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:35:51] "GET /metrics HTTP/1.1" 200 36529 "" "Prometheus/2.33.4" 2023-08-28T02:35:52.420 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:35:52.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:52 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-08-28T02:35:52.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:52 smithi143 ceph-mon[183059]: osdmap e331: 8 total, 8 up, 7 in 2023-08-28T02:35:52.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:52 smithi143 ceph-mon[183059]: failed to encode map e331 with expected crc 2023-08-28T02:35:52.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:52 smithi143 ceph-mon[183059]: failed to encode map e331 with expected crc 2023-08-28T02:35:52.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:52 smithi143 ceph-mon[183059]: failed to encode map e331 with expected crc 2023-08-28T02:35:52.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:52 smithi143 ceph-mon[183059]: failed to encode map e331 with expected crc 2023-08-28T02:35:52.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:52 smithi143 ceph-mon[183059]: pgmap v853: 89 pgs: 89 active+clean; 2.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 53 MiB/s wr, 841 op/s 2023-08-28T02:35:52.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:52 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-08-28T02:35:52.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:52 smithi143 ceph-mon[189871]: osdmap e331: 8 total, 8 up, 7 in 2023-08-28T02:35:52.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:52 smithi143 ceph-mon[189871]: failed to encode map e331 with expected crc 2023-08-28T02:35:52.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:52 smithi143 ceph-mon[189871]: failed to encode map e331 with expected crc 2023-08-28T02:35:52.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:52 smithi143 ceph-mon[189871]: failed to encode map e331 with expected crc 2023-08-28T02:35:52.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:52 smithi143 ceph-mon[189871]: failed to encode map e331 with expected crc 2023-08-28T02:35:52.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:52 smithi143 ceph-mon[189871]: pgmap v853: 89 pgs: 89 active+clean; 2.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 53 MiB/s wr, 841 op/s 2023-08-28T02:35:52.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:52 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-08-28T02:35:52.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:52 smithi169 ceph-mon[159415]: osdmap e331: 8 total, 8 up, 7 in 2023-08-28T02:35:52.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:52 smithi169 ceph-mon[159415]: failed to encode map e331 with expected crc 2023-08-28T02:35:52.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:52 smithi169 ceph-mon[159415]: failed to encode map e331 with expected crc 2023-08-28T02:35:52.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:52 smithi169 ceph-mon[159415]: failed to encode map e331 with expected crc 2023-08-28T02:35:52.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:52 smithi169 ceph-mon[159415]: failed to encode map e331 with expected crc 2023-08-28T02:35:52.739 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:52 smithi169 ceph-mon[159415]: pgmap v853: 89 pgs: 89 active+clean; 2.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 53 MiB/s wr, 841 op/s 2023-08-28T02:35:53.188 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 30 16 44522 44506 92.6718 66.375 0.0105678 0.0107853 2023-08-28T02:35:53.200 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:35:53.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:53.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:35:53.308 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:35:53.328 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:35:53.337 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:53.362 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd reweight 6 1 2023-08-28T02:35:53.398 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:35:53.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:53.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:35:53.504 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:35:53.538 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:53.598 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:35:53.603 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:53 smithi169 ceph-mon[159415]: failed to encode map e331 with expected crc 2023-08-28T02:35:53.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:53.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:35:53.654 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:53 smithi143 ceph-mon[183059]: failed to encode map e331 with expected crc 2023-08-28T02:35:53.654 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:53 smithi143 ceph-mon[189871]: failed to encode map e331 with expected crc 2023-08-28T02:35:53.704 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:35:53.781 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:35:53.812 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:53.923 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:35:53.923 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [ 2023-08-28T02:35:53.923 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.923 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.45647.0:45329 8.c 8:385e75fc:::benchmark_data_smithi143_419289_object45328:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e331)", 2023-08-28T02:35:53.923 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:35:53.910474+0000", 2023-08-28T02:35:53.923 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.012891628, 2023-08-28T02:35:53.924 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.012927652, 2023-08-28T02:35:53.924 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:35:53.924 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "waiting for sub ops", 2023-08-28T02:35:53.924 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:35:53.924 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.45647", 2023-08-28T02:35:53.924 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3230872490", 2023-08-28T02:35:53.924 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 45329 2023-08-28T02:35:53.924 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:35:53.924 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:35:53.924 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.924 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:35:53.924 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:53.910474+0000", 2023-08-28T02:35:53.925 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:35:53.925 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:35:53.925 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.925 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:35:53.925 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:53.910474+0000", 2023-08-28T02:35:53.925 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:35:53.925 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:35:53.925 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.925 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:35:53.925 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:53.910472+0000", 2023-08-28T02:35:53.925 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:35:53.925 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:35:53.925 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.926 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:35:53.926 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:53.910497+0000", 2023-08-28T02:35:53.926 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.5406e-05 2023-08-28T02:35:53.926 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:35:53.926 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.926 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:35:53.926 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:53.910510+0000", 2023-08-28T02:35:53.926 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.31e-05 2023-08-28T02:35:53.926 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:35:53.926 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.926 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:35:53.926 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:53.910519+0000", 2023-08-28T02:35:53.926 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.8710000000000003e-06 2023-08-28T02:35:53.927 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:35:53.927 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.927 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:35:53.927 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:53.910578+0000", 2023-08-28T02:35:53.927 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.9321999999999999e-05 2023-08-28T02:35:53.927 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:35:53.927 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.927 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:35:53.927 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:53.910738+0000", 2023-08-28T02:35:53.927 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00015966600000000001 2023-08-28T02:35:53.927 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:35:53.927 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.927 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 3,5", 2023-08-28T02:35:53.928 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:53.910894+0000", 2023-08-28T02:35:53.928 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00015537099999999999 2023-08-28T02:35:53.928 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:35:53.928 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.928 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:35:53.928 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:53.913135+0000", 2023-08-28T02:35:53.928 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0022410360000000001 2023-08-28T02:35:53.928 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:35:53.928 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.928 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:53.928 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:53.914280+0000", 2023-08-28T02:35:53.928 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.001145215 2023-08-28T02:35:53.929 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:35:53.929 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:35:53.929 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:35:53.929 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:35:53.929 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.929 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.45647.0:45330 8.9 e331/318 8:9f923b62:::benchmark_data_smithi143_419289_object45329:head v 331'2846, mlcod=331'2844)", 2023-08-28T02:35:53.929 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:35:53.912775+0000", 2023-08-28T02:35:53.929 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.010590613, 2023-08-28T02:35:53.929 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.010718596, 2023-08-28T02:35:53.929 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:35:53.929 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "started", 2023-08-28T02:35:53.929 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:35:53.929 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.930 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:35:53.930 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:53.912775+0000", 2023-08-28T02:35:53.930 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:35:53.930 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:35:53.930 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.930 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:35:53.930 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:53.912775+0000", 2023-08-28T02:35:53.930 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:35:53.930 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:35:53.930 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.930 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:35:53.930 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:53.912774+0000", 2023-08-28T02:35:53.931 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:35:53.931 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:35:53.931 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.931 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:35:53.931 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:53.912812+0000", 2023-08-28T02:35:53.931 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.7598999999999997e-05 2023-08-28T02:35:53.931 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:35:53.931 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.931 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:35:53.931 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:53.912822+0000", 2023-08-28T02:35:53.931 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0285999999999999e-05 2023-08-28T02:35:53.931 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:35:53.931 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.932 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:35:53.932 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:53.912829+0000", 2023-08-28T02:35:53.932 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.5239999999999997e-06 2023-08-28T02:35:53.932 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:35:53.932 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.932 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:35:53.932 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:53.912877+0000", 2023-08-28T02:35:53.932 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.8176000000000001e-05 2023-08-28T02:35:53.932 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:35:53.932 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.932 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:35:53.933 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:53.912896+0000", 2023-08-28T02:35:53.933 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.8668000000000002e-05 2023-08-28T02:35:53.933 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:35:53.933 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:35:53.933 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:35:53.933 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:35:53.933 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.933 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.45647.0:45334 8.6 8:6ed1199e:::benchmark_data_smithi143_419289_object45333:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e331)", 2023-08-28T02:35:53.933 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:35:53.914884+0000", 2023-08-28T02:35:53.933 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.0084811980000000006, 2023-08-28T02:35:53.933 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0086741639999999998, 2023-08-28T02:35:53.934 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:35:53.934 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "waiting for sub ops", 2023-08-28T02:35:53.934 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:35:53.934 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.45647", 2023-08-28T02:35:53.934 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3230872490", 2023-08-28T02:35:53.934 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 45334 2023-08-28T02:35:53.934 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:35:53.934 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:35:53.934 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.934 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:35:53.934 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:53.914884+0000", 2023-08-28T02:35:53.934 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:35:53.934 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:35:53.935 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.935 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:35:53.935 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:53.914884+0000", 2023-08-28T02:35:53.935 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:35:53.935 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:35:53.935 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.935 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:35:53.935 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:53.914883+0000", 2023-08-28T02:35:53.935 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:35:53.935 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:35:53.935 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.935 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:35:53.936 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:53.914908+0000", 2023-08-28T02:35:53.936 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.4876999999999999e-05 2023-08-28T02:35:53.936 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:35:53.936 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.936 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:35:53.936 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:53.914926+0000", 2023-08-28T02:35:53.936 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.7716000000000001e-05 2023-08-28T02:35:53.936 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:35:53.936 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.936 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:35:53.936 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:53.914935+0000", 2023-08-28T02:35:53.936 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.5640000000000002e-06 2023-08-28T02:35:53.937 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:35:53.937 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.937 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:35:53.937 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:53.914977+0000", 2023-08-28T02:35:53.937 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.2367999999999998e-05 2023-08-28T02:35:53.937 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:35:53.937 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.937 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:35:53.937 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:53.915195+0000", 2023-08-28T02:35:53.937 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00021751 2023-08-28T02:35:53.937 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:35:53.937 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.937 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 2,6", 2023-08-28T02:35:53.937 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:53.915323+0000", 2023-08-28T02:35:53.938 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00012809799999999999 2023-08-28T02:35:53.938 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:35:53.938 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.938 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:53.938 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:53.920060+0000", 2023-08-28T02:35:53.938 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0047371590000000003 2023-08-28T02:35:53.938 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:35:53.938 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.938 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:35:53.938 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:53.923058+0000", 2023-08-28T02:35:53.938 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0029977459999999999 2023-08-28T02:35:53.938 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:35:53.938 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:35:53.939 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:35:53.939 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:35:53.939 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.939 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.45647.0:45336 8.9 e331/318 8:90b1a37e:::benchmark_data_smithi143_419289_object45335:head v 331'2847, mlcod=331'2844)", 2023-08-28T02:35:53.939 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:35:53.915394+0000", 2023-08-28T02:35:53.939 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.0079710370000000003, 2023-08-28T02:35:53.939 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0082477309999999995, 2023-08-28T02:35:53.939 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:35:53.939 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "started", 2023-08-28T02:35:53.939 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:35:53.939 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.939 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:35:53.940 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:53.915394+0000", 2023-08-28T02:35:53.940 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:35:53.940 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:35:53.940 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.940 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:35:53.940 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:53.915394+0000", 2023-08-28T02:35:53.940 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:35:53.940 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:35:53.940 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.940 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:35:53.940 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:53.915394+0000", 2023-08-28T02:35:53.940 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:35:53.940 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:35:53.941 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.941 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:35:53.941 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:53.915467+0000", 2023-08-28T02:35:53.941 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.3270999999999997e-05 2023-08-28T02:35:53.941 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:35:53.941 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.941 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:35:53.941 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:53.915478+0000", 2023-08-28T02:35:53.941 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0512e-05 2023-08-28T02:35:53.941 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:35:53.941 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.941 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:35:53.941 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:53.915485+0000", 2023-08-28T02:35:53.942 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.7430000000000002e-06 2023-08-28T02:35:53.942 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:35:53.942 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.942 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:35:53.942 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:53.920091+0000", 2023-08-28T02:35:53.942 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0046053550000000002 2023-08-28T02:35:53.942 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:35:53.942 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.942 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:35:53.942 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:53.920108+0000", 2023-08-28T02:35:53.942 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.7782e-05 2023-08-28T02:35:53.942 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:35:53.942 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:35:53.943 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:35:53.943 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:35:53.943 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.943 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.45647.0:45340 8.2 e331/318 8:4e8333a5:::benchmark_data_smithi143_419289_object45339:head v 331'2883, mlcod=331'2881)", 2023-08-28T02:35:53.943 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:35:53.919798+0000", 2023-08-28T02:35:53.943 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.0035676480000000001, 2023-08-28T02:35:53.943 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0039074540000000003, 2023-08-28T02:35:53.943 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:35:53.943 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "started", 2023-08-28T02:35:53.943 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:35:53.944 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.944 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:35:53.944 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:53.919798+0000", 2023-08-28T02:35:53.944 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:35:53.944 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:35:53.944 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.944 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:35:53.944 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:53.919798+0000", 2023-08-28T02:35:53.944 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:35:53.944 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:35:53.944 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.944 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:35:53.944 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:53.919796+0000", 2023-08-28T02:35:53.945 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:35:53.945 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:35:53.945 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.945 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:35:53.945 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:53.919842+0000", 2023-08-28T02:35:53.945 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.5734e-05 2023-08-28T02:35:53.945 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:35:53.945 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.945 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:35:53.945 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:53.919862+0000", 2023-08-28T02:35:53.945 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.9647e-05 2023-08-28T02:35:53.945 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:35:53.946 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.946 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:35:53.946 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:53.919870+0000", 2023-08-28T02:35:53.946 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.1540000000000002e-06 2023-08-28T02:35:53.946 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:35:53.946 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.946 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:35:53.946 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:53.919914+0000", 2023-08-28T02:35:53.946 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.4226e-05 2023-08-28T02:35:53.946 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:35:53.946 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:35:53.946 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:35:53.947 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:53.919939+0000", 2023-08-28T02:35:53.947 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.4632999999999999e-05 2023-08-28T02:35:53.947 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:35:53.947 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:35:53.947 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:35:53.947 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:35:53.947 INFO:teuthology.orchestra.run.smithi143.stdout: ], 2023-08-28T02:35:53.947 INFO:teuthology.orchestra.run.smithi143.stdout: "num_ops": 5 2023-08-28T02:35:53.947 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:35:53.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:53.948 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:35:53.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:35:54.049 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:35:54.080 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:54.151 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:35:54.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:54.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:35:54.190 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 31 16 45669 45653 91.99 71.6875 0.00477004 0.0108654 2023-08-28T02:35:54.255 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:35:54.285 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:54.349 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:35:54.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:54.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:35:54.459 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:35:54.502 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:54.506 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:54 smithi143 ceph-mon[183059]: pgmap v854: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-08-28T02:35:54.506 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:54 smithi143 ceph-mon[189871]: pgmap v854: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-08-28T02:35:54.569 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:35:54.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:54.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:35:54.675 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:35:54.709 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:54 smithi169 ceph-mon[159415]: pgmap v854: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-08-28T02:35:54.714 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:54.780 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:35:54.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:54.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:35:54.888 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:35:54.921 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:54.969 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:35:54.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:54.976 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:35:55.077 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:35:55.103 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:55.159 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:35:55.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:55.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:35:55.190 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 32 16 46898 46882 91.5157 76.8125 0.00510094 0.0109102 2023-08-28T02:35:55.265 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:35:55.301 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:55.356 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:35:55.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:55.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:35:55.462 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:35:55.503 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:55.566 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:35:55.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:55.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:35:55.673 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:35:55.698 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:55.746 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:35:55.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:55.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:35:55.855 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:35:55.889 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:55.956 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:35:55.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:55.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:35:56.068 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:35:56.103 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:56.168 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:35:56.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:56.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:35:56.190 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 33 16 48076 48060 90.9736 73.625 0.0108909 0.0109873 2023-08-28T02:35:56.273 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:35:56.305 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:56.333 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.0 dump_blocked_ops 2023-08-28T02:35:56.366 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:35:56.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:56.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:35:56.471 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:35:56.500 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:56.558 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:35:56.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:56.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:35:56.566 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:56 smithi143 ceph-mon[183059]: pgmap v855: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-08-28T02:35:56.566 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:56 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:35:56.566 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:56 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:35:56.567 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:56 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:35:56.663 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:35:56.692 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:56.733 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:35:56.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:56.739 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:35:56.840 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:35:56.864 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:56 smithi169 ceph-mon[159415]: pgmap v855: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-08-28T02:35:56.864 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:56 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:35:56.864 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:56 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:35:56.864 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:56 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:35:56.866 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:56.908 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:35:56.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:56.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:35:56.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:56 smithi143 ceph-mon[189871]: pgmap v855: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-08-28T02:35:56.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:56 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:35:56.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:56 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:35:56.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:56 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:35:57.018 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:35:57.107 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:57.190 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 34 15 49043 49028 90.0775 60.5 0.0098819 0.0110966 2023-08-28T02:35:57.347 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:35:57.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:57.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:35:57.423 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:35:57.460 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:35:57.533 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:57.741 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:35:57.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:57.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:35:57.860 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:35:57.963 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:58.086 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:35:58.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:58.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:35:58.190 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 35 16 50189 50173 89.5486 71.5625 0.0107133 0.0111619 2023-08-28T02:35:58.191 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:35:58.225 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:58.286 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:35:58.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:58.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:35:58.372 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:35:58.394 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:35:58.422 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:35:58.425 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:58.479 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:35:58.480 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:35:58.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:35:58.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:35:58.527 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:35:58.563 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:35:58.586 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:35:58.596 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:35:58.625 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:58 smithi143 ceph-mon[183059]: pgmap v856: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-08-28T02:35:58.625 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:58 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/661619855' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:35:58.625 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:58 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:35:58.626 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:58 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/4092576723' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:35:58.626 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:35:58.649 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is set 2023-08-28T02:35:58.667 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:35:58.667 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:35:58.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:58 smithi169 ceph-mon[159415]: pgmap v856: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-08-28T02:35:58.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:58 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/661619855' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:35:58.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:58 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:35:58.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:58 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/4092576723' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:35:58.756 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:35:58.756 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:35:58.756 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:35:58.756 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:35:58.756 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:35:58.757 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:35:58.757 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:35:58.757 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:35:58.757 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:35:58.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:58 smithi143 ceph-mon[189871]: pgmap v856: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-08-28T02:35:58.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:58 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/661619855' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:35:58.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:58 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:35:58.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:58 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/4092576723' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:35:59.191 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 36 16 51397 51381 89.1583 75.5 0.00804164 0.0112115 2023-08-28T02:35:59.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:59 smithi143 ceph-mon[183059]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:35:59.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:59 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:35:59.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:59 smithi143 ceph-mon[183059]: osdmap e332: 8 total, 8 up, 7 in 2023-08-28T02:35:59.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:59 smithi143 ceph-mon[183059]: failed to encode map e332 with expected crc 2023-08-28T02:35:59.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:35:59 smithi143 ceph-mon[183059]: failed to encode map e332 with expected crc 2023-08-28T02:35:59.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:59 smithi143 ceph-mon[189871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:35:59.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:59 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:35:59.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:59 smithi143 ceph-mon[189871]: osdmap e332: 8 total, 8 up, 7 in 2023-08-28T02:35:59.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:59 smithi143 ceph-mon[189871]: failed to encode map e332 with expected crc 2023-08-28T02:35:59.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:35:59 smithi143 ceph-mon[189871]: failed to encode map e332 with expected crc 2023-08-28T02:35:59.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:59 smithi169 ceph-mon[159415]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:35:59.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:59 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:35:59.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:59 smithi169 ceph-mon[159415]: osdmap e332: 8 total, 8 up, 7 in 2023-08-28T02:35:59.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:59 smithi169 ceph-mon[159415]: failed to encode map e332 with expected crc 2023-08-28T02:35:59.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:35:59 smithi169 ceph-mon[159415]: failed to encode map e332 with expected crc 2023-08-28T02:36:00.191 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 37 16 52287 52271 88.2506 55.625 0.00959783 0.0113259 2023-08-28T02:36:00.252 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:36:00.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:00.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:36:00.359 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:36:00.436 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:00.729 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:36:00.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:00.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:36:00.840 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:36:00.868 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:00.910 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:36:00.913 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:00 smithi169 ceph-mon[159415]: pgmap v858: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-08-28T02:36:00.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:00.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:36:00.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:00 smithi143 ceph-mon[183059]: pgmap v858: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-08-28T02:36:00.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:00 smithi143 ceph-mon[189871]: pgmap v858: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-08-28T02:36:01.015 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:36:01.051 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:01.105 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:36:01.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:01.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:36:01.191 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 38 16 53048 53032 87.1801 47.5625 0.0154277 0.0114359 2023-08-28T02:36:01.211 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:36:01.250 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:01.323 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:36:01.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:01.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:36:01.430 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:36:01.464 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:01.531 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:36:01.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:01.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:36:01.641 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:36:01.672 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:01.733 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:36:01.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:01.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:36:01.838 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:36:01.873 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:01.931 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:36:01.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:01.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:36:02.037 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:36:02.065 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:02.125 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:36:02.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:02.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:36:02.192 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 39 16 54287 54271 86.9302 77.4375 0.0138406 0.0114967 2023-08-28T02:36:02.203 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:36:01 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:36:01] "GET /metrics HTTP/1.1" 200 36531 "" "Prometheus/2.33.4" 2023-08-28T02:36:02.234 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:36:02.270 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:02.336 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:36:02.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:02.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:36:02.443 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:36:02.480 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:02.540 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:36:02.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:02.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:36:02.646 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:36:02.658 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:02 smithi143 ceph-mon[183059]: pgmap v859: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-08-28T02:36:02.658 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:02 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3205449679' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-08-28T02:36:02.675 INFO:teuthology.orchestra.run.smithi143.stderr:reweighted osd.6 to 1 (10000) 2023-08-28T02:36:02.676 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:02.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:02 smithi169 ceph-mon[159415]: pgmap v859: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-08-28T02:36:02.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:02 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3205449679' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-08-28T02:36:02.743 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:36:02.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:02.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:36:02.849 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:36:02.888 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:02.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:02 smithi143 ceph-mon[189871]: pgmap v859: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-08-28T02:36:02.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:02 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3205449679' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-08-28T02:36:02.964 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:36:02.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:02.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:36:03.070 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:36:03.114 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:03.185 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:36:03.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:03.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:36:03.192 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:2023-08-28T02:36:03.193342+0000 min lat: 0.00106302 max lat: 1.01701 avg lat: 0.0115175 2023-08-28T02:36:03.192 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-08-28T02:36:03.193 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 40 16 55584 55568 86.7834 81.0625 0.0110327 0.0115175 2023-08-28T02:36:03.293 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:36:03.359 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:03.635 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:36:03.756 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset noscrub 2023-08-28T02:36:03.926 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:36:03.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:03.934 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:03 smithi143 ceph-mon[183059]: failed to encode map e333 with expected crc 2023-08-28T02:36:03.934 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:03 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3205449679' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-08-28T02:36:03.934 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:03 smithi143 ceph-mon[183059]: osdmap e333: 8 total, 8 up, 7 in 2023-08-28T02:36:03.934 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:03 smithi143 ceph-mon[183059]: failed to encode map e333 with expected crc 2023-08-28T02:36:03.934 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:03 smithi143 ceph-mon[183059]: failed to encode map e333 with expected crc 2023-08-28T02:36:03.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:36:03.935 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:03 smithi143 ceph-mon[189871]: failed to encode map e333 with expected crc 2023-08-28T02:36:03.935 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:03 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3205449679' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-08-28T02:36:03.935 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:03 smithi143 ceph-mon[189871]: osdmap e333: 8 total, 8 up, 7 in 2023-08-28T02:36:03.935 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:03 smithi143 ceph-mon[189871]: failed to encode map e333 with expected crc 2023-08-28T02:36:03.935 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:03 smithi143 ceph-mon[189871]: failed to encode map e333 with expected crc 2023-08-28T02:36:03.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:03 smithi169 ceph-mon[159415]: failed to encode map e333 with expected crc 2023-08-28T02:36:03.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:03 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3205449679' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-08-28T02:36:03.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:03 smithi169 ceph-mon[159415]: osdmap e333: 8 total, 8 up, 7 in 2023-08-28T02:36:03.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:03 smithi169 ceph-mon[159415]: failed to encode map e333 with expected crc 2023-08-28T02:36:03.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:03 smithi169 ceph-mon[159415]: failed to encode map e333 with expected crc 2023-08-28T02:36:04.035 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:36:04.067 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:04.121 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:36:04.122 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd reweight 1 1 2023-08-28T02:36:04.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:04.129 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:36:04.192 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 41 16 56545 56529 86.1317 60.0625 0.0159472 0.011604 2023-08-28T02:36:04.230 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:36:04.260 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:04.314 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:36:04.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:04.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:36:04.420 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:36:04.552 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:04.601 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:36:04.625 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:36:04.625 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:36:04.649 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:36:04.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:04.665 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:36:04.765 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:36:04.800 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:04.860 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:36:04.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:04.864 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:36:04.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:04 smithi143 ceph-mon[183059]: pgmap v861: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-08-28T02:36:04.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:04 smithi143 ceph-mon[189871]: pgmap v861: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-08-28T02:36:04.965 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:36:04.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:04 smithi169 ceph-mon[159415]: pgmap v861: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-08-28T02:36:05.000 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:05.060 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:36:05.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:05.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:36:05.166 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:36:05.192 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 42 16 57559 57543 85.5899 63.375 0.00768433 0.0116784 2023-08-28T02:36:05.195 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:05.267 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:36:05.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:05.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:36:05.375 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:36:05.401 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:05.442 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:36:05.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:05.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:36:05.548 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:36:05.603 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:06.025 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:36:06.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:06.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:36:06.129 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:36:06.140 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:36:06.160 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:06.189 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.0 dump_historic_ops 2023-08-28T02:36:06.198 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 43 16 58531 58515 85.0124 60.75 0.0128341 0.0117579 2023-08-28T02:36:06.217 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:36:06.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:06.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:36:06.327 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:36:06.419 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:06.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:06 smithi143 ceph-mon[183059]: pgmap v862: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-08-28T02:36:06.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:06 smithi143 ceph-mon[189871]: pgmap v862: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-08-28T02:36:06.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:06 smithi169 ceph-mon[159415]: pgmap v862: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-08-28T02:36:07.048 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:36:07.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:07.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:36:07.153 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:36:07.187 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:07.192 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 44 16 59410 59394 84.329 54.9375 0.00830982 0.0118519 2023-08-28T02:36:07.247 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:36:07.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:07.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:36:07.300 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:36:07.354 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:36:07.413 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:07.718 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:36:07.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:07.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:36:07.829 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:36:07.904 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:08.192 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 45 16 60185 60169 83.5316 48.4375 0.0035157 0.0119114 2023-08-28T02:36:08.902 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:36:08.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:08.923 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:36:08.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:08 smithi143 ceph-mon[183059]: pgmap v863: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 65 MiB/s wr, 1.05k op/s 2023-08-28T02:36:08.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:08 smithi143 ceph-mon[189871]: pgmap v863: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 65 MiB/s wr, 1.05k op/s 2023-08-28T02:36:08.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:08 smithi169 ceph-mon[159415]: pgmap v863: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 65 MiB/s wr, 1.05k op/s 2023-08-28T02:36:09.024 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:36:09.077 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:36:09.091 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:09.192 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 46 16 61316 61300 83.2523 70.6875 0.0204107 0.0120056 2023-08-28T02:36:09.379 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:36:09.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:09.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:36:09.435 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:36:09.480 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:36:09.491 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:36:09.512 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:36:09.553 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:36:09.570 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:09.590 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:36:09.625 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:36:09.667 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:36:09.701 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:36:09.708 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:36:09.708 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:36:09.708 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:36:09.708 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:36:09.708 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:36:09.708 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:36:09.708 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:36:09.708 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:36:09.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:09 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1506024176' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:36:10.195 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 47 16 62263 62247 82.735 59.1875 0.0147408 0.0120801 2023-08-28T02:36:10.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:09 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1506024176' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:36:10.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:09 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1506024176' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:36:10.737 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is unset 2023-08-28T02:36:10.740 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:36:10.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:10.744 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:36:10.845 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:36:10.935 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:10 smithi143 ceph-mon[183059]: pgmap v864: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-08-28T02:36:11.009 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:10 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2720054112' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:36:11.010 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:10 smithi169 ceph-mon[159415]: pgmap v864: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-08-28T02:36:11.010 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:10 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2720054112' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:36:11.010 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:10 smithi143 ceph-mon[189871]: pgmap v864: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-08-28T02:36:11.010 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:10 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2720054112' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:36:11.010 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:11.198 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 48 16 63290 63274 82.3451 64.1875 0.0142686 0.0121388 2023-08-28T02:36:11.762 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:36:11.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:11.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:36:11.871 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:36:11.907 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:11.930 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:11 smithi143 ceph-mon[183059]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:36:11.930 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:11 smithi143 ceph-mon[183059]: failed to encode map e334 with expected crc 2023-08-28T02:36:11.930 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:11 smithi143 ceph-mon[183059]: failed to encode map e334 with expected crc 2023-08-28T02:36:11.930 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:11 smithi143 ceph-mon[183059]: failed to encode map e334 with expected crc 2023-08-28T02:36:11.930 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:11 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2720054112' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:36:11.930 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:11 smithi143 ceph-mon[183059]: osdmap e334: 8 total, 8 up, 7 in 2023-08-28T02:36:11.930 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:11 smithi143 ceph-mon[183059]: failed to encode map e334 with expected crc 2023-08-28T02:36:11.931 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:11 smithi143 ceph-mon[183059]: failed to encode map e334 with expected crc 2023-08-28T02:36:11.931 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:11 smithi143 ceph-mon[183059]: failed to encode map e334 with expected crc 2023-08-28T02:36:11.931 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:11 smithi143 ceph-mon[183059]: failed to encode map e334 with expected crc 2023-08-28T02:36:11.931 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:11 smithi143 ceph-mon[183059]: failed to encode map e334 with expected crc 2023-08-28T02:36:11.931 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:11 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:36:11.931 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:11 smithi143 ceph-mon[189871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:36:11.931 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:11 smithi143 ceph-mon[189871]: failed to encode map e334 with expected crc 2023-08-28T02:36:11.931 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:11 smithi143 ceph-mon[189871]: failed to encode map e334 with expected crc 2023-08-28T02:36:11.932 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:11 smithi143 ceph-mon[189871]: failed to encode map e334 with expected crc 2023-08-28T02:36:11.932 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:11 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2720054112' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:36:11.932 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:11 smithi143 ceph-mon[189871]: osdmap e334: 8 total, 8 up, 7 in 2023-08-28T02:36:11.932 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:11 smithi143 ceph-mon[189871]: failed to encode map e334 with expected crc 2023-08-28T02:36:11.932 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:11 smithi143 ceph-mon[189871]: failed to encode map e334 with expected crc 2023-08-28T02:36:11.932 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:11 smithi143 ceph-mon[189871]: failed to encode map e334 with expected crc 2023-08-28T02:36:11.932 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:11 smithi143 ceph-mon[189871]: failed to encode map e334 with expected crc 2023-08-28T02:36:11.932 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:11 smithi143 ceph-mon[189871]: failed to encode map e334 with expected crc 2023-08-28T02:36:11.932 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:11 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:36:11.932 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:36:11 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:36:11] "GET /metrics HTTP/1.1" 200 36530 "" "Prometheus/2.33.4" 2023-08-28T02:36:11.977 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:36:11.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:11.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:36:11.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:11 smithi169 ceph-mon[159415]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:36:11.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:11 smithi169 ceph-mon[159415]: failed to encode map e334 with expected crc 2023-08-28T02:36:11.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:11 smithi169 ceph-mon[159415]: failed to encode map e334 with expected crc 2023-08-28T02:36:11.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:11 smithi169 ceph-mon[159415]: failed to encode map e334 with expected crc 2023-08-28T02:36:11.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:11 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2720054112' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:36:11.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:11 smithi169 ceph-mon[159415]: osdmap e334: 8 total, 8 up, 7 in 2023-08-28T02:36:11.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:11 smithi169 ceph-mon[159415]: failed to encode map e334 with expected crc 2023-08-28T02:36:11.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:11 smithi169 ceph-mon[159415]: failed to encode map e334 with expected crc 2023-08-28T02:36:11.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:11 smithi169 ceph-mon[159415]: failed to encode map e334 with expected crc 2023-08-28T02:36:11.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:11 smithi169 ceph-mon[159415]: failed to encode map e334 with expected crc 2023-08-28T02:36:11.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:11 smithi169 ceph-mon[159415]: failed to encode map e334 with expected crc 2023-08-28T02:36:11.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:11 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:36:12.087 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:36:12.142 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:12.197 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 49 16 64297 64281 81.9491 62.9375 0.00740747 0.0121915 2023-08-28T02:36:13.198 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 50 16 65231 65215 81.4775 58.375 0.00319277 0.0122287 2023-08-28T02:36:13.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:12 smithi143 ceph-mon[183059]: pgmap v866: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 62 MiB/s wr, 995 op/s 2023-08-28T02:36:13.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:12 smithi143 ceph-mon[189871]: pgmap v866: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 62 MiB/s wr, 995 op/s 2023-08-28T02:36:13.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:12 smithi169 ceph-mon[159415]: pgmap v866: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 62 MiB/s wr, 995 op/s 2023-08-28T02:36:13.868 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:36:13.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:13.882 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:13 smithi143 ceph-mon[183059]: pgmap v867: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 62 MiB/s wr, 998 op/s 2023-08-28T02:36:13.882 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:13 smithi143 ceph-mon[189871]: pgmap v867: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 62 MiB/s wr, 998 op/s 2023-08-28T02:36:13.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:36:13.984 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:36:14.017 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:13 smithi169 ceph-mon[159415]: pgmap v867: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 62 MiB/s wr, 998 op/s 2023-08-28T02:36:14.021 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:14.092 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:36:14.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:14.096 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:36:14.197 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:36:14.198 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 51 16 66211 66195 81.0809 61.25 0.00937165 0.0123282 2023-08-28T02:36:14.226 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:14.288 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:36:14.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:14.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:36:14.395 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:36:14.430 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:14.468 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:36:14.468 INFO:teuthology.orchestra.run.smithi143.stdout: "size": 20, 2023-08-28T02:36:14.468 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 600, 2023-08-28T02:36:14.468 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [ 2023-08-28T02:36:14.468 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.468 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:78148 6.a 6:510b0ad3:::benchmark_data_smithi143_305742_object78147:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e221)", 2023-08-28T02:36:14.468 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:28:25.332701+0000", 2023-08-28T02:36:14.468 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 469.133935157, 2023-08-28T02:36:14.469 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.148737852, 2023-08-28T02:36:14.469 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:36:14.469 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "reached pg", 2023-08-28T02:36:14.469 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:36:14.469 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:36:14.469 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:36:14.469 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 78148 2023-08-28T02:36:14.469 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.469 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:36:14.469 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.469 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:36:14.469 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.332701+0000", 2023-08-28T02:36:14.470 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:14.470 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.470 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.470 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:36:14.470 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.332701+0000", 2023-08-28T02:36:14.470 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:14.470 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.470 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.470 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:36:14.470 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.332699+0000", 2023-08-28T02:36:14.470 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:36:14.470 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.470 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.471 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:36:14.471 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.332728+0000", 2023-08-28T02:36:14.471 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.9193999999999999e-05 2023-08-28T02:36:14.471 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.471 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.471 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:36:14.471 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.332742+0000", 2023-08-28T02:36:14.471 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3518e-05 2023-08-28T02:36:14.471 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.471 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.471 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:36:14.472 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.332752+0000", 2023-08-28T02:36:14.472 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0026e-05 2023-08-28T02:36:14.472 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.472 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.472 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:36:14.472 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.334218+0000", 2023-08-28T02:36:14.472 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0014661159999999999 2023-08-28T02:36:14.472 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.472 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.472 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:36:14.472 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.334231+0000", 2023-08-28T02:36:14.472 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.358e-05 2023-08-28T02:36:14.473 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.473 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.473 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:36:14.473 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.470078+0000", 2023-08-28T02:36:14.473 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.135846471 2023-08-28T02:36:14.473 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.473 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.473 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for degraded object", 2023-08-28T02:36:14.473 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.471483+0000", 2023-08-28T02:36:14.473 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.001405585 2023-08-28T02:36:14.473 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.473 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.473 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:36:14.474 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.481350+0000", 2023-08-28T02:36:14.474 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0098665560000000003 2023-08-28T02:36:14.474 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.474 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.474 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:36:14.474 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.481439+0000", 2023-08-28T02:36:14.474 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.8771999999999996e-05 2023-08-28T02:36:14.474 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:14.474 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:36:14.474 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:14.474 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.474 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.475 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:78161 6.a 6:5fab5d5c:::benchmark_data_smithi143_305742_object78160:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e221)", 2023-08-28T02:36:14.475 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:28:25.332788+0000", 2023-08-28T02:36:14.475 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 469.13384808199999, 2023-08-28T02:36:14.475 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1547578279999999, 2023-08-28T02:36:14.475 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:36:14.475 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:36:14.475 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:36:14.475 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:36:14.475 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:36:14.475 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 78161 2023-08-28T02:36:14.475 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.475 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:36:14.476 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.476 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:36:14.476 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.332788+0000", 2023-08-28T02:36:14.476 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:14.476 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.476 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.476 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:36:14.476 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.332788+0000", 2023-08-28T02:36:14.476 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:14.476 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.476 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.476 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:36:14.477 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.332787+0000", 2023-08-28T02:36:14.477 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:36:14.477 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.477 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.477 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:36:14.477 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.332814+0000", 2023-08-28T02:36:14.477 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.6265999999999999e-05 2023-08-28T02:36:14.477 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.477 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.477 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:36:14.477 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.332821+0000", 2023-08-28T02:36:14.477 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.6000000000000001e-06 2023-08-28T02:36:14.477 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.478 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.478 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:36:14.478 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.332827+0000", 2023-08-28T02:36:14.478 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.6969999999999998e-06 2023-08-28T02:36:14.478 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.478 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.478 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:36:14.478 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.334262+0000", 2023-08-28T02:36:14.478 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0014349409999999999 2023-08-28T02:36:14.478 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.478 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.478 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:36:14.479 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.334271+0000", 2023-08-28T02:36:14.479 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.6850000000000007e-06 2023-08-28T02:36:14.479 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.479 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.479 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:36:14.479 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.471538+0000", 2023-08-28T02:36:14.479 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1372679139999999 2023-08-28T02:36:14.479 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.479 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.479 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:36:14.479 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.471689+0000", 2023-08-28T02:36:14.479 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00015092799999999999 2023-08-28T02:36:14.479 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.480 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.480 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 2,5,6", 2023-08-28T02:36:14.480 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.471864+0000", 2023-08-28T02:36:14.480 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00017473599999999999 2023-08-28T02:36:14.480 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.480 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.480 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:36:14.480 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.481479+0000", 2023-08-28T02:36:14.480 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0096152310000000001 2023-08-28T02:36:14.480 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.480 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.480 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:36:14.481 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.481550+0000", 2023-08-28T02:36:14.481 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.0742999999999997e-05 2023-08-28T02:36:14.481 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.481 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.481 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:36:14.481 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.481731+0000", 2023-08-28T02:36:14.481 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000180562 2023-08-28T02:36:14.481 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.481 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.481 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:36:14.481 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.487363+0000", 2023-08-28T02:36:14.481 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0056318879999999998 2023-08-28T02:36:14.481 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.482 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.482 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:36:14.482 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.487424+0000", 2023-08-28T02:36:14.482 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.1050000000000007e-05 2023-08-28T02:36:14.482 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.482 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.482 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:36:14.482 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.487546+0000", 2023-08-28T02:36:14.482 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00012208100000000001 2023-08-28T02:36:14.482 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:14.482 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:36:14.482 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:14.482 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.483 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.483 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:78184 6.a 6:5da46a44:::benchmark_data_smithi143_305742_object78183:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e221)", 2023-08-28T02:36:14.483 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:28:25.332832+0000", 2023-08-28T02:36:14.483 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 469.133803806, 2023-08-28T02:36:14.483 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1550407680000001, 2023-08-28T02:36:14.483 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:36:14.483 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:36:14.483 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:36:14.483 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:36:14.483 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:36:14.483 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 78184 2023-08-28T02:36:14.484 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.484 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:36:14.484 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.484 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:36:14.484 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.332832+0000", 2023-08-28T02:36:14.484 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:14.484 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.484 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.484 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:36:14.484 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.332832+0000", 2023-08-28T02:36:14.484 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:14.484 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.484 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.485 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:36:14.485 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.332832+0000", 2023-08-28T02:36:14.485 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:36:14.485 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.485 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.485 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:36:14.485 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.332852+0000", 2023-08-28T02:36:14.485 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.0690999999999999e-05 2023-08-28T02:36:14.485 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.485 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.485 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:36:14.485 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.332859+0000", 2023-08-28T02:36:14.486 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.5520000000000001e-06 2023-08-28T02:36:14.486 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.486 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.486 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:36:14.486 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.332864+0000", 2023-08-28T02:36:14.486 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.8899999999999998e-06 2023-08-28T02:36:14.486 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.486 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.486 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:36:14.486 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.334297+0000", 2023-08-28T02:36:14.486 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0014326619999999999 2023-08-28T02:36:14.486 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.487 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.487 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:36:14.487 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.334307+0000", 2023-08-28T02:36:14.487 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.083e-05 2023-08-28T02:36:14.487 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.487 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.487 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:36:14.487 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.474666+0000", 2023-08-28T02:36:14.487 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.140358242 2023-08-28T02:36:14.487 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.487 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.487 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:36:14.488 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.474849+0000", 2023-08-28T02:36:14.488 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00018383599999999999 2023-08-28T02:36:14.488 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.488 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.488 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 2,5,6", 2023-08-28T02:36:14.488 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.475057+0000", 2023-08-28T02:36:14.488 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00020736300000000001 2023-08-28T02:36:14.488 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.488 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.488 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:36:14.488 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.481516+0000", 2023-08-28T02:36:14.488 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0064590189999999999 2023-08-28T02:36:14.489 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.489 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.489 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:36:14.489 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.481586+0000", 2023-08-28T02:36:14.489 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.9698999999999998e-05 2023-08-28T02:36:14.489 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.489 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.489 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:36:14.489 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.481737+0000", 2023-08-28T02:36:14.489 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00015137500000000001 2023-08-28T02:36:14.489 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.489 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.489 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:36:14.490 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.487703+0000", 2023-08-28T02:36:14.490 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0059662990000000004 2023-08-28T02:36:14.490 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.490 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.490 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:36:14.490 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.487758+0000", 2023-08-28T02:36:14.490 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.4861999999999998e-05 2023-08-28T02:36:14.490 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.490 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.490 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:36:14.490 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.487873+0000", 2023-08-28T02:36:14.490 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000114827 2023-08-28T02:36:14.490 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:14.491 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:36:14.491 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:14.491 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.491 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.491 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:78190 6.a 6:5f0de539:::benchmark_data_smithi143_305742_object78189:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e221)", 2023-08-28T02:36:14.491 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:28:25.484849+0000", 2023-08-28T02:36:14.491 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 468.981787118, 2023-08-28T02:36:14.491 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.003235442, 2023-08-28T02:36:14.491 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:36:14.491 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:36:14.491 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:36:14.491 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:36:14.492 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:36:14.492 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 78190 2023-08-28T02:36:14.492 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.492 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:36:14.492 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.492 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:36:14.492 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.484849+0000", 2023-08-28T02:36:14.492 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:14.492 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.492 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.492 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:36:14.492 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.484849+0000", 2023-08-28T02:36:14.493 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:14.493 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.493 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.493 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:36:14.493 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.484847+0000", 2023-08-28T02:36:14.493 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:36:14.493 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.493 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.493 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:36:14.493 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.484876+0000", 2023-08-28T02:36:14.493 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.8646e-05 2023-08-28T02:36:14.493 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.493 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.494 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:36:14.494 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.484897+0000", 2023-08-28T02:36:14.494 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.1569e-05 2023-08-28T02:36:14.494 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.494 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.494 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:36:14.494 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.484909+0000", 2023-08-28T02:36:14.494 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1476e-05 2023-08-28T02:36:14.494 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.494 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.494 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:36:14.494 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.484978+0000", 2023-08-28T02:36:14.495 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.8848000000000002e-05 2023-08-28T02:36:14.495 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.495 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.495 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:36:14.495 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.484992+0000", 2023-08-28T02:36:14.495 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.4099e-05 2023-08-28T02:36:14.495 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.495 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.495 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:36:14.495 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.477959+0000", 2023-08-28T02:36:14.495 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.99296681499999995 2023-08-28T02:36:14.495 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.496 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.496 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:36:14.496 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.478115+0000", 2023-08-28T02:36:14.496 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000156362 2023-08-28T02:36:14.496 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.496 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.496 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 2,5,6", 2023-08-28T02:36:14.496 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.478294+0000", 2023-08-28T02:36:14.496 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000178896 2023-08-28T02:36:14.496 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.496 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.496 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:36:14.497 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.482309+0000", 2023-08-28T02:36:14.497 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0040148800000000002 2023-08-28T02:36:14.497 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.497 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.497 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:36:14.497 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.482672+0000", 2023-08-28T02:36:14.497 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00036328400000000001 2023-08-28T02:36:14.497 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.497 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.497 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:36:14.497 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.486362+0000", 2023-08-28T02:36:14.497 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0036900869999999999 2023-08-28T02:36:14.497 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.498 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.498 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:36:14.498 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.487919+0000", 2023-08-28T02:36:14.498 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0015570429999999999 2023-08-28T02:36:14.498 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.498 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.498 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:36:14.498 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.487973+0000", 2023-08-28T02:36:14.498 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.3764999999999998e-05 2023-08-28T02:36:14.498 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.498 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.498 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:36:14.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:14.500 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.488084+0000", 2023-08-28T02:36:14.500 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00011128699999999999 2023-08-28T02:36:14.500 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:14.500 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:36:14.500 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:14.501 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.501 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.501 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.55270.0:59173 7.f 7:f5708084:::benchmark_data_smithi143_360738_object59172:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e268)", 2023-08-28T02:36:14.501 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:31:42.245385+0000", 2023-08-28T02:36:14.501 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 272.22125128699997, 2023-08-28T02:36:14.501 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.28729396899999998, 2023-08-28T02:36:14.501 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:36:14.501 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:36:14.501 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:36:14.501 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.55270", 2023-08-28T02:36:14.501 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:36:14.502 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 59173 2023-08-28T02:36:14.502 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.502 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:36:14.502 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.502 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:36:14.502 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:42.245385+0000", 2023-08-28T02:36:14.502 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:14.502 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.502 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.502 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:36:14.502 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:42.245385+0000", 2023-08-28T02:36:14.502 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:14.503 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.503 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.503 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:36:14.503 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:42.245383+0000", 2023-08-28T02:36:14.503 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:36:14.503 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.503 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.503 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:36:14.503 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:42.245407+0000", 2023-08-28T02:36:14.503 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.3558999999999999e-05 2023-08-28T02:36:14.503 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.503 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.504 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:36:14.504 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:42.245419+0000", 2023-08-28T02:36:14.504 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.2422999999999999e-05 2023-08-28T02:36:14.504 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.504 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.504 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:36:14.504 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:42.245427+0000", 2023-08-28T02:36:14.504 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.9510000000000006e-06 2023-08-28T02:36:14.504 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.504 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.504 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:36:14.504 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:42.245656+0000", 2023-08-28T02:36:14.505 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00022931000000000001 2023-08-28T02:36:14.505 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.505 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.505 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:36:14.505 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:42.245768+0000", 2023-08-28T02:36:14.505 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000111413 2023-08-28T02:36:14.505 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.505 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.505 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 3,7", 2023-08-28T02:36:14.505 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:42.245872+0000", 2023-08-28T02:36:14.505 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00010375899999999999 2023-08-28T02:36:14.505 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.506 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.506 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:36:14.506 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:42.247777+0000", 2023-08-28T02:36:14.506 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0019051440000000001 2023-08-28T02:36:14.506 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.506 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.506 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:36:14.506 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:42.248446+0000", 2023-08-28T02:36:14.506 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00066982200000000004 2023-08-28T02:36:14.506 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.506 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.506 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:36:14.507 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:42.532605+0000", 2023-08-28T02:36:14.507 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.28415805100000002 2023-08-28T02:36:14.507 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.507 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.507 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:36:14.507 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:42.532638+0000", 2023-08-28T02:36:14.507 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.3479000000000003e-05 2023-08-28T02:36:14.507 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.507 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.507 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:36:14.507 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:42.532679+0000", 2023-08-28T02:36:14.507 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.0587000000000003e-05 2023-08-28T02:36:14.508 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:14.508 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:36:14.508 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:14.508 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.508 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.508 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.55270.0:59964 7.6 7:60d81a83:::benchmark_data_smithi143_360738_object59963:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e270)", 2023-08-28T02:36:14.508 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:31:43.482241+0000", 2023-08-28T02:36:14.508 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 270.984395178, 2023-08-28T02:36:14.508 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.97283552100000004, 2023-08-28T02:36:14.508 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:36:14.508 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "reached pg", 2023-08-28T02:36:14.508 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:36:14.508 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.55270", 2023-08-28T02:36:14.509 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:36:14.509 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 59964 2023-08-28T02:36:14.509 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.509 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:36:14.509 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.509 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:36:14.509 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.482241+0000", 2023-08-28T02:36:14.509 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:14.509 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.509 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.509 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:36:14.509 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.482241+0000", 2023-08-28T02:36:14.509 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:14.510 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.510 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.510 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:36:14.510 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.482239+0000", 2023-08-28T02:36:14.510 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:36:14.510 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.510 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.510 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:36:14.510 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.482289+0000", 2023-08-28T02:36:14.510 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.9725000000000002e-05 2023-08-28T02:36:14.510 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.510 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.510 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:36:14.511 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.482307+0000", 2023-08-28T02:36:14.511 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.8346e-05 2023-08-28T02:36:14.511 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.511 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.511 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:36:14.511 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.482318+0000", 2023-08-28T02:36:14.511 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0604999999999999e-05 2023-08-28T02:36:14.511 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.511 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.511 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:36:14.511 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.482431+0000", 2023-08-28T02:36:14.511 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000113798 2023-08-28T02:36:14.512 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.512 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.512 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op must wait for map", 2023-08-28T02:36:14.512 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.482439+0000", 2023-08-28T02:36:14.512 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.4599999999999997e-06 2023-08-28T02:36:14.512 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.512 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.512 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:36:14.512 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.658797+0000", 2023-08-28T02:36:14.512 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.176358709 2023-08-28T02:36:14.512 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.513 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.513 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:36:14.513 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.658815+0000", 2023-08-28T02:36:14.513 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.7076999999999999e-05 2023-08-28T02:36:14.513 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.513 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.513 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:36:14.513 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:44.454972+0000", 2023-08-28T02:36:14.513 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.79615752699999998 2023-08-28T02:36:14.513 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.513 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.513 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:36:14.513 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:44.455076+0000", 2023-08-28T02:36:14.514 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000104194 2023-08-28T02:36:14.514 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:14.514 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:36:14.514 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:14.514 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.514 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.514 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.55270.0:59979 7.16 7:6eab6166:::benchmark_data_smithi143_360738_object59978:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e270)", 2023-08-28T02:36:14.514 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:31:43.482329+0000", 2023-08-28T02:36:14.514 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 270.98430644400003, 2023-08-28T02:36:14.514 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0088134209999999, 2023-08-28T02:36:14.514 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:36:14.514 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "reached pg", 2023-08-28T02:36:14.515 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:36:14.515 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.55270", 2023-08-28T02:36:14.515 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:36:14.515 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 59979 2023-08-28T02:36:14.515 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.515 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:36:14.515 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.515 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:36:14.515 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.482329+0000", 2023-08-28T02:36:14.515 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:14.515 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.515 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.515 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:36:14.516 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.482329+0000", 2023-08-28T02:36:14.516 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:14.516 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.516 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.516 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:36:14.516 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.482329+0000", 2023-08-28T02:36:14.516 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:36:14.516 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.516 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.516 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:36:14.516 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.482373+0000", 2023-08-28T02:36:14.516 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.3764999999999999e-05 2023-08-28T02:36:14.517 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.517 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.517 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:36:14.517 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.482383+0000", 2023-08-28T02:36:14.517 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0122000000000001e-05 2023-08-28T02:36:14.517 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.517 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.517 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:36:14.517 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.482388+0000", 2023-08-28T02:36:14.517 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.4870000000000002e-06 2023-08-28T02:36:14.517 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.517 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.517 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:36:14.518 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.652362+0000", 2023-08-28T02:36:14.518 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.169973178 2023-08-28T02:36:14.518 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.518 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.518 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:36:14.518 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.652373+0000", 2023-08-28T02:36:14.518 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1633e-05 2023-08-28T02:36:14.518 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.518 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.518 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:36:14.518 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:44.491068+0000", 2023-08-28T02:36:14.518 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.83869483700000003 2023-08-28T02:36:14.519 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.519 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.519 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:36:14.519 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:44.491143+0000", 2023-08-28T02:36:14.519 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.4843000000000002e-05 2023-08-28T02:36:14.519 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:14.519 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:36:14.519 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:14.519 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.519 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.519 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.55270.0:74727 7.6 7:670e8844:::benchmark_data_smithi143_360738_object74726:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e278)", 2023-08-28T02:36:14.519 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:32:04.065992+0000", 2023-08-28T02:36:14.519 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 250.400644349, 2023-08-28T02:36:14.520 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.37220715900000001, 2023-08-28T02:36:14.520 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:36:14.520 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:36:14.520 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:36:14.520 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.55270", 2023-08-28T02:36:14.520 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:36:14.520 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 74727 2023-08-28T02:36:14.520 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.520 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:36:14.520 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.520 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:36:14.520 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:04.065992+0000", 2023-08-28T02:36:14.520 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:14.521 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.521 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.521 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:36:14.521 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:04.065992+0000", 2023-08-28T02:36:14.521 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:14.521 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.521 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.521 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:36:14.521 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:04.065991+0000", 2023-08-28T02:36:14.521 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:36:14.521 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.521 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.521 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:36:14.522 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:04.066015+0000", 2023-08-28T02:36:14.522 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.4199e-05 2023-08-28T02:36:14.522 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.522 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.522 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:36:14.522 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:04.066022+0000", 2023-08-28T02:36:14.522 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.0110000000000003e-06 2023-08-28T02:36:14.522 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.522 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.522 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:36:14.522 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:04.066028+0000", 2023-08-28T02:36:14.522 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.3890000000000004e-06 2023-08-28T02:36:14.522 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.523 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.523 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:36:14.523 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:04.067925+0000", 2023-08-28T02:36:14.523 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0018972349999999999 2023-08-28T02:36:14.523 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.523 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.523 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:36:14.523 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:04.068082+0000", 2023-08-28T02:36:14.523 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00015756799999999999 2023-08-28T02:36:14.523 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.523 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.523 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 3,5", 2023-08-28T02:36:14.523 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:04.068243+0000", 2023-08-28T02:36:14.524 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000160087 2023-08-28T02:36:14.524 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.524 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.524 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:36:14.524 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:04.073712+0000", 2023-08-28T02:36:14.524 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0054692270000000001 2023-08-28T02:36:14.524 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.524 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.524 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:36:14.524 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:04.076387+0000", 2023-08-28T02:36:14.524 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0026753760000000001 2023-08-28T02:36:14.524 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.524 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.525 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:36:14.525 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:04.436679+0000", 2023-08-28T02:36:14.525 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.36029140799999998 2023-08-28T02:36:14.525 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.525 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.525 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:36:14.525 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:04.436734+0000", 2023-08-28T02:36:14.525 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.5612999999999998e-05 2023-08-28T02:36:14.525 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.525 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.525 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:36:14.525 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:04.438199+0000", 2023-08-28T02:36:14.525 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0014645369999999999 2023-08-28T02:36:14.526 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:14.526 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:36:14.526 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:14.526 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.526 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.526 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.55270.0:74729 7.6 7:612e4b7b:::benchmark_data_smithi143_360738_object74728:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e278)", 2023-08-28T02:36:14.526 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:32:04.069850+0000", 2023-08-28T02:36:14.526 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 250.39678591099999, 2023-08-28T02:36:14.526 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.36954840700000002, 2023-08-28T02:36:14.526 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:36:14.526 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:36:14.526 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:36:14.527 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.55270", 2023-08-28T02:36:14.527 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:36:14.527 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 74729 2023-08-28T02:36:14.527 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.527 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:36:14.527 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.527 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:36:14.527 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:04.069850+0000", 2023-08-28T02:36:14.527 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:14.527 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.527 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.527 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:36:14.528 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:04.069850+0000", 2023-08-28T02:36:14.528 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:14.528 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.528 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.528 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:36:14.528 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:04.069849+0000", 2023-08-28T02:36:14.528 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:36:14.528 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.528 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.528 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:36:14.528 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:04.069877+0000", 2023-08-28T02:36:14.528 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.8557000000000001e-05 2023-08-28T02:36:14.528 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.529 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.529 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:36:14.529 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:04.069888+0000", 2023-08-28T02:36:14.529 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0981999999999999e-05 2023-08-28T02:36:14.529 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.529 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.529 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:36:14.529 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:04.069896+0000", 2023-08-28T02:36:14.529 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.7400000000000004e-06 2023-08-28T02:36:14.529 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.529 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.529 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:36:14.530 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:04.073752+0000", 2023-08-28T02:36:14.530 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.003855884 2023-08-28T02:36:14.530 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.530 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.530 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:36:14.530 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:04.073914+0000", 2023-08-28T02:36:14.530 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000162351 2023-08-28T02:36:14.530 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.530 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.530 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 3,5", 2023-08-28T02:36:14.530 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:04.074085+0000", 2023-08-28T02:36:14.530 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00017091299999999999 2023-08-28T02:36:14.531 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.531 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.531 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:36:14.531 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:04.076392+0000", 2023-08-28T02:36:14.531 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0023068950000000002 2023-08-28T02:36:14.531 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.531 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.531 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:36:14.531 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:04.078039+0000", 2023-08-28T02:36:14.531 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.001646837 2023-08-28T02:36:14.531 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.531 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.532 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:36:14.532 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:04.438254+0000", 2023-08-28T02:36:14.532 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.36021553699999997 2023-08-28T02:36:14.532 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.532 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.532 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:36:14.532 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:04.438311+0000", 2023-08-28T02:36:14.532 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.6493000000000003e-05 2023-08-28T02:36:14.532 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.532 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.532 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:36:14.532 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:04.439398+0000", 2023-08-28T02:36:14.533 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0010874470000000001 2023-08-28T02:36:14.533 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:14.533 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:36:14.533 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:14.533 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.533 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.533 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.55270.0:74740 7.6 7:622f5b34:::benchmark_data_smithi143_360738_object74739:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e278)", 2023-08-28T02:36:14.533 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:32:04.079744+0000", 2023-08-28T02:36:14.533 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 250.38689228199999, 2023-08-28T02:36:14.533 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.36942792699999999, 2023-08-28T02:36:14.533 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:36:14.533 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:36:14.533 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:36:14.534 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.55270", 2023-08-28T02:36:14.534 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:36:14.534 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 74740 2023-08-28T02:36:14.534 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.534 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:36:14.534 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.534 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:36:14.534 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:04.079744+0000", 2023-08-28T02:36:14.534 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:14.534 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.534 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.534 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:36:14.535 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:04.079744+0000", 2023-08-28T02:36:14.535 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:14.535 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.535 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.535 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:36:14.535 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:04.079742+0000", 2023-08-28T02:36:14.535 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:36:14.535 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.535 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.535 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:36:14.535 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:04.079766+0000", 2023-08-28T02:36:14.535 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.3881000000000001e-05 2023-08-28T02:36:14.536 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.536 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.536 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:36:14.536 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:04.309083+0000", 2023-08-28T02:36:14.536 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.229317506 2023-08-28T02:36:14.536 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.536 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.536 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:36:14.536 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:04.309112+0000", 2023-08-28T02:36:14.536 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.8153e-05 2023-08-28T02:36:14.536 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.536 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.536 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:36:14.537 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:04.309252+0000", 2023-08-28T02:36:14.537 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00014071999999999999 2023-08-28T02:36:14.537 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.537 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.537 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:36:14.537 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:04.309451+0000", 2023-08-28T02:36:14.537 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00019867499999999999 2023-08-28T02:36:14.537 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.537 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.537 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 3,5", 2023-08-28T02:36:14.537 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:04.309594+0000", 2023-08-28T02:36:14.537 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00014310000000000001 2023-08-28T02:36:14.538 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.538 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.538 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:36:14.538 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:04.311191+0000", 2023-08-28T02:36:14.538 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.001597214 2023-08-28T02:36:14.538 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.538 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.538 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:36:14.538 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:04.312083+0000", 2023-08-28T02:36:14.538 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00089129699999999999 2023-08-28T02:36:14.538 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.538 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.538 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:36:14.539 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:04.447601+0000", 2023-08-28T02:36:14.539 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.135518521 2023-08-28T02:36:14.539 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.539 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.539 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:36:14.539 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:04.447644+0000", 2023-08-28T02:36:14.539 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.2877999999999998e-05 2023-08-28T02:36:14.539 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.539 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.539 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:36:14.539 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:04.449172+0000", 2023-08-28T02:36:14.539 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00152764 2023-08-28T02:36:14.540 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:14.540 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:36:14.540 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:14.540 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.540 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.540 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.55270.0:77038 7.f 7:fcb7506c:::benchmark_data_smithi143_360738_object77037:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e279)", 2023-08-28T02:36:14.540 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:32:07.329807+0000", 2023-08-28T02:36:14.540 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 247.13682869900001, 2023-08-28T02:36:14.540 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.297336885, 2023-08-28T02:36:14.540 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:36:14.540 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:36:14.540 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:36:14.541 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.55270", 2023-08-28T02:36:14.541 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:36:14.541 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 77038 2023-08-28T02:36:14.541 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.541 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:36:14.541 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.541 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:36:14.541 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:07.329807+0000", 2023-08-28T02:36:14.541 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:14.541 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.541 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.541 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:36:14.541 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:07.329807+0000", 2023-08-28T02:36:14.542 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:14.542 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.542 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.542 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:36:14.542 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:07.329806+0000", 2023-08-28T02:36:14.542 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:36:14.542 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.542 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.542 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:36:14.542 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:07.329829+0000", 2023-08-28T02:36:14.542 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.2756e-05 2023-08-28T02:36:14.542 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.543 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.543 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:36:14.543 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:07.329838+0000", 2023-08-28T02:36:14.543 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.687e-06 2023-08-28T02:36:14.543 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.543 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.543 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:36:14.543 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:07.329843+0000", 2023-08-28T02:36:14.543 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.2360000000000004e-06 2023-08-28T02:36:14.543 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.543 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.543 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:36:14.543 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:07.329893+0000", 2023-08-28T02:36:14.544 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.0043999999999997e-05 2023-08-28T02:36:14.544 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.544 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.544 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:36:14.544 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:07.330068+0000", 2023-08-28T02:36:14.544 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000175553 2023-08-28T02:36:14.544 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.544 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.544 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 3,4", 2023-08-28T02:36:14.544 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:07.330241+0000", 2023-08-28T02:36:14.544 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00017298500000000001 2023-08-28T02:36:14.544 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.545 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.545 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:36:14.545 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:07.333370+0000", 2023-08-28T02:36:14.545 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0031284939999999999 2023-08-28T02:36:14.545 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.545 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.545 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:36:14.545 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:07.341727+0000", 2023-08-28T02:36:14.545 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0083574419999999996 2023-08-28T02:36:14.545 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.545 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.545 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:36:14.546 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:07.624485+0000", 2023-08-28T02:36:14.546 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.28275725800000001 2023-08-28T02:36:14.546 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.546 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.546 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:36:14.546 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:07.624541+0000", 2023-08-28T02:36:14.546 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.6437999999999997e-05 2023-08-28T02:36:14.546 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.546 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.546 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:36:14.546 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:07.627144+0000", 2023-08-28T02:36:14.546 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0026030459999999999 2023-08-28T02:36:14.547 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:14.547 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:36:14.547 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:14.547 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.547 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.547 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.55270.0:77047 7.15 e279/277 7:a9594299:::benchmark_data_smithi143_360738_object77046:head v 279'4222, mlcod=0'0)", 2023-08-28T02:36:14.547 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:32:07.346098+0000", 2023-08-28T02:36:14.547 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 247.120538372, 2023-08-28T02:36:14.547 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.28439605299999998, 2023-08-28T02:36:14.547 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:36:14.547 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:36:14.547 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:36:14.548 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.548 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:36:14.548 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:07.346098+0000", 2023-08-28T02:36:14.548 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:14.548 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.548 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.548 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:36:14.548 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:07.346098+0000", 2023-08-28T02:36:14.548 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:14.548 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.548 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.548 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:36:14.549 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:07.346097+0000", 2023-08-28T02:36:14.549 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:36:14.549 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.549 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.549 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:36:14.549 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:07.346134+0000", 2023-08-28T02:36:14.549 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.7419000000000002e-05 2023-08-28T02:36:14.549 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.549 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.549 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:36:14.549 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:07.346147+0000", 2023-08-28T02:36:14.549 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3312000000000001e-05 2023-08-28T02:36:14.550 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.550 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.550 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:36:14.550 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:07.346156+0000", 2023-08-28T02:36:14.550 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.1710000000000002e-06 2023-08-28T02:36:14.550 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.550 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.550 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:36:14.550 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:07.624441+0000", 2023-08-28T02:36:14.550 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.27828547599999998 2023-08-28T02:36:14.550 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.550 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.551 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:36:14.551 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:07.624472+0000", 2023-08-28T02:36:14.551 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.1195000000000002e-05 2023-08-28T02:36:14.551 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.551 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.551 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:36:14.551 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:07.630455+0000", 2023-08-28T02:36:14.551 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0059823410000000004 2023-08-28T02:36:14.551 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.551 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.551 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:36:14.551 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:07.630494+0000", 2023-08-28T02:36:14.551 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.9053e-05 2023-08-28T02:36:14.552 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:14.552 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:36:14.552 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:14.552 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.552 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.552 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.55270.0:91015 7.f 7:f3165709:::benchmark_data_smithi143_360738_object91014:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e285)", 2023-08-28T02:36:14.552 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:32:28.941760+0000", 2023-08-28T02:36:14.552 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 225.52487547699999, 2023-08-28T02:36:14.552 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.28012970300000001, 2023-08-28T02:36:14.552 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:36:14.552 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:36:14.552 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:36:14.552 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.55270", 2023-08-28T02:36:14.553 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:36:14.553 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 91015 2023-08-28T02:36:14.553 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.553 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:36:14.553 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.553 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:36:14.553 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:28.941760+0000", 2023-08-28T02:36:14.553 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:14.553 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.553 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.553 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:36:14.554 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:28.941760+0000", 2023-08-28T02:36:14.554 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:14.554 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.554 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.554 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:36:14.554 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:28.941760+0000", 2023-08-28T02:36:14.554 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:36:14.554 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.554 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.554 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:36:14.554 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:28.941803+0000", 2023-08-28T02:36:14.554 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.2834999999999999e-05 2023-08-28T02:36:14.554 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.555 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.555 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:36:14.555 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:28.941810+0000", 2023-08-28T02:36:14.555 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.4649999999999997e-06 2023-08-28T02:36:14.555 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.555 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.555 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:36:14.555 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:28.941816+0000", 2023-08-28T02:36:14.555 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.083e-06 2023-08-28T02:36:14.555 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.555 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.556 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:36:14.556 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:28.941912+0000", 2023-08-28T02:36:14.556 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.5740000000000002e-05 2023-08-28T02:36:14.556 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.556 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.556 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:36:14.556 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:28.942068+0000", 2023-08-28T02:36:14.556 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00015557400000000001 2023-08-28T02:36:14.556 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.556 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.556 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 3,4", 2023-08-28T02:36:14.556 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:28.942230+0000", 2023-08-28T02:36:14.557 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00016269099999999999 2023-08-28T02:36:14.557 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.557 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.557 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:36:14.557 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:28.949193+0000", 2023-08-28T02:36:14.557 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0069626899999999997 2023-08-28T02:36:14.557 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.557 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.557 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:36:14.557 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:28.951280+0000", 2023-08-28T02:36:14.557 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0020867609999999999 2023-08-28T02:36:14.557 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.558 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.558 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:36:14.558 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:29.218873+0000", 2023-08-28T02:36:14.558 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.267593211 2023-08-28T02:36:14.558 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.558 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.558 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:36:14.558 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:29.218919+0000", 2023-08-28T02:36:14.558 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.6511999999999998e-05 2023-08-28T02:36:14.558 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.558 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.558 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:36:14.559 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:32:29.221890+0000", 2023-08-28T02:36:14.559 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0029706680000000001 2023-08-28T02:36:14.559 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:14.559 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:36:14.559 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:14.559 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.559 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.559 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.55270.0:183816 7.6 7:64a7abea:::benchmark_data_smithi143_360738_object91046:head [delete] snapc 0=[] ondisk+write+known_if_redirected e312)", 2023-08-28T02:36:14.559 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:34:58.223179+0000", 2023-08-28T02:36:14.559 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 76.243456491000003, 2023-08-28T02:36:14.559 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.51700689099999997, 2023-08-28T02:36:14.559 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:36:14.560 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:36:14.560 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:36:14.560 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.55270", 2023-08-28T02:36:14.560 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:36:14.560 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 183816 2023-08-28T02:36:14.560 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.560 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:36:14.560 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.560 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:36:14.560 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.223179+0000", 2023-08-28T02:36:14.560 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:14.560 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.560 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.561 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:36:14.561 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.223179+0000", 2023-08-28T02:36:14.561 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:14.561 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.561 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.561 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:36:14.561 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.223178+0000", 2023-08-28T02:36:14.561 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:36:14.561 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.561 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.561 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:36:14.561 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.223185+0000", 2023-08-28T02:36:14.561 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.9249999999999998e-06 2023-08-28T02:36:14.562 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.562 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.562 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:36:14.562 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.223202+0000", 2023-08-28T02:36:14.562 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.7431e-05 2023-08-28T02:36:14.562 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.562 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.562 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:36:14.562 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.223211+0000", 2023-08-28T02:36:14.562 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.8300000000000002e-06 2023-08-28T02:36:14.562 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.562 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.562 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:36:14.563 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.223291+0000", 2023-08-28T02:36:14.563 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.9850000000000003e-05 2023-08-28T02:36:14.563 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.563 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.563 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:36:14.563 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.223318+0000", 2023-08-28T02:36:14.563 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.7069000000000001e-05 2023-08-28T02:36:14.563 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.563 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.563 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:36:14.563 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.730695+0000", 2023-08-28T02:36:14.563 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.50737642599999999 2023-08-28T02:36:14.563 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.564 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.564 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:36:14.564 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.730805+0000", 2023-08-28T02:36:14.564 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000109931 2023-08-28T02:36:14.564 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.564 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.564 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 3,5", 2023-08-28T02:36:14.564 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.731191+0000", 2023-08-28T02:36:14.564 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00038677399999999999 2023-08-28T02:36:14.564 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.564 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.564 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:36:14.564 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.740048+0000", 2023-08-28T02:36:14.565 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0088567790000000004 2023-08-28T02:36:14.565 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.565 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.565 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:36:14.565 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.740083+0000", 2023-08-28T02:36:14.565 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.4829000000000001e-05 2023-08-28T02:36:14.565 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.565 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.565 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:36:14.565 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.740118+0000", 2023-08-28T02:36:14.565 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.4785e-05 2023-08-28T02:36:14.565 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.566 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.566 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:36:14.566 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.740157+0000", 2023-08-28T02:36:14.566 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.8954e-05 2023-08-28T02:36:14.566 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.566 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.566 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:36:14.566 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.740186+0000", 2023-08-28T02:36:14.566 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.9675e-05 2023-08-28T02:36:14.566 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:14.566 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:36:14.566 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:14.566 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.567 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.567 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.55270.0:183824 7.6 7:605eaa27:::benchmark_data_smithi143_360738_object91054:head [delete] snapc 0=[] ondisk+write+known_if_redirected e312)", 2023-08-28T02:36:14.567 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:34:58.231583+0000", 2023-08-28T02:36:14.567 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 76.235052683999996, 2023-08-28T02:36:14.567 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.51869616900000004, 2023-08-28T02:36:14.567 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:36:14.567 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:36:14.567 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:36:14.567 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.55270", 2023-08-28T02:36:14.567 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:36:14.567 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 183824 2023-08-28T02:36:14.567 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.568 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:36:14.568 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.568 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:36:14.568 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.231583+0000", 2023-08-28T02:36:14.568 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:14.568 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.568 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.568 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:36:14.568 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.231583+0000", 2023-08-28T02:36:14.568 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:14.568 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.568 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.568 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:36:14.569 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.231583+0000", 2023-08-28T02:36:14.569 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:36:14.569 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.569 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.569 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:36:14.569 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.231586+0000", 2023-08-28T02:36:14.569 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.9249999999999999e-06 2023-08-28T02:36:14.569 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.569 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.569 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:36:14.569 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.231592+0000", 2023-08-28T02:36:14.569 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.3230000000000003e-06 2023-08-28T02:36:14.570 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.570 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.570 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:36:14.570 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.231597+0000", 2023-08-28T02:36:14.570 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.1490000000000002e-06 2023-08-28T02:36:14.570 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.570 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.570 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:36:14.570 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.231662+0000", 2023-08-28T02:36:14.570 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.4727999999999994e-05 2023-08-28T02:36:14.570 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.570 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.571 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:36:14.571 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.231681+0000", 2023-08-28T02:36:14.571 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.9091000000000001e-05 2023-08-28T02:36:14.571 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.571 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.571 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:36:14.571 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.736887+0000", 2023-08-28T02:36:14.571 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.50520572200000002 2023-08-28T02:36:14.571 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.571 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.571 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:36:14.571 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.736963+0000", 2023-08-28T02:36:14.571 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.6503999999999998e-05 2023-08-28T02:36:14.572 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.572 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.572 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 3,5", 2023-08-28T02:36:14.572 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.737395+0000", 2023-08-28T02:36:14.572 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00043215499999999998 2023-08-28T02:36:14.572 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.572 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.572 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:36:14.572 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.740199+0000", 2023-08-28T02:36:14.572 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0028034829999999998 2023-08-28T02:36:14.572 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.572 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.573 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:36:14.573 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.740257+0000", 2023-08-28T02:36:14.573 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.7930999999999998e-05 2023-08-28T02:36:14.573 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.573 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.573 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:36:14.573 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.750203+0000", 2023-08-28T02:36:14.573 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0099462439999999999 2023-08-28T02:36:14.573 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.573 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.573 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:36:14.573 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.750245+0000", 2023-08-28T02:36:14.573 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.2324999999999999e-05 2023-08-28T02:36:14.574 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.574 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.574 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:36:14.574 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.750279+0000", 2023-08-28T02:36:14.574 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.4059e-05 2023-08-28T02:36:14.574 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:14.574 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:36:14.574 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:14.574 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.574 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.574 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.55270.0:183830 7.6 7:621df927:::benchmark_data_smithi143_360738_object91060:head [delete] snapc 0=[] ondisk+write+known_if_redirected e312)", 2023-08-28T02:36:14.574 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:34:58.250690+0000", 2023-08-28T02:36:14.575 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 76.215945998999999, 2023-08-28T02:36:14.575 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.50291049600000004, 2023-08-28T02:36:14.575 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:36:14.575 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:36:14.575 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:36:14.575 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.55270", 2023-08-28T02:36:14.575 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:36:14.575 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 183830 2023-08-28T02:36:14.575 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.575 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:36:14.575 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.575 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:36:14.576 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.250690+0000", 2023-08-28T02:36:14.576 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:14.576 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.576 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.576 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:36:14.576 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.250690+0000", 2023-08-28T02:36:14.576 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:14.576 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.576 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.576 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:36:14.576 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.250689+0000", 2023-08-28T02:36:14.576 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:36:14.577 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.577 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.577 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:36:14.577 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.250695+0000", 2023-08-28T02:36:14.577 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.9789999999999998e-06 2023-08-28T02:36:14.577 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.577 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.577 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:36:14.577 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.250705+0000", 2023-08-28T02:36:14.577 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0713000000000001e-05 2023-08-28T02:36:14.577 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.577 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.578 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:36:14.578 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.250711+0000", 2023-08-28T02:36:14.578 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.2260000000000002e-06 2023-08-28T02:36:14.578 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.578 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.578 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:36:14.578 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.250769+0000", 2023-08-28T02:36:14.578 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.7429999999999997e-05 2023-08-28T02:36:14.578 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.578 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.578 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:36:14.578 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.250783+0000", 2023-08-28T02:36:14.579 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.4436e-05 2023-08-28T02:36:14.579 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.579 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.579 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:36:14.579 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.740299+0000", 2023-08-28T02:36:14.579 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.48951565000000002 2023-08-28T02:36:14.579 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.579 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.579 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:36:14.579 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.740404+0000", 2023-08-28T02:36:14.579 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000104813 2023-08-28T02:36:14.579 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.579 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.580 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 3,5", 2023-08-28T02:36:14.580 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.740857+0000", 2023-08-28T02:36:14.580 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00045316300000000002 2023-08-28T02:36:14.580 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.580 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.580 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:36:14.580 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.750296+0000", 2023-08-28T02:36:14.580 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0094393029999999996 2023-08-28T02:36:14.580 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.580 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.580 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:36:14.580 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.750331+0000", 2023-08-28T02:36:14.581 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.4371e-05 2023-08-28T02:36:14.581 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.581 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.581 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:36:14.581 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.753510+0000", 2023-08-28T02:36:14.581 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.003179243 2023-08-28T02:36:14.581 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.581 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.581 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:36:14.581 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.753561+0000", 2023-08-28T02:36:14.581 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.1041000000000001e-05 2023-08-28T02:36:14.581 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.581 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.582 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:36:14.582 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.753600+0000", 2023-08-28T02:36:14.582 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.9527000000000003e-05 2023-08-28T02:36:14.582 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:14.582 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:36:14.582 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:14.582 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.582 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.582 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.55270.0:183848 7.6 7:6465d94b:::benchmark_data_smithi143_360738_object91078:head [delete] snapc 0=[] ondisk+write+known_if_redirected e312)", 2023-08-28T02:36:14.582 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:34:58.286258+0000", 2023-08-28T02:36:14.582 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 76.180377784000001, 2023-08-28T02:36:14.582 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.47374068000000003, 2023-08-28T02:36:14.583 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:36:14.583 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:36:14.583 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:36:14.583 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.55270", 2023-08-28T02:36:14.583 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:36:14.583 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 183848 2023-08-28T02:36:14.583 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.583 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:36:14.583 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.583 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:36:14.583 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.286258+0000", 2023-08-28T02:36:14.583 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:14.583 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.584 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.584 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:36:14.584 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.286258+0000", 2023-08-28T02:36:14.584 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:14.584 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.584 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.584 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:36:14.584 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.286257+0000", 2023-08-28T02:36:14.584 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:36:14.584 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.584 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.584 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:36:14.585 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.286264+0000", 2023-08-28T02:36:14.585 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.0700000000000001e-06 2023-08-28T02:36:14.585 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.585 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.585 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:36:14.585 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.286282+0000", 2023-08-28T02:36:14.585 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.7699000000000001e-05 2023-08-28T02:36:14.585 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.585 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.585 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:36:14.585 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.286289+0000", 2023-08-28T02:36:14.585 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.1470000000000004e-06 2023-08-28T02:36:14.585 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.586 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.586 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:36:14.586 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.286336+0000", 2023-08-28T02:36:14.586 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.6845999999999998e-05 2023-08-28T02:36:14.586 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.586 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.586 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:36:14.586 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.286345+0000", 2023-08-28T02:36:14.586 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.7249999999999992e-06 2023-08-28T02:36:14.586 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.586 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.586 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:36:14.587 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.743730+0000", 2023-08-28T02:36:14.587 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.45738426500000001 2023-08-28T02:36:14.587 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.587 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.587 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:36:14.587 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.743878+0000", 2023-08-28T02:36:14.587 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000148904 2023-08-28T02:36:14.587 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.587 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.587 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 3,5", 2023-08-28T02:36:14.587 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.744351+0000", 2023-08-28T02:36:14.587 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00047262399999999999 2023-08-28T02:36:14.588 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.588 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.588 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:36:14.588 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.750302+0000", 2023-08-28T02:36:14.588 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0059504329999999998 2023-08-28T02:36:14.588 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.588 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.588 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:36:14.588 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.750362+0000", 2023-08-28T02:36:14.588 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.0319000000000003e-05 2023-08-28T02:36:14.588 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.588 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.589 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:36:14.589 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.759902+0000", 2023-08-28T02:36:14.589 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0095404119999999999 2023-08-28T02:36:14.589 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.589 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.589 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:36:14.589 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.759953+0000", 2023-08-28T02:36:14.589 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.0980999999999999e-05 2023-08-28T02:36:14.589 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.589 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.589 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:36:14.589 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.759999+0000", 2023-08-28T02:36:14.589 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.5605000000000001e-05 2023-08-28T02:36:14.590 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:14.590 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:36:14.590 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:14.590 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.590 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.590 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.55270.0:183863 7.6 7:6624c7a9:::benchmark_data_smithi143_360738_object91093:head [delete] snapc 0=[] ondisk+write+known_if_redirected e312)", 2023-08-28T02:36:14.590 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:34:58.318108+0000", 2023-08-28T02:36:14.590 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 76.148527442000002, 2023-08-28T02:36:14.590 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.44403041599999998, 2023-08-28T02:36:14.590 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:36:14.590 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:36:14.590 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:36:14.591 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.55270", 2023-08-28T02:36:14.591 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:36:14.591 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 183863 2023-08-28T02:36:14.591 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.591 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:36:14.591 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.591 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:36:14.591 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.318108+0000", 2023-08-28T02:36:14.591 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:14.591 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.591 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.591 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:36:14.592 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.318108+0000", 2023-08-28T02:36:14.592 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:14.592 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.592 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.592 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:36:14.592 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.318107+0000", 2023-08-28T02:36:14.592 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:36:14.592 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.592 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.592 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:36:14.592 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.318115+0000", 2023-08-28T02:36:14.592 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.464e-06 2023-08-28T02:36:14.593 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.593 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.593 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:36:14.593 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.318131+0000", 2023-08-28T02:36:14.593 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.6291e-05 2023-08-28T02:36:14.593 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.593 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.593 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:36:14.593 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.318141+0000", 2023-08-28T02:36:14.593 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0084999999999999e-05 2023-08-28T02:36:14.593 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.593 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.594 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:36:14.594 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.318194+0000", 2023-08-28T02:36:14.594 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.2573e-05 2023-08-28T02:36:14.594 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.594 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.594 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:36:14.594 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.318206+0000", 2023-08-28T02:36:14.594 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.2184000000000001e-05 2023-08-28T02:36:14.594 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.594 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.594 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:36:14.594 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.746981+0000", 2023-08-28T02:36:14.594 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.42877542499999999 2023-08-28T02:36:14.595 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.595 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.595 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:36:14.595 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.747105+0000", 2023-08-28T02:36:14.595 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000123819 2023-08-28T02:36:14.595 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.595 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.595 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 3,5", 2023-08-28T02:36:14.595 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.747602+0000", 2023-08-28T02:36:14.595 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00049703099999999999 2023-08-28T02:36:14.595 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.595 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.596 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:36:14.596 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.750428+0000", 2023-08-28T02:36:14.596 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.002825611 2023-08-28T02:36:14.596 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.596 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.596 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:36:14.596 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.760036+0000", 2023-08-28T02:36:14.596 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0096087910000000002 2023-08-28T02:36:14.596 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.596 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.596 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:36:14.596 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.762046+0000", 2023-08-28T02:36:14.596 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.002009837 2023-08-28T02:36:14.597 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.597 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.597 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:36:14.597 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.762096+0000", 2023-08-28T02:36:14.597 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.9490999999999998e-05 2023-08-28T02:36:14.597 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.597 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.597 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:36:14.597 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:34:58.762139+0000", 2023-08-28T02:36:14.597 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.3158000000000003e-05 2023-08-28T02:36:14.597 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:14.597 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:36:14.598 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:14.598 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.598 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.598 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.45647.0:40973 8.6 8:6c845e2e:::benchmark_data_smithi143_419289_object40972:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e330)", 2023-08-28T02:36:14.598 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:35:49.708196+0000", 2023-08-28T02:36:14.598 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 24.758440293, 2023-08-28T02:36:14.598 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.27692169799999999, 2023-08-28T02:36:14.598 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:36:14.598 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:36:14.598 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:36:14.598 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.45647", 2023-08-28T02:36:14.598 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3230872490", 2023-08-28T02:36:14.598 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 40973 2023-08-28T02:36:14.599 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.599 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:36:14.599 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.599 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:36:14.599 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:49.708196+0000", 2023-08-28T02:36:14.599 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:14.599 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.599 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.599 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:36:14.599 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:49.708196+0000", 2023-08-28T02:36:14.599 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:14.599 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.600 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.600 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:36:14.600 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:49.708195+0000", 2023-08-28T02:36:14.600 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:36:14.600 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.600 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.600 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:36:14.600 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:49.708217+0000", 2023-08-28T02:36:14.600 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.2323e-05 2023-08-28T02:36:14.600 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.600 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.600 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:36:14.601 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:49.708224+0000", 2023-08-28T02:36:14.601 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.9129999999999997e-06 2023-08-28T02:36:14.601 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.601 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.601 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:36:14.601 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:49.708229+0000", 2023-08-28T02:36:14.601 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.8640000000000004e-06 2023-08-28T02:36:14.601 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.601 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.601 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:36:14.601 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:49.708389+0000", 2023-08-28T02:36:14.601 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00015973800000000001 2023-08-28T02:36:14.602 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.602 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.602 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:36:14.602 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:49.708513+0000", 2023-08-28T02:36:14.602 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00012395400000000001 2023-08-28T02:36:14.602 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.602 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.602 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 2,6", 2023-08-28T02:36:14.602 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:49.708706+0000", 2023-08-28T02:36:14.602 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000192974 2023-08-28T02:36:14.602 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.602 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.603 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:36:14.603 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:49.713375+0000", 2023-08-28T02:36:14.603 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0046695299999999999 2023-08-28T02:36:14.603 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.603 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.603 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:36:14.603 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:49.713696+0000", 2023-08-28T02:36:14.603 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00032035199999999998 2023-08-28T02:36:14.603 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.603 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.603 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:36:14.603 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:49.985017+0000", 2023-08-28T02:36:14.604 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.27132078199999998 2023-08-28T02:36:14.604 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.604 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.604 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:36:14.604 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:49.985070+0000", 2023-08-28T02:36:14.604 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.3699000000000002e-05 2023-08-28T02:36:14.604 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.604 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.604 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:36:14.604 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:49.985117+0000", 2023-08-28T02:36:14.604 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.7120999999999998e-05 2023-08-28T02:36:14.604 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:14.604 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:36:14.605 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:14.605 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.605 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.605 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.45647.0:40978 8.e 8:7f42d463:::benchmark_data_smithi143_419289_object40977:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e330)", 2023-08-28T02:36:14.605 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:35:49.711242+0000", 2023-08-28T02:36:14.605 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 24.755394082999999, 2023-08-28T02:36:14.605 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.27639176999999998, 2023-08-28T02:36:14.605 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:36:14.605 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:36:14.605 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:36:14.605 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.45647", 2023-08-28T02:36:14.605 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3230872490", 2023-08-28T02:36:14.606 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 40978 2023-08-28T02:36:14.606 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.606 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:36:14.606 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.606 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:36:14.606 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:49.711242+0000", 2023-08-28T02:36:14.606 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:14.606 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.606 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.606 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:36:14.606 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:49.711242+0000", 2023-08-28T02:36:14.606 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:14.607 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.607 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.607 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:36:14.607 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:49.711240+0000", 2023-08-28T02:36:14.607 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:36:14.607 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.607 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.607 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:36:14.607 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:49.711264+0000", 2023-08-28T02:36:14.607 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.4042000000000001e-05 2023-08-28T02:36:14.607 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.607 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.608 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:36:14.608 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:49.711275+0000", 2023-08-28T02:36:14.608 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1066e-05 2023-08-28T02:36:14.608 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.608 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.608 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:36:14.608 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:49.711282+0000", 2023-08-28T02:36:14.608 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.9330000000000004e-06 2023-08-28T02:36:14.608 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.608 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.608 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:36:14.608 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:49.711329+0000", 2023-08-28T02:36:14.608 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.6965000000000002e-05 2023-08-28T02:36:14.609 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.609 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.609 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:36:14.609 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:49.711501+0000", 2023-08-28T02:36:14.609 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00017164900000000001 2023-08-28T02:36:14.609 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.609 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.609 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 2,4", 2023-08-28T02:36:14.609 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:49.711652+0000", 2023-08-28T02:36:14.609 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000151321 2023-08-28T02:36:14.609 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.609 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.610 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:36:14.610 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:49.713701+0000", 2023-08-28T02:36:14.610 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0020488250000000002 2023-08-28T02:36:14.610 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.610 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.610 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:36:14.610 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:49.717604+0000", 2023-08-28T02:36:14.610 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0039032379999999998 2023-08-28T02:36:14.610 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.610 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.610 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:36:14.610 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:49.987545+0000", 2023-08-28T02:36:14.611 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.269940703 2023-08-28T02:36:14.611 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.611 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.611 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:36:14.611 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:49.987597+0000", 2023-08-28T02:36:14.611 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.1851999999999998e-05 2023-08-28T02:36:14.611 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:14.611 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:14.611 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:36:14.611 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:49.987634+0000", 2023-08-28T02:36:14.611 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.6862e-05 2023-08-28T02:36:14.611 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:14.611 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:36:14.612 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:14.612 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:14.612 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:36:14.612 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:36:14.612 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:36:14.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:36:14.713 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:36:14.769 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:14 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3343859822' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-08-28T02:36:14.769 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:14 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-08-28T02:36:14.779 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:14.794 INFO:teuthology.orchestra.run.smithi143.stderr:reweighted osd.1 to 1 (10000) 2023-08-28T02:36:14.896 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:36:14.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:14.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:36:15.006 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:36:15.053 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:14 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3343859822' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-08-28T02:36:15.053 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:14 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-08-28T02:36:15.058 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:15.165 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:36:15.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:15.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:36:15.198 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 52 16 67273 67257 80.7977 66.375 0.0181239 0.0123714 2023-08-28T02:36:15.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:14 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3343859822' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-08-28T02:36:15.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:14 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-08-28T02:36:15.273 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:36:15.299 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:15.340 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:36:15.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:15.345 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:36:15.405 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset nodeep-scrub 2023-08-28T02:36:15.446 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:36:15.477 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:15.517 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:36:15.524 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:36:15.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:15.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:36:15.629 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:36:15.732 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:16.198 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 53 16 68047 68031 80.1861 48.375 0.00541707 0.0124657 2023-08-28T02:36:16.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:15 smithi143 ceph-mon[183059]: failed to encode map e335 with expected crc 2023-08-28T02:36:16.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:15 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-08-28T02:36:16.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:15 smithi143 ceph-mon[183059]: osdmap e335: 8 total, 8 up, 7 in 2023-08-28T02:36:16.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:15 smithi143 ceph-mon[183059]: failed to encode map e335 with expected crc 2023-08-28T02:36:16.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:15 smithi143 ceph-mon[183059]: failed to encode map e335 with expected crc 2023-08-28T02:36:16.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:15 smithi143 ceph-mon[183059]: pgmap v869: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-08-28T02:36:16.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:15 smithi143 ceph-mon[189871]: failed to encode map e335 with expected crc 2023-08-28T02:36:16.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:15 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-08-28T02:36:16.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:15 smithi143 ceph-mon[189871]: osdmap e335: 8 total, 8 up, 7 in 2023-08-28T02:36:16.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:15 smithi143 ceph-mon[189871]: failed to encode map e335 with expected crc 2023-08-28T02:36:16.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:15 smithi143 ceph-mon[189871]: failed to encode map e335 with expected crc 2023-08-28T02:36:16.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:15 smithi143 ceph-mon[189871]: pgmap v869: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-08-28T02:36:16.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:15 smithi169 ceph-mon[159415]: failed to encode map e335 with expected crc 2023-08-28T02:36:16.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:15 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-08-28T02:36:16.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:15 smithi169 ceph-mon[159415]: osdmap e335: 8 total, 8 up, 7 in 2023-08-28T02:36:16.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:15 smithi169 ceph-mon[159415]: failed to encode map e335 with expected crc 2023-08-28T02:36:16.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:15 smithi169 ceph-mon[159415]: failed to encode map e335 with expected crc 2023-08-28T02:36:16.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:15 smithi169 ceph-mon[159415]: pgmap v869: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-08-28T02:36:16.487 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:36:16.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:16.493 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:36:16.520 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.1 dump_ops_in_flight 2023-08-28T02:36:16.594 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:36:16.596 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-08-28T02:36:16.596 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph pg dump --format=json 2023-08-28T02:36:16.714 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:16.860 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:36:16.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:16.868 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:36:16.903 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:36:16.963 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:36:16.969 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:36:17.080 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:17.198 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 54 16 68806 68790 79.5798 47.4375 0.0168689 0.0125597 2023-08-28T02:36:17.315 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:36:17.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:17.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:36:17.421 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:36:17.506 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:18.091 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:36:18.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:18.099 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:36:18.198 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 55 16 69758 69742 79.2148 59.5 0.0236137 0.0126178 2023-08-28T02:36:18.201 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:36:18.227 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:18.269 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:36:18.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:18.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:36:18.376 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:36:18.401 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:18.443 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:36:18.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:18.447 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:36:18.547 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:36:18.577 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:18.638 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:36:18.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:18.642 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:36:18.743 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:36:18.777 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:18.861 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:36:18.867 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:18 smithi169 ceph-mon[159415]: pgmap v870: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 57 MiB/s wr, 911 op/s 2023-08-28T02:36:18.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:18.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:36:18.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:18 smithi143 ceph-mon[183059]: pgmap v870: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 57 MiB/s wr, 911 op/s 2023-08-28T02:36:18.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:18 smithi143 ceph-mon[189871]: pgmap v870: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 57 MiB/s wr, 911 op/s 2023-08-28T02:36:18.969 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:36:18.997 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:19.201 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 56 16 70378 70362 78.4886 38.75 0.0200808 0.012736 2023-08-28T02:36:19.515 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:36:19.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:19.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:36:19.549 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:36:19.624 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:36:19.636 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:36:19.658 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:19.720 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:36:19.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:19.724 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:36:19.824 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:36:19.830 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:19 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:36:19.834 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:19 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:36:19.925 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:19.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:19 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:36:20.201 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 57 16 71345 71329 78.1718 60.4375 0.0098682 0.0127877 2023-08-28T02:36:20.267 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:36:20.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:20.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:36:20.380 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:36:20.413 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:20.525 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:36:20.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:20.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:36:20.631 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:36:20.635 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:36:20.670 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:20 smithi169 ceph-mon[159415]: pgmap v871: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 59 MiB/s wr, 938 op/s 2023-08-28T02:36:20.670 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:20 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/64361638' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:36:20.673 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:20.678 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:36:20.720 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:36:20.734 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:36:20.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:20.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:36:20.766 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:36:20.817 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:36:20.841 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:36:20.846 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:20 smithi143 ceph-mon[183059]: pgmap v871: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 59 MiB/s wr, 938 op/s 2023-08-28T02:36:20.846 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:20 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/64361638' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:36:20.848 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:20 smithi143 ceph-mon[189871]: pgmap v871: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 59 MiB/s wr, 938 op/s 2023-08-28T02:36:20.848 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:20 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/64361638' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:36:20.859 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:36:20.914 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:36:20.916 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:20.949 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:36:20.951 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:36:20.952 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:36:20.952 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:36:20.952 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:36:20.952 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:36:20.952 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:36:20.952 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:36:20.952 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:36:21.202 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 58 16 72225 72209 77.7724 55 0.0169181 0.0128526 2023-08-28T02:36:21.646 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is unset 2023-08-28T02:36:21.936 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:21 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1297295516' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:36:21.936 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:21 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:36:21.937 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:36:21 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:36:21] "GET /metrics HTTP/1.1" 200 36530 "" "Prometheus/2.33.4" 2023-08-28T02:36:21.937 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:21 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1297295516' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:36:21.937 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:21 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:36:21.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:21 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1297295516' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:36:21.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:21 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:36:22.202 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 59 16 72754 72738 77.0149 33.0625 0.00310004 0.0129681 2023-08-28T02:36:22.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:22 smithi143 ceph-mon[183059]: pgmap v872: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 58 MiB/s wr, 925 op/s 2023-08-28T02:36:22.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:22 smithi143 ceph-mon[183059]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:36:22.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:22 smithi143 ceph-mon[183059]: Cluster is now healthy 2023-08-28T02:36:22.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:22 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:36:22.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:22 smithi143 ceph-mon[183059]: osdmap e336: 8 total, 8 up, 7 in 2023-08-28T02:36:22.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:22 smithi143 ceph-mon[183059]: failed to encode map e336 with expected crc 2023-08-28T02:36:22.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:22 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:36:22.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:22 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:36:22.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:22 smithi143 ceph-mon[189871]: pgmap v872: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 58 MiB/s wr, 925 op/s 2023-08-28T02:36:22.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:22 smithi143 ceph-mon[189871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:36:22.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:22 smithi143 ceph-mon[189871]: Cluster is now healthy 2023-08-28T02:36:22.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:22 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:36:22.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:22 smithi143 ceph-mon[189871]: osdmap e336: 8 total, 8 up, 7 in 2023-08-28T02:36:22.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:22 smithi143 ceph-mon[189871]: failed to encode map e336 with expected crc 2023-08-28T02:36:22.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:36:22.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:22 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:36:22.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:22 smithi169 ceph-mon[159415]: pgmap v872: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 58 MiB/s wr, 925 op/s 2023-08-28T02:36:22.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:22 smithi169 ceph-mon[159415]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:36:22.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:22 smithi169 ceph-mon[159415]: Cluster is now healthy 2023-08-28T02:36:22.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:22 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:36:22.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:22 smithi169 ceph-mon[159415]: osdmap e336: 8 total, 8 up, 7 in 2023-08-28T02:36:22.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:22 smithi169 ceph-mon[159415]: failed to encode map e336 with expected crc 2023-08-28T02:36:22.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:22 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:36:22.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:22 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:36:23.202 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:2023-08-28T02:36:23.203426+0000 min lat: 0.00106302 max lat: 1.01701 avg lat: 0.0130217 2023-08-28T02:36:23.202 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-08-28T02:36:23.202 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 60 16 73729 73713 76.7469 60.9375 0.0408348 0.0130217 2023-08-28T02:36:23.747 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:36:23.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:23.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:36:23.857 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:36:23.900 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:23.960 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:36:23.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:23.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:36:24.066 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:36:24.104 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:24.159 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:36:24.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:24.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:36:24.202 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 61 16 74605 74589 76.3862 54.75 0.00943091 0.0130853 2023-08-28T02:36:24.264 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:36:24.297 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:24.351 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:36:24.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:24.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:36:24.456 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:36:24.517 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:24.660 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:24 smithi143 ceph-mon[183059]: pgmap v874: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 48 MiB/s wr, 762 op/s 2023-08-28T02:36:24.888 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:36:24.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:24.906 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:36:24.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:24 smithi143 ceph-mon[189871]: pgmap v874: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 48 MiB/s wr, 762 op/s 2023-08-28T02:36:24.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:24 smithi169 ceph-mon[159415]: pgmap v874: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 48 MiB/s wr, 762 op/s 2023-08-28T02:36:25.007 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:36:25.011 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:36:25.011 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:36:25.037 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:25.093 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:36:25.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:25.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:36:25.199 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:36:25.204 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 62 16 75473 75457 76.0292 54.25 0.00973803 0.0131467 2023-08-28T02:36:25.249 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:36:25.278 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:25.285 INFO:teuthology.orchestra.run.smithi143.stderr:dumped all 2023-08-28T02:36:25.829 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set noscrub 2023-08-28T02:36:26.051 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:36:26.202 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 63 16 76205 76189 75.5487 45.75 0.00937164 0.0132315 2023-08-28T02:36:26.622 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:36:26.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:26.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:36:26.731 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:36:26.757 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:26.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:26 smithi143 ceph-mon[183059]: from='client.55693 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:36:26.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:26 smithi143 ceph-mon[183059]: pgmap v875: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 53 MiB/s wr, 847 op/s 2023-08-28T02:36:26.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:26 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:36:26.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:26 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:36:26.958 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:26 smithi143 ceph-mon[189871]: from='client.55693 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:36:26.958 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:26 smithi143 ceph-mon[189871]: pgmap v875: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 53 MiB/s wr, 847 op/s 2023-08-28T02:36:26.958 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:26 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:36:26.959 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:26 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:36:26.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:26 smithi169 ceph-mon[159415]: from='client.55693 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:36:26.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:26 smithi169 ceph-mon[159415]: pgmap v875: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 53 MiB/s wr, 847 op/s 2023-08-28T02:36:26.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:26 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:36:26.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:26 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:36:27.147 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.1 dump_blocked_ops 2023-08-28T02:36:27.202 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 64 16 77100 77084 75.2423 55.9375 0.00754973 0.0132844 2023-08-28T02:36:27.311 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:36:27.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:27.323 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:36:27.424 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:36:27.456 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:27.509 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:36:27.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:27.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:36:27.522 INFO:teuthology.orchestra.run.smithi143.stdout:{"pg_ready":true,"pg_map":{"version":874,"stamp":"2023-08-28T02:36:23.511114+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4713813939,"num_objects":71882,"num_object_clones":0,"num_object_copies":215646,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":71882,"num_whiteouts":0,"num_read":6274,"num_read_kb":13707,"num_write":72960,"num_write_kb":4613797,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":297,"num_bytes_recovered":14969132,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":73024,"ondisk_log_size":73024,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":17535984,"kb_used_data":13808120,"kb_used_omap":29,"kb_used_meta":3727778,"kb_avail":638680080,"statfs":{"total":671965249536,"available":654008401920,"internally_reserved":0,"allocated":14139514880,"data_stored":14130486772,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30674,"internal_metadata":3817244718},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,5,7,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":436535296,"num_objects":6661,"num_object_clones":0,"num_object_copies":19983,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6661,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6661,"num_write_kb":426304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.732427"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":355,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.807107+0000","last_change":"2023-08-28T02:32:58.740384+0000","last_active":"2023-08-28T02:36:14.807107+0000","last_peered":"2023-08-28T02:36:14.807107+0000","last_clean":"2023-08-28T02:36:14.807107+0000","last_became_active":"2023-08-28T02:32:58.739959+0000","last_became_peered":"2023-08-28T02:32:58.739959+0000","last_unstale":"2023-08-28T02:36:14.807107+0000","last_undegraded":"2023-08-28T02:36:14.807107+0000","last_fullsized":"2023-08-28T02:36:14.807107+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":241,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-08-28T02:36:10.723468+0000","last_change":"2023-08-28T02:32:58.634375+0000","last_active":"2023-08-28T02:36:10.723468+0000","last_peered":"2023-08-28T02:36:10.723468+0000","last_clean":"2023-08-28T02:36:10.723468+0000","last_became_active":"2023-08-28T02:32:58.632726+0000","last_became_peered":"2023-08-28T02:32:58.632726+0000","last_unstale":"2023-08-28T02:36:10.723468+0000","last_undegraded":"2023-08-28T02:36:10.723468+0000","last_fullsized":"2023-08-28T02:36:10.723468+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:20:27.730725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":56,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.789066+0000","last_change":"2023-08-28T02:32:58.628387+0000","last_active":"2023-08-28T02:36:14.789066+0000","last_peered":"2023-08-28T02:36:14.789066+0000","last_clean":"2023-08-28T02:36:14.789066+0000","last_became_active":"2023-08-28T02:32:58.628071+0000","last_became_peered":"2023-08-28T02:32:58.628071+0000","last_unstale":"2023-08-28T02:36:14.789066+0000","last_undegraded":"2023-08-28T02:36:14.789066+0000","last_fullsized":"2023-08-28T02:36:14.789066+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:25:55.264433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":352,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-08-28T02:36:10.727675+0000","last_change":"2023-08-28T02:32:58.772285+0000","last_active":"2023-08-28T02:36:10.727675+0000","last_peered":"2023-08-28T02:36:10.727675+0000","last_clean":"2023-08-28T02:36:10.727675+0000","last_became_active":"2023-08-28T02:32:58.771439+0000","last_became_peered":"2023-08-28T02:32:58.771439+0000","last_unstale":"2023-08-28T02:36:10.727675+0000","last_undegraded":"2023-08-28T02:36:10.727675+0000","last_fullsized":"2023-08-28T02:36:10.727675+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:10:48.541785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":233,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-08-28T02:36:10.733988+0000","last_change":"2023-08-28T02:32:58.779602+0000","last_active":"2023-08-28T02:36:10.733988+0000","last_peered":"2023-08-28T02:36:10.733988+0000","last_clean":"2023-08-28T02:36:10.733988+0000","last_became_active":"2023-08-28T02:32:58.778945+0000","last_became_peered":"2023-08-28T02:32:58.778945+0000","last_unstale":"2023-08-28T02:36:10.733988+0000","last_undegraded":"2023-08-28T02:36:10.733988+0000","last_fullsized":"2023-08-28T02:36:10.733988+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:35:55.832928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":142,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.793841+0000","last_change":"2023-08-28T02:32:58.626766+0000","last_active":"2023-08-28T02:36:14.793841+0000","last_peered":"2023-08-28T02:36:14.793841+0000","last_clean":"2023-08-28T02:36:14.793841+0000","last_became_active":"2023-08-28T02:32:58.626415+0000","last_became_peered":"2023-08-28T02:32:58.626415+0000","last_unstale":"2023-08-28T02:36:14.793841+0000","last_undegraded":"2023-08-28T02:36:14.793841+0000","last_fullsized":"2023-08-28T02:36:14.793841+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":305,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-08-28T02:36:10.744184+0000","last_change":"2023-08-28T02:23:59.227592+0000","last_active":"2023-08-28T02:36:10.744184+0000","last_peered":"2023-08-28T02:36:10.744184+0000","last_clean":"2023-08-28T02:36:10.744184+0000","last_became_active":"2023-08-28T02:23:59.227342+0000","last_became_peered":"2023-08-28T02:23:59.227342+0000","last_unstale":"2023-08-28T02:36:10.744184+0000","last_undegraded":"2023-08-28T02:36:10.744184+0000","last_fullsized":"2023-08-28T02:36:10.744184+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:08:19.874396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":251,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-08-28T02:36:10.734067+0000","last_change":"2023-08-28T02:32:58.779435+0000","last_active":"2023-08-28T02:36:10.734067+0000","last_peered":"2023-08-28T02:36:10.734067+0000","last_clean":"2023-08-28T02:36:10.734067+0000","last_became_active":"2023-08-28T02:32:58.778629+0000","last_became_peered":"2023-08-28T02:32:58.778629+0000","last_unstale":"2023-08-28T02:36:10.734067+0000","last_undegraded":"2023-08-28T02:36:10.734067+0000","last_fullsized":"2023-08-28T02:36:10.734067+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:25:32.193898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":344,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.787998+0000","last_change":"2023-08-28T02:35:05.992277+0000","last_active":"2023-08-28T02:36:14.787998+0000","last_peered":"2023-08-28T02:36:14.787998+0000","last_clean":"2023-08-28T02:36:14.787998+0000","last_became_active":"2023-08-28T02:35:05.991954+0000","last_became_peered":"2023-08-28T02:35:05.991954+0000","last_unstale":"2023-08-28T02:36:14.787998+0000","last_undegraded":"2023-08-28T02:36:14.787998+0000","last_fullsized":"2023-08-28T02:36:14.787998+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:41:10.177837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":224,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-08-28T02:36:10.723566+0000","last_change":"2023-08-28T02:31:55.525178+0000","last_active":"2023-08-28T02:36:10.723566+0000","last_peered":"2023-08-28T02:36:10.723566+0000","last_clean":"2023-08-28T02:36:10.723566+0000","last_became_active":"2023-08-28T02:31:55.524801+0000","last_became_peered":"2023-08-28T02:31:55.524801+0000","last_unstale":"2023-08-28T02:36:10.723566+0000","last_undegraded":"2023-08-28T02:36:10.723566+0000","last_fullsized":"2023-08-28T02:36:10.723566+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:47:30.078960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":102,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-08-28T02:36:10.725004+0000","last_change":"2023-08-28T02:31:55.532306+0000","last_active":"2023-08-28T02:36:10.725004+0000","last_peered":"2023-08-28T02:36:10.725004+0000","last_clean":"2023-08-28T02:36:10.725004+0000","last_became_active":"2023-08-28T02:31:55.524276+0000","last_became_peered":"2023-08-28T02:31:55.524276+0000","last_unstale":"2023-08-28T02:36:10.725004+0000","last_undegraded":"2023-08-28T02:36:10.725004+0000","last_fullsized":"2023-08-28T02:36:10.725004+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:27:06.850784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":67,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.793495+0000","last_change":"2023-08-28T02:35:45.112374+0000","last_active":"2023-08-28T02:36:14.793495+0000","last_peered":"2023-08-28T02:36:14.793495+0000","last_clean":"2023-08-28T02:36:14.793495+0000","last_became_active":"2023-08-28T02:35:45.112024+0000","last_became_peered":"2023-08-28T02:35:45.112024+0000","last_unstale":"2023-08-28T02:36:14.793495+0000","last_undegraded":"2023-08-28T02:36:14.793495+0000","last_fullsized":"2023-08-28T02:36:14.793495+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"86'22","reported_seq":405,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.807033+0000","last_change":"2023-08-28T02:31:55.538001+0000","last_active":"2023-08-28T02:36:14.807033+0000","last_peered":"2023-08-28T02:36:14.807033+0000","last_clean":"2023-08-28T02:36:14.807033+0000","last_became_active":"2023-08-28T02:31:55.518604+0000","last_became_peered":"2023-08-28T02:31:55.518604+0000","last_unstale":"2023-08-28T02:36:14.807033+0000","last_undegraded":"2023-08-28T02:36:14.807033+0000","last_fullsized":"2023-08-28T02:36:14.807033+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":379,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.806514+0000","last_change":"2023-08-28T02:32:58.629451+0000","last_active":"2023-08-28T02:36:14.806514+0000","last_peered":"2023-08-28T02:36:14.806514+0000","last_clean":"2023-08-28T02:36:14.806514+0000","last_became_active":"2023-08-28T02:32:58.628767+0000","last_became_peered":"2023-08-28T02:32:58.628767+0000","last_unstale":"2023-08-28T02:36:14.806514+0000","last_undegraded":"2023-08-28T02:36:14.806514+0000","last_fullsized":"2023-08-28T02:36:14.806514+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":109,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.806591+0000","last_change":"2023-08-28T02:31:55.528552+0000","last_active":"2023-08-28T02:36:14.806591+0000","last_peered":"2023-08-28T02:36:14.806591+0000","last_clean":"2023-08-28T02:36:14.806591+0000","last_became_active":"2023-08-28T02:31:55.518079+0000","last_became_peered":"2023-08-28T02:31:55.518079+0000","last_unstale":"2023-08-28T02:36:14.806591+0000","last_undegraded":"2023-08-28T02:36:14.806591+0000","last_fullsized":"2023-08-28T02:36:14.806591+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":242,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-08-28T02:36:10.724026+0000","last_change":"2023-08-28T02:32:58.680534+0000","last_active":"2023-08-28T02:36:10.724026+0000","last_peered":"2023-08-28T02:36:10.724026+0000","last_clean":"2023-08-28T02:36:10.724026+0000","last_became_active":"2023-08-28T02:32:58.680145+0000","last_became_peered":"2023-08-28T02:32:58.680145+0000","last_unstale":"2023-08-28T02:36:10.724026+0000","last_undegraded":"2023-08-28T02:36:10.724026+0000","last_fullsized":"2023-08-28T02:36:10.724026+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:56:40.113013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":350,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-08-28T02:36:10.726495+0000","last_change":"2023-08-28T02:32:58.746929+0000","last_active":"2023-08-28T02:36:10.726495+0000","last_peered":"2023-08-28T02:36:10.726495+0000","last_clean":"2023-08-28T02:36:10.726495+0000","last_became_active":"2023-08-28T02:32:58.746590+0000","last_became_peered":"2023-08-28T02:32:58.746590+0000","last_unstale":"2023-08-28T02:36:10.726495+0000","last_undegraded":"2023-08-28T02:36:10.726495+0000","last_fullsized":"2023-08-28T02:36:10.726495+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:51:11.974154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":334,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-08-28T02:36:10.724914+0000","last_change":"2023-08-28T02:35:05.993977+0000","last_active":"2023-08-28T02:36:10.724914+0000","last_peered":"2023-08-28T02:36:10.724914+0000","last_clean":"2023-08-28T02:36:10.724914+0000","last_became_active":"2023-08-28T02:35:05.993610+0000","last_became_peered":"2023-08-28T02:35:05.993610+0000","last_unstale":"2023-08-28T02:36:10.724914+0000","last_undegraded":"2023-08-28T02:36:10.724914+0000","last_fullsized":"2023-08-28T02:36:10.724914+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:37:10.901664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":165,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.806246+0000","last_change":"2023-08-28T02:31:55.517851+0000","last_active":"2023-08-28T02:36:14.806246+0000","last_peered":"2023-08-28T02:36:14.806246+0000","last_clean":"2023-08-28T02:36:14.806246+0000","last_became_active":"2023-08-28T02:31:55.517320+0000","last_became_peered":"2023-08-28T02:31:55.517320+0000","last_unstale":"2023-08-28T02:36:14.806246+0000","last_undegraded":"2023-08-28T02:36:14.806246+0000","last_fullsized":"2023-08-28T02:36:14.806246+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":333,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-08-28T02:36:10.727748+0000","last_change":"2023-08-28T02:31:55.528250+0000","last_active":"2023-08-28T02:36:10.727748+0000","last_peered":"2023-08-28T02:36:10.727748+0000","last_clean":"2023-08-28T02:36:10.727748+0000","last_became_active":"2023-08-28T02:31:55.527950+0000","last_became_peered":"2023-08-28T02:31:55.527950+0000","last_unstale":"2023-08-28T02:36:10.727748+0000","last_undegraded":"2023-08-28T02:36:10.727748+0000","last_fullsized":"2023-08-28T02:36:10.727748+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:39:46.120437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":134,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.793393+0000","last_change":"2023-08-28T02:28:25.703333+0000","last_active":"2023-08-28T02:36:14.793393+0000","last_peered":"2023-08-28T02:36:14.793393+0000","last_clean":"2023-08-28T02:36:14.793393+0000","last_became_active":"2023-08-28T02:28:25.343515+0000","last_became_peered":"2023-08-28T02:28:25.343515+0000","last_unstale":"2023-08-28T02:36:14.793393+0000","last_undegraded":"2023-08-28T02:36:14.793393+0000","last_fullsized":"2023-08-28T02:36:14.793393+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":251,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-08-28T02:36:10.725030+0000","last_change":"2023-08-28T02:35:45.097133+0000","last_active":"2023-08-28T02:36:10.725030+0000","last_peered":"2023-08-28T02:36:10.725030+0000","last_clean":"2023-08-28T02:36:10.725030+0000","last_became_active":"2023-08-28T02:35:45.096766+0000","last_became_peered":"2023-08-28T02:35:45.096766+0000","last_unstale":"2023-08-28T02:36:10.725030+0000","last_undegraded":"2023-08-28T02:36:10.725030+0000","last_fullsized":"2023-08-28T02:36:10.725030+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:27:29.558704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"336'4555","reported_seq":4581,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.985869+0000","last_change":"2023-08-28T02:35:14.499701+0000","last_active":"2023-08-28T02:36:21.985869+0000","last_peered":"2023-08-28T02:36:21.985869+0000","last_clean":"2023-08-28T02:36:21.985869+0000","last_became_active":"2023-08-28T02:35:14.499400+0000","last_became_peered":"2023-08-28T02:35:14.499400+0000","last_unstale":"2023-08-28T02:36:21.985869+0000","last_undegraded":"2023-08-28T02:36:21.985869+0000","last_fullsized":"2023-08-28T02:36:21.985869+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":4555,"log_dups_size":0,"ondisk_log_size":4555,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298516480,"num_objects":4555,"num_object_clones":0,"num_object_copies":13665,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4555,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4555,"num_write_kb":291520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":197,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-08-28T02:36:10.717372+0000","last_change":"2023-08-28T02:32:58.782840+0000","last_active":"2023-08-28T02:36:10.717372+0000","last_peered":"2023-08-28T02:36:10.717372+0000","last_clean":"2023-08-28T02:36:10.717372+0000","last_became_active":"2023-08-28T02:32:58.683096+0000","last_became_peered":"2023-08-28T02:32:58.683096+0000","last_unstale":"2023-08-28T02:36:10.717372+0000","last_undegraded":"2023-08-28T02:36:10.717372+0000","last_fullsized":"2023-08-28T02:36:10.717372+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:10:08.037078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":295,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.793333+0000","last_change":"2023-08-28T02:35:06.546129+0000","last_active":"2023-08-28T02:36:14.793333+0000","last_peered":"2023-08-28T02:36:14.793333+0000","last_clean":"2023-08-28T02:36:14.793333+0000","last_became_active":"2023-08-28T02:35:06.545789+0000","last_became_peered":"2023-08-28T02:35:06.545789+0000","last_unstale":"2023-08-28T02:36:14.793333+0000","last_undegraded":"2023-08-28T02:36:14.793333+0000","last_fullsized":"2023-08-28T02:36:14.793333+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.5","version":"336'4565","reported_seq":4591,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:22.230494+0000","last_change":"2023-08-28T02:35:14.500308+0000","last_active":"2023-08-28T02:36:22.230494+0000","last_peered":"2023-08-28T02:36:22.230494+0000","last_clean":"2023-08-28T02:36:22.230494+0000","last_became_active":"2023-08-28T02:35:14.499954+0000","last_became_peered":"2023-08-28T02:35:14.499954+0000","last_unstale":"2023-08-28T02:36:22.230494+0000","last_undegraded":"2023-08-28T02:36:22.230494+0000","last_fullsized":"2023-08-28T02:36:22.230494+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":4565,"log_dups_size":0,"ondisk_log_size":4565,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299171840,"num_objects":4565,"num_object_clones":0,"num_object_copies":13695,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4565,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4565,"num_write_kb":292160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":131,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.807152+0000","last_change":"2023-08-28T02:32:58.856906+0000","last_active":"2023-08-28T02:36:14.807152+0000","last_peered":"2023-08-28T02:36:14.807152+0000","last_clean":"2023-08-28T02:36:14.807152+0000","last_became_active":"2023-08-28T02:32:58.740028+0000","last_became_peered":"2023-08-28T02:32:58.740028+0000","last_unstale":"2023-08-28T02:36:14.807152+0000","last_undegraded":"2023-08-28T02:36:14.807152+0000","last_fullsized":"2023-08-28T02:36:14.807152+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":389,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.806175+0000","last_change":"2023-08-28T02:32:58.740396+0000","last_active":"2023-08-28T02:36:14.806175+0000","last_peered":"2023-08-28T02:36:14.806175+0000","last_clean":"2023-08-28T02:36:14.806175+0000","last_became_active":"2023-08-28T02:32:58.739941+0000","last_became_peered":"2023-08-28T02:32:58.739941+0000","last_unstale":"2023-08-28T02:36:14.806175+0000","last_undegraded":"2023-08-28T02:36:14.806175+0000","last_fullsized":"2023-08-28T02:36:14.806175+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"335'4469","reported_seq":4495,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:19.533022+0000","last_change":"2023-08-28T02:35:14.495274+0000","last_active":"2023-08-28T02:36:19.533022+0000","last_peered":"2023-08-28T02:36:19.533022+0000","last_clean":"2023-08-28T02:36:19.533022+0000","last_became_active":"2023-08-28T02:35:14.494642+0000","last_became_peered":"2023-08-28T02:35:14.494642+0000","last_unstale":"2023-08-28T02:36:19.533022+0000","last_undegraded":"2023-08-28T02:36:19.533022+0000","last_fullsized":"2023-08-28T02:36:19.533022+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":4469,"log_dups_size":0,"ondisk_log_size":4469,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:28:37.992837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292880384,"num_objects":4469,"num_object_clones":0,"num_object_copies":13407,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4469,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4469,"num_write_kb":286016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":134,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.793767+0000","last_change":"2023-08-28T02:35:05.994859+0000","last_active":"2023-08-28T02:36:14.793767+0000","last_peered":"2023-08-28T02:36:14.793767+0000","last_clean":"2023-08-28T02:36:14.793767+0000","last_became_active":"2023-08-28T02:35:05.994374+0000","last_became_peered":"2023-08-28T02:35:05.994374+0000","last_unstale":"2023-08-28T02:36:14.793767+0000","last_undegraded":"2023-08-28T02:36:14.793767+0000","last_fullsized":"2023-08-28T02:36:14.793767+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":160,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.793477+0000","last_change":"2023-08-28T02:35:45.112365+0000","last_active":"2023-08-28T02:36:14.793477+0000","last_peered":"2023-08-28T02:36:14.793477+0000","last_clean":"2023-08-28T02:36:14.793477+0000","last_became_active":"2023-08-28T02:35:45.112008+0000","last_became_peered":"2023-08-28T02:35:45.112008+0000","last_unstale":"2023-08-28T02:36:14.793477+0000","last_undegraded":"2023-08-28T02:36:14.793477+0000","last_fullsized":"2023-08-28T02:36:14.793477+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"336'4656","reported_seq":4683,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:23.238849+0000","last_change":"2023-08-28T02:35:14.498797+0000","last_active":"2023-08-28T02:36:23.238849+0000","last_peered":"2023-08-28T02:36:23.238849+0000","last_clean":"2023-08-28T02:36:23.238849+0000","last_became_active":"2023-08-28T02:35:14.498551+0000","last_became_peered":"2023-08-28T02:35:14.498551+0000","last_unstale":"2023-08-28T02:36:23.238849+0000","last_undegraded":"2023-08-28T02:36:23.238849+0000","last_fullsized":"2023-08-28T02:36:23.238849+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":4656,"log_dups_size":0,"ondisk_log_size":4656,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:43:07.729642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305135616,"num_objects":4656,"num_object_clones":0,"num_object_copies":13968,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4656,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4656,"num_write_kb":297984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":339,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-08-28T02:36:10.724150+0000","last_change":"2023-08-28T02:28:25.445982+0000","last_active":"2023-08-28T02:36:10.724150+0000","last_peered":"2023-08-28T02:36:10.724150+0000","last_clean":"2023-08-28T02:36:10.724150+0000","last_became_active":"2023-08-28T02:28:25.443577+0000","last_became_peered":"2023-08-28T02:28:25.443577+0000","last_unstale":"2023-08-28T02:36:10.724150+0000","last_undegraded":"2023-08-28T02:36:10.724150+0000","last_fullsized":"2023-08-28T02:36:10.724150+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:24:03.381751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":335,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-08-28T02:36:10.727910+0000","last_change":"2023-08-28T02:32:58.887874+0000","last_active":"2023-08-28T02:36:10.727910+0000","last_peered":"2023-08-28T02:36:10.727910+0000","last_clean":"2023-08-28T02:36:10.727910+0000","last_became_active":"2023-08-28T02:32:58.887377+0000","last_became_peered":"2023-08-28T02:32:58.887377+0000","last_unstale":"2023-08-28T02:36:10.727910+0000","last_undegraded":"2023-08-28T02:36:10.727910+0000","last_fullsized":"2023-08-28T02:36:10.727910+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:08:07.234984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"335'4344","reported_seq":4370,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:18.813053+0000","last_change":"2023-08-28T02:35:14.503271+0000","last_active":"2023-08-28T02:36:18.813053+0000","last_peered":"2023-08-28T02:36:18.813053+0000","last_clean":"2023-08-28T02:36:18.813053+0000","last_became_active":"2023-08-28T02:35:14.502940+0000","last_became_peered":"2023-08-28T02:35:14.502940+0000","last_unstale":"2023-08-28T02:36:18.813053+0000","last_undegraded":"2023-08-28T02:36:18.813053+0000","last_fullsized":"2023-08-28T02:36:18.813053+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":4344,"log_dups_size":0,"ondisk_log_size":4344,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:07:49.114269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284688384,"num_objects":4344,"num_object_clones":0,"num_object_copies":13032,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4344,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4344,"num_write_kb":278016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":201,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-08-28T02:36:10.723525+0000","last_change":"2023-08-28T02:23:59.230059+0000","last_active":"2023-08-28T02:36:10.723525+0000","last_peered":"2023-08-28T02:36:10.723525+0000","last_clean":"2023-08-28T02:36:10.723525+0000","last_became_active":"2023-08-28T02:23:59.217783+0000","last_became_peered":"2023-08-28T02:23:59.217783+0000","last_unstale":"2023-08-28T02:36:10.723525+0000","last_undegraded":"2023-08-28T02:36:10.723525+0000","last_fullsized":"2023-08-28T02:36:10.723525+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:00:12.145395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":356,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.806997+0000","last_change":"2023-08-28T02:28:25.351698+0000","last_active":"2023-08-28T02:36:14.806997+0000","last_peered":"2023-08-28T02:36:14.806997+0000","last_clean":"2023-08-28T02:36:14.806997+0000","last_became_active":"2023-08-28T02:28:25.351349+0000","last_became_peered":"2023-08-28T02:28:25.351349+0000","last_unstale":"2023-08-28T02:36:14.806997+0000","last_undegraded":"2023-08-28T02:36:14.806997+0000","last_fullsized":"2023-08-28T02:36:14.806997+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"336'4575","reported_seq":4602,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:23.217961+0000","last_change":"2023-08-28T02:35:14.494905+0000","last_active":"2023-08-28T02:36:23.217961+0000","last_peered":"2023-08-28T02:36:23.217961+0000","last_clean":"2023-08-28T02:36:23.217961+0000","last_became_active":"2023-08-28T02:35:14.494589+0000","last_became_peered":"2023-08-28T02:35:14.494589+0000","last_unstale":"2023-08-28T02:36:23.217961+0000","last_undegraded":"2023-08-28T02:36:23.217961+0000","last_fullsized":"2023-08-28T02:36:23.217961+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":4575,"log_dups_size":0,"ondisk_log_size":4575,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:26:28.302635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299827200,"num_objects":4575,"num_object_clones":0,"num_object_copies":13725,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4575,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4575,"num_write_kb":292800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":142,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.793420+0000","last_change":"2023-08-28T02:32:58.629249+0000","last_active":"2023-08-28T02:36:14.793420+0000","last_peered":"2023-08-28T02:36:14.793420+0000","last_clean":"2023-08-28T02:36:14.793420+0000","last_became_active":"2023-08-28T02:32:58.628583+0000","last_became_peered":"2023-08-28T02:32:58.628583+0000","last_unstale":"2023-08-28T02:36:14.793420+0000","last_undegraded":"2023-08-28T02:36:14.793420+0000","last_fullsized":"2023-08-28T02:36:14.793420+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":409,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.807494+0000","last_change":"2023-08-28T02:35:45.100048+0000","last_active":"2023-08-28T02:36:14.807494+0000","last_peered":"2023-08-28T02:36:14.807494+0000","last_clean":"2023-08-28T02:36:14.807494+0000","last_became_active":"2023-08-28T02:35:45.099690+0000","last_became_peered":"2023-08-28T02:35:45.099690+0000","last_unstale":"2023-08-28T02:36:14.807494+0000","last_undegraded":"2023-08-28T02:36:14.807494+0000","last_fullsized":"2023-08-28T02:36:14.807494+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"336'4549","reported_seq":4575,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.989779+0000","last_change":"2023-08-28T02:35:14.498910+0000","last_active":"2023-08-28T02:36:21.989779+0000","last_peered":"2023-08-28T02:36:21.989779+0000","last_clean":"2023-08-28T02:36:21.989779+0000","last_became_active":"2023-08-28T02:35:14.498631+0000","last_became_peered":"2023-08-28T02:35:14.498631+0000","last_unstale":"2023-08-28T02:36:21.989779+0000","last_undegraded":"2023-08-28T02:36:21.989779+0000","last_fullsized":"2023-08-28T02:36:21.989779+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":4549,"log_dups_size":0,"ondisk_log_size":4549,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298123264,"num_objects":4549,"num_object_clones":0,"num_object_copies":13647,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4549,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4549,"num_write_kb":291136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":272,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.806314+0000","last_change":"2023-08-28T02:32:58.740311+0000","last_active":"2023-08-28T02:36:14.806314+0000","last_peered":"2023-08-28T02:36:14.806314+0000","last_clean":"2023-08-28T02:36:14.806314+0000","last_became_active":"2023-08-28T02:32:58.739897+0000","last_became_peered":"2023-08-28T02:32:58.739897+0000","last_unstale":"2023-08-28T02:36:14.806314+0000","last_undegraded":"2023-08-28T02:36:14.806314+0000","last_fullsized":"2023-08-28T02:36:14.806314+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":322,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-08-28T02:36:10.727816+0000","last_change":"2023-08-28T02:32:58.766795+0000","last_active":"2023-08-28T02:36:10.727816+0000","last_peered":"2023-08-28T02:36:10.727816+0000","last_clean":"2023-08-28T02:36:10.727816+0000","last_became_active":"2023-08-28T02:32:58.766441+0000","last_became_peered":"2023-08-28T02:32:58.766441+0000","last_unstale":"2023-08-28T02:36:10.727816+0000","last_undegraded":"2023-08-28T02:36:10.727816+0000","last_fullsized":"2023-08-28T02:36:10.727816+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:58:13.963961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.3","version":"336'4661","reported_seq":4688,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:23.227438+0000","last_change":"2023-08-28T02:35:14.498188+0000","last_active":"2023-08-28T02:36:23.227438+0000","last_peered":"2023-08-28T02:36:23.227438+0000","last_clean":"2023-08-28T02:36:23.227438+0000","last_became_active":"2023-08-28T02:35:14.496278+0000","last_became_peered":"2023-08-28T02:35:14.496278+0000","last_unstale":"2023-08-28T02:36:23.227438+0000","last_undegraded":"2023-08-28T02:36:23.227438+0000","last_fullsized":"2023-08-28T02:36:23.227438+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":4661,"log_dups_size":0,"ondisk_log_size":4661,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:43:23.633921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305463296,"num_objects":4661,"num_object_clones":0,"num_object_copies":13983,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4661,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4661,"num_write_kb":298304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":242,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-08-28T02:36:10.734183+0000","last_change":"2023-08-28T02:32:58.779257+0000","last_active":"2023-08-28T02:36:10.734183+0000","last_peered":"2023-08-28T02:36:10.734183+0000","last_clean":"2023-08-28T02:36:10.734183+0000","last_became_active":"2023-08-28T02:32:58.778295+0000","last_became_peered":"2023-08-28T02:32:58.778295+0000","last_unstale":"2023-08-28T02:36:10.734183+0000","last_undegraded":"2023-08-28T02:36:10.734183+0000","last_fullsized":"2023-08-28T02:36:10.734183+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:50:49.444715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":56,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.794231+0000","last_change":"2023-08-28T02:32:58.755765+0000","last_active":"2023-08-28T02:36:14.794231+0000","last_peered":"2023-08-28T02:36:14.794231+0000","last_clean":"2023-08-28T02:36:14.794231+0000","last_became_active":"2023-08-28T02:32:58.755438+0000","last_became_peered":"2023-08-28T02:32:58.755438+0000","last_unstale":"2023-08-28T02:36:14.794231+0000","last_undegraded":"2023-08-28T02:36:14.794231+0000","last_fullsized":"2023-08-28T02:36:14.794231+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:14:11.498611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"335'4428","reported_seq":4454,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:19.549752+0000","last_change":"2023-08-28T02:35:14.497511+0000","last_active":"2023-08-28T02:36:19.549752+0000","last_peered":"2023-08-28T02:36:19.549752+0000","last_clean":"2023-08-28T02:36:19.549752+0000","last_became_active":"2023-08-28T02:35:14.497180+0000","last_became_peered":"2023-08-28T02:35:14.497180+0000","last_unstale":"2023-08-28T02:36:19.549752+0000","last_undegraded":"2023-08-28T02:36:19.549752+0000","last_fullsized":"2023-08-28T02:36:19.549752+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":4428,"log_dups_size":0,"ondisk_log_size":4428,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:59:26.376998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":290193408,"num_objects":4428,"num_object_clones":0,"num_object_copies":13284,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4428,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4428,"num_write_kb":283392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"78'126","reported_seq":740,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.787655+0000","last_change":"2023-08-28T02:35:09.735906+0000","last_active":"2023-08-28T02:36:14.787655+0000","last_peered":"2023-08-28T02:36:14.787655+0000","last_clean":"2023-08-28T02:36:14.787655+0000","last_became_active":"2023-08-28T02:32:59.660388+0000","last_became_peered":"2023-08-28T02:32:59.660388+0000","last_unstale":"2023-08-28T02:36:14.787655+0000","last_undegraded":"2023-08-28T02:36:14.787655+0000","last_fullsized":"2023-08-28T02:36:14.787655+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:19:53.565088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":743,"num_read_kb":1331,"num_write":125,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":176,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-08-28T02:36:10.724830+0000","last_change":"2023-08-28T02:35:05.991133+0000","last_active":"2023-08-28T02:36:10.724830+0000","last_peered":"2023-08-28T02:36:10.724830+0000","last_clean":"2023-08-28T02:36:10.724830+0000","last_became_active":"2023-08-28T02:35:05.990829+0000","last_became_peered":"2023-08-28T02:35:05.990829+0000","last_unstale":"2023-08-28T02:36:10.724830+0000","last_undegraded":"2023-08-28T02:36:10.724830+0000","last_fullsized":"2023-08-28T02:36:10.724830+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:40:28.077023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":17,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-08-28T02:36:10.716845+0000","last_change":"2023-08-28T02:35:45.099619+0000","last_active":"2023-08-28T02:36:10.716845+0000","last_peered":"2023-08-28T02:36:10.716845+0000","last_clean":"2023-08-28T02:36:10.716845+0000","last_became_active":"2023-08-28T02:35:45.099285+0000","last_became_peered":"2023-08-28T02:35:45.099285+0000","last_unstale":"2023-08-28T02:36:10.716845+0000","last_undegraded":"2023-08-28T02:36:10.716845+0000","last_fullsized":"2023-08-28T02:36:10.716845+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:24:58.305957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.8","version":"335'4464","reported_seq":4490,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:18.825002+0000","last_change":"2023-08-28T02:35:14.497638+0000","last_active":"2023-08-28T02:36:18.825002+0000","last_peered":"2023-08-28T02:36:18.825002+0000","last_clean":"2023-08-28T02:36:18.825002+0000","last_became_active":"2023-08-28T02:35:14.497323+0000","last_became_peered":"2023-08-28T02:35:14.497323+0000","last_unstale":"2023-08-28T02:36:18.825002+0000","last_undegraded":"2023-08-28T02:36:18.825002+0000","last_fullsized":"2023-08-28T02:36:18.825002+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":4464,"log_dups_size":0,"ondisk_log_size":4464,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:30:06.078735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292552704,"num_objects":4464,"num_object_clones":0,"num_object_copies":13392,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4464,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4464,"num_write_kb":285696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"78'204","reported_seq":1420,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.794362+0000","last_change":"2023-08-28T02:32:59.644574+0000","last_active":"2023-08-28T02:36:14.794362+0000","last_peered":"2023-08-28T02:36:14.794362+0000","last_clean":"2023-08-28T02:36:14.794362+0000","last_became_active":"2023-08-28T02:32:59.644053+0000","last_became_peered":"2023-08-28T02:32:59.644053+0000","last_unstale":"2023-08-28T02:36:14.794362+0000","last_undegraded":"2023-08-28T02:36:14.794362+0000","last_fullsized":"2023-08-28T02:36:14.794362+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":204,"log_dups_size":0,"ondisk_log_size":204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:55:51.460483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1521,"num_read_kb":2511,"num_write":178,"num_write_kb":1175,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":330,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.806660+0000","last_change":"2023-08-28T02:28:25.352417+0000","last_active":"2023-08-28T02:36:14.806660+0000","last_peered":"2023-08-28T02:36:14.806660+0000","last_clean":"2023-08-28T02:36:14.806660+0000","last_became_active":"2023-08-28T02:28:25.352216+0000","last_became_peered":"2023-08-28T02:28:25.352216+0000","last_unstale":"2023-08-28T02:36:14.806660+0000","last_undegraded":"2023-08-28T02:36:14.806660+0000","last_fullsized":"2023-08-28T02:36:14.806660+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":56,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.787585+0000","last_change":"2023-08-28T02:32:58.720576+0000","last_active":"2023-08-28T02:36:14.787585+0000","last_peered":"2023-08-28T02:36:14.787585+0000","last_clean":"2023-08-28T02:36:14.787585+0000","last_became_active":"2023-08-28T02:32:58.719924+0000","last_became_peered":"2023-08-28T02:32:58.719924+0000","last_unstale":"2023-08-28T02:36:14.787585+0000","last_undegraded":"2023-08-28T02:36:14.787585+0000","last_fullsized":"2023-08-28T02:36:14.787585+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:28:50.697438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"335'4390","reported_seq":4416,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:18.815164+0000","last_change":"2023-08-28T02:35:14.503279+0000","last_active":"2023-08-28T02:36:18.815164+0000","last_peered":"2023-08-28T02:36:18.815164+0000","last_clean":"2023-08-28T02:36:18.815164+0000","last_became_active":"2023-08-28T02:35:14.502953+0000","last_became_peered":"2023-08-28T02:35:14.502953+0000","last_unstale":"2023-08-28T02:36:18.815164+0000","last_undegraded":"2023-08-28T02:36:18.815164+0000","last_fullsized":"2023-08-28T02:36:18.815164+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":4390,"log_dups_size":0,"ondisk_log_size":4390,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:55:29.284519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":287703040,"num_objects":4390,"num_object_clones":0,"num_object_copies":13170,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4390,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4390,"num_write_kb":280960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"78'47","reported_seq":554,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-08-28T02:36:10.717174+0000","last_change":"2023-08-28T02:24:05.799647+0000","last_active":"2023-08-28T02:36:10.717174+0000","last_peered":"2023-08-28T02:36:10.717174+0000","last_clean":"2023-08-28T02:36:10.717174+0000","last_became_active":"2023-08-28T02:23:59.210481+0000","last_became_peered":"2023-08-28T02:23:59.210481+0000","last_unstale":"2023-08-28T02:36:10.717174+0000","last_undegraded":"2023-08-28T02:36:10.717174+0000","last_fullsized":"2023-08-28T02:36:10.717174+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":47,"log_dups_size":0,"ondisk_log_size":47,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:40:18.542375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":162,"num_read_kb":1281,"num_write":48,"num_write_kb":1153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":56,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.788397+0000","last_change":"2023-08-28T02:32:58.622079+0000","last_active":"2023-08-28T02:36:14.788397+0000","last_peered":"2023-08-28T02:36:14.788397+0000","last_clean":"2023-08-28T02:36:14.788397+0000","last_became_active":"2023-08-28T02:32:58.621819+0000","last_became_peered":"2023-08-28T02:32:58.621819+0000","last_unstale":"2023-08-28T02:36:14.788397+0000","last_undegraded":"2023-08-28T02:36:14.788397+0000","last_fullsized":"2023-08-28T02:36:14.788397+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:03:36.676500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":56,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.795926+0000","last_change":"2023-08-28T02:32:58.724424+0000","last_active":"2023-08-28T02:36:14.795926+0000","last_peered":"2023-08-28T02:36:14.795926+0000","last_clean":"2023-08-28T02:36:14.795926+0000","last_became_active":"2023-08-28T02:32:58.724127+0000","last_became_peered":"2023-08-28T02:32:58.724127+0000","last_unstale":"2023-08-28T02:36:14.795926+0000","last_undegraded":"2023-08-28T02:36:14.795926+0000","last_fullsized":"2023-08-28T02:36:14.795926+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:02:25.178712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"336'4573","reported_seq":4599,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:22.144267+0000","last_change":"2023-08-28T02:35:14.500884+0000","last_active":"2023-08-28T02:36:22.144267+0000","last_peered":"2023-08-28T02:36:22.144267+0000","last_clean":"2023-08-28T02:36:22.144267+0000","last_became_active":"2023-08-28T02:35:14.500643+0000","last_became_peered":"2023-08-28T02:35:14.500643+0000","last_unstale":"2023-08-28T02:36:22.144267+0000","last_undegraded":"2023-08-28T02:36:22.144267+0000","last_fullsized":"2023-08-28T02:36:22.144267+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":4573,"log_dups_size":0,"ondisk_log_size":4573,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299696128,"num_objects":4573,"num_object_clones":0,"num_object_copies":13719,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4573,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4573,"num_write_kb":292672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"78'114","reported_seq":388,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-08-28T02:36:10.717003+0000","last_change":"2023-08-28T02:35:05.992001+0000","last_active":"2023-08-28T02:36:10.717003+0000","last_peered":"2023-08-28T02:36:10.717003+0000","last_clean":"2023-08-28T02:36:10.717003+0000","last_became_active":"2023-08-28T02:35:05.991513+0000","last_became_peered":"2023-08-28T02:35:05.991513+0000","last_unstale":"2023-08-28T02:36:10.717003+0000","last_undegraded":"2023-08-28T02:36:10.717003+0000","last_fullsized":"2023-08-28T02:36:10.717003+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":114,"log_dups_size":0,"ondisk_log_size":114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:00:42.194692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":358,"num_read_kb":1352,"num_write":102,"num_write_kb":1104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"86'139","reported_seq":420,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-08-28T02:36:10.726151+0000","last_change":"2023-08-28T02:35:09.566656+0000","last_active":"2023-08-28T02:36:10.726151+0000","last_peered":"2023-08-28T02:36:10.726151+0000","last_clean":"2023-08-28T02:36:10.726151+0000","last_became_active":"2023-08-28T02:32:59.651089+0000","last_became_peered":"2023-08-28T02:32:59.651089+0000","last_unstale":"2023-08-28T02:36:10.726151+0000","last_undegraded":"2023-08-28T02:36:10.726151+0000","last_fullsized":"2023-08-28T02:36:10.726151+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_clean_scrub_stamp":"2023-08-28T02:08:02.043718+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:31:19.261767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":215,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-08-28T02:36:10.723723+0000","last_change":"2023-08-28T02:28:25.347600+0000","last_active":"2023-08-28T02:36:10.723723+0000","last_peered":"2023-08-28T02:36:10.723723+0000","last_clean":"2023-08-28T02:36:10.723723+0000","last_became_active":"2023-08-28T02:28:25.347237+0000","last_became_peered":"2023-08-28T02:28:25.347237+0000","last_unstale":"2023-08-28T02:36:10.723723+0000","last_undegraded":"2023-08-28T02:36:10.723723+0000","last_fullsized":"2023-08-28T02:36:10.723723+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:29:26.767559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":133,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.793870+0000","last_change":"2023-08-28T02:31:55.517732+0000","last_active":"2023-08-28T02:36:14.793870+0000","last_peered":"2023-08-28T02:36:14.793870+0000","last_clean":"2023-08-28T02:36:14.793870+0000","last_became_active":"2023-08-28T02:31:55.517347+0000","last_became_peered":"2023-08-28T02:31:55.517347+0000","last_unstale":"2023-08-28T02:36:14.793870+0000","last_undegraded":"2023-08-28T02:36:14.793870+0000","last_fullsized":"2023-08-28T02:36:14.793870+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"336'4514","reported_seq":4541,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:23.217886+0000","last_change":"2023-08-28T02:35:14.493898+0000","last_active":"2023-08-28T02:36:23.217886+0000","last_peered":"2023-08-28T02:36:23.217886+0000","last_clean":"2023-08-28T02:36:23.217886+0000","last_became_active":"2023-08-28T02:35:14.493649+0000","last_became_peered":"2023-08-28T02:35:14.493649+0000","last_unstale":"2023-08-28T02:36:23.217886+0000","last_undegraded":"2023-08-28T02:36:23.217886+0000","last_fullsized":"2023-08-28T02:36:23.217886+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":4514,"log_dups_size":0,"ondisk_log_size":4514,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:49:46.405255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295829504,"num_objects":4514,"num_object_clones":0,"num_object_copies":13542,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4514,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4514,"num_write_kb":288896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'148","reported_seq":791,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.788522+0000","last_change":"2023-08-28T02:35:10.122879+0000","last_active":"2023-08-28T02:36:14.788522+0000","last_peered":"2023-08-28T02:36:14.788522+0000","last_clean":"2023-08-28T02:36:14.788522+0000","last_became_active":"2023-08-28T02:32:59.661273+0000","last_became_peered":"2023-08-28T02:32:59.661273+0000","last_unstale":"2023-08-28T02:36:14.788522+0000","last_undegraded":"2023-08-28T02:36:14.788522+0000","last_fullsized":"2023-08-28T02:36:14.788522+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:07:46.974854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1092,"num_read_kb":1955,"num_write":127,"num_write_kb":5177,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":306,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-08-28T02:36:10.730965+0000","last_change":"2023-08-28T02:23:59.226986+0000","last_active":"2023-08-28T02:36:10.730965+0000","last_peered":"2023-08-28T02:36:10.730965+0000","last_clean":"2023-08-28T02:36:10.730965+0000","last_became_active":"2023-08-28T02:23:59.226529+0000","last_became_peered":"2023-08-28T02:23:59.226529+0000","last_unstale":"2023-08-28T02:36:10.730965+0000","last_undegraded":"2023-08-28T02:36:10.730965+0000","last_fullsized":"2023-08-28T02:36:10.730965+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:22:09.193619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":369,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-08-28T02:36:10.727513+0000","last_change":"2023-08-28T02:32:58.744332+0000","last_active":"2023-08-28T02:36:10.727513+0000","last_peered":"2023-08-28T02:36:10.727513+0000","last_clean":"2023-08-28T02:36:10.727513+0000","last_became_active":"2023-08-28T02:32:58.744025+0000","last_became_peered":"2023-08-28T02:32:58.744025+0000","last_unstale":"2023-08-28T02:36:10.727513+0000","last_undegraded":"2023-08-28T02:36:10.727513+0000","last_fullsized":"2023-08-28T02:36:10.727513+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:41:15.957049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"335'4410","reported_seq":4594,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:18.842034+0000","last_change":"2023-08-28T02:35:24.994481+0000","last_active":"2023-08-28T02:36:18.842034+0000","last_peered":"2023-08-28T02:36:18.842034+0000","last_clean":"2023-08-28T02:36:18.842034+0000","last_became_active":"2023-08-28T02:35:24.668942+0000","last_became_peered":"2023-08-28T02:35:24.668942+0000","last_unstale":"2023-08-28T02:36:18.842034+0000","last_undegraded":"2023-08-28T02:36:18.842034+0000","last_fullsized":"2023-08-28T02:36:18.842034+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":4410,"log_dups_size":0,"ondisk_log_size":4410,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:16:02.211038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289013760,"num_objects":4410,"num_object_clones":0,"num_object_copies":13230,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4410,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4410,"num_write_kb":282240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":73,"num_bytes_recovered":4784128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"78'103","reported_seq":666,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-08-28T02:36:10.726540+0000","last_change":"2023-08-28T02:35:09.798248+0000","last_active":"2023-08-28T02:36:10.726540+0000","last_peered":"2023-08-28T02:36:10.726540+0000","last_clean":"2023-08-28T02:36:10.726540+0000","last_became_active":"2023-08-28T02:32:59.656276+0000","last_became_peered":"2023-08-28T02:32:59.656276+0000","last_unstale":"2023-08-28T02:36:10.726540+0000","last_undegraded":"2023-08-28T02:36:10.726540+0000","last_fullsized":"2023-08-28T02:36:10.726540+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:34:10.622592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":803,"num_read_kb":2080,"num_write":73,"num_write_kb":1244,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":56,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.787520+0000","last_change":"2023-08-28T02:32:58.720401+0000","last_active":"2023-08-28T02:36:14.787520+0000","last_peered":"2023-08-28T02:36:14.787520+0000","last_clean":"2023-08-28T02:36:14.787520+0000","last_became_active":"2023-08-28T02:32:58.719518+0000","last_became_peered":"2023-08-28T02:32:58.719518+0000","last_unstale":"2023-08-28T02:36:14.787520+0000","last_undegraded":"2023-08-28T02:36:14.787520+0000","last_fullsized":"2023-08-28T02:36:14.787520+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:46:28.947812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":319,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-08-28T02:36:10.728139+0000","last_change":"2023-08-28T02:32:58.887691+0000","last_active":"2023-08-28T02:36:10.728139+0000","last_peered":"2023-08-28T02:36:10.728139+0000","last_clean":"2023-08-28T02:36:10.728139+0000","last_became_active":"2023-08-28T02:32:58.886906+0000","last_became_peered":"2023-08-28T02:32:58.886906+0000","last_unstale":"2023-08-28T02:36:10.728139+0000","last_undegraded":"2023-08-28T02:36:10.728139+0000","last_fullsized":"2023-08-28T02:36:10.728139+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:39:29.229171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"335'4348","reported_seq":4374,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:19.548926+0000","last_change":"2023-08-28T02:35:14.497565+0000","last_active":"2023-08-28T02:36:19.548926+0000","last_peered":"2023-08-28T02:36:19.548926+0000","last_clean":"2023-08-28T02:36:19.548926+0000","last_became_active":"2023-08-28T02:35:14.497285+0000","last_became_peered":"2023-08-28T02:35:14.497285+0000","last_unstale":"2023-08-28T02:36:19.548926+0000","last_undegraded":"2023-08-28T02:36:19.548926+0000","last_fullsized":"2023-08-28T02:36:19.548926+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":4348,"log_dups_size":0,"ondisk_log_size":4348,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:04:31.778861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284950528,"num_objects":4348,"num_object_clones":0,"num_object_copies":13044,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4348,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4348,"num_write_kb":278272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":214,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-08-28T02:36:10.725058+0000","last_change":"2023-08-28T02:31:55.535099+0000","last_active":"2023-08-28T02:36:10.725058+0000","last_peered":"2023-08-28T02:36:10.725058+0000","last_clean":"2023-08-28T02:36:10.725058+0000","last_became_active":"2023-08-28T02:31:55.527768+0000","last_became_peered":"2023-08-28T02:31:55.527768+0000","last_unstale":"2023-08-28T02:36:10.725058+0000","last_undegraded":"2023-08-28T02:36:10.725058+0000","last_fullsized":"2023-08-28T02:36:10.725058+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:43:30.804793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'112","reported_seq":489,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.788081+0000","last_change":"2023-08-28T02:32:59.643465+0000","last_active":"2023-08-28T02:36:14.788081+0000","last_peered":"2023-08-28T02:36:14.788081+0000","last_clean":"2023-08-28T02:36:14.788081+0000","last_became_active":"2023-08-28T02:32:59.642979+0000","last_became_peered":"2023-08-28T02:32:59.642979+0000","last_unstale":"2023-08-28T02:36:14.788081+0000","last_undegraded":"2023-08-28T02:36:14.788081+0000","last_fullsized":"2023-08-28T02:36:14.788081+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:48:45.923015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":444,"num_read_kb":473,"num_write":94,"num_write_kb":127,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":225,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-08-28T02:36:10.734125+0000","last_change":"2023-08-28T02:32:58.833408+0000","last_active":"2023-08-28T02:36:10.734125+0000","last_peered":"2023-08-28T02:36:10.734125+0000","last_clean":"2023-08-28T02:36:10.734125+0000","last_became_active":"2023-08-28T02:32:58.821347+0000","last_became_peered":"2023-08-28T02:32:58.821347+0000","last_unstale":"2023-08-28T02:36:10.734125+0000","last_undegraded":"2023-08-28T02:36:10.734125+0000","last_fullsized":"2023-08-28T02:36:10.734125+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:41:25.683179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"335'4353","reported_seq":4379,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:19.981257+0000","last_change":"2023-08-28T02:35:14.495405+0000","last_active":"2023-08-28T02:36:19.981257+0000","last_peered":"2023-08-28T02:36:19.981257+0000","last_clean":"2023-08-28T02:36:19.981257+0000","last_became_active":"2023-08-28T02:35:14.494591+0000","last_became_peered":"2023-08-28T02:35:14.494591+0000","last_unstale":"2023-08-28T02:36:19.981257+0000","last_undegraded":"2023-08-28T02:36:19.981257+0000","last_fullsized":"2023-08-28T02:36:19.981257+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":4353,"log_dups_size":0,"ondisk_log_size":4353,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:47:27.839376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":285278208,"num_objects":4353,"num_object_clones":0,"num_object_copies":13059,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4353,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4353,"num_write_kb":278592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"78'122","reported_seq":679,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.793843+0000","last_change":"2023-08-28T02:33:00.672889+0000","last_active":"2023-08-28T02:36:14.793843+0000","last_peered":"2023-08-28T02:36:14.793843+0000","last_clean":"2023-08-28T02:36:14.793843+0000","last_became_active":"2023-08-28T02:33:00.672494+0000","last_became_peered":"2023-08-28T02:33:00.672494+0000","last_unstale":"2023-08-28T02:36:14.793843+0000","last_undegraded":"2023-08-28T02:36:14.793843+0000","last_fullsized":"2023-08-28T02:36:14.793843+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":857,"num_read_kb":2224,"num_write":99,"num_write_kb":2312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":344,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-08-28T02:36:10.727598+0000","last_change":"2023-08-28T02:32:58.795995+0000","last_active":"2023-08-28T02:36:10.727598+0000","last_peered":"2023-08-28T02:36:10.727598+0000","last_clean":"2023-08-28T02:36:10.727598+0000","last_became_active":"2023-08-28T02:32:58.771872+0000","last_became_peered":"2023-08-28T02:32:58.771872+0000","last_unstale":"2023-08-28T02:36:10.727598+0000","last_undegraded":"2023-08-28T02:36:10.727598+0000","last_fullsized":"2023-08-28T02:36:10.727598+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:16:02.965467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":305,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.806000+0000","last_change":"2023-08-28T02:28:25.351840+0000","last_active":"2023-08-28T02:36:14.806000+0000","last_peered":"2023-08-28T02:36:14.806000+0000","last_clean":"2023-08-28T02:36:14.806000+0000","last_became_active":"2023-08-28T02:28:25.351399+0000","last_became_peered":"2023-08-28T02:28:25.351399+0000","last_unstale":"2023-08-28T02:36:14.806000+0000","last_undegraded":"2023-08-28T02:36:14.806000+0000","last_fullsized":"2023-08-28T02:36:14.806000+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":321,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-08-28T02:36:10.728063+0000","last_change":"2023-08-28T02:32:58.751818+0000","last_active":"2023-08-28T02:36:10.728063+0000","last_peered":"2023-08-28T02:36:10.728063+0000","last_clean":"2023-08-28T02:36:10.728063+0000","last_became_active":"2023-08-28T02:32:58.743388+0000","last_became_peered":"2023-08-28T02:32:58.743388+0000","last_unstale":"2023-08-28T02:36:10.728063+0000","last_undegraded":"2023-08-28T02:36:10.728063+0000","last_fullsized":"2023-08-28T02:36:10.728063+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:52:12.910174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":22,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-08-28T02:36:10.718173+0000","last_change":"2023-08-28T02:35:24.666417+0000","last_active":"2023-08-28T02:36:10.718173+0000","last_peered":"2023-08-28T02:36:10.718173+0000","last_clean":"2023-08-28T02:36:10.718173+0000","last_became_active":"2023-08-28T02:35:24.666107+0000","last_became_peered":"2023-08-28T02:35:24.666107+0000","last_unstale":"2023-08-28T02:36:10.718173+0000","last_undegraded":"2023-08-28T02:36:10.718173+0000","last_fullsized":"2023-08-28T02:36:10.718173+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:22:55.497120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":168,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.806919+0000","last_change":"2023-08-28T02:32:58.839847+0000","last_active":"2023-08-28T02:36:14.806919+0000","last_peered":"2023-08-28T02:36:14.806919+0000","last_clean":"2023-08-28T02:36:14.806919+0000","last_became_active":"2023-08-28T02:32:58.628159+0000","last_became_peered":"2023-08-28T02:32:58.628159+0000","last_unstale":"2023-08-28T02:36:14.806919+0000","last_undegraded":"2023-08-28T02:36:14.806919+0000","last_fullsized":"2023-08-28T02:36:14.806919+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":371,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.806776+0000","last_change":"2023-08-28T02:32:58.629596+0000","last_active":"2023-08-28T02:36:14.806776+0000","last_peered":"2023-08-28T02:36:14.806776+0000","last_clean":"2023-08-28T02:36:14.806776+0000","last_became_active":"2023-08-28T02:32:58.629136+0000","last_became_peered":"2023-08-28T02:32:58.629136+0000","last_unstale":"2023-08-28T02:36:14.806776+0000","last_undegraded":"2023-08-28T02:36:14.806776+0000","last_fullsized":"2023-08-28T02:36:14.806776+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":155,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.807412+0000","last_change":"2023-08-28T02:28:25.705331+0000","last_active":"2023-08-28T02:36:14.807412+0000","last_peered":"2023-08-28T02:36:14.807412+0000","last_clean":"2023-08-28T02:36:14.807412+0000","last_became_active":"2023-08-28T02:28:25.397480+0000","last_became_peered":"2023-08-28T02:28:25.397480+0000","last_unstale":"2023-08-28T02:36:14.807412+0000","last_undegraded":"2023-08-28T02:36:14.807412+0000","last_fullsized":"2023-08-28T02:36:14.807412+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":335,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.806952+0000","last_change":"2023-08-28T02:23:59.207929+0000","last_active":"2023-08-28T02:36:14.806952+0000","last_peered":"2023-08-28T02:36:14.806952+0000","last_clean":"2023-08-28T02:36:14.806952+0000","last_became_active":"2023-08-28T02:23:59.207589+0000","last_became_peered":"2023-08-28T02:23:59.207589+0000","last_unstale":"2023-08-28T02:36:14.806952+0000","last_undegraded":"2023-08-28T02:36:14.806952+0000","last_fullsized":"2023-08-28T02:36:14.806952+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":225,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-08-28T02:36:10.725222+0000","last_change":"2023-08-28T02:28:25.347988+0000","last_active":"2023-08-28T02:36:10.725222+0000","last_peered":"2023-08-28T02:36:10.725222+0000","last_clean":"2023-08-28T02:36:10.725222+0000","last_became_active":"2023-08-28T02:28:25.347632+0000","last_became_peered":"2023-08-28T02:28:25.347632+0000","last_unstale":"2023-08-28T02:36:10.725222+0000","last_undegraded":"2023-08-28T02:36:10.725222+0000","last_fullsized":"2023-08-28T02:36:10.725222+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:56:20.202662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":197,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-08-28T02:36:10.723639+0000","last_change":"2023-08-28T02:23:59.217794+0000","last_active":"2023-08-28T02:36:10.723639+0000","last_peered":"2023-08-28T02:36:10.723639+0000","last_clean":"2023-08-28T02:36:10.723639+0000","last_became_active":"2023-08-28T02:23:59.217231+0000","last_became_peered":"2023-08-28T02:23:59.217231+0000","last_unstale":"2023-08-28T02:36:10.723639+0000","last_undegraded":"2023-08-28T02:36:10.723639+0000","last_fullsized":"2023-08-28T02:36:10.723639+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:36:26.297410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":322,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-08-28T02:36:10.744273+0000","last_change":"2023-08-28T02:35:24.680274+0000","last_active":"2023-08-28T02:36:10.744273+0000","last_peered":"2023-08-28T02:36:10.744273+0000","last_clean":"2023-08-28T02:36:10.744273+0000","last_became_active":"2023-08-28T02:35:24.673318+0000","last_became_peered":"2023-08-28T02:35:24.673318+0000","last_unstale":"2023-08-28T02:36:10.744273+0000","last_undegraded":"2023-08-28T02:36:10.744273+0000","last_fullsized":"2023-08-28T02:36:10.744273+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:22:46.371303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":394,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.805822+0000","last_change":"2023-08-28T02:32:58.630270+0000","last_active":"2023-08-28T02:36:14.805822+0000","last_peered":"2023-08-28T02:36:14.805822+0000","last_clean":"2023-08-28T02:36:14.805822+0000","last_became_active":"2023-08-28T02:32:58.629958+0000","last_became_peered":"2023-08-28T02:32:58.629958+0000","last_unstale":"2023-08-28T02:36:14.805822+0000","last_undegraded":"2023-08-28T02:36:14.805822+0000","last_fullsized":"2023-08-28T02:36:14.805822+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":62,"num_bytes_recovered":25254,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":21462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26810,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":4709023744,"num_objects":71854,"num_object_clones":0,"num_object_copies":215562,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":71854,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":71854,"num_write_kb":4598656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":73,"num_bytes_recovered":4784128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14153678848,"data_stored":14153678848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":71854,"ondisk_log_size":71854,"up":48,"acting":48,"num_store_stats":7},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":156,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16789504,"data_stored":16777273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":1335734829074,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626756,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2064972,"kb_used_data":1755460,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":91680180,"statfs":{"total":95995035648,"available":93880504320,"internally_reserved":0,"allocated":1797591040,"data_stored":1796710458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:36:20 2023","interfaces":[{"interface":"back","average":{"1min":1.0980000000000001,"5min":3.0529999999999999,"15min":3.6560000000000001},"min":{"1min":0.23000000000000001,"5min":0.215,"15min":0.20799999999999999},"max":{"1min":3.5760000000000001,"5min":121.78,"15min":121.78},"last":0.23499999999999999},{"interface":"front","average":{"1min":1.254,"5min":3.6309999999999998,"15min":3.7549999999999999},"min":{"1min":0.29999999999999999,"5min":0.23699999999999999,"15min":0.23200000000000001},"max":{"1min":7.4370000000000003,"5min":121.79900000000001,"15min":121.79900000000001},"last":0.38700000000000001}]},{"osd":1,"last update":"Mon Aug 28 02:36:20 2023","interfaces":[{"interface":"back","average":{"1min":1.373,"5min":1.2589999999999999,"15min":2.6880000000000002},"min":{"1min":0.315,"5min":0.23799999999999999,"15min":0.23699999999999999},"max":{"1min":4.1959999999999997,"5min":8.2639999999999993,"15min":141.87299999999999},"last":0.32200000000000001},{"interface":"front","average":{"1min":1.5640000000000001,"5min":1.484,"15min":2.8450000000000002},"min":{"1min":0.25,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":4.8440000000000003,"5min":8.5180000000000007,"15min":141.904},"last":2.9369999999999998}]},{"osd":2,"last update":"Mon Aug 28 02:36:04 2023","interfaces":[{"interface":"back","average":{"1min":5.5019999999999998,"5min":5.6580000000000004,"15min":4.173},"min":{"1min":0.34899999999999998,"5min":0.29499999999999998,"15min":0.27500000000000002},"max":{"1min":91.641000000000005,"5min":234.876,"15min":234.876},"last":0.27600000000000002},{"interface":"front","average":{"1min":5.4139999999999997,"5min":5.4320000000000004,"15min":4.1130000000000004},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":91.655000000000001,"5min":234.482,"15min":234.482},"last":4.7110000000000003}]},{"osd":3,"last update":"Mon Aug 28 02:36:20 2023","interfaces":[{"interface":"back","average":{"1min":1.8480000000000001,"5min":3.3069999999999999,"15min":3.2069999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":7.2750000000000004,"5min":143.19499999999999,"15min":157.625},"last":3.363},{"interface":"front","average":{"1min":2.1070000000000002,"5min":3.3079999999999998,"15min":3.2149999999999999},"min":{"1min":0.38200000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":7.2869999999999999,"5min":143.20699999999999,"15min":157.637},"last":3.3719999999999999}]},{"osd":4,"last update":"Mon Aug 28 02:35:24 2023","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.63100000000000001,"15min":1.1040000000000001},"min":{"1min":0.25600000000000001,"5min":0.251,"15min":0.251},"max":{"1min":0.70099999999999996,"5min":7.3090000000000002,"15min":18.308},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.70499999999999996,"5min":0.71499999999999997,"15min":1.206},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":2.907,"5min":4.7690000000000001,"15min":18.263000000000002},"last":0.41799999999999998}]},{"osd":5,"last update":"Mon Aug 28 02:35:24 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.67400000000000004,"15min":1.1819999999999999},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.26300000000000001},"max":{"1min":3.0739999999999998,"5min":6.0049999999999999,"15min":20.759},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.65000000000000002,"15min":1.0029999999999999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.65200000000000002,"5min":7.2370000000000001,"15min":7.3780000000000001},"last":0.54300000000000004}]},{"osd":7,"last update":"Mon Aug 28 02:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":1.3180000000000001,"5min":1.3180000000000001,"15min":1.3180000000000001},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.52400000000000002}]}]},{"osd":1,"up_from":98,"seq":420906795209,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2576132,"kb_used_data":2073852,"kb_used_omap":7,"kb_used_meta":502264,"kb_avail":91169020,"statfs":{"total":95995035648,"available":93357076480,"internally_reserved":0,"allocated":2123624448,"data_stored":2121892343,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":514318667},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,5],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:36:18 2023","interfaces":[{"interface":"back","average":{"1min":3.3809999999999998,"5min":6.6779999999999999,"15min":4.4089999999999998},"min":{"1min":0.23799999999999999,"5min":0.189,"15min":0.17499999999999999},"max":{"1min":36.838000000000001,"5min":129.346,"15min":129.346},"last":2.8290000000000002},{"interface":"front","average":{"1min":2.9279999999999999,"5min":6.6210000000000004,"15min":4.4329999999999998},"min":{"1min":0.19800000000000001,"5min":0.19700000000000001,"15min":0.188},"max":{"1min":36.622,"5min":120.774,"15min":128.685},"last":0.23699999999999999}]},{"osd":2,"last update":"Mon Aug 28 02:35:43 2023","interfaces":[{"interface":"back","average":{"1min":2.2269999999999999,"5min":3.883,"15min":4.1429999999999998},"min":{"1min":0.218,"5min":0.189,"15min":0.189},"max":{"1min":10.818,"5min":119.276,"15min":135.25999999999999},"last":0.56200000000000006},{"interface":"front","average":{"1min":1.0549999999999999,"5min":3.492,"15min":3.8100000000000001},"min":{"1min":0.221,"5min":0.218,"15min":0.218},"max":{"1min":6.8879999999999999,"5min":121.849,"15min":135.10300000000001},"last":2.7930000000000001}]},{"osd":3,"last update":"Mon Aug 28 02:36:18 2023","interfaces":[{"interface":"back","average":{"1min":1.6839999999999999,"5min":7.7859999999999996,"15min":4.4699999999999998},"min":{"1min":0.21299999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":4.29,"5min":126.95699999999999,"15min":126.95699999999999},"last":2.7280000000000002},{"interface":"front","average":{"1min":2.141,"5min":7.4969999999999999,"15min":4.5890000000000004},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":10.477,"5min":128.00200000000001,"15min":128.00200000000001},"last":0.435}]},{"osd":4,"last update":"Mon Aug 28 02:36:18 2023","interfaces":[{"interface":"back","average":{"1min":2.7170000000000001,"5min":1.7410000000000001,"15min":1.6899999999999999},"min":{"1min":0.29699999999999999,"5min":0.28199999999999997,"15min":0.27500000000000002},"max":{"1min":24.809999999999999,"5min":24.809999999999999,"15min":24.809999999999999},"last":0.496},{"interface":"front","average":{"1min":2.403,"5min":1.546,"15min":1.484},"min":{"1min":0.378,"5min":0.29499999999999998,"15min":0.25600000000000001},"max":{"1min":24.835999999999999,"5min":24.835999999999999,"15min":24.835999999999999},"last":0.75800000000000001}]},{"osd":5,"last update":"Mon Aug 28 02:36:18 2023","interfaces":[{"interface":"back","average":{"1min":8.0570000000000004,"5min":2.9809999999999999,"15min":2.2010000000000001},"min":{"1min":0.34399999999999997,"5min":0.252,"15min":0.252},"max":{"1min":82.722999999999999,"5min":82.722999999999999,"15min":82.722999999999999},"last":1.7849999999999999},{"interface":"front","average":{"1min":8.4969999999999999,"5min":2.9540000000000002,"15min":2.081},"min":{"1min":0.32900000000000001,"5min":0.307,"15min":0.307},"max":{"1min":82.733999999999995,"5min":82.733999999999995,"15min":82.733999999999995},"last":2.569}]},{"osd":6,"last update":"Mon Aug 28 02:36:18 2023","interfaces":[{"interface":"back","average":{"1min":3.024,"5min":2.0230000000000001,"15min":1.994},"min":{"1min":0.41999999999999998,"5min":0.311,"15min":0.311},"max":{"1min":19.527000000000001,"5min":19.527000000000001,"15min":19.527000000000001},"last":1.087},{"interface":"front","average":{"1min":3.4969999999999999,"5min":2.1259999999999999,"15min":1.9550000000000001},"min":{"1min":0.38,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":19.516999999999999,"5min":19.516999999999999,"15min":19.516999999999999},"last":2.528}]},{"osd":7,"last update":"Mon Aug 28 02:35:57 2023","interfaces":[{"interface":"back","average":{"1min":0.91100000000000003,"5min":0.91100000000000003,"15min":0.91100000000000003},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":2.7829999999999999,"5min":2.7829999999999999,"15min":2.7829999999999999},"last":1.8129999999999999},{"interface":"front","average":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":3.4239999999999999,"5min":3.4239999999999999,"15min":3.4239999999999999},"last":2.7549999999999999}]}]},{"osd":0,"up_from":90,"seq":386547056854,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2842768,"kb_used_data":2249480,"kb_used_omap":8,"kb_used_meta":593271,"kb_avail":90902384,"statfs":{"total":95995035648,"available":93084041216,"internally_reserved":0,"allocated":2303467520,"data_stored":2301891790,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":607509865},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Aug 28 02:35:26 2023","interfaces":[{"interface":"back","average":{"1min":1.613,"5min":2.5179999999999998,"15min":2.6139999999999999},"min":{"1min":0.24399999999999999,"5min":0.16500000000000001,"15min":0.16500000000000001},"max":{"1min":10.702999999999999,"5min":61.932000000000002,"15min":61.932000000000002},"last":0.78700000000000003},{"interface":"front","average":{"1min":3.181,"5min":3.746,"15min":3.056},"min":{"1min":0.22900000000000001,"5min":0.19500000000000001,"15min":0.17000000000000001},"max":{"1min":26.492000000000001,"5min":62.061999999999998,"15min":62.061999999999998},"last":2.0670000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:35:45 2023","interfaces":[{"interface":"back","average":{"1min":2.5110000000000001,"5min":4.5430000000000001,"15min":3.5990000000000002},"min":{"1min":0.22900000000000001,"5min":0.20599999999999999,"15min":0.19500000000000001},"max":{"1min":19.416,"5min":86.269000000000005,"15min":100.718},"last":1.266},{"interface":"front","average":{"1min":3.8809999999999998,"5min":5.5979999999999999,"15min":4.1520000000000001},"min":{"1min":0.26200000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":26.536000000000001,"5min":86.353999999999999,"15min":100.962},"last":1.889}]},{"osd":3,"last update":"Mon Aug 28 02:35:31 2023","interfaces":[{"interface":"back","average":{"1min":2.9620000000000002,"5min":5.9000000000000004,"15min":4.7889999999999997},"min":{"1min":0.28499999999999998,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":14.666,"5min":126.896,"15min":237.43899999999999},"last":0.90900000000000003},{"interface":"front","average":{"1min":2.984,"5min":5.9820000000000002,"15min":4.7220000000000004},"min":{"1min":0.26600000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":10.894,"5min":127.01600000000001,"15min":237.48099999999999},"last":4.9370000000000003}]},{"osd":4,"last update":"Mon Aug 28 02:35:22 2023","interfaces":[{"interface":"back","average":{"1min":1.1359999999999999,"5min":1.538,"15min":1.575},"min":{"1min":0.25700000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":3.7090000000000001,"5min":8.9559999999999995,"15min":9.6649999999999991},"last":0.82399999999999995},{"interface":"front","average":{"1min":2.242,"5min":1.8919999999999999,"15min":1.8600000000000001},"min":{"1min":0.34699999999999998,"5min":0.31,"15min":0.309},"max":{"1min":10.848000000000001,"5min":10.848000000000001,"15min":27.271999999999998},"last":0.93400000000000005}]},{"osd":5,"last update":"Mon Aug 28 02:35:22 2023","interfaces":[{"interface":"back","average":{"1min":2.399,"5min":1.901,"15min":1.976},"min":{"1min":0.32000000000000001,"5min":0.29899999999999999,"15min":0.29599999999999999},"max":{"1min":10.779999999999999,"5min":10.779999999999999,"15min":44.000999999999998},"last":0.95299999999999996},{"interface":"front","average":{"1min":1.3560000000000001,"5min":1.5580000000000001,"15min":1.79},"min":{"1min":0.308,"5min":0.307,"15min":0.30599999999999999},"max":{"1min":4.5990000000000002,"5min":9.0310000000000006,"15min":43.953000000000003},"last":0.877}]},{"osd":6,"last update":"Mon Aug 28 02:35:22 2023","interfaces":[{"interface":"back","average":{"1min":2.2490000000000001,"5min":2.0800000000000001,"15min":2.0600000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":10.823,"5min":10.823,"15min":32.417999999999999},"last":0.97299999999999998},{"interface":"front","average":{"1min":1.339,"5min":1.5209999999999999,"15min":1.833},"min":{"1min":0.33700000000000002,"5min":0.32700000000000001,"15min":0.30399999999999999},"max":{"1min":4.7119999999999997,"5min":9.0009999999999994,"15min":32.130000000000003},"last":0.84499999999999997}]},{"osd":7,"last update":"Mon Aug 28 02:35:57 2023","interfaces":[{"interface":"back","average":{"1min":1.147,"5min":1.147,"15min":1.147},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":3.242,"5min":3.242,"15min":3.242},"last":0.85999999999999999},{"interface":"front","average":{"1min":1.9259999999999999,"5min":1.9259999999999999,"15min":1.9259999999999999},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":16.529,"5min":16.529,"15min":16.529},"last":1.0389999999999999}]}]},{"osd":2,"up_from":104,"seq":446676598978,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2561396,"kb_used_data":1977004,"kb_used_omap":10,"kb_used_meta":584373,"kb_avail":91183756,"statfs":{"total":95995035648,"available":93372166144,"internally_reserved":0,"allocated":2024452096,"data_stored":2022902418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10792,"internal_metadata":598398424},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:35:49 2023","interfaces":[{"interface":"back","average":{"1min":2.0600000000000001,"5min":7.1950000000000003,"15min":4.5519999999999996},"min":{"1min":0.21099999999999999,"5min":0.191,"15min":0.17599999999999999},"max":{"1min":17.791,"5min":185.93799999999999,"15min":185.93799999999999},"last":3.2610000000000001},{"interface":"front","average":{"1min":1.647,"5min":6.6210000000000004,"15min":5.5579999999999998},"min":{"1min":0.252,"5min":0.22,"15min":0.19900000000000001},"max":{"1min":6.569,"5min":187.90700000000001,"15min":187.90700000000001},"last":10.744999999999999}]},{"osd":1,"last update":"Mon Aug 28 02:35:49 2023","interfaces":[{"interface":"back","average":{"1min":4.702,"5min":2.625,"15min":2.4940000000000002},"min":{"1min":0.23100000000000001,"5min":0.221,"15min":0.20999999999999999},"max":{"1min":66.010999999999996,"5min":66.010999999999996,"15min":66.010999999999996},"last":5.7770000000000001},{"interface":"front","average":{"1min":4.0510000000000002,"5min":2.4540000000000002,"15min":2.395},"min":{"1min":0.21299999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":63.770000000000003,"5min":63.770000000000003,"15min":63.770000000000003},"last":13.558}]},{"osd":3,"last update":"Mon Aug 28 02:35:18 2023","interfaces":[{"interface":"back","average":{"1min":3.6320000000000001,"5min":5.6459999999999999,"15min":4.032},"min":{"1min":0.253,"5min":0.23100000000000001,"15min":0.22700000000000001},"max":{"1min":25.536999999999999,"5min":134.07599999999999,"15min":134.07599999999999},"last":5.8170000000000002},{"interface":"front","average":{"1min":2.77,"5min":5.4610000000000003,"15min":3.9929999999999999},"min":{"1min":0.29899999999999999,"5min":0.29299999999999998,"15min":0.223},"max":{"1min":21.399000000000001,"5min":134.172,"15min":134.172},"last":13.585000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.83599999999999997,"5min":1.319,"15min":2.903},"min":{"1min":0.29999999999999999,"5min":0.26800000000000002,"15min":0.26600000000000001},"max":{"1min":3.391,"5min":13.273,"15min":207.34100000000001},"last":13.449999999999999},{"interface":"front","average":{"1min":1.6419999999999999,"5min":2.0910000000000002,"15min":2.8940000000000001},"min":{"1min":0.33800000000000002,"5min":0.309,"15min":0.27400000000000002},"max":{"1min":17.768999999999998,"5min":22.303999999999998,"15min":207.352},"last":6.0419999999999998}]},{"osd":5,"last update":"Mon Aug 28 02:35:49 2023","interfaces":[{"interface":"back","average":{"1min":1.1160000000000001,"5min":1.5429999999999999,"15min":2.847},"min":{"1min":0.36199999999999999,"5min":0.315,"15min":0.29299999999999998},"max":{"1min":3.968,"5min":13.335000000000001,"15min":128.696},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.76500000000000001,"5min":1.389,"15min":2.871},"min":{"1min":0.34599999999999997,"5min":0.30399999999999999,"15min":0.30299999999999999},"max":{"1min":2.0539999999999998,"5min":13.303000000000001,"15min":128.84100000000001},"last":13.404999999999999}]},{"osd":6,"last update":"Mon Aug 28 02:35:49 2023","interfaces":[{"interface":"back","average":{"1min":1.6359999999999999,"5min":2.2120000000000002,"15min":1.887},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.29299999999999998},"max":{"1min":17.719000000000001,"5min":22.25,"15min":22.25},"last":6.0800000000000001},{"interface":"front","average":{"1min":1.081,"5min":1.466,"15min":1.7769999999999999},"min":{"1min":0.35599999999999998,"5min":0.30199999999999999,"15min":0.29399999999999998},"max":{"1min":3.4390000000000001,"5min":13.311,"15min":13.311},"last":0.71599999999999997}]},{"osd":7,"last update":"Mon Aug 28 02:35:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":1.0189999999999999,"15min":1.0189999999999999},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":2.8559999999999999,"5min":2.8559999999999999,"15min":2.8559999999999999},"last":0.74099999999999999},{"interface":"front","average":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":3.3410000000000002,"5min":3.3410000000000002,"15min":3.3410000000000002},"last":13.478999999999999}]}]},{"osd":3,"up_from":112,"seq":481036337340,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2251564,"kb_used_data":1692580,"kb_used_omap":3,"kb_used_meta":558972,"kb_avail":91493588,"statfs":{"total":95995035648,"available":93689434112,"internally_reserved":0,"allocated":1733201920,"data_stored":1731653438,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":572388258},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:36:17 2023","interfaces":[{"interface":"back","average":{"1min":3.1669999999999998,"5min":5.1580000000000004,"15min":4.7169999999999996},"min":{"1min":0.20799999999999999,"5min":0.19500000000000001,"15min":0.193},"max":{"1min":24.561,"5min":138.39699999999999,"15min":164.25299999999999},"last":4.2519999999999998},{"interface":"front","average":{"1min":2.0640000000000001,"5min":2.9079999999999999,"15min":3.9169999999999998},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.192},"max":{"1min":24.048999999999999,"5min":24.048999999999999,"15min":164.55000000000001},"last":0.84499999999999997}]},{"osd":1,"last update":"Mon Aug 28 02:36:17 2023","interfaces":[{"interface":"back","average":{"1min":7.6219999999999999,"5min":4.0919999999999996,"15min":3.1669999999999998},"min":{"1min":0.222,"5min":0.222,"15min":0.20699999999999999},"max":{"1min":105.04600000000001,"5min":105.04600000000001,"15min":117.34699999999999},"last":6.6859999999999999},{"interface":"front","average":{"1min":7.6379999999999999,"5min":5.7039999999999997,"15min":3.8340000000000001},"min":{"1min":0.29099999999999998,"5min":0.23599999999999999,"15min":0.224},"max":{"1min":105.55800000000001,"5min":143.54400000000001,"15min":143.54400000000001},"last":4.4080000000000004}]},{"osd":2,"last update":"Mon Aug 28 02:36:17 2023","interfaces":[{"interface":"back","average":{"1min":2.052,"5min":5.3099999999999996,"15min":4.5599999999999996},"min":{"1min":0.26700000000000002,"5min":0.20100000000000001,"15min":0.19900000000000001},"max":{"1min":9.9789999999999992,"5min":143.50299999999999,"15min":143.50299999999999},"last":4.6280000000000001},{"interface":"front","average":{"1min":1.583,"5min":2.9039999999999999,"15min":3.6589999999999998},"min":{"1min":0.23699999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":9.9120000000000008,"5min":12.646000000000001,"15min":120.44499999999999},"last":0.28699999999999998}]},{"osd":4,"last update":"Mon Aug 28 02:36:17 2023","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":1.5620000000000001,"15min":1.827},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":2.8980000000000001,"5min":7.8940000000000001,"15min":58.067999999999998},"last":0.34399999999999997},{"interface":"front","average":{"1min":1.101,"5min":1.7609999999999999,"15min":1.9390000000000001},"min":{"1min":0.38400000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":2.8519999999999999,"5min":10.613,"15min":58.088000000000001},"last":1.282}]},{"osd":5,"last update":"Mon Aug 28 02:36:17 2023","interfaces":[{"interface":"back","average":{"1min":1.0669999999999999,"5min":1.9259999999999999,"15min":2.1099999999999999},"min":{"1min":0.39300000000000002,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.8769999999999998,"5min":10.722,"15min":102.904},"last":1.2529999999999999},{"interface":"front","average":{"1min":1.3819999999999999,"5min":3.798,"15min":2.7589999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":4.319,"5min":138.328,"15min":138.328},"last":4.319}]},{"osd":6,"last update":"Mon Aug 28 02:36:17 2023","interfaces":[{"interface":"back","average":{"1min":1.48,"5min":3.7749999999999999,"15min":2.5350000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.314},"max":{"1min":4.351,"5min":138.37,"15min":138.37},"last":4.351},{"interface":"front","average":{"1min":1.179,"5min":1.655,"15min":1.7749999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.314},"max":{"1min":5.5060000000000002,"5min":7.9400000000000004,"15min":72.030000000000001},"last":0.32100000000000001}]},{"osd":7,"last update":"Mon Aug 28 02:35:57 2023","interfaces":[{"interface":"back","average":{"1min":1.155,"5min":1.155,"15min":1.155},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":3.46,"5min":3.46,"15min":3.46},"last":4.3789999999999996},{"interface":"front","average":{"1min":1.2869999999999999,"5min":1.2869999999999999,"15min":1.2869999999999999},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":5.4560000000000004,"5min":5.4560000000000004,"15min":5.4560000000000004},"last":0.373}]}]},{"osd":4,"up_from":30,"seq":128849019214,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2051648,"kb_used_data":1457848,"kb_used_omap":0,"kb_used_meta":593792,"kb_avail":91693504,"statfs":{"total":95995035648,"available":93894148096,"internally_reserved":0,"allocated":1492836352,"data_stored":1491959817,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":608043008},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:35:24 2023","interfaces":[{"interface":"back","average":{"1min":2.1899999999999999,"5min":7.0650000000000004,"15min":4.8550000000000004},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.219},"max":{"1min":14.787000000000001,"5min":131.893,"15min":138.67400000000001},"last":0.91300000000000003},{"interface":"front","average":{"1min":2.2149999999999999,"5min":6.8109999999999999,"15min":4.7080000000000002},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":11.548,"5min":131.96700000000001,"15min":134.14599999999999},"last":3.032}]},{"osd":1,"last update":"Mon Aug 28 02:35:24 2023","interfaces":[{"interface":"back","average":{"1min":1.383,"5min":1.897,"15min":1.8500000000000001},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.221},"max":{"1min":9.9909999999999997,"5min":13.090999999999999,"15min":53.622999999999998},"last":0.69199999999999995},{"interface":"front","average":{"1min":1.099,"5min":1.3939999999999999,"15min":1.6839999999999999},"min":{"1min":0.30599999999999999,"5min":0.245,"15min":0.245},"max":{"1min":5.335,"5min":6.782,"15min":53.774000000000001},"last":3.4209999999999998}]},{"osd":2,"last update":"Mon Aug 28 02:35:50 2023","interfaces":[{"interface":"back","average":{"1min":0.98199999999999998,"5min":3.5270000000000001,"15min":3.8180000000000001},"min":{"1min":0.29699999999999999,"5min":0.255,"15min":0.255},"max":{"1min":5.6319999999999997,"5min":107.786,"15min":107.786},"last":0.77400000000000002},{"interface":"front","average":{"1min":1.0289999999999999,"5min":3.504,"15min":3.7210000000000001},"min":{"1min":0.312,"5min":0.27300000000000002,"15min":0.25},"max":{"1min":5.2169999999999996,"5min":107.599,"15min":107.599},"last":0.96099999999999997}]},{"osd":3,"last update":"Mon Aug 28 02:35:24 2023","interfaces":[{"interface":"back","average":{"1min":3.7440000000000002,"5min":8.8379999999999992,"15min":5.2889999999999997},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":48.427999999999997,"5min":203.583,"15min":203.583},"last":0.90300000000000002},{"interface":"front","average":{"1min":3.879,"5min":9.0760000000000005,"15min":5.4290000000000003},"min":{"1min":0.34000000000000002,"5min":0.33000000000000002,"15min":0.32500000000000001},"max":{"1min":48.353000000000002,"5min":203.41399999999999,"15min":203.41399999999999},"last":0.61399999999999999}]},{"osd":5,"last update":"Mon Aug 28 02:36:03 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.45200000000000001,"15min":1.4410000000000001},"min":{"1min":0.26800000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":2.218,"5min":2.3149999999999999,"15min":124.718},"last":1.4399999999999999},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.55200000000000005,"15min":1.6579999999999999},"min":{"1min":0.27200000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.5620000000000001,"5min":2.444,"15min":124.694},"last":0.749}]},{"osd":6,"last update":"Mon Aug 28 02:35:24 2023","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":1.038,"15min":1.123},"min":{"1min":0.28599999999999998,"5min":0.25,"15min":0.25},"max":{"1min":1.306,"5min":14.907999999999999,"15min":14.907999999999999},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.432,"5min":0.73399999999999999,"15min":0.88900000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.61699999999999999,"5min":5.1609999999999996,"15min":7.3170000000000002},"last":0.84399999999999997}]},{"osd":7,"last update":"Mon Aug 28 02:35:55 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":2.1989999999999998,"5min":2.1989999999999998,"15min":2.1989999999999998},"last":0.71799999999999997},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":1.6100000000000001,"5min":1.6100000000000001,"15min":1.6100000000000001},"last":0.97599999999999998}]}]},{"osd":5,"up_from":36,"seq":154618822986,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3187504,"kb_used_data":2601896,"kb_used_omap":0,"kb_used_meta":585600,"kb_avail":90557648,"statfs":{"total":95995035648,"available":92731031552,"internally_reserved":0,"allocated":2664341504,"data_stored":2663476508,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":599654400},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:36:13 2023","interfaces":[{"interface":"back","average":{"1min":5.1289999999999996,"5min":4.782,"15min":4.399},"min":{"1min":0.216,"5min":0.216,"15min":0.20300000000000001},"max":{"1min":64.590999999999994,"5min":128.72800000000001,"15min":131.06100000000001},"last":1.2150000000000001},{"interface":"front","average":{"1min":5.1520000000000001,"5min":4.8019999999999996,"15min":4.6130000000000004},"min":{"1min":0.245,"5min":0.245,"15min":0.22900000000000001},"max":{"1min":64.578999999999994,"5min":128.75200000000001,"15min":131.262},"last":1.2250000000000001}]},{"osd":1,"last update":"Mon Aug 28 02:35:26 2023","interfaces":[{"interface":"back","average":{"1min":9.7219999999999995,"5min":2.952,"15min":3.1960000000000002},"min":{"1min":0.315,"5min":0.27300000000000002,"15min":0.216},"max":{"1min":98.632000000000005,"5min":98.632000000000005,"15min":183.97900000000001},"last":2.0590000000000002},{"interface":"front","average":{"1min":9.3040000000000003,"5min":2.968,"15min":3.3599999999999999},"min":{"1min":0.28599999999999998,"5min":0.26400000000000001,"15min":0.23200000000000001},"max":{"1min":98.560000000000002,"5min":98.560000000000002,"15min":184.33699999999999},"last":1.2729999999999999}]},{"osd":2,"last update":"Mon Aug 28 02:35:55 2023","interfaces":[{"interface":"back","average":{"1min":0.76300000000000001,"5min":2.944,"15min":4.7590000000000003},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.252},"max":{"1min":3.3690000000000002,"5min":78.834000000000003,"15min":176.69499999999999},"last":0.66900000000000004},{"interface":"front","average":{"1min":0.90700000000000003,"5min":3.0009999999999999,"15min":5.0149999999999997},"min":{"1min":0.35399999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.3940000000000001,"5min":78.798000000000002,"15min":176.71199999999999},"last":0.68500000000000005}]},{"osd":3,"last update":"Mon Aug 28 02:35:26 2023","interfaces":[{"interface":"back","average":{"1min":2.3500000000000001,"5min":6.5129999999999999,"15min":3.5859999999999999},"min":{"1min":0.316,"5min":0.29699999999999999,"15min":0.26100000000000001},"max":{"1min":19.190999999999999,"5min":128.571,"15min":128.571},"last":14.202},{"interface":"front","average":{"1min":1.1100000000000001,"5min":6.2060000000000004,"15min":3.8159999999999998},"min":{"1min":0.34599999999999997,"5min":0.28699999999999998,"15min":0.26200000000000001},"max":{"1min":4.1699999999999999,"5min":128.684,"15min":128.684},"last":2.3759999999999999}]},{"osd":4,"last update":"Mon Aug 28 02:36:04 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.55200000000000005,"15min":1.335},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":2.145,"5min":3.0259999999999998,"15min":85.769999999999996},"last":0.28199999999999997},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.499,"15min":1.3100000000000001},"min":{"1min":0.29899999999999999,"5min":0.248,"15min":0.248},"max":{"1min":0.94699999999999995,"5min":2.7690000000000001,"15min":85.825000000000003},"last":0.34799999999999998}]},{"osd":6,"last update":"Mon Aug 28 02:35:26 2023","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.65300000000000002,"15min":1.0529999999999999},"min":{"1min":0.28699999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":1.3300000000000001,"5min":4.1449999999999996,"15min":14.803000000000001},"last":0.313},{"interface":"front","average":{"1min":0.76600000000000001,"5min":0.70199999999999996,"15min":0.97299999999999998},"min":{"1min":0.307,"5min":0.29599999999999999,"15min":0.28799999999999998},"max":{"1min":7.4740000000000002,"5min":7.4740000000000002,"15min":8.6479999999999997},"last":0.40899999999999997}]},{"osd":7,"last update":"Mon Aug 28 02:35:55 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":2.0049999999999999,"5min":2.0049999999999999,"15min":2.0049999999999999},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":2.7839999999999998,"5min":2.7839999999999998,"15min":2.7839999999999998},"last":0.41999999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2323054592,"data_stored":2323054592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2114519040,"data_stored":2114519040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2030239744,"data_stored":2030239744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1727463424,"data_stored":1727463424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1492123648,"data_stored":1492123648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2670985216,"data_stored":2670985216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1795293184,"data_stored":1795293184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-08-28T02:36:27.529 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph pg dump --format=json 2023-08-28T02:36:27.609 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:36:27.617 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:36:27.646 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:27.705 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:36:27.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:27.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:36:27.726 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:36:27.813 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:36:27.946 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:28.092 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:36:28.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:28.099 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:36:28.200 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:36:28.202 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 65 16 77876 77860 74.831 48.5 0.0183729 0.013359 2023-08-28T02:36:28.232 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:28.288 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:36:28.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:28.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:36:28.393 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:36:28.485 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:28.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:28 smithi143 ceph-mon[183059]: pgmap v876: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 52 MiB/s wr, 837 op/s 2023-08-28T02:36:28.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:28 smithi143 ceph-mon[189871]: pgmap v876: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 52 MiB/s wr, 837 op/s 2023-08-28T02:36:28.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:28 smithi169 ceph-mon[159415]: pgmap v876: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 52 MiB/s wr, 837 op/s 2023-08-28T02:36:29.089 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:36:29.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:29.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:36:29.192 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:36:29.202 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 66 16 78520 78504 74.3072 40.25 0.010688 0.0134275 2023-08-28T02:36:29.219 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:29.624 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:36:29.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:29.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:36:29.735 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:36:29.779 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:29.820 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:36:29.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:29.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:36:29.926 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:36:29.995 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:30.206 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 67 16 79483 79467 74.0924 60.1875 0.017984 0.0134903 2023-08-28T02:36:30.401 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:36:30.566 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:36:30.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:30.582 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:36:30.682 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:30 smithi143 ceph-mon[183059]: pgmap v877: 89 pgs: 89 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 51 MiB/s wr, 822 op/s 2023-08-28T02:36:30.683 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:36:30.769 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:36:30.770 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:30.943 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:30 smithi143 ceph-mon[189871]: pgmap v877: 89 pgs: 89 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 51 MiB/s wr, 822 op/s 2023-08-28T02:36:30.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:30 smithi169 ceph-mon[159415]: pgmap v877: 89 pgs: 89 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 51 MiB/s wr, 822 op/s 2023-08-28T02:36:31.159 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:36:31.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:31.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:36:31.207 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 68 16 80324 80308 73.7751 52.5625 0.0105002 0.0135485 2023-08-28T02:36:31.268 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:36:31.268 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:36:31.299 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:31.305 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:36:31.342 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:36:31.360 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:36:31.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:31.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:36:31.387 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:36:31.432 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:36:31.466 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:36:31.468 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:36:31.525 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:36:31.564 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:31.573 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:36:31.592 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:36:31.592 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:36:31.592 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:36:31.592 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:36:31.592 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:36:31.592 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:36:31.592 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:36:31.592 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:36:31.799 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:31 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/115257098' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:36:31.800 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:31 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/115257098' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:36:31.927 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:36:31.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:31.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:36:31.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:31 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/115257098' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:36:32.050 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:36:32.091 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:32.118 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:36:31 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:36:31] "GET /metrics HTTP/1.1" 200 36532 "" "Prometheus/2.33.4" 2023-08-28T02:36:32.173 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:36:32.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:32.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:36:32.207 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 69 16 81155 81139 73.4586 51.9375 0.0134903 0.0136079 2023-08-28T02:36:32.282 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:36:32.318 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:32.380 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:36:32.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:32.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:36:32.495 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:36:32.529 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:32.605 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:36:32.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:32.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:36:32.706 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is set 2023-08-28T02:36:32.712 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:36:32.743 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:32.806 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:36:32.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:32.811 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:36:32.912 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:36:32.945 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:32.953 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:32 smithi143 ceph-mon[183059]: pgmap v878: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 51 MiB/s wr, 818 op/s 2023-08-28T02:36:32.953 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:32 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3460506410' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:36:32.953 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:32 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:36:32.957 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:32 smithi143 ceph-mon[189871]: pgmap v878: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 51 MiB/s wr, 818 op/s 2023-08-28T02:36:32.957 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:32 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3460506410' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:36:32.958 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:32 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:36:32.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:32 smithi169 ceph-mon[159415]: pgmap v878: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 51 MiB/s wr, 818 op/s 2023-08-28T02:36:32.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:32 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3460506410' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:36:32.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:32 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:36:33.021 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:36:33.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:33.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:36:33.126 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:36:33.166 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:33.207 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 70 16 82111 82095 73.2628 59.75 0.0158976 0.0136318 2023-08-28T02:36:33.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:33 smithi143 ceph-mon[183059]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:36:33.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:33 smithi143 ceph-mon[183059]: failed to encode map e337 with expected crc 2023-08-28T02:36:33.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:33 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:36:33.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:33 smithi143 ceph-mon[183059]: osdmap e337: 8 total, 8 up, 7 in 2023-08-28T02:36:33.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:33 smithi143 ceph-mon[183059]: failed to encode map e337 with expected crc 2023-08-28T02:36:33.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:33 smithi143 ceph-mon[183059]: failed to encode map e337 with expected crc 2023-08-28T02:36:33.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:33 smithi143 ceph-mon[183059]: failed to encode map e337 with expected crc 2023-08-28T02:36:33.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:33 smithi143 ceph-mon[189871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:36:33.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:33 smithi143 ceph-mon[189871]: failed to encode map e337 with expected crc 2023-08-28T02:36:33.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:33 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:36:33.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:33 smithi143 ceph-mon[189871]: osdmap e337: 8 total, 8 up, 7 in 2023-08-28T02:36:33.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:33 smithi143 ceph-mon[189871]: failed to encode map e337 with expected crc 2023-08-28T02:36:33.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:33 smithi143 ceph-mon[189871]: failed to encode map e337 with expected crc 2023-08-28T02:36:33.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:33 smithi143 ceph-mon[189871]: failed to encode map e337 with expected crc 2023-08-28T02:36:33.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:33 smithi169 ceph-mon[159415]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:36:33.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:33 smithi169 ceph-mon[159415]: failed to encode map e337 with expected crc 2023-08-28T02:36:33.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:33 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:36:33.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:33 smithi169 ceph-mon[159415]: osdmap e337: 8 total, 8 up, 7 in 2023-08-28T02:36:33.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:33 smithi169 ceph-mon[159415]: failed to encode map e337 with expected crc 2023-08-28T02:36:33.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:33 smithi169 ceph-mon[159415]: failed to encode map e337 with expected crc 2023-08-28T02:36:33.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:33 smithi169 ceph-mon[159415]: failed to encode map e337 with expected crc 2023-08-28T02:36:34.208 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 71 16 82832 82816 72.8656 45.0625 0.0100036 0.0137188 2023-08-28T02:36:34.361 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:36:34.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:34.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:36:34.468 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:36:34.497 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:34.538 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:36:34.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:34.544 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:36:34.645 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:36:34.719 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:34.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:34 smithi169 ceph-mon[159415]: pgmap v880: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 52 MiB/s wr, 829 op/s 2023-08-28T02:36:35.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:34 smithi143 ceph-mon[183059]: pgmap v880: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 52 MiB/s wr, 829 op/s 2023-08-28T02:36:35.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:34 smithi143 ceph-mon[189871]: pgmap v880: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 52 MiB/s wr, 829 op/s 2023-08-28T02:36:35.207 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 72 16 83571 83555 72.4952 46.1875 0.017067 0.0137883 2023-08-28T02:36:35.514 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:36:35.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:35.520 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:36:35.621 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:36:35.650 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:35.712 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:36:35.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:35.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:36:35.819 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:36:35.849 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:35.910 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:36:35.910 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [], 2023-08-28T02:36:35.910 INFO:teuthology.orchestra.run.smithi143.stdout: "complaint_time": 30, 2023-08-28T02:36:35.910 INFO:teuthology.orchestra.run.smithi143.stdout: "num_blocked_ops": 0 2023-08-28T02:36:35.910 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:36:35.910 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:36:35.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:35.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:36:36.017 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:36:36.043 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:36.099 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:36:36.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:36.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:36:36.208 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:36:36.209 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 73 16 84414 84398 72.2239 52.6875 0.0229717 0.0138406 2023-08-28T02:36:36.239 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:36.296 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:36:36.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:36.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:36:36.403 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:36:36.439 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:36:36.461 INFO:teuthology.orchestra.run.smithi143.stderr:dumped all 2023-08-28T02:36:36.502 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:36.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:36 smithi143 ceph-mon[183059]: pgmap v881: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 51 MiB/s wr, 823 op/s 2023-08-28T02:36:36.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:36 smithi143 ceph-mon[189871]: pgmap v881: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 51 MiB/s wr, 823 op/s 2023-08-28T02:36:36.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:36 smithi169 ceph-mon[159415]: pgmap v881: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 51 MiB/s wr, 823 op/s 2023-08-28T02:36:37.208 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 74 16 85303 85287 71.9988 55.5625 0.0217534 0.013884 2023-08-28T02:36:37.235 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:36:37.302 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:36:37.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:37.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:36:37.408 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:36:37.409 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set nodeep-scrub 2023-08-28T02:36:37.514 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:37.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:37 smithi143 ceph-mon[183059]: from='client.45725 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:36:37.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:37 smithi143 ceph-mon[189871]: from='client.45725 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:36:37.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:37 smithi169 ceph-mon[159415]: from='client.45725 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:36:38.086 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.1 dump_historic_ops 2023-08-28T02:36:38.208 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 75 16 86009 85993 71.6272 44.125 0.020231 0.0139557 2023-08-28T02:36:38.320 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:36:38.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:38.328 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:36:38.429 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:36:38.458 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:38.520 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:36:38.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:38.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:36:38.561 INFO:teuthology.orchestra.run.smithi143.stdout:{"pg_ready":true,"pg_map":{"version":881,"stamp":"2023-08-28T02:36:35.529754+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5392963507,"num_objects":82245,"num_object_clones":0,"num_object_copies":246735,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":82245,"num_whiteouts":0,"num_read":6274,"num_read_kb":13707,"num_write":83323,"num_write_kb":5277029,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":297,"num_bytes_recovered":14969132,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":83387,"ondisk_log_size":83387,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":19461120,"kb_used_data":15701256,"kb_used_omap":29,"kb_used_meta":3759778,"kb_avail":636754944,"statfs":{"total":671965249536,"available":652037062656,"internally_reserved":0,"allocated":16078086144,"data_stored":16069042124,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30674,"internal_metadata":3850012718},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,2,3,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":540344320,"num_objects":8245,"num_object_clones":0,"num_object_copies":24735,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8245,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8245,"num_write_kb":527680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.015563"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":355,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.807107+0000","last_change":"2023-08-28T02:32:58.740384+0000","last_active":"2023-08-28T02:36:14.807107+0000","last_peered":"2023-08-28T02:36:14.807107+0000","last_clean":"2023-08-28T02:36:14.807107+0000","last_became_active":"2023-08-28T02:32:58.739959+0000","last_became_peered":"2023-08-28T02:32:58.739959+0000","last_unstale":"2023-08-28T02:36:14.807107+0000","last_undegraded":"2023-08-28T02:36:14.807107+0000","last_fullsized":"2023-08-28T02:36:14.807107+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":244,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:28.843457+0000","last_change":"2023-08-28T02:32:58.634375+0000","last_active":"2023-08-28T02:36:28.843457+0000","last_peered":"2023-08-28T02:36:28.843457+0000","last_clean":"2023-08-28T02:36:28.843457+0000","last_became_active":"2023-08-28T02:32:58.632726+0000","last_became_peered":"2023-08-28T02:32:58.632726+0000","last_unstale":"2023-08-28T02:36:28.843457+0000","last_undegraded":"2023-08-28T02:36:28.843457+0000","last_fullsized":"2023-08-28T02:36:28.843457+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:20:27.730725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":57,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.641929+0000","last_change":"2023-08-28T02:32:58.628387+0000","last_active":"2023-08-28T02:36:21.641929+0000","last_peered":"2023-08-28T02:36:21.641929+0000","last_clean":"2023-08-28T02:36:21.641929+0000","last_became_active":"2023-08-28T02:32:58.628071+0000","last_became_peered":"2023-08-28T02:32:58.628071+0000","last_unstale":"2023-08-28T02:36:21.641929+0000","last_undegraded":"2023-08-28T02:36:21.641929+0000","last_fullsized":"2023-08-28T02:36:21.641929+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:25:55.264433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":354,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.675367+0000","last_change":"2023-08-28T02:32:58.772285+0000","last_active":"2023-08-28T02:36:21.675367+0000","last_peered":"2023-08-28T02:36:21.675367+0000","last_clean":"2023-08-28T02:36:21.675367+0000","last_became_active":"2023-08-28T02:32:58.771439+0000","last_became_peered":"2023-08-28T02:32:58.771439+0000","last_unstale":"2023-08-28T02:36:21.675367+0000","last_undegraded":"2023-08-28T02:36:21.675367+0000","last_fullsized":"2023-08-28T02:36:21.675367+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:10:48.541785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":235,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.669649+0000","last_change":"2023-08-28T02:32:58.779602+0000","last_active":"2023-08-28T02:36:21.669649+0000","last_peered":"2023-08-28T02:36:21.669649+0000","last_clean":"2023-08-28T02:36:21.669649+0000","last_became_active":"2023-08-28T02:32:58.778945+0000","last_became_peered":"2023-08-28T02:32:58.778945+0000","last_unstale":"2023-08-28T02:36:21.669649+0000","last_undegraded":"2023-08-28T02:36:21.669649+0000","last_fullsized":"2023-08-28T02:36:21.669649+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:35:55.832928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":142,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.793841+0000","last_change":"2023-08-28T02:32:58.626766+0000","last_active":"2023-08-28T02:36:14.793841+0000","last_peered":"2023-08-28T02:36:14.793841+0000","last_clean":"2023-08-28T02:36:14.793841+0000","last_became_active":"2023-08-28T02:32:58.626415+0000","last_became_peered":"2023-08-28T02:32:58.626415+0000","last_unstale":"2023-08-28T02:36:14.793841+0000","last_undegraded":"2023-08-28T02:36:14.793841+0000","last_fullsized":"2023-08-28T02:36:14.793841+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":307,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.675313+0000","last_change":"2023-08-28T02:23:59.227592+0000","last_active":"2023-08-28T02:36:21.675313+0000","last_peered":"2023-08-28T02:36:21.675313+0000","last_clean":"2023-08-28T02:36:21.675313+0000","last_became_active":"2023-08-28T02:23:59.227342+0000","last_became_peered":"2023-08-28T02:23:59.227342+0000","last_unstale":"2023-08-28T02:36:21.675313+0000","last_undegraded":"2023-08-28T02:36:21.675313+0000","last_fullsized":"2023-08-28T02:36:21.675313+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:08:19.874396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":253,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.669722+0000","last_change":"2023-08-28T02:32:58.779435+0000","last_active":"2023-08-28T02:36:21.669722+0000","last_peered":"2023-08-28T02:36:21.669722+0000","last_clean":"2023-08-28T02:36:21.669722+0000","last_became_active":"2023-08-28T02:32:58.778629+0000","last_became_peered":"2023-08-28T02:32:58.778629+0000","last_unstale":"2023-08-28T02:36:21.669722+0000","last_undegraded":"2023-08-28T02:36:21.669722+0000","last_fullsized":"2023-08-28T02:36:21.669722+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:25:32.193898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":345,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.642423+0000","last_change":"2023-08-28T02:35:05.992277+0000","last_active":"2023-08-28T02:36:21.642423+0000","last_peered":"2023-08-28T02:36:21.642423+0000","last_clean":"2023-08-28T02:36:21.642423+0000","last_became_active":"2023-08-28T02:35:05.991954+0000","last_became_peered":"2023-08-28T02:35:05.991954+0000","last_unstale":"2023-08-28T02:36:21.642423+0000","last_undegraded":"2023-08-28T02:36:21.642423+0000","last_fullsized":"2023-08-28T02:36:21.642423+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:41:10.177837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":226,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.667205+0000","last_change":"2023-08-28T02:31:55.525178+0000","last_active":"2023-08-28T02:36:21.667205+0000","last_peered":"2023-08-28T02:36:21.667205+0000","last_clean":"2023-08-28T02:36:21.667205+0000","last_became_active":"2023-08-28T02:31:55.524801+0000","last_became_peered":"2023-08-28T02:31:55.524801+0000","last_unstale":"2023-08-28T02:36:21.667205+0000","last_undegraded":"2023-08-28T02:36:21.667205+0000","last_fullsized":"2023-08-28T02:36:21.667205+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:47:30.078960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":104,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.667623+0000","last_change":"2023-08-28T02:31:55.532306+0000","last_active":"2023-08-28T02:36:21.667623+0000","last_peered":"2023-08-28T02:36:21.667623+0000","last_clean":"2023-08-28T02:36:21.667623+0000","last_became_active":"2023-08-28T02:31:55.524276+0000","last_became_peered":"2023-08-28T02:31:55.524276+0000","last_unstale":"2023-08-28T02:36:21.667623+0000","last_undegraded":"2023-08-28T02:36:21.667623+0000","last_fullsized":"2023-08-28T02:36:21.667623+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:27:06.850784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":67,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.793495+0000","last_change":"2023-08-28T02:35:45.112374+0000","last_active":"2023-08-28T02:36:14.793495+0000","last_peered":"2023-08-28T02:36:14.793495+0000","last_clean":"2023-08-28T02:36:14.793495+0000","last_became_active":"2023-08-28T02:35:45.112024+0000","last_became_peered":"2023-08-28T02:35:45.112024+0000","last_unstale":"2023-08-28T02:36:14.793495+0000","last_undegraded":"2023-08-28T02:36:14.793495+0000","last_fullsized":"2023-08-28T02:36:14.793495+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"86'22","reported_seq":405,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.807033+0000","last_change":"2023-08-28T02:31:55.538001+0000","last_active":"2023-08-28T02:36:14.807033+0000","last_peered":"2023-08-28T02:36:14.807033+0000","last_clean":"2023-08-28T02:36:14.807033+0000","last_became_active":"2023-08-28T02:31:55.518604+0000","last_became_peered":"2023-08-28T02:31:55.518604+0000","last_unstale":"2023-08-28T02:36:14.807033+0000","last_undegraded":"2023-08-28T02:36:14.807033+0000","last_fullsized":"2023-08-28T02:36:14.807033+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":379,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.806514+0000","last_change":"2023-08-28T02:32:58.629451+0000","last_active":"2023-08-28T02:36:14.806514+0000","last_peered":"2023-08-28T02:36:14.806514+0000","last_clean":"2023-08-28T02:36:14.806514+0000","last_became_active":"2023-08-28T02:32:58.628767+0000","last_became_peered":"2023-08-28T02:32:58.628767+0000","last_unstale":"2023-08-28T02:36:14.806514+0000","last_undegraded":"2023-08-28T02:36:14.806514+0000","last_fullsized":"2023-08-28T02:36:14.806514+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":109,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.806591+0000","last_change":"2023-08-28T02:31:55.528552+0000","last_active":"2023-08-28T02:36:14.806591+0000","last_peered":"2023-08-28T02:36:14.806591+0000","last_clean":"2023-08-28T02:36:14.806591+0000","last_became_active":"2023-08-28T02:31:55.518079+0000","last_became_peered":"2023-08-28T02:31:55.518079+0000","last_unstale":"2023-08-28T02:36:14.806591+0000","last_undegraded":"2023-08-28T02:36:14.806591+0000","last_fullsized":"2023-08-28T02:36:14.806591+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":244,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.669471+0000","last_change":"2023-08-28T02:32:58.680534+0000","last_active":"2023-08-28T02:36:21.669471+0000","last_peered":"2023-08-28T02:36:21.669471+0000","last_clean":"2023-08-28T02:36:21.669471+0000","last_became_active":"2023-08-28T02:32:58.680145+0000","last_became_peered":"2023-08-28T02:32:58.680145+0000","last_unstale":"2023-08-28T02:36:21.669471+0000","last_undegraded":"2023-08-28T02:36:21.669471+0000","last_fullsized":"2023-08-28T02:36:21.669471+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:56:40.113013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":352,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.674689+0000","last_change":"2023-08-28T02:32:58.746929+0000","last_active":"2023-08-28T02:36:21.674689+0000","last_peered":"2023-08-28T02:36:21.674689+0000","last_clean":"2023-08-28T02:36:21.674689+0000","last_became_active":"2023-08-28T02:32:58.746590+0000","last_became_peered":"2023-08-28T02:32:58.746590+0000","last_unstale":"2023-08-28T02:36:21.674689+0000","last_undegraded":"2023-08-28T02:36:21.674689+0000","last_fullsized":"2023-08-28T02:36:21.674689+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:51:11.974154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":336,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.667549+0000","last_change":"2023-08-28T02:35:05.993977+0000","last_active":"2023-08-28T02:36:21.667549+0000","last_peered":"2023-08-28T02:36:21.667549+0000","last_clean":"2023-08-28T02:36:21.667549+0000","last_became_active":"2023-08-28T02:35:05.993610+0000","last_became_peered":"2023-08-28T02:35:05.993610+0000","last_unstale":"2023-08-28T02:36:21.667549+0000","last_undegraded":"2023-08-28T02:36:21.667549+0000","last_fullsized":"2023-08-28T02:36:21.667549+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:37:10.901664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":165,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.806246+0000","last_change":"2023-08-28T02:31:55.517851+0000","last_active":"2023-08-28T02:36:14.806246+0000","last_peered":"2023-08-28T02:36:14.806246+0000","last_clean":"2023-08-28T02:36:14.806246+0000","last_became_active":"2023-08-28T02:31:55.517320+0000","last_became_peered":"2023-08-28T02:31:55.517320+0000","last_unstale":"2023-08-28T02:36:14.806246+0000","last_undegraded":"2023-08-28T02:36:14.806246+0000","last_fullsized":"2023-08-28T02:36:14.806246+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":335,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.675494+0000","last_change":"2023-08-28T02:31:55.528250+0000","last_active":"2023-08-28T02:36:21.675494+0000","last_peered":"2023-08-28T02:36:21.675494+0000","last_clean":"2023-08-28T02:36:21.675494+0000","last_became_active":"2023-08-28T02:31:55.527950+0000","last_became_peered":"2023-08-28T02:31:55.527950+0000","last_unstale":"2023-08-28T02:36:21.675494+0000","last_undegraded":"2023-08-28T02:36:21.675494+0000","last_fullsized":"2023-08-28T02:36:21.675494+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:39:46.120437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":134,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.793393+0000","last_change":"2023-08-28T02:28:25.703333+0000","last_active":"2023-08-28T02:36:14.793393+0000","last_peered":"2023-08-28T02:36:14.793393+0000","last_clean":"2023-08-28T02:36:14.793393+0000","last_became_active":"2023-08-28T02:28:25.343515+0000","last_became_peered":"2023-08-28T02:28:25.343515+0000","last_unstale":"2023-08-28T02:36:14.793393+0000","last_undegraded":"2023-08-28T02:36:14.793393+0000","last_fullsized":"2023-08-28T02:36:14.793393+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":253,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.667626+0000","last_change":"2023-08-28T02:35:45.097133+0000","last_active":"2023-08-28T02:36:21.667626+0000","last_peered":"2023-08-28T02:36:21.667626+0000","last_clean":"2023-08-28T02:36:21.667626+0000","last_became_active":"2023-08-28T02:35:45.096766+0000","last_became_peered":"2023-08-28T02:35:45.096766+0000","last_unstale":"2023-08-28T02:36:21.667626+0000","last_undegraded":"2023-08-28T02:36:21.667626+0000","last_fullsized":"2023-08-28T02:36:21.667626+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:27:29.558704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"336'5042","reported_seq":5068,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:31.999702+0000","last_change":"2023-08-28T02:35:14.499701+0000","last_active":"2023-08-28T02:36:31.999702+0000","last_peered":"2023-08-28T02:36:31.999702+0000","last_clean":"2023-08-28T02:36:31.999702+0000","last_became_active":"2023-08-28T02:35:14.499400+0000","last_became_peered":"2023-08-28T02:35:14.499400+0000","last_unstale":"2023-08-28T02:36:31.999702+0000","last_undegraded":"2023-08-28T02:36:31.999702+0000","last_fullsized":"2023-08-28T02:36:31.999702+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":5042,"log_dups_size":0,"ondisk_log_size":5042,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330432512,"num_objects":5042,"num_object_clones":0,"num_object_copies":15126,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5042,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5042,"num_write_kb":322688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":200,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:30.021744+0000","last_change":"2023-08-28T02:32:58.782840+0000","last_active":"2023-08-28T02:36:30.021744+0000","last_peered":"2023-08-28T02:36:30.021744+0000","last_clean":"2023-08-28T02:36:30.021744+0000","last_became_active":"2023-08-28T02:32:58.683096+0000","last_became_peered":"2023-08-28T02:32:58.683096+0000","last_unstale":"2023-08-28T02:36:30.021744+0000","last_undegraded":"2023-08-28T02:36:30.021744+0000","last_fullsized":"2023-08-28T02:36:30.021744+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:10:08.037078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":295,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.793333+0000","last_change":"2023-08-28T02:35:06.546129+0000","last_active":"2023-08-28T02:36:14.793333+0000","last_peered":"2023-08-28T02:36:14.793333+0000","last_clean":"2023-08-28T02:36:14.793333+0000","last_became_active":"2023-08-28T02:35:06.545789+0000","last_became_peered":"2023-08-28T02:35:06.545789+0000","last_unstale":"2023-08-28T02:36:14.793333+0000","last_undegraded":"2023-08-28T02:36:14.793333+0000","last_fullsized":"2023-08-28T02:36:14.793333+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.5","version":"336'5079","reported_seq":5105,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:32.221860+0000","last_change":"2023-08-28T02:35:14.500308+0000","last_active":"2023-08-28T02:36:32.221860+0000","last_peered":"2023-08-28T02:36:32.221860+0000","last_clean":"2023-08-28T02:36:32.221860+0000","last_became_active":"2023-08-28T02:35:14.499954+0000","last_became_peered":"2023-08-28T02:35:14.499954+0000","last_unstale":"2023-08-28T02:36:32.221860+0000","last_undegraded":"2023-08-28T02:36:32.221860+0000","last_fullsized":"2023-08-28T02:36:32.221860+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":5079,"log_dups_size":0,"ondisk_log_size":5079,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332857344,"num_objects":5079,"num_object_clones":0,"num_object_copies":15237,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5079,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5079,"num_write_kb":325056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":131,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.807152+0000","last_change":"2023-08-28T02:32:58.856906+0000","last_active":"2023-08-28T02:36:14.807152+0000","last_peered":"2023-08-28T02:36:14.807152+0000","last_clean":"2023-08-28T02:36:14.807152+0000","last_became_active":"2023-08-28T02:32:58.740028+0000","last_became_peered":"2023-08-28T02:32:58.740028+0000","last_unstale":"2023-08-28T02:36:14.807152+0000","last_undegraded":"2023-08-28T02:36:14.807152+0000","last_fullsized":"2023-08-28T02:36:14.807152+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":389,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.806175+0000","last_change":"2023-08-28T02:32:58.740396+0000","last_active":"2023-08-28T02:36:14.806175+0000","last_peered":"2023-08-28T02:36:14.806175+0000","last_clean":"2023-08-28T02:36:14.806175+0000","last_became_active":"2023-08-28T02:32:58.739941+0000","last_became_peered":"2023-08-28T02:32:58.739941+0000","last_unstale":"2023-08-28T02:36:14.806175+0000","last_undegraded":"2023-08-28T02:36:14.806175+0000","last_fullsized":"2023-08-28T02:36:14.806175+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"337'5265","reported_seq":5292,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-08-28T02:36:34.615542+0000","last_change":"2023-08-28T02:35:14.495274+0000","last_active":"2023-08-28T02:36:34.615542+0000","last_peered":"2023-08-28T02:36:34.615542+0000","last_clean":"2023-08-28T02:36:34.615542+0000","last_became_active":"2023-08-28T02:35:14.494642+0000","last_became_peered":"2023-08-28T02:35:14.494642+0000","last_unstale":"2023-08-28T02:36:34.615542+0000","last_undegraded":"2023-08-28T02:36:34.615542+0000","last_fullsized":"2023-08-28T02:36:34.615542+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":5265,"log_dups_size":0,"ondisk_log_size":5265,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:28:37.992837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345047040,"num_objects":5265,"num_object_clones":0,"num_object_copies":15795,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5265,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5265,"num_write_kb":336960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":134,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.793767+0000","last_change":"2023-08-28T02:35:05.994859+0000","last_active":"2023-08-28T02:36:14.793767+0000","last_peered":"2023-08-28T02:36:14.793767+0000","last_clean":"2023-08-28T02:36:14.793767+0000","last_became_active":"2023-08-28T02:35:05.994374+0000","last_became_peered":"2023-08-28T02:35:05.994374+0000","last_unstale":"2023-08-28T02:36:14.793767+0000","last_undegraded":"2023-08-28T02:36:14.793767+0000","last_fullsized":"2023-08-28T02:36:14.793767+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":160,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.793477+0000","last_change":"2023-08-28T02:35:45.112365+0000","last_active":"2023-08-28T02:36:14.793477+0000","last_peered":"2023-08-28T02:36:14.793477+0000","last_clean":"2023-08-28T02:36:14.793477+0000","last_became_active":"2023-08-28T02:35:45.112008+0000","last_became_peered":"2023-08-28T02:35:45.112008+0000","last_unstale":"2023-08-28T02:36:14.793477+0000","last_undegraded":"2023-08-28T02:36:14.793477+0000","last_fullsized":"2023-08-28T02:36:14.793477+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"337'5201","reported_seq":5227,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-08-28T02:36:33.303886+0000","last_change":"2023-08-28T02:35:14.498797+0000","last_active":"2023-08-28T02:36:33.303886+0000","last_peered":"2023-08-28T02:36:33.303886+0000","last_clean":"2023-08-28T02:36:33.303886+0000","last_became_active":"2023-08-28T02:35:14.498551+0000","last_became_peered":"2023-08-28T02:35:14.498551+0000","last_unstale":"2023-08-28T02:36:33.303886+0000","last_undegraded":"2023-08-28T02:36:33.303886+0000","last_fullsized":"2023-08-28T02:36:33.303886+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":5201,"log_dups_size":0,"ondisk_log_size":5201,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:43:07.729642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340852736,"num_objects":5201,"num_object_clones":0,"num_object_copies":15603,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5201,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5201,"num_write_kb":332864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":341,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.669694+0000","last_change":"2023-08-28T02:28:25.445982+0000","last_active":"2023-08-28T02:36:21.669694+0000","last_peered":"2023-08-28T02:36:21.669694+0000","last_clean":"2023-08-28T02:36:21.669694+0000","last_became_active":"2023-08-28T02:28:25.443577+0000","last_became_peered":"2023-08-28T02:28:25.443577+0000","last_unstale":"2023-08-28T02:36:21.669694+0000","last_undegraded":"2023-08-28T02:36:21.669694+0000","last_fullsized":"2023-08-28T02:36:21.669694+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:24:03.381751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":337,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.674713+0000","last_change":"2023-08-28T02:32:58.887874+0000","last_active":"2023-08-28T02:36:21.674713+0000","last_peered":"2023-08-28T02:36:21.674713+0000","last_clean":"2023-08-28T02:36:21.674713+0000","last_became_active":"2023-08-28T02:32:58.887377+0000","last_became_peered":"2023-08-28T02:32:58.887377+0000","last_unstale":"2023-08-28T02:36:21.674713+0000","last_undegraded":"2023-08-28T02:36:21.674713+0000","last_fullsized":"2023-08-28T02:36:21.674713+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:08:07.234984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"337'5120","reported_seq":5148,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-08-28T02:36:33.813049+0000","last_change":"2023-08-28T02:35:14.503271+0000","last_active":"2023-08-28T02:36:33.813049+0000","last_peered":"2023-08-28T02:36:33.813049+0000","last_clean":"2023-08-28T02:36:33.813049+0000","last_became_active":"2023-08-28T02:35:14.502940+0000","last_became_peered":"2023-08-28T02:35:14.502940+0000","last_unstale":"2023-08-28T02:36:33.813049+0000","last_undegraded":"2023-08-28T02:36:33.813049+0000","last_fullsized":"2023-08-28T02:36:33.813049+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":5120,"log_dups_size":0,"ondisk_log_size":5120,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:07:49.114269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335544320,"num_objects":5120,"num_object_clones":0,"num_object_copies":15360,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5120,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5120,"num_write_kb":327680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":203,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.669723+0000","last_change":"2023-08-28T02:23:59.230059+0000","last_active":"2023-08-28T02:36:21.669723+0000","last_peered":"2023-08-28T02:36:21.669723+0000","last_clean":"2023-08-28T02:36:21.669723+0000","last_became_active":"2023-08-28T02:23:59.217783+0000","last_became_peered":"2023-08-28T02:23:59.217783+0000","last_unstale":"2023-08-28T02:36:21.669723+0000","last_undegraded":"2023-08-28T02:36:21.669723+0000","last_fullsized":"2023-08-28T02:36:21.669723+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:00:12.145395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":356,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.806997+0000","last_change":"2023-08-28T02:28:25.351698+0000","last_active":"2023-08-28T02:36:14.806997+0000","last_peered":"2023-08-28T02:36:14.806997+0000","last_clean":"2023-08-28T02:36:14.806997+0000","last_became_active":"2023-08-28T02:28:25.351349+0000","last_became_peered":"2023-08-28T02:28:25.351349+0000","last_unstale":"2023-08-28T02:36:14.806997+0000","last_undegraded":"2023-08-28T02:36:14.806997+0000","last_fullsized":"2023-08-28T02:36:14.806997+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"337'5121","reported_seq":5149,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-08-28T02:36:33.293152+0000","last_change":"2023-08-28T02:35:14.494905+0000","last_active":"2023-08-28T02:36:33.293152+0000","last_peered":"2023-08-28T02:36:33.293152+0000","last_clean":"2023-08-28T02:36:33.293152+0000","last_became_active":"2023-08-28T02:35:14.494589+0000","last_became_peered":"2023-08-28T02:35:14.494589+0000","last_unstale":"2023-08-28T02:36:33.293152+0000","last_undegraded":"2023-08-28T02:36:33.293152+0000","last_fullsized":"2023-08-28T02:36:33.293152+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":5121,"log_dups_size":0,"ondisk_log_size":5121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:26:28.302635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335609856,"num_objects":5121,"num_object_clones":0,"num_object_copies":15363,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5121,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5121,"num_write_kb":327744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":142,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.793420+0000","last_change":"2023-08-28T02:32:58.629249+0000","last_active":"2023-08-28T02:36:14.793420+0000","last_peered":"2023-08-28T02:36:14.793420+0000","last_clean":"2023-08-28T02:36:14.793420+0000","last_became_active":"2023-08-28T02:32:58.628583+0000","last_became_peered":"2023-08-28T02:32:58.628583+0000","last_unstale":"2023-08-28T02:36:14.793420+0000","last_undegraded":"2023-08-28T02:36:14.793420+0000","last_fullsized":"2023-08-28T02:36:14.793420+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":409,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.807494+0000","last_change":"2023-08-28T02:35:45.100048+0000","last_active":"2023-08-28T02:36:14.807494+0000","last_peered":"2023-08-28T02:36:14.807494+0000","last_clean":"2023-08-28T02:36:14.807494+0000","last_became_active":"2023-08-28T02:35:45.099690+0000","last_became_peered":"2023-08-28T02:35:45.099690+0000","last_unstale":"2023-08-28T02:36:14.807494+0000","last_undegraded":"2023-08-28T02:36:14.807494+0000","last_fullsized":"2023-08-28T02:36:14.807494+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"336'5095","reported_seq":5121,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:32.010238+0000","last_change":"2023-08-28T02:35:14.498910+0000","last_active":"2023-08-28T02:36:32.010238+0000","last_peered":"2023-08-28T02:36:32.010238+0000","last_clean":"2023-08-28T02:36:32.010238+0000","last_became_active":"2023-08-28T02:35:14.498631+0000","last_became_peered":"2023-08-28T02:35:14.498631+0000","last_unstale":"2023-08-28T02:36:32.010238+0000","last_undegraded":"2023-08-28T02:36:32.010238+0000","last_fullsized":"2023-08-28T02:36:32.010238+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":5095,"log_dups_size":0,"ondisk_log_size":5095,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333905920,"num_objects":5095,"num_object_clones":0,"num_object_copies":15285,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5095,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5095,"num_write_kb":326080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":272,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.806314+0000","last_change":"2023-08-28T02:32:58.740311+0000","last_active":"2023-08-28T02:36:14.806314+0000","last_peered":"2023-08-28T02:36:14.806314+0000","last_clean":"2023-08-28T02:36:14.806314+0000","last_became_active":"2023-08-28T02:32:58.739897+0000","last_became_peered":"2023-08-28T02:32:58.739897+0000","last_unstale":"2023-08-28T02:36:14.806314+0000","last_undegraded":"2023-08-28T02:36:14.806314+0000","last_fullsized":"2023-08-28T02:36:14.806314+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":324,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.675567+0000","last_change":"2023-08-28T02:32:58.766795+0000","last_active":"2023-08-28T02:36:21.675567+0000","last_peered":"2023-08-28T02:36:21.675567+0000","last_clean":"2023-08-28T02:36:21.675567+0000","last_became_active":"2023-08-28T02:32:58.766441+0000","last_became_peered":"2023-08-28T02:32:58.766441+0000","last_unstale":"2023-08-28T02:36:21.675567+0000","last_undegraded":"2023-08-28T02:36:21.675567+0000","last_fullsized":"2023-08-28T02:36:21.675567+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:58:13.963961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.3","version":"337'5154","reported_seq":5182,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-08-28T02:36:33.098765+0000","last_change":"2023-08-28T02:35:14.498188+0000","last_active":"2023-08-28T02:36:33.098765+0000","last_peered":"2023-08-28T02:36:33.098765+0000","last_clean":"2023-08-28T02:36:33.098765+0000","last_became_active":"2023-08-28T02:35:14.496278+0000","last_became_peered":"2023-08-28T02:35:14.496278+0000","last_unstale":"2023-08-28T02:36:33.098765+0000","last_undegraded":"2023-08-28T02:36:33.098765+0000","last_fullsized":"2023-08-28T02:36:33.098765+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":5154,"log_dups_size":0,"ondisk_log_size":5154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:43:23.633921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":337772544,"num_objects":5154,"num_object_clones":0,"num_object_copies":15462,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5154,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5154,"num_write_kb":329856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":244,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.670013+0000","last_change":"2023-08-28T02:32:58.779257+0000","last_active":"2023-08-28T02:36:21.670013+0000","last_peered":"2023-08-28T02:36:21.670013+0000","last_clean":"2023-08-28T02:36:21.670013+0000","last_became_active":"2023-08-28T02:32:58.778295+0000","last_became_peered":"2023-08-28T02:32:58.778295+0000","last_unstale":"2023-08-28T02:36:21.670013+0000","last_undegraded":"2023-08-28T02:36:21.670013+0000","last_fullsized":"2023-08-28T02:36:21.670013+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:50:49.444715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":57,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.642852+0000","last_change":"2023-08-28T02:32:58.755765+0000","last_active":"2023-08-28T02:36:21.642852+0000","last_peered":"2023-08-28T02:36:21.642852+0000","last_clean":"2023-08-28T02:36:21.642852+0000","last_became_active":"2023-08-28T02:32:58.755438+0000","last_became_peered":"2023-08-28T02:32:58.755438+0000","last_unstale":"2023-08-28T02:36:21.642852+0000","last_undegraded":"2023-08-28T02:36:21.642852+0000","last_fullsized":"2023-08-28T02:36:21.642852+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:14:11.498611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"337'5158","reported_seq":5186,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-08-28T02:36:34.586758+0000","last_change":"2023-08-28T02:35:14.497511+0000","last_active":"2023-08-28T02:36:34.586758+0000","last_peered":"2023-08-28T02:36:34.586758+0000","last_clean":"2023-08-28T02:36:34.586758+0000","last_became_active":"2023-08-28T02:35:14.497180+0000","last_became_peered":"2023-08-28T02:35:14.497180+0000","last_unstale":"2023-08-28T02:36:34.586758+0000","last_undegraded":"2023-08-28T02:36:34.586758+0000","last_fullsized":"2023-08-28T02:36:34.586758+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":5158,"log_dups_size":0,"ondisk_log_size":5158,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:59:26.376998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":338034688,"num_objects":5158,"num_object_clones":0,"num_object_copies":15474,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5158,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5158,"num_write_kb":330112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"78'126","reported_seq":741,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.642577+0000","last_change":"2023-08-28T02:35:09.735906+0000","last_active":"2023-08-28T02:36:21.642577+0000","last_peered":"2023-08-28T02:36:21.642577+0000","last_clean":"2023-08-28T02:36:21.642577+0000","last_became_active":"2023-08-28T02:32:59.660388+0000","last_became_peered":"2023-08-28T02:32:59.660388+0000","last_unstale":"2023-08-28T02:36:21.642577+0000","last_undegraded":"2023-08-28T02:36:21.642577+0000","last_fullsized":"2023-08-28T02:36:21.642577+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:19:53.565088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":743,"num_read_kb":1331,"num_write":125,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":178,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.667484+0000","last_change":"2023-08-28T02:35:05.991133+0000","last_active":"2023-08-28T02:36:21.667484+0000","last_peered":"2023-08-28T02:36:21.667484+0000","last_clean":"2023-08-28T02:36:21.667484+0000","last_became_active":"2023-08-28T02:35:05.990829+0000","last_became_peered":"2023-08-28T02:35:05.990829+0000","last_unstale":"2023-08-28T02:36:21.667484+0000","last_undegraded":"2023-08-28T02:36:21.667484+0000","last_fullsized":"2023-08-28T02:36:21.667484+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:40:28.077023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":19,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.641646+0000","last_change":"2023-08-28T02:35:45.099619+0000","last_active":"2023-08-28T02:36:21.641646+0000","last_peered":"2023-08-28T02:36:21.641646+0000","last_clean":"2023-08-28T02:36:21.641646+0000","last_became_active":"2023-08-28T02:35:45.099285+0000","last_became_peered":"2023-08-28T02:35:45.099285+0000","last_unstale":"2023-08-28T02:36:21.641646+0000","last_undegraded":"2023-08-28T02:36:21.641646+0000","last_fullsized":"2023-08-28T02:36:21.641646+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:24:58.305957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.8","version":"337'5234","reported_seq":5262,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-08-28T02:36:33.825936+0000","last_change":"2023-08-28T02:35:14.497638+0000","last_active":"2023-08-28T02:36:33.825936+0000","last_peered":"2023-08-28T02:36:33.825936+0000","last_clean":"2023-08-28T02:36:33.825936+0000","last_became_active":"2023-08-28T02:35:14.497323+0000","last_became_peered":"2023-08-28T02:35:14.497323+0000","last_unstale":"2023-08-28T02:36:33.825936+0000","last_undegraded":"2023-08-28T02:36:33.825936+0000","last_fullsized":"2023-08-28T02:36:33.825936+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":5234,"log_dups_size":0,"ondisk_log_size":5234,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:30:06.078735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343015424,"num_objects":5234,"num_object_clones":0,"num_object_copies":15702,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5234,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5234,"num_write_kb":334976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"78'204","reported_seq":1421,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.642760+0000","last_change":"2023-08-28T02:32:59.644574+0000","last_active":"2023-08-28T02:36:21.642760+0000","last_peered":"2023-08-28T02:36:21.642760+0000","last_clean":"2023-08-28T02:36:21.642760+0000","last_became_active":"2023-08-28T02:32:59.644053+0000","last_became_peered":"2023-08-28T02:32:59.644053+0000","last_unstale":"2023-08-28T02:36:21.642760+0000","last_undegraded":"2023-08-28T02:36:21.642760+0000","last_fullsized":"2023-08-28T02:36:21.642760+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":204,"log_dups_size":0,"ondisk_log_size":204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:55:51.460483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1521,"num_read_kb":2511,"num_write":178,"num_write_kb":1175,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":330,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.806660+0000","last_change":"2023-08-28T02:28:25.352417+0000","last_active":"2023-08-28T02:36:14.806660+0000","last_peered":"2023-08-28T02:36:14.806660+0000","last_clean":"2023-08-28T02:36:14.806660+0000","last_became_active":"2023-08-28T02:28:25.352216+0000","last_became_peered":"2023-08-28T02:28:25.352216+0000","last_unstale":"2023-08-28T02:36:14.806660+0000","last_undegraded":"2023-08-28T02:36:14.806660+0000","last_fullsized":"2023-08-28T02:36:14.806660+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":57,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.642480+0000","last_change":"2023-08-28T02:32:58.720576+0000","last_active":"2023-08-28T02:36:21.642480+0000","last_peered":"2023-08-28T02:36:21.642480+0000","last_clean":"2023-08-28T02:36:21.642480+0000","last_became_active":"2023-08-28T02:32:58.719924+0000","last_became_peered":"2023-08-28T02:32:58.719924+0000","last_unstale":"2023-08-28T02:36:21.642480+0000","last_undegraded":"2023-08-28T02:36:21.642480+0000","last_fullsized":"2023-08-28T02:36:21.642480+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:28:50.697438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"337'5150","reported_seq":5176,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-08-28T02:36:33.839221+0000","last_change":"2023-08-28T02:35:14.503279+0000","last_active":"2023-08-28T02:36:33.839221+0000","last_peered":"2023-08-28T02:36:33.839221+0000","last_clean":"2023-08-28T02:36:33.839221+0000","last_became_active":"2023-08-28T02:35:14.502953+0000","last_became_peered":"2023-08-28T02:35:14.502953+0000","last_unstale":"2023-08-28T02:36:33.839221+0000","last_undegraded":"2023-08-28T02:36:33.839221+0000","last_fullsized":"2023-08-28T02:36:33.839221+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":5150,"log_dups_size":0,"ondisk_log_size":5150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:55:29.284519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":337510400,"num_objects":5150,"num_object_clones":0,"num_object_copies":15450,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5150,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5150,"num_write_kb":329600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"78'47","reported_seq":556,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.641384+0000","last_change":"2023-08-28T02:24:05.799647+0000","last_active":"2023-08-28T02:36:21.641384+0000","last_peered":"2023-08-28T02:36:21.641384+0000","last_clean":"2023-08-28T02:36:21.641384+0000","last_became_active":"2023-08-28T02:23:59.210481+0000","last_became_peered":"2023-08-28T02:23:59.210481+0000","last_unstale":"2023-08-28T02:36:21.641384+0000","last_undegraded":"2023-08-28T02:36:21.641384+0000","last_fullsized":"2023-08-28T02:36:21.641384+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":47,"log_dups_size":0,"ondisk_log_size":47,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:40:18.542375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":162,"num_read_kb":1281,"num_write":48,"num_write_kb":1153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":57,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.641680+0000","last_change":"2023-08-28T02:32:58.622079+0000","last_active":"2023-08-28T02:36:21.641680+0000","last_peered":"2023-08-28T02:36:21.641680+0000","last_clean":"2023-08-28T02:36:21.641680+0000","last_became_active":"2023-08-28T02:32:58.621819+0000","last_became_peered":"2023-08-28T02:32:58.621819+0000","last_unstale":"2023-08-28T02:36:21.641680+0000","last_undegraded":"2023-08-28T02:36:21.641680+0000","last_fullsized":"2023-08-28T02:36:21.641680+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:03:36.676500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":57,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.642923+0000","last_change":"2023-08-28T02:32:58.724424+0000","last_active":"2023-08-28T02:36:21.642923+0000","last_peered":"2023-08-28T02:36:21.642923+0000","last_clean":"2023-08-28T02:36:21.642923+0000","last_became_active":"2023-08-28T02:32:58.724127+0000","last_became_peered":"2023-08-28T02:32:58.724127+0000","last_unstale":"2023-08-28T02:36:21.642923+0000","last_undegraded":"2023-08-28T02:36:21.642923+0000","last_fullsized":"2023-08-28T02:36:21.642923+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:02:25.178712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"336'5115","reported_seq":5141,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:32.224267+0000","last_change":"2023-08-28T02:35:14.500884+0000","last_active":"2023-08-28T02:36:32.224267+0000","last_peered":"2023-08-28T02:36:32.224267+0000","last_clean":"2023-08-28T02:36:32.224267+0000","last_became_active":"2023-08-28T02:35:14.500643+0000","last_became_peered":"2023-08-28T02:35:14.500643+0000","last_unstale":"2023-08-28T02:36:32.224267+0000","last_undegraded":"2023-08-28T02:36:32.224267+0000","last_fullsized":"2023-08-28T02:36:32.224267+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":5115,"log_dups_size":0,"ondisk_log_size":5115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335216640,"num_objects":5115,"num_object_clones":0,"num_object_copies":15345,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5115,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5115,"num_write_kb":327360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"78'114","reported_seq":390,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.641502+0000","last_change":"2023-08-28T02:35:05.992001+0000","last_active":"2023-08-28T02:36:21.641502+0000","last_peered":"2023-08-28T02:36:21.641502+0000","last_clean":"2023-08-28T02:36:21.641502+0000","last_became_active":"2023-08-28T02:35:05.991513+0000","last_became_peered":"2023-08-28T02:35:05.991513+0000","last_unstale":"2023-08-28T02:36:21.641502+0000","last_undegraded":"2023-08-28T02:36:21.641502+0000","last_fullsized":"2023-08-28T02:36:21.641502+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":114,"log_dups_size":0,"ondisk_log_size":114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:00:42.194692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":358,"num_read_kb":1352,"num_write":102,"num_write_kb":1104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"86'139","reported_seq":422,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.677134+0000","last_change":"2023-08-28T02:35:09.566656+0000","last_active":"2023-08-28T02:36:21.677134+0000","last_peered":"2023-08-28T02:36:21.677134+0000","last_clean":"2023-08-28T02:36:21.677134+0000","last_became_active":"2023-08-28T02:32:59.651089+0000","last_became_peered":"2023-08-28T02:32:59.651089+0000","last_unstale":"2023-08-28T02:36:21.677134+0000","last_undegraded":"2023-08-28T02:36:21.677134+0000","last_fullsized":"2023-08-28T02:36:21.677134+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_clean_scrub_stamp":"2023-08-28T02:08:02.043718+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:31:19.261767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":218,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:28.843383+0000","last_change":"2023-08-28T02:28:25.347600+0000","last_active":"2023-08-28T02:36:28.843383+0000","last_peered":"2023-08-28T02:36:28.843383+0000","last_clean":"2023-08-28T02:36:28.843383+0000","last_became_active":"2023-08-28T02:28:25.347237+0000","last_became_peered":"2023-08-28T02:28:25.347237+0000","last_unstale":"2023-08-28T02:36:28.843383+0000","last_undegraded":"2023-08-28T02:36:28.843383+0000","last_fullsized":"2023-08-28T02:36:28.843383+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:29:26.767559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":133,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.793870+0000","last_change":"2023-08-28T02:31:55.517732+0000","last_active":"2023-08-28T02:36:14.793870+0000","last_peered":"2023-08-28T02:36:14.793870+0000","last_clean":"2023-08-28T02:36:14.793870+0000","last_became_active":"2023-08-28T02:31:55.517347+0000","last_became_peered":"2023-08-28T02:31:55.517347+0000","last_unstale":"2023-08-28T02:36:14.793870+0000","last_undegraded":"2023-08-28T02:36:14.793870+0000","last_fullsized":"2023-08-28T02:36:14.793870+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"337'5044","reported_seq":5072,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-08-28T02:36:33.303366+0000","last_change":"2023-08-28T02:35:14.493898+0000","last_active":"2023-08-28T02:36:33.303366+0000","last_peered":"2023-08-28T02:36:33.303366+0000","last_clean":"2023-08-28T02:36:33.303366+0000","last_became_active":"2023-08-28T02:35:14.493649+0000","last_became_peered":"2023-08-28T02:35:14.493649+0000","last_unstale":"2023-08-28T02:36:33.303366+0000","last_undegraded":"2023-08-28T02:36:33.303366+0000","last_fullsized":"2023-08-28T02:36:33.303366+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":5044,"log_dups_size":0,"ondisk_log_size":5044,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:49:46.405255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330563584,"num_objects":5044,"num_object_clones":0,"num_object_copies":15132,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5044,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5044,"num_write_kb":322816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'148","reported_seq":792,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.641808+0000","last_change":"2023-08-28T02:35:10.122879+0000","last_active":"2023-08-28T02:36:21.641808+0000","last_peered":"2023-08-28T02:36:21.641808+0000","last_clean":"2023-08-28T02:36:21.641808+0000","last_became_active":"2023-08-28T02:32:59.661273+0000","last_became_peered":"2023-08-28T02:32:59.661273+0000","last_unstale":"2023-08-28T02:36:21.641808+0000","last_undegraded":"2023-08-28T02:36:21.641808+0000","last_fullsized":"2023-08-28T02:36:21.641808+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:07:46.974854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1092,"num_read_kb":1955,"num_write":127,"num_write_kb":5177,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":308,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.675889+0000","last_change":"2023-08-28T02:23:59.226986+0000","last_active":"2023-08-28T02:36:21.675889+0000","last_peered":"2023-08-28T02:36:21.675889+0000","last_clean":"2023-08-28T02:36:21.675889+0000","last_became_active":"2023-08-28T02:23:59.226529+0000","last_became_peered":"2023-08-28T02:23:59.226529+0000","last_unstale":"2023-08-28T02:36:21.675889+0000","last_undegraded":"2023-08-28T02:36:21.675889+0000","last_fullsized":"2023-08-28T02:36:21.675889+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:22:09.193619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":371,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.675056+0000","last_change":"2023-08-28T02:32:58.744332+0000","last_active":"2023-08-28T02:36:21.675056+0000","last_peered":"2023-08-28T02:36:21.675056+0000","last_clean":"2023-08-28T02:36:21.675056+0000","last_became_active":"2023-08-28T02:32:58.744025+0000","last_became_peered":"2023-08-28T02:32:58.744025+0000","last_unstale":"2023-08-28T02:36:21.675056+0000","last_undegraded":"2023-08-28T02:36:21.675056+0000","last_fullsized":"2023-08-28T02:36:21.675056+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:41:15.957049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"337'5208","reported_seq":5393,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-08-28T02:36:33.837160+0000","last_change":"2023-08-28T02:35:24.994481+0000","last_active":"2023-08-28T02:36:33.837160+0000","last_peered":"2023-08-28T02:36:33.837160+0000","last_clean":"2023-08-28T02:36:33.837160+0000","last_became_active":"2023-08-28T02:35:24.668942+0000","last_became_peered":"2023-08-28T02:35:24.668942+0000","last_unstale":"2023-08-28T02:36:33.837160+0000","last_undegraded":"2023-08-28T02:36:33.837160+0000","last_fullsized":"2023-08-28T02:36:33.837160+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":5208,"log_dups_size":0,"ondisk_log_size":5208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:16:02.211038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":341311488,"num_objects":5208,"num_object_clones":0,"num_object_copies":15624,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5208,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5208,"num_write_kb":333312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":73,"num_bytes_recovered":4784128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"78'103","reported_seq":668,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.675203+0000","last_change":"2023-08-28T02:35:09.798248+0000","last_active":"2023-08-28T02:36:21.675203+0000","last_peered":"2023-08-28T02:36:21.675203+0000","last_clean":"2023-08-28T02:36:21.675203+0000","last_became_active":"2023-08-28T02:32:59.656276+0000","last_became_peered":"2023-08-28T02:32:59.656276+0000","last_unstale":"2023-08-28T02:36:21.675203+0000","last_undegraded":"2023-08-28T02:36:21.675203+0000","last_fullsized":"2023-08-28T02:36:21.675203+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:34:10.622592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":803,"num_read_kb":2080,"num_write":73,"num_write_kb":1244,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":57,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.642386+0000","last_change":"2023-08-28T02:32:58.720401+0000","last_active":"2023-08-28T02:36:21.642386+0000","last_peered":"2023-08-28T02:36:21.642386+0000","last_clean":"2023-08-28T02:36:21.642386+0000","last_became_active":"2023-08-28T02:32:58.719518+0000","last_became_peered":"2023-08-28T02:32:58.719518+0000","last_unstale":"2023-08-28T02:36:21.642386+0000","last_undegraded":"2023-08-28T02:36:21.642386+0000","last_fullsized":"2023-08-28T02:36:21.642386+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:46:28.947812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":321,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.674969+0000","last_change":"2023-08-28T02:32:58.887691+0000","last_active":"2023-08-28T02:36:21.674969+0000","last_peered":"2023-08-28T02:36:21.674969+0000","last_clean":"2023-08-28T02:36:21.674969+0000","last_became_active":"2023-08-28T02:32:58.886906+0000","last_became_peered":"2023-08-28T02:32:58.886906+0000","last_unstale":"2023-08-28T02:36:21.674969+0000","last_undegraded":"2023-08-28T02:36:21.674969+0000","last_fullsized":"2023-08-28T02:36:21.674969+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:39:29.229171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"337'5080","reported_seq":5108,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-08-28T02:36:34.598289+0000","last_change":"2023-08-28T02:35:14.497565+0000","last_active":"2023-08-28T02:36:34.598289+0000","last_peered":"2023-08-28T02:36:34.598289+0000","last_clean":"2023-08-28T02:36:34.598289+0000","last_became_active":"2023-08-28T02:35:14.497285+0000","last_became_peered":"2023-08-28T02:35:14.497285+0000","last_unstale":"2023-08-28T02:36:34.598289+0000","last_undegraded":"2023-08-28T02:36:34.598289+0000","last_fullsized":"2023-08-28T02:36:34.598289+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":5080,"log_dups_size":0,"ondisk_log_size":5080,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:04:31.778861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332922880,"num_objects":5080,"num_object_clones":0,"num_object_copies":15240,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5080,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5080,"num_write_kb":325120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":216,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.667711+0000","last_change":"2023-08-28T02:31:55.535099+0000","last_active":"2023-08-28T02:36:21.667711+0000","last_peered":"2023-08-28T02:36:21.667711+0000","last_clean":"2023-08-28T02:36:21.667711+0000","last_became_active":"2023-08-28T02:31:55.527768+0000","last_became_peered":"2023-08-28T02:31:55.527768+0000","last_unstale":"2023-08-28T02:36:21.667711+0000","last_undegraded":"2023-08-28T02:36:21.667711+0000","last_fullsized":"2023-08-28T02:36:21.667711+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:43:30.804793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'112","reported_seq":490,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.642523+0000","last_change":"2023-08-28T02:32:59.643465+0000","last_active":"2023-08-28T02:36:21.642523+0000","last_peered":"2023-08-28T02:36:21.642523+0000","last_clean":"2023-08-28T02:36:21.642523+0000","last_became_active":"2023-08-28T02:32:59.642979+0000","last_became_peered":"2023-08-28T02:32:59.642979+0000","last_unstale":"2023-08-28T02:36:21.642523+0000","last_undegraded":"2023-08-28T02:36:21.642523+0000","last_fullsized":"2023-08-28T02:36:21.642523+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:48:45.923015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":444,"num_read_kb":473,"num_write":94,"num_write_kb":127,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":227,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.669796+0000","last_change":"2023-08-28T02:32:58.833408+0000","last_active":"2023-08-28T02:36:21.669796+0000","last_peered":"2023-08-28T02:36:21.669796+0000","last_clean":"2023-08-28T02:36:21.669796+0000","last_became_active":"2023-08-28T02:32:58.821347+0000","last_became_peered":"2023-08-28T02:32:58.821347+0000","last_unstale":"2023-08-28T02:36:21.669796+0000","last_undegraded":"2023-08-28T02:36:21.669796+0000","last_fullsized":"2023-08-28T02:36:21.669796+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:41:25.683179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"337'5151","reported_seq":5179,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-08-28T02:36:34.907876+0000","last_change":"2023-08-28T02:35:14.495405+0000","last_active":"2023-08-28T02:36:34.907876+0000","last_peered":"2023-08-28T02:36:34.907876+0000","last_clean":"2023-08-28T02:36:34.907876+0000","last_became_active":"2023-08-28T02:35:14.494591+0000","last_became_peered":"2023-08-28T02:35:14.494591+0000","last_unstale":"2023-08-28T02:36:34.907876+0000","last_undegraded":"2023-08-28T02:36:34.907876+0000","last_fullsized":"2023-08-28T02:36:34.907876+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":5151,"log_dups_size":0,"ondisk_log_size":5151,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:47:27.839376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":337575936,"num_objects":5151,"num_object_clones":0,"num_object_copies":15453,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5151,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5151,"num_write_kb":329664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"78'122","reported_seq":679,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.793843+0000","last_change":"2023-08-28T02:33:00.672889+0000","last_active":"2023-08-28T02:36:14.793843+0000","last_peered":"2023-08-28T02:36:14.793843+0000","last_clean":"2023-08-28T02:36:14.793843+0000","last_became_active":"2023-08-28T02:33:00.672494+0000","last_became_peered":"2023-08-28T02:33:00.672494+0000","last_unstale":"2023-08-28T02:36:14.793843+0000","last_undegraded":"2023-08-28T02:36:14.793843+0000","last_fullsized":"2023-08-28T02:36:14.793843+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":857,"num_read_kb":2224,"num_write":99,"num_write_kb":2312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":346,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.675292+0000","last_change":"2023-08-28T02:32:58.795995+0000","last_active":"2023-08-28T02:36:21.675292+0000","last_peered":"2023-08-28T02:36:21.675292+0000","last_clean":"2023-08-28T02:36:21.675292+0000","last_became_active":"2023-08-28T02:32:58.771872+0000","last_became_peered":"2023-08-28T02:32:58.771872+0000","last_unstale":"2023-08-28T02:36:21.675292+0000","last_undegraded":"2023-08-28T02:36:21.675292+0000","last_fullsized":"2023-08-28T02:36:21.675292+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:16:02.965467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":305,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.806000+0000","last_change":"2023-08-28T02:28:25.351840+0000","last_active":"2023-08-28T02:36:14.806000+0000","last_peered":"2023-08-28T02:36:14.806000+0000","last_clean":"2023-08-28T02:36:14.806000+0000","last_became_active":"2023-08-28T02:28:25.351399+0000","last_became_peered":"2023-08-28T02:28:25.351399+0000","last_unstale":"2023-08-28T02:36:14.806000+0000","last_undegraded":"2023-08-28T02:36:14.806000+0000","last_fullsized":"2023-08-28T02:36:14.806000+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":323,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.674881+0000","last_change":"2023-08-28T02:32:58.751818+0000","last_active":"2023-08-28T02:36:21.674881+0000","last_peered":"2023-08-28T02:36:21.674881+0000","last_clean":"2023-08-28T02:36:21.674881+0000","last_became_active":"2023-08-28T02:32:58.743388+0000","last_became_peered":"2023-08-28T02:32:58.743388+0000","last_unstale":"2023-08-28T02:36:21.674881+0000","last_undegraded":"2023-08-28T02:36:21.674881+0000","last_fullsized":"2023-08-28T02:36:21.674881+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:52:12.910174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":24,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.642066+0000","last_change":"2023-08-28T02:35:24.666417+0000","last_active":"2023-08-28T02:36:21.642066+0000","last_peered":"2023-08-28T02:36:21.642066+0000","last_clean":"2023-08-28T02:36:21.642066+0000","last_became_active":"2023-08-28T02:35:24.666107+0000","last_became_peered":"2023-08-28T02:35:24.666107+0000","last_unstale":"2023-08-28T02:36:21.642066+0000","last_undegraded":"2023-08-28T02:36:21.642066+0000","last_fullsized":"2023-08-28T02:36:21.642066+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:22:55.497120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":168,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.806919+0000","last_change":"2023-08-28T02:32:58.839847+0000","last_active":"2023-08-28T02:36:14.806919+0000","last_peered":"2023-08-28T02:36:14.806919+0000","last_clean":"2023-08-28T02:36:14.806919+0000","last_became_active":"2023-08-28T02:32:58.628159+0000","last_became_peered":"2023-08-28T02:32:58.628159+0000","last_unstale":"2023-08-28T02:36:14.806919+0000","last_undegraded":"2023-08-28T02:36:14.806919+0000","last_fullsized":"2023-08-28T02:36:14.806919+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":371,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.806776+0000","last_change":"2023-08-28T02:32:58.629596+0000","last_active":"2023-08-28T02:36:14.806776+0000","last_peered":"2023-08-28T02:36:14.806776+0000","last_clean":"2023-08-28T02:36:14.806776+0000","last_became_active":"2023-08-28T02:32:58.629136+0000","last_became_peered":"2023-08-28T02:32:58.629136+0000","last_unstale":"2023-08-28T02:36:14.806776+0000","last_undegraded":"2023-08-28T02:36:14.806776+0000","last_fullsized":"2023-08-28T02:36:14.806776+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":155,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.807412+0000","last_change":"2023-08-28T02:28:25.705331+0000","last_active":"2023-08-28T02:36:14.807412+0000","last_peered":"2023-08-28T02:36:14.807412+0000","last_clean":"2023-08-28T02:36:14.807412+0000","last_became_active":"2023-08-28T02:28:25.397480+0000","last_became_peered":"2023-08-28T02:28:25.397480+0000","last_unstale":"2023-08-28T02:36:14.807412+0000","last_undegraded":"2023-08-28T02:36:14.807412+0000","last_fullsized":"2023-08-28T02:36:14.807412+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":335,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.806952+0000","last_change":"2023-08-28T02:23:59.207929+0000","last_active":"2023-08-28T02:36:14.806952+0000","last_peered":"2023-08-28T02:36:14.806952+0000","last_clean":"2023-08-28T02:36:14.806952+0000","last_became_active":"2023-08-28T02:23:59.207589+0000","last_became_peered":"2023-08-28T02:23:59.207589+0000","last_unstale":"2023-08-28T02:36:14.806952+0000","last_undegraded":"2023-08-28T02:36:14.806952+0000","last_fullsized":"2023-08-28T02:36:14.806952+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":227,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.667895+0000","last_change":"2023-08-28T02:28:25.347988+0000","last_active":"2023-08-28T02:36:21.667895+0000","last_peered":"2023-08-28T02:36:21.667895+0000","last_clean":"2023-08-28T02:36:21.667895+0000","last_became_active":"2023-08-28T02:28:25.347632+0000","last_became_peered":"2023-08-28T02:28:25.347632+0000","last_unstale":"2023-08-28T02:36:21.667895+0000","last_undegraded":"2023-08-28T02:36:21.667895+0000","last_fullsized":"2023-08-28T02:36:21.667895+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:56:20.202662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":199,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.667241+0000","last_change":"2023-08-28T02:23:59.217794+0000","last_active":"2023-08-28T02:36:21.667241+0000","last_peered":"2023-08-28T02:36:21.667241+0000","last_clean":"2023-08-28T02:36:21.667241+0000","last_became_active":"2023-08-28T02:23:59.217231+0000","last_became_peered":"2023-08-28T02:23:59.217231+0000","last_unstale":"2023-08-28T02:36:21.667241+0000","last_undegraded":"2023-08-28T02:36:21.667241+0000","last_fullsized":"2023-08-28T02:36:21.667241+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:36:26.297410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":324,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.675397+0000","last_change":"2023-08-28T02:35:24.680274+0000","last_active":"2023-08-28T02:36:21.675397+0000","last_peered":"2023-08-28T02:36:21.675397+0000","last_clean":"2023-08-28T02:36:21.675397+0000","last_became_active":"2023-08-28T02:35:24.673318+0000","last_became_peered":"2023-08-28T02:35:24.673318+0000","last_unstale":"2023-08-28T02:36:21.675397+0000","last_undegraded":"2023-08-28T02:36:21.675397+0000","last_fullsized":"2023-08-28T02:36:21.675397+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:22:46.371303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":394,"reported_epoch":335,"state":"active+clean","last_fresh":"2023-08-28T02:36:14.805822+0000","last_change":"2023-08-28T02:32:58.630270+0000","last_active":"2023-08-28T02:36:14.805822+0000","last_peered":"2023-08-28T02:36:14.805822+0000","last_clean":"2023-08-28T02:36:14.805822+0000","last_became_active":"2023-08-28T02:32:58.629958+0000","last_became_peered":"2023-08-28T02:32:58.629958+0000","last_unstale":"2023-08-28T02:36:14.805822+0000","last_undegraded":"2023-08-28T02:36:14.805822+0000","last_fullsized":"2023-08-28T02:36:14.805822+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":62,"num_bytes_recovered":25254,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":21462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26810,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":5388173312,"num_objects":82217,"num_object_clones":0,"num_object_copies":246651,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":82217,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":82217,"num_write_kb":5261888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":73,"num_bytes_recovered":4784128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16134963200,"data_stored":16134963200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82217,"ondisk_log_size":82217,"up":48,"acting":48,"num_store_stats":7},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":156,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16789504,"data_stored":16777273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":1335734829077,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626758,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2260492,"kb_used_data":1955076,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":91484660,"statfs":{"total":95995035648,"available":93680291840,"internally_reserved":0,"allocated":2001997824,"data_stored":2001117242,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:36:20 2023","interfaces":[{"interface":"back","average":{"1min":1.0980000000000001,"5min":3.0529999999999999,"15min":3.6560000000000001},"min":{"1min":0.23000000000000001,"5min":0.215,"15min":0.20799999999999999},"max":{"1min":3.5760000000000001,"5min":121.78,"15min":121.78},"last":2.1989999999999998},{"interface":"front","average":{"1min":1.254,"5min":3.6309999999999998,"15min":3.7549999999999999},"min":{"1min":0.29999999999999999,"5min":0.23699999999999999,"15min":0.23200000000000001},"max":{"1min":7.4370000000000003,"5min":121.79900000000001,"15min":121.79900000000001},"last":3.5720000000000001}]},{"osd":1,"last update":"Mon Aug 28 02:36:20 2023","interfaces":[{"interface":"back","average":{"1min":1.373,"5min":1.2589999999999999,"15min":2.6880000000000002},"min":{"1min":0.315,"5min":0.23799999999999999,"15min":0.23699999999999999},"max":{"1min":4.1959999999999997,"5min":8.2639999999999993,"15min":141.87299999999999},"last":3.3439999999999999},{"interface":"front","average":{"1min":1.5640000000000001,"5min":1.484,"15min":2.8450000000000002},"min":{"1min":0.25,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":4.8440000000000003,"5min":8.5180000000000007,"15min":141.904},"last":1.9810000000000001}]},{"osd":2,"last update":"Mon Aug 28 02:36:04 2023","interfaces":[{"interface":"back","average":{"1min":5.5019999999999998,"5min":5.6580000000000004,"15min":4.173},"min":{"1min":0.34899999999999998,"5min":0.29499999999999998,"15min":0.27500000000000002},"max":{"1min":91.641000000000005,"5min":234.876,"15min":234.876},"last":1.5640000000000001},{"interface":"front","average":{"1min":5.4139999999999997,"5min":5.4320000000000004,"15min":4.1130000000000004},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":91.655000000000001,"5min":234.482,"15min":234.482},"last":1.407}]},{"osd":3,"last update":"Mon Aug 28 02:36:20 2023","interfaces":[{"interface":"back","average":{"1min":1.8480000000000001,"5min":3.3069999999999999,"15min":3.2069999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":7.2750000000000004,"5min":143.19499999999999,"15min":157.625},"last":2.0880000000000001},{"interface":"front","average":{"1min":2.1070000000000002,"5min":3.3079999999999998,"15min":3.2149999999999999},"min":{"1min":0.38200000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":7.2869999999999999,"5min":143.20699999999999,"15min":157.637},"last":2.0950000000000002}]},{"osd":4,"last update":"Mon Aug 28 02:36:25 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.50600000000000001,"15min":1.103},"min":{"1min":0.28999999999999998,"5min":0.251,"15min":0.251},"max":{"1min":1.645,"5min":3.8610000000000002,"15min":18.308},"last":0.442},{"interface":"front","average":{"1min":0.73199999999999998,"5min":0.68799999999999994,"15min":1.2150000000000001},"min":{"1min":0.28100000000000003,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":2.1019999999999999,"5min":3.8450000000000002,"15min":18.263000000000002},"last":0.35599999999999998}]},{"osd":5,"last update":"Mon Aug 28 02:36:25 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.61099999999999999,"15min":1.1870000000000001},"min":{"1min":0.29699999999999999,"5min":0.29299999999999998,"15min":0.26300000000000001},"max":{"1min":2.8639999999999999,"5min":4.2800000000000002,"15min":20.759},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.52700000000000002,"15min":0.997},"min":{"1min":0.32100000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":1.399,"5min":2.972,"15min":7.3780000000000001},"last":0.38100000000000001}]},{"osd":7,"last update":"Mon Aug 28 02:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":1.3180000000000001,"5min":1.3180000000000001,"15min":1.3180000000000001},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.52400000000000002}]}]},{"osd":1,"up_from":98,"seq":420906795211,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2794756,"kb_used_data":2284284,"kb_used_omap":7,"kb_used_meta":510456,"kb_avail":90950396,"statfs":{"total":95995035648,"available":93133205504,"internally_reserved":0,"allocated":2339106816,"data_stored":2337374711,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":522707275},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:36:18 2023","interfaces":[{"interface":"back","average":{"1min":3.3809999999999998,"5min":6.6779999999999999,"15min":4.4089999999999998},"min":{"1min":0.23799999999999999,"5min":0.189,"15min":0.17499999999999999},"max":{"1min":36.838000000000001,"5min":129.346,"15min":129.346},"last":1.8440000000000001},{"interface":"front","average":{"1min":2.9279999999999999,"5min":6.6210000000000004,"15min":4.4329999999999998},"min":{"1min":0.19800000000000001,"5min":0.19700000000000001,"15min":0.188},"max":{"1min":36.622,"5min":120.774,"15min":128.685},"last":0.19900000000000001}]},{"osd":2,"last update":"Mon Aug 28 02:35:43 2023","interfaces":[{"interface":"back","average":{"1min":2.2269999999999999,"5min":3.883,"15min":4.1429999999999998},"min":{"1min":0.218,"5min":0.189,"15min":0.189},"max":{"1min":10.818,"5min":119.276,"15min":135.25999999999999},"last":0.81899999999999995},{"interface":"front","average":{"1min":1.0549999999999999,"5min":3.492,"15min":3.8100000000000001},"min":{"1min":0.221,"5min":0.218,"15min":0.218},"max":{"1min":6.8879999999999999,"5min":121.849,"15min":135.10300000000001},"last":0.51000000000000001}]},{"osd":3,"last update":"Mon Aug 28 02:36:18 2023","interfaces":[{"interface":"back","average":{"1min":1.6839999999999999,"5min":7.7859999999999996,"15min":4.4699999999999998},"min":{"1min":0.21299999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":4.29,"5min":126.95699999999999,"15min":126.95699999999999},"last":0.27500000000000002},{"interface":"front","average":{"1min":2.141,"5min":7.4969999999999999,"15min":4.5890000000000004},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":10.477,"5min":128.00200000000001,"15min":128.00200000000001},"last":0.72599999999999998}]},{"osd":4,"last update":"Mon Aug 28 02:36:18 2023","interfaces":[{"interface":"back","average":{"1min":2.7170000000000001,"5min":1.7410000000000001,"15min":1.6899999999999999},"min":{"1min":0.29699999999999999,"5min":0.28199999999999997,"15min":0.27500000000000002},"max":{"1min":24.809999999999999,"5min":24.809999999999999,"15min":24.809999999999999},"last":0.79100000000000004},{"interface":"front","average":{"1min":2.403,"5min":1.546,"15min":1.484},"min":{"1min":0.378,"5min":0.29499999999999998,"15min":0.25600000000000001},"max":{"1min":24.835999999999999,"5min":24.835999999999999,"15min":24.835999999999999},"last":0.317}]},{"osd":5,"last update":"Mon Aug 28 02:36:18 2023","interfaces":[{"interface":"back","average":{"1min":8.0570000000000004,"5min":2.9809999999999999,"15min":2.2010000000000001},"min":{"1min":0.34399999999999997,"5min":0.252,"15min":0.252},"max":{"1min":82.722999999999999,"5min":82.722999999999999,"15min":82.722999999999999},"last":0.30499999999999999},{"interface":"front","average":{"1min":8.4969999999999999,"5min":2.9540000000000002,"15min":2.081},"min":{"1min":0.32900000000000001,"5min":0.307,"15min":0.307},"max":{"1min":82.733999999999995,"5min":82.733999999999995,"15min":82.733999999999995},"last":0.76200000000000001}]},{"osd":6,"last update":"Mon Aug 28 02:36:18 2023","interfaces":[{"interface":"back","average":{"1min":3.024,"5min":2.0230000000000001,"15min":1.994},"min":{"1min":0.41999999999999998,"5min":0.311,"15min":0.311},"max":{"1min":19.527000000000001,"5min":19.527000000000001,"15min":19.527000000000001},"last":0.36899999999999999},{"interface":"front","average":{"1min":3.4969999999999999,"5min":2.1259999999999999,"15min":1.9550000000000001},"min":{"1min":0.38,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":19.516999999999999,"5min":19.516999999999999,"15min":19.516999999999999},"last":0.84799999999999998}]},{"osd":7,"last update":"Mon Aug 28 02:35:57 2023","interfaces":[{"interface":"back","average":{"1min":0.91100000000000003,"5min":0.91100000000000003,"15min":0.91100000000000003},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":2.7829999999999999,"5min":2.7829999999999999,"15min":2.7829999999999999},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":3.4239999999999999,"5min":3.4239999999999999,"15min":3.4239999999999999},"last":0.45800000000000002}]}]},{"osd":0,"up_from":90,"seq":386547056857,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3265060,"kb_used_data":2649628,"kb_used_omap":8,"kb_used_meta":615415,"kb_avail":90480092,"statfs":{"total":95995035648,"available":92651614208,"internally_reserved":0,"allocated":2713219072,"data_stored":2711639364,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":630185321},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Aug 28 02:36:27 2023","interfaces":[{"interface":"back","average":{"1min":2.4620000000000002,"5min":2.7480000000000002,"15min":2.7549999999999999},"min":{"1min":0.16500000000000001,"5min":0.16500000000000001,"15min":0.16500000000000001},"max":{"1min":15.728,"5min":61.932000000000002,"15min":61.932000000000002},"last":2.653},{"interface":"front","average":{"1min":3.3849999999999998,"5min":3.98,"15min":3.2570000000000001},"min":{"1min":0.28599999999999998,"5min":0.19500000000000001,"15min":0.17000000000000001},"max":{"1min":16.443999999999999,"5min":62.061999999999998,"15min":62.061999999999998},"last":0.48899999999999999}]},{"osd":2,"last update":"Mon Aug 28 02:35:45 2023","interfaces":[{"interface":"back","average":{"1min":2.5110000000000001,"5min":4.5430000000000001,"15min":3.5990000000000002},"min":{"1min":0.22900000000000001,"5min":0.20599999999999999,"15min":0.19500000000000001},"max":{"1min":19.416,"5min":86.269000000000005,"15min":100.718},"last":3.4159999999999999},{"interface":"front","average":{"1min":3.8809999999999998,"5min":5.5979999999999999,"15min":4.1520000000000001},"min":{"1min":0.26200000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":26.536000000000001,"5min":86.353999999999999,"15min":100.962},"last":0.68000000000000005}]},{"osd":3,"last update":"Mon Aug 28 02:36:33 2023","interfaces":[{"interface":"back","average":{"1min":2.4159999999999999,"5min":6.1790000000000003,"15min":4.9260000000000002},"min":{"1min":0.40999999999999998,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":9.1319999999999997,"5min":126.896,"15min":237.43899999999999},"last":2.8980000000000001},{"interface":"front","average":{"1min":1.833,"5min":6.0460000000000003,"15min":4.8179999999999996},"min":{"1min":0.308,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":4.9370000000000003,"5min":127.01600000000001,"15min":237.48099999999999},"last":2.6749999999999998}]},{"osd":4,"last update":"Mon Aug 28 02:36:23 2023","interfaces":[{"interface":"back","average":{"1min":1.796,"5min":1.5229999999999999,"15min":1.6539999999999999},"min":{"1min":0.42099999999999999,"5min":0.252,"15min":0.23999999999999999},"max":{"1min":14.209,"5min":14.209,"15min":14.209},"last":2.8740000000000001},{"interface":"front","average":{"1min":2.0649999999999999,"5min":2.0310000000000001,"15min":1.956},"min":{"1min":0.34000000000000002,"5min":0.31,"15min":0.309},"max":{"1min":14.241,"5min":14.241,"15min":27.271999999999998},"last":0.51800000000000002}]},{"osd":5,"last update":"Mon Aug 28 02:36:23 2023","interfaces":[{"interface":"back","average":{"1min":1.47,"5min":1.956,"15min":2.0339999999999998},"min":{"1min":0.38100000000000001,"5min":0.29899999999999999,"15min":0.29599999999999999},"max":{"1min":3.827,"5min":10.779999999999999,"15min":44.000999999999998},"last":0.54200000000000004},{"interface":"front","average":{"1min":1.2210000000000001,"5min":1.486,"15min":1.8320000000000001},"min":{"1min":0.45500000000000002,"5min":0.307,"15min":0.307},"max":{"1min":3.2170000000000001,"5min":9.0310000000000006,"15min":43.953000000000003},"last":2.9119999999999999}]},{"osd":6,"last update":"Mon Aug 28 02:36:23 2023","interfaces":[{"interface":"back","average":{"1min":1.3819999999999999,"5min":1.9970000000000001,"15min":2.113},"min":{"1min":0.36199999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":3.8610000000000002,"5min":10.823,"15min":32.417999999999999},"last":0.56100000000000005},{"interface":"front","average":{"1min":1.355,"5min":1.51,"15min":1.8799999999999999},"min":{"1min":0.52500000000000002,"5min":0.32900000000000001,"15min":0.30399999999999999},"max":{"1min":3.266,"5min":9.0009999999999994,"15min":32.130000000000003},"last":3.0129999999999999}]},{"osd":7,"last update":"Mon Aug 28 02:35:57 2023","interfaces":[{"interface":"back","average":{"1min":1.147,"5min":1.147,"15min":1.147},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":3.242,"5min":3.242,"15min":3.242},"last":2.9420000000000002},{"interface":"front","average":{"1min":1.9259999999999999,"5min":1.9259999999999999,"15min":1.9259999999999999},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":16.529,"5min":16.529,"15min":16.529},"last":0.45900000000000002}]}]},{"osd":2,"up_from":104,"seq":446676598981,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2928076,"kb_used_data":2325636,"kb_used_omap":10,"kb_used_meta":602421,"kb_avail":90817076,"statfs":{"total":95995035648,"available":92996685824,"internally_reserved":0,"allocated":2381451264,"data_stored":2379893988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10792,"internal_metadata":616879576},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:35:49 2023","interfaces":[{"interface":"back","average":{"1min":2.0600000000000001,"5min":7.1950000000000003,"15min":4.5519999999999996},"min":{"1min":0.21099999999999999,"5min":0.191,"15min":0.17599999999999999},"max":{"1min":17.791,"5min":185.93799999999999,"15min":185.93799999999999},"last":0.26300000000000001},{"interface":"front","average":{"1min":1.647,"5min":6.6210000000000004,"15min":5.5579999999999998},"min":{"1min":0.252,"5min":0.22,"15min":0.19900000000000001},"max":{"1min":6.569,"5min":187.90700000000001,"15min":187.90700000000001},"last":0.246}]},{"osd":1,"last update":"Mon Aug 28 02:35:49 2023","interfaces":[{"interface":"back","average":{"1min":4.702,"5min":2.625,"15min":2.4940000000000002},"min":{"1min":0.23100000000000001,"5min":0.221,"15min":0.20999999999999999},"max":{"1min":66.010999999999996,"5min":66.010999999999996,"15min":66.010999999999996},"last":3.1640000000000001},{"interface":"front","average":{"1min":4.0510000000000002,"5min":2.4540000000000002,"15min":2.395},"min":{"1min":0.21299999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":63.770000000000003,"5min":63.770000000000003,"15min":63.770000000000003},"last":3.1480000000000001}]},{"osd":3,"last update":"Mon Aug 28 02:36:23 2023","interfaces":[{"interface":"back","average":{"1min":4.8390000000000004,"5min":5.6470000000000002,"15min":4.3280000000000003},"min":{"1min":0.23499999999999999,"5min":0.23100000000000001,"15min":0.22700000000000001},"max":{"1min":57.338000000000001,"5min":134.07599999999999,"15min":134.07599999999999},"last":36.603999999999999},{"interface":"front","average":{"1min":4.6769999999999996,"5min":5.5549999999999997,"15min":4.2789999999999999},"min":{"1min":0.29599999999999999,"5min":0.29299999999999998,"15min":0.223},"max":{"1min":57.375999999999998,"5min":134.172,"15min":134.172},"last":34.984999999999999}]},{"osd":4,"last update":"Mon Aug 28 02:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.83599999999999997,"5min":1.319,"15min":2.903},"min":{"1min":0.29999999999999999,"5min":0.26800000000000002,"15min":0.26600000000000001},"max":{"1min":3.391,"5min":13.273,"15min":207.34100000000001},"last":0.33300000000000002},{"interface":"front","average":{"1min":1.6419999999999999,"5min":2.0910000000000002,"15min":2.8940000000000001},"min":{"1min":0.33800000000000002,"5min":0.309,"15min":0.27400000000000002},"max":{"1min":17.768999999999998,"5min":22.303999999999998,"15min":207.352},"last":0.44600000000000001}]},{"osd":5,"last update":"Mon Aug 28 02:35:49 2023","interfaces":[{"interface":"back","average":{"1min":1.1160000000000001,"5min":1.5429999999999999,"15min":2.847},"min":{"1min":0.36199999999999999,"5min":0.315,"15min":0.29299999999999998},"max":{"1min":3.968,"5min":13.335000000000001,"15min":128.696},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.76500000000000001,"5min":1.389,"15min":2.871},"min":{"1min":0.34599999999999997,"5min":0.30399999999999999,"15min":0.30299999999999999},"max":{"1min":2.0539999999999998,"5min":13.303000000000001,"15min":128.84100000000001},"last":0.35999999999999999}]},{"osd":6,"last update":"Mon Aug 28 02:35:49 2023","interfaces":[{"interface":"back","average":{"1min":1.6359999999999999,"5min":2.2120000000000002,"15min":1.887},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.29299999999999998},"max":{"1min":17.719000000000001,"5min":22.25,"15min":22.25},"last":2.0310000000000001},{"interface":"front","average":{"1min":1.081,"5min":1.466,"15min":1.7769999999999999},"min":{"1min":0.35599999999999998,"5min":0.30199999999999999,"15min":0.29399999999999998},"max":{"1min":3.4390000000000001,"5min":13.311,"15min":13.311},"last":2.0179999999999998}]},{"osd":7,"last update":"Mon Aug 28 02:35:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":1.0189999999999999,"15min":1.0189999999999999},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":2.8559999999999999,"5min":2.8559999999999999,"15min":2.8559999999999999},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":3.3410000000000002,"5min":3.3410000000000002,"15min":3.3410000000000002},"last":0.42099999999999999}]}]},{"osd":3,"up_from":112,"seq":481036337343,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2534400,"kb_used_data":1979512,"kb_used_omap":3,"kb_used_meta":554876,"kb_avail":91210752,"statfs":{"total":95995035648,"available":93399810048,"internally_reserved":0,"allocated":2027020288,"data_stored":2025467828,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":568193954},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:36:17 2023","interfaces":[{"interface":"back","average":{"1min":3.1669999999999998,"5min":5.1580000000000004,"15min":4.7169999999999996},"min":{"1min":0.20799999999999999,"5min":0.19500000000000001,"15min":0.193},"max":{"1min":24.561,"5min":138.39699999999999,"15min":164.25299999999999},"last":2.96},{"interface":"front","average":{"1min":2.0640000000000001,"5min":2.9079999999999999,"15min":3.9169999999999998},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.192},"max":{"1min":24.048999999999999,"5min":24.048999999999999,"15min":164.55000000000001},"last":1.7350000000000001}]},{"osd":1,"last update":"Mon Aug 28 02:36:17 2023","interfaces":[{"interface":"back","average":{"1min":7.6219999999999999,"5min":4.0919999999999996,"15min":3.1669999999999998},"min":{"1min":0.222,"5min":0.222,"15min":0.20699999999999999},"max":{"1min":105.04600000000001,"5min":105.04600000000001,"15min":117.34699999999999},"last":1.6879999999999999},{"interface":"front","average":{"1min":7.6379999999999999,"5min":5.7039999999999997,"15min":3.8340000000000001},"min":{"1min":0.29099999999999998,"5min":0.23599999999999999,"15min":0.224},"max":{"1min":105.55800000000001,"5min":143.54400000000001,"15min":143.54400000000001},"last":2.9900000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:36:17 2023","interfaces":[{"interface":"back","average":{"1min":2.052,"5min":5.3099999999999996,"15min":4.5599999999999996},"min":{"1min":0.26700000000000002,"5min":0.20100000000000001,"15min":0.19900000000000001},"max":{"1min":9.9789999999999992,"5min":143.50299999999999,"15min":143.50299999999999},"last":3.4220000000000002},{"interface":"front","average":{"1min":1.583,"5min":2.9039999999999999,"15min":3.6589999999999998},"min":{"1min":0.23699999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":9.9120000000000008,"5min":12.646000000000001,"15min":120.44499999999999},"last":3.383}]},{"osd":4,"last update":"Mon Aug 28 02:36:17 2023","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":1.5620000000000001,"15min":1.827},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":2.8980000000000001,"5min":7.8940000000000001,"15min":58.067999999999998},"last":1.7649999999999999},{"interface":"front","average":{"1min":1.101,"5min":1.7609999999999999,"15min":1.9390000000000001},"min":{"1min":0.38400000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":2.8519999999999999,"5min":10.613,"15min":58.088000000000001},"last":1.7170000000000001}]},{"osd":5,"last update":"Mon Aug 28 02:36:17 2023","interfaces":[{"interface":"back","average":{"1min":1.0669999999999999,"5min":1.9259999999999999,"15min":2.1099999999999999},"min":{"1min":0.39300000000000002,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.8769999999999998,"5min":10.722,"15min":102.904},"last":1.7470000000000001},{"interface":"front","average":{"1min":1.3819999999999999,"5min":3.798,"15min":2.7589999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":4.319,"5min":138.328,"15min":138.328},"last":3.0419999999999998}]},{"osd":6,"last update":"Mon Aug 28 02:36:17 2023","interfaces":[{"interface":"back","average":{"1min":1.48,"5min":3.7749999999999999,"15min":2.5350000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.314},"max":{"1min":4.351,"5min":138.37,"15min":138.37},"last":3.0830000000000002},{"interface":"front","average":{"1min":1.179,"5min":1.655,"15min":1.7749999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.314},"max":{"1min":5.5060000000000002,"5min":7.9400000000000004,"15min":72.030000000000001},"last":1.796}]},{"osd":7,"last update":"Mon Aug 28 02:35:57 2023","interfaces":[{"interface":"back","average":{"1min":1.155,"5min":1.155,"15min":1.155},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":3.46,"5min":3.46,"15min":3.46},"last":3.1120000000000001},{"interface":"front","average":{"1min":1.2869999999999999,"5min":1.2869999999999999,"15min":1.2869999999999999},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":5.4560000000000004,"5min":5.4560000000000004,"15min":5.4560000000000004},"last":1.821}]}]},{"osd":4,"up_from":30,"seq":128849019216,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2198408,"kb_used_data":1612800,"kb_used_omap":0,"kb_used_meta":585600,"kb_avail":91546744,"statfs":{"total":95995035648,"available":93743865856,"internally_reserved":0,"allocated":1651507200,"data_stored":1650630486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":599654400},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:36:25 2023","interfaces":[{"interface":"back","average":{"1min":7.6079999999999997,"5min":7.9610000000000003,"15min":5.3259999999999996},"min":{"1min":0.23400000000000001,"5min":0.23100000000000001,"15min":0.219},"max":{"1min":61.972999999999999,"5min":131.893,"15min":138.67400000000001},"last":1.7549999999999999},{"interface":"front","average":{"1min":7.8399999999999999,"5min":8.1769999999999996,"15min":5.1950000000000003},"min":{"1min":0.23799999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":61.459000000000003,"5min":131.96700000000001,"15min":134.14599999999999},"last":1.8680000000000001}]},{"osd":1,"last update":"Mon Aug 28 02:36:25 2023","interfaces":[{"interface":"back","average":{"1min":9.9469999999999992,"5min":3.3050000000000002,"15min":2.4780000000000002},"min":{"1min":0.27800000000000002,"5min":0.24399999999999999,"15min":0.221},"max":{"1min":105.604,"5min":105.604,"15min":105.604},"last":0.35999999999999999},{"interface":"front","average":{"1min":9.6080000000000005,"5min":3.113,"15min":2.2869999999999999},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":105.883,"5min":105.883,"15min":105.883},"last":6.2169999999999996}]},{"osd":2,"last update":"Mon Aug 28 02:35:50 2023","interfaces":[{"interface":"back","average":{"1min":0.98199999999999998,"5min":3.5270000000000001,"15min":3.8180000000000001},"min":{"1min":0.29699999999999999,"5min":0.255,"15min":0.255},"max":{"1min":5.6319999999999997,"5min":107.786,"15min":107.786},"last":0.39500000000000002},{"interface":"front","average":{"1min":1.0289999999999999,"5min":3.504,"15min":3.7210000000000001},"min":{"1min":0.312,"5min":0.27300000000000002,"15min":0.25},"max":{"1min":5.2169999999999996,"5min":107.599,"15min":107.599},"last":1.9350000000000001}]},{"osd":3,"last update":"Mon Aug 28 02:36:25 2023","interfaces":[{"interface":"back","average":{"1min":8.5619999999999994,"5min":9.8659999999999997,"15min":5.827},"min":{"1min":0.27700000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":137.995,"5min":203.583,"15min":203.583},"last":0.371},{"interface":"front","average":{"1min":8.9250000000000007,"5min":10.34,"15min":5.9850000000000003},"min":{"1min":0.33100000000000002,"5min":0.33000000000000002,"15min":0.32500000000000001},"max":{"1min":137.65899999999999,"5min":203.41399999999999,"15min":203.41399999999999},"last":0.42199999999999999}]},{"osd":5,"last update":"Mon Aug 28 02:36:03 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.45200000000000001,"15min":1.4410000000000001},"min":{"1min":0.26800000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":2.218,"5min":2.3149999999999999,"15min":124.718},"last":1.99},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.55200000000000005,"15min":1.6579999999999999},"min":{"1min":0.27200000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.5620000000000001,"5min":2.444,"15min":124.694},"last":0.40200000000000002}]},{"osd":6,"last update":"Mon Aug 28 02:36:25 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.625,"15min":1.1279999999999999},"min":{"1min":0.28499999999999998,"5min":0.25,"15min":0.25},"max":{"1min":2.1339999999999999,"5min":3.395,"15min":14.907999999999999},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.751,"5min":0.625,"15min":0.89800000000000002},"min":{"1min":0.36199999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":2.2650000000000001,"5min":3.1219999999999999,"15min":7.3170000000000002},"last":1.907}]},{"osd":7,"last update":"Mon Aug 28 02:35:55 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":2.1989999999999998,"5min":2.1989999999999998,"15min":2.1989999999999998},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":1.6100000000000001,"5min":1.6100000000000001,"15min":1.6100000000000001},"last":0.58399999999999996}]}]},{"osd":5,"up_from":36,"seq":154618822988,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3479928,"kb_used_data":2894320,"kb_used_omap":0,"kb_used_meta":585600,"kb_avail":90265224,"statfs":{"total":95995035648,"available":92431589376,"internally_reserved":0,"allocated":2963783680,"data_stored":2962918505,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":599654400},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:36:13 2023","interfaces":[{"interface":"back","average":{"1min":5.1289999999999996,"5min":4.782,"15min":4.399},"min":{"1min":0.216,"5min":0.216,"15min":0.20300000000000001},"max":{"1min":64.590999999999994,"5min":128.72800000000001,"15min":131.06100000000001},"last":0.34799999999999998},{"interface":"front","average":{"1min":5.1520000000000001,"5min":4.8019999999999996,"15min":4.6130000000000004},"min":{"1min":0.245,"5min":0.245,"15min":0.22900000000000001},"max":{"1min":64.578999999999994,"5min":128.75200000000001,"15min":131.262},"last":0.35699999999999998}]},{"osd":1,"last update":"Mon Aug 28 02:36:27 2023","interfaces":[{"interface":"back","average":{"1min":4.3529999999999998,"5min":3.5539999999999998,"15min":3.4510000000000001},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.216},"max":{"1min":55.463999999999999,"5min":98.632000000000005,"15min":183.97900000000001},"last":1.256},{"interface":"front","average":{"1min":4.2649999999999997,"5min":3.548,"15min":3.6080000000000001},"min":{"1min":0.28000000000000003,"5min":0.26400000000000001,"15min":0.23200000000000001},"max":{"1min":55.546999999999997,"5min":98.560000000000002,"15min":184.33699999999999},"last":1.6120000000000001}]},{"osd":2,"last update":"Mon Aug 28 02:35:55 2023","interfaces":[{"interface":"back","average":{"1min":0.76300000000000001,"5min":2.944,"15min":4.7590000000000003},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.252},"max":{"1min":3.3690000000000002,"5min":78.834000000000003,"15min":176.69499999999999},"last":0.29299999999999998},{"interface":"front","average":{"1min":0.90700000000000003,"5min":3.0009999999999999,"15min":5.0149999999999997},"min":{"1min":0.35399999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.3940000000000001,"5min":78.798000000000002,"15min":176.71199999999999},"last":0.38800000000000001}]},{"osd":3,"last update":"Mon Aug 28 02:36:27 2023","interfaces":[{"interface":"back","average":{"1min":2.2480000000000002,"5min":6.5819999999999999,"15min":3.6949999999999998},"min":{"1min":0.30499999999999999,"5min":0.29699999999999999,"15min":0.26100000000000001},"max":{"1min":14.202,"5min":128.571,"15min":128.571},"last":0.55400000000000005},{"interface":"front","average":{"1min":1.6579999999999999,"5min":6.2140000000000004,"15min":3.8849999999999998},"min":{"1min":0.32900000000000001,"5min":0.28699999999999998,"15min":0.26200000000000001},"max":{"1min":4.1909999999999998,"5min":128.684,"15min":128.684},"last":1.71}]},{"osd":4,"last update":"Mon Aug 28 02:36:04 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.55200000000000005,"15min":1.335},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":2.145,"5min":3.0259999999999998,"15min":85.769999999999996},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.499,"15min":1.3100000000000001},"min":{"1min":0.29899999999999999,"5min":0.248,"15min":0.248},"max":{"1min":0.94699999999999995,"5min":2.7690000000000001,"15min":85.825000000000003},"last":0.379}]},{"osd":6,"last update":"Mon Aug 28 02:36:27 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.55100000000000005,"15min":1.054},"min":{"1min":0.27800000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":2.181,"5min":3.9990000000000001,"15min":14.803000000000001},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.65400000000000003,"15min":0.97099999999999997},"min":{"1min":0.376,"5min":0.29599999999999999,"15min":0.28799999999999998},"max":{"1min":1.4379999999999999,"5min":7.4740000000000002,"15min":8.6479999999999997},"last":0.44400000000000001}]},{"osd":7,"last update":"Mon Aug 28 02:35:55 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":2.0049999999999999,"5min":2.0049999999999999,"15min":2.0049999999999999},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":2.7839999999999998,"5min":2.7839999999999998,"15min":2.7839999999999998},"last":0.495}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2728394752,"data_stored":2728394752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2355101696,"data_stored":2355101696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2382823424,"data_stored":2382823424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2030567424,"data_stored":2030567424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1657143296,"data_stored":1657143296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2977038336,"data_stored":2977038336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2003894272,"data_stored":2003894272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-08-28T02:36:38.567 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-08-28T02:36:38.567 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-08-28T02:36:38.568 INFO:tasks.thrashosds.thrasher:Growing pool device_health_metrics 2023-08-28T02:36:38.568 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph pg dump --format=json 2023-08-28T02:36:38.625 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:36:38.655 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:38.696 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:36:38.712 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:36:38.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:38.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:36:38.817 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:36:38.853 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:36:38.908 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:38.953 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:38 smithi143 ceph-mon[183059]: pgmap v882: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 51 MiB/s wr, 819 op/s 2023-08-28T02:36:38.953 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:38 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:36:38.953 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:38 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:36:38.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:38 smithi143 ceph-mon[189871]: pgmap v882: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 51 MiB/s wr, 819 op/s 2023-08-28T02:36:38.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:38 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:36:38.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:38 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:36:38.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:38 smithi169 ceph-mon[159415]: pgmap v882: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 51 MiB/s wr, 819 op/s 2023-08-28T02:36:38.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:38 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:36:38.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:38 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:36:39.208 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 76 16 86687 86671 71.2424 42.375 0.0104841 0.0140305 2023-08-28T02:36:39.213 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:36:39.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:39.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:36:39.323 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:36:39.408 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:39.475 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:36:40.032 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:39 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:36:40.032 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:39 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:36:40.032 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:39 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:36:40.033 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:39 smithi143 ceph-mon[183059]: pgmap v883: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 51 MiB/s wr, 812 op/s 2023-08-28T02:36:40.033 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:39 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:36:40.033 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:39 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:36:40.033 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:39 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:36:40.033 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:39 smithi143 ceph-mon[189871]: pgmap v883: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 51 MiB/s wr, 812 op/s 2023-08-28T02:36:40.208 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 77 16 87331 87315 70.84 40.25 0.00561341 0.0141083 2023-08-28T02:36:40.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:39 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:36:40.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:39 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:36:40.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:39 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:36:40.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:39 smithi169 ceph-mon[159415]: pgmap v883: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 51 MiB/s wr, 812 op/s 2023-08-28T02:36:41.047 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:36:41.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:41.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:36:41.156 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:36:41.186 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:41.208 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 78 16 88204 88188 70.6313 54.5625 0.00444701 0.0141501 2023-08-28T02:36:41.231 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:36:41.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:41.237 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:36:41.338 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:36:41.424 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:41.859 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:36:41.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:41.863 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:36:41.888 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:36:41.964 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:36:41.993 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:42.050 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:36:42.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:42.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:36:42.155 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:36:42.177 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:42 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:36:42.177 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:42 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:36:42.178 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:42 smithi143 ceph-mon[183059]: pgmap v884: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 51 MiB/s wr, 811 op/s 2023-08-28T02:36:42.180 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:42 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:36:42.180 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:42 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:36:42.180 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:42 smithi143 ceph-mon[189871]: pgmap v884: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 51 MiB/s wr, 811 op/s 2023-08-28T02:36:42.180 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:36:41 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:36:41] "GET /metrics HTTP/1.1" 200 36530 "" "Prometheus/2.33.4" 2023-08-28T02:36:42.208 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 79 16 89003 88987 70.3694 49.9375 0.0132683 0.0142051 2023-08-28T02:36:42.234 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:42.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:42 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:36:42.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:42 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:36:42.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:42 smithi169 ceph-mon[159415]: pgmap v884: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 51 MiB/s wr, 811 op/s 2023-08-28T02:36:43.173 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:36:43.208 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:2023-08-28T02:36:43.209845+0000 min lat: 0.00106302 max lat: 1.01701 avg lat: 0.014279 2023-08-28T02:36:43.208 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-08-28T02:36:43.208 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 80 16 89662 89646 70.0047 41.1875 0.0144839 0.014279 2023-08-28T02:36:43.210 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:36:43.246 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:36:43.281 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:36:43.316 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:36:43.352 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:36:43.391 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:36:43.434 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:36:43.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:43 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3375949098' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:36:43.454 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:43 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3375949098' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:36:43.476 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:36:43.476 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:36:43.476 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:36:43.476 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:36:43.476 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:36:43.476 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:36:43.476 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:36:43.477 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:36:43.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:43 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3375949098' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:36:43.912 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:36:43.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:43.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:36:44.027 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:36:44.069 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:44.131 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:36:44.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:44.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:36:44.211 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 81 16 90546 90530 69.8204 55.25 0.0146591 0.0143161 2023-08-28T02:36:44.240 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:36:44.270 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:44.310 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:36:44.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:44.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:36:44.416 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:36:44.445 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:44 smithi169 ceph-mon[159415]: pgmap v885: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 52 MiB/s wr, 827 op/s 2023-08-28T02:36:44.448 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:44.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:44 smithi143 ceph-mon[183059]: pgmap v885: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 52 MiB/s wr, 827 op/s 2023-08-28T02:36:44.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:44 smithi143 ceph-mon[189871]: pgmap v885: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 52 MiB/s wr, 827 op/s 2023-08-28T02:36:44.504 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:36:44.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:44.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:36:44.614 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:36:44.645 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:45.213 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 82 16 91192 91176 69.4595 40.375 0.0232922 0.0143917 2023-08-28T02:36:45.221 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is set 2023-08-28T02:36:45.457 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:36:45.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:45.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:36:45.476 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:45 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/4213158630' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:36:45.476 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:45 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:36:45.480 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:45 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/4213158630' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:36:45.481 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:45 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:36:45.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:45 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/4213158630' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:36:45.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:45 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:36:45.577 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:36:45.610 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:45.651 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:36:45.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:45.657 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:36:45.758 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:36:45.786 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:46.214 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 83 16 91789 91773 69.0723 37.3125 0.0105591 0.0144727 2023-08-28T02:36:46.291 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:36:46.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:46.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:36:46.403 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:36:46.488 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:46 smithi169 ceph-mon[159415]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:36:46.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:46 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:36:46.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:46 smithi169 ceph-mon[159415]: osdmap e338: 8 total, 8 up, 7 in 2023-08-28T02:36:46.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:46 smithi169 ceph-mon[159415]: failed to encode map e338 with expected crc 2023-08-28T02:36:46.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:46 smithi169 ceph-mon[159415]: failed to encode map e338 with expected crc 2023-08-28T02:36:46.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:46 smithi169 ceph-mon[159415]: failed to encode map e338 with expected crc 2023-08-28T02:36:46.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:46 smithi169 ceph-mon[159415]: failed to encode map e338 with expected crc 2023-08-28T02:36:46.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:46 smithi169 ceph-mon[159415]: failed to encode map e338 with expected crc 2023-08-28T02:36:46.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:46 smithi169 ceph-mon[159415]: failed to encode map e338 with expected crc 2023-08-28T02:36:46.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:46 smithi169 ceph-mon[159415]: failed to encode map e338 with expected crc 2023-08-28T02:36:46.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:46 smithi169 ceph-mon[159415]: failed to encode map e338 with expected crc 2023-08-28T02:36:46.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:46 smithi169 ceph-mon[159415]: pgmap v887: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 48 MiB/s wr, 773 op/s 2023-08-28T02:36:46.497 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:46 smithi143 ceph-mon[183059]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:36:46.497 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:46 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:36:46.498 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:46 smithi143 ceph-mon[183059]: osdmap e338: 8 total, 8 up, 7 in 2023-08-28T02:36:46.498 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:46 smithi143 ceph-mon[183059]: failed to encode map e338 with expected crc 2023-08-28T02:36:46.498 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:46 smithi143 ceph-mon[183059]: failed to encode map e338 with expected crc 2023-08-28T02:36:46.498 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:46 smithi143 ceph-mon[183059]: failed to encode map e338 with expected crc 2023-08-28T02:36:46.498 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:46 smithi143 ceph-mon[183059]: failed to encode map e338 with expected crc 2023-08-28T02:36:46.498 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:46 smithi143 ceph-mon[183059]: failed to encode map e338 with expected crc 2023-08-28T02:36:46.498 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:46 smithi143 ceph-mon[183059]: failed to encode map e338 with expected crc 2023-08-28T02:36:46.498 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:46 smithi143 ceph-mon[183059]: failed to encode map e338 with expected crc 2023-08-28T02:36:46.498 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:46 smithi143 ceph-mon[183059]: failed to encode map e338 with expected crc 2023-08-28T02:36:46.498 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:46 smithi143 ceph-mon[183059]: pgmap v887: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 48 MiB/s wr, 773 op/s 2023-08-28T02:36:46.499 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:46 smithi143 ceph-mon[189871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:36:46.499 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:46 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:36:46.499 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:46 smithi143 ceph-mon[189871]: osdmap e338: 8 total, 8 up, 7 in 2023-08-28T02:36:46.499 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:46 smithi143 ceph-mon[189871]: failed to encode map e338 with expected crc 2023-08-28T02:36:46.499 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:46 smithi143 ceph-mon[189871]: failed to encode map e338 with expected crc 2023-08-28T02:36:46.499 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:46 smithi143 ceph-mon[189871]: failed to encode map e338 with expected crc 2023-08-28T02:36:46.499 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:46 smithi143 ceph-mon[189871]: failed to encode map e338 with expected crc 2023-08-28T02:36:46.499 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:46 smithi143 ceph-mon[189871]: failed to encode map e338 with expected crc 2023-08-28T02:36:46.499 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:46 smithi143 ceph-mon[189871]: failed to encode map e338 with expected crc 2023-08-28T02:36:46.499 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:46 smithi143 ceph-mon[189871]: failed to encode map e338 with expected crc 2023-08-28T02:36:46.499 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:46 smithi143 ceph-mon[189871]: failed to encode map e338 with expected crc 2023-08-28T02:36:46.499 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:46 smithi143 ceph-mon[189871]: pgmap v887: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 48 MiB/s wr, 773 op/s 2023-08-28T02:36:46.517 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:46.659 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:36:46.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:46.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:36:46.770 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:36:46.799 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:46.852 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:36:46.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:46.856 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:36:46.957 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:36:46.989 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:46.990 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:36:46.990 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:36:47.046 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:36:47.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:47.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:36:47.151 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:36:47.156 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:36:47.174 INFO:teuthology.orchestra.run.smithi143.stderr:dumped all 2023-08-28T02:36:47.213 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 84 16 92585 92569 68.8423 49.75 0.0186406 0.0145206 2023-08-28T02:36:47.217 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:47.324 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:36:47.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:47.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:36:47.429 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:36:47.430 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:36:47.516 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:48.213 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 85 16 93201 93185 68.4855 38.5 0.00881314 0.0145944 2023-08-28T02:36:48.501 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset noscrub 2023-08-28T02:36:48.656 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:36:48.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:48.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:36:48.765 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.2 dump_ops_in_flight 2023-08-28T02:36:48.768 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:36:48.807 INFO:teuthology.orchestra.run.smithi143.stdout:{"pg_ready":true,"pg_map":{"version":887,"stamp":"2023-08-28T02:36:45.537403+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5900343219,"num_objects":89987,"num_object_clones":0,"num_object_copies":269961,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":89987,"num_whiteouts":0,"num_read":6274,"num_read_kb":13707,"num_write":91065,"num_write_kb":5772517,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":297,"num_bytes_recovered":14969132,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":91129,"ondisk_log_size":91129,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":21006180,"kb_used_data":17209388,"kb_used_omap":29,"kb_used_meta":3796642,"kb_avail":635209884,"statfs":{"total":671965249536,"available":650454921216,"internally_reserved":0,"allocated":17622413312,"data_stored":17613364956,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30674,"internal_metadata":3887761454},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,1,7,4,1,0,0,10],"upper_bound":512},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":507379712,"num_objects":7742,"num_object_clones":0,"num_object_copies":23226,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7742,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7742,"num_write_kb":495488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.007648"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":356,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.670012+0000","last_change":"2023-08-28T02:32:58.740384+0000","last_active":"2023-08-28T02:36:21.670012+0000","last_peered":"2023-08-28T02:36:21.670012+0000","last_clean":"2023-08-28T02:36:21.670012+0000","last_became_active":"2023-08-28T02:32:58.739959+0000","last_became_peered":"2023-08-28T02:32:58.739959+0000","last_unstale":"2023-08-28T02:36:21.670012+0000","last_undegraded":"2023-08-28T02:36:21.670012+0000","last_fullsized":"2023-08-28T02:36:21.670012+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":244,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:28.843457+0000","last_change":"2023-08-28T02:32:58.634375+0000","last_active":"2023-08-28T02:36:28.843457+0000","last_peered":"2023-08-28T02:36:28.843457+0000","last_clean":"2023-08-28T02:36:28.843457+0000","last_became_active":"2023-08-28T02:32:58.632726+0000","last_became_peered":"2023-08-28T02:32:58.632726+0000","last_unstale":"2023-08-28T02:36:28.843457+0000","last_undegraded":"2023-08-28T02:36:28.843457+0000","last_fullsized":"2023-08-28T02:36:28.843457+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:20:27.730725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":57,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.641929+0000","last_change":"2023-08-28T02:32:58.628387+0000","last_active":"2023-08-28T02:36:21.641929+0000","last_peered":"2023-08-28T02:36:21.641929+0000","last_clean":"2023-08-28T02:36:21.641929+0000","last_became_active":"2023-08-28T02:32:58.628071+0000","last_became_peered":"2023-08-28T02:32:58.628071+0000","last_unstale":"2023-08-28T02:36:21.641929+0000","last_undegraded":"2023-08-28T02:36:21.641929+0000","last_fullsized":"2023-08-28T02:36:21.641929+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:25:55.264433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":355,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-08-28T02:36:32.705949+0000","last_change":"2023-08-28T02:32:58.772285+0000","last_active":"2023-08-28T02:36:32.705949+0000","last_peered":"2023-08-28T02:36:32.705949+0000","last_clean":"2023-08-28T02:36:32.705949+0000","last_became_active":"2023-08-28T02:32:58.771439+0000","last_became_peered":"2023-08-28T02:32:58.771439+0000","last_unstale":"2023-08-28T02:36:32.705949+0000","last_undegraded":"2023-08-28T02:36:32.705949+0000","last_fullsized":"2023-08-28T02:36:32.705949+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:10:48.541785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":235,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.669649+0000","last_change":"2023-08-28T02:32:58.779602+0000","last_active":"2023-08-28T02:36:21.669649+0000","last_peered":"2023-08-28T02:36:21.669649+0000","last_clean":"2023-08-28T02:36:21.669649+0000","last_became_active":"2023-08-28T02:32:58.778945+0000","last_became_peered":"2023-08-28T02:32:58.778945+0000","last_unstale":"2023-08-28T02:36:21.669649+0000","last_undegraded":"2023-08-28T02:36:21.669649+0000","last_fullsized":"2023-08-28T02:36:21.669649+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:35:55.832928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":143,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.668662+0000","last_change":"2023-08-28T02:32:58.626766+0000","last_active":"2023-08-28T02:36:21.668662+0000","last_peered":"2023-08-28T02:36:21.668662+0000","last_clean":"2023-08-28T02:36:21.668662+0000","last_became_active":"2023-08-28T02:32:58.626415+0000","last_became_peered":"2023-08-28T02:32:58.626415+0000","last_unstale":"2023-08-28T02:36:21.668662+0000","last_undegraded":"2023-08-28T02:36:21.668662+0000","last_fullsized":"2023-08-28T02:36:21.668662+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":307,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.675313+0000","last_change":"2023-08-28T02:23:59.227592+0000","last_active":"2023-08-28T02:36:21.675313+0000","last_peered":"2023-08-28T02:36:21.675313+0000","last_clean":"2023-08-28T02:36:21.675313+0000","last_became_active":"2023-08-28T02:23:59.227342+0000","last_became_peered":"2023-08-28T02:23:59.227342+0000","last_unstale":"2023-08-28T02:36:21.675313+0000","last_undegraded":"2023-08-28T02:36:21.675313+0000","last_fullsized":"2023-08-28T02:36:21.675313+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:08:19.874396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":253,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.669722+0000","last_change":"2023-08-28T02:32:58.779435+0000","last_active":"2023-08-28T02:36:21.669722+0000","last_peered":"2023-08-28T02:36:21.669722+0000","last_clean":"2023-08-28T02:36:21.669722+0000","last_became_active":"2023-08-28T02:32:58.778629+0000","last_became_peered":"2023-08-28T02:32:58.778629+0000","last_unstale":"2023-08-28T02:36:21.669722+0000","last_undegraded":"2023-08-28T02:36:21.669722+0000","last_fullsized":"2023-08-28T02:36:21.669722+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:25:32.193898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":345,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.642423+0000","last_change":"2023-08-28T02:35:05.992277+0000","last_active":"2023-08-28T02:36:21.642423+0000","last_peered":"2023-08-28T02:36:21.642423+0000","last_clean":"2023-08-28T02:36:21.642423+0000","last_became_active":"2023-08-28T02:35:05.991954+0000","last_became_peered":"2023-08-28T02:35:05.991954+0000","last_unstale":"2023-08-28T02:36:21.642423+0000","last_undegraded":"2023-08-28T02:36:21.642423+0000","last_fullsized":"2023-08-28T02:36:21.642423+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:41:10.177837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":226,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.667205+0000","last_change":"2023-08-28T02:31:55.525178+0000","last_active":"2023-08-28T02:36:21.667205+0000","last_peered":"2023-08-28T02:36:21.667205+0000","last_clean":"2023-08-28T02:36:21.667205+0000","last_became_active":"2023-08-28T02:31:55.524801+0000","last_became_peered":"2023-08-28T02:31:55.524801+0000","last_unstale":"2023-08-28T02:36:21.667205+0000","last_undegraded":"2023-08-28T02:36:21.667205+0000","last_fullsized":"2023-08-28T02:36:21.667205+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:47:30.078960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":104,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.667623+0000","last_change":"2023-08-28T02:31:55.532306+0000","last_active":"2023-08-28T02:36:21.667623+0000","last_peered":"2023-08-28T02:36:21.667623+0000","last_clean":"2023-08-28T02:36:21.667623+0000","last_became_active":"2023-08-28T02:31:55.524276+0000","last_became_peered":"2023-08-28T02:31:55.524276+0000","last_unstale":"2023-08-28T02:36:21.667623+0000","last_undegraded":"2023-08-28T02:36:21.667623+0000","last_fullsized":"2023-08-28T02:36:21.667623+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:27:06.850784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":68,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.668276+0000","last_change":"2023-08-28T02:35:45.112374+0000","last_active":"2023-08-28T02:36:21.668276+0000","last_peered":"2023-08-28T02:36:21.668276+0000","last_clean":"2023-08-28T02:36:21.668276+0000","last_became_active":"2023-08-28T02:35:45.112024+0000","last_became_peered":"2023-08-28T02:35:45.112024+0000","last_unstale":"2023-08-28T02:36:21.668276+0000","last_undegraded":"2023-08-28T02:36:21.668276+0000","last_fullsized":"2023-08-28T02:36:21.668276+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"86'22","reported_seq":406,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.667631+0000","last_change":"2023-08-28T02:31:55.538001+0000","last_active":"2023-08-28T02:36:21.667631+0000","last_peered":"2023-08-28T02:36:21.667631+0000","last_clean":"2023-08-28T02:36:21.667631+0000","last_became_active":"2023-08-28T02:31:55.518604+0000","last_became_peered":"2023-08-28T02:31:55.518604+0000","last_unstale":"2023-08-28T02:36:21.667631+0000","last_undegraded":"2023-08-28T02:36:21.667631+0000","last_fullsized":"2023-08-28T02:36:21.667631+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":380,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.666842+0000","last_change":"2023-08-28T02:32:58.629451+0000","last_active":"2023-08-28T02:36:21.666842+0000","last_peered":"2023-08-28T02:36:21.666842+0000","last_clean":"2023-08-28T02:36:21.666842+0000","last_became_active":"2023-08-28T02:32:58.628767+0000","last_became_peered":"2023-08-28T02:32:58.628767+0000","last_unstale":"2023-08-28T02:36:21.666842+0000","last_undegraded":"2023-08-28T02:36:21.666842+0000","last_fullsized":"2023-08-28T02:36:21.666842+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":110,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.666930+0000","last_change":"2023-08-28T02:31:55.528552+0000","last_active":"2023-08-28T02:36:21.666930+0000","last_peered":"2023-08-28T02:36:21.666930+0000","last_clean":"2023-08-28T02:36:21.666930+0000","last_became_active":"2023-08-28T02:31:55.518079+0000","last_became_peered":"2023-08-28T02:31:55.518079+0000","last_unstale":"2023-08-28T02:36:21.666930+0000","last_undegraded":"2023-08-28T02:36:21.666930+0000","last_fullsized":"2023-08-28T02:36:21.666930+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":244,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.669471+0000","last_change":"2023-08-28T02:32:58.680534+0000","last_active":"2023-08-28T02:36:21.669471+0000","last_peered":"2023-08-28T02:36:21.669471+0000","last_clean":"2023-08-28T02:36:21.669471+0000","last_became_active":"2023-08-28T02:32:58.680145+0000","last_became_peered":"2023-08-28T02:32:58.680145+0000","last_unstale":"2023-08-28T02:36:21.669471+0000","last_undegraded":"2023-08-28T02:36:21.669471+0000","last_fullsized":"2023-08-28T02:36:21.669471+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:56:40.113013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":352,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.674689+0000","last_change":"2023-08-28T02:32:58.746929+0000","last_active":"2023-08-28T02:36:21.674689+0000","last_peered":"2023-08-28T02:36:21.674689+0000","last_clean":"2023-08-28T02:36:21.674689+0000","last_became_active":"2023-08-28T02:32:58.746590+0000","last_became_peered":"2023-08-28T02:32:58.746590+0000","last_unstale":"2023-08-28T02:36:21.674689+0000","last_undegraded":"2023-08-28T02:36:21.674689+0000","last_fullsized":"2023-08-28T02:36:21.674689+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:51:11.974154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":336,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.667549+0000","last_change":"2023-08-28T02:35:05.993977+0000","last_active":"2023-08-28T02:36:21.667549+0000","last_peered":"2023-08-28T02:36:21.667549+0000","last_clean":"2023-08-28T02:36:21.667549+0000","last_became_active":"2023-08-28T02:35:05.993610+0000","last_became_peered":"2023-08-28T02:35:05.993610+0000","last_unstale":"2023-08-28T02:36:21.667549+0000","last_undegraded":"2023-08-28T02:36:21.667549+0000","last_fullsized":"2023-08-28T02:36:21.667549+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:37:10.901664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":166,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.666248+0000","last_change":"2023-08-28T02:31:55.517851+0000","last_active":"2023-08-28T02:36:21.666248+0000","last_peered":"2023-08-28T02:36:21.666248+0000","last_clean":"2023-08-28T02:36:21.666248+0000","last_became_active":"2023-08-28T02:31:55.517320+0000","last_became_peered":"2023-08-28T02:31:55.517320+0000","last_unstale":"2023-08-28T02:36:21.666248+0000","last_undegraded":"2023-08-28T02:36:21.666248+0000","last_fullsized":"2023-08-28T02:36:21.666248+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":335,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.675494+0000","last_change":"2023-08-28T02:31:55.528250+0000","last_active":"2023-08-28T02:36:21.675494+0000","last_peered":"2023-08-28T02:36:21.675494+0000","last_clean":"2023-08-28T02:36:21.675494+0000","last_became_active":"2023-08-28T02:31:55.527950+0000","last_became_peered":"2023-08-28T02:31:55.527950+0000","last_unstale":"2023-08-28T02:36:21.675494+0000","last_undegraded":"2023-08-28T02:36:21.675494+0000","last_fullsized":"2023-08-28T02:36:21.675494+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:39:46.120437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":135,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.668103+0000","last_change":"2023-08-28T02:28:25.703333+0000","last_active":"2023-08-28T02:36:21.668103+0000","last_peered":"2023-08-28T02:36:21.668103+0000","last_clean":"2023-08-28T02:36:21.668103+0000","last_became_active":"2023-08-28T02:28:25.343515+0000","last_became_peered":"2023-08-28T02:28:25.343515+0000","last_unstale":"2023-08-28T02:36:21.668103+0000","last_undegraded":"2023-08-28T02:36:21.668103+0000","last_fullsized":"2023-08-28T02:36:21.668103+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":253,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.667626+0000","last_change":"2023-08-28T02:35:45.097133+0000","last_active":"2023-08-28T02:36:21.667626+0000","last_peered":"2023-08-28T02:36:21.667626+0000","last_clean":"2023-08-28T02:36:21.667626+0000","last_became_active":"2023-08-28T02:35:45.096766+0000","last_became_peered":"2023-08-28T02:35:45.096766+0000","last_unstale":"2023-08-28T02:36:21.667626+0000","last_undegraded":"2023-08-28T02:36:21.667626+0000","last_fullsized":"2023-08-28T02:36:21.667626+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:27:29.558704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"337'5553","reported_seq":5580,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-08-28T02:36:42.022769+0000","last_change":"2023-08-28T02:35:14.499701+0000","last_active":"2023-08-28T02:36:42.022769+0000","last_peered":"2023-08-28T02:36:42.022769+0000","last_clean":"2023-08-28T02:36:42.022769+0000","last_became_active":"2023-08-28T02:35:14.499400+0000","last_became_peered":"2023-08-28T02:35:14.499400+0000","last_unstale":"2023-08-28T02:36:42.022769+0000","last_undegraded":"2023-08-28T02:36:42.022769+0000","last_fullsized":"2023-08-28T02:36:42.022769+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":5553,"log_dups_size":0,"ondisk_log_size":5553,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363921408,"num_objects":5553,"num_object_clones":0,"num_object_copies":16659,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5553,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5553,"num_write_kb":355392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":200,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:30.021744+0000","last_change":"2023-08-28T02:32:58.782840+0000","last_active":"2023-08-28T02:36:30.021744+0000","last_peered":"2023-08-28T02:36:30.021744+0000","last_clean":"2023-08-28T02:36:30.021744+0000","last_became_active":"2023-08-28T02:32:58.683096+0000","last_became_peered":"2023-08-28T02:32:58.683096+0000","last_unstale":"2023-08-28T02:36:30.021744+0000","last_undegraded":"2023-08-28T02:36:30.021744+0000","last_fullsized":"2023-08-28T02:36:30.021744+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:10:08.037078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":296,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.668018+0000","last_change":"2023-08-28T02:35:06.546129+0000","last_active":"2023-08-28T02:36:21.668018+0000","last_peered":"2023-08-28T02:36:21.668018+0000","last_clean":"2023-08-28T02:36:21.668018+0000","last_became_active":"2023-08-28T02:35:06.545789+0000","last_became_peered":"2023-08-28T02:35:06.545789+0000","last_unstale":"2023-08-28T02:36:21.668018+0000","last_undegraded":"2023-08-28T02:36:21.668018+0000","last_fullsized":"2023-08-28T02:36:21.668018+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.5","version":"337'5573","reported_seq":5600,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-08-28T02:36:42.224995+0000","last_change":"2023-08-28T02:35:14.500308+0000","last_active":"2023-08-28T02:36:42.224995+0000","last_peered":"2023-08-28T02:36:42.224995+0000","last_clean":"2023-08-28T02:36:42.224995+0000","last_became_active":"2023-08-28T02:35:14.499954+0000","last_became_peered":"2023-08-28T02:35:14.499954+0000","last_unstale":"2023-08-28T02:36:42.224995+0000","last_undegraded":"2023-08-28T02:36:42.224995+0000","last_fullsized":"2023-08-28T02:36:42.224995+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":5573,"log_dups_size":0,"ondisk_log_size":5573,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365232128,"num_objects":5573,"num_object_clones":0,"num_object_copies":16719,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5573,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5573,"num_write_kb":356672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":132,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.666480+0000","last_change":"2023-08-28T02:32:58.856906+0000","last_active":"2023-08-28T02:36:21.666480+0000","last_peered":"2023-08-28T02:36:21.666480+0000","last_clean":"2023-08-28T02:36:21.666480+0000","last_became_active":"2023-08-28T02:32:58.740028+0000","last_became_peered":"2023-08-28T02:32:58.740028+0000","last_unstale":"2023-08-28T02:36:21.666480+0000","last_undegraded":"2023-08-28T02:36:21.666480+0000","last_fullsized":"2023-08-28T02:36:21.666480+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":390,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.666175+0000","last_change":"2023-08-28T02:32:58.740396+0000","last_active":"2023-08-28T02:36:21.666175+0000","last_peered":"2023-08-28T02:36:21.666175+0000","last_clean":"2023-08-28T02:36:21.666175+0000","last_became_active":"2023-08-28T02:32:58.739941+0000","last_became_peered":"2023-08-28T02:32:58.739941+0000","last_unstale":"2023-08-28T02:36:21.666175+0000","last_undegraded":"2023-08-28T02:36:21.666175+0000","last_fullsized":"2023-08-28T02:36:21.666175+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"337'5743","reported_seq":5771,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-08-28T02:36:44.575917+0000","last_change":"2023-08-28T02:35:14.495274+0000","last_active":"2023-08-28T02:36:44.575917+0000","last_peered":"2023-08-28T02:36:44.575917+0000","last_clean":"2023-08-28T02:36:44.575917+0000","last_became_active":"2023-08-28T02:35:14.494642+0000","last_became_peered":"2023-08-28T02:35:14.494642+0000","last_unstale":"2023-08-28T02:36:44.575917+0000","last_undegraded":"2023-08-28T02:36:44.575917+0000","last_fullsized":"2023-08-28T02:36:44.575917+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":5743,"log_dups_size":0,"ondisk_log_size":5743,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:28:37.992837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":376373248,"num_objects":5743,"num_object_clones":0,"num_object_copies":17229,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5743,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5743,"num_write_kb":367552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":135,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.668613+0000","last_change":"2023-08-28T02:35:05.994859+0000","last_active":"2023-08-28T02:36:21.668613+0000","last_peered":"2023-08-28T02:36:21.668613+0000","last_clean":"2023-08-28T02:36:21.668613+0000","last_became_active":"2023-08-28T02:35:05.994374+0000","last_became_peered":"2023-08-28T02:35:05.994374+0000","last_unstale":"2023-08-28T02:36:21.668613+0000","last_undegraded":"2023-08-28T02:36:21.668613+0000","last_fullsized":"2023-08-28T02:36:21.668613+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":161,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.668202+0000","last_change":"2023-08-28T02:35:45.112365+0000","last_active":"2023-08-28T02:36:21.668202+0000","last_peered":"2023-08-28T02:36:21.668202+0000","last_clean":"2023-08-28T02:36:21.668202+0000","last_became_active":"2023-08-28T02:35:45.112008+0000","last_became_peered":"2023-08-28T02:35:45.112008+0000","last_unstale":"2023-08-28T02:36:21.668202+0000","last_undegraded":"2023-08-28T02:36:21.668202+0000","last_fullsized":"2023-08-28T02:36:21.668202+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"337'5669","reported_seq":5697,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-08-28T02:36:43.275662+0000","last_change":"2023-08-28T02:35:14.498797+0000","last_active":"2023-08-28T02:36:43.275662+0000","last_peered":"2023-08-28T02:36:43.275662+0000","last_clean":"2023-08-28T02:36:43.275662+0000","last_became_active":"2023-08-28T02:35:14.498551+0000","last_became_peered":"2023-08-28T02:35:14.498551+0000","last_unstale":"2023-08-28T02:36:43.275662+0000","last_undegraded":"2023-08-28T02:36:43.275662+0000","last_fullsized":"2023-08-28T02:36:43.275662+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":5669,"log_dups_size":0,"ondisk_log_size":5669,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:43:07.729642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":371523584,"num_objects":5669,"num_object_clones":0,"num_object_copies":17007,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5669,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5669,"num_write_kb":362816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":341,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.669694+0000","last_change":"2023-08-28T02:28:25.445982+0000","last_active":"2023-08-28T02:36:21.669694+0000","last_peered":"2023-08-28T02:36:21.669694+0000","last_clean":"2023-08-28T02:36:21.669694+0000","last_became_active":"2023-08-28T02:28:25.443577+0000","last_became_peered":"2023-08-28T02:28:25.443577+0000","last_unstale":"2023-08-28T02:36:21.669694+0000","last_undegraded":"2023-08-28T02:36:21.669694+0000","last_fullsized":"2023-08-28T02:36:21.669694+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:24:03.381751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":338,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-08-28T02:36:32.705152+0000","last_change":"2023-08-28T02:32:58.887874+0000","last_active":"2023-08-28T02:36:32.705152+0000","last_peered":"2023-08-28T02:36:32.705152+0000","last_clean":"2023-08-28T02:36:32.705152+0000","last_became_active":"2023-08-28T02:32:58.887377+0000","last_became_peered":"2023-08-28T02:32:58.887377+0000","last_unstale":"2023-08-28T02:36:32.705152+0000","last_undegraded":"2023-08-28T02:36:32.705152+0000","last_fullsized":"2023-08-28T02:36:32.705152+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:08:07.234984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"337'5608","reported_seq":5636,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-08-28T02:36:43.751336+0000","last_change":"2023-08-28T02:35:14.503271+0000","last_active":"2023-08-28T02:36:43.751336+0000","last_peered":"2023-08-28T02:36:43.751336+0000","last_clean":"2023-08-28T02:36:43.751336+0000","last_became_active":"2023-08-28T02:35:14.502940+0000","last_became_peered":"2023-08-28T02:35:14.502940+0000","last_unstale":"2023-08-28T02:36:43.751336+0000","last_undegraded":"2023-08-28T02:36:43.751336+0000","last_fullsized":"2023-08-28T02:36:43.751336+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":5608,"log_dups_size":0,"ondisk_log_size":5608,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:07:49.114269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367525888,"num_objects":5608,"num_object_clones":0,"num_object_copies":16824,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5608,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5608,"num_write_kb":358912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":203,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.669723+0000","last_change":"2023-08-28T02:23:59.230059+0000","last_active":"2023-08-28T02:36:21.669723+0000","last_peered":"2023-08-28T02:36:21.669723+0000","last_clean":"2023-08-28T02:36:21.669723+0000","last_became_active":"2023-08-28T02:23:59.217783+0000","last_became_peered":"2023-08-28T02:23:59.217783+0000","last_unstale":"2023-08-28T02:36:21.669723+0000","last_undegraded":"2023-08-28T02:36:21.669723+0000","last_fullsized":"2023-08-28T02:36:21.669723+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:00:12.145395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":357,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.667537+0000","last_change":"2023-08-28T02:28:25.351698+0000","last_active":"2023-08-28T02:36:21.667537+0000","last_peered":"2023-08-28T02:36:21.667537+0000","last_clean":"2023-08-28T02:36:21.667537+0000","last_became_active":"2023-08-28T02:28:25.351349+0000","last_became_peered":"2023-08-28T02:28:25.351349+0000","last_unstale":"2023-08-28T02:36:21.667537+0000","last_undegraded":"2023-08-28T02:36:21.667537+0000","last_fullsized":"2023-08-28T02:36:21.667537+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"337'5597","reported_seq":5625,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-08-28T02:36:43.207606+0000","last_change":"2023-08-28T02:35:14.494905+0000","last_active":"2023-08-28T02:36:43.207606+0000","last_peered":"2023-08-28T02:36:43.207606+0000","last_clean":"2023-08-28T02:36:43.207606+0000","last_became_active":"2023-08-28T02:35:14.494589+0000","last_became_peered":"2023-08-28T02:35:14.494589+0000","last_unstale":"2023-08-28T02:36:43.207606+0000","last_undegraded":"2023-08-28T02:36:43.207606+0000","last_fullsized":"2023-08-28T02:36:43.207606+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":5597,"log_dups_size":0,"ondisk_log_size":5597,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:26:28.302635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":366804992,"num_objects":5597,"num_object_clones":0,"num_object_copies":16791,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5597,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5597,"num_write_kb":358208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":143,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.668061+0000","last_change":"2023-08-28T02:32:58.629249+0000","last_active":"2023-08-28T02:36:21.668061+0000","last_peered":"2023-08-28T02:36:21.668061+0000","last_clean":"2023-08-28T02:36:21.668061+0000","last_became_active":"2023-08-28T02:32:58.628583+0000","last_became_peered":"2023-08-28T02:32:58.628583+0000","last_unstale":"2023-08-28T02:36:21.668061+0000","last_undegraded":"2023-08-28T02:36:21.668061+0000","last_fullsized":"2023-08-28T02:36:21.668061+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":410,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.667779+0000","last_change":"2023-08-28T02:35:45.100048+0000","last_active":"2023-08-28T02:36:21.667779+0000","last_peered":"2023-08-28T02:36:21.667779+0000","last_clean":"2023-08-28T02:36:21.667779+0000","last_became_active":"2023-08-28T02:35:45.099690+0000","last_became_peered":"2023-08-28T02:35:45.099690+0000","last_unstale":"2023-08-28T02:36:21.667779+0000","last_undegraded":"2023-08-28T02:36:21.667779+0000","last_fullsized":"2023-08-28T02:36:21.667779+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"337'5580","reported_seq":5607,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-08-28T02:36:42.001599+0000","last_change":"2023-08-28T02:35:14.498910+0000","last_active":"2023-08-28T02:36:42.001599+0000","last_peered":"2023-08-28T02:36:42.001599+0000","last_clean":"2023-08-28T02:36:42.001599+0000","last_became_active":"2023-08-28T02:35:14.498631+0000","last_became_peered":"2023-08-28T02:35:14.498631+0000","last_unstale":"2023-08-28T02:36:42.001599+0000","last_undegraded":"2023-08-28T02:36:42.001599+0000","last_fullsized":"2023-08-28T02:36:42.001599+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":5580,"log_dups_size":0,"ondisk_log_size":5580,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365690880,"num_objects":5580,"num_object_clones":0,"num_object_copies":16740,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5580,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5580,"num_write_kb":357120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":273,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.666317+0000","last_change":"2023-08-28T02:32:58.740311+0000","last_active":"2023-08-28T02:36:21.666317+0000","last_peered":"2023-08-28T02:36:21.666317+0000","last_clean":"2023-08-28T02:36:21.666317+0000","last_became_active":"2023-08-28T02:32:58.739897+0000","last_became_peered":"2023-08-28T02:32:58.739897+0000","last_unstale":"2023-08-28T02:36:21.666317+0000","last_undegraded":"2023-08-28T02:36:21.666317+0000","last_fullsized":"2023-08-28T02:36:21.666317+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":324,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.675567+0000","last_change":"2023-08-28T02:32:58.766795+0000","last_active":"2023-08-28T02:36:21.675567+0000","last_peered":"2023-08-28T02:36:21.675567+0000","last_clean":"2023-08-28T02:36:21.675567+0000","last_became_active":"2023-08-28T02:32:58.766441+0000","last_became_peered":"2023-08-28T02:32:58.766441+0000","last_unstale":"2023-08-28T02:36:21.675567+0000","last_undegraded":"2023-08-28T02:36:21.675567+0000","last_fullsized":"2023-08-28T02:36:21.675567+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:58:13.963961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.3","version":"337'5646","reported_seq":5674,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-08-28T02:36:43.314621+0000","last_change":"2023-08-28T02:35:14.498188+0000","last_active":"2023-08-28T02:36:43.314621+0000","last_peered":"2023-08-28T02:36:43.314621+0000","last_clean":"2023-08-28T02:36:43.314621+0000","last_became_active":"2023-08-28T02:35:14.496278+0000","last_became_peered":"2023-08-28T02:35:14.496278+0000","last_unstale":"2023-08-28T02:36:43.314621+0000","last_undegraded":"2023-08-28T02:36:43.314621+0000","last_fullsized":"2023-08-28T02:36:43.314621+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":5646,"log_dups_size":0,"ondisk_log_size":5646,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:43:23.633921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":370016256,"num_objects":5646,"num_object_clones":0,"num_object_copies":16938,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5646,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5646,"num_write_kb":361344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":244,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.670013+0000","last_change":"2023-08-28T02:32:58.779257+0000","last_active":"2023-08-28T02:36:21.670013+0000","last_peered":"2023-08-28T02:36:21.670013+0000","last_clean":"2023-08-28T02:36:21.670013+0000","last_became_active":"2023-08-28T02:32:58.778295+0000","last_became_peered":"2023-08-28T02:32:58.778295+0000","last_unstale":"2023-08-28T02:36:21.670013+0000","last_undegraded":"2023-08-28T02:36:21.670013+0000","last_fullsized":"2023-08-28T02:36:21.670013+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:50:49.444715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":57,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.642852+0000","last_change":"2023-08-28T02:32:58.755765+0000","last_active":"2023-08-28T02:36:21.642852+0000","last_peered":"2023-08-28T02:36:21.642852+0000","last_clean":"2023-08-28T02:36:21.642852+0000","last_became_active":"2023-08-28T02:32:58.755438+0000","last_became_peered":"2023-08-28T02:32:58.755438+0000","last_unstale":"2023-08-28T02:36:21.642852+0000","last_undegraded":"2023-08-28T02:36:21.642852+0000","last_fullsized":"2023-08-28T02:36:21.642852+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:14:11.498611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"337'5670","reported_seq":5698,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-08-28T02:36:44.587558+0000","last_change":"2023-08-28T02:35:14.497511+0000","last_active":"2023-08-28T02:36:44.587558+0000","last_peered":"2023-08-28T02:36:44.587558+0000","last_clean":"2023-08-28T02:36:44.587558+0000","last_became_active":"2023-08-28T02:35:14.497180+0000","last_became_peered":"2023-08-28T02:35:14.497180+0000","last_unstale":"2023-08-28T02:36:44.587558+0000","last_undegraded":"2023-08-28T02:36:44.587558+0000","last_fullsized":"2023-08-28T02:36:44.587558+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":5670,"log_dups_size":0,"ondisk_log_size":5670,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:59:26.376998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":371589120,"num_objects":5670,"num_object_clones":0,"num_object_copies":17010,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5670,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5670,"num_write_kb":362880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"78'126","reported_seq":741,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.642577+0000","last_change":"2023-08-28T02:35:09.735906+0000","last_active":"2023-08-28T02:36:21.642577+0000","last_peered":"2023-08-28T02:36:21.642577+0000","last_clean":"2023-08-28T02:36:21.642577+0000","last_became_active":"2023-08-28T02:32:59.660388+0000","last_became_peered":"2023-08-28T02:32:59.660388+0000","last_unstale":"2023-08-28T02:36:21.642577+0000","last_undegraded":"2023-08-28T02:36:21.642577+0000","last_fullsized":"2023-08-28T02:36:21.642577+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:19:53.565088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":743,"num_read_kb":1331,"num_write":125,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":178,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.667484+0000","last_change":"2023-08-28T02:35:05.991133+0000","last_active":"2023-08-28T02:36:21.667484+0000","last_peered":"2023-08-28T02:36:21.667484+0000","last_clean":"2023-08-28T02:36:21.667484+0000","last_became_active":"2023-08-28T02:35:05.990829+0000","last_became_peered":"2023-08-28T02:35:05.990829+0000","last_unstale":"2023-08-28T02:36:21.667484+0000","last_undegraded":"2023-08-28T02:36:21.667484+0000","last_fullsized":"2023-08-28T02:36:21.667484+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:40:28.077023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":19,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.641646+0000","last_change":"2023-08-28T02:35:45.099619+0000","last_active":"2023-08-28T02:36:21.641646+0000","last_peered":"2023-08-28T02:36:21.641646+0000","last_clean":"2023-08-28T02:36:21.641646+0000","last_became_active":"2023-08-28T02:35:45.099285+0000","last_became_peered":"2023-08-28T02:35:45.099285+0000","last_unstale":"2023-08-28T02:36:21.641646+0000","last_undegraded":"2023-08-28T02:36:21.641646+0000","last_fullsized":"2023-08-28T02:36:21.641646+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:24:58.305957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.8","version":"337'5701","reported_seq":5729,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-08-28T02:36:43.789350+0000","last_change":"2023-08-28T02:35:14.497638+0000","last_active":"2023-08-28T02:36:43.789350+0000","last_peered":"2023-08-28T02:36:43.789350+0000","last_clean":"2023-08-28T02:36:43.789350+0000","last_became_active":"2023-08-28T02:35:14.497323+0000","last_became_peered":"2023-08-28T02:35:14.497323+0000","last_unstale":"2023-08-28T02:36:43.789350+0000","last_undegraded":"2023-08-28T02:36:43.789350+0000","last_fullsized":"2023-08-28T02:36:43.789350+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":5701,"log_dups_size":0,"ondisk_log_size":5701,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:30:06.078735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373620736,"num_objects":5701,"num_object_clones":0,"num_object_copies":17103,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5701,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5701,"num_write_kb":364864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"78'204","reported_seq":1421,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.642760+0000","last_change":"2023-08-28T02:32:59.644574+0000","last_active":"2023-08-28T02:36:21.642760+0000","last_peered":"2023-08-28T02:36:21.642760+0000","last_clean":"2023-08-28T02:36:21.642760+0000","last_became_active":"2023-08-28T02:32:59.644053+0000","last_became_peered":"2023-08-28T02:32:59.644053+0000","last_unstale":"2023-08-28T02:36:21.642760+0000","last_undegraded":"2023-08-28T02:36:21.642760+0000","last_fullsized":"2023-08-28T02:36:21.642760+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":204,"log_dups_size":0,"ondisk_log_size":204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:55:51.460483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1521,"num_read_kb":2511,"num_write":178,"num_write_kb":1175,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":331,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.666926+0000","last_change":"2023-08-28T02:28:25.352417+0000","last_active":"2023-08-28T02:36:21.666926+0000","last_peered":"2023-08-28T02:36:21.666926+0000","last_clean":"2023-08-28T02:36:21.666926+0000","last_became_active":"2023-08-28T02:28:25.352216+0000","last_became_peered":"2023-08-28T02:28:25.352216+0000","last_unstale":"2023-08-28T02:36:21.666926+0000","last_undegraded":"2023-08-28T02:36:21.666926+0000","last_fullsized":"2023-08-28T02:36:21.666926+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":57,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.642480+0000","last_change":"2023-08-28T02:32:58.720576+0000","last_active":"2023-08-28T02:36:21.642480+0000","last_peered":"2023-08-28T02:36:21.642480+0000","last_clean":"2023-08-28T02:36:21.642480+0000","last_became_active":"2023-08-28T02:32:58.719924+0000","last_became_peered":"2023-08-28T02:32:58.719924+0000","last_unstale":"2023-08-28T02:36:21.642480+0000","last_undegraded":"2023-08-28T02:36:21.642480+0000","last_fullsized":"2023-08-28T02:36:21.642480+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:28:50.697438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"337'5640","reported_seq":5668,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-08-28T02:36:43.820365+0000","last_change":"2023-08-28T02:35:14.503279+0000","last_active":"2023-08-28T02:36:43.820365+0000","last_peered":"2023-08-28T02:36:43.820365+0000","last_clean":"2023-08-28T02:36:43.820365+0000","last_became_active":"2023-08-28T02:35:14.502953+0000","last_became_peered":"2023-08-28T02:35:14.502953+0000","last_unstale":"2023-08-28T02:36:43.820365+0000","last_undegraded":"2023-08-28T02:36:43.820365+0000","last_fullsized":"2023-08-28T02:36:43.820365+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":5640,"log_dups_size":0,"ondisk_log_size":5640,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:55:29.284519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":369623040,"num_objects":5640,"num_object_clones":0,"num_object_copies":16920,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5640,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5640,"num_write_kb":360960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"78'47","reported_seq":556,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.641384+0000","last_change":"2023-08-28T02:24:05.799647+0000","last_active":"2023-08-28T02:36:21.641384+0000","last_peered":"2023-08-28T02:36:21.641384+0000","last_clean":"2023-08-28T02:36:21.641384+0000","last_became_active":"2023-08-28T02:23:59.210481+0000","last_became_peered":"2023-08-28T02:23:59.210481+0000","last_unstale":"2023-08-28T02:36:21.641384+0000","last_undegraded":"2023-08-28T02:36:21.641384+0000","last_fullsized":"2023-08-28T02:36:21.641384+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":47,"log_dups_size":0,"ondisk_log_size":47,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:40:18.542375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":162,"num_read_kb":1281,"num_write":48,"num_write_kb":1153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":57,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.641680+0000","last_change":"2023-08-28T02:32:58.622079+0000","last_active":"2023-08-28T02:36:21.641680+0000","last_peered":"2023-08-28T02:36:21.641680+0000","last_clean":"2023-08-28T02:36:21.641680+0000","last_became_active":"2023-08-28T02:32:58.621819+0000","last_became_peered":"2023-08-28T02:32:58.621819+0000","last_unstale":"2023-08-28T02:36:21.641680+0000","last_undegraded":"2023-08-28T02:36:21.641680+0000","last_fullsized":"2023-08-28T02:36:21.641680+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:03:36.676500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":57,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.642923+0000","last_change":"2023-08-28T02:32:58.724424+0000","last_active":"2023-08-28T02:36:21.642923+0000","last_peered":"2023-08-28T02:36:21.642923+0000","last_clean":"2023-08-28T02:36:21.642923+0000","last_became_active":"2023-08-28T02:32:58.724127+0000","last_became_peered":"2023-08-28T02:32:58.724127+0000","last_unstale":"2023-08-28T02:36:21.642923+0000","last_undegraded":"2023-08-28T02:36:21.642923+0000","last_fullsized":"2023-08-28T02:36:21.642923+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:02:25.178712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"337'5620","reported_seq":5647,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-08-28T02:36:42.226097+0000","last_change":"2023-08-28T02:35:14.500884+0000","last_active":"2023-08-28T02:36:42.226097+0000","last_peered":"2023-08-28T02:36:42.226097+0000","last_clean":"2023-08-28T02:36:42.226097+0000","last_became_active":"2023-08-28T02:35:14.500643+0000","last_became_peered":"2023-08-28T02:35:14.500643+0000","last_unstale":"2023-08-28T02:36:42.226097+0000","last_undegraded":"2023-08-28T02:36:42.226097+0000","last_fullsized":"2023-08-28T02:36:42.226097+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":5620,"log_dups_size":0,"ondisk_log_size":5620,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":368312320,"num_objects":5620,"num_object_clones":0,"num_object_copies":16860,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5620,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5620,"num_write_kb":359680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"78'114","reported_seq":390,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.641502+0000","last_change":"2023-08-28T02:35:05.992001+0000","last_active":"2023-08-28T02:36:21.641502+0000","last_peered":"2023-08-28T02:36:21.641502+0000","last_clean":"2023-08-28T02:36:21.641502+0000","last_became_active":"2023-08-28T02:35:05.991513+0000","last_became_peered":"2023-08-28T02:35:05.991513+0000","last_unstale":"2023-08-28T02:36:21.641502+0000","last_undegraded":"2023-08-28T02:36:21.641502+0000","last_fullsized":"2023-08-28T02:36:21.641502+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":114,"log_dups_size":0,"ondisk_log_size":114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:00:42.194692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":358,"num_read_kb":1352,"num_write":102,"num_write_kb":1104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"86'139","reported_seq":422,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.677134+0000","last_change":"2023-08-28T02:35:09.566656+0000","last_active":"2023-08-28T02:36:21.677134+0000","last_peered":"2023-08-28T02:36:21.677134+0000","last_clean":"2023-08-28T02:36:21.677134+0000","last_became_active":"2023-08-28T02:32:59.651089+0000","last_became_peered":"2023-08-28T02:32:59.651089+0000","last_unstale":"2023-08-28T02:36:21.677134+0000","last_undegraded":"2023-08-28T02:36:21.677134+0000","last_fullsized":"2023-08-28T02:36:21.677134+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_clean_scrub_stamp":"2023-08-28T02:08:02.043718+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:31:19.261767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":218,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:28.843383+0000","last_change":"2023-08-28T02:28:25.347600+0000","last_active":"2023-08-28T02:36:28.843383+0000","last_peered":"2023-08-28T02:36:28.843383+0000","last_clean":"2023-08-28T02:36:28.843383+0000","last_became_active":"2023-08-28T02:28:25.347237+0000","last_became_peered":"2023-08-28T02:28:25.347237+0000","last_unstale":"2023-08-28T02:36:28.843383+0000","last_undegraded":"2023-08-28T02:36:28.843383+0000","last_fullsized":"2023-08-28T02:36:28.843383+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:29:26.767559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":134,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.668827+0000","last_change":"2023-08-28T02:31:55.517732+0000","last_active":"2023-08-28T02:36:21.668827+0000","last_peered":"2023-08-28T02:36:21.668827+0000","last_clean":"2023-08-28T02:36:21.668827+0000","last_became_active":"2023-08-28T02:31:55.517347+0000","last_became_peered":"2023-08-28T02:31:55.517347+0000","last_unstale":"2023-08-28T02:36:21.668827+0000","last_undegraded":"2023-08-28T02:36:21.668827+0000","last_fullsized":"2023-08-28T02:36:21.668827+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"337'5524","reported_seq":5552,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-08-28T02:36:43.274300+0000","last_change":"2023-08-28T02:35:14.493898+0000","last_active":"2023-08-28T02:36:43.274300+0000","last_peered":"2023-08-28T02:36:43.274300+0000","last_clean":"2023-08-28T02:36:43.274300+0000","last_became_active":"2023-08-28T02:35:14.493649+0000","last_became_peered":"2023-08-28T02:35:14.493649+0000","last_unstale":"2023-08-28T02:36:43.274300+0000","last_undegraded":"2023-08-28T02:36:43.274300+0000","last_fullsized":"2023-08-28T02:36:43.274300+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":5524,"log_dups_size":0,"ondisk_log_size":5524,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:49:46.405255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362020864,"num_objects":5524,"num_object_clones":0,"num_object_copies":16572,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5524,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5524,"num_write_kb":353536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'148","reported_seq":792,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.641808+0000","last_change":"2023-08-28T02:35:10.122879+0000","last_active":"2023-08-28T02:36:21.641808+0000","last_peered":"2023-08-28T02:36:21.641808+0000","last_clean":"2023-08-28T02:36:21.641808+0000","last_became_active":"2023-08-28T02:32:59.661273+0000","last_became_peered":"2023-08-28T02:32:59.661273+0000","last_unstale":"2023-08-28T02:36:21.641808+0000","last_undegraded":"2023-08-28T02:36:21.641808+0000","last_fullsized":"2023-08-28T02:36:21.641808+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:07:46.974854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1092,"num_read_kb":1955,"num_write":127,"num_write_kb":5177,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":308,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.675889+0000","last_change":"2023-08-28T02:23:59.226986+0000","last_active":"2023-08-28T02:36:21.675889+0000","last_peered":"2023-08-28T02:36:21.675889+0000","last_clean":"2023-08-28T02:36:21.675889+0000","last_became_active":"2023-08-28T02:23:59.226529+0000","last_became_peered":"2023-08-28T02:23:59.226529+0000","last_unstale":"2023-08-28T02:36:21.675889+0000","last_undegraded":"2023-08-28T02:36:21.675889+0000","last_fullsized":"2023-08-28T02:36:21.675889+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:22:09.193619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":371,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.675056+0000","last_change":"2023-08-28T02:32:58.744332+0000","last_active":"2023-08-28T02:36:21.675056+0000","last_peered":"2023-08-28T02:36:21.675056+0000","last_clean":"2023-08-28T02:36:21.675056+0000","last_became_active":"2023-08-28T02:32:58.744025+0000","last_became_peered":"2023-08-28T02:32:58.744025+0000","last_unstale":"2023-08-28T02:36:21.675056+0000","last_undegraded":"2023-08-28T02:36:21.675056+0000","last_fullsized":"2023-08-28T02:36:21.675056+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:41:15.957049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"337'5686","reported_seq":5872,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-08-28T02:36:43.821699+0000","last_change":"2023-08-28T02:35:24.994481+0000","last_active":"2023-08-28T02:36:43.821699+0000","last_peered":"2023-08-28T02:36:43.821699+0000","last_clean":"2023-08-28T02:36:43.821699+0000","last_became_active":"2023-08-28T02:35:24.668942+0000","last_became_peered":"2023-08-28T02:35:24.668942+0000","last_unstale":"2023-08-28T02:36:43.821699+0000","last_undegraded":"2023-08-28T02:36:43.821699+0000","last_fullsized":"2023-08-28T02:36:43.821699+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":5686,"log_dups_size":0,"ondisk_log_size":5686,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:16:02.211038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":372637696,"num_objects":5686,"num_object_clones":0,"num_object_copies":17058,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5686,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5686,"num_write_kb":363904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":73,"num_bytes_recovered":4784128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"78'103","reported_seq":668,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.675203+0000","last_change":"2023-08-28T02:35:09.798248+0000","last_active":"2023-08-28T02:36:21.675203+0000","last_peered":"2023-08-28T02:36:21.675203+0000","last_clean":"2023-08-28T02:36:21.675203+0000","last_became_active":"2023-08-28T02:32:59.656276+0000","last_became_peered":"2023-08-28T02:32:59.656276+0000","last_unstale":"2023-08-28T02:36:21.675203+0000","last_undegraded":"2023-08-28T02:36:21.675203+0000","last_fullsized":"2023-08-28T02:36:21.675203+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:34:10.622592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":803,"num_read_kb":2080,"num_write":73,"num_write_kb":1244,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":57,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.642386+0000","last_change":"2023-08-28T02:32:58.720401+0000","last_active":"2023-08-28T02:36:21.642386+0000","last_peered":"2023-08-28T02:36:21.642386+0000","last_clean":"2023-08-28T02:36:21.642386+0000","last_became_active":"2023-08-28T02:32:58.719518+0000","last_became_peered":"2023-08-28T02:32:58.719518+0000","last_unstale":"2023-08-28T02:36:21.642386+0000","last_undegraded":"2023-08-28T02:36:21.642386+0000","last_fullsized":"2023-08-28T02:36:21.642386+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:46:28.947812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":321,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.674969+0000","last_change":"2023-08-28T02:32:58.887691+0000","last_active":"2023-08-28T02:36:21.674969+0000","last_peered":"2023-08-28T02:36:21.674969+0000","last_clean":"2023-08-28T02:36:21.674969+0000","last_became_active":"2023-08-28T02:32:58.886906+0000","last_became_peered":"2023-08-28T02:32:58.886906+0000","last_unstale":"2023-08-28T02:36:21.674969+0000","last_undegraded":"2023-08-28T02:36:21.674969+0000","last_fullsized":"2023-08-28T02:36:21.674969+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:39:29.229171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"337'5534","reported_seq":5562,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-08-28T02:36:44.556574+0000","last_change":"2023-08-28T02:35:14.497565+0000","last_active":"2023-08-28T02:36:44.556574+0000","last_peered":"2023-08-28T02:36:44.556574+0000","last_clean":"2023-08-28T02:36:44.556574+0000","last_became_active":"2023-08-28T02:35:14.497285+0000","last_became_peered":"2023-08-28T02:35:14.497285+0000","last_unstale":"2023-08-28T02:36:44.556574+0000","last_undegraded":"2023-08-28T02:36:44.556574+0000","last_fullsized":"2023-08-28T02:36:44.556574+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":5534,"log_dups_size":0,"ondisk_log_size":5534,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:04:31.778861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362676224,"num_objects":5534,"num_object_clones":0,"num_object_copies":16602,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5534,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5534,"num_write_kb":354176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":216,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.667711+0000","last_change":"2023-08-28T02:31:55.535099+0000","last_active":"2023-08-28T02:36:21.667711+0000","last_peered":"2023-08-28T02:36:21.667711+0000","last_clean":"2023-08-28T02:36:21.667711+0000","last_became_active":"2023-08-28T02:31:55.527768+0000","last_became_peered":"2023-08-28T02:31:55.527768+0000","last_unstale":"2023-08-28T02:36:21.667711+0000","last_undegraded":"2023-08-28T02:36:21.667711+0000","last_fullsized":"2023-08-28T02:36:21.667711+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:43:30.804793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'112","reported_seq":490,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.642523+0000","last_change":"2023-08-28T02:32:59.643465+0000","last_active":"2023-08-28T02:36:21.642523+0000","last_peered":"2023-08-28T02:36:21.642523+0000","last_clean":"2023-08-28T02:36:21.642523+0000","last_became_active":"2023-08-28T02:32:59.642979+0000","last_became_peered":"2023-08-28T02:32:59.642979+0000","last_unstale":"2023-08-28T02:36:21.642523+0000","last_undegraded":"2023-08-28T02:36:21.642523+0000","last_fullsized":"2023-08-28T02:36:21.642523+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:48:45.923015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":444,"num_read_kb":473,"num_write":94,"num_write_kb":127,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":227,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.669796+0000","last_change":"2023-08-28T02:32:58.833408+0000","last_active":"2023-08-28T02:36:21.669796+0000","last_peered":"2023-08-28T02:36:21.669796+0000","last_clean":"2023-08-28T02:36:21.669796+0000","last_became_active":"2023-08-28T02:32:58.821347+0000","last_became_peered":"2023-08-28T02:32:58.821347+0000","last_unstale":"2023-08-28T02:36:21.669796+0000","last_undegraded":"2023-08-28T02:36:21.669796+0000","last_fullsized":"2023-08-28T02:36:21.669796+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:41:25.683179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"337'5615","reported_seq":5643,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-08-28T02:36:44.963320+0000","last_change":"2023-08-28T02:35:14.495405+0000","last_active":"2023-08-28T02:36:44.963320+0000","last_peered":"2023-08-28T02:36:44.963320+0000","last_clean":"2023-08-28T02:36:44.963320+0000","last_became_active":"2023-08-28T02:35:14.494591+0000","last_became_peered":"2023-08-28T02:35:14.494591+0000","last_unstale":"2023-08-28T02:36:44.963320+0000","last_undegraded":"2023-08-28T02:36:44.963320+0000","last_fullsized":"2023-08-28T02:36:44.963320+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":5615,"log_dups_size":0,"ondisk_log_size":5615,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:47:27.839376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367984640,"num_objects":5615,"num_object_clones":0,"num_object_copies":16845,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5615,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5615,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"78'122","reported_seq":680,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.668741+0000","last_change":"2023-08-28T02:33:00.672889+0000","last_active":"2023-08-28T02:36:21.668741+0000","last_peered":"2023-08-28T02:36:21.668741+0000","last_clean":"2023-08-28T02:36:21.668741+0000","last_became_active":"2023-08-28T02:33:00.672494+0000","last_became_peered":"2023-08-28T02:33:00.672494+0000","last_unstale":"2023-08-28T02:36:21.668741+0000","last_undegraded":"2023-08-28T02:36:21.668741+0000","last_fullsized":"2023-08-28T02:36:21.668741+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":857,"num_read_kb":2224,"num_write":99,"num_write_kb":2312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":346,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.675292+0000","last_change":"2023-08-28T02:32:58.795995+0000","last_active":"2023-08-28T02:36:21.675292+0000","last_peered":"2023-08-28T02:36:21.675292+0000","last_clean":"2023-08-28T02:36:21.675292+0000","last_became_active":"2023-08-28T02:32:58.771872+0000","last_became_peered":"2023-08-28T02:32:58.771872+0000","last_unstale":"2023-08-28T02:36:21.675292+0000","last_undegraded":"2023-08-28T02:36:21.675292+0000","last_fullsized":"2023-08-28T02:36:21.675292+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:16:02.965467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":306,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.665990+0000","last_change":"2023-08-28T02:28:25.351840+0000","last_active":"2023-08-28T02:36:21.665990+0000","last_peered":"2023-08-28T02:36:21.665990+0000","last_clean":"2023-08-28T02:36:21.665990+0000","last_became_active":"2023-08-28T02:28:25.351399+0000","last_became_peered":"2023-08-28T02:28:25.351399+0000","last_unstale":"2023-08-28T02:36:21.665990+0000","last_undegraded":"2023-08-28T02:36:21.665990+0000","last_fullsized":"2023-08-28T02:36:21.665990+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":323,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.674881+0000","last_change":"2023-08-28T02:32:58.751818+0000","last_active":"2023-08-28T02:36:21.674881+0000","last_peered":"2023-08-28T02:36:21.674881+0000","last_clean":"2023-08-28T02:36:21.674881+0000","last_became_active":"2023-08-28T02:32:58.743388+0000","last_became_peered":"2023-08-28T02:32:58.743388+0000","last_unstale":"2023-08-28T02:36:21.674881+0000","last_undegraded":"2023-08-28T02:36:21.674881+0000","last_fullsized":"2023-08-28T02:36:21.674881+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:52:12.910174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":24,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.642066+0000","last_change":"2023-08-28T02:35:24.666417+0000","last_active":"2023-08-28T02:36:21.642066+0000","last_peered":"2023-08-28T02:36:21.642066+0000","last_clean":"2023-08-28T02:36:21.642066+0000","last_became_active":"2023-08-28T02:35:24.666107+0000","last_became_peered":"2023-08-28T02:35:24.666107+0000","last_unstale":"2023-08-28T02:36:21.642066+0000","last_undegraded":"2023-08-28T02:36:21.642066+0000","last_fullsized":"2023-08-28T02:36:21.642066+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:22:55.497120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":169,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.667455+0000","last_change":"2023-08-28T02:32:58.839847+0000","last_active":"2023-08-28T02:36:21.667455+0000","last_peered":"2023-08-28T02:36:21.667455+0000","last_clean":"2023-08-28T02:36:21.667455+0000","last_became_active":"2023-08-28T02:32:58.628159+0000","last_became_peered":"2023-08-28T02:32:58.628159+0000","last_unstale":"2023-08-28T02:36:21.667455+0000","last_undegraded":"2023-08-28T02:36:21.667455+0000","last_fullsized":"2023-08-28T02:36:21.667455+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":372,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.667013+0000","last_change":"2023-08-28T02:32:58.629596+0000","last_active":"2023-08-28T02:36:21.667013+0000","last_peered":"2023-08-28T02:36:21.667013+0000","last_clean":"2023-08-28T02:36:21.667013+0000","last_became_active":"2023-08-28T02:32:58.629136+0000","last_became_peered":"2023-08-28T02:32:58.629136+0000","last_unstale":"2023-08-28T02:36:21.667013+0000","last_undegraded":"2023-08-28T02:36:21.667013+0000","last_fullsized":"2023-08-28T02:36:21.667013+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":156,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.667909+0000","last_change":"2023-08-28T02:28:25.705331+0000","last_active":"2023-08-28T02:36:21.667909+0000","last_peered":"2023-08-28T02:36:21.667909+0000","last_clean":"2023-08-28T02:36:21.667909+0000","last_became_active":"2023-08-28T02:28:25.397480+0000","last_became_peered":"2023-08-28T02:28:25.397480+0000","last_unstale":"2023-08-28T02:36:21.667909+0000","last_undegraded":"2023-08-28T02:36:21.667909+0000","last_fullsized":"2023-08-28T02:36:21.667909+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":336,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.669869+0000","last_change":"2023-08-28T02:23:59.207929+0000","last_active":"2023-08-28T02:36:21.669869+0000","last_peered":"2023-08-28T02:36:21.669869+0000","last_clean":"2023-08-28T02:36:21.669869+0000","last_became_active":"2023-08-28T02:23:59.207589+0000","last_became_peered":"2023-08-28T02:23:59.207589+0000","last_unstale":"2023-08-28T02:36:21.669869+0000","last_undegraded":"2023-08-28T02:36:21.669869+0000","last_fullsized":"2023-08-28T02:36:21.669869+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":227,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.667895+0000","last_change":"2023-08-28T02:28:25.347988+0000","last_active":"2023-08-28T02:36:21.667895+0000","last_peered":"2023-08-28T02:36:21.667895+0000","last_clean":"2023-08-28T02:36:21.667895+0000","last_became_active":"2023-08-28T02:28:25.347632+0000","last_became_peered":"2023-08-28T02:28:25.347632+0000","last_unstale":"2023-08-28T02:36:21.667895+0000","last_undegraded":"2023-08-28T02:36:21.667895+0000","last_fullsized":"2023-08-28T02:36:21.667895+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:56:20.202662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":199,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.667241+0000","last_change":"2023-08-28T02:23:59.217794+0000","last_active":"2023-08-28T02:36:21.667241+0000","last_peered":"2023-08-28T02:36:21.667241+0000","last_clean":"2023-08-28T02:36:21.667241+0000","last_became_active":"2023-08-28T02:23:59.217231+0000","last_became_peered":"2023-08-28T02:23:59.217231+0000","last_unstale":"2023-08-28T02:36:21.667241+0000","last_undegraded":"2023-08-28T02:36:21.667241+0000","last_fullsized":"2023-08-28T02:36:21.667241+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:36:26.297410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":324,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.675397+0000","last_change":"2023-08-28T02:35:24.680274+0000","last_active":"2023-08-28T02:36:21.675397+0000","last_peered":"2023-08-28T02:36:21.675397+0000","last_clean":"2023-08-28T02:36:21.675397+0000","last_became_active":"2023-08-28T02:35:24.673318+0000","last_became_peered":"2023-08-28T02:35:24.673318+0000","last_unstale":"2023-08-28T02:36:21.675397+0000","last_undegraded":"2023-08-28T02:36:21.675397+0000","last_fullsized":"2023-08-28T02:36:21.675397+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:22:46.371303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":395,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-08-28T02:36:21.665918+0000","last_change":"2023-08-28T02:32:58.630270+0000","last_active":"2023-08-28T02:36:21.665918+0000","last_peered":"2023-08-28T02:36:21.665918+0000","last_clean":"2023-08-28T02:36:21.665918+0000","last_became_active":"2023-08-28T02:32:58.629958+0000","last_became_peered":"2023-08-28T02:32:58.629958+0000","last_unstale":"2023-08-28T02:36:21.665918+0000","last_undegraded":"2023-08-28T02:36:21.665918+0000","last_fullsized":"2023-08-28T02:36:21.665918+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":62,"num_bytes_recovered":25254,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":21462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26810,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":5895553024,"num_objects":89959,"num_object_clones":0,"num_object_copies":269877,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":89959,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":89959,"num_write_kb":5757376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":73,"num_bytes_recovered":4784128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17660182528,"data_stored":17660182528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":89959,"ondisk_log_size":89959,"up":48,"acting":48,"num_store_stats":7},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":156,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16789504,"data_stored":16777273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":1335734829079,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626760,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2447572,"kb_used_data":2146252,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":91297580,"statfs":{"total":95995035648,"available":93488721920,"internally_reserved":0,"allocated":2197762048,"data_stored":2196881287,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:36:20 2023","interfaces":[{"interface":"back","average":{"1min":1.0980000000000001,"5min":3.0529999999999999,"15min":3.6560000000000001},"min":{"1min":0.23000000000000001,"5min":0.215,"15min":0.20799999999999999},"max":{"1min":3.5760000000000001,"5min":121.78,"15min":121.78},"last":0.25800000000000001},{"interface":"front","average":{"1min":1.254,"5min":3.6309999999999998,"15min":3.7549999999999999},"min":{"1min":0.29999999999999999,"5min":0.23699999999999999,"15min":0.23200000000000001},"max":{"1min":7.4370000000000003,"5min":121.79900000000001,"15min":121.79900000000001},"last":2.1669999999999998}]},{"osd":1,"last update":"Mon Aug 28 02:36:20 2023","interfaces":[{"interface":"back","average":{"1min":1.373,"5min":1.2589999999999999,"15min":2.6880000000000002},"min":{"1min":0.315,"5min":0.23799999999999999,"15min":0.23699999999999999},"max":{"1min":4.1959999999999997,"5min":8.2639999999999993,"15min":141.87299999999999},"last":1.3240000000000001},{"interface":"front","average":{"1min":1.5640000000000001,"5min":1.484,"15min":2.8450000000000002},"min":{"1min":0.25,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":4.8440000000000003,"5min":8.5180000000000007,"15min":141.904},"last":0.77300000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:36:04 2023","interfaces":[{"interface":"back","average":{"1min":5.5019999999999998,"5min":5.6580000000000004,"15min":4.173},"min":{"1min":0.34899999999999998,"5min":0.29499999999999998,"15min":0.27500000000000002},"max":{"1min":91.641000000000005,"5min":234.876,"15min":234.876},"last":1.274},{"interface":"front","average":{"1min":5.4139999999999997,"5min":5.4320000000000004,"15min":4.1130000000000004},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":91.655000000000001,"5min":234.482,"15min":234.482},"last":2.0449999999999999}]},{"osd":3,"last update":"Mon Aug 28 02:36:20 2023","interfaces":[{"interface":"back","average":{"1min":1.8480000000000001,"5min":3.3069999999999999,"15min":3.2069999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":7.2750000000000004,"5min":143.19499999999999,"15min":157.625},"last":2.3889999999999998},{"interface":"front","average":{"1min":2.1070000000000002,"5min":3.3079999999999998,"15min":3.2149999999999999},"min":{"1min":0.38200000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":7.2869999999999999,"5min":143.20699999999999,"15min":157.637},"last":2.3980000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:36:25 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.50600000000000001,"15min":1.103},"min":{"1min":0.28999999999999998,"5min":0.251,"15min":0.251},"max":{"1min":1.645,"5min":3.8610000000000002,"15min":18.308},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.73199999999999998,"5min":0.68799999999999994,"15min":1.2150000000000001},"min":{"1min":0.28100000000000003,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":2.1019999999999999,"5min":3.8450000000000002,"15min":18.263000000000002},"last":0.71699999999999997}]},{"osd":5,"last update":"Mon Aug 28 02:36:25 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.61099999999999999,"15min":1.1870000000000001},"min":{"1min":0.29699999999999999,"5min":0.29299999999999998,"15min":0.26300000000000001},"max":{"1min":2.8639999999999999,"5min":4.2800000000000002,"15min":20.759},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.52700000000000002,"15min":0.997},"min":{"1min":0.32100000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":1.399,"5min":2.972,"15min":7.3780000000000001},"last":0.502}]},{"osd":7,"last update":"Mon Aug 28 02:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":1.3180000000000001,"5min":1.3180000000000001,"15min":1.3180000000000001},"last":0.751},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.47599999999999998}]}]},{"osd":1,"up_from":98,"seq":420906795213,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3017360,"kb_used_data":2510920,"kb_used_omap":7,"kb_used_meta":506360,"kb_avail":90727792,"statfs":{"total":95995035648,"available":92905259008,"internally_reserved":0,"allocated":2571182080,"data_stored":2569446176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":518512971},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,10],"upper_bound":512},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:36:18 2023","interfaces":[{"interface":"back","average":{"1min":3.3809999999999998,"5min":6.6779999999999999,"15min":4.4089999999999998},"min":{"1min":0.23799999999999999,"5min":0.189,"15min":0.17499999999999999},"max":{"1min":36.838000000000001,"5min":129.346,"15min":129.346},"last":4.327},{"interface":"front","average":{"1min":2.9279999999999999,"5min":6.6210000000000004,"15min":4.4329999999999998},"min":{"1min":0.19800000000000001,"5min":0.19700000000000001,"15min":0.188},"max":{"1min":36.622,"5min":120.774,"15min":128.685},"last":1.5089999999999999}]},{"osd":2,"last update":"Mon Aug 28 02:35:43 2023","interfaces":[{"interface":"back","average":{"1min":2.2269999999999999,"5min":3.883,"15min":4.1429999999999998},"min":{"1min":0.218,"5min":0.189,"15min":0.189},"max":{"1min":10.818,"5min":119.276,"15min":135.25999999999999},"last":0.91400000000000003},{"interface":"front","average":{"1min":1.0549999999999999,"5min":3.492,"15min":3.8100000000000001},"min":{"1min":0.221,"5min":0.218,"15min":0.218},"max":{"1min":6.8879999999999999,"5min":121.849,"15min":135.10300000000001},"last":11.074}]},{"osd":3,"last update":"Mon Aug 28 02:36:18 2023","interfaces":[{"interface":"back","average":{"1min":1.6839999999999999,"5min":7.7859999999999996,"15min":4.4699999999999998},"min":{"1min":0.21299999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":4.29,"5min":126.95699999999999,"15min":126.95699999999999},"last":4.665},{"interface":"front","average":{"1min":2.141,"5min":7.4969999999999999,"15min":4.5890000000000004},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":10.477,"5min":128.00200000000001,"15min":128.00200000000001},"last":2.9329999999999998}]},{"osd":4,"last update":"Mon Aug 28 02:36:18 2023","interfaces":[{"interface":"back","average":{"1min":2.7170000000000001,"5min":1.7410000000000001,"15min":1.6899999999999999},"min":{"1min":0.29699999999999999,"5min":0.28199999999999997,"15min":0.27500000000000002},"max":{"1min":24.809999999999999,"5min":24.809999999999999,"15min":24.809999999999999},"last":0.97199999999999998},{"interface":"front","average":{"1min":2.403,"5min":1.546,"15min":1.484},"min":{"1min":0.378,"5min":0.29499999999999998,"15min":0.25600000000000001},"max":{"1min":24.835999999999999,"5min":24.835999999999999,"15min":24.835999999999999},"last":4.2220000000000004}]},{"osd":5,"last update":"Mon Aug 28 02:36:18 2023","interfaces":[{"interface":"back","average":{"1min":8.0570000000000004,"5min":2.9809999999999999,"15min":2.2010000000000001},"min":{"1min":0.34399999999999997,"5min":0.252,"15min":0.252},"max":{"1min":82.722999999999999,"5min":82.722999999999999,"15min":82.722999999999999},"last":1.431},{"interface":"front","average":{"1min":8.4969999999999999,"5min":2.9540000000000002,"15min":2.081},"min":{"1min":0.32900000000000001,"5min":0.307,"15min":0.307},"max":{"1min":82.733999999999995,"5min":82.733999999999995,"15min":82.733999999999995},"last":0.94399999999999995}]},{"osd":6,"last update":"Mon Aug 28 02:36:18 2023","interfaces":[{"interface":"back","average":{"1min":3.024,"5min":2.0230000000000001,"15min":1.994},"min":{"1min":0.41999999999999998,"5min":0.311,"15min":0.311},"max":{"1min":19.527000000000001,"5min":19.527000000000001,"15min":19.527000000000001},"last":1.4630000000000001},{"interface":"front","average":{"1min":3.4969999999999999,"5min":2.1259999999999999,"15min":1.9550000000000001},"min":{"1min":0.38,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":19.516999999999999,"5min":19.516999999999999,"15min":19.516999999999999},"last":1.1859999999999999}]},{"osd":7,"last update":"Mon Aug 28 02:35:57 2023","interfaces":[{"interface":"back","average":{"1min":0.91100000000000003,"5min":0.91100000000000003,"15min":0.91100000000000003},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":2.7829999999999999,"5min":2.7829999999999999,"15min":2.7829999999999999},"last":1.488},{"interface":"front","average":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":3.4239999999999999,"5min":3.4239999999999999,"15min":3.4239999999999999},"last":4.2590000000000003}]}]},{"osd":0,"up_from":90,"seq":386547056859,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3531620,"kb_used_data":2903900,"kb_used_omap":8,"kb_used_meta":627703,"kb_avail":90213532,"statfs":{"total":95995035648,"available":92378656768,"internally_reserved":0,"allocated":2973593600,"data_stored":2972013892,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":642768233},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Aug 28 02:36:27 2023","interfaces":[{"interface":"back","average":{"1min":2.4620000000000002,"5min":2.7480000000000002,"15min":2.7549999999999999},"min":{"1min":0.16500000000000001,"5min":0.16500000000000001,"15min":0.16500000000000001},"max":{"1min":15.728,"5min":61.932000000000002,"15min":61.932000000000002},"last":0.39900000000000002},{"interface":"front","average":{"1min":3.3849999999999998,"5min":3.98,"15min":3.2570000000000001},"min":{"1min":0.28599999999999998,"5min":0.19500000000000001,"15min":0.17000000000000001},"max":{"1min":16.443999999999999,"5min":62.061999999999998,"15min":62.061999999999998},"last":0.38400000000000001}]},{"osd":2,"last update":"Mon Aug 28 02:35:45 2023","interfaces":[{"interface":"back","average":{"1min":2.5110000000000001,"5min":4.5430000000000001,"15min":3.5990000000000002},"min":{"1min":0.22900000000000001,"5min":0.20599999999999999,"15min":0.19500000000000001},"max":{"1min":19.416,"5min":86.269000000000005,"15min":100.718},"last":3.2759999999999998},{"interface":"front","average":{"1min":3.8809999999999998,"5min":5.5979999999999999,"15min":4.1520000000000001},"min":{"1min":0.26200000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":26.536000000000001,"5min":86.353999999999999,"15min":100.962},"last":0.52000000000000002}]},{"osd":3,"last update":"Mon Aug 28 02:36:33 2023","interfaces":[{"interface":"back","average":{"1min":2.4159999999999999,"5min":6.1790000000000003,"15min":4.9260000000000002},"min":{"1min":0.40999999999999998,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":9.1319999999999997,"5min":126.896,"15min":237.43899999999999},"last":0.61799999999999999},{"interface":"front","average":{"1min":1.833,"5min":6.0460000000000003,"15min":4.8179999999999996},"min":{"1min":0.308,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":4.9370000000000003,"5min":127.01600000000001,"15min":237.48099999999999},"last":0.499}]},{"osd":4,"last update":"Mon Aug 28 02:36:23 2023","interfaces":[{"interface":"back","average":{"1min":1.796,"5min":1.5229999999999999,"15min":1.6539999999999999},"min":{"1min":0.42099999999999999,"5min":0.252,"15min":0.23999999999999999},"max":{"1min":14.209,"5min":14.209,"15min":14.209},"last":0.36099999999999999},{"interface":"front","average":{"1min":2.0649999999999999,"5min":2.0310000000000001,"15min":1.956},"min":{"1min":0.34000000000000002,"5min":0.31,"15min":0.309},"max":{"1min":14.241,"5min":14.241,"15min":27.271999999999998},"last":0.53200000000000003}]},{"osd":5,"last update":"Mon Aug 28 02:36:23 2023","interfaces":[{"interface":"back","average":{"1min":1.47,"5min":1.956,"15min":2.0339999999999998},"min":{"1min":0.38100000000000001,"5min":0.29899999999999999,"15min":0.29599999999999999},"max":{"1min":3.827,"5min":10.779999999999999,"15min":44.000999999999998},"last":0.56299999999999994},{"interface":"front","average":{"1min":1.2210000000000001,"5min":1.486,"15min":1.8320000000000001},"min":{"1min":0.45500000000000002,"5min":0.307,"15min":0.307},"max":{"1min":3.2170000000000001,"5min":9.0310000000000006,"15min":43.953000000000003},"last":0.32800000000000001}]},{"osd":6,"last update":"Mon Aug 28 02:36:23 2023","interfaces":[{"interface":"back","average":{"1min":1.3819999999999999,"5min":1.9970000000000001,"15min":2.113},"min":{"1min":0.36199999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":3.8610000000000002,"5min":10.823,"15min":32.417999999999999},"last":0.58899999999999997},{"interface":"front","average":{"1min":1.355,"5min":1.51,"15min":1.8799999999999999},"min":{"1min":0.52500000000000002,"5min":0.32900000000000001,"15min":0.30399999999999999},"max":{"1min":3.266,"5min":9.0009999999999994,"15min":32.130000000000003},"last":0.434}]},{"osd":7,"last update":"Mon Aug 28 02:35:57 2023","interfaces":[{"interface":"back","average":{"1min":1.147,"5min":1.147,"15min":1.147},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":3.242,"5min":3.242,"15min":3.242},"last":0.46100000000000002},{"interface":"front","average":{"1min":1.9259999999999999,"5min":1.9259999999999999,"15min":1.9259999999999999},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":16.529,"5min":16.529,"15min":16.529},"last":0.45200000000000001}]}]},{"osd":2,"up_from":104,"seq":446676598983,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3159244,"kb_used_data":2544516,"kb_used_omap":10,"kb_used_meta":614709,"kb_avail":90585908,"statfs":{"total":95995035648,"available":92759969792,"internally_reserved":0,"allocated":2605584384,"data_stored":2604027108,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10792,"internal_metadata":629462488},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,2,3,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:35:49 2023","interfaces":[{"interface":"back","average":{"1min":2.0600000000000001,"5min":7.1950000000000003,"15min":4.5519999999999996},"min":{"1min":0.21099999999999999,"5min":0.191,"15min":0.17599999999999999},"max":{"1min":17.791,"5min":185.93799999999999,"15min":185.93799999999999},"last":2.0640000000000001},{"interface":"front","average":{"1min":1.647,"5min":6.6210000000000004,"15min":5.5579999999999998},"min":{"1min":0.252,"5min":0.22,"15min":0.19900000000000001},"max":{"1min":6.569,"5min":187.90700000000001,"15min":187.90700000000001},"last":0.59299999999999997}]},{"osd":1,"last update":"Mon Aug 28 02:35:49 2023","interfaces":[{"interface":"back","average":{"1min":4.702,"5min":2.625,"15min":2.4940000000000002},"min":{"1min":0.23100000000000001,"5min":0.221,"15min":0.20999999999999999},"max":{"1min":66.010999999999996,"5min":66.010999999999996,"15min":66.010999999999996},"last":0.27600000000000002},{"interface":"front","average":{"1min":4.0510000000000002,"5min":2.4540000000000002,"15min":2.395},"min":{"1min":0.21299999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":63.770000000000003,"5min":63.770000000000003,"15min":63.770000000000003},"last":8.1519999999999992}]},{"osd":3,"last update":"Mon Aug 28 02:36:23 2023","interfaces":[{"interface":"back","average":{"1min":4.8390000000000004,"5min":5.6470000000000002,"15min":4.3280000000000003},"min":{"1min":0.23499999999999999,"5min":0.23100000000000001,"15min":0.22700000000000001},"max":{"1min":57.338000000000001,"5min":134.07599999999999,"15min":134.07599999999999},"last":15.109999999999999},{"interface":"front","average":{"1min":4.6769999999999996,"5min":5.5549999999999997,"15min":4.2789999999999999},"min":{"1min":0.29599999999999999,"5min":0.29299999999999998,"15min":0.223},"max":{"1min":57.375999999999998,"5min":134.172,"15min":134.172},"last":8.2400000000000002}]},{"osd":4,"last update":"Mon Aug 28 02:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.83599999999999997,"5min":1.319,"15min":2.903},"min":{"1min":0.29999999999999999,"5min":0.26800000000000002,"15min":0.26600000000000001},"max":{"1min":3.391,"5min":13.273,"15min":207.34100000000001},"last":2.3170000000000002},{"interface":"front","average":{"1min":1.6419999999999999,"5min":2.0910000000000002,"15min":2.8940000000000001},"min":{"1min":0.33800000000000002,"5min":0.309,"15min":0.27400000000000002},"max":{"1min":17.768999999999998,"5min":22.303999999999998,"15min":207.352},"last":2.1419999999999999}]},{"osd":5,"last update":"Mon Aug 28 02:35:49 2023","interfaces":[{"interface":"back","average":{"1min":1.1160000000000001,"5min":1.5429999999999999,"15min":2.847},"min":{"1min":0.36199999999999999,"5min":0.315,"15min":0.29299999999999998},"max":{"1min":3.968,"5min":13.335000000000001,"15min":128.696},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.76500000000000001,"5min":1.389,"15min":2.871},"min":{"1min":0.34599999999999997,"5min":0.30399999999999999,"15min":0.30299999999999999},"max":{"1min":2.0539999999999998,"5min":13.303000000000001,"15min":128.84100000000001},"last":3.1480000000000001}]},{"osd":6,"last update":"Mon Aug 28 02:35:49 2023","interfaces":[{"interface":"back","average":{"1min":1.6359999999999999,"5min":2.2120000000000002,"15min":1.887},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.29299999999999998},"max":{"1min":17.719000000000001,"5min":22.25,"15min":22.25},"last":2.1099999999999999},{"interface":"front","average":{"1min":1.081,"5min":1.466,"15min":1.7769999999999999},"min":{"1min":0.35599999999999998,"5min":0.30199999999999999,"15min":0.29399999999999998},"max":{"1min":3.4390000000000001,"5min":13.311,"15min":13.311},"last":0.47799999999999998}]},{"osd":7,"last update":"Mon Aug 28 02:35:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":1.0189999999999999,"15min":1.0189999999999999},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":2.8559999999999999,"5min":2.8559999999999999,"15min":2.8559999999999999},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":3.3410000000000002,"5min":3.3410000000000002,"15min":3.3410000000000002},"last":8.2050000000000001}]}]},{"osd":3,"up_from":112,"seq":481036337345,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2721472,"kb_used_data":2158392,"kb_used_omap":3,"kb_used_meta":563068,"kb_avail":91023680,"statfs":{"total":95995035648,"available":93208248320,"internally_reserved":0,"allocated":2210193408,"data_stored":2208640948,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":576582562},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:36:17 2023","interfaces":[{"interface":"back","average":{"1min":3.1669999999999998,"5min":5.1580000000000004,"15min":4.7169999999999996},"min":{"1min":0.20799999999999999,"5min":0.19500000000000001,"15min":0.193},"max":{"1min":24.561,"5min":138.39699999999999,"15min":164.25299999999999},"last":1.8480000000000001},{"interface":"front","average":{"1min":2.0640000000000001,"5min":2.9079999999999999,"15min":3.9169999999999998},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.192},"max":{"1min":24.048999999999999,"5min":24.048999999999999,"15min":164.55000000000001},"last":2.5529999999999999}]},{"osd":1,"last update":"Mon Aug 28 02:36:17 2023","interfaces":[{"interface":"back","average":{"1min":7.6219999999999999,"5min":4.0919999999999996,"15min":3.1669999999999998},"min":{"1min":0.222,"5min":0.222,"15min":0.20699999999999999},"max":{"1min":105.04600000000001,"5min":105.04600000000001,"15min":117.34699999999999},"last":0.33000000000000002},{"interface":"front","average":{"1min":7.6379999999999999,"5min":5.7039999999999997,"15min":3.8340000000000001},"min":{"1min":0.29099999999999998,"5min":0.23599999999999999,"15min":0.224},"max":{"1min":105.55800000000001,"5min":143.54400000000001,"15min":143.54400000000001},"last":1.8220000000000001}]},{"osd":2,"last update":"Mon Aug 28 02:36:17 2023","interfaces":[{"interface":"back","average":{"1min":2.052,"5min":5.3099999999999996,"15min":4.5599999999999996},"min":{"1min":0.26700000000000002,"5min":0.20100000000000001,"15min":0.19900000000000001},"max":{"1min":9.9789999999999992,"5min":143.50299999999999,"15min":143.50299999999999},"last":1.915},{"interface":"front","average":{"1min":1.583,"5min":2.9039999999999999,"15min":3.6589999999999998},"min":{"1min":0.23699999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":9.9120000000000008,"5min":12.646000000000001,"15min":120.44499999999999},"last":2.504}]},{"osd":4,"last update":"Mon Aug 28 02:36:17 2023","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":1.5620000000000001,"15min":1.827},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":2.8980000000000001,"5min":7.8940000000000001,"15min":58.067999999999998},"last":2.653},{"interface":"front","average":{"1min":1.101,"5min":1.7609999999999999,"15min":1.9390000000000001},"min":{"1min":0.38400000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":2.8519999999999999,"5min":10.613,"15min":58.088000000000001},"last":0.39700000000000002}]},{"osd":5,"last update":"Mon Aug 28 02:36:17 2023","interfaces":[{"interface":"back","average":{"1min":1.0669999999999999,"5min":1.9259999999999999,"15min":2.1099999999999999},"min":{"1min":0.39300000000000002,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.8769999999999998,"5min":10.722,"15min":102.904},"last":0.371},{"interface":"front","average":{"1min":1.3819999999999999,"5min":3.798,"15min":2.7589999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":4.319,"5min":138.328,"15min":138.328},"last":1.8620000000000001}]},{"osd":6,"last update":"Mon Aug 28 02:36:17 2023","interfaces":[{"interface":"back","average":{"1min":1.48,"5min":3.7749999999999999,"15min":2.5350000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.314},"max":{"1min":4.351,"5min":138.37,"15min":138.37},"last":1.877},{"interface":"front","average":{"1min":1.179,"5min":1.655,"15min":1.7749999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.314},"max":{"1min":5.5060000000000002,"5min":7.9400000000000004,"15min":72.030000000000001},"last":2.698}]},{"osd":7,"last update":"Mon Aug 28 02:35:57 2023","interfaces":[{"interface":"back","average":{"1min":1.155,"5min":1.155,"15min":1.155},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":3.46,"5min":3.46,"15min":3.46},"last":1.8959999999999999},{"interface":"front","average":{"1min":1.2869999999999999,"5min":1.2869999999999999,"15min":1.2869999999999999},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":5.4560000000000004,"5min":5.4560000000000004,"15min":5.4560000000000004},"last":2.6099999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019218,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2372432,"kb_used_data":1774536,"kb_used_omap":0,"kb_used_meta":597888,"kb_avail":91372720,"statfs":{"total":95995035648,"available":93565665280,"internally_reserved":0,"allocated":1817124864,"data_stored":1816247971,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":612237312},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:36:25 2023","interfaces":[{"interface":"back","average":{"1min":7.6079999999999997,"5min":7.9610000000000003,"15min":5.3259999999999996},"min":{"1min":0.23400000000000001,"5min":0.23100000000000001,"15min":0.219},"max":{"1min":61.972999999999999,"5min":131.893,"15min":138.67400000000001},"last":1.3009999999999999},{"interface":"front","average":{"1min":7.8399999999999999,"5min":8.1769999999999996,"15min":5.1950000000000003},"min":{"1min":0.23799999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":61.459000000000003,"5min":131.96700000000001,"15min":134.14599999999999},"last":3.0950000000000002}]},{"osd":1,"last update":"Mon Aug 28 02:36:25 2023","interfaces":[{"interface":"back","average":{"1min":9.9469999999999992,"5min":3.3050000000000002,"15min":2.4780000000000002},"min":{"1min":0.27800000000000002,"5min":0.24399999999999999,"15min":0.221},"max":{"1min":105.604,"5min":105.604,"15min":105.604},"last":0.39700000000000002},{"interface":"front","average":{"1min":9.6080000000000005,"5min":3.113,"15min":2.2869999999999999},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":105.883,"5min":105.883,"15min":105.883},"last":0.46100000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:35:50 2023","interfaces":[{"interface":"back","average":{"1min":0.98199999999999998,"5min":3.5270000000000001,"15min":3.8180000000000001},"min":{"1min":0.29699999999999999,"5min":0.255,"15min":0.255},"max":{"1min":5.6319999999999997,"5min":107.786,"15min":107.786},"last":0.26600000000000001},{"interface":"front","average":{"1min":1.0289999999999999,"5min":3.504,"15min":3.7210000000000001},"min":{"1min":0.312,"5min":0.27300000000000002,"15min":0.25},"max":{"1min":5.2169999999999996,"5min":107.599,"15min":107.599},"last":1.2130000000000001}]},{"osd":3,"last update":"Mon Aug 28 02:36:25 2023","interfaces":[{"interface":"back","average":{"1min":8.5619999999999994,"5min":9.8659999999999997,"15min":5.827},"min":{"1min":0.27700000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":137.995,"5min":203.583,"15min":203.583},"last":1.6160000000000001},{"interface":"front","average":{"1min":8.9250000000000007,"5min":10.34,"15min":5.9850000000000003},"min":{"1min":0.33100000000000002,"5min":0.33000000000000002,"15min":0.32500000000000001},"max":{"1min":137.65899999999999,"5min":203.41399999999999,"15min":203.41399999999999},"last":1.6100000000000001}]},{"osd":5,"last update":"Mon Aug 28 02:36:03 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.45200000000000001,"15min":1.4410000000000001},"min":{"1min":0.26800000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":2.218,"5min":2.3149999999999999,"15min":124.718},"last":0.255},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.55200000000000005,"15min":1.6579999999999999},"min":{"1min":0.27200000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.5620000000000001,"5min":2.444,"15min":124.694},"last":0.44900000000000001}]},{"osd":6,"last update":"Mon Aug 28 02:36:25 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.625,"15min":1.1279999999999999},"min":{"1min":0.28499999999999998,"5min":0.25,"15min":0.25},"max":{"1min":2.1339999999999999,"5min":3.395,"15min":14.907999999999999},"last":0.29499999999999998},{"interface":"front","average":{"1min":0.751,"5min":0.625,"15min":0.89800000000000002},"min":{"1min":0.36199999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":2.2650000000000001,"5min":3.1219999999999999,"15min":7.3170000000000002},"last":0.35799999999999998}]},{"osd":7,"last update":"Mon Aug 28 02:35:55 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":2.1989999999999998,"5min":2.1989999999999998,"15min":2.1989999999999998},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":1.6100000000000001,"5min":1.6100000000000001,"15min":1.6100000000000001},"last":0.58699999999999997}]}]},{"osd":5,"up_from":36,"seq":154618822990,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3756480,"kb_used_data":3170872,"kb_used_omap":0,"kb_used_meta":585600,"kb_avail":89988672,"statfs":{"total":95995035648,"available":92148400128,"internally_reserved":0,"allocated":3246972928,"data_stored":3246107574,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":599654400},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:36:13 2023","interfaces":[{"interface":"back","average":{"1min":5.1289999999999996,"5min":4.782,"15min":4.399},"min":{"1min":0.216,"5min":0.216,"15min":0.20300000000000001},"max":{"1min":64.590999999999994,"5min":128.72800000000001,"15min":131.06100000000001},"last":0.41399999999999998},{"interface":"front","average":{"1min":5.1520000000000001,"5min":4.8019999999999996,"15min":4.6130000000000004},"min":{"1min":0.245,"5min":0.245,"15min":0.22900000000000001},"max":{"1min":64.578999999999994,"5min":128.75200000000001,"15min":131.262},"last":0.42699999999999999}]},{"osd":1,"last update":"Mon Aug 28 02:36:27 2023","interfaces":[{"interface":"back","average":{"1min":4.3529999999999998,"5min":3.5539999999999998,"15min":3.4510000000000001},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.216},"max":{"1min":55.463999999999999,"5min":98.632000000000005,"15min":183.97900000000001},"last":0.44700000000000001},{"interface":"front","average":{"1min":4.2649999999999997,"5min":3.548,"15min":3.6080000000000001},"min":{"1min":0.28000000000000003,"5min":0.26400000000000001,"15min":0.23200000000000001},"max":{"1min":55.546999999999997,"5min":98.560000000000002,"15min":184.33699999999999},"last":4.016}]},{"osd":2,"last update":"Mon Aug 28 02:35:55 2023","interfaces":[{"interface":"back","average":{"1min":0.76300000000000001,"5min":2.944,"15min":4.7590000000000003},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.252},"max":{"1min":3.3690000000000002,"5min":78.834000000000003,"15min":176.69499999999999},"last":4.6120000000000001},{"interface":"front","average":{"1min":0.90700000000000003,"5min":3.0009999999999999,"15min":5.0149999999999997},"min":{"1min":0.35399999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.3940000000000001,"5min":78.798000000000002,"15min":176.71199999999999},"last":4.6459999999999999}]},{"osd":3,"last update":"Mon Aug 28 02:36:27 2023","interfaces":[{"interface":"back","average":{"1min":2.2480000000000002,"5min":6.5819999999999999,"15min":3.6949999999999998},"min":{"1min":0.30499999999999999,"5min":0.29699999999999999,"15min":0.26100000000000001},"max":{"1min":14.202,"5min":128.571,"15min":128.571},"last":1.159},{"interface":"front","average":{"1min":1.6579999999999999,"5min":6.2140000000000004,"15min":3.8849999999999998},"min":{"1min":0.32900000000000001,"5min":0.28699999999999998,"15min":0.26200000000000001},"max":{"1min":4.1909999999999998,"5min":128.684,"15min":128.684},"last":0.98399999999999999}]},{"osd":4,"last update":"Mon Aug 28 02:36:04 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.55200000000000005,"15min":1.335},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":2.145,"5min":3.0259999999999998,"15min":85.769999999999996},"last":0.72999999999999998},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.499,"15min":1.3100000000000001},"min":{"1min":0.29899999999999999,"5min":0.248,"15min":0.248},"max":{"1min":0.94699999999999995,"5min":2.7690000000000001,"15min":85.825000000000003},"last":0.72299999999999998}]},{"osd":6,"last update":"Mon Aug 28 02:36:27 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.55100000000000005,"15min":1.054},"min":{"1min":0.27800000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":2.181,"5min":3.9990000000000001,"15min":14.803000000000001},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.65400000000000003,"15min":0.97099999999999997},"min":{"1min":0.376,"5min":0.29599999999999999,"15min":0.28799999999999998},"max":{"1min":1.4379999999999999,"5min":7.4740000000000002,"15min":8.6479999999999997},"last":0.68999999999999995}]},{"osd":7,"last update":"Mon Aug 28 02:35:55 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":2.0049999999999999,"5min":2.0049999999999999,"15min":2.0049999999999999},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":2.7839999999999998,"5min":2.7839999999999998,"15min":2.7839999999999998},"last":0.66200000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2978414592,"data_stored":2978414592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2575040512,"data_stored":2575040512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2605187072,"data_stored":2605187072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2217017344,"data_stored":2217017344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1820327936,"data_stored":1820327936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3266904064,"data_stored":3266904064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2197291008,"data_stored":2197291008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-08-28T02:36:48.813 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-08-28T02:36:48.813 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd pool set device_health_metrics pg_num 31 2023-08-28T02:36:48.848 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:48 smithi143 ceph-mon[183059]: from='client.35724 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:36:48.848 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:48 smithi143 ceph-mon[183059]: pgmap v888: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 48 MiB/s wr, 760 op/s 2023-08-28T02:36:48.848 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:48 smithi143 ceph-mon[189871]: from='client.35724 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:36:48.848 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:48 smithi143 ceph-mon[189871]: pgmap v888: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 48 MiB/s wr, 760 op/s 2023-08-28T02:36:48.879 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:48.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:48 smithi169 ceph-mon[159415]: from='client.35724 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:36:48.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:48 smithi169 ceph-mon[159415]: pgmap v888: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 48 MiB/s wr, 760 op/s 2023-08-28T02:36:49.124 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:36:49.161 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:36:49.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:49.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:36:49.213 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 86 16 93960 93944 68.2408 47.4375 0.0413461 0.0146486 2023-08-28T02:36:49.279 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:36:49.310 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:49.368 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:36:49.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:49.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:36:49.477 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:36:49.586 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:49.708 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:36:49.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:49.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:36:49.814 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:36:49.847 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:36:49.914 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:50.219 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 87 16 94744 94728 68.0159 49 0.0087354 0.0146971 2023-08-28T02:36:50.822 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:36:50.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:50.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:36:50.930 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:36:50.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:50 smithi143 ceph-mon[189871]: pgmap v889: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 46 MiB/s wr, 737 op/s 2023-08-28T02:36:50.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:50 smithi143 ceph-mon[183059]: pgmap v889: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 46 MiB/s wr, 737 op/s 2023-08-28T02:36:50.984 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:50.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:50 smithi169 ceph-mon[159415]: pgmap v889: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 46 MiB/s wr, 737 op/s 2023-08-28T02:36:51.002 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:36:51.011 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:36:51.119 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:36:51.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:51.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:36:51.218 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 88 16 95422 95406 67.7246 42.375 0.0150405 0.0147607 2023-08-28T02:36:51.229 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:36:51.303 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:51.864 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:36:51.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:51.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:36:51.976 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:36:52.007 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:52.071 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:36:52.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:52.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:36:52.148 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:36:51 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:36:51] "GET /metrics HTTP/1.1" 200 36530 "" "Prometheus/2.33.4" 2023-08-28T02:36:52.176 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:36:52.212 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:52.218 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 89 16 96269 96253 67.5585 52.9375 0.00669725 0.0147968 2023-08-28T02:36:52.256 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:36:52.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:52.262 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:36:52.363 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:36:52.425 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:52.516 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:36:52.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:52.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:36:52.623 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:36:52.662 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:52.694 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:52 smithi143 ceph-mon[189871]: pgmap v890: 89 pgs: 89 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 46 MiB/s wr, 733 op/s 2023-08-28T02:36:52.696 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:52 smithi143 ceph-mon[183059]: pgmap v890: 89 pgs: 89 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 46 MiB/s wr, 733 op/s 2023-08-28T02:36:52.721 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:36:52.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:52.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:36:52.836 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:36:52.916 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:52.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:52 smithi169 ceph-mon[159415]: pgmap v890: 89 pgs: 89 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 46 MiB/s wr, 733 op/s 2023-08-28T02:36:53.219 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 90 16 96907 96891 67.2509 39.875 0.0233923 0.0148436 2023-08-28T02:36:53.248 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:36:53.267 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:36:53.284 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:36:53.302 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:36:53.320 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:36:53.337 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:36:53.355 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:36:53.372 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:36:53.373 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:36:53.374 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:36:53.374 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:36:53.374 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:36:53.374 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:36:53.374 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:36:53.374 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:36:53.374 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:36:53.698 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:53 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3503222042' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:36:53.698 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:53 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3503222042' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:36:53.733 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:36:53.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:53.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:36:53.836 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:36:53.863 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:53.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:53 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3503222042' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:36:54.219 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Total time run: 90.0595 2023-08-28T02:36:54.219 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Total writes made: 96907 2023-08-28T02:36:54.219 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Write size: 65536 2023-08-28T02:36:54.219 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Object size: 65536 2023-08-28T02:36:54.219 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Bandwidth (MB/sec): 67.2521 2023-08-28T02:36:54.219 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Stddev Bandwidth: 26.4514 2023-08-28T02:36:54.220 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Max bandwidth (MB/sec): 163.062 2023-08-28T02:36:54.220 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Min bandwidth (MB/sec): 33.0625 2023-08-28T02:36:54.220 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Average IOPS: 1076 2023-08-28T02:36:54.220 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Stddev IOPS: 423.222 2023-08-28T02:36:54.220 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Max IOPS: 2609 2023-08-28T02:36:54.220 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Min IOPS: 529 2023-08-28T02:36:54.220 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Average Latency(s): 0.0148657 2023-08-28T02:36:54.220 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Stddev Latency(s): 0.0249496 2023-08-28T02:36:54.220 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Max latency(s): 1.01701 2023-08-28T02:36:54.220 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Min latency(s): 0.00106302 2023-08-28T02:36:54.226 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Cleaning up (deleting benchmark objects) 2023-08-28T02:36:54.815 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:36:54.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:54.826 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:54 smithi143 ceph-mon[183059]: pgmap v891: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 45 MiB/s wr, 724 op/s 2023-08-28T02:36:54.827 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:54 smithi143 ceph-mon[189871]: pgmap v891: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 45 MiB/s wr, 724 op/s 2023-08-28T02:36:54.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:36:54.928 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:36:54.957 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:54 smithi169 ceph-mon[159415]: pgmap v891: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 45 MiB/s wr, 724 op/s 2023-08-28T02:36:54.960 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:55.036 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:36:55.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:55.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:36:55.146 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:36:55.224 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:55.412 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:36:55.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:55.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:36:55.526 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:36:55.618 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:55.749 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:36:55.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:55.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:36:55.819 INFO:teuthology.orchestra.run.smithi143.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-08-28T02:36:55.846 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:36:55.846 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [ 2023-08-28T02:36:55.846 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.846 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.45647.0:98226 8.e e338/318 8:7e3d61ac:::benchmark_data_smithi143_419289_object1316:head v 338'6168, mlcod=338'6165)", 2023-08-28T02:36:55.846 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:36:55.837552+0000", 2023-08-28T02:36:55.847 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.0090800010000000007, 2023-08-28T02:36:55.847 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0091035860000000003, 2023-08-28T02:36:55.847 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:36:55.847 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "started", 2023-08-28T02:36:55.847 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:36:55.847 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.847 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:36:55.847 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.837552+0000", 2023-08-28T02:36:55.847 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:55.847 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.847 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.847 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:36:55.848 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.837552+0000", 2023-08-28T02:36:55.848 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:55.848 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.848 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.848 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:36:55.848 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.837552+0000", 2023-08-28T02:36:55.848 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:36:55.848 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.848 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.848 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:36:55.848 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.837556+0000", 2023-08-28T02:36:55.848 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.2030000000000002e-06 2023-08-28T02:36:55.848 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.849 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.849 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:36:55.849 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.837562+0000", 2023-08-28T02:36:55.849 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.9340000000000003e-06 2023-08-28T02:36:55.849 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.849 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.849 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:36:55.849 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.837566+0000", 2023-08-28T02:36:55.849 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.5820000000000003e-06 2023-08-28T02:36:55.849 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.849 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.849 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:36:55.850 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.843774+0000", 2023-08-28T02:36:55.850 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0062073850000000002 2023-08-28T02:36:55.850 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.850 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.850 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:36:55.850 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.843791+0000", 2023-08-28T02:36:55.850 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.7655999999999998e-05 2023-08-28T02:36:55.850 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:55.850 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:36:55.850 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:55.850 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.851 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.851 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.45647.0:98229 8.6 e338/318)", 2023-08-28T02:36:55.851 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:36:55.837571+0000", 2023-08-28T02:36:55.851 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.0090611570000000002, 2023-08-28T02:36:55.851 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0091239719999999993, 2023-08-28T02:36:55.851 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:36:55.851 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "queued for pg", 2023-08-28T02:36:55.851 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:36:55.851 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.851 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:36:55.851 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.837571+0000", 2023-08-28T02:36:55.851 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:55.851 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.852 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.852 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:36:55.852 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.837571+0000", 2023-08-28T02:36:55.852 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:55.852 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.852 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.852 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:36:55.852 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.837571+0000", 2023-08-28T02:36:55.852 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:36:55.852 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.852 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.852 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:36:55.853 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.837579+0000", 2023-08-28T02:36:55.853 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.4109999999999996e-06 2023-08-28T02:36:55.853 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.853 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.853 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:36:55.853 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.837586+0000", 2023-08-28T02:36:55.853 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.4930000000000003e-06 2023-08-28T02:36:55.853 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.853 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.853 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:36:55.853 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.837590+0000", 2023-08-28T02:36:55.853 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.5990000000000004e-06 2023-08-28T02:36:55.853 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:55.854 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:36:55.854 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:55.854 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.854 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.854 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.45647.0:98236 8.4 e338/318 8:2acdf36d:::benchmark_data_smithi143_419289_object1326:head v 338'6132, mlcod=338'6130)", 2023-08-28T02:36:55.854 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:36:55.842853+0000", 2023-08-28T02:36:55.854 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.0037790570000000002, 2023-08-28T02:36:55.854 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0038713710000000002, 2023-08-28T02:36:55.854 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:36:55.854 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "started", 2023-08-28T02:36:55.854 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:36:55.854 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.855 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:36:55.855 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.842853+0000", 2023-08-28T02:36:55.855 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:55.855 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.855 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.855 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:36:55.855 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.842853+0000", 2023-08-28T02:36:55.855 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:55.855 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.855 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.855 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:36:55.855 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.842852+0000", 2023-08-28T02:36:55.856 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:36:55.856 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.856 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.856 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:36:55.856 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.842860+0000", 2023-08-28T02:36:55.856 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.79e-06 2023-08-28T02:36:55.856 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.856 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.856 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:36:55.856 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.842877+0000", 2023-08-28T02:36:55.856 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.685e-05 2023-08-28T02:36:55.856 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.856 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.857 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:36:55.857 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.842885+0000", 2023-08-28T02:36:55.857 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.9139999999999997e-06 2023-08-28T02:36:55.857 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.857 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.857 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:36:55.857 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.844680+0000", 2023-08-28T02:36:55.857 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0017947169999999999 2023-08-28T02:36:55.857 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.857 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.857 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:36:55.857 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.844699+0000", 2023-08-28T02:36:55.858 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.8459999999999999e-05 2023-08-28T02:36:55.858 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:55.858 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:36:55.858 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:55.858 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.858 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.858 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.45647.0:98210 8.6 e338/318 8:62953666:::benchmark_data_smithi143_419289_object1300:head v 338'6220, mlcod=338'6217)", 2023-08-28T02:36:55.858 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:36:55.821758+0000", 2023-08-28T02:36:55.858 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.024873897999999998, 2023-08-28T02:36:55.858 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.025002342, 2023-08-28T02:36:55.858 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:36:55.858 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "started", 2023-08-28T02:36:55.858 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:36:55.859 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.859 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:36:55.859 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.821758+0000", 2023-08-28T02:36:55.859 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:55.859 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.859 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.859 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:36:55.859 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.821758+0000", 2023-08-28T02:36:55.859 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:55.859 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.859 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.859 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:36:55.860 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.821758+0000", 2023-08-28T02:36:55.860 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:36:55.860 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.860 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.860 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:36:55.860 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.821762+0000", 2023-08-28T02:36:55.860 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.3409999999999996e-06 2023-08-28T02:36:55.860 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.860 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.860 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:36:55.860 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.821769+0000", 2023-08-28T02:36:55.860 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.3499999999999999e-06 2023-08-28T02:36:55.860 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.861 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.861 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:36:55.861 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.821774+0000", 2023-08-28T02:36:55.861 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.9189999999999998e-06 2023-08-28T02:36:55.861 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.861 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.861 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:36:55.861 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.827429+0000", 2023-08-28T02:36:55.861 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0056544799999999999 2023-08-28T02:36:55.861 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.861 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.861 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:36:55.862 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.827448+0000", 2023-08-28T02:36:55.862 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.91e-05 2023-08-28T02:36:55.862 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:55.862 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:36:55.862 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:55.862 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.862 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.862 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.45647.0:98219 8.e e338/318 8:75af3cce:::benchmark_data_smithi143_419289_object1309:head v 338'6167, mlcod=338'6165)", 2023-08-28T02:36:55.862 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:36:55.821779+0000", 2023-08-28T02:36:55.862 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.024853409999999999, 2023-08-28T02:36:55.862 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.025017133, 2023-08-28T02:36:55.862 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:36:55.863 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "started", 2023-08-28T02:36:55.863 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:36:55.863 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.863 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:36:55.863 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.821779+0000", 2023-08-28T02:36:55.863 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:55.863 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.863 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.863 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:36:55.863 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.821779+0000", 2023-08-28T02:36:55.863 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:55.863 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.863 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.864 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:36:55.864 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.821778+0000", 2023-08-28T02:36:55.864 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967296 2023-08-28T02:36:55.864 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.864 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.864 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:36:55.864 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.821787+0000", 2023-08-28T02:36:55.864 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.7190000000000007e-06 2023-08-28T02:36:55.864 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.864 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.864 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:36:55.864 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.821801+0000", 2023-08-28T02:36:55.865 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3783e-05 2023-08-28T02:36:55.865 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.865 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.865 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:36:55.865 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.821806+0000", 2023-08-28T02:36:55.865 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.7899999999999999e-06 2023-08-28T02:36:55.865 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.865 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.865 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:36:55.865 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.836329+0000", 2023-08-28T02:36:55.865 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.014523279 2023-08-28T02:36:55.865 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.865 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.866 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:36:55.866 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.836351+0000", 2023-08-28T02:36:55.866 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.2076e-05 2023-08-28T02:36:55.866 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:55.866 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:36:55.866 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:55.866 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.866 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.866 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.45647.0:98218 8.6 e338/318 8:6557a936:::benchmark_data_smithi143_419289_object1308:head v 338'6221, mlcod=338'6217)", 2023-08-28T02:36:55.866 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:36:55.821810+0000", 2023-08-28T02:36:55.866 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.024821685999999999, 2023-08-28T02:36:55.866 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.025018946, 2023-08-28T02:36:55.867 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:36:55.867 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "started", 2023-08-28T02:36:55.867 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:36:55.867 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.867 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:36:55.867 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.821810+0000", 2023-08-28T02:36:55.867 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:55.867 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.867 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.867 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:36:55.867 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.821810+0000", 2023-08-28T02:36:55.867 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:55.867 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.868 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.868 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:36:55.868 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.821810+0000", 2023-08-28T02:36:55.868 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:36:55.868 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.868 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.868 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:36:55.868 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.821814+0000", 2023-08-28T02:36:55.868 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.6669999999999998e-06 2023-08-28T02:36:55.868 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.868 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.868 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:36:55.869 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.821819+0000", 2023-08-28T02:36:55.869 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.3329999999999996e-06 2023-08-28T02:36:55.869 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.869 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.869 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:36:55.869 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.821823+0000", 2023-08-28T02:36:55.869 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.2690000000000001e-06 2023-08-28T02:36:55.869 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.869 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.869 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:36:55.869 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.840501+0000", 2023-08-28T02:36:55.869 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.018677454999999999 2023-08-28T02:36:55.870 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.870 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.870 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:36:55.870 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.840521+0000", 2023-08-28T02:36:55.870 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.0343000000000002e-05 2023-08-28T02:36:55.870 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:55.870 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:36:55.870 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:55.870 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.870 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.870 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.45647.0:98224 8.6 e338/318)", 2023-08-28T02:36:55.870 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:36:55.837516+0000", 2023-08-28T02:36:55.870 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.0091161130000000003, 2023-08-28T02:36:55.871 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0093448049999999994, 2023-08-28T02:36:55.871 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:36:55.871 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "queued for pg", 2023-08-28T02:36:55.871 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:36:55.871 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.871 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:36:55.871 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.837516+0000", 2023-08-28T02:36:55.871 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:55.871 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.871 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.871 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:36:55.871 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.837516+0000", 2023-08-28T02:36:55.872 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:36:55.872 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.872 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.872 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:36:55.872 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.837515+0000", 2023-08-28T02:36:55.872 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:36:55.872 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.872 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.872 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:36:55.872 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.837524+0000", 2023-08-28T02:36:55.872 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.456e-06 2023-08-28T02:36:55.872 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.873 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.873 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:36:55.873 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.837537+0000", 2023-08-28T02:36:55.873 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.31e-05 2023-08-28T02:36:55.873 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:36:55.873 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:36:55.873 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:36:55.873 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:55.837544+0000", 2023-08-28T02:36:55.873 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.7109999999999995e-06 2023-08-28T02:36:55.873 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:55.873 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:36:55.873 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:55.873 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:36:55.874 INFO:teuthology.orchestra.run.smithi143.stdout: ], 2023-08-28T02:36:55.874 INFO:teuthology.orchestra.run.smithi143.stdout: "num_ops": 7 2023-08-28T02:36:55.874 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:36:55.874 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:36:55.991 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:56.659 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is unset 2023-08-28T02:36:56.738 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:36:56.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:56 smithi143 ceph-mon[183059]: pgmap v892: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 99 B/s rd, 41 MiB/s wr, 666 op/s 2023-08-28T02:36:56.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:56 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3503355552' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "31"}]: dispatch 2023-08-28T02:36:56.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:56 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "31"}]: dispatch 2023-08-28T02:36:56.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:56 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:36:56.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:56 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3909468016' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:36:56.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:56 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:36:56.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:56 smithi143 ceph-mon[189871]: pgmap v892: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 99 B/s rd, 41 MiB/s wr, 666 op/s 2023-08-28T02:36:56.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:56 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3503355552' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "31"}]: dispatch 2023-08-28T02:36:56.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:56 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "31"}]: dispatch 2023-08-28T02:36:56.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:56 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:36:56.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:56 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3909468016' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:36:56.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:56 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:36:56.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:56 smithi169 ceph-mon[159415]: pgmap v892: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 99 B/s rd, 41 MiB/s wr, 666 op/s 2023-08-28T02:36:56.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:56 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3503355552' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "31"}]: dispatch 2023-08-28T02:36:56.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:56 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "31"}]: dispatch 2023-08-28T02:36:56.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:56 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:36:56.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:56 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3909468016' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:36:56.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:56 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:36:57.630 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:36:57.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:57.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:36:57.735 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:36:57.765 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:57.794 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-08-28T02:36:57.833 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:36:57.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:57.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:36:57.902 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:57 smithi143 ceph-mon[183059]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:36:57.903 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:57 smithi143 ceph-mon[183059]: failed to encode map e339 with expected crc 2023-08-28T02:36:57.903 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:57 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:36:57.903 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:57 smithi143 ceph-mon[183059]: osdmap e339: 8 total, 8 up, 7 in 2023-08-28T02:36:57.903 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:57 smithi143 ceph-mon[183059]: failed to encode map e339 with expected crc 2023-08-28T02:36:57.903 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:57 smithi143 ceph-mon[183059]: failed to encode map e339 with expected crc 2023-08-28T02:36:57.904 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:57 smithi143 ceph-mon[189871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:36:57.904 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:57 smithi143 ceph-mon[189871]: failed to encode map e339 with expected crc 2023-08-28T02:36:57.904 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:57 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:36:57.904 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:57 smithi143 ceph-mon[189871]: osdmap e339: 8 total, 8 up, 7 in 2023-08-28T02:36:57.904 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:57 smithi143 ceph-mon[189871]: failed to encode map e339 with expected crc 2023-08-28T02:36:57.904 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:57 smithi143 ceph-mon[189871]: failed to encode map e339 with expected crc 2023-08-28T02:36:57.936 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:36:57.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:57 smithi169 ceph-mon[159415]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:36:57.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:57 smithi169 ceph-mon[159415]: failed to encode map e339 with expected crc 2023-08-28T02:36:57.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:57 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:36:57.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:57 smithi169 ceph-mon[159415]: osdmap e339: 8 total, 8 up, 7 in 2023-08-28T02:36:57.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:57 smithi169 ceph-mon[159415]: failed to encode map e339 with expected crc 2023-08-28T02:36:57.979 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:57 smithi169 ceph-mon[159415]: failed to encode map e339 with expected crc 2023-08-28T02:36:57.983 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:58.024 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:36:58.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:58.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:36:58.130 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:36:58.170 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:58.218 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:36:58.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:58.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:36:58.326 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:36:58.406 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:58.526 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.2 dump_blocked_ops 2023-08-28T02:36:58.546 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:36:58.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:58.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:36:58.655 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:36:58.661 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:36:58 smithi143 ceph-mon[189871]: pgmap v894: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 38 MiB/s wr, 678 op/s 2023-08-28T02:36:58.690 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:58.740 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:36:58.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:58.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:36:58.760 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:36:58.850 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:36:58.882 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:58.940 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:36:58.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:58.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:36:58.948 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:36:58 smithi169 ceph-mon[159415]: pgmap v894: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 38 MiB/s wr, 678 op/s 2023-08-28T02:36:58.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:36:58 smithi143 ceph-mon[183059]: pgmap v894: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 38 MiB/s wr, 678 op/s 2023-08-28T02:36:59.047 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:36:59.094 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:59.164 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:36:59.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:59.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:36:59.271 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:36:59.313 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:59.375 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:36:59.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:59.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:36:59.481 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:36:59.511 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:59.572 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:36:59.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:59.582 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:36:59.683 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:36:59.756 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:36:59.897 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:36:59.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:36:59.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:37:00.008 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:37:00.074 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:00.353 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:37:00.373 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:37:00.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:00.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:37:00.482 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:37:00.547 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:00.595 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset nodeep-scrub 2023-08-28T02:37:00.664 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:37:00.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:00.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:37:00.778 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:37:00.810 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:00.878 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:37:00.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:00.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:37:00.919 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:00 smithi143 ceph-mon[189871]: pgmap v895: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 27 MiB/s wr, 667 op/s 2023-08-28T02:37:00.921 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:00 smithi143 ceph-mon[183059]: pgmap v895: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 27 MiB/s wr, 667 op/s 2023-08-28T02:37:00.987 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:37:00.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:00 smithi169 ceph-mon[159415]: pgmap v895: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 27 MiB/s wr, 667 op/s 2023-08-28T02:37:01.083 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:01.584 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:37:01.584 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:37:01.584 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:37:01.631 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:37:01.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:01.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:37:01.682 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:37:01.715 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:37:01.738 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:37:01.760 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:37:01.799 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:37:01.816 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:01.836 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:37:01.855 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:37:01.855 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:37:01.855 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:37:01.855 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:37:01.856 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:37:01.856 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:37:01.856 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:37:01.856 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:37:01.880 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:01 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1029991282' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:37:01.885 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:01 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1029991282' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:37:01.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:01 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1029991282' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:37:02.206 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:37:01 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:37:01] "GET /metrics HTTP/1.1" 200 36534 "" "Prometheus/2.33.4" 2023-08-28T02:37:02.585 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:37:02.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:02.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:37:02.693 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:37:02.745 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:02.795 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 6, 1] out_osds: [7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-08-28T02:37:02.795 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-08-28T02:37:02.795 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2023-08-28T02:37:02.795 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-08-28T02:37:02.795 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-08-28T02:37:02.796 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.2 config set filestore_inject_stall 3 2023-08-28T02:37:02.857 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:02 smithi143 ceph-mon[183059]: pgmap v896: 89 pgs: 89 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 21 MiB/s wr, 664 op/s 2023-08-28T02:37:02.858 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:02 smithi143 ceph-mon[189871]: pgmap v896: 89 pgs: 89 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 21 MiB/s wr, 664 op/s 2023-08-28T02:37:02.951 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:37:02.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:03.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:37:03.006 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:02 smithi169 ceph-mon[159415]: pgmap v896: 89 pgs: 89 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 21 MiB/s wr, 664 op/s 2023-08-28T02:37:03.106 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:37:03.192 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:03.293 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:37:03.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:03.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:37:03.348 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:37:03.405 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:37:03.434 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:03.476 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:37:03.476 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:37:03.491 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:37:03.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:03.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:37:03.596 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:37:03.686 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:04.123 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:37:04.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:04.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:37:04.233 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:37:04.287 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:04.348 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:37:04.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:04.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:37:04.455 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:37:04.542 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:04.722 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:37:04.726 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:04 smithi169 ceph-mon[159415]: pgmap v897: 89 pgs: 89 active+clean; 5.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 10 MiB/s wr, 667 op/s 2023-08-28T02:37:04.730 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:37:04.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:04.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:37:04.750 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.2 dump_historic_ops 2023-08-28T02:37:04.833 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:37:04.842 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:04 smithi143 ceph-mon[183059]: pgmap v897: 89 pgs: 89 active+clean; 5.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 10 MiB/s wr, 667 op/s 2023-08-28T02:37:04.844 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:04 smithi143 ceph-mon[189871]: pgmap v897: 89 pgs: 89 active+clean; 5.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 102 B/s rd, 10 MiB/s wr, 667 op/s 2023-08-28T02:37:04.891 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:05.006 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:37:05.500 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:37:05.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:05.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:37:05.604 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:37:05.685 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:06.113 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:37:06.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:06.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:37:06.222 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:37:06.257 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:06.323 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:37:06.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:06.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:37:06.435 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:37:06.470 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:06.527 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:37:06.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:06.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:37:06.635 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:37:06.745 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:06.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:06 smithi143 ceph-mon[183059]: pgmap v898: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 1.2 MiB/s wr, 697 op/s 2023-08-28T02:37:06.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:06 smithi143 ceph-mon[189871]: pgmap v898: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 1.2 MiB/s wr, 697 op/s 2023-08-28T02:37:06.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:06 smithi169 ceph-mon[159415]: pgmap v898: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 1.2 MiB/s wr, 697 op/s 2023-08-28T02:37:07.618 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:37:07.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:07.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:37:07.714 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:37:07.725 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:37:07.779 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:37:07.788 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is unset 2023-08-28T02:37:07.833 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:07.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:07 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2106015937' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:37:07.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:07 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2106015937' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:37:08.070 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:37:08.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:08.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:37:08.181 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:37:08.205 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:07 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2106015937' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:37:08.207 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:08.250 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:37:08.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:08.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:37:08.357 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:37:08.393 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:08.451 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:37:08.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:08.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:37:08.559 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:37:08.591 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:08.662 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:37:08.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:08.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:37:08.772 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:37:08.808 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:08.868 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:37:08.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:08.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:37:08.973 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:37:09.005 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:09.012 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:08 smithi143 ceph-mon[189871]: pgmap v899: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 1.1 MiB/s wr, 729 op/s 2023-08-28T02:37:09.012 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:08 smithi143 ceph-mon[189871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:37:09.013 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:08 smithi143 ceph-mon[189871]: Cluster is now healthy 2023-08-28T02:37:09.013 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:08 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2106015937' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:37:09.013 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:08 smithi143 ceph-mon[189871]: osdmap e340: 8 total, 8 up, 7 in 2023-08-28T02:37:09.013 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:08 smithi143 ceph-mon[189871]: failed to encode map e340 with expected crc 2023-08-28T02:37:09.013 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:08 smithi143 ceph-mon[189871]: failed to encode map e340 with expected crc 2023-08-28T02:37:09.013 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:08 smithi143 ceph-mon[189871]: failed to encode map e340 with expected crc 2023-08-28T02:37:09.013 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:08 smithi143 ceph-mon[189871]: failed to encode map e340 with expected crc 2023-08-28T02:37:09.013 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:08 smithi143 ceph-mon[183059]: pgmap v899: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 1.1 MiB/s wr, 729 op/s 2023-08-28T02:37:09.013 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:08 smithi143 ceph-mon[183059]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:37:09.014 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:08 smithi143 ceph-mon[183059]: Cluster is now healthy 2023-08-28T02:37:09.014 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:08 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2106015937' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:37:09.014 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:08 smithi143 ceph-mon[183059]: osdmap e340: 8 total, 8 up, 7 in 2023-08-28T02:37:09.014 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:08 smithi143 ceph-mon[183059]: failed to encode map e340 with expected crc 2023-08-28T02:37:09.014 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:08 smithi143 ceph-mon[183059]: failed to encode map e340 with expected crc 2023-08-28T02:37:09.014 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:08 smithi143 ceph-mon[183059]: failed to encode map e340 with expected crc 2023-08-28T02:37:09.014 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:08 smithi143 ceph-mon[183059]: failed to encode map e340 with expected crc 2023-08-28T02:37:09.066 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:37:09.069 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:08 smithi169 ceph-mon[159415]: pgmap v899: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 1.1 MiB/s wr, 729 op/s 2023-08-28T02:37:09.069 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:08 smithi169 ceph-mon[159415]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:37:09.069 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:08 smithi169 ceph-mon[159415]: Cluster is now healthy 2023-08-28T02:37:09.069 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:08 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2106015937' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:37:09.069 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:08 smithi169 ceph-mon[159415]: osdmap e340: 8 total, 8 up, 7 in 2023-08-28T02:37:09.070 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:08 smithi169 ceph-mon[159415]: failed to encode map e340 with expected crc 2023-08-28T02:37:09.070 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:08 smithi169 ceph-mon[159415]: failed to encode map e340 with expected crc 2023-08-28T02:37:09.070 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:08 smithi169 ceph-mon[159415]: failed to encode map e340 with expected crc 2023-08-28T02:37:09.070 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:08 smithi169 ceph-mon[159415]: failed to encode map e340 with expected crc 2023-08-28T02:37:09.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:09.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:37:09.171 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:37:09.203 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:09.238 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:37:09.238 INFO:teuthology.orchestra.run.smithi143.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-08-28T02:37:09.239 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:37:09.271 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:37:09.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:09.277 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:37:09.378 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:37:09.477 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:09.644 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:37:09.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:09.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:37:09.755 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:37:09.805 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:10.824 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set noscrub 2023-08-28T02:37:11.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:10 smithi143 ceph-mon[183059]: pgmap v901: 89 pgs: 89 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 746 op/s 2023-08-28T02:37:11.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:10 smithi143 ceph-mon[189871]: pgmap v901: 89 pgs: 89 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 746 op/s 2023-08-28T02:37:11.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:10 smithi169 ceph-mon[159415]: pgmap v901: 89 pgs: 89 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 746 op/s 2023-08-28T02:37:11.443 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:37:11.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:11.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:37:11.558 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:37:11.634 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:11.819 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:37:11.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:11.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:37:11.926 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:37:11.955 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:11.981 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:37:12.011 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:37:12.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:12.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:37:12.018 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:37:11 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:37:11] "GET /metrics HTTP/1.1" 200 36536 "" "Prometheus/2.33.4" 2023-08-28T02:37:12.119 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:37:12.123 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:37:12.123 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:37:12.155 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:12.208 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:37:12.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:12.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:37:12.219 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:12 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:37:12.219 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:12 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:37:12.219 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:12 smithi169 ceph-mon[159415]: pgmap v902: 89 pgs: 89 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 750 op/s 2023-08-28T02:37:12.303 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:37:12.319 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:37:12.344 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:37:12.353 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:12.397 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:37:12.429 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:37:12.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:12.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:37:12.448 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:37:12.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:12 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:37:12.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:12 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:37:12.450 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:12 smithi143 ceph-mon[189871]: pgmap v902: 89 pgs: 89 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 750 op/s 2023-08-28T02:37:12.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:12 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:37:12.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:12 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:37:12.453 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:12 smithi143 ceph-mon[183059]: pgmap v902: 89 pgs: 89 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 750 op/s 2023-08-28T02:37:12.489 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:37:12.519 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:37:12.538 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:37:12.550 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:37:12.581 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:37:12.605 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:12.609 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:37:12.609 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:37:12.609 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:37:12.609 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:37:12.610 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:37:12.610 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:37:12.610 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:37:12.610 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:37:13.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:13 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2563978535' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:37:13.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:13 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2563978535' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:37:13.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:13 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2563978535' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:37:13.560 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:37:13.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:13.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:37:13.633 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.3 dump_ops_in_flight 2023-08-28T02:37:13.674 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:37:13.710 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:13.775 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:37:13.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:13.781 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:37:13.882 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:37:13.907 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:13.984 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:37:13.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:13.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:37:14.097 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:37:14.174 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:14.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:14 smithi143 ceph-mon[183059]: pgmap v903: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 747 op/s 2023-08-28T02:37:14.457 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:14 smithi143 ceph-mon[189871]: pgmap v903: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 747 op/s 2023-08-28T02:37:14.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:14 smithi169 ceph-mon[159415]: pgmap v903: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 747 op/s 2023-08-28T02:37:14.633 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:37:14.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:14.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:37:14.743 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:37:14.769 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:14.810 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:37:14.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:14.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:37:14.915 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:37:14.950 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:15.003 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:37:15.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:15.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:37:15.109 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:37:15.141 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:15.206 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:37:15.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:15.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:37:15.316 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:37:15.392 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:15.461 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:37:15.516 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:37:15.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:15.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:37:15.623 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:37:15.721 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:15.841 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:37:15.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:15.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:37:15.948 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:37:15.981 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:16.040 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:37:16.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:16.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:37:16.150 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:37:16.230 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:16.413 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:37:16.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:16.420 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:37:16.459 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 6, 1] out_osds: [7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-08-28T02:37:16.459 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-08-28T02:37:16.459 INFO:tasks.thrashosds.thrasher:Growing pool device_health_metrics 2023-08-28T02:37:16.459 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph pg dump --format=json 2023-08-28T02:37:16.522 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:37:16.614 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:16.621 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:37:16.706 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is set 2023-08-28T02:37:16.788 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:37:16.803 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:16 smithi143 ceph-mon[183059]: pgmap v904: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 731 op/s 2023-08-28T02:37:16.803 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:16 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/520974727' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:37:16.803 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:16 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:37:16.803 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:16 smithi143 ceph-mon[189871]: pgmap v904: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 731 op/s 2023-08-28T02:37:16.803 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:16 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/520974727' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:37:16.804 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:16 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:37:16.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:16.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:37:16.905 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:37:16.932 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:16 smithi169 ceph-mon[159415]: pgmap v904: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 731 op/s 2023-08-28T02:37:16.932 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:16 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/520974727' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:37:16.933 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:16 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:37:16.935 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:16.992 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:37:16.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:16.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:37:17.100 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:37:17.138 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:17.213 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:37:17.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:17.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:37:17.320 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:37:17.386 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:17.957 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:17 smithi143 ceph-mon[183059]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:37:17.957 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:17 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:37:17.957 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:17 smithi143 ceph-mon[183059]: osdmap e341: 8 total, 8 up, 7 in 2023-08-28T02:37:17.957 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:17 smithi143 ceph-mon[183059]: failed to encode map e341 with expected crc 2023-08-28T02:37:17.957 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:17 smithi143 ceph-mon[183059]: failed to encode map e341 with expected crc 2023-08-28T02:37:17.957 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:17 smithi143 ceph-mon[183059]: failed to encode map e341 with expected crc 2023-08-28T02:37:17.957 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:17 smithi143 ceph-mon[183059]: failed to encode map e341 with expected crc 2023-08-28T02:37:17.958 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:17 smithi143 ceph-mon[189871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:37:17.958 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:17 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:37:17.958 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:17 smithi143 ceph-mon[189871]: osdmap e341: 8 total, 8 up, 7 in 2023-08-28T02:37:17.958 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:17 smithi143 ceph-mon[189871]: failed to encode map e341 with expected crc 2023-08-28T02:37:17.958 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:17 smithi143 ceph-mon[189871]: failed to encode map e341 with expected crc 2023-08-28T02:37:17.958 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:17 smithi143 ceph-mon[189871]: failed to encode map e341 with expected crc 2023-08-28T02:37:17.958 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:17 smithi143 ceph-mon[189871]: failed to encode map e341 with expected crc 2023-08-28T02:37:17.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:17 smithi169 ceph-mon[159415]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:37:17.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:17 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:37:17.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:17 smithi169 ceph-mon[159415]: osdmap e341: 8 total, 8 up, 7 in 2023-08-28T02:37:17.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:17 smithi169 ceph-mon[159415]: failed to encode map e341 with expected crc 2023-08-28T02:37:17.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:17 smithi169 ceph-mon[159415]: failed to encode map e341 with expected crc 2023-08-28T02:37:17.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:17 smithi169 ceph-mon[159415]: failed to encode map e341 with expected crc 2023-08-28T02:37:17.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:17 smithi169 ceph-mon[159415]: failed to encode map e341 with expected crc 2023-08-28T02:37:18.001 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:37:18.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:18.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:37:18.113 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:37:18.206 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:18.401 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:37:18.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:18.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:37:18.519 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:37:18.523 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:37:18.523 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:37:18.634 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:18.674 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:37:18.674 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [ 2023-08-28T02:37:18.674 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.674 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.45647.0:114852 8.8 8:1d8b669d:::benchmark_data_smithi143_419289_object17942:head [delete] snapc 0=[] ondisk+write+known_if_redirected e341)", 2023-08-28T02:37:18.674 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:37:18.657163+0000", 2023-08-28T02:37:18.674 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.011906271, 2023-08-28T02:37:18.674 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.011943871999999999, 2023-08-28T02:37:18.674 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:37:18.674 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "waiting for sub ops", 2023-08-28T02:37:18.675 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:37:18.675 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.45647", 2023-08-28T02:37:18.675 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3230872490", 2023-08-28T02:37:18.675 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 114852 2023-08-28T02:37:18.675 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.675 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:37:18.675 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.675 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:37:18.675 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.657163+0000", 2023-08-28T02:37:18.675 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:18.675 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.675 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.676 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:37:18.676 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.657163+0000", 2023-08-28T02:37:18.676 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:18.676 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.676 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.676 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:37:18.676 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.657161+0000", 2023-08-28T02:37:18.676 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:37:18.676 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.676 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.676 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:37:18.676 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.657171+0000", 2023-08-28T02:37:18.676 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0054999999999999e-05 2023-08-28T02:37:18.677 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.677 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.677 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:37:18.677 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.657194+0000", 2023-08-28T02:37:18.677 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.3442e-05 2023-08-28T02:37:18.677 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.677 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.677 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:37:18.677 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.657206+0000", 2023-08-28T02:37:18.677 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1496e-05 2023-08-28T02:37:18.677 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.677 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.678 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:37:18.678 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.657274+0000", 2023-08-28T02:37:18.678 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.7702999999999999e-05 2023-08-28T02:37:18.678 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.678 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.678 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:37:18.678 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.657412+0000", 2023-08-28T02:37:18.678 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00013801199999999999 2023-08-28T02:37:18.678 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.678 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.678 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:37:18.678 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.657544+0000", 2023-08-28T02:37:18.678 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00013222200000000001 2023-08-28T02:37:18.678 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.679 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.679 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:37:18.679 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.662119+0000", 2023-08-28T02:37:18.679 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0045753670000000003 2023-08-28T02:37:18.679 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.679 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.679 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:18.679 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.666262+0000", 2023-08-28T02:37:18.679 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0041424080000000002 2023-08-28T02:37:18.679 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:18.679 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:37:18.679 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:18.679 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.680 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.680 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.45647.0:114854 8.0 8:04a75bed:::benchmark_data_smithi143_419289_object17944:head [delete] snapc 0=[] ondisk+write+known_if_redirected e341)", 2023-08-28T02:37:18.680 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:37:18.661585+0000", 2023-08-28T02:37:18.680 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.0074844630000000002, 2023-08-28T02:37:18.680 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.007620262, 2023-08-28T02:37:18.680 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:37:18.680 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "waiting for sub ops", 2023-08-28T02:37:18.680 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:37:18.680 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.45647", 2023-08-28T02:37:18.680 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3230872490", 2023-08-28T02:37:18.680 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 114854 2023-08-28T02:37:18.680 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.680 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:37:18.681 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.681 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:37:18.681 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.661585+0000", 2023-08-28T02:37:18.681 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:18.681 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.681 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.681 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:37:18.681 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.661585+0000", 2023-08-28T02:37:18.681 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:18.681 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.681 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.681 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:37:18.681 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.661583+0000", 2023-08-28T02:37:18.682 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:37:18.682 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.682 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.682 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:37:18.682 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.661591+0000", 2023-08-28T02:37:18.682 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.1890000000000007e-06 2023-08-28T02:37:18.682 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.682 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.682 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:37:18.682 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.661611+0000", 2023-08-28T02:37:18.682 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.9995000000000001e-05 2023-08-28T02:37:18.682 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.683 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.683 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:37:18.683 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.661620+0000", 2023-08-28T02:37:18.683 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.5860000000000005e-06 2023-08-28T02:37:18.683 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.683 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.683 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:37:18.683 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.662147+0000", 2023-08-28T02:37:18.683 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000526703 2023-08-28T02:37:18.683 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.683 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.683 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:37:18.683 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.662371+0000", 2023-08-28T02:37:18.684 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000224723 2023-08-28T02:37:18.684 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.684 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.684 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,5", 2023-08-28T02:37:18.684 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.662491+0000", 2023-08-28T02:37:18.684 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000119801 2023-08-28T02:37:18.684 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:18.684 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:37:18.684 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:18.684 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.684 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.684 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.45647.0:114855 8.8 8:14d91847:::benchmark_data_smithi143_419289_object17945:head [delete] snapc 0=[] ondisk+write+known_if_redirected e341)", 2023-08-28T02:37:18.684 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:37:18.661631+0000", 2023-08-28T02:37:18.684 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.0074383799999999996, 2023-08-28T02:37:18.685 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.007652926, 2023-08-28T02:37:18.685 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:37:18.685 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "waiting for sub ops", 2023-08-28T02:37:18.685 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:37:18.685 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.45647", 2023-08-28T02:37:18.685 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3230872490", 2023-08-28T02:37:18.685 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 114855 2023-08-28T02:37:18.685 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.685 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:37:18.685 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.685 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:37:18.685 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.661631+0000", 2023-08-28T02:37:18.685 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:18.686 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.686 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.686 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:37:18.686 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.661631+0000", 2023-08-28T02:37:18.686 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:18.686 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.686 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.686 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:37:18.686 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.661630+0000", 2023-08-28T02:37:18.686 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:37:18.686 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.686 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.687 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:37:18.687 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.661634+0000", 2023-08-28T02:37:18.687 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.597e-06 2023-08-28T02:37:18.687 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.687 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.687 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:37:18.687 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.661641+0000", 2023-08-28T02:37:18.687 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.6989999999999997e-06 2023-08-28T02:37:18.687 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.687 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.687 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:37:18.687 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.661646+0000", 2023-08-28T02:37:18.687 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.9899999999999997e-06 2023-08-28T02:37:18.688 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.688 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.688 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:37:18.688 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.666288+0000", 2023-08-28T02:37:18.688 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0046424229999999997 2023-08-28T02:37:18.688 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.688 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.688 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:37:18.688 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.666399+0000", 2023-08-28T02:37:18.688 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000110685 2023-08-28T02:37:18.688 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.688 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.688 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:37:18.688 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.666520+0000", 2023-08-28T02:37:18.689 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00012087900000000001 2023-08-28T02:37:18.689 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:18.689 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:37:18.689 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:18.689 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.689 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.689 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.45647.0:114860 8.f 8:f80173d8:::benchmark_data_smithi143_419289_object17950:head [delete] snapc 0=[] ondisk+write+known_if_redirected e341)", 2023-08-28T02:37:18.689 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:37:18.661651+0000", 2023-08-28T02:37:18.689 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.007418755, 2023-08-28T02:37:18.689 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0077125850000000001, 2023-08-28T02:37:18.689 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:37:18.689 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "waiting for sub ops", 2023-08-28T02:37:18.689 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:37:18.689 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.45647", 2023-08-28T02:37:18.690 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3230872490", 2023-08-28T02:37:18.690 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 114860 2023-08-28T02:37:18.690 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.690 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:37:18.690 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.690 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:37:18.690 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.661651+0000", 2023-08-28T02:37:18.690 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:18.690 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.690 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.690 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:37:18.690 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.661651+0000", 2023-08-28T02:37:18.691 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:18.691 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.691 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.691 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:37:18.691 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.661650+0000", 2023-08-28T02:37:18.691 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:37:18.691 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.691 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.691 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:37:18.691 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.661653+0000", 2023-08-28T02:37:18.691 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.5969999999999999e-06 2023-08-28T02:37:18.691 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.691 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.692 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:37:18.692 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.661658+0000", 2023-08-28T02:37:18.692 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.4419999999999997e-06 2023-08-28T02:37:18.692 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.692 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.692 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:37:18.692 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.661663+0000", 2023-08-28T02:37:18.692 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.4299999999999999e-06 2023-08-28T02:37:18.692 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.692 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.692 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:37:18.692 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.661714+0000", 2023-08-28T02:37:18.693 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.1823999999999997e-05 2023-08-28T02:37:18.693 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.693 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.693 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:37:18.693 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.661808+0000", 2023-08-28T02:37:18.693 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.3117999999999999e-05 2023-08-28T02:37:18.693 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.693 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.693 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,2", 2023-08-28T02:37:18.693 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.661885+0000", 2023-08-28T02:37:18.693 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.7119000000000005e-05 2023-08-28T02:37:18.693 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.693 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.694 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:37:18.694 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.667353+0000", 2023-08-28T02:37:18.694 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0054684950000000003 2023-08-28T02:37:18.694 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:18.694 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:37:18.694 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:18.694 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.694 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.694 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.45647.0:114861 8.8 8.a3b74308 (undecoded) ondisk+write+known_if_redirected e341)", 2023-08-28T02:37:18.694 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:37:18.661673+0000", 2023-08-28T02:37:18.694 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.0073961729999999998, 2023-08-28T02:37:18.694 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0077673350000000002, 2023-08-28T02:37:18.694 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:37:18.695 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "queued for pg", 2023-08-28T02:37:18.695 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:37:18.695 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.45647", 2023-08-28T02:37:18.695 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3230872490", 2023-08-28T02:37:18.695 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 114861 2023-08-28T02:37:18.695 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.695 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:37:18.695 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.695 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:37:18.695 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.661673+0000", 2023-08-28T02:37:18.695 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:18.695 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.695 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.696 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:37:18.696 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.661673+0000", 2023-08-28T02:37:18.696 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:18.696 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.696 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.696 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:37:18.696 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.661673+0000", 2023-08-28T02:37:18.696 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:37:18.696 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.696 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.696 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:37:18.696 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.661676+0000", 2023-08-28T02:37:18.697 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.7980000000000001e-06 2023-08-28T02:37:18.697 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.697 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.697 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:37:18.697 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.661681+0000", 2023-08-28T02:37:18.697 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.6790000000000002e-06 2023-08-28T02:37:18.697 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.697 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.697 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:37:18.697 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.661686+0000", 2023-08-28T02:37:18.697 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.4349999999999999e-06 2023-08-28T02:37:18.697 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:18.697 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:37:18.697 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:18.698 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.698 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.698 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop_reply(client.45647.0:114854 8.0 e341/318)", 2023-08-28T02:37:18.698 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:37:18.666532+0000", 2023-08-28T02:37:18.698 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.0025370639999999999, 2023-08-28T02:37:18.698 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0029636559999999998, 2023-08-28T02:37:18.698 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:37:18.698 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "queued for pg", 2023-08-28T02:37:18.698 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:37:18.698 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.698 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:37:18.698 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.666532+0000", 2023-08-28T02:37:18.699 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:18.699 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.699 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.699 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:37:18.699 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.666532+0000", 2023-08-28T02:37:18.699 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:18.699 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.699 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.699 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:37:18.699 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.666531+0000", 2023-08-28T02:37:18.699 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:37:18.699 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.700 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.700 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:37:18.700 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.666538+0000", 2023-08-28T02:37:18.700 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.3679999999999996e-06 2023-08-28T02:37:18.700 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.700 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.700 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:37:18.700 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.666551+0000", 2023-08-28T02:37:18.700 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.292e-05 2023-08-28T02:37:18.700 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:18.700 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:18.700 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:37:18.700 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:18.666560+0000", 2023-08-28T02:37:18.700 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.0429999999999996e-06 2023-08-28T02:37:18.701 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:18.701 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:37:18.701 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:18.701 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:18.701 INFO:teuthology.orchestra.run.smithi143.stdout: ], 2023-08-28T02:37:18.701 INFO:teuthology.orchestra.run.smithi143.stdout: "num_ops": 6 2023-08-28T02:37:18.701 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:37:18.879 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:37:18.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:18.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:37:18.886 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:18 smithi143 ceph-mon[183059]: pgmap v906: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 739 op/s 2023-08-28T02:37:18.887 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:18 smithi143 ceph-mon[189871]: pgmap v906: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 739 op/s 2023-08-28T02:37:18.986 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:37:18.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:18 smithi169 ceph-mon[159415]: pgmap v906: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 739 op/s 2023-08-28T02:37:19.035 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:19.095 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:37:19.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:19.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:37:19.202 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:37:19.227 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:19.269 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:37:19.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:19.273 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:37:19.373 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:37:19.400 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:20.012 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:37:20.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:20.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:37:20.118 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set nodeep-scrub 2023-08-28T02:37:20.134 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:37:20.161 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:20.176 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.3 dump_blocked_ops 2023-08-28T02:37:20.219 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:37:20.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:20.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:37:20.326 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:37:20.353 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:20.401 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:37:20.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:20.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:37:20.506 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:37:20.535 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:20.606 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:37:20.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:20.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:37:20.680 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:20 smithi143 ceph-mon[183059]: pgmap v907: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 704 op/s 2023-08-28T02:37:20.712 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:37:20.738 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:20.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:20 smithi143 ceph-mon[189871]: pgmap v907: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 704 op/s 2023-08-28T02:37:20.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:20 smithi169 ceph-mon[159415]: pgmap v907: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 704 op/s 2023-08-28T02:37:21.417 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:37:21.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:21.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:37:21.531 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:37:21.557 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:22.004 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:37:22.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:22.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:37:22.019 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:37:21 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:37:21] "GET /metrics HTTP/1.1" 200 36536 "" "Prometheus/2.33.4" 2023-08-28T02:37:22.120 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:37:22.209 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:22.210 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:37:22.417 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:37:22.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:22.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:37:22.526 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:37:22.609 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:22.637 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:37:22.837 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:37:22.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:22.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:37:22.897 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:37:22.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:22 smithi143 ceph-mon[189871]: pgmap v908: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 709 op/s 2023-08-28T02:37:22.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:22 smithi143 ceph-mon[183059]: pgmap v908: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 709 op/s 2023-08-28T02:37:22.953 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:37:22.955 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:37:22.980 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:22 smithi169 ceph-mon[159415]: pgmap v908: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 709 op/s 2023-08-28T02:37:22.982 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:22.998 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:37:23.038 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:37:23.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:23.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:37:23.069 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:37:23.115 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:37:23.124 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:37:23.137 INFO:teuthology.orchestra.run.smithi143.stderr:dumped all 2023-08-28T02:37:23.144 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:37:23.164 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:37:23.195 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:37:23.219 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:23.242 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:37:23.243 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:37:23.243 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:37:23.243 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:37:23.243 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:37:23.243 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:37:23.243 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:37:23.243 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:37:23.243 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:37:23.606 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:37:23.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:23.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:37:23.715 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:37:23.795 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:23.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:23 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/550871226' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:37:23.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:23 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/550871226' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:37:23.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:23 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/550871226' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:37:24.954 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:24 smithi143 ceph-mon[183059]: from='client.55762 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:37:24.955 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:24 smithi143 ceph-mon[183059]: pgmap v909: 89 pgs: 89 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 700 op/s 2023-08-28T02:37:24.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:24 smithi143 ceph-mon[189871]: from='client.55762 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:37:24.955 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:24 smithi143 ceph-mon[189871]: pgmap v909: 89 pgs: 89 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 700 op/s 2023-08-28T02:37:24.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:24 smithi169 ceph-mon[159415]: from='client.55762 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:37:24.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:24 smithi169 ceph-mon[159415]: pgmap v909: 89 pgs: 89 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 700 op/s 2023-08-28T02:37:26.022 INFO:teuthology.orchestra.run.smithi143.stdout:{"pg_ready":true,"pg_map":{"version":908,"stamp":"2023-08-28T02:37:21.565316+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5184100299,"num_objects":79059,"num_object_clones":0,"num_object_copies":237177,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":79059,"num_whiteouts":0,"num_read":6275,"num_read_kb":13708,"num_write":115891,"num_write_kb":6217190,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":297,"num_bytes_recovered":14969132,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":113555,"ondisk_log_size":113555,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":19151556,"kb_used_data":15304588,"kb_used_omap":29,"kb_used_meta":3846690,"kb_avail":637064508,"statfs":{"total":671965249536,"available":652354056192,"internally_reserved":0,"allocated":15671898112,"data_stored":15662778407,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30674,"internal_metadata":3939010606},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,4,8,8,7,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-465502208,"num_objects":-7103,"num_object_clones":0,"num_object_copies":-21309,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-7103,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7103,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006207"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":360,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.833777+0000","last_change":"2023-08-28T02:32:58.740384+0000","last_active":"2023-08-28T02:37:07.833777+0000","last_peered":"2023-08-28T02:37:07.833777+0000","last_clean":"2023-08-28T02:37:07.833777+0000","last_became_active":"2023-08-28T02:32:58.739959+0000","last_became_peered":"2023-08-28T02:32:58.739959+0000","last_unstale":"2023-08-28T02:37:07.833777+0000","last_undegraded":"2023-08-28T02:37:07.833777+0000","last_fullsized":"2023-08-28T02:37:07.833777+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":248,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.777978+0000","last_change":"2023-08-28T02:32:58.634375+0000","last_active":"2023-08-28T02:37:07.777978+0000","last_peered":"2023-08-28T02:37:07.777978+0000","last_clean":"2023-08-28T02:37:07.777978+0000","last_became_active":"2023-08-28T02:32:58.632726+0000","last_became_peered":"2023-08-28T02:32:58.632726+0000","last_unstale":"2023-08-28T02:37:07.777978+0000","last_undegraded":"2023-08-28T02:37:07.777978+0000","last_fullsized":"2023-08-28T02:37:07.777978+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:20:27.730725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":61,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.780063+0000","last_change":"2023-08-28T02:32:58.628387+0000","last_active":"2023-08-28T02:37:07.780063+0000","last_peered":"2023-08-28T02:37:07.780063+0000","last_clean":"2023-08-28T02:37:07.780063+0000","last_became_active":"2023-08-28T02:32:58.628071+0000","last_became_peered":"2023-08-28T02:32:58.628071+0000","last_unstale":"2023-08-28T02:37:07.780063+0000","last_undegraded":"2023-08-28T02:37:07.780063+0000","last_fullsized":"2023-08-28T02:37:07.780063+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:25:55.264433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":359,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.795471+0000","last_change":"2023-08-28T02:32:58.772285+0000","last_active":"2023-08-28T02:37:07.795471+0000","last_peered":"2023-08-28T02:37:07.795471+0000","last_clean":"2023-08-28T02:37:07.795471+0000","last_became_active":"2023-08-28T02:32:58.771439+0000","last_became_peered":"2023-08-28T02:32:58.771439+0000","last_unstale":"2023-08-28T02:37:07.795471+0000","last_undegraded":"2023-08-28T02:37:07.795471+0000","last_fullsized":"2023-08-28T02:37:07.795471+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:10:48.541785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":239,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.776210+0000","last_change":"2023-08-28T02:32:58.779602+0000","last_active":"2023-08-28T02:37:07.776210+0000","last_peered":"2023-08-28T02:37:07.776210+0000","last_clean":"2023-08-28T02:37:07.776210+0000","last_became_active":"2023-08-28T02:32:58.778945+0000","last_became_peered":"2023-08-28T02:32:58.778945+0000","last_unstale":"2023-08-28T02:37:07.776210+0000","last_undegraded":"2023-08-28T02:37:07.776210+0000","last_fullsized":"2023-08-28T02:37:07.776210+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:35:55.832928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":147,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.790935+0000","last_change":"2023-08-28T02:32:58.626766+0000","last_active":"2023-08-28T02:37:07.790935+0000","last_peered":"2023-08-28T02:37:07.790935+0000","last_clean":"2023-08-28T02:37:07.790935+0000","last_became_active":"2023-08-28T02:32:58.626415+0000","last_became_peered":"2023-08-28T02:32:58.626415+0000","last_unstale":"2023-08-28T02:37:07.790935+0000","last_undegraded":"2023-08-28T02:37:07.790935+0000","last_fullsized":"2023-08-28T02:37:07.790935+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":311,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.795407+0000","last_change":"2023-08-28T02:23:59.227592+0000","last_active":"2023-08-28T02:37:07.795407+0000","last_peered":"2023-08-28T02:37:07.795407+0000","last_clean":"2023-08-28T02:37:07.795407+0000","last_became_active":"2023-08-28T02:23:59.227342+0000","last_became_peered":"2023-08-28T02:23:59.227342+0000","last_unstale":"2023-08-28T02:37:07.795407+0000","last_undegraded":"2023-08-28T02:37:07.795407+0000","last_fullsized":"2023-08-28T02:37:07.795407+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:08:19.874396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":257,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.776281+0000","last_change":"2023-08-28T02:32:58.779435+0000","last_active":"2023-08-28T02:37:07.776281+0000","last_peered":"2023-08-28T02:37:07.776281+0000","last_clean":"2023-08-28T02:37:07.776281+0000","last_became_active":"2023-08-28T02:32:58.778629+0000","last_became_peered":"2023-08-28T02:32:58.778629+0000","last_unstale":"2023-08-28T02:37:07.776281+0000","last_undegraded":"2023-08-28T02:37:07.776281+0000","last_fullsized":"2023-08-28T02:37:07.776281+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:25:32.193898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":349,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.783186+0000","last_change":"2023-08-28T02:35:05.992277+0000","last_active":"2023-08-28T02:37:07.783186+0000","last_peered":"2023-08-28T02:37:07.783186+0000","last_clean":"2023-08-28T02:37:07.783186+0000","last_became_active":"2023-08-28T02:35:05.991954+0000","last_became_peered":"2023-08-28T02:35:05.991954+0000","last_unstale":"2023-08-28T02:37:07.783186+0000","last_undegraded":"2023-08-28T02:37:07.783186+0000","last_fullsized":"2023-08-28T02:37:07.783186+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:41:10.177837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":230,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.777592+0000","last_change":"2023-08-28T02:31:55.525178+0000","last_active":"2023-08-28T02:37:07.777592+0000","last_peered":"2023-08-28T02:37:07.777592+0000","last_clean":"2023-08-28T02:37:07.777592+0000","last_became_active":"2023-08-28T02:31:55.524801+0000","last_became_peered":"2023-08-28T02:31:55.524801+0000","last_unstale":"2023-08-28T02:37:07.777592+0000","last_undegraded":"2023-08-28T02:37:07.777592+0000","last_fullsized":"2023-08-28T02:37:07.777592+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:47:30.078960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":109,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.778171+0000","last_change":"2023-08-28T02:31:55.532306+0000","last_active":"2023-08-28T02:37:07.778171+0000","last_peered":"2023-08-28T02:37:07.778171+0000","last_clean":"2023-08-28T02:37:07.778171+0000","last_became_active":"2023-08-28T02:31:55.524276+0000","last_became_peered":"2023-08-28T02:31:55.524276+0000","last_unstale":"2023-08-28T02:37:07.778171+0000","last_undegraded":"2023-08-28T02:37:07.778171+0000","last_fullsized":"2023-08-28T02:37:07.778171+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:27:06.850784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":72,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.791016+0000","last_change":"2023-08-28T02:35:45.112374+0000","last_active":"2023-08-28T02:37:07.791016+0000","last_peered":"2023-08-28T02:37:07.791016+0000","last_clean":"2023-08-28T02:37:07.791016+0000","last_became_active":"2023-08-28T02:35:45.112024+0000","last_became_peered":"2023-08-28T02:35:45.112024+0000","last_unstale":"2023-08-28T02:37:07.791016+0000","last_undegraded":"2023-08-28T02:37:07.791016+0000","last_fullsized":"2023-08-28T02:37:07.791016+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"86'22","reported_seq":410,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.833897+0000","last_change":"2023-08-28T02:31:55.538001+0000","last_active":"2023-08-28T02:37:07.833897+0000","last_peered":"2023-08-28T02:37:07.833897+0000","last_clean":"2023-08-28T02:37:07.833897+0000","last_became_active":"2023-08-28T02:31:55.518604+0000","last_became_peered":"2023-08-28T02:31:55.518604+0000","last_unstale":"2023-08-28T02:37:07.833897+0000","last_undegraded":"2023-08-28T02:37:07.833897+0000","last_fullsized":"2023-08-28T02:37:07.833897+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":384,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.833773+0000","last_change":"2023-08-28T02:32:58.629451+0000","last_active":"2023-08-28T02:37:07.833773+0000","last_peered":"2023-08-28T02:37:07.833773+0000","last_clean":"2023-08-28T02:37:07.833773+0000","last_became_active":"2023-08-28T02:32:58.628767+0000","last_became_peered":"2023-08-28T02:32:58.628767+0000","last_unstale":"2023-08-28T02:37:07.833773+0000","last_undegraded":"2023-08-28T02:37:07.833773+0000","last_fullsized":"2023-08-28T02:37:07.833773+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":114,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.833876+0000","last_change":"2023-08-28T02:31:55.528552+0000","last_active":"2023-08-28T02:37:07.833876+0000","last_peered":"2023-08-28T02:37:07.833876+0000","last_clean":"2023-08-28T02:37:07.833876+0000","last_became_active":"2023-08-28T02:31:55.518079+0000","last_became_peered":"2023-08-28T02:31:55.518079+0000","last_unstale":"2023-08-28T02:37:07.833876+0000","last_undegraded":"2023-08-28T02:37:07.833876+0000","last_fullsized":"2023-08-28T02:37:07.833876+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":248,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.778334+0000","last_change":"2023-08-28T02:32:58.680534+0000","last_active":"2023-08-28T02:37:07.778334+0000","last_peered":"2023-08-28T02:37:07.778334+0000","last_clean":"2023-08-28T02:37:07.778334+0000","last_became_active":"2023-08-28T02:32:58.680145+0000","last_became_peered":"2023-08-28T02:32:58.680145+0000","last_unstale":"2023-08-28T02:37:07.778334+0000","last_undegraded":"2023-08-28T02:37:07.778334+0000","last_fullsized":"2023-08-28T02:37:07.778334+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:56:40.113013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":357,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-08-28T02:37:16.686289+0000","last_change":"2023-08-28T02:32:58.746929+0000","last_active":"2023-08-28T02:37:16.686289+0000","last_peered":"2023-08-28T02:37:16.686289+0000","last_clean":"2023-08-28T02:37:16.686289+0000","last_became_active":"2023-08-28T02:32:58.746590+0000","last_became_peered":"2023-08-28T02:32:58.746590+0000","last_unstale":"2023-08-28T02:37:16.686289+0000","last_undegraded":"2023-08-28T02:37:16.686289+0000","last_fullsized":"2023-08-28T02:37:16.686289+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:51:11.974154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":340,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.778113+0000","last_change":"2023-08-28T02:35:05.993977+0000","last_active":"2023-08-28T02:37:07.778113+0000","last_peered":"2023-08-28T02:37:07.778113+0000","last_clean":"2023-08-28T02:37:07.778113+0000","last_became_active":"2023-08-28T02:35:05.993610+0000","last_became_peered":"2023-08-28T02:35:05.993610+0000","last_unstale":"2023-08-28T02:37:07.778113+0000","last_undegraded":"2023-08-28T02:37:07.778113+0000","last_fullsized":"2023-08-28T02:37:07.778113+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:37:10.901664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":170,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.833683+0000","last_change":"2023-08-28T02:31:55.517851+0000","last_active":"2023-08-28T02:37:07.833683+0000","last_peered":"2023-08-28T02:37:07.833683+0000","last_clean":"2023-08-28T02:37:07.833683+0000","last_became_active":"2023-08-28T02:31:55.517320+0000","last_became_peered":"2023-08-28T02:31:55.517320+0000","last_unstale":"2023-08-28T02:37:07.833683+0000","last_undegraded":"2023-08-28T02:37:07.833683+0000","last_fullsized":"2023-08-28T02:37:07.833683+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":339,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.795535+0000","last_change":"2023-08-28T02:31:55.528250+0000","last_active":"2023-08-28T02:37:07.795535+0000","last_peered":"2023-08-28T02:37:07.795535+0000","last_clean":"2023-08-28T02:37:07.795535+0000","last_became_active":"2023-08-28T02:31:55.527950+0000","last_became_peered":"2023-08-28T02:31:55.527950+0000","last_unstale":"2023-08-28T02:37:07.795535+0000","last_undegraded":"2023-08-28T02:37:07.795535+0000","last_fullsized":"2023-08-28T02:37:07.795535+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:39:46.120437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":139,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.790955+0000","last_change":"2023-08-28T02:28:25.703333+0000","last_active":"2023-08-28T02:37:07.790955+0000","last_peered":"2023-08-28T02:37:07.790955+0000","last_clean":"2023-08-28T02:37:07.790955+0000","last_became_active":"2023-08-28T02:28:25.343515+0000","last_became_peered":"2023-08-28T02:28:25.343515+0000","last_unstale":"2023-08-28T02:37:07.790955+0000","last_undegraded":"2023-08-28T02:37:07.790955+0000","last_fullsized":"2023-08-28T02:37:07.790955+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":257,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.779259+0000","last_change":"2023-08-28T02:35:45.097133+0000","last_active":"2023-08-28T02:37:07.779259+0000","last_peered":"2023-08-28T02:37:07.779259+0000","last_clean":"2023-08-28T02:37:07.779259+0000","last_became_active":"2023-08-28T02:35:45.096766+0000","last_became_peered":"2023-08-28T02:35:45.096766+0000","last_unstale":"2023-08-28T02:37:07.779259+0000","last_undegraded":"2023-08-28T02:37:07.779259+0000","last_fullsized":"2023-08-28T02:37:07.779259+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:27:29.558704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"341'7082","reported_seq":7113,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-08-28T02:37:17.013819+0000","last_change":"2023-08-28T02:35:14.499701+0000","last_active":"2023-08-28T02:37:17.013819+0000","last_peered":"2023-08-28T02:37:17.013819+0000","last_clean":"2023-08-28T02:37:17.013819+0000","last_became_active":"2023-08-28T02:35:14.499400+0000","last_became_peered":"2023-08-28T02:35:14.499400+0000","last_unstale":"2023-08-28T02:37:17.013819+0000","last_undegraded":"2023-08-28T02:37:17.013819+0000","last_fullsized":"2023-08-28T02:37:17.013819+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":7082,"log_dups_size":0,"ondisk_log_size":7082,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328597504,"num_objects":5014,"num_object_clones":0,"num_object_copies":15042,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5014,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7082,"num_write_kb":387072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":204,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.796956+0000","last_change":"2023-08-28T02:32:58.782840+0000","last_active":"2023-08-28T02:37:07.796956+0000","last_peered":"2023-08-28T02:37:07.796956+0000","last_clean":"2023-08-28T02:37:07.796956+0000","last_became_active":"2023-08-28T02:32:58.683096+0000","last_became_peered":"2023-08-28T02:32:58.683096+0000","last_unstale":"2023-08-28T02:37:07.796956+0000","last_undegraded":"2023-08-28T02:37:07.796956+0000","last_fullsized":"2023-08-28T02:37:07.796956+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:10:08.037078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":300,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.790638+0000","last_change":"2023-08-28T02:35:06.546129+0000","last_active":"2023-08-28T02:37:07.790638+0000","last_peered":"2023-08-28T02:37:07.790638+0000","last_clean":"2023-08-28T02:37:07.790638+0000","last_became_active":"2023-08-28T02:35:06.545789+0000","last_became_peered":"2023-08-28T02:35:06.545789+0000","last_unstale":"2023-08-28T02:37:07.790638+0000","last_undegraded":"2023-08-28T02:37:07.790638+0000","last_fullsized":"2023-08-28T02:37:07.790638+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.5","version":"341'7079","reported_seq":7110,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-08-28T02:37:17.238464+0000","last_change":"2023-08-28T02:35:14.500308+0000","last_active":"2023-08-28T02:37:17.238464+0000","last_peered":"2023-08-28T02:37:17.238464+0000","last_clean":"2023-08-28T02:37:17.238464+0000","last_became_active":"2023-08-28T02:35:14.499954+0000","last_became_peered":"2023-08-28T02:35:14.499954+0000","last_unstale":"2023-08-28T02:37:17.238464+0000","last_undegraded":"2023-08-28T02:37:17.238464+0000","last_fullsized":"2023-08-28T02:37:17.238464+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":7079,"log_dups_size":0,"ondisk_log_size":7079,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":331546624,"num_objects":5059,"num_object_clones":0,"num_object_copies":15177,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5059,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7079,"num_write_kb":388416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":136,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.834041+0000","last_change":"2023-08-28T02:32:58.856906+0000","last_active":"2023-08-28T02:37:07.834041+0000","last_peered":"2023-08-28T02:37:07.834041+0000","last_clean":"2023-08-28T02:37:07.834041+0000","last_became_active":"2023-08-28T02:32:58.740028+0000","last_became_peered":"2023-08-28T02:32:58.740028+0000","last_unstale":"2023-08-28T02:37:07.834041+0000","last_undegraded":"2023-08-28T02:37:07.834041+0000","last_fullsized":"2023-08-28T02:37:07.834041+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":394,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.833636+0000","last_change":"2023-08-28T02:32:58.740396+0000","last_active":"2023-08-28T02:37:07.833636+0000","last_peered":"2023-08-28T02:37:07.833636+0000","last_clean":"2023-08-28T02:37:07.833636+0000","last_became_active":"2023-08-28T02:32:58.739941+0000","last_became_peered":"2023-08-28T02:32:58.739941+0000","last_unstale":"2023-08-28T02:37:07.833636+0000","last_undegraded":"2023-08-28T02:37:07.833636+0000","last_fullsized":"2023-08-28T02:37:07.833636+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"341'7301","reported_seq":7333,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-08-28T02:37:19.869692+0000","last_change":"2023-08-28T02:35:14.495274+0000","last_active":"2023-08-28T02:37:19.869692+0000","last_peered":"2023-08-28T02:37:19.869692+0000","last_clean":"2023-08-28T02:37:19.869692+0000","last_became_active":"2023-08-28T02:35:14.494642+0000","last_became_peered":"2023-08-28T02:35:14.494642+0000","last_unstale":"2023-08-28T02:37:19.869692+0000","last_undegraded":"2023-08-28T02:37:19.869692+0000","last_fullsized":"2023-08-28T02:37:19.869692+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":7301,"log_dups_size":0,"ondisk_log_size":7301,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:28:37.992837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":326041600,"num_objects":4975,"num_object_clones":0,"num_object_copies":14925,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4975,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7301,"num_write_kb":392832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":139,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.790879+0000","last_change":"2023-08-28T02:35:05.994859+0000","last_active":"2023-08-28T02:37:07.790879+0000","last_peered":"2023-08-28T02:37:07.790879+0000","last_clean":"2023-08-28T02:37:07.790879+0000","last_became_active":"2023-08-28T02:35:05.994374+0000","last_became_peered":"2023-08-28T02:35:05.994374+0000","last_unstale":"2023-08-28T02:37:07.790879+0000","last_undegraded":"2023-08-28T02:37:07.790879+0000","last_fullsized":"2023-08-28T02:37:07.790879+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":165,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.790940+0000","last_change":"2023-08-28T02:35:45.112365+0000","last_active":"2023-08-28T02:37:07.790940+0000","last_peered":"2023-08-28T02:37:07.790940+0000","last_clean":"2023-08-28T02:37:07.790940+0000","last_became_active":"2023-08-28T02:35:45.112008+0000","last_became_peered":"2023-08-28T02:35:45.112008+0000","last_unstale":"2023-08-28T02:37:07.790940+0000","last_undegraded":"2023-08-28T02:37:07.790940+0000","last_fullsized":"2023-08-28T02:37:07.790940+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"341'7238","reported_seq":7270,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-08-28T02:37:18.347139+0000","last_change":"2023-08-28T02:35:14.498797+0000","last_active":"2023-08-28T02:37:18.347139+0000","last_peered":"2023-08-28T02:37:18.347139+0000","last_clean":"2023-08-28T02:37:18.347139+0000","last_became_active":"2023-08-28T02:35:14.498551+0000","last_became_peered":"2023-08-28T02:35:14.498551+0000","last_unstale":"2023-08-28T02:37:18.347139+0000","last_undegraded":"2023-08-28T02:37:18.347139+0000","last_fullsized":"2023-08-28T02:37:18.347139+0000","mapping_epoch":318,"log_start":"324'700","ondisk_log_start":"324'700","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":6538,"log_dups_size":0,"ondisk_log_size":6538,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:43:07.729642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327417856,"num_objects":4996,"num_object_clones":0,"num_object_copies":14988,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4996,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7238,"num_write_kb":391488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":346,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:13.872654+0000","last_change":"2023-08-28T02:28:25.445982+0000","last_active":"2023-08-28T02:37:13.872654+0000","last_peered":"2023-08-28T02:37:13.872654+0000","last_clean":"2023-08-28T02:37:13.872654+0000","last_became_active":"2023-08-28T02:28:25.443577+0000","last_became_peered":"2023-08-28T02:28:25.443577+0000","last_unstale":"2023-08-28T02:37:13.872654+0000","last_undegraded":"2023-08-28T02:37:13.872654+0000","last_fullsized":"2023-08-28T02:37:13.872654+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:24:03.381751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":342,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.797968+0000","last_change":"2023-08-28T02:32:58.887874+0000","last_active":"2023-08-28T02:37:07.797968+0000","last_peered":"2023-08-28T02:37:07.797968+0000","last_clean":"2023-08-28T02:37:07.797968+0000","last_became_active":"2023-08-28T02:32:58.887377+0000","last_became_peered":"2023-08-28T02:32:58.887377+0000","last_unstale":"2023-08-28T02:37:07.797968+0000","last_undegraded":"2023-08-28T02:37:07.797968+0000","last_fullsized":"2023-08-28T02:37:07.797968+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:08:07.234984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"341'7191","reported_seq":7223,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-08-28T02:37:18.859250+0000","last_change":"2023-08-28T02:35:14.503271+0000","last_active":"2023-08-28T02:37:18.859250+0000","last_peered":"2023-08-28T02:37:18.859250+0000","last_clean":"2023-08-28T02:37:18.859250+0000","last_became_active":"2023-08-28T02:35:14.502940+0000","last_became_peered":"2023-08-28T02:35:14.502940+0000","last_unstale":"2023-08-28T02:37:18.859250+0000","last_undegraded":"2023-08-28T02:37:18.859250+0000","last_fullsized":"2023-08-28T02:37:18.859250+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":7191,"log_dups_size":0,"ondisk_log_size":7191,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:07:49.114269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319619072,"num_objects":4877,"num_object_clones":0,"num_object_copies":14631,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4877,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7191,"num_write_kb":386176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":207,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.781396+0000","last_change":"2023-08-28T02:23:59.230059+0000","last_active":"2023-08-28T02:37:07.781396+0000","last_peered":"2023-08-28T02:37:07.781396+0000","last_clean":"2023-08-28T02:37:07.781396+0000","last_became_active":"2023-08-28T02:23:59.217783+0000","last_became_peered":"2023-08-28T02:23:59.217783+0000","last_unstale":"2023-08-28T02:37:07.781396+0000","last_undegraded":"2023-08-28T02:37:07.781396+0000","last_fullsized":"2023-08-28T02:37:07.781396+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:00:12.145395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":361,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.833878+0000","last_change":"2023-08-28T02:28:25.351698+0000","last_active":"2023-08-28T02:37:07.833878+0000","last_peered":"2023-08-28T02:37:07.833878+0000","last_clean":"2023-08-28T02:37:07.833878+0000","last_became_active":"2023-08-28T02:28:25.351349+0000","last_became_peered":"2023-08-28T02:28:25.351349+0000","last_unstale":"2023-08-28T02:37:07.833878+0000","last_undegraded":"2023-08-28T02:37:07.833878+0000","last_fullsized":"2023-08-28T02:37:07.833878+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"341'7094","reported_seq":7126,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-08-28T02:37:18.362856+0000","last_change":"2023-08-28T02:35:14.494905+0000","last_active":"2023-08-28T02:37:18.362856+0000","last_peered":"2023-08-28T02:37:18.362856+0000","last_clean":"2023-08-28T02:37:18.362856+0000","last_became_active":"2023-08-28T02:35:14.494589+0000","last_became_peered":"2023-08-28T02:35:14.494589+0000","last_unstale":"2023-08-28T02:37:18.362856+0000","last_undegraded":"2023-08-28T02:37:18.362856+0000","last_fullsized":"2023-08-28T02:37:18.362856+0000","mapping_epoch":318,"log_start":"324'500","ondisk_log_start":"324'500","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":6594,"log_dups_size":0,"ondisk_log_size":6594,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:26:28.302635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":321912832,"num_objects":4912,"num_object_clones":0,"num_object_copies":14736,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4912,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7094,"num_write_kb":384192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":147,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.790787+0000","last_change":"2023-08-28T02:32:58.629249+0000","last_active":"2023-08-28T02:37:07.790787+0000","last_peered":"2023-08-28T02:37:07.790787+0000","last_clean":"2023-08-28T02:37:07.790787+0000","last_became_active":"2023-08-28T02:32:58.628583+0000","last_became_peered":"2023-08-28T02:32:58.628583+0000","last_unstale":"2023-08-28T02:37:07.790787+0000","last_undegraded":"2023-08-28T02:37:07.790787+0000","last_fullsized":"2023-08-28T02:37:07.790787+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":414,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.834004+0000","last_change":"2023-08-28T02:35:45.100048+0000","last_active":"2023-08-28T02:37:07.834004+0000","last_peered":"2023-08-28T02:37:07.834004+0000","last_clean":"2023-08-28T02:37:07.834004+0000","last_became_active":"2023-08-28T02:35:45.099690+0000","last_became_peered":"2023-08-28T02:35:45.099690+0000","last_unstale":"2023-08-28T02:37:07.834004+0000","last_undegraded":"2023-08-28T02:37:07.834004+0000","last_fullsized":"2023-08-28T02:37:07.834004+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"341'7167","reported_seq":7198,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-08-28T02:37:17.001759+0000","last_change":"2023-08-28T02:35:14.498910+0000","last_active":"2023-08-28T02:37:17.001759+0000","last_peered":"2023-08-28T02:37:17.001759+0000","last_clean":"2023-08-28T02:37:17.001759+0000","last_became_active":"2023-08-28T02:35:14.498631+0000","last_became_peered":"2023-08-28T02:35:14.498631+0000","last_unstale":"2023-08-28T02:37:17.001759+0000","last_undegraded":"2023-08-28T02:37:17.001759+0000","last_fullsized":"2023-08-28T02:37:17.001759+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":7167,"log_dups_size":0,"ondisk_log_size":7167,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":325255168,"num_objects":4963,"num_object_clones":0,"num_object_copies":14889,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4963,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7167,"num_write_kb":388160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":277,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.833732+0000","last_change":"2023-08-28T02:32:58.740311+0000","last_active":"2023-08-28T02:37:07.833732+0000","last_peered":"2023-08-28T02:37:07.833732+0000","last_clean":"2023-08-28T02:37:07.833732+0000","last_became_active":"2023-08-28T02:32:58.739897+0000","last_became_peered":"2023-08-28T02:32:58.739897+0000","last_unstale":"2023-08-28T02:37:07.833732+0000","last_undegraded":"2023-08-28T02:37:07.833732+0000","last_fullsized":"2023-08-28T02:37:07.833732+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":328,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.795606+0000","last_change":"2023-08-28T02:32:58.766795+0000","last_active":"2023-08-28T02:37:07.795606+0000","last_peered":"2023-08-28T02:37:07.795606+0000","last_clean":"2023-08-28T02:37:07.795606+0000","last_became_active":"2023-08-28T02:32:58.766441+0000","last_became_peered":"2023-08-28T02:32:58.766441+0000","last_unstale":"2023-08-28T02:37:07.795606+0000","last_undegraded":"2023-08-28T02:37:07.795606+0000","last_fullsized":"2023-08-28T02:37:07.795606+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:58:13.963961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.3","version":"341'7201","reported_seq":7233,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-08-28T02:37:18.374159+0000","last_change":"2023-08-28T02:35:14.498188+0000","last_active":"2023-08-28T02:37:18.374159+0000","last_peered":"2023-08-28T02:37:18.374159+0000","last_clean":"2023-08-28T02:37:18.374159+0000","last_became_active":"2023-08-28T02:35:14.496278+0000","last_became_peered":"2023-08-28T02:35:14.496278+0000","last_unstale":"2023-08-28T02:37:18.374159+0000","last_undegraded":"2023-08-28T02:37:18.374159+0000","last_fullsized":"2023-08-28T02:37:18.374159+0000","mapping_epoch":318,"log_start":"324'600","ondisk_log_start":"324'600","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":6601,"log_dups_size":0,"ondisk_log_size":6601,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:43:23.633921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324468736,"num_objects":4951,"num_object_clones":0,"num_object_copies":14853,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4951,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7201,"num_write_kb":388864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":249,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.776423+0000","last_change":"2023-08-28T02:32:58.779257+0000","last_active":"2023-08-28T02:37:07.776423+0000","last_peered":"2023-08-28T02:37:07.776423+0000","last_clean":"2023-08-28T02:37:07.776423+0000","last_became_active":"2023-08-28T02:32:58.778295+0000","last_became_peered":"2023-08-28T02:32:58.778295+0000","last_unstale":"2023-08-28T02:37:07.776423+0000","last_undegraded":"2023-08-28T02:37:07.776423+0000","last_fullsized":"2023-08-28T02:37:07.776423+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:50:49.444715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":61,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.782579+0000","last_change":"2023-08-28T02:32:58.755765+0000","last_active":"2023-08-28T02:37:07.782579+0000","last_peered":"2023-08-28T02:37:07.782579+0000","last_clean":"2023-08-28T02:37:07.782579+0000","last_became_active":"2023-08-28T02:32:58.755438+0000","last_became_peered":"2023-08-28T02:32:58.755438+0000","last_unstale":"2023-08-28T02:37:07.782579+0000","last_undegraded":"2023-08-28T02:37:07.782579+0000","last_fullsized":"2023-08-28T02:37:07.782579+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:14:11.498611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"341'7261","reported_seq":7293,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-08-28T02:37:19.860780+0000","last_change":"2023-08-28T02:35:14.497511+0000","last_active":"2023-08-28T02:37:19.860780+0000","last_peered":"2023-08-28T02:37:19.860780+0000","last_clean":"2023-08-28T02:37:19.860780+0000","last_became_active":"2023-08-28T02:35:14.497180+0000","last_became_peered":"2023-08-28T02:35:14.497180+0000","last_unstale":"2023-08-28T02:37:19.860780+0000","last_undegraded":"2023-08-28T02:37:19.860780+0000","last_fullsized":"2023-08-28T02:37:19.860780+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":7261,"log_dups_size":0,"ondisk_log_size":7261,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:59:26.376998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":321191936,"num_objects":4901,"num_object_clones":0,"num_object_copies":14703,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4901,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7261,"num_write_kb":389184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"78'126","reported_seq":746,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:13.384325+0000","last_change":"2023-08-28T02:35:09.735906+0000","last_active":"2023-08-28T02:37:13.384325+0000","last_peered":"2023-08-28T02:37:13.384325+0000","last_clean":"2023-08-28T02:37:13.384325+0000","last_became_active":"2023-08-28T02:32:59.660388+0000","last_became_peered":"2023-08-28T02:32:59.660388+0000","last_unstale":"2023-08-28T02:37:13.384325+0000","last_undegraded":"2023-08-28T02:37:13.384325+0000","last_fullsized":"2023-08-28T02:37:13.384325+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:19:53.565088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":743,"num_read_kb":1331,"num_write":125,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":183,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-08-28T02:37:16.673541+0000","last_change":"2023-08-28T02:35:05.991133+0000","last_active":"2023-08-28T02:37:16.673541+0000","last_peered":"2023-08-28T02:37:16.673541+0000","last_clean":"2023-08-28T02:37:16.673541+0000","last_became_active":"2023-08-28T02:35:05.990829+0000","last_became_peered":"2023-08-28T02:35:05.990829+0000","last_unstale":"2023-08-28T02:37:16.673541+0000","last_undegraded":"2023-08-28T02:37:16.673541+0000","last_fullsized":"2023-08-28T02:37:16.673541+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:40:28.077023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":23,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.797173+0000","last_change":"2023-08-28T02:35:45.099619+0000","last_active":"2023-08-28T02:37:07.797173+0000","last_peered":"2023-08-28T02:37:07.797173+0000","last_clean":"2023-08-28T02:37:07.797173+0000","last_became_active":"2023-08-28T02:35:45.099285+0000","last_became_peered":"2023-08-28T02:35:45.099285+0000","last_unstale":"2023-08-28T02:37:07.797173+0000","last_undegraded":"2023-08-28T02:37:07.797173+0000","last_fullsized":"2023-08-28T02:37:07.797173+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:24:58.305957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.8","version":"341'7330","reported_seq":7361,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-08-28T02:37:18.859065+0000","last_change":"2023-08-28T02:35:14.497638+0000","last_active":"2023-08-28T02:37:18.859065+0000","last_peered":"2023-08-28T02:37:18.859065+0000","last_clean":"2023-08-28T02:37:18.859065+0000","last_became_active":"2023-08-28T02:35:14.497323+0000","last_became_peered":"2023-08-28T02:35:14.497323+0000","last_unstale":"2023-08-28T02:37:18.859065+0000","last_undegraded":"2023-08-28T02:37:18.859065+0000","last_fullsized":"2023-08-28T02:37:18.859065+0000","mapping_epoch":318,"log_start":"324'100","ondisk_log_start":"324'100","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":7230,"log_dups_size":0,"ondisk_log_size":7230,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:30:06.078735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323354624,"num_objects":4934,"num_object_clones":0,"num_object_copies":14802,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4934,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7330,"num_write_kb":392448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"78'204","reported_seq":1426,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:08.371619+0000","last_change":"2023-08-28T02:32:59.644574+0000","last_active":"2023-08-28T02:37:08.371619+0000","last_peered":"2023-08-28T02:37:08.371619+0000","last_clean":"2023-08-28T02:37:08.371619+0000","last_became_active":"2023-08-28T02:32:59.644053+0000","last_became_peered":"2023-08-28T02:32:59.644053+0000","last_unstale":"2023-08-28T02:37:08.371619+0000","last_undegraded":"2023-08-28T02:37:08.371619+0000","last_fullsized":"2023-08-28T02:37:08.371619+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":204,"log_dups_size":0,"ondisk_log_size":204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:55:51.460483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1521,"num_read_kb":2511,"num_write":178,"num_write_kb":1175,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":335,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.833919+0000","last_change":"2023-08-28T02:28:25.352417+0000","last_active":"2023-08-28T02:37:07.833919+0000","last_peered":"2023-08-28T02:37:07.833919+0000","last_clean":"2023-08-28T02:37:07.833919+0000","last_became_active":"2023-08-28T02:28:25.352216+0000","last_became_peered":"2023-08-28T02:28:25.352216+0000","last_unstale":"2023-08-28T02:37:07.833919+0000","last_undegraded":"2023-08-28T02:37:07.833919+0000","last_fullsized":"2023-08-28T02:37:07.833919+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":62,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:13.384308+0000","last_change":"2023-08-28T02:32:58.720576+0000","last_active":"2023-08-28T02:37:13.384308+0000","last_peered":"2023-08-28T02:37:13.384308+0000","last_clean":"2023-08-28T02:37:13.384308+0000","last_became_active":"2023-08-28T02:32:58.719924+0000","last_became_peered":"2023-08-28T02:32:58.719924+0000","last_unstale":"2023-08-28T02:37:13.384308+0000","last_undegraded":"2023-08-28T02:37:13.384308+0000","last_fullsized":"2023-08-28T02:37:13.384308+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:28:50.697438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"341'7171","reported_seq":7203,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-08-28T02:37:18.803488+0000","last_change":"2023-08-28T02:35:14.503279+0000","last_active":"2023-08-28T02:37:18.803488+0000","last_peered":"2023-08-28T02:37:18.803488+0000","last_clean":"2023-08-28T02:37:18.803488+0000","last_became_active":"2023-08-28T02:35:14.502953+0000","last_became_peered":"2023-08-28T02:35:14.502953+0000","last_unstale":"2023-08-28T02:37:18.803488+0000","last_undegraded":"2023-08-28T02:37:18.803488+0000","last_fullsized":"2023-08-28T02:37:18.803488+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":7171,"log_dups_size":0,"ondisk_log_size":7171,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:55:29.284519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":325910528,"num_objects":4973,"num_object_clones":0,"num_object_copies":14919,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4973,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7171,"num_write_kb":388608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"78'47","reported_seq":561,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.797510+0000","last_change":"2023-08-28T02:24:05.799647+0000","last_active":"2023-08-28T02:37:07.797510+0000","last_peered":"2023-08-28T02:37:07.797510+0000","last_clean":"2023-08-28T02:37:07.797510+0000","last_became_active":"2023-08-28T02:23:59.210481+0000","last_became_peered":"2023-08-28T02:23:59.210481+0000","last_unstale":"2023-08-28T02:37:07.797510+0000","last_undegraded":"2023-08-28T02:37:07.797510+0000","last_fullsized":"2023-08-28T02:37:07.797510+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":47,"log_dups_size":0,"ondisk_log_size":47,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:40:18.542375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":162,"num_read_kb":1281,"num_write":48,"num_write_kb":1153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":62,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.779902+0000","last_change":"2023-08-28T02:32:58.622079+0000","last_active":"2023-08-28T02:37:07.779902+0000","last_peered":"2023-08-28T02:37:07.779902+0000","last_clean":"2023-08-28T02:37:07.779902+0000","last_became_active":"2023-08-28T02:32:58.621819+0000","last_became_peered":"2023-08-28T02:32:58.621819+0000","last_unstale":"2023-08-28T02:37:07.779902+0000","last_undegraded":"2023-08-28T02:37:07.779902+0000","last_fullsized":"2023-08-28T02:37:07.779902+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:03:36.676500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":62,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-08-28T02:37:16.661658+0000","last_change":"2023-08-28T02:32:58.724424+0000","last_active":"2023-08-28T02:37:16.661658+0000","last_peered":"2023-08-28T02:37:16.661658+0000","last_clean":"2023-08-28T02:37:16.661658+0000","last_became_active":"2023-08-28T02:32:58.724127+0000","last_became_peered":"2023-08-28T02:32:58.724127+0000","last_unstale":"2023-08-28T02:37:16.661658+0000","last_undegraded":"2023-08-28T02:37:16.661658+0000","last_fullsized":"2023-08-28T02:37:16.661658+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:02:25.178712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"341'7191","reported_seq":7222,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-08-28T02:37:17.238793+0000","last_change":"2023-08-28T02:35:14.500884+0000","last_active":"2023-08-28T02:37:17.238793+0000","last_peered":"2023-08-28T02:37:17.238793+0000","last_clean":"2023-08-28T02:37:17.238793+0000","last_became_active":"2023-08-28T02:35:14.500643+0000","last_became_peered":"2023-08-28T02:35:14.500643+0000","last_unstale":"2023-08-28T02:37:17.238793+0000","last_undegraded":"2023-08-28T02:37:17.238793+0000","last_fullsized":"2023-08-28T02:37:17.238793+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":7191,"log_dups_size":0,"ondisk_log_size":7191,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332070912,"num_objects":5067,"num_object_clones":0,"num_object_copies":15201,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5067,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7191,"num_write_kb":392256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"78'114","reported_seq":395,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:15.043394+0000","last_change":"2023-08-28T02:35:05.992001+0000","last_active":"2023-08-28T02:37:15.043394+0000","last_peered":"2023-08-28T02:37:15.043394+0000","last_clean":"2023-08-28T02:37:15.043394+0000","last_became_active":"2023-08-28T02:35:05.991513+0000","last_became_peered":"2023-08-28T02:35:05.991513+0000","last_unstale":"2023-08-28T02:37:15.043394+0000","last_undegraded":"2023-08-28T02:37:15.043394+0000","last_fullsized":"2023-08-28T02:37:15.043394+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":114,"log_dups_size":0,"ondisk_log_size":114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:00:42.194692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":358,"num_read_kb":1352,"num_write":102,"num_write_kb":1104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"86'139","reported_seq":426,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.795028+0000","last_change":"2023-08-28T02:35:09.566656+0000","last_active":"2023-08-28T02:37:07.795028+0000","last_peered":"2023-08-28T02:37:07.795028+0000","last_clean":"2023-08-28T02:37:07.795028+0000","last_became_active":"2023-08-28T02:32:59.651089+0000","last_became_peered":"2023-08-28T02:32:59.651089+0000","last_unstale":"2023-08-28T02:37:07.795028+0000","last_undegraded":"2023-08-28T02:37:07.795028+0000","last_fullsized":"2023-08-28T02:37:07.795028+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_clean_scrub_stamp":"2023-08-28T02:08:02.043718+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:31:19.261767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":222,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.777777+0000","last_change":"2023-08-28T02:28:25.347600+0000","last_active":"2023-08-28T02:37:07.777777+0000","last_peered":"2023-08-28T02:37:07.777777+0000","last_clean":"2023-08-28T02:37:07.777777+0000","last_became_active":"2023-08-28T02:28:25.347237+0000","last_became_peered":"2023-08-28T02:28:25.347237+0000","last_unstale":"2023-08-28T02:37:07.777777+0000","last_undegraded":"2023-08-28T02:37:07.777777+0000","last_fullsized":"2023-08-28T02:37:07.777777+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:29:26.767559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":138,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.791125+0000","last_change":"2023-08-28T02:31:55.517732+0000","last_active":"2023-08-28T02:37:07.791125+0000","last_peered":"2023-08-28T02:37:07.791125+0000","last_clean":"2023-08-28T02:37:07.791125+0000","last_became_active":"2023-08-28T02:31:55.517347+0000","last_became_peered":"2023-08-28T02:31:55.517347+0000","last_unstale":"2023-08-28T02:37:07.791125+0000","last_undegraded":"2023-08-28T02:37:07.791125+0000","last_fullsized":"2023-08-28T02:37:07.791125+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"341'7093","reported_seq":7124,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-08-28T02:37:18.392245+0000","last_change":"2023-08-28T02:35:14.493898+0000","last_active":"2023-08-28T02:37:18.392245+0000","last_peered":"2023-08-28T02:37:18.392245+0000","last_clean":"2023-08-28T02:37:18.392245+0000","last_became_active":"2023-08-28T02:35:14.493649+0000","last_became_peered":"2023-08-28T02:35:14.493649+0000","last_unstale":"2023-08-28T02:37:18.392245+0000","last_undegraded":"2023-08-28T02:37:18.392245+0000","last_fullsized":"2023-08-28T02:37:18.392245+0000","mapping_epoch":318,"log_start":"324'500","ondisk_log_start":"324'500","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":6593,"log_dups_size":0,"ondisk_log_size":6593,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:49:46.405255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318963712,"num_objects":4867,"num_object_clones":0,"num_object_copies":14601,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4867,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7093,"num_write_kb":382720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'148","reported_seq":796,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.779992+0000","last_change":"2023-08-28T02:35:10.122879+0000","last_active":"2023-08-28T02:37:07.779992+0000","last_peered":"2023-08-28T02:37:07.779992+0000","last_clean":"2023-08-28T02:37:07.779992+0000","last_became_active":"2023-08-28T02:32:59.661273+0000","last_became_peered":"2023-08-28T02:32:59.661273+0000","last_unstale":"2023-08-28T02:37:07.779992+0000","last_undegraded":"2023-08-28T02:37:07.779992+0000","last_fullsized":"2023-08-28T02:37:07.779992+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:07:46.974854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1092,"num_read_kb":1955,"num_write":127,"num_write_kb":5177,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":312,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.797884+0000","last_change":"2023-08-28T02:23:59.226986+0000","last_active":"2023-08-28T02:37:07.797884+0000","last_peered":"2023-08-28T02:37:07.797884+0000","last_clean":"2023-08-28T02:37:07.797884+0000","last_became_active":"2023-08-28T02:23:59.226529+0000","last_became_peered":"2023-08-28T02:23:59.226529+0000","last_unstale":"2023-08-28T02:37:07.797884+0000","last_undegraded":"2023-08-28T02:37:07.797884+0000","last_fullsized":"2023-08-28T02:37:07.797884+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:22:09.193619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":375,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.795319+0000","last_change":"2023-08-28T02:32:58.744332+0000","last_active":"2023-08-28T02:37:07.795319+0000","last_peered":"2023-08-28T02:37:07.795319+0000","last_clean":"2023-08-28T02:37:07.795319+0000","last_became_active":"2023-08-28T02:32:58.744025+0000","last_became_peered":"2023-08-28T02:32:58.744025+0000","last_unstale":"2023-08-28T02:37:07.795319+0000","last_undegraded":"2023-08-28T02:37:07.795319+0000","last_fullsized":"2023-08-28T02:37:07.795319+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:41:15.957049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"341'7239","reported_seq":7429,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-08-28T02:37:18.835720+0000","last_change":"2023-08-28T02:35:24.994481+0000","last_active":"2023-08-28T02:37:18.835720+0000","last_peered":"2023-08-28T02:37:18.835720+0000","last_clean":"2023-08-28T02:37:18.835720+0000","last_became_active":"2023-08-28T02:35:24.668942+0000","last_became_peered":"2023-08-28T02:35:24.668942+0000","last_unstale":"2023-08-28T02:37:18.835720+0000","last_undegraded":"2023-08-28T02:37:18.835720+0000","last_fullsized":"2023-08-28T02:37:18.835720+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":7239,"log_dups_size":0,"ondisk_log_size":7239,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:16:02.211038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":326828032,"num_objects":4987,"num_object_clones":0,"num_object_copies":14961,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4987,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7239,"num_write_kb":391232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":73,"num_bytes_recovered":4784128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"78'103","reported_seq":672,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.795695+0000","last_change":"2023-08-28T02:35:09.798248+0000","last_active":"2023-08-28T02:37:07.795695+0000","last_peered":"2023-08-28T02:37:07.795695+0000","last_clean":"2023-08-28T02:37:07.795695+0000","last_became_active":"2023-08-28T02:32:59.656276+0000","last_became_peered":"2023-08-28T02:32:59.656276+0000","last_unstale":"2023-08-28T02:37:07.795695+0000","last_undegraded":"2023-08-28T02:37:07.795695+0000","last_fullsized":"2023-08-28T02:37:07.795695+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:34:10.622592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":803,"num_read_kb":2080,"num_write":73,"num_write_kb":1244,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":62,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:13.384290+0000","last_change":"2023-08-28T02:32:58.720401+0000","last_active":"2023-08-28T02:37:13.384290+0000","last_peered":"2023-08-28T02:37:13.384290+0000","last_clean":"2023-08-28T02:37:13.384290+0000","last_became_active":"2023-08-28T02:32:58.719518+0000","last_became_peered":"2023-08-28T02:32:58.719518+0000","last_unstale":"2023-08-28T02:37:13.384290+0000","last_undegraded":"2023-08-28T02:37:13.384290+0000","last_fullsized":"2023-08-28T02:37:13.384290+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:46:28.947812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":325,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.798199+0000","last_change":"2023-08-28T02:32:58.887691+0000","last_active":"2023-08-28T02:37:07.798199+0000","last_peered":"2023-08-28T02:37:07.798199+0000","last_clean":"2023-08-28T02:37:07.798199+0000","last_became_active":"2023-08-28T02:32:58.886906+0000","last_became_peered":"2023-08-28T02:32:58.886906+0000","last_unstale":"2023-08-28T02:37:07.798199+0000","last_undegraded":"2023-08-28T02:37:07.798199+0000","last_fullsized":"2023-08-28T02:37:07.798199+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:39:29.229171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"341'7005","reported_seq":7037,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-08-28T02:37:19.869582+0000","last_change":"2023-08-28T02:35:14.497565+0000","last_active":"2023-08-28T02:37:19.869582+0000","last_peered":"2023-08-28T02:37:19.869582+0000","last_clean":"2023-08-28T02:37:19.869582+0000","last_became_active":"2023-08-28T02:35:14.497285+0000","last_became_peered":"2023-08-28T02:35:14.497285+0000","last_unstale":"2023-08-28T02:37:19.869582+0000","last_undegraded":"2023-08-28T02:37:19.869582+0000","last_fullsized":"2023-08-28T02:37:19.869582+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":7005,"log_dups_size":0,"ondisk_log_size":7005,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:04:31.778861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311361536,"num_objects":4751,"num_object_clones":0,"num_object_copies":14253,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4751,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7005,"num_write_kb":376192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":221,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.778976+0000","last_change":"2023-08-28T02:31:55.535099+0000","last_active":"2023-08-28T02:37:07.778976+0000","last_peered":"2023-08-28T02:37:07.778976+0000","last_clean":"2023-08-28T02:37:07.778976+0000","last_became_active":"2023-08-28T02:31:55.527768+0000","last_became_peered":"2023-08-28T02:31:55.527768+0000","last_unstale":"2023-08-28T02:37:07.778976+0000","last_undegraded":"2023-08-28T02:37:07.778976+0000","last_fullsized":"2023-08-28T02:37:07.778976+0000","mapping_epoch":276,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":277,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:43:30.804793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'112","reported_seq":495,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-08-28T02:37:16.662220+0000","last_change":"2023-08-28T02:32:59.643465+0000","last_active":"2023-08-28T02:37:16.662220+0000","last_peered":"2023-08-28T02:37:16.662220+0000","last_clean":"2023-08-28T02:37:16.662220+0000","last_became_active":"2023-08-28T02:32:59.642979+0000","last_became_peered":"2023-08-28T02:32:59.642979+0000","last_unstale":"2023-08-28T02:37:16.662220+0000","last_undegraded":"2023-08-28T02:37:16.662220+0000","last_fullsized":"2023-08-28T02:37:16.662220+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:48:45.923015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":444,"num_read_kb":473,"num_write":94,"num_write_kb":127,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":231,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.776353+0000","last_change":"2023-08-28T02:32:58.833408+0000","last_active":"2023-08-28T02:37:07.776353+0000","last_peered":"2023-08-28T02:37:07.776353+0000","last_clean":"2023-08-28T02:37:07.776353+0000","last_became_active":"2023-08-28T02:32:58.821347+0000","last_became_peered":"2023-08-28T02:32:58.821347+0000","last_unstale":"2023-08-28T02:37:07.776353+0000","last_undegraded":"2023-08-28T02:37:07.776353+0000","last_fullsized":"2023-08-28T02:37:07.776353+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:41:25.683179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"341'7142","reported_seq":7175,"reported_epoch":341,"state":"active+clean","last_fresh":"2023-08-28T02:37:19.986474+0000","last_change":"2023-08-28T02:35:14.495405+0000","last_active":"2023-08-28T02:37:19.986474+0000","last_peered":"2023-08-28T02:37:19.986474+0000","last_clean":"2023-08-28T02:37:19.986474+0000","last_became_active":"2023-08-28T02:35:14.494591+0000","last_became_peered":"2023-08-28T02:35:14.494591+0000","last_unstale":"2023-08-28T02:37:19.986474+0000","last_undegraded":"2023-08-28T02:37:19.986474+0000","last_fullsized":"2023-08-28T02:37:19.986474+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":7142,"log_dups_size":0,"ondisk_log_size":7142,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:47:27.839376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314769432,"num_objects":4804,"num_object_clones":0,"num_object_copies":14412,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4804,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7142,"num_write_kb":382209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"78'122","reported_seq":684,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.791088+0000","last_change":"2023-08-28T02:33:00.672889+0000","last_active":"2023-08-28T02:37:07.791088+0000","last_peered":"2023-08-28T02:37:07.791088+0000","last_clean":"2023-08-28T02:37:07.791088+0000","last_became_active":"2023-08-28T02:33:00.672494+0000","last_became_peered":"2023-08-28T02:33:00.672494+0000","last_unstale":"2023-08-28T02:37:07.791088+0000","last_undegraded":"2023-08-28T02:37:07.791088+0000","last_fullsized":"2023-08-28T02:37:07.791088+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":294,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":857,"num_read_kb":2224,"num_write":99,"num_write_kb":2312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":350,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.795399+0000","last_change":"2023-08-28T02:32:58.795995+0000","last_active":"2023-08-28T02:37:07.795399+0000","last_peered":"2023-08-28T02:37:07.795399+0000","last_clean":"2023-08-28T02:37:07.795399+0000","last_became_active":"2023-08-28T02:32:58.771872+0000","last_became_peered":"2023-08-28T02:32:58.771872+0000","last_unstale":"2023-08-28T02:37:07.795399+0000","last_undegraded":"2023-08-28T02:37:07.795399+0000","last_fullsized":"2023-08-28T02:37:07.795399+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:16:02.965467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":310,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.833546+0000","last_change":"2023-08-28T02:28:25.351840+0000","last_active":"2023-08-28T02:37:07.833546+0000","last_peered":"2023-08-28T02:37:07.833546+0000","last_clean":"2023-08-28T02:37:07.833546+0000","last_became_active":"2023-08-28T02:28:25.351399+0000","last_became_peered":"2023-08-28T02:28:25.351399+0000","last_unstale":"2023-08-28T02:37:07.833546+0000","last_undegraded":"2023-08-28T02:37:07.833546+0000","last_fullsized":"2023-08-28T02:37:07.833546+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":327,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.798129+0000","last_change":"2023-08-28T02:32:58.751818+0000","last_active":"2023-08-28T02:37:07.798129+0000","last_peered":"2023-08-28T02:37:07.798129+0000","last_clean":"2023-08-28T02:37:07.798129+0000","last_became_active":"2023-08-28T02:32:58.743388+0000","last_became_peered":"2023-08-28T02:32:58.743388+0000","last_unstale":"2023-08-28T02:37:07.798129+0000","last_undegraded":"2023-08-28T02:37:07.798129+0000","last_fullsized":"2023-08-28T02:37:07.798129+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:52:12.910174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":28,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.797717+0000","last_change":"2023-08-28T02:35:24.666417+0000","last_active":"2023-08-28T02:37:07.797717+0000","last_peered":"2023-08-28T02:37:07.797717+0000","last_clean":"2023-08-28T02:37:07.797717+0000","last_became_active":"2023-08-28T02:35:24.666107+0000","last_became_peered":"2023-08-28T02:35:24.666107+0000","last_unstale":"2023-08-28T02:37:07.797717+0000","last_undegraded":"2023-08-28T02:37:07.797717+0000","last_fullsized":"2023-08-28T02:37:07.797717+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:22:55.497120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":173,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.833830+0000","last_change":"2023-08-28T02:32:58.839847+0000","last_active":"2023-08-28T02:37:07.833830+0000","last_peered":"2023-08-28T02:37:07.833830+0000","last_clean":"2023-08-28T02:37:07.833830+0000","last_became_active":"2023-08-28T02:32:58.628159+0000","last_became_peered":"2023-08-28T02:32:58.628159+0000","last_unstale":"2023-08-28T02:37:07.833830+0000","last_undegraded":"2023-08-28T02:37:07.833830+0000","last_fullsized":"2023-08-28T02:37:07.833830+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":376,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.834112+0000","last_change":"2023-08-28T02:32:58.629596+0000","last_active":"2023-08-28T02:37:07.834112+0000","last_peered":"2023-08-28T02:37:07.834112+0000","last_clean":"2023-08-28T02:37:07.834112+0000","last_became_active":"2023-08-28T02:32:58.629136+0000","last_became_peered":"2023-08-28T02:32:58.629136+0000","last_unstale":"2023-08-28T02:37:07.834112+0000","last_undegraded":"2023-08-28T02:37:07.834112+0000","last_fullsized":"2023-08-28T02:37:07.834112+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":160,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.834095+0000","last_change":"2023-08-28T02:28:25.705331+0000","last_active":"2023-08-28T02:37:07.834095+0000","last_peered":"2023-08-28T02:37:07.834095+0000","last_clean":"2023-08-28T02:37:07.834095+0000","last_became_active":"2023-08-28T02:28:25.397480+0000","last_became_peered":"2023-08-28T02:28:25.397480+0000","last_unstale":"2023-08-28T02:37:07.834095+0000","last_undegraded":"2023-08-28T02:37:07.834095+0000","last_fullsized":"2023-08-28T02:37:07.834095+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":340,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.833656+0000","last_change":"2023-08-28T02:23:59.207929+0000","last_active":"2023-08-28T02:37:07.833656+0000","last_peered":"2023-08-28T02:37:07.833656+0000","last_clean":"2023-08-28T02:37:07.833656+0000","last_became_active":"2023-08-28T02:23:59.207589+0000","last_became_peered":"2023-08-28T02:23:59.207589+0000","last_unstale":"2023-08-28T02:37:07.833656+0000","last_undegraded":"2023-08-28T02:37:07.833656+0000","last_fullsized":"2023-08-28T02:37:07.833656+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":231,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.778425+0000","last_change":"2023-08-28T02:28:25.347988+0000","last_active":"2023-08-28T02:37:07.778425+0000","last_peered":"2023-08-28T02:37:07.778425+0000","last_clean":"2023-08-28T02:37:07.778425+0000","last_became_active":"2023-08-28T02:28:25.347632+0000","last_became_peered":"2023-08-28T02:28:25.347632+0000","last_unstale":"2023-08-28T02:37:07.778425+0000","last_undegraded":"2023-08-28T02:37:07.778425+0000","last_fullsized":"2023-08-28T02:37:07.778425+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:56:20.202662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":203,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.777920+0000","last_change":"2023-08-28T02:23:59.217794+0000","last_active":"2023-08-28T02:37:07.777920+0000","last_peered":"2023-08-28T02:37:07.777920+0000","last_clean":"2023-08-28T02:37:07.777920+0000","last_became_active":"2023-08-28T02:23:59.217231+0000","last_became_peered":"2023-08-28T02:23:59.217231+0000","last_unstale":"2023-08-28T02:37:07.777920+0000","last_undegraded":"2023-08-28T02:37:07.777920+0000","last_fullsized":"2023-08-28T02:37:07.777920+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:36:26.297410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":328,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.795455+0000","last_change":"2023-08-28T02:35:24.680274+0000","last_active":"2023-08-28T02:37:07.795455+0000","last_peered":"2023-08-28T02:37:07.795455+0000","last_clean":"2023-08-28T02:37:07.795455+0000","last_became_active":"2023-08-28T02:35:24.673318+0000","last_became_peered":"2023-08-28T02:35:24.673318+0000","last_unstale":"2023-08-28T02:37:07.795455+0000","last_undegraded":"2023-08-28T02:37:07.795455+0000","last_fullsized":"2023-08-28T02:37:07.795455+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:22:46.371303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":399,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-08-28T02:37:07.833510+0000","last_change":"2023-08-28T02:32:58.630270+0000","last_active":"2023-08-28T02:37:07.833510+0000","last_peered":"2023-08-28T02:37:07.833510+0000","last_clean":"2023-08-28T02:37:07.833510+0000","last_became_active":"2023-08-28T02:32:58.629958+0000","last_became_peered":"2023-08-28T02:32:58.629958+0000","last_unstale":"2023-08-28T02:37:07.833510+0000","last_undegraded":"2023-08-28T02:37:07.833510+0000","last_fullsized":"2023-08-28T02:37:07.833510+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":62,"num_bytes_recovered":25254,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":21462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26810,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":5179310104,"num_objects":79031,"num_object_clones":0,"num_object_copies":237093,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":79031,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":114785,"num_write_kb":6202049,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":73,"num_bytes_recovered":4784128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15562256384,"data_stored":15562244168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":112385,"ondisk_log_size":112385,"up":48,"acting":48,"num_store_stats":7},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":156,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16789504,"data_stored":16777273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":1335734829086,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":42,"seq":180388626767,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2247800,"kb_used_data":1942384,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":91497352,"statfs":{"total":95995035648,"available":93693288448,"internally_reserved":0,"allocated":1989001216,"data_stored":1988115667,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:36:20 2023","interfaces":[{"interface":"back","average":{"1min":1.0980000000000001,"5min":3.0529999999999999,"15min":3.6560000000000001},"min":{"1min":0.23000000000000001,"5min":0.215,"15min":0.20799999999999999},"max":{"1min":3.5760000000000001,"5min":121.78,"15min":121.78},"last":1.3069999999999999},{"interface":"front","average":{"1min":1.254,"5min":3.6309999999999998,"15min":3.7549999999999999},"min":{"1min":0.29999999999999999,"5min":0.23699999999999999,"15min":0.23200000000000001},"max":{"1min":7.4370000000000003,"5min":121.79900000000001,"15min":121.79900000000001},"last":3.3650000000000002}]},{"osd":1,"last update":"Mon Aug 28 02:36:20 2023","interfaces":[{"interface":"back","average":{"1min":1.373,"5min":1.2589999999999999,"15min":2.6880000000000002},"min":{"1min":0.315,"5min":0.23799999999999999,"15min":0.23699999999999999},"max":{"1min":4.1959999999999997,"5min":8.2639999999999993,"15min":141.87299999999999},"last":0.33700000000000002},{"interface":"front","average":{"1min":1.5640000000000001,"5min":1.484,"15min":2.8450000000000002},"min":{"1min":0.25,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":4.8440000000000003,"5min":8.5180000000000007,"15min":141.904},"last":0.28999999999999998}]},{"osd":2,"last update":"Mon Aug 28 02:37:09 2023","interfaces":[{"interface":"back","average":{"1min":1.2210000000000001,"5min":5.6150000000000002,"15min":4.2140000000000004},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":2.839,"5min":234.876,"15min":234.876},"last":1.097},{"interface":"front","average":{"1min":1.7769999999999999,"5min":5.5949999999999998,"15min":4.1929999999999996},"min":{"1min":0.30299999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":6.5119999999999996,"5min":234.482,"15min":234.482},"last":1.5189999999999999}]},{"osd":3,"last update":"Mon Aug 28 02:36:20 2023","interfaces":[{"interface":"back","average":{"1min":1.8480000000000001,"5min":3.3069999999999999,"15min":3.2069999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":7.2750000000000004,"5min":143.19499999999999,"15min":157.625},"last":1.238},{"interface":"front","average":{"1min":2.1070000000000002,"5min":3.3079999999999998,"15min":3.2149999999999999},"min":{"1min":0.38200000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":7.2869999999999999,"5min":143.20699999999999,"15min":157.637},"last":1.25}]},{"osd":4,"last update":"Mon Aug 28 02:36:25 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.50600000000000001,"15min":1.103},"min":{"1min":0.28999999999999998,"5min":0.251,"15min":0.251},"max":{"1min":1.645,"5min":3.8610000000000002,"15min":18.308},"last":1.002},{"interface":"front","average":{"1min":0.73199999999999998,"5min":0.68799999999999994,"15min":1.2150000000000001},"min":{"1min":0.28100000000000003,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":2.1019999999999999,"5min":3.8450000000000002,"15min":18.263000000000002},"last":0.32400000000000001}]},{"osd":5,"last update":"Mon Aug 28 02:36:25 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.61099999999999999,"15min":1.1870000000000001},"min":{"1min":0.29699999999999999,"5min":0.29299999999999998,"15min":0.26300000000000001},"max":{"1min":2.8639999999999999,"5min":4.2800000000000002,"15min":20.759},"last":0.30399999999999999},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.52700000000000002,"15min":0.997},"min":{"1min":0.32100000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":1.399,"5min":2.972,"15min":7.3780000000000001},"last":0.39400000000000002}]},{"osd":7,"last update":"Mon Aug 28 02:36:58 2023","interfaces":[{"interface":"back","average":{"1min":0.997,"5min":0.64400000000000002,"15min":0.58499999999999996},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":3.0419999999999998,"5min":3.0419999999999998,"15min":3.0419999999999998},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.53200000000000003,"15min":0.51500000000000001},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":2.3450000000000002,"5min":2.3450000000000002,"15min":2.3450000000000002},"last":0.45900000000000002}]}]},{"osd":1,"up_from":98,"seq":420906795220,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2760064,"kb_used_data":2235576,"kb_used_omap":7,"kb_used_meta":524408,"kb_avail":90985088,"statfs":{"total":95995035648,"available":93168730112,"internally_reserved":0,"allocated":2289229824,"data_stored":2287478724,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":536994123},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,2,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:36:18 2023","interfaces":[{"interface":"back","average":{"1min":3.3809999999999998,"5min":6.6779999999999999,"15min":4.4089999999999998},"min":{"1min":0.23799999999999999,"5min":0.189,"15min":0.17499999999999999},"max":{"1min":36.838000000000001,"5min":129.346,"15min":129.346},"last":0.56899999999999995},{"interface":"front","average":{"1min":2.9279999999999999,"5min":6.6210000000000004,"15min":4.4329999999999998},"min":{"1min":0.19800000000000001,"5min":0.19700000000000001,"15min":0.188},"max":{"1min":36.622,"5min":120.774,"15min":128.685},"last":4.4409999999999998}]},{"osd":2,"last update":"Mon Aug 28 02:36:48 2023","interfaces":[{"interface":"back","average":{"1min":2.3050000000000002,"5min":4.0629999999999997,"15min":4.274},"min":{"1min":0.22900000000000001,"5min":0.189,"15min":0.189},"max":{"1min":7.9279999999999999,"5min":119.276,"15min":135.25999999999999},"last":4.7569999999999997},{"interface":"front","average":{"1min":2.641,"5min":3.7919999999999998,"15min":3.9609999999999999},"min":{"1min":0.251,"5min":0.221,"15min":0.218},"max":{"1min":11.074,"5min":121.849,"15min":135.10300000000001},"last":2.323}]},{"osd":3,"last update":"Mon Aug 28 02:36:18 2023","interfaces":[{"interface":"back","average":{"1min":1.6839999999999999,"5min":7.7859999999999996,"15min":4.4699999999999998},"min":{"1min":0.21299999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":4.29,"5min":126.95699999999999,"15min":126.95699999999999},"last":4.6559999999999997},{"interface":"front","average":{"1min":2.141,"5min":7.4969999999999999,"15min":4.5890000000000004},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":10.477,"5min":128.00200000000001,"15min":128.00200000000001},"last":4.5990000000000002}]},{"osd":4,"last update":"Mon Aug 28 02:36:18 2023","interfaces":[{"interface":"back","average":{"1min":2.7170000000000001,"5min":1.7410000000000001,"15min":1.6899999999999999},"min":{"1min":0.29699999999999999,"5min":0.28199999999999997,"15min":0.27500000000000002},"max":{"1min":24.809999999999999,"5min":24.809999999999999,"15min":24.809999999999999},"last":4.2690000000000001},{"interface":"front","average":{"1min":2.403,"5min":1.546,"15min":1.484},"min":{"1min":0.378,"5min":0.29499999999999998,"15min":0.25600000000000001},"max":{"1min":24.835999999999999,"5min":24.835999999999999,"15min":24.835999999999999},"last":0.51900000000000002}]},{"osd":5,"last update":"Mon Aug 28 02:36:18 2023","interfaces":[{"interface":"back","average":{"1min":8.0570000000000004,"5min":2.9809999999999999,"15min":2.2010000000000001},"min":{"1min":0.34399999999999997,"5min":0.252,"15min":0.252},"max":{"1min":82.722999999999999,"5min":82.722999999999999,"15min":82.722999999999999},"last":4.3579999999999997},{"interface":"front","average":{"1min":8.4969999999999999,"5min":2.9540000000000002,"15min":2.081},"min":{"1min":0.32900000000000001,"5min":0.307,"15min":0.307},"max":{"1min":82.733999999999995,"5min":82.733999999999995,"15min":82.733999999999995},"last":4.306}]},{"osd":6,"last update":"Mon Aug 28 02:36:18 2023","interfaces":[{"interface":"back","average":{"1min":3.024,"5min":2.0230000000000001,"15min":1.994},"min":{"1min":0.41999999999999998,"5min":0.311,"15min":0.311},"max":{"1min":19.527000000000001,"5min":19.527000000000001,"15min":19.527000000000001},"last":4.399},{"interface":"front","average":{"1min":3.4969999999999999,"5min":2.1259999999999999,"15min":1.9550000000000001},"min":{"1min":0.38,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":19.516999999999999,"5min":19.516999999999999,"15min":19.516999999999999},"last":4.3250000000000002}]},{"osd":7,"last update":"Mon Aug 28 02:37:02 2023","interfaces":[{"interface":"back","average":{"1min":1.4179999999999999,"5min":1.012,"15min":0.94399999999999995},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":6.4909999999999997,"5min":6.4909999999999997,"15min":6.4909999999999997},"last":4.4199999999999999},{"interface":"front","average":{"1min":1.788,"5min":1.006,"15min":0.876},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":6.3860000000000001,"5min":6.3860000000000001,"15min":6.3860000000000001},"last":0.54700000000000004}]}]},{"osd":0,"up_from":90,"seq":386547056866,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3157712,"kb_used_data":2524232,"kb_used_omap":8,"kb_used_meta":633463,"kb_avail":90587440,"statfs":{"total":95995035648,"available":92761538560,"internally_reserved":0,"allocated":2584813568,"data_stored":2583222284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":648666473},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Aug 28 02:36:27 2023","interfaces":[{"interface":"back","average":{"1min":2.4620000000000002,"5min":2.7480000000000002,"15min":2.7549999999999999},"min":{"1min":0.16500000000000001,"5min":0.16500000000000001,"15min":0.16500000000000001},"max":{"1min":15.728,"5min":61.932000000000002,"15min":61.932000000000002},"last":13.923999999999999},{"interface":"front","average":{"1min":3.3849999999999998,"5min":3.98,"15min":3.2570000000000001},"min":{"1min":0.28599999999999998,"5min":0.19500000000000001,"15min":0.17000000000000001},"max":{"1min":16.443999999999999,"5min":62.061999999999998,"15min":62.061999999999998},"last":0.55900000000000005}]},{"osd":2,"last update":"Mon Aug 28 02:36:49 2023","interfaces":[{"interface":"back","average":{"1min":1.8400000000000001,"5min":4.7300000000000004,"15min":3.698},"min":{"1min":0.254,"5min":0.22800000000000001,"15min":0.19500000000000001},"max":{"1min":3.6960000000000002,"5min":86.269000000000005,"15min":100.718},"last":0.45000000000000001},{"interface":"front","average":{"1min":3.0110000000000001,"5min":5.774,"15min":4.3259999999999996},"min":{"1min":0.24399999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":15.337,"5min":86.353999999999999,"15min":100.962},"last":0.67000000000000004}]},{"osd":3,"last update":"Mon Aug 28 02:36:33 2023","interfaces":[{"interface":"back","average":{"1min":2.4159999999999999,"5min":6.1790000000000003,"15min":4.9260000000000002},"min":{"1min":0.40999999999999998,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":9.1319999999999997,"5min":126.896,"15min":237.43899999999999},"last":3.1349999999999998},{"interface":"front","average":{"1min":1.833,"5min":6.0460000000000003,"15min":4.8179999999999996},"min":{"1min":0.308,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":4.9370000000000003,"5min":127.01600000000001,"15min":237.48099999999999},"last":3.1549999999999998}]},{"osd":4,"last update":"Mon Aug 28 02:36:23 2023","interfaces":[{"interface":"back","average":{"1min":1.796,"5min":1.5229999999999999,"15min":1.6539999999999999},"min":{"1min":0.42099999999999999,"5min":0.252,"15min":0.23999999999999999},"max":{"1min":14.209,"5min":14.209,"15min":14.209},"last":0.52400000000000002},{"interface":"front","average":{"1min":2.0649999999999999,"5min":2.0310000000000001,"15min":1.956},"min":{"1min":0.34000000000000002,"5min":0.31,"15min":0.309},"max":{"1min":14.241,"5min":14.241,"15min":27.271999999999998},"last":3.0390000000000001}]},{"osd":5,"last update":"Mon Aug 28 02:36:23 2023","interfaces":[{"interface":"back","average":{"1min":1.47,"5min":1.956,"15min":2.0339999999999998},"min":{"1min":0.38100000000000001,"5min":0.29899999999999999,"15min":0.29599999999999999},"max":{"1min":3.827,"5min":10.779999999999999,"15min":44.000999999999998},"last":3.1120000000000001},{"interface":"front","average":{"1min":1.2210000000000001,"5min":1.486,"15min":1.8320000000000001},"min":{"1min":0.45500000000000002,"5min":0.307,"15min":0.307},"max":{"1min":3.2170000000000001,"5min":9.0310000000000006,"15min":43.953000000000003},"last":0.49199999999999999}]},{"osd":6,"last update":"Mon Aug 28 02:36:23 2023","interfaces":[{"interface":"back","average":{"1min":1.3819999999999999,"5min":1.9970000000000001,"15min":2.113},"min":{"1min":0.36199999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":3.8610000000000002,"5min":10.823,"15min":32.417999999999999},"last":3.0840000000000001},{"interface":"front","average":{"1min":1.355,"5min":1.51,"15min":1.8799999999999999},"min":{"1min":0.52500000000000002,"5min":0.32900000000000001,"15min":0.30399999999999999},"max":{"1min":3.266,"5min":9.0009999999999994,"15min":32.130000000000003},"last":0.55000000000000004}]},{"osd":7,"last update":"Mon Aug 28 02:37:01 2023","interfaces":[{"interface":"back","average":{"1min":1.46,"5min":1.2090000000000001,"15min":1.167},"min":{"1min":0.46000000000000002,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":6.6669999999999998,"5min":6.6669999999999998,"15min":6.6669999999999998},"last":0.57399999999999995},{"interface":"front","average":{"1min":1.5009999999999999,"5min":1.841,"15min":1.897},"min":{"1min":0.39100000000000001,"5min":0.37,"15min":0.37},"max":{"1min":6.6059999999999999,"5min":16.529,"15min":16.529},"last":0.63300000000000001}]}]},{"osd":2,"up_from":104,"seq":446676598990,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2807168,"kb_used_data":2188216,"kb_used_omap":10,"kb_used_meta":618805,"kb_avail":90937984,"statfs":{"total":95995035648,"available":93120495616,"internally_reserved":0,"allocated":2240733184,"data_stored":2239156640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10792,"internal_metadata":633656792},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:36:50 2023","interfaces":[{"interface":"back","average":{"1min":10.484,"5min":9.0630000000000006,"15min":5.2300000000000004},"min":{"1min":0.20899999999999999,"5min":0.191,"15min":0.17599999999999999},"max":{"1min":166.13,"5min":185.93799999999999,"15min":185.93799999999999},"last":4.3220000000000001},{"interface":"front","average":{"1min":10.879,"5min":8.5429999999999993,"15min":6.2590000000000003},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":170.19200000000001,"5min":187.90700000000001,"15min":187.90700000000001},"last":0.997}]},{"osd":1,"last update":"Mon Aug 28 02:36:50 2023","interfaces":[{"interface":"back","average":{"1min":10.595000000000001,"5min":4.5140000000000002,"15min":3.1760000000000002},"min":{"1min":0.27600000000000002,"5min":0.221,"15min":0.20999999999999999},"max":{"1min":118.11799999999999,"5min":118.11799999999999,"15min":118.11799999999999},"last":0.54600000000000004},{"interface":"front","average":{"1min":11.292,"5min":4.556,"15min":3.1259999999999999},"min":{"1min":0.38200000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":118.941,"5min":118.941,"15min":118.941},"last":9.2189999999999994}]},{"osd":3,"last update":"Mon Aug 28 02:36:23 2023","interfaces":[{"interface":"back","average":{"1min":4.8390000000000004,"5min":5.6470000000000002,"15min":4.3280000000000003},"min":{"1min":0.23499999999999999,"5min":0.23100000000000001,"15min":0.22700000000000001},"max":{"1min":57.338000000000001,"5min":134.07599999999999,"15min":134.07599999999999},"last":8.9380000000000006},{"interface":"front","average":{"1min":4.6769999999999996,"5min":5.5549999999999997,"15min":4.2789999999999999},"min":{"1min":0.29599999999999999,"5min":0.29299999999999998,"15min":0.223},"max":{"1min":57.375999999999998,"5min":134.172,"15min":134.172},"last":9.2539999999999996}]},{"osd":4,"last update":"Mon Aug 28 02:36:50 2023","interfaces":[{"interface":"back","average":{"1min":1.9239999999999999,"5min":1.556,"15min":2.9910000000000001},"min":{"1min":0.316,"5min":0.26800000000000002,"15min":0.26600000000000001},"max":{"1min":13.449999999999999,"5min":13.449999999999999,"15min":207.34100000000001},"last":9.1099999999999994},{"interface":"front","average":{"1min":1.4910000000000001,"5min":2.242,"15min":2.9500000000000002},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27400000000000002},"max":{"1min":6.0419999999999998,"5min":22.303999999999998,"15min":207.352},"last":4.25}]},{"osd":5,"last update":"Mon Aug 28 02:36:50 2023","interfaces":[{"interface":"back","average":{"1min":0.89100000000000001,"5min":1.458,"15min":2.8650000000000002},"min":{"1min":0.314,"5min":0.314,"15min":0.29299999999999998},"max":{"1min":2.9159999999999999,"5min":13.335000000000001,"15min":128.696},"last":0.58199999999999996},{"interface":"front","average":{"1min":2.0790000000000002,"5min":1.6459999999999999,"15min":2.972},"min":{"1min":0.35999999999999999,"5min":0.30399999999999999,"15min":0.30299999999999999},"max":{"1min":13.404999999999999,"5min":13.404999999999999,"15min":128.84100000000001},"last":9.0670000000000002}]},{"osd":6,"last update":"Mon Aug 28 02:36:50 2023","interfaces":[{"interface":"back","average":{"1min":1.726,"5min":2.3730000000000002,"15min":1.962},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29299999999999998},"max":{"1min":6.0800000000000001,"5min":22.25,"15min":22.25},"last":4.2830000000000004},{"interface":"front","average":{"1min":1.272,"5min":1.4530000000000001,"15min":1.821},"min":{"1min":0.35799999999999998,"5min":0.30199999999999999,"15min":0.29399999999999998},"max":{"1min":7.8689999999999998,"5min":13.311,"15min":13.311},"last":0.64100000000000001}]},{"osd":7,"last update":"Mon Aug 28 02:36:55 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":1.0049999999999999,"15min":1.014},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":2.9430000000000001,"5min":2.9430000000000001,"15min":2.9430000000000001},"last":0.61199999999999999},{"interface":"front","average":{"1min":2.7789999999999999,"5min":1.2789999999999999,"15min":1.0289999999999999},"min":{"1min":0.42099999999999999,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":13.478999999999999,"5min":13.478999999999999,"15min":13.478999999999999},"last":9.1890000000000001}]}]},{"osd":3,"up_from":112,"seq":481036337352,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2495600,"kb_used_data":1914472,"kb_used_omap":3,"kb_used_meta":581116,"kb_avail":91249552,"statfs":{"total":95995035648,"available":93439541248,"internally_reserved":0,"allocated":1960419328,"data_stored":1958851672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":595063714},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,2,3,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:36:17 2023","interfaces":[{"interface":"back","average":{"1min":3.1669999999999998,"5min":5.1580000000000004,"15min":4.7169999999999996},"min":{"1min":0.20799999999999999,"5min":0.19500000000000001,"15min":0.193},"max":{"1min":24.561,"5min":138.39699999999999,"15min":164.25299999999999},"last":6.8380000000000001},{"interface":"front","average":{"1min":2.0640000000000001,"5min":2.9079999999999999,"15min":3.9169999999999998},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.192},"max":{"1min":24.048999999999999,"5min":24.048999999999999,"15min":164.55000000000001},"last":3.1179999999999999}]},{"osd":1,"last update":"Mon Aug 28 02:36:17 2023","interfaces":[{"interface":"back","average":{"1min":7.6219999999999999,"5min":4.0919999999999996,"15min":3.1669999999999998},"min":{"1min":0.222,"5min":0.222,"15min":0.20699999999999999},"max":{"1min":105.04600000000001,"5min":105.04600000000001,"15min":117.34699999999999},"last":6.6950000000000003},{"interface":"front","average":{"1min":7.6379999999999999,"5min":5.7039999999999997,"15min":3.8340000000000001},"min":{"1min":0.29099999999999998,"5min":0.23599999999999999,"15min":0.224},"max":{"1min":105.55800000000001,"5min":143.54400000000001,"15min":143.54400000000001},"last":6.7409999999999997}]},{"osd":2,"last update":"Mon Aug 28 02:36:17 2023","interfaces":[{"interface":"back","average":{"1min":2.052,"5min":5.3099999999999996,"15min":4.5599999999999996},"min":{"1min":0.26700000000000002,"5min":0.20100000000000001,"15min":0.19900000000000001},"max":{"1min":9.9789999999999992,"5min":143.50299999999999,"15min":143.50299999999999},"last":6.8639999999999999},{"interface":"front","average":{"1min":1.583,"5min":2.9039999999999999,"15min":3.6589999999999998},"min":{"1min":0.23699999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":9.9120000000000008,"5min":12.646000000000001,"15min":120.44499999999999},"last":11.368}]},{"osd":4,"last update":"Mon Aug 28 02:36:17 2023","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":1.5620000000000001,"15min":1.827},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":2.8980000000000001,"5min":7.8940000000000001,"15min":58.067999999999998},"last":3.0950000000000002},{"interface":"front","average":{"1min":1.101,"5min":1.7609999999999999,"15min":1.9390000000000001},"min":{"1min":0.38400000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":2.8519999999999999,"5min":10.613,"15min":58.088000000000001},"last":6.7270000000000003}]},{"osd":5,"last update":"Mon Aug 28 02:36:17 2023","interfaces":[{"interface":"back","average":{"1min":1.0669999999999999,"5min":1.9259999999999999,"15min":2.1099999999999999},"min":{"1min":0.39300000000000002,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.8769999999999998,"5min":10.722,"15min":102.904},"last":6.7590000000000003},{"interface":"front","average":{"1min":1.3819999999999999,"5min":3.798,"15min":2.7589999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":4.319,"5min":138.328,"15min":138.328},"last":6.6440000000000001}]},{"osd":6,"last update":"Mon Aug 28 02:36:17 2023","interfaces":[{"interface":"back","average":{"1min":1.48,"5min":3.7749999999999999,"15min":2.5350000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.314},"max":{"1min":4.351,"5min":138.37,"15min":138.37},"last":6.6760000000000002},{"interface":"front","average":{"1min":1.179,"5min":1.655,"15min":1.7749999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.314},"max":{"1min":5.5060000000000002,"5min":7.9400000000000004,"15min":72.030000000000001},"last":3.036}]},{"osd":7,"last update":"Mon Aug 28 02:37:03 2023","interfaces":[{"interface":"back","average":{"1min":1.75,"5min":1.274,"15min":1.194},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":5.0090000000000003,"5min":5.0090000000000003,"15min":5.0090000000000003},"last":6.774},{"interface":"front","average":{"1min":1.6659999999999999,"5min":1.3620000000000001,"15min":1.3120000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":5.9180000000000001,"5min":5.9180000000000001,"15min":5.9180000000000001},"last":3.0670000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019225,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2211568,"kb_used_data":1617768,"kb_used_omap":0,"kb_used_meta":593792,"kb_avail":91533584,"statfs":{"total":95995035648,"available":93730390016,"internally_reserved":0,"allocated":1656594432,"data_stored":1655716823,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":608043008},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:36:25 2023","interfaces":[{"interface":"back","average":{"1min":7.6079999999999997,"5min":7.9610000000000003,"15min":5.3259999999999996},"min":{"1min":0.23400000000000001,"5min":0.23100000000000001,"15min":0.219},"max":{"1min":61.972999999999999,"5min":131.893,"15min":138.67400000000001},"last":2.9129999999999998},{"interface":"front","average":{"1min":7.8399999999999999,"5min":8.1769999999999996,"15min":5.1950000000000003},"min":{"1min":0.23799999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":61.459000000000003,"5min":131.96700000000001,"15min":134.14599999999999},"last":0.26300000000000001}]},{"osd":1,"last update":"Mon Aug 28 02:36:25 2023","interfaces":[{"interface":"back","average":{"1min":9.9469999999999992,"5min":3.3050000000000002,"15min":2.4780000000000002},"min":{"1min":0.27800000000000002,"5min":0.24399999999999999,"15min":0.221},"max":{"1min":105.604,"5min":105.604,"15min":105.604},"last":2.7759999999999998},{"interface":"front","average":{"1min":9.6080000000000005,"5min":3.113,"15min":2.2869999999999999},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":105.883,"5min":105.883,"15min":105.883},"last":2.6739999999999999}]},{"osd":2,"last update":"Mon Aug 28 02:36:53 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":3.444,"15min":3.843},"min":{"1min":0.26600000000000001,"5min":0.255,"15min":0.255},"max":{"1min":6.1769999999999996,"5min":107.786,"15min":107.786},"last":0.38900000000000001},{"interface":"front","average":{"1min":1.238,"5min":3.573,"15min":3.7669999999999999},"min":{"1min":0.29099999999999998,"5min":0.27300000000000002,"15min":0.25},"max":{"1min":6.5899999999999999,"5min":107.599,"15min":107.599},"last":2.5049999999999999}]},{"osd":3,"last update":"Mon Aug 28 02:36:25 2023","interfaces":[{"interface":"back","average":{"1min":8.5619999999999994,"5min":9.8659999999999997,"15min":5.827},"min":{"1min":0.27700000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":137.995,"5min":203.583,"15min":203.583},"last":2.6379999999999999},{"interface":"front","average":{"1min":8.9250000000000007,"5min":10.34,"15min":5.9850000000000003},"min":{"1min":0.33100000000000002,"5min":0.33000000000000002,"15min":0.32500000000000001},"max":{"1min":137.65899999999999,"5min":203.41399999999999,"15min":203.41399999999999},"last":2.8090000000000002}]},{"osd":5,"last update":"Mon Aug 28 02:37:03 2023","interfaces":[{"interface":"back","average":{"1min":0.871,"5min":0.53400000000000003,"15min":1.4650000000000001},"min":{"1min":0.255,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":2.7919999999999998,"5min":2.7919999999999998,"15min":124.718},"last":0.317},{"interface":"front","average":{"1min":0.80600000000000005,"5min":0.58499999999999996,"15min":1.679},"min":{"1min":0.34799999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":2.5230000000000001,"5min":2.5230000000000001,"15min":124.694},"last":0.40699999999999997}]},{"osd":6,"last update":"Mon Aug 28 02:36:25 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.625,"15min":1.1279999999999999},"min":{"1min":0.28499999999999998,"5min":0.25,"15min":0.25},"max":{"1min":2.1339999999999999,"5min":3.395,"15min":14.907999999999999},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.751,"5min":0.625,"15min":0.89800000000000002},"min":{"1min":0.36199999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":2.2650000000000001,"5min":3.1219999999999999,"15min":7.3170000000000002},"last":0.34899999999999998}]},{"osd":7,"last update":"Mon Aug 28 02:36:58 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":0.57699999999999996,"15min":0.53800000000000003},"min":{"1min":0.32000000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":2.8660000000000001,"5min":2.8660000000000001,"15min":2.8660000000000001},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.67900000000000005,"5min":0.60499999999999998,"15min":0.59299999999999997},"min":{"1min":0.38,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":2.2629999999999999,"5min":2.2629999999999999,"15min":2.2629999999999999},"last":0.375}]}]},{"osd":5,"up_from":36,"seq":154618822997,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3471644,"kb_used_data":2881940,"kb_used_omap":0,"kb_used_meta":589696,"kb_avail":90273508,"statfs":{"total":95995035648,"available":92440072192,"internally_reserved":0,"allocated":2951106560,"data_stored":2950236597,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603848704},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,4],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:37:14 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":3.8889999999999998,"15min":4.4379999999999997},"min":{"1min":0.23400000000000001,"5min":0.216,"15min":0.20300000000000001},"max":{"1min":5.46,"5min":128.72800000000001,"15min":131.06100000000001},"last":3.5670000000000002},{"interface":"front","average":{"1min":1.4330000000000001,"5min":3.9809999999999999,"15min":4.6719999999999997},"min":{"1min":0.316,"5min":0.245,"15min":0.22900000000000001},"max":{"1min":7.609,"5min":128.75200000000001,"15min":131.262},"last":3.5590000000000002}]},{"osd":1,"last update":"Mon Aug 28 02:36:27 2023","interfaces":[{"interface":"back","average":{"1min":4.3529999999999998,"5min":3.5539999999999998,"15min":3.4510000000000001},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.216},"max":{"1min":55.463999999999999,"5min":98.632000000000005,"15min":183.97900000000001},"last":3.794},{"interface":"front","average":{"1min":4.2649999999999997,"5min":3.548,"15min":3.6080000000000001},"min":{"1min":0.28000000000000003,"5min":0.26400000000000001,"15min":0.23200000000000001},"max":{"1min":55.546999999999997,"5min":98.560000000000002,"15min":184.33699999999999},"last":2.7829999999999999}]},{"osd":2,"last update":"Mon Aug 28 02:36:57 2023","interfaces":[{"interface":"back","average":{"1min":1.458,"5min":2.9729999999999999,"15min":4.8170000000000002},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.252},"max":{"1min":5.6070000000000002,"5min":78.834000000000003,"15min":176.69499999999999},"last":2.5710000000000002},{"interface":"front","average":{"1min":4.2149999999999999,"5min":3.5649999999999999,"15min":5.2569999999999997},"min":{"1min":0.34699999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":50.279000000000003,"5min":78.798000000000002,"15min":176.71199999999999},"last":2.629}]},{"osd":3,"last update":"Mon Aug 28 02:36:27 2023","interfaces":[{"interface":"back","average":{"1min":2.2480000000000002,"5min":6.5819999999999999,"15min":3.6949999999999998},"min":{"1min":0.30499999999999999,"5min":0.29699999999999999,"15min":0.26100000000000001},"max":{"1min":14.202,"5min":128.571,"15min":128.571},"last":0.30299999999999999},{"interface":"front","average":{"1min":1.6579999999999999,"5min":6.2140000000000004,"15min":3.8849999999999998},"min":{"1min":0.32900000000000001,"5min":0.28699999999999998,"15min":0.26200000000000001},"max":{"1min":4.1909999999999998,"5min":128.684,"15min":128.684},"last":3.5230000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:37:05 2023","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.48499999999999999,"15min":1.3240000000000001},"min":{"1min":0.27200000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.72999999999999998,"5min":3.0259999999999998,"15min":85.769999999999996},"last":2.7080000000000002},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.48799999999999999,"15min":1.306},"min":{"1min":0.34200000000000003,"5min":0.248,"15min":0.248},"max":{"1min":1.5660000000000001,"5min":2.7690000000000001,"15min":85.825000000000003},"last":0.375}]},{"osd":6,"last update":"Mon Aug 28 02:36:27 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.55100000000000005,"15min":1.054},"min":{"1min":0.27800000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":2.181,"5min":3.9990000000000001,"15min":14.803000000000001},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.65400000000000003,"15min":0.97099999999999997},"min":{"1min":0.376,"5min":0.29599999999999999,"15min":0.28799999999999998},"max":{"1min":1.4379999999999999,"5min":7.4740000000000002,"15min":8.6479999999999997},"last":2.6019999999999999}]},{"osd":7,"last update":"Mon Aug 28 02:36:57 2023","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.57999999999999996,"15min":0.59499999999999997},"min":{"1min":0.38300000000000001,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.65000000000000002,"5min":2.0049999999999999,"15min":2.0049999999999999},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.76300000000000001,"5min":0.71499999999999997,"15min":0.70699999999999996},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":3.3199999999999998,"5min":3.3199999999999998,"15min":3.3199999999999998},"last":0.57499999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2559770624,"data_stored":2559770624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2268921856,"data_stored":2268921856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2235568128,"data_stored":2235564056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1935671296,"data_stored":1935671296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1648492544,"data_stored":1648492544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2935885824,"data_stored":2935881752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1977946112,"data_stored":1977942040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-08-28T02:37:26.028 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-08-28T02:37:26.028 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd pool set device_health_metrics pg_num 41 2023-08-28T02:37:26.191 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:37:26.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:26.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:37:26.307 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:37:26.431 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:26.598 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:37:26.598 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:37:26.615 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:37:26.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:26.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:37:26.723 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:37:26.750 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:26.828 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:37:26.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:26.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:37:26.936 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:37:26.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:26 smithi143 ceph-mon[183059]: pgmap v910: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 693 op/s 2023-08-28T02:37:26.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:26 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:37:26.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:26 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:37:26.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:26 smithi143 ceph-mon[189871]: pgmap v910: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 693 op/s 2023-08-28T02:37:26.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:26 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:37:26.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:26 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:37:26.980 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:27.036 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:37:27.044 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:26 smithi169 ceph-mon[159415]: pgmap v910: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 693 op/s 2023-08-28T02:37:27.044 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:26 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:37:27.044 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:26 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:37:27.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:27.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:37:27.107 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:37:27.146 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:37:27.185 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:27.247 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:37:27.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:27.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:37:27.352 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:37:27.378 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:27.420 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:37:27.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:27.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:37:27.525 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:37:27.560 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:27.631 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:37:27.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:27.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:37:27.738 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:37:27.770 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:27.810 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is set 2023-08-28T02:37:27.840 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:37:27.847 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:37:27.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:27.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:37:27.890 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.3 dump_historic_ops 2023-08-28T02:37:27.949 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:37:27.978 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:27 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/4044323307' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:37:27.981 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:28.037 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:37:28.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:28.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:37:28.102 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:27 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/4044323307' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:37:28.106 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:27 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/4044323307' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:37:28.144 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:37:28.236 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:28.356 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:37:28.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:28.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:37:28.463 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:37:28.498 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:28.566 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:37:28.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:28.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:37:28.673 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:37:28.758 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:28 smithi169 ceph-mon[159415]: pgmap v911: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 714 op/s 2023-08-28T02:37:28.758 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:28 smithi169 ceph-mon[159415]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:37:28.758 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:28 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/4044323307' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:37:28.759 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:28 smithi169 ceph-mon[159415]: osdmap e342: 8 total, 8 up, 7 in 2023-08-28T02:37:28.759 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:28 smithi169 ceph-mon[159415]: failed to encode map e342 with expected crc 2023-08-28T02:37:28.759 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:28 smithi169 ceph-mon[159415]: failed to encode map e342 with expected crc 2023-08-28T02:37:28.765 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:28.928 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:37:29.012 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:37:29.019 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:28 smithi143 ceph-mon[183059]: pgmap v911: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 714 op/s 2023-08-28T02:37:29.019 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:28 smithi143 ceph-mon[183059]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:37:29.019 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:28 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/4044323307' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:37:29.019 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:28 smithi143 ceph-mon[183059]: osdmap e342: 8 total, 8 up, 7 in 2023-08-28T02:37:29.019 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:28 smithi143 ceph-mon[183059]: failed to encode map e342 with expected crc 2023-08-28T02:37:29.019 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:28 smithi143 ceph-mon[183059]: failed to encode map e342 with expected crc 2023-08-28T02:37:29.019 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:28 smithi143 ceph-mon[189871]: pgmap v911: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 714 op/s 2023-08-28T02:37:29.019 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:28 smithi143 ceph-mon[189871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:37:29.020 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:28 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/4044323307' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:37:29.020 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:28 smithi143 ceph-mon[189871]: osdmap e342: 8 total, 8 up, 7 in 2023-08-28T02:37:29.020 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:28 smithi143 ceph-mon[189871]: failed to encode map e342 with expected crc 2023-08-28T02:37:29.020 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:28 smithi143 ceph-mon[189871]: failed to encode map e342 with expected crc 2023-08-28T02:37:29.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:29.034 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:37:29.134 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:37:29.173 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:29.231 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:37:29.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:29.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:37:29.340 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:37:29.383 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:29.446 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:37:29.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:29.455 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:37:29.556 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:37:29.584 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:30.130 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:37:30.136 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:29 smithi143 ceph-mon[189871]: failed to encode map e342 with expected crc 2023-08-28T02:37:30.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:30.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:37:30.141 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:29 smithi143 ceph-mon[183059]: failed to encode map e342 with expected crc 2023-08-28T02:37:30.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:29 smithi169 ceph-mon[159415]: failed to encode map e342 with expected crc 2023-08-28T02:37:30.238 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:37:30.269 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:30.336 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:37:30.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:30.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:37:30.447 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:37:30.526 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:30.556 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:37:30.955 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset noscrub 2023-08-28T02:37:31.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:30 smithi143 ceph-mon[183059]: pgmap v913: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 682 op/s 2023-08-28T02:37:31.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:30 smithi143 ceph-mon[189871]: pgmap v913: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 682 op/s 2023-08-28T02:37:31.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:30 smithi169 ceph-mon[159415]: pgmap v913: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 682 op/s 2023-08-28T02:37:31.647 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:37:31.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:31.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:37:31.758 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:37:31.790 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:31.850 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:37:31.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:31.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:37:31.956 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:37:31.993 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:32.052 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:37:32.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:32.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:37:32.148 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:37:31 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:37:31] "GET /metrics HTTP/1.1" 200 36538 "" "Prometheus/2.33.4" 2023-08-28T02:37:32.162 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:37:32.194 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:32.256 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:37:32.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:32.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:37:32.273 INFO:teuthology.orchestra.run.smithi143.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-08-28T02:37:32.365 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:37:32.436 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:33.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:32 smithi143 ceph-mon[183059]: pgmap v914: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 669 op/s 2023-08-28T02:37:33.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:32 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2709281067' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "41"}]: dispatch 2023-08-28T02:37:33.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:32 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "41"}]: dispatch 2023-08-28T02:37:33.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:32 smithi143 ceph-mon[189871]: pgmap v914: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 669 op/s 2023-08-28T02:37:33.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:32 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2709281067' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "41"}]: dispatch 2023-08-28T02:37:33.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:32 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "41"}]: dispatch 2023-08-28T02:37:33.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:32 smithi169 ceph-mon[159415]: pgmap v914: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 669 op/s 2023-08-28T02:37:33.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:32 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2709281067' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "41"}]: dispatch 2023-08-28T02:37:33.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:32 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "41"}]: dispatch 2023-08-28T02:37:33.941 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:37:33.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:34.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:37:34.062 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:37:34.071 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:37:34.106 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:37:34.138 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:37:34.147 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:37:34.163 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:37:34.166 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-08-28T02:37:34.193 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:37:34.195 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:34.231 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:37:34.250 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:37:34.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:34.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:37:34.272 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:37:34.311 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:37:34.347 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:37:34.347 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:37:34.347 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:37:34.347 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:37:34.347 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:37:34.347 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:37:34.347 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:37:34.347 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:37:34.356 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:37:34.394 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:34.451 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:37:34.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:34.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:37:34.562 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:37:34.591 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:34.645 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:37:34.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:34.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:37:34.752 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:37:34.836 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:35.016 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:37:35.016 INFO:teuthology.orchestra.run.smithi143.stdout: "size": 20, 2023-08-28T02:37:35.016 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 600, 2023-08-28T02:37:35.016 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [ 2023-08-28T02:37:35.017 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.017 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:78158 6.c 6:3c2b9257:::benchmark_data_smithi143_305742_object78157:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e220)", 2023-08-28T02:37:35.017 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:28:24.351475+0000", 2023-08-28T02:37:35.017 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 550.49608481099995, 2023-08-28T02:37:35.017 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.115024359, 2023-08-28T02:37:35.017 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:37:35.018 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:37:35.018 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:37:35.018 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:37:35.018 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:37:35.018 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 78158 2023-08-28T02:37:35.018 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.018 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:37:35.018 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.019 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:37:35.019 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.351475+0000", 2023-08-28T02:37:35.019 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:35.019 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.019 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.020 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:37:35.020 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.351475+0000", 2023-08-28T02:37:35.020 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:35.020 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.020 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.020 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:37:35.020 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.351473+0000", 2023-08-28T02:37:35.020 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999976 2023-08-28T02:37:35.020 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.020 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.020 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:37:35.020 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.351505+0000", 2023-08-28T02:37:35.020 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.2289999999999997e-05 2023-08-28T02:37:35.021 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.021 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.021 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:37:35.021 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.351530+0000", 2023-08-28T02:37:35.021 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.4946000000000001e-05 2023-08-28T02:37:35.021 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.021 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.021 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:37:35.021 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.351544+0000", 2023-08-28T02:37:35.021 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3298e-05 2023-08-28T02:37:35.021 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.021 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.021 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:37:35.022 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.352280+0000", 2023-08-28T02:37:35.022 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00073618100000000003 2023-08-28T02:37:35.022 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.022 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.022 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:37:35.022 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.352291+0000", 2023-08-28T02:37:35.022 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0964e-05 2023-08-28T02:37:35.022 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.022 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.023 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:37:35.023 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.331636+0000", 2023-08-28T02:37:35.023 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.97934522000000002 2023-08-28T02:37:35.023 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.023 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.023 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:37:35.024 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.331647+0000", 2023-08-28T02:37:35.024 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1392e-05 2023-08-28T02:37:35.024 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.024 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.024 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:37:35.024 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.452208+0000", 2023-08-28T02:37:35.024 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.120560625 2023-08-28T02:37:35.024 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.024 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.024 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:37:35.024 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.452384+0000", 2023-08-28T02:37:35.024 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00017561200000000001 2023-08-28T02:37:35.025 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.025 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.025 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 5,6,7", 2023-08-28T02:37:35.025 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.452555+0000", 2023-08-28T02:37:35.025 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00017118799999999999 2023-08-28T02:37:35.025 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.025 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.025 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:35.025 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.455618+0000", 2023-08-28T02:37:35.025 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0030633460000000002 2023-08-28T02:37:35.025 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.025 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.025 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:37:35.026 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.455688+0000", 2023-08-28T02:37:35.026 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.9611999999999997e-05 2023-08-28T02:37:35.026 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.026 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.026 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:35.026 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.458168+0000", 2023-08-28T02:37:35.026 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0024801239999999998 2023-08-28T02:37:35.026 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.026 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.026 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:35.026 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.466293+0000", 2023-08-28T02:37:35.026 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0081255110000000002 2023-08-28T02:37:35.026 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.027 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.027 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:37:35.027 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.466366+0000", 2023-08-28T02:37:35.027 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.2871000000000001e-05 2023-08-28T02:37:35.027 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.027 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.027 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:37:35.027 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.466500+0000", 2023-08-28T02:37:35.027 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00013335899999999999 2023-08-28T02:37:35.027 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:35.027 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:37:35.027 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:35.027 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.028 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.028 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:78162 6.c 6:318d1535:::benchmark_data_smithi143_305742_object78161:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e220)", 2023-08-28T02:37:35.028 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:28:24.353558+0000", 2023-08-28T02:37:35.028 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 550.49400240199998, 2023-08-28T02:37:35.028 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.1131857620000001, 2023-08-28T02:37:35.028 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:37:35.028 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:37:35.028 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:37:35.028 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:37:35.028 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:37:35.028 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 78162 2023-08-28T02:37:35.028 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.028 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:37:35.029 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.029 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:37:35.029 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.353558+0000", 2023-08-28T02:37:35.029 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:35.029 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.029 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.029 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:37:35.029 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.353558+0000", 2023-08-28T02:37:35.029 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:35.029 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.029 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.029 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:37:35.029 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.353556+0000", 2023-08-28T02:37:35.030 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:37:35.030 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.030 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.030 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:37:35.030 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.353594+0000", 2023-08-28T02:37:35.030 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.7595999999999997e-05 2023-08-28T02:37:35.030 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.030 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.030 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:37:35.030 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.353608+0000", 2023-08-28T02:37:35.030 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.4416000000000001e-05 2023-08-28T02:37:35.030 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.030 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.031 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:37:35.031 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.353617+0000", 2023-08-28T02:37:35.031 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.9069999999999996e-06 2023-08-28T02:37:35.031 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.031 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.031 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:37:35.031 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.353671+0000", 2023-08-28T02:37:35.031 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.3871e-05 2023-08-28T02:37:35.031 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.031 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.031 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:37:35.031 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:24.353681+0000", 2023-08-28T02:37:35.031 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0713999999999999e-05 2023-08-28T02:37:35.032 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.032 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.032 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:37:35.032 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.331885+0000", 2023-08-28T02:37:35.032 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.97820372 2023-08-28T02:37:35.032 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.032 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.032 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:37:35.032 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.331895+0000", 2023-08-28T02:37:35.032 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.9920000000000002e-06 2023-08-28T02:37:35.032 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.032 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.032 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:37:35.032 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.455726+0000", 2023-08-28T02:37:35.033 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1238307970000001 2023-08-28T02:37:35.033 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.033 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.033 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:37:35.033 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.455870+0000", 2023-08-28T02:37:35.033 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000143814 2023-08-28T02:37:35.033 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.033 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.033 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 5,6,7", 2023-08-28T02:37:35.033 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.456053+0000", 2023-08-28T02:37:35.033 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000183234 2023-08-28T02:37:35.033 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.034 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.034 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:37:35.034 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.461802+0000", 2023-08-28T02:37:35.034 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0057491449999999998 2023-08-28T02:37:35.034 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.034 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.034 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:35.034 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.464367+0000", 2023-08-28T02:37:35.034 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0025651480000000002 2023-08-28T02:37:35.034 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.034 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.034 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:35.034 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.464470+0000", 2023-08-28T02:37:35.034 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00010294 2023-08-28T02:37:35.035 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.035 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.035 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:35.035 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.466571+0000", 2023-08-28T02:37:35.035 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0021002669999999998 2023-08-28T02:37:35.035 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.035 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.035 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:37:35.035 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.466630+0000", 2023-08-28T02:37:35.035 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.9827000000000002e-05 2023-08-28T02:37:35.035 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.035 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.035 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:37:35.036 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.466743+0000", 2023-08-28T02:37:35.036 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000113047 2023-08-28T02:37:35.036 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:35.036 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:37:35.036 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:35.036 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.036 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.036 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:78191 6.c 6:388e80c8:::benchmark_data_smithi143_305742_object78190:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e221)", 2023-08-28T02:37:35.036 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:28:25.485179+0000", 2023-08-28T02:37:35.036 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 549.36238124299996, 2023-08-28T02:37:35.037 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.98181897100000004, 2023-08-28T02:37:35.037 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:37:35.037 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:37:35.037 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:37:35.037 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:37:35.037 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:37:35.037 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 78191 2023-08-28T02:37:35.037 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.037 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:37:35.037 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.037 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:37:35.037 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.485179+0000", 2023-08-28T02:37:35.038 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:35.038 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.038 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.038 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:37:35.038 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.485179+0000", 2023-08-28T02:37:35.038 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:35.038 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.038 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.038 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:37:35.038 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.485177+0000", 2023-08-28T02:37:35.038 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:37:35.038 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.039 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.039 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:37:35.039 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.485203+0000", 2023-08-28T02:37:35.039 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.6021e-05 2023-08-28T02:37:35.039 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.039 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.039 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:37:35.039 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.485220+0000", 2023-08-28T02:37:35.039 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.7220000000000001e-05 2023-08-28T02:37:35.039 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.039 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.039 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:37:35.040 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.485230+0000", 2023-08-28T02:37:35.040 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.3940000000000002e-06 2023-08-28T02:37:35.040 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.040 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.040 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:37:35.040 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.485298+0000", 2023-08-28T02:37:35.040 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.8418999999999997e-05 2023-08-28T02:37:35.040 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.040 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.040 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:37:35.040 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.485312+0000", 2023-08-28T02:37:35.040 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.4382000000000001e-05 2023-08-28T02:37:35.040 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.041 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.041 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:37:35.041 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.458189+0000", 2023-08-28T02:37:35.041 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.97287682399999997 2023-08-28T02:37:35.041 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.041 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.041 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:37:35.041 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.458284+0000", 2023-08-28T02:37:35.041 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.4900000000000003e-05 2023-08-28T02:37:35.041 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.041 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.041 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 5,6,7", 2023-08-28T02:37:35.042 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.458384+0000", 2023-08-28T02:37:35.042 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.9763e-05 2023-08-28T02:37:35.042 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.042 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.042 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:37:35.042 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.461806+0000", 2023-08-28T02:37:35.042 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0034222649999999999 2023-08-28T02:37:35.042 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.042 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.042 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:35.042 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.464511+0000", 2023-08-28T02:37:35.042 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0027045929999999999 2023-08-28T02:37:35.042 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.043 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.043 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:35.043 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.464548+0000", 2023-08-28T02:37:35.043 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.7626999999999997e-05 2023-08-28T02:37:35.043 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.043 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.043 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:35.043 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.466803+0000", 2023-08-28T02:37:35.043 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0022543279999999999 2023-08-28T02:37:35.043 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.043 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.043 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:37:35.043 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.466866+0000", 2023-08-28T02:37:35.044 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.3870999999999999e-05 2023-08-28T02:37:35.044 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.044 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.044 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:37:35.044 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.466998+0000", 2023-08-28T02:37:35.044 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00013130699999999999 2023-08-28T02:37:35.044 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:35.044 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:37:35.044 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:35.044 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.044 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.044 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35076.0:78192 6.c 6:36cee94b:::benchmark_data_smithi143_305742_object78191:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e221)", 2023-08-28T02:37:35.044 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:28:25.494860+0000", 2023-08-28T02:37:35.045 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 549.35269999000002, 2023-08-28T02:37:35.045 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.97873924599999995, 2023-08-28T02:37:35.045 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:37:35.045 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:37:35.045 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:37:35.045 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35076", 2023-08-28T02:37:35.045 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:37:35.045 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 78192 2023-08-28T02:37:35.045 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.045 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:37:35.045 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.045 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:37:35.046 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.494860+0000", 2023-08-28T02:37:35.046 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:35.046 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.046 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.046 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:37:35.046 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.494860+0000", 2023-08-28T02:37:35.046 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:35.046 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.046 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.046 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:37:35.046 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.494859+0000", 2023-08-28T02:37:35.046 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:37:35.046 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.047 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.047 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:37:35.047 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.494878+0000", 2023-08-28T02:37:35.047 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.8918000000000001e-05 2023-08-28T02:37:35.047 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.047 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.047 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:37:35.047 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.494891+0000", 2023-08-28T02:37:35.047 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3122000000000001e-05 2023-08-28T02:37:35.047 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.047 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.047 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:37:35.048 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.494897+0000", 2023-08-28T02:37:35.048 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.1929999999999998e-06 2023-08-28T02:37:35.048 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.048 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.048 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:37:35.048 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.494945+0000", 2023-08-28T02:37:35.048 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.8053999999999998e-05 2023-08-28T02:37:35.048 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.048 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.048 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:37:35.048 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:25.494956+0000", 2023-08-28T02:37:35.048 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0399e-05 2023-08-28T02:37:35.049 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.049 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.049 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:37:35.049 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.459931+0000", 2023-08-28T02:37:35.049 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.96497512699999999 2023-08-28T02:37:35.049 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.049 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.049 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:37:35.049 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.460075+0000", 2023-08-28T02:37:35.049 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000144571 2023-08-28T02:37:35.049 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.049 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.050 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 5,6,7", 2023-08-28T02:37:35.050 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.460181+0000", 2023-08-28T02:37:35.050 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000105993 2023-08-28T02:37:35.050 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.050 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.050 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:37:35.050 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.461924+0000", 2023-08-28T02:37:35.050 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.001743047 2023-08-28T02:37:35.050 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.050 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.050 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:35.050 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.464585+0000", 2023-08-28T02:37:35.050 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0026605359999999998 2023-08-28T02:37:35.051 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.051 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.051 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:35.051 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.464630+0000", 2023-08-28T02:37:35.051 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.4801e-05 2023-08-28T02:37:35.051 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.051 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.051 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:35.051 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.473424+0000", 2023-08-28T02:37:35.051 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0087939469999999999 2023-08-28T02:37:35.051 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.052 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.052 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:37:35.052 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.473484+0000", 2023-08-28T02:37:35.052 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.0270999999999999e-05 2023-08-28T02:37:35.052 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.052 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.052 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:37:35.052 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:28:26.473599+0000", 2023-08-28T02:37:35.052 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000115362 2023-08-28T02:37:35.052 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:35.052 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:37:35.052 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:35.052 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.052 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.053 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.55270.0:59980 7.3 7:c246cc2e:::benchmark_data_smithi143_360738_object59979:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e270)", 2023-08-28T02:37:35.053 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:31:43.482132+0000", 2023-08-28T02:37:35.053 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 351.36542763699998, 2023-08-28T02:37:35.053 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.981863826, 2023-08-28T02:37:35.053 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:37:35.053 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "reached pg", 2023-08-28T02:37:35.053 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:37:35.053 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.55270", 2023-08-28T02:37:35.053 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:37:35.053 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 59980 2023-08-28T02:37:35.053 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.054 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:37:35.054 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.054 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:37:35.054 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.482132+0000", 2023-08-28T02:37:35.054 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:35.054 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.054 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.054 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:37:35.054 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.482132+0000", 2023-08-28T02:37:35.054 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:35.054 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.055 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.055 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:37:35.055 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.482131+0000", 2023-08-28T02:37:35.055 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:37:35.055 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.055 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.055 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:37:35.055 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.482158+0000", 2023-08-28T02:37:35.055 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.7708999999999998e-05 2023-08-28T02:37:35.055 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.055 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.055 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:37:35.055 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.482178+0000", 2023-08-28T02:37:35.056 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.9840999999999999e-05 2023-08-28T02:37:35.056 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.056 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.056 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:37:35.056 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.482188+0000", 2023-08-28T02:37:35.056 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.7610000000000002e-06 2023-08-28T02:37:35.056 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.056 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.056 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:37:35.056 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.483028+0000", 2023-08-28T02:37:35.056 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00084029899999999997 2023-08-28T02:37:35.056 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.056 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.057 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op must wait for map", 2023-08-28T02:37:35.057 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.483034+0000", 2023-08-28T02:37:35.057 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.3290000000000001e-06 2023-08-28T02:37:35.057 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.057 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.057 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:37:35.057 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.641234+0000", 2023-08-28T02:37:35.057 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.15820019499999999 2023-08-28T02:37:35.057 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.057 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.057 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:37:35.057 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.641247+0000", 2023-08-28T02:37:35.057 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3149999999999999e-05 2023-08-28T02:37:35.058 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.058 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.058 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:37:35.058 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:44.463896+0000", 2023-08-28T02:37:35.058 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.82264939100000001 2023-08-28T02:37:35.058 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.058 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.058 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:37:35.058 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:44.463996+0000", 2023-08-28T02:37:35.058 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.9920000000000006e-05 2023-08-28T02:37:35.058 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:35.058 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:37:35.059 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:35.059 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.059 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.059 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.55270.0:59989 7.3 7:c10c4578:::benchmark_data_smithi143_360738_object59988:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e270)", 2023-08-28T02:37:35.059 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:31:43.497113+0000", 2023-08-28T02:37:35.059 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 351.35044659699997, 2023-08-28T02:37:35.059 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.98866365899999997, 2023-08-28T02:37:35.059 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:37:35.059 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:37:35.059 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:37:35.059 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.55270", 2023-08-28T02:37:35.059 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:37:35.060 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 59989 2023-08-28T02:37:35.060 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.060 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:37:35.060 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.060 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:37:35.060 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.497113+0000", 2023-08-28T02:37:35.060 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:35.060 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.060 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.060 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:37:35.060 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.497113+0000", 2023-08-28T02:37:35.060 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:35.061 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.061 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.061 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:37:35.061 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.497111+0000", 2023-08-28T02:37:35.061 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999971 2023-08-28T02:37:35.061 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.061 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.061 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:37:35.061 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.497159+0000", 2023-08-28T02:37:35.061 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.8668000000000002e-05 2023-08-28T02:37:35.061 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.061 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.061 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:37:35.062 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.497184+0000", 2023-08-28T02:37:35.062 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.4264999999999999e-05 2023-08-28T02:37:35.062 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.062 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.062 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:37:35.062 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.497198+0000", 2023-08-28T02:37:35.062 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3845999999999999e-05 2023-08-28T02:37:35.062 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.062 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.062 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:37:35.062 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.641768+0000", 2023-08-28T02:37:35.062 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.144570584 2023-08-28T02:37:35.062 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.063 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.063 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for peered", 2023-08-28T02:37:35.063 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:43.641775+0000", 2023-08-28T02:37:35.063 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.7329999999999997e-06 2023-08-28T02:37:35.063 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.063 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.063 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:37:35.063 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:44.467475+0000", 2023-08-28T02:37:35.063 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.82569998700000002 2023-08-28T02:37:35.063 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.063 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.063 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:37:35.064 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:44.467619+0000", 2023-08-28T02:37:35.064 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000143819 2023-08-28T02:37:35.064 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.064 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.064 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 2,4", 2023-08-28T02:37:35.064 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:44.467768+0000", 2023-08-28T02:37:35.064 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000149087 2023-08-28T02:37:35.064 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.064 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.064 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:37:35.064 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:44.473197+0000", 2023-08-28T02:37:35.064 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0054293579999999996 2023-08-28T02:37:35.064 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.065 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.065 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:35.065 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:44.476440+0000", 2023-08-28T02:37:35.065 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0032426199999999999 2023-08-28T02:37:35.065 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.065 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.065 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:35.065 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:44.485699+0000", 2023-08-28T02:37:35.065 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0092596469999999993 2023-08-28T02:37:35.065 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.065 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.065 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:37:35.066 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:44.485744+0000", 2023-08-28T02:37:35.066 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.4656000000000001e-05 2023-08-28T02:37:35.066 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.066 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.066 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:37:35.066 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:31:44.485777+0000", 2023-08-28T02:37:35.066 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.3037000000000002e-05 2023-08-28T02:37:35.066 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:35.066 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:37:35.066 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:35.066 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.066 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.066 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.45647.0:43335 8.0 8:01713a22:::benchmark_data_smithi143_419289_object43334:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e331)", 2023-08-28T02:37:35.066 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:35:51.779377+0000", 2023-08-28T02:37:35.067 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 103.06818350899999, 2023-08-28T02:37:35.067 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.30633328199999998, 2023-08-28T02:37:35.067 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:37:35.067 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:37:35.067 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:37:35.067 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.45647", 2023-08-28T02:37:35.067 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3230872490", 2023-08-28T02:37:35.067 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 43335 2023-08-28T02:37:35.067 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.067 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:37:35.067 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.067 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:37:35.067 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:51.779377+0000", 2023-08-28T02:37:35.068 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:35.068 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.068 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.068 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:37:35.068 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:51.779377+0000", 2023-08-28T02:37:35.068 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:35.068 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.068 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.068 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:37:35.068 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:51.779376+0000", 2023-08-28T02:37:35.068 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:37:35.068 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.069 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.069 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:37:35.069 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:51.779403+0000", 2023-08-28T02:37:35.069 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.7016e-05 2023-08-28T02:37:35.069 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.069 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.069 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:37:35.069 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:51.779414+0000", 2023-08-28T02:37:35.069 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1586000000000001e-05 2023-08-28T02:37:35.069 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.069 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.069 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:37:35.069 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:51.779422+0000", 2023-08-28T02:37:35.070 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.3499999999999999e-06 2023-08-28T02:37:35.070 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.070 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.070 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:37:35.070 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:52.017874+0000", 2023-08-28T02:37:35.070 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.23845287300000001 2023-08-28T02:37:35.070 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.070 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.070 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:37:35.070 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:52.018099+0000", 2023-08-28T02:37:35.070 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00022465500000000001 2023-08-28T02:37:35.070 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.071 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.071 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,5", 2023-08-28T02:37:35.071 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:52.018275+0000", 2023-08-28T02:37:35.071 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00017615 2023-08-28T02:37:35.071 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.071 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.071 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:35.071 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:52.020345+0000", 2023-08-28T02:37:35.071 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0020697269999999999 2023-08-28T02:37:35.071 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.071 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.071 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:37:35.072 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:52.026004+0000", 2023-08-28T02:37:35.072 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0056590299999999998 2023-08-28T02:37:35.072 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.072 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.072 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:35.072 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:52.085620+0000", 2023-08-28T02:37:35.072 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.059615509999999997 2023-08-28T02:37:35.072 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.072 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.072 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:37:35.072 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:52.085671+0000", 2023-08-28T02:37:35.072 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.1753999999999999e-05 2023-08-28T02:37:35.072 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.072 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.073 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:37:35.073 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:52.085710+0000", 2023-08-28T02:37:35.073 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.8552e-05 2023-08-28T02:37:35.073 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:35.073 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:37:35.073 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:35.073 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.073 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.073 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.45647.0:43340 8.f 8:ffbebdde:::benchmark_data_smithi143_419289_object43339:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e331)", 2023-08-28T02:37:35.073 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:35:51.782741+0000", 2023-08-28T02:37:35.073 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 103.064818684, 2023-08-28T02:37:35.073 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.30236935199999998, 2023-08-28T02:37:35.073 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:37:35.074 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:37:35.074 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:37:35.074 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.45647", 2023-08-28T02:37:35.074 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3230872490", 2023-08-28T02:37:35.074 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 43340 2023-08-28T02:37:35.074 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.074 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:37:35.074 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.074 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:37:35.074 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:51.782741+0000", 2023-08-28T02:37:35.074 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:35.074 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.075 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.075 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:37:35.075 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:51.782741+0000", 2023-08-28T02:37:35.075 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:35.075 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.075 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.075 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:37:35.075 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:51.782740+0000", 2023-08-28T02:37:35.075 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:37:35.075 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.075 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.076 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:37:35.076 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:51.782768+0000", 2023-08-28T02:37:35.076 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.8042999999999999e-05 2023-08-28T02:37:35.076 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.076 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.076 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:37:35.076 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:52.017825+0000", 2023-08-28T02:37:35.076 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.235057247 2023-08-28T02:37:35.076 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.076 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.076 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:37:35.076 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:52.017846+0000", 2023-08-28T02:37:35.076 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.0917999999999999e-05 2023-08-28T02:37:35.077 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.077 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.077 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:37:35.077 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:52.018153+0000", 2023-08-28T02:37:35.077 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00030716499999999998 2023-08-28T02:37:35.077 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.077 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.077 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:37:35.077 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:52.018281+0000", 2023-08-28T02:37:35.077 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000127782 2023-08-28T02:37:35.077 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.077 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.077 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,2", 2023-08-28T02:37:35.077 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:52.018406+0000", 2023-08-28T02:37:35.078 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00012502800000000001 2023-08-28T02:37:35.078 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.078 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.078 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:37:35.078 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:52.021895+0000", 2023-08-28T02:37:35.078 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0034886740000000002 2023-08-28T02:37:35.078 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.078 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.078 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:35.078 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:52.022027+0000", 2023-08-28T02:37:35.078 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00013228900000000001 2023-08-28T02:37:35.078 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.078 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.079 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:35.079 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:52.085000+0000", 2023-08-28T02:37:35.079 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.062972755000000005 2023-08-28T02:37:35.079 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.079 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.079 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:37:35.079 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:52.085068+0000", 2023-08-28T02:37:35.079 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.8456999999999999e-05 2023-08-28T02:37:35.079 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.079 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.079 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:37:35.079 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:35:52.085111+0000", 2023-08-28T02:37:35.079 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.2580000000000002e-05 2023-08-28T02:37:35.080 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:35.080 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:37:35.080 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:35.080 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.080 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.080 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.45647.0:69734 8.b 8:d14e1d38:::benchmark_data_smithi143_419289_object69733:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e335)", 2023-08-28T02:37:35.080 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:36:18.174948+0000", 2023-08-28T02:37:35.080 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 76.672612290999993, 2023-08-28T02:37:35.080 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.30353469100000002, 2023-08-28T02:37:35.080 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:37:35.080 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:37:35.080 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:37:35.080 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.45647", 2023-08-28T02:37:35.081 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3230872490", 2023-08-28T02:37:35.081 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 69734 2023-08-28T02:37:35.081 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.081 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:37:35.081 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.081 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:37:35.081 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.174948+0000", 2023-08-28T02:37:35.081 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:35.081 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.081 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.081 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:37:35.081 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.174948+0000", 2023-08-28T02:37:35.081 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:35.081 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.082 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.082 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:37:35.082 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.174947+0000", 2023-08-28T02:37:35.082 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:37:35.082 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.082 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.082 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:37:35.082 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.174969+0000", 2023-08-28T02:37:35.082 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.2670000000000001e-05 2023-08-28T02:37:35.082 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.082 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.082 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:37:35.082 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.174981+0000", 2023-08-28T02:37:35.083 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.172e-05 2023-08-28T02:37:35.083 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.083 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.083 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:37:35.083 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.174989+0000", 2023-08-28T02:37:35.083 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.9359999999999999e-06 2023-08-28T02:37:35.083 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.083 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.083 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:37:35.083 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.175063+0000", 2023-08-28T02:37:35.083 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.4107e-05 2023-08-28T02:37:35.083 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.083 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.084 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:37:35.084 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.175198+0000", 2023-08-28T02:37:35.084 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00013449800000000001 2023-08-28T02:37:35.084 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.084 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.084 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,4", 2023-08-28T02:37:35.084 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.175342+0000", 2023-08-28T02:37:35.084 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00014475100000000001 2023-08-28T02:37:35.084 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.084 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.084 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:37:35.084 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.178208+0000", 2023-08-28T02:37:35.084 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.002865308 2023-08-28T02:37:35.085 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.085 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.085 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:35.085 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.181451+0000", 2023-08-28T02:37:35.085 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0032432989999999998 2023-08-28T02:37:35.085 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.085 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.085 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:35.085 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.478398+0000", 2023-08-28T02:37:35.085 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.296946924 2023-08-28T02:37:35.085 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.085 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.085 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:37:35.086 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.478444+0000", 2023-08-28T02:37:35.086 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.6412000000000002e-05 2023-08-28T02:37:35.086 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.086 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.086 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:37:35.086 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.478482+0000", 2023-08-28T02:37:35.086 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.8145000000000002e-05 2023-08-28T02:37:35.086 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:35.087 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:37:35.087 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:35.087 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.087 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.087 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.45647.0:69738 8.8 8:1d549489:::benchmark_data_smithi143_419289_object69737:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e335)", 2023-08-28T02:37:35.087 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:36:18.178265+0000", 2023-08-28T02:37:35.087 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 76.669295411999997, 2023-08-28T02:37:35.087 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.30018081200000002, 2023-08-28T02:37:35.087 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:37:35.087 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:37:35.088 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:37:35.088 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.45647", 2023-08-28T02:37:35.088 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3230872490", 2023-08-28T02:37:35.088 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 69738 2023-08-28T02:37:35.088 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.088 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:37:35.088 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.088 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:37:35.088 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.178265+0000", 2023-08-28T02:37:35.088 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:35.088 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.088 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.088 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:37:35.089 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.178265+0000", 2023-08-28T02:37:35.089 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:35.089 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.089 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.089 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:37:35.089 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.178263+0000", 2023-08-28T02:37:35.089 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:37:35.089 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.089 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.089 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:37:35.089 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.178288+0000", 2023-08-28T02:37:35.089 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.5184000000000001e-05 2023-08-28T02:37:35.089 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.090 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.090 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:37:35.090 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.178301+0000", 2023-08-28T02:37:35.090 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.2843000000000001e-05 2023-08-28T02:37:35.090 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.090 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.090 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:37:35.090 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.178309+0000", 2023-08-28T02:37:35.090 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.0590000000000003e-06 2023-08-28T02:37:35.090 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.090 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.090 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:37:35.090 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.178384+0000", 2023-08-28T02:37:35.091 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.5082999999999999e-05 2023-08-28T02:37:35.091 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.091 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.091 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:37:35.091 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.178521+0000", 2023-08-28T02:37:35.091 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00013647600000000001 2023-08-28T02:37:35.091 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.091 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.091 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:37:35.091 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.178661+0000", 2023-08-28T02:37:35.091 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000139822 2023-08-28T02:37:35.091 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.092 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.092 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:37:35.092 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.184146+0000", 2023-08-28T02:37:35.092 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0054850890000000003 2023-08-28T02:37:35.092 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.092 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.092 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:35.092 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.184743+0000", 2023-08-28T02:37:35.092 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00059688500000000002 2023-08-28T02:37:35.092 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.092 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.092 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:35.092 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.478348+0000", 2023-08-28T02:37:35.092 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.29360530400000001 2023-08-28T02:37:35.093 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.093 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.093 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:37:35.093 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.478401+0000", 2023-08-28T02:37:35.093 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.2676000000000002e-05 2023-08-28T02:37:35.093 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.093 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.093 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:37:35.093 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.478445+0000", 2023-08-28T02:37:35.093 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.4811000000000002e-05 2023-08-28T02:37:35.093 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:35.093 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:37:35.093 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:35.093 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.094 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.094 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.45647.0:69740 8.0 8:0c3fede6:::benchmark_data_smithi143_419289_object69739:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e335)", 2023-08-28T02:37:35.094 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:36:18.178401+0000", 2023-08-28T02:37:35.094 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 76.669158709000001, 2023-08-28T02:37:35.094 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.30025965300000002, 2023-08-28T02:37:35.094 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:37:35.094 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:37:35.094 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:37:35.094 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.45647", 2023-08-28T02:37:35.094 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3230872490", 2023-08-28T02:37:35.094 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 69740 2023-08-28T02:37:35.094 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.094 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:37:35.094 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.095 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:37:35.095 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.178401+0000", 2023-08-28T02:37:35.095 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:35.095 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.095 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.095 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:37:35.095 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.178401+0000", 2023-08-28T02:37:35.095 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:35.095 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.095 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.095 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:37:35.095 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.178400+0000", 2023-08-28T02:37:35.095 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:37:35.096 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.096 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.096 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:37:35.096 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.178424+0000", 2023-08-28T02:37:35.096 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.3991000000000001e-05 2023-08-28T02:37:35.096 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.096 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.096 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:37:35.096 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.178435+0000", 2023-08-28T02:37:35.096 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.0611999999999999e-05 2023-08-28T02:37:35.096 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.096 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.096 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:37:35.096 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.178442+0000", 2023-08-28T02:37:35.097 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.3629999999999996e-06 2023-08-28T02:37:35.097 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.097 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.097 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:37:35.097 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.178528+0000", 2023-08-28T02:37:35.097 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.5692000000000006e-05 2023-08-28T02:37:35.097 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.097 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.097 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:37:35.097 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.178653+0000", 2023-08-28T02:37:35.097 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00012562900000000001 2023-08-28T02:37:35.097 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.097 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.098 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,5", 2023-08-28T02:37:35.098 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.181395+0000", 2023-08-28T02:37:35.098 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0027415489999999998 2023-08-28T02:37:35.098 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.098 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.098 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:35.098 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.478308+0000", 2023-08-28T02:37:35.098 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.29691279500000001 2023-08-28T02:37:35.098 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.098 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.098 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:35.098 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.478482+0000", 2023-08-28T02:37:35.098 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00017467900000000001 2023-08-28T02:37:35.099 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.099 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.099 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:37:35.099 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.478569+0000", 2023-08-28T02:37:35.099 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.7013000000000002e-05 2023-08-28T02:37:35.099 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.099 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.099 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:37:35.099 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.478620+0000", 2023-08-28T02:37:35.099 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.0853999999999998e-05 2023-08-28T02:37:35.099 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.099 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.099 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:37:35.100 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.478661+0000", 2023-08-28T02:37:35.100 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.0676999999999997e-05 2023-08-28T02:37:35.100 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:35.100 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:37:35.100 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:35.100 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.100 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.100 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.45647.0:69742 8.f 8:f526aedc:::benchmark_data_smithi143_419289_object69741:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e335)", 2023-08-28T02:37:35.100 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:36:18.178449+0000", 2023-08-28T02:37:35.100 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 76.669111399000002, 2023-08-28T02:37:35.100 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.30023462099999998, 2023-08-28T02:37:35.100 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:37:35.100 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:37:35.100 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:37:35.101 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.45647", 2023-08-28T02:37:35.101 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3230872490", 2023-08-28T02:37:35.101 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 69742 2023-08-28T02:37:35.101 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.101 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:37:35.101 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.101 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:37:35.101 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.178449+0000", 2023-08-28T02:37:35.101 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:35.101 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.101 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.101 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:37:35.101 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.178449+0000", 2023-08-28T02:37:35.102 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:35.102 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.102 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.102 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:37:35.102 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.178448+0000", 2023-08-28T02:37:35.102 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:37:35.102 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.102 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.102 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:37:35.102 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.178473+0000", 2023-08-28T02:37:35.102 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.4420999999999999e-05 2023-08-28T02:37:35.102 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.102 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.102 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:37:35.103 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.178480+0000", 2023-08-28T02:37:35.103 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.7929999999999998e-06 2023-08-28T02:37:35.103 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.103 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.103 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:37:35.103 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.178486+0000", 2023-08-28T02:37:35.103 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.9529999999999996e-06 2023-08-28T02:37:35.103 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.103 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.103 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:37:35.103 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.178959+0000", 2023-08-28T02:37:35.103 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00047245799999999998 2023-08-28T02:37:35.104 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.104 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.104 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:37:35.104 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.179080+0000", 2023-08-28T02:37:35.104 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00012125699999999999 2023-08-28T02:37:35.104 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.104 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.104 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,2", 2023-08-28T02:37:35.104 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.179202+0000", 2023-08-28T02:37:35.104 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000121657 2023-08-28T02:37:35.104 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.104 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.104 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:37:35.104 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.182027+0000", 2023-08-28T02:37:35.105 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0028248930000000002 2023-08-28T02:37:35.105 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.105 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.105 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:35.105 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.182247+0000", 2023-08-28T02:37:35.105 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00022005899999999999 2023-08-28T02:37:35.105 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.105 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.105 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:35.105 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.478618+0000", 2023-08-28T02:37:35.105 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.29637111300000002 2023-08-28T02:37:35.105 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.105 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.106 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:37:35.106 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.478658+0000", 2023-08-28T02:37:35.106 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.0173000000000003e-05 2023-08-28T02:37:35.106 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.106 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.106 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:37:35.106 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:36:18.478683+0000", 2023-08-28T02:37:35.106 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.5420999999999999e-05 2023-08-28T02:37:35.106 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:35.106 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:37:35.106 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:35.106 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.106 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.107 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.45647.0:116769 8.8 8:188f03ba:::benchmark_data_smithi143_419289_object19859:head [delete] snapc 0=[] ondisk+write+known_if_redirected e341)", 2023-08-28T02:37:35.107 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:37:21.444250+0000", 2023-08-28T02:37:35.107 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 13.403310504, 2023-08-28T02:37:35.107 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.19071837999999999, 2023-08-28T02:37:35.107 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:37:35.107 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:37:35.107 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:37:35.107 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.45647", 2023-08-28T02:37:35.107 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3230872490", 2023-08-28T02:37:35.107 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 116769 2023-08-28T02:37:35.107 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.107 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:37:35.107 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.108 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:37:35.108 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.444250+0000", 2023-08-28T02:37:35.108 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:35.108 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.108 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.108 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:37:35.108 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.444250+0000", 2023-08-28T02:37:35.108 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:35.108 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.108 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.108 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:37:35.108 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.444248+0000", 2023-08-28T02:37:35.108 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:37:35.108 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.109 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.109 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:37:35.109 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.444257+0000", 2023-08-28T02:37:35.109 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.6200000000000005e-06 2023-08-28T02:37:35.109 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.109 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.109 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:37:35.109 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.444283+0000", 2023-08-28T02:37:35.109 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.5953e-05 2023-08-28T02:37:35.109 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.109 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.109 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:37:35.109 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.444292+0000", 2023-08-28T02:37:35.109 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.8700000000000004e-06 2023-08-28T02:37:35.110 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.110 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.110 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:37:35.110 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.448087+0000", 2023-08-28T02:37:35.110 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0037950190000000002 2023-08-28T02:37:35.110 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.110 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.110 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:37:35.110 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.448226+0000", 2023-08-28T02:37:35.110 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000138174 2023-08-28T02:37:35.110 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.110 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.110 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:37:35.111 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.448344+0000", 2023-08-28T02:37:35.111 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000118882 2023-08-28T02:37:35.111 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.111 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.111 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:37:35.111 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.455912+0000", 2023-08-28T02:37:35.111 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0075675660000000004 2023-08-28T02:37:35.111 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.111 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.111 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:35.111 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.460213+0000", 2023-08-28T02:37:35.111 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0043005969999999998 2023-08-28T02:37:35.111 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.112 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.112 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:35.112 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.634890+0000", 2023-08-28T02:37:35.112 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.174677841 2023-08-28T02:37:35.112 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.112 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.112 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:37:35.112 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.634932+0000", 2023-08-28T02:37:35.112 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.1468999999999999e-05 2023-08-28T02:37:35.112 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.112 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.113 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:37:35.113 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.634968+0000", 2023-08-28T02:37:35.113 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.6003000000000001e-05 2023-08-28T02:37:35.113 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:35.113 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:37:35.113 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:35.113 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.113 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.113 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.45647.0:116761 8.c e341/318 8:3d3eeb21:::benchmark_data_smithi143_419289_object19851:head v 341'7074, mlcod=341'7070)", 2023-08-28T02:37:35.113 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:37:21.447467+0000", 2023-08-28T02:37:35.113 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 13.400093108, 2023-08-28T02:37:35.113 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.18222854099999999, 2023-08-28T02:37:35.113 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:37:35.114 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:37:35.114 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:37:35.114 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.114 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:37:35.114 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.447467+0000", 2023-08-28T02:37:35.114 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:35.114 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.114 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.114 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:37:35.114 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.447467+0000", 2023-08-28T02:37:35.114 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:35.114 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.114 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.115 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:37:35.115 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.447467+0000", 2023-08-28T02:37:35.115 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:37:35.115 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.115 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.115 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:37:35.115 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.447470+0000", 2023-08-28T02:37:35.115 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.5719999999999999e-06 2023-08-28T02:37:35.115 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.115 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.115 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:37:35.115 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.447484+0000", 2023-08-28T02:37:35.115 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3385e-05 2023-08-28T02:37:35.116 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.116 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.116 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:37:35.116 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.447488+0000", 2023-08-28T02:37:35.116 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.2520000000000001e-06 2023-08-28T02:37:35.116 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.116 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.116 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:37:35.116 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.451416+0000", 2023-08-28T02:37:35.116 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.003928244 2023-08-28T02:37:35.116 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.116 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.116 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:37:35.117 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.451434+0000", 2023-08-28T02:37:35.117 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.7620000000000001e-05 2023-08-28T02:37:35.117 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.117 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.117 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:37:35.117 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.461885+0000", 2023-08-28T02:37:35.117 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.010451201 2023-08-28T02:37:35.117 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.117 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.117 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:37:35.117 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.629695+0000", 2023-08-28T02:37:35.117 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.167810509 2023-08-28T02:37:35.117 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:35.118 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:37:35.118 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:35.118 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.118 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.118 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.45647.0:116770 8.8 8:1731c4fa:::benchmark_data_smithi143_419289_object19860:head [delete] snapc 0=[] ondisk+write+known_if_redirected e341)", 2023-08-28T02:37:35.118 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:37:21.448138+0000", 2023-08-28T02:37:35.118 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 13.399422345, 2023-08-28T02:37:35.118 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.18693753099999999, 2023-08-28T02:37:35.118 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:37:35.118 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:37:35.118 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:37:35.118 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.45647", 2023-08-28T02:37:35.118 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3230872490", 2023-08-28T02:37:35.119 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 116770 2023-08-28T02:37:35.119 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.119 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:37:35.119 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.119 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:37:35.119 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.448138+0000", 2023-08-28T02:37:35.119 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:35.119 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.119 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.119 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:37:35.119 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.448138+0000", 2023-08-28T02:37:35.119 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:35.120 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.120 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.120 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:37:35.120 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.448136+0000", 2023-08-28T02:37:35.120 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:37:35.120 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.120 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.120 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:37:35.120 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.448142+0000", 2023-08-28T02:37:35.120 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.0039999999999996e-06 2023-08-28T02:37:35.120 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.120 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.120 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:37:35.121 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.448154+0000", 2023-08-28T02:37:35.121 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1365e-05 2023-08-28T02:37:35.121 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.121 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.121 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:37:35.121 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.448161+0000", 2023-08-28T02:37:35.121 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.7139999999999993e-06 2023-08-28T02:37:35.121 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.121 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.121 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:37:35.121 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.452043+0000", 2023-08-28T02:37:35.121 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0038814079999999998 2023-08-28T02:37:35.121 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.121 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.122 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:37:35.122 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.452169+0000", 2023-08-28T02:37:35.122 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00012588000000000001 2023-08-28T02:37:35.122 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.122 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.122 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:37:35.122 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.452283+0000", 2023-08-28T02:37:35.122 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000114208 2023-08-28T02:37:35.122 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.122 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.122 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:35.122 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.460242+0000", 2023-08-28T02:37:35.122 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0079588850000000006 2023-08-28T02:37:35.122 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.123 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.123 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:37:35.123 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.634814+0000", 2023-08-28T02:37:35.123 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.17457198500000001 2023-08-28T02:37:35.123 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.123 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.123 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:35.123 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.634999+0000", 2023-08-28T02:37:35.123 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00018582399999999999 2023-08-28T02:37:35.123 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.123 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.123 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:37:35.123 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.635046+0000", 2023-08-28T02:37:35.124 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.6496999999999999e-05 2023-08-28T02:37:35.124 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.124 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.124 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:37:35.124 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.635075+0000", 2023-08-28T02:37:35.124 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.9312999999999999e-05 2023-08-28T02:37:35.124 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:35.124 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:37:35.125 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:35.125 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.125 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.125 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.45647.0:116768 8.c e341/318 8:3e0df643:::benchmark_data_smithi143_419289_object19858:head v 341'7075, mlcod=341'7070)", 2023-08-28T02:37:35.125 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:37:21.452113+0000", 2023-08-28T02:37:35.125 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 13.395447411999999, 2023-08-28T02:37:35.125 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.17771979399999999, 2023-08-28T02:37:35.126 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:37:35.126 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:37:35.126 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:37:35.126 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.126 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:37:35.126 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.452113+0000", 2023-08-28T02:37:35.126 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:35.126 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.126 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.126 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:37:35.126 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.452113+0000", 2023-08-28T02:37:35.126 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:35.127 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.127 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.127 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:37:35.127 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.452112+0000", 2023-08-28T02:37:35.127 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:37:35.127 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.127 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.127 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:37:35.127 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.452119+0000", 2023-08-28T02:37:35.127 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.2729999999999997e-06 2023-08-28T02:37:35.127 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.127 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.128 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:37:35.128 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.452128+0000", 2023-08-28T02:37:35.128 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.8359999999999998e-06 2023-08-28T02:37:35.128 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.128 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.128 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:37:35.128 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.452134+0000", 2023-08-28T02:37:35.128 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.6720000000000001e-06 2023-08-28T02:37:35.128 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.128 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.128 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:37:35.128 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.457455+0000", 2023-08-28T02:37:35.128 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0053216519999999996 2023-08-28T02:37:35.129 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.129 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.129 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:37:35.129 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.457474+0000", 2023-08-28T02:37:35.129 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.9099000000000001e-05 2023-08-28T02:37:35.129 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.129 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.129 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:37:35.129 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.629811+0000", 2023-08-28T02:37:35.129 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.17233696600000001 2023-08-28T02:37:35.129 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.129 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.129 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:37:35.130 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.629832+0000", 2023-08-28T02:37:35.130 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.1111000000000002e-05 2023-08-28T02:37:35.130 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:35.130 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:37:35.130 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:35.130 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.130 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.130 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.45647.0:116772 8.b 8:d8a40377:::benchmark_data_smithi143_419289_object19862:head [delete] snapc 0=[] ondisk+write+known_if_redirected e341)", 2023-08-28T02:37:35.130 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:37:21.453145+0000", 2023-08-28T02:37:35.130 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 13.394415327000001, 2023-08-28T02:37:35.130 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.18094517299999999, 2023-08-28T02:37:35.130 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:37:35.130 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:37:35.130 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:37:35.131 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.45647", 2023-08-28T02:37:35.131 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3230872490", 2023-08-28T02:37:35.131 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 116772 2023-08-28T02:37:35.131 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.131 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:37:35.131 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.131 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:37:35.131 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.453145+0000", 2023-08-28T02:37:35.131 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:35.131 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.131 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.131 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:37:35.131 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.453145+0000", 2023-08-28T02:37:35.132 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:35.132 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.132 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.132 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:37:35.132 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.453143+0000", 2023-08-28T02:37:35.132 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:37:35.132 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.132 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.132 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:37:35.132 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.453150+0000", 2023-08-28T02:37:35.132 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.5729999999999996e-06 2023-08-28T02:37:35.132 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.132 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.132 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:37:35.133 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.453162+0000", 2023-08-28T02:37:35.133 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1642e-05 2023-08-28T02:37:35.133 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.133 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.133 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:37:35.133 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.453170+0000", 2023-08-28T02:37:35.133 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.0930000000000003e-06 2023-08-28T02:37:35.133 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.133 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.133 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:37:35.133 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.453247+0000", 2023-08-28T02:37:35.133 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.7338999999999999e-05 2023-08-28T02:37:35.133 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.134 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.134 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:37:35.134 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.453353+0000", 2023-08-28T02:37:35.134 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000106488 2023-08-28T02:37:35.134 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.134 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.134 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,4", 2023-08-28T02:37:35.134 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.453460+0000", 2023-08-28T02:37:35.134 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00010612599999999999 2023-08-28T02:37:35.134 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.134 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.134 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:35.134 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.461445+0000", 2023-08-28T02:37:35.134 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0079858900000000007 2023-08-28T02:37:35.135 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.135 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.135 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:35.135 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.461477+0000", 2023-08-28T02:37:35.135 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.1520999999999999e-05 2023-08-28T02:37:35.135 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.135 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.135 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:37:35.135 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.633998+0000", 2023-08-28T02:37:35.135 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.17252115200000001 2023-08-28T02:37:35.135 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.135 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.135 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:37:35.136 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.634051+0000", 2023-08-28T02:37:35.136 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.2828999999999999e-05 2023-08-28T02:37:35.136 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.136 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.136 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:37:35.136 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.634090+0000", 2023-08-28T02:37:35.136 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.8949000000000003e-05 2023-08-28T02:37:35.136 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:35.136 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:37:35.136 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:35.136 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.136 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.136 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.45647.0:116774 8.f 8:fa1b4a51:::benchmark_data_smithi143_419289_object19864:head [delete] snapc 0=[] ondisk+write+known_if_redirected e341)", 2023-08-28T02:37:35.136 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:37:21.453186+0000", 2023-08-28T02:37:35.137 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 13.394374173999999, 2023-08-28T02:37:35.137 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.180463601, 2023-08-28T02:37:35.137 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:37:35.137 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:37:35.137 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:37:35.137 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.45647", 2023-08-28T02:37:35.137 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3230872490", 2023-08-28T02:37:35.137 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 116774 2023-08-28T02:37:35.137 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.137 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:37:35.137 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.137 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:37:35.138 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.453186+0000", 2023-08-28T02:37:35.138 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:35.138 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.138 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.138 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:37:35.138 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.453186+0000", 2023-08-28T02:37:35.138 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:35.138 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.138 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.138 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:37:35.138 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.453185+0000", 2023-08-28T02:37:35.138 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:37:35.138 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.139 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.139 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:37:35.139 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.453188+0000", 2023-08-28T02:37:35.139 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.1190000000000001e-06 2023-08-28T02:37:35.139 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.139 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.139 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:37:35.139 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.453196+0000", 2023-08-28T02:37:35.139 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.126e-06 2023-08-28T02:37:35.139 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.139 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.139 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:37:35.140 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.453201+0000", 2023-08-28T02:37:35.140 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.609e-06 2023-08-28T02:37:35.140 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.140 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.140 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:37:35.140 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.453244+0000", 2023-08-28T02:37:35.140 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.3121999999999998e-05 2023-08-28T02:37:35.140 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.140 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.140 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:37:35.140 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.453352+0000", 2023-08-28T02:37:35.140 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000107624 2023-08-28T02:37:35.140 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.141 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.141 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,2", 2023-08-28T02:37:35.141 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.453457+0000", 2023-08-28T02:37:35.141 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000105041 2023-08-28T02:37:35.141 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.141 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.141 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:37:35.141 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.633454+0000", 2023-08-28T02:37:35.141 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.17999667699999999 2023-08-28T02:37:35.141 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.141 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.142 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:35.142 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.633514+0000", 2023-08-28T02:37:35.142 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.0597000000000003e-05 2023-08-28T02:37:35.142 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.142 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.142 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:35.142 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.633556+0000", 2023-08-28T02:37:35.142 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.1600999999999997e-05 2023-08-28T02:37:35.142 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.142 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.142 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:37:35.142 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.633607+0000", 2023-08-28T02:37:35.142 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.1487999999999999e-05 2023-08-28T02:37:35.142 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.143 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.143 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:37:35.143 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.633649+0000", 2023-08-28T02:37:35.143 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.2110000000000002e-05 2023-08-28T02:37:35.143 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:35.143 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:37:35.143 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:35.143 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.143 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.143 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.45647.0:116780 8.8 8:1b4144c9:::benchmark_data_smithi143_419289_object19870:head [delete] snapc 0=[] ondisk+write+known_if_redirected e341)", 2023-08-28T02:37:35.143 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:37:21.460874+0000", 2023-08-28T02:37:35.143 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 13.386686543, 2023-08-28T02:37:35.143 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.18621083999999999, 2023-08-28T02:37:35.144 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:37:35.144 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:37:35.144 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:37:35.144 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.45647", 2023-08-28T02:37:35.144 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3230872490", 2023-08-28T02:37:35.144 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 116780 2023-08-28T02:37:35.144 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.144 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:37:35.144 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.144 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:37:35.144 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.460874+0000", 2023-08-28T02:37:35.144 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:35.144 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.144 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.145 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:37:35.145 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.460874+0000", 2023-08-28T02:37:35.145 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:35.145 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.145 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.145 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:37:35.145 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.460872+0000", 2023-08-28T02:37:35.145 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:37:35.145 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.145 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.145 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:37:35.145 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.460879+0000", 2023-08-28T02:37:35.145 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.807e-06 2023-08-28T02:37:35.145 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.146 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.146 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:37:35.146 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.460893+0000", 2023-08-28T02:37:35.146 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3912e-05 2023-08-28T02:37:35.146 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.146 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.146 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:37:35.146 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.460902+0000", 2023-08-28T02:37:35.146 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.4570000000000005e-06 2023-08-28T02:37:35.146 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.146 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.146 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:37:35.146 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.461507+0000", 2023-08-28T02:37:35.147 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00060472900000000003 2023-08-28T02:37:35.147 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.147 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.147 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:37:35.147 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.461618+0000", 2023-08-28T02:37:35.147 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000111776 2023-08-28T02:37:35.147 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.147 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.147 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:37:35.147 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.461728+0000", 2023-08-28T02:37:35.147 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000109751 2023-08-28T02:37:35.148 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.148 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.148 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:35.148 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.634856+0000", 2023-08-28T02:37:35.148 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.17312777800000001 2023-08-28T02:37:35.148 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.148 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.148 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:35.148 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.635109+0000", 2023-08-28T02:37:35.148 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00025318299999999998 2023-08-28T02:37:35.148 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.148 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.148 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:37:35.149 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.646959+0000", 2023-08-28T02:37:35.149 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.011849564999999999 2023-08-28T02:37:35.149 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.149 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.149 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:37:35.149 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.647027+0000", 2023-08-28T02:37:35.149 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.8538e-05 2023-08-28T02:37:35.149 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.149 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.149 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:37:35.149 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.647084+0000", 2023-08-28T02:37:35.149 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.7241000000000003e-05 2023-08-28T02:37:35.150 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:35.150 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:37:35.150 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:35.150 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.150 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.150 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.45647.0:116783 8.b 8:defafc2c:::benchmark_data_smithi143_419289_object19873:head [delete] snapc 0=[] ondisk+write+known_if_redirected e341)", 2023-08-28T02:37:35.150 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:37:21.465013+0000", 2023-08-28T02:37:35.150 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 13.382547553, 2023-08-28T02:37:35.150 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.17764312099999999, 2023-08-28T02:37:35.150 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:37:35.150 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:37:35.150 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:37:35.150 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.45647", 2023-08-28T02:37:35.151 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3230872490", 2023-08-28T02:37:35.151 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 116783 2023-08-28T02:37:35.151 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.151 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:37:35.151 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.151 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:37:35.151 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.465013+0000", 2023-08-28T02:37:35.151 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:35.151 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.151 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.151 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:37:35.151 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.465013+0000", 2023-08-28T02:37:35.152 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:37:35.152 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.152 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.152 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:37:35.152 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.465011+0000", 2023-08-28T02:37:35.152 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:37:35.152 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.152 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.152 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:37:35.152 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.465018+0000", 2023-08-28T02:37:35.152 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.2139999999999999e-06 2023-08-28T02:37:35.152 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.152 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.152 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:37:35.153 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.629639+0000", 2023-08-28T02:37:35.153 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.164620724 2023-08-28T02:37:35.153 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.153 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.153 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:37:35.153 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.629657+0000", 2023-08-28T02:37:35.153 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.8428999999999999e-05 2023-08-28T02:37:35.153 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.153 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.153 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:37:35.153 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.629922+0000", 2023-08-28T02:37:35.154 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00026484700000000002 2023-08-28T02:37:35.154 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.154 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.154 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:37:35.154 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.630102+0000", 2023-08-28T02:37:35.154 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000180237 2023-08-28T02:37:35.154 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.154 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.154 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 1,4", 2023-08-28T02:37:35.154 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.630241+0000", 2023-08-28T02:37:35.154 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00013856299999999999 2023-08-28T02:37:35.155 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.155 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.155 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:35.155 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.634185+0000", 2023-08-28T02:37:35.155 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.003944187 2023-08-28T02:37:35.155 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.155 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.155 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:35.155 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.639040+0000", 2023-08-28T02:37:35.155 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0048551189999999998 2023-08-28T02:37:35.155 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.155 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.155 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:37:35.156 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.642573+0000", 2023-08-28T02:37:35.156 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00353258 2023-08-28T02:37:35.156 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.156 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.156 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "commit_sent", 2023-08-28T02:37:35.156 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.642616+0000", 2023-08-28T02:37:35.156 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.3123e-05 2023-08-28T02:37:35.156 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:37:35.156 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:37:35.156 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "done", 2023-08-28T02:37:35.156 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:37:21.642656+0000", 2023-08-28T02:37:35.156 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.9678999999999997e-05 2023-08-28T02:37:35.156 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:35.157 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:37:35.157 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:35.157 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:37:35.157 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:37:35.157 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:37:35.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:34 smithi143 ceph-mon[183059]: pgmap v915: 89 pgs: 89 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 670 op/s 2023-08-28T02:37:35.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:34 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3751065593' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:37:35.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:34 smithi143 ceph-mon[189871]: pgmap v915: 89 pgs: 89 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 670 op/s 2023-08-28T02:37:35.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:34 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3751065593' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:37:35.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:34 smithi169 ceph-mon[159415]: pgmap v915: 89 pgs: 89 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 670 op/s 2023-08-28T02:37:35.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:34 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3751065593' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:37:35.267 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:37:35.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:35.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:37:35.381 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:37:35.423 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:35.500 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:37:35.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:35.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:37:35.609 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:37:35.686 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:36.118 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:37:36.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:36.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:37:36.223 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:37:36.254 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:36.321 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:37:36.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:36.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:37:36.429 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:37:36.490 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:37.044 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:36 smithi143 ceph-mon[183059]: pgmap v916: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 678 op/s 2023-08-28T02:37:37.044 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:36 smithi143 ceph-mon[189871]: pgmap v916: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 678 op/s 2023-08-28T02:37:37.064 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.4 dump_ops_in_flight 2023-08-28T02:37:37.074 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:37:37.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:37.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:37:37.099 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:36 smithi169 ceph-mon[159415]: pgmap v916: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 678 op/s 2023-08-28T02:37:37.192 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:37:37.241 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:37.327 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:37:37.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:37.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:37:37.436 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:37:37.490 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:37.563 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:37:37.564 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:37:37.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:37.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:37:37.669 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:37:37.706 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:37.888 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:37:37.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:37.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:37:37.996 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:37:38.035 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:38.257 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:37:38.260 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:37:38.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:38.265 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:37:38.366 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:37:38.486 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:38.636 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:37:38.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:38.650 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:37:38.751 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:37:38.783 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:38.840 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:37:38.841 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is unset 2023-08-28T02:37:38.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:38.845 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:37:38.946 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:37:38.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:38 smithi143 ceph-mon[183059]: pgmap v917: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 685 op/s 2023-08-28T02:37:38.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:38 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2535071724' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:37:38.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:38 smithi143 ceph-mon[189871]: pgmap v917: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 685 op/s 2023-08-28T02:37:38.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:38 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2535071724' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:37:38.982 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:39.041 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:37:39.045 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:38 smithi169 ceph-mon[159415]: pgmap v917: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 685 op/s 2023-08-28T02:37:39.046 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:38 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2535071724' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:37:39.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:39.046 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:37:39.147 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:37:39.167 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 6, 1] out_osds: [7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-08-28T02:37:39.168 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-08-28T02:37:39.168 INFO:tasks.thrashosds.thrasher:Adding osd 7 2023-08-28T02:37:39.168 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd in 7 2023-08-28T02:37:39.192 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:39.248 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:37:39.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:39.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:37:39.353 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:37:39.396 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:39.462 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:37:39.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:39.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:37:39.565 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:37:39.727 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:40.086 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:37:40.087 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:39 smithi143 ceph-mon[183059]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:37:40.087 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:39 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2535071724' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:37:40.087 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:39 smithi143 ceph-mon[183059]: osdmap e343: 8 total, 8 up, 7 in 2023-08-28T02:37:40.087 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:39 smithi143 ceph-mon[183059]: failed to encode map e343 with expected crc 2023-08-28T02:37:40.087 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:39 smithi143 ceph-mon[183059]: failed to encode map e343 with expected crc 2023-08-28T02:37:40.087 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:39 smithi143 ceph-mon[183059]: failed to encode map e343 with expected crc 2023-08-28T02:37:40.087 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:39 smithi143 ceph-mon[183059]: failed to encode map e343 with expected crc 2023-08-28T02:37:40.088 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:39 smithi143 ceph-mon[183059]: failed to encode map e343 with expected crc 2023-08-28T02:37:40.088 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:39 smithi143 ceph-mon[183059]: failed to encode map e343 with expected crc 2023-08-28T02:37:40.088 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:39 smithi143 ceph-mon[183059]: failed to encode map e343 with expected crc 2023-08-28T02:37:40.088 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:39 smithi143 ceph-mon[183059]: failed to encode map e343 with expected crc 2023-08-28T02:37:40.088 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:39 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:37:40.088 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:39 smithi143 ceph-mon[189871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:37:40.088 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:39 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2535071724' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:37:40.088 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:39 smithi143 ceph-mon[189871]: osdmap e343: 8 total, 8 up, 7 in 2023-08-28T02:37:40.088 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:39 smithi143 ceph-mon[189871]: failed to encode map e343 with expected crc 2023-08-28T02:37:40.088 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:39 smithi143 ceph-mon[189871]: failed to encode map e343 with expected crc 2023-08-28T02:37:40.089 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:39 smithi143 ceph-mon[189871]: failed to encode map e343 with expected crc 2023-08-28T02:37:40.089 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:39 smithi143 ceph-mon[189871]: failed to encode map e343 with expected crc 2023-08-28T02:37:40.089 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:39 smithi143 ceph-mon[189871]: failed to encode map e343 with expected crc 2023-08-28T02:37:40.089 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:39 smithi143 ceph-mon[189871]: failed to encode map e343 with expected crc 2023-08-28T02:37:40.089 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:39 smithi143 ceph-mon[189871]: failed to encode map e343 with expected crc 2023-08-28T02:37:40.089 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:39 smithi143 ceph-mon[189871]: failed to encode map e343 with expected crc 2023-08-28T02:37:40.089 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:39 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:37:40.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:40.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:37:40.190 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:37:40.194 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:37:40.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:39 smithi169 ceph-mon[159415]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:37:40.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:39 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2535071724' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:37:40.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:39 smithi169 ceph-mon[159415]: osdmap e343: 8 total, 8 up, 7 in 2023-08-28T02:37:40.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:39 smithi169 ceph-mon[159415]: failed to encode map e343 with expected crc 2023-08-28T02:37:40.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:39 smithi169 ceph-mon[159415]: failed to encode map e343 with expected crc 2023-08-28T02:37:40.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:39 smithi169 ceph-mon[159415]: failed to encode map e343 with expected crc 2023-08-28T02:37:40.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:39 smithi169 ceph-mon[159415]: failed to encode map e343 with expected crc 2023-08-28T02:37:40.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:39 smithi169 ceph-mon[159415]: failed to encode map e343 with expected crc 2023-08-28T02:37:40.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:39 smithi169 ceph-mon[159415]: failed to encode map e343 with expected crc 2023-08-28T02:37:40.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:39 smithi169 ceph-mon[159415]: failed to encode map e343 with expected crc 2023-08-28T02:37:40.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:39 smithi169 ceph-mon[159415]: failed to encode map e343 with expected crc 2023-08-28T02:37:40.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:39 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:37:40.284 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:40.389 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:37:40.431 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:37:40.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:40.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:37:40.542 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:37:40.638 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:40.759 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:37:40.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:40.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:37:40.865 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:37:40.963 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:41.050 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:37:41.051 INFO:teuthology.orchestra.run.smithi169.stdout: "ops": [ 2023-08-28T02:37:41.051 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:41.051 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_repop(client.45647.0:130061 8.5 e343/318 8:ad2d478e:::benchmark_data_smithi143_419289_object33151:head v 343'8133, mlcod=343'8131)", 2023-08-28T02:37:41.051 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:37:41.050554+0000", 2023-08-28T02:37:41.051 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 0.00035828200000000002, 2023-08-28T02:37:41.051 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00038779, 2023-08-28T02:37:41.051 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:37:41.051 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "started", 2023-08-28T02:37:41.051 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:37:41.052 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:41.052 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:37:41.052 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:41.050554+0000", 2023-08-28T02:37:41.052 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:37:41.052 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:41.052 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:41.052 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:37:41.052 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:41.050554+0000", 2023-08-28T02:37:41.052 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:37:41.052 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:41.052 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:41.052 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:37:41.053 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:41.050553+0000", 2023-08-28T02:37:41.053 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1687e-05 2023-08-28T02:37:41.053 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:41.053 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:41.053 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:37:41.053 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:41.050565+0000", 2023-08-28T02:37:41.053 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.8757e-05 2023-08-28T02:37:41.053 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:41.053 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:41.053 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:37:41.053 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:41.050583+0000", 2023-08-28T02:37:41.054 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.4200000000000007e-06 2023-08-28T02:37:41.054 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:41.054 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:41.054 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:37:41.054 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:41.050592+0000", 2023-08-28T02:37:41.054 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.3099000000000001e-05 2023-08-28T02:37:41.054 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:41.054 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:41.054 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:37:41.055 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:41.050645+0000", 2023-08-28T02:37:41.055 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.3751999999999999e-05 2023-08-28T02:37:41.055 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:41.055 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:41.055 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:37:41.055 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:41.050669+0000", 2023-08-28T02:37:41.055 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000114379 2023-08-28T02:37:41.055 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:37:41.055 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:37:41.055 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:37:41.055 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:37:41.055 INFO:teuthology.orchestra.run.smithi169.stdout: ], 2023-08-28T02:37:41.056 INFO:teuthology.orchestra.run.smithi169.stdout: "num_ops": 1 2023-08-28T02:37:41.056 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:37:41.074 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:40 smithi169 ceph-mon[159415]: pgmap v919: 89 pgs: 89 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 700 op/s 2023-08-28T02:37:41.074 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:40 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:37:41.075 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:40 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:37:41.075 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:40 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:37:41.075 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:40 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:37:41.165 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:40 smithi143 ceph-mon[183059]: pgmap v919: 89 pgs: 89 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 700 op/s 2023-08-28T02:37:41.165 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:40 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:37:41.165 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:40 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:37:41.165 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:40 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:37:41.165 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:40 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:37:41.165 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:40 smithi143 ceph-mon[189871]: pgmap v919: 89 pgs: 89 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 700 op/s 2023-08-28T02:37:41.165 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:40 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:37:41.165 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:40 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:37:41.165 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:40 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:37:41.166 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:40 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:37:41.832 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:37:41.838 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:41 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:37:41.838 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:41 smithi143 ceph-mon[183059]: pgmap v920: 89 pgs: 89 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 701 op/s 2023-08-28T02:37:41.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:41.839 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:41 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:37:41.839 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:41 smithi143 ceph-mon[189871]: pgmap v920: 89 pgs: 89 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 701 op/s 2023-08-28T02:37:41.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:37:41.940 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:37:41.962 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.4 dump_blocked_ops 2023-08-28T02:37:41.998 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:42.082 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:37:42.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:42.093 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:37:42.094 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:41 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:37:42.094 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:41 smithi169 ceph-mon[159415]: pgmap v920: 89 pgs: 89 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 701 op/s 2023-08-28T02:37:42.134 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:37:41 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:37:41] "GET /metrics HTTP/1.1" 200 36535 "" "Prometheus/2.33.4" 2023-08-28T02:37:42.195 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:37:42.230 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:42.232 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset nodeep-scrub 2023-08-28T02:37:42.541 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:37:42.624 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:37:42.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:42.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:37:42.734 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:37:42.769 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:43.213 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:37:43.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:43.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:37:43.319 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:37:43.354 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:37:43.362 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:43.532 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:37:43.587 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:37:43.634 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:37:43.660 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:37:43.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:43 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2682136006' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:37:43.802 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:37:43.849 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:37:43.889 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:43 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2682136006' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:37:43.889 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:43 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2682136006' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:37:43.898 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:37:43.946 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:37:43.959 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:37:43.959 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:37:43.959 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:37:43.959 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:37:43.959 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:37:43.959 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:37:43.960 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:37:43.960 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:37:44.034 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:37:44.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:44.037 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:37:44.137 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:37:44.170 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:43 smithi143 conmon[183035]: 2023-08-28T02:37:43.895+0000 7fadc3887700 -1 mon.a@0(leader).osd e343 definitely_dead 0 2023-08-28T02:37:44.231 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:44.592 INFO:teuthology.orchestra.run.smithi143.stderr:marked in osd.7. 2023-08-28T02:37:44.716 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:37:44.716 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:37:44.734 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:44 smithi169 ceph-mon[159415]: pgmap v921: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 687 op/s 2023-08-28T02:37:44.735 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:44 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/424612603' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-08-28T02:37:44.735 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:44 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-08-28T02:37:44.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:44 smithi143 ceph-mon[183059]: pgmap v921: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 687 op/s 2023-08-28T02:37:44.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:44 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/424612603' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-08-28T02:37:44.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:44 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-08-28T02:37:44.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:44 smithi143 ceph-mon[189871]: pgmap v921: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 687 op/s 2023-08-28T02:37:44.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:44 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/424612603' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-08-28T02:37:44.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:44 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-08-28T02:37:45.821 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:45 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-08-28T02:37:45.821 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:45 smithi169 ceph-mon[159415]: osdmap e344: 8 total, 8 up, 8 in 2023-08-28T02:37:45.821 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:45 smithi169 ceph-mon[159415]: failed to encode map e344 with expected crc 2023-08-28T02:37:45.822 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:45 smithi169 ceph-mon[159415]: failed to encode map e344 with expected crc 2023-08-28T02:37:45.822 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:45 smithi169 ceph-mon[159415]: failed to encode map e344 with expected crc 2023-08-28T02:37:45.822 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:45 smithi169 ceph-mon[159415]: failed to encode map e344 with expected crc 2023-08-28T02:37:45.822 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:45 smithi169 ceph-mon[159415]: failed to encode map e344 with expected crc 2023-08-28T02:37:45.822 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:45 smithi169 ceph-mon[159415]: failed to encode map e344 with expected crc 2023-08-28T02:37:45.822 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:45 smithi169 ceph-mon[159415]: failed to encode map e344 with expected crc 2023-08-28T02:37:45.842 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.4 dump_historic_ops 2023-08-28T02:37:45.894 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:45 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-08-28T02:37:45.895 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:45 smithi143 ceph-mon[183059]: osdmap e344: 8 total, 8 up, 8 in 2023-08-28T02:37:45.895 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:45 smithi143 ceph-mon[183059]: failed to encode map e344 with expected crc 2023-08-28T02:37:45.895 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:45 smithi143 ceph-mon[183059]: failed to encode map e344 with expected crc 2023-08-28T02:37:45.895 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:45 smithi143 ceph-mon[183059]: failed to encode map e344 with expected crc 2023-08-28T02:37:45.895 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:45 smithi143 ceph-mon[183059]: failed to encode map e344 with expected crc 2023-08-28T02:37:45.895 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:45 smithi143 ceph-mon[183059]: failed to encode map e344 with expected crc 2023-08-28T02:37:45.895 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:45 smithi143 ceph-mon[183059]: failed to encode map e344 with expected crc 2023-08-28T02:37:45.895 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:45 smithi143 ceph-mon[183059]: failed to encode map e344 with expected crc 2023-08-28T02:37:45.895 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:45 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-08-28T02:37:45.896 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:45 smithi143 ceph-mon[189871]: osdmap e344: 8 total, 8 up, 8 in 2023-08-28T02:37:45.896 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:45 smithi143 ceph-mon[189871]: failed to encode map e344 with expected crc 2023-08-28T02:37:45.896 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:45 smithi143 ceph-mon[189871]: failed to encode map e344 with expected crc 2023-08-28T02:37:45.896 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:45 smithi143 ceph-mon[189871]: failed to encode map e344 with expected crc 2023-08-28T02:37:45.896 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:45 smithi143 ceph-mon[189871]: failed to encode map e344 with expected crc 2023-08-28T02:37:45.896 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:45 smithi143 ceph-mon[189871]: failed to encode map e344 with expected crc 2023-08-28T02:37:45.896 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:45 smithi143 ceph-mon[189871]: failed to encode map e344 with expected crc 2023-08-28T02:37:45.896 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:45 smithi143 ceph-mon[189871]: failed to encode map e344 with expected crc 2023-08-28T02:37:46.140 INFO:tasks.thrashosds.thrasher:Added osd 7 2023-08-28T02:37:46.143 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:37:46.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:46.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:37:46.248 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:37:46.278 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:46.376 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:37:46.377 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:37:46.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:46.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:37:46.481 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:37:46.513 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:46.557 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:37:46.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:46.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:37:46.661 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:37:46.693 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:46.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:46 smithi143 ceph-mon[183059]: failed to encode map e345 with expected crc 2023-08-28T02:37:46.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:46 smithi143 ceph-mon[183059]: failed to encode map e345 with expected crc 2023-08-28T02:37:46.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:46 smithi143 ceph-mon[183059]: osdmap e345: 8 total, 8 up, 8 in 2023-08-28T02:37:46.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:46 smithi143 ceph-mon[183059]: failed to encode map e345 with expected crc 2023-08-28T02:37:46.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:46 smithi143 ceph-mon[183059]: failed to encode map e345 with expected crc 2023-08-28T02:37:46.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:46 smithi143 ceph-mon[183059]: failed to encode map e345 with expected crc 2023-08-28T02:37:46.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:46 smithi143 ceph-mon[183059]: failed to encode map e345 with expected crc 2023-08-28T02:37:46.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:46 smithi143 ceph-mon[183059]: failed to encode map e345 with expected crc 2023-08-28T02:37:46.701 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:46 smithi143 ceph-mon[183059]: pgmap v924: 89 pgs: 8 peering, 81 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 686 op/s 2023-08-28T02:37:46.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:46 smithi143 ceph-mon[189871]: failed to encode map e345 with expected crc 2023-08-28T02:37:46.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:46 smithi143 ceph-mon[189871]: failed to encode map e345 with expected crc 2023-08-28T02:37:46.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:46 smithi143 ceph-mon[189871]: osdmap e345: 8 total, 8 up, 8 in 2023-08-28T02:37:46.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:46 smithi143 ceph-mon[189871]: failed to encode map e345 with expected crc 2023-08-28T02:37:46.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:46 smithi143 ceph-mon[189871]: failed to encode map e345 with expected crc 2023-08-28T02:37:46.701 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:46 smithi143 ceph-mon[189871]: failed to encode map e345 with expected crc 2023-08-28T02:37:46.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:46 smithi143 ceph-mon[189871]: failed to encode map e345 with expected crc 2023-08-28T02:37:46.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:46 smithi143 ceph-mon[189871]: failed to encode map e345 with expected crc 2023-08-28T02:37:46.702 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:46 smithi143 ceph-mon[189871]: pgmap v924: 89 pgs: 8 peering, 81 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 686 op/s 2023-08-28T02:37:46.903 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:46 smithi169 ceph-mon[159415]: failed to encode map e345 with expected crc 2023-08-28T02:37:46.903 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:46 smithi169 ceph-mon[159415]: failed to encode map e345 with expected crc 2023-08-28T02:37:46.903 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:46 smithi169 ceph-mon[159415]: osdmap e345: 8 total, 8 up, 8 in 2023-08-28T02:37:46.903 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:46 smithi169 ceph-mon[159415]: failed to encode map e345 with expected crc 2023-08-28T02:37:46.904 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:46 smithi169 ceph-mon[159415]: failed to encode map e345 with expected crc 2023-08-28T02:37:46.904 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:46 smithi169 ceph-mon[159415]: failed to encode map e345 with expected crc 2023-08-28T02:37:46.904 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:46 smithi169 ceph-mon[159415]: failed to encode map e345 with expected crc 2023-08-28T02:37:46.904 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:46 smithi169 ceph-mon[159415]: failed to encode map e345 with expected crc 2023-08-28T02:37:46.904 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:46 smithi169 ceph-mon[159415]: pgmap v924: 89 pgs: 8 peering, 81 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 686 op/s 2023-08-28T02:37:47.076 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:37:47.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:47.079 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:37:47.180 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:37:47.262 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:47.359 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:37:47.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:47.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:37:47.463 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:37:47.508 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:47.566 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:47 smithi143 ceph-mon[183059]: failed to encode map e346 with expected crc 2023-08-28T02:37:47.566 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:47 smithi143 ceph-mon[183059]: 8.8 starting backfill to osd.7 from (0'0,0'0] MAX to 343'8420 2023-08-28T02:37:47.566 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:47 smithi143 ceph-mon[183059]: 8.c starting backfill to osd.7 from (0'0,0'0] MAX to 343'7984 2023-08-28T02:37:47.566 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:47 smithi143 ceph-mon[183059]: osdmap e346: 8 total, 8 up, 8 in 2023-08-28T02:37:47.566 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:47 smithi143 ceph-mon[183059]: failed to encode map e346 with expected crc 2023-08-28T02:37:47.566 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:47 smithi143 ceph-mon[183059]: failed to encode map e346 with expected crc 2023-08-28T02:37:47.652 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is unset 2023-08-28T02:37:47.782 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:37:47.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:47.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:37:47.790 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:47 smithi169 ceph-mon[159415]: failed to encode map e346 with expected crc 2023-08-28T02:37:47.790 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:47 smithi169 ceph-mon[159415]: 8.8 starting backfill to osd.7 from (0'0,0'0] MAX to 343'8420 2023-08-28T02:37:47.790 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:47 smithi169 ceph-mon[159415]: 8.c starting backfill to osd.7 from (0'0,0'0] MAX to 343'7984 2023-08-28T02:37:47.790 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:47 smithi169 ceph-mon[159415]: osdmap e346: 8 total, 8 up, 8 in 2023-08-28T02:37:47.790 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:47 smithi169 ceph-mon[159415]: failed to encode map e346 with expected crc 2023-08-28T02:37:47.790 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:47 smithi169 ceph-mon[159415]: failed to encode map e346 with expected crc 2023-08-28T02:37:47.790 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:47 smithi169 ceph-mon[159415]: failed to encode map e346 with expected crc 2023-08-28T02:37:47.790 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:47 smithi169 ceph-mon[159415]: 8.6 starting backfill to osd.7 from (0'0,0'0] MAX to 343'8376 2023-08-28T02:37:47.791 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:47 smithi169 ceph-mon[159415]: failed to encode map e346 with expected crc 2023-08-28T02:37:47.791 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:47 smithi169 ceph-mon[159415]: failed to encode map e346 with expected crc 2023-08-28T02:37:47.791 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:47 smithi169 ceph-mon[159415]: failed to encode map e346 with expected crc 2023-08-28T02:37:47.791 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:47 smithi169 ceph-mon[159415]: failed to encode map e346 with expected crc 2023-08-28T02:37:47.791 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:47 smithi169 ceph-mon[159415]: failed to encode map e346 with expected crc 2023-08-28T02:37:47.791 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:47 smithi169 ceph-mon[159415]: failed to encode map e346 with expected crc 2023-08-28T02:37:47.791 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:47 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/808990814' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:37:47.890 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:37:47.891 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:47 smithi143 ceph-mon[183059]: failed to encode map e346 with expected crc 2023-08-28T02:37:47.891 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:47 smithi143 ceph-mon[183059]: 8.6 starting backfill to osd.7 from (0'0,0'0] MAX to 343'8376 2023-08-28T02:37:47.891 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:47 smithi143 ceph-mon[183059]: failed to encode map e346 with expected crc 2023-08-28T02:37:47.891 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:47 smithi143 ceph-mon[183059]: failed to encode map e346 with expected crc 2023-08-28T02:37:47.891 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:47 smithi143 ceph-mon[183059]: failed to encode map e346 with expected crc 2023-08-28T02:37:47.891 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:47 smithi143 ceph-mon[183059]: failed to encode map e346 with expected crc 2023-08-28T02:37:47.891 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:47 smithi143 ceph-mon[183059]: failed to encode map e346 with expected crc 2023-08-28T02:37:47.891 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:47 smithi143 ceph-mon[183059]: failed to encode map e346 with expected crc 2023-08-28T02:37:47.892 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:47 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/808990814' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:37:47.892 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:47 smithi143 ceph-mon[189871]: failed to encode map e346 with expected crc 2023-08-28T02:37:47.892 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:47 smithi143 ceph-mon[189871]: 8.8 starting backfill to osd.7 from (0'0,0'0] MAX to 343'8420 2023-08-28T02:37:47.892 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:47 smithi143 ceph-mon[189871]: 8.c starting backfill to osd.7 from (0'0,0'0] MAX to 343'7984 2023-08-28T02:37:47.892 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:47 smithi143 ceph-mon[189871]: osdmap e346: 8 total, 8 up, 8 in 2023-08-28T02:37:47.892 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:47 smithi143 ceph-mon[189871]: failed to encode map e346 with expected crc 2023-08-28T02:37:47.892 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:47 smithi143 ceph-mon[189871]: failed to encode map e346 with expected crc 2023-08-28T02:37:47.892 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:47 smithi143 ceph-mon[189871]: failed to encode map e346 with expected crc 2023-08-28T02:37:47.892 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:47 smithi143 ceph-mon[189871]: 8.6 starting backfill to osd.7 from (0'0,0'0] MAX to 343'8376 2023-08-28T02:37:47.892 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:47 smithi143 ceph-mon[189871]: failed to encode map e346 with expected crc 2023-08-28T02:37:47.892 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:47 smithi143 ceph-mon[189871]: failed to encode map e346 with expected crc 2023-08-28T02:37:47.893 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:47 smithi143 ceph-mon[189871]: failed to encode map e346 with expected crc 2023-08-28T02:37:47.893 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:47 smithi143 ceph-mon[189871]: failed to encode map e346 with expected crc 2023-08-28T02:37:47.893 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:47 smithi143 ceph-mon[189871]: failed to encode map e346 with expected crc 2023-08-28T02:37:47.893 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:47 smithi143 ceph-mon[189871]: failed to encode map e346 with expected crc 2023-08-28T02:37:47.893 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:47 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/808990814' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:37:47.915 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:37:47.967 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:48.202 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:37:48.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:48.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:37:48.305 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:37:48.336 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:48.383 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:37:48.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:48.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:37:48.454 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:37:48.454 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:37:48.488 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:37:48.524 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:48.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:48 smithi169 ceph-mon[159415]: pgmap v926: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 8 peering, 79 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 547 op/s; 6066/186168 objects degraded (3.258%); 788 B/s, 2 objects/s recovering 2023-08-28T02:37:48.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:48 smithi169 ceph-mon[159415]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:37:48.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:48 smithi169 ceph-mon[159415]: Cluster is now healthy 2023-08-28T02:37:48.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:48 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/808990814' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:37:48.740 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:48 smithi169 ceph-mon[159415]: failed to encode map e347 with expected crc 2023-08-28T02:37:48.741 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:48 smithi169 ceph-mon[159415]: osdmap e347: 8 total, 8 up, 8 in 2023-08-28T02:37:48.741 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:48 smithi169 ceph-mon[159415]: failed to encode map e347 with expected crc 2023-08-28T02:37:48.741 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:48 smithi169 ceph-mon[159415]: failed to encode map e347 with expected crc 2023-08-28T02:37:48.741 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:48 smithi169 ceph-mon[159415]: failed to encode map e347 with expected crc 2023-08-28T02:37:48.741 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:48 smithi169 ceph-mon[159415]: failed to encode map e347 with expected crc 2023-08-28T02:37:48.741 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:48 smithi169 ceph-mon[159415]: failed to encode map e347 with expected crc 2023-08-28T02:37:48.741 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:48 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:37:48.741 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:48 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:37:48.932 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:37:48.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:48.942 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:48 smithi143 ceph-mon[183059]: pgmap v926: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 8 peering, 79 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 547 op/s; 6066/186168 objects degraded (3.258%); 788 B/s, 2 objects/s recovering 2023-08-28T02:37:48.942 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:48 smithi143 ceph-mon[183059]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:37:48.942 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:48 smithi143 ceph-mon[183059]: Cluster is now healthy 2023-08-28T02:37:48.943 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:48 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/808990814' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:37:48.943 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:48 smithi143 ceph-mon[183059]: failed to encode map e347 with expected crc 2023-08-28T02:37:48.943 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:48 smithi143 ceph-mon[183059]: osdmap e347: 8 total, 8 up, 8 in 2023-08-28T02:37:48.943 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:48 smithi143 ceph-mon[183059]: failed to encode map e347 with expected crc 2023-08-28T02:37:48.943 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:48 smithi143 ceph-mon[183059]: failed to encode map e347 with expected crc 2023-08-28T02:37:48.943 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:48 smithi143 ceph-mon[183059]: failed to encode map e347 with expected crc 2023-08-28T02:37:48.943 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:48 smithi143 ceph-mon[183059]: failed to encode map e347 with expected crc 2023-08-28T02:37:48.943 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:48 smithi143 ceph-mon[183059]: failed to encode map e347 with expected crc 2023-08-28T02:37:48.943 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:48 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:37:48.943 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:48 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:37:48.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:37:48.945 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:48 smithi143 ceph-mon[189871]: pgmap v926: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 8 peering, 79 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 547 op/s; 6066/186168 objects degraded (3.258%); 788 B/s, 2 objects/s recovering 2023-08-28T02:37:48.946 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:48 smithi143 ceph-mon[189871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:37:48.946 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:48 smithi143 ceph-mon[189871]: Cluster is now healthy 2023-08-28T02:37:48.946 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:48 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/808990814' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:37:48.946 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:48 smithi143 ceph-mon[189871]: failed to encode map e347 with expected crc 2023-08-28T02:37:48.946 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:48 smithi143 ceph-mon[189871]: osdmap e347: 8 total, 8 up, 8 in 2023-08-28T02:37:48.946 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:48 smithi143 ceph-mon[189871]: failed to encode map e347 with expected crc 2023-08-28T02:37:48.946 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:48 smithi143 ceph-mon[189871]: failed to encode map e347 with expected crc 2023-08-28T02:37:48.946 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:48 smithi143 ceph-mon[189871]: failed to encode map e347 with expected crc 2023-08-28T02:37:48.946 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:48 smithi143 ceph-mon[189871]: failed to encode map e347 with expected crc 2023-08-28T02:37:48.946 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:48 smithi143 ceph-mon[189871]: failed to encode map e347 with expected crc 2023-08-28T02:37:48.946 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:48 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:37:48.946 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:48 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:37:49.045 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:37:49.091 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:49.231 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:37:49.310 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:37:49.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:49.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:37:49.415 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:37:49.440 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.5 dump_ops_in_flight 2023-08-28T02:37:49.471 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:49.563 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:37:49.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:49.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:37:49.682 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:37:49.716 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:49 smithi169 ceph-mon[159415]: Health check failed: Degraded data redundancy: 6066/186168 objects degraded (3.258%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:37:49.716 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:49 smithi169 ceph-mon[159415]: osdmap e348: 8 total, 8 up, 8 in 2023-08-28T02:37:49.716 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:49 smithi169 ceph-mon[159415]: failed to encode map e348 with expected crc 2023-08-28T02:37:49.716 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:49 smithi169 ceph-mon[159415]: failed to encode map e348 with expected crc 2023-08-28T02:37:49.717 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:49 smithi169 ceph-mon[159415]: failed to encode map e348 with expected crc 2023-08-28T02:37:49.717 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:49 smithi169 ceph-mon[159415]: failed to encode map e348 with expected crc 2023-08-28T02:37:49.717 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:49 smithi169 ceph-mon[159415]: failed to encode map e348 with expected crc 2023-08-28T02:37:49.720 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:49.799 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:37:49.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:49.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:37:49.909 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:37:49.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:49 smithi143 ceph-mon[183059]: Health check failed: Degraded data redundancy: 6066/186168 objects degraded (3.258%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:37:49.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:49 smithi143 ceph-mon[183059]: osdmap e348: 8 total, 8 up, 8 in 2023-08-28T02:37:49.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:49 smithi143 ceph-mon[183059]: failed to encode map e348 with expected crc 2023-08-28T02:37:49.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:49 smithi143 ceph-mon[183059]: failed to encode map e348 with expected crc 2023-08-28T02:37:49.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:49 smithi143 ceph-mon[183059]: failed to encode map e348 with expected crc 2023-08-28T02:37:49.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:49 smithi143 ceph-mon[183059]: failed to encode map e348 with expected crc 2023-08-28T02:37:49.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:49 smithi143 ceph-mon[183059]: failed to encode map e348 with expected crc 2023-08-28T02:37:49.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:49 smithi143 ceph-mon[189871]: Health check failed: Degraded data redundancy: 6066/186168 objects degraded (3.258%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:37:49.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:49 smithi143 ceph-mon[189871]: osdmap e348: 8 total, 8 up, 8 in 2023-08-28T02:37:49.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:49 smithi143 ceph-mon[189871]: failed to encode map e348 with expected crc 2023-08-28T02:37:49.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:49 smithi143 ceph-mon[189871]: failed to encode map e348 with expected crc 2023-08-28T02:37:49.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:49 smithi143 ceph-mon[189871]: failed to encode map e348 with expected crc 2023-08-28T02:37:49.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:49 smithi143 ceph-mon[189871]: failed to encode map e348 with expected crc 2023-08-28T02:37:49.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:49 smithi143 ceph-mon[189871]: failed to encode map e348 with expected crc 2023-08-28T02:37:49.956 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:50.032 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:37:50.048 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:37:50.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:50.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:37:50.156 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:37:50.185 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:50.368 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:37:50.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:50.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:37:50.472 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:37:50.541 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:50.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:50 smithi169 ceph-mon[159415]: pgmap v929: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 9 peering, 77 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 291 op/s; 6066/183912 objects degraded (3.298%); 3715/183912 objects misplaced (2.020%); 1.2 MiB/s, 24 objects/s recovering 2023-08-28T02:37:50.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:50 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:37:50.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:50 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:37:50.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:50 smithi143 ceph-mon[189871]: pgmap v929: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 9 peering, 77 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 291 op/s; 6066/183912 objects degraded (3.298%); 3715/183912 objects misplaced (2.020%); 1.2 MiB/s, 24 objects/s recovering 2023-08-28T02:37:50.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:50 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:37:50.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:50 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:37:50.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:50 smithi143 ceph-mon[183059]: pgmap v929: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 9 peering, 77 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 291 op/s; 6066/183912 objects degraded (3.298%); 3715/183912 objects misplaced (2.020%); 1.2 MiB/s, 24 objects/s recovering 2023-08-28T02:37:50.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:50 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:37:50.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:50 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:37:51.053 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set noscrub 2023-08-28T02:37:51.140 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 6, 1, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-08-28T02:37:51.140 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-08-28T02:37:51.141 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-08-28T02:37:51.141 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph pg dump --format=json 2023-08-28T02:37:51.339 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:37:51.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:51.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:37:51.449 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:37:51.570 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:51.697 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:37:51.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:51.702 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:37:51.795 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:51 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:37:51.795 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:51 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:37:51.795 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:51 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:37:51.803 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:37:51.881 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:51 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:37:51.881 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:51 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:37:51.881 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:51 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:37:51.884 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:51 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:37:51.884 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:51 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:37:51.884 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:51 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:37:51.892 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:52.059 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:37:52.059 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:37:52.129 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:37:52.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:52.135 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:37:52.136 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:37:51 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:37:51] "GET /metrics HTTP/1.1" 200 36535 "" "Prometheus/2.33.4" 2023-08-28T02:37:52.236 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:37:52.271 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:37:52.331 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:37:52.344 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:52.474 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:37:52.474 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:37:52.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:52.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:37:52.519 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:37:52.565 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:37:52.579 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:37:52.602 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:37:52.650 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:52.660 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:37:52.701 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:37:52.752 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:37:52.786 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:37:52.824 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:37:52.824 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:37:52.825 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:37:52.825 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:37:52.825 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:37:52.825 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:37:52.825 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:37:52.825 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:37:52.925 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:52 smithi169 ceph-mon[159415]: pgmap v930: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 peering, 82 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 293 op/s; 6112/182124 objects degraded (3.356%); 14885/182124 objects misplaced (8.173%); 1.6 MiB/s, 30 objects/s recovering 2023-08-28T02:37:52.925 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:52 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/657304744' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:37:52.937 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:37:52.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:52.946 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:37:52.960 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.5 dump_blocked_ops 2023-08-28T02:37:53.047 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:37:53.048 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:52 smithi143 ceph-mon[183059]: pgmap v930: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 peering, 82 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 293 op/s; 6112/182124 objects degraded (3.356%); 14885/182124 objects misplaced (8.173%); 1.6 MiB/s, 30 objects/s recovering 2023-08-28T02:37:53.048 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:52 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/657304744' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:37:53.049 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:52 smithi143 ceph-mon[189871]: pgmap v930: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 peering, 82 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 293 op/s; 6112/182124 objects degraded (3.356%); 14885/182124 objects misplaced (8.173%); 1.6 MiB/s, 30 objects/s recovering 2023-08-28T02:37:53.049 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:52 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/657304744' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:37:53.110 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:53.274 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:37:53.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:53.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:37:53.389 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:37:53.452 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:53.596 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:37:53.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:53.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:37:53.645 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:37:53.704 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:37:53.757 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:53.936 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:53 smithi169 ceph-mon[159415]: pgmap v931: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 469 op/s; 6112/177543 objects degraded (3.443%); 14885/177543 objects misplaced (8.384%); 1.4 MiB/s, 26 objects/s recovering 2023-08-28T02:37:53.936 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:53 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 6112/177543 objects degraded (3.443%), 2 pgs degraded (PG_DEGRADED) 2023-08-28T02:37:54.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:53 smithi143 ceph-mon[183059]: pgmap v931: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 469 op/s; 6112/177543 objects degraded (3.443%); 14885/177543 objects misplaced (8.384%); 1.4 MiB/s, 26 objects/s recovering 2023-08-28T02:37:54.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:53 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 6112/177543 objects degraded (3.443%), 2 pgs degraded (PG_DEGRADED) 2023-08-28T02:37:54.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:53 smithi143 ceph-mon[189871]: pgmap v931: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 469 op/s; 6112/177543 objects degraded (3.443%); 14885/177543 objects misplaced (8.384%); 1.4 MiB/s, 26 objects/s recovering 2023-08-28T02:37:54.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:53 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 6112/177543 objects degraded (3.443%), 2 pgs degraded (PG_DEGRADED) 2023-08-28T02:37:54.512 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:37:54.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:54.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:37:54.620 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:37:54.664 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:55.276 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:37:55.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:55.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:37:55.380 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:37:55.438 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:55.540 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:37:55.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:55.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:37:55.582 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:37:55.583 INFO:teuthology.orchestra.run.smithi169.stdout: "ops": [], 2023-08-28T02:37:55.583 INFO:teuthology.orchestra.run.smithi169.stdout: "complaint_time": 30, 2023-08-28T02:37:55.583 INFO:teuthology.orchestra.run.smithi169.stdout: "num_blocked_ops": 0 2023-08-28T02:37:55.583 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:37:55.644 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:37:55.691 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:56.388 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.5 dump_historic_ops 2023-08-28T02:37:56.502 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:37:56.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:56.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:37:56.609 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:37:56.660 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:56 smithi169 ceph-mon[159415]: pgmap v932: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 550 op/s; 6191/172950 objects degraded (3.580%); 14082/172950 objects misplaced (8.142%); 2.2 MiB/s, 37 objects/s recovering 2023-08-28T02:37:56.661 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:56 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:37:56.661 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:56 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:37:56.661 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:56 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:37:56.674 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:56 smithi143 ceph-mon[183059]: pgmap v932: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 550 op/s; 6191/172950 objects degraded (3.580%); 14082/172950 objects misplaced (8.142%); 2.2 MiB/s, 37 objects/s recovering 2023-08-28T02:37:56.674 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:56 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:37:56.674 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:56 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:37:56.674 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:56 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:37:56.674 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:56 smithi143 ceph-mon[189871]: pgmap v932: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 550 op/s; 6191/172950 objects degraded (3.580%); 14082/172950 objects misplaced (8.142%); 2.2 MiB/s, 37 objects/s recovering 2023-08-28T02:37:56.674 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:56 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:37:56.675 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:56 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:37:56.675 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:56 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:37:56.684 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:56.871 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:37:56.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:56.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:37:56.979 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:37:56.995 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:37:57.036 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:57.181 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:37:57.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:57.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:37:57.300 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:37:57.347 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:57.470 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:37:57.479 INFO:teuthology.orchestra.run.smithi143.stderr:dumped all 2023-08-28T02:37:57.538 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:37:57.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:57.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:37:57.582 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:37:57.645 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:37:57.727 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is set 2023-08-28T02:37:57.796 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:57 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/238594318' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:37:57.796 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:57 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:37:57.797 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:57 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/238594318' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:37:57.797 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:57 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:37:57.820 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:57.907 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:37:57.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:57.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:37:57.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:57 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/238594318' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:37:57.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:57 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:37:58.011 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:37:58.039 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:58.354 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:37:58.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:58.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:37:58.457 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:37:58.485 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:58.560 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:37:58.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:58.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:37:58.667 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:37:58.719 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:58.929 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:37:58.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:58.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:37:58.934 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:58 smithi169 ceph-mon[159415]: from='client.45812 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:37:58.934 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:58 smithi169 ceph-mon[159415]: pgmap v933: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 518 op/s; 6191/170706 objects degraded (3.627%); 14082/170706 objects misplaced (8.249%); 1.8 MiB/s, 30 objects/s recovering 2023-08-28T02:37:58.935 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:58 smithi169 ceph-mon[159415]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:37:58.935 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:58 smithi169 ceph-mon[159415]: failed to encode map e349 with expected crc 2023-08-28T02:37:58.935 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:58 smithi169 ceph-mon[159415]: failed to encode map e349 with expected crc 2023-08-28T02:37:58.935 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:58 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:37:58.935 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:58 smithi169 ceph-mon[159415]: osdmap e349: 8 total, 8 up, 8 in 2023-08-28T02:37:58.935 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:58 smithi169 ceph-mon[159415]: failed to encode map e349 with expected crc 2023-08-28T02:37:58.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:58 smithi143 ceph-mon[183059]: from='client.45812 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:37:58.953 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:58 smithi143 ceph-mon[183059]: pgmap v933: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 518 op/s; 6191/170706 objects degraded (3.627%); 14082/170706 objects misplaced (8.249%); 1.8 MiB/s, 30 objects/s recovering 2023-08-28T02:37:58.953 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:58 smithi143 ceph-mon[183059]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:37:58.953 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:58 smithi143 ceph-mon[183059]: failed to encode map e349 with expected crc 2023-08-28T02:37:58.953 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:58 smithi143 ceph-mon[183059]: failed to encode map e349 with expected crc 2023-08-28T02:37:58.953 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:58 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:37:58.953 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:58 smithi143 ceph-mon[183059]: osdmap e349: 8 total, 8 up, 8 in 2023-08-28T02:37:58.953 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:58 smithi143 ceph-mon[183059]: failed to encode map e349 with expected crc 2023-08-28T02:37:58.958 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:37:58.958 INFO:teuthology.orchestra.run.smithi169.stdout: "size": 20, 2023-08-28T02:37:58.959 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 600, 2023-08-28T02:37:58.959 INFO:teuthology.orchestra.run.smithi169.stdout: "ops": [ 2023-08-28T02:37:58.959 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.959 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:65345 6.a 6:5cafef95:::benchmark_data_smithi143_305742_object65344:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-08-28T02:37:58.959 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:28:07.393170+0000", 2023-08-28T02:37:58.959 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 591.56202924199999, 2023-08-28T02:37:58.959 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.97206194499999998, 2023-08-28T02:37:58.959 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:37:58.959 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:37:58.959 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:37:58.959 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:37:58.960 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:37:58.960 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 65345 2023-08-28T02:37:58.960 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.960 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:37:58.960 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.960 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:37:58.960 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.393170+0000", 2023-08-28T02:37:58.960 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:37:58.960 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.960 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.960 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:37:58.960 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.393170+0000", 2023-08-28T02:37:58.960 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999976 2023-08-28T02:37:58.961 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.961 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.961 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:37:58.961 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.393168+0000", 2023-08-28T02:37:58.961 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.392e-05 2023-08-28T02:37:58.961 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.961 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.961 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:37:58.961 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.393212+0000", 2023-08-28T02:37:58.961 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1738e-05 2023-08-28T02:37:58.961 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.961 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.961 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:37:58.962 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.393224+0000", 2023-08-28T02:37:58.962 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.5649999999999998e-06 2023-08-28T02:37:58.962 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.962 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.962 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:37:58.962 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.393230+0000", 2023-08-28T02:37:58.962 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.4586000000000002e-05 2023-08-28T02:37:58.962 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.962 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.962 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:37:58.962 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.393285+0000", 2023-08-28T02:37:58.962 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.8400000000000007e-06 2023-08-28T02:37:58.962 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.963 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.963 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:37:58.963 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.393295+0000", 2023-08-28T02:37:58.963 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.96419019500000003 2023-08-28T02:37:58.963 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.963 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.963 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:37:58.963 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.357485+0000", 2023-08-28T02:37:58.963 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00013377699999999999 2023-08-28T02:37:58.963 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.963 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.963 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:37:58.963 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.357619+0000", 2023-08-28T02:37:58.964 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012924900000000001 2023-08-28T02:37:58.964 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.964 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.964 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,2", 2023-08-28T02:37:58.964 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.357748+0000", 2023-08-28T02:37:58.964 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.006065857 2023-08-28T02:37:58.964 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.964 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.964 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:37:58.964 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.363814+0000", 2023-08-28T02:37:58.964 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0012912659999999999 2023-08-28T02:37:58.964 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.964 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.965 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:58.965 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.365105+0000", 2023-08-28T02:37:58.965 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.3149000000000003e-05 2023-08-28T02:37:58.965 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.965 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.965 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:58.965 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.365148+0000", 2023-08-28T02:37:58.965 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.8458999999999997e-05 2023-08-28T02:37:58.965 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.965 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.965 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:37:58.965 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.365207+0000", 2023-08-28T02:37:58.966 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.5528e-05 2023-08-28T02:37:58.966 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.966 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.966 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:37:58.966 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.365232+0000", 2023-08-28T02:37:58.966 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.97206194499999998 2023-08-28T02:37:58.966 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:37:58.966 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:37:58.966 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:37:58.966 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.966 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.966 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:65370 6.a 6:5b0efaa9:::benchmark_data_smithi143_305742_object65369:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-08-28T02:37:58.967 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:28:07.421258+0000", 2023-08-28T02:37:58.967 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 591.53394127000001, 2023-08-28T02:37:58.967 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.95615907099999997, 2023-08-28T02:37:58.967 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:37:58.967 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:37:58.967 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:37:58.967 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:37:58.967 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:37:58.967 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 65370 2023-08-28T02:37:58.967 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.967 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:37:58.967 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.967 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:37:58.968 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.421258+0000", 2023-08-28T02:37:58.968 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:37:58.968 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.968 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.968 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:37:58.968 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.421258+0000", 2023-08-28T02:37:58.968 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:37:58.968 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.968 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.968 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:37:58.968 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.421257+0000", 2023-08-28T02:37:58.969 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.1618000000000004e-05 2023-08-28T02:37:58.969 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.969 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.969 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:37:58.969 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.421338+0000", 2023-08-28T02:37:58.969 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.7480000000000005e-06 2023-08-28T02:37:58.969 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.969 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.969 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:37:58.969 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.421348+0000", 2023-08-28T02:37:58.969 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.7060000000000003e-06 2023-08-28T02:37:58.969 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.969 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.970 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:37:58.970 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.421353+0000", 2023-08-28T02:37:58.970 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.9262000000000003e-05 2023-08-28T02:37:58.970 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.970 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.970 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:37:58.970 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.421412+0000", 2023-08-28T02:37:58.970 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0037000000000001e-05 2023-08-28T02:37:58.970 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.970 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.970 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:37:58.970 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.421422+0000", 2023-08-28T02:37:58.970 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.93841324500000001 2023-08-28T02:37:58.971 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.971 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.971 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:37:58.971 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.359835+0000", 2023-08-28T02:37:58.971 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00010686199999999999 2023-08-28T02:37:58.971 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.971 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.971 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:37:58.971 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.359942+0000", 2023-08-28T02:37:58.971 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000114066 2023-08-28T02:37:58.971 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.971 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.972 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,2", 2023-08-28T02:37:58.972 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.360056+0000", 2023-08-28T02:37:58.972 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0051866639999999997 2023-08-28T02:37:58.972 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.972 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.972 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:37:58.972 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.365243+0000", 2023-08-28T02:37:58.972 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0065147149999999999 2023-08-28T02:37:58.972 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.972 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.972 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:58.972 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.371758+0000", 2023-08-28T02:37:58.972 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0055873479999999998 2023-08-28T02:37:58.973 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.973 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.973 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:58.973 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377345+0000", 2023-08-28T02:37:58.973 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.3902000000000001e-05 2023-08-28T02:37:58.973 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.973 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.973 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:37:58.973 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377389+0000", 2023-08-28T02:37:58.973 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.8276000000000002e-05 2023-08-28T02:37:58.973 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.973 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.974 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:37:58.974 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377417+0000", 2023-08-28T02:37:58.974 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.95615907099999997 2023-08-28T02:37:58.974 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:37:58.974 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:37:58.974 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:37:58.974 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.974 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.974 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:65382 6.a 6:591f19d2:::benchmark_data_smithi143_305742_object65381:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-08-28T02:37:58.974 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:28:07.430975+0000", 2023-08-28T02:37:58.974 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 591.52422410899999, 2023-08-28T02:37:58.974 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.94658987900000002, 2023-08-28T02:37:58.974 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:37:58.975 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:37:58.975 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:37:58.975 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:37:58.975 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:37:58.975 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 65382 2023-08-28T02:37:58.975 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.975 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:37:58.975 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.975 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:37:58.975 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.430975+0000", 2023-08-28T02:37:58.975 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:37:58.975 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.976 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.976 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:37:58.976 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.430975+0000", 2023-08-28T02:37:58.976 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:37:58.976 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.976 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.976 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:37:58.976 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.430974+0000", 2023-08-28T02:37:58.976 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.9980000000000004e-05 2023-08-28T02:37:58.976 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.976 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.977 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:37:58.977 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.431044+0000", 2023-08-28T02:37:58.977 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.115e-06 2023-08-28T02:37:58.977 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.977 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.977 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:37:58.977 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.431053+0000", 2023-08-28T02:37:58.977 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.6990000000000002e-06 2023-08-28T02:37:58.977 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.977 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.977 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:37:58.977 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.431058+0000", 2023-08-28T02:37:58.978 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.4104999999999994e-05 2023-08-28T02:37:58.978 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.978 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.978 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:37:58.978 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.431152+0000", 2023-08-28T02:37:58.978 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.5532999999999998e-05 2023-08-28T02:37:58.978 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.978 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.978 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:37:58.978 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.431168+0000", 2023-08-28T02:37:58.978 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.92994798499999998 2023-08-28T02:37:58.978 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.979 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.979 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:37:58.979 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.361116+0000", 2023-08-28T02:37:58.979 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000128291 2023-08-28T02:37:58.979 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.979 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.979 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:37:58.979 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.361244+0000", 2023-08-28T02:37:58.979 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00016667299999999999 2023-08-28T02:37:58.979 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.979 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.979 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,2", 2023-08-28T02:37:58.980 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.361411+0000", 2023-08-28T02:37:58.980 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0038359240000000001 2023-08-28T02:37:58.980 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.980 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.980 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:37:58.980 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.365247+0000", 2023-08-28T02:37:58.980 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0093736270000000007 2023-08-28T02:37:58.980 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.980 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.980 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:58.980 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.374620+0000", 2023-08-28T02:37:58.980 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0028494010000000001 2023-08-28T02:37:58.980 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.981 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.981 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:58.981 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377470+0000", 2023-08-28T02:37:58.981 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.486e-05 2023-08-28T02:37:58.981 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.981 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.981 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:37:58.981 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377525+0000", 2023-08-28T02:37:58.981 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.0658999999999998e-05 2023-08-28T02:37:58.981 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.981 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.981 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:37:58.982 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377565+0000", 2023-08-28T02:37:58.982 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.94658987900000002 2023-08-28T02:37:58.982 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:37:58.982 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:37:58.982 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:37:58.982 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.982 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.982 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:65411 6.a 6:59a52dc1:::benchmark_data_smithi143_305742_object65410:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-08-28T02:37:58.982 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:28:07.493832+0000", 2023-08-28T02:37:58.982 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 591.46136748000004, 2023-08-28T02:37:58.982 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.88382154300000004, 2023-08-28T02:37:58.982 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:37:58.983 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:37:58.983 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:37:58.983 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:37:58.983 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:37:58.983 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 65411 2023-08-28T02:37:58.983 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.983 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:37:58.983 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.983 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:37:58.983 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493832+0000", 2023-08-28T02:37:58.983 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:37:58.983 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.983 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.984 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:37:58.984 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493832+0000", 2023-08-28T02:37:58.984 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999995 2023-08-28T02:37:58.984 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.984 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.984 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:37:58.984 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493831+0000", 2023-08-28T02:37:58.984 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.7842999999999997e-05 2023-08-28T02:37:58.984 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.984 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.984 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:37:58.984 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493869+0000", 2023-08-28T02:37:58.985 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.6300000000000003e-06 2023-08-28T02:37:58.985 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.985 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.985 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:37:58.985 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493875+0000", 2023-08-28T02:37:58.985 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.6119999999999999e-06 2023-08-28T02:37:58.985 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.985 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.985 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:37:58.985 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493879+0000", 2023-08-28T02:37:58.985 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.4725999999999997e-05 2023-08-28T02:37:58.985 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.985 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.986 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:37:58.986 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493933+0000", 2023-08-28T02:37:58.986 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.5399999999999996e-06 2023-08-28T02:37:58.986 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.986 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.986 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:37:58.986 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493942+0000", 2023-08-28T02:37:58.986 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.868725466 2023-08-28T02:37:58.986 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.986 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.986 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:37:58.986 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.362667+0000", 2023-08-28T02:37:58.987 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012190500000000001 2023-08-28T02:37:58.987 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.987 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.987 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:37:58.987 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.362789+0000", 2023-08-28T02:37:58.987 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000101784 2023-08-28T02:37:58.987 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.987 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.987 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,2", 2023-08-28T02:37:58.987 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.362891+0000", 2023-08-28T02:37:58.987 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0023590709999999999 2023-08-28T02:37:58.987 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.987 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.988 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:37:58.988 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.365250+0000", 2023-08-28T02:37:58.988 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0094366910000000005 2023-08-28T02:37:58.988 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.988 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.988 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:58.988 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.374687+0000", 2023-08-28T02:37:58.988 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0029174880000000002 2023-08-28T02:37:58.988 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.988 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.988 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:58.988 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377604+0000", 2023-08-28T02:37:58.988 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.8928000000000002e-05 2023-08-28T02:37:58.989 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.989 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.989 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:37:58.989 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377633+0000", 2023-08-28T02:37:58.989 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.0352000000000001e-05 2023-08-28T02:37:58.989 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.989 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.989 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:37:58.989 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377653+0000", 2023-08-28T02:37:58.989 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.88382154300000004 2023-08-28T02:37:58.989 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:37:58.989 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:37:58.989 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:37:58.990 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.990 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.990 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:65413 6.a 6:51617f2c:::benchmark_data_smithi143_305742_object65412:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-08-28T02:37:58.990 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:28:07.493890+0000", 2023-08-28T02:37:58.990 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 591.46130941399997, 2023-08-28T02:37:58.990 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.88389938400000001, 2023-08-28T02:37:58.990 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:37:58.990 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:37:58.990 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:37:58.990 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:37:58.990 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:37:58.990 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 65413 2023-08-28T02:37:58.991 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.991 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:37:58.991 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.991 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:37:58.991 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493890+0000", 2023-08-28T02:37:58.991 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:37:58.991 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.991 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.991 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:37:58.991 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493890+0000", 2023-08-28T02:37:58.991 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999995 2023-08-28T02:37:58.992 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.992 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.992 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:37:58.992 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493889+0000", 2023-08-28T02:37:58.992 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.5259999999999998e-05 2023-08-28T02:37:58.992 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.992 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.992 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:37:58.992 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493925+0000", 2023-08-28T02:37:58.992 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.6899999999999997e-06 2023-08-28T02:37:58.992 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.992 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.992 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:37:58.993 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493930+0000", 2023-08-28T02:37:58.993 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.9160000000000003e-06 2023-08-28T02:37:58.993 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.993 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.993 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:37:58.993 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493934+0000", 2023-08-28T02:37:58.993 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.5824e-05 2023-08-28T02:37:58.993 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.993 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.993 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:37:58.993 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493980+0000", 2023-08-28T02:37:58.993 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0889e-05 2023-08-28T02:37:58.993 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.994 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.994 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:37:58.994 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.493991+0000", 2023-08-28T02:37:58.994 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.86984277300000001 2023-08-28T02:37:58.994 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.994 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.994 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:37:58.994 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.363834+0000", 2023-08-28T02:37:58.994 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.9104999999999996e-05 2023-08-28T02:37:58.994 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.994 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.994 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:37:58.994 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.363913+0000", 2023-08-28T02:37:58.995 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.9914999999999995e-05 2023-08-28T02:37:58.995 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.995 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.995 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,2", 2023-08-28T02:37:58.995 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.364013+0000", 2023-08-28T02:37:58.995 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.003811933 2023-08-28T02:37:58.995 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.995 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.995 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:37:58.995 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.367825+0000", 2023-08-28T02:37:58.995 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0069041399999999996 2023-08-28T02:37:58.995 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.995 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.996 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:58.996 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.374729+0000", 2023-08-28T02:37:58.996 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0029736039999999999 2023-08-28T02:37:58.996 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.996 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.996 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:58.996 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377703+0000", 2023-08-28T02:37:58.996 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.8238999999999997e-05 2023-08-28T02:37:58.996 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.996 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.996 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:37:58.996 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377751+0000", 2023-08-28T02:37:58.997 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.8593000000000001e-05 2023-08-28T02:37:58.997 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.997 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.997 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:37:58.997 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377789+0000", 2023-08-28T02:37:58.997 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.88389938400000001 2023-08-28T02:37:58.997 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:37:58.997 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:37:58.997 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:37:58.997 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.997 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.997 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:65445 6.a 6:554d4ed8:::benchmark_data_smithi143_305742_object65444:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-08-28T02:37:58.997 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:28:07.533277+0000", 2023-08-28T02:37:58.998 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 591.421922344, 2023-08-28T02:37:58.998 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.84459700800000004, 2023-08-28T02:37:58.998 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:37:58.998 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:37:58.998 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:37:58.998 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:37:58.998 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:37:58.998 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 65445 2023-08-28T02:37:58.998 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.998 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:37:58.998 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.998 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:37:58.999 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533277+0000", 2023-08-28T02:37:58.999 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:37:58.999 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.999 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.999 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:37:58.999 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533277+0000", 2023-08-28T02:37:58.999 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:37:58.999 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:58.999 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:58.999 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:37:58.999 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533276+0000", 2023-08-28T02:37:59.000 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.8158999999999996e-05 2023-08-28T02:37:59.000 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.000 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.000 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:37:59.000 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533344+0000", 2023-08-28T02:37:59.000 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.1166e-05 2023-08-28T02:37:59.000 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.000 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.000 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:37:59.000 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533365+0000", 2023-08-28T02:37:59.000 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.2080000000000002e-06 2023-08-28T02:37:59.000 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.001 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.001 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:37:59.001 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533369+0000", 2023-08-28T02:37:59.001 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.5699999999999998e-05 2023-08-28T02:37:59.001 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.001 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.001 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:37:59.001 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533435+0000", 2023-08-28T02:37:59.001 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0674e-05 2023-08-28T02:37:59.001 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.001 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.001 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:37:59.002 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533446+0000", 2023-08-28T02:37:59.002 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.83182787300000005 2023-08-28T02:37:59.002 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.002 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.002 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:37:59.002 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.365274+0000", 2023-08-28T02:37:59.002 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000119156 2023-08-28T02:37:59.002 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.002 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.002 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:37:59.002 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.365393+0000", 2023-08-28T02:37:59.002 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000164434 2023-08-28T02:37:59.002 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.003 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.003 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,2", 2023-08-28T02:37:59.003 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.365557+0000", 2023-08-28T02:37:59.003 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0062156579999999998 2023-08-28T02:37:59.003 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.003 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.003 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:37:59.003 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.371773+0000", 2023-08-28T02:37:59.003 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0055333500000000002 2023-08-28T02:37:59.003 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.003 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.003 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:59.004 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377306+0000", 2023-08-28T02:37:59.004 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00051990900000000004 2023-08-28T02:37:59.004 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.004 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.004 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:59.004 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377826+0000", 2023-08-28T02:37:59.004 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.7752000000000001e-05 2023-08-28T02:37:59.004 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.004 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.004 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:37:59.004 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377854+0000", 2023-08-28T02:37:59.005 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.012e-05 2023-08-28T02:37:59.005 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.005 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.005 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:37:59.005 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377874+0000", 2023-08-28T02:37:59.005 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.84459700800000004 2023-08-28T02:37:59.005 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:37:59.005 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:37:59.005 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:37:59.005 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.005 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.005 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:65447 6.a 6:5c3c3d34:::benchmark_data_smithi143_305742_object65446:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-08-28T02:37:59.005 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:28:07.533387+0000", 2023-08-28T02:37:59.006 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 591.42181243100003, 2023-08-28T02:37:59.006 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.858966224, 2023-08-28T02:37:59.006 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:37:59.006 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:37:59.006 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:37:59.006 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:37:59.006 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:37:59.006 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 65447 2023-08-28T02:37:59.006 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.006 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:37:59.006 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.006 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:37:59.006 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533387+0000", 2023-08-28T02:37:59.007 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:37:59.007 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.007 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.007 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:37:59.007 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533387+0000", 2023-08-28T02:37:59.007 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:37:59.007 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.007 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.007 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:37:59.007 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533386+0000", 2023-08-28T02:37:59.007 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.5514999999999999e-05 2023-08-28T02:37:59.007 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.007 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.008 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:37:59.008 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533452+0000", 2023-08-28T02:37:59.008 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.9199999999999998e-06 2023-08-28T02:37:59.008 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.008 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.008 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:37:59.008 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533459+0000", 2023-08-28T02:37:59.008 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.9409999999999998e-06 2023-08-28T02:37:59.008 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.008 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.008 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:37:59.008 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533464+0000", 2023-08-28T02:37:59.009 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.6830999999999999e-05 2023-08-28T02:37:59.009 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.009 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.009 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:37:59.009 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533510+0000", 2023-08-28T02:37:59.009 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.3380999999999999e-05 2023-08-28T02:37:59.009 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.009 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.009 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:37:59.009 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533524+0000", 2023-08-28T02:37:59.009 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.83310036300000001 2023-08-28T02:37:59.009 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.010 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.010 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:37:59.010 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.366624+0000", 2023-08-28T02:37:59.010 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012930600000000001 2023-08-28T02:37:59.010 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.010 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.010 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:37:59.010 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.366753+0000", 2023-08-28T02:37:59.010 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000102413 2023-08-28T02:37:59.010 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.010 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.011 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,2", 2023-08-28T02:37:59.011 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.366856+0000", 2023-08-28T02:37:59.011 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0077729440000000004 2023-08-28T02:37:59.011 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.011 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.011 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:37:59.011 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.374629+0000", 2023-08-28T02:37:59.011 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0032932349999999998 2023-08-28T02:37:59.011 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.011 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.011 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:59.011 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377922+0000", 2023-08-28T02:37:59.012 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.01433544 2023-08-28T02:37:59.012 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.012 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.012 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:59.012 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.392257+0000", 2023-08-28T02:37:59.012 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.2425e-05 2023-08-28T02:37:59.012 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.012 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.012 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:37:59.012 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.392310+0000", 2023-08-28T02:37:59.012 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.3294999999999998e-05 2023-08-28T02:37:59.012 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.012 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.013 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:37:59.013 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.392353+0000", 2023-08-28T02:37:59.013 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.858966224 2023-08-28T02:37:59.013 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:37:59.013 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:37:59.013 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:37:59.013 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.013 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.013 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:65449 6.a 6:53253814:::benchmark_data_smithi143_305742_object65448:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-08-28T02:37:59.013 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:28:07.533542+0000", 2023-08-28T02:37:59.013 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 591.42165700099997, 2023-08-28T02:37:59.013 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.85898421800000002, 2023-08-28T02:37:59.013 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:37:59.014 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:37:59.014 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:37:59.014 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:37:59.014 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:37:59.014 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 65449 2023-08-28T02:37:59.014 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.014 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:37:59.014 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.014 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:37:59.014 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533542+0000", 2023-08-28T02:37:59.014 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:37:59.014 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.014 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.015 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:37:59.015 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533542+0000", 2023-08-28T02:37:59.015 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999995 2023-08-28T02:37:59.015 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.015 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.015 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:37:59.015 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533542+0000", 2023-08-28T02:37:59.015 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.8541999999999999e-05 2023-08-28T02:37:59.015 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.015 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.016 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:37:59.016 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533570+0000", 2023-08-28T02:37:59.016 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.267e-06 2023-08-28T02:37:59.016 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.016 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.016 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:37:59.016 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533578+0000", 2023-08-28T02:37:59.016 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.6210000000000003e-06 2023-08-28T02:37:59.016 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.016 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.016 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:37:59.016 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533582+0000", 2023-08-28T02:37:59.017 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.1579000000000002e-05 2023-08-28T02:37:59.017 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.017 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.017 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:37:59.017 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533634+0000", 2023-08-28T02:37:59.017 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.8409999999999995e-06 2023-08-28T02:37:59.017 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.017 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.017 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:37:59.017 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.533644+0000", 2023-08-28T02:37:59.017 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.83422247699999996 2023-08-28T02:37:59.017 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.018 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.018 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:37:59.018 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.367866+0000", 2023-08-28T02:37:59.018 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00013198600000000001 2023-08-28T02:37:59.018 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.018 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.018 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:37:59.018 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.367998+0000", 2023-08-28T02:37:59.018 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00016983100000000001 2023-08-28T02:37:59.018 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.018 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.018 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,2", 2023-08-28T02:37:59.018 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.368168+0000", 2023-08-28T02:37:59.019 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0064664470000000002 2023-08-28T02:37:59.019 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.019 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.019 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:37:59.019 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.374634+0000", 2023-08-28T02:37:59.019 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0074225960000000001 2023-08-28T02:37:59.019 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.019 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.019 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:59.019 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.382057+0000", 2023-08-28T02:37:59.019 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.010359019000000001 2023-08-28T02:37:59.019 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.020 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.020 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:59.020 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.392416+0000", 2023-08-28T02:37:59.020 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.0843e-05 2023-08-28T02:37:59.020 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.020 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.020 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:37:59.020 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.392477+0000", 2023-08-28T02:37:59.020 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.9842999999999997e-05 2023-08-28T02:37:59.020 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.020 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.020 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:37:59.021 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.392527+0000", 2023-08-28T02:37:59.021 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.85898421800000002 2023-08-28T02:37:59.021 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:37:59.021 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:37:59.021 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:37:59.021 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.021 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.021 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:65517 6.a 6:57043958:::benchmark_data_smithi143_305742_object65516:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-08-28T02:37:59.021 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:28:07.651846+0000", 2023-08-28T02:37:59.021 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 591.30335306799998, 2023-08-28T02:37:59.021 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.74079570299999997, 2023-08-28T02:37:59.021 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:37:59.022 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:37:59.022 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:37:59.022 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:37:59.022 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:37:59.022 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 65517 2023-08-28T02:37:59.022 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.022 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:37:59.022 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.022 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:37:59.022 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.651846+0000", 2023-08-28T02:37:59.023 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:37:59.023 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.023 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.023 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:37:59.023 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.651846+0000", 2023-08-28T02:37:59.023 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:37:59.023 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.023 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.023 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:37:59.023 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.651845+0000", 2023-08-28T02:37:59.024 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.3328000000000002e-05 2023-08-28T02:37:59.024 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.024 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.024 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:37:59.024 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.651899+0000", 2023-08-28T02:37:59.024 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1822e-05 2023-08-28T02:37:59.024 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.024 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.024 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:37:59.024 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.651910+0000", 2023-08-28T02:37:59.024 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.1270000000000002e-06 2023-08-28T02:37:59.024 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.025 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.025 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:37:59.025 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.651915+0000", 2023-08-28T02:37:59.025 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.2360000000000005e-05 2023-08-28T02:37:59.025 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.025 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.025 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:37:59.025 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.651988+0000", 2023-08-28T02:37:59.025 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.1109999999999997e-06 2023-08-28T02:37:59.025 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.025 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.025 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:37:59.026 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.651997+0000", 2023-08-28T02:37:59.026 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.71717974100000004 2023-08-28T02:37:59.026 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.026 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.026 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:37:59.026 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.369177+0000", 2023-08-28T02:37:59.026 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012856099999999999 2023-08-28T02:37:59.026 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.026 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.026 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:37:59.026 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.369305+0000", 2023-08-28T02:37:59.026 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00010799200000000001 2023-08-28T02:37:59.027 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.027 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.027 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,2", 2023-08-28T02:37:59.027 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.369413+0000", 2023-08-28T02:37:59.027 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.005225861 2023-08-28T02:37:59.027 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.027 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.027 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:37:59.027 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.374639+0000", 2023-08-28T02:37:59.027 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.013565384999999999 2023-08-28T02:37:59.027 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.027 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.028 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:59.028 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.388205+0000", 2023-08-28T02:37:59.028 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0043731919999999997 2023-08-28T02:37:59.028 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.028 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.028 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:59.028 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.392578+0000", 2023-08-28T02:37:59.028 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.9850000000000001e-05 2023-08-28T02:37:59.028 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.028 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.028 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:37:59.028 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.392608+0000", 2023-08-28T02:37:59.028 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.4477999999999998e-05 2023-08-28T02:37:59.029 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.029 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.029 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:37:59.029 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.392642+0000", 2023-08-28T02:37:59.029 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.74079570299999997 2023-08-28T02:37:59.029 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:37:59.029 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:37:59.029 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:37:59.029 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.029 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.029 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:65528 6.a 6:584fd7a4:::benchmark_data_smithi143_305742_object65527:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-08-28T02:37:59.029 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:28:07.678223+0000", 2023-08-28T02:37:59.030 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 591.27697605699996, 2023-08-28T02:37:59.030 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.71583466399999995, 2023-08-28T02:37:59.030 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:37:59.030 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:37:59.030 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:37:59.030 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:37:59.030 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:37:59.030 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 65528 2023-08-28T02:37:59.030 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.030 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:37:59.030 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.031 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:37:59.031 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.678223+0000", 2023-08-28T02:37:59.031 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:37:59.031 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.031 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.031 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:37:59.031 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.678223+0000", 2023-08-28T02:37:59.031 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:37:59.031 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.031 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.031 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:37:59.031 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.678222+0000", 2023-08-28T02:37:59.032 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.5943999999999999e-05 2023-08-28T02:37:59.032 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.032 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.032 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:37:59.032 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.678268+0000", 2023-08-28T02:37:59.032 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0148e-05 2023-08-28T02:37:59.032 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.032 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.032 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:37:59.032 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.678278+0000", 2023-08-28T02:37:59.032 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.0350000000000001e-06 2023-08-28T02:37:59.032 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.032 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.032 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:37:59.033 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.678286+0000", 2023-08-28T02:37:59.033 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.7000000000000003e-05 2023-08-28T02:37:59.033 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.033 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.033 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:37:59.033 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.678343+0000", 2023-08-28T02:37:59.033 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.8419999999999994e-06 2023-08-28T02:37:59.033 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.033 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.033 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:37:59.033 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.678352+0000", 2023-08-28T02:37:59.033 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.69213458500000002 2023-08-28T02:37:59.034 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.034 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.034 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:37:59.034 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.370487+0000", 2023-08-28T02:37:59.034 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012174400000000001 2023-08-28T02:37:59.034 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.034 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.034 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:37:59.034 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.370608+0000", 2023-08-28T02:37:59.034 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00016430399999999999 2023-08-28T02:37:59.034 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.034 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.035 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,2", 2023-08-28T02:37:59.035 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.370773+0000", 2023-08-28T02:37:59.035 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0039615850000000001 2023-08-28T02:37:59.035 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.035 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.035 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:37:59.035 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.374734+0000", 2023-08-28T02:37:59.035 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.017946527 2023-08-28T02:37:59.035 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.035 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.035 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:59.036 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.392681+0000", 2023-08-28T02:37:59.036 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.001296632 2023-08-28T02:37:59.036 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.036 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.036 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:59.036 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.393977+0000", 2023-08-28T02:37:59.036 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.4098999999999999e-05 2023-08-28T02:37:59.036 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.036 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.036 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:37:59.036 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.394022+0000", 2023-08-28T02:37:59.036 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.6463e-05 2023-08-28T02:37:59.036 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.037 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.037 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:37:59.037 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.394058+0000", 2023-08-28T02:37:59.037 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.71583466399999995 2023-08-28T02:37:59.037 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:37:59.037 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:37:59.037 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:37:59.037 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.037 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.037 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:65590 6.a 6:52c121f0:::benchmark_data_smithi143_305742_object65589:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-08-28T02:37:59.037 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:28:07.793373+0000", 2023-08-28T02:37:59.037 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 591.16182653800001, 2023-08-28T02:37:59.038 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.60446993100000002, 2023-08-28T02:37:59.038 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:37:59.038 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:37:59.038 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:37:59.038 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:37:59.038 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:37:59.038 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 65590 2023-08-28T02:37:59.038 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.038 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:37:59.038 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.038 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:37:59.038 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.793373+0000", 2023-08-28T02:37:59.039 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:37:59.039 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.039 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.039 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:37:59.039 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.793373+0000", 2023-08-28T02:37:59.039 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:37:59.039 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.039 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.039 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:37:59.039 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.793372+0000", 2023-08-28T02:37:59.039 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.9734000000000002e-05 2023-08-28T02:37:59.039 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.040 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.040 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:37:59.040 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.793421+0000", 2023-08-28T02:37:59.040 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2872e-05 2023-08-28T02:37:59.040 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.040 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.040 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:37:59.040 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.793434+0000", 2023-08-28T02:37:59.040 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.4409999999999998e-06 2023-08-28T02:37:59.040 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.040 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.040 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:37:59.040 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.793441+0000", 2023-08-28T02:37:59.041 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.2967000000000003e-05 2023-08-28T02:37:59.041 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.041 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.041 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:37:59.041 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.793494+0000", 2023-08-28T02:37:59.041 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.2439999999999996e-06 2023-08-28T02:37:59.041 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.041 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.041 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:37:59.041 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.793501+0000", 2023-08-28T02:37:59.041 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.57831116699999996 2023-08-28T02:37:59.041 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.042 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.042 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:37:59.042 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.371812+0000", 2023-08-28T02:37:59.042 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000123433 2023-08-28T02:37:59.042 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.042 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.042 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:37:59.042 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.371936+0000", 2023-08-28T02:37:59.042 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00016441700000000001 2023-08-28T02:37:59.042 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.042 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.042 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,2", 2023-08-28T02:37:59.043 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.372100+0000", 2023-08-28T02:37:59.043 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0026377420000000002 2023-08-28T02:37:59.043 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.043 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.043 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:37:59.043 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.374738+0000", 2023-08-28T02:37:59.043 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.019368895000000001 2023-08-28T02:37:59.043 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.043 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.043 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:59.043 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.394107+0000", 2023-08-28T02:37:59.043 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.003625634 2023-08-28T02:37:59.043 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.044 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.044 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:59.044 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.397732+0000", 2023-08-28T02:37:59.044 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.1490999999999998e-05 2023-08-28T02:37:59.044 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.044 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.044 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:37:59.044 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.397794+0000", 2023-08-28T02:37:59.044 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.9030000000000003e-05 2023-08-28T02:37:59.044 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.044 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.045 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:37:59.045 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.397843+0000", 2023-08-28T02:37:59.045 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.60446993100000002 2023-08-28T02:37:59.045 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:37:59.045 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:37:59.045 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:37:59.045 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.045 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.045 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.35076.0:65630 6.a 6:54d1eb7a:::benchmark_data_smithi143_305742_object65629:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e214)", 2023-08-28T02:37:59.045 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:28:07.879482+0000", 2023-08-28T02:37:59.045 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 591.07571784000004, 2023-08-28T02:37:59.045 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.51854038599999996, 2023-08-28T02:37:59.046 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:37:59.046 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:37:59.046 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:37:59.046 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.35076", 2023-08-28T02:37:59.046 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3909308270", 2023-08-28T02:37:59.046 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 65630 2023-08-28T02:37:59.046 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.046 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:37:59.046 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.046 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:37:59.046 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.879482+0000", 2023-08-28T02:37:59.046 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:37:59.047 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.047 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.047 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:37:59.047 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.879482+0000", 2023-08-28T02:37:59.047 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999976 2023-08-28T02:37:59.047 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.047 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.047 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:37:59.047 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.879479+0000", 2023-08-28T02:37:59.047 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.5662999999999999e-05 2023-08-28T02:37:59.047 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.047 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.047 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:37:59.048 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.879535+0000", 2023-08-28T02:37:59.048 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.6812000000000001e-05 2023-08-28T02:37:59.048 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.048 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.048 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:37:59.048 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.879552+0000", 2023-08-28T02:37:59.048 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.8879999999999997e-06 2023-08-28T02:37:59.048 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.048 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.048 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:37:59.048 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.879559+0000", 2023-08-28T02:37:59.048 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.8687000000000002e-05 2023-08-28T02:37:59.049 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.049 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.049 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:37:59.049 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.879628+0000", 2023-08-28T02:37:59.049 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2177999999999999e-05 2023-08-28T02:37:59.049 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.049 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.049 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:37:59.049 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:07.879640+0000", 2023-08-28T02:37:59.049 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.49364018300000001 2023-08-28T02:37:59.049 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.049 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.049 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:37:59.050 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.373281+0000", 2023-08-28T02:37:59.050 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000123545 2023-08-28T02:37:59.050 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.050 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.050 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:37:59.050 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.373404+0000", 2023-08-28T02:37:59.050 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00016205899999999999 2023-08-28T02:37:59.050 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.050 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.050 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,2", 2023-08-28T02:37:59.050 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.373566+0000", 2023-08-28T02:37:59.050 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.003690292 2023-08-28T02:37:59.051 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.051 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.051 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:37:59.051 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.377256+0000", 2023-08-28T02:37:59.051 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.016889149999999999 2023-08-28T02:37:59.051 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.051 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.051 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:59.051 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.394146+0000", 2023-08-28T02:37:59.051 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0037648989999999999 2023-08-28T02:37:59.051 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.051 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.052 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:37:59.052 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.397910+0000", 2023-08-28T02:37:59.052 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.1008999999999998e-05 2023-08-28T02:37:59.052 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.052 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.052 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:37:59.052 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.397971+0000", 2023-08-28T02:37:59.052 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.0457000000000002e-05 2023-08-28T02:37:59.052 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.052 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.052 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:37:59.052 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:28:08.398022+0000", 2023-08-28T02:37:59.053 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.51854038599999996 2023-08-28T02:37:59.053 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:37:59.053 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:37:59.053 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:37:59.053 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.053 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.053 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:59974 7.0 7:055944e3:::benchmark_data_smithi143_360738_object59973:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e270)", 2023-08-28T02:37:59.053 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:31:43.477791+0000", 2023-08-28T02:37:59.053 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 375.47740865100002, 2023-08-28T02:37:59.053 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.99199723799999995, 2023-08-28T02:37:59.053 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:37:59.053 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "reached pg", 2023-08-28T02:37:59.053 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:37:59.054 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:37:59.054 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:37:59.054 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 59974 2023-08-28T02:37:59.054 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.054 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:37:59.054 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.054 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:37:59.054 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.477791+0000", 2023-08-28T02:37:59.054 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:37:59.054 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.054 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.054 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:37:59.055 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.477791+0000", 2023-08-28T02:37:59.055 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999967 2023-08-28T02:37:59.055 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.055 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.055 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:37:59.055 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.477787+0000", 2023-08-28T02:37:59.055 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.3393999999999995e-05 2023-08-28T02:37:59.055 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.055 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.055 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:37:59.055 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.477871+0000", 2023-08-28T02:37:59.055 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.5343000000000003e-05 2023-08-28T02:37:59.055 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.056 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.056 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:37:59.056 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.477916+0000", 2023-08-28T02:37:59.056 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2707999999999999e-05 2023-08-28T02:37:59.056 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.056 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.056 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:37:59.056 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.477929+0000", 2023-08-28T02:37:59.056 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.011155768999999999 2023-08-28T02:37:59.056 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.056 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.056 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:37:59.057 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.489085+0000", 2023-08-28T02:37:59.057 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.8269999999999993e-06 2023-08-28T02:37:59.057 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.057 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.057 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:37:59.057 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.489094+0000", 2023-08-28T02:37:59.057 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.98061759000000004 2023-08-28T02:37:59.057 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.057 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.057 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:37:59.057 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.469712+0000", 2023-08-28T02:37:59.058 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.6050000000000005e-05 2023-08-28T02:37:59.058 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.058 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.058 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:37:59.058 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.469788+0000", 2023-08-28T02:37:59.058 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.99199723799999995 2023-08-28T02:37:59.058 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:37:59.058 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:37:59.058 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:37:59.058 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.058 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.058 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:59977 7.0 7:00ce683a:::benchmark_data_smithi143_360738_object59976:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e270)", 2023-08-28T02:37:59.058 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:31:43.477944+0000", 2023-08-28T02:37:59.059 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 375.47725510399999, 2023-08-28T02:37:59.059 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.99213284300000004, 2023-08-28T02:37:59.059 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:37:59.059 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "reached pg", 2023-08-28T02:37:59.059 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:37:59.059 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:37:59.059 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:37:59.059 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 59977 2023-08-28T02:37:59.059 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.059 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:37:59.059 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.060 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:37:59.060 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.477944+0000", 2023-08-28T02:37:59.060 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:37:59.060 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.060 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.060 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:37:59.060 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.477944+0000", 2023-08-28T02:37:59.060 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999995 2023-08-28T02:37:59.060 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.060 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.060 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:37:59.060 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.477944+0000", 2023-08-28T02:37:59.061 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.2744000000000002e-05 2023-08-28T02:37:59.061 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.061 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.061 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:37:59.061 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.477996+0000", 2023-08-28T02:37:59.061 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.0423e-05 2023-08-28T02:37:59.061 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.061 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.061 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:37:59.061 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.478027+0000", 2023-08-28T02:37:59.061 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.2459999999999997e-06 2023-08-28T02:37:59.061 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.061 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.062 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:37:59.062 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.478034+0000", 2023-08-28T02:37:59.062 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.011097246 2023-08-28T02:37:59.062 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.062 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.062 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:37:59.062 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.489131+0000", 2023-08-28T02:37:59.062 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.455e-06 2023-08-28T02:37:59.062 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.062 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.062 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:37:59.062 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.489138+0000", 2023-08-28T02:37:59.063 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.98088636200000001 2023-08-28T02:37:59.063 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.063 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.063 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:37:59.063 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.470024+0000", 2023-08-28T02:37:59.063 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.2932000000000001e-05 2023-08-28T02:37:59.063 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.063 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.063 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:37:59.063 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.470077+0000", 2023-08-28T02:37:59.063 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.99213284300000004 2023-08-28T02:37:59.063 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:37:59.063 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:37:59.064 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:37:59.064 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.064 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.064 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:59978 7.10 7:0e447851:::benchmark_data_smithi143_360738_object59977:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e270)", 2023-08-28T02:37:59.064 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:31:43.478040+0000", 2023-08-28T02:37:59.064 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 375.47715972200001, 2023-08-28T02:37:59.064 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0064679560000001, 2023-08-28T02:37:59.064 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:37:59.064 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "reached pg", 2023-08-28T02:37:59.064 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:37:59.064 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:37:59.064 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:37:59.064 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 59978 2023-08-28T02:37:59.065 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.065 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:37:59.065 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.065 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:37:59.065 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.478040+0000", 2023-08-28T02:37:59.065 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:37:59.065 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.065 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.065 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:37:59.065 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.478040+0000", 2023-08-28T02:37:59.065 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999995 2023-08-28T02:37:59.065 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.066 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.066 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:37:59.066 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.478039+0000", 2023-08-28T02:37:59.066 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.7616e-05 2023-08-28T02:37:59.066 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.066 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.066 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:37:59.066 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.478077+0000", 2023-08-28T02:37:59.066 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.9510000000000006e-06 2023-08-28T02:37:59.066 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.066 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.066 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:37:59.067 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.478085+0000", 2023-08-28T02:37:59.067 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.6410000000000001e-06 2023-08-28T02:37:59.067 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.067 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.067 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:37:59.067 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.478089+0000", 2023-08-28T02:37:59.067 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.011218619000000001 2023-08-28T02:37:59.067 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.067 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.067 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:37:59.067 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.489308+0000", 2023-08-28T02:37:59.067 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.6300000000000003e-06 2023-08-28T02:37:59.068 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.068 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.068 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:37:59.068 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.489313+0000", 2023-08-28T02:37:59.068 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.99510897799999998 2023-08-28T02:37:59.068 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.068 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.068 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:37:59.068 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.484422+0000", 2023-08-28T02:37:59.068 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.5159000000000002e-05 2023-08-28T02:37:59.070 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.070 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.070 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:37:59.070 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.484508+0000", 2023-08-28T02:37:59.070 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0064679560000001 2023-08-28T02:37:59.070 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:37:59.070 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:37:59.070 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:37:59.070 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.070 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.070 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:59984 7.0 7:01f850c8:::benchmark_data_smithi143_360738_object59983:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e270)", 2023-08-28T02:37:59.071 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:31:43.478095+0000", 2023-08-28T02:37:59.071 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 375.47710479199998, 2023-08-28T02:37:59.071 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.992082256, 2023-08-28T02:37:59.071 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:37:59.071 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "reached pg", 2023-08-28T02:37:59.071 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:37:59.071 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:37:59.071 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:37:59.071 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 59984 2023-08-28T02:37:59.071 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.071 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:37:59.072 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.072 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:37:59.072 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.478095+0000", 2023-08-28T02:37:59.072 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:37:59.072 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.072 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.072 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:37:59.072 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.478095+0000", 2023-08-28T02:37:59.072 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999995 2023-08-28T02:37:59.072 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.072 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.072 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:37:59.072 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.478094+0000", 2023-08-28T02:37:59.073 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.0431999999999999e-05 2023-08-28T02:37:59.073 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.073 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.073 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:37:59.073 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.478125+0000", 2023-08-28T02:37:59.073 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.8859999999999998e-06 2023-08-28T02:37:59.073 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.073 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.073 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:37:59.073 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.478131+0000", 2023-08-28T02:37:59.073 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.4510000000000002e-06 2023-08-28T02:37:59.074 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.074 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.074 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:37:59.074 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.478136+0000", 2023-08-28T02:37:59.074 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.011034641 2023-08-28T02:37:59.074 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.074 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.074 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:37:59.074 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.489170+0000", 2023-08-28T02:37:59.074 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.1199999999999996e-06 2023-08-28T02:37:59.074 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.074 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.075 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:37:59.075 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.489178+0000", 2023-08-28T02:37:59.075 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.98092388699999999 2023-08-28T02:37:59.075 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.075 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.075 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:37:59.075 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.470101+0000", 2023-08-28T02:37:59.075 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.5358999999999995e-05 2023-08-28T02:37:59.075 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.075 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.075 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:37:59.075 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.470177+0000", 2023-08-28T02:37:59.076 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.992082256 2023-08-28T02:37:59.076 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:37:59.076 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:37:59.076 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:37:59.076 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.076 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.076 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_repop(client.45647.0:136726 8.d e348/345 8:b32ce506:::benchmark_data_smithi143_419289_object39816:head v 348'8412, mlcod=330'2434)", 2023-08-28T02:37:59.076 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:37:54.521984+0000", 2023-08-28T02:37:59.076 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 4.4332154829999997, 2023-08-28T02:37:59.076 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.13352926500000001, 2023-08-28T02:37:59.076 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:37:59.076 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:37:59.077 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:37:59.077 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.077 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:37:59.077 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:54.521984+0000", 2023-08-28T02:37:59.077 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:37:59.077 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.077 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.077 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:37:59.077 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:54.521984+0000", 2023-08-28T02:37:59.077 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:37:59.077 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.077 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.077 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:37:59.078 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:54.521982+0000", 2023-08-28T02:37:59.078 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.3699999999999999e-05 2023-08-28T02:37:59.078 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.078 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.078 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:37:59.078 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:54.521996+0000", 2023-08-28T02:37:59.078 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.115631989 2023-08-28T02:37:59.078 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.078 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.078 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:37:59.078 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:54.637628+0000", 2023-08-28T02:37:59.078 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.8979999999999996e-06 2023-08-28T02:37:59.079 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.079 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.079 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:37:59.079 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:54.637636+0000", 2023-08-28T02:37:59.079 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00025963599999999999 2023-08-28T02:37:59.079 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.079 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.079 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:37:59.079 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:54.637896+0000", 2023-08-28T02:37:59.079 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.4627e-05 2023-08-28T02:37:59.079 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.080 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.080 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:37:59.080 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:54.637920+0000", 2023-08-28T02:37:59.080 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.017552116999999999 2023-08-28T02:37:59.080 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.080 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.080 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:37:59.080 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:54.655472+0000", 2023-08-28T02:37:59.080 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.0873e-05 2023-08-28T02:37:59.080 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.080 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.080 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:37:59.081 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:54.655513+0000", 2023-08-28T02:37:59.081 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.13352926500000001 2023-08-28T02:37:59.081 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:37:59.081 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:37:59.081 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:37:59.081 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.081 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.081 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_repop(client.45647.0:138753 8.d e349/345 8:b9d598ab:::benchmark_data_smithi143_419289_object41843:head v 349'8551, mlcod=330'2573)", 2023-08-28T02:37:59.081 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:37:57.899625+0000", 2023-08-28T02:37:59.081 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 1.055574214, 2023-08-28T02:37:59.081 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.17459303100000001, 2023-08-28T02:37:59.081 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:37:59.082 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:37:59.082 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:37:59.082 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.082 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:37:59.082 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:57.899625+0000", 2023-08-28T02:37:59.082 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:37:59.082 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.082 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.082 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:37:59.082 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:57.899625+0000", 2023-08-28T02:37:59.082 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:37:59.083 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.083 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.083 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:37:59.083 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:57.899624+0000", 2023-08-28T02:37:59.083 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.3018e-05 2023-08-28T02:37:59.083 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.083 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.083 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:37:59.083 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:57.899637+0000", 2023-08-28T02:37:59.083 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.3400999999999999e-05 2023-08-28T02:37:59.083 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.083 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.083 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:37:59.084 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:57.899650+0000", 2023-08-28T02:37:59.084 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.4799999999999998e-06 2023-08-28T02:37:59.084 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.084 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.084 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:37:59.084 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:57.899656+0000", 2023-08-28T02:37:59.084 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.16461827300000001 2023-08-28T02:37:59.084 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.084 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.084 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:37:59.084 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:58.064275+0000", 2023-08-28T02:37:59.084 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.294e-05 2023-08-28T02:37:59.085 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.085 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.085 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:37:59.085 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:58.064308+0000", 2023-08-28T02:37:59.085 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0098717070000000004 2023-08-28T02:37:59.085 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.085 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.085 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:37:59.085 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:58.074179+0000", 2023-08-28T02:37:59.085 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.8866999999999999e-05 2023-08-28T02:37:59.085 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.085 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.086 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:37:59.086 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:58.074218+0000", 2023-08-28T02:37:59.086 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.17459303100000001 2023-08-28T02:37:59.086 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:37:59.086 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:37:59.086 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:37:59.086 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.086 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.086 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_repop(client.45647.0:138767 8.8 e349/345 8:186376c6:::benchmark_data_smithi143_419289_object41857:head v 349'8845, mlcod=349'8843)", 2023-08-28T02:37:59.086 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:37:57.917765+0000", 2023-08-28T02:37:59.086 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 1.037434883, 2023-08-28T02:37:59.086 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.161207551, 2023-08-28T02:37:59.086 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:37:59.087 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:37:59.087 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:37:59.087 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.087 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:37:59.087 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:57.917765+0000", 2023-08-28T02:37:59.087 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:37:59.087 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.087 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.087 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:37:59.087 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:57.917765+0000", 2023-08-28T02:37:59.087 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:37:59.087 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.088 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.088 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:37:59.088 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:57.917763+0000", 2023-08-28T02:37:59.088 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0526000000000001e-05 2023-08-28T02:37:59.088 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.088 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.088 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:37:59.088 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:57.917774+0000", 2023-08-28T02:37:59.088 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.14621696200000001 2023-08-28T02:37:59.088 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.088 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.088 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:37:59.089 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:58.063991+0000", 2023-08-28T02:37:59.089 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.3631999999999999e-05 2023-08-28T02:37:59.089 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.089 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.089 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:37:59.089 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:58.064004+0000", 2023-08-28T02:37:59.089 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.001694217 2023-08-28T02:37:59.089 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.089 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.089 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:37:59.089 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:58.065699+0000", 2023-08-28T02:37:59.090 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.4907999999999999e-05 2023-08-28T02:37:59.090 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.090 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.090 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:37:59.090 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:58.065724+0000", 2023-08-28T02:37:59.090 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.013205285000000001 2023-08-28T02:37:59.090 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.090 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.090 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:37:59.090 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:58.078929+0000", 2023-08-28T02:37:59.090 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.3167000000000002e-05 2023-08-28T02:37:59.090 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.090 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.091 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:37:59.091 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:58.078972+0000", 2023-08-28T02:37:59.091 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.161207551 2023-08-28T02:37:59.091 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:37:59.091 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:37:59.091 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:37:59.091 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.091 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.091 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_repop(client.45647.0:138766 8.d e349/345 8:b620298b:::benchmark_data_smithi143_419289_object41856:head v 349'8552, mlcod=330'2575)", 2023-08-28T02:37:59.091 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:37:57.924440+0000", 2023-08-28T02:37:59.091 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 1.0307592000000001, 2023-08-28T02:37:59.092 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.15460950700000001, 2023-08-28T02:37:59.092 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:37:59.092 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:37:59.092 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:37:59.092 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.092 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:37:59.092 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:57.924440+0000", 2023-08-28T02:37:59.092 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:37:59.092 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.092 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.092 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:37:59.093 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:57.924440+0000", 2023-08-28T02:37:59.093 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:37:59.093 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.093 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.093 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:37:59.093 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:57.924439+0000", 2023-08-28T02:37:59.093 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.3261999999999999e-05 2023-08-28T02:37:59.093 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.093 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.093 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:37:59.093 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:57.924452+0000", 2023-08-28T02:37:59.093 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.13775354400000001 2023-08-28T02:37:59.093 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.094 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.094 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:37:59.094 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:58.062206+0000", 2023-08-28T02:37:59.094 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.6784000000000001e-05 2023-08-28T02:37:59.094 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.094 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.094 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:37:59.094 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:58.062223+0000", 2023-08-28T02:37:59.094 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0080636149999999997 2023-08-28T02:37:59.094 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.094 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.095 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:37:59.095 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:58.070286+0000", 2023-08-28T02:37:59.095 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.4134e-05 2023-08-28T02:37:59.095 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.095 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.095 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:37:59.095 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:58.070310+0000", 2023-08-28T02:37:59.095 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0087060739999999994 2023-08-28T02:37:59.095 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.095 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.095 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:37:59.095 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:58.079017+0000", 2023-08-28T02:37:59.095 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.3164000000000003e-05 2023-08-28T02:37:59.095 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:37:59.096 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:37:59.096 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:37:59.096 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:58.079050+0000", 2023-08-28T02:37:59.096 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.15460950700000001 2023-08-28T02:37:59.096 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:37:59.096 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:37:59.096 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:37:59.096 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:37:59.096 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:37:59.096 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:37:59.097 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:37:59.097 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:58 smithi143 ceph-mon[189871]: from='client.45812 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:37:59.097 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:58 smithi143 ceph-mon[189871]: pgmap v933: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 518 op/s; 6191/170706 objects degraded (3.627%); 14082/170706 objects misplaced (8.249%); 1.8 MiB/s, 30 objects/s recovering 2023-08-28T02:37:59.097 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:58 smithi143 ceph-mon[189871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:37:59.098 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:58 smithi143 ceph-mon[189871]: failed to encode map e349 with expected crc 2023-08-28T02:37:59.098 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:58 smithi143 ceph-mon[189871]: failed to encode map e349 with expected crc 2023-08-28T02:37:59.098 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:58 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:37:59.098 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:58 smithi143 ceph-mon[189871]: osdmap e349: 8 total, 8 up, 8 in 2023-08-28T02:37:59.098 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:58 smithi143 ceph-mon[189871]: failed to encode map e349 with expected crc 2023-08-28T02:37:59.166 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:37:59.826 INFO:teuthology.orchestra.run.smithi143.stdout:{"pg_ready":true,"pg_map":{"version":932,"stamp":"2023-08-28T02:37:55.583342+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3781040075,"num_objects":57650,"num_object_clones":0,"num_object_copies":172950,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6191,"num_objects_misplaced":14082,"num_objects_unfound":0,"num_objects_dirty":57650,"num_whiteouts":0,"num_read":6275,"num_read_kb":13708,"num_write":137300,"num_write_kb":6217190,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":612,"num_bytes_recovered":33652070,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":123964,"ondisk_log_size":123964,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":273,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":15442136,"kb_used_data":11238552,"kb_used_omap":38,"kb_used_meta":4203289,"kb_avail":734519080,"statfs":{"total":767960285184,"available":752147537920,"internally_reserved":0,"allocated":11508277248,"data_stored":11498131905,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":39201,"internal_metadata":4304168671},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,1,6,6,3,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-288751616,"num_objects":-4406,"num_object_clones":0,"num_object_copies":-13218,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":125,"num_objects_misplaced":14082,"num_objects_unfound":0,"num_objects_dirty":-4406,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4406,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":299,"num_bytes_recovered":18678204,"num_keys_recovered":-10,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.009255"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":367,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.661326+0000","last_change":"2023-08-28T02:32:58.740384+0000","last_active":"2023-08-28T02:37:47.661326+0000","last_peered":"2023-08-28T02:37:47.661326+0000","last_clean":"2023-08-28T02:37:47.661326+0000","last_became_active":"2023-08-28T02:32:58.739959+0000","last_became_peered":"2023-08-28T02:32:58.739959+0000","last_unstale":"2023-08-28T02:37:47.661326+0000","last_undegraded":"2023-08-28T02:37:47.661326+0000","last_fullsized":"2023-08-28T02:37:47.661326+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":255,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.641068+0000","last_change":"2023-08-28T02:32:58.634375+0000","last_active":"2023-08-28T02:37:47.641068+0000","last_peered":"2023-08-28T02:37:47.641068+0000","last_clean":"2023-08-28T02:37:47.641068+0000","last_became_active":"2023-08-28T02:32:58.632726+0000","last_became_peered":"2023-08-28T02:32:58.632726+0000","last_unstale":"2023-08-28T02:37:47.641068+0000","last_undegraded":"2023-08-28T02:37:47.641068+0000","last_fullsized":"2023-08-28T02:37:47.641068+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:20:27.730725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":69,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-08-28T02:37:48.641210+0000","last_change":"2023-08-28T02:32:58.628387+0000","last_active":"2023-08-28T02:37:48.641210+0000","last_peered":"2023-08-28T02:37:48.641210+0000","last_clean":"2023-08-28T02:37:48.641210+0000","last_became_active":"2023-08-28T02:32:58.628071+0000","last_became_peered":"2023-08-28T02:32:58.628071+0000","last_unstale":"2023-08-28T02:37:48.641210+0000","last_undegraded":"2023-08-28T02:37:48.641210+0000","last_fullsized":"2023-08-28T02:37:48.641210+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:25:55.264433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":375,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.643052+0000","last_change":"2023-08-28T02:37:45.580751+0000","last_active":"2023-08-28T02:37:47.643052+0000","last_peered":"2023-08-28T02:37:47.643052+0000","last_clean":"2023-08-28T02:37:47.643052+0000","last_became_active":"2023-08-28T02:37:45.580446+0000","last_became_peered":"2023-08-28T02:37:45.580446+0000","last_unstale":"2023-08-28T02:37:47.643052+0000","last_undegraded":"2023-08-28T02:37:47.643052+0000","last_fullsized":"2023-08-28T02:37:47.643052+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:59:21.094880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":246,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.640503+0000","last_change":"2023-08-28T02:32:58.779602+0000","last_active":"2023-08-28T02:37:47.640503+0000","last_peered":"2023-08-28T02:37:47.640503+0000","last_clean":"2023-08-28T02:37:47.640503+0000","last_became_active":"2023-08-28T02:32:58.778945+0000","last_became_peered":"2023-08-28T02:32:58.778945+0000","last_unstale":"2023-08-28T02:37:47.640503+0000","last_undegraded":"2023-08-28T02:37:47.640503+0000","last_fullsized":"2023-08-28T02:37:47.640503+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:35:55.832928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":154,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.640642+0000","last_change":"2023-08-28T02:32:58.626766+0000","last_active":"2023-08-28T02:37:47.640642+0000","last_peered":"2023-08-28T02:37:47.640642+0000","last_clean":"2023-08-28T02:37:47.640642+0000","last_became_active":"2023-08-28T02:32:58.626415+0000","last_became_peered":"2023-08-28T02:32:58.626415+0000","last_unstale":"2023-08-28T02:37:47.640642+0000","last_undegraded":"2023-08-28T02:37:47.640642+0000","last_fullsized":"2023-08-28T02:37:47.640642+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":318,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.636282+0000","last_change":"2023-08-28T02:23:59.227592+0000","last_active":"2023-08-28T02:37:47.636282+0000","last_peered":"2023-08-28T02:37:47.636282+0000","last_clean":"2023-08-28T02:37:47.636282+0000","last_became_active":"2023-08-28T02:23:59.227342+0000","last_became_peered":"2023-08-28T02:23:59.227342+0000","last_unstale":"2023-08-28T02:37:47.636282+0000","last_undegraded":"2023-08-28T02:37:47.636282+0000","last_fullsized":"2023-08-28T02:37:47.636282+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:08:19.874396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":264,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.640575+0000","last_change":"2023-08-28T02:32:58.779435+0000","last_active":"2023-08-28T02:37:47.640575+0000","last_peered":"2023-08-28T02:37:47.640575+0000","last_clean":"2023-08-28T02:37:47.640575+0000","last_became_active":"2023-08-28T02:32:58.778629+0000","last_became_peered":"2023-08-28T02:32:58.778629+0000","last_unstale":"2023-08-28T02:37:47.640575+0000","last_undegraded":"2023-08-28T02:37:47.640575+0000","last_fullsized":"2023-08-28T02:37:47.640575+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:25:32.193898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":13,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.647879+0000","last_change":"2023-08-28T02:37:45.579563+0000","last_active":"2023-08-28T02:37:47.647879+0000","last_peered":"2023-08-28T02:37:47.647879+0000","last_clean":"2023-08-28T02:37:47.647879+0000","last_became_active":"2023-08-28T02:37:45.579133+0000","last_became_peered":"2023-08-28T02:37:45.579133+0000","last_unstale":"2023-08-28T02:37:47.647879+0000","last_undegraded":"2023-08-28T02:37:47.647879+0000","last_fullsized":"2023-08-28T02:37:47.647879+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":246,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.640715+0000","last_change":"2023-08-28T02:37:45.580277+0000","last_active":"2023-08-28T02:37:47.640715+0000","last_peered":"2023-08-28T02:37:47.640715+0000","last_clean":"2023-08-28T02:37:47.640715+0000","last_became_active":"2023-08-28T02:37:45.579888+0000","last_became_peered":"2023-08-28T02:37:45.579888+0000","last_unstale":"2023-08-28T02:37:47.640715+0000","last_undegraded":"2023-08-28T02:37:47.640715+0000","last_fullsized":"2023-08-28T02:37:47.640715+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:18:54.085955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":19,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.661527+0000","last_change":"2023-08-28T02:37:46.016429+0000","last_active":"2023-08-28T02:37:47.661527+0000","last_peered":"2023-08-28T02:37:47.661527+0000","last_clean":"2023-08-28T02:37:47.661527+0000","last_became_active":"2023-08-28T02:37:45.581897+0000","last_became_peered":"2023-08-28T02:37:45.581897+0000","last_unstale":"2023-08-28T02:37:47.661527+0000","last_undegraded":"2023-08-28T02:37:47.661527+0000","last_fullsized":"2023-08-28T02:37:47.661527+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":79,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.642518+0000","last_change":"2023-08-28T02:35:45.112374+0000","last_active":"2023-08-28T02:37:47.642518+0000","last_peered":"2023-08-28T02:37:47.642518+0000","last_clean":"2023-08-28T02:37:47.642518+0000","last_became_active":"2023-08-28T02:35:45.112024+0000","last_became_peered":"2023-08-28T02:35:45.112024+0000","last_unstale":"2023-08-28T02:37:47.642518+0000","last_undegraded":"2023-08-28T02:37:47.642518+0000","last_fullsized":"2023-08-28T02:37:47.642518+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"86'22","reported_seq":436,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.660956+0000","last_change":"2023-08-28T02:37:45.597318+0000","last_active":"2023-08-28T02:37:47.660956+0000","last_peered":"2023-08-28T02:37:47.660956+0000","last_clean":"2023-08-28T02:37:47.660956+0000","last_became_active":"2023-08-28T02:37:45.578060+0000","last_became_peered":"2023-08-28T02:37:45.578060+0000","last_unstale":"2023-08-28T02:37:47.660956+0000","last_undegraded":"2023-08-28T02:37:47.660956+0000","last_fullsized":"2023-08-28T02:37:47.660956+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":28404,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":391,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.663856+0000","last_change":"2023-08-28T02:32:58.629451+0000","last_active":"2023-08-28T02:37:47.663856+0000","last_peered":"2023-08-28T02:37:47.663856+0000","last_clean":"2023-08-28T02:37:47.663856+0000","last_became_active":"2023-08-28T02:32:58.628767+0000","last_became_peered":"2023-08-28T02:32:58.628767+0000","last_unstale":"2023-08-28T02:37:47.663856+0000","last_undegraded":"2023-08-28T02:37:47.663856+0000","last_fullsized":"2023-08-28T02:37:47.663856+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":19,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.643880+0000","last_change":"2023-08-28T02:37:45.842784+0000","last_active":"2023-08-28T02:37:47.643880+0000","last_peered":"2023-08-28T02:37:47.643880+0000","last_clean":"2023-08-28T02:37:47.643880+0000","last_became_active":"2023-08-28T02:37:45.576303+0000","last_became_peered":"2023-08-28T02:37:45.576303+0000","last_unstale":"2023-08-28T02:37:47.643880+0000","last_undegraded":"2023-08-28T02:37:47.643880+0000","last_fullsized":"2023-08-28T02:37:47.643880+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":264,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.641309+0000","last_change":"2023-08-28T02:37:45.579069+0000","last_active":"2023-08-28T02:37:47.641309+0000","last_peered":"2023-08-28T02:37:47.641309+0000","last_clean":"2023-08-28T02:37:47.641309+0000","last_became_active":"2023-08-28T02:37:45.578728+0000","last_became_peered":"2023-08-28T02:37:45.578728+0000","last_unstale":"2023-08-28T02:37:47.641309+0000","last_undegraded":"2023-08-28T02:37:47.641309+0000","last_fullsized":"2023-08-28T02:37:47.641309+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:58:05.172104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":373,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.637446+0000","last_change":"2023-08-28T02:37:45.579678+0000","last_active":"2023-08-28T02:37:47.637446+0000","last_peered":"2023-08-28T02:37:47.637446+0000","last_clean":"2023-08-28T02:37:47.637446+0000","last_became_active":"2023-08-28T02:37:45.579390+0000","last_became_peered":"2023-08-28T02:37:45.579390+0000","last_unstale":"2023-08-28T02:37:47.637446+0000","last_undegraded":"2023-08-28T02:37:47.637446+0000","last_fullsized":"2023-08-28T02:37:47.637446+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:31:04.974167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.660539+0000","last_change":"2023-08-28T02:37:45.584893+0000","last_active":"2023-08-28T02:37:47.660539+0000","last_peered":"2023-08-28T02:37:47.660539+0000","last_clean":"2023-08-28T02:37:47.660539+0000","last_became_active":"2023-08-28T02:37:45.584568+0000","last_became_peered":"2023-08-28T02:37:45.584568+0000","last_unstale":"2023-08-28T02:37:47.660539+0000","last_undegraded":"2023-08-28T02:37:47.660539+0000","last_fullsized":"2023-08-28T02:37:47.660539+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":13,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.643598+0000","last_change":"2023-08-28T02:37:45.577769+0000","last_active":"2023-08-28T02:37:47.643598+0000","last_peered":"2023-08-28T02:37:47.643598+0000","last_clean":"2023-08-28T02:37:47.643598+0000","last_became_active":"2023-08-28T02:37:45.577446+0000","last_became_peered":"2023-08-28T02:37:45.577446+0000","last_unstale":"2023-08-28T02:37:47.643598+0000","last_undegraded":"2023-08-28T02:37:47.643598+0000","last_fullsized":"2023-08-28T02:37:47.643598+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":355,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.642335+0000","last_change":"2023-08-28T02:37:45.580634+0000","last_active":"2023-08-28T02:37:47.642335+0000","last_peered":"2023-08-28T02:37:47.642335+0000","last_clean":"2023-08-28T02:37:47.642335+0000","last_became_active":"2023-08-28T02:37:45.580322+0000","last_became_peered":"2023-08-28T02:37:45.580322+0000","last_unstale":"2023-08-28T02:37:47.642335+0000","last_undegraded":"2023-08-28T02:37:47.642335+0000","last_fullsized":"2023-08-28T02:37:47.642335+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:24:37.014061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":146,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.639147+0000","last_change":"2023-08-28T02:28:25.703333+0000","last_active":"2023-08-28T02:37:47.639147+0000","last_peered":"2023-08-28T02:37:47.639147+0000","last_clean":"2023-08-28T02:37:47.639147+0000","last_became_active":"2023-08-28T02:28:25.343515+0000","last_became_peered":"2023-08-28T02:28:25.343515+0000","last_unstale":"2023-08-28T02:37:47.639147+0000","last_undegraded":"2023-08-28T02:37:47.639147+0000","last_fullsized":"2023-08-28T02:37:47.639147+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":273,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.642372+0000","last_change":"2023-08-28T02:37:45.579694+0000","last_active":"2023-08-28T02:37:47.642372+0000","last_peered":"2023-08-28T02:37:47.642372+0000","last_clean":"2023-08-28T02:37:47.642372+0000","last_became_active":"2023-08-28T02:37:45.579219+0000","last_became_peered":"2023-08-28T02:37:45.579219+0000","last_unstale":"2023-08-28T02:37:47.642372+0000","last_undegraded":"2023-08-28T02:37:47.642372+0000","last_fullsized":"2023-08-28T02:37:47.642372+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:22:51.406552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"348'8436","reported_seq":8474,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-08-28T02:37:52.035431+0000","last_change":"2023-08-28T02:35:14.499701+0000","last_active":"2023-08-28T02:37:52.035431+0000","last_peered":"2023-08-28T02:37:52.035431+0000","last_clean":"2023-08-28T02:37:52.035431+0000","last_became_active":"2023-08-28T02:35:14.499400+0000","last_became_peered":"2023-08-28T02:35:14.499400+0000","last_unstale":"2023-08-28T02:37:52.035431+0000","last_undegraded":"2023-08-28T02:37:52.035431+0000","last_fullsized":"2023-08-28T02:37:52.035431+0000","mapping_epoch":318,"log_start":"324'100","ondisk_log_start":"324'100","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":8336,"log_dups_size":0,"ondisk_log_size":8336,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239861760,"num_objects":3660,"num_object_clones":0,"num_object_copies":10980,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3660,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8436,"num_write_kb":387072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":17,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.654440+0000","last_change":"2023-08-28T02:37:46.103528+0000","last_active":"2023-08-28T02:37:47.654440+0000","last_peered":"2023-08-28T02:37:47.654440+0000","last_clean":"2023-08-28T02:37:47.654440+0000","last_became_active":"2023-08-28T02:37:45.583986+0000","last_became_peered":"2023-08-28T02:37:45.583986+0000","last_unstale":"2023-08-28T02:37:47.654440+0000","last_undegraded":"2023-08-28T02:37:47.654440+0000","last_fullsized":"2023-08-28T02:37:47.654440+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.648769+0000","last_change":"2023-08-28T02:37:45.585246+0000","last_active":"2023-08-28T02:37:47.648769+0000","last_peered":"2023-08-28T02:37:47.648769+0000","last_clean":"2023-08-28T02:37:47.648769+0000","last_became_active":"2023-08-28T02:37:45.584391+0000","last_became_peered":"2023-08-28T02:37:45.584391+0000","last_unstale":"2023-08-28T02:37:47.648769+0000","last_undegraded":"2023-08-28T02:37:47.648769+0000","last_fullsized":"2023-08-28T02:37:47.648769+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.5","version":"348'8450","reported_seq":8499,"reported_epoch":348,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:37:52.255201+0000","last_change":"2023-08-28T02:37:47.102802+0000","last_active":"2023-08-28T02:37:52.255201+0000","last_peered":"2023-08-28T02:37:52.255201+0000","last_clean":"2023-08-28T02:37:44.558549+0000","last_became_active":"2023-08-28T02:37:47.079792+0000","last_became_peered":"2023-08-28T02:37:47.079792+0000","last_unstale":"2023-08-28T02:37:52.255201+0000","last_undegraded":"2023-08-28T02:37:46.622481+0000","last_fullsized":"2023-08-28T02:37:46.586056+0000","mapping_epoch":345,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":8450,"log_dups_size":0,"ondisk_log_size":8450,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":241696768,"num_objects":3688,"num_object_clones":0,"num_object_copies":11064,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6066,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3688,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8450,"num_write_kb":388416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4],"avail_no_missing":["6","0","4"],"object_location_counts":[{"shards":"0,4,6","objects":3688},{"shards":"4,6","objects":-186}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":17,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.648019+0000","last_change":"2023-08-28T02:37:45.669932+0000","last_active":"2023-08-28T02:37:47.648019+0000","last_peered":"2023-08-28T02:37:47.648019+0000","last_clean":"2023-08-28T02:37:47.648019+0000","last_became_active":"2023-08-28T02:37:45.572825+0000","last_became_peered":"2023-08-28T02:37:45.572825+0000","last_unstale":"2023-08-28T02:37:47.648019+0000","last_undegraded":"2023-08-28T02:37:47.648019+0000","last_fullsized":"2023-08-28T02:37:47.648019+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":401,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.658249+0000","last_change":"2023-08-28T02:32:58.740396+0000","last_active":"2023-08-28T02:37:47.658249+0000","last_peered":"2023-08-28T02:37:47.658249+0000","last_clean":"2023-08-28T02:37:47.658249+0000","last_became_active":"2023-08-28T02:32:58.739941+0000","last_became_peered":"2023-08-28T02:32:58.739941+0000","last_unstale":"2023-08-28T02:37:47.658249+0000","last_undegraded":"2023-08-28T02:37:47.658249+0000","last_fullsized":"2023-08-28T02:37:47.658249+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"348'8676","reported_seq":8726,"reported_epoch":348,"state":"active+remapped+backfill_wait","last_fresh":"2023-08-28T02:37:54.932148+0000","last_change":"2023-08-28T02:37:46.828548+0000","last_active":"2023-08-28T02:37:54.932148+0000","last_peered":"2023-08-28T02:37:54.932148+0000","last_clean":"2023-08-28T02:37:44.550616+0000","last_became_active":"2023-08-28T02:37:46.807673+0000","last_became_peered":"2023-08-28T02:37:46.807673+0000","last_unstale":"2023-08-28T02:37:54.932148+0000","last_undegraded":"2023-08-28T02:37:54.932148+0000","last_fullsized":"2023-08-28T02:37:54.932148+0000","mapping_epoch":345,"log_start":"324'800","ondisk_log_start":"324'800","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":7876,"log_dups_size":0,"ondisk_log_size":7876,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:41:59.898437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235929600,"num_objects":3600,"num_object_clones":0,"num_object_copies":10800,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3600,"num_objects_unfound":0,"num_objects_dirty":3600,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8676,"num_write_kb":392832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,2],"avail_no_missing":["0","2","6"],"object_location_counts":[{"shards":"0,2,6","objects":3600}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":17,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.660489+0000","last_change":"2023-08-28T02:37:46.190365+0000","last_active":"2023-08-28T02:37:47.660489+0000","last_peered":"2023-08-28T02:37:47.660489+0000","last_clean":"2023-08-28T02:37:47.660489+0000","last_became_active":"2023-08-28T02:37:45.584408+0000","last_became_peered":"2023-08-28T02:37:45.584408+0000","last_unstale":"2023-08-28T02:37:47.660489+0000","last_undegraded":"2023-08-28T02:37:47.660489+0000","last_fullsized":"2023-08-28T02:37:47.660489+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":181,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.642448+0000","last_change":"2023-08-28T02:37:45.579428+0000","last_active":"2023-08-28T02:37:47.642448+0000","last_peered":"2023-08-28T02:37:47.642448+0000","last_clean":"2023-08-28T02:37:47.642448+0000","last_became_active":"2023-08-28T02:37:45.579090+0000","last_became_peered":"2023-08-28T02:37:45.579090+0000","last_unstale":"2023-08-28T02:37:47.642448+0000","last_undegraded":"2023-08-28T02:37:47.642448+0000","last_fullsized":"2023-08-28T02:37:47.642448+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"348'8612","reported_seq":8652,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-08-28T02:37:53.444223+0000","last_change":"2023-08-28T02:35:14.498797+0000","last_active":"2023-08-28T02:37:53.444223+0000","last_peered":"2023-08-28T02:37:53.444223+0000","last_clean":"2023-08-28T02:37:53.444223+0000","last_became_active":"2023-08-28T02:35:14.498551+0000","last_became_peered":"2023-08-28T02:35:14.498551+0000","last_unstale":"2023-08-28T02:37:53.444223+0000","last_undegraded":"2023-08-28T02:37:53.444223+0000","last_fullsized":"2023-08-28T02:37:53.444223+0000","mapping_epoch":318,"log_start":"326'1800","ondisk_log_start":"326'1800","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":6812,"log_dups_size":0,"ondisk_log_size":6812,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:43:07.729642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237371392,"num_objects":3622,"num_object_clones":0,"num_object_copies":10866,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3622,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8612,"num_write_kb":391488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":353,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.641129+0000","last_change":"2023-08-28T02:28:25.445982+0000","last_active":"2023-08-28T02:37:47.641129+0000","last_peered":"2023-08-28T02:37:47.641129+0000","last_clean":"2023-08-28T02:37:47.641129+0000","last_became_active":"2023-08-28T02:28:25.443577+0000","last_became_peered":"2023-08-28T02:28:25.443577+0000","last_unstale":"2023-08-28T02:37:47.641129+0000","last_undegraded":"2023-08-28T02:37:47.641129+0000","last_fullsized":"2023-08-28T02:37:47.641129+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:24:03.381751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":349,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.637438+0000","last_change":"2023-08-28T02:32:58.887874+0000","last_active":"2023-08-28T02:37:47.637438+0000","last_peered":"2023-08-28T02:37:47.637438+0000","last_clean":"2023-08-28T02:37:47.637438+0000","last_became_active":"2023-08-28T02:32:58.887377+0000","last_became_peered":"2023-08-28T02:32:58.887377+0000","last_unstale":"2023-08-28T02:37:47.637438+0000","last_undegraded":"2023-08-28T02:37:47.637438+0000","last_fullsized":"2023-08-28T02:37:47.637438+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:08:07.234984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"348'8552","reported_seq":8592,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-08-28T02:37:53.768535+0000","last_change":"2023-08-28T02:35:14.503271+0000","last_active":"2023-08-28T02:37:53.768535+0000","last_peered":"2023-08-28T02:37:53.768535+0000","last_clean":"2023-08-28T02:37:53.768535+0000","last_became_active":"2023-08-28T02:35:14.502940+0000","last_became_peered":"2023-08-28T02:35:14.502940+0000","last_unstale":"2023-08-28T02:37:53.768535+0000","last_undegraded":"2023-08-28T02:37:53.768535+0000","last_fullsized":"2023-08-28T02:37:53.768535+0000","mapping_epoch":318,"log_start":"324'1100","ondisk_log_start":"324'1100","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":7452,"log_dups_size":0,"ondisk_log_size":7452,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:07:49.114269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":230424576,"num_objects":3516,"num_object_clones":0,"num_object_copies":10548,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3516,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8552,"num_write_kb":386176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":215,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.641538+0000","last_change":"2023-08-28T02:23:59.230059+0000","last_active":"2023-08-28T02:37:47.641538+0000","last_peered":"2023-08-28T02:37:47.641538+0000","last_clean":"2023-08-28T02:37:47.641538+0000","last_became_active":"2023-08-28T02:23:59.217783+0000","last_became_peered":"2023-08-28T02:23:59.217783+0000","last_unstale":"2023-08-28T02:37:47.641538+0000","last_undegraded":"2023-08-28T02:37:47.641538+0000","last_fullsized":"2023-08-28T02:37:47.641538+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:00:12.145395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":368,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.660934+0000","last_change":"2023-08-28T02:28:25.351698+0000","last_active":"2023-08-28T02:37:47.660934+0000","last_peered":"2023-08-28T02:37:47.660934+0000","last_clean":"2023-08-28T02:37:47.660934+0000","last_became_active":"2023-08-28T02:28:25.351349+0000","last_became_peered":"2023-08-28T02:28:25.351349+0000","last_unstale":"2023-08-28T02:37:47.660934+0000","last_undegraded":"2023-08-28T02:37:47.660934+0000","last_fullsized":"2023-08-28T02:37:47.660934+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"348'8469","reported_seq":8509,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-08-28T02:37:53.483476+0000","last_change":"2023-08-28T02:35:14.494905+0000","last_active":"2023-08-28T02:37:53.483476+0000","last_peered":"2023-08-28T02:37:53.483476+0000","last_clean":"2023-08-28T02:37:53.483476+0000","last_became_active":"2023-08-28T02:35:14.494589+0000","last_became_peered":"2023-08-28T02:35:14.494589+0000","last_unstale":"2023-08-28T02:37:53.483476+0000","last_undegraded":"2023-08-28T02:37:53.483476+0000","last_fullsized":"2023-08-28T02:37:53.483476+0000","mapping_epoch":318,"log_start":"326'1600","ondisk_log_start":"326'1600","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":6869,"log_dups_size":0,"ondisk_log_size":6869,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:26:28.302635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":231800832,"num_objects":3537,"num_object_clones":0,"num_object_copies":10611,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3537,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8469,"num_write_kb":384192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":154,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.642279+0000","last_change":"2023-08-28T02:32:58.629249+0000","last_active":"2023-08-28T02:37:47.642279+0000","last_peered":"2023-08-28T02:37:47.642279+0000","last_clean":"2023-08-28T02:37:47.642279+0000","last_became_active":"2023-08-28T02:32:58.628583+0000","last_became_peered":"2023-08-28T02:32:58.628583+0000","last_unstale":"2023-08-28T02:37:47.642279+0000","last_undegraded":"2023-08-28T02:37:47.642279+0000","last_fullsized":"2023-08-28T02:37:47.642279+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":421,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.660937+0000","last_change":"2023-08-28T02:35:45.100048+0000","last_active":"2023-08-28T02:37:47.660937+0000","last_peered":"2023-08-28T02:37:47.660937+0000","last_clean":"2023-08-28T02:37:47.660937+0000","last_became_active":"2023-08-28T02:35:45.099690+0000","last_became_peered":"2023-08-28T02:35:45.099690+0000","last_unstale":"2023-08-28T02:37:47.660937+0000","last_undegraded":"2023-08-28T02:37:47.660937+0000","last_fullsized":"2023-08-28T02:37:47.660937+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"348'8497","reported_seq":8535,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-08-28T02:37:52.006058+0000","last_change":"2023-08-28T02:35:14.498910+0000","last_active":"2023-08-28T02:37:52.006058+0000","last_peered":"2023-08-28T02:37:52.006058+0000","last_clean":"2023-08-28T02:37:52.006058+0000","last_became_active":"2023-08-28T02:35:14.498631+0000","last_became_peered":"2023-08-28T02:35:14.498631+0000","last_unstale":"2023-08-28T02:37:52.006058+0000","last_undegraded":"2023-08-28T02:37:52.006058+0000","last_fullsized":"2023-08-28T02:37:52.006058+0000","mapping_epoch":318,"log_start":"324'100","ondisk_log_start":"324'100","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":8397,"log_dups_size":0,"ondisk_log_size":8397,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":238092288,"num_objects":3633,"num_object_clones":0,"num_object_copies":10899,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3633,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8497,"num_write_kb":388160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":293,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.658365+0000","last_change":"2023-08-28T02:37:45.573586+0000","last_active":"2023-08-28T02:37:47.658365+0000","last_peered":"2023-08-28T02:37:47.658365+0000","last_clean":"2023-08-28T02:37:47.658365+0000","last_became_active":"2023-08-28T02:37:45.573239+0000","last_became_peered":"2023-08-28T02:37:45.573239+0000","last_unstale":"2023-08-28T02:37:47.658365+0000","last_undegraded":"2023-08-28T02:37:47.658365+0000","last_fullsized":"2023-08-28T02:37:47.658365+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":336,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.642409+0000","last_change":"2023-08-28T02:32:58.766795+0000","last_active":"2023-08-28T02:37:47.642409+0000","last_peered":"2023-08-28T02:37:47.642409+0000","last_clean":"2023-08-28T02:37:47.642409+0000","last_became_active":"2023-08-28T02:32:58.766441+0000","last_became_peered":"2023-08-28T02:32:58.766441+0000","last_unstale":"2023-08-28T02:37:47.642409+0000","last_undegraded":"2023-08-28T02:37:47.642409+0000","last_fullsized":"2023-08-28T02:37:47.642409+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:58:13.963961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.3","version":"348'8531","reported_seq":8571,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-08-28T02:37:53.428001+0000","last_change":"2023-08-28T02:35:14.498188+0000","last_active":"2023-08-28T02:37:53.428001+0000","last_peered":"2023-08-28T02:37:53.428001+0000","last_clean":"2023-08-28T02:37:53.428001+0000","last_became_active":"2023-08-28T02:35:14.496278+0000","last_became_peered":"2023-08-28T02:35:14.496278+0000","last_unstale":"2023-08-28T02:37:53.428001+0000","last_undegraded":"2023-08-28T02:37:53.428001+0000","last_fullsized":"2023-08-28T02:37:53.428001+0000","mapping_epoch":318,"log_start":"326'1700","ondisk_log_start":"326'1700","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":6831,"log_dups_size":0,"ondisk_log_size":6831,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:43:23.633921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237305856,"num_objects":3621,"num_object_clones":0,"num_object_copies":10863,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3621,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8531,"num_write_kb":388864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":265,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.640715+0000","last_change":"2023-08-28T02:37:45.579112+0000","last_active":"2023-08-28T02:37:47.640715+0000","last_peered":"2023-08-28T02:37:47.640715+0000","last_clean":"2023-08-28T02:37:47.640715+0000","last_became_active":"2023-08-28T02:37:45.578787+0000","last_became_peered":"2023-08-28T02:37:45.578787+0000","last_unstale":"2023-08-28T02:37:47.640715+0000","last_undegraded":"2023-08-28T02:37:47.640715+0000","last_fullsized":"2023-08-28T02:37:47.640715+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:20:59.994447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":69,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.644456+0000","last_change":"2023-08-28T02:32:58.755765+0000","last_active":"2023-08-28T02:37:47.644456+0000","last_peered":"2023-08-28T02:37:47.644456+0000","last_clean":"2023-08-28T02:37:47.644456+0000","last_became_active":"2023-08-28T02:32:58.755438+0000","last_became_peered":"2023-08-28T02:32:58.755438+0000","last_unstale":"2023-08-28T02:37:47.644456+0000","last_undegraded":"2023-08-28T02:37:47.644456+0000","last_fullsized":"2023-08-28T02:37:47.644456+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:14:11.498611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"348'8546","reported_seq":8585,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-08-28T02:37:54.936938+0000","last_change":"2023-08-28T02:35:14.497511+0000","last_active":"2023-08-28T02:37:54.936938+0000","last_peered":"2023-08-28T02:37:54.936938+0000","last_clean":"2023-08-28T02:37:54.936938+0000","last_became_active":"2023-08-28T02:35:14.497180+0000","last_became_peered":"2023-08-28T02:35:14.497180+0000","last_unstale":"2023-08-28T02:37:54.936938+0000","last_undegraded":"2023-08-28T02:37:54.936938+0000","last_fullsized":"2023-08-28T02:37:54.936938+0000","mapping_epoch":318,"log_start":"324'700","ondisk_log_start":"324'700","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":7846,"log_dups_size":0,"ondisk_log_size":7846,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:59:26.376998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236978176,"num_objects":3616,"num_object_clones":0,"num_object_copies":10848,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3616,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8546,"num_write_kb":389184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"78'126","reported_seq":794,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-08-28T02:37:48.647489+0000","last_change":"2023-08-28T02:37:48.647489+0000","last_active":"2023-08-28T02:37:48.647489+0000","last_peered":"2023-08-28T02:37:48.647489+0000","last_clean":"2023-08-28T02:37:48.647489+0000","last_became_active":"2023-08-28T02:37:48.647002+0000","last_became_peered":"2023-08-28T02:37:48.647002+0000","last_unstale":"2023-08-28T02:37:48.647489+0000","last_undegraded":"2023-08-28T02:37:48.647489+0000","last_fullsized":"2023-08-28T02:37:48.647489+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:04:10.819878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":743,"num_read_kb":1331,"num_write":125,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":190,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.641131+0000","last_change":"2023-08-28T02:35:05.991133+0000","last_active":"2023-08-28T02:37:47.641131+0000","last_peered":"2023-08-28T02:37:47.641131+0000","last_clean":"2023-08-28T02:37:47.641131+0000","last_became_active":"2023-08-28T02:35:05.990829+0000","last_became_peered":"2023-08-28T02:35:05.990829+0000","last_unstale":"2023-08-28T02:37:47.641131+0000","last_undegraded":"2023-08-28T02:37:47.641131+0000","last_fullsized":"2023-08-28T02:37:47.641131+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:40:28.077023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":31,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-08-28T02:37:48.640672+0000","last_change":"2023-08-28T02:35:45.099619+0000","last_active":"2023-08-28T02:37:48.640672+0000","last_peered":"2023-08-28T02:37:48.640672+0000","last_clean":"2023-08-28T02:37:48.640672+0000","last_became_active":"2023-08-28T02:35:45.099285+0000","last_became_peered":"2023-08-28T02:35:45.099285+0000","last_unstale":"2023-08-28T02:37:48.640672+0000","last_undegraded":"2023-08-28T02:37:48.640672+0000","last_fullsized":"2023-08-28T02:37:48.640672+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:24:58.305957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.8","version":"348'8706","reported_seq":8891,"reported_epoch":348,"state":"active+remapped+backfill_wait","last_fresh":"2023-08-28T02:37:53.792718+0000","last_change":"2023-08-28T02:37:47.073785+0000","last_active":"2023-08-28T02:37:53.792718+0000","last_peered":"2023-08-28T02:37:53.792718+0000","last_clean":"2023-08-28T02:37:44.530880+0000","last_became_active":"2023-08-28T02:37:46.790659+0000","last_became_peered":"2023-08-28T02:37:46.790659+0000","last_unstale":"2023-08-28T02:37:53.792718+0000","last_undegraded":"2023-08-28T02:37:53.792718+0000","last_fullsized":"2023-08-28T02:37:53.792718+0000","mapping_epoch":345,"log_start":"324'1200","ondisk_log_start":"324'1200","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":7506,"log_dups_size":0,"ondisk_log_size":7506,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:37:46.944577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":233177088,"num_objects":3558,"num_object_clones":0,"num_object_copies":10674,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3496,"num_objects_unfound":0,"num_objects_dirty":3558,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8706,"num_write_kb":392448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":66,"num_bytes_recovered":4325376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,0],"avail_no_missing":["3","0","5"],"object_location_counts":[{"shards":"0,3,5","objects":3558}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"78'204","reported_seq":1433,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.639523+0000","last_change":"2023-08-28T02:32:59.644574+0000","last_active":"2023-08-28T02:37:47.639523+0000","last_peered":"2023-08-28T02:37:47.639523+0000","last_clean":"2023-08-28T02:37:47.639523+0000","last_became_active":"2023-08-28T02:32:59.644053+0000","last_became_peered":"2023-08-28T02:32:59.644053+0000","last_unstale":"2023-08-28T02:37:47.639523+0000","last_undegraded":"2023-08-28T02:37:47.639523+0000","last_fullsized":"2023-08-28T02:37:47.639523+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":204,"log_dups_size":0,"ondisk_log_size":204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:55:51.460483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1521,"num_read_kb":2511,"num_write":178,"num_write_kb":1175,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":342,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.663948+0000","last_change":"2023-08-28T02:28:25.352417+0000","last_active":"2023-08-28T02:37:47.663948+0000","last_peered":"2023-08-28T02:37:47.663948+0000","last_clean":"2023-08-28T02:37:47.663948+0000","last_became_active":"2023-08-28T02:28:25.352216+0000","last_became_peered":"2023-08-28T02:28:25.352216+0000","last_unstale":"2023-08-28T02:37:47.663948+0000","last_undegraded":"2023-08-28T02:37:47.663948+0000","last_fullsized":"2023-08-28T02:37:47.663948+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":69,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.643094+0000","last_change":"2023-08-28T02:32:58.720576+0000","last_active":"2023-08-28T02:37:47.643094+0000","last_peered":"2023-08-28T02:37:47.643094+0000","last_clean":"2023-08-28T02:37:47.643094+0000","last_became_active":"2023-08-28T02:32:58.719924+0000","last_became_peered":"2023-08-28T02:32:58.719924+0000","last_unstale":"2023-08-28T02:37:47.643094+0000","last_undegraded":"2023-08-28T02:37:47.643094+0000","last_fullsized":"2023-08-28T02:37:47.643094+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:28:50.697438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"348'8488","reported_seq":8528,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-08-28T02:37:54.011120+0000","last_change":"2023-08-28T02:35:14.503279+0000","last_active":"2023-08-28T02:37:54.011120+0000","last_peered":"2023-08-28T02:37:54.011120+0000","last_clean":"2023-08-28T02:37:54.011120+0000","last_became_active":"2023-08-28T02:35:14.502953+0000","last_became_peered":"2023-08-28T02:35:14.502953+0000","last_unstale":"2023-08-28T02:37:54.011120+0000","last_undegraded":"2023-08-28T02:37:54.011120+0000","last_fullsized":"2023-08-28T02:37:54.011120+0000","mapping_epoch":318,"log_start":"324'1100","ondisk_log_start":"324'1100","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":7388,"log_dups_size":0,"ondisk_log_size":7388,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:55:29.284519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239599616,"num_objects":3656,"num_object_clones":0,"num_object_copies":10968,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3656,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8488,"num_write_kb":388608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"78'47","reported_seq":568,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.633114+0000","last_change":"2023-08-28T02:24:05.799647+0000","last_active":"2023-08-28T02:37:47.633114+0000","last_peered":"2023-08-28T02:37:47.633114+0000","last_clean":"2023-08-28T02:37:47.633114+0000","last_became_active":"2023-08-28T02:23:59.210481+0000","last_became_peered":"2023-08-28T02:23:59.210481+0000","last_unstale":"2023-08-28T02:37:47.633114+0000","last_undegraded":"2023-08-28T02:37:47.633114+0000","last_fullsized":"2023-08-28T02:37:47.633114+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":47,"log_dups_size":0,"ondisk_log_size":47,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:40:18.542375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":162,"num_read_kb":1281,"num_write":48,"num_write_kb":1153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":69,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.638249+0000","last_change":"2023-08-28T02:32:58.622079+0000","last_active":"2023-08-28T02:37:47.638249+0000","last_peered":"2023-08-28T02:37:47.638249+0000","last_clean":"2023-08-28T02:37:47.638249+0000","last_became_active":"2023-08-28T02:32:58.621819+0000","last_became_peered":"2023-08-28T02:32:58.621819+0000","last_unstale":"2023-08-28T02:37:47.638249+0000","last_undegraded":"2023-08-28T02:37:47.638249+0000","last_fullsized":"2023-08-28T02:37:47.638249+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:03:36.676500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":69,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.644536+0000","last_change":"2023-08-28T02:32:58.724424+0000","last_active":"2023-08-28T02:37:47.644536+0000","last_peered":"2023-08-28T02:37:47.644536+0000","last_clean":"2023-08-28T02:37:47.644536+0000","last_became_active":"2023-08-28T02:32:58.724127+0000","last_became_peered":"2023-08-28T02:32:58.724127+0000","last_unstale":"2023-08-28T02:37:47.644536+0000","last_undegraded":"2023-08-28T02:37:47.644536+0000","last_fullsized":"2023-08-28T02:37:47.644536+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:02:25.178712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"348'8544","reported_seq":8582,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-08-28T02:37:52.200305+0000","last_change":"2023-08-28T02:35:14.500884+0000","last_active":"2023-08-28T02:37:52.200305+0000","last_peered":"2023-08-28T02:37:52.200305+0000","last_clean":"2023-08-28T02:37:52.200305+0000","last_became_active":"2023-08-28T02:35:14.500643+0000","last_became_peered":"2023-08-28T02:35:14.500643+0000","last_unstale":"2023-08-28T02:37:52.200305+0000","last_undegraded":"2023-08-28T02:37:52.200305+0000","last_fullsized":"2023-08-28T02:37:52.200305+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":8544,"log_dups_size":0,"ondisk_log_size":8544,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243400704,"num_objects":3714,"num_object_clones":0,"num_object_copies":11142,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3714,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8544,"num_write_kb":392256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"78'114","reported_seq":435,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-08-28T02:37:48.667554+0000","last_change":"2023-08-28T02:37:48.667554+0000","last_active":"2023-08-28T02:37:48.667554+0000","last_peered":"2023-08-28T02:37:48.667554+0000","last_clean":"2023-08-28T02:37:48.667554+0000","last_became_active":"2023-08-28T02:37:48.666659+0000","last_became_peered":"2023-08-28T02:37:48.666659+0000","last_unstale":"2023-08-28T02:37:48.667554+0000","last_undegraded":"2023-08-28T02:37:48.667554+0000","last_fullsized":"2023-08-28T02:37:48.667554+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":114,"log_dups_size":0,"ondisk_log_size":114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:47:57.511598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":358,"num_read_kb":1352,"num_write":102,"num_write_kb":1104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"86'139","reported_seq":442,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-08-28T02:37:48.651113+0000","last_change":"2023-08-28T02:37:48.651113+0000","last_active":"2023-08-28T02:37:48.651113+0000","last_peered":"2023-08-28T02:37:48.651113+0000","last_clean":"2023-08-28T02:37:48.651113+0000","last_became_active":"2023-08-28T02:37:48.650691+0000","last_became_peered":"2023-08-28T02:37:48.650691+0000","last_unstale":"2023-08-28T02:37:48.651113+0000","last_undegraded":"2023-08-28T02:37:48.651113+0000","last_fullsized":"2023-08-28T02:37:48.651113+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_clean_scrub_stamp":"2023-08-28T02:08:02.043718+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":229,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.640857+0000","last_change":"2023-08-28T02:28:25.347600+0000","last_active":"2023-08-28T02:37:47.640857+0000","last_peered":"2023-08-28T02:37:47.640857+0000","last_clean":"2023-08-28T02:37:47.640857+0000","last_became_active":"2023-08-28T02:28:25.347237+0000","last_became_peered":"2023-08-28T02:28:25.347237+0000","last_unstale":"2023-08-28T02:37:47.640857+0000","last_undegraded":"2023-08-28T02:37:47.640857+0000","last_fullsized":"2023-08-28T02:37:47.640857+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:29:26.767559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":154,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.642355+0000","last_change":"2023-08-28T02:37:45.581261+0000","last_active":"2023-08-28T02:37:47.642355+0000","last_peered":"2023-08-28T02:37:47.642355+0000","last_clean":"2023-08-28T02:37:47.642355+0000","last_became_active":"2023-08-28T02:37:45.580927+0000","last_became_peered":"2023-08-28T02:37:45.580927+0000","last_unstale":"2023-08-28T02:37:47.642355+0000","last_undegraded":"2023-08-28T02:37:47.642355+0000","last_fullsized":"2023-08-28T02:37:47.642355+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"348'8393","reported_seq":8433,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-08-28T02:37:53.467593+0000","last_change":"2023-08-28T02:35:14.493898+0000","last_active":"2023-08-28T02:37:53.467593+0000","last_peered":"2023-08-28T02:37:53.467593+0000","last_clean":"2023-08-28T02:37:53.467593+0000","last_became_active":"2023-08-28T02:35:14.493649+0000","last_became_peered":"2023-08-28T02:35:14.493649+0000","last_unstale":"2023-08-28T02:37:53.467593+0000","last_undegraded":"2023-08-28T02:37:53.467593+0000","last_fullsized":"2023-08-28T02:37:53.467593+0000","mapping_epoch":318,"log_start":"326'1600","ondisk_log_start":"326'1600","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":6793,"log_dups_size":0,"ondisk_log_size":6793,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:49:46.405255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":233766912,"num_objects":3567,"num_object_clones":0,"num_object_copies":10701,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3567,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8393,"num_write_kb":382720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'148","reported_seq":666,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-08-28T02:37:48.650996+0000","last_change":"2023-08-28T02:37:48.650996+0000","last_active":"2023-08-28T02:37:48.650996+0000","last_peered":"2023-08-28T02:37:48.650996+0000","last_clean":"2023-08-28T02:37:48.650996+0000","last_became_active":"2023-08-28T02:37:48.650016+0000","last_became_peered":"2023-08-28T02:37:48.650016+0000","last_unstale":"2023-08-28T02:37:48.650996+0000","last_undegraded":"2023-08-28T02:37:48.650996+0000","last_fullsized":"2023-08-28T02:37:48.650996+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1092,"num_read_kb":1955,"num_write":127,"num_write_kb":5177,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":320,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.637802+0000","last_change":"2023-08-28T02:23:59.226986+0000","last_active":"2023-08-28T02:37:47.637802+0000","last_peered":"2023-08-28T02:37:47.637802+0000","last_clean":"2023-08-28T02:37:47.637802+0000","last_became_active":"2023-08-28T02:23:59.226529+0000","last_became_peered":"2023-08-28T02:23:59.226529+0000","last_unstale":"2023-08-28T02:37:47.637802+0000","last_undegraded":"2023-08-28T02:37:47.637802+0000","last_fullsized":"2023-08-28T02:37:47.637802+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:22:09.193619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":382,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.642126+0000","last_change":"2023-08-28T02:32:58.744332+0000","last_active":"2023-08-28T02:37:47.642126+0000","last_peered":"2023-08-28T02:37:47.642126+0000","last_clean":"2023-08-28T02:37:47.642126+0000","last_became_active":"2023-08-28T02:32:58.744025+0000","last_became_peered":"2023-08-28T02:32:58.744025+0000","last_unstale":"2023-08-28T02:37:47.642126+0000","last_undegraded":"2023-08-28T02:37:47.642126+0000","last_fullsized":"2023-08-28T02:37:47.642126+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:41:15.957049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"348'8596","reported_seq":8794,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-08-28T02:37:54.025025+0000","last_change":"2023-08-28T02:35:24.994481+0000","last_active":"2023-08-28T02:37:54.025025+0000","last_peered":"2023-08-28T02:37:54.025025+0000","last_clean":"2023-08-28T02:37:54.025025+0000","last_became_active":"2023-08-28T02:35:24.668942+0000","last_became_peered":"2023-08-28T02:35:24.668942+0000","last_unstale":"2023-08-28T02:37:54.025025+0000","last_undegraded":"2023-08-28T02:37:54.025025+0000","last_fullsized":"2023-08-28T02:37:54.025025+0000","mapping_epoch":323,"log_start":"325'1200","ondisk_log_start":"325'1200","created":318,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":7396,"log_dups_size":0,"ondisk_log_size":7396,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:16:02.211038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237895680,"num_objects":3630,"num_object_clones":0,"num_object_copies":10890,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3630,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8596,"num_write_kb":391232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":73,"num_bytes_recovered":4784128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"78'103","reported_seq":688,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-08-28T02:37:48.651919+0000","last_change":"2023-08-28T02:37:48.651919+0000","last_active":"2023-08-28T02:37:48.651919+0000","last_peered":"2023-08-28T02:37:48.651919+0000","last_clean":"2023-08-28T02:37:48.651919+0000","last_became_active":"2023-08-28T02:37:48.651437+0000","last_became_peered":"2023-08-28T02:37:48.651437+0000","last_unstale":"2023-08-28T02:37:48.651919+0000","last_undegraded":"2023-08-28T02:37:48.651919+0000","last_fullsized":"2023-08-28T02:37:48.651919+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":803,"num_read_kb":2080,"num_write":73,"num_write_kb":1244,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":69,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.643029+0000","last_change":"2023-08-28T02:32:58.720401+0000","last_active":"2023-08-28T02:37:47.643029+0000","last_peered":"2023-08-28T02:37:47.643029+0000","last_clean":"2023-08-28T02:37:47.643029+0000","last_became_active":"2023-08-28T02:32:58.719518+0000","last_became_peered":"2023-08-28T02:32:58.719518+0000","last_unstale":"2023-08-28T02:37:47.643029+0000","last_undegraded":"2023-08-28T02:37:47.643029+0000","last_fullsized":"2023-08-28T02:37:47.643029+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:46:28.947812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":342,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.637690+0000","last_change":"2023-08-28T02:37:45.579740+0000","last_active":"2023-08-28T02:37:47.637690+0000","last_peered":"2023-08-28T02:37:47.637690+0000","last_clean":"2023-08-28T02:37:47.637690+0000","last_became_active":"2023-08-28T02:37:45.579412+0000","last_became_peered":"2023-08-28T02:37:45.579412+0000","last_unstale":"2023-08-28T02:37:47.637690+0000","last_undegraded":"2023-08-28T02:37:47.637690+0000","last_fullsized":"2023-08-28T02:37:47.637690+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:31:18.483532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"348'8269","reported_seq":8345,"reported_epoch":348,"state":"active+remapped+backfill_wait","last_fresh":"2023-08-28T02:37:54.876376+0000","last_change":"2023-08-28T02:37:47.122442+0000","last_active":"2023-08-28T02:37:54.876376+0000","last_peered":"2023-08-28T02:37:54.876376+0000","last_clean":"2023-08-28T02:37:44.513990+0000","last_became_active":"2023-08-28T02:37:46.801002+0000","last_became_peered":"2023-08-28T02:37:46.801002+0000","last_unstale":"2023-08-28T02:37:54.876376+0000","last_undegraded":"2023-08-28T02:37:54.876376+0000","last_fullsized":"2023-08-28T02:37:54.876376+0000","mapping_epoch":345,"log_start":"324'400","ondisk_log_start":"324'400","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":7869,"log_dups_size":0,"ondisk_log_size":7869,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:51:20.134517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228524032,"num_objects":3487,"num_object_clones":0,"num_object_copies":10461,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3476,"num_objects_unfound":0,"num_objects_dirty":3487,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8269,"num_write_kb":376192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":786432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,5],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,3,5","objects":3487}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":241,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.642459+0000","last_change":"2023-08-28T02:37:45.590339+0000","last_active":"2023-08-28T02:37:47.642459+0000","last_peered":"2023-08-28T02:37:47.642459+0000","last_clean":"2023-08-28T02:37:47.642459+0000","last_became_active":"2023-08-28T02:37:45.578751+0000","last_became_peered":"2023-08-28T02:37:45.578751+0000","last_unstale":"2023-08-28T02:37:47.642459+0000","last_undegraded":"2023-08-28T02:37:47.642459+0000","last_fullsized":"2023-08-28T02:37:47.642459+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:27:24.226372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'112","reported_seq":502,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.642756+0000","last_change":"2023-08-28T02:32:59.643465+0000","last_active":"2023-08-28T02:37:47.642756+0000","last_peered":"2023-08-28T02:37:47.642756+0000","last_clean":"2023-08-28T02:37:47.642756+0000","last_became_active":"2023-08-28T02:32:59.642979+0000","last_became_peered":"2023-08-28T02:32:59.642979+0000","last_unstale":"2023-08-28T02:37:47.642756+0000","last_undegraded":"2023-08-28T02:37:47.642756+0000","last_fullsized":"2023-08-28T02:37:47.642756+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:48:45.923015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":444,"num_read_kb":473,"num_write":94,"num_write_kb":127,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":247,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.640641+0000","last_change":"2023-08-28T02:37:45.578790+0000","last_active":"2023-08-28T02:37:47.640641+0000","last_peered":"2023-08-28T02:37:47.640641+0000","last_clean":"2023-08-28T02:37:47.640641+0000","last_became_active":"2023-08-28T02:37:45.578473+0000","last_became_peered":"2023-08-28T02:37:45.578473+0000","last_unstale":"2023-08-28T02:37:47.640641+0000","last_undegraded":"2023-08-28T02:37:47.640641+0000","last_fullsized":"2023-08-28T02:37:47.640641+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:23:51.722772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"348'8429","reported_seq":8872,"reported_epoch":348,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-08-28T02:37:55.129061+0000","last_change":"2023-08-28T02:37:47.146950+0000","last_active":"2023-08-28T02:37:55.129061+0000","last_peered":"2023-08-28T02:37:55.129061+0000","last_clean":"2023-08-28T02:37:44.560462+0000","last_became_active":"2023-08-28T02:37:47.075781+0000","last_became_peered":"2023-08-28T02:37:47.075781+0000","last_unstale":"2023-08-28T02:37:55.129061+0000","last_undegraded":"2023-08-28T02:37:46.612092+0000","last_fullsized":"2023-08-28T02:37:46.565881+0000","mapping_epoch":345,"log_start":"0'0","ondisk_log_start":"0'0","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":8429,"log_dups_size":0,"ondisk_log_size":8429,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T14:09:04.679007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":230424600,"num_objects":3517,"num_object_clones":0,"num_object_copies":10551,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":125,"num_objects_misplaced":3510,"num_objects_unfound":0,"num_objects_dirty":3517,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8429,"num_write_kb":382209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":199,"num_bytes_recovered":12976128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[2,5],"avail_no_missing":["2","5","6"],"object_location_counts":[{"shards":"2,5","objects":-118},{"shards":"2,5,6","objects":3510}],"blocked_by":[],"up_primary":7,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"78'122","reported_seq":728,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-08-28T02:37:48.652349+0000","last_change":"2023-08-28T02:37:48.652349+0000","last_active":"2023-08-28T02:37:48.652349+0000","last_peered":"2023-08-28T02:37:48.652349+0000","last_clean":"2023-08-28T02:37:48.652349+0000","last_became_active":"2023-08-28T02:37:48.651826+0000","last_became_peered":"2023-08-28T02:37:48.651826+0000","last_unstale":"2023-08-28T02:37:48.652349+0000","last_undegraded":"2023-08-28T02:37:48.652349+0000","last_fullsized":"2023-08-28T02:37:48.652349+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":857,"num_read_kb":2224,"num_write":99,"num_write_kb":2312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":357,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.642231+0000","last_change":"2023-08-28T02:32:58.795995+0000","last_active":"2023-08-28T02:37:47.642231+0000","last_peered":"2023-08-28T02:37:47.642231+0000","last_clean":"2023-08-28T02:37:47.642231+0000","last_became_active":"2023-08-28T02:32:58.771872+0000","last_became_peered":"2023-08-28T02:32:58.771872+0000","last_unstale":"2023-08-28T02:37:47.642231+0000","last_undegraded":"2023-08-28T02:37:47.642231+0000","last_fullsized":"2023-08-28T02:37:47.642231+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:16:02.965467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":317,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.658065+0000","last_change":"2023-08-28T02:28:25.351840+0000","last_active":"2023-08-28T02:37:47.658065+0000","last_peered":"2023-08-28T02:37:47.658065+0000","last_clean":"2023-08-28T02:37:47.658065+0000","last_became_active":"2023-08-28T02:28:25.351399+0000","last_became_peered":"2023-08-28T02:28:25.351399+0000","last_unstale":"2023-08-28T02:37:47.658065+0000","last_undegraded":"2023-08-28T02:37:47.658065+0000","last_fullsized":"2023-08-28T02:37:47.658065+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":343,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.637598+0000","last_change":"2023-08-28T02:37:45.581817+0000","last_active":"2023-08-28T02:37:47.637598+0000","last_peered":"2023-08-28T02:37:47.637598+0000","last_clean":"2023-08-28T02:37:47.637598+0000","last_became_active":"2023-08-28T02:37:45.581475+0000","last_became_peered":"2023-08-28T02:37:45.581475+0000","last_unstale":"2023-08-28T02:37:47.637598+0000","last_undegraded":"2023-08-28T02:37:47.637598+0000","last_fullsized":"2023-08-28T02:37:47.637598+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:00:31.628265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":36,"reported_epoch":348,"state":"active+clean","last_fresh":"2023-08-28T02:37:48.642372+0000","last_change":"2023-08-28T02:35:24.666417+0000","last_active":"2023-08-28T02:37:48.642372+0000","last_peered":"2023-08-28T02:37:48.642372+0000","last_clean":"2023-08-28T02:37:48.642372+0000","last_became_active":"2023-08-28T02:35:24.666107+0000","last_became_peered":"2023-08-28T02:35:24.666107+0000","last_unstale":"2023-08-28T02:37:48.642372+0000","last_undegraded":"2023-08-28T02:37:48.642372+0000","last_fullsized":"2023-08-28T02:37:48.642372+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:22:55.497120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":180,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.660857+0000","last_change":"2023-08-28T02:32:58.839847+0000","last_active":"2023-08-28T02:37:47.660857+0000","last_peered":"2023-08-28T02:37:47.660857+0000","last_clean":"2023-08-28T02:37:47.660857+0000","last_became_active":"2023-08-28T02:32:58.628159+0000","last_became_peered":"2023-08-28T02:32:58.628159+0000","last_unstale":"2023-08-28T02:37:47.660857+0000","last_undegraded":"2023-08-28T02:37:47.660857+0000","last_fullsized":"2023-08-28T02:37:47.660857+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":383,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.664023+0000","last_change":"2023-08-28T02:32:58.629596+0000","last_active":"2023-08-28T02:37:47.664023+0000","last_peered":"2023-08-28T02:37:47.664023+0000","last_clean":"2023-08-28T02:37:47.664023+0000","last_became_active":"2023-08-28T02:32:58.629136+0000","last_became_peered":"2023-08-28T02:32:58.629136+0000","last_unstale":"2023-08-28T02:37:47.664023+0000","last_undegraded":"2023-08-28T02:37:47.664023+0000","last_fullsized":"2023-08-28T02:37:47.664023+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":167,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.661837+0000","last_change":"2023-08-28T02:28:25.705331+0000","last_active":"2023-08-28T02:37:47.661837+0000","last_peered":"2023-08-28T02:37:47.661837+0000","last_clean":"2023-08-28T02:37:47.661837+0000","last_became_active":"2023-08-28T02:28:25.397480+0000","last_became_peered":"2023-08-28T02:28:25.397480+0000","last_unstale":"2023-08-28T02:37:47.661837+0000","last_undegraded":"2023-08-28T02:37:47.661837+0000","last_fullsized":"2023-08-28T02:37:47.661837+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":347,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.661178+0000","last_change":"2023-08-28T02:23:59.207929+0000","last_active":"2023-08-28T02:37:47.661178+0000","last_peered":"2023-08-28T02:37:47.661178+0000","last_clean":"2023-08-28T02:37:47.661178+0000","last_became_active":"2023-08-28T02:23:59.207589+0000","last_became_peered":"2023-08-28T02:23:59.207589+0000","last_unstale":"2023-08-28T02:37:47.661178+0000","last_undegraded":"2023-08-28T02:37:47.661178+0000","last_fullsized":"2023-08-28T02:37:47.661178+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":238,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.641310+0000","last_change":"2023-08-28T02:28:25.347988+0000","last_active":"2023-08-28T02:37:47.641310+0000","last_peered":"2023-08-28T02:37:47.641310+0000","last_clean":"2023-08-28T02:37:47.641310+0000","last_became_active":"2023-08-28T02:28:25.347632+0000","last_became_peered":"2023-08-28T02:28:25.347632+0000","last_unstale":"2023-08-28T02:37:47.641310+0000","last_undegraded":"2023-08-28T02:37:47.641310+0000","last_fullsized":"2023-08-28T02:37:47.641310+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:56:20.202662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":210,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.640780+0000","last_change":"2023-08-28T02:23:59.217794+0000","last_active":"2023-08-28T02:37:47.640780+0000","last_peered":"2023-08-28T02:37:47.640780+0000","last_clean":"2023-08-28T02:37:47.640780+0000","last_became_active":"2023-08-28T02:23:59.217231+0000","last_became_peered":"2023-08-28T02:23:59.217231+0000","last_unstale":"2023-08-28T02:37:47.640780+0000","last_undegraded":"2023-08-28T02:37:47.640780+0000","last_fullsized":"2023-08-28T02:37:47.640780+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:36:26.297410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":336,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.636371+0000","last_change":"2023-08-28T02:35:24.680274+0000","last_active":"2023-08-28T02:37:47.636371+0000","last_peered":"2023-08-28T02:37:47.636371+0000","last_clean":"2023-08-28T02:37:47.636371+0000","last_became_active":"2023-08-28T02:35:24.673318+0000","last_became_peered":"2023-08-28T02:35:24.673318+0000","last_unstale":"2023-08-28T02:37:47.636371+0000","last_undegraded":"2023-08-28T02:37:47.636371+0000","last_fullsized":"2023-08-28T02:37:47.636371+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:22:46.371303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":406,"reported_epoch":347,"state":"active+clean","last_fresh":"2023-08-28T02:37:47.658034+0000","last_change":"2023-08-28T02:32:58.630270+0000","last_active":"2023-08-28T02:37:47.658034+0000","last_peered":"2023-08-28T02:37:47.658034+0000","last_clean":"2023-08-28T02:37:47.658034+0000","last_became_active":"2023-08-28T02:32:58.629958+0000","last_became_peered":"2023-08-28T02:32:58.629958+0000","last_unstale":"2023-08-28T02:37:47.658034+0000","last_undegraded":"2023-08-28T02:37:47.658034+0000","last_fullsized":"2023-08-28T02:37:47.658034+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":29888,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35337,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":3776249880,"num_objects":57622,"num_object_clones":0,"num_object_copies":172866,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6191,"num_objects_misplaced":14082,"num_objects_unfound":0,"num_objects_dirty":57622,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":136194,"num_write_kb":6202049,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":353,"num_bytes_recovered":22872064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11391283200,"data_stored":11391270984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":122794,"ondisk_log_size":122794,"up":48,"acting":46,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":191,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":311,"seq":1335734829093,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":339964,"kb_used_data":12916,"kb_used_omap":8,"kb_used_meta":327031,"kb_avail":93405188,"statfs":{"total":95995035648,"available":95646912512,"internally_reserved":0,"allocated":13225984,"data_stored":12328254,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8527,"internal_metadata":334880433},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,1,2,2,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:37:01 2023","interfaces":[{"interface":"back","average":{"1min":14.039,"5min":4.0199999999999996,"15min":2.3500000000000001},"min":{"1min":0.25700000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":140.35499999999999,"5min":140.35499999999999,"15min":140.35499999999999},"last":7.694},{"interface":"front","average":{"1min":13.738,"5min":3.798,"15min":2.141},"min":{"1min":0.29799999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":140.40199999999999,"5min":140.40199999999999,"15min":140.40199999999999},"last":2.0840000000000001}]},{"osd":1,"last update":"Mon Aug 28 02:37:01 2023","interfaces":[{"interface":"back","average":{"1min":17.434000000000001,"5min":8.4629999999999992,"15min":6.968},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":151.048,"5min":151.048,"15min":151.048},"last":7.7590000000000003},{"interface":"front","average":{"1min":17.559999999999999,"5min":8.3610000000000007,"15min":6.8280000000000003},"min":{"1min":0.35399999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":150.56299999999999,"5min":150.56299999999999,"15min":150.56299999999999},"last":2.1110000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:37:01 2023","interfaces":[{"interface":"back","average":{"1min":2.7000000000000002,"5min":3.496,"15min":3.629},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":18.492999999999999,"5min":46.042999999999999,"15min":46.042999999999999},"last":1.7230000000000001},{"interface":"front","average":{"1min":1.911,"5min":3.2650000000000001,"15min":3.4910000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":18.468,"5min":46.020000000000003,"15min":46.020000000000003},"last":7.7880000000000003}]},{"osd":3,"last update":"Mon Aug 28 02:37:01 2023","interfaces":[{"interface":"back","average":{"1min":7.5490000000000004,"5min":4.9180000000000001,"15min":4.4800000000000004},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":129.20099999999999,"5min":129.20099999999999,"15min":129.20099999999999},"last":1.6930000000000001},{"interface":"front","average":{"1min":8.3499999999999996,"5min":5.6360000000000001,"15min":5.1840000000000002},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":129.512,"5min":129.512,"15min":129.512},"last":7.7350000000000003}]},{"osd":4,"last update":"Mon Aug 28 02:37:01 2023","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.92800000000000005,"15min":1.02},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.73899999999999999,"5min":14.984999999999999,"15min":14.984999999999999},"last":2.137},{"interface":"front","average":{"1min":0.39400000000000002,"5min":1.079,"15min":1.1930000000000001},"min":{"1min":0.30299999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.63100000000000001,"5min":15.15,"15min":15.15},"last":0.66300000000000003}]},{"osd":5,"last update":"Mon Aug 28 02:37:01 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":1.9390000000000001,"15min":2.181},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":1.591,"5min":36.476999999999997,"15min":36.476999999999997},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.56200000000000006,"5min":2.0230000000000001,"15min":2.2669999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":2.7040000000000002,"5min":36.872,"15min":36.872},"last":7.8789999999999996}]},{"osd":6,"last update":"Mon Aug 28 02:37:01 2023","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":2.8220000000000001,"15min":3.218},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.56000000000000005,"5min":53.301000000000002,"15min":53.301000000000002},"last":2.294},{"interface":"front","average":{"1min":0.55100000000000005,"5min":2.9889999999999999,"15min":3.395},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":2.1349999999999998,"5min":57.915999999999997,"15min":57.915999999999997},"last":0.53100000000000003}]}]},{"osd":6,"up_from":42,"seq":180388626774,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1737588,"kb_used_data":1436268,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92007564,"statfs":{"total":95995035648,"available":94215745536,"internally_reserved":0,"allocated":1470738432,"data_stored":1469848476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:37:22 2023","interfaces":[{"interface":"back","average":{"1min":1.607,"5min":2.3719999999999999,"15min":3.726},"min":{"1min":0.23599999999999999,"5min":0.215,"15min":0.20799999999999999},"max":{"1min":14.09,"5min":121.78,"15min":121.78},"last":3.024},{"interface":"front","average":{"1min":2.746,"5min":3.0680000000000001,"15min":3.9020000000000001},"min":{"1min":0.27000000000000002,"5min":0.23699999999999999,"15min":0.23200000000000001},"max":{"1min":12.612,"5min":121.79900000000001,"15min":121.79900000000001},"last":3.2549999999999999}]},{"osd":1,"last update":"Mon Aug 28 02:37:22 2023","interfaces":[{"interface":"back","average":{"1min":3.3900000000000001,"5min":1.7490000000000001,"15min":2.8719999999999999},"min":{"1min":0.27500000000000002,"5min":0.245,"15min":0.23699999999999999},"max":{"1min":16.736000000000001,"5min":16.736000000000001,"15min":141.87299999999999},"last":8.3620000000000001},{"interface":"front","average":{"1min":2.7210000000000001,"5min":1.827,"15min":2.984},"min":{"1min":0.28999999999999998,"5min":0.25,"15min":0.24299999999999999},"max":{"1min":16.797000000000001,"5min":16.797000000000001,"15min":141.904},"last":1.4410000000000001}]},{"osd":2,"last update":"Mon Aug 28 02:37:09 2023","interfaces":[{"interface":"back","average":{"1min":1.2210000000000001,"5min":5.6150000000000002,"15min":4.2140000000000004},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":2.839,"5min":234.876,"15min":234.876},"last":1.5289999999999999},{"interface":"front","average":{"1min":1.7769999999999999,"5min":5.5949999999999998,"15min":4.1929999999999996},"min":{"1min":0.30299999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":6.5119999999999996,"5min":234.482,"15min":234.482},"last":3.4750000000000001}]},{"osd":3,"last update":"Mon Aug 28 02:37:22 2023","interfaces":[{"interface":"back","average":{"1min":3.859,"5min":3.794,"15min":3.4199999999999999},"min":{"1min":0.36899999999999999,"5min":0.313,"15min":0.313},"max":{"1min":17.276,"5min":143.19499999999999,"15min":157.625},"last":2.9319999999999999},{"interface":"front","average":{"1min":3.7890000000000001,"5min":3.7970000000000002,"15min":3.4220000000000002},"min":{"1min":0.38,"5min":0.35899999999999999,"15min":0.27400000000000002},"max":{"1min":17.300000000000001,"5min":143.20699999999999,"15min":157.637},"last":2.9399999999999999}]},{"osd":4,"last update":"Mon Aug 28 02:37:26 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.52200000000000002,"15min":1.1120000000000001},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":2.931,"5min":3.8610000000000002,"15min":18.308},"last":0.30099999999999999},{"interface":"front","average":{"1min":1.075,"5min":0.753,"15min":1.25},"min":{"1min":0.309,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":6.2560000000000002,"5min":6.2560000000000002,"15min":18.263000000000002},"last":1.47}]},{"osd":5,"last update":"Mon Aug 28 02:37:26 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.58599999999999997,"15min":1.1930000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":2.7549999999999999,"5min":3.3159999999999998,"15min":20.759},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.52700000000000002,"15min":0.996},"min":{"1min":0.32100000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":2.1000000000000001,"5min":2.972,"15min":7.3780000000000001},"last":0.37}]},{"osd":7,"last update":"Mon Aug 28 02:36:58 2023","interfaces":[{"interface":"back","average":{"1min":0.997,"5min":0.64400000000000002,"15min":0.58499999999999996},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":3.0419999999999998,"5min":3.0419999999999998,"15min":3.0419999999999998},"last":1.502},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.53200000000000003,"15min":0.51500000000000001},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":2.3450000000000002,"5min":2.3450000000000002,"15min":2.3450000000000002},"last":0.45100000000000001}]}]},{"osd":1,"up_from":98,"seq":420906795227,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2169812,"kb_used_data":1627340,"kb_used_omap":7,"kb_used_meta":542456,"kb_avail":91575340,"statfs":{"total":95995035648,"available":93773148160,"internally_reserved":0,"allocated":1666396160,"data_stored":1664620287,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":555475275},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:37:19 2023","interfaces":[{"interface":"back","average":{"1min":1.931,"5min":6.25,"15min":4.5170000000000003},"min":{"1min":0.185,"5min":0.185,"15min":0.17499999999999999},"max":{"1min":7.2480000000000002,"5min":129.346,"15min":129.346},"last":5.4109999999999996},{"interface":"front","average":{"1min":2.0830000000000002,"5min":6.0099999999999998,"15min":4.5490000000000004},"min":{"1min":0.19900000000000001,"5min":0.19800000000000001,"15min":0.188},"max":{"1min":8.2929999999999993,"5min":120.774,"15min":128.685},"last":1.0049999999999999}]},{"osd":2,"last update":"Mon Aug 28 02:37:51 2023","interfaces":[{"interface":"back","average":{"1min":2.6840000000000002,"5min":3.8519999999999999,"15min":4.4059999999999997},"min":{"1min":0.29399999999999998,"5min":0.189,"15min":0.189},"max":{"1min":14.912000000000001,"5min":119.276,"15min":135.25999999999999},"last":0.38500000000000001},{"interface":"front","average":{"1min":1.736,"5min":3.4279999999999999,"15min":4.0220000000000002},"min":{"1min":0.27800000000000002,"5min":0.221,"15min":0.218},"max":{"1min":5.452,"5min":121.849,"15min":135.10300000000001},"last":5.452}]},{"osd":3,"last update":"Mon Aug 28 02:37:19 2023","interfaces":[{"interface":"back","average":{"1min":3.5830000000000002,"5min":7.6429999999999998,"15min":4.6849999999999996},"min":{"1min":0.27500000000000002,"5min":0.21299999999999999,"15min":0.19700000000000001},"max":{"1min":10.683,"5min":126.95699999999999,"15min":126.95699999999999},"last":0.84899999999999998},{"interface":"front","average":{"1min":3.4159999999999999,"5min":7.4379999999999997,"15min":4.7859999999999996},"min":{"1min":0.27400000000000002,"5min":0.217,"15min":0.217},"max":{"1min":10.938000000000001,"5min":128.00200000000001,"15min":128.00200000000001},"last":0.41599999999999998}]},{"osd":4,"last update":"Mon Aug 28 02:37:19 2023","interfaces":[{"interface":"back","average":{"1min":1.974,"5min":1.903,"15min":1.776},"min":{"1min":0.32400000000000001,"5min":0.28199999999999997,"15min":0.27500000000000002},"max":{"1min":11,"5min":24.809999999999999,"15min":24.809999999999999},"last":0.67100000000000004},{"interface":"front","average":{"1min":1.347,"5min":1.5509999999999999,"15min":1.5329999999999999},"min":{"1min":0.317,"5min":0.29499999999999998,"15min":0.25600000000000001},"max":{"1min":4.2220000000000004,"5min":24.835999999999999,"15min":24.835999999999999},"last":0.30499999999999999}]},{"osd":5,"last update":"Mon Aug 28 02:37:19 2023","interfaces":[{"interface":"back","average":{"1min":1.4470000000000001,"5min":2.9460000000000002,"15min":2.258},"min":{"1min":0.30499999999999999,"5min":0.252,"15min":0.252},"max":{"1min":6.4379999999999997,"5min":82.722999999999999,"15min":82.722999999999999},"last":0.88500000000000001},{"interface":"front","average":{"1min":2.915,"5min":3.2959999999999998,"15min":2.23},"min":{"1min":0.36299999999999999,"5min":0.315,"15min":0.307},"max":{"1min":14.566000000000001,"5min":82.733999999999995,"15min":82.733999999999995},"last":0.60699999999999998}]},{"osd":6,"last update":"Mon Aug 28 02:37:19 2023","interfaces":[{"interface":"back","average":{"1min":1.7070000000000001,"5min":2.0310000000000001,"15min":2.0699999999999998},"min":{"1min":0.36899999999999999,"5min":0.33600000000000002,"15min":0.311},"max":{"1min":6.5270000000000001,"5min":19.527000000000001,"15min":19.527000000000001},"last":0.90900000000000003},{"interface":"front","average":{"1min":2.593,"5min":2.3559999999999999,"15min":2.0840000000000001},"min":{"1min":0.46400000000000002,"5min":0.34899999999999998,"15min":0.32200000000000001},"max":{"1min":11.031000000000001,"5min":19.516999999999999,"15min":19.516999999999999},"last":0.70299999999999996}]},{"osd":7,"last update":"Mon Aug 28 02:37:02 2023","interfaces":[{"interface":"back","average":{"1min":1.4179999999999999,"5min":1.012,"15min":0.94399999999999995},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":6.4909999999999997,"5min":6.4909999999999997,"15min":6.4909999999999997},"last":0.93500000000000005},{"interface":"front","average":{"1min":1.788,"5min":1.006,"15min":0.876},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":6.3860000000000001,"5min":6.3860000000000001,"15min":6.3860000000000001},"last":0.42999999999999999}]}]},{"osd":0,"up_from":90,"seq":386547056873,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2490664,"kb_used_data":1853088,"kb_used_omap":8,"kb_used_meta":637559,"kb_avail":91254488,"statfs":{"total":95995035648,"available":93444595712,"internally_reserved":0,"allocated":1897562112,"data_stored":1895942435,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":652860777},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,4,3,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Aug 28 02:37:28 2023","interfaces":[{"interface":"back","average":{"1min":15.891,"5min":5.0430000000000001,"15min":3.7909999999999999},"min":{"1min":0.23799999999999999,"5min":0.16500000000000001,"15min":0.16500000000000001},"max":{"1min":150.00700000000001,"5min":150.00700000000001,"15min":150.00700000000001},"last":2.1760000000000002},{"interface":"front","average":{"1min":14.218,"5min":5.6509999999999998,"15min":4.1829999999999998},"min":{"1min":0.214,"5min":0.19500000000000001,"15min":0.17000000000000001},"max":{"1min":150.27699999999999,"5min":150.27699999999999,"15min":150.27699999999999},"last":2.3500000000000001}]},{"osd":2,"last update":"Mon Aug 28 02:37:50 2023","interfaces":[{"interface":"back","average":{"1min":3.0819999999999999,"5min":4.3499999999999996,"15min":3.8490000000000002},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.19500000000000001},"max":{"1min":24.030999999999999,"5min":86.269000000000005,"15min":100.718},"last":2.339},{"interface":"front","average":{"1min":3.0899999999999999,"5min":5.367,"15min":4.4610000000000003},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.20000000000000001},"max":{"1min":28.050999999999998,"5min":86.353999999999999,"15min":100.962},"last":0.68899999999999995}]},{"osd":3,"last update":"Mon Aug 28 02:37:35 2023","interfaces":[{"interface":"back","average":{"1min":13.976000000000001,"5min":7.3339999999999996,"15min":5.8330000000000002},"min":{"1min":0.23699999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":196.702,"5min":196.702,"15min":237.43899999999999},"last":0.47899999999999998},{"interface":"front","average":{"1min":15.141999999999999,"5min":7.508,"15min":5.8010000000000002},"min":{"1min":0.26800000000000002,"5min":0.26300000000000001,"15min":0.24399999999999999},"max":{"1min":197.75399999999999,"5min":197.75399999999999,"15min":237.48099999999999},"last":0.38800000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:37:24 2023","interfaces":[{"interface":"back","average":{"1min":1.6850000000000001,"5min":1.581,"15min":1.7250000000000001},"min":{"1min":0.35499999999999998,"5min":0.25700000000000001,"15min":0.23999999999999999},"max":{"1min":9.8510000000000009,"5min":14.209,"15min":14.209},"last":2.2109999999999999},{"interface":"front","average":{"1min":3.9630000000000001,"5min":2.5510000000000002,"15min":2.1779999999999999},"min":{"1min":0.33400000000000002,"5min":0.33100000000000002,"15min":0.309},"max":{"1min":11.473000000000001,"5min":14.241,"15min":27.271999999999998},"last":0.41599999999999998}]},{"osd":5,"last update":"Mon Aug 28 02:37:24 2023","interfaces":[{"interface":"back","average":{"1min":3.5649999999999999,"5min":2.427,"15min":2.2360000000000002},"min":{"1min":0.309,"5min":0.29899999999999999,"15min":0.29599999999999999},"max":{"1min":11.429,"5min":11.429,"15min":44.000999999999998},"last":0.441},{"interface":"front","average":{"1min":1.708,"5min":1.5409999999999999,"15min":1.905},"min":{"1min":0.32800000000000001,"5min":0.307,"15min":0.307},"max":{"1min":9.8149999999999995,"5min":9.8149999999999995,"15min":43.953000000000003},"last":2.2829999999999999}]},{"osd":6,"last update":"Mon Aug 28 02:37:24 2023","interfaces":[{"interface":"back","average":{"1min":3.5859999999999999,"5min":2.3849999999999998,"15min":2.3130000000000002},"min":{"1min":0.35599999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":11.500999999999999,"5min":11.500999999999999,"15min":32.417999999999999},"last":0.46000000000000002},{"interface":"front","average":{"1min":2.222,"5min":1.6679999999999999,"15min":1.9850000000000001},"min":{"1min":0.36599999999999999,"5min":0.32900000000000001,"15min":0.30399999999999999},"max":{"1min":9.9049999999999994,"5min":9.9049999999999994,"15min":32.130000000000003},"last":2.2469999999999999}]},{"osd":7,"last update":"Mon Aug 28 02:37:01 2023","interfaces":[{"interface":"back","average":{"1min":1.46,"5min":1.2090000000000001,"15min":1.167},"min":{"1min":0.46000000000000002,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":6.6669999999999998,"5min":6.6669999999999998,"15min":6.6669999999999998},"last":2.3109999999999999},{"interface":"front","average":{"1min":1.5009999999999999,"5min":1.841,"15min":1.897},"min":{"1min":0.39100000000000001,"5min":0.37,"15min":0.37},"max":{"1min":6.6059999999999999,"5min":16.529,"15min":16.529},"last":0.59499999999999997}]}]},{"osd":2,"up_from":104,"seq":446676598997,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2262928,"kb_used_data":1612104,"kb_used_omap":10,"kb_used_meta":650805,"kb_avail":91482224,"statfs":{"total":95995035648,"available":93677797376,"internally_reserved":0,"allocated":1650794496,"data_stored":1649193107,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10792,"internal_metadata":666424792},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:37:50 2023","interfaces":[{"interface":"back","average":{"1min":19.539999999999999,"5min":8.391,"15min":6.4870000000000001},"min":{"1min":0.16400000000000001,"5min":0.16400000000000001,"15min":0.16400000000000001},"max":{"1min":236.58199999999999,"5min":236.58199999999999,"15min":236.58199999999999},"last":3.875},{"interface":"front","average":{"1min":19.969000000000001,"5min":8.0280000000000005,"15min":7.5359999999999996},"min":{"1min":0.18099999999999999,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":236.524,"5min":236.524,"15min":236.524},"last":4.5}]},{"osd":1,"last update":"Mon Aug 28 02:37:50 2023","interfaces":[{"interface":"back","average":{"1min":9.4039999999999999,"5min":5.9420000000000002,"15min":3.645},"min":{"1min":0.222,"5min":0.221,"15min":0.20999999999999999},"max":{"1min":142.11000000000001,"5min":142.11000000000001,"15min":142.11000000000001},"last":8.3919999999999995},{"interface":"front","average":{"1min":8.4870000000000001,"5min":5.8970000000000002,"15min":3.544},"min":{"1min":0.23999999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":141.309,"5min":141.309,"15min":141.309},"last":0.56000000000000005}]},{"osd":3,"last update":"Mon Aug 28 02:37:24 2023","interfaces":[{"interface":"back","average":{"1min":27.613,"5min":10.163,"15min":6.1420000000000003},"min":{"1min":0.26000000000000001,"5min":0.23100000000000001,"15min":0.22700000000000001},"max":{"1min":165.07900000000001,"5min":165.07900000000001,"15min":165.07900000000001},"last":8.4320000000000004},{"interface":"front","average":{"1min":26.82,"5min":9.9700000000000006,"15min":6.0410000000000004},"min":{"1min":0.34499999999999997,"5min":0.29299999999999998,"15min":0.223},"max":{"1min":162.232,"5min":162.232,"15min":162.232},"last":0.65500000000000003}]},{"osd":4,"last update":"Mon Aug 28 02:37:50 2023","interfaces":[{"interface":"back","average":{"1min":2.2709999999999999,"5min":1.681,"15min":3.0899999999999999},"min":{"1min":0.27600000000000002,"5min":0.26800000000000002,"15min":0.26600000000000001},"max":{"1min":9.1099999999999994,"5min":13.449999999999999,"15min":207.34100000000001},"last":0.59099999999999997},{"interface":"front","average":{"1min":1.98,"5min":2.2250000000000001,"15min":3.0099999999999998},"min":{"1min":0.33700000000000002,"5min":0.27900000000000003,"15min":0.27400000000000002},"max":{"1min":7.3159999999999998,"5min":22.303999999999998,"15min":207.352},"last":3.9430000000000001}]},{"osd":5,"last update":"Mon Aug 28 02:37:50 2023","interfaces":[{"interface":"back","average":{"1min":2.278,"5min":1.536,"15min":2.9550000000000001},"min":{"1min":0.316,"5min":0.314,"15min":0.29299999999999998},"max":{"1min":8.1760000000000002,"5min":8.5440000000000005,"15min":128.696},"last":4.5869999999999997},{"interface":"front","average":{"1min":2.5950000000000002,"5min":1.7949999999999999,"15min":3.0979999999999999},"min":{"1min":0.318,"5min":0.312,"15min":0.30299999999999999},"max":{"1min":9.0670000000000002,"5min":13.404999999999999,"15min":128.84100000000001},"last":0.63400000000000001}]},{"osd":6,"last update":"Mon Aug 28 02:37:50 2023","interfaces":[{"interface":"back","average":{"1min":1.8340000000000001,"5min":2.3029999999999999,"15min":2.02},"min":{"1min":0.307,"5min":0.29699999999999999,"15min":0.29299999999999998},"max":{"1min":7.2839999999999998,"5min":22.25,"15min":22.25},"last":3.9140000000000001},{"interface":"front","average":{"1min":2.8889999999999998,"5min":1.657,"15min":1.9430000000000001},"min":{"1min":0.39900000000000002,"5min":0.30199999999999999,"15min":0.29399999999999998},"max":{"1min":8.1310000000000002,"5min":8.4979999999999993,"15min":13.311},"last":4.6219999999999999}]},{"osd":7,"last update":"Mon Aug 28 02:36:55 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":1.0049999999999999,"15min":1.014},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":2.9430000000000001,"5min":2.9430000000000001,"15min":2.9430000000000001},"last":4.6630000000000003},{"interface":"front","average":{"1min":2.7789999999999999,"5min":1.2789999999999999,"15min":1.0289999999999999},"min":{"1min":0.42099999999999999,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":13.478999999999999,"5min":13.478999999999999,"15min":13.478999999999999},"last":0.61399999999999999}]}]},{"osd":3,"up_from":112,"seq":481036337359,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1965384,"kb_used_data":1392384,"kb_used_omap":3,"kb_used_meta":572924,"kb_avail":91779768,"statfs":{"total":95995035648,"available":93982482432,"internally_reserved":0,"allocated":1425801216,"data_stored":1424205187,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":586675106},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:37:21 2023","interfaces":[{"interface":"back","average":{"1min":2.5710000000000002,"5min":5.0540000000000003,"15min":4.8650000000000002},"min":{"1min":0.23200000000000001,"5min":0.19500000000000001,"15min":0.193},"max":{"1min":9.2469999999999999,"5min":138.39699999999999,"15min":164.25299999999999},"last":3.8050000000000002},{"interface":"front","average":{"1min":3.4470000000000001,"5min":3.109,"15min":4.125},"min":{"1min":0.23000000000000001,"5min":0.20100000000000001,"15min":0.192},"max":{"1min":13.448,"5min":24.048999999999999,"15min":164.55000000000001},"last":3.5840000000000001}]},{"osd":1,"last update":"Mon Aug 28 02:37:21 2023","interfaces":[{"interface":"back","average":{"1min":4.3609999999999998,"5min":4.4569999999999999,"15min":3.4350000000000001},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":36.076999999999998,"5min":105.04600000000001,"15min":117.34699999999999},"last":3.2919999999999998},{"interface":"front","average":{"1min":5.6680000000000001,"5min":6.3520000000000003,"15min":4.1859999999999999},"min":{"1min":0.28699999999999998,"5min":0.27300000000000002,"15min":0.224},"max":{"1min":36.225999999999999,"5min":143.54400000000001,"15min":143.54400000000001},"last":3.8940000000000001}]},{"osd":2,"last update":"Mon Aug 28 02:37:21 2023","interfaces":[{"interface":"back","average":{"1min":2.855,"5min":5.3150000000000004,"15min":4.7249999999999996},"min":{"1min":0.33800000000000002,"5min":0.23799999999999999,"15min":0.19900000000000001},"max":{"1min":10.129,"5min":143.50299999999999,"15min":143.50299999999999},"last":3.867},{"interface":"front","average":{"1min":2.8290000000000002,"5min":2.964,"15min":3.823},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":14.494999999999999,"5min":14.494999999999999,"15min":120.44499999999999},"last":3.2360000000000002}]},{"osd":4,"last update":"Mon Aug 28 02:37:21 2023","interfaces":[{"interface":"back","average":{"1min":2.0409999999999999,"5min":1.6819999999999999,"15min":1.921},"min":{"1min":0.25800000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":12.151999999999999,"5min":12.151999999999999,"15min":58.067999999999998},"last":3.2759999999999998},{"interface":"front","average":{"1min":1.7789999999999999,"5min":1.841,"15min":2.0150000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":6.7270000000000003,"5min":10.613,"15min":58.088000000000001},"last":2.5659999999999998}]},{"osd":5,"last update":"Mon Aug 28 02:37:21 2023","interfaces":[{"interface":"back","average":{"1min":1.601,"5min":1.9530000000000001,"15min":2.1739999999999999},"min":{"1min":0.317,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":6.7590000000000003,"5min":10.722,"15min":102.904},"last":2.863},{"interface":"front","average":{"1min":1.653,"5min":3.8420000000000001,"15min":2.827},"min":{"1min":0.35299999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":6.6440000000000001,"5min":138.328,"15min":138.328},"last":7.3470000000000004}]},{"osd":6,"last update":"Mon Aug 28 02:37:21 2023","interfaces":[{"interface":"back","average":{"1min":2.0299999999999998,"5min":3.8690000000000002,"15min":2.6280000000000001},"min":{"1min":0.436,"5min":0.33000000000000002,"15min":0.314},"max":{"1min":6.6760000000000002,"5min":138.37,"15min":138.37},"last":4.1970000000000001},{"interface":"front","average":{"1min":1.9319999999999999,"5min":1.7250000000000001,"15min":1.8640000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":12.193,"5min":12.193,"15min":72.030000000000001},"last":3.3100000000000001}]},{"osd":7,"last update":"Mon Aug 28 02:37:03 2023","interfaces":[{"interface":"back","average":{"1min":1.75,"5min":1.274,"15min":1.194},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":5.0090000000000003,"5min":5.0090000000000003,"15min":5.0090000000000003},"last":7.3099999999999996},{"interface":"front","average":{"1min":1.6659999999999999,"5min":1.3620000000000001,"15min":1.3120000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":5.9180000000000001,"5min":5.9180000000000001,"15min":5.9180000000000001},"last":3.411}]}]},{"osd":4,"up_from":30,"seq":128849019232,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1774064,"kb_used_data":1188456,"kb_used_omap":0,"kb_used_meta":585600,"kb_avail":91971088,"statfs":{"total":95995035648,"available":94178394112,"internally_reserved":0,"allocated":1216978944,"data_stored":1216092866,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":599654400},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:37:27 2023","interfaces":[{"interface":"back","average":{"1min":7.9160000000000004,"5min":8.3469999999999995,"15min":5.8200000000000003},"min":{"1min":0.247,"5min":0.23100000000000001,"15min":0.219},"max":{"1min":121.60899999999999,"5min":131.893,"15min":138.67400000000001},"last":3.734},{"interface":"front","average":{"1min":7.7080000000000002,"5min":8.4049999999999994,"15min":5.6779999999999999},"min":{"1min":0.24399999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":121.621,"5min":131.96700000000001,"15min":134.14599999999999},"last":0.29199999999999998}]},{"osd":1,"last update":"Mon Aug 28 02:37:27 2023","interfaces":[{"interface":"back","average":{"1min":8.3539999999999992,"5min":4.7050000000000001,"15min":3.0019999999999998},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.221},"max":{"1min":131.845,"5min":131.845,"15min":131.845},"last":0.503},{"interface":"front","average":{"1min":8.4809999999999999,"5min":4.5389999999999997,"15min":2.8180000000000001},"min":{"1min":0.316,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":132.304,"5min":132.304,"15min":132.304},"last":1.3480000000000001}]},{"osd":2,"last update":"Mon Aug 28 02:36:53 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":3.444,"15min":3.843},"min":{"1min":0.26600000000000001,"5min":0.255,"15min":0.255},"max":{"1min":6.1769999999999996,"5min":107.786,"15min":107.786},"last":5.5270000000000001},{"interface":"front","average":{"1min":1.238,"5min":3.573,"15min":3.7669999999999999},"min":{"1min":0.29099999999999998,"5min":0.27300000000000002,"15min":0.25},"max":{"1min":6.5899999999999999,"5min":107.599,"15min":107.599},"last":1.706}]},{"osd":3,"last update":"Mon Aug 28 02:37:27 2023","interfaces":[{"interface":"back","average":{"1min":2.0990000000000002,"5min":7.4379999999999997,"15min":5.9329999999999998},"min":{"1min":0.33300000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":6.3399999999999999,"5min":137.995,"15min":203.583},"last":1.8859999999999999},{"interface":"front","average":{"1min":1.3740000000000001,"5min":7.4909999999999997,"15min":6.0369999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":6.1029999999999998,"5min":137.65899999999999,"15min":203.41399999999999},"last":0.46500000000000002}]},{"osd":5,"last update":"Mon Aug 28 02:37:03 2023","interfaces":[{"interface":"back","average":{"1min":0.871,"5min":0.53400000000000003,"15min":1.4650000000000001},"min":{"1min":0.255,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":2.7919999999999998,"5min":2.7919999999999998,"15min":124.718},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.80600000000000005,"5min":0.58499999999999996,"15min":1.679},"min":{"1min":0.34799999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":2.5230000000000001,"5min":2.5230000000000001,"15min":124.694},"last":0.41199999999999998}]},{"osd":6,"last update":"Mon Aug 28 02:37:27 2023","interfaces":[{"interface":"back","average":{"1min":0.76800000000000002,"5min":0.64300000000000002,"15min":1.1419999999999999},"min":{"1min":0.29499999999999998,"5min":0.25,"15min":0.25},"max":{"1min":3.7069999999999999,"5min":3.7069999999999999,"15min":14.907999999999999},"last":0.42699999999999999},{"interface":"front","average":{"1min":1.177,"5min":0.71999999999999997,"15min":0.93799999999999994},"min":{"1min":0.314,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":3.9769999999999999,"5min":3.9769999999999999,"15min":7.3170000000000002},"last":0.437}]},{"osd":7,"last update":"Mon Aug 28 02:36:58 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":0.57699999999999996,"15min":0.53800000000000003},"min":{"1min":0.32000000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":2.8660000000000001,"5min":2.8660000000000001,"15min":2.8660000000000001},"last":0.371},{"interface":"front","average":{"1min":0.67900000000000005,"5min":0.60499999999999998,"15min":0.59299999999999997},"min":{"1min":0.38,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":2.2629999999999999,"5min":2.2629999999999999,"15min":2.2629999999999999},"last":0.44900000000000001}]}]},{"osd":5,"up_from":36,"seq":154618823004,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2701732,"kb_used_data":2115996,"kb_used_omap":0,"kb_used_meta":585600,"kb_avail":91043420,"statfs":{"total":95995035648,"available":93228462080,"internally_reserved":0,"allocated":2166779904,"data_stored":2165901293,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":599654400},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:37:14 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":3.8889999999999998,"15min":4.4379999999999997},"min":{"1min":0.23400000000000001,"5min":0.216,"15min":0.20300000000000001},"max":{"1min":5.46,"5min":128.72800000000001,"15min":131.06100000000001},"last":4.8819999999999997},{"interface":"front","average":{"1min":1.4330000000000001,"5min":3.9809999999999999,"15min":4.6719999999999997},"min":{"1min":0.316,"5min":0.245,"15min":0.22900000000000001},"max":{"1min":7.609,"5min":128.75200000000001,"15min":131.262},"last":4.9039999999999999}]},{"osd":1,"last update":"Mon Aug 28 02:37:29 2023","interfaces":[{"interface":"back","average":{"1min":20.154,"5min":7.4329999999999998,"15min":4.7619999999999996},"min":{"1min":0.24199999999999999,"5min":0.22800000000000001,"15min":0.216},"max":{"1min":148.01400000000001,"5min":148.01400000000001,"15min":183.97900000000001},"last":0.38600000000000001},{"interface":"front","average":{"1min":20.516999999999999,"5min":7.4870000000000001,"15min":4.9420000000000002},"min":{"1min":0.27400000000000002,"5min":0.26400000000000001,"15min":0.23200000000000001},"max":{"1min":148.08000000000001,"5min":148.08000000000001,"15min":184.33699999999999},"last":10.984999999999999}]},{"osd":2,"last update":"Mon Aug 28 02:36:57 2023","interfaces":[{"interface":"back","average":{"1min":1.458,"5min":2.9729999999999999,"15min":4.8170000000000002},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.252},"max":{"1min":5.6070000000000002,"5min":78.834000000000003,"15min":176.69499999999999},"last":9.6560000000000006},{"interface":"front","average":{"1min":4.2149999999999999,"5min":3.5649999999999999,"15min":5.2569999999999997},"min":{"1min":0.34699999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":50.279000000000003,"5min":78.798000000000002,"15min":176.71199999999999},"last":7.149}]},{"osd":3,"last update":"Mon Aug 28 02:37:29 2023","interfaces":[{"interface":"back","average":{"1min":5.7750000000000004,"5min":5.1799999999999997,"15min":4.0389999999999997},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.26100000000000001},"max":{"1min":87.233999999999995,"5min":109.63,"15min":128.571},"last":2.7530000000000001},{"interface":"front","average":{"1min":6.508,"5min":4.7610000000000001,"15min":4.2770000000000001},"min":{"1min":0.36399999999999999,"5min":0.28699999999999998,"15min":0.26200000000000001},"max":{"1min":87.337000000000003,"5min":109.55800000000001,"15min":128.684},"last":6.1580000000000004}]},{"osd":4,"last update":"Mon Aug 28 02:37:05 2023","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.48499999999999999,"15min":1.3240000000000001},"min":{"1min":0.27200000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.72999999999999998,"5min":3.0259999999999998,"15min":85.769999999999996},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.48799999999999999,"15min":1.306},"min":{"1min":0.34200000000000003,"5min":0.248,"15min":0.248},"max":{"1min":1.5660000000000001,"5min":2.7690000000000001,"15min":85.825000000000003},"last":0.54800000000000004}]},{"osd":6,"last update":"Mon Aug 28 02:37:29 2023","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.57799999999999996,"15min":1.0589999999999999},"min":{"1min":0.30599999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":3.274,"5min":3.9990000000000001,"15min":14.803000000000001},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.73799999999999999,"5min":0.65100000000000002,"15min":0.97899999999999998},"min":{"1min":0.33000000000000002,"5min":0.29599999999999999,"15min":0.28799999999999998},"max":{"1min":2.6019999999999999,"5min":7.4740000000000002,"15min":8.6479999999999997},"last":0.57399999999999995}]},{"osd":7,"last update":"Mon Aug 28 02:36:57 2023","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.57999999999999996,"15min":0.59499999999999997},"min":{"1min":0.38300000000000001,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.65000000000000002,"5min":2.0049999999999999,"15min":2.0049999999999999},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.76300000000000001,"5min":0.71499999999999997,"15min":0.70699999999999996},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":3.3199999999999998,"5min":3.3199999999999998,"15min":3.3199999999999998},"last":0.46200000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8527,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1882324992,"data_stored":1882324992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1650917376,"data_stored":1650917376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1627918336,"data_stored":1627914264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1408172032,"data_stored":1408172032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1208352768,"data_stored":1208352768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2148339712,"data_stored":2148335640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1460277248,"data_stored":1460273176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4980736,"data_stored":4980736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-08-28T02:37:59.832 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd pool set device_health_metrics pgp_num 41 2023-08-28T02:37:59.893 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.6 dump_ops_in_flight 2023-08-28T02:37:59.976 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:37:59.980 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:37:59 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 6191/166359 objects degraded (3.721%), 2 pgs degraded (PG_DEGRADED) 2023-08-28T02:37:59.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:37:59.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:37:59.982 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:37:59 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 6191/166359 objects degraded (3.721%), 2 pgs degraded (PG_DEGRADED) 2023-08-28T02:37:59.983 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:37:59 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 6191/166359 objects degraded (3.721%), 2 pgs degraded (PG_DEGRADED) 2023-08-28T02:38:00.083 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:38:00.187 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:00.199 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:38:00.307 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:38:00.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:00.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:38:00.419 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:38:00.485 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:00.496 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:38:00.621 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:38:00.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:00.630 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:38:00.731 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:38:00.798 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:00.864 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:38:00.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:00.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:38:00.924 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:00 smithi143 ceph-mon[183059]: pgmap v935: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 584 op/s; 6191/166359 objects degraded (3.721%); 13911/166359 objects misplaced (8.362%); 1.3 MiB/s, 21 objects/s recovering 2023-08-28T02:38:00.925 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:00 smithi143 ceph-mon[189871]: pgmap v935: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 584 op/s; 6191/166359 objects degraded (3.721%); 13911/166359 objects misplaced (8.362%); 1.3 MiB/s, 21 objects/s recovering 2023-08-28T02:38:00.967 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:38:00.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:00 smithi169 ceph-mon[159415]: pgmap v935: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 584 op/s; 6191/166359 objects degraded (3.721%); 13911/166359 objects misplaced (8.362%); 1.3 MiB/s, 21 objects/s recovering 2023-08-28T02:38:01.015 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:01.277 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:38:01.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:01.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:38:01.390 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:38:01.416 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:01.458 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:38:01.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:01.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:38:01.561 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:38:01.605 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:01.691 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:38:01.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:01.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:38:01.798 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:38:01.898 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:02.003 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:38:02.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:02.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:38:02.031 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:38:02.092 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set nodeep-scrub 2023-08-28T02:38:02.119 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:38:02.160 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:02.167 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:38:01 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:38:01] "GET /metrics HTTP/1.1" 200 36574 "" "Prometheus/2.33.4" 2023-08-28T02:38:02.333 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:38:02.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:02.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:38:02.438 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:38:02.503 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:02.589 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:38:02.589 INFO:teuthology.orchestra.run.smithi169.stdout: "ops": [ 2023-08-28T02:38:02.589 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.590 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.45647.0:141406 8.9 8:932ac54f:::benchmark_data_smithi143_419289_object44496:head [delete] snapc 0=[] ondisk+write+known_if_redirected e349)", 2023-08-28T02:38:02.590 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:38:02.569860+0000", 2023-08-28T02:38:02.590 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 0.01829136, 2023-08-28T02:38:02.590 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.018330879000000001, 2023-08-28T02:38:02.590 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:38:02.590 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "waiting for sub ops", 2023-08-28T02:38:02.590 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:38:02.590 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.45647", 2023-08-28T02:38:02.590 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3230872490", 2023-08-28T02:38:02.590 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 141406 2023-08-28T02:38:02.590 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.591 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:38:02.591 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.591 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:38:02.591 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.569860+0000", 2023-08-28T02:38:02.591 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:38:02.591 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.591 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.591 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:38:02.591 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.569860+0000", 2023-08-28T02:38:02.591 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:38:02.591 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.592 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.592 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:38:02.592 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.569859+0000", 2023-08-28T02:38:02.592 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.9699999999999999e-06 2023-08-28T02:38:02.592 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.592 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.592 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:38:02.592 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.569866+0000", 2023-08-28T02:38:02.592 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1814000000000001e-05 2023-08-28T02:38:02.592 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.592 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.592 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:38:02.593 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.569878+0000", 2023-08-28T02:38:02.593 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.3300000000000004e-06 2023-08-28T02:38:02.593 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.593 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.593 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:38:02.593 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.569885+0000", 2023-08-28T02:38:02.593 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.3540000000000005e-05 2023-08-28T02:38:02.593 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.593 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.593 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:38:02.593 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.569948+0000", 2023-08-28T02:38:02.593 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000119589 2023-08-28T02:38:02.594 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.594 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.594 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:38:02.594 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.570068+0000", 2023-08-28T02:38:02.594 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00013997 2023-08-28T02:38:02.594 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.594 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.594 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:38:02.594 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.570208+0000", 2023-08-28T02:38:02.594 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0085158609999999996 2023-08-28T02:38:02.594 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.594 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.595 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:38:02.595 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.578724+0000", 2023-08-28T02:38:02.595 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0078120580000000002 2023-08-28T02:38:02.595 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.595 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.595 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:38:02.595 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.586536+0000", 2023-08-28T02:38:02.595 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.016675483000000001 2023-08-28T02:38:02.595 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:38:02.595 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:38:02.595 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:38:02.596 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.596 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.596 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.45647.0:141413 8.9 8:9212070b:::benchmark_data_smithi143_419289_object44503:head [delete] snapc 0=[] ondisk+write+known_if_redirected e349)", 2023-08-28T02:38:02.596 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:38:02.581204+0000", 2023-08-28T02:38:02.596 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 0.0069470349999999998, 2023-08-28T02:38:02.596 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0070855459999999999, 2023-08-28T02:38:02.596 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:38:02.596 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "waiting for sub ops", 2023-08-28T02:38:02.596 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:38:02.596 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.45647", 2023-08-28T02:38:02.596 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3230872490", 2023-08-28T02:38:02.596 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 141413 2023-08-28T02:38:02.597 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.597 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:38:02.597 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.597 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:38:02.597 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.581204+0000", 2023-08-28T02:38:02.597 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:38:02.597 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.597 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.597 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:38:02.597 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.581204+0000", 2023-08-28T02:38:02.597 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:38:02.597 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.598 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.598 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:38:02.598 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.581202+0000", 2023-08-28T02:38:02.598 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.6180000000000001e-05 2023-08-28T02:38:02.598 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.598 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.598 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:38:02.598 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.581219+0000", 2023-08-28T02:38:02.598 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.8406e-05 2023-08-28T02:38:02.598 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.598 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.598 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:38:02.599 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.581237+0000", 2023-08-28T02:38:02.599 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.3389999999999993e-06 2023-08-28T02:38:02.599 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.599 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.599 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:38:02.599 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.581245+0000", 2023-08-28T02:38:02.599 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00103192 2023-08-28T02:38:02.599 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.599 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.599 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:38:02.599 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.582277+0000", 2023-08-28T02:38:02.599 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00013509199999999999 2023-08-28T02:38:02.600 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.600 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.600 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:38:02.600 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.582412+0000", 2023-08-28T02:38:02.600 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000137293 2023-08-28T02:38:02.600 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.600 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.600 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:38:02.600 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.582550+0000", 2023-08-28T02:38:02.600 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00134512 2023-08-28T02:38:02.600 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:38:02.600 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:38:02.601 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:38:02.601 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.601 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.601 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_repop(client.45647.0:141411 8.3 e349/318 8:c25d412c:::benchmark_data_smithi143_419289_object44501:head v 349'8861, mlcod=349'8859)", 2023-08-28T02:38:02.601 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:38:02.582245+0000", 2023-08-28T02:38:02.601 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 0.0059067130000000001, 2023-08-28T02:38:02.601 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.006120889, 2023-08-28T02:38:02.601 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:38:02.601 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "started", 2023-08-28T02:38:02.601 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:38:02.601 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.601 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:38:02.602 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.582245+0000", 2023-08-28T02:38:02.602 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:38:02.602 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.602 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.602 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:38:02.602 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.582245+0000", 2023-08-28T02:38:02.602 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:38:02.602 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.602 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.602 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:38:02.602 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.582244+0000", 2023-08-28T02:38:02.602 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.3239999999999992e-06 2023-08-28T02:38:02.603 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.603 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.603 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:38:02.603 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.582253+0000", 2023-08-28T02:38:02.603 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1144999999999999e-05 2023-08-28T02:38:02.603 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.603 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.603 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:38:02.603 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.582264+0000", 2023-08-28T02:38:02.603 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.8560000000000003e-06 2023-08-28T02:38:02.603 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.603 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.603 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:38:02.604 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.582270+0000", 2023-08-28T02:38:02.604 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.6443999999999998e-05 2023-08-28T02:38:02.604 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.604 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.604 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:38:02.604 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.582317+0000", 2023-08-28T02:38:02.604 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.1416999999999998e-05 2023-08-28T02:38:02.604 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.604 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.604 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:38:02.604 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.582338+0000", 2023-08-28T02:38:02.605 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.3128999999999996e-05 2023-08-28T02:38:02.605 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:38:02.605 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:38:02.605 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:38:02.605 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.605 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.605 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.45647.0:141394 8.5 8:a98dc952:::benchmark_data_smithi143_419289_object44484:head [delete] snapc 0=[] ondisk+write+known_if_redirected e349)", 2023-08-28T02:38:02.605 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:38:02.547289+0000", 2023-08-28T02:38:02.605 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 0.040862075999999997, 2023-08-28T02:38:02.605 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.041138754999999999, 2023-08-28T02:38:02.605 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:38:02.606 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "waiting for sub ops", 2023-08-28T02:38:02.606 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:38:02.606 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.45647", 2023-08-28T02:38:02.606 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3230872490", 2023-08-28T02:38:02.606 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 141394 2023-08-28T02:38:02.606 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.606 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:38:02.606 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.606 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:38:02.606 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.547289+0000", 2023-08-28T02:38:02.606 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:38:02.606 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.607 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.607 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:38:02.607 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.547289+0000", 2023-08-28T02:38:02.607 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999995 2023-08-28T02:38:02.607 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.607 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.607 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:38:02.607 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.547289+0000", 2023-08-28T02:38:02.607 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.4400000000000001e-06 2023-08-28T02:38:02.607 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.607 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.608 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:38:02.608 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.547292+0000", 2023-08-28T02:38:02.608 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.4600000000000002e-06 2023-08-28T02:38:02.608 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.608 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.608 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:38:02.608 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.547298+0000", 2023-08-28T02:38:02.608 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.2699999999999998e-06 2023-08-28T02:38:02.608 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.608 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.608 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:38:02.608 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.547302+0000", 2023-08-28T02:38:02.609 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0054463139999999998 2023-08-28T02:38:02.609 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.609 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.609 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:38:02.609 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.552748+0000", 2023-08-28T02:38:02.609 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00018639700000000001 2023-08-28T02:38:02.609 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.609 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.609 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:38:02.609 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.552935+0000", 2023-08-28T02:38:02.609 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000180621 2023-08-28T02:38:02.609 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.609 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.610 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 4,7", 2023-08-28T02:38:02.610 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.553115+0000", 2023-08-28T02:38:02.610 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.015438367 2023-08-28T02:38:02.610 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.610 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.610 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:38:02.610 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.568554+0000", 2023-08-28T02:38:02.610 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0042795289999999998 2023-08-28T02:38:02.610 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.610 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.610 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:38:02.610 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.572833+0000", 2023-08-28T02:38:02.611 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.025543928 2023-08-28T02:38:02.611 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:38:02.611 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:38:02.611 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:38:02.611 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.611 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.611 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.45647.0:141397 8.5 8:ad38d6ce:::benchmark_data_smithi143_419289_object44487:head [delete] snapc 0=[] ondisk+write+known_if_redirected e349)", 2023-08-28T02:38:02.611 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:38:02.558458+0000", 2023-08-28T02:38:02.611 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 0.029693990999999999, 2023-08-28T02:38:02.611 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.03005675, 2023-08-28T02:38:02.611 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:38:02.612 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "waiting for sub ops", 2023-08-28T02:38:02.612 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:38:02.612 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.45647", 2023-08-28T02:38:02.612 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3230872490", 2023-08-28T02:38:02.612 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 141397 2023-08-28T02:38:02.612 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.612 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:38:02.612 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.612 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:38:02.612 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.558458+0000", 2023-08-28T02:38:02.612 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:38:02.612 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.612 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.612 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:38:02.613 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.558458+0000", 2023-08-28T02:38:02.613 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999976 2023-08-28T02:38:02.613 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.613 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.613 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:38:02.613 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.558455+0000", 2023-08-28T02:38:02.613 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.188e-05 2023-08-28T02:38:02.613 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.613 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.613 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:38:02.613 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.558467+0000", 2023-08-28T02:38:02.613 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.878e-05 2023-08-28T02:38:02.614 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.614 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.614 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:38:02.614 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.558486+0000", 2023-08-28T02:38:02.614 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.3330000000000002e-06 2023-08-28T02:38:02.614 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.614 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.614 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:38:02.614 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.558492+0000", 2023-08-28T02:38:02.614 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.7620000000000001e-05 2023-08-28T02:38:02.614 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.614 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.615 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:38:02.615 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.558540+0000", 2023-08-28T02:38:02.615 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00021995399999999999 2023-08-28T02:38:02.615 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.615 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.615 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:38:02.615 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.558760+0000", 2023-08-28T02:38:02.615 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0048348469999999998 2023-08-28T02:38:02.615 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.615 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.615 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 4,7", 2023-08-28T02:38:02.616 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.563595+0000", 2023-08-28T02:38:02.616 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0092462320000000001 2023-08-28T02:38:02.616 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.616 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.616 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:38:02.616 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.572841+0000", 2023-08-28T02:38:02.616 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0069066309999999999 2023-08-28T02:38:02.616 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:02.616 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:02.616 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:38:02.616 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:02.579748+0000", 2023-08-28T02:38:02.617 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.021289994999999999 2023-08-28T02:38:02.617 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:38:02.617 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:38:02.617 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:38:02.617 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:38:02.617 INFO:teuthology.orchestra.run.smithi169.stdout: ], 2023-08-28T02:38:02.617 INFO:teuthology.orchestra.run.smithi169.stdout: "num_ops": 5 2023-08-28T02:38:02.617 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:38:02.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:02.618 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:38:02.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:38:02.719 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:38:02.751 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:02 smithi143 ceph-mon[183059]: pgmap v936: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 6245/164154 objects degraded (3.804%); 13360/164154 objects misplaced (8.139%); 1.7 MiB/s, 26 objects/s recovering 2023-08-28T02:38:02.845 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:02.913 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:38:02.975 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:38:02.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:02 smithi169 ceph-mon[159415]: pgmap v936: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 6245/164154 objects degraded (3.804%); 13360/164154 objects misplaced (8.139%); 1.7 MiB/s, 26 objects/s recovering 2023-08-28T02:38:03.005 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:38:03.026 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:38:03.093 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:38:03.094 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:02 smithi143 ceph-mon[189871]: pgmap v936: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 6245/164154 objects degraded (3.804%); 13360/164154 objects misplaced (8.139%); 1.7 MiB/s, 26 objects/s recovering 2023-08-28T02:38:03.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:03.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:38:03.147 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:38:03.188 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:38:03.197 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:38:03.212 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:38:03.226 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:03.237 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:38:03.238 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:38:03.238 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:38:03.238 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:38:03.238 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:38:03.238 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:38:03.239 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:38:03.239 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:38:03.239 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:38:03.407 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:38:03.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:03.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:38:03.504 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.6 dump_blocked_ops 2023-08-28T02:38:03.511 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:38:03.560 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:04.071 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:38:04.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:03 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3816908079' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:38:04.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:03 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3816908079' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:38:04.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:03 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3816908079' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:38:04.947 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:38:04.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:04.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:38:04.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:04 smithi169 ceph-mon[159415]: pgmap v937: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 6245/159867 objects degraded (3.906%); 13360/159867 objects misplaced (8.357%); 1.7 MiB/s, 26 objects/s recovering 2023-08-28T02:38:05.057 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:38:05.085 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:05.128 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:38:05.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:05.130 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:38:05.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:04 smithi143 ceph-mon[183059]: pgmap v937: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 6245/159867 objects degraded (3.906%); 13360/159867 objects misplaced (8.357%); 1.7 MiB/s, 26 objects/s recovering 2023-08-28T02:38:05.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:04 smithi143 ceph-mon[189871]: pgmap v937: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 6245/159867 objects degraded (3.906%); 13360/159867 objects misplaced (8.357%); 1.7 MiB/s, 26 objects/s recovering 2023-08-28T02:38:05.231 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:38:05.295 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:05.528 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:38:05.608 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:38:05.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:05.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:38:05.719 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:38:05.826 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:05.953 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:38:05.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:05.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:38:05.975 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:05 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 6276/155478 objects degraded (4.037%), 2 pgs degraded (PG_DEGRADED) 2023-08-28T02:38:05.975 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:05 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 6276/155478 objects degraded (4.037%), 2 pgs degraded (PG_DEGRADED) 2023-08-28T02:38:06.060 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:38:06.158 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:38:06.158 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:38:06.160 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:06.182 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:05 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 6276/155478 objects degraded (4.037%), 2 pgs degraded (PG_DEGRADED) 2023-08-28T02:38:06.378 INFO:teuthology.orchestra.run.smithi143.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-08-28T02:38:06.420 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:38:06.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:06.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:38:06.525 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:38:06.557 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:07.025 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.6 dump_historic_ops 2023-08-28T02:38:07.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:06 smithi143 ceph-mon[183059]: pgmap v938: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 6276/155478 objects degraded (4.037%); 12644/155478 objects misplaced (8.132%); 1.8 MiB/s, 28 objects/s recovering 2023-08-28T02:38:07.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:06 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3080120617' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "41"}]: dispatch 2023-08-28T02:38:07.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:06 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "41"}]: dispatch 2023-08-28T02:38:07.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:06 smithi143 ceph-mon[189871]: pgmap v938: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 6276/155478 objects degraded (4.037%); 12644/155478 objects misplaced (8.132%); 1.8 MiB/s, 28 objects/s recovering 2023-08-28T02:38:07.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:06 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3080120617' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "41"}]: dispatch 2023-08-28T02:38:07.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:06 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "41"}]: dispatch 2023-08-28T02:38:07.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:06 smithi169 ceph-mon[159415]: pgmap v938: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 6276/155478 objects degraded (4.037%); 12644/155478 objects misplaced (8.132%); 1.8 MiB/s, 28 objects/s recovering 2023-08-28T02:38:07.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:06 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3080120617' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "41"}]: dispatch 2023-08-28T02:38:07.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:06 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "41"}]: dispatch 2023-08-28T02:38:07.556 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:38:07.617 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:38:07.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:07 smithi143 ceph-mon[189871]: pgmap v939: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 6276/153288 objects degraded (4.094%); 12644/153288 objects misplaced (8.249%); 1.8 MiB/s, 28 objects/s recovering 2023-08-28T02:38:07.953 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:07 smithi143 ceph-mon[183059]: pgmap v939: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 6276/153288 objects degraded (4.094%); 12644/153288 objects misplaced (8.249%); 1.8 MiB/s, 28 objects/s recovering 2023-08-28T02:38:08.113 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:38:08.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:08.124 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:38:08.157 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:07 smithi169 ceph-mon[159415]: pgmap v939: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 6276/153288 objects degraded (4.094%); 12644/153288 objects misplaced (8.249%); 1.8 MiB/s, 28 objects/s recovering 2023-08-28T02:38:08.225 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:38:08.254 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:08.412 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:38:08.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:08.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:38:08.521 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:38:08.605 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:08.759 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:38:08.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:08.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:38:08.786 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-08-28T02:38:08.865 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:38:08.924 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:09.066 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:38:09.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:09.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:38:09.121 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:38:09.178 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:38:09.212 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:09.399 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:38:09.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:09.403 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:38:09.504 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:38:09.601 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:09.652 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:38:09.652 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:38:09.699 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is set 2023-08-28T02:38:09.899 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:38:09.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:09.912 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:38:09.938 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:09 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3885779394' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:38:09.938 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:09 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:38:09.939 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:09 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3885779394' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:38:09.940 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:09 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:38:09.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:09 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3885779394' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:38:09.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:09 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:38:10.013 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:38:10.069 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:10.574 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.7 dump_ops_in_flight 2023-08-28T02:38:10.932 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:38:10.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:10.942 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:10 smithi143 ceph-mon[183059]: pgmap v940: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 6276/148923 objects degraded (4.214%); 12472/148923 objects misplaced (8.375%); 1.5 MiB/s, 24 objects/s recovering 2023-08-28T02:38:10.942 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:10 smithi143 ceph-mon[183059]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:38:10.943 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:10 smithi143 ceph-mon[183059]: failed to encode map e350 with expected crc 2023-08-28T02:38:10.943 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:10 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:38:10.943 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:10 smithi143 ceph-mon[183059]: osdmap e350: 8 total, 8 up, 8 in 2023-08-28T02:38:10.943 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:10 smithi143 ceph-mon[183059]: failed to encode map e350 with expected crc 2023-08-28T02:38:10.943 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:10 smithi143 ceph-mon[183059]: failed to encode map e350 with expected crc 2023-08-28T02:38:10.943 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:10 smithi143 ceph-mon[183059]: failed to encode map e350 with expected crc 2023-08-28T02:38:10.943 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:10 smithi143 ceph-mon[183059]: failed to encode map e350 with expected crc 2023-08-28T02:38:10.943 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:10 smithi143 ceph-mon[183059]: failed to encode map e350 with expected crc 2023-08-28T02:38:10.943 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:10 smithi143 ceph-mon[189871]: pgmap v940: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 6276/148923 objects degraded (4.214%); 12472/148923 objects misplaced (8.375%); 1.5 MiB/s, 24 objects/s recovering 2023-08-28T02:38:10.943 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:10 smithi143 ceph-mon[189871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:38:10.944 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:10 smithi143 ceph-mon[189871]: failed to encode map e350 with expected crc 2023-08-28T02:38:10.944 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:10 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:38:10.944 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:10 smithi143 ceph-mon[189871]: osdmap e350: 8 total, 8 up, 8 in 2023-08-28T02:38:10.944 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:10 smithi143 ceph-mon[189871]: failed to encode map e350 with expected crc 2023-08-28T02:38:10.944 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:10 smithi143 ceph-mon[189871]: failed to encode map e350 with expected crc 2023-08-28T02:38:10.944 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:10 smithi143 ceph-mon[189871]: failed to encode map e350 with expected crc 2023-08-28T02:38:10.944 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:10 smithi143 ceph-mon[189871]: failed to encode map e350 with expected crc 2023-08-28T02:38:10.944 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:10 smithi143 ceph-mon[189871]: failed to encode map e350 with expected crc 2023-08-28T02:38:10.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:38:10.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:10 smithi169 ceph-mon[159415]: pgmap v940: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 6276/148923 objects degraded (4.214%); 12472/148923 objects misplaced (8.375%); 1.5 MiB/s, 24 objects/s recovering 2023-08-28T02:38:10.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:10 smithi169 ceph-mon[159415]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:38:10.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:10 smithi169 ceph-mon[159415]: failed to encode map e350 with expected crc 2023-08-28T02:38:10.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:10 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:38:10.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:10 smithi169 ceph-mon[159415]: osdmap e350: 8 total, 8 up, 8 in 2023-08-28T02:38:10.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:10 smithi169 ceph-mon[159415]: failed to encode map e350 with expected crc 2023-08-28T02:38:10.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:10 smithi169 ceph-mon[159415]: failed to encode map e350 with expected crc 2023-08-28T02:38:10.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:10 smithi169 ceph-mon[159415]: failed to encode map e350 with expected crc 2023-08-28T02:38:10.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:10 smithi169 ceph-mon[159415]: failed to encode map e350 with expected crc 2023-08-28T02:38:10.991 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:10 smithi169 ceph-mon[159415]: failed to encode map e350 with expected crc 2023-08-28T02:38:11.045 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:38:11.070 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:38:11.100 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:11.210 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:38:11.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:11.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:38:11.316 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:38:11.381 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:11.781 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:38:11.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:11.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:38:11.890 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:38:11.934 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:38:11 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:38:11] "GET /metrics HTTP/1.1" 200 36558 "" "Prometheus/2.33.4" 2023-08-28T02:38:11.993 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:12.192 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:38:12.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:12 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:38:12.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:12 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:38:12.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:12 smithi143 ceph-mon[183059]: pgmap v942: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 6291/146727 objects degraded (4.288%); 11944/146727 objects misplaced (8.140%); 1.8 MiB/s, 29 objects/s recovering 2023-08-28T02:38:12.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:12 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 6276/148923 objects degraded (4.214%), 2 pgs degraded (PG_DEGRADED) 2023-08-28T02:38:12.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:12 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:38:12.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:12 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:38:12.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:12 smithi143 ceph-mon[189871]: pgmap v942: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 6291/146727 objects degraded (4.288%); 11944/146727 objects misplaced (8.140%); 1.8 MiB/s, 29 objects/s recovering 2023-08-28T02:38:12.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:12 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 6276/148923 objects degraded (4.214%), 2 pgs degraded (PG_DEGRADED) 2023-08-28T02:38:12.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:12.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:38:12.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:12 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:38:12.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:12 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:38:12.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:12 smithi169 ceph-mon[159415]: pgmap v942: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 6291/146727 objects degraded (4.288%); 11944/146727 objects misplaced (8.140%); 1.8 MiB/s, 29 objects/s recovering 2023-08-28T02:38:12.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:12 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 6276/148923 objects degraded (4.214%), 2 pgs degraded (PG_DEGRADED) 2023-08-28T02:38:12.250 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:38:12.287 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:38:12.305 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:38:12.337 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:38:12.353 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:12.379 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:38:12.420 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:38:12.446 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:38:12.477 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:38:12.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:12.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:38:12.493 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:38:12.525 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:38:12.550 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:38:12.550 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:38:12.551 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:38:12.551 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:38:12.551 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:38:12.551 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:38:12.551 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:38:12.551 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:38:12.588 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:38:12.698 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:12.838 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:38:12.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:12.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:38:12.943 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:38:13.003 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:13.046 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:38:13.046 INFO:teuthology.orchestra.run.smithi169.stdout: "ops": [ 2023-08-28T02:38:13.046 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:13.046 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_repop(client.45647.0:147461 8.6 e350/345 8:6c57dab6:::benchmark_data_smithi143_419289_object50551:head v 350'9322, mlcod=350'9318)", 2023-08-28T02:38:13.047 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:38:13.043809+0000", 2023-08-28T02:38:13.047 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 0.0031881380000000001, 2023-08-28T02:38:13.047 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0032289929999999999, 2023-08-28T02:38:13.047 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:38:13.047 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "started", 2023-08-28T02:38:13.047 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:38:13.047 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:13.047 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:38:13.047 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:13.043809+0000", 2023-08-28T02:38:13.047 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:38:13.048 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:13.048 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:13.048 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:38:13.048 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:13.043809+0000", 2023-08-28T02:38:13.048 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:38:13.048 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:13.048 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:13.048 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:38:13.048 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:13.043808+0000", 2023-08-28T02:38:13.048 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0128e-05 2023-08-28T02:38:13.048 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:13.049 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:13.049 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:38:13.049 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:13.043818+0000", 2023-08-28T02:38:13.049 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.9014000000000001e-05 2023-08-28T02:38:13.049 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:13.049 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:13.049 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:38:13.049 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:13.043837+0000", 2023-08-28T02:38:13.049 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.9459999999999998e-06 2023-08-28T02:38:13.049 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:13.049 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:13.049 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:38:13.050 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:13.043845+0000", 2023-08-28T02:38:13.050 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.9938999999999996e-05 2023-08-28T02:38:13.050 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:13.050 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:13.050 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:38:13.050 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:13.043915+0000", 2023-08-28T02:38:13.050 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.3139999999999999e-05 2023-08-28T02:38:13.050 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:38:13.050 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:38:13.050 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:38:13.050 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:13.043938+0000", 2023-08-28T02:38:13.050 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012881500000000001 2023-08-28T02:38:13.050 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:38:13.050 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:38:13.051 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:38:13.051 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:38:13.051 INFO:teuthology.orchestra.run.smithi169.stdout: ], 2023-08-28T02:38:13.051 INFO:teuthology.orchestra.run.smithi169.stdout: "num_ops": 1 2023-08-28T02:38:13.051 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:38:13.073 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:13 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3323706462' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:38:13.076 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:38:13.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:13.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:38:13.181 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:38:13.205 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:13 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3323706462' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:38:13.205 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:13 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3323706462' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:38:13.240 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:13.282 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:38:13.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:13.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:38:13.385 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:38:13.417 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:13.473 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:38:13.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:13.485 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:38:13.585 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:38:13.632 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:13.756 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset noscrub 2023-08-28T02:38:13.786 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 6, 1, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-08-28T02:38:13.787 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-08-28T02:38:13.787 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-08-28T02:38:13.787 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-08-28T02:38:13.787 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-08-28T02:38:13.787 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-08-28T02:38:13.943 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:38:13.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:13.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:38:13.980 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.7 dump_blocked_ops 2023-08-28T02:38:14.052 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:38:14.080 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:14 smithi143 ceph-mon[183059]: pgmap v943: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 6291/142512 objects degraded (4.414%); 11944/142512 objects misplaced (8.381%); 1.8 MiB/s, 29 objects/s recovering 2023-08-28T02:38:14.139 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:14.245 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:14 smithi169 ceph-mon[159415]: pgmap v943: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 6291/142512 objects degraded (4.414%); 11944/142512 objects misplaced (8.381%); 1.8 MiB/s, 29 objects/s recovering 2023-08-28T02:38:14.272 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:38:14.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:14.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:38:14.386 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:38:14.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:14 smithi143 ceph-mon[189871]: pgmap v943: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 6291/142512 objects degraded (4.414%); 11944/142512 objects misplaced (8.381%); 1.8 MiB/s, 29 objects/s recovering 2023-08-28T02:38:14.452 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:38:14.477 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:38:14.488 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:14.638 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:38:14.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:14.648 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:38:14.743 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:38:14.749 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:38:14.841 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:15.053 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:38:15.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:15.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:38:15.172 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:38:15.252 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:15.349 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:38:15.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:15.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:38:15.462 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:38:15.518 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:15.597 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:38:15.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:15.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:38:15.701 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:38:15.737 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:15.810 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:38:15.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:15.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:38:15.923 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:38:15.951 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:38:15.952 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:16.364 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:38:16.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:16.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:38:16.478 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:38:16.524 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:16.881 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:38:16.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:16.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:38:16.938 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:16 smithi143 ceph-mon[189871]: pgmap v944: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 2.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 6309/138099 objects degraded (4.568%); 11238/138099 objects misplaced (8.138%); 1.8 MiB/s, 29 objects/s recovering 2023-08-28T02:38:16.938 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:16 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 6309/138099 objects degraded (4.568%), 2 pgs degraded (PG_DEGRADED) 2023-08-28T02:38:16.939 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:16 smithi143 ceph-mon[183059]: pgmap v944: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 2.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 6309/138099 objects degraded (4.568%); 11238/138099 objects misplaced (8.138%); 1.8 MiB/s, 29 objects/s recovering 2023-08-28T02:38:16.939 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:16 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 6309/138099 objects degraded (4.568%), 2 pgs degraded (PG_DEGRADED) 2023-08-28T02:38:16.992 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:16 smithi169 ceph-mon[159415]: pgmap v944: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 2.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 6309/138099 objects degraded (4.568%); 11238/138099 objects misplaced (8.138%); 1.8 MiB/s, 29 objects/s recovering 2023-08-28T02:38:16.992 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:16 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 6309/138099 objects degraded (4.568%), 2 pgs degraded (PG_DEGRADED) 2023-08-28T02:38:16.998 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:38:17.086 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:17.171 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:38:17.252 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:38:17.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:17.255 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:38:17.356 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:38:17.421 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:17.527 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:38:17.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:17.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:38:17.554 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:38:17.554 INFO:teuthology.orchestra.run.smithi169.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-08-28T02:38:17.554 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:38:17.652 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:38:17.672 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:17 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2410931747' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:38:17.740 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is unset 2023-08-28T02:38:17.765 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:17.772 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:17 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2410931747' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:38:17.772 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:17 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2410931747' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:38:17.815 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:38:17.815 INFO:teuthology.orchestra.run.smithi169.stdout: "ops": [], 2023-08-28T02:38:17.815 INFO:teuthology.orchestra.run.smithi169.stdout: "complaint_time": 30, 2023-08-28T02:38:17.816 INFO:teuthology.orchestra.run.smithi169.stdout: "num_blocked_ops": 0 2023-08-28T02:38:17.816 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:38:17.932 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:38:17.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:17.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:38:18.050 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:38:18.162 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:18.521 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:38:18.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:18.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:38:18.636 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:38:18.703 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:18.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:18 smithi143 ceph-mon[183059]: pgmap v945: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 6309/135870 objects degraded (4.643%); 11238/135870 objects misplaced (8.271%); 1.8 MiB/s, 29 objects/s recovering 2023-08-28T02:38:18.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:18 smithi143 ceph-mon[183059]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:38:18.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:18 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2410931747' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:38:18.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:18 smithi143 ceph-mon[183059]: osdmap e351: 8 total, 8 up, 8 in 2023-08-28T02:38:18.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:18 smithi143 ceph-mon[183059]: failed to encode map e351 with expected crc 2023-08-28T02:38:18.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:18 smithi143 ceph-mon[183059]: failed to encode map e351 with expected crc 2023-08-28T02:38:18.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:18 smithi143 ceph-mon[183059]: failed to encode map e351 with expected crc 2023-08-28T02:38:18.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:18 smithi143 ceph-mon[183059]: failed to encode map e351 with expected crc 2023-08-28T02:38:18.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:18 smithi143 ceph-mon[183059]: failed to encode map e351 with expected crc 2023-08-28T02:38:18.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:18 smithi143 ceph-mon[189871]: pgmap v945: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 6309/135870 objects degraded (4.643%); 11238/135870 objects misplaced (8.271%); 1.8 MiB/s, 29 objects/s recovering 2023-08-28T02:38:18.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:18 smithi143 ceph-mon[189871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:38:18.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:18 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2410931747' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:38:18.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:18 smithi143 ceph-mon[189871]: osdmap e351: 8 total, 8 up, 8 in 2023-08-28T02:38:18.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:18 smithi143 ceph-mon[189871]: failed to encode map e351 with expected crc 2023-08-28T02:38:18.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:18 smithi143 ceph-mon[189871]: failed to encode map e351 with expected crc 2023-08-28T02:38:18.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:18 smithi143 ceph-mon[189871]: failed to encode map e351 with expected crc 2023-08-28T02:38:18.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:18 smithi143 ceph-mon[189871]: failed to encode map e351 with expected crc 2023-08-28T02:38:18.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:18 smithi143 ceph-mon[189871]: failed to encode map e351 with expected crc 2023-08-28T02:38:18.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:18 smithi169 ceph-mon[159415]: pgmap v945: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 6309/135870 objects degraded (4.643%); 11238/135870 objects misplaced (8.271%); 1.8 MiB/s, 29 objects/s recovering 2023-08-28T02:38:18.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:18 smithi169 ceph-mon[159415]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:38:18.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:18 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2410931747' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:38:18.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:18 smithi169 ceph-mon[159415]: osdmap e351: 8 total, 8 up, 8 in 2023-08-28T02:38:18.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:18 smithi169 ceph-mon[159415]: failed to encode map e351 with expected crc 2023-08-28T02:38:18.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:18 smithi169 ceph-mon[159415]: failed to encode map e351 with expected crc 2023-08-28T02:38:18.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:18 smithi169 ceph-mon[159415]: failed to encode map e351 with expected crc 2023-08-28T02:38:18.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:18 smithi169 ceph-mon[159415]: failed to encode map e351 with expected crc 2023-08-28T02:38:18.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:18 smithi169 ceph-mon[159415]: failed to encode map e351 with expected crc 2023-08-28T02:38:19.023 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:38:19.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:19.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:38:19.144 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:38:19.158 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.7 dump_historic_ops 2023-08-28T02:38:19.171 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:19.214 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:38:19.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:19.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:38:19.317 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:38:19.384 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:19.707 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:38:20.067 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:38:20.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:20.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:38:20.172 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:38:20.314 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:20.521 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:38:20.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:20.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:38:20.634 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:38:20.693 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:20 smithi143 ceph-mon[189871]: pgmap v947: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 6309/131313 objects degraded (4.805%); 11032/131313 objects misplaced (8.401%); 1.8 MiB/s, 29 objects/s recovering 2023-08-28T02:38:20.701 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:20.738 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:20 smithi169 ceph-mon[159415]: pgmap v947: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 6309/131313 objects degraded (4.805%); 11032/131313 objects misplaced (8.401%); 1.8 MiB/s, 29 objects/s recovering 2023-08-28T02:38:20.848 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:38:20.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:20.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:38:20.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:20 smithi143 ceph-mon[183059]: pgmap v947: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 6309/131313 objects degraded (4.805%); 11032/131313 objects misplaced (8.401%); 1.8 MiB/s, 29 objects/s recovering 2023-08-28T02:38:20.961 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:38:21.063 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:21.135 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:38:21.185 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:38:21.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:21.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:38:21.197 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:38:21.232 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:38:21.259 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:38:21.291 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:38:21.298 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:38:21.353 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:21.356 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:38:21.406 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:38:21.432 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:38:21.452 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:38:21.452 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:38:21.453 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:38:21.453 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:38:21.453 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:38:21.453 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:38:21.453 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:38:21.453 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:38:21.458 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:38:21.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:21.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:38:21.568 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:38:21.652 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:21.908 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:38:21.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:21.919 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:21 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3018681993' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:38:21.919 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:21 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 6309/131313 objects degraded (4.805%), 2 pgs degraded (PG_DEGRADED) 2023-08-28T02:38:21.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:38:21.928 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:21 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3018681993' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:38:21.928 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:21 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 6309/131313 objects degraded (4.805%), 2 pgs degraded (PG_DEGRADED) 2023-08-28T02:38:21.928 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:21 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3018681993' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:38:21.928 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:21 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 6309/131313 objects degraded (4.805%), 2 pgs degraded (PG_DEGRADED) 2023-08-28T02:38:21.933 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:38:21.933 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:38:22.021 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:38:22.107 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:22.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:38:21 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:38:21] "GET /metrics HTTP/1.1" 200 36558 "" "Prometheus/2.33.4" 2023-08-28T02:38:22.358 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:38:22.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:22.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:38:22.465 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:38:22.516 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset nodeep-scrub 2023-08-28T02:38:22.530 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:22.765 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:38:22.775 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:22 smithi169 ceph-mon[159415]: pgmap v948: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 6361/129057 objects degraded (4.929%); 10469/129057 objects misplaced (8.112%); 1.8 MiB/s, 29 objects/s recovering 2023-08-28T02:38:22.776 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.0 dump_ops_in_flight 2023-08-28T02:38:22.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:22.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:38:22.879 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:38:22.944 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:23.087 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:38:23.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:23.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:38:23.106 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:22 smithi143 ceph-mon[183059]: pgmap v948: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 6361/129057 objects degraded (4.929%); 10469/129057 objects misplaced (8.112%); 1.8 MiB/s, 29 objects/s recovering 2023-08-28T02:38:23.108 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:22 smithi143 ceph-mon[189871]: pgmap v948: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 6361/129057 objects degraded (4.929%); 10469/129057 objects misplaced (8.112%); 1.8 MiB/s, 29 objects/s recovering 2023-08-28T02:38:23.206 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:38:23.250 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:23.304 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:38:23.314 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:38:23.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:23.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:38:23.425 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:38:23.505 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:23.763 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:38:23.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:23.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:38:23.869 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:38:23.913 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:24.005 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:38:24.016 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:38:24.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:24.022 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:38:24.110 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 4, 5, 3, 6, 1, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-08-28T02:38:24.111 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-08-28T02:38:24.111 INFO:tasks.thrashosds.thrasher:Removing osd 2, in_osds are: [0, 2, 4, 5, 3, 6, 1, 7] 2023-08-28T02:38:24.111 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd out 2 2023-08-28T02:38:24.123 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:38:24.228 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:24.891 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:38:24.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:24 smithi169 ceph-mon[159415]: pgmap v949: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 6361/124566 objects degraded (5.107%); 10469/124566 objects misplaced (8.404%); 1.8 MiB/s, 29 objects/s recovering 2023-08-28T02:38:25.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:24 smithi143 ceph-mon[183059]: pgmap v949: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 6361/124566 objects degraded (5.107%); 10469/124566 objects misplaced (8.404%); 1.8 MiB/s, 29 objects/s recovering 2023-08-28T02:38:25.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:24 smithi143 ceph-mon[189871]: pgmap v949: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 6361/124566 objects degraded (5.107%); 10469/124566 objects misplaced (8.404%); 1.8 MiB/s, 29 objects/s recovering 2023-08-28T02:38:25.522 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:38:25.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:25.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:38:25.630 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:38:25.706 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:25.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:25.912 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:38:25.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:38:26.013 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:38:26.103 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:27.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:26 smithi143 ceph-mon[189871]: pgmap v950: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 2.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 6385/120240 objects degraded (5.310%); 9716/120240 objects misplaced (8.081%); 2.0 MiB/s, 31 objects/s recovering 2023-08-28T02:38:27.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:26 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:38:27.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:26 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 6385/120240 objects degraded (5.310%), 2 pgs degraded (PG_DEGRADED) 2023-08-28T02:38:27.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:26 smithi143 ceph-mon[183059]: pgmap v950: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 2.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 6385/120240 objects degraded (5.310%); 9716/120240 objects misplaced (8.081%); 2.0 MiB/s, 31 objects/s recovering 2023-08-28T02:38:27.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:26 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:38:27.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:26 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 6385/120240 objects degraded (5.310%), 2 pgs degraded (PG_DEGRADED) 2023-08-28T02:38:27.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:26 smithi169 ceph-mon[159415]: pgmap v950: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 2.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 6385/120240 objects degraded (5.310%); 9716/120240 objects misplaced (8.081%); 2.0 MiB/s, 31 objects/s recovering 2023-08-28T02:38:27.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:26 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:38:27.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:26 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 6385/120240 objects degraded (5.310%), 2 pgs degraded (PG_DEGRADED) 2023-08-28T02:38:27.269 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:38:27.501 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:38:27.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:27.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:38:27.612 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:38:27.687 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:38:27.715 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:27.765 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:27 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/4266147265' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:38:27.769 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:27 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/4266147265' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:38:27.805 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is unset 2023-08-28T02:38:27.864 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:38:27.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:27.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:38:27.973 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:38:28.011 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:28.094 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:38:28.097 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:27 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/4266147265' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:38:28.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:28.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:38:28.199 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:38:28.246 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:28.354 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:38:28.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:28.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:38:28.464 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:38:28.504 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:28.571 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:38:28.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:28.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:38:28.687 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:38:28.717 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:28.789 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:38:28.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:28.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:38:28.895 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:38:28.922 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:28.965 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:38:28.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:28 smithi169 ceph-mon[159415]: pgmap v951: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 6385/118029 objects degraded (5.410%); 9716/118029 objects misplaced (8.232%); 2.0 MiB/s, 31 objects/s recovering 2023-08-28T02:38:28.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:28 smithi169 ceph-mon[159415]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:38:28.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:28 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/4266147265' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:38:28.975 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:28 smithi169 ceph-mon[159415]: osdmap e352: 8 total, 8 up, 8 in 2023-08-28T02:38:28.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:28 smithi169 ceph-mon[159415]: failed to encode map e352 with expected crc 2023-08-28T02:38:28.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:28 smithi169 ceph-mon[159415]: failed to encode map e352 with expected crc 2023-08-28T02:38:28.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:28 smithi169 ceph-mon[159415]: failed to encode map e352 with expected crc 2023-08-28T02:38:28.976 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:28 smithi169 ceph-mon[159415]: failed to encode map e352 with expected crc 2023-08-28T02:38:28.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:28.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:38:29.078 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:38:29.156 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:28 smithi143 ceph-mon[183059]: pgmap v951: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 6385/118029 objects degraded (5.410%); 9716/118029 objects misplaced (8.232%); 2.0 MiB/s, 31 objects/s recovering 2023-08-28T02:38:29.156 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:28 smithi143 ceph-mon[183059]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:38:29.156 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:28 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/4266147265' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:38:29.156 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:28 smithi143 ceph-mon[183059]: osdmap e352: 8 total, 8 up, 8 in 2023-08-28T02:38:29.156 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:28 smithi143 ceph-mon[183059]: failed to encode map e352 with expected crc 2023-08-28T02:38:29.156 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:28 smithi143 ceph-mon[183059]: failed to encode map e352 with expected crc 2023-08-28T02:38:29.156 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:28 smithi143 ceph-mon[183059]: failed to encode map e352 with expected crc 2023-08-28T02:38:29.156 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:28 smithi143 ceph-mon[183059]: failed to encode map e352 with expected crc 2023-08-28T02:38:29.156 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:28 smithi143 ceph-mon[189871]: pgmap v951: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 6385/118029 objects degraded (5.410%); 9716/118029 objects misplaced (8.232%); 2.0 MiB/s, 31 objects/s recovering 2023-08-28T02:38:29.157 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:28 smithi143 ceph-mon[189871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:38:29.157 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:28 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/4266147265' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:38:29.157 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:28 smithi143 ceph-mon[189871]: osdmap e352: 8 total, 8 up, 8 in 2023-08-28T02:38:29.157 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:28 smithi143 ceph-mon[189871]: failed to encode map e352 with expected crc 2023-08-28T02:38:29.157 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:28 smithi143 ceph-mon[189871]: failed to encode map e352 with expected crc 2023-08-28T02:38:29.157 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:28 smithi143 ceph-mon[189871]: failed to encode map e352 with expected crc 2023-08-28T02:38:29.157 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:28 smithi143 ceph-mon[189871]: failed to encode map e352 with expected crc 2023-08-28T02:38:29.171 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:29.997 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:38:30.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:30.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:38:30.105 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:38:30.169 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:30.242 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:38:30.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:30.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:38:30.302 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:38:30.302 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:38:30.356 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:38:30.450 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:30.953 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:30 smithi143 ceph-mon[183059]: pgmap v953: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 6385/113625 objects degraded (5.619%); 9553/113625 objects misplaced (8.407%); 2.0 MiB/s, 31 objects/s recovering 2023-08-28T02:38:30.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:30 smithi143 ceph-mon[189871]: pgmap v953: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 6385/113625 objects degraded (5.619%); 9553/113625 objects misplaced (8.407%); 2.0 MiB/s, 31 objects/s recovering 2023-08-28T02:38:31.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:30 smithi169 ceph-mon[159415]: pgmap v953: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 6385/113625 objects degraded (5.619%); 9553/113625 objects misplaced (8.407%); 2.0 MiB/s, 31 objects/s recovering 2023-08-28T02:38:31.700 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:38:31.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:31.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:38:31.821 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:38:31.840 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.0 dump_blocked_ops 2023-08-28T02:38:31.866 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:31.940 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:38:31.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:31.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:38:32.049 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:38:32.127 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:31 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 6385/113625 objects degraded (5.619%), 2 pgs degraded (PG_DEGRADED) 2023-08-28T02:38:32.127 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:31 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 6385/113625 objects degraded (5.619%), 2 pgs degraded (PG_DEGRADED) 2023-08-28T02:38:32.127 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:38:31 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:38:31] "GET /metrics HTTP/1.1" 200 36559 "" "Prometheus/2.33.4" 2023-08-28T02:38:32.130 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:32.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:31 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 6385/113625 objects degraded (5.619%), 2 pgs degraded (PG_DEGRADED) 2023-08-28T02:38:32.270 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:38:32.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:32.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:38:32.372 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set noscrub 2023-08-28T02:38:32.374 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:38:32.411 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:32.488 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:38:32.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:32.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:38:32.594 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:38:32.633 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:32.699 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:38:32.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:32.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:38:32.810 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:38:32.851 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:32.917 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:38:32.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:32.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:38:32.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:32 smithi169 ceph-mon[159415]: pgmap v954: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 6390/111333 objects degraded (5.740%); 9003/111333 objects misplaced (8.087%); 2.1 MiB/s, 33 objects/s recovering 2023-08-28T02:38:33.041 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:38:33.074 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:33.111 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:38:33.139 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:38:33.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:33.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:38:33.161 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:38:33.195 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:38:33.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:32 smithi143 ceph-mon[183059]: pgmap v954: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 6390/111333 objects degraded (5.740%); 9003/111333 objects misplaced (8.087%); 2.1 MiB/s, 33 objects/s recovering 2023-08-28T02:38:33.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:32 smithi143 ceph-mon[189871]: pgmap v954: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 6390/111333 objects degraded (5.740%); 9003/111333 objects misplaced (8.087%); 2.1 MiB/s, 33 objects/s recovering 2023-08-28T02:38:33.228 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:38:33.245 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:38:33.251 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:38:33.287 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:38:33.308 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:33.348 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:38:33.380 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:38:33.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:33.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:38:33.403 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:38:33.452 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:38:33.470 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:38:33.470 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:38:33.470 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:38:33.470 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:38:33.470 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:38:33.470 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:38:33.470 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:38:33.470 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:38:33.488 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:38:33.568 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:33.673 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:38:33.826 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:38:33.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:33.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:38:33.943 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:38:34.073 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:34.103 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:33 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1372250247' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:38:34.104 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:34 smithi143 conmon[183035]: 2023-08-28T02:38:34.100+0000 7fadc3887700 -1 mon.a@0(leader).osd e352 definitely_dead 0 2023-08-28T02:38:34.104 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:33 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1372250247' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:38:34.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:33 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1372250247' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:38:34.894 INFO:teuthology.orchestra.run.smithi143.stderr:marked out osd.2. 2023-08-28T02:38:35.139 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:38:35.142 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:34 smithi143 ceph-mon[183059]: pgmap v955: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 6390/107112 objects degraded (5.966%); 9003/107112 objects misplaced (8.405%); 2.1 MiB/s, 33 objects/s recovering 2023-08-28T02:38:35.142 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:34 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/541740192' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-08-28T02:38:35.142 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:34 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-08-28T02:38:35.142 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:34 smithi143 ceph-mon[183059]: Client client.admin marked osd.2 out, while it was still marked up 2023-08-28T02:38:35.142 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:34 smithi143 ceph-mon[189871]: pgmap v955: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 6390/107112 objects degraded (5.966%); 9003/107112 objects misplaced (8.405%); 2.1 MiB/s, 33 objects/s recovering 2023-08-28T02:38:35.142 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:34 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/541740192' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-08-28T02:38:35.142 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:34 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-08-28T02:38:35.143 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:34 smithi143 ceph-mon[189871]: Client client.admin marked osd.2 out, while it was still marked up 2023-08-28T02:38:35.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:35.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:38:35.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:34 smithi169 ceph-mon[159415]: pgmap v955: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 84 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 6390/107112 objects degraded (5.966%); 9003/107112 objects misplaced (8.405%); 2.1 MiB/s, 33 objects/s recovering 2023-08-28T02:38:35.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:34 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/541740192' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-08-28T02:38:35.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:34 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-08-28T02:38:35.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:34 smithi169 ceph-mon[159415]: Client client.admin marked osd.2 out, while it was still marked up 2023-08-28T02:38:35.245 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:38:35.271 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:35.318 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:38:35.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:35.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:38:35.429 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:38:35.457 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:36.069 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:35 smithi143 ceph-mon[183059]: failed to encode map e353 with expected crc 2023-08-28T02:38:36.069 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:35 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-08-28T02:38:36.069 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:35 smithi143 ceph-mon[183059]: osdmap e353: 8 total, 8 up, 7 in 2023-08-28T02:38:36.069 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:35 smithi143 ceph-mon[183059]: failed to encode map e353 with expected crc 2023-08-28T02:38:36.069 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:35 smithi143 ceph-mon[183059]: pgmap v957: 89 pgs: 1 remapped+peering, 3 peering, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 81 active+clean; 2.1 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 628 op/s; 6066/102924 objects degraded (5.894%); 6224/102924 objects misplaced (6.047%); 2.6 MiB/s, 41 objects/s recovering 2023-08-28T02:38:36.070 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:35 smithi143 ceph-mon[189871]: failed to encode map e353 with expected crc 2023-08-28T02:38:36.070 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:35 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-08-28T02:38:36.070 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:35 smithi143 ceph-mon[189871]: osdmap e353: 8 total, 8 up, 7 in 2023-08-28T02:38:36.070 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:35 smithi143 ceph-mon[189871]: failed to encode map e353 with expected crc 2023-08-28T02:38:36.070 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:35 smithi143 ceph-mon[189871]: pgmap v957: 89 pgs: 1 remapped+peering, 3 peering, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 81 active+clean; 2.1 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 628 op/s; 6066/102924 objects degraded (5.894%); 6224/102924 objects misplaced (6.047%); 2.6 MiB/s, 41 objects/s recovering 2023-08-28T02:38:36.209 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:38:36.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:36.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:38:36.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:35 smithi169 ceph-mon[159415]: failed to encode map e353 with expected crc 2023-08-28T02:38:36.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:35 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-08-28T02:38:36.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:35 smithi169 ceph-mon[159415]: osdmap e353: 8 total, 8 up, 7 in 2023-08-28T02:38:36.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:35 smithi169 ceph-mon[159415]: failed to encode map e353 with expected crc 2023-08-28T02:38:36.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:35 smithi169 ceph-mon[159415]: pgmap v957: 89 pgs: 1 remapped+peering, 3 peering, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 81 active+clean; 2.1 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 628 op/s; 6066/102924 objects degraded (5.894%); 6224/102924 objects misplaced (6.047%); 2.6 MiB/s, 41 objects/s recovering 2023-08-28T02:38:36.313 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:38:36.358 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:37.165 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:38:37.173 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:36 smithi143 ceph-mon[183059]: failed to encode map e354 with expected crc 2023-08-28T02:38:37.173 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:36 smithi143 ceph-mon[183059]: failed to encode map e354 with expected crc 2023-08-28T02:38:37.173 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:36 smithi143 ceph-mon[183059]: failed to encode map e354 with expected crc 2023-08-28T02:38:37.173 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:36 smithi143 ceph-mon[183059]: osdmap e354: 8 total, 8 up, 7 in 2023-08-28T02:38:37.173 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:36 smithi143 ceph-mon[183059]: failed to encode map e354 with expected crc 2023-08-28T02:38:37.173 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:36 smithi143 ceph-mon[183059]: failed to encode map e354 with expected crc 2023-08-28T02:38:37.173 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:36 smithi143 ceph-mon[183059]: failed to encode map e354 with expected crc 2023-08-28T02:38:37.173 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:36 smithi143 ceph-mon[183059]: failed to encode map e354 with expected crc 2023-08-28T02:38:37.173 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:36 smithi143 ceph-mon[183059]: failed to encode map e354 with expected crc 2023-08-28T02:38:37.173 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:36 smithi143 ceph-mon[183059]: failed to encode map e354 with expected crc 2023-08-28T02:38:37.174 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:36 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 6066/102924 objects degraded (5.894%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:38:37.174 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:36 smithi143 ceph-mon[189871]: failed to encode map e354 with expected crc 2023-08-28T02:38:37.174 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:36 smithi143 ceph-mon[189871]: failed to encode map e354 with expected crc 2023-08-28T02:38:37.174 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:36 smithi143 ceph-mon[189871]: failed to encode map e354 with expected crc 2023-08-28T02:38:37.174 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:36 smithi143 ceph-mon[189871]: osdmap e354: 8 total, 8 up, 7 in 2023-08-28T02:38:37.174 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:36 smithi143 ceph-mon[189871]: failed to encode map e354 with expected crc 2023-08-28T02:38:37.174 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:36 smithi143 ceph-mon[189871]: failed to encode map e354 with expected crc 2023-08-28T02:38:37.174 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:36 smithi143 ceph-mon[189871]: failed to encode map e354 with expected crc 2023-08-28T02:38:37.174 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:36 smithi143 ceph-mon[189871]: failed to encode map e354 with expected crc 2023-08-28T02:38:37.174 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:36 smithi143 ceph-mon[189871]: failed to encode map e354 with expected crc 2023-08-28T02:38:37.174 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:36 smithi143 ceph-mon[189871]: failed to encode map e354 with expected crc 2023-08-28T02:38:37.175 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:36 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 6066/102924 objects degraded (5.894%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:38:37.205 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:38:37.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:37.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:38:37.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:36 smithi169 ceph-mon[159415]: failed to encode map e354 with expected crc 2023-08-28T02:38:37.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:36 smithi169 ceph-mon[159415]: failed to encode map e354 with expected crc 2023-08-28T02:38:37.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:36 smithi169 ceph-mon[159415]: failed to encode map e354 with expected crc 2023-08-28T02:38:37.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:36 smithi169 ceph-mon[159415]: osdmap e354: 8 total, 8 up, 7 in 2023-08-28T02:38:37.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:36 smithi169 ceph-mon[159415]: failed to encode map e354 with expected crc 2023-08-28T02:38:37.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:36 smithi169 ceph-mon[159415]: failed to encode map e354 with expected crc 2023-08-28T02:38:37.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:36 smithi169 ceph-mon[159415]: failed to encode map e354 with expected crc 2023-08-28T02:38:37.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:36 smithi169 ceph-mon[159415]: failed to encode map e354 with expected crc 2023-08-28T02:38:37.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:36 smithi169 ceph-mon[159415]: failed to encode map e354 with expected crc 2023-08-28T02:38:37.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:36 smithi169 ceph-mon[159415]: failed to encode map e354 with expected crc 2023-08-28T02:38:37.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:36 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 6066/102924 objects degraded (5.894%), 1 pg degraded (PG_DEGRADED) 2023-08-28T02:38:37.310 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:38:37.337 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:37.392 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:38:37.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:37.396 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:38:37.497 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:38:37.544 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:37.887 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:37 smithi143 ceph-mon[183059]: failed to encode map e355 with expected crc 2023-08-28T02:38:37.888 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:37 smithi143 ceph-mon[183059]: osdmap e355: 8 total, 8 up, 7 in 2023-08-28T02:38:37.888 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:37 smithi143 ceph-mon[183059]: failed to encode map e355 with expected crc 2023-08-28T02:38:37.888 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:37 smithi143 ceph-mon[183059]: failed to encode map e355 with expected crc 2023-08-28T02:38:37.888 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:37 smithi143 ceph-mon[183059]: 8.e starting backfill to osd.7 from (0'0,0'0] MAX to 352'10058 2023-08-28T02:38:37.888 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:37 smithi143 ceph-mon[183059]: 8.1 starting backfill to osd.5 from (0'0,0'0] MAX to 352'9920 2023-08-28T02:38:37.888 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:37 smithi143 ceph-mon[183059]: 8.4 starting backfill to osd.6 from (0'0,0'0] MAX to 352'10008 2023-08-28T02:38:37.888 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:37 smithi143 ceph-mon[183059]: 8.a starting backfill to osd.3 from (0'0,0'0] MAX to 352'9853 2023-08-28T02:38:37.888 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:37 smithi143 ceph-mon[183059]: 8.f starting backfill to osd.4 from (0'0,0'0] MAX to 352'10100 2023-08-28T02:38:37.889 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:37 smithi143 ceph-mon[183059]: pgmap v960: 89 pgs: 1 remapped+peering, 3 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 79 active+clean; 2.1 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 536 op/s; 1782/101667 objects degraded (1.753%); 10240/101667 objects misplaced (10.072%); 3.5 MiB/s, 56 objects/s recovering 2023-08-28T02:38:37.924 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:38:37.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:37.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:38:38.035 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:38:38.109 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:38.185 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:37 smithi143 ceph-mon[189871]: failed to encode map e355 with expected crc 2023-08-28T02:38:38.185 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:37 smithi143 ceph-mon[189871]: osdmap e355: 8 total, 8 up, 7 in 2023-08-28T02:38:38.185 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:37 smithi143 ceph-mon[189871]: failed to encode map e355 with expected crc 2023-08-28T02:38:38.185 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:37 smithi143 ceph-mon[189871]: failed to encode map e355 with expected crc 2023-08-28T02:38:38.185 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:37 smithi143 ceph-mon[189871]: 8.e starting backfill to osd.7 from (0'0,0'0] MAX to 352'10058 2023-08-28T02:38:38.185 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:37 smithi143 ceph-mon[189871]: 8.1 starting backfill to osd.5 from (0'0,0'0] MAX to 352'9920 2023-08-28T02:38:38.185 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:37 smithi143 ceph-mon[189871]: 8.4 starting backfill to osd.6 from (0'0,0'0] MAX to 352'10008 2023-08-28T02:38:38.185 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:37 smithi143 ceph-mon[189871]: 8.a starting backfill to osd.3 from (0'0,0'0] MAX to 352'9853 2023-08-28T02:38:38.185 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:37 smithi143 ceph-mon[189871]: 8.f starting backfill to osd.4 from (0'0,0'0] MAX to 352'10100 2023-08-28T02:38:38.185 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:37 smithi143 ceph-mon[189871]: pgmap v960: 89 pgs: 1 remapped+peering, 3 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 79 active+clean; 2.1 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 536 op/s; 1782/101667 objects degraded (1.753%); 10240/101667 objects misplaced (10.072%); 3.5 MiB/s, 56 objects/s recovering 2023-08-28T02:38:38.228 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:38:38.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:37 smithi169 ceph-mon[159415]: failed to encode map e355 with expected crc 2023-08-28T02:38:38.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:37 smithi169 ceph-mon[159415]: osdmap e355: 8 total, 8 up, 7 in 2023-08-28T02:38:38.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:37 smithi169 ceph-mon[159415]: failed to encode map e355 with expected crc 2023-08-28T02:38:38.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:37 smithi169 ceph-mon[159415]: failed to encode map e355 with expected crc 2023-08-28T02:38:38.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:37 smithi169 ceph-mon[159415]: 8.e starting backfill to osd.7 from (0'0,0'0] MAX to 352'10058 2023-08-28T02:38:38.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:37 smithi169 ceph-mon[159415]: 8.1 starting backfill to osd.5 from (0'0,0'0] MAX to 352'9920 2023-08-28T02:38:38.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:37 smithi169 ceph-mon[159415]: 8.4 starting backfill to osd.6 from (0'0,0'0] MAX to 352'10008 2023-08-28T02:38:38.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:37 smithi169 ceph-mon[159415]: 8.a starting backfill to osd.3 from (0'0,0'0] MAX to 352'9853 2023-08-28T02:38:38.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:37 smithi169 ceph-mon[159415]: 8.f starting backfill to osd.4 from (0'0,0'0] MAX to 352'10100 2023-08-28T02:38:38.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:37 smithi169 ceph-mon[159415]: pgmap v960: 89 pgs: 1 remapped+peering, 3 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 79 active+clean; 2.1 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 536 op/s; 1782/101667 objects degraded (1.753%); 10240/101667 objects misplaced (10.072%); 3.5 MiB/s, 56 objects/s recovering 2023-08-28T02:38:38.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:38.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:38:38.340 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:38:38.445 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:38.507 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:38:38.507 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [], 2023-08-28T02:38:38.507 INFO:teuthology.orchestra.run.smithi143.stdout: "complaint_time": 30, 2023-08-28T02:38:38.507 INFO:teuthology.orchestra.run.smithi143.stdout: "num_blocked_ops": 0 2023-08-28T02:38:38.508 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:38:38.585 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:38:38.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:38.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:38:38.692 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:38:38.810 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:38.917 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:38:38.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:38.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:38:39.021 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:38:39.048 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:39.064 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is set 2023-08-28T02:38:39.094 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:38:39.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:39.097 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:38:39.198 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:38:39.224 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:39.307 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:38:39.318 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:39 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3909112325' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:38:39.318 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:39 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:38:39.318 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:39 smithi169 ceph-mon[159415]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:38:39.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:39.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:38:39.420 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:38:39.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:39 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3909112325' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:38:39.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:39 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:38:39.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:39 smithi143 ceph-mon[189871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:38:39.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:39 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3909112325' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:38:39.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:39 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:38:39.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:39 smithi143 ceph-mon[183059]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:38:39.530 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:39.644 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:38:39.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:39.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:38:39.764 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:38:39.848 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:39.921 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:38:39.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:39.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:38:40.025 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:38:40.056 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:40.199 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:38:40.227 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.0 dump_historic_ops 2023-08-28T02:38:40.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:40 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:38:40.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:40 smithi169 ceph-mon[159415]: osdmap e356: 8 total, 8 up, 7 in 2023-08-28T02:38:40.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:40 smithi169 ceph-mon[159415]: failed to encode map e356 with expected crc 2023-08-28T02:38:40.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:40 smithi169 ceph-mon[159415]: failed to encode map e356 with expected crc 2023-08-28T02:38:40.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:40 smithi169 ceph-mon[159415]: pgmap v962: 89 pgs: 1 remapped+peering, 3 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 75 active+clean; 2.0 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 398 op/s; 1789/99390 objects degraded (1.800%); 16300/99390 objects misplaced (16.400%); 4.5 MiB/s, 2 keys/s, 71 objects/s recovering 2023-08-28T02:38:40.333 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:40 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:38:40.333 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:40 smithi143 ceph-mon[183059]: osdmap e356: 8 total, 8 up, 7 in 2023-08-28T02:38:40.334 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:40 smithi143 ceph-mon[183059]: failed to encode map e356 with expected crc 2023-08-28T02:38:40.334 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:40 smithi143 ceph-mon[183059]: failed to encode map e356 with expected crc 2023-08-28T02:38:40.334 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:40 smithi143 ceph-mon[183059]: pgmap v962: 89 pgs: 1 remapped+peering, 3 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 75 active+clean; 2.0 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 398 op/s; 1789/99390 objects degraded (1.800%); 16300/99390 objects misplaced (16.400%); 4.5 MiB/s, 2 keys/s, 71 objects/s recovering 2023-08-28T02:38:40.334 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:40 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:38:40.334 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:40 smithi143 ceph-mon[189871]: osdmap e356: 8 total, 8 up, 7 in 2023-08-28T02:38:40.334 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:40 smithi143 ceph-mon[189871]: failed to encode map e356 with expected crc 2023-08-28T02:38:40.334 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:40 smithi143 ceph-mon[189871]: failed to encode map e356 with expected crc 2023-08-28T02:38:40.334 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:40 smithi143 ceph-mon[189871]: pgmap v962: 89 pgs: 1 remapped+peering, 3 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 75 active+clean; 2.0 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 398 op/s; 1789/99390 objects degraded (1.800%); 16300/99390 objects misplaced (16.400%); 4.5 MiB/s, 2 keys/s, 71 objects/s recovering 2023-08-28T02:38:40.347 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:38:40.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:40.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:38:40.454 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:38:40.505 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:40.581 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:38:40.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:40.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:38:40.693 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:38:40.770 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:40.897 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:38:40.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:40.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:38:41.010 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:38:41.078 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:41.129 INFO:tasks.thrashosds.thrasher:in_osds: [0, 4, 5, 3, 6, 1, 7] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-08-28T02:38:41.129 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-08-28T02:38:41.129 INFO:tasks.thrashosds.thrasher:Growing pool device_health_metrics 2023-08-28T02:38:41.130 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph pg dump --format=json 2023-08-28T02:38:41.207 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:38:41.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:41.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:38:41.312 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:38:41.350 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:41 smithi169 ceph-mon[159415]: failed to encode map e356 with expected crc 2023-08-28T02:38:41.350 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:41 smithi169 ceph-mon[159415]: failed to encode map e356 with expected crc 2023-08-28T02:38:41.350 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:41 smithi169 ceph-mon[159415]: failed to encode map e356 with expected crc 2023-08-28T02:38:41.351 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:41 smithi169 ceph-mon[159415]: failed to encode map e356 with expected crc 2023-08-28T02:38:41.351 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:41 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:38:41.355 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:41.451 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:38:41.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:41 smithi143 ceph-mon[183059]: failed to encode map e356 with expected crc 2023-08-28T02:38:41.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:41 smithi143 ceph-mon[183059]: failed to encode map e356 with expected crc 2023-08-28T02:38:41.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:41 smithi143 ceph-mon[183059]: failed to encode map e356 with expected crc 2023-08-28T02:38:41.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:41 smithi143 ceph-mon[183059]: failed to encode map e356 with expected crc 2023-08-28T02:38:41.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:41 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:38:41.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:41 smithi143 ceph-mon[189871]: failed to encode map e356 with expected crc 2023-08-28T02:38:41.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:41 smithi143 ceph-mon[189871]: failed to encode map e356 with expected crc 2023-08-28T02:38:41.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:41 smithi143 ceph-mon[189871]: failed to encode map e356 with expected crc 2023-08-28T02:38:41.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:41 smithi143 ceph-mon[189871]: failed to encode map e356 with expected crc 2023-08-28T02:38:41.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:41 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:38:41.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:41.471 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:38:41.571 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:38:41.598 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:41.658 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:38:41.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:41.666 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:38:41.767 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:38:41.814 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:42.001 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:38:42.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:42.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:38:42.008 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:38:42.089 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:38:41 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:38:41] "GET /metrics HTTP/1.1" 200 36573 "" "Prometheus/2.33.4" 2023-08-28T02:38:42.090 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:42 smithi143 ceph-mon[183059]: pgmap v963: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 2.0 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 256 op/s; 3523/98304 objects degraded (3.584%); 22695/98304 objects misplaced (23.087%); 1.9 MiB/s, 1 keys/s, 29 objects/s recovering 2023-08-28T02:38:42.090 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:42 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 1789/99390 objects degraded (1.800%), 3 pgs degraded (PG_DEGRADED) 2023-08-28T02:38:42.091 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:42 smithi143 ceph-mon[189871]: pgmap v963: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 2.0 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 256 op/s; 3523/98304 objects degraded (3.584%); 22695/98304 objects misplaced (23.087%); 1.9 MiB/s, 1 keys/s, 29 objects/s recovering 2023-08-28T02:38:42.091 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:42 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 1789/99390 objects degraded (1.800%), 3 pgs degraded (PG_DEGRADED) 2023-08-28T02:38:42.109 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:38:42.176 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:38:42.204 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:42.371 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:38:42.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:42.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:38:42.475 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:38:42.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:42 smithi169 ceph-mon[159415]: pgmap v963: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 2.0 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 256 op/s; 3523/98304 objects degraded (3.584%); 22695/98304 objects misplaced (23.087%); 1.9 MiB/s, 1 keys/s, 29 objects/s recovering 2023-08-28T02:38:42.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:42 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 1789/99390 objects degraded (1.800%), 3 pgs degraded (PG_DEGRADED) 2023-08-28T02:38:42.534 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:42.544 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set nodeep-scrub 2023-08-28T02:38:42.640 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:38:42.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:42.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:38:42.755 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:38:42.913 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:43.336 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:43 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/666086585' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:38:43.336 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:43 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/666086585' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:38:43.489 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:43 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/666086585' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:38:43.653 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:38:43.653 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:38:43.653 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:38:43.653 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:38:43.654 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:38:43.654 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:38:43.654 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:38:43.654 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:38:43.654 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:38:43.654 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:38:43.654 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:38:43.654 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:38:43.654 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:38:43.654 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:38:43.655 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:38:43.655 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:38:44.345 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:38:44.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:44.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:38:44.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:44 smithi143 ceph-mon[183059]: pgmap v964: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 1.9 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 374 op/s; 3523/94218 objects degraded (3.739%); 21982/94218 objects misplaced (23.331%); 3.0 MiB/s, 1 keys/s, 47 objects/s recovering 2023-08-28T02:38:44.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:44 smithi143 ceph-mon[189871]: pgmap v964: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 1.9 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 374 op/s; 3523/94218 objects degraded (3.739%); 21982/94218 objects misplaced (23.331%); 3.0 MiB/s, 1 keys/s, 47 objects/s recovering 2023-08-28T02:38:44.453 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:38:44.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:44 smithi169 ceph-mon[159415]: pgmap v964: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 1.9 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 374 op/s; 3523/94218 objects degraded (3.739%); 21982/94218 objects misplaced (23.331%); 3.0 MiB/s, 1 keys/s, 47 objects/s recovering 2023-08-28T02:38:44.504 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:45.330 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:38:45.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:45.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:38:45.436 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:38:45.478 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:45.551 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:38:45.578 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:38:45.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:45.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:38:45.690 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:38:45.774 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:45.894 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:38:45.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:45.905 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:38:46.006 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:38:46.071 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:46.158 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:38:46.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:46.162 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:38:46.263 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:38:46.335 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:46.353 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:38:46.353 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:38:46.368 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:38:46.390 INFO:teuthology.orchestra.run.smithi143.stderr:dumped all 2023-08-28T02:38:46.431 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:38:46.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:46.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:38:46.548 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:38:46.605 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:46.683 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:46 smithi143 ceph-mon[189871]: pgmap v965: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 1.8 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 493 op/s; 3523/89979 objects degraded (3.915%); 21077/89979 objects misplaced (23.424%); 2.6 MiB/s, 1 keys/s, 42 objects/s recovering 2023-08-28T02:38:46.683 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:46 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 3523/89979 objects degraded (3.915%), 4 pgs degraded (PG_DEGRADED) 2023-08-28T02:38:46.683 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:46 smithi143 ceph-mon[183059]: pgmap v965: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 1.8 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 493 op/s; 3523/89979 objects degraded (3.915%); 21077/89979 objects misplaced (23.424%); 2.6 MiB/s, 1 keys/s, 42 objects/s recovering 2023-08-28T02:38:46.683 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:46 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 3523/89979 objects degraded (3.915%), 4 pgs degraded (PG_DEGRADED) 2023-08-28T02:38:46.728 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:38:46.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:46.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:38:46.842 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:38:46.905 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:46 smithi169 ceph-mon[159415]: pgmap v965: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 1.8 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 493 op/s; 3523/89979 objects degraded (3.915%); 21077/89979 objects misplaced (23.424%); 2.6 MiB/s, 1 keys/s, 42 objects/s recovering 2023-08-28T02:38:46.905 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:46 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 3523/89979 objects degraded (3.915%), 4 pgs degraded (PG_DEGRADED) 2023-08-28T02:38:46.915 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:47.036 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:38:47.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:47.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:38:47.145 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:38:47.213 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:47.290 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:38:47.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:47 smithi143 ceph-mon[183059]: from='client.45857 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:38:47.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:47 smithi143 ceph-mon[189871]: from='client.45857 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:38:47.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:47 smithi169 ceph-mon[159415]: from='client.45857 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:38:47.991 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:38:48.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:48.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:38:48.106 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:38:48.133 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:48.198 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:38:48.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:48.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:38:48.304 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:38:48.381 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:48.497 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:38:48.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:48.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:38:48.584 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.1 dump_ops_in_flight 2023-08-28T02:38:48.602 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:38:48.719 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:48.735 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:48 smithi169 ceph-mon[159415]: pgmap v966: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 1.8 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 465 op/s; 3523/87690 objects degraded (4.018%); 20736/87690 objects misplaced (23.647%); 2.2 MiB/s, 0 keys/s, 35 objects/s recovering 2023-08-28T02:38:48.846 INFO:teuthology.orchestra.run.smithi143.stdout:{"pg_ready":true,"pg_map":{"version":965,"stamp":"2023-08-28T02:38:45.618087+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1968510923,"num_objects":29993,"num_object_clones":0,"num_object_copies":89979,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3523,"num_objects_misplaced":21077,"num_objects_unfound":0,"num_objects_dirty":29993,"num_whiteouts":0,"num_read":6275,"num_read_kb":13708,"num_write":164957,"num_write_kb":6217190,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2206,"num_bytes_recovered":138248136,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124003,"ondisk_log_size":124003,"up":267,"acting":263,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":270,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":8890692,"kb_used_data":5293772,"kb_used_omap":27,"kb_used_meta":3596772,"kb_avail":647325372,"statfs":{"total":671965249536,"available":662861180928,"internally_reserved":0,"allocated":5420822528,"data_stored":5412100474,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28409,"internal_metadata":3683094791},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,1,3,4,1,0,0,2],"upper_bound":512},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-282787840,"num_objects":-4315,"num_object_clones":0,"num_object_copies":-12945,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-2543,"num_objects_misplaced":14853,"num_objects_unfound":0,"num_objects_dirty":-4315,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4315,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":369,"num_bytes_recovered":24314466,"num_keys_recovered":10,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.750352"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":384,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.875246+0000","last_change":"2023-08-28T02:38:35.886897+0000","last_active":"2023-08-28T02:38:36.875246+0000","last_peered":"2023-08-28T02:38:36.875246+0000","last_clean":"2023-08-28T02:38:36.875246+0000","last_became_active":"2023-08-28T02:38:35.886527+0000","last_became_peered":"2023-08-28T02:38:35.886527+0000","last_unstale":"2023-08-28T02:38:36.875246+0000","last_undegraded":"2023-08-28T02:38:36.875246+0000","last_fullsized":"2023-08-28T02:38:36.875246+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":263,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.880953+0000","last_change":"2023-08-28T02:32:58.634375+0000","last_active":"2023-08-28T02:38:36.880953+0000","last_peered":"2023-08-28T02:38:36.880953+0000","last_clean":"2023-08-28T02:38:36.880953+0000","last_became_active":"2023-08-28T02:32:58.632726+0000","last_became_peered":"2023-08-28T02:32:58.632726+0000","last_unstale":"2023-08-28T02:38:36.880953+0000","last_undegraded":"2023-08-28T02:38:36.880953+0000","last_fullsized":"2023-08-28T02:38:36.880953+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:20:27.730725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":77,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.903966+0000","last_change":"2023-08-28T02:32:58.628387+0000","last_active":"2023-08-28T02:38:36.903966+0000","last_peered":"2023-08-28T02:38:36.903966+0000","last_clean":"2023-08-28T02:38:36.903966+0000","last_became_active":"2023-08-28T02:32:58.628071+0000","last_became_peered":"2023-08-28T02:32:58.628071+0000","last_unstale":"2023-08-28T02:38:36.903966+0000","last_undegraded":"2023-08-28T02:38:36.903966+0000","last_fullsized":"2023-08-28T02:38:36.903966+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:25:55.264433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":384,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-08-28T02:38:39.060263+0000","last_change":"2023-08-28T02:37:45.580751+0000","last_active":"2023-08-28T02:38:39.060263+0000","last_peered":"2023-08-28T02:38:39.060263+0000","last_clean":"2023-08-28T02:38:39.060263+0000","last_became_active":"2023-08-28T02:37:45.580446+0000","last_became_peered":"2023-08-28T02:37:45.580446+0000","last_unstale":"2023-08-28T02:38:39.060263+0000","last_undegraded":"2023-08-28T02:38:39.060263+0000","last_fullsized":"2023-08-28T02:38:39.060263+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:59:21.094880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":255,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.876664+0000","last_change":"2023-08-28T02:32:58.779602+0000","last_active":"2023-08-28T02:38:36.876664+0000","last_peered":"2023-08-28T02:38:36.876664+0000","last_clean":"2023-08-28T02:38:36.876664+0000","last_became_active":"2023-08-28T02:32:58.778945+0000","last_became_peered":"2023-08-28T02:32:58.778945+0000","last_unstale":"2023-08-28T02:38:36.876664+0000","last_undegraded":"2023-08-28T02:38:36.876664+0000","last_fullsized":"2023-08-28T02:38:36.876664+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:35:55.832928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":162,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.876830+0000","last_change":"2023-08-28T02:32:58.626766+0000","last_active":"2023-08-28T02:38:36.876830+0000","last_peered":"2023-08-28T02:38:36.876830+0000","last_clean":"2023-08-28T02:38:36.876830+0000","last_became_active":"2023-08-28T02:32:58.626415+0000","last_became_peered":"2023-08-28T02:32:58.626415+0000","last_unstale":"2023-08-28T02:38:36.876830+0000","last_undegraded":"2023-08-28T02:38:36.876830+0000","last_fullsized":"2023-08-28T02:38:36.876830+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":327,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.873490+0000","last_change":"2023-08-28T02:23:59.227592+0000","last_active":"2023-08-28T02:38:36.873490+0000","last_peered":"2023-08-28T02:38:36.873490+0000","last_clean":"2023-08-28T02:38:36.873490+0000","last_became_active":"2023-08-28T02:23:59.227342+0000","last_became_peered":"2023-08-28T02:23:59.227342+0000","last_unstale":"2023-08-28T02:38:36.873490+0000","last_undegraded":"2023-08-28T02:38:36.873490+0000","last_fullsized":"2023-08-28T02:38:36.873490+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:08:19.874396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":273,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.876751+0000","last_change":"2023-08-28T02:32:58.779435+0000","last_active":"2023-08-28T02:38:36.876751+0000","last_peered":"2023-08-28T02:38:36.876751+0000","last_clean":"2023-08-28T02:38:36.876751+0000","last_became_active":"2023-08-28T02:32:58.778629+0000","last_became_peered":"2023-08-28T02:32:58.778629+0000","last_unstale":"2023-08-28T02:38:36.876751+0000","last_undegraded":"2023-08-28T02:38:36.876751+0000","last_fullsized":"2023-08-28T02:38:36.876751+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:25:32.193898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":21,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.873558+0000","last_change":"2023-08-28T02:37:45.579563+0000","last_active":"2023-08-28T02:38:36.873558+0000","last_peered":"2023-08-28T02:38:36.873558+0000","last_clean":"2023-08-28T02:38:36.873558+0000","last_became_active":"2023-08-28T02:37:45.579133+0000","last_became_peered":"2023-08-28T02:37:45.579133+0000","last_unstale":"2023-08-28T02:38:36.873558+0000","last_undegraded":"2023-08-28T02:38:36.873558+0000","last_fullsized":"2023-08-28T02:38:36.873558+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":255,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.881599+0000","last_change":"2023-08-28T02:37:45.580277+0000","last_active":"2023-08-28T02:38:36.881599+0000","last_peered":"2023-08-28T02:38:36.881599+0000","last_clean":"2023-08-28T02:38:36.881599+0000","last_became_active":"2023-08-28T02:37:45.579888+0000","last_became_peered":"2023-08-28T02:37:45.579888+0000","last_unstale":"2023-08-28T02:38:36.881599+0000","last_undegraded":"2023-08-28T02:38:36.881599+0000","last_fullsized":"2023-08-28T02:38:36.881599+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:18:54.085955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":27,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.873905+0000","last_change":"2023-08-28T02:37:46.016429+0000","last_active":"2023-08-28T02:38:36.873905+0000","last_peered":"2023-08-28T02:38:36.873905+0000","last_clean":"2023-08-28T02:38:36.873905+0000","last_became_active":"2023-08-28T02:37:45.581897+0000","last_became_peered":"2023-08-28T02:37:45.581897+0000","last_unstale":"2023-08-28T02:38:36.873905+0000","last_undegraded":"2023-08-28T02:38:36.873905+0000","last_fullsized":"2023-08-28T02:38:36.873905+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":96,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.876123+0000","last_change":"2023-08-28T02:38:35.885068+0000","last_active":"2023-08-28T02:38:36.876123+0000","last_peered":"2023-08-28T02:38:36.876123+0000","last_clean":"2023-08-28T02:38:36.876123+0000","last_became_active":"2023-08-28T02:38:35.884670+0000","last_became_peered":"2023-08-28T02:38:35.884670+0000","last_unstale":"2023-08-28T02:38:36.876123+0000","last_undegraded":"2023-08-28T02:38:36.876123+0000","last_fullsized":"2023-08-28T02:38:36.876123+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"86'22","reported_seq":444,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.875162+0000","last_change":"2023-08-28T02:37:45.597318+0000","last_active":"2023-08-28T02:38:36.875162+0000","last_peered":"2023-08-28T02:38:36.875162+0000","last_clean":"2023-08-28T02:38:36.875162+0000","last_became_active":"2023-08-28T02:37:45.578060+0000","last_became_peered":"2023-08-28T02:37:45.578060+0000","last_unstale":"2023-08-28T02:38:36.875162+0000","last_undegraded":"2023-08-28T02:38:36.875162+0000","last_fullsized":"2023-08-28T02:38:36.875162+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":28404,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":399,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.873743+0000","last_change":"2023-08-28T02:32:58.629451+0000","last_active":"2023-08-28T02:38:36.873743+0000","last_peered":"2023-08-28T02:38:36.873743+0000","last_clean":"2023-08-28T02:38:36.873743+0000","last_became_active":"2023-08-28T02:32:58.628767+0000","last_became_peered":"2023-08-28T02:32:58.628767+0000","last_unstale":"2023-08-28T02:38:36.873743+0000","last_undegraded":"2023-08-28T02:38:36.873743+0000","last_fullsized":"2023-08-28T02:38:36.873743+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":42,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.873476+0000","last_change":"2023-08-28T02:38:36.016884+0000","last_active":"2023-08-28T02:38:36.873476+0000","last_peered":"2023-08-28T02:38:36.873476+0000","last_clean":"2023-08-28T02:38:36.873476+0000","last_became_active":"2023-08-28T02:38:35.882996+0000","last_became_peered":"2023-08-28T02:38:35.882996+0000","last_unstale":"2023-08-28T02:38:36.873476+0000","last_undegraded":"2023-08-28T02:38:36.873476+0000","last_fullsized":"2023-08-28T02:38:36.873476+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":273,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.876592+0000","last_change":"2023-08-28T02:37:45.579069+0000","last_active":"2023-08-28T02:38:36.876592+0000","last_peered":"2023-08-28T02:38:36.876592+0000","last_clean":"2023-08-28T02:38:36.876592+0000","last_became_active":"2023-08-28T02:37:45.578728+0000","last_became_peered":"2023-08-28T02:37:45.578728+0000","last_unstale":"2023-08-28T02:38:36.876592+0000","last_undegraded":"2023-08-28T02:38:36.876592+0000","last_fullsized":"2023-08-28T02:38:36.876592+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:58:05.172104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":381,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.873037+0000","last_change":"2023-08-28T02:37:45.579678+0000","last_active":"2023-08-28T02:38:36.873037+0000","last_peered":"2023-08-28T02:38:36.873037+0000","last_clean":"2023-08-28T02:38:36.873037+0000","last_became_active":"2023-08-28T02:37:45.579390+0000","last_became_peered":"2023-08-28T02:37:45.579390+0000","last_unstale":"2023-08-28T02:38:36.873037+0000","last_undegraded":"2023-08-28T02:38:36.873037+0000","last_fullsized":"2023-08-28T02:38:36.873037+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:31:04.974167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":21,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.873831+0000","last_change":"2023-08-28T02:37:45.584893+0000","last_active":"2023-08-28T02:38:36.873831+0000","last_peered":"2023-08-28T02:38:36.873831+0000","last_clean":"2023-08-28T02:38:36.873831+0000","last_became_active":"2023-08-28T02:37:45.584568+0000","last_became_peered":"2023-08-28T02:37:45.584568+0000","last_unstale":"2023-08-28T02:38:36.873831+0000","last_undegraded":"2023-08-28T02:38:36.873831+0000","last_fullsized":"2023-08-28T02:38:36.873831+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":21,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.873161+0000","last_change":"2023-08-28T02:37:45.577769+0000","last_active":"2023-08-28T02:38:36.873161+0000","last_peered":"2023-08-28T02:38:36.873161+0000","last_clean":"2023-08-28T02:38:36.873161+0000","last_became_active":"2023-08-28T02:37:45.577446+0000","last_became_peered":"2023-08-28T02:37:45.577446+0000","last_unstale":"2023-08-28T02:38:36.873161+0000","last_undegraded":"2023-08-28T02:38:36.873161+0000","last_fullsized":"2023-08-28T02:38:36.873161+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":364,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.884874+0000","last_change":"2023-08-28T02:37:45.580634+0000","last_active":"2023-08-28T02:38:36.884874+0000","last_peered":"2023-08-28T02:38:36.884874+0000","last_clean":"2023-08-28T02:38:36.884874+0000","last_became_active":"2023-08-28T02:37:45.580322+0000","last_became_peered":"2023-08-28T02:37:45.580322+0000","last_unstale":"2023-08-28T02:38:36.884874+0000","last_undegraded":"2023-08-28T02:38:36.884874+0000","last_fullsized":"2023-08-28T02:38:36.884874+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:24:37.014061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":154,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.877131+0000","last_change":"2023-08-28T02:28:25.703333+0000","last_active":"2023-08-28T02:38:36.877131+0000","last_peered":"2023-08-28T02:38:36.877131+0000","last_clean":"2023-08-28T02:38:36.877131+0000","last_became_active":"2023-08-28T02:28:25.343515+0000","last_became_peered":"2023-08-28T02:28:25.343515+0000","last_unstale":"2023-08-28T02:38:36.877131+0000","last_undegraded":"2023-08-28T02:38:36.877131+0000","last_fullsized":"2023-08-28T02:38:36.877131+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":282,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.899177+0000","last_change":"2023-08-28T02:37:45.579694+0000","last_active":"2023-08-28T02:38:36.899177+0000","last_peered":"2023-08-28T02:38:36.899177+0000","last_clean":"2023-08-28T02:38:36.899177+0000","last_became_active":"2023-08-28T02:37:45.579219+0000","last_became_peered":"2023-08-28T02:37:45.579219+0000","last_unstale":"2023-08-28T02:38:36.899177+0000","last_undegraded":"2023-08-28T02:38:36.899177+0000","last_fullsized":"2023-08-28T02:38:36.899177+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:22:51.406552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"356'10180","reported_seq":10236,"reported_epoch":356,"state":"active+remapped+backfill_wait","last_fresh":"2023-08-28T02:38:42.038722+0000","last_change":"2023-08-28T02:38:37.060359+0000","last_active":"2023-08-28T02:38:42.038722+0000","last_peered":"2023-08-28T02:38:42.038722+0000","last_clean":"2023-08-28T02:38:34.857311+0000","last_became_active":"2023-08-28T02:38:37.023094+0000","last_became_peered":"2023-08-28T02:38:37.023094+0000","last_unstale":"2023-08-28T02:38:42.038722+0000","last_undegraded":"2023-08-28T02:38:42.038722+0000","last_fullsized":"2023-08-28T02:38:42.038722+0000","mapping_epoch":354,"log_start":"326'2000","ondisk_log_start":"326'2000","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":8180,"log_dups_size":0,"ondisk_log_size":8180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125566976,"num_objects":1916,"num_object_clones":0,"num_object_copies":5748,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1916,"num_objects_unfound":0,"num_objects_dirty":1916,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10180,"num_write_kb":387072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,2],"avail_no_missing":["4","2","5"],"object_location_counts":[{"shards":"2,4,5","objects":1916}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":38,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.874603+0000","last_change":"2023-08-28T02:38:35.929952+0000","last_active":"2023-08-28T02:38:36.874603+0000","last_peered":"2023-08-28T02:38:36.874603+0000","last_clean":"2023-08-28T02:38:36.874603+0000","last_became_active":"2023-08-28T02:38:35.882215+0000","last_became_peered":"2023-08-28T02:38:35.882215+0000","last_unstale":"2023-08-28T02:38:36.874603+0000","last_undegraded":"2023-08-28T02:38:36.874603+0000","last_fullsized":"2023-08-28T02:38:36.874603+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":21,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.873326+0000","last_change":"2023-08-28T02:37:45.585246+0000","last_active":"2023-08-28T02:38:36.873326+0000","last_peered":"2023-08-28T02:38:36.873326+0000","last_clean":"2023-08-28T02:38:36.873326+0000","last_became_active":"2023-08-28T02:37:45.584391+0000","last_became_peered":"2023-08-28T02:37:45.584391+0000","last_unstale":"2023-08-28T02:38:36.873326+0000","last_undegraded":"2023-08-28T02:38:36.873326+0000","last_fullsized":"2023-08-28T02:38:36.873326+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.5","version":"356'10186","reported_seq":10973,"reported_epoch":356,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-08-28T02:38:42.299047+0000","last_change":"2023-08-28T02:38:35.930624+0000","last_active":"2023-08-28T02:38:42.299047+0000","last_peered":"2023-08-28T02:38:42.299047+0000","last_clean":"2023-08-28T02:37:44.558549+0000","last_became_active":"2023-08-28T02:37:47.079792+0000","last_became_peered":"2023-08-28T02:37:47.079792+0000","last_unstale":"2023-08-28T02:38:42.299047+0000","last_undegraded":"2023-08-28T02:37:46.622481+0000","last_fullsized":"2023-08-28T02:37:46.586056+0000","mapping_epoch":345,"log_start":"330'2446","ondisk_log_start":"330'2446","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":7740,"log_dups_size":0,"ondisk_log_size":7740,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127926272,"num_objects":1952,"num_object_clones":0,"num_object_copies":5856,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1773,"num_objects_misplaced":1738,"num_objects_unfound":0,"num_objects_dirty":1952,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10186,"num_write_kb":388416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":366,"num_bytes_recovered":23789568,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4],"avail_no_missing":["6","0","4"],"object_location_counts":[{"shards":"0,4,6","objects":1738},{"shards":"4,6","objects":-1559}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":25,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.873476+0000","last_change":"2023-08-28T02:37:45.669932+0000","last_active":"2023-08-28T02:38:36.873476+0000","last_peered":"2023-08-28T02:38:36.873476+0000","last_clean":"2023-08-28T02:38:36.873476+0000","last_became_active":"2023-08-28T02:37:45.572825+0000","last_became_peered":"2023-08-28T02:37:45.572825+0000","last_unstale":"2023-08-28T02:38:36.873476+0000","last_undegraded":"2023-08-28T02:38:36.873476+0000","last_fullsized":"2023-08-28T02:38:36.873476+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":409,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.873436+0000","last_change":"2023-08-28T02:32:58.740396+0000","last_active":"2023-08-28T02:38:36.873436+0000","last_peered":"2023-08-28T02:38:36.873436+0000","last_clean":"2023-08-28T02:38:36.873436+0000","last_became_active":"2023-08-28T02:32:58.739941+0000","last_became_peered":"2023-08-28T02:32:58.739941+0000","last_unstale":"2023-08-28T02:38:36.873436+0000","last_undegraded":"2023-08-28T02:38:36.873436+0000","last_fullsized":"2023-08-28T02:38:36.873436+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"356'10418","reported_seq":10476,"reported_epoch":356,"state":"active+remapped+backfill_wait","last_fresh":"2023-08-28T02:38:45.066519+0000","last_change":"2023-08-28T02:37:46.828548+0000","last_active":"2023-08-28T02:38:45.066519+0000","last_peered":"2023-08-28T02:38:45.066519+0000","last_clean":"2023-08-28T02:37:44.550616+0000","last_became_active":"2023-08-28T02:37:46.807673+0000","last_became_peered":"2023-08-28T02:37:46.807673+0000","last_unstale":"2023-08-28T02:38:45.066519+0000","last_undegraded":"2023-08-28T02:38:45.066519+0000","last_fullsized":"2023-08-28T02:38:45.066519+0000","mapping_epoch":345,"log_start":"331'3024","ondisk_log_start":"331'3024","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":7394,"log_dups_size":0,"ondisk_log_size":7394,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:41:59.898437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121765888,"num_objects":1858,"num_object_clones":0,"num_object_copies":5574,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1858,"num_objects_unfound":0,"num_objects_dirty":1858,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10418,"num_write_kb":392832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,2],"avail_no_missing":["0","2","6"],"object_location_counts":[{"shards":"0,2,6","objects":1858}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":25,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.873679+0000","last_change":"2023-08-28T02:37:46.190365+0000","last_active":"2023-08-28T02:38:36.873679+0000","last_peered":"2023-08-28T02:38:36.873679+0000","last_clean":"2023-08-28T02:38:36.873679+0000","last_became_active":"2023-08-28T02:37:45.584408+0000","last_became_peered":"2023-08-28T02:37:45.584408+0000","last_unstale":"2023-08-28T02:38:36.873679+0000","last_undegraded":"2023-08-28T02:38:36.873679+0000","last_fullsized":"2023-08-28T02:38:36.873679+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":198,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.876043+0000","last_change":"2023-08-28T02:38:35.888252+0000","last_active":"2023-08-28T02:38:36.876043+0000","last_peered":"2023-08-28T02:38:36.876043+0000","last_clean":"2023-08-28T02:38:36.876043+0000","last_became_active":"2023-08-28T02:38:35.887901+0000","last_became_peered":"2023-08-28T02:38:35.887901+0000","last_unstale":"2023-08-28T02:38:36.876043+0000","last_undegraded":"2023-08-28T02:38:36.876043+0000","last_fullsized":"2023-08-28T02:38:36.876043+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"356'10357","reported_seq":10405,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-08-28T02:38:43.513577+0000","last_change":"2023-08-28T02:35:14.498797+0000","last_active":"2023-08-28T02:38:43.513577+0000","last_peered":"2023-08-28T02:38:43.513577+0000","last_clean":"2023-08-28T02:38:43.513577+0000","last_became_active":"2023-08-28T02:35:14.498551+0000","last_became_peered":"2023-08-28T02:35:14.498551+0000","last_unstale":"2023-08-28T02:38:43.513577+0000","last_undegraded":"2023-08-28T02:38:43.513577+0000","last_fullsized":"2023-08-28T02:38:43.513577+0000","mapping_epoch":318,"log_start":"331'3208","ondisk_log_start":"331'3208","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":7149,"log_dups_size":0,"ondisk_log_size":7149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:43:07.729642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123011072,"num_objects":1877,"num_object_clones":0,"num_object_copies":5631,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1877,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10357,"num_write_kb":391488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":361,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.883240+0000","last_change":"2023-08-28T02:28:25.445982+0000","last_active":"2023-08-28T02:38:36.883240+0000","last_peered":"2023-08-28T02:38:36.883240+0000","last_clean":"2023-08-28T02:38:36.883240+0000","last_became_active":"2023-08-28T02:28:25.443577+0000","last_became_peered":"2023-08-28T02:28:25.443577+0000","last_unstale":"2023-08-28T02:38:36.883240+0000","last_undegraded":"2023-08-28T02:38:36.883240+0000","last_fullsized":"2023-08-28T02:38:36.883240+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:24:03.381751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":357,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.872870+0000","last_change":"2023-08-28T02:32:58.887874+0000","last_active":"2023-08-28T02:38:36.872870+0000","last_peered":"2023-08-28T02:38:36.872870+0000","last_clean":"2023-08-28T02:38:36.872870+0000","last_became_active":"2023-08-28T02:32:58.887377+0000","last_became_peered":"2023-08-28T02:32:58.887377+0000","last_unstale":"2023-08-28T02:38:36.872870+0000","last_undegraded":"2023-08-28T02:38:36.872870+0000","last_fullsized":"2023-08-28T02:38:36.872870+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:08:07.234984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"356'10216","reported_seq":10264,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-08-28T02:38:44.078991+0000","last_change":"2023-08-28T02:35:14.503271+0000","last_active":"2023-08-28T02:38:44.078991+0000","last_peered":"2023-08-28T02:38:44.078991+0000","last_clean":"2023-08-28T02:38:44.078991+0000","last_became_active":"2023-08-28T02:35:14.502940+0000","last_became_peered":"2023-08-28T02:35:14.502940+0000","last_unstale":"2023-08-28T02:38:44.078991+0000","last_undegraded":"2023-08-28T02:38:44.078991+0000","last_fullsized":"2023-08-28T02:38:44.078991+0000","mapping_epoch":318,"log_start":"330'2698","ondisk_log_start":"330'2698","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":7518,"log_dups_size":0,"ondisk_log_size":7518,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:07:49.114269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121372672,"num_objects":1852,"num_object_clones":0,"num_object_copies":5556,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1852,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10216,"num_write_kb":386176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":223,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.883004+0000","last_change":"2023-08-28T02:23:59.230059+0000","last_active":"2023-08-28T02:38:36.883004+0000","last_peered":"2023-08-28T02:38:36.883004+0000","last_clean":"2023-08-28T02:38:36.883004+0000","last_became_active":"2023-08-28T02:23:59.217783+0000","last_became_peered":"2023-08-28T02:23:59.217783+0000","last_unstale":"2023-08-28T02:38:36.883004+0000","last_undegraded":"2023-08-28T02:38:36.883004+0000","last_fullsized":"2023-08-28T02:38:36.883004+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:00:12.145395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":376,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.875063+0000","last_change":"2023-08-28T02:28:25.351698+0000","last_active":"2023-08-28T02:38:36.875063+0000","last_peered":"2023-08-28T02:38:36.875063+0000","last_clean":"2023-08-28T02:38:36.875063+0000","last_became_active":"2023-08-28T02:28:25.351349+0000","last_became_peered":"2023-08-28T02:28:25.351349+0000","last_unstale":"2023-08-28T02:38:36.875063+0000","last_undegraded":"2023-08-28T02:38:36.875063+0000","last_fullsized":"2023-08-28T02:38:36.875063+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"356'10152","reported_seq":10208,"reported_epoch":356,"state":"active+remapped+backfill_wait","last_fresh":"2023-08-28T02:38:43.592995+0000","last_change":"2023-08-28T02:38:37.063228+0000","last_active":"2023-08-28T02:38:43.592995+0000","last_peered":"2023-08-28T02:38:43.592995+0000","last_clean":"2023-08-28T02:38:34.841553+0000","last_became_active":"2023-08-28T02:38:37.003296+0000","last_became_peered":"2023-08-28T02:38:37.003296+0000","last_unstale":"2023-08-28T02:38:43.592995+0000","last_undegraded":"2023-08-28T02:38:43.592995+0000","last_fullsized":"2023-08-28T02:38:43.592995+0000","mapping_epoch":354,"log_start":"331'2978","ondisk_log_start":"331'2978","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":7174,"log_dups_size":0,"ondisk_log_size":7174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:00:11.875820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121503744,"num_objects":1854,"num_object_clones":0,"num_object_copies":5562,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1854,"num_objects_unfound":0,"num_objects_dirty":1854,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10152,"num_write_kb":384192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,2],"avail_no_missing":["1","0","2"],"object_location_counts":[{"shards":"0,1,2","objects":1854}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":162,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.875895+0000","last_change":"2023-08-28T02:32:58.629249+0000","last_active":"2023-08-28T02:38:36.875895+0000","last_peered":"2023-08-28T02:38:36.875895+0000","last_clean":"2023-08-28T02:38:36.875895+0000","last_became_active":"2023-08-28T02:32:58.628583+0000","last_became_peered":"2023-08-28T02:32:58.628583+0000","last_unstale":"2023-08-28T02:38:36.875895+0000","last_undegraded":"2023-08-28T02:38:36.875895+0000","last_fullsized":"2023-08-28T02:38:36.875895+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":438,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.874797+0000","last_change":"2023-08-28T02:38:35.950952+0000","last_active":"2023-08-28T02:38:36.874797+0000","last_peered":"2023-08-28T02:38:36.874797+0000","last_clean":"2023-08-28T02:38:36.874797+0000","last_became_active":"2023-08-28T02:38:35.950478+0000","last_became_peered":"2023-08-28T02:38:35.950478+0000","last_unstale":"2023-08-28T02:38:36.874797+0000","last_undegraded":"2023-08-28T02:38:36.874797+0000","last_fullsized":"2023-08-28T02:38:36.874797+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"356'10199","reported_seq":10245,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-08-28T02:38:41.978855+0000","last_change":"2023-08-28T02:35:14.498910+0000","last_active":"2023-08-28T02:38:41.978855+0000","last_peered":"2023-08-28T02:38:41.978855+0000","last_clean":"2023-08-28T02:38:41.978855+0000","last_became_active":"2023-08-28T02:35:14.498631+0000","last_became_peered":"2023-08-28T02:35:14.498631+0000","last_unstale":"2023-08-28T02:38:41.978855+0000","last_undegraded":"2023-08-28T02:38:41.978855+0000","last_fullsized":"2023-08-28T02:38:41.978855+0000","mapping_epoch":318,"log_start":"326'2027","ondisk_log_start":"326'2027","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":8172,"log_dups_size":0,"ondisk_log_size":8172,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126550016,"num_objects":1931,"num_object_clones":0,"num_object_copies":5793,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1931,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10199,"num_write_kb":388160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":310,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.873574+0000","last_change":"2023-08-28T02:38:35.885723+0000","last_active":"2023-08-28T02:38:36.873574+0000","last_peered":"2023-08-28T02:38:36.873574+0000","last_clean":"2023-08-28T02:38:36.873574+0000","last_became_active":"2023-08-28T02:38:35.885381+0000","last_became_peered":"2023-08-28T02:38:35.885381+0000","last_unstale":"2023-08-28T02:38:36.873574+0000","last_undegraded":"2023-08-28T02:38:36.873574+0000","last_fullsized":"2023-08-28T02:38:36.873574+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":344,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.884952+0000","last_change":"2023-08-28T02:32:58.766795+0000","last_active":"2023-08-28T02:38:36.884952+0000","last_peered":"2023-08-28T02:38:36.884952+0000","last_clean":"2023-08-28T02:38:36.884952+0000","last_became_active":"2023-08-28T02:32:58.766441+0000","last_became_peered":"2023-08-28T02:32:58.766441+0000","last_unstale":"2023-08-28T02:38:36.884952+0000","last_undegraded":"2023-08-28T02:38:36.884952+0000","last_fullsized":"2023-08-28T02:38:36.884952+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:58:13.963961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.3","version":"356'10296","reported_seq":10344,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-08-28T02:38:43.547417+0000","last_change":"2023-08-28T02:35:14.498188+0000","last_active":"2023-08-28T02:38:43.547417+0000","last_peered":"2023-08-28T02:38:43.547417+0000","last_clean":"2023-08-28T02:38:43.547417+0000","last_became_active":"2023-08-28T02:35:14.496278+0000","last_became_peered":"2023-08-28T02:35:14.496278+0000","last_unstale":"2023-08-28T02:38:43.547417+0000","last_undegraded":"2023-08-28T02:38:43.547417+0000","last_fullsized":"2023-08-28T02:38:43.547417+0000","mapping_epoch":318,"log_start":"331'3133","ondisk_log_start":"331'3133","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":7163,"log_dups_size":0,"ondisk_log_size":7163,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:43:23.633921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121634816,"num_objects":1856,"num_object_clones":0,"num_object_copies":5568,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1856,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10296,"num_write_kb":388864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":273,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.883111+0000","last_change":"2023-08-28T02:37:45.579112+0000","last_active":"2023-08-28T02:38:36.883111+0000","last_peered":"2023-08-28T02:38:36.883111+0000","last_clean":"2023-08-28T02:38:36.883111+0000","last_became_active":"2023-08-28T02:37:45.578787+0000","last_became_peered":"2023-08-28T02:37:45.578787+0000","last_unstale":"2023-08-28T02:38:36.883111+0000","last_undegraded":"2023-08-28T02:38:36.883111+0000","last_fullsized":"2023-08-28T02:38:36.883111+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:20:59.994447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":77,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.874325+0000","last_change":"2023-08-28T02:32:58.755765+0000","last_active":"2023-08-28T02:38:36.874325+0000","last_peered":"2023-08-28T02:38:36.874325+0000","last_clean":"2023-08-28T02:38:36.874325+0000","last_became_active":"2023-08-28T02:32:58.755438+0000","last_became_peered":"2023-08-28T02:32:58.755438+0000","last_unstale":"2023-08-28T02:38:36.874325+0000","last_undegraded":"2023-08-28T02:38:36.874325+0000","last_fullsized":"2023-08-28T02:38:36.874325+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:14:11.498611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"356'10329","reported_seq":10386,"reported_epoch":356,"state":"active+remapped+backfill_wait","last_fresh":"2023-08-28T02:38:45.038020+0000","last_change":"2023-08-28T02:38:37.052869+0000","last_active":"2023-08-28T02:38:45.038020+0000","last_peered":"2023-08-28T02:38:45.038020+0000","last_clean":"2023-08-28T02:38:34.861093+0000","last_became_active":"2023-08-28T02:38:37.005189+0000","last_became_peered":"2023-08-28T02:38:37.005189+0000","last_unstale":"2023-08-28T02:38:45.038020+0000","last_undegraded":"2023-08-28T02:38:45.038020+0000","last_fullsized":"2023-08-28T02:38:45.038020+0000","mapping_epoch":354,"log_start":"331'2941","ondisk_log_start":"331'2941","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":7388,"log_dups_size":0,"ondisk_log_size":7388,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:20:40.565888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120127488,"num_objects":1833,"num_object_clones":0,"num_object_copies":5499,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1833,"num_objects_unfound":0,"num_objects_dirty":1833,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10329,"num_write_kb":389184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,2],"avail_no_missing":["0","2","4"],"object_location_counts":[{"shards":"0,2,4","objects":1833}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"78'126","reported_seq":802,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.874313+0000","last_change":"2023-08-28T02:37:48.647489+0000","last_active":"2023-08-28T02:38:36.874313+0000","last_peered":"2023-08-28T02:38:36.874313+0000","last_clean":"2023-08-28T02:38:36.874313+0000","last_became_active":"2023-08-28T02:37:48.647002+0000","last_became_peered":"2023-08-28T02:37:48.647002+0000","last_unstale":"2023-08-28T02:38:36.874313+0000","last_undegraded":"2023-08-28T02:38:36.874313+0000","last_fullsized":"2023-08-28T02:38:36.874313+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:04:10.819878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":743,"num_read_kb":1331,"num_write":125,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":207,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.881058+0000","last_change":"2023-08-28T02:38:35.898925+0000","last_active":"2023-08-28T02:38:36.881058+0000","last_peered":"2023-08-28T02:38:36.881058+0000","last_clean":"2023-08-28T02:38:36.881058+0000","last_became_active":"2023-08-28T02:38:35.898546+0000","last_became_peered":"2023-08-28T02:38:35.898546+0000","last_unstale":"2023-08-28T02:38:36.881058+0000","last_undegraded":"2023-08-28T02:38:36.881058+0000","last_fullsized":"2023-08-28T02:38:36.881058+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:25:31.558360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:38.589392+0000","last_change":"2023-08-28T02:38:35.883771+0000","last_active":"2023-08-28T02:38:38.589392+0000","last_peered":"2023-08-28T02:38:38.589392+0000","last_clean":"2023-08-28T02:38:38.589392+0000","last_became_active":"2023-08-28T02:38:35.883052+0000","last_became_peered":"2023-08-28T02:38:35.883052+0000","last_unstale":"2023-08-28T02:38:38.589392+0000","last_undegraded":"2023-08-28T02:38:38.589392+0000","last_fullsized":"2023-08-28T02:38:38.589392+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:54:17.468119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"356'10383","reported_seq":10595,"reported_epoch":356,"state":"active+remapped+backfill_wait","last_fresh":"2023-08-28T02:38:43.976412+0000","last_change":"2023-08-28T02:38:35.930610+0000","last_active":"2023-08-28T02:38:43.976412+0000","last_peered":"2023-08-28T02:38:43.976412+0000","last_clean":"2023-08-28T02:37:44.530880+0000","last_became_active":"2023-08-28T02:37:46.790659+0000","last_became_peered":"2023-08-28T02:37:46.790659+0000","last_unstale":"2023-08-28T02:38:43.976412+0000","last_undegraded":"2023-08-28T02:38:43.976412+0000","last_fullsized":"2023-08-28T02:38:43.976412+0000","mapping_epoch":345,"log_start":"331'2858","ondisk_log_start":"331'2858","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":7525,"log_dups_size":0,"ondisk_log_size":7525,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:37:46.944577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123273216,"num_objects":1881,"num_object_clones":0,"num_object_copies":5643,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1841,"num_objects_unfound":0,"num_objects_dirty":1881,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10383,"num_write_kb":392448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":74,"num_bytes_recovered":4849664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,0],"avail_no_missing":["3","0","5"],"object_location_counts":[{"shards":"0,3,5","objects":1881}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"78'204","reported_seq":1441,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.890016+0000","last_change":"2023-08-28T02:32:59.644574+0000","last_active":"2023-08-28T02:38:36.890016+0000","last_peered":"2023-08-28T02:38:36.890016+0000","last_clean":"2023-08-28T02:38:36.890016+0000","last_became_active":"2023-08-28T02:32:59.644053+0000","last_became_peered":"2023-08-28T02:32:59.644053+0000","last_unstale":"2023-08-28T02:38:36.890016+0000","last_undegraded":"2023-08-28T02:38:36.890016+0000","last_fullsized":"2023-08-28T02:38:36.890016+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":204,"log_dups_size":0,"ondisk_log_size":204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:55:51.460483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1521,"num_read_kb":2511,"num_write":178,"num_write_kb":1175,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":350,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.873913+0000","last_change":"2023-08-28T02:28:25.352417+0000","last_active":"2023-08-28T02:38:36.873913+0000","last_peered":"2023-08-28T02:38:36.873913+0000","last_clean":"2023-08-28T02:38:36.873913+0000","last_became_active":"2023-08-28T02:28:25.352216+0000","last_became_peered":"2023-08-28T02:28:25.352216+0000","last_unstale":"2023-08-28T02:38:36.873913+0000","last_undegraded":"2023-08-28T02:38:36.873913+0000","last_fullsized":"2023-08-28T02:38:36.873913+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":86,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.874249+0000","last_change":"2023-08-28T02:38:35.883370+0000","last_active":"2023-08-28T02:38:36.874249+0000","last_peered":"2023-08-28T02:38:36.874249+0000","last_clean":"2023-08-28T02:38:36.874249+0000","last_became_active":"2023-08-28T02:38:35.883049+0000","last_became_peered":"2023-08-28T02:38:35.883049+0000","last_unstale":"2023-08-28T02:38:36.874249+0000","last_undegraded":"2023-08-28T02:38:36.874249+0000","last_fullsized":"2023-08-28T02:38:36.874249+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:09:38.259839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"356'10247","reported_seq":10295,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-08-28T02:38:44.054362+0000","last_change":"2023-08-28T02:35:14.503279+0000","last_active":"2023-08-28T02:38:44.054362+0000","last_peered":"2023-08-28T02:38:44.054362+0000","last_clean":"2023-08-28T02:38:44.054362+0000","last_became_active":"2023-08-28T02:35:14.502953+0000","last_became_peered":"2023-08-28T02:35:14.502953+0000","last_unstale":"2023-08-28T02:38:44.054362+0000","last_undegraded":"2023-08-28T02:38:44.054362+0000","last_fullsized":"2023-08-28T02:38:44.054362+0000","mapping_epoch":318,"log_start":"331'2721","ondisk_log_start":"331'2721","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":7526,"log_dups_size":0,"ondisk_log_size":7526,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:55:29.284519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124321792,"num_objects":1897,"num_object_clones":0,"num_object_copies":5691,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1897,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10247,"num_write_kb":388608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"78'47","reported_seq":372,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-08-28T02:38:39.058591+0000","last_change":"2023-08-28T02:38:35.942552+0000","last_active":"2023-08-28T02:38:39.058591+0000","last_peered":"2023-08-28T02:38:39.058591+0000","last_clean":"2023-08-28T02:38:39.058591+0000","last_became_active":"2023-08-28T02:38:35.885504+0000","last_became_peered":"2023-08-28T02:38:35.885504+0000","last_unstale":"2023-08-28T02:38:39.058591+0000","last_undegraded":"2023-08-28T02:38:39.058591+0000","last_fullsized":"2023-08-28T02:38:39.058591+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":47,"log_dups_size":0,"ondisk_log_size":47,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:05:08.556625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":162,"num_read_kb":1281,"num_write":48,"num_write_kb":1153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":86,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.873304+0000","last_change":"2023-08-28T02:38:35.883555+0000","last_active":"2023-08-28T02:38:36.873304+0000","last_peered":"2023-08-28T02:38:36.873304+0000","last_clean":"2023-08-28T02:38:36.873304+0000","last_became_active":"2023-08-28T02:38:35.883221+0000","last_became_peered":"2023-08-28T02:38:35.883221+0000","last_unstale":"2023-08-28T02:38:36.873304+0000","last_undegraded":"2023-08-28T02:38:36.873304+0000","last_fullsized":"2023-08-28T02:38:36.873304+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:37:47.771594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":77,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.874411+0000","last_change":"2023-08-28T02:32:58.724424+0000","last_active":"2023-08-28T02:38:36.874411+0000","last_peered":"2023-08-28T02:38:36.874411+0000","last_clean":"2023-08-28T02:38:36.874411+0000","last_became_active":"2023-08-28T02:32:58.724127+0000","last_became_peered":"2023-08-28T02:32:58.724127+0000","last_unstale":"2023-08-28T02:38:36.874411+0000","last_undegraded":"2023-08-28T02:38:36.874411+0000","last_fullsized":"2023-08-28T02:38:36.874411+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:02:25.178712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"356'10307","reported_seq":10353,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-08-28T02:38:42.248112+0000","last_change":"2023-08-28T02:35:14.500884+0000","last_active":"2023-08-28T02:38:42.248112+0000","last_peered":"2023-08-28T02:38:42.248112+0000","last_clean":"2023-08-28T02:38:42.248112+0000","last_became_active":"2023-08-28T02:35:14.500643+0000","last_became_peered":"2023-08-28T02:35:14.500643+0000","last_unstale":"2023-08-28T02:38:42.248112+0000","last_undegraded":"2023-08-28T02:38:42.248112+0000","last_fullsized":"2023-08-28T02:38:42.248112+0000","mapping_epoch":318,"log_start":"330'2531","ondisk_log_start":"330'2531","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":7776,"log_dups_size":0,"ondisk_log_size":7776,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127860736,"num_objects":1951,"num_object_clones":0,"num_object_copies":5853,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1951,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10307,"num_write_kb":392256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"78'114","reported_seq":414,"reported_epoch":355,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:38:36.884225+0000","last_change":"2023-08-28T02:38:36.884225+0000","last_active":"2023-08-28T02:38:36.884225+0000","last_peered":"2023-08-28T02:38:36.884225+0000","last_clean":"2023-08-28T02:37:38.799203+0000","last_became_active":"2023-08-28T02:38:36.883700+0000","last_became_peered":"2023-08-28T02:38:36.883700+0000","last_unstale":"2023-08-28T02:38:36.884225+0000","last_undegraded":"2023-08-28T02:38:36.874234+0000","last_fullsized":"2023-08-28T02:38:36.873408+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":114,"log_dups_size":0,"ondisk_log_size":114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":358,"num_read_kb":1352,"num_write":102,"num_write_kb":1104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5],"avail_no_missing":["7","2","5"],"object_location_counts":[{"shards":"2,5,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"86'139","reported_seq":450,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.873056+0000","last_change":"2023-08-28T02:37:48.651113+0000","last_active":"2023-08-28T02:38:36.873056+0000","last_peered":"2023-08-28T02:38:36.873056+0000","last_clean":"2023-08-28T02:38:36.873056+0000","last_became_active":"2023-08-28T02:37:48.650691+0000","last_became_peered":"2023-08-28T02:37:48.650691+0000","last_unstale":"2023-08-28T02:38:36.873056+0000","last_undegraded":"2023-08-28T02:38:36.873056+0000","last_fullsized":"2023-08-28T02:38:36.873056+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_clean_scrub_stamp":"2023-08-28T02:08:02.043718+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":238,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.881316+0000","last_change":"2023-08-28T02:28:25.347600+0000","last_active":"2023-08-28T02:38:36.881316+0000","last_peered":"2023-08-28T02:38:36.881316+0000","last_clean":"2023-08-28T02:38:36.881316+0000","last_became_active":"2023-08-28T02:28:25.347237+0000","last_became_peered":"2023-08-28T02:28:25.347237+0000","last_unstale":"2023-08-28T02:38:36.881316+0000","last_undegraded":"2023-08-28T02:38:36.881316+0000","last_fullsized":"2023-08-28T02:38:36.881316+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:29:26.767559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":162,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.884339+0000","last_change":"2023-08-28T02:37:45.581261+0000","last_active":"2023-08-28T02:38:36.884339+0000","last_peered":"2023-08-28T02:38:36.884339+0000","last_clean":"2023-08-28T02:38:36.884339+0000","last_became_active":"2023-08-28T02:37:45.580927+0000","last_became_peered":"2023-08-28T02:37:45.580927+0000","last_unstale":"2023-08-28T02:38:36.884339+0000","last_undegraded":"2023-08-28T02:38:36.884339+0000","last_fullsized":"2023-08-28T02:38:36.884339+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"356'10063","reported_seq":10121,"reported_epoch":356,"state":"active+remapped+backfill_wait","last_fresh":"2023-08-28T02:38:43.476916+0000","last_change":"2023-08-28T02:38:37.028262+0000","last_active":"2023-08-28T02:38:43.476916+0000","last_peered":"2023-08-28T02:38:43.476916+0000","last_clean":"2023-08-28T02:38:34.861056+0000","last_became_active":"2023-08-28T02:38:36.988695+0000","last_became_peered":"2023-08-28T02:38:36.988695+0000","last_unstale":"2023-08-28T02:38:43.476916+0000","last_undegraded":"2023-08-28T02:38:43.476916+0000","last_fullsized":"2023-08-28T02:38:43.476916+0000","mapping_epoch":354,"log_start":"331'2911","ondisk_log_start":"331'2911","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":7152,"log_dups_size":0,"ondisk_log_size":7152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:09:45.928327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124321792,"num_objects":1897,"num_object_clones":0,"num_object_copies":5691,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1897,"num_objects_unfound":0,"num_objects_dirty":1897,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10063,"num_write_kb":382720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,2],"avail_no_missing":["1","2","5"],"object_location_counts":[{"shards":"1,2,5","objects":1897}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'148","reported_seq":674,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.873809+0000","last_change":"2023-08-28T02:37:48.650996+0000","last_active":"2023-08-28T02:38:36.873809+0000","last_peered":"2023-08-28T02:38:36.873809+0000","last_clean":"2023-08-28T02:38:36.873809+0000","last_became_active":"2023-08-28T02:37:48.650016+0000","last_became_peered":"2023-08-28T02:37:48.650016+0000","last_unstale":"2023-08-28T02:38:36.873809+0000","last_undegraded":"2023-08-28T02:38:36.873809+0000","last_fullsized":"2023-08-28T02:38:36.873809+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1092,"num_read_kb":1955,"num_write":127,"num_write_kb":5177,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":328,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.873200+0000","last_change":"2023-08-28T02:23:59.226986+0000","last_active":"2023-08-28T02:38:36.873200+0000","last_peered":"2023-08-28T02:38:36.873200+0000","last_clean":"2023-08-28T02:38:36.873200+0000","last_became_active":"2023-08-28T02:23:59.226529+0000","last_became_peered":"2023-08-28T02:23:59.226529+0000","last_unstale":"2023-08-28T02:38:36.873200+0000","last_undegraded":"2023-08-28T02:38:36.873200+0000","last_fullsized":"2023-08-28T02:38:36.873200+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:22:09.193619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":400,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.883516+0000","last_change":"2023-08-28T02:38:35.885714+0000","last_active":"2023-08-28T02:38:36.883516+0000","last_peered":"2023-08-28T02:38:36.883516+0000","last_clean":"2023-08-28T02:38:36.883516+0000","last_became_active":"2023-08-28T02:38:35.885391+0000","last_became_peered":"2023-08-28T02:38:35.885391+0000","last_unstale":"2023-08-28T02:38:36.883516+0000","last_undegraded":"2023-08-28T02:38:36.883516+0000","last_fullsized":"2023-08-28T02:38:36.883516+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:02:57.365844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"356'10332","reported_seq":10547,"reported_epoch":356,"state":"active+remapped+backfill_wait","last_fresh":"2023-08-28T02:38:44.072290+0000","last_change":"2023-08-28T02:38:37.076874+0000","last_active":"2023-08-28T02:38:44.072290+0000","last_peered":"2023-08-28T02:38:44.072290+0000","last_clean":"2023-08-28T02:38:34.867302+0000","last_became_active":"2023-08-28T02:38:37.016339+0000","last_became_peered":"2023-08-28T02:38:37.016339+0000","last_unstale":"2023-08-28T02:38:44.072290+0000","last_undegraded":"2023-08-28T02:38:44.072290+0000","last_fullsized":"2023-08-28T02:38:44.072290+0000","mapping_epoch":354,"log_start":"331'2800","ondisk_log_start":"331'2800","created":318,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":7532,"log_dups_size":0,"ondisk_log_size":7532,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:01:21.215657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124125184,"num_objects":1894,"num_object_clones":0,"num_object_copies":5682,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1894,"num_objects_unfound":0,"num_objects_dirty":1894,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10332,"num_write_kb":391232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":73,"num_bytes_recovered":4784128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,2],"avail_no_missing":["3","1","2"],"object_location_counts":[{"shards":"1,2,3","objects":1894}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"78'103","reported_seq":696,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.873763+0000","last_change":"2023-08-28T02:37:48.651919+0000","last_active":"2023-08-28T02:38:36.873763+0000","last_peered":"2023-08-28T02:38:36.873763+0000","last_clean":"2023-08-28T02:38:36.873763+0000","last_became_active":"2023-08-28T02:37:48.651437+0000","last_became_peered":"2023-08-28T02:37:48.651437+0000","last_unstale":"2023-08-28T02:38:36.873763+0000","last_undegraded":"2023-08-28T02:38:36.873763+0000","last_fullsized":"2023-08-28T02:38:36.873763+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":803,"num_read_kb":2080,"num_write":73,"num_write_kb":1244,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":86,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.874158+0000","last_change":"2023-08-28T02:38:35.882757+0000","last_active":"2023-08-28T02:38:36.874158+0000","last_peered":"2023-08-28T02:38:36.874158+0000","last_clean":"2023-08-28T02:38:36.874158+0000","last_became_active":"2023-08-28T02:38:35.882423+0000","last_became_peered":"2023-08-28T02:38:35.882423+0000","last_unstale":"2023-08-28T02:38:36.874158+0000","last_undegraded":"2023-08-28T02:38:36.874158+0000","last_fullsized":"2023-08-28T02:38:36.874158+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:54:44.915353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":350,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.873413+0000","last_change":"2023-08-28T02:37:45.579740+0000","last_active":"2023-08-28T02:38:36.873413+0000","last_peered":"2023-08-28T02:38:36.873413+0000","last_clean":"2023-08-28T02:38:36.873413+0000","last_became_active":"2023-08-28T02:37:45.579412+0000","last_became_peered":"2023-08-28T02:37:45.579412+0000","last_unstale":"2023-08-28T02:38:36.873413+0000","last_undegraded":"2023-08-28T02:38:36.873413+0000","last_fullsized":"2023-08-28T02:38:36.873413+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:31:18.483532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"356'10054","reported_seq":10137,"reported_epoch":356,"state":"active+remapped+backfill_wait","last_fresh":"2023-08-28T02:38:45.101076+0000","last_change":"2023-08-28T02:37:47.122442+0000","last_active":"2023-08-28T02:38:45.101076+0000","last_peered":"2023-08-28T02:38:45.101076+0000","last_clean":"2023-08-28T02:37:44.513990+0000","last_became_active":"2023-08-28T02:37:46.801002+0000","last_became_peered":"2023-08-28T02:37:46.801002+0000","last_unstale":"2023-08-28T02:38:45.101076+0000","last_undegraded":"2023-08-28T02:38:45.101076+0000","last_fullsized":"2023-08-28T02:38:45.101076+0000","mapping_epoch":345,"log_start":"331'2642","ondisk_log_start":"331'2642","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":7412,"log_dups_size":0,"ondisk_log_size":7412,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:51:20.134517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111542272,"num_objects":1702,"num_object_clones":0,"num_object_copies":5106,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1697,"num_objects_unfound":0,"num_objects_dirty":1702,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10054,"num_write_kb":376192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":786432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,5],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,3,5","objects":1702}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":249,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.899268+0000","last_change":"2023-08-28T02:37:45.590339+0000","last_active":"2023-08-28T02:38:36.899268+0000","last_peered":"2023-08-28T02:38:36.899268+0000","last_clean":"2023-08-28T02:38:36.899268+0000","last_became_active":"2023-08-28T02:37:45.578751+0000","last_became_peered":"2023-08-28T02:37:45.578751+0000","last_unstale":"2023-08-28T02:38:36.899268+0000","last_undegraded":"2023-08-28T02:38:36.899268+0000","last_fullsized":"2023-08-28T02:38:36.899268+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:27:24.226372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'112","reported_seq":510,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.874568+0000","last_change":"2023-08-28T02:32:59.643465+0000","last_active":"2023-08-28T02:38:36.874568+0000","last_peered":"2023-08-28T02:38:36.874568+0000","last_clean":"2023-08-28T02:38:36.874568+0000","last_became_active":"2023-08-28T02:32:59.642979+0000","last_became_peered":"2023-08-28T02:32:59.642979+0000","last_unstale":"2023-08-28T02:38:36.874568+0000","last_undegraded":"2023-08-28T02:38:36.874568+0000","last_fullsized":"2023-08-28T02:38:36.874568+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:48:45.923015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":444,"num_read_kb":473,"num_write":94,"num_write_kb":127,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":256,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.883282+0000","last_change":"2023-08-28T02:37:45.578790+0000","last_active":"2023-08-28T02:38:36.883282+0000","last_peered":"2023-08-28T02:38:36.883282+0000","last_clean":"2023-08-28T02:38:36.883282+0000","last_became_active":"2023-08-28T02:37:45.578473+0000","last_became_peered":"2023-08-28T02:37:45.578473+0000","last_unstale":"2023-08-28T02:38:36.883282+0000","last_undegraded":"2023-08-28T02:38:36.883282+0000","last_fullsized":"2023-08-28T02:38:36.883282+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:23:51.722772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"356'10132","reported_seq":13041,"reported_epoch":356,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:38:45.281730+0000","last_change":"2023-08-28T02:38:36.180937+0000","last_active":"2023-08-28T02:38:45.281730+0000","last_peered":"2023-08-28T02:38:45.281730+0000","last_clean":"2023-08-28T02:37:44.560462+0000","last_became_active":"2023-08-28T02:38:36.091055+0000","last_became_peered":"2023-08-28T02:38:36.091055+0000","last_unstale":"2023-08-28T02:38:45.281730+0000","last_undegraded":"2023-08-28T02:38:36.038772+0000","last_fullsized":"2023-08-28T02:38:35.872670+0000","mapping_epoch":353,"log_start":"324'100","ondisk_log_start":"324'100","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":10032,"log_dups_size":0,"ondisk_log_size":10032,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:55:48.264648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118816792,"num_objects":1814,"num_object_clones":0,"num_object_copies":5442,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1734,"num_objects_misplaced":4549,"num_objects_unfound":0,"num_objects_dirty":1814,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10132,"num_write_kb":382209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1424,"num_bytes_recovered":93257728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":-265},{"shards":"2,5,6","objects":1814}],"blocked_by":[],"up_primary":7,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"78'122","reported_seq":744,"reported_epoch":355,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:38:36.904311+0000","last_change":"2023-08-28T02:38:36.904311+0000","last_active":"2023-08-28T02:38:36.904311+0000","last_peered":"2023-08-28T02:38:36.904311+0000","last_clean":"2023-08-28T02:38:27.810878+0000","last_became_active":"2023-08-28T02:38:36.903810+0000","last_became_peered":"2023-08-28T02:38:36.903810+0000","last_unstale":"2023-08-28T02:38:36.904311+0000","last_undegraded":"2023-08-28T02:38:36.876395+0000","last_fullsized":"2023-08-28T02:38:36.875861+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":857,"num_read_kb":2224,"num_write":99,"num_write_kb":2312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7],"avail_no_missing":["6","2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":366,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.883606+0000","last_change":"2023-08-28T02:32:58.795995+0000","last_active":"2023-08-28T02:38:36.883606+0000","last_peered":"2023-08-28T02:38:36.883606+0000","last_clean":"2023-08-28T02:38:36.883606+0000","last_became_active":"2023-08-28T02:32:58.771872+0000","last_became_peered":"2023-08-28T02:32:58.771872+0000","last_unstale":"2023-08-28T02:38:36.883606+0000","last_undegraded":"2023-08-28T02:38:36.883606+0000","last_fullsized":"2023-08-28T02:38:36.883606+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:16:02.965467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":325,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.873223+0000","last_change":"2023-08-28T02:28:25.351840+0000","last_active":"2023-08-28T02:38:36.873223+0000","last_peered":"2023-08-28T02:38:36.873223+0000","last_clean":"2023-08-28T02:38:36.873223+0000","last_became_active":"2023-08-28T02:28:25.351399+0000","last_became_peered":"2023-08-28T02:28:25.351399+0000","last_unstale":"2023-08-28T02:38:36.873223+0000","last_undegraded":"2023-08-28T02:38:36.873223+0000","last_fullsized":"2023-08-28T02:38:36.873223+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":352,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-08-28T02:38:39.058972+0000","last_change":"2023-08-28T02:37:45.581817+0000","last_active":"2023-08-28T02:38:39.058972+0000","last_peered":"2023-08-28T02:38:39.058972+0000","last_clean":"2023-08-28T02:38:39.058972+0000","last_became_active":"2023-08-28T02:37:45.581475+0000","last_became_peered":"2023-08-28T02:37:45.581475+0000","last_unstale":"2023-08-28T02:38:39.058972+0000","last_undegraded":"2023-08-28T02:38:39.058972+0000","last_fullsized":"2023-08-28T02:38:39.058972+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:00:31.628265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":398,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:38.589446+0000","last_change":"2023-08-28T02:38:35.882453+0000","last_active":"2023-08-28T02:38:38.589446+0000","last_peered":"2023-08-28T02:38:38.589446+0000","last_clean":"2023-08-28T02:38:38.589446+0000","last_became_active":"2023-08-28T02:38:35.882128+0000","last_became_peered":"2023-08-28T02:38:35.882128+0000","last_unstale":"2023-08-28T02:38:38.589446+0000","last_undegraded":"2023-08-28T02:38:38.589446+0000","last_fullsized":"2023-08-28T02:38:38.589446+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:07:17.115788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":188,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.874984+0000","last_change":"2023-08-28T02:32:58.839847+0000","last_active":"2023-08-28T02:38:36.874984+0000","last_peered":"2023-08-28T02:38:36.874984+0000","last_clean":"2023-08-28T02:38:36.874984+0000","last_became_active":"2023-08-28T02:32:58.628159+0000","last_became_peered":"2023-08-28T02:32:58.628159+0000","last_unstale":"2023-08-28T02:38:36.874984+0000","last_undegraded":"2023-08-28T02:38:36.874984+0000","last_fullsized":"2023-08-28T02:38:36.874984+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":391,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.873998+0000","last_change":"2023-08-28T02:32:58.629596+0000","last_active":"2023-08-28T02:38:36.873998+0000","last_peered":"2023-08-28T02:38:36.873998+0000","last_clean":"2023-08-28T02:38:36.873998+0000","last_became_active":"2023-08-28T02:32:58.629136+0000","last_became_peered":"2023-08-28T02:32:58.629136+0000","last_unstale":"2023-08-28T02:38:36.873998+0000","last_undegraded":"2023-08-28T02:38:36.873998+0000","last_fullsized":"2023-08-28T02:38:36.873998+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":175,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.874907+0000","last_change":"2023-08-28T02:28:25.705331+0000","last_active":"2023-08-28T02:38:36.874907+0000","last_peered":"2023-08-28T02:38:36.874907+0000","last_clean":"2023-08-28T02:38:36.874907+0000","last_became_active":"2023-08-28T02:28:25.397480+0000","last_became_peered":"2023-08-28T02:28:25.397480+0000","last_unstale":"2023-08-28T02:38:36.874907+0000","last_undegraded":"2023-08-28T02:38:36.874907+0000","last_fullsized":"2023-08-28T02:38:36.874907+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":355,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.875093+0000","last_change":"2023-08-28T02:23:59.207929+0000","last_active":"2023-08-28T02:38:36.875093+0000","last_peered":"2023-08-28T02:38:36.875093+0000","last_clean":"2023-08-28T02:38:36.875093+0000","last_became_active":"2023-08-28T02:23:59.207589+0000","last_became_peered":"2023-08-28T02:23:59.207589+0000","last_unstale":"2023-08-28T02:38:36.875093+0000","last_undegraded":"2023-08-28T02:38:36.875093+0000","last_fullsized":"2023-08-28T02:38:36.875093+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":256,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.876689+0000","last_change":"2023-08-28T02:38:35.890747+0000","last_active":"2023-08-28T02:38:36.876689+0000","last_peered":"2023-08-28T02:38:36.876689+0000","last_clean":"2023-08-28T02:38:36.876689+0000","last_became_active":"2023-08-28T02:38:35.890440+0000","last_became_peered":"2023-08-28T02:38:35.890440+0000","last_unstale":"2023-08-28T02:38:36.876689+0000","last_undegraded":"2023-08-28T02:38:36.876689+0000","last_fullsized":"2023-08-28T02:38:36.876689+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:29:47.306581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":228,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.881453+0000","last_change":"2023-08-28T02:38:35.882097+0000","last_active":"2023-08-28T02:38:36.881453+0000","last_peered":"2023-08-28T02:38:36.881453+0000","last_clean":"2023-08-28T02:38:36.881453+0000","last_became_active":"2023-08-28T02:38:35.881687+0000","last_became_peered":"2023-08-28T02:38:35.881687+0000","last_unstale":"2023-08-28T02:38:36.881453+0000","last_undegraded":"2023-08-28T02:38:36.881453+0000","last_fullsized":"2023-08-28T02:38:36.881453+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:23:54.674952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":359,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.873540+0000","last_change":"2023-08-28T02:38:35.920047+0000","last_active":"2023-08-28T02:38:36.873540+0000","last_peered":"2023-08-28T02:38:36.873540+0000","last_clean":"2023-08-28T02:38:36.873540+0000","last_became_active":"2023-08-28T02:38:35.888575+0000","last_became_peered":"2023-08-28T02:38:35.888575+0000","last_unstale":"2023-08-28T02:38:36.873540+0000","last_undegraded":"2023-08-28T02:38:36.873540+0000","last_fullsized":"2023-08-28T02:38:36.873540+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:35:57.784343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":414,"reported_epoch":355,"state":"active+clean","last_fresh":"2023-08-28T02:38:36.873325+0000","last_change":"2023-08-28T02:32:58.630270+0000","last_active":"2023-08-28T02:38:36.873325+0000","last_peered":"2023-08-28T02:38:36.873325+0000","last_clean":"2023-08-28T02:38:36.873325+0000","last_became_active":"2023-08-28T02:32:58.629958+0000","last_became_peered":"2023-08-28T02:32:58.629958+0000","last_unstale":"2023-08-28T02:38:36.873325+0000","last_undegraded":"2023-08-28T02:38:36.873325+0000","last_fullsized":"2023-08-28T02:38:36.873325+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":30498,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":17122,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35337,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":1963720728,"num_objects":29965,"num_object_clones":0,"num_object_copies":89895,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3507,"num_objects_misplaced":21077,"num_objects_unfound":0,"num_objects_dirty":29965,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":163851,"num_write_kb":6202049,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1949,"num_bytes_recovered":127467520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6175338496,"data_stored":6175326280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":122833,"ondisk_log_size":122833,"up":48,"acting":46,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":184,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":311,"seq":1335734829103,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":356104,"kb_used_data":24960,"kb_used_omap":8,"kb_used_meta":331127,"kb_avail":93389048,"statfs":{"total":95995035648,"available":95630385152,"internally_reserved":0,"allocated":25559040,"data_stored":24649167,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8527,"internal_metadata":339074737},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:38:03 2023","interfaces":[{"interface":"back","average":{"1min":3.6389999999999998,"5min":4.4450000000000003,"15min":2.492},"min":{"1min":0.33000000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":18.5,"5min":140.35499999999999,"15min":140.35499999999999},"last":7.5490000000000004},{"interface":"front","average":{"1min":3.1579999999999999,"5min":4.1669999999999998,"15min":2.2639999999999998},"min":{"1min":0.39200000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":18.521000000000001,"5min":140.40199999999999,"15min":140.40199999999999},"last":7.5369999999999999}]},{"osd":1,"last update":"Mon Aug 28 02:38:03 2023","interfaces":[{"interface":"back","average":{"1min":3.9049999999999998,"5min":8,"15min":6.8140000000000001},"min":{"1min":0.28399999999999997,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":34.420999999999999,"5min":151.048,"15min":151.048},"last":0.505},{"interface":"front","average":{"1min":4.5830000000000002,"5min":8.0649999999999995,"15min":6.7290000000000001},"min":{"1min":0.28899999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":38.045999999999999,"5min":150.56299999999999,"15min":150.56299999999999},"last":0.40500000000000003}]},{"osd":2,"last update":"Mon Aug 28 02:38:03 2023","interfaces":[{"interface":"back","average":{"1min":2.8559999999999999,"5min":3.3279999999999998,"15min":3.573},"min":{"1min":0.34799999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":15.880000000000001,"5min":46.042999999999999,"15min":46.042999999999999},"last":6.2270000000000003},{"interface":"front","average":{"1min":2.0750000000000002,"5min":2.9590000000000001,"15min":3.3889999999999998},"min":{"1min":0.36599999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":7.7880000000000003,"5min":46.020000000000003,"15min":46.020000000000003},"last":5.1769999999999996}]},{"osd":3,"last update":"Mon Aug 28 02:38:03 2023","interfaces":[{"interface":"back","average":{"1min":4.6360000000000001,"5min":4.9930000000000003,"15min":4.5049999999999999},"min":{"1min":0.34999999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":18.463999999999999,"5min":129.20099999999999,"15min":129.20099999999999},"last":0.38200000000000001},{"interface":"front","average":{"1min":4.0670000000000002,"5min":5.4580000000000002,"15min":5.1239999999999997},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":18.399000000000001,"5min":129.512,"15min":129.512},"last":2.0579999999999998}]},{"osd":4,"last update":"Mon Aug 28 02:38:03 2023","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":0.89900000000000002,"15min":1.01},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":4.7400000000000002,"5min":14.984999999999999,"15min":14.984999999999999},"last":1.032},{"interface":"front","average":{"1min":1.6579999999999999,"5min":1.161,"15min":1.2210000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":21.18,"5min":21.18,"15min":21.18},"last":0.317}]},{"osd":5,"last update":"Mon Aug 28 02:38:03 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":1.663,"15min":2.089},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":5.3380000000000001,"5min":36.476999999999997,"15min":36.476999999999997},"last":1.103},{"interface":"front","average":{"1min":1.159,"5min":1.7769999999999999,"15min":2.1850000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":7.8789999999999996,"5min":36.872,"15min":36.872},"last":3.8170000000000002}]},{"osd":6,"last update":"Mon Aug 28 02:38:03 2023","interfaces":[{"interface":"back","average":{"1min":0.75,"5min":2.2890000000000001,"15min":3.04},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":2.294,"5min":53.301000000000002,"15min":53.301000000000002},"last":0.373},{"interface":"front","average":{"1min":1.698,"5min":2.609,"15min":3.2690000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":15.949,"5min":57.915999999999997,"15min":57.915999999999997},"last":5.04}]}]},{"osd":6,"up_from":42,"seq":180388626784,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1165640,"kb_used_data":864320,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92579512,"statfs":{"total":95995035648,"available":94801420288,"internally_reserved":0,"allocated":885063680,"data_stored":884153977,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,1,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:38:26 2023","interfaces":[{"interface":"back","average":{"1min":13.147,"5min":3.5830000000000002,"15min":4.4980000000000002},"min":{"1min":0.27200000000000002,"5min":0.215,"15min":0.20799999999999999},"max":{"1min":185.09800000000001,"5min":185.09800000000001,"15min":185.09800000000001},"last":2.5670000000000002},{"interface":"front","average":{"1min":13.528,"5min":4.2409999999999997,"15min":4.7480000000000002},"min":{"1min":0.28299999999999997,"5min":0.246,"15min":0.23200000000000001},"max":{"1min":185.017,"5min":185.017,"15min":185.017},"last":0.26800000000000002}]},{"osd":1,"last update":"Mon Aug 28 02:38:26 2023","interfaces":[{"interface":"back","average":{"1min":5.1130000000000004,"5min":2.5190000000000001,"15min":2.9550000000000001},"min":{"1min":0.26400000000000001,"5min":0.245,"15min":0.23699999999999999},"max":{"1min":62.981000000000002,"5min":62.981000000000002,"15min":141.87299999999999},"last":3.0329999999999999},{"interface":"front","average":{"1min":6.891,"5min":2.895,"15min":3.1789999999999998},"min":{"1min":0.35699999999999998,"5min":0.25,"15min":0.24299999999999999},"max":{"1min":63.725999999999999,"5min":63.725999999999999,"15min":141.904},"last":7.6479999999999997}]},{"osd":2,"last update":"Mon Aug 28 02:38:13 2023","interfaces":[{"interface":"back","average":{"1min":10.586,"5min":4.3860000000000001,"15min":4.8819999999999997},"min":{"1min":0.35399999999999998,"5min":0.255,"15min":0.255},"max":{"1min":149.321,"5min":149.321,"15min":234.876},"last":0.434},{"interface":"front","average":{"1min":10.167999999999999,"5min":4.3570000000000002,"15min":4.8360000000000003},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":149.34,"5min":149.34,"15min":234.482},"last":0.70499999999999996}]},{"osd":3,"last update":"Mon Aug 28 02:38:26 2023","interfaces":[{"interface":"back","average":{"1min":12.75,"5min":4.5389999999999997,"15min":4.2030000000000003},"min":{"1min":0.36899999999999999,"5min":0.313,"15min":0.313},"max":{"1min":167.233,"5min":167.233,"15min":167.233},"last":3.621},{"interface":"front","average":{"1min":12.99,"5min":4.5940000000000003,"15min":4.1849999999999996},"min":{"1min":0.47599999999999998,"5min":0.35899999999999999,"15min":0.27400000000000002},"max":{"1min":167.25,"5min":167.25,"15min":167.25},"last":3.6299999999999999}]},{"osd":4,"last update":"Mon Aug 28 02:38:29 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":0.52000000000000002,"15min":1.087},"min":{"1min":0.29499999999999998,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":4.5110000000000001,"5min":4.5110000000000001,"15min":18.308},"last":0.36399999999999999},{"interface":"front","average":{"1min":1.71,"5min":0.93000000000000005,"15min":1.2909999999999999},"min":{"1min":0.28599999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":10.678000000000001,"5min":10.678000000000001,"15min":18.263000000000002},"last":0.34499999999999997}]},{"osd":5,"last update":"Mon Aug 28 02:38:29 2023","interfaces":[{"interface":"back","average":{"1min":1.55,"5min":0.79100000000000004,"15min":1.1719999999999999},"min":{"1min":0.307,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":17.419,"5min":17.419,"15min":20.759},"last":0.74299999999999999},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.495,"15min":0.94599999999999995},"min":{"1min":0.32700000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":1.581,"5min":2.1000000000000001,"15min":7.2880000000000003},"last":0.40100000000000002}]},{"osd":7,"last update":"Mon Aug 28 02:37:58 2023","interfaces":[{"interface":"back","average":{"1min":0.83199999999999996,"5min":0.69899999999999995,"15min":0.60299999999999998},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":2.6930000000000001,"5min":3.0419999999999998,"15min":3.0419999999999998},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.624,"5min":0.55500000000000005,"15min":0.52300000000000002},"min":{"1min":0.40999999999999998,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":2.8359999999999999,"5min":2.8359999999999999,"15min":2.8359999999999999},"last":0.58799999999999997}]}]},{"osd":1,"up_from":98,"seq":420906795237,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1388096,"kb_used_data":853752,"kb_used_omap":7,"kb_used_meta":534264,"kb_avail":92357056,"statfs":{"total":95995035648,"available":94573625344,"internally_reserved":0,"allocated":874242048,"data_stored":872425685,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":547086667},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,2],"upper_bound":512},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:38:22 2023","interfaces":[{"interface":"back","average":{"1min":6.1989999999999998,"5min":5.9489999999999998,"15min":4.8319999999999999},"min":{"1min":0.20499999999999999,"5min":0.185,"15min":0.17499999999999999},"max":{"1min":86.706000000000003,"5min":129.346,"15min":129.346},"last":3.0470000000000002},{"interface":"front","average":{"1min":5.9320000000000004,"5min":5.8689999999999998,"15min":4.7919999999999998},"min":{"1min":0.247,"5min":0.19800000000000001,"15min":0.188},"max":{"1min":86.596999999999994,"5min":120.774,"15min":128.685},"last":3.073}]},{"osd":2,"last update":"Mon Aug 28 02:37:51 2023","interfaces":[{"interface":"back","average":{"1min":2.6840000000000002,"5min":3.8519999999999999,"15min":4.4059999999999997},"min":{"1min":0.29399999999999998,"5min":0.189,"15min":0.189},"max":{"1min":14.912000000000001,"5min":119.276,"15min":135.25999999999999},"last":0.24399999999999999},{"interface":"front","average":{"1min":1.736,"5min":3.4279999999999999,"15min":4.0220000000000002},"min":{"1min":0.27800000000000002,"5min":0.221,"15min":0.218},"max":{"1min":5.452,"5min":121.849,"15min":135.10300000000001},"last":0.27300000000000002}]},{"osd":3,"last update":"Mon Aug 28 02:38:22 2023","interfaces":[{"interface":"back","average":{"1min":1.952,"5min":6.0309999999999997,"15min":4.7050000000000001},"min":{"1min":0.23400000000000001,"5min":0.21299999999999999,"15min":0.19700000000000001},"max":{"1min":6.6440000000000001,"5min":126.95699999999999,"15min":126.95699999999999},"last":16.239999999999998},{"interface":"front","average":{"1min":1.879,"5min":5.7489999999999997,"15min":4.8040000000000003},"min":{"1min":0.28299999999999997,"5min":0.217,"15min":0.217},"max":{"1min":5.7709999999999999,"5min":128.00200000000001,"15min":128.00200000000001},"last":12.265000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:38:22 2023","interfaces":[{"interface":"back","average":{"1min":1.9550000000000001,"5min":2.0459999999999998,"15min":1.831},"min":{"1min":0.318,"5min":0.28199999999999997,"15min":0.27500000000000002},"max":{"1min":8.4779999999999998,"5min":24.809999999999999,"15min":24.809999999999999},"last":0.32300000000000001},{"interface":"front","average":{"1min":2.3239999999999998,"5min":1.6160000000000001,"15min":1.6279999999999999},"min":{"1min":0.30499999999999999,"5min":0.29499999999999998,"15min":0.25600000000000001},"max":{"1min":10.34,"5min":24.835999999999999,"15min":24.835999999999999},"last":0.38}]},{"osd":5,"last update":"Mon Aug 28 02:38:22 2023","interfaces":[{"interface":"back","average":{"1min":2.6890000000000001,"5min":3.1859999999999999,"15min":2.3199999999999998},"min":{"1min":0.32100000000000001,"5min":0.252,"15min":0.252},"max":{"1min":11.587,"5min":82.722999999999999,"15min":82.722999999999999},"last":0.626},{"interface":"front","average":{"1min":2.5539999999999998,"5min":3.5310000000000001,"15min":2.319},"min":{"1min":0.34300000000000003,"5min":0.315,"15min":0.307},"max":{"1min":11.223000000000001,"5min":82.733999999999995,"15min":82.733999999999995},"last":0.34100000000000003}]},{"osd":6,"last update":"Mon Aug 28 02:38:22 2023","interfaces":[{"interface":"back","average":{"1min":2.653,"5min":2.2719999999999998,"15min":2.0739999999999998},"min":{"1min":0.35499999999999998,"5min":0.33600000000000002,"15min":0.311},"max":{"1min":11.516,"5min":19.527000000000001,"15min":19.527000000000001},"last":0.59099999999999997},{"interface":"front","average":{"1min":2.9660000000000002,"5min":2.6709999999999998,"15min":2.202},"min":{"1min":0.38200000000000001,"5min":0.34899999999999998,"15min":0.32200000000000001},"max":{"1min":11.19,"5min":19.516999999999999,"15min":19.516999999999999},"last":0.47399999999999998}]},{"osd":7,"last update":"Mon Aug 28 02:38:04 2023","interfaces":[{"interface":"back","average":{"1min":2.105,"5min":1.2509999999999999,"15min":1.024},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":8.3680000000000003,"5min":8.3680000000000003,"15min":8.3680000000000003},"last":0.65000000000000002},{"interface":"front","average":{"1min":1.8779999999999999,"5min":1.2190000000000001,"15min":0.94699999999999995},"min":{"1min":0.42099999999999999,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":9.4900000000000002,"5min":9.4900000000000002,"15min":9.4900000000000002},"last":0.54400000000000004}]}]},{"osd":0,"up_from":90,"seq":386547056883,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1730832,"kb_used_data":1083400,"kb_used_omap":8,"kb_used_meta":647415,"kb_avail":92014320,"statfs":{"total":95995035648,"available":94222663680,"internally_reserved":0,"allocated":1109401600,"data_stored":1107745053,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":662953321},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Aug 28 02:38:28 2023","interfaces":[{"interface":"back","average":{"1min":7.5039999999999996,"5min":6.1029999999999998,"15min":4.1689999999999996},"min":{"1min":0.248,"5min":0.16500000000000001,"15min":0.16500000000000001},"max":{"1min":60.268999999999998,"5min":150.00700000000001,"15min":150.00700000000001},"last":0.221},{"interface":"front","average":{"1min":5.0529999999999999,"5min":5.8700000000000001,"15min":4.4550000000000001},"min":{"1min":0.26300000000000001,"5min":0.19500000000000001,"15min":0.17699999999999999},"max":{"1min":55.457000000000001,"5min":150.27699999999999,"15min":150.27699999999999},"last":1.133}]},{"osd":2,"last update":"Mon Aug 28 02:37:50 2023","interfaces":[{"interface":"back","average":{"1min":3.0819999999999999,"5min":4.3499999999999996,"15min":3.8490000000000002},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.19500000000000001},"max":{"1min":24.030999999999999,"5min":86.269000000000005,"15min":100.718},"last":0.315},{"interface":"front","average":{"1min":3.0899999999999999,"5min":5.367,"15min":4.4610000000000003},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.20000000000000001},"max":{"1min":28.050999999999998,"5min":86.353999999999999,"15min":100.962},"last":1.095}]},{"osd":3,"last update":"Mon Aug 28 02:38:35 2023","interfaces":[{"interface":"back","average":{"1min":3.9249999999999998,"5min":6.2640000000000002,"15min":5.9950000000000001},"min":{"1min":0.26700000000000002,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":9.9339999999999993,"5min":196.702,"15min":237.43899999999999},"last":12.250999999999999},{"interface":"front","average":{"1min":4.194,"5min":6.5119999999999996,"15min":5.9800000000000004},"min":{"1min":0.28100000000000003,"5min":0.26300000000000001,"15min":0.24399999999999999},"max":{"1min":16.827999999999999,"5min":197.75399999999999,"15min":237.48099999999999},"last":5.657}]},{"osd":4,"last update":"Mon Aug 28 02:38:28 2023","interfaces":[{"interface":"back","average":{"1min":12.749000000000001,"5min":3.8300000000000001,"15min":2.4940000000000002},"min":{"1min":0.29299999999999998,"5min":0.25700000000000001,"15min":0.23999999999999999},"max":{"1min":223.64599999999999,"5min":223.64599999999999,"15min":223.64599999999999},"last":2.4689999999999999},{"interface":"front","average":{"1min":2.335,"5min":2.5209999999999999,"15min":2.2240000000000002},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.309},"max":{"1min":9.9809999999999999,"5min":14.241,"15min":27.271999999999998},"last":5.2969999999999997}]},{"osd":5,"last update":"Mon Aug 28 02:38:28 2023","interfaces":[{"interface":"back","average":{"1min":3.125,"5min":2.5339999999999998,"15min":2.327},"min":{"1min":0.34999999999999998,"5min":0.29899999999999999,"15min":0.29599999999999999},"max":{"1min":10.911,"5min":11.429,"15min":44.000999999999998},"last":5.423},{"interface":"front","average":{"1min":2.6629999999999998,"5min":1.764,"15min":1.982},"min":{"1min":0.316,"5min":0.308,"15min":0.307},"max":{"1min":13.103,"5min":13.103,"15min":43.953000000000003},"last":2.573}]},{"osd":6,"last update":"Mon Aug 28 02:38:28 2023","interfaces":[{"interface":"back","average":{"1min":3.504,"5min":2.5819999999999999,"15min":2.4220000000000002},"min":{"1min":0.374,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":16.777999999999999,"5min":16.777999999999999,"15min":32.417999999999999},"last":5.383},{"interface":"front","average":{"1min":2.4689999999999999,"5min":1.847,"15min":2.0720000000000001},"min":{"1min":0.35299999999999998,"5min":0.33700000000000002,"15min":0.30399999999999999},"max":{"1min":7.8419999999999996,"5min":9.9049999999999994,"15min":32.130000000000003},"last":2.5489999999999999}]},{"osd":7,"last update":"Mon Aug 28 02:38:02 2023","interfaces":[{"interface":"back","average":{"1min":2.3599999999999999,"5min":1.452,"15min":1.248},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":9.9280000000000008,"5min":9.9280000000000008,"15min":9.9280000000000008},"last":2.5179999999999998},{"interface":"front","average":{"1min":1.3720000000000001,"5min":1.73,"15min":1.8600000000000001},"min":{"1min":0.41999999999999998,"5min":0.37,"15min":0.37},"max":{"1min":5.1619999999999999,"5min":16.529,"15min":16.529},"last":1.157}]}]},{"osd":2,"up_from":0,"seq":446676599007,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":112,"seq":481036337369,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1319924,"kb_used_data":720748,"kb_used_omap":3,"kb_used_meta":599164,"kb_avail":92425228,"statfs":{"total":95995035648,"available":94643433472,"internally_reserved":0,"allocated":738045952,"data_stored":736409433,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":613544866},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:38:22 2023","interfaces":[{"interface":"back","average":{"1min":3.3330000000000002,"5min":4.6429999999999998,"15min":5.0149999999999997},"min":{"1min":0.21199999999999999,"5min":0.19500000000000001,"15min":0.193},"max":{"1min":14.023999999999999,"5min":138.39699999999999,"15min":164.25299999999999},"last":2.3250000000000002},{"interface":"front","average":{"1min":3.2509999999999999,"5min":3.1859999999999999,"15min":4.2709999999999999},"min":{"1min":0.432,"5min":0.20100000000000001,"15min":0.192},"max":{"1min":14.73,"5min":24.048999999999999,"15min":164.55000000000001},"last":3.452}]},{"osd":1,"last update":"Mon Aug 28 02:38:22 2023","interfaces":[{"interface":"back","average":{"1min":3.3500000000000001,"5min":4.4130000000000003,"15min":3.6040000000000001},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":13.289999999999999,"5min":105.04600000000001,"15min":117.34699999999999},"last":6.2229999999999999},{"interface":"front","average":{"1min":4.9189999999999996,"5min":6.5709999999999997,"15min":4.4500000000000002},"min":{"1min":0.34699999999999998,"5min":0.28699999999999998,"15min":0.224},"max":{"1min":14.301,"5min":143.54400000000001,"15min":143.54400000000001},"last":4.7300000000000004}]},{"osd":2,"last update":"Mon Aug 28 02:38:22 2023","interfaces":[{"interface":"back","average":{"1min":6.46,"5min":5.5449999999999999,"15min":5.0860000000000003},"min":{"1min":0.38800000000000001,"5min":0.23799999999999999,"15min":0.19900000000000001},"max":{"1min":42.417999999999999,"5min":143.50299999999999,"15min":143.50299999999999},"last":2.1970000000000001},{"interface":"front","average":{"1min":5.3170000000000002,"5min":3.4209999999999998,"15min":4.1210000000000004},"min":{"1min":0.377,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":42.229999999999997,"5min":42.229999999999997,"15min":120.44499999999999},"last":3.4860000000000002}]},{"osd":4,"last update":"Mon Aug 28 02:38:22 2023","interfaces":[{"interface":"back","average":{"1min":11.298,"5min":3.6909999999999998,"15min":2.6059999999999999},"min":{"1min":0.36399999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":158.739,"5min":158.739,"15min":158.739},"last":4.9909999999999997},{"interface":"front","average":{"1min":10.269,"5min":3.323,"15min":2.6309999999999998},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":158.72,"5min":158.72,"15min":158.72},"last":6.1529999999999996}]},{"osd":5,"last update":"Mon Aug 28 02:38:22 2023","interfaces":[{"interface":"back","average":{"1min":10.497,"5min":3.419,"15min":2.8149999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.29099999999999998},"max":{"1min":163.55199999999999,"5min":163.55199999999999,"15min":163.55199999999999},"last":6.1180000000000003},{"interface":"front","average":{"1min":12.454000000000001,"5min":5.5709999999999997,"15min":3.5870000000000002},"min":{"1min":0.42099999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":163.45699999999999,"5min":163.45699999999999,"15min":163.45699999999999},"last":2.133}]},{"osd":6,"last update":"Mon Aug 28 02:38:22 2023","interfaces":[{"interface":"back","average":{"1min":3.387,"5min":3.8599999999999999,"15min":2.7759999999999998},"min":{"1min":0.41299999999999998,"5min":0.33000000000000002,"15min":0.314},"max":{"1min":9.3300000000000001,"5min":138.37,"15min":138.37},"last":2.1059999999999999},{"interface":"front","average":{"1min":2.6120000000000001,"5min":1.9910000000000001,"15min":1.9750000000000001},"min":{"1min":0.40500000000000003,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":8.2029999999999994,"5min":12.193,"15min":72.030000000000001},"last":3.536}]},{"osd":7,"last update":"Mon Aug 28 02:38:05 2023","interfaces":[{"interface":"back","average":{"1min":3.4620000000000002,"5min":1.7350000000000001,"15min":1.3480000000000001},"min":{"1min":0.42199999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":7.7069999999999999,"5min":7.7069999999999999,"15min":7.7069999999999999},"last":2.1549999999999998},{"interface":"front","average":{"1min":2.8879999999999999,"5min":1.6830000000000001,"15min":1.419},"min":{"1min":0.41899999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":12.226000000000001,"5min":12.226000000000001,"15min":12.226000000000001},"last":3.512}]}]},{"osd":4,"up_from":30,"seq":128849019242,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1215680,"kb_used_data":625976,"kb_used_omap":0,"kb_used_meta":589696,"kb_avail":92529472,"statfs":{"total":95995035648,"available":94750179328,"internally_reserved":0,"allocated":640999424,"data_stored":640097129,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603848704},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:38:29 2023","interfaces":[{"interface":"back","average":{"1min":12.731999999999999,"5min":6.923,"15min":6.6180000000000003},"min":{"1min":0.24299999999999999,"5min":0.23100000000000001,"15min":0.219},"max":{"1min":168.75800000000001,"5min":168.75800000000001,"15min":168.75800000000001},"last":3.2040000000000002},{"interface":"front","average":{"1min":13.162000000000001,"5min":7.1820000000000004,"15min":6.4850000000000003},"min":{"1min":0.23999999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":168.84200000000001,"5min":168.84200000000001,"15min":168.84200000000001},"last":1.1539999999999999}]},{"osd":1,"last update":"Mon Aug 28 02:38:29 2023","interfaces":[{"interface":"back","average":{"1min":11.567,"5min":6.7199999999999998,"15min":3.6899999999999999},"min":{"1min":0.252,"5min":0.23300000000000001,"15min":0.221},"max":{"1min":107.545,"5min":131.845,"15min":131.845},"last":3.294},{"interface":"front","average":{"1min":9.702,"5min":6.0819999999999999,"15min":3.3980000000000001},"min":{"1min":0.33100000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":107.563,"5min":132.304,"15min":132.304},"last":1.2430000000000001}]},{"osd":2,"last update":"Mon Aug 28 02:37:54 2023","interfaces":[{"interface":"back","average":{"1min":10.226000000000001,"5min":4.415,"15min":4.4859999999999998},"min":{"1min":0.28100000000000003,"5min":0.255,"15min":0.255},"max":{"1min":133.06399999999999,"5min":133.06399999999999,"15min":133.06399999999999},"last":3.3439999999999999},{"interface":"front","average":{"1min":11.221,"5min":4.7480000000000002,"15min":4.4720000000000004},"min":{"1min":0.371,"5min":0.27700000000000002,"15min":0.27300000000000002},"max":{"1min":133.08699999999999,"5min":133.08699999999999,"15min":133.08699999999999},"last":2.8330000000000002}]},{"osd":3,"last update":"Mon Aug 28 02:38:29 2023","interfaces":[{"interface":"back","average":{"1min":9.4120000000000008,"5min":6.4770000000000003,"15min":6.492},"min":{"1min":0.317,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":150.958,"5min":150.958,"15min":203.583},"last":2.7650000000000001},{"interface":"front","average":{"1min":10.08,"5min":6.4850000000000003,"15min":6.6139999999999999},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":150.63499999999999,"5min":150.63499999999999,"15min":203.41399999999999},"last":3.319}]},{"osd":5,"last update":"Mon Aug 28 02:38:06 2023","interfaces":[{"interface":"back","average":{"1min":1.256,"5min":0.67500000000000004,"15min":1.4910000000000001},"min":{"1min":0.26700000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":7.4829999999999997,"5min":7.4829999999999997,"15min":124.718},"last":1.2210000000000001},{"interface":"front","average":{"1min":1.6459999999999999,"5min":0.80700000000000005,"15min":1.72},"min":{"1min":0.28000000000000003,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":6.2869999999999999,"5min":6.2869999999999999,"15min":124.694},"last":2.7879999999999998}]},{"osd":6,"last update":"Mon Aug 28 02:38:29 2023","interfaces":[{"interface":"back","average":{"1min":9.1509999999999998,"5min":2.3159999999999998,"15min":1.6679999999999999},"min":{"1min":0.34300000000000003,"5min":0.25,"15min":0.25},"max":{"1min":148.87100000000001,"5min":148.87100000000001,"15min":148.87100000000001},"last":3.242},{"interface":"front","average":{"1min":9.6579999999999995,"5min":2.488,"15min":1.5169999999999999},"min":{"1min":0.314,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":148.88300000000001,"5min":148.88300000000001,"15min":148.88300000000001},"last":1.194}]},{"osd":7,"last update":"Mon Aug 28 02:37:59 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":0.67700000000000005,"15min":0.57099999999999995},"min":{"1min":0.32900000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":3.5369999999999999,"5min":3.5369999999999999,"15min":3.5369999999999999},"last":3.2690000000000001},{"interface":"front","average":{"1min":1.46,"5min":0.78000000000000003,"15min":0.65100000000000002},"min":{"1min":0.33000000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":7.117,"5min":7.117,"15min":7.117},"last":2.7120000000000002}]}]},{"osd":5,"up_from":36,"seq":154618823014,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1714416,"kb_used_data":1120616,"kb_used_omap":0,"kb_used_meta":593792,"kb_avail":92030736,"statfs":{"total":95995035648,"available":94239473664,"internally_reserved":0,"allocated":1147510784,"data_stored":1146620030,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":608043008},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:38:18 2023","interfaces":[{"interface":"back","average":{"1min":3.7170000000000001,"5min":4.3179999999999996,"15min":4.0999999999999996},"min":{"1min":0.23000000000000001,"5min":0.216,"15min":0.214},"max":{"1min":38.328000000000003,"5min":128.72800000000001,"15min":128.72800000000001},"last":0.371},{"interface":"front","average":{"1min":4.8049999999999997,"5min":4.641,"15min":4.3959999999999999},"min":{"1min":0.318,"5min":0.245,"15min":0.22900000000000001},"max":{"1min":38.314,"5min":128.75200000000001,"15min":128.75200000000001},"last":8.7599999999999998}]},{"osd":1,"last update":"Mon Aug 28 02:38:31 2023","interfaces":[{"interface":"back","average":{"1min":2.383,"5min":7.548,"15min":4.0209999999999999},"min":{"1min":0.28199999999999997,"5min":0.22800000000000001,"15min":0.216},"max":{"1min":9.5579999999999998,"5min":148.01400000000001,"15min":148.01400000000001},"last":0.72499999999999998},{"interface":"front","average":{"1min":2.7349999999999999,"5min":7.7300000000000004,"15min":4.2389999999999999},"min":{"1min":0.32000000000000001,"5min":0.26400000000000001,"15min":0.23200000000000001},"max":{"1min":10.984999999999999,"5min":148.08000000000001,"15min":148.08000000000001},"last":8.8059999999999992}]},{"osd":2,"last update":"Mon Aug 28 02:38:02 2023","interfaces":[{"interface":"back","average":{"1min":2.3119999999999998,"5min":2.5430000000000001,"15min":4.6139999999999999},"min":{"1min":0.28899999999999998,"5min":0.27500000000000002,"15min":0.25800000000000001},"max":{"1min":9.6560000000000006,"5min":78.834000000000003,"15min":176.69499999999999},"last":0.39800000000000002},{"interface":"front","average":{"1min":3.0030000000000001,"5min":3.2330000000000001,"15min":5.0990000000000002},"min":{"1min":0.36099999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":10.4,"5min":78.798000000000002,"15min":176.71199999999999},"last":8.8900000000000006}]},{"osd":3,"last update":"Mon Aug 28 02:38:31 2023","interfaces":[{"interface":"back","average":{"1min":2.3860000000000001,"5min":3.2309999999999999,"15min":4.1539999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.26100000000000001},"max":{"1min":8.0749999999999993,"5min":87.233999999999995,"15min":128.571},"last":0.41999999999999998},{"interface":"front","average":{"1min":3.274,"5min":3.2240000000000002,"15min":4.4139999999999997},"min":{"1min":0.41199999999999998,"5min":0.28699999999999998,"15min":0.26200000000000001},"max":{"1min":16.564,"5min":87.337000000000003,"15min":128.684},"last":8.9830000000000005}]},{"osd":4,"last update":"Mon Aug 28 02:38:09 2023","interfaces":[{"interface":"back","average":{"1min":1.113,"5min":0.59999999999999998,"15min":1.3240000000000001},"min":{"1min":0.28199999999999997,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":5.5819999999999999,"5min":5.5819999999999999,"15min":85.769999999999996},"last":5.2800000000000002},{"interface":"front","average":{"1min":1.01,"5min":0.59299999999999997,"15min":1.3089999999999999},"min":{"1min":0.30199999999999999,"5min":0.248,"15min":0.248},"max":{"1min":8.1189999999999998,"5min":8.1189999999999998,"15min":85.825000000000003},"last":3.9279999999999999}]},{"osd":6,"last update":"Mon Aug 28 02:38:31 2023","interfaces":[{"interface":"back","average":{"1min":1.3360000000000001,"5min":0.67300000000000004,"15min":1.034},"min":{"1min":0.31,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":5.6150000000000002,"5min":5.6150000000000002,"15min":14.803000000000001},"last":5.3129999999999997},{"interface":"front","average":{"1min":1.0629999999999999,"5min":0.70899999999999996,"15min":0.99099999999999999},"min":{"1min":0.35199999999999998,"5min":0.29599999999999999,"15min":0.28799999999999998},"max":{"1min":8.9480000000000004,"5min":8.9480000000000004,"15min":8.9480000000000004},"last":3.4689999999999999}]},{"osd":7,"last update":"Mon Aug 28 02:38:02 2023","interfaces":[{"interface":"back","average":{"1min":1.446,"5min":0.749,"15min":0.65100000000000002},"min":{"1min":0.37,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":8.9100000000000001,"5min":8.9100000000000001,"15min":8.9100000000000001},"last":3.9910000000000001},{"interface":"front","average":{"1min":1.0169999999999999,"5min":0.77700000000000002,"15min":0.72699999999999998},"min":{"1min":0.39800000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":4.5650000000000004,"5min":4.5650000000000004,"15min":4.5650000000000004},"last":8.9199999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8527,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1085079552,"data_stored":1085079552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":862322688,"data_stored":862322688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":836898816,"data_stored":836894744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":730398720,"data_stored":730398720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":636878848,"data_stored":636878848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1131614208,"data_stored":1131610136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":875106304,"data_stored":875102232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":17039360,"data_stored":17039360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-08-28T02:38:48.854 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-08-28T02:38:48.854 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd pool set device_health_metrics pg_num 51 2023-08-28T02:38:48.855 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:48 smithi143 ceph-mon[189871]: pgmap v966: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 1.8 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 465 op/s; 3523/87690 objects degraded (4.018%); 20736/87690 objects misplaced (23.647%); 2.2 MiB/s, 0 keys/s, 35 objects/s recovering 2023-08-28T02:38:48.855 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:48 smithi143 ceph-mon[183059]: pgmap v966: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 1.8 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 465 op/s; 3523/87690 objects degraded (4.018%); 20736/87690 objects misplaced (23.647%); 2.2 MiB/s, 0 keys/s, 35 objects/s recovering 2023-08-28T02:38:49.013 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:38:49.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:49.032 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:38:49.133 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:38:49.191 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:49.296 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:38:49.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:49.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:38:49.309 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:38:49.406 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:38:49.504 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:49.526 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:38:49.752 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:38:49.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:49.772 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:38:49.873 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:38:49.944 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:50.789 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is set 2023-08-28T02:38:51.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:50 smithi143 ceph-mon[183059]: pgmap v967: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 1.7 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 554 op/s; 3523/82992 objects degraded (4.245%); 19953/82992 objects misplaced (24.042%); 2.1 MiB/s, 33 objects/s recovering 2023-08-28T02:38:51.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:50 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3853063373' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:38:51.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:50 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:38:51.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:50 smithi143 ceph-mon[189871]: pgmap v967: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 1.7 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 554 op/s; 3523/82992 objects degraded (4.245%); 19953/82992 objects misplaced (24.042%); 2.1 MiB/s, 33 objects/s recovering 2023-08-28T02:38:51.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:50 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3853063373' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:38:51.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:50 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:38:51.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:50 smithi169 ceph-mon[159415]: pgmap v967: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 1.7 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 554 op/s; 3523/82992 objects degraded (4.245%); 19953/82992 objects misplaced (24.042%); 2.1 MiB/s, 33 objects/s recovering 2023-08-28T02:38:51.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:50 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3853063373' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:38:51.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:50 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:38:52.201 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:38:51 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:38:51] "GET /metrics HTTP/1.1" 200 36573 "" "Prometheus/2.33.4" 2023-08-28T02:38:52.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:51 smithi143 ceph-mon[183059]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:38:52.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:51 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:38:52.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:51 smithi143 ceph-mon[183059]: osdmap e357: 8 total, 8 up, 7 in 2023-08-28T02:38:52.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:51 smithi143 ceph-mon[183059]: failed to encode map e357 with expected crc 2023-08-28T02:38:52.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:51 smithi143 ceph-mon[183059]: failed to encode map e357 with expected crc 2023-08-28T02:38:52.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:51 smithi143 ceph-mon[183059]: failed to encode map e357 with expected crc 2023-08-28T02:38:52.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:51 smithi143 ceph-mon[183059]: failed to encode map e357 with expected crc 2023-08-28T02:38:52.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:51 smithi143 ceph-mon[183059]: failed to encode map e357 with expected crc 2023-08-28T02:38:52.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:51 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 3523/82992 objects degraded (4.245%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:38:52.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:51 smithi143 ceph-mon[189871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:38:52.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:51 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:38:52.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:51 smithi143 ceph-mon[189871]: osdmap e357: 8 total, 8 up, 7 in 2023-08-28T02:38:52.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:51 smithi143 ceph-mon[189871]: failed to encode map e357 with expected crc 2023-08-28T02:38:52.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:51 smithi143 ceph-mon[189871]: failed to encode map e357 with expected crc 2023-08-28T02:38:52.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:51 smithi143 ceph-mon[189871]: failed to encode map e357 with expected crc 2023-08-28T02:38:52.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:51 smithi143 ceph-mon[189871]: failed to encode map e357 with expected crc 2023-08-28T02:38:52.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:51 smithi143 ceph-mon[189871]: failed to encode map e357 with expected crc 2023-08-28T02:38:52.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:51 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 3523/82992 objects degraded (4.245%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:38:52.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:51 smithi169 ceph-mon[159415]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:38:52.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:51 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:38:52.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:51 smithi169 ceph-mon[159415]: osdmap e357: 8 total, 8 up, 7 in 2023-08-28T02:38:52.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:51 smithi169 ceph-mon[159415]: failed to encode map e357 with expected crc 2023-08-28T02:38:52.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:51 smithi169 ceph-mon[159415]: failed to encode map e357 with expected crc 2023-08-28T02:38:52.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:51 smithi169 ceph-mon[159415]: failed to encode map e357 with expected crc 2023-08-28T02:38:52.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:51 smithi169 ceph-mon[159415]: failed to encode map e357 with expected crc 2023-08-28T02:38:52.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:51 smithi169 ceph-mon[159415]: failed to encode map e357 with expected crc 2023-08-28T02:38:52.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:51 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 3523/82992 objects degraded (4.245%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:38:52.562 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:38:52.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:52.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:38:52.668 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:38:52.735 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:38:52.741 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:52.816 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:38:52.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:52.834 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:38:52.935 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:38:52.977 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:52 smithi143 ceph-mon[183059]: pgmap v969: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 1.6 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 581 op/s; 3523/80844 objects degraded (4.358%); 19416/80844 objects misplaced (24.017%); 2.2 MiB/s, 34 objects/s recovering 2023-08-28T02:38:52.977 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:52 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:38:52.978 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:52 smithi143 ceph-mon[189871]: pgmap v969: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 1.6 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 581 op/s; 3523/80844 objects degraded (4.358%); 19416/80844 objects misplaced (24.017%); 2.2 MiB/s, 34 objects/s recovering 2023-08-28T02:38:52.978 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:52 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:38:52.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:52 smithi169 ceph-mon[159415]: pgmap v969: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 1.6 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 581 op/s; 3523/80844 objects degraded (4.358%); 19416/80844 objects misplaced (24.017%); 2.2 MiB/s, 34 objects/s recovering 2023-08-28T02:38:52.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:52 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:38:53.012 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:53.126 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:38:53.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:53.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:38:53.238 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:38:53.246 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:38:53.282 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:38:53.282 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:38:53.303 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:53.303 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:38:53.337 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:38:53.382 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:38:53.429 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:38:53.492 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:38:53.529 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:38:53.571 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:38:53.580 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:38:53.580 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:38:53.581 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:38:53.581 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:38:53.581 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:38:53.581 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:38:53.581 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:38:53.581 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:38:53.732 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:38:53.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:53.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:38:53.836 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:38:53.892 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:53 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/619635219' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:38:53.892 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:53 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/619635219' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:38:53.904 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:54.246 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:53 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/619635219' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:38:54.722 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset noscrub 2023-08-28T02:38:54.799 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:38:54.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:54.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:38:54.909 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:38:54.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:54 smithi143 ceph-mon[183059]: pgmap v970: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 1.6 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 587 op/s; 3523/76587 objects degraded (4.600%); 18689/76587 objects misplaced (24.402%); 2.1 MiB/s, 33 objects/s recovering 2023-08-28T02:38:54.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:54 smithi143 ceph-mon[189871]: pgmap v970: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 1.6 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 587 op/s; 3523/76587 objects degraded (4.600%); 18689/76587 objects misplaced (24.402%); 2.1 MiB/s, 33 objects/s recovering 2023-08-28T02:38:54.974 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:55.172 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:38:55.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:55.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:38:55.180 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:54 smithi169 ceph-mon[159415]: pgmap v970: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 1.6 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 587 op/s; 3523/76587 objects degraded (4.600%); 18689/76587 objects misplaced (24.402%); 2.1 MiB/s, 33 objects/s recovering 2023-08-28T02:38:55.280 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:38:55.360 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:55.811 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.1 dump_blocked_ops 2023-08-28T02:38:56.369 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:38:56.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:56.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:38:56.509 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:38:56.584 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:38:56.714 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:56.919 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:38:56.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:56.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:38:56.931 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:56 smithi143 ceph-mon[183059]: pgmap v971: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 1.5 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 591 op/s; 3523/72225 objects degraded (4.878%); 17784/72225 objects misplaced (24.623%); 2.1 MiB/s, 33 objects/s recovering 2023-08-28T02:38:56.931 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:56 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:38:56.932 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:56 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 3523/72225 objects degraded (4.878%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:38:56.934 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:56 smithi143 ceph-mon[189871]: pgmap v971: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 1.5 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 591 op/s; 3523/72225 objects degraded (4.878%); 17784/72225 objects misplaced (24.623%); 2.1 MiB/s, 33 objects/s recovering 2023-08-28T02:38:56.934 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:56 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:38:56.934 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:56 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 3523/72225 objects degraded (4.878%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:38:56.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:56 smithi169 ceph-mon[159415]: pgmap v971: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 1.5 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 591 op/s; 3523/72225 objects degraded (4.878%); 17784/72225 objects misplaced (24.623%); 2.1 MiB/s, 33 objects/s recovering 2023-08-28T02:38:56.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:56 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:38:56.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:56 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 3523/72225 objects degraded (4.878%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:38:57.032 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:38:57.083 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:57.155 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:38:57.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:57.163 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:38:57.196 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:38:57.264 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:38:57.344 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:57.408 INFO:teuthology.orchestra.run.smithi143.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-08-28T02:38:57.459 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:38:57.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:57.470 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:38:57.571 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:38:57.607 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:57.715 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:38:57.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:57.724 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:38:57.825 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:38:57.908 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:58.054 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:38:58.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:58.066 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:38:58.167 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:38:58.226 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:58 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:38:58.226 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:58 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:38:58.226 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:58 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/534364788' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "51"}]: dispatch 2023-08-28T02:38:58.226 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:38:58 smithi169 ceph-mon[159415]: pgmap v972: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 1.4 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 586 op/s; 3523/70080 objects degraded (5.027%); 17479/70080 objects misplaced (24.941%); 2.0 MiB/s, 32 objects/s recovering 2023-08-28T02:38:58.235 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:58.348 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:38:58.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:58.353 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:38:58.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:58 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:38:58.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:58 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:38:58.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:58 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/534364788' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "51"}]: dispatch 2023-08-28T02:38:58.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:38:58 smithi143 ceph-mon[183059]: pgmap v972: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 1.4 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 586 op/s; 3523/70080 objects degraded (5.027%); 17479/70080 objects misplaced (24.941%); 2.0 MiB/s, 32 objects/s recovering 2023-08-28T02:38:58.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:58 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:38:58.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:58 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:38:58.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:58 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/534364788' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "51"}]: dispatch 2023-08-28T02:38:58.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:38:58 smithi143 ceph-mon[189871]: pgmap v972: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 1.4 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 586 op/s; 3523/70080 objects degraded (5.027%); 17479/70080 objects misplaced (24.941%); 2.0 MiB/s, 32 objects/s recovering 2023-08-28T02:38:58.454 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:38:58.506 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:58.644 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:38:58.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:58.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:38:58.763 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:38:58.826 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:58.910 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-08-28T02:38:58.912 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:38:58.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:58.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:38:58.937 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:38:59.022 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:38:59.103 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:59.116 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:38:59.207 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:38:59.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:59.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:38:59.316 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:38:59.410 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:38:59.692 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:38:59.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:38:59.696 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:38:59.797 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:38:59.865 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:00.049 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:39:00.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:00.054 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:39:00.155 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:39:00.701 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is unset 2023-08-28T02:39:00.721 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:00.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:00 smithi143 ceph-mon[183059]: pgmap v973: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 1.3 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 572 op/s; 3523/65805 objects degraded (5.354%); 16756/65805 objects misplaced (25.463%); 2.0 MiB/s, 32 objects/s recovering 2023-08-28T02:39:00.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:00 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3940566251' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:39:00.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:00 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:39:00.953 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:00 smithi143 ceph-mon[189871]: pgmap v973: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 1.3 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 572 op/s; 3523/65805 objects degraded (5.354%); 16756/65805 objects misplaced (25.463%); 2.0 MiB/s, 32 objects/s recovering 2023-08-28T02:39:00.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:00 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3940566251' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:39:00.954 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:00 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:39:01.072 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:00 smithi169 ceph-mon[159415]: pgmap v973: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 1.3 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 572 op/s; 3523/65805 objects degraded (5.354%); 16756/65805 objects misplaced (25.463%); 2.0 MiB/s, 32 objects/s recovering 2023-08-28T02:39:01.072 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:00 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3940566251' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:39:01.072 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:00 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:39:01.164 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:39:01.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:01.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:39:01.274 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:39:01.325 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:01.893 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:01 smithi143 ceph-mon[189871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:39:01.893 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:01 smithi143 ceph-mon[189871]: failed to encode map e358 with expected crc 2023-08-28T02:39:01.894 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:01 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:39:01.894 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:01 smithi143 ceph-mon[189871]: osdmap e358: 8 total, 8 up, 7 in 2023-08-28T02:39:01.894 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:01 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 3523/65805 objects degraded (5.354%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:39:01.897 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:01 smithi143 ceph-mon[183059]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:39:01.897 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:01 smithi143 ceph-mon[183059]: failed to encode map e358 with expected crc 2023-08-28T02:39:01.897 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:01 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:39:01.897 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:01 smithi143 ceph-mon[183059]: osdmap e358: 8 total, 8 up, 7 in 2023-08-28T02:39:01.897 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:01 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 3523/65805 objects degraded (5.354%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:39:01.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:01 smithi169 ceph-mon[159415]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:39:01.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:01 smithi169 ceph-mon[159415]: failed to encode map e358 with expected crc 2023-08-28T02:39:01.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:01 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:39:01.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:01 smithi169 ceph-mon[159415]: osdmap e358: 8 total, 8 up, 7 in 2023-08-28T02:39:01.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:01 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 3523/65805 objects degraded (5.354%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:39:02.162 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:39:02.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:02.181 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:39:01 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:39:01] "GET /metrics HTTP/1.1" 200 36575 "" "Prometheus/2.33.4" 2023-08-28T02:39:02.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:39:02.282 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:39:02.358 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:02.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:02 smithi169 ceph-mon[159415]: pgmap v975: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 1.3 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 569 op/s; 3523/63762 objects degraded (5.525%); 16255/63762 objects misplaced (25.493%); 2.0 MiB/s, 32 objects/s recovering 2023-08-28T02:39:03.019 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:39:03.019 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [], 2023-08-28T02:39:03.020 INFO:teuthology.orchestra.run.smithi143.stdout: "complaint_time": 30, 2023-08-28T02:39:03.020 INFO:teuthology.orchestra.run.smithi143.stdout: "num_blocked_ops": 0 2023-08-28T02:39:03.020 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:39:03.039 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:39:03.051 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:02 smithi143 ceph-mon[183059]: pgmap v975: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 1.3 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 569 op/s; 3523/63762 objects degraded (5.525%); 16255/63762 objects misplaced (25.493%); 2.0 MiB/s, 32 objects/s recovering 2023-08-28T02:39:03.052 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:02 smithi143 ceph-mon[189871]: pgmap v975: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 1.3 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 569 op/s; 3523/63762 objects degraded (5.525%); 16255/63762 objects misplaced (25.493%); 2.0 MiB/s, 32 objects/s recovering 2023-08-28T02:39:03.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:03.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:39:03.153 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:39:03.451 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:03.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:03.593 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:39:03.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:39:03.694 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:39:03.722 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:03.912 INFO:tasks.thrashosds.thrasher:in_osds: [0, 4, 5, 3, 6, 1, 7] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-08-28T02:39:03.912 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-08-28T02:39:03.912 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-08-28T02:39:03.912 INFO:tasks.thrashosds.thrasher:Setting osd 6 primary_affinity to 1.000000 2023-08-28T02:39:03.913 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd primary-affinity 6 1 2023-08-28T02:39:03.951 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:39:04.025 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:39:04.075 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:39:04.111 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:39:04.161 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:39:04.192 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:39:04.210 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:39:04.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:04.217 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:39:04.247 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:39:04.282 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:39:04.295 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:39:04.295 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:39:04.295 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:39:04.295 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:39:04.295 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:39:04.295 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:39:04.295 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:39:04.295 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:39:04.318 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:39:04.366 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:04.384 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.1 dump_historic_ops 2023-08-28T02:39:04.438 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:39:04.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:04.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:39:04.543 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:39:04.637 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:04.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:04 smithi143 ceph-mon[183059]: pgmap v976: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 1.2 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 573 op/s; 3523/59379 objects degraded (5.933%); 15501/59379 objects misplaced (26.105%); 2.1 MiB/s, 33 objects/s recovering 2023-08-28T02:39:04.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:04 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2048651137' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:39:04.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:04 smithi143 ceph-mon[189871]: pgmap v976: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 1.2 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 573 op/s; 3523/59379 objects degraded (5.933%); 15501/59379 objects misplaced (26.105%); 2.1 MiB/s, 33 objects/s recovering 2023-08-28T02:39:04.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:04 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2048651137' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:39:04.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:04 smithi169 ceph-mon[159415]: pgmap v976: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 1.2 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 573 op/s; 3523/59379 objects degraded (5.933%); 15501/59379 objects misplaced (26.105%); 2.1 MiB/s, 33 objects/s recovering 2023-08-28T02:39:04.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:04 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2048651137' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:39:05.353 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset nodeep-scrub 2023-08-28T02:39:05.449 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:39:05.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:05.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:39:05.575 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:39:05.593 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:39:05.688 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:05.855 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:39:05.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:05.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:39:05.911 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:39:05.973 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:39:06.084 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:06.345 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:39:06.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:06.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:39:06.450 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:39:06.573 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:07.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:06 smithi143 ceph-mon[189871]: pgmap v977: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 1.1 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 564 op/s; 3523/55284 objects degraded (6.373%); 14665/55284 objects misplaced (26.527%); 2.1 MiB/s, 33 objects/s recovering 2023-08-28T02:39:07.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:06 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 3523/55284 objects degraded (6.373%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:39:07.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:06 smithi143 ceph-mon[183059]: pgmap v977: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 1.1 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 564 op/s; 3523/55284 objects degraded (6.373%); 14665/55284 objects misplaced (26.527%); 2.1 MiB/s, 33 objects/s recovering 2023-08-28T02:39:07.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:06 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 3523/55284 objects degraded (6.373%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:39:07.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:06 smithi169 ceph-mon[159415]: pgmap v977: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 1.1 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 564 op/s; 3523/55284 objects degraded (6.373%); 14665/55284 objects misplaced (26.527%); 2.1 MiB/s, 33 objects/s recovering 2023-08-28T02:39:07.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:06 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 3523/55284 objects degraded (6.373%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:39:07.458 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:39:07.896 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:39:07.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:07.915 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:39:08.016 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:39:08.107 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:08.858 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:39:08.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:08.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:39:08.880 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:39:08.976 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:39:09.036 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:08 smithi169 ceph-mon[159415]: pgmap v978: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 1.1 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 563 op/s; 3523/53160 objects degraded (6.627%); 14283/53160 objects misplaced (26.868%); 2.5 MiB/s, 40 objects/s recovering 2023-08-28T02:39:09.036 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:08 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:39:09.036 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:08 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:39:09.036 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:09.114 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:08 smithi143 ceph-mon[183059]: pgmap v978: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 1.1 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 563 op/s; 3523/53160 objects degraded (6.627%); 14283/53160 objects misplaced (26.868%); 2.5 MiB/s, 40 objects/s recovering 2023-08-28T02:39:09.114 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:08 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:39:09.114 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:08 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:39:09.115 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:08 smithi143 ceph-mon[189871]: pgmap v978: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 1.1 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 563 op/s; 3523/53160 objects degraded (6.627%); 14283/53160 objects misplaced (26.868%); 2.5 MiB/s, 40 objects/s recovering 2023-08-28T02:39:09.115 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:08 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:39:09.115 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:08 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:39:09.149 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:39:09.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:09.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:39:09.267 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:39:09.325 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:09.365 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:39:09.468 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:39:09.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:09.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:39:09.573 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:39:09.617 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:09.691 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:39:09.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:09.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:39:09.781 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:09 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:39:09.781 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:09 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:39:09.781 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:09 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:39:09.800 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:39:09.804 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:39:09.804 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:39:09.904 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:09.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:09 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:39:09.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:09 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:39:09.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:09 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:39:10.105 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:09 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:39:10.105 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:09 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:39:10.105 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:09 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:39:10.819 INFO:teuthology.orchestra.run.smithi143.stderr:set osd.6 primary-affinity to 1 (8655362) 2023-08-28T02:39:11.069 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:39:11.072 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:10 smithi143 ceph-mon[183059]: pgmap v979: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 1022 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 562 op/s; 3523/48915 objects degraded (7.202%); 13589/48915 objects misplaced (27.781%); 2.5 MiB/s, 40 objects/s recovering 2023-08-28T02:39:11.072 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:10 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/4061783637' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-08-28T02:39:11.072 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:10 smithi143 ceph-mon[189871]: pgmap v979: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 1022 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 562 op/s; 3523/48915 objects degraded (7.202%); 13589/48915 objects misplaced (27.781%); 2.5 MiB/s, 40 objects/s recovering 2023-08-28T02:39:11.072 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:10 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/4061783637' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-08-28T02:39:11.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:11.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:39:11.178 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:39:11.205 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:11.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:10 smithi169 ceph-mon[159415]: pgmap v979: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 1022 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 562 op/s; 3523/48915 objects degraded (7.202%); 13589/48915 objects misplaced (27.781%); 2.5 MiB/s, 40 objects/s recovering 2023-08-28T02:39:11.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:10 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/4061783637' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-08-28T02:39:11.927 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:11 smithi143 ceph-mon[183059]: failed to encode map e359 with expected crc 2023-08-28T02:39:11.927 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:11 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/4061783637' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-08-28T02:39:11.927 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:11 smithi143 ceph-mon[183059]: osdmap e359: 8 total, 8 up, 7 in 2023-08-28T02:39:11.927 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:11 smithi143 ceph-mon[183059]: failed to encode map e359 with expected crc 2023-08-28T02:39:11.927 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:11 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:39:11.927 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:11 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 3523/48915 objects degraded (7.202%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:39:11.927 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:11 smithi143 ceph-mon[189871]: failed to encode map e359 with expected crc 2023-08-28T02:39:11.928 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:11 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/4061783637' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-08-28T02:39:11.928 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:11 smithi143 ceph-mon[189871]: osdmap e359: 8 total, 8 up, 7 in 2023-08-28T02:39:11.928 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:11 smithi143 ceph-mon[189871]: failed to encode map e359 with expected crc 2023-08-28T02:39:11.928 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:11 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:39:11.928 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:11 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 3523/48915 objects degraded (7.202%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:39:12.201 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:39:11 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:39:11] "GET /metrics HTTP/1.1" 200 36578 "" "Prometheus/2.33.4" 2023-08-28T02:39:12.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:11 smithi169 ceph-mon[159415]: failed to encode map e359 with expected crc 2023-08-28T02:39:12.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:11 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/4061783637' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-08-28T02:39:12.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:11 smithi169 ceph-mon[159415]: osdmap e359: 8 total, 8 up, 7 in 2023-08-28T02:39:12.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:11 smithi169 ceph-mon[159415]: failed to encode map e359 with expected crc 2023-08-28T02:39:12.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:11 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:39:12.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:11 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 3523/48915 objects degraded (7.202%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:39:12.548 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.2 dump_ops_in_flight 2023-08-28T02:39:12.708 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:39:12.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:12.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:39:12.814 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:39:12.865 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:12.967 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:39:12.974 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:12 smithi143 ceph-mon[189871]: pgmap v981: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 977 MiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 566 op/s; 3523/46755 objects degraded (7.535%); 13061/46755 objects misplaced (27.935%); 2.5 MiB/s, 40 objects/s recovering 2023-08-28T02:39:12.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:12.974 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:12 smithi143 ceph-mon[183059]: pgmap v981: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 977 MiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 566 op/s; 3523/46755 objects degraded (7.535%); 13061/46755 objects misplaced (27.935%); 2.5 MiB/s, 40 objects/s recovering 2023-08-28T02:39:12.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:39:13.076 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:39:13.127 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:12 smithi169 ceph-mon[159415]: pgmap v981: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 977 MiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 566 op/s; 3523/46755 objects degraded (7.535%); 13061/46755 objects misplaced (27.935%); 2.5 MiB/s, 40 objects/s recovering 2023-08-28T02:39:13.139 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:13.309 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:39:13.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:13.317 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:39:13.418 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:39:13.472 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:13.741 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:39:13.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:13.746 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:39:13.846 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:39:13.938 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:14.086 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:39:14.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:14.098 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:39:14.198 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:39:14.283 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:39:14.297 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:14.408 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:39:14.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:14.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:39:14.484 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:39:14.515 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:39:14.516 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:39:14.554 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:39:14.578 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:14.582 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:39:14.634 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:39:14.693 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:39:14.703 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:39:14.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:14.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:39:14.736 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:39:14.778 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:39:14.787 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:39:14.788 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:39:14.788 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:39:14.788 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:39:14.788 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:39:14.788 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:39:14.788 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:39:14.788 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:39:14.813 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:39:14.842 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:14 smithi169 ceph-mon[159415]: pgmap v982: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 889 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 3523/42531 objects degraded (8.283%); 12334/42531 objects misplaced (29.000%); 2.7 MiB/s, 43 objects/s recovering 2023-08-28T02:39:14.843 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:14 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2290490057' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:39:14.843 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:14 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/244829359' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:39:14.874 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is unset 2023-08-28T02:39:14.881 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:15.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:15.028 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:39:15.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:39:15.129 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:39:15.172 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:14 smithi143 ceph-mon[183059]: pgmap v982: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 889 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 3523/42531 objects degraded (8.283%); 12334/42531 objects misplaced (29.000%); 2.7 MiB/s, 43 objects/s recovering 2023-08-28T02:39:15.173 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:14 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2290490057' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:39:15.173 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:14 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/244829359' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:39:15.179 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:14 smithi143 ceph-mon[189871]: pgmap v982: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 889 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 3523/42531 objects degraded (8.283%); 12334/42531 objects misplaced (29.000%); 2.7 MiB/s, 43 objects/s recovering 2023-08-28T02:39:15.179 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:14 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2290490057' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:39:15.179 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:14 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/244829359' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:39:15.188 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:16.141 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:15 smithi143 ceph-mon[183059]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:39:16.142 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:15 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/244829359' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:39:16.142 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:15 smithi143 ceph-mon[183059]: osdmap e360: 8 total, 8 up, 7 in 2023-08-28T02:39:16.142 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:15 smithi143 ceph-mon[183059]: failed to encode map e360 with expected crc 2023-08-28T02:39:16.142 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:15 smithi143 ceph-mon[183059]: failed to encode map e360 with expected crc 2023-08-28T02:39:16.142 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:15 smithi143 ceph-mon[183059]: failed to encode map e360 with expected crc 2023-08-28T02:39:16.142 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:15 smithi143 ceph-mon[183059]: failed to encode map e360 with expected crc 2023-08-28T02:39:16.142 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:15 smithi143 ceph-mon[183059]: failed to encode map e360 with expected crc 2023-08-28T02:39:16.142 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:15 smithi143 ceph-mon[183059]: failed to encode map e360 with expected crc 2023-08-28T02:39:16.142 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:15 smithi143 ceph-mon[183059]: failed to encode map e360 with expected crc 2023-08-28T02:39:16.142 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:15 smithi143 ceph-mon[189871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:39:16.143 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:15 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/244829359' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:39:16.143 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:15 smithi143 ceph-mon[189871]: osdmap e360: 8 total, 8 up, 7 in 2023-08-28T02:39:16.143 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:15 smithi143 ceph-mon[189871]: failed to encode map e360 with expected crc 2023-08-28T02:39:16.143 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:15 smithi143 ceph-mon[189871]: failed to encode map e360 with expected crc 2023-08-28T02:39:16.143 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:15 smithi143 ceph-mon[189871]: failed to encode map e360 with expected crc 2023-08-28T02:39:16.143 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:15 smithi143 ceph-mon[189871]: failed to encode map e360 with expected crc 2023-08-28T02:39:16.143 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:15 smithi143 ceph-mon[189871]: failed to encode map e360 with expected crc 2023-08-28T02:39:16.143 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:15 smithi143 ceph-mon[189871]: failed to encode map e360 with expected crc 2023-08-28T02:39:16.143 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:15 smithi143 ceph-mon[189871]: failed to encode map e360 with expected crc 2023-08-28T02:39:16.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:15 smithi169 ceph-mon[159415]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:39:16.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:15 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/244829359' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:39:16.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:15 smithi169 ceph-mon[159415]: osdmap e360: 8 total, 8 up, 7 in 2023-08-28T02:39:16.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:15 smithi169 ceph-mon[159415]: failed to encode map e360 with expected crc 2023-08-28T02:39:16.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:15 smithi169 ceph-mon[159415]: failed to encode map e360 with expected crc 2023-08-28T02:39:16.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:15 smithi169 ceph-mon[159415]: failed to encode map e360 with expected crc 2023-08-28T02:39:16.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:15 smithi169 ceph-mon[159415]: failed to encode map e360 with expected crc 2023-08-28T02:39:16.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:15 smithi169 ceph-mon[159415]: failed to encode map e360 with expected crc 2023-08-28T02:39:16.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:15 smithi169 ceph-mon[159415]: failed to encode map e360 with expected crc 2023-08-28T02:39:16.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:15 smithi169 ceph-mon[159415]: failed to encode map e360 with expected crc 2023-08-28T02:39:17.131 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:16 smithi143 ceph-mon[183059]: pgmap v984: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 796 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 628 op/s; 3523/38055 objects degraded (9.258%); 11406/38055 objects misplaced (29.972%); 1.8 MiB/s, 28 objects/s recovering 2023-08-28T02:39:17.132 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:16 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 3523/38055 objects degraded (9.258%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:39:17.132 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:16 smithi143 ceph-mon[189871]: pgmap v984: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 796 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 628 op/s; 3523/38055 objects degraded (9.258%); 11406/38055 objects misplaced (29.972%); 1.8 MiB/s, 28 objects/s recovering 2023-08-28T02:39:17.132 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:16 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 3523/38055 objects degraded (9.258%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:39:17.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:16 smithi169 ceph-mon[159415]: pgmap v984: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 796 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 628 op/s; 3523/38055 objects degraded (9.258%); 11406/38055 objects misplaced (29.972%); 1.8 MiB/s, 28 objects/s recovering 2023-08-28T02:39:17.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:16 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 3523/38055 objects degraded (9.258%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:39:17.252 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:39:17.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:17.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:39:17.365 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:39:17.414 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:17.505 INFO:tasks.thrashosds.thrasher:in_osds: [0, 4, 5, 3, 6, 1, 7] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-08-28T02:39:17.506 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-08-28T02:39:17.506 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-08-28T02:39:17.506 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-08-28T02:39:17.506 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-08-28T02:39:17.507 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-08-28T02:39:17.524 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:39:17.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:17.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:39:17.632 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:39:17.695 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:17.819 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:39:17.823 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:39:17.823 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [ 2023-08-28T02:39:17.823 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.823 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.45647.0:183592 8.d 8:b578949d:::benchmark_data_smithi143_419289_object86682:head [delete] snapc 0=[] ondisk+write+known_if_redirected e360)", 2023-08-28T02:39:17.823 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:39:17.771431+0000", 2023-08-28T02:39:17.823 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.051290827999999997, 2023-08-28T02:39:17.824 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.051321645999999999, 2023-08-28T02:39:17.824 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:39:17.824 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "waiting for sub ops", 2023-08-28T02:39:17.824 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:39:17.824 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.45647", 2023-08-28T02:39:17.824 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3230872490", 2023-08-28T02:39:17.824 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 183592 2023-08-28T02:39:17.824 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.824 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:39:17.824 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.825 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:39:17.825 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.771431+0000", 2023-08-28T02:39:17.825 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:39:17.825 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.825 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.825 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:39:17.825 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.771431+0000", 2023-08-28T02:39:17.825 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:39:17.825 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.825 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.825 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:39:17.826 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.771429+0000", 2023-08-28T02:39:17.826 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:39:17.826 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.826 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.826 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:39:17.826 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.771438+0000", 2023-08-28T02:39:17.826 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.8109999999999992e-06 2023-08-28T02:39:17.826 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.826 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.826 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:39:17.826 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.771457+0000", 2023-08-28T02:39:17.826 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.8916e-05 2023-08-28T02:39:17.827 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.827 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.827 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:39:17.827 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.771467+0000", 2023-08-28T02:39:17.827 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.046e-05 2023-08-28T02:39:17.827 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.827 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.827 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:39:17.827 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.771548+0000", 2023-08-28T02:39:17.827 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.1314000000000001e-05 2023-08-28T02:39:17.827 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.827 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.828 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:39:17.828 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.771717+0000", 2023-08-28T02:39:17.828 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000168579 2023-08-28T02:39:17.828 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.828 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.828 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 5,6,7", 2023-08-28T02:39:17.828 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.771884+0000", 2023-08-28T02:39:17.828 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00016714900000000001 2023-08-28T02:39:17.828 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.828 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.828 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:39:17.828 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.777471+0000", 2023-08-28T02:39:17.829 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0055870160000000002 2023-08-28T02:39:17.829 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.829 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.829 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:39:17.829 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.790213+0000", 2023-08-28T02:39:17.829 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.012741522 2023-08-28T02:39:17.829 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.829 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.829 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:39:17.829 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.798106+0000", 2023-08-28T02:39:17.829 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0078938770000000005 2023-08-28T02:39:17.829 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:39:17.829 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:39:17.830 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:39:17.830 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.830 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.830 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.45647.0:183595 8.d 8:bd259feb:::benchmark_data_smithi143_419289_object86685:head [delete] snapc 0=[] ondisk+write+known_if_redirected e360)", 2023-08-28T02:39:17.830 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:39:17.778398+0000", 2023-08-28T02:39:17.830 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.044323271999999997, 2023-08-28T02:39:17.830 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.044453594999999999, 2023-08-28T02:39:17.830 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:39:17.830 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "waiting for sub ops", 2023-08-28T02:39:17.830 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:39:17.830 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.45647", 2023-08-28T02:39:17.831 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3230872490", 2023-08-28T02:39:17.831 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 183595 2023-08-28T02:39:17.831 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.831 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:39:17.831 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.831 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:39:17.831 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.778398+0000", 2023-08-28T02:39:17.831 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:39:17.831 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.831 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.831 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:39:17.831 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.778398+0000", 2023-08-28T02:39:17.831 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:39:17.832 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.832 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.832 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:39:17.832 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.778397+0000", 2023-08-28T02:39:17.832 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:39:17.832 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.832 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.832 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:39:17.832 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.778403+0000", 2023-08-28T02:39:17.832 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.772e-06 2023-08-28T02:39:17.832 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.832 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.833 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:39:17.833 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.778424+0000", 2023-08-28T02:39:17.833 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.1021000000000001e-05 2023-08-28T02:39:17.833 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.833 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.833 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:39:17.833 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.778433+0000", 2023-08-28T02:39:17.833 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.2070000000000001e-06 2023-08-28T02:39:17.833 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.833 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.833 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:39:17.833 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.778482+0000", 2023-08-28T02:39:17.834 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.8659999999999998e-05 2023-08-28T02:39:17.834 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.834 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.834 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:39:17.834 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.778669+0000", 2023-08-28T02:39:17.834 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00018749000000000001 2023-08-28T02:39:17.834 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.834 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.834 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 5,6,7", 2023-08-28T02:39:17.834 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.778836+0000", 2023-08-28T02:39:17.834 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00016652999999999999 2023-08-28T02:39:17.834 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.835 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.835 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:39:17.835 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.789652+0000", 2023-08-28T02:39:17.835 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.010816144999999999 2023-08-28T02:39:17.835 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.835 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.835 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:39:17.835 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.800389+0000", 2023-08-28T02:39:17.835 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.010737078000000001 2023-08-28T02:39:17.835 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.835 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.835 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:39:17.836 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.804531+0000", 2023-08-28T02:39:17.836 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.004142522 2023-08-28T02:39:17.836 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:39:17.836 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:39:17.836 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:39:17.836 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.836 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.836 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.45647.0:183600 8.d 8:bca09400:::benchmark_data_smithi143_419289_object86690:head [delete] snapc 0=[] ondisk+write+known_if_redirected e360)", 2023-08-28T02:39:17.836 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:39:17.779806+0000", 2023-08-28T02:39:17.836 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.042915828000000003, 2023-08-28T02:39:17.836 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.043136079000000001, 2023-08-28T02:39:17.836 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:39:17.837 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "waiting for sub ops", 2023-08-28T02:39:17.837 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:39:17.837 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.45647", 2023-08-28T02:39:17.837 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3230872490", 2023-08-28T02:39:17.837 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 183600 2023-08-28T02:39:17.837 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.837 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:39:17.837 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.837 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:39:17.837 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.779806+0000", 2023-08-28T02:39:17.837 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:39:17.837 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.837 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.838 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:39:17.838 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.779806+0000", 2023-08-28T02:39:17.838 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:39:17.838 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.838 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.838 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:39:17.838 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.779804+0000", 2023-08-28T02:39:17.838 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:39:17.838 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.838 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.838 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:39:17.838 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.779811+0000", 2023-08-28T02:39:17.839 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.9600000000000003e-06 2023-08-28T02:39:17.839 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.839 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.839 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:39:17.839 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.779823+0000", 2023-08-28T02:39:17.839 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.2159e-05 2023-08-28T02:39:17.839 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.839 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.839 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:39:17.839 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.779831+0000", 2023-08-28T02:39:17.839 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.14e-06 2023-08-28T02:39:17.839 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.839 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.840 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:39:17.840 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.784001+0000", 2023-08-28T02:39:17.840 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0041695170000000002 2023-08-28T02:39:17.840 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.840 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.840 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:39:17.840 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.784210+0000", 2023-08-28T02:39:17.840 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00020961100000000001 2023-08-28T02:39:17.840 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.840 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.840 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 5,6,7", 2023-08-28T02:39:17.840 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.784344+0000", 2023-08-28T02:39:17.841 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000133413 2023-08-28T02:39:17.841 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.841 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.841 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:39:17.841 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.789771+0000", 2023-08-28T02:39:17.841 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.005426958 2023-08-28T02:39:17.841 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.841 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.841 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:39:17.841 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.800425+0000", 2023-08-28T02:39:17.841 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.010653898 2023-08-28T02:39:17.841 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.841 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.842 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:39:17.842 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.804571+0000", 2023-08-28T02:39:17.842 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0041468240000000003 2023-08-28T02:39:17.842 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:39:17.842 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:39:17.842 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:39:17.842 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.842 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.842 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.45647.0:183615 8.d 8:bfe731c3:::benchmark_data_smithi143_419289_object86705:head [delete] snapc 0=[] ondisk+write+known_if_redirected e360)", 2023-08-28T02:39:17.842 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:39:17.805156+0000", 2023-08-28T02:39:17.842 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.017565570999999999, 2023-08-28T02:39:17.842 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.017890352000000002, 2023-08-28T02:39:17.843 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:39:17.843 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "waiting for sub ops", 2023-08-28T02:39:17.843 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:39:17.843 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.45647", 2023-08-28T02:39:17.843 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3230872490", 2023-08-28T02:39:17.843 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 183615 2023-08-28T02:39:17.843 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.843 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:39:17.843 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.843 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:39:17.843 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.805156+0000", 2023-08-28T02:39:17.843 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:39:17.844 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.844 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.844 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:39:17.844 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.805156+0000", 2023-08-28T02:39:17.844 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:39:17.844 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.844 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.844 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:39:17.844 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.805154+0000", 2023-08-28T02:39:17.844 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999981 2023-08-28T02:39:17.844 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.844 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.844 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:39:17.845 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.805162+0000", 2023-08-28T02:39:17.845 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.5599999999999996e-06 2023-08-28T02:39:17.845 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.845 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.845 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:39:17.845 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.805177+0000", 2023-08-28T02:39:17.845 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.5118e-05 2023-08-28T02:39:17.845 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.845 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.845 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:39:17.845 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.805185+0000", 2023-08-28T02:39:17.845 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.7050000000000005e-06 2023-08-28T02:39:17.846 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.846 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.846 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:39:17.846 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.805235+0000", 2023-08-28T02:39:17.846 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.9966999999999998e-05 2023-08-28T02:39:17.846 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.846 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.846 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:39:17.846 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.805372+0000", 2023-08-28T02:39:17.846 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00013658799999999999 2023-08-28T02:39:17.846 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.846 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.846 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 5,6,7", 2023-08-28T02:39:17.847 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.805498+0000", 2023-08-28T02:39:17.847 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000125623 2023-08-28T02:39:17.847 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.847 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.847 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "op_commit", 2023-08-28T02:39:17.847 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.810368+0000", 2023-08-28T02:39:17.847 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0048707300000000002 2023-08-28T02:39:17.847 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.847 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.847 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:39:17.847 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.816086+0000", 2023-08-28T02:39:17.847 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0057180190000000004 2023-08-28T02:39:17.848 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:39:17.848 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:39:17.848 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:39:17.848 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.848 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.848 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.45647.0:183622 8.1 e360/354 8:8feb1860:::benchmark_data_smithi143_419289_object86712:head v 360'11408, mlcod=360'11406)", 2023-08-28T02:39:17.848 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:39:17.822511+0000", 2023-08-28T02:39:17.848 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.00021108699999999999, 2023-08-28T02:39:17.848 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00061793299999999998, 2023-08-28T02:39:17.848 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:39:17.848 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "started", 2023-08-28T02:39:17.848 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:39:17.848 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.849 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:39:17.849 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.822511+0000", 2023-08-28T02:39:17.849 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:39:17.849 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.849 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.849 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:39:17.849 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.822511+0000", 2023-08-28T02:39:17.849 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:39:17.849 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.849 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.849 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:39:17.849 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.822510+0000", 2023-08-28T02:39:17.849 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:39:17.850 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.850 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.850 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:39:17.850 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.822518+0000", 2023-08-28T02:39:17.850 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.1440000000000003e-06 2023-08-28T02:39:17.850 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.850 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.850 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:39:17.850 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.822529+0000", 2023-08-28T02:39:17.850 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.1260000000000001e-05 2023-08-28T02:39:17.850 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.850 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.850 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:39:17.851 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.822536+0000", 2023-08-28T02:39:17.851 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.5529999999999998e-06 2023-08-28T02:39:17.851 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.851 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.851 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:39:17.851 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.822584+0000", 2023-08-28T02:39:17.851 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.8498000000000002e-05 2023-08-28T02:39:17.851 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:39:17.851 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:39:17.851 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:39:17.851 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:39:17.822603+0000", 2023-08-28T02:39:17.851 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.8865999999999999e-05 2023-08-28T02:39:17.852 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:39:17.852 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:39:17.852 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:39:17.852 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:39:17.852 INFO:teuthology.orchestra.run.smithi143.stdout: ], 2023-08-28T02:39:17.852 INFO:teuthology.orchestra.run.smithi143.stdout: "num_ops": 5 2023-08-28T02:39:17.852 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:39:17.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:17.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:39:17.954 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:39:18.070 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:18.897 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:39:18.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:18.904 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:39:18.973 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.2 dump_blocked_ops 2023-08-28T02:39:19.005 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:39:19.094 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:19.134 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:39:19.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:18 smithi143 ceph-mon[183059]: pgmap v985: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 745 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 552 op/s; 3523/35637 objects degraded (9.886%); 10980/35637 objects misplaced (30.811%); 3.4 MiB/s, 54 objects/s recovering 2023-08-28T02:39:19.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:18 smithi143 ceph-mon[189871]: pgmap v985: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 745 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 552 op/s; 3523/35637 objects degraded (9.886%); 10980/35637 objects misplaced (30.811%); 3.4 MiB/s, 54 objects/s recovering 2023-08-28T02:39:19.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:18 smithi169 ceph-mon[159415]: pgmap v985: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 745 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 552 op/s; 3523/35637 objects degraded (9.886%); 10980/35637 objects misplaced (30.811%); 3.4 MiB/s, 54 objects/s recovering 2023-08-28T02:39:19.238 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set noscrub 2023-08-28T02:39:19.255 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:39:19.267 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:39:19.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:19.284 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:39:19.384 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:39:19.422 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:19.528 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:39:19.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:19.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:39:19.635 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:39:19.672 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:19.769 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:39:19.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:19.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:39:19.877 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:39:19.984 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:19.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:19 smithi169 ceph-mon[159415]: pgmap v986: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 656 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 580 op/s; 3523/31344 objects degraded (11.240%); 10253/31344 objects misplaced (32.711%); 3.1 MiB/s, 49 objects/s recovering 2023-08-28T02:39:20.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:19 smithi143 ceph-mon[183059]: pgmap v986: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 656 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 580 op/s; 3523/31344 objects degraded (11.240%); 10253/31344 objects misplaced (32.711%); 3.1 MiB/s, 49 objects/s recovering 2023-08-28T02:39:20.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:19 smithi143 ceph-mon[189871]: pgmap v986: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 656 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 580 op/s; 3523/31344 objects degraded (11.240%); 10253/31344 objects misplaced (32.711%); 3.1 MiB/s, 49 objects/s recovering 2023-08-28T02:39:20.239 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:39:20.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:20.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:39:20.350 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:39:20.378 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:20.422 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:39:20.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:20.429 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:39:20.530 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:39:20.578 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:20.663 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:39:20.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:20.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:39:20.774 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:39:20.834 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:20.957 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:39:20.972 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:39:20.985 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:39:21.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:21.009 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:39:21.110 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:39:21.138 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:39:21.183 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:21.578 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:39:21.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:21.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:39:21.686 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:39:21.762 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:22.064 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:39:22.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:22.071 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:21 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 3523/31344 objects degraded (11.240%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:39:22.072 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:39:21 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:39:21] "GET /metrics HTTP/1.1" 200 36578 "" "Prometheus/2.33.4" 2023-08-28T02:39:22.072 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:21 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 3523/31344 objects degraded (11.240%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:39:22.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:39:22.173 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:39:22.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:21 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 3523/31344 objects degraded (11.240%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:39:22.247 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:23.204 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:22 smithi143 ceph-mon[183059]: pgmap v987: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 611 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 584 op/s; 3523/29199 objects degraded (12.065%); 9714/29199 objects misplaced (33.268%); 2.7 MiB/s, 43 objects/s recovering 2023-08-28T02:39:23.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:22 smithi143 ceph-mon[189871]: pgmap v987: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 611 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 584 op/s; 3523/29199 objects degraded (12.065%); 9714/29199 objects misplaced (33.268%); 2.7 MiB/s, 43 objects/s recovering 2023-08-28T02:39:23.241 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:22 smithi169 ceph-mon[159415]: pgmap v987: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 611 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 584 op/s; 3523/29199 objects degraded (12.065%); 9714/29199 objects misplaced (33.268%); 2.7 MiB/s, 43 objects/s recovering 2023-08-28T02:39:23.379 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:39:23.380 INFO:teuthology.orchestra.run.smithi143.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-08-28T02:39:23.380 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:39:24.550 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:39:24.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:24.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:39:24.660 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:39:24.737 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:24.922 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:24 smithi143 ceph-mon[183059]: pgmap v988: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 519 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 591 op/s; 3523/24765 objects degraded (14.226%); 8910/24765 objects misplaced (35.978%); 2.8 MiB/s, 45 objects/s recovering 2023-08-28T02:39:24.927 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:24 smithi143 ceph-mon[189871]: pgmap v988: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 519 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 591 op/s; 3523/24765 objects degraded (14.226%); 8910/24765 objects misplaced (35.978%); 2.8 MiB/s, 45 objects/s recovering 2023-08-28T02:39:25.087 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:39:25.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:25.091 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:39:25.192 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:39:25.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:24 smithi169 ceph-mon[159415]: pgmap v988: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 519 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 591 op/s; 3523/24765 objects degraded (14.226%); 8910/24765 objects misplaced (35.978%); 2.8 MiB/s, 45 objects/s recovering 2023-08-28T02:39:25.254 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:25.382 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:39:25.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:25.398 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:39:25.499 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:39:25.545 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:25.733 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:39:25.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:25.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:39:25.837 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:39:25.942 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:26.011 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:39:26.011 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [], 2023-08-28T02:39:26.011 INFO:teuthology.orchestra.run.smithi143.stdout: "complaint_time": 30, 2023-08-28T02:39:26.011 INFO:teuthology.orchestra.run.smithi143.stdout: "num_blocked_ops": 0 2023-08-28T02:39:26.011 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:39:26.227 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:39:26.254 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:39:26.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:26.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:39:26.273 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:39:26.307 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:39:26.338 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:39:26.360 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:39:26.363 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:39:26.383 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:39:26.395 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:26.400 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:39:26.418 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:39:26.419 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:39:26.419 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:39:26.419 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:39:26.419 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:39:26.419 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:39:26.419 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:39:26.419 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:39:26.419 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:39:26.443 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:39:26.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:26.447 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:39:26.548 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:39:26.624 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:26.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:26 smithi143 ceph-mon[183059]: pgmap v989: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 421 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 3523/20091 objects degraded (17.535%); 7944/20091 objects misplaced (39.540%); 2.6 MiB/s, 41 objects/s recovering 2023-08-28T02:39:26.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:26 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:39:26.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:26 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:39:26.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:26 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/4184389376' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:39:26.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:26 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 3523/20091 objects degraded (17.535%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:39:26.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:26 smithi143 ceph-mon[189871]: pgmap v989: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 421 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 3523/20091 objects degraded (17.535%); 7944/20091 objects misplaced (39.540%); 2.6 MiB/s, 41 objects/s recovering 2023-08-28T02:39:26.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:26 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:39:26.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:26 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:39:26.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:26 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/4184389376' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:39:26.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:26 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 3523/20091 objects degraded (17.535%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:39:27.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:26 smithi169 ceph-mon[159415]: pgmap v989: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 421 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 3523/20091 objects degraded (17.535%); 7944/20091 objects misplaced (39.540%); 2.6 MiB/s, 41 objects/s recovering 2023-08-28T02:39:27.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:26 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:39:27.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:26 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:39:27.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:26 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/4184389376' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:39:27.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:26 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 3523/20091 objects degraded (17.535%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:39:27.716 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.2 dump_historic_ops 2023-08-28T02:39:27.915 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:39:27.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:27.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:39:28.023 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:39:28.087 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:28.193 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:39:28.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:28.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:39:28.300 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:39:28.377 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:28.473 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:39:28.546 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:39:28.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:28.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:39:28.653 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:39:28.712 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:28.812 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:39:28.812 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is set 2023-08-28T02:39:28.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:28.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:39:28.922 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:39:28.980 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:28 smithi169 ceph-mon[159415]: pgmap v990: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 374 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 561 op/s; 3523/17808 objects degraded (19.783%); 7561/17808 objects misplaced (42.458%); 3.3 MiB/s, 57 objects/s recovering 2023-08-28T02:39:28.980 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:28 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3375035211' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:39:28.994 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:29.057 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:39:29.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:29.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:39:29.164 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:39:29.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:28 smithi143 ceph-mon[183059]: pgmap v990: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 374 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 561 op/s; 3523/17808 objects degraded (19.783%); 7561/17808 objects misplaced (42.458%); 3.3 MiB/s, 57 objects/s recovering 2023-08-28T02:39:29.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:28 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3375035211' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:39:29.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:28 smithi143 ceph-mon[189871]: pgmap v990: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 374 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 561 op/s; 3523/17808 objects degraded (19.783%); 7561/17808 objects misplaced (42.458%); 3.3 MiB/s, 57 objects/s recovering 2023-08-28T02:39:29.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:28 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3375035211' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:39:29.236 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:29.327 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:39:29.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:29.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:39:29.442 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:39:29.502 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:29.682 INFO:tasks.thrashosds.thrasher:in_osds: [0, 4, 5, 3, 6, 1, 7] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-08-28T02:39:29.682 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-08-28T02:39:29.682 INFO:tasks.thrashosds.thrasher:Growing pool device_health_metrics 2023-08-28T02:39:29.683 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph pg dump --format=json 2023-08-28T02:39:29.789 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:29 smithi143 ceph-mon[189871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:39:29.789 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:29 smithi143 ceph-mon[189871]: failed to encode map e361 with expected crc 2023-08-28T02:39:29.789 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:29 smithi143 ceph-mon[189871]: failed to encode map e361 with expected crc 2023-08-28T02:39:29.789 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:29 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3375035211' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:39:29.790 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:29 smithi143 ceph-mon[189871]: osdmap e361: 8 total, 8 up, 7 in 2023-08-28T02:39:29.790 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:29 smithi143 ceph-mon[189871]: failed to encode map e361 with expected crc 2023-08-28T02:39:29.790 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:29 smithi143 ceph-mon[189871]: failed to encode map e361 with expected crc 2023-08-28T02:39:29.790 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:29 smithi143 ceph-mon[189871]: failed to encode map e361 with expected crc 2023-08-28T02:39:29.790 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:29 smithi143 ceph-mon[189871]: failed to encode map e361 with expected crc 2023-08-28T02:39:30.026 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:39:30.067 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:29 smithi143 ceph-mon[183059]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:39:30.067 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:29 smithi143 ceph-mon[183059]: failed to encode map e361 with expected crc 2023-08-28T02:39:30.067 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:29 smithi143 ceph-mon[183059]: failed to encode map e361 with expected crc 2023-08-28T02:39:30.067 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:29 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3375035211' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:39:30.068 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:29 smithi143 ceph-mon[183059]: osdmap e361: 8 total, 8 up, 7 in 2023-08-28T02:39:30.068 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:29 smithi143 ceph-mon[183059]: failed to encode map e361 with expected crc 2023-08-28T02:39:30.068 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:29 smithi143 ceph-mon[183059]: failed to encode map e361 with expected crc 2023-08-28T02:39:30.068 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:29 smithi143 ceph-mon[183059]: failed to encode map e361 with expected crc 2023-08-28T02:39:30.068 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:29 smithi143 ceph-mon[183059]: failed to encode map e361 with expected crc 2023-08-28T02:39:30.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:29 smithi169 ceph-mon[159415]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:39:30.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:29 smithi169 ceph-mon[159415]: failed to encode map e361 with expected crc 2023-08-28T02:39:30.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:29 smithi169 ceph-mon[159415]: failed to encode map e361 with expected crc 2023-08-28T02:39:30.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:29 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3375035211' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:39:30.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:29 smithi169 ceph-mon[159415]: osdmap e361: 8 total, 8 up, 7 in 2023-08-28T02:39:30.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:29 smithi169 ceph-mon[159415]: failed to encode map e361 with expected crc 2023-08-28T02:39:30.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:29 smithi169 ceph-mon[159415]: failed to encode map e361 with expected crc 2023-08-28T02:39:30.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:29 smithi169 ceph-mon[159415]: failed to encode map e361 with expected crc 2023-08-28T02:39:30.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:29 smithi169 ceph-mon[159415]: failed to encode map e361 with expected crc 2023-08-28T02:39:30.754 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:39:30.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:30.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:39:30.861 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:39:30.903 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:30.988 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:39:30.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:30.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:39:30.997 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:39:31.096 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:39:31.102 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:30 smithi143 ceph-mon[183059]: pgmap v992: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 281 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 599 op/s; 3523/13359 objects degraded (26.372%); 6842/13359 objects misplaced (51.216%); 2.6 MiB/s, 47 objects/s recovering 2023-08-28T02:39:31.103 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:30 smithi143 ceph-mon[189871]: pgmap v992: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 281 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 599 op/s; 3523/13359 objects degraded (26.372%); 6842/13359 objects misplaced (51.216%); 2.6 MiB/s, 47 objects/s recovering 2023-08-28T02:39:31.138 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:30 smithi169 ceph-mon[159415]: pgmap v992: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 281 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 599 op/s; 3523/13359 objects degraded (26.372%); 6842/13359 objects misplaced (51.216%); 2.6 MiB/s, 47 objects/s recovering 2023-08-28T02:39:31.142 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:31.256 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:39:31.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:31.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:39:31.376 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:39:31.416 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:39:31.459 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:39:31.459 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:39:31.516 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:31.675 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:39:31.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:31.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:39:31.790 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:39:31.847 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:31.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:31 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 3523/13359 objects degraded (26.372%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:39:31.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:31 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 3523/13359 objects degraded (26.372%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:39:31.951 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:39:31 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:39:31] "GET /metrics HTTP/1.1" 200 36574 "" "Prometheus/2.33.4" 2023-08-28T02:39:31.965 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:39:31.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:31.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:39:32.075 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:39:32.135 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:32.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:31 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 3523/13359 objects degraded (26.372%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:39:32.747 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set nodeep-scrub 2023-08-28T02:39:33.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:32 smithi143 ceph-mon[183059]: pgmap v993: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 234 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 603 op/s; 3523/11088 objects degraded (31.773%); 6281/11088 objects misplaced (56.647%); 2.6 MiB/s, 47 objects/s recovering 2023-08-28T02:39:33.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:32 smithi143 ceph-mon[189871]: pgmap v993: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 234 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 603 op/s; 3523/11088 objects degraded (31.773%); 6281/11088 objects misplaced (56.647%); 2.6 MiB/s, 47 objects/s recovering 2023-08-28T02:39:33.203 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:39:33.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:33.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:39:33.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:32 smithi169 ceph-mon[159415]: pgmap v993: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 234 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 603 op/s; 3523/11088 objects degraded (31.773%); 6281/11088 objects misplaced (56.647%); 2.6 MiB/s, 47 objects/s recovering 2023-08-28T02:39:33.311 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:39:33.387 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:34.441 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.3 dump_ops_in_flight 2023-08-28T02:39:34.459 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:39:34.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:34.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:39:34.579 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:39:34.663 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:34.742 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Removed 96907 objects 2023-08-28T02:39:34.742 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Clean up completed and total clean up time :160.515 2023-08-28T02:39:34.753 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:39:34.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:34.757 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:39:34.858 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:39:34.887 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:34.895 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:39:34.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:34 smithi143 ceph-mon[183059]: pgmap v994: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 137 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 611 op/s; 3523/6426 objects degraded (54.824%); 5701/6426 objects misplaced (88.718%); 1.1 MiB/s, 47 objects/s recovering 2023-08-28T02:39:34.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:34 smithi143 ceph-mon[189871]: pgmap v994: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 137 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 611 op/s; 3523/6426 objects degraded (54.824%); 5701/6426 objects misplaced (88.718%); 1.1 MiB/s, 47 objects/s recovering 2023-08-28T02:39:34.953 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:39:34.974 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:39:34.993 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:39:35.012 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:39:35.030 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:39:35.053 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:39:35.068 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:39:35.086 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:39:35.087 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:39:35.087 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:39:35.088 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:39:35.088 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:39:35.088 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:39:35.088 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:39:35.088 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:39:35.088 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:39:35.094 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:39:35.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:35.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:39:35.199 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:39:35.223 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:34 smithi169 ceph-mon[159415]: pgmap v994: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 137 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 611 op/s; 3523/6426 objects degraded (54.824%); 5701/6426 objects misplaced (88.718%); 1.1 MiB/s, 47 objects/s recovering 2023-08-28T02:39:35.226 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:35.271 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:39:35.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:35.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:39:35.374 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:39:35.375 INFO:teuthology.orchestra.run.smithi143.stderr:dumped all 2023-08-28T02:39:35.380 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:39:35.409 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:35.826 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:35 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/560412498' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:39:36.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:35 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/560412498' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:39:36.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:35 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/560412498' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:39:36.295 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:39:36.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:36.300 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:39:36.401 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:39:36.427 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:36.471 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:39:36.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:36.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:39:36.577 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:39:36.607 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:36.758 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:39:36.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:36.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:39:36.862 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:39:36.871 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:36 smithi143 ceph-mon[183059]: from='client.35886 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:39:36.871 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:36 smithi143 ceph-mon[183059]: pgmap v995: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 43 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 604 op/s; 3523/1950 objects degraded (180.667%); 4760/1950 objects misplaced (244.103%); 1.1 MiB/s, 47 objects/s recovering 2023-08-28T02:39:36.871 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:36 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 3523/1950 objects degraded (180.667%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:39:36.871 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:36 smithi143 ceph-mon[189871]: from='client.35886 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:39:36.872 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:36 smithi143 ceph-mon[189871]: pgmap v995: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 43 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 604 op/s; 3523/1950 objects degraded (180.667%); 4760/1950 objects misplaced (244.103%); 1.1 MiB/s, 47 objects/s recovering 2023-08-28T02:39:36.872 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:36 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 3523/1950 objects degraded (180.667%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:39:36.888 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:36.934 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:39:36.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:36.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:39:37.043 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:39:37.071 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:37.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:36 smithi169 ceph-mon[159415]: from='client.35886 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:39:37.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:36 smithi169 ceph-mon[159415]: pgmap v995: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 43 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 604 op/s; 3523/1950 objects degraded (180.667%); 4760/1950 objects misplaced (244.103%); 1.1 MiB/s, 47 objects/s recovering 2023-08-28T02:39:37.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:36 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 3523/1950 objects degraded (180.667%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:39:37.352 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:39:37.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:37.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:39:37.423 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:39:37.449 INFO:teuthology.orchestra.run.smithi143.stdout:{"pg_ready":true,"pg_map":{"version":994,"stamp":"2023-08-28T02:39:33.655506+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":143267787,"num_objects":2142,"num_object_clones":0,"num_object_copies":6426,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3523,"num_objects_misplaced":5701,"num_objects_unfound":0,"num_objects_dirty":2142,"num_whiteouts":0,"num_read":6275,"num_read_kb":13708,"num_write":192808,"num_write_kb":6217190,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4241,"num_bytes_recovered":252149704,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124154,"ondisk_log_size":124154,"up":267,"acting":263,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":270,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":4718220,"kb_used_data":778196,"kb_used_omap":27,"kb_used_meta":3939876,"kb_avail":651497844,"statfs":{"total":671965249536,"available":667133792256,"internally_reserved":0,"allocated":796872704,"data_stored":787970786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28409,"internal_metadata":4034433287},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-400621568,"num_objects":-6113,"num_object_clones":0,"num_object_copies":-18339,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-3209,"num_objects_unfound":0,"num_objects_dirty":-6113,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6113,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":474,"num_bytes_recovered":11599872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002956"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":389,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.848355+0000","last_change":"2023-08-28T02:38:35.886897+0000","last_active":"2023-08-28T02:39:14.848355+0000","last_peered":"2023-08-28T02:39:14.848355+0000","last_clean":"2023-08-28T02:39:14.848355+0000","last_became_active":"2023-08-28T02:38:35.886527+0000","last_became_peered":"2023-08-28T02:38:35.886527+0000","last_unstale":"2023-08-28T02:39:14.848355+0000","last_undegraded":"2023-08-28T02:39:14.848355+0000","last_fullsized":"2023-08-28T02:39:14.848355+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":269,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.842233+0000","last_change":"2023-08-28T02:32:58.634375+0000","last_active":"2023-08-28T02:39:14.842233+0000","last_peered":"2023-08-28T02:39:14.842233+0000","last_clean":"2023-08-28T02:39:14.842233+0000","last_became_active":"2023-08-28T02:32:58.632726+0000","last_became_peered":"2023-08-28T02:32:58.632726+0000","last_unstale":"2023-08-28T02:39:14.842233+0000","last_undegraded":"2023-08-28T02:39:14.842233+0000","last_fullsized":"2023-08-28T02:39:14.842233+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:20:27.730725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":82,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.843402+0000","last_change":"2023-08-28T02:32:58.628387+0000","last_active":"2023-08-28T02:39:14.843402+0000","last_peered":"2023-08-28T02:39:14.843402+0000","last_clean":"2023-08-28T02:39:14.843402+0000","last_became_active":"2023-08-28T02:32:58.628071+0000","last_became_peered":"2023-08-28T02:32:58.628071+0000","last_unstale":"2023-08-28T02:39:14.843402+0000","last_undegraded":"2023-08-28T02:39:14.843402+0000","last_fullsized":"2023-08-28T02:39:14.843402+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:25:55.264433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":389,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.851127+0000","last_change":"2023-08-28T02:37:45.580751+0000","last_active":"2023-08-28T02:39:14.851127+0000","last_peered":"2023-08-28T02:39:14.851127+0000","last_clean":"2023-08-28T02:39:14.851127+0000","last_became_active":"2023-08-28T02:37:45.580446+0000","last_became_peered":"2023-08-28T02:37:45.580446+0000","last_unstale":"2023-08-28T02:39:14.851127+0000","last_undegraded":"2023-08-28T02:39:14.851127+0000","last_fullsized":"2023-08-28T02:39:14.851127+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:59:21.094880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":260,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.843455+0000","last_change":"2023-08-28T02:32:58.779602+0000","last_active":"2023-08-28T02:39:14.843455+0000","last_peered":"2023-08-28T02:39:14.843455+0000","last_clean":"2023-08-28T02:39:14.843455+0000","last_became_active":"2023-08-28T02:32:58.778945+0000","last_became_peered":"2023-08-28T02:32:58.778945+0000","last_unstale":"2023-08-28T02:39:14.843455+0000","last_undegraded":"2023-08-28T02:39:14.843455+0000","last_fullsized":"2023-08-28T02:39:14.843455+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:35:55.832928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":167,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.855244+0000","last_change":"2023-08-28T02:32:58.626766+0000","last_active":"2023-08-28T02:39:14.855244+0000","last_peered":"2023-08-28T02:39:14.855244+0000","last_clean":"2023-08-28T02:39:14.855244+0000","last_became_active":"2023-08-28T02:32:58.626415+0000","last_became_peered":"2023-08-28T02:32:58.626415+0000","last_unstale":"2023-08-28T02:39:14.855244+0000","last_undegraded":"2023-08-28T02:39:14.855244+0000","last_fullsized":"2023-08-28T02:39:14.855244+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":332,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.850767+0000","last_change":"2023-08-28T02:23:59.227592+0000","last_active":"2023-08-28T02:39:14.850767+0000","last_peered":"2023-08-28T02:39:14.850767+0000","last_clean":"2023-08-28T02:39:14.850767+0000","last_became_active":"2023-08-28T02:23:59.227342+0000","last_became_peered":"2023-08-28T02:23:59.227342+0000","last_unstale":"2023-08-28T02:39:14.850767+0000","last_undegraded":"2023-08-28T02:39:14.850767+0000","last_fullsized":"2023-08-28T02:39:14.850767+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:08:19.874396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":278,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.843533+0000","last_change":"2023-08-28T02:32:58.779435+0000","last_active":"2023-08-28T02:39:14.843533+0000","last_peered":"2023-08-28T02:39:14.843533+0000","last_clean":"2023-08-28T02:39:14.843533+0000","last_became_active":"2023-08-28T02:32:58.778629+0000","last_became_peered":"2023-08-28T02:32:58.778629+0000","last_unstale":"2023-08-28T02:39:14.843533+0000","last_undegraded":"2023-08-28T02:39:14.843533+0000","last_fullsized":"2023-08-28T02:39:14.843533+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:25:32.193898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":25,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-08-28T02:39:10.812153+0000","last_change":"2023-08-28T02:37:45.579563+0000","last_active":"2023-08-28T02:39:10.812153+0000","last_peered":"2023-08-28T02:39:10.812153+0000","last_clean":"2023-08-28T02:39:10.812153+0000","last_became_active":"2023-08-28T02:37:45.579133+0000","last_became_peered":"2023-08-28T02:37:45.579133+0000","last_unstale":"2023-08-28T02:39:10.812153+0000","last_undegraded":"2023-08-28T02:39:10.812153+0000","last_fullsized":"2023-08-28T02:39:10.812153+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":260,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.843021+0000","last_change":"2023-08-28T02:37:45.580277+0000","last_active":"2023-08-28T02:39:14.843021+0000","last_peered":"2023-08-28T02:39:14.843021+0000","last_clean":"2023-08-28T02:39:14.843021+0000","last_became_active":"2023-08-28T02:37:45.579888+0000","last_became_peered":"2023-08-28T02:37:45.579888+0000","last_unstale":"2023-08-28T02:39:14.843021+0000","last_undegraded":"2023-08-28T02:39:14.843021+0000","last_fullsized":"2023-08-28T02:39:14.843021+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:18:54.085955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":31,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-08-28T02:39:10.811758+0000","last_change":"2023-08-28T02:37:46.016429+0000","last_active":"2023-08-28T02:39:10.811758+0000","last_peered":"2023-08-28T02:39:10.811758+0000","last_clean":"2023-08-28T02:39:10.811758+0000","last_became_active":"2023-08-28T02:37:45.581897+0000","last_became_peered":"2023-08-28T02:37:45.581897+0000","last_unstale":"2023-08-28T02:39:10.811758+0000","last_undegraded":"2023-08-28T02:39:10.811758+0000","last_fullsized":"2023-08-28T02:39:10.811758+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":101,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.854633+0000","last_change":"2023-08-28T02:38:35.885068+0000","last_active":"2023-08-28T02:39:14.854633+0000","last_peered":"2023-08-28T02:39:14.854633+0000","last_clean":"2023-08-28T02:39:14.854633+0000","last_became_active":"2023-08-28T02:38:35.884670+0000","last_became_peered":"2023-08-28T02:38:35.884670+0000","last_unstale":"2023-08-28T02:39:14.854633+0000","last_undegraded":"2023-08-28T02:39:14.854633+0000","last_fullsized":"2023-08-28T02:39:14.854633+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"86'22","reported_seq":449,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.849695+0000","last_change":"2023-08-28T02:37:45.597318+0000","last_active":"2023-08-28T02:39:14.849695+0000","last_peered":"2023-08-28T02:39:14.849695+0000","last_clean":"2023-08-28T02:39:14.849695+0000","last_became_active":"2023-08-28T02:37:45.578060+0000","last_became_peered":"2023-08-28T02:37:45.578060+0000","last_unstale":"2023-08-28T02:39:14.849695+0000","last_undegraded":"2023-08-28T02:39:14.849695+0000","last_fullsized":"2023-08-28T02:39:14.849695+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":28404,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":404,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.848035+0000","last_change":"2023-08-28T02:32:58.629451+0000","last_active":"2023-08-28T02:39:14.848035+0000","last_peered":"2023-08-28T02:39:14.848035+0000","last_clean":"2023-08-28T02:39:14.848035+0000","last_became_active":"2023-08-28T02:32:58.628767+0000","last_became_peered":"2023-08-28T02:32:58.628767+0000","last_unstale":"2023-08-28T02:39:14.848035+0000","last_undegraded":"2023-08-28T02:39:14.848035+0000","last_fullsized":"2023-08-28T02:39:14.848035+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":46,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-08-28T02:39:10.811780+0000","last_change":"2023-08-28T02:38:36.016884+0000","last_active":"2023-08-28T02:39:10.811780+0000","last_peered":"2023-08-28T02:39:10.811780+0000","last_clean":"2023-08-28T02:39:10.811780+0000","last_became_active":"2023-08-28T02:38:35.882996+0000","last_became_peered":"2023-08-28T02:38:35.882996+0000","last_unstale":"2023-08-28T02:39:10.811780+0000","last_undegraded":"2023-08-28T02:39:10.811780+0000","last_fullsized":"2023-08-28T02:39:10.811780+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":278,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.842578+0000","last_change":"2023-08-28T02:37:45.579069+0000","last_active":"2023-08-28T02:39:14.842578+0000","last_peered":"2023-08-28T02:39:14.842578+0000","last_clean":"2023-08-28T02:39:14.842578+0000","last_became_active":"2023-08-28T02:37:45.578728+0000","last_became_peered":"2023-08-28T02:37:45.578728+0000","last_unstale":"2023-08-28T02:39:14.842578+0000","last_undegraded":"2023-08-28T02:39:14.842578+0000","last_fullsized":"2023-08-28T02:39:14.842578+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:58:05.172104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":386,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.847820+0000","last_change":"2023-08-28T02:37:45.579678+0000","last_active":"2023-08-28T02:39:14.847820+0000","last_peered":"2023-08-28T02:39:14.847820+0000","last_clean":"2023-08-28T02:39:14.847820+0000","last_became_active":"2023-08-28T02:37:45.579390+0000","last_became_peered":"2023-08-28T02:37:45.579390+0000","last_unstale":"2023-08-28T02:39:14.847820+0000","last_undegraded":"2023-08-28T02:39:14.847820+0000","last_fullsized":"2023-08-28T02:39:14.847820+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:31:04.974167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":25,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-08-28T02:39:10.811688+0000","last_change":"2023-08-28T02:37:45.584893+0000","last_active":"2023-08-28T02:39:10.811688+0000","last_peered":"2023-08-28T02:39:10.811688+0000","last_clean":"2023-08-28T02:39:10.811688+0000","last_became_active":"2023-08-28T02:37:45.584568+0000","last_became_peered":"2023-08-28T02:37:45.584568+0000","last_unstale":"2023-08-28T02:39:10.811688+0000","last_undegraded":"2023-08-28T02:39:10.811688+0000","last_fullsized":"2023-08-28T02:39:10.811688+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":25,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-08-28T02:39:10.811456+0000","last_change":"2023-08-28T02:37:45.577769+0000","last_active":"2023-08-28T02:39:10.811456+0000","last_peered":"2023-08-28T02:39:10.811456+0000","last_clean":"2023-08-28T02:39:10.811456+0000","last_became_active":"2023-08-28T02:37:45.577446+0000","last_became_peered":"2023-08-28T02:37:45.577446+0000","last_unstale":"2023-08-28T02:39:10.811456+0000","last_undegraded":"2023-08-28T02:39:10.811456+0000","last_fullsized":"2023-08-28T02:39:10.811456+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":369,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.851202+0000","last_change":"2023-08-28T02:37:45.580634+0000","last_active":"2023-08-28T02:39:14.851202+0000","last_peered":"2023-08-28T02:39:14.851202+0000","last_clean":"2023-08-28T02:39:14.851202+0000","last_became_active":"2023-08-28T02:37:45.580322+0000","last_became_peered":"2023-08-28T02:37:45.580322+0000","last_unstale":"2023-08-28T02:39:14.851202+0000","last_undegraded":"2023-08-28T02:39:14.851202+0000","last_fullsized":"2023-08-28T02:39:14.851202+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:24:37.014061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":159,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.856959+0000","last_change":"2023-08-28T02:28:25.703333+0000","last_active":"2023-08-28T02:39:14.856959+0000","last_peered":"2023-08-28T02:39:14.856959+0000","last_clean":"2023-08-28T02:39:14.856959+0000","last_became_active":"2023-08-28T02:28:25.343515+0000","last_became_peered":"2023-08-28T02:28:25.343515+0000","last_unstale":"2023-08-28T02:39:14.856959+0000","last_undegraded":"2023-08-28T02:39:14.856959+0000","last_fullsized":"2023-08-28T02:39:14.856959+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":287,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.844093+0000","last_change":"2023-08-28T02:37:45.579694+0000","last_active":"2023-08-28T02:39:14.844093+0000","last_peered":"2023-08-28T02:39:14.844093+0000","last_clean":"2023-08-28T02:39:14.844093+0000","last_became_active":"2023-08-28T02:37:45.579219+0000","last_became_peered":"2023-08-28T02:37:45.579219+0000","last_unstale":"2023-08-28T02:39:14.844093+0000","last_undegraded":"2023-08-28T02:39:14.844093+0000","last_fullsized":"2023-08-28T02:39:14.844093+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:22:51.406552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"361'11999","reported_seq":12060,"reported_epoch":361,"state":"active+remapped+backfill_wait","last_fresh":"2023-08-28T02:39:32.091864+0000","last_change":"2023-08-28T02:38:37.060359+0000","last_active":"2023-08-28T02:39:32.091864+0000","last_peered":"2023-08-28T02:39:32.091864+0000","last_clean":"2023-08-28T02:38:34.857311+0000","last_became_active":"2023-08-28T02:38:37.023094+0000","last_became_peered":"2023-08-28T02:38:37.023094+0000","last_unstale":"2023-08-28T02:39:32.091864+0000","last_undegraded":"2023-08-28T02:39:32.091864+0000","last_fullsized":"2023-08-28T02:39:32.091864+0000","mapping_epoch":354,"log_start":"333'3800","ondisk_log_start":"333'3800","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":8199,"log_dups_size":0,"ondisk_log_size":8199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6356992,"num_objects":97,"num_object_clones":0,"num_object_copies":291,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":97,"num_objects_unfound":0,"num_objects_dirty":97,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11999,"num_write_kb":387072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,2],"avail_no_missing":["4","2","5"],"object_location_counts":[{"shards":"2,4,5","objects":97}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":42,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-08-28T02:39:10.811957+0000","last_change":"2023-08-28T02:38:35.929952+0000","last_active":"2023-08-28T02:39:10.811957+0000","last_peered":"2023-08-28T02:39:10.811957+0000","last_clean":"2023-08-28T02:39:10.811957+0000","last_became_active":"2023-08-28T02:38:35.882215+0000","last_became_peered":"2023-08-28T02:38:35.882215+0000","last_unstale":"2023-08-28T02:39:10.811957+0000","last_undegraded":"2023-08-28T02:39:10.811957+0000","last_fullsized":"2023-08-28T02:39:10.811957+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":25,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-08-28T02:39:10.811447+0000","last_change":"2023-08-28T02:37:45.585246+0000","last_active":"2023-08-28T02:39:10.811447+0000","last_peered":"2023-08-28T02:39:10.811447+0000","last_clean":"2023-08-28T02:39:10.811447+0000","last_became_active":"2023-08-28T02:37:45.584391+0000","last_became_peered":"2023-08-28T02:37:45.584391+0000","last_unstale":"2023-08-28T02:39:10.811447+0000","last_undegraded":"2023-08-28T02:39:10.811447+0000","last_fullsized":"2023-08-28T02:39:10.811447+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.5","version":"361'12040","reported_seq":16902,"reported_epoch":361,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-08-28T02:39:32.454530+0000","last_change":"2023-08-28T02:38:35.930624+0000","last_active":"2023-08-28T02:39:32.454530+0000","last_peered":"2023-08-28T02:39:32.454530+0000","last_clean":"2023-08-28T02:37:44.558549+0000","last_became_active":"2023-08-28T02:37:47.079792+0000","last_became_peered":"2023-08-28T02:37:47.079792+0000","last_unstale":"2023-08-28T02:39:32.454530+0000","last_undegraded":"2023-08-28T02:37:46.622481+0000","last_fullsized":"2023-08-28T02:37:46.586056+0000","mapping_epoch":345,"log_start":"335'4346","ondisk_log_start":"335'4346","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":7694,"log_dups_size":0,"ondisk_log_size":7694,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6422528,"num_objects":98,"num_object_clones":0,"num_object_copies":294,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1773,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":98,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12040,"num_write_kb":388416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2401,"num_bytes_recovered":137691136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4],"avail_no_missing":["6","0","4"],"object_location_counts":[{"shards":"4,6","objects":-1675}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":29,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-08-28T02:39:10.812111+0000","last_change":"2023-08-28T02:37:45.669932+0000","last_active":"2023-08-28T02:39:10.812111+0000","last_peered":"2023-08-28T02:39:10.812111+0000","last_clean":"2023-08-28T02:39:10.812111+0000","last_became_active":"2023-08-28T02:37:45.572825+0000","last_became_peered":"2023-08-28T02:37:45.572825+0000","last_unstale":"2023-08-28T02:39:10.812111+0000","last_undegraded":"2023-08-28T02:39:10.812111+0000","last_fullsized":"2023-08-28T02:39:10.812111+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":414,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.848106+0000","last_change":"2023-08-28T02:32:58.740396+0000","last_active":"2023-08-28T02:39:14.848106+0000","last_peered":"2023-08-28T02:39:14.848106+0000","last_clean":"2023-08-28T02:39:14.848106+0000","last_became_active":"2023-08-28T02:32:58.739941+0000","last_became_peered":"2023-08-28T02:32:58.739941+0000","last_unstale":"2023-08-28T02:39:14.848106+0000","last_undegraded":"2023-08-28T02:39:14.848106+0000","last_fullsized":"2023-08-28T02:39:14.848106+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"361'12094","reported_seq":12157,"reported_epoch":361,"state":"active+remapped+backfill_wait","last_fresh":"2023-08-28T02:39:30.099259+0000","last_change":"2023-08-28T02:37:46.828548+0000","last_active":"2023-08-28T02:39:30.099259+0000","last_peered":"2023-08-28T02:39:30.099259+0000","last_clean":"2023-08-28T02:37:44.550616+0000","last_became_active":"2023-08-28T02:37:46.807673+0000","last_became_peered":"2023-08-28T02:37:46.807673+0000","last_unstale":"2023-08-28T02:39:30.099259+0000","last_undegraded":"2023-08-28T02:39:30.099259+0000","last_fullsized":"2023-08-28T02:39:30.099259+0000","mapping_epoch":345,"log_start":"336'4724","ondisk_log_start":"336'4724","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":7370,"log_dups_size":0,"ondisk_log_size":7370,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:41:59.898437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11927552,"num_objects":182,"num_object_clones":0,"num_object_copies":546,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":182,"num_objects_unfound":0,"num_objects_dirty":182,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12094,"num_write_kb":392832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,2],"avail_no_missing":["0","2","6"],"object_location_counts":[{"shards":"0,2,6","objects":182}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":29,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-08-28T02:39:10.811543+0000","last_change":"2023-08-28T02:37:46.190365+0000","last_active":"2023-08-28T02:39:10.811543+0000","last_peered":"2023-08-28T02:39:10.811543+0000","last_clean":"2023-08-28T02:39:10.811543+0000","last_became_active":"2023-08-28T02:37:45.584408+0000","last_became_peered":"2023-08-28T02:37:45.584408+0000","last_unstale":"2023-08-28T02:39:10.811543+0000","last_undegraded":"2023-08-28T02:39:10.811543+0000","last_fullsized":"2023-08-28T02:39:10.811543+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":203,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.854559+0000","last_change":"2023-08-28T02:38:35.888252+0000","last_active":"2023-08-28T02:39:14.854559+0000","last_peered":"2023-08-28T02:39:14.854559+0000","last_clean":"2023-08-28T02:39:14.854559+0000","last_became_active":"2023-08-28T02:38:35.887901+0000","last_became_peered":"2023-08-28T02:38:35.887901+0000","last_unstale":"2023-08-28T02:39:14.854559+0000","last_undegraded":"2023-08-28T02:39:14.854559+0000","last_fullsized":"2023-08-28T02:39:14.854559+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"361'12194","reported_seq":12247,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-08-28T02:39:33.640261+0000","last_change":"2023-08-28T02:35:14.498797+0000","last_active":"2023-08-28T02:39:33.640261+0000","last_peered":"2023-08-28T02:39:33.640261+0000","last_clean":"2023-08-28T02:39:33.640261+0000","last_became_active":"2023-08-28T02:35:14.498551+0000","last_became_peered":"2023-08-28T02:35:14.498551+0000","last_unstale":"2023-08-28T02:39:33.640261+0000","last_undegraded":"2023-08-28T02:39:33.640261+0000","last_fullsized":"2023-08-28T02:39:33.640261+0000","mapping_epoch":318,"log_start":"336'5008","ondisk_log_start":"336'5008","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":7186,"log_dups_size":0,"ondisk_log_size":7186,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:43:07.729642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2621440,"num_objects":40,"num_object_clones":0,"num_object_copies":120,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":40,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12194,"num_write_kb":391488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":367,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-08-28T02:39:28.798816+0000","last_change":"2023-08-28T02:28:25.445982+0000","last_active":"2023-08-28T02:39:28.798816+0000","last_peered":"2023-08-28T02:39:28.798816+0000","last_clean":"2023-08-28T02:39:28.798816+0000","last_became_active":"2023-08-28T02:28:25.443577+0000","last_became_peered":"2023-08-28T02:28:25.443577+0000","last_unstale":"2023-08-28T02:39:28.798816+0000","last_undegraded":"2023-08-28T02:39:28.798816+0000","last_fullsized":"2023-08-28T02:39:28.798816+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:24:03.381751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":363,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.847643+0000","last_change":"2023-08-28T02:32:58.887874+0000","last_active":"2023-08-28T02:39:14.847643+0000","last_peered":"2023-08-28T02:39:14.847643+0000","last_clean":"2023-08-28T02:39:14.847643+0000","last_became_active":"2023-08-28T02:32:58.887377+0000","last_became_peered":"2023-08-28T02:32:58.887377+0000","last_unstale":"2023-08-28T02:39:14.847643+0000","last_undegraded":"2023-08-28T02:39:14.847643+0000","last_fullsized":"2023-08-28T02:39:14.847643+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:08:07.234984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"361'11871","reported_seq":11925,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-08-28T02:39:29.164145+0000","last_change":"2023-08-28T02:35:14.503271+0000","last_active":"2023-08-28T02:39:29.164145+0000","last_peered":"2023-08-28T02:39:29.164145+0000","last_clean":"2023-08-28T02:39:29.164145+0000","last_became_active":"2023-08-28T02:35:14.502940+0000","last_became_peered":"2023-08-28T02:35:14.502940+0000","last_unstale":"2023-08-28T02:39:29.164145+0000","last_undegraded":"2023-08-28T02:39:29.164145+0000","last_fullsized":"2023-08-28T02:39:29.164145+0000","mapping_epoch":318,"log_start":"335'4298","ondisk_log_start":"335'4298","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":7573,"log_dups_size":0,"ondisk_log_size":7573,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:07:49.114269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12910592,"num_objects":197,"num_object_clones":0,"num_object_copies":591,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":197,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11871,"num_write_kb":386176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":228,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.843487+0000","last_change":"2023-08-28T02:23:59.230059+0000","last_active":"2023-08-28T02:39:14.843487+0000","last_peered":"2023-08-28T02:39:14.843487+0000","last_clean":"2023-08-28T02:39:14.843487+0000","last_became_active":"2023-08-28T02:23:59.217783+0000","last_became_peered":"2023-08-28T02:23:59.217783+0000","last_unstale":"2023-08-28T02:39:14.843487+0000","last_undegraded":"2023-08-28T02:39:14.843487+0000","last_fullsized":"2023-08-28T02:39:14.843487+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:00:12.145395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":381,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.849617+0000","last_change":"2023-08-28T02:28:25.351698+0000","last_active":"2023-08-28T02:39:14.849617+0000","last_peered":"2023-08-28T02:39:14.849617+0000","last_clean":"2023-08-28T02:39:14.849617+0000","last_became_active":"2023-08-28T02:28:25.351349+0000","last_became_peered":"2023-08-28T02:28:25.351349+0000","last_unstale":"2023-08-28T02:39:14.849617+0000","last_undegraded":"2023-08-28T02:39:14.849617+0000","last_fullsized":"2023-08-28T02:39:14.849617+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"361'11966","reported_seq":12029,"reported_epoch":361,"state":"active+remapped+backfill_wait","last_fresh":"2023-08-28T02:39:33.604203+0000","last_change":"2023-08-28T02:38:37.063228+0000","last_active":"2023-08-28T02:39:33.604203+0000","last_peered":"2023-08-28T02:39:33.604203+0000","last_clean":"2023-08-28T02:38:34.841553+0000","last_became_active":"2023-08-28T02:38:37.003296+0000","last_became_peered":"2023-08-28T02:38:37.003296+0000","last_unstale":"2023-08-28T02:39:33.604203+0000","last_undegraded":"2023-08-28T02:39:33.604203+0000","last_fullsized":"2023-08-28T02:39:33.604203+0000","mapping_epoch":354,"log_start":"336'4778","ondisk_log_start":"336'4778","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":7188,"log_dups_size":0,"ondisk_log_size":7188,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:00:11.875820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2621440,"num_objects":40,"num_object_clones":0,"num_object_copies":120,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":40,"num_objects_unfound":0,"num_objects_dirty":40,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11966,"num_write_kb":384192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,2],"avail_no_missing":["1","0","2"],"object_location_counts":[{"shards":"0,1,2","objects":40}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":167,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.854423+0000","last_change":"2023-08-28T02:32:58.629249+0000","last_active":"2023-08-28T02:39:14.854423+0000","last_peered":"2023-08-28T02:39:14.854423+0000","last_clean":"2023-08-28T02:39:14.854423+0000","last_became_active":"2023-08-28T02:32:58.628583+0000","last_became_peered":"2023-08-28T02:32:58.628583+0000","last_unstale":"2023-08-28T02:39:14.854423+0000","last_undegraded":"2023-08-28T02:39:14.854423+0000","last_fullsized":"2023-08-28T02:39:14.854423+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":443,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.848186+0000","last_change":"2023-08-28T02:38:35.950952+0000","last_active":"2023-08-28T02:39:14.848186+0000","last_peered":"2023-08-28T02:39:14.848186+0000","last_clean":"2023-08-28T02:39:14.848186+0000","last_became_active":"2023-08-28T02:38:35.950478+0000","last_became_peered":"2023-08-28T02:38:35.950478+0000","last_unstale":"2023-08-28T02:39:14.848186+0000","last_undegraded":"2023-08-28T02:39:14.848186+0000","last_fullsized":"2023-08-28T02:39:14.848186+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"361'12021","reported_seq":12072,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-08-28T02:39:32.066753+0000","last_change":"2023-08-28T02:35:14.498910+0000","last_active":"2023-08-28T02:39:32.066753+0000","last_peered":"2023-08-28T02:39:32.066753+0000","last_clean":"2023-08-28T02:39:32.066753+0000","last_became_active":"2023-08-28T02:35:14.498631+0000","last_became_peered":"2023-08-28T02:35:14.498631+0000","last_unstale":"2023-08-28T02:39:32.066753+0000","last_undegraded":"2023-08-28T02:39:32.066753+0000","last_fullsized":"2023-08-28T02:39:32.066753+0000","mapping_epoch":318,"log_start":"333'3827","ondisk_log_start":"333'3827","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":8194,"log_dups_size":0,"ondisk_log_size":8194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7143424,"num_objects":109,"num_object_clones":0,"num_object_copies":327,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":109,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12021,"num_write_kb":388160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":315,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.848103+0000","last_change":"2023-08-28T02:38:35.885723+0000","last_active":"2023-08-28T02:39:14.848103+0000","last_peered":"2023-08-28T02:39:14.848103+0000","last_clean":"2023-08-28T02:39:14.848103+0000","last_became_active":"2023-08-28T02:38:35.885381+0000","last_became_peered":"2023-08-28T02:38:35.885381+0000","last_unstale":"2023-08-28T02:39:14.848103+0000","last_undegraded":"2023-08-28T02:39:14.848103+0000","last_fullsized":"2023-08-28T02:39:14.848103+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":349,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.851214+0000","last_change":"2023-08-28T02:32:58.766795+0000","last_active":"2023-08-28T02:39:14.851214+0000","last_peered":"2023-08-28T02:39:14.851214+0000","last_clean":"2023-08-28T02:39:14.851214+0000","last_became_active":"2023-08-28T02:32:58.766441+0000","last_became_peered":"2023-08-28T02:32:58.766441+0000","last_unstale":"2023-08-28T02:39:14.851214+0000","last_undegraded":"2023-08-28T02:39:14.851214+0000","last_fullsized":"2023-08-28T02:39:14.851214+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:58:13.963961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.3","version":"361'12113","reported_seq":12167,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-08-28T02:39:33.610244+0000","last_change":"2023-08-28T02:35:14.498188+0000","last_active":"2023-08-28T02:39:33.610244+0000","last_peered":"2023-08-28T02:39:33.610244+0000","last_clean":"2023-08-28T02:39:33.610244+0000","last_became_active":"2023-08-28T02:35:14.496278+0000","last_became_peered":"2023-08-28T02:35:14.496278+0000","last_unstale":"2023-08-28T02:39:33.610244+0000","last_undegraded":"2023-08-28T02:39:33.610244+0000","last_fullsized":"2023-08-28T02:39:33.610244+0000","mapping_epoch":318,"log_start":"336'4933","ondisk_log_start":"336'4933","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":7180,"log_dups_size":0,"ondisk_log_size":7180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:43:23.633921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2555904,"num_objects":39,"num_object_clones":0,"num_object_copies":117,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":39,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12113,"num_write_kb":388864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":279,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.843685+0000","last_change":"2023-08-28T02:37:45.579112+0000","last_active":"2023-08-28T02:39:14.843685+0000","last_peered":"2023-08-28T02:39:14.843685+0000","last_clean":"2023-08-28T02:39:14.843685+0000","last_became_active":"2023-08-28T02:37:45.578787+0000","last_became_peered":"2023-08-28T02:37:45.578787+0000","last_unstale":"2023-08-28T02:39:14.843685+0000","last_undegraded":"2023-08-28T02:39:14.843685+0000","last_fullsized":"2023-08-28T02:39:14.843685+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:20:59.994447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":82,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.845145+0000","last_change":"2023-08-28T02:32:58.755765+0000","last_active":"2023-08-28T02:39:14.845145+0000","last_peered":"2023-08-28T02:39:14.845145+0000","last_clean":"2023-08-28T02:39:14.845145+0000","last_became_active":"2023-08-28T02:32:58.755438+0000","last_became_peered":"2023-08-28T02:32:58.755438+0000","last_unstale":"2023-08-28T02:39:14.845145+0000","last_undegraded":"2023-08-28T02:39:14.845145+0000","last_fullsized":"2023-08-28T02:39:14.845145+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:14:11.498611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"361'11966","reported_seq":12028,"reported_epoch":361,"state":"active+remapped+backfill_wait","last_fresh":"2023-08-28T02:39:30.123750+0000","last_change":"2023-08-28T02:38:37.052869+0000","last_active":"2023-08-28T02:39:30.123750+0000","last_peered":"2023-08-28T02:39:30.123750+0000","last_clean":"2023-08-28T02:38:34.861093+0000","last_became_active":"2023-08-28T02:38:37.005189+0000","last_became_peered":"2023-08-28T02:38:37.005189+0000","last_unstale":"2023-08-28T02:39:30.123750+0000","last_undegraded":"2023-08-28T02:39:30.123750+0000","last_fullsized":"2023-08-28T02:39:30.123750+0000","mapping_epoch":354,"log_start":"336'4641","ondisk_log_start":"336'4641","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":7325,"log_dups_size":0,"ondisk_log_size":7325,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:20:40.565888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12845056,"num_objects":196,"num_object_clones":0,"num_object_copies":588,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":196,"num_objects_unfound":0,"num_objects_dirty":196,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11966,"num_write_kb":389184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,2],"avail_no_missing":["0","2","4"],"object_location_counts":[{"shards":"0,2,4","objects":196}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"78'126","reported_seq":808,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:18.615538+0000","last_change":"2023-08-28T02:37:48.647489+0000","last_active":"2023-08-28T02:39:18.615538+0000","last_peered":"2023-08-28T02:39:18.615538+0000","last_clean":"2023-08-28T02:39:18.615538+0000","last_became_active":"2023-08-28T02:37:48.647002+0000","last_became_peered":"2023-08-28T02:37:48.647002+0000","last_unstale":"2023-08-28T02:39:18.615538+0000","last_undegraded":"2023-08-28T02:39:18.615538+0000","last_fullsized":"2023-08-28T02:39:18.615538+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:04:10.819878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":743,"num_read_kb":1331,"num_write":125,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":213,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:24.188769+0000","last_change":"2023-08-28T02:38:35.898925+0000","last_active":"2023-08-28T02:39:24.188769+0000","last_peered":"2023-08-28T02:39:24.188769+0000","last_clean":"2023-08-28T02:39:24.188769+0000","last_became_active":"2023-08-28T02:38:35.898546+0000","last_became_peered":"2023-08-28T02:38:35.898546+0000","last_unstale":"2023-08-28T02:39:24.188769+0000","last_undegraded":"2023-08-28T02:39:24.188769+0000","last_fullsized":"2023-08-28T02:39:24.188769+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:25:31.558360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":18,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.845336+0000","last_change":"2023-08-28T02:38:35.883771+0000","last_active":"2023-08-28T02:39:14.845336+0000","last_peered":"2023-08-28T02:39:14.845336+0000","last_clean":"2023-08-28T02:39:14.845336+0000","last_became_active":"2023-08-28T02:38:35.883052+0000","last_became_peered":"2023-08-28T02:38:35.883052+0000","last_unstale":"2023-08-28T02:39:14.845336+0000","last_undegraded":"2023-08-28T02:39:14.845336+0000","last_fullsized":"2023-08-28T02:39:14.845336+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:54:17.468119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"361'12042","reported_seq":12259,"reported_epoch":361,"state":"active+remapped+backfill_wait","last_fresh":"2023-08-28T02:39:29.158409+0000","last_change":"2023-08-28T02:38:35.930610+0000","last_active":"2023-08-28T02:39:29.158409+0000","last_peered":"2023-08-28T02:39:29.158409+0000","last_clean":"2023-08-28T02:37:44.530880+0000","last_became_active":"2023-08-28T02:37:46.790659+0000","last_became_peered":"2023-08-28T02:37:46.790659+0000","last_unstale":"2023-08-28T02:39:29.158409+0000","last_undegraded":"2023-08-28T02:39:29.158409+0000","last_fullsized":"2023-08-28T02:39:29.158409+0000","mapping_epoch":345,"log_start":"335'4458","ondisk_log_start":"335'4458","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":7584,"log_dups_size":0,"ondisk_log_size":7584,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:37:46.944577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14548992,"num_objects":222,"num_object_clones":0,"num_object_copies":666,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":217,"num_objects_unfound":0,"num_objects_dirty":222,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12042,"num_write_kb":392448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":74,"num_bytes_recovered":4849664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,0],"avail_no_missing":["3","0","5"],"object_location_counts":[{"shards":"0,3,5","objects":222}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"78'204","reported_seq":1447,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.845707+0000","last_change":"2023-08-28T02:32:59.644574+0000","last_active":"2023-08-28T02:39:14.845707+0000","last_peered":"2023-08-28T02:39:14.845707+0000","last_clean":"2023-08-28T02:39:14.845707+0000","last_became_active":"2023-08-28T02:32:59.644053+0000","last_became_peered":"2023-08-28T02:32:59.644053+0000","last_unstale":"2023-08-28T02:39:14.845707+0000","last_undegraded":"2023-08-28T02:39:14.845707+0000","last_fullsized":"2023-08-28T02:39:14.845707+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":204,"log_dups_size":0,"ondisk_log_size":204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:55:51.460483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1521,"num_read_kb":2511,"num_write":178,"num_write_kb":1175,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":355,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.848137+0000","last_change":"2023-08-28T02:28:25.352417+0000","last_active":"2023-08-28T02:39:14.848137+0000","last_peered":"2023-08-28T02:39:14.848137+0000","last_clean":"2023-08-28T02:39:14.848137+0000","last_became_active":"2023-08-28T02:28:25.352216+0000","last_became_peered":"2023-08-28T02:28:25.352216+0000","last_unstale":"2023-08-28T02:39:14.848137+0000","last_undegraded":"2023-08-28T02:39:14.848137+0000","last_fullsized":"2023-08-28T02:39:14.848137+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":92,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:18.615517+0000","last_change":"2023-08-28T02:38:35.883370+0000","last_active":"2023-08-28T02:39:18.615517+0000","last_peered":"2023-08-28T02:39:18.615517+0000","last_clean":"2023-08-28T02:39:18.615517+0000","last_became_active":"2023-08-28T02:38:35.883049+0000","last_became_peered":"2023-08-28T02:38:35.883049+0000","last_unstale":"2023-08-28T02:39:18.615517+0000","last_undegraded":"2023-08-28T02:39:18.615517+0000","last_fullsized":"2023-08-28T02:39:18.615517+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:09:38.259839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"361'11904","reported_seq":11957,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-08-28T02:39:29.171512+0000","last_change":"2023-08-28T02:35:14.503279+0000","last_active":"2023-08-28T02:39:29.171512+0000","last_peered":"2023-08-28T02:39:29.171512+0000","last_clean":"2023-08-28T02:39:29.171512+0000","last_became_active":"2023-08-28T02:35:14.502953+0000","last_became_peered":"2023-08-28T02:35:14.502953+0000","last_unstale":"2023-08-28T02:39:29.171512+0000","last_undegraded":"2023-08-28T02:39:29.171512+0000","last_fullsized":"2023-08-28T02:39:29.171512+0000","mapping_epoch":318,"log_start":"335'4321","ondisk_log_start":"335'4321","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":7583,"log_dups_size":0,"ondisk_log_size":7583,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:55:29.284519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15728640,"num_objects":240,"num_object_clones":0,"num_object_copies":720,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":240,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11904,"num_write_kb":388608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"78'47","reported_seq":377,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.842943+0000","last_change":"2023-08-28T02:38:35.942552+0000","last_active":"2023-08-28T02:39:14.842943+0000","last_peered":"2023-08-28T02:39:14.842943+0000","last_clean":"2023-08-28T02:39:14.842943+0000","last_became_active":"2023-08-28T02:38:35.885504+0000","last_became_peered":"2023-08-28T02:38:35.885504+0000","last_unstale":"2023-08-28T02:39:14.842943+0000","last_undegraded":"2023-08-28T02:39:14.842943+0000","last_fullsized":"2023-08-28T02:39:14.842943+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":47,"log_dups_size":0,"ondisk_log_size":47,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:05:08.556625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":162,"num_read_kb":1281,"num_write":48,"num_write_kb":1153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":92,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.848602+0000","last_change":"2023-08-28T02:38:35.883555+0000","last_active":"2023-08-28T02:39:14.848602+0000","last_peered":"2023-08-28T02:39:14.848602+0000","last_clean":"2023-08-28T02:39:14.848602+0000","last_became_active":"2023-08-28T02:38:35.883221+0000","last_became_peered":"2023-08-28T02:38:35.883221+0000","last_unstale":"2023-08-28T02:39:14.848602+0000","last_undegraded":"2023-08-28T02:39:14.848602+0000","last_fullsized":"2023-08-28T02:39:14.848602+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:37:47.771594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":83,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:28.647571+0000","last_change":"2023-08-28T02:32:58.724424+0000","last_active":"2023-08-28T02:39:28.647571+0000","last_peered":"2023-08-28T02:39:28.647571+0000","last_clean":"2023-08-28T02:39:28.647571+0000","last_became_active":"2023-08-28T02:32:58.724127+0000","last_became_peered":"2023-08-28T02:32:58.724127+0000","last_unstale":"2023-08-28T02:39:28.647571+0000","last_undegraded":"2023-08-28T02:39:28.647571+0000","last_fullsized":"2023-08-28T02:39:28.647571+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:02:25.178712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"361'12175","reported_seq":12226,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-08-28T02:39:32.434872+0000","last_change":"2023-08-28T02:35:14.500884+0000","last_active":"2023-08-28T02:39:32.434872+0000","last_peered":"2023-08-28T02:39:32.434872+0000","last_clean":"2023-08-28T02:39:32.434872+0000","last_became_active":"2023-08-28T02:35:14.500643+0000","last_became_peered":"2023-08-28T02:35:14.500643+0000","last_unstale":"2023-08-28T02:39:32.434872+0000","last_undegraded":"2023-08-28T02:39:32.434872+0000","last_fullsized":"2023-08-28T02:39:32.434872+0000","mapping_epoch":318,"log_start":"335'4431","ondisk_log_start":"335'4431","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":7744,"log_dups_size":0,"ondisk_log_size":7744,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5439488,"num_objects":83,"num_object_clones":0,"num_object_copies":249,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":83,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12175,"num_write_kb":392256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"78'114","reported_seq":418,"reported_epoch":359,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:39:10.811677+0000","last_change":"2023-08-28T02:38:36.884225+0000","last_active":"2023-08-28T02:39:10.811677+0000","last_peered":"2023-08-28T02:39:10.811677+0000","last_clean":"2023-08-28T02:37:38.799203+0000","last_became_active":"2023-08-28T02:38:36.883700+0000","last_became_peered":"2023-08-28T02:38:36.883700+0000","last_unstale":"2023-08-28T02:39:10.811677+0000","last_undegraded":"2023-08-28T02:38:36.874234+0000","last_fullsized":"2023-08-28T02:38:36.873408+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":114,"log_dups_size":0,"ondisk_log_size":114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":358,"num_read_kb":1352,"num_write":102,"num_write_kb":1104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5],"avail_no_missing":["7","2","5"],"object_location_counts":[{"shards":"2,5,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"86'139","reported_seq":454,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-08-28T02:39:10.811216+0000","last_change":"2023-08-28T02:37:48.651113+0000","last_active":"2023-08-28T02:39:10.811216+0000","last_peered":"2023-08-28T02:39:10.811216+0000","last_clean":"2023-08-28T02:39:10.811216+0000","last_became_active":"2023-08-28T02:37:48.650691+0000","last_became_peered":"2023-08-28T02:37:48.650691+0000","last_unstale":"2023-08-28T02:39:10.811216+0000","last_undegraded":"2023-08-28T02:39:10.811216+0000","last_fullsized":"2023-08-28T02:39:10.811216+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_clean_scrub_stamp":"2023-08-28T02:08:02.043718+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":243,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.842909+0000","last_change":"2023-08-28T02:28:25.347600+0000","last_active":"2023-08-28T02:39:14.842909+0000","last_peered":"2023-08-28T02:39:14.842909+0000","last_clean":"2023-08-28T02:39:14.842909+0000","last_became_active":"2023-08-28T02:28:25.347237+0000","last_became_peered":"2023-08-28T02:28:25.347237+0000","last_unstale":"2023-08-28T02:39:14.842909+0000","last_undegraded":"2023-08-28T02:39:14.842909+0000","last_fullsized":"2023-08-28T02:39:14.842909+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:29:26.767559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":167,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.856402+0000","last_change":"2023-08-28T02:37:45.581261+0000","last_active":"2023-08-28T02:39:14.856402+0000","last_peered":"2023-08-28T02:39:14.856402+0000","last_clean":"2023-08-28T02:39:14.856402+0000","last_became_active":"2023-08-28T02:37:45.580927+0000","last_became_peered":"2023-08-28T02:37:45.580927+0000","last_unstale":"2023-08-28T02:39:14.856402+0000","last_undegraded":"2023-08-28T02:39:14.856402+0000","last_fullsized":"2023-08-28T02:39:14.856402+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"361'11923","reported_seq":11986,"reported_epoch":361,"state":"active+remapped+backfill_wait","last_fresh":"2023-08-28T02:39:33.610236+0000","last_change":"2023-08-28T02:38:37.028262+0000","last_active":"2023-08-28T02:39:33.610236+0000","last_peered":"2023-08-28T02:39:33.610236+0000","last_clean":"2023-08-28T02:38:34.861056+0000","last_became_active":"2023-08-28T02:38:36.988695+0000","last_became_peered":"2023-08-28T02:38:36.988695+0000","last_unstale":"2023-08-28T02:39:33.610236+0000","last_undegraded":"2023-08-28T02:39:33.610236+0000","last_fullsized":"2023-08-28T02:39:33.610236+0000","mapping_epoch":354,"log_start":"336'4711","ondisk_log_start":"336'4711","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":7212,"log_dups_size":0,"ondisk_log_size":7212,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:09:45.928327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2424832,"num_objects":37,"num_object_clones":0,"num_object_copies":111,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":37,"num_objects_unfound":0,"num_objects_dirty":37,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11923,"num_write_kb":382720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,2],"avail_no_missing":["1","2","5"],"object_location_counts":[{"shards":"1,2,5","objects":37}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"78'148","reported_seq":678,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-08-28T02:39:10.811214+0000","last_change":"2023-08-28T02:37:48.650996+0000","last_active":"2023-08-28T02:39:10.811214+0000","last_peered":"2023-08-28T02:39:10.811214+0000","last_clean":"2023-08-28T02:39:10.811214+0000","last_became_active":"2023-08-28T02:37:48.650016+0000","last_became_peered":"2023-08-28T02:37:48.650016+0000","last_unstale":"2023-08-28T02:39:10.811214+0000","last_undegraded":"2023-08-28T02:39:10.811214+0000","last_fullsized":"2023-08-28T02:39:10.811214+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1092,"num_read_kb":1955,"num_write":127,"num_write_kb":5177,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":334,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-08-28T02:39:28.796244+0000","last_change":"2023-08-28T02:23:59.226986+0000","last_active":"2023-08-28T02:39:28.796244+0000","last_peered":"2023-08-28T02:39:28.796244+0000","last_clean":"2023-08-28T02:39:28.796244+0000","last_became_active":"2023-08-28T02:23:59.226529+0000","last_became_peered":"2023-08-28T02:23:59.226529+0000","last_unstale":"2023-08-28T02:39:28.796244+0000","last_undegraded":"2023-08-28T02:39:28.796244+0000","last_fullsized":"2023-08-28T02:39:28.796244+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:22:09.193619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":405,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.851000+0000","last_change":"2023-08-28T02:38:35.885714+0000","last_active":"2023-08-28T02:39:14.851000+0000","last_peered":"2023-08-28T02:39:14.851000+0000","last_clean":"2023-08-28T02:39:14.851000+0000","last_became_active":"2023-08-28T02:38:35.885391+0000","last_became_peered":"2023-08-28T02:38:35.885391+0000","last_unstale":"2023-08-28T02:39:14.851000+0000","last_undegraded":"2023-08-28T02:39:14.851000+0000","last_fullsized":"2023-08-28T02:39:14.851000+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:02:57.365844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"361'12001","reported_seq":12221,"reported_epoch":361,"state":"active+remapped+backfill_wait","last_fresh":"2023-08-28T02:39:29.147511+0000","last_change":"2023-08-28T02:38:37.076874+0000","last_active":"2023-08-28T02:39:29.147511+0000","last_peered":"2023-08-28T02:39:29.147511+0000","last_clean":"2023-08-28T02:38:34.867302+0000","last_became_active":"2023-08-28T02:38:37.016339+0000","last_became_peered":"2023-08-28T02:38:37.016339+0000","last_unstale":"2023-08-28T02:39:29.147511+0000","last_undegraded":"2023-08-28T02:39:29.147511+0000","last_fullsized":"2023-08-28T02:39:29.147511+0000","mapping_epoch":354,"log_start":"335'4500","ondisk_log_start":"335'4500","created":318,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":7501,"log_dups_size":0,"ondisk_log_size":7501,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:01:21.215657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14745600,"num_objects":225,"num_object_clones":0,"num_object_copies":675,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":225,"num_objects_unfound":0,"num_objects_dirty":225,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12001,"num_write_kb":391232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":73,"num_bytes_recovered":4784128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,2],"avail_no_missing":["3","1","2"],"object_location_counts":[{"shards":"1,2,3","objects":225}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"78'103","reported_seq":700,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-08-28T02:39:10.811623+0000","last_change":"2023-08-28T02:37:48.651919+0000","last_active":"2023-08-28T02:39:10.811623+0000","last_peered":"2023-08-28T02:39:10.811623+0000","last_clean":"2023-08-28T02:39:10.811623+0000","last_became_active":"2023-08-28T02:37:48.651437+0000","last_became_peered":"2023-08-28T02:37:48.651437+0000","last_unstale":"2023-08-28T02:39:10.811623+0000","last_undegraded":"2023-08-28T02:39:10.811623+0000","last_fullsized":"2023-08-28T02:39:10.811623+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":803,"num_read_kb":2080,"num_write":73,"num_write_kb":1244,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":92,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:18.615502+0000","last_change":"2023-08-28T02:38:35.882757+0000","last_active":"2023-08-28T02:39:18.615502+0000","last_peered":"2023-08-28T02:39:18.615502+0000","last_clean":"2023-08-28T02:39:18.615502+0000","last_became_active":"2023-08-28T02:38:35.882423+0000","last_became_peered":"2023-08-28T02:38:35.882423+0000","last_unstale":"2023-08-28T02:39:18.615502+0000","last_undegraded":"2023-08-28T02:39:18.615502+0000","last_fullsized":"2023-08-28T02:39:18.615502+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:54:44.915353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":356,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-08-28T02:39:28.796503+0000","last_change":"2023-08-28T02:37:45.579740+0000","last_active":"2023-08-28T02:39:28.796503+0000","last_peered":"2023-08-28T02:39:28.796503+0000","last_clean":"2023-08-28T02:39:28.796503+0000","last_became_active":"2023-08-28T02:37:45.579412+0000","last_became_peered":"2023-08-28T02:37:45.579412+0000","last_unstale":"2023-08-28T02:39:28.796503+0000","last_undegraded":"2023-08-28T02:39:28.796503+0000","last_fullsized":"2023-08-28T02:39:28.796503+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:31:18.483532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"361'11598","reported_seq":11688,"reported_epoch":361,"state":"active+remapped+backfill_wait","last_fresh":"2023-08-28T02:39:30.060678+0000","last_change":"2023-08-28T02:37:47.122442+0000","last_active":"2023-08-28T02:39:30.060678+0000","last_peered":"2023-08-28T02:39:30.060678+0000","last_clean":"2023-08-28T02:37:44.513990+0000","last_became_active":"2023-08-28T02:37:46.801002+0000","last_became_peered":"2023-08-28T02:37:46.801002+0000","last_unstale":"2023-08-28T02:39:30.060678+0000","last_undegraded":"2023-08-28T02:39:30.060678+0000","last_fullsized":"2023-08-28T02:39:30.060678+0000","mapping_epoch":345,"log_start":"335'4242","ondisk_log_start":"335'4242","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":7356,"log_dups_size":0,"ondisk_log_size":7356,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:51:20.134517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10354688,"num_objects":158,"num_object_clones":0,"num_object_copies":474,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":158,"num_objects_unfound":0,"num_objects_dirty":158,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11598,"num_write_kb":376192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":786432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,5],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,3,5","objects":158}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":255,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.844185+0000","last_change":"2023-08-28T02:37:45.590339+0000","last_active":"2023-08-28T02:39:14.844185+0000","last_peered":"2023-08-28T02:39:14.844185+0000","last_clean":"2023-08-28T02:39:14.844185+0000","last_became_active":"2023-08-28T02:37:45.578751+0000","last_became_peered":"2023-08-28T02:37:45.578751+0000","last_unstale":"2023-08-28T02:39:14.844185+0000","last_undegraded":"2023-08-28T02:39:14.844185+0000","last_fullsized":"2023-08-28T02:39:14.844185+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:27:24.226372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'112","reported_seq":516,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:28.647556+0000","last_change":"2023-08-28T02:32:59.643465+0000","last_active":"2023-08-28T02:39:28.647556+0000","last_peered":"2023-08-28T02:39:28.647556+0000","last_clean":"2023-08-28T02:39:28.647556+0000","last_became_active":"2023-08-28T02:32:59.642979+0000","last_became_peered":"2023-08-28T02:32:59.642979+0000","last_unstale":"2023-08-28T02:39:28.647556+0000","last_undegraded":"2023-08-28T02:39:28.647556+0000","last_fullsized":"2023-08-28T02:39:28.647556+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:48:45.923015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":444,"num_read_kb":473,"num_write":94,"num_write_kb":127,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":261,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.843606+0000","last_change":"2023-08-28T02:37:45.578790+0000","last_active":"2023-08-28T02:39:14.843606+0000","last_peered":"2023-08-28T02:39:14.843606+0000","last_clean":"2023-08-28T02:39:14.843606+0000","last_became_active":"2023-08-28T02:37:45.578473+0000","last_became_peered":"2023-08-28T02:37:45.578473+0000","last_unstale":"2023-08-28T02:39:14.843606+0000","last_undegraded":"2023-08-28T02:39:14.843606+0000","last_fullsized":"2023-08-28T02:39:14.843606+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:23:51.722772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"361'11795","reported_seq":14711,"reported_epoch":361,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:39:30.316485+0000","last_change":"2023-08-28T02:38:36.180937+0000","last_active":"2023-08-28T02:39:30.316485+0000","last_peered":"2023-08-28T02:39:30.316485+0000","last_clean":"2023-08-28T02:37:44.560462+0000","last_became_active":"2023-08-28T02:38:36.091055+0000","last_became_peered":"2023-08-28T02:38:36.091055+0000","last_unstale":"2023-08-28T02:39:30.316485+0000","last_undegraded":"2023-08-28T02:38:36.038772+0000","last_fullsized":"2023-08-28T02:38:35.872670+0000","mapping_epoch":353,"log_start":"326'1700","ondisk_log_start":"326'1700","created":318,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:35:13.482146+0000","last_clean_scrub_stamp":"2023-08-28T02:35:13.482146+0000","objects_scrubbed":0,"log_size":10095,"log_dups_size":0,"ondisk_log_size":10095,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:55:48.264648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":9830424,"num_objects":151,"num_object_clones":0,"num_object_copies":453,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1734,"num_objects_misplaced":4549,"num_objects_unfound":0,"num_objects_dirty":151,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11795,"num_write_kb":382209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1424,"num_bytes_recovered":93257728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":-1928},{"shards":"2,5,6","objects":151}],"blocked_by":[],"up_primary":7,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"78'122","reported_seq":749,"reported_epoch":360,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:39:14.855056+0000","last_change":"2023-08-28T02:38:36.904311+0000","last_active":"2023-08-28T02:39:14.855056+0000","last_peered":"2023-08-28T02:39:14.855056+0000","last_clean":"2023-08-28T02:38:27.810878+0000","last_became_active":"2023-08-28T02:38:36.903810+0000","last_became_peered":"2023-08-28T02:38:36.903810+0000","last_unstale":"2023-08-28T02:39:14.855056+0000","last_undegraded":"2023-08-28T02:38:36.876395+0000","last_fullsized":"2023-08-28T02:38:36.875861+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":857,"num_read_kb":2224,"num_write":99,"num_write_kb":2312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7],"avail_no_missing":["6","2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":371,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.851163+0000","last_change":"2023-08-28T02:32:58.795995+0000","last_active":"2023-08-28T02:39:14.851163+0000","last_peered":"2023-08-28T02:39:14.851163+0000","last_clean":"2023-08-28T02:39:14.851163+0000","last_became_active":"2023-08-28T02:32:58.771872+0000","last_became_peered":"2023-08-28T02:32:58.771872+0000","last_unstale":"2023-08-28T02:39:14.851163+0000","last_undegraded":"2023-08-28T02:39:14.851163+0000","last_fullsized":"2023-08-28T02:39:14.851163+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:16:02.965467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":330,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.847833+0000","last_change":"2023-08-28T02:28:25.351840+0000","last_active":"2023-08-28T02:39:14.847833+0000","last_peered":"2023-08-28T02:39:14.847833+0000","last_clean":"2023-08-28T02:39:14.847833+0000","last_became_active":"2023-08-28T02:28:25.351399+0000","last_became_peered":"2023-08-28T02:28:25.351399+0000","last_unstale":"2023-08-28T02:39:14.847833+0000","last_undegraded":"2023-08-28T02:39:14.847833+0000","last_fullsized":"2023-08-28T02:39:14.847833+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":357,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.847788+0000","last_change":"2023-08-28T02:37:45.581817+0000","last_active":"2023-08-28T02:39:14.847788+0000","last_peered":"2023-08-28T02:39:14.847788+0000","last_clean":"2023-08-28T02:39:14.847788+0000","last_became_active":"2023-08-28T02:37:45.581475+0000","last_became_peered":"2023-08-28T02:37:45.581475+0000","last_unstale":"2023-08-28T02:39:14.847788+0000","last_undegraded":"2023-08-28T02:39:14.847788+0000","last_fullsized":"2023-08-28T02:39:14.847788+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:00:31.628265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":403,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.843360+0000","last_change":"2023-08-28T02:38:35.882453+0000","last_active":"2023-08-28T02:39:14.843360+0000","last_peered":"2023-08-28T02:39:14.843360+0000","last_clean":"2023-08-28T02:39:14.843360+0000","last_became_active":"2023-08-28T02:38:35.882128+0000","last_became_peered":"2023-08-28T02:38:35.882128+0000","last_unstale":"2023-08-28T02:39:14.843360+0000","last_undegraded":"2023-08-28T02:39:14.843360+0000","last_fullsized":"2023-08-28T02:39:14.843360+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:07:17.115788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":193,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.849537+0000","last_change":"2023-08-28T02:32:58.839847+0000","last_active":"2023-08-28T02:39:14.849537+0000","last_peered":"2023-08-28T02:39:14.849537+0000","last_clean":"2023-08-28T02:39:14.849537+0000","last_became_active":"2023-08-28T02:32:58.628159+0000","last_became_peered":"2023-08-28T02:32:58.628159+0000","last_unstale":"2023-08-28T02:39:14.849537+0000","last_undegraded":"2023-08-28T02:39:14.849537+0000","last_fullsized":"2023-08-28T02:39:14.849537+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":396,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.848289+0000","last_change":"2023-08-28T02:32:58.629596+0000","last_active":"2023-08-28T02:39:14.848289+0000","last_peered":"2023-08-28T02:39:14.848289+0000","last_clean":"2023-08-28T02:39:14.848289+0000","last_became_active":"2023-08-28T02:32:58.629136+0000","last_became_peered":"2023-08-28T02:32:58.629136+0000","last_unstale":"2023-08-28T02:39:14.848289+0000","last_undegraded":"2023-08-28T02:39:14.848289+0000","last_fullsized":"2023-08-28T02:39:14.848289+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":180,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.848361+0000","last_change":"2023-08-28T02:28:25.705331+0000","last_active":"2023-08-28T02:39:14.848361+0000","last_peered":"2023-08-28T02:39:14.848361+0000","last_clean":"2023-08-28T02:39:14.848361+0000","last_became_active":"2023-08-28T02:28:25.397480+0000","last_became_peered":"2023-08-28T02:28:25.397480+0000","last_unstale":"2023-08-28T02:39:14.848361+0000","last_undegraded":"2023-08-28T02:39:14.848361+0000","last_fullsized":"2023-08-28T02:39:14.848361+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":360,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.848205+0000","last_change":"2023-08-28T02:23:59.207929+0000","last_active":"2023-08-28T02:39:14.848205+0000","last_peered":"2023-08-28T02:39:14.848205+0000","last_clean":"2023-08-28T02:39:14.848205+0000","last_became_active":"2023-08-28T02:23:59.207589+0000","last_became_peered":"2023-08-28T02:23:59.207589+0000","last_unstale":"2023-08-28T02:39:14.848205+0000","last_undegraded":"2023-08-28T02:39:14.848205+0000","last_fullsized":"2023-08-28T02:39:14.848205+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":261,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.842567+0000","last_change":"2023-08-28T02:38:35.890747+0000","last_active":"2023-08-28T02:39:14.842567+0000","last_peered":"2023-08-28T02:39:14.842567+0000","last_clean":"2023-08-28T02:39:14.842567+0000","last_became_active":"2023-08-28T02:38:35.890440+0000","last_became_peered":"2023-08-28T02:38:35.890440+0000","last_unstale":"2023-08-28T02:39:14.842567+0000","last_undegraded":"2023-08-28T02:39:14.842567+0000","last_fullsized":"2023-08-28T02:39:14.842567+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:29:47.306581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":233,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.843093+0000","last_change":"2023-08-28T02:38:35.882097+0000","last_active":"2023-08-28T02:39:14.843093+0000","last_peered":"2023-08-28T02:39:14.843093+0000","last_clean":"2023-08-28T02:39:14.843093+0000","last_became_active":"2023-08-28T02:38:35.881687+0000","last_became_peered":"2023-08-28T02:38:35.881687+0000","last_unstale":"2023-08-28T02:39:14.843093+0000","last_undegraded":"2023-08-28T02:39:14.843093+0000","last_fullsized":"2023-08-28T02:39:14.843093+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:23:54.674952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":365,"reported_epoch":361,"state":"active+clean","last_fresh":"2023-08-28T02:39:28.796521+0000","last_change":"2023-08-28T02:38:35.920047+0000","last_active":"2023-08-28T02:39:28.796521+0000","last_peered":"2023-08-28T02:39:28.796521+0000","last_clean":"2023-08-28T02:39:28.796521+0000","last_became_active":"2023-08-28T02:38:35.888575+0000","last_became_peered":"2023-08-28T02:38:35.888575+0000","last_unstale":"2023-08-28T02:39:28.796521+0000","last_undegraded":"2023-08-28T02:39:28.796521+0000","last_fullsized":"2023-08-28T02:39:28.796521+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:35:57.784343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":419,"reported_epoch":360,"state":"active+clean","last_fresh":"2023-08-28T02:39:14.847835+0000","last_change":"2023-08-28T02:32:58.630270+0000","last_active":"2023-08-28T02:39:14.847835+0000","last_peered":"2023-08-28T02:39:14.847835+0000","last_clean":"2023-08-28T02:39:14.847835+0000","last_became_active":"2023-08-28T02:32:58.629958+0000","last_became_peered":"2023-08-28T02:32:58.629958+0000","last_unstale":"2023-08-28T02:39:14.847835+0000","last_undegraded":"2023-08-28T02:39:14.847835+0000","last_fullsized":"2023-08-28T02:39:14.847835+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":30498,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":17122,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35337,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":138477592,"num_objects":2114,"num_object_clones":0,"num_object_copies":6342,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3507,"num_objects_misplaced":5701,"num_objects_unfound":0,"num_objects_dirty":2114,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":191702,"num_write_kb":6202049,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3984,"num_bytes_recovered":241369088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":776089600,"data_stored":776077384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":122984,"ondisk_log_size":122984,"up":48,"acting":46,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":184,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":311,"seq":1335734829112,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":349432,"kb_used_data":26480,"kb_used_omap":8,"kb_used_meta":322935,"kb_avail":93395720,"statfs":{"total":95995035648,"available":95637217280,"internally_reserved":0,"allocated":27115520,"data_stored":26189719,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8527,"internal_metadata":330686129},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:39:04 2023","interfaces":[{"interface":"back","average":{"1min":9.3629999999999995,"5min":6.0140000000000002,"15min":3.0150000000000001},"min":{"1min":0.246,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":86.337000000000003,"5min":140.35499999999999,"15min":140.35499999999999},"last":2.0899999999999999},{"interface":"front","average":{"1min":9.3889999999999993,"5min":5.782,"15min":2.802},"min":{"1min":0.33400000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":86.358000000000004,"5min":140.40199999999999,"15min":140.40199999999999},"last":2.0800000000000001}]},{"osd":1,"last update":"Mon Aug 28 02:39:04 2023","interfaces":[{"interface":"back","average":{"1min":8.0850000000000009,"5min":8.3729999999999993,"15min":6.9379999999999997},"min":{"1min":0.25,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":127.152,"5min":151.048,"15min":151.048},"last":1.367},{"interface":"front","average":{"1min":8.5199999999999996,"5min":8.5570000000000004,"15min":6.8929999999999998},"min":{"1min":0.28899999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":129.465,"5min":150.56299999999999,"15min":150.56299999999999},"last":4.7889999999999997}]},{"osd":2,"last update":"Mon Aug 28 02:39:04 2023","interfaces":[{"interface":"back","average":{"1min":3.5179999999999998,"5min":3.2930000000000001,"15min":3.5609999999999999},"min":{"1min":0.34300000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":10.912000000000001,"5min":46.042999999999999,"15min":46.042999999999999},"last":4.1619999999999999},{"interface":"front","average":{"1min":2.29,"5min":2.6960000000000002,"15min":3.3010000000000002},"min":{"1min":0.34300000000000003,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":8.343,"5min":46.020000000000003,"15min":46.020000000000003},"last":4.069}]},{"osd":3,"last update":"Mon Aug 28 02:39:04 2023","interfaces":[{"interface":"back","average":{"1min":9.1180000000000003,"5min":5.9649999999999999,"15min":4.8289999999999997},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":125.30800000000001,"5min":129.20099999999999,"15min":129.20099999999999},"last":1.7250000000000001},{"interface":"front","average":{"1min":8.4629999999999992,"5min":6.1589999999999998,"15min":5.3579999999999997},"min":{"1min":0.39400000000000002,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":125.245,"5min":129.512,"15min":129.512},"last":8.1370000000000005}]},{"osd":4,"last update":"Mon Aug 28 02:39:04 2023","interfaces":[{"interface":"back","average":{"1min":2.105,"5min":1.107,"15min":1.079},"min":{"1min":0.27600000000000002,"5min":0.25,"15min":0.25},"max":{"1min":12.444000000000001,"5min":14.984999999999999,"15min":14.984999999999999},"last":1.228},{"interface":"front","average":{"1min":2.7879999999999998,"5min":1.468,"15min":1.323},"min":{"1min":0.317,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":11.83,"5min":21.18,"15min":21.18},"last":1.6639999999999999}]},{"osd":5,"last update":"Mon Aug 28 02:39:04 2023","interfaces":[{"interface":"back","average":{"1min":1.659,"5min":1.5349999999999999,"15min":2.0470000000000002},"min":{"1min":0.307,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":8.3670000000000009,"5min":36.476999999999997,"15min":36.476999999999997},"last":1.3029999999999999},{"interface":"front","average":{"1min":2.177,"5min":1.7350000000000001,"15min":2.1709999999999998},"min":{"1min":0.36599999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":8.6199999999999992,"5min":36.872,"15min":36.872},"last":2.3490000000000002}]},{"osd":6,"last update":"Mon Aug 28 02:39:04 2023","interfaces":[{"interface":"back","average":{"1min":11.016999999999999,"5min":3.8090000000000002,"15min":3.5470000000000002},"min":{"1min":0.373,"5min":0.307,"15min":0.307},"max":{"1min":204.48400000000001,"5min":204.48400000000001,"15min":204.48400000000001},"last":1.2549999999999999},{"interface":"front","average":{"1min":12.648,"5min":4.4189999999999996,"15min":3.8719999999999999},"min":{"1min":0.379,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":204.52099999999999,"5min":204.52099999999999,"15min":204.52099999999999},"last":1.6950000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626794,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":745476,"kb_used_data":159868,"kb_used_omap":0,"kb_used_meta":585600,"kb_avail":92999676,"statfs":{"total":95995035648,"available":95231668224,"internally_reserved":0,"allocated":163704832,"data_stored":162775219,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":599654400},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:39:26 2023","interfaces":[{"interface":"back","average":{"1min":2.52,"5min":3.8450000000000002,"15min":4.3799999999999999},"min":{"1min":0.23499999999999999,"5min":0.215,"15min":0.20799999999999999},"max":{"1min":10.516,"5min":185.09800000000001,"15min":185.09800000000001},"last":0.82499999999999996},{"interface":"front","average":{"1min":2.5630000000000002,"5min":4.3949999999999996,"15min":4.5830000000000002},"min":{"1min":0.26800000000000002,"5min":0.255,"15min":0.23200000000000001},"max":{"1min":9.157,"5min":185.017,"15min":185.017},"last":0.77900000000000003}]},{"osd":1,"last update":"Mon Aug 28 02:39:26 2023","interfaces":[{"interface":"back","average":{"1min":3.1349999999999998,"5min":2.8999999999999999,"15min":3.0249999999999999},"min":{"1min":0.28199999999999997,"5min":0.247,"15min":0.23799999999999999},"max":{"1min":14.522,"5min":62.981000000000002,"15min":141.87299999999999},"last":0.80600000000000005},{"interface":"front","average":{"1min":3.4409999999999998,"5min":3.177,"15min":3.2320000000000002},"min":{"1min":0.25700000000000001,"5min":0.25,"15min":0.24299999999999999},"max":{"1min":9.2240000000000002,"5min":63.725999999999999,"15min":141.904},"last":4.3109999999999999}]},{"osd":2,"last update":"Mon Aug 28 02:39:16 2023","interfaces":[{"interface":"back","average":{"1min":2.8010000000000002,"5min":4.4560000000000004,"15min":4.9630000000000001},"min":{"1min":0.32400000000000001,"5min":0.255,"15min":0.255},"max":{"1min":9.1980000000000004,"5min":149.321,"15min":234.876},"last":0.755},{"interface":"front","average":{"1min":2.6019999999999999,"5min":4.3760000000000003,"15min":4.8620000000000001},"min":{"1min":0.33500000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":7.8739999999999997,"5min":149.34,"15min":234.482},"last":0.5}]},{"osd":3,"last update":"Mon Aug 28 02:39:26 2023","interfaces":[{"interface":"back","average":{"1min":8.2200000000000006,"5min":5.734,"15min":4.6070000000000002},"min":{"1min":0.35699999999999998,"5min":0.313,"15min":0.313},"max":{"1min":100.28100000000001,"5min":167.233,"15min":167.233},"last":4.6379999999999999},{"interface":"front","average":{"1min":8.0600000000000005,"5min":5.7530000000000001,"15min":4.556},"min":{"1min":0.39600000000000002,"5min":0.373,"15min":0.27400000000000002},"max":{"1min":100.203,"5min":167.25,"15min":167.25},"last":4.6280000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:39:30 2023","interfaces":[{"interface":"back","average":{"1min":2.5129999999999999,"5min":0.95399999999999996,"15min":1.0600000000000001},"min":{"1min":0.27700000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":9.6519999999999992,"5min":9.6519999999999992,"15min":18.308},"last":0.40600000000000003},{"interface":"front","average":{"1min":4.1130000000000004,"5min":1.667,"15min":1.319},"min":{"1min":0.26400000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":26.465,"5min":26.465,"15min":26.465},"last":1.04}]},{"osd":5,"last update":"Mon Aug 28 02:39:30 2023","interfaces":[{"interface":"back","average":{"1min":3.9510000000000001,"5min":1.4610000000000001,"15min":1.155},"min":{"1min":0.32400000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":16.484999999999999,"5min":17.419,"15min":17.419},"last":0.70399999999999996},{"interface":"front","average":{"1min":2.4510000000000001,"5min":0.90600000000000003,"15min":0.98899999999999999},"min":{"1min":0.39000000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":9.6940000000000008,"5min":9.6940000000000008,"15min":9.6940000000000008},"last":3.1080000000000001}]},{"osd":7,"last update":"Mon Aug 28 02:39:00 2023","interfaces":[{"interface":"back","average":{"1min":1.5429999999999999,"5min":0.89600000000000002,"15min":0.66900000000000004},"min":{"1min":0.46300000000000002,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":10.723000000000001,"5min":10.723000000000001,"15min":10.723000000000001},"last":0.72999999999999998},{"interface":"front","average":{"1min":1.6020000000000001,"5min":0.77400000000000002,"15min":0.59599999999999997},"min":{"1min":0.40699999999999997,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":7.75,"5min":7.75,"15min":7.75},"last":2.52}]}]},{"osd":1,"up_from":98,"seq":420906795247,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":602256,"kb_used_data":37640,"kb_used_omap":7,"kb_used_meta":564600,"kb_avail":93142896,"statfs":{"total":95995035648,"available":95378325504,"internally_reserved":0,"allocated":38543360,"data_stored":36688560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":578150731},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:39:23 2023","interfaces":[{"interface":"back","average":{"1min":2.6099999999999999,"5min":4.5140000000000002,"15min":4.8399999999999999},"min":{"1min":0.216,"5min":0.185,"15min":0.17499999999999999},"max":{"1min":9.0220000000000002,"5min":114.56999999999999,"15min":129.346},"last":3.1269999999999998},{"interface":"front","average":{"1min":2.6779999999999999,"5min":4.5759999999999996,"15min":4.8380000000000001},"min":{"1min":0.22500000000000001,"5min":0.19800000000000001,"15min":0.188},"max":{"1min":10.912000000000001,"5min":114.402,"15min":128.685},"last":4.7489999999999997}]},{"osd":2,"last update":"Mon Aug 28 02:38:55 2023","interfaces":[{"interface":"back","average":{"1min":4.8319999999999999,"5min":4.4290000000000003,"15min":4.5640000000000001},"min":{"1min":0.24399999999999999,"5min":0.189,"15min":0.189},"max":{"1min":25.283000000000001,"5min":119.276,"15min":135.25999999999999},"last":0.40500000000000003},{"interface":"front","average":{"1min":4.3819999999999997,"5min":3.734,"15min":4.1619999999999999},"min":{"1min":0.23000000000000001,"5min":0.221,"15min":0.218},"max":{"1min":24.850999999999999,"5min":121.849,"15min":135.10300000000001},"last":3.1669999999999998}]},{"osd":3,"last update":"Mon Aug 28 02:39:23 2023","interfaces":[{"interface":"back","average":{"1min":4.5049999999999999,"5min":4.2279999999999998,"15min":4.9009999999999998},"min":{"1min":0.27300000000000002,"5min":0.21299999999999999,"15min":0.19700000000000001},"max":{"1min":16.239999999999998,"5min":126.95699999999999,"15min":126.95699999999999},"last":0.48699999999999999},{"interface":"front","average":{"1min":5.859,"5min":4.532,"15min":5.0800000000000001},"min":{"1min":0.23999999999999999,"5min":0.217,"15min":0.217},"max":{"1min":17.332999999999998,"5min":128.00200000000001,"15min":128.00200000000001},"last":0.73999999999999999}]},{"osd":4,"last update":"Mon Aug 28 02:39:23 2023","interfaces":[{"interface":"back","average":{"1min":4.3819999999999997,"5min":2.5510000000000002,"15min":2.0150000000000001},"min":{"1min":0.29799999999999999,"5min":0.28199999999999997,"15min":0.27500000000000002},"max":{"1min":20.731000000000002,"5min":24.809999999999999,"15min":24.809999999999999},"last":4.8330000000000002},{"interface":"front","average":{"1min":3.3769999999999998,"5min":2.0830000000000002,"15min":1.748},"min":{"1min":0.34100000000000003,"5min":0.29499999999999998,"15min":0.29399999999999998},"max":{"1min":19.187999999999999,"5min":24.835999999999999,"15min":24.835999999999999},"last":3.2090000000000001}]},{"osd":5,"last update":"Mon Aug 28 02:39:23 2023","interfaces":[{"interface":"back","average":{"1min":2.8940000000000001,"5min":3.3839999999999999,"15min":2.3919999999999999},"min":{"1min":0.32300000000000001,"5min":0.252,"15min":0.252},"max":{"1min":11.323,"5min":82.722999999999999,"15min":82.722999999999999},"last":4.7889999999999997},{"interface":"front","average":{"1min":3.9159999999999999,"5min":3.9380000000000002,"15min":2.4750000000000001},"min":{"1min":0.33000000000000002,"5min":0.315,"15min":0.307},"max":{"1min":13.667999999999999,"5min":82.733999999999995,"15min":82.733999999999995},"last":4.8639999999999999}]},{"osd":6,"last update":"Mon Aug 28 02:39:23 2023","interfaces":[{"interface":"back","average":{"1min":2.8889999999999998,"5min":2.4660000000000002,"15min":2.113},"min":{"1min":0.36099999999999999,"5min":0.33600000000000002,"15min":0.311},"max":{"1min":9.7059999999999995,"5min":19.527000000000001,"15min":19.527000000000001},"last":0.52100000000000002},{"interface":"front","average":{"1min":4.1219999999999999,"5min":3.036,"15min":2.2610000000000001},"min":{"1min":0.35699999999999998,"5min":0.34899999999999998,"15min":0.32200000000000001},"max":{"1min":13.233000000000001,"5min":19.516999999999999,"15min":19.516999999999999},"last":6.0940000000000003}]},{"osd":7,"last update":"Mon Aug 28 02:39:04 2023","interfaces":[{"interface":"back","average":{"1min":2.9990000000000001,"5min":1.6679999999999999,"15min":1.163},"min":{"1min":0.42699999999999999,"5min":0.38,"15min":0.38},"max":{"1min":10.872,"5min":10.872,"15min":10.872},"last":5.2839999999999998},{"interface":"front","average":{"1min":2.3530000000000002,"5min":1.528,"15min":1.05},"min":{"1min":0.42199999999999999,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":7.5730000000000004,"5min":9.4900000000000002,"15min":9.4900000000000002},"last":5.0860000000000003}]}]},{"osd":0,"up_from":90,"seq":386547056892,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1013016,"kb_used_data":348816,"kb_used_omap":8,"kb_used_meta":664183,"kb_avail":92732136,"statfs":{"total":95995035648,"available":94957707264,"internally_reserved":0,"allocated":357187584,"data_stored":355500006,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":680123753},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Aug 28 02:38:28 2023","interfaces":[{"interface":"back","average":{"1min":7.5039999999999996,"5min":6.1029999999999998,"15min":4.1689999999999996},"min":{"1min":0.248,"5min":0.16500000000000001,"15min":0.16500000000000001},"max":{"1min":60.268999999999998,"5min":150.00700000000001,"15min":150.00700000000001},"last":1.865},{"interface":"front","average":{"1min":5.0529999999999999,"5min":5.8700000000000001,"15min":4.4550000000000001},"min":{"1min":0.26300000000000001,"5min":0.19500000000000001,"15min":0.17699999999999999},"max":{"1min":55.457000000000001,"5min":150.27699999999999,"15min":150.27699999999999},"last":3.6850000000000001}]},{"osd":2,"last update":"Mon Aug 28 02:38:53 2023","interfaces":[{"interface":"back","average":{"1min":3.3029999999999999,"5min":4.4219999999999997,"15min":3.9590000000000001},"min":{"1min":0.216,"5min":0.20999999999999999,"15min":0.19500000000000001},"max":{"1min":8.6110000000000007,"5min":86.269000000000005,"15min":100.718},"last":0.44700000000000001},{"interface":"front","average":{"1min":2.9300000000000002,"5min":4.9459999999999997,"15min":4.532},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.20000000000000001},"max":{"1min":14.18,"5min":86.353999999999999,"15min":100.962},"last":3.621}]},{"osd":3,"last update":"Mon Aug 28 02:38:35 2023","interfaces":[{"interface":"back","average":{"1min":3.9249999999999998,"5min":6.2640000000000002,"15min":5.9950000000000001},"min":{"1min":0.26700000000000002,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":9.9339999999999993,"5min":196.702,"15min":237.43899999999999},"last":2.7160000000000002},{"interface":"front","average":{"1min":4.194,"5min":6.5119999999999996,"15min":5.9800000000000004},"min":{"1min":0.28100000000000003,"5min":0.26300000000000001,"15min":0.24399999999999999},"max":{"1min":16.827999999999999,"5min":197.75399999999999,"15min":237.48099999999999},"last":7.2039999999999997}]},{"osd":4,"last update":"Mon Aug 28 02:38:28 2023","interfaces":[{"interface":"back","average":{"1min":12.749000000000001,"5min":3.8300000000000001,"15min":2.4940000000000002},"min":{"1min":0.29299999999999998,"5min":0.25700000000000001,"15min":0.23999999999999999},"max":{"1min":223.64599999999999,"5min":223.64599999999999,"15min":223.64599999999999},"last":2.6970000000000001},{"interface":"front","average":{"1min":2.335,"5min":2.5209999999999999,"15min":2.2240000000000002},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.309},"max":{"1min":9.9809999999999999,"5min":14.241,"15min":27.271999999999998},"last":2.6749999999999998}]},{"osd":5,"last update":"Mon Aug 28 02:38:28 2023","interfaces":[{"interface":"back","average":{"1min":3.125,"5min":2.5339999999999998,"15min":2.327},"min":{"1min":0.34999999999999998,"5min":0.29899999999999999,"15min":0.29599999999999999},"max":{"1min":10.911,"5min":11.429,"15min":44.000999999999998},"last":7.2450000000000001},{"interface":"front","average":{"1min":2.6629999999999998,"5min":1.764,"15min":1.982},"min":{"1min":0.316,"5min":0.308,"15min":0.307},"max":{"1min":13.103,"5min":13.103,"15min":43.953000000000003},"last":2.9049999999999998}]},{"osd":6,"last update":"Mon Aug 28 02:38:28 2023","interfaces":[{"interface":"back","average":{"1min":3.504,"5min":2.5819999999999999,"15min":2.4220000000000002},"min":{"1min":0.374,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":16.777999999999999,"5min":16.777999999999999,"15min":32.417999999999999},"last":9.5410000000000004},{"interface":"front","average":{"1min":2.4689999999999999,"5min":1.847,"15min":2.0720000000000001},"min":{"1min":0.35299999999999998,"5min":0.33700000000000002,"15min":0.30399999999999999},"max":{"1min":7.8419999999999996,"5min":9.9049999999999994,"15min":32.130000000000003},"last":5.6349999999999998}]},{"osd":7,"last update":"Mon Aug 28 02:39:03 2023","interfaces":[{"interface":"back","average":{"1min":3.5870000000000002,"5min":1.9399999999999999,"15min":1.411},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":13.071,"5min":13.071,"15min":13.071},"last":6.9749999999999996},{"interface":"front","average":{"1min":2.3519999999999999,"5min":1.8149999999999999,"15min":1.889},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":12.121,"5min":16.529,"15min":16.529},"last":3.6589999999999998}]}]},{"osd":2,"up_from":0,"seq":446676599016,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":112,"seq":481036337378,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":707904,"kb_used_data":88824,"kb_used_omap":3,"kb_used_meta":619068,"kb_avail":93037248,"statfs":{"total":95995035648,"available":95270141952,"internally_reserved":0,"allocated":90955776,"data_stored":89284519,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":633926562},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:39:27 2023","interfaces":[{"interface":"back","average":{"1min":5.1500000000000004,"5min":5.0119999999999996,"15min":4.9029999999999996},"min":{"1min":0.14999999999999999,"5min":0.14999999999999999,"15min":0.14999999999999999},"max":{"1min":59.256999999999998,"5min":138.39699999999999,"15min":164.25299999999999},"last":59.256999999999998},{"interface":"front","average":{"1min":5.016,"5min":3.4529999999999998,"15min":4.1109999999999998},"min":{"1min":0.20200000000000001,"5min":0.20100000000000001,"15min":0.192},"max":{"1min":59.341999999999999,"5min":59.341999999999999,"15min":164.55000000000001},"last":59.341999999999999}]},{"osd":1,"last update":"Mon Aug 28 02:39:27 2023","interfaces":[{"interface":"back","average":{"1min":2.242,"5min":4.2729999999999997,"15min":2.8620000000000001},"min":{"1min":0.214,"5min":0.191,"15min":0.191},"max":{"1min":6.7160000000000002,"5min":105.04600000000001,"15min":105.04600000000001},"last":0.38300000000000001},{"interface":"front","average":{"1min":2.5800000000000001,"5min":6.3940000000000001,"15min":3.7090000000000001},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":8.4659999999999993,"5min":143.54400000000001,"15min":143.54400000000001},"last":0.40100000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:39:27 2023","interfaces":[{"interface":"back","average":{"1min":1.6970000000000001,"5min":4.9370000000000003,"15min":4.3819999999999997},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.19900000000000001},"max":{"1min":11.18,"5min":143.50299999999999,"15min":143.50299999999999},"last":0.439},{"interface":"front","average":{"1min":1.403,"5min":3.0939999999999999,"15min":3.3969999999999998},"min":{"1min":0.374,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":4.5720000000000001,"5min":42.229999999999997,"15min":120.44499999999999},"last":0.41999999999999998}]},{"osd":4,"last update":"Mon Aug 28 02:39:27 2023","interfaces":[{"interface":"back","average":{"1min":1.9990000000000001,"5min":3.8069999999999999,"15min":2.5990000000000002},"min":{"1min":0.316,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":6.0030000000000001,"5min":158.739,"15min":158.739},"last":0.54600000000000004},{"interface":"front","average":{"1min":2.8879999999999999,"5min":3.504,"15min":2.7069999999999999},"min":{"1min":0.34499999999999997,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":7.46,"5min":158.72,"15min":158.72},"last":0.56299999999999994}]},{"osd":5,"last update":"Mon Aug 28 02:39:27 2023","interfaces":[{"interface":"back","average":{"1min":2.4169999999999998,"5min":3.4910000000000001,"15min":2.8809999999999998},"min":{"1min":0.316,"5min":0.315,"15min":0.29099999999999998},"max":{"1min":6.1980000000000004,"5min":163.55199999999999,"15min":163.55199999999999},"last":0.53600000000000003},{"interface":"front","average":{"1min":2.2669999999999999,"5min":5.5049999999999999,"15min":3.6059999999999999},"min":{"1min":0.41399999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":8.3940000000000001,"5min":163.45699999999999,"15min":163.45699999999999},"last":0.55500000000000005}]},{"osd":6,"last update":"Mon Aug 28 02:39:27 2023","interfaces":[{"interface":"back","average":{"1min":2.5649999999999999,"5min":3.8490000000000002,"15min":2.581},"min":{"1min":0.371,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":11.324999999999999,"5min":138.37,"15min":138.37},"last":0.47199999999999998},{"interface":"front","average":{"1min":1.9910000000000001,"5min":2.0979999999999999,"15min":1.746},"min":{"1min":0.34899999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":4.4980000000000002,"5min":12.193,"15min":12.193},"last":0.50900000000000001}]},{"osd":7,"last update":"Mon Aug 28 02:39:08 2023","interfaces":[{"interface":"back","average":{"1min":3.6000000000000001,"5min":2.2240000000000002,"15min":1.5109999999999999},"min":{"1min":0.45400000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":9.2530000000000001,"5min":9.2530000000000001,"15min":9.2530000000000001},"last":0.498},{"interface":"front","average":{"1min":2.661,"5min":1.9570000000000001,"15min":1.51},"min":{"1min":0.47599999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":9.3490000000000002,"5min":12.226000000000001,"15min":12.226000000000001},"last":0.57399999999999995}]}]},{"osd":4,"up_from":30,"seq":128849019252,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":633532,"kb_used_data":39732,"kb_used_omap":0,"kb_used_meta":593792,"kb_avail":93111620,"statfs":{"total":95995035648,"available":95346298880,"internally_reserved":0,"allocated":40685568,"data_stored":39763363,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":608043008},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:39:31 2023","interfaces":[{"interface":"back","average":{"1min":1.9470000000000001,"5min":6.4779999999999998,"15min":6.4930000000000003},"min":{"1min":0.23599999999999999,"5min":0.23100000000000001,"15min":0.219},"max":{"1min":7.4059999999999997,"5min":168.75800000000001,"15min":168.75800000000001},"last":0.23599999999999999},{"interface":"front","average":{"1min":1.97,"5min":6.5789999999999997,"15min":6.4169999999999998},"min":{"1min":0.27300000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":7.8339999999999996,"5min":168.84200000000001,"15min":168.84200000000001},"last":1.097}]},{"osd":1,"last update":"Mon Aug 28 02:39:31 2023","interfaces":[{"interface":"back","average":{"1min":2.1589999999999998,"5min":6.6820000000000004,"15min":3.5640000000000001},"min":{"1min":0.23699999999999999,"5min":0.23300000000000001,"15min":0.221},"max":{"1min":7.4470000000000001,"5min":131.845,"15min":131.845},"last":3.9399999999999999},{"interface":"front","average":{"1min":2.5659999999999998,"5min":6.2910000000000004,"15min":3.3149999999999999},"min":{"1min":0.33000000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":12.254,"5min":132.304,"15min":132.304},"last":1.665}]},{"osd":2,"last update":"Mon Aug 28 02:38:55 2023","interfaces":[{"interface":"back","average":{"1min":3.3399999999999999,"5min":3.4209999999999998,"15min":4.3410000000000002},"min":{"1min":0.32000000000000001,"5min":0.26600000000000001,"15min":0.255},"max":{"1min":9.8130000000000006,"5min":133.06399999999999,"15min":133.06399999999999},"last":3.8210000000000002},{"interface":"front","average":{"1min":2.3690000000000002,"5min":3.585,"15min":4.2809999999999997},"min":{"1min":0.27900000000000003,"5min":0.27700000000000002,"15min":0.27300000000000002},"max":{"1min":8.8689999999999998,"5min":133.08699999999999,"15min":133.08699999999999},"last":1.1850000000000001}]},{"osd":3,"last update":"Mon Aug 28 02:39:31 2023","interfaces":[{"interface":"back","average":{"1min":3.7120000000000002,"5min":5.5049999999999999,"15min":6.6509999999999998},"min":{"1min":0.317,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":17.178999999999998,"5min":150.958,"15min":203.583},"last":0.375},{"interface":"front","average":{"1min":3.4340000000000002,"5min":5.5380000000000003,"15min":6.7229999999999999},"min":{"1min":0.318,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":11.942,"5min":150.63499999999999,"15min":203.41399999999999},"last":3.911}]},{"osd":5,"last update":"Mon Aug 28 02:39:07 2023","interfaces":[{"interface":"back","average":{"1min":1.907,"5min":0.96999999999999997,"15min":1.52},"min":{"1min":0.28799999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":8.8330000000000002,"5min":8.8330000000000002,"15min":124.718},"last":1.1319999999999999},{"interface":"front","average":{"1min":2.512,"5min":1.1890000000000001,"15min":1.7749999999999999},"min":{"1min":0.33100000000000002,"5min":0.26500000000000001,"15min":0.24399999999999999},"max":{"1min":11.632,"5min":11.632,"15min":124.694},"last":0.34399999999999997}]},{"osd":6,"last update":"Mon Aug 28 02:39:31 2023","interfaces":[{"interface":"back","average":{"1min":2.7330000000000001,"5min":2.7469999999999999,"15min":1.722},"min":{"1min":0.29499999999999998,"5min":0.28499999999999998,"15min":0.25},"max":{"1min":9.8979999999999997,"5min":148.87100000000001,"15min":148.87100000000001},"last":0.30099999999999999},{"interface":"front","average":{"1min":2.2490000000000001,"5min":2.8530000000000002,"15min":1.613},"min":{"1min":0.32100000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":9.8559999999999999,"5min":148.88300000000001,"15min":148.88300000000001},"last":1.157}]},{"osd":7,"last update":"Mon Aug 28 02:39:01 2023","interfaces":[{"interface":"back","average":{"1min":2.0600000000000001,"5min":0.98499999999999999,"15min":0.67400000000000004},"min":{"1min":0.31900000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.5519999999999996,"5min":7.5519999999999996,"15min":7.5519999999999996},"last":3.8700000000000001},{"interface":"front","average":{"1min":1.7070000000000001,"5min":1.004,"15min":0.72599999999999998},"min":{"1min":0.373,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":5.3390000000000004,"5min":7.117,"15min":7.117},"last":0.49399999999999999}]}]},{"osd":5,"up_from":36,"seq":154618823024,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":666604,"kb_used_data":76836,"kb_used_omap":0,"kb_used_meta":589696,"kb_avail":93078548,"statfs":{"total":95995035648,"available":95312433152,"internally_reserved":0,"allocated":78680064,"data_stored":77769400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603848704},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:39:19 2023","interfaces":[{"interface":"back","average":{"1min":3.9079999999999999,"5min":3.0419999999999998,"15min":4.1159999999999997},"min":{"1min":0.22,"5min":0.216,"15min":0.214},"max":{"1min":55.448999999999998,"5min":64.590999999999994,"15min":128.72800000000001},"last":6.5410000000000004},{"interface":"front","average":{"1min":4.7510000000000003,"5min":3.504,"15min":4.4669999999999996},"min":{"1min":0.26400000000000001,"5min":0.245,"15min":0.24199999999999999},"max":{"1min":55.436999999999998,"5min":64.578999999999994,"15min":128.75200000000001},"last":6.9409999999999998}]},{"osd":1,"last update":"Mon Aug 28 02:38:31 2023","interfaces":[{"interface":"back","average":{"1min":2.383,"5min":7.548,"15min":4.0209999999999999},"min":{"1min":0.28199999999999997,"5min":0.22800000000000001,"15min":0.216},"max":{"1min":9.5579999999999998,"5min":148.01400000000001,"15min":148.01400000000001},"last":2.391},{"interface":"front","average":{"1min":2.7349999999999999,"5min":7.7300000000000004,"15min":4.2389999999999999},"min":{"1min":0.32000000000000001,"5min":0.26400000000000001,"15min":0.23200000000000001},"max":{"1min":10.984999999999999,"5min":148.08000000000001,"15min":148.08000000000001},"last":7.0590000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:39:02 2023","interfaces":[{"interface":"back","average":{"1min":1.7090000000000001,"5min":2.4060000000000001,"15min":4.4210000000000003},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":8.5299999999999994,"5min":78.834000000000003,"15min":176.69499999999999},"last":6.5839999999999996},{"interface":"front","average":{"1min":2.7789999999999999,"5min":3.3170000000000002,"15min":4.9359999999999999},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.27600000000000002},"max":{"1min":8.8900000000000006,"5min":78.798000000000002,"15min":176.71199999999999},"last":7.0919999999999996}]},{"osd":3,"last update":"Mon Aug 28 02:38:31 2023","interfaces":[{"interface":"back","average":{"1min":2.3860000000000001,"5min":3.2309999999999999,"15min":4.1539999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.26100000000000001},"max":{"1min":8.0749999999999993,"5min":87.233999999999995,"15min":128.571},"last":6.5629999999999997},{"interface":"front","average":{"1min":3.274,"5min":3.2240000000000002,"15min":4.4139999999999997},"min":{"1min":0.41199999999999998,"5min":0.28699999999999998,"15min":0.26200000000000001},"max":{"1min":16.564,"5min":87.337000000000003,"15min":128.684},"last":7.1189999999999998}]},{"osd":4,"last update":"Mon Aug 28 02:39:12 2023","interfaces":[{"interface":"back","average":{"1min":1.768,"5min":0.83699999999999997,"15min":1.3320000000000001},"min":{"1min":0.27800000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":11.206,"5min":11.206,"15min":85.769999999999996},"last":2.2589999999999999},{"interface":"front","average":{"1min":1.7270000000000001,"5min":0.81699999999999995,"15min":1.3049999999999999},"min":{"1min":0.318,"5min":0.27100000000000002,"15min":0.248},"max":{"1min":9.4930000000000003,"5min":9.4930000000000003,"15min":85.825000000000003},"last":6.4989999999999997}]},{"osd":6,"last update":"Mon Aug 28 02:38:31 2023","interfaces":[{"interface":"back","average":{"1min":1.3360000000000001,"5min":0.67300000000000004,"15min":1.034},"min":{"1min":0.31,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":5.6150000000000002,"5min":5.6150000000000002,"15min":14.803000000000001},"last":7.0330000000000004},{"interface":"front","average":{"1min":1.0629999999999999,"5min":0.70899999999999996,"15min":0.99099999999999999},"min":{"1min":0.35199999999999998,"5min":0.29599999999999999,"15min":0.28799999999999998},"max":{"1min":8.9480000000000004,"5min":8.9480000000000004,"15min":8.9480000000000004},"last":6.9809999999999999}]},{"osd":7,"last update":"Mon Aug 28 02:39:02 2023","interfaces":[{"interface":"back","average":{"1min":2.2389999999999999,"5min":1.0760000000000001,"15min":0.76000000000000001},"min":{"1min":0.40500000000000003,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":11.699,"5min":11.699,"15min":11.699},"last":7.0090000000000003},{"interface":"front","average":{"1min":1.9370000000000001,"5min":1.024,"15min":0.81000000000000005},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":8.9199999999999999,"5min":8.9199999999999999,"15min":8.9199999999999999},"last":6.9089999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8527,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":336068608,"data_stored":336068608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":17170432,"data_stored":17170432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":80154624,"data_stored":80150552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":85524480,"data_stored":85524480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":35520512,"data_stored":35520512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":56823808,"data_stored":56819736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":147591168,"data_stored":147587096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":17235968,"data_stored":17235968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-08-28T02:39:37.455 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-08-28T02:39:37.456 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd pool set device_health_metrics pg_num 61 2023-08-28T02:39:37.456 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:39:37.482 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:37.530 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:39:37.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:37.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:39:37.637 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:39:37.666 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:37.721 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:39:37.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:37.725 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:39:37.825 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:39:37.852 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:37.899 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:39:37.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:37.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:39:38.009 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:39:38.012 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:39:38.043 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:38.142 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:39:38.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:38.145 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:39:38.246 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:39:38.272 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:38.315 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:39:38.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:38.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:39:38.419 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:39:38.446 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:38.491 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:39:38.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:38.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:39:38.595 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:39:38.616 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:39:38.622 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:38.669 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:39:38.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:38.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:39:38.775 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:39:38.802 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:38.839 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is set 2023-08-28T02:39:38.850 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:39:38.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:38.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:39:38.944 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:38 smithi143 ceph-mon[183059]: pgmap v996: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 19 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 567 op/s; 3523/789 objects degraded (446.515%); 4663/789 objects misplaced (591.001%); 0 B/s, 77 objects/s recovering 2023-08-28T02:39:38.944 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:38 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2336609994' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:39:38.944 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:38 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:39:38.944 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:38 smithi143 ceph-mon[189871]: pgmap v996: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 19 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 567 op/s; 3523/789 objects degraded (446.515%); 4663/789 objects misplaced (591.001%); 0 B/s, 77 objects/s recovering 2023-08-28T02:39:38.944 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:38 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2336609994' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:39:38.944 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:38 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:39:38.954 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:39:38.984 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:38.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:38 smithi169 ceph-mon[159415]: pgmap v996: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 19 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 567 op/s; 3523/789 objects degraded (446.515%); 4663/789 objects misplaced (591.001%); 0 B/s, 77 objects/s recovering 2023-08-28T02:39:38.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:38 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2336609994' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:39:38.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:38 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:39:39.316 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:39:39.316 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [], 2023-08-28T02:39:39.316 INFO:teuthology.orchestra.run.smithi143.stdout: "num_ops": 0 2023-08-28T02:39:39.316 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:39:39.786 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:39:39.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:39.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:39:39.889 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:39:39.918 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:40.106 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:39 smithi143 ceph-mon[183059]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:39:40.106 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:39 smithi143 ceph-mon[183059]: failed to encode map e362 with expected crc 2023-08-28T02:39:40.106 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:39 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:39:40.106 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:39 smithi143 ceph-mon[183059]: osdmap e362: 8 total, 8 up, 7 in 2023-08-28T02:39:40.106 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:39 smithi143 ceph-mon[183059]: failed to encode map e362 with expected crc 2023-08-28T02:39:40.106 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:39 smithi143 ceph-mon[189871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:39:40.106 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:39 smithi143 ceph-mon[189871]: failed to encode map e362 with expected crc 2023-08-28T02:39:40.106 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:39 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:39:40.106 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:39 smithi143 ceph-mon[189871]: osdmap e362: 8 total, 8 up, 7 in 2023-08-28T02:39:40.107 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:39 smithi143 ceph-mon[189871]: failed to encode map e362 with expected crc 2023-08-28T02:39:40.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:39 smithi169 ceph-mon[159415]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:39:40.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:39 smithi169 ceph-mon[159415]: failed to encode map e362 with expected crc 2023-08-28T02:39:40.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:39 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:39:40.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:39 smithi169 ceph-mon[159415]: osdmap e362: 8 total, 8 up, 7 in 2023-08-28T02:39:40.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:39 smithi169 ceph-mon[159415]: failed to encode map e362 with expected crc 2023-08-28T02:39:40.463 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:39:40.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:40.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:39:40.566 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:39:40.594 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:40.806 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.3 dump_blocked_ops 2023-08-28T02:39:40.943 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:39:40.972 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:39:40.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:40.981 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:39:41.081 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:39:41.114 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:40 smithi143 ceph-mon[183059]: pgmap v998: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 4.6 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 442 op/s; 3523/84 objects degraded (4194.048%); 4549/84 objects misplaced (5415.476%); 0 B/s, 77 objects/s recovering 2023-08-28T02:39:41.114 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:40 smithi143 ceph-mon[189871]: pgmap v998: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 4.6 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 442 op/s; 3523/84 objects degraded (4194.048%); 4549/84 objects misplaced (5415.476%); 0 B/s, 77 objects/s recovering 2023-08-28T02:39:41.117 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:41.165 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:39:41.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:41.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:39:41.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:40 smithi169 ceph-mon[159415]: pgmap v998: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 4.6 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 442 op/s; 3523/84 objects degraded (4194.048%); 4549/84 objects misplaced (5415.476%); 0 B/s, 77 objects/s recovering 2023-08-28T02:39:41.268 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:39:41.295 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:41.338 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:39:41.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:41.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:39:41.445 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:39:41.478 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:41.625 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:39:41.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:41.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:39:41.728 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:39:41.729 INFO:teuthology.orchestra.run.smithi143.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-08-28T02:39:41.756 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:41.799 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:39:41.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:41.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:39:41.902 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:39:41.928 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:41.973 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:39:41.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:41.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:39:42.073 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:39:41 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:39:41] "GET /metrics HTTP/1.1" 200 36556 "" "Prometheus/2.33.4" 2023-08-28T02:39:42.079 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:39:42.106 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:42.127 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset noscrub 2023-08-28T02:39:42.151 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:39:42.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:42.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:39:42.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:42 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:39:42.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:42 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:39:42.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:42 smithi169 ceph-mon[159415]: pgmap v999: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 4.6 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 366 op/s; 3523/84 objects degraded (4194.048%); 4549/84 objects misplaced (5415.476%); 0 B/s, 77 objects/s recovering 2023-08-28T02:39:42.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:42 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 3523/84 objects degraded (4194.048%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:39:42.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:42 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/5131509' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "61"}]: dispatch 2023-08-28T02:39:42.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:42 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "61"}]: dispatch 2023-08-28T02:39:42.259 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:39:42.285 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:42.322 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:42 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:39:42.322 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:42 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:39:42.322 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:42 smithi143 ceph-mon[183059]: pgmap v999: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 4.6 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 366 op/s; 3523/84 objects degraded (4194.048%); 4549/84 objects misplaced (5415.476%); 0 B/s, 77 objects/s recovering 2023-08-28T02:39:42.322 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:42 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 3523/84 objects degraded (4194.048%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:39:42.323 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:42 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/5131509' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "61"}]: dispatch 2023-08-28T02:39:42.323 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:42 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "61"}]: dispatch 2023-08-28T02:39:42.323 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:42 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:39:42.323 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:42 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:39:42.323 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:42 smithi143 ceph-mon[189871]: pgmap v999: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 4.6 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 366 op/s; 3523/84 objects degraded (4194.048%); 4549/84 objects misplaced (5415.476%); 0 B/s, 77 objects/s recovering 2023-08-28T02:39:42.323 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:42 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 3523/84 objects degraded (4194.048%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:39:42.323 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:42 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/5131509' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "61"}]: dispatch 2023-08-28T02:39:42.323 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:42 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "61"}]: dispatch 2023-08-28T02:39:42.682 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:39:42.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:42.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:39:42.786 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:39:42.821 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:42.846 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-08-28T02:39:42.846 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2023-08-28T02:39:42.847 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2023-08-28T02:39:42.876 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:39:43.036 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:39:43.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:43.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:39:43.141 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:39:43.167 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:43.183 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:39:43.215 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:39:43.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:43.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:39:43.297 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:39:43.320 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:39:43.321 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:39:43.342 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:39:43.351 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:43.364 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:39:43.389 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:39:43.407 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:39:43.424 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:39:43.441 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:39:43.443 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:39:43.443 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:39:43.443 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:39:43.443 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:39:43.443 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:39:43.443 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:39:43.443 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:39:43.444 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:39:43.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:43 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3558717215' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:39:43.524 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:43 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3558717215' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:39:43.525 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:43 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3558717215' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:39:43.751 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:39:43.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:43.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:39:43.855 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:39:43.881 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:43.926 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:39:43.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:43.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:39:44.031 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:39:44.060 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:44.626 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:44 smithi143 ceph-mon[183059]: pgmap v1000: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 4.6 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 211 op/s; 3523/84 objects degraded (4194.048%); 4549/84 objects misplaced (5415.476%); 0 B/s, 160 objects/s recovering 2023-08-28T02:39:44.626 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:44 smithi143 ceph-mon[189871]: pgmap v1000: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 4.6 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 211 op/s; 3523/84 objects degraded (4194.048%); 4549/84 objects misplaced (5415.476%); 0 B/s, 160 objects/s recovering 2023-08-28T02:39:44.707 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:39:44.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:44.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:39:44.737 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:44 smithi169 ceph-mon[159415]: pgmap v1000: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 4.6 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 211 op/s; 3523/84 objects degraded (4194.048%); 4549/84 objects misplaced (5415.476%); 0 B/s, 160 objects/s recovering 2023-08-28T02:39:44.811 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:39:44.839 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:44.883 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:39:44.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:44.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:39:44.992 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:39:45.018 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:45.063 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:39:45.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:45.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:39:45.170 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:39:45.197 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:45.426 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:39:45.431 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:39:45.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:45.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:39:45.537 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:39:45.564 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:45.605 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:39:45.606 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:39:45.608 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:39:45.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:45.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:39:45.717 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:39:45.743 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:45.788 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:39:45.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:45.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:39:45.893 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:39:45.925 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:46.449 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:39:46.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:46.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:39:46.554 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:39:46.558 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.3 dump_historic_ops 2023-08-28T02:39:46.580 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:46.627 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:39:46.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:46.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:39:46.644 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:39:46.733 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:39:46.760 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:46.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:46 smithi169 ceph-mon[159415]: pgmap v1001: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 4.6 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 62 op/s; 3523/84 objects degraded (4194.048%); 4549/84 objects misplaced (5415.476%); 0 B/s, 160 objects/s recovering 2023-08-28T02:39:47.111 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:39:47.113 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:46 smithi143 ceph-mon[183059]: pgmap v1001: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 4.6 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 62 op/s; 3523/84 objects degraded (4194.048%); 4549/84 objects misplaced (5415.476%); 0 B/s, 160 objects/s recovering 2023-08-28T02:39:47.113 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:46 smithi143 ceph-mon[189871]: pgmap v1001: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 4.6 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 62 op/s; 3523/84 objects degraded (4194.048%); 4549/84 objects misplaced (5415.476%); 0 B/s, 160 objects/s recovering 2023-08-28T02:39:47.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:47.114 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:39:47.215 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:39:47.249 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:47.346 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:39:47.429 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:39:47.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:47.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:39:47.532 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:39:47.559 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:47.631 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:39:47.706 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:39:47.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:47.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:39:47.726 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is unset 2023-08-28T02:39:47.813 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:39:47.842 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:47.847 INFO:tasks.thrashosds.thrasher:in_osds: [0, 4, 5, 3, 6, 1, 7] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-08-28T02:39:47.847 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-08-28T02:39:47.847 INFO:tasks.thrashosds.thrasher:Adding osd 2 2023-08-28T02:39:47.847 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd in 2 2023-08-28T02:39:47.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:47 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1274806233' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:39:47.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:47 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:39:47.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:47 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1274806233' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:39:47.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:47 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:39:47.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:47 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1274806233' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:39:47.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:47 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:39:48.749 INFO:teuthology.orchestra.run.smithi143.stderr:pool 'unique_pool_3' removed 2023-08-28T02:39:48.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:48 smithi169 ceph-mon[159415]: pgmap v1002: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 4.6 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 23 op/s; 2595/84 objects degraded (3089.286%); 4549/84 objects misplaced (5415.476%); 0 B/s, 229 objects/s recovering 2023-08-28T02:39:48.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:48 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 2595/84 objects degraded (3089.286%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:39:48.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:48 smithi169 ceph-mon[159415]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:39:48.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:48 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:39:48.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:48 smithi169 ceph-mon[159415]: osdmap e363: 8 total, 8 up, 7 in 2023-08-28T02:39:48.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:48 smithi169 ceph-mon[159415]: failed to encode map e363 with expected crc 2023-08-28T02:39:48.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:48 smithi169 ceph-mon[159415]: failed to encode map e363 with expected crc 2023-08-28T02:39:48.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:48 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1033848834' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-08-28T02:39:48.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:48 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-08-28T02:39:49.049 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:39:49.051 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:48 smithi143 ceph-mon[183059]: pgmap v1002: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 4.6 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 23 op/s; 2595/84 objects degraded (3089.286%); 4549/84 objects misplaced (5415.476%); 0 B/s, 229 objects/s recovering 2023-08-28T02:39:49.051 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:48 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 2595/84 objects degraded (3089.286%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:39:49.051 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:48 smithi143 ceph-mon[183059]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:39:49.051 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:48 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:39:49.051 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:48 smithi143 ceph-mon[183059]: osdmap e363: 8 total, 8 up, 7 in 2023-08-28T02:39:49.051 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:48 smithi143 ceph-mon[183059]: failed to encode map e363 with expected crc 2023-08-28T02:39:49.051 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:48 smithi143 ceph-mon[183059]: failed to encode map e363 with expected crc 2023-08-28T02:39:49.052 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:48 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1033848834' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-08-28T02:39:49.052 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:48 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-08-28T02:39:49.052 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:48 smithi143 ceph-mon[189871]: pgmap v1002: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 77 active+clean; 4.6 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 23 op/s; 2595/84 objects degraded (3089.286%); 4549/84 objects misplaced (5415.476%); 0 B/s, 229 objects/s recovering 2023-08-28T02:39:49.052 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:48 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 2595/84 objects degraded (3089.286%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-08-28T02:39:49.052 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:48 smithi143 ceph-mon[189871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:39:49.052 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:48 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:39:49.052 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:48 smithi143 ceph-mon[189871]: osdmap e363: 8 total, 8 up, 7 in 2023-08-28T02:39:49.052 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:48 smithi143 ceph-mon[189871]: failed to encode map e363 with expected crc 2023-08-28T02:39:49.052 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:48 smithi143 ceph-mon[189871]: failed to encode map e363 with expected crc 2023-08-28T02:39:49.052 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:48 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1033848834' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-08-28T02:39:49.053 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:48 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-08-28T02:39:49.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:49.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:39:49.154 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:39:49.199 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:49.252 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:39:49.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:49.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:39:49.360 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:39:49.390 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:49.835 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:39:49.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:49.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:39:49.939 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:39:49.966 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:49.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:49 smithi169 ceph-mon[159415]: failed to encode map e364 with expected crc 2023-08-28T02:39:49.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:49 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-08-28T02:39:49.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:49 smithi169 ceph-mon[159415]: osdmap e364: 8 total, 8 up, 7 in 2023-08-28T02:39:49.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:49 smithi169 ceph-mon[159415]: failed to encode map e364 with expected crc 2023-08-28T02:39:49.994 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:49 smithi143 ceph-mon[183059]: failed to encode map e364 with expected crc 2023-08-28T02:39:49.994 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:49 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-08-28T02:39:49.994 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:49 smithi143 ceph-mon[183059]: osdmap e364: 8 total, 8 up, 7 in 2023-08-28T02:39:49.994 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:49 smithi143 ceph-mon[183059]: failed to encode map e364 with expected crc 2023-08-28T02:39:49.994 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:49 smithi143 ceph-mon[189871]: failed to encode map e364 with expected crc 2023-08-28T02:39:49.995 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:49 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-08-28T02:39:49.995 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:49 smithi143 ceph-mon[189871]: osdmap e364: 8 total, 8 up, 7 in 2023-08-28T02:39:49.995 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:49 smithi143 ceph-mon[189871]: failed to encode map e364 with expected crc 2023-08-28T02:39:50.008 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:39:50.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:50.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:39:50.113 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:39:50.138 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:50.180 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:39:50.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:50.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:39:50.270 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-08-28T02:39:50.271 INFO:tasks.radosbench:Beginning radosbench... 2023-08-28T02:39:50.271 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_4 2023-08-28T02:39:50.271 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd pool create unique_pool_4 16 2023-08-28T02:39:50.288 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:39:50.314 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:50.355 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:39:50.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:50.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:39:50.461 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:39:50.487 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:50.528 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:39:50.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:50.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:39:50.633 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:39:50.662 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:50.710 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:39:50.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:50.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:39:50.814 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:39:50.840 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:50.882 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:39:50.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:50.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:39:50.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:50 smithi143 ceph-mon[183059]: pgmap v1005: 73 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.6 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 16/84 objects degraded (19.048%) 2023-08-28T02:39:50.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:50 smithi143 ceph-mon[183059]: failed to encode map e365 with expected crc 2023-08-28T02:39:50.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:50 smithi143 ceph-mon[183059]: failed to encode map e365 with expected crc 2023-08-28T02:39:50.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:50 smithi143 ceph-mon[183059]: failed to encode map e365 with expected crc 2023-08-28T02:39:50.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:50 smithi143 ceph-mon[183059]: osdmap e365: 8 total, 8 up, 7 in 2023-08-28T02:39:50.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:50 smithi143 ceph-mon[183059]: failed to encode map e365 with expected crc 2023-08-28T02:39:50.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:50 smithi143 ceph-mon[189871]: pgmap v1005: 73 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.6 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 16/84 objects degraded (19.048%) 2023-08-28T02:39:50.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:50 smithi143 ceph-mon[189871]: failed to encode map e365 with expected crc 2023-08-28T02:39:50.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:50 smithi143 ceph-mon[189871]: failed to encode map e365 with expected crc 2023-08-28T02:39:50.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:50 smithi143 ceph-mon[189871]: failed to encode map e365 with expected crc 2023-08-28T02:39:50.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:50 smithi143 ceph-mon[189871]: osdmap e365: 8 total, 8 up, 7 in 2023-08-28T02:39:50.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:50 smithi143 ceph-mon[189871]: failed to encode map e365 with expected crc 2023-08-28T02:39:50.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:50 smithi169 ceph-mon[159415]: pgmap v1005: 73 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.6 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 16/84 objects degraded (19.048%) 2023-08-28T02:39:50.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:50 smithi169 ceph-mon[159415]: failed to encode map e365 with expected crc 2023-08-28T02:39:50.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:50 smithi169 ceph-mon[159415]: failed to encode map e365 with expected crc 2023-08-28T02:39:50.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:50 smithi169 ceph-mon[159415]: failed to encode map e365 with expected crc 2023-08-28T02:39:50.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:50 smithi169 ceph-mon[159415]: osdmap e365: 8 total, 8 up, 7 in 2023-08-28T02:39:50.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:50 smithi169 ceph-mon[159415]: failed to encode map e365 with expected crc 2023-08-28T02:39:50.989 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:39:51.017 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:39:51.017 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:39:51.019 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:51.105 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:39:51.228 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:39:51.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:51.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:39:51.268 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:39:51.283 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:39:51.306 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:39:51.326 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:39:51.331 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:39:51.349 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:39:51.368 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:51.373 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:39:51.394 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:39:51.414 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:39:51.432 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:39:51.433 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:39:51.434 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:39:51.434 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:39:51.434 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:39:51.434 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:39:51.434 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:39:51.434 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:39:51.434 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:39:51.794 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset nodeep-scrub 2023-08-28T02:39:51.927 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:51 smithi143 ceph-mon[183059]: failed to encode map e366 with expected crc 2023-08-28T02:39:51.927 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:51 smithi143 ceph-mon[183059]: failed to encode map e366 with expected crc 2023-08-28T02:39:51.927 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:51 smithi143 ceph-mon[183059]: failed to encode map e366 with expected crc 2023-08-28T02:39:51.927 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:51 smithi143 ceph-mon[183059]: osdmap e366: 8 total, 8 up, 7 in 2023-08-28T02:39:51.928 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:51 smithi143 ceph-mon[183059]: failed to encode map e366 with expected crc 2023-08-28T02:39:51.928 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:51 smithi143 ceph-mon[183059]: failed to encode map e366 with expected crc 2023-08-28T02:39:51.928 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:51 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3689333366' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:39:51.928 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:51 smithi143 ceph-mon[189871]: failed to encode map e366 with expected crc 2023-08-28T02:39:51.928 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:51 smithi143 ceph-mon[189871]: failed to encode map e366 with expected crc 2023-08-28T02:39:51.928 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:51 smithi143 ceph-mon[189871]: failed to encode map e366 with expected crc 2023-08-28T02:39:51.928 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:51 smithi143 ceph-mon[189871]: osdmap e366: 8 total, 8 up, 7 in 2023-08-28T02:39:51.928 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:51 smithi143 ceph-mon[189871]: failed to encode map e366 with expected crc 2023-08-28T02:39:51.928 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:51 smithi143 ceph-mon[189871]: failed to encode map e366 with expected crc 2023-08-28T02:39:51.928 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:51 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3689333366' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:39:51.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:51 smithi169 ceph-mon[159415]: failed to encode map e366 with expected crc 2023-08-28T02:39:51.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:51 smithi169 ceph-mon[159415]: failed to encode map e366 with expected crc 2023-08-28T02:39:51.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:51 smithi169 ceph-mon[159415]: failed to encode map e366 with expected crc 2023-08-28T02:39:51.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:51 smithi169 ceph-mon[159415]: osdmap e366: 8 total, 8 up, 7 in 2023-08-28T02:39:51.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:51 smithi169 ceph-mon[159415]: failed to encode map e366 with expected crc 2023-08-28T02:39:51.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:51 smithi169 ceph-mon[159415]: failed to encode map e366 with expected crc 2023-08-28T02:39:51.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:51 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3689333366' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:39:52.020 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:39:52.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:52.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:39:52.125 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:39:52.152 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:52.200 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:39:51 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:39:51] "GET /metrics HTTP/1.1" 200 36556 "" "Prometheus/2.33.4" 2023-08-28T02:39:52.411 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.4 dump_ops_in_flight 2023-08-28T02:39:53.094 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:39:53.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:52 smithi143 ceph-mon[183059]: pgmap v1008: 73 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.6 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 16/84 objects degraded (19.048%) 2023-08-28T02:39:53.200 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:52 smithi143 ceph-mon[189871]: pgmap v1008: 73 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.6 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 16/84 objects degraded (19.048%) 2023-08-28T02:39:53.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:52 smithi169 ceph-mon[159415]: pgmap v1008: 73 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.6 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 16/84 objects degraded (19.048%) 2023-08-28T02:39:53.352 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:39:53.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:53.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:39:53.456 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:39:53.484 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:53.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:53 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 7/84 objects degraded (8.333%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:39:53.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:53 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 7/84 objects degraded (8.333%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:39:53.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:53 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 7/84 objects degraded (8.333%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-08-28T02:39:54.181 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:39:54.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:54.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:39:54.285 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:39:54.321 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:54.339 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:39:54.371 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:39:54.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:54.374 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:39:54.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:54 smithi143 conmon[183035]: 2023-08-28T02:39:54.242+0000 7fadc3887700 -1 mon.a@0(leader).osd e366 definitely_dead 0 2023-08-28T02:39:54.475 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:39:54.504 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:54.551 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:39:54.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:54.554 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:39:54.655 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:39:54.687 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:54.805 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:54 smithi143 ceph-mon[183059]: pgmap v1009: 73 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 4.6 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 7/84 objects degraded (8.333%) 2023-08-28T02:39:54.805 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:54 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/540754605' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-08-28T02:39:54.811 INFO:teuthology.orchestra.run.smithi143.stderr:marked in osd.2. 2023-08-28T02:39:54.835 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:39:54.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:54.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:39:54.941 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:39:54.967 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:55.101 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:54 smithi143 ceph-mon[189871]: pgmap v1009: 73 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 4.6 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 7/84 objects degraded (8.333%) 2023-08-28T02:39:55.102 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:54 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/540754605' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-08-28T02:39:55.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:54 smithi169 ceph-mon[159415]: pgmap v1009: 73 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 4.6 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 7/84 objects degraded (8.333%) 2023-08-28T02:39:55.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:54 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/540754605' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-08-28T02:39:55.258 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:39:55.816 INFO:teuthology.orchestra.run.smithi143.stderr:pool 'unique_pool_4' created 2023-08-28T02:39:56.020 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:39:56.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:56.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:39:56.125 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:39:56.153 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:55 smithi169 ceph-mon[159415]: failed to encode map e367 with expected crc 2023-08-28T02:39:56.153 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:55 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/540754605' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-08-28T02:39:56.153 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:55 smithi169 ceph-mon[159415]: osdmap e367: 8 total, 8 up, 8 in 2023-08-28T02:39:56.153 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:55 smithi169 ceph-mon[159415]: failed to encode map e367 with expected crc 2023-08-28T02:39:56.153 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:55 smithi169 ceph-mon[159415]: failed to encode map e367 with expected crc 2023-08-28T02:39:56.153 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:55 smithi169 ceph-mon[159415]: failed to encode map e367 with expected crc 2023-08-28T02:39:56.153 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:55 smithi169 ceph-mon[159415]: failed to encode map e367 with expected crc 2023-08-28T02:39:56.153 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:55 smithi169 ceph-mon[159415]: failed to encode map e367 with expected crc 2023-08-28T02:39:56.153 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:55 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1672121305' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-08-28T02:39:56.153 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:55 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-08-28T02:39:56.153 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:55 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-08-28T02:39:56.154 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:55 smithi169 ceph-mon[159415]: osdmap e368: 8 total, 8 up, 8 in 2023-08-28T02:39:56.156 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:56.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:55 smithi143 ceph-mon[183059]: failed to encode map e367 with expected crc 2023-08-28T02:39:56.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:55 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/540754605' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-08-28T02:39:56.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:55 smithi143 ceph-mon[183059]: osdmap e367: 8 total, 8 up, 8 in 2023-08-28T02:39:56.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:55 smithi143 ceph-mon[183059]: failed to encode map e367 with expected crc 2023-08-28T02:39:56.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:55 smithi143 ceph-mon[183059]: failed to encode map e367 with expected crc 2023-08-28T02:39:56.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:55 smithi143 ceph-mon[183059]: failed to encode map e367 with expected crc 2023-08-28T02:39:56.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:55 smithi143 ceph-mon[183059]: failed to encode map e367 with expected crc 2023-08-28T02:39:56.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:55 smithi143 ceph-mon[183059]: failed to encode map e367 with expected crc 2023-08-28T02:39:56.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:55 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1672121305' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-08-28T02:39:56.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:55 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-08-28T02:39:56.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:55 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-08-28T02:39:56.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:55 smithi143 ceph-mon[183059]: osdmap e368: 8 total, 8 up, 8 in 2023-08-28T02:39:56.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:55 smithi143 ceph-mon[189871]: failed to encode map e367 with expected crc 2023-08-28T02:39:56.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:55 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/540754605' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-08-28T02:39:56.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:55 smithi143 ceph-mon[189871]: osdmap e367: 8 total, 8 up, 8 in 2023-08-28T02:39:56.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:55 smithi143 ceph-mon[189871]: failed to encode map e367 with expected crc 2023-08-28T02:39:56.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:55 smithi143 ceph-mon[189871]: failed to encode map e367 with expected crc 2023-08-28T02:39:56.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:55 smithi143 ceph-mon[189871]: failed to encode map e367 with expected crc 2023-08-28T02:39:56.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:55 smithi143 ceph-mon[189871]: failed to encode map e367 with expected crc 2023-08-28T02:39:56.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:55 smithi143 ceph-mon[189871]: failed to encode map e367 with expected crc 2023-08-28T02:39:56.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:55 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1672121305' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-08-28T02:39:56.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:55 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-08-28T02:39:56.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:55 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-08-28T02:39:56.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:55 smithi143 ceph-mon[189871]: osdmap e368: 8 total, 8 up, 8 in 2023-08-28T02:39:56.203 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:39:56.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:56.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:39:56.308 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:39:56.315 INFO:tasks.thrashosds.thrasher:Added osd 2 2023-08-28T02:39:56.335 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:56.614 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd pool application enable unique_pool_4 rados --yes-i-really-mean-it || true 2023-08-28T02:39:56.634 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:39:56.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:56.639 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:39:56.736 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:39:56.740 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:39:56.772 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:56.844 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:56 smithi143 ceph-mon[189871]: pgmap v1011: 73 pgs: 1 unknown, 4 peering, 68 active+clean; 4.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-08-28T02:39:56.844 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:56 smithi143 ceph-mon[189871]: failed to encode map e368 with expected crc 2023-08-28T02:39:56.844 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:56 smithi143 ceph-mon[189871]: failed to encode map e368 with expected crc 2023-08-28T02:39:56.844 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:56 smithi143 ceph-mon[189871]: failed to encode map e368 with expected crc 2023-08-28T02:39:56.844 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:56 smithi143 ceph-mon[189871]: failed to encode map e368 with expected crc 2023-08-28T02:39:56.844 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:56 smithi143 ceph-mon[189871]: failed to encode map e368 with expected crc 2023-08-28T02:39:56.845 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:56 smithi143 ceph-mon[189871]: failed to encode map e368 with expected crc 2023-08-28T02:39:56.845 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:56 smithi143 ceph-mon[189871]: failed to encode map e368 with expected crc 2023-08-28T02:39:56.845 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:56 smithi143 ceph-mon[189871]: failed to encode map e368 with expected crc 2023-08-28T02:39:56.845 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:56 smithi143 ceph-mon[189871]: failed to encode map e368 with expected crc 2023-08-28T02:39:56.845 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:56 smithi143 ceph-mon[189871]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-08-28T02:39:56.845 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:56 smithi143 ceph-mon[189871]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7/84 objects degraded (8.333%), 1 pg degraded, 1 pg undersized) 2023-08-28T02:39:56.845 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:56 smithi143 ceph-mon[189871]: failed to encode map e368 with expected crc 2023-08-28T02:39:56.845 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:56 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:39:56.845 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:56 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:39:56.845 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:56 smithi143 ceph-mon[189871]: osdmap e369: 8 total, 8 up, 8 in 2023-08-28T02:39:56.845 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:56 smithi143 ceph-mon[183059]: pgmap v1011: 73 pgs: 1 unknown, 4 peering, 68 active+clean; 4.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-08-28T02:39:56.846 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:56 smithi143 ceph-mon[183059]: failed to encode map e368 with expected crc 2023-08-28T02:39:56.846 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:56 smithi143 ceph-mon[183059]: failed to encode map e368 with expected crc 2023-08-28T02:39:56.846 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:56 smithi143 ceph-mon[183059]: failed to encode map e368 with expected crc 2023-08-28T02:39:56.846 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:56 smithi143 ceph-mon[183059]: failed to encode map e368 with expected crc 2023-08-28T02:39:56.846 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:56 smithi143 ceph-mon[183059]: failed to encode map e368 with expected crc 2023-08-28T02:39:56.846 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:56 smithi143 ceph-mon[183059]: failed to encode map e368 with expected crc 2023-08-28T02:39:56.846 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:56 smithi143 ceph-mon[183059]: failed to encode map e368 with expected crc 2023-08-28T02:39:56.846 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:56 smithi143 ceph-mon[183059]: failed to encode map e368 with expected crc 2023-08-28T02:39:56.846 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:56 smithi143 ceph-mon[183059]: failed to encode map e368 with expected crc 2023-08-28T02:39:56.846 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:56 smithi143 ceph-mon[183059]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-08-28T02:39:56.846 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:56 smithi143 ceph-mon[183059]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7/84 objects degraded (8.333%), 1 pg degraded, 1 pg undersized) 2023-08-28T02:39:56.846 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:56 smithi143 ceph-mon[183059]: failed to encode map e368 with expected crc 2023-08-28T02:39:56.847 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:56 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:39:56.847 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:56 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:39:56.847 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:56 smithi143 ceph-mon[183059]: osdmap e369: 8 total, 8 up, 8 in 2023-08-28T02:39:56.903 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:39:56.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:56.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:39:57.005 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:39:57.037 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:57.135 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:39:57.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:57.138 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:39:57.200 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:56 smithi169 ceph-mon[159415]: pgmap v1011: 73 pgs: 1 unknown, 4 peering, 68 active+clean; 4.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-08-28T02:39:57.200 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:56 smithi169 ceph-mon[159415]: failed to encode map e368 with expected crc 2023-08-28T02:39:57.200 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:56 smithi169 ceph-mon[159415]: failed to encode map e368 with expected crc 2023-08-28T02:39:57.200 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:56 smithi169 ceph-mon[159415]: failed to encode map e368 with expected crc 2023-08-28T02:39:57.200 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:56 smithi169 ceph-mon[159415]: failed to encode map e368 with expected crc 2023-08-28T02:39:57.200 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:56 smithi169 ceph-mon[159415]: failed to encode map e368 with expected crc 2023-08-28T02:39:57.200 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:56 smithi169 ceph-mon[159415]: failed to encode map e368 with expected crc 2023-08-28T02:39:57.200 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:56 smithi169 ceph-mon[159415]: failed to encode map e368 with expected crc 2023-08-28T02:39:57.201 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:56 smithi169 ceph-mon[159415]: failed to encode map e368 with expected crc 2023-08-28T02:39:57.201 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:56 smithi169 ceph-mon[159415]: failed to encode map e368 with expected crc 2023-08-28T02:39:57.201 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:56 smithi169 ceph-mon[159415]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-08-28T02:39:57.201 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:56 smithi169 ceph-mon[159415]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7/84 objects degraded (8.333%), 1 pg degraded, 1 pg undersized) 2023-08-28T02:39:57.201 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:56 smithi169 ceph-mon[159415]: failed to encode map e368 with expected crc 2023-08-28T02:39:57.201 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:56 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:39:57.201 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:56 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:39:57.201 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:56 smithi169 ceph-mon[159415]: osdmap e369: 8 total, 8 up, 8 in 2023-08-28T02:39:57.239 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:39:57.267 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:57.409 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:39:57.409 INFO:teuthology.orchestra.run.smithi169.stdout: "ops": [], 2023-08-28T02:39:57.409 INFO:teuthology.orchestra.run.smithi169.stdout: "num_ops": 0 2023-08-28T02:39:57.410 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:39:57.563 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:39:57.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:57.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:39:57.668 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:39:57.698 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:57.742 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:39:57.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:57.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:39:57.810 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is unset 2023-08-28T02:39:57.845 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:39:57.871 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:57.914 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:39:57.916 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:57 smithi169 ceph-mon[159415]: failed to encode map e369 with expected crc 2023-08-28T02:39:57.916 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:57 smithi169 ceph-mon[159415]: failed to encode map e369 with expected crc 2023-08-28T02:39:57.916 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:57 smithi169 ceph-mon[159415]: failed to encode map e369 with expected crc 2023-08-28T02:39:57.916 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:57 smithi169 ceph-mon[159415]: failed to encode map e369 with expected crc 2023-08-28T02:39:57.916 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:57 smithi169 ceph-mon[159415]: failed to encode map e369 with expected crc 2023-08-28T02:39:57.916 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:57 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1121591534' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:39:57.916 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:57 smithi169 ceph-mon[159415]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:39:57.916 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:57 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1121591534' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:39:57.916 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:57 smithi169 ceph-mon[159415]: osdmap e370: 8 total, 8 up, 8 in 2023-08-28T02:39:57.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:57.917 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:39:58.018 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:39:58.045 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:58.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:57 smithi143 ceph-mon[189871]: failed to encode map e369 with expected crc 2023-08-28T02:39:58.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:57 smithi143 ceph-mon[189871]: failed to encode map e369 with expected crc 2023-08-28T02:39:58.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:57 smithi143 ceph-mon[189871]: failed to encode map e369 with expected crc 2023-08-28T02:39:58.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:57 smithi143 ceph-mon[189871]: failed to encode map e369 with expected crc 2023-08-28T02:39:58.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:57 smithi143 ceph-mon[189871]: failed to encode map e369 with expected crc 2023-08-28T02:39:58.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:57 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1121591534' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:39:58.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:57 smithi143 ceph-mon[189871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:39:58.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:57 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1121591534' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:39:58.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:57 smithi143 ceph-mon[189871]: osdmap e370: 8 total, 8 up, 8 in 2023-08-28T02:39:58.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:57 smithi143 ceph-mon[183059]: failed to encode map e369 with expected crc 2023-08-28T02:39:58.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:57 smithi143 ceph-mon[183059]: failed to encode map e369 with expected crc 2023-08-28T02:39:58.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:57 smithi143 ceph-mon[183059]: failed to encode map e369 with expected crc 2023-08-28T02:39:58.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:57 smithi143 ceph-mon[183059]: failed to encode map e369 with expected crc 2023-08-28T02:39:58.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:57 smithi143 ceph-mon[183059]: failed to encode map e369 with expected crc 2023-08-28T02:39:58.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:57 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1121591534' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:39:58.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:57 smithi143 ceph-mon[183059]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:39:58.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:57 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1121591534' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:39:58.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:57 smithi143 ceph-mon[183059]: osdmap e370: 8 total, 8 up, 8 in 2023-08-28T02:39:58.257 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:39:58.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:58.263 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:39:58.264 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.4 dump_blocked_ops 2023-08-28T02:39:58.358 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:39:58.364 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:39:58.390 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:58.433 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:39:58.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:58.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:39:58.540 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:39:58.568 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:58.614 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:39:58.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:58.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:39:58.660 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:39:58.720 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:39:58.759 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:58.907 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:39:58.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:39:58.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:39:59.010 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:39:59.037 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:58 smithi143 ceph-mon[183059]: pgmap v1014: 89 pgs: 1 active+undersized+remapped, 17 unknown, 4 peering, 67 active+clean; 4.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-08-28T02:39:59.037 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:58 smithi143 ceph-mon[183059]: failed to encode map e370 with expected crc 2023-08-28T02:39:59.037 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:58 smithi143 ceph-mon[183059]: failed to encode map e370 with expected crc 2023-08-28T02:39:59.037 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:58 smithi143 ceph-mon[183059]: failed to encode map e370 with expected crc 2023-08-28T02:39:59.037 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:58 smithi143 ceph-mon[183059]: failed to encode map e370 with expected crc 2023-08-28T02:39:59.037 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:58 smithi143 ceph-mon[183059]: osdmap e371: 8 total, 8 up, 8 in 2023-08-28T02:39:59.037 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:58 smithi143 ceph-mon[189871]: pgmap v1014: 89 pgs: 1 active+undersized+remapped, 17 unknown, 4 peering, 67 active+clean; 4.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-08-28T02:39:59.038 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:58 smithi143 ceph-mon[189871]: failed to encode map e370 with expected crc 2023-08-28T02:39:59.038 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:58 smithi143 ceph-mon[189871]: failed to encode map e370 with expected crc 2023-08-28T02:39:59.038 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:58 smithi143 ceph-mon[189871]: failed to encode map e370 with expected crc 2023-08-28T02:39:59.038 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:58 smithi143 ceph-mon[189871]: failed to encode map e370 with expected crc 2023-08-28T02:39:59.038 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:58 smithi143 ceph-mon[189871]: osdmap e371: 8 total, 8 up, 8 in 2023-08-28T02:39:59.047 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:39:59.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:58 smithi169 ceph-mon[159415]: pgmap v1014: 89 pgs: 1 active+undersized+remapped, 17 unknown, 4 peering, 67 active+clean; 4.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-08-28T02:39:59.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:58 smithi169 ceph-mon[159415]: failed to encode map e370 with expected crc 2023-08-28T02:39:59.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:58 smithi169 ceph-mon[159415]: failed to encode map e370 with expected crc 2023-08-28T02:39:59.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:58 smithi169 ceph-mon[159415]: failed to encode map e370 with expected crc 2023-08-28T02:39:59.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:58 smithi169 ceph-mon[159415]: failed to encode map e370 with expected crc 2023-08-28T02:39:59.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:58 smithi169 ceph-mon[159415]: osdmap e371: 8 total, 8 up, 8 in 2023-08-28T02:39:59.460 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:39:59.460 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:39:59.460 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:39:59.460 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:39:59.460 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:39:59.460 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:39:59.461 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:39:59.461 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:39:59.461 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:39:59.461 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:39:59.461 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:39:59.461 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:39:59.462 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:39:59.462 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:39:59.462 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:39:59.462 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:40:00.086 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:59 smithi143 ceph-mon[183059]: failed to encode map e371 with expected crc 2023-08-28T02:40:00.086 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:59 smithi143 ceph-mon[183059]: failed to encode map e371 with expected crc 2023-08-28T02:40:00.086 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:59 smithi143 ceph-mon[183059]: failed to encode map e371 with expected crc 2023-08-28T02:40:00.086 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:59 smithi143 ceph-mon[183059]: failed to encode map e371 with expected crc 2023-08-28T02:40:00.086 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:59 smithi143 ceph-mon[183059]: failed to encode map e371 with expected crc 2023-08-28T02:40:00.086 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:59 smithi143 ceph-mon[183059]: failed to encode map e371 with expected crc 2023-08-28T02:40:00.087 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:39:59 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1660801617' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:40:00.087 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:59 smithi143 ceph-mon[189871]: failed to encode map e371 with expected crc 2023-08-28T02:40:00.087 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:59 smithi143 ceph-mon[189871]: failed to encode map e371 with expected crc 2023-08-28T02:40:00.087 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:59 smithi143 ceph-mon[189871]: failed to encode map e371 with expected crc 2023-08-28T02:40:00.087 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:59 smithi143 ceph-mon[189871]: failed to encode map e371 with expected crc 2023-08-28T02:40:00.087 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:59 smithi143 ceph-mon[189871]: failed to encode map e371 with expected crc 2023-08-28T02:40:00.087 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:59 smithi143 ceph-mon[189871]: failed to encode map e371 with expected crc 2023-08-28T02:40:00.087 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:39:59 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1660801617' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:40:00.127 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:59 smithi169 ceph-mon[159415]: failed to encode map e371 with expected crc 2023-08-28T02:40:00.127 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:59 smithi169 ceph-mon[159415]: failed to encode map e371 with expected crc 2023-08-28T02:40:00.127 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:59 smithi169 ceph-mon[159415]: failed to encode map e371 with expected crc 2023-08-28T02:40:00.127 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:59 smithi169 ceph-mon[159415]: failed to encode map e371 with expected crc 2023-08-28T02:40:00.127 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:59 smithi169 ceph-mon[159415]: failed to encode map e371 with expected crc 2023-08-28T02:40:00.127 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:59 smithi169 ceph-mon[159415]: failed to encode map e371 with expected crc 2023-08-28T02:40:00.127 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:39:59 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1660801617' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:40:00.247 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:40:00.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:00.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:40:00.351 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:40:00.386 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:00.431 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:40:00.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:00.434 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:40:00.456 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:40:00.456 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:40:00.532 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set noscrub 2023-08-28T02:40:00.535 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:40:00.562 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:00.756 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:40:00.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:00.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:40:00.859 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:40:00.886 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:00.890 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:40:00.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:00 smithi169 ceph-mon[159415]: pgmap v1017: 89 pgs: 1 active+undersized+remapped, 5 unknown, 4 peering, 79 active+clean; 4.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 8 B/s, 2 keys/s, 2 objects/s recovering 2023-08-28T02:40:00.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:00 smithi169 ceph-mon[159415]: Health detail: HEALTH_WARN Reduced data availability: 1 pg inactive, 2 pgs peering 2023-08-28T02:40:00.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:00 smithi169 ceph-mon[159415]: [WRN] PG_AVAILABILITY: Reduced data availability: 1 pg inactive, 2 pgs peering 2023-08-28T02:40:00.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:00 smithi169 ceph-mon[159415]: pg 2.1 is stuck peering for 27m, current state peering, last acting [2,3,0] 2023-08-28T02:40:00.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:00 smithi169 ceph-mon[159415]: pg 4.1c is stuck peering for 7m, current state peering, last acting [2,1,3] 2023-08-28T02:40:01.092 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:00 smithi143 ceph-mon[183059]: pgmap v1017: 89 pgs: 1 active+undersized+remapped, 5 unknown, 4 peering, 79 active+clean; 4.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 8 B/s, 2 keys/s, 2 objects/s recovering 2023-08-28T02:40:01.092 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:00 smithi143 ceph-mon[183059]: Health detail: HEALTH_WARN Reduced data availability: 1 pg inactive, 2 pgs peering 2023-08-28T02:40:01.092 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:00 smithi143 ceph-mon[183059]: [WRN] PG_AVAILABILITY: Reduced data availability: 1 pg inactive, 2 pgs peering 2023-08-28T02:40:01.092 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:00 smithi143 ceph-mon[183059]: pg 2.1 is stuck peering for 27m, current state peering, last acting [2,3,0] 2023-08-28T02:40:01.092 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:00 smithi143 ceph-mon[183059]: pg 4.1c is stuck peering for 7m, current state peering, last acting [2,1,3] 2023-08-28T02:40:01.093 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:00 smithi143 ceph-mon[189871]: pgmap v1017: 89 pgs: 1 active+undersized+remapped, 5 unknown, 4 peering, 79 active+clean; 4.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 8 B/s, 2 keys/s, 2 objects/s recovering 2023-08-28T02:40:01.093 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:00 smithi143 ceph-mon[189871]: Health detail: HEALTH_WARN Reduced data availability: 1 pg inactive, 2 pgs peering 2023-08-28T02:40:01.093 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:00 smithi143 ceph-mon[189871]: [WRN] PG_AVAILABILITY: Reduced data availability: 1 pg inactive, 2 pgs peering 2023-08-28T02:40:01.093 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:00 smithi143 ceph-mon[189871]: pg 2.1 is stuck peering for 27m, current state peering, last acting [2,3,0] 2023-08-28T02:40:01.093 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:00 smithi143 ceph-mon[189871]: pg 4.1c is stuck peering for 7m, current state peering, last acting [2,1,3] 2023-08-28T02:40:01.217 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:40:01.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:01.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:40:01.290 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.4 dump_historic_ops 2023-08-28T02:40:01.316 INFO:tasks.thrashosds.thrasher:in_osds: [0, 4, 5, 3, 6, 1, 7, 2] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-08-28T02:40:01.316 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-08-28T02:40:01.317 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-08-28T02:40:01.317 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-08-28T02:40:01.317 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-08-28T02:40:01.317 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-08-28T02:40:01.321 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:40:01.350 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:01.445 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:40:01.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:01.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:40:01.549 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:40:01.575 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:01.619 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:40:01.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:01.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:40:01.723 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:40:01.749 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:40:01.751 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:01.797 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:40:01.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:01.801 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:40:01.801 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:40:01.854 INFO:teuthology.orchestra.run.smithi143.stderr:enabled application 'rados' on pool 'unique_pool_4' 2023-08-28T02:40:01.888 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:01 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/823209420' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-28T02:40:01.888 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:01 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-28T02:40:01.889 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:01 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/823209420' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-28T02:40:01.889 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:01 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-28T02:40:01.901 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:40:01.926 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:02.015 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:01 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/823209420' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-28T02:40:02.015 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:01 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-28T02:40:02.022 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:40:02.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:02.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:40:02.127 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:40:02.151 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:40:01 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:40:01] "GET /metrics HTTP/1.1" 200 36474 "" "Prometheus/2.33.4" 2023-08-28T02:40:02.154 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:02.374 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:40:02.543 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:40:02.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:02.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:40:02.646 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:40:02.676 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:02.920 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:40:03.022 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:40:03.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:03.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:40:03.127 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:40:03.151 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:02 smithi169 ceph-mon[159415]: pgmap v1018: 89 pgs: 1 active+undersized+remapped, 88 active+clean; 4.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 231 KiB/s rd, 188 KiB/s wr, 78 op/s; 100 B/s, 1 keys/s, 4 objects/s recovering 2023-08-28T02:40:03.151 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:02 smithi169 ceph-mon[159415]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-08-28T02:40:03.151 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:02 smithi169 ceph-mon[159415]: Cluster is now healthy 2023-08-28T02:40:03.151 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:02 smithi169 ceph-mon[159415]: failed to encode map e372 with expected crc 2023-08-28T02:40:03.151 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:02 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-08-28T02:40:03.151 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:02 smithi169 ceph-mon[159415]: osdmap e372: 8 total, 8 up, 8 in 2023-08-28T02:40:03.153 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:03.158 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:02 smithi143 ceph-mon[183059]: pgmap v1018: 89 pgs: 1 active+undersized+remapped, 88 active+clean; 4.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 231 KiB/s rd, 188 KiB/s wr, 78 op/s; 100 B/s, 1 keys/s, 4 objects/s recovering 2023-08-28T02:40:03.158 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:02 smithi143 ceph-mon[183059]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-08-28T02:40:03.158 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:02 smithi143 ceph-mon[183059]: Cluster is now healthy 2023-08-28T02:40:03.158 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:02 smithi143 ceph-mon[183059]: failed to encode map e372 with expected crc 2023-08-28T02:40:03.158 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:02 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-08-28T02:40:03.158 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:02 smithi143 ceph-mon[183059]: osdmap e372: 8 total, 8 up, 8 in 2023-08-28T02:40:03.158 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:02 smithi143 ceph-mon[189871]: pgmap v1018: 89 pgs: 1 active+undersized+remapped, 88 active+clean; 4.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 231 KiB/s rd, 188 KiB/s wr, 78 op/s; 100 B/s, 1 keys/s, 4 objects/s recovering 2023-08-28T02:40:03.158 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:02 smithi143 ceph-mon[189871]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-08-28T02:40:03.159 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:02 smithi143 ceph-mon[189871]: Cluster is now healthy 2023-08-28T02:40:03.159 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:02 smithi143 ceph-mon[189871]: failed to encode map e372 with expected crc 2023-08-28T02:40:03.159 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:02 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-08-28T02:40:03.159 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:02 smithi143 ceph-mon[189871]: osdmap e372: 8 total, 8 up, 8 in 2023-08-28T02:40:03.247 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:40:03.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:03.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:40:03.351 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:40:03.379 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:03.679 DEBUG:teuthology.orchestra.run.smithi143:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_4 bench 90 write' 2023-08-28T02:40:03.682 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-08-28T02:40:03.682 INFO:teuthology.orchestra.run:waiting for 3000 2023-08-28T02:40:03.728 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:hints = 1 2023-08-28T02:40:03.729 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-08-28T02:40:03.729 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:Object prefix: benchmark_data_smithi143_479678 2023-08-28T02:40:03.729 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-08-28T02:40:03.729 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 0 0 0 0 0 0 - 0 2023-08-28T02:40:03.824 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:40:03.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:03.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:40:03.883 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is set 2023-08-28T02:40:03.933 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:40:03.983 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:04.087 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:03 smithi169 ceph-mon[159415]: failed to encode map e372 with expected crc 2023-08-28T02:40:04.087 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:03 smithi169 ceph-mon[159415]: failed to encode map e372 with expected crc 2023-08-28T02:40:04.087 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:03 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/216007424' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:40:04.109 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:03 smithi143 ceph-mon[183059]: failed to encode map e372 with expected crc 2023-08-28T02:40:04.109 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:03 smithi143 ceph-mon[183059]: failed to encode map e372 with expected crc 2023-08-28T02:40:04.109 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:03 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/216007424' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:40:04.110 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:03 smithi143 ceph-mon[189871]: failed to encode map e372 with expected crc 2023-08-28T02:40:04.110 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:03 smithi143 ceph-mon[189871]: failed to encode map e372 with expected crc 2023-08-28T02:40:04.110 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:03 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/216007424' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:40:04.209 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:40:04.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:04.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:40:04.251 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:40:04.251 INFO:teuthology.orchestra.run.smithi169.stdout: "success": "" 2023-08-28T02:40:04.251 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:40:04.314 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:40:04.355 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:04.729 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 1 16 2443 2427 151.606 151.688 0.00277548 0.00657611 2023-08-28T02:40:05.073 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:40:05.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:05.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:40:05.121 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:04 smithi143 ceph-mon[183059]: failed to encode map e372 with expected crc 2023-08-28T02:40:05.121 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:04 smithi143 ceph-mon[183059]: failed to encode map e372 with expected crc 2023-08-28T02:40:05.122 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:04 smithi143 ceph-mon[183059]: pgmap v1020: 89 pgs: 89 active+clean; 4.6 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 225 KiB/s rd, 184 KiB/s wr, 76 op/s; 97 B/s, 1 keys/s, 3 objects/s recovering 2023-08-28T02:40:05.122 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:04 smithi143 ceph-mon[183059]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:40:05.122 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:04 smithi143 ceph-mon[183059]: failed to encode map e373 with expected crc 2023-08-28T02:40:05.122 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:04 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/216007424' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:40:05.122 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:04 smithi143 ceph-mon[183059]: osdmap e373: 8 total, 8 up, 8 in 2023-08-28T02:40:05.122 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:04 smithi143 ceph-mon[183059]: failed to encode map e373 with expected crc 2023-08-28T02:40:05.122 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:04 smithi143 ceph-mon[183059]: failed to encode map e373 with expected crc 2023-08-28T02:40:05.122 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:04 smithi143 ceph-mon[183059]: failed to encode map e373 with expected crc 2023-08-28T02:40:05.131 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:04 smithi143 ceph-mon[189871]: failed to encode map e372 with expected crc 2023-08-28T02:40:05.131 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:04 smithi143 ceph-mon[189871]: failed to encode map e372 with expected crc 2023-08-28T02:40:05.131 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:04 smithi143 ceph-mon[189871]: pgmap v1020: 89 pgs: 89 active+clean; 4.6 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 225 KiB/s rd, 184 KiB/s wr, 76 op/s; 97 B/s, 1 keys/s, 3 objects/s recovering 2023-08-28T02:40:05.131 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:04 smithi143 ceph-mon[189871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:40:05.131 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:04 smithi143 ceph-mon[189871]: failed to encode map e373 with expected crc 2023-08-28T02:40:05.131 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:04 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/216007424' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:40:05.131 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:04 smithi143 ceph-mon[189871]: osdmap e373: 8 total, 8 up, 8 in 2023-08-28T02:40:05.131 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:04 smithi143 ceph-mon[189871]: failed to encode map e373 with expected crc 2023-08-28T02:40:05.132 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:04 smithi143 ceph-mon[189871]: failed to encode map e373 with expected crc 2023-08-28T02:40:05.132 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:04 smithi143 ceph-mon[189871]: failed to encode map e373 with expected crc 2023-08-28T02:40:05.176 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:40:05.219 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:04 smithi169 ceph-mon[159415]: failed to encode map e372 with expected crc 2023-08-28T02:40:05.219 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:04 smithi169 ceph-mon[159415]: failed to encode map e372 with expected crc 2023-08-28T02:40:05.219 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:04 smithi169 ceph-mon[159415]: pgmap v1020: 89 pgs: 89 active+clean; 4.6 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 225 KiB/s rd, 184 KiB/s wr, 76 op/s; 97 B/s, 1 keys/s, 3 objects/s recovering 2023-08-28T02:40:05.219 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:04 smithi169 ceph-mon[159415]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:40:05.219 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:04 smithi169 ceph-mon[159415]: failed to encode map e373 with expected crc 2023-08-28T02:40:05.219 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:04 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/216007424' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:40:05.219 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:04 smithi169 ceph-mon[159415]: osdmap e373: 8 total, 8 up, 8 in 2023-08-28T02:40:05.219 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:04 smithi169 ceph-mon[159415]: failed to encode map e373 with expected crc 2023-08-28T02:40:05.220 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:04 smithi169 ceph-mon[159415]: failed to encode map e373 with expected crc 2023-08-28T02:40:05.220 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:04 smithi169 ceph-mon[159415]: failed to encode map e373 with expected crc 2023-08-28T02:40:05.223 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:05.367 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:40:05.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:05.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:40:05.472 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:40:05.518 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:05.598 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:40:05.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:05.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:40:05.637 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:40:05.671 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:40:05.688 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:40:05.688 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:40:05.704 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:40:05.708 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:40:05.730 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 2 16 5254 5238 163.527 175.688 0.00430896 0.00610041 2023-08-28T02:40:05.754 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:40:05.760 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:05.790 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:40:05.829 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:40:05.862 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:40:05.863 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:40:05.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:05.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:40:05.897 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:40:05.900 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:40:05.900 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:40:05.900 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:40:05.900 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:40:05.900 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:40:05.900 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:40:05.900 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:40:05.900 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:40:05.969 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:40:06.012 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:06.059 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:05 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3827610079' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:40:06.060 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:05 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3827610079' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:40:06.090 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:40:06.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:06.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:40:06.194 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:40:06.229 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:06.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:05 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3827610079' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:40:06.273 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:40:06.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:06.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:40:06.376 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:40:06.402 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:06.468 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:40:06.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:06.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:40:06.574 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:40:06.626 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:06.714 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:40:06.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:06.721 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:40:06.730 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 3 16 7676 7660 159.475 151.375 0.00383064 0.00626178 2023-08-28T02:40:06.822 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:40:06.837 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.5 dump_ops_in_flight 2023-08-28T02:40:06.848 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:06.891 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:40:06.892 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:06 smithi143 ceph-mon[183059]: pgmap v1022: 89 pgs: 89 active+clean; 85 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 197 KiB/s rd, 12 MiB/s wr, 255 op/s; 80 B/s, 2 objects/s recovering 2023-08-28T02:40:06.893 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:06 smithi143 ceph-mon[189871]: pgmap v1022: 89 pgs: 89 active+clean; 85 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 197 KiB/s rd, 12 MiB/s wr, 255 op/s; 80 B/s, 2 objects/s recovering 2023-08-28T02:40:06.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:06.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:40:06.994 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:40:07.019 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:07.083 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:40:07.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:07.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:40:07.158 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set nodeep-scrub 2023-08-28T02:40:07.190 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:40:07.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:06 smithi169 ceph-mon[159415]: pgmap v1022: 89 pgs: 89 active+clean; 85 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 197 KiB/s rd, 12 MiB/s wr, 255 op/s; 80 B/s, 2 objects/s recovering 2023-08-28T02:40:07.246 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:07.255 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:40:07.392 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:40:07.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:07.400 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:40:07.500 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:40:07.548 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:07.640 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:40:07.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:07.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:40:07.730 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 4 16 9797 9781 152.747 132.562 0.00447799 0.00654009 2023-08-28T02:40:07.747 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:40:07.804 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:07.949 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:40:07.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:07.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:40:08.043 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:40:08.061 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:40:08.149 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:08.222 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:40:08.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:08.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:40:08.327 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:40:08.380 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:08.425 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:40:08.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:08.428 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:40:08.528 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:40:08.555 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:08.596 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:40:08.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:08.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:40:08.700 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:40:08.725 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:08.730 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 5 16 11514 11498 143.66 107.312 0.00196232 0.00646382 2023-08-28T02:40:08.778 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:40:08.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:08.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:40:08.881 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:40:08.925 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:08.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:08 smithi143 ceph-mon[189871]: pgmap v1023: 89 pgs: 89 active+clean; 85 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 169 KiB/s rd, 10 MiB/s wr, 218 op/s; 69 B/s, 2 objects/s recovering 2023-08-28T02:40:08.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:08 smithi143 ceph-mon[183059]: pgmap v1023: 89 pgs: 89 active+clean; 85 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 169 KiB/s rd, 10 MiB/s wr, 218 op/s; 69 B/s, 2 objects/s recovering 2023-08-28T02:40:09.134 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:40:09.136 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:08 smithi169 ceph-mon[159415]: pgmap v1023: 89 pgs: 89 active+clean; 85 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 169 KiB/s rd, 10 MiB/s wr, 218 op/s; 69 B/s, 2 objects/s recovering 2023-08-28T02:40:09.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:09.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:40:09.238 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:40:09.293 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:09.409 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:40:09.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:09.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:40:09.513 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:40:09.543 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:09.731 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 6 16 13387 13371 139.226 117.062 0.00423246 0.00717645 2023-08-28T02:40:09.750 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:40:09.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:09.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:40:09.794 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:40:09.794 INFO:teuthology.orchestra.run.smithi169.stdout: "ops": [], 2023-08-28T02:40:09.794 INFO:teuthology.orchestra.run.smithi169.stdout: "num_ops": 0 2023-08-28T02:40:09.794 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:40:09.856 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:40:09.875 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:40:09.916 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:10.022 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:09 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:40:10.094 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:40:10.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:10.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:40:10.199 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:40:10.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:09 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:40:10.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:09 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:40:10.301 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:10.442 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:40:10.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:10.452 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:40:10.480 INFO:tasks.thrashosds.thrasher:in_osds: [0, 4, 5, 3, 6, 1, 7, 2] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-08-28T02:40:10.480 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-08-28T02:40:10.480 INFO:tasks.thrashosds.thrasher:Removing osd 4, in_osds are: [0, 4, 5, 3, 6, 1, 7, 2] 2023-08-28T02:40:10.480 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd out 4 2023-08-28T02:40:10.553 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:40:10.579 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:10.636 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:40:10.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:10.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:40:10.730 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 7 16 15503 15487 138.228 132.25 0.00474534 0.00722711 2023-08-28T02:40:10.740 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:40:10.776 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:10.897 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:10 smithi169 ceph-mon[159415]: pgmap v1024: 89 pgs: 89 active+clean; 617 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 77 MiB/s wr, 1.22k op/s 2023-08-28T02:40:11.012 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:40:11.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:11.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:40:11.118 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:40:11.128 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.5 dump_blocked_ops 2023-08-28T02:40:11.171 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:10 smithi143 ceph-mon[183059]: pgmap v1024: 89 pgs: 89 active+clean; 617 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 77 MiB/s wr, 1.22k op/s 2023-08-28T02:40:11.171 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:10 smithi143 ceph-mon[189871]: pgmap v1024: 89 pgs: 89 active+clean; 617 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 77 MiB/s wr, 1.22k op/s 2023-08-28T02:40:11.176 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:11.266 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:40:11.448 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:40:11.480 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:40:11.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:11.492 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:40:11.592 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:40:11.648 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:11.731 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 8 16 17318 17302 135.129 113.438 0.0041627 0.00739061 2023-08-28T02:40:11.789 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:40:11.913 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:40:11.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:11.920 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:40:12.020 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:40:12.066 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:40:11 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:40:11] "GET /metrics HTTP/1.1" 200 36525 "" "Prometheus/2.33.4" 2023-08-28T02:40:12.079 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:12.114 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is set 2023-08-28T02:40:12.173 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:40:12.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:12.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:40:12.276 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:40:12.327 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:12 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:40:12.327 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:12 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:40:12.327 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:12 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/18813015' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:40:12.327 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:12 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:40:12.327 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:12 smithi169 ceph-mon[159415]: pgmap v1025: 89 pgs: 89 active+clean; 792 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-08-28T02:40:12.334 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:12 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:40:12.334 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:12 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:40:12.334 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:12 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/18813015' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:40:12.334 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:12 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:40:12.334 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:12 smithi143 ceph-mon[183059]: pgmap v1025: 89 pgs: 89 active+clean; 792 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-08-28T02:40:12.335 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:12 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:40:12.335 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:12 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:40:12.335 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:12 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/18813015' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:40:12.335 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:12 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:40:12.335 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:12 smithi143 ceph-mon[189871]: pgmap v1025: 89 pgs: 89 active+clean; 792 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-08-28T02:40:12.338 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:12.731 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 9 16 19178 19162 133.03 116.25 0.00605384 0.00751259 2023-08-28T02:40:13.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:13 smithi169 ceph-mon[159415]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:40:13.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:13 smithi169 ceph-mon[159415]: failed to encode map e374 with expected crc 2023-08-28T02:40:13.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:13 smithi169 ceph-mon[159415]: failed to encode map e374 with expected crc 2023-08-28T02:40:13.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:13 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/18813015' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:40:13.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:13 smithi169 ceph-mon[159415]: osdmap e374: 8 total, 8 up, 8 in 2023-08-28T02:40:13.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:13 smithi169 ceph-mon[159415]: failed to encode map e374 with expected crc 2023-08-28T02:40:13.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:13 smithi169 ceph-mon[159415]: failed to encode map e374 with expected crc 2023-08-28T02:40:13.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:13 smithi169 ceph-mon[159415]: failed to encode map e374 with expected crc 2023-08-28T02:40:13.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:13 smithi169 ceph-mon[159415]: failed to encode map e374 with expected crc 2023-08-28T02:40:13.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:13 smithi143 ceph-mon[183059]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:40:13.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:13 smithi143 ceph-mon[183059]: failed to encode map e374 with expected crc 2023-08-28T02:40:13.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:13 smithi143 ceph-mon[183059]: failed to encode map e374 with expected crc 2023-08-28T02:40:13.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:13 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/18813015' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:40:13.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:13 smithi143 ceph-mon[183059]: osdmap e374: 8 total, 8 up, 8 in 2023-08-28T02:40:13.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:13 smithi143 ceph-mon[183059]: failed to encode map e374 with expected crc 2023-08-28T02:40:13.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:13 smithi143 ceph-mon[183059]: failed to encode map e374 with expected crc 2023-08-28T02:40:13.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:13 smithi143 ceph-mon[183059]: failed to encode map e374 with expected crc 2023-08-28T02:40:13.452 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:13 smithi143 ceph-mon[183059]: failed to encode map e374 with expected crc 2023-08-28T02:40:13.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:13 smithi143 ceph-mon[189871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:40:13.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:13 smithi143 ceph-mon[189871]: failed to encode map e374 with expected crc 2023-08-28T02:40:13.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:13 smithi143 ceph-mon[189871]: failed to encode map e374 with expected crc 2023-08-28T02:40:13.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:13 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/18813015' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:40:13.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:13 smithi143 ceph-mon[189871]: osdmap e374: 8 total, 8 up, 8 in 2023-08-28T02:40:13.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:13 smithi143 ceph-mon[189871]: failed to encode map e374 with expected crc 2023-08-28T02:40:13.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:13 smithi143 ceph-mon[189871]: failed to encode map e374 with expected crc 2023-08-28T02:40:13.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:13 smithi143 ceph-mon[189871]: failed to encode map e374 with expected crc 2023-08-28T02:40:13.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:13 smithi143 ceph-mon[189871]: failed to encode map e374 with expected crc 2023-08-28T02:40:13.727 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:40:13.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:13.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:40:13.732 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 10 16 20958 20942 130.835 111.25 0.00364484 0.00763907 2023-08-28T02:40:13.832 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:40:13.899 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:14.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:14 smithi143 ceph-mon[183059]: pgmap v1027: 89 pgs: 89 active+clean; 939 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-08-28T02:40:14.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:14 smithi143 ceph-mon[189871]: pgmap v1027: 89 pgs: 89 active+clean; 939 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-08-28T02:40:14.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:14 smithi169 ceph-mon[159415]: pgmap v1027: 89 pgs: 89 active+clean; 939 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-08-28T02:40:14.732 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 11 15 22816 22801 129.503 116.188 0.00873141 0.00771688 2023-08-28T02:40:14.837 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:40:14.838 INFO:teuthology.orchestra.run.smithi169.stdout: "ops": [], 2023-08-28T02:40:14.838 INFO:teuthology.orchestra.run.smithi169.stdout: "complaint_time": 30, 2023-08-28T02:40:14.838 INFO:teuthology.orchestra.run.smithi169.stdout: "num_blocked_ops": 0 2023-08-28T02:40:14.838 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:40:15.733 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 12 16 24265 24249 126.253 90.5 0.00179403 0.00789742 2023-08-28T02:40:16.262 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.5 dump_historic_ops 2023-08-28T02:40:16.391 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:40:16.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:16.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:40:16.500 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:40:16.569 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:16.735 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 13 16 25683 25667 123.338 88.625 0.00945617 0.00810187 2023-08-28T02:40:16.754 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:40:16.760 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:16 smithi143 ceph-mon[183059]: pgmap v1028: 89 pgs: 89 active+clean; 1.3 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2023-08-28T02:40:16.760 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:16 smithi143 ceph-mon[189871]: pgmap v1028: 89 pgs: 89 active+clean; 1.3 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2023-08-28T02:40:16.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:16.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:40:16.862 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:40:16.901 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:16.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:16 smithi169 ceph-mon[159415]: pgmap v1028: 89 pgs: 89 active+clean; 1.3 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2023-08-28T02:40:17.001 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:40:17.017 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:40:17.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:17.023 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:40:17.047 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:40:17.087 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:40:17.123 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:40:17.127 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:40:17.128 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:40:17.160 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:40:17.170 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:17.195 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:40:17.252 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:40:17.293 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:40:17.320 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:40:17.320 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:40:17.320 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:40:17.320 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:40:17.320 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:40:17.320 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:40:17.320 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:40:17.320 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:40:17.456 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:40:17.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:17.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:40:17.561 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:40:17.642 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:17.700 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:17 smithi143 conmon[183035]: 2023-08-28T02:40:17.539+0000 7fadc3887700 -1 mon.a@0(leader).osd e374 definitely_dead 0 2023-08-28T02:40:17.736 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 14 16 27347 27331 121.945 104 0.00577731 0.00819462 2023-08-28T02:40:17.773 INFO:teuthology.orchestra.run.smithi143.stderr:marked out osd.4. 2023-08-28T02:40:18.081 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:40:18.082 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:17 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3398933022' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:40:18.083 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:17 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/835897321' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-08-28T02:40:18.083 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:17 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-08-28T02:40:18.083 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:17 smithi169 ceph-mon[159415]: Client client.admin marked osd.4 out, while it was still marked up 2023-08-28T02:40:18.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:18.084 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:40:18.185 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:40:18.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:17 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3398933022' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:40:18.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:17 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/835897321' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-08-28T02:40:18.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:17 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-08-28T02:40:18.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:17 smithi143 ceph-mon[183059]: Client client.admin marked osd.4 out, while it was still marked up 2023-08-28T02:40:18.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:17 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3398933022' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:40:18.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:17 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/835897321' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-08-28T02:40:18.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:17 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-08-28T02:40:18.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:17 smithi143 ceph-mon[189871]: Client client.admin marked osd.4 out, while it was still marked up 2023-08-28T02:40:18.210 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset noscrub 2023-08-28T02:40:18.217 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:18.411 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:40:18.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:18.414 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:40:18.514 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:40:18.543 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:18.736 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 15 16 27434 27418 114.181 5.4375 0.00189729 0.00819423 2023-08-28T02:40:18.831 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:18 smithi143 ceph-mon[183059]: pgmap v1029: 89 pgs: 89 active+clean; 1.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2023-08-28T02:40:18.831 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:18 smithi143 ceph-mon[183059]: failed to encode map e375 with expected crc 2023-08-28T02:40:18.831 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:18 smithi143 ceph-mon[183059]: failed to encode map e375 with expected crc 2023-08-28T02:40:18.831 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:18 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-08-28T02:40:18.831 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:18 smithi143 ceph-mon[183059]: osdmap e375: 8 total, 8 up, 7 in 2023-08-28T02:40:18.831 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:18 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:40:18.831 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:18 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:40:18.832 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:18 smithi143 ceph-mon[189871]: pgmap v1029: 89 pgs: 89 active+clean; 1.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2023-08-28T02:40:18.832 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:18 smithi143 ceph-mon[189871]: failed to encode map e375 with expected crc 2023-08-28T02:40:18.832 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:18 smithi143 ceph-mon[189871]: failed to encode map e375 with expected crc 2023-08-28T02:40:18.832 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:18 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-08-28T02:40:18.832 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:18 smithi143 ceph-mon[189871]: osdmap e375: 8 total, 8 up, 7 in 2023-08-28T02:40:18.832 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:18 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:40:18.832 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:18 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:40:19.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:18 smithi169 ceph-mon[159415]: pgmap v1029: 89 pgs: 89 active+clean; 1.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2023-08-28T02:40:19.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:18 smithi169 ceph-mon[159415]: failed to encode map e375 with expected crc 2023-08-28T02:40:19.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:18 smithi169 ceph-mon[159415]: failed to encode map e375 with expected crc 2023-08-28T02:40:19.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:18 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-08-28T02:40:19.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:18 smithi169 ceph-mon[159415]: osdmap e375: 8 total, 8 up, 7 in 2023-08-28T02:40:19.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:18 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:40:19.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:18 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:40:19.339 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:40:19.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:19.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:40:19.443 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:40:19.473 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:19.493 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:40:19.493 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:40:19.519 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:40:19.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:19.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:40:19.622 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:40:19.649 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:19.736 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 16 16 27434 27418 107.047 0 - 0.00819423 2023-08-28T02:40:19.745 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:40:19.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:19.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:40:19.849 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:40:19.896 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:19.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:19 smithi169 ceph-mon[159415]: failed to encode map e376 with expected crc 2023-08-28T02:40:19.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:19 smithi169 ceph-mon[159415]: failed to encode map e376 with expected crc 2023-08-28T02:40:19.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:19 smithi169 ceph-mon[159415]: failed to encode map e376 with expected crc 2023-08-28T02:40:19.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:19 smithi169 ceph-mon[159415]: osdmap e376: 8 total, 8 up, 7 in 2023-08-28T02:40:19.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:19 smithi169 ceph-mon[159415]: failed to encode map e376 with expected crc 2023-08-28T02:40:19.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:19 smithi169 ceph-mon[159415]: failed to encode map e376 with expected crc 2023-08-28T02:40:19.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:19 smithi169 ceph-mon[159415]: failed to encode map e376 with expected crc 2023-08-28T02:40:19.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:19 smithi169 ceph-mon[159415]: failed to encode map e376 with expected crc 2023-08-28T02:40:19.992 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:40:19.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:19.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:40:20.099 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:40:20.148 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:20.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:19 smithi143 ceph-mon[189871]: failed to encode map e376 with expected crc 2023-08-28T02:40:20.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:19 smithi143 ceph-mon[189871]: failed to encode map e376 with expected crc 2023-08-28T02:40:20.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:19 smithi143 ceph-mon[189871]: failed to encode map e376 with expected crc 2023-08-28T02:40:20.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:19 smithi143 ceph-mon[189871]: osdmap e376: 8 total, 8 up, 7 in 2023-08-28T02:40:20.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:19 smithi143 ceph-mon[189871]: failed to encode map e376 with expected crc 2023-08-28T02:40:20.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:19 smithi143 ceph-mon[189871]: failed to encode map e376 with expected crc 2023-08-28T02:40:20.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:19 smithi143 ceph-mon[189871]: failed to encode map e376 with expected crc 2023-08-28T02:40:20.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:19 smithi143 ceph-mon[189871]: failed to encode map e376 with expected crc 2023-08-28T02:40:20.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:19 smithi143 ceph-mon[183059]: failed to encode map e376 with expected crc 2023-08-28T02:40:20.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:19 smithi143 ceph-mon[183059]: failed to encode map e376 with expected crc 2023-08-28T02:40:20.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:19 smithi143 ceph-mon[183059]: failed to encode map e376 with expected crc 2023-08-28T02:40:20.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:19 smithi143 ceph-mon[183059]: osdmap e376: 8 total, 8 up, 7 in 2023-08-28T02:40:20.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:19 smithi143 ceph-mon[183059]: failed to encode map e376 with expected crc 2023-08-28T02:40:20.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:19 smithi143 ceph-mon[183059]: failed to encode map e376 with expected crc 2023-08-28T02:40:20.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:19 smithi143 ceph-mon[183059]: failed to encode map e376 with expected crc 2023-08-28T02:40:20.203 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:19 smithi143 ceph-mon[183059]: failed to encode map e376 with expected crc 2023-08-28T02:40:20.271 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.6 dump_ops_in_flight 2023-08-28T02:40:20.281 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:40:20.306 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:40:20.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:20.317 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:40:20.418 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:40:20.454 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:20.525 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:40:20.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:20.530 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:40:20.631 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:40:20.664 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:20.721 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:40:20.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:20.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:40:20.737 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 17 16 28657 28641 105.247 38.2188 0.00932554 0.00949373 2023-08-28T02:40:20.773 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:40:20.828 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:40:20.867 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:20.904 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:20 smithi143 ceph-mon[183059]: pgmap v1032: 89 pgs: 1 active+recovery_wait+degraded, 4 activating+degraded, 3 remapped+peering, 8 peering, 73 active+clean; 1.6 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 114 MiB/s wr, 1.82k op/s; 10/79215 objects degraded (0.013%) 2023-08-28T02:40:20.904 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:20 smithi143 ceph-mon[183059]: Health check failed: Reduced data availability: 4 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-08-28T02:40:20.904 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:20 smithi143 ceph-mon[183059]: Health check failed: Degraded data redundancy: 10/79215 objects degraded (0.013%), 5 pgs degraded (PG_DEGRADED) 2023-08-28T02:40:20.904 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:20 smithi143 ceph-mon[183059]: failed to encode map e377 with expected crc 2023-08-28T02:40:20.904 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:20 smithi143 ceph-mon[183059]: osdmap e377: 8 total, 8 up, 7 in 2023-08-28T02:40:20.904 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:20 smithi143 ceph-mon[183059]: failed to encode map e377 with expected crc 2023-08-28T02:40:20.904 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:20 smithi143 ceph-mon[183059]: failed to encode map e377 with expected crc 2023-08-28T02:40:20.904 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:20 smithi143 ceph-mon[183059]: failed to encode map e377 with expected crc 2023-08-28T02:40:20.904 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:20 smithi143 ceph-mon[189871]: pgmap v1032: 89 pgs: 1 active+recovery_wait+degraded, 4 activating+degraded, 3 remapped+peering, 8 peering, 73 active+clean; 1.6 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 114 MiB/s wr, 1.82k op/s; 10/79215 objects degraded (0.013%) 2023-08-28T02:40:20.905 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:20 smithi143 ceph-mon[189871]: Health check failed: Reduced data availability: 4 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-08-28T02:40:20.905 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:20 smithi143 ceph-mon[189871]: Health check failed: Degraded data redundancy: 10/79215 objects degraded (0.013%), 5 pgs degraded (PG_DEGRADED) 2023-08-28T02:40:20.905 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:20 smithi143 ceph-mon[189871]: failed to encode map e377 with expected crc 2023-08-28T02:40:20.905 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:20 smithi143 ceph-mon[189871]: osdmap e377: 8 total, 8 up, 7 in 2023-08-28T02:40:20.905 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:20 smithi143 ceph-mon[189871]: failed to encode map e377 with expected crc 2023-08-28T02:40:20.905 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:20 smithi143 ceph-mon[189871]: failed to encode map e377 with expected crc 2023-08-28T02:40:20.905 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:20 smithi143 ceph-mon[189871]: failed to encode map e377 with expected crc 2023-08-28T02:40:20.982 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:40:20.985 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:20 smithi169 ceph-mon[159415]: pgmap v1032: 89 pgs: 1 active+recovery_wait+degraded, 4 activating+degraded, 3 remapped+peering, 8 peering, 73 active+clean; 1.6 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 114 MiB/s wr, 1.82k op/s; 10/79215 objects degraded (0.013%) 2023-08-28T02:40:20.985 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:20 smithi169 ceph-mon[159415]: Health check failed: Reduced data availability: 4 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-08-28T02:40:20.985 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:20 smithi169 ceph-mon[159415]: Health check failed: Degraded data redundancy: 10/79215 objects degraded (0.013%), 5 pgs degraded (PG_DEGRADED) 2023-08-28T02:40:20.985 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:20 smithi169 ceph-mon[159415]: failed to encode map e377 with expected crc 2023-08-28T02:40:20.985 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:20 smithi169 ceph-mon[159415]: osdmap e377: 8 total, 8 up, 7 in 2023-08-28T02:40:20.985 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:20 smithi169 ceph-mon[159415]: failed to encode map e377 with expected crc 2023-08-28T02:40:20.985 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:20 smithi169 ceph-mon[159415]: failed to encode map e377 with expected crc 2023-08-28T02:40:20.985 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:20 smithi169 ceph-mon[159415]: failed to encode map e377 with expected crc 2023-08-28T02:40:20.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:20.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:40:21.087 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:40:21.160 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:21.481 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:40:21.737 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 18 16 30129 30113 104.51 92 0.00565271 0.0095576 2023-08-28T02:40:21.765 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:40:21.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:21.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:40:21.795 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:21 smithi143 ceph-mon[189871]: failed to encode map e378 with expected crc 2023-08-28T02:40:21.795 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:21 smithi143 ceph-mon[189871]: osdmap e378: 8 total, 8 up, 7 in 2023-08-28T02:40:21.795 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:21 smithi143 ceph-mon[189871]: failed to encode map e378 with expected crc 2023-08-28T02:40:21.795 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:21 smithi143 ceph-mon[189871]: failed to encode map e378 with expected crc 2023-08-28T02:40:21.795 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:21 smithi143 ceph-mon[189871]: failed to encode map e378 with expected crc 2023-08-28T02:40:21.795 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:21 smithi143 ceph-mon[189871]: failed to encode map e378 with expected crc 2023-08-28T02:40:21.872 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:40:21.899 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:21.942 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:40:21.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:21.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:40:22.045 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:40:22.104 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:21 smithi143 ceph-mon[183059]: failed to encode map e378 with expected crc 2023-08-28T02:40:22.104 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:21 smithi143 ceph-mon[183059]: osdmap e378: 8 total, 8 up, 7 in 2023-08-28T02:40:22.104 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:21 smithi143 ceph-mon[183059]: failed to encode map e378 with expected crc 2023-08-28T02:40:22.104 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:21 smithi143 ceph-mon[183059]: failed to encode map e378 with expected crc 2023-08-28T02:40:22.104 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:21 smithi143 ceph-mon[183059]: failed to encode map e378 with expected crc 2023-08-28T02:40:22.104 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:21 smithi143 ceph-mon[183059]: failed to encode map e378 with expected crc 2023-08-28T02:40:22.105 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:40:21 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:40:21] "GET /metrics HTTP/1.1" 200 36525 "" "Prometheus/2.33.4" 2023-08-28T02:40:22.109 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:22.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:21 smithi169 ceph-mon[159415]: failed to encode map e378 with expected crc 2023-08-28T02:40:22.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:21 smithi169 ceph-mon[159415]: osdmap e378: 8 total, 8 up, 7 in 2023-08-28T02:40:22.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:21 smithi169 ceph-mon[159415]: failed to encode map e378 with expected crc 2023-08-28T02:40:22.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:21 smithi169 ceph-mon[159415]: failed to encode map e378 with expected crc 2023-08-28T02:40:22.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:21 smithi169 ceph-mon[159415]: failed to encode map e378 with expected crc 2023-08-28T02:40:22.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:21 smithi169 ceph-mon[159415]: failed to encode map e378 with expected crc 2023-08-28T02:40:22.737 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 19 16 31326 31310 102.947 74.8125 0.0155853 0.00970599 2023-08-28T02:40:22.879 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:40:22.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:22.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:40:22.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:22 smithi143 ceph-mon[183059]: pgmap v1035: 89 pgs: 1 active+recovery_wait+degraded, 4 activating+degraded, 3 remapped+peering, 8 peering, 73 active+clean; 1.7 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 91 MiB/s wr, 1.46k op/s; 10/82734 objects degraded (0.012%) 2023-08-28T02:40:22.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:22 smithi143 ceph-mon[183059]: osdmap e379: 8 total, 8 up, 7 in 2023-08-28T02:40:22.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:22 smithi143 ceph-mon[183059]: failed to encode map e379 with expected crc 2023-08-28T02:40:22.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:22 smithi143 ceph-mon[183059]: failed to encode map e379 with expected crc 2023-08-28T02:40:22.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:22 smithi143 ceph-mon[183059]: failed to encode map e379 with expected crc 2023-08-28T02:40:22.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:22 smithi143 ceph-mon[183059]: failed to encode map e379 with expected crc 2023-08-28T02:40:22.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:22 smithi143 ceph-mon[183059]: failed to encode map e379 with expected crc 2023-08-28T02:40:22.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:22 smithi143 ceph-mon[183059]: failed to encode map e379 with expected crc 2023-08-28T02:40:22.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:22 smithi143 ceph-mon[183059]: failed to encode map e379 with expected crc 2023-08-28T02:40:22.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:22 smithi143 ceph-mon[183059]: failed to encode map e379 with expected crc 2023-08-28T02:40:22.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:22 smithi143 ceph-mon[189871]: pgmap v1035: 89 pgs: 1 active+recovery_wait+degraded, 4 activating+degraded, 3 remapped+peering, 8 peering, 73 active+clean; 1.7 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 91 MiB/s wr, 1.46k op/s; 10/82734 objects degraded (0.012%) 2023-08-28T02:40:22.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:22 smithi143 ceph-mon[189871]: osdmap e379: 8 total, 8 up, 7 in 2023-08-28T02:40:22.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:22 smithi143 ceph-mon[189871]: failed to encode map e379 with expected crc 2023-08-28T02:40:22.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:22 smithi143 ceph-mon[189871]: failed to encode map e379 with expected crc 2023-08-28T02:40:22.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:22 smithi143 ceph-mon[189871]: failed to encode map e379 with expected crc 2023-08-28T02:40:22.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:22 smithi143 ceph-mon[189871]: failed to encode map e379 with expected crc 2023-08-28T02:40:22.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:22 smithi143 ceph-mon[189871]: failed to encode map e379 with expected crc 2023-08-28T02:40:22.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:22 smithi143 ceph-mon[189871]: failed to encode map e379 with expected crc 2023-08-28T02:40:22.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:22 smithi143 ceph-mon[189871]: failed to encode map e379 with expected crc 2023-08-28T02:40:22.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:22 smithi143 ceph-mon[189871]: failed to encode map e379 with expected crc 2023-08-28T02:40:22.966 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:40:22.966 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:40:22.985 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:40:22.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:22 smithi169 ceph-mon[159415]: pgmap v1035: 89 pgs: 1 active+recovery_wait+degraded, 4 activating+degraded, 3 remapped+peering, 8 peering, 73 active+clean; 1.7 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 91 MiB/s wr, 1.46k op/s; 10/82734 objects degraded (0.012%) 2023-08-28T02:40:22.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:22 smithi169 ceph-mon[159415]: osdmap e379: 8 total, 8 up, 7 in 2023-08-28T02:40:22.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:22 smithi169 ceph-mon[159415]: failed to encode map e379 with expected crc 2023-08-28T02:40:22.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:22 smithi169 ceph-mon[159415]: failed to encode map e379 with expected crc 2023-08-28T02:40:22.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:22 smithi169 ceph-mon[159415]: failed to encode map e379 with expected crc 2023-08-28T02:40:22.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:22 smithi169 ceph-mon[159415]: failed to encode map e379 with expected crc 2023-08-28T02:40:22.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:22 smithi169 ceph-mon[159415]: failed to encode map e379 with expected crc 2023-08-28T02:40:22.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:22 smithi169 ceph-mon[159415]: failed to encode map e379 with expected crc 2023-08-28T02:40:22.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:22 smithi169 ceph-mon[159415]: failed to encode map e379 with expected crc 2023-08-28T02:40:22.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:22 smithi169 ceph-mon[159415]: failed to encode map e379 with expected crc 2023-08-28T02:40:23.020 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:23.332 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:40:23.393 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:40:23.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:23.397 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:40:23.498 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:40:23.578 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:23.737 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:2023-08-28T02:40:23.738474+0000 min lat: 0.00102595 max lat: 2.1786 avg lat: 0.00981993 2023-08-28T02:40:23.737 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-08-28T02:40:23.737 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 20 16 32594 32578 101.762 79.25 0.0103827 0.00981993 2023-08-28T02:40:23.794 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:40:23.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:23.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:40:23.908 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:40:23.939 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:24.048 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:40:24.049 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.6 dump_blocked_ops 2023-08-28T02:40:24.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:24.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:40:24.156 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:40:24.192 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:24 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:40:24.192 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:24 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:40:24.192 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:24 smithi169 ceph-mon[159415]: pgmap v1037: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 4 activating+degraded, 3 remapped+peering, 80 active+clean; 1.8 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s; 10/86526 objects degraded (0.012%); 1457/86526 objects misplaced (1.684%); 3.2 MiB/s, 53 objects/s recovering 2023-08-28T02:40:24.192 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:24 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:40:24.192 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:24 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:40:24.192 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:24 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:40:24.198 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:24.268 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:40:24.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:24.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:40:24.384 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:40:24.421 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:24.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:24 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:40:24.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:24 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:40:24.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:24 smithi143 ceph-mon[183059]: pgmap v1037: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 4 activating+degraded, 3 remapped+peering, 80 active+clean; 1.8 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s; 10/86526 objects degraded (0.012%); 1457/86526 objects misplaced (1.684%); 3.2 MiB/s, 53 objects/s recovering 2023-08-28T02:40:24.450 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:24 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:40:24.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:24 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:40:24.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:24 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:40:24.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:24 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:40:24.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:24 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:40:24.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:24 smithi143 ceph-mon[189871]: pgmap v1037: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 4 activating+degraded, 3 remapped+peering, 80 active+clean; 1.8 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s; 10/86526 objects degraded (0.012%); 1457/86526 objects misplaced (1.684%); 3.2 MiB/s, 53 objects/s recovering 2023-08-28T02:40:24.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:24 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:40:24.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:24 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:40:24.451 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:24 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:40:24.487 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:40:24.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:24.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:40:24.585 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:40:24.605 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:40:24.642 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:24.737 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 21 16 33893 33877 100.782 81.1875 0.00992727 0.00991601 2023-08-28T02:40:24.753 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:40:24.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:24.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:40:24.857 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:40:24.880 INFO:tasks.thrashosds.thrasher:in_osds: [0, 5, 3, 6, 1, 7, 2] out_osds: [4] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-08-28T02:40:24.880 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-08-28T02:40:24.880 INFO:tasks.thrashosds.thrasher:Removing osd 2, in_osds are: [0, 5, 3, 6, 1, 7, 2] 2023-08-28T02:40:24.881 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd out 2 2023-08-28T02:40:24.898 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:24.963 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:40:24.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:24.967 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:40:25.068 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:40:25.113 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:25.197 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is unset 2023-08-28T02:40:25.204 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:25 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3633391993' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:40:25.204 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:25 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:40:25.204 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:25 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3633391993' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:40:25.205 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:25 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:40:25.227 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:40:25.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:25.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:40:25.332 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:40:25.363 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:25.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:25 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3633391993' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:40:25.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:25 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:40:25.737 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 22 16 34857 34841 98.9401 60.25 0.0110907 0.0100998 2023-08-28T02:40:26.000 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:40:26.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:26.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:40:26.104 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:40:26.162 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:26.324 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:26 smithi169 ceph-mon[159415]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:40:26.324 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:26 smithi169 ceph-mon[159415]: failed to encode map e380 with expected crc 2023-08-28T02:40:26.324 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:26 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:40:26.324 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:26 smithi169 ceph-mon[159415]: osdmap e380: 8 total, 8 up, 7 in 2023-08-28T02:40:26.324 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:26 smithi169 ceph-mon[159415]: failed to encode map e380 with expected crc 2023-08-28T02:40:26.324 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:26 smithi169 ceph-mon[159415]: failed to encode map e380 with expected crc 2023-08-28T02:40:26.324 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:26 smithi169 ceph-mon[159415]: failed to encode map e380 with expected crc 2023-08-28T02:40:26.324 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:26 smithi169 ceph-mon[159415]: pgmap v1039: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 86 active+clean; 2.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s; 3417/99531 objects degraded (3.433%); 1457/99531 objects misplaced (1.464%); 2.7 MiB/s, 1 keys/s, 48 objects/s recovering 2023-08-28T02:40:26.324 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:26 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:40:26.324 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:26 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:40:26.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:26 smithi143 ceph-mon[183059]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:40:26.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:26 smithi143 ceph-mon[183059]: failed to encode map e380 with expected crc 2023-08-28T02:40:26.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:26 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:40:26.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:26 smithi143 ceph-mon[183059]: osdmap e380: 8 total, 8 up, 7 in 2023-08-28T02:40:26.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:26 smithi143 ceph-mon[183059]: failed to encode map e380 with expected crc 2023-08-28T02:40:26.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:26 smithi143 ceph-mon[183059]: failed to encode map e380 with expected crc 2023-08-28T02:40:26.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:26 smithi143 ceph-mon[183059]: failed to encode map e380 with expected crc 2023-08-28T02:40:26.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:26 smithi143 ceph-mon[183059]: pgmap v1039: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 86 active+clean; 2.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s; 3417/99531 objects degraded (3.433%); 1457/99531 objects misplaced (1.464%); 2.7 MiB/s, 1 keys/s, 48 objects/s recovering 2023-08-28T02:40:26.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:26 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:40:26.451 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:26 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:40:26.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:26 smithi143 ceph-mon[189871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:40:26.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:26 smithi143 ceph-mon[189871]: failed to encode map e380 with expected crc 2023-08-28T02:40:26.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:26 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:40:26.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:26 smithi143 ceph-mon[189871]: osdmap e380: 8 total, 8 up, 7 in 2023-08-28T02:40:26.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:26 smithi143 ceph-mon[189871]: failed to encode map e380 with expected crc 2023-08-28T02:40:26.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:26 smithi143 ceph-mon[189871]: failed to encode map e380 with expected crc 2023-08-28T02:40:26.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:26 smithi143 ceph-mon[189871]: failed to encode map e380 with expected crc 2023-08-28T02:40:26.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:26 smithi143 ceph-mon[189871]: pgmap v1039: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 86 active+clean; 2.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s; 3417/99531 objects degraded (3.433%); 1457/99531 objects misplaced (1.464%); 2.7 MiB/s, 1 keys/s, 48 objects/s recovering 2023-08-28T02:40:26.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:26 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:40:26.452 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:26 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:40:26.524 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:40:26.525 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:40:26.737 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 23 16 35953 35937 97.6168 68.5 0.0143211 0.0102394 2023-08-28T02:40:26.790 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:40:26.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:26.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:40:26.898 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:40:26.926 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:26.972 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:40:27.028 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:40:27.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:27.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:40:27.132 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:40:27.200 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:27.391 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:27 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 3417/99531 objects degraded (3.433%), 2 pgs degraded (PG_DEGRADED) 2023-08-28T02:40:27.392 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:27 smithi143 ceph-mon[183059]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-08-28T02:40:27.392 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:27 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 3417/99531 objects degraded (3.433%), 2 pgs degraded (PG_DEGRADED) 2023-08-28T02:40:27.392 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:27 smithi143 ceph-mon[189871]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-08-28T02:40:27.398 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:40:27.420 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:40:27.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:27.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:40:27.435 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.6 dump_historic_ops 2023-08-28T02:40:27.446 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:40:27.464 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:27 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 3417/99531 objects degraded (3.433%), 2 pgs degraded (PG_DEGRADED) 2023-08-28T02:40:27.464 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:27 smithi169 ceph-mon[159415]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-08-28T02:40:27.482 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:40:27.526 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:40:27.529 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:40:27.572 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:40:27.606 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:40:27.610 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:27.642 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:40:27.687 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:40:27.703 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:40:27.703 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:40:27.703 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:40:27.703 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:40:27.703 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:40:27.704 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:40:27.704 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:40:27.704 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:40:27.737 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 24 16 36943 36927 96.1277 61.875 0.0121946 0.0103972 2023-08-28T02:40:27.871 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:40:28.337 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:40:28.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:28.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:40:28.447 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:40:28.484 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:28 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2022243464' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:40:28.484 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:28 smithi169 ceph-mon[159415]: pgmap v1040: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 86 active+clean; 2.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 51 MiB/s wr, 812 op/s; 3417/99531 objects degraded (3.433%); 1457/99531 objects misplaced (1.464%); 2.3 MiB/s, 1 keys/s, 41 objects/s recovering 2023-08-28T02:40:28.489 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:28.562 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:28 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2022243464' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:40:28.562 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:28 smithi143 ceph-mon[183059]: pgmap v1040: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 86 active+clean; 2.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 51 MiB/s wr, 812 op/s; 3417/99531 objects degraded (3.433%); 1457/99531 objects misplaced (1.464%); 2.3 MiB/s, 1 keys/s, 41 objects/s recovering 2023-08-28T02:40:28.563 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:28 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2022243464' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:40:28.563 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:28 smithi143 ceph-mon[189871]: pgmap v1040: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 86 active+clean; 2.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 51 MiB/s wr, 812 op/s; 3417/99531 objects degraded (3.433%); 1457/99531 objects misplaced (1.464%); 2.3 MiB/s, 1 keys/s, 41 objects/s recovering 2023-08-28T02:40:28.595 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:40:28.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:28.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:40:28.700 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:40:28.737 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 25 16 38107 38091 95.1926 72.75 0.00772934 0.0104992 2023-08-28T02:40:28.758 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:28.890 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset nodeep-scrub 2023-08-28T02:40:29.399 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:40:29.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:29.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:40:29.512 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:40:29.551 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:29.653 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:40:29.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:29.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:40:29.737 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 26 16 38992 38976 93.659 55.3125 0.00495173 0.0106328 2023-08-28T02:40:29.757 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:40:29.813 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:30.093 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:40:30.094 INFO:teuthology.orchestra.run.smithi169.stdout: "size": 20, 2023-08-28T02:40:30.094 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 600, 2023-08-28T02:40:30.094 INFO:teuthology.orchestra.run.smithi169.stdout: "ops": [ 2023-08-28T02:40:30.094 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.094 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:59156 7.7 7:e651d29d:::benchmark_data_smithi143_360738_object59155:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e268)", 2023-08-28T02:40:30.095 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:31:42.218748+0000", 2023-08-28T02:40:30.095 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 527.871284414, 2023-08-28T02:40:30.095 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.30541675200000001, 2023-08-28T02:40:30.095 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:40:30.095 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:40:30.095 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:40:30.095 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:40:30.096 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:40:30.096 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 59156 2023-08-28T02:40:30.096 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.096 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:40:30.096 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.096 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:40:30.097 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.218748+0000", 2023-08-28T02:40:30.097 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:40:30.097 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.097 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.097 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:40:30.097 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.218748+0000", 2023-08-28T02:40:30.098 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:40:30.098 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.098 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.098 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:40:30.098 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.218747+0000", 2023-08-28T02:40:30.098 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.2304000000000003e-05 2023-08-28T02:40:30.099 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.099 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.099 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:40:30.099 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.218840+0000", 2023-08-28T02:40:30.099 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.1790000000000008e-06 2023-08-28T02:40:30.099 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.099 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.100 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:40:30.100 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.218848+0000", 2023-08-28T02:40:30.100 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.1410000000000005e-06 2023-08-28T02:40:30.100 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.100 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.100 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:40:30.101 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.218856+0000", 2023-08-28T02:40:30.101 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.7073000000000001e-05 2023-08-28T02:40:30.101 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.101 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.101 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:40:30.101 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.218903+0000", 2023-08-28T02:40:30.101 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000116227 2023-08-28T02:40:30.102 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.102 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.102 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:40:30.102 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.219019+0000", 2023-08-28T02:40:30.102 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00016372199999999999 2023-08-28T02:40:30.102 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.103 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.103 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,3", 2023-08-28T02:40:30.103 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.219183+0000", 2023-08-28T02:40:30.103 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0017889679999999999 2023-08-28T02:40:30.103 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.103 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.104 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:40:30.104 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.220972+0000", 2023-08-28T02:40:30.104 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.025971195999999998 2023-08-28T02:40:30.104 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.104 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.104 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:40:30.104 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.246943+0000", 2023-08-28T02:40:30.105 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.27711988100000001 2023-08-28T02:40:30.105 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.105 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.105 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:40:30.105 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.524063+0000", 2023-08-28T02:40:30.105 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.3491000000000001e-05 2023-08-28T02:40:30.106 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.106 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.106 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:40:30.106 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.524107+0000", 2023-08-28T02:40:30.106 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.8653999999999998e-05 2023-08-28T02:40:30.106 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.106 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.107 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:40:30.107 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.524165+0000", 2023-08-28T02:40:30.107 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.30541675200000001 2023-08-28T02:40:30.107 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:40:30.107 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:40:30.107 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:40:30.108 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.108 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.108 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:59160 7.7 7:efb4b659:::benchmark_data_smithi143_360738_object59159:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e268)", 2023-08-28T02:40:30.108 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:31:42.221041+0000", 2023-08-28T02:40:30.108 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 527.868992198, 2023-08-28T02:40:30.108 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.30908384300000002, 2023-08-28T02:40:30.108 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:40:30.108 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:40:30.109 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:40:30.109 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:40:30.109 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:40:30.109 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 59160 2023-08-28T02:40:30.109 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.109 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:40:30.109 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.109 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:40:30.109 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.221041+0000", 2023-08-28T02:40:30.109 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:40:30.109 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.110 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.110 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:40:30.110 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.221041+0000", 2023-08-28T02:40:30.110 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999995 2023-08-28T02:40:30.110 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.110 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.110 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:40:30.110 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.221040+0000", 2023-08-28T02:40:30.110 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.7115e-05 2023-08-28T02:40:30.110 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.110 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.110 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:40:30.111 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.221077+0000", 2023-08-28T02:40:30.111 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.04e-06 2023-08-28T02:40:30.111 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.111 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.111 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:40:30.111 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.221082+0000", 2023-08-28T02:40:30.111 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.1319999999999998e-06 2023-08-28T02:40:30.111 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.111 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.111 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:40:30.111 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.221086+0000", 2023-08-28T02:40:30.112 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.5927000000000003e-05 2023-08-28T02:40:30.112 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.112 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.112 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:40:30.112 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.221131+0000", 2023-08-28T02:40:30.112 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012290200000000001 2023-08-28T02:40:30.112 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.112 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.112 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:40:30.112 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.221254+0000", 2023-08-28T02:40:30.112 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000191273 2023-08-28T02:40:30.112 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.112 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.113 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,3", 2023-08-28T02:40:30.113 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.221446+0000", 2023-08-28T02:40:30.113 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00321025 2023-08-28T02:40:30.113 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.113 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.113 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:40:30.113 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.224656+0000", 2023-08-28T02:40:30.113 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.022316244999999998 2023-08-28T02:40:30.113 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.113 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.113 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:40:30.113 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.246972+0000", 2023-08-28T02:40:30.113 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.283081268 2023-08-28T02:40:30.114 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.114 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.114 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:40:30.114 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.530053+0000", 2023-08-28T02:40:30.114 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.4394999999999998e-05 2023-08-28T02:40:30.114 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.114 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.114 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:40:30.114 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.530088+0000", 2023-08-28T02:40:30.115 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.6699999999999998e-05 2023-08-28T02:40:30.115 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.115 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.115 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:40:30.115 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.530125+0000", 2023-08-28T02:40:30.115 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.30908384300000002 2023-08-28T02:40:30.115 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:40:30.115 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:40:30.115 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:40:30.116 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.116 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.116 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:59162 7.7 7:eedf689e:::benchmark_data_smithi143_360738_object59161:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e268)", 2023-08-28T02:40:30.116 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:31:42.221096+0000", 2023-08-28T02:40:30.116 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 527.86893734199998, 2023-08-28T02:40:30.116 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.311517979, 2023-08-28T02:40:30.116 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:40:30.116 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:40:30.117 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:40:30.117 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:40:30.117 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:40:30.117 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 59162 2023-08-28T02:40:30.117 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.118 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:40:30.118 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.118 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:40:30.118 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.221096+0000", 2023-08-28T02:40:30.118 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:40:30.118 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.118 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.118 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:40:30.118 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.221096+0000", 2023-08-28T02:40:30.118 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999995 2023-08-28T02:40:30.118 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.118 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.118 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:40:30.119 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.221095+0000", 2023-08-28T02:40:30.119 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.6899000000000001e-05 2023-08-28T02:40:30.119 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.119 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.119 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:40:30.119 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.221122+0000", 2023-08-28T02:40:30.119 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.1060000000000002e-06 2023-08-28T02:40:30.119 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.119 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.119 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:40:30.119 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.221126+0000", 2023-08-28T02:40:30.120 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.6129999999999998e-06 2023-08-28T02:40:30.120 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.120 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.120 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:40:30.120 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.221129+0000", 2023-08-28T02:40:30.120 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.001819312 2023-08-28T02:40:30.120 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.120 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.120 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:40:30.120 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.222948+0000", 2023-08-28T02:40:30.120 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00014037199999999999 2023-08-28T02:40:30.120 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.121 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.121 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:40:30.121 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.223088+0000", 2023-08-28T02:40:30.121 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00016837199999999999 2023-08-28T02:40:30.121 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.121 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.121 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,3", 2023-08-28T02:40:30.121 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.223257+0000", 2023-08-28T02:40:30.121 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0015392699999999999 2023-08-28T02:40:30.121 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.121 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.121 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:40:30.121 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.224796+0000", 2023-08-28T02:40:30.122 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.022328396 2023-08-28T02:40:30.122 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.122 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.122 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:40:30.122 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.247124+0000", 2023-08-28T02:40:30.122 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.28539166999999999 2023-08-28T02:40:30.122 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.122 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.122 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:40:30.122 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.532516+0000", 2023-08-28T02:40:30.122 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.3111e-05 2023-08-28T02:40:30.122 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.122 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.123 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:40:30.123 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.532569+0000", 2023-08-28T02:40:30.123 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.4298999999999997e-05 2023-08-28T02:40:30.123 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.123 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.123 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:40:30.123 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.532614+0000", 2023-08-28T02:40:30.123 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.311517979 2023-08-28T02:40:30.123 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:40:30.123 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:40:30.123 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:40:30.123 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.124 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.124 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:59172 7.c 7:348f4251:::benchmark_data_smithi143_360738_object59171:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e268)", 2023-08-28T02:40:30.124 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:31:42.244405+0000", 2023-08-28T02:40:30.124 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 527.84562808099997, 2023-08-28T02:40:30.124 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.28357411100000002, 2023-08-28T02:40:30.124 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:40:30.124 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:40:30.124 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:40:30.124 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:40:30.124 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:40:30.124 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 59172 2023-08-28T02:40:30.124 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.125 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:40:30.125 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.125 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:40:30.125 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.244405+0000", 2023-08-28T02:40:30.125 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:40:30.125 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.125 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.125 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:40:30.125 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.244405+0000", 2023-08-28T02:40:30.125 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:40:30.125 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.125 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.125 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:40:30.126 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.244404+0000", 2023-08-28T02:40:30.126 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.3516000000000006e-05 2023-08-28T02:40:30.126 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.126 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.126 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:40:30.126 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.244468+0000", 2023-08-28T02:40:30.126 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1113e-05 2023-08-28T02:40:30.126 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.126 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.126 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:40:30.126 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.244479+0000", 2023-08-28T02:40:30.126 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.9339999999999997e-06 2023-08-28T02:40:30.126 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.127 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.127 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:40:30.127 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.244484+0000", 2023-08-28T02:40:30.127 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.7425000000000001e-05 2023-08-28T02:40:30.127 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.127 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.127 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:40:30.127 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.244531+0000", 2023-08-28T02:40:30.127 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00013291299999999999 2023-08-28T02:40:30.127 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.127 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.128 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:40:30.128 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.244664+0000", 2023-08-28T02:40:30.128 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00016822899999999999 2023-08-28T02:40:30.128 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.128 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.128 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 3,7", 2023-08-28T02:40:30.128 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.244832+0000", 2023-08-28T02:40:30.128 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0024013200000000002 2023-08-28T02:40:30.128 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.128 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.128 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:40:30.128 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.247233+0000", 2023-08-28T02:40:30.128 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.001143853 2023-08-28T02:40:30.129 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.129 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.129 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:40:30.129 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.248377+0000", 2023-08-28T02:40:30.129 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.27954063299999998 2023-08-28T02:40:30.129 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.129 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.129 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:40:30.129 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.527918+0000", 2023-08-28T02:40:30.129 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.5543000000000002e-05 2023-08-28T02:40:30.129 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.129 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.130 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:40:30.130 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.527953+0000", 2023-08-28T02:40:30.130 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.5420000000000001e-05 2023-08-28T02:40:30.130 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.130 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.130 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:40:30.130 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:42.527979+0000", 2023-08-28T02:40:30.130 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.28357411100000002 2023-08-28T02:40:30.130 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:40:30.130 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:40:30.130 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:40:30.130 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.130 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.131 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:59983 7.17 7:e8265a08:::benchmark_data_smithi143_360738_object59982:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e270)", 2023-08-28T02:40:30.131 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:31:43.479107+0000", 2023-08-28T02:40:30.131 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 526.61092624299999, 2023-08-28T02:40:30.131 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.99602258499999996, 2023-08-28T02:40:30.131 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:40:30.131 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "reached pg", 2023-08-28T02:40:30.131 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:40:30.131 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:40:30.131 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:40:30.131 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 59983 2023-08-28T02:40:30.131 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.131 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:40:30.131 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.132 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:40:30.132 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.479107+0000", 2023-08-28T02:40:30.132 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:40:30.132 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.132 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.132 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:40:30.132 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.479107+0000", 2023-08-28T02:40:30.132 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:40:30.132 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.132 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.132 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:40:30.132 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.479105+0000", 2023-08-28T02:40:30.132 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.4614999999999998e-05 2023-08-28T02:40:30.133 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.133 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.133 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:40:30.133 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.479159+0000", 2023-08-28T02:40:30.133 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.5138e-05 2023-08-28T02:40:30.133 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.133 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.133 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:40:30.133 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.479174+0000", 2023-08-28T02:40:30.133 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.9370000000000004e-06 2023-08-28T02:40:30.133 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.133 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.133 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:40:30.134 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.479182+0000", 2023-08-28T02:40:30.134 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.043864379000000002 2023-08-28T02:40:30.134 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.134 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.134 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:40:30.134 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.523047+0000", 2023-08-28T02:40:30.134 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.3399999999999998e-06 2023-08-28T02:40:30.134 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.134 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.134 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:40:30.134 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.523055+0000", 2023-08-28T02:40:30.134 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.95200016600000004 2023-08-28T02:40:30.134 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.134 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.135 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:40:30.135 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.475055+0000", 2023-08-28T02:40:30.135 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.4060000000000006e-05 2023-08-28T02:40:30.135 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.135 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.135 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:40:30.135 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.475129+0000", 2023-08-28T02:40:30.135 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.99602258499999996 2023-08-28T02:40:30.135 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:40:30.135 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:40:30.135 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:40:30.135 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.136 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.136 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:59986 7.17 7:ee7c11b0:::benchmark_data_smithi143_360738_object59985:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e270)", 2023-08-28T02:40:30.136 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:31:43.479190+0000", 2023-08-28T02:40:30.136 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 526.61084287400001, 2023-08-28T02:40:30.136 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.011258244, 2023-08-28T02:40:30.136 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:40:30.136 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:40:30.136 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:40:30.136 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:40:30.136 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:40:30.136 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 59986 2023-08-28T02:40:30.136 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.136 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:40:30.137 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.137 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:40:30.137 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.479190+0000", 2023-08-28T02:40:30.137 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:40:30.137 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.137 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.137 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:40:30.137 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.479190+0000", 2023-08-28T02:40:30.137 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999995 2023-08-28T02:40:30.137 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.137 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.137 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:40:30.137 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.479189+0000", 2023-08-28T02:40:30.138 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.6390999999999998e-05 2023-08-28T02:40:30.138 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.138 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.138 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:40:30.138 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.479226+0000", 2023-08-28T02:40:30.138 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.0899999999999999e-06 2023-08-28T02:40:30.138 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.138 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.138 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:40:30.138 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.479233+0000", 2023-08-28T02:40:30.138 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.4750000000000004e-06 2023-08-28T02:40:30.138 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.138 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.138 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:40:30.139 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.479237+0000", 2023-08-28T02:40:30.139 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.043836669000000002 2023-08-28T02:40:30.139 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.139 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.139 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:40:30.139 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.523074+0000", 2023-08-28T02:40:30.139 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.6149999999999999e-06 2023-08-28T02:40:30.139 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.139 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.139 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:40:30.139 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.523079+0000", 2023-08-28T02:40:30.139 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.95230524400000005 2023-08-28T02:40:30.139 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.140 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.140 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:40:30.140 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.475384+0000", 2023-08-28T02:40:30.140 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012201699999999999 2023-08-28T02:40:30.140 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.140 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.140 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:40:30.140 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.475506+0000", 2023-08-28T02:40:30.140 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00013047800000000001 2023-08-28T02:40:30.140 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.140 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.141 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,3", 2023-08-28T02:40:30.141 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.475636+0000", 2023-08-28T02:40:30.141 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0035618910000000002 2023-08-28T02:40:30.141 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.141 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.141 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:40:30.141 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.479198+0000", 2023-08-28T02:40:30.141 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0041008520000000003 2023-08-28T02:40:30.141 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.141 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.141 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:40:30.142 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.483299+0000", 2023-08-28T02:40:30.142 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0070752510000000003 2023-08-28T02:40:30.142 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.142 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.142 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:40:30.142 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.490374+0000", 2023-08-28T02:40:30.142 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.6100999999999999e-05 2023-08-28T02:40:30.142 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.142 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.142 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:40:30.142 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.490411+0000", 2023-08-28T02:40:30.142 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.7731000000000002e-05 2023-08-28T02:40:30.142 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.143 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.143 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:40:30.143 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.490448+0000", 2023-08-28T02:40:30.143 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.011258244 2023-08-28T02:40:30.143 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:40:30.143 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:40:30.143 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:40:30.143 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.143 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.143 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:59987 7.17 7:ea8b344b:::benchmark_data_smithi143_360738_object59986:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e270)", 2023-08-28T02:40:30.143 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:31:43.490101+0000", 2023-08-28T02:40:30.144 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 526.59993234399997, 2023-08-28T02:40:30.144 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0004954230000001, 2023-08-28T02:40:30.144 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:40:30.144 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:40:30.144 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:40:30.144 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:40:30.144 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:40:30.144 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 59987 2023-08-28T02:40:30.144 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.144 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:40:30.144 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.144 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:40:30.144 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.490101+0000", 2023-08-28T02:40:30.144 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:40:30.145 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.145 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.145 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:40:30.145 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.490101+0000", 2023-08-28T02:40:30.145 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:40:30.145 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.145 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.145 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:40:30.145 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.490099+0000", 2023-08-28T02:40:30.145 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.5648000000000001e-05 2023-08-28T02:40:30.145 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.145 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.145 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:40:30.146 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.490144+0000", 2023-08-28T02:40:30.146 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.7496999999999999e-05 2023-08-28T02:40:30.146 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.146 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.146 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:40:30.146 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.490162+0000", 2023-08-28T02:40:30.146 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.0639999999999996e-06 2023-08-28T02:40:30.146 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.146 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.146 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:40:30.146 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.490169+0000", 2023-08-28T02:40:30.146 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.032926438000000002 2023-08-28T02:40:30.147 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.147 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.147 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:40:30.147 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.523095+0000", 2023-08-28T02:40:30.147 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.6530000000000002e-06 2023-08-28T02:40:30.147 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.147 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.147 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for peered", 2023-08-28T02:40:30.147 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:43.523100+0000", 2023-08-28T02:40:30.147 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.95330794900000004 2023-08-28T02:40:30.147 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.147 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.148 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:40:30.148 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.476408+0000", 2023-08-28T02:40:30.148 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00012834599999999999 2023-08-28T02:40:30.148 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.148 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.148 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:40:30.148 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.476536+0000", 2023-08-28T02:40:30.148 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00072788400000000004 2023-08-28T02:40:30.148 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.148 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.148 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,3", 2023-08-28T02:40:30.149 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.477264+0000", 2023-08-28T02:40:30.149 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.001939807 2023-08-28T02:40:30.149 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.149 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.149 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:40:30.149 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.479204+0000", 2023-08-28T02:40:30.149 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0042188959999999998 2023-08-28T02:40:30.149 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.149 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.149 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:40:30.149 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.483423+0000", 2023-08-28T02:40:30.149 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0070798140000000002 2023-08-28T02:40:30.149 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.150 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.150 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:40:30.150 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.490503+0000", 2023-08-28T02:40:30.150 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.4703999999999997e-05 2023-08-28T02:40:30.150 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.150 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.150 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:40:30.150 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.490558+0000", 2023-08-28T02:40:30.150 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.8408999999999998e-05 2023-08-28T02:40:30.150 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.150 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.150 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:40:30.150 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:31:44.490596+0000", 2023-08-28T02:40:30.151 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.0004954230000001 2023-08-28T02:40:30.151 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:40:30.151 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:40:30.151 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:40:30.151 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.151 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.151 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:73185 7.c 7:3902c7aa:::benchmark_data_smithi143_360738_object73184:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e278)", 2023-08-28T02:40:30.151 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:32:01.861018+0000", 2023-08-28T02:40:30.151 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 508.22901531000002, 2023-08-28T02:40:30.151 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.30387867000000002, 2023-08-28T02:40:30.151 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:40:30.151 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:40:30.152 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:40:30.152 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:40:30.152 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:40:30.152 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 73185 2023-08-28T02:40:30.152 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.152 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:40:30.152 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.152 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:40:30.152 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:01.861018+0000", 2023-08-28T02:40:30.152 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:40:30.152 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.152 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.152 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:40:30.153 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:01.861018+0000", 2023-08-28T02:40:30.153 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:40:30.153 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.153 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.153 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:40:30.153 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:01.861017+0000", 2023-08-28T02:40:30.153 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.9664999999999999e-05 2023-08-28T02:40:30.153 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.153 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.153 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:40:30.153 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:01.861036+0000", 2023-08-28T02:40:30.153 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.9590000000000003e-06 2023-08-28T02:40:30.153 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.153 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.154 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:40:30.154 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:01.861041+0000", 2023-08-28T02:40:30.154 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.19e-06 2023-08-28T02:40:30.154 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.154 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.154 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:40:30.154 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:01.861045+0000", 2023-08-28T02:40:30.154 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.3587999999999997e-05 2023-08-28T02:40:30.154 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.154 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.154 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:40:30.154 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:01.861088+0000", 2023-08-28T02:40:30.154 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000118669 2023-08-28T02:40:30.154 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.155 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.155 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:40:30.155 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:01.861207+0000", 2023-08-28T02:40:30.155 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000120389 2023-08-28T02:40:30.155 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.155 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.155 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 2,3", 2023-08-28T02:40:30.155 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:01.861327+0000", 2023-08-28T02:40:30.155 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.002126976 2023-08-28T02:40:30.155 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.155 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.155 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:40:30.156 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:01.863454+0000", 2023-08-28T02:40:30.156 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0044199699999999996 2023-08-28T02:40:30.156 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.156 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.156 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:40:30.156 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:01.867874+0000", 2023-08-28T02:40:30.156 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.29321470100000002 2023-08-28T02:40:30.156 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.156 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.156 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:40:30.156 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:02.161089+0000", 2023-08-28T02:40:30.156 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.0504000000000001e-05 2023-08-28T02:40:30.156 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.156 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.157 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:40:30.157 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:02.161159+0000", 2023-08-28T02:40:30.157 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0037368029999999999 2023-08-28T02:40:30.157 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.157 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.157 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:40:30.157 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:02.164896+0000", 2023-08-28T02:40:30.157 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.30387867000000002 2023-08-28T02:40:30.157 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:40:30.157 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:40:30.157 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:40:30.157 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.157 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.158 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:73196 7.c 7:3aac1676:::benchmark_data_smithi143_360738_object73195:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e278)", 2023-08-28T02:40:30.158 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:32:01.871375+0000", 2023-08-28T02:40:30.158 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 508.21865773299999, 2023-08-28T02:40:30.158 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.29715278499999997, 2023-08-28T02:40:30.158 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:40:30.158 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:40:30.158 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:40:30.158 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:40:30.158 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:40:30.158 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 73196 2023-08-28T02:40:30.158 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.158 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:40:30.158 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.158 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:40:30.159 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:01.871375+0000", 2023-08-28T02:40:30.159 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:40:30.159 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.159 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.159 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:40:30.159 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:01.871375+0000", 2023-08-28T02:40:30.159 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:40:30.159 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.159 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.159 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:40:30.159 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:01.871374+0000", 2023-08-28T02:40:30.159 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.8094000000000001e-05 2023-08-28T02:40:30.159 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.160 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.160 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:40:30.160 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:01.871462+0000", 2023-08-28T02:40:30.160 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.1780000000000003e-06 2023-08-28T02:40:30.160 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.160 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.160 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:40:30.160 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:01.871471+0000", 2023-08-28T02:40:30.160 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.9480000000000001e-06 2023-08-28T02:40:30.160 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.160 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.160 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:40:30.160 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:01.871474+0000", 2023-08-28T02:40:30.160 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.8983000000000003e-05 2023-08-28T02:40:30.161 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.161 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.161 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:40:30.161 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:01.871513+0000", 2023-08-28T02:40:30.161 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 9.7836000000000003e-05 2023-08-28T02:40:30.161 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.161 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.161 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:40:30.161 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:01.871611+0000", 2023-08-28T02:40:30.161 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000142748 2023-08-28T02:40:30.161 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.161 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.161 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 2,3", 2023-08-28T02:40:30.161 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:01.871754+0000", 2023-08-28T02:40:30.162 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0020802260000000001 2023-08-28T02:40:30.162 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.162 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.162 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:40:30.162 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:01.873834+0000", 2023-08-28T02:40:30.162 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0030338280000000001 2023-08-28T02:40:30.162 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.162 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.162 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:40:30.162 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:01.876868+0000", 2023-08-28T02:40:30.162 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.288150458 2023-08-28T02:40:30.162 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.163 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.163 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:40:30.163 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:02.165019+0000", 2023-08-28T02:40:30.163 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.9417000000000003e-05 2023-08-28T02:40:30.163 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.163 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.163 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:40:30.163 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:02.165088+0000", 2023-08-28T02:40:30.163 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0034399349999999999 2023-08-28T02:40:30.163 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.163 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.163 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:40:30.163 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:02.168528+0000", 2023-08-28T02:40:30.164 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.29715278499999997 2023-08-28T02:40:30.164 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:40:30.164 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:40:30.164 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:40:30.164 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.164 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.164 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:74735 7.11 7:8c3c856e:::benchmark_data_smithi143_360738_object74734:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e278)", 2023-08-28T02:40:30.164 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:32:04.074263+0000", 2023-08-28T02:40:30.164 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 506.01576968500001, 2023-08-28T02:40:30.164 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.36364874699999999, 2023-08-28T02:40:30.164 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:40:30.164 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:40:30.164 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:40:30.165 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:40:30.165 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:40:30.165 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 74735 2023-08-28T02:40:30.165 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.165 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:40:30.165 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.165 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:40:30.165 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.074263+0000", 2023-08-28T02:40:30.165 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:40:30.165 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.165 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.165 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:40:30.165 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.074263+0000", 2023-08-28T02:40:30.166 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:40:30.166 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.166 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.166 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:40:30.166 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.074262+0000", 2023-08-28T02:40:30.166 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.2809000000000003e-05 2023-08-28T02:40:30.166 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.166 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.166 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:40:30.166 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.074305+0000", 2023-08-28T02:40:30.166 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.135e-05 2023-08-28T02:40:30.166 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.166 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.167 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:40:30.167 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.074316+0000", 2023-08-28T02:40:30.167 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.7039999999999994e-06 2023-08-28T02:40:30.167 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.167 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.167 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:40:30.167 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.074324+0000", 2023-08-28T02:40:30.167 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.9311000000000002e-05 2023-08-28T02:40:30.167 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.167 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.167 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:40:30.167 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.074383+0000", 2023-08-28T02:40:30.167 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00013908 2023-08-28T02:40:30.168 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.168 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.168 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:40:30.168 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.074522+0000", 2023-08-28T02:40:30.168 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00013703000000000001 2023-08-28T02:40:30.168 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.168 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.168 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 1,3", 2023-08-28T02:40:30.168 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.074659+0000", 2023-08-28T02:40:30.168 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0013023920000000001 2023-08-28T02:40:30.168 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.168 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.168 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:40:30.168 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.075962+0000", 2023-08-28T02:40:30.169 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0023683659999999998 2023-08-28T02:40:30.169 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.169 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.169 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:40:30.169 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.078330+0000", 2023-08-28T02:40:30.169 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.35812925699999998 2023-08-28T02:40:30.169 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.169 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.169 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:40:30.169 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.436459+0000", 2023-08-28T02:40:30.169 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.4033999999999999e-05 2023-08-28T02:40:30.169 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.169 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.170 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:40:30.170 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.436503+0000", 2023-08-28T02:40:30.170 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0014087240000000001 2023-08-28T02:40:30.170 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.170 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.170 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:40:30.170 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.437912+0000", 2023-08-28T02:40:30.170 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.36364874699999999 2023-08-28T02:40:30.170 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:40:30.170 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:40:30.170 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:40:30.170 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.170 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.171 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:74737 7.c 7:30dd3236:::benchmark_data_smithi143_360738_object74736:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e278)", 2023-08-28T02:40:30.171 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:32:04.074337+0000", 2023-08-28T02:40:30.171 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 506.01569551199998, 2023-08-28T02:40:30.171 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.36470671900000001, 2023-08-28T02:40:30.171 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:40:30.171 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:40:30.171 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:40:30.171 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:40:30.171 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:40:30.171 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 74737 2023-08-28T02:40:30.171 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.171 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:40:30.172 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.172 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:40:30.172 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.074337+0000", 2023-08-28T02:40:30.172 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:40:30.172 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.172 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.172 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:40:30.172 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.074337+0000", 2023-08-28T02:40:30.172 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999995 2023-08-28T02:40:30.172 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.172 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.172 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:40:30.172 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.074337+0000", 2023-08-28T02:40:30.173 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.5392e-05 2023-08-28T02:40:30.173 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.173 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.173 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:40:30.173 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.074362+0000", 2023-08-28T02:40:30.173 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.7339999999999999e-06 2023-08-28T02:40:30.173 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.173 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.173 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:40:30.173 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.074367+0000", 2023-08-28T02:40:30.173 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.0759999999999996e-06 2023-08-28T02:40:30.173 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.173 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.174 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:40:30.174 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.074371+0000", 2023-08-28T02:40:30.174 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.5117999999999998e-05 2023-08-28T02:40:30.174 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.174 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.174 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:40:30.174 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.074406+0000", 2023-08-28T02:40:30.174 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000115262 2023-08-28T02:40:30.174 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.174 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.174 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:40:30.174 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.074521+0000", 2023-08-28T02:40:30.174 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000138424 2023-08-28T02:40:30.175 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.175 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.175 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 2,3", 2023-08-28T02:40:30.175 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.074660+0000", 2023-08-28T02:40:30.175 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0020618849999999998 2023-08-28T02:40:30.175 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.175 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.175 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:40:30.175 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.076722+0000", 2023-08-28T02:40:30.175 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.28327240100000001 2023-08-28T02:40:30.175 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.175 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.175 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:40:30.176 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.359994+0000", 2023-08-28T02:40:30.176 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.076505733000000006 2023-08-28T02:40:30.176 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.176 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.176 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:40:30.176 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.436500+0000", 2023-08-28T02:40:30.176 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.8109999999999998e-05 2023-08-28T02:40:30.176 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.176 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.176 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:40:30.176 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.436548+0000", 2023-08-28T02:40:30.176 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0024961839999999998 2023-08-28T02:40:30.176 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.176 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.177 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:40:30.177 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.439044+0000", 2023-08-28T02:40:30.177 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.36470671900000001 2023-08-28T02:40:30.177 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:40:30.177 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:40:30.177 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:40:30.177 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.177 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.177 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:74743 7.c 7:3b888767:::benchmark_data_smithi143_360738_object74742:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e278)", 2023-08-28T02:40:30.177 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:32:04.079960+0000", 2023-08-28T02:40:30.177 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 506.01007240299998, 2023-08-28T02:40:30.177 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.363101022, 2023-08-28T02:40:30.178 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:40:30.178 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:40:30.178 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:40:30.178 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:40:30.178 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:40:30.178 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 74743 2023-08-28T02:40:30.178 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.178 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:40:30.178 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.178 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:40:30.178 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.079960+0000", 2023-08-28T02:40:30.178 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:40:30.178 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.179 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.179 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:40:30.179 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.079960+0000", 2023-08-28T02:40:30.179 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:40:30.179 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.179 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.179 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:40:30.179 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.079959+0000", 2023-08-28T02:40:30.179 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.8719999999999999e-05 2023-08-28T02:40:30.179 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.179 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.179 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:40:30.179 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.079988+0000", 2023-08-28T02:40:30.179 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1620999999999999e-05 2023-08-28T02:40:30.179 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.180 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.180 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:40:30.180 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.080000+0000", 2023-08-28T02:40:30.180 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.4839999999999998e-06 2023-08-28T02:40:30.180 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.180 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.180 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:40:30.180 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.080004+0000", 2023-08-28T02:40:30.180 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.3571000000000001e-05 2023-08-28T02:40:30.181 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.181 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.182 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:40:30.182 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.080038+0000", 2023-08-28T02:40:30.182 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000134991 2023-08-28T02:40:30.182 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.182 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.182 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:40:30.182 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.080173+0000", 2023-08-28T02:40:30.182 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00014020800000000001 2023-08-28T02:40:30.182 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.182 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.182 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 2,3", 2023-08-28T02:40:30.182 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.080313+0000", 2023-08-28T02:40:30.183 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00187014 2023-08-28T02:40:30.183 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.183 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.183 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:40:30.183 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.082183+0000", 2023-08-28T02:40:30.183 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.28375494000000001 2023-08-28T02:40:30.183 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.183 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.183 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:40:30.183 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.365938+0000", 2023-08-28T02:40:30.183 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.075251154000000001 2023-08-28T02:40:30.183 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.184 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.184 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:40:30.184 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.441189+0000", 2023-08-28T02:40:30.184 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.6300000000000001e-05 2023-08-28T02:40:30.184 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.184 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.184 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:40:30.184 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.441236+0000", 2023-08-28T02:40:30.184 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0018259249999999999 2023-08-28T02:40:30.184 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.184 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.184 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:40:30.184 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:04.443061+0000", 2023-08-28T02:40:30.185 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.363101022 2023-08-28T02:40:30.185 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:40:30.185 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:40:30.185 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:40:30.185 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.185 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.185 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:105956 7.c 7:3e062ca2:::benchmark_data_smithi143_360738_object13186:head [delete] snapc 0=[] ondisk+write+known_if_redirected e289)", 2023-08-28T02:40:30.185 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:32:51.395239+0000", 2023-08-28T02:40:30.185 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 458.69479389999998, 2023-08-28T02:40:30.185 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.27643455700000003, 2023-08-28T02:40:30.185 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:40:30.185 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:40:30.185 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:40:30.186 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:40:30.186 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:40:30.186 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 105956 2023-08-28T02:40:30.186 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.186 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:40:30.186 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.186 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:40:30.186 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.395239+0000", 2023-08-28T02:40:30.186 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:40:30.186 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.186 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.186 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:40:30.186 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.395239+0000", 2023-08-28T02:40:30.187 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:40:30.187 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.187 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.187 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:40:30.187 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.395238+0000", 2023-08-28T02:40:30.187 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.2100000000000003e-06 2023-08-28T02:40:30.187 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.187 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.187 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:40:30.187 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.395242+0000", 2023-08-28T02:40:30.188 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.2470000000000008e-06 2023-08-28T02:40:30.188 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.188 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.188 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:40:30.188 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.395250+0000", 2023-08-28T02:40:30.188 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.3158e-05 2023-08-28T02:40:30.188 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.188 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.189 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:40:30.189 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.395263+0000", 2023-08-28T02:40:30.189 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.4830000000000001e-05 2023-08-28T02:40:30.189 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.189 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.189 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:40:30.189 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.395328+0000", 2023-08-28T02:40:30.189 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00017673999999999999 2023-08-28T02:40:30.189 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.189 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.189 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:40:30.189 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.395505+0000", 2023-08-28T02:40:30.189 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000195909 2023-08-28T02:40:30.190 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.190 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.190 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 2,3", 2023-08-28T02:40:30.190 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.395701+0000", 2023-08-28T02:40:30.190 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0030780730000000002 2023-08-28T02:40:30.190 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.190 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.190 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:40:30.190 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.398779+0000", 2023-08-28T02:40:30.190 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0088778889999999999 2023-08-28T02:40:30.190 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.190 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.190 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:40:30.191 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.407657+0000", 2023-08-28T02:40:30.191 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.26167327899999998 2023-08-28T02:40:30.191 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.191 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.191 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:40:30.191 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.669330+0000", 2023-08-28T02:40:30.191 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.9592999999999999e-05 2023-08-28T02:40:30.191 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.191 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.191 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:40:30.191 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.669370+0000", 2023-08-28T02:40:30.191 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0023037980000000001 2023-08-28T02:40:30.191 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.192 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.192 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:40:30.192 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.671674+0000", 2023-08-28T02:40:30.192 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.27643455700000003 2023-08-28T02:40:30.192 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:40:30.192 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:40:30.192 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:40:30.192 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.192 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.192 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.55270.0:105970 7.c 7:3d49304d:::benchmark_data_smithi143_360738_object13200:head [delete] snapc 0=[] ondisk+write+known_if_redirected e289)", 2023-08-28T02:40:30.192 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:32:51.413324+0000", 2023-08-28T02:40:30.193 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 458.67670919099999, 2023-08-28T02:40:30.193 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.27555664499999999, 2023-08-28T02:40:30.193 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:40:30.193 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:40:30.193 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:40:30.193 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.55270", 2023-08-28T02:40:30.193 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3337694554", 2023-08-28T02:40:30.193 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 105970 2023-08-28T02:40:30.193 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.193 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:40:30.193 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.193 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:40:30.193 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.413324+0000", 2023-08-28T02:40:30.194 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:40:30.194 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.194 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.194 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:40:30.194 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.413324+0000", 2023-08-28T02:40:30.194 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:40:30.194 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.194 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.194 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:40:30.194 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.413322+0000", 2023-08-28T02:40:30.194 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.9150000000000001e-06 2023-08-28T02:40:30.194 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.195 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.195 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:40:30.195 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.413328+0000", 2023-08-28T02:40:30.195 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2621e-05 2023-08-28T02:40:30.195 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.195 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.195 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:40:30.195 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.413341+0000", 2023-08-28T02:40:30.195 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.353e-06 2023-08-28T02:40:30.195 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.195 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.195 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:40:30.195 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.413347+0000", 2023-08-28T02:40:30.195 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.1079999999999999e-05 2023-08-28T02:40:30.196 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.196 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.196 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:40:30.196 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.413398+0000", 2023-08-28T02:40:30.196 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00016214199999999999 2023-08-28T02:40:30.196 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.196 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.196 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:40:30.196 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.413560+0000", 2023-08-28T02:40:30.196 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000131607 2023-08-28T02:40:30.196 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.196 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.196 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 2,3", 2023-08-28T02:40:30.196 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.413692+0000", 2023-08-28T02:40:30.197 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.003280331 2023-08-28T02:40:30.197 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.197 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.197 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:40:30.197 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.416972+0000", 2023-08-28T02:40:30.197 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000391078 2023-08-28T02:40:30.197 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.197 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.197 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:40:30.197 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.417363+0000", 2023-08-28T02:40:30.197 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.26862200200000003 2023-08-28T02:40:30.197 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.197 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.198 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:40:30.198 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.685985+0000", 2023-08-28T02:40:30.198 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.6096e-05 2023-08-28T02:40:30.198 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.198 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.198 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:40:30.198 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.686042+0000", 2023-08-28T02:40:30.198 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0028387550000000001 2023-08-28T02:40:30.198 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.198 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.198 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:40:30.198 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:32:51.688880+0000", 2023-08-28T02:40:30.198 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.27555664499999999 2023-08-28T02:40:30.199 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:40:30.199 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:40:30.199 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:40:30.199 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.199 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.199 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.45647.0:129205 8.5 8:aaa5ed1e:::benchmark_data_smithi143_419289_object32295:head [delete] snapc 0=[] ondisk+write+known_if_redirected e343)", 2023-08-28T02:40:30.199 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:37:39.436505+0000", 2023-08-28T02:40:30.199 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 170.65352826500001, 2023-08-28T02:40:30.199 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.32594509999999999, 2023-08-28T02:40:30.199 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:40:30.199 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:40:30.199 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:40:30.199 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.45647", 2023-08-28T02:40:30.200 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3230872490", 2023-08-28T02:40:30.200 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 129205 2023-08-28T02:40:30.200 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.200 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:40:30.200 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.200 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:40:30.200 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:39.436505+0000", 2023-08-28T02:40:30.200 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:40:30.200 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.200 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.200 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:40:30.200 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:39.436505+0000", 2023-08-28T02:40:30.200 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:40:30.201 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.201 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.201 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:40:30.201 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:39.436503+0000", 2023-08-28T02:40:30.201 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.5300000000000002e-06 2023-08-28T02:40:30.201 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.201 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.201 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:40:30.201 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:39.436510+0000", 2023-08-28T02:40:30.201 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.4408e-05 2023-08-28T02:40:30.201 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.201 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.202 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:40:30.202 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:39.436524+0000", 2023-08-28T02:40:30.202 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.1630000000000001e-06 2023-08-28T02:40:30.202 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.202 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.202 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:40:30.202 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:39.436531+0000", 2023-08-28T02:40:30.202 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.1660000000000002e-05 2023-08-28T02:40:30.202 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.202 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.202 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:40:30.202 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:39.436602+0000", 2023-08-28T02:40:30.202 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.9005000000000006e-05 2023-08-28T02:40:30.203 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.203 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.203 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:40:30.203 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:39.436691+0000", 2023-08-28T02:40:30.203 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000100898 2023-08-28T02:40:30.203 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.203 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.203 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,4", 2023-08-28T02:40:30.203 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:39.436792+0000", 2023-08-28T02:40:30.203 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.017029181000000001 2023-08-28T02:40:30.203 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.203 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.203 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:40:30.204 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:39.453821+0000", 2023-08-28T02:40:30.204 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0010434590000000001 2023-08-28T02:40:30.204 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.204 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.204 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:40:30.204 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:39.454865+0000", 2023-08-28T02:40:30.204 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.30748156300000001 2023-08-28T02:40:30.204 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.204 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.204 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:40:30.204 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:39.762346+0000", 2023-08-28T02:40:30.204 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.7025999999999999e-05 2023-08-28T02:40:30.204 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.205 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.205 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:40:30.205 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:39.762403+0000", 2023-08-28T02:40:30.205 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.6319999999999997e-05 2023-08-28T02:40:30.205 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.205 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.205 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:40:30.205 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:39.762450+0000", 2023-08-28T02:40:30.205 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.32594509999999999 2023-08-28T02:40:30.205 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:40:30.205 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:40:30.205 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:40:30.205 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.206 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.206 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.45647.0:129211 8.5 8:af288149:::benchmark_data_smithi143_419289_object32301:head [delete] snapc 0=[] ondisk+write+known_if_redirected e343)", 2023-08-28T02:40:30.206 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:37:39.443229+0000", 2023-08-28T02:40:30.206 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 170.64680354800001, 2023-08-28T02:40:30.206 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.31931388399999999, 2023-08-28T02:40:30.206 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:40:30.206 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:40:30.206 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:40:30.206 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.45647", 2023-08-28T02:40:30.206 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3230872490", 2023-08-28T02:40:30.206 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 129211 2023-08-28T02:40:30.206 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.206 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:40:30.207 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.207 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:40:30.207 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:39.443229+0000", 2023-08-28T02:40:30.207 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:40:30.207 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.207 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.207 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:40:30.207 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:39.443229+0000", 2023-08-28T02:40:30.207 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:40:30.207 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.207 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.207 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:40:30.208 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:39.443227+0000", 2023-08-28T02:40:30.208 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.5599999999999994e-06 2023-08-28T02:40:30.208 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.208 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.208 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:40:30.208 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:39.443236+0000", 2023-08-28T02:40:30.208 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2576e-05 2023-08-28T02:40:30.208 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.208 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.208 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:40:30.208 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:39.443248+0000", 2023-08-28T02:40:30.208 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.1810000000000004e-06 2023-08-28T02:40:30.209 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.209 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.209 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:40:30.209 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:39.443256+0000", 2023-08-28T02:40:30.209 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0016697330000000001 2023-08-28T02:40:30.209 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.209 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.209 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:40:30.209 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:39.444925+0000", 2023-08-28T02:40:30.209 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000133347 2023-08-28T02:40:30.209 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.209 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.210 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:40:30.210 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:39.445059+0000", 2023-08-28T02:40:30.210 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000141156 2023-08-28T02:40:30.210 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.210 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.210 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,4", 2023-08-28T02:40:30.210 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:39.445200+0000", 2023-08-28T02:40:30.210 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0092931349999999992 2023-08-28T02:40:30.210 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.210 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.210 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:40:30.211 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:39.454493+0000", 2023-08-28T02:40:30.211 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.001891193 2023-08-28T02:40:30.211 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.211 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.211 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:40:30.211 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:39.456384+0000", 2023-08-28T02:40:30.211 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.30610307199999998 2023-08-28T02:40:30.211 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.211 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.211 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:40:30.211 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:39.762487+0000", 2023-08-28T02:40:30.211 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.5491000000000003e-05 2023-08-28T02:40:30.211 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.212 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.212 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:40:30.212 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:39.762523+0000", 2023-08-28T02:40:30.212 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.052e-05 2023-08-28T02:40:30.212 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.212 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.212 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:40:30.212 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:39.762543+0000", 2023-08-28T02:40:30.212 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.31931388399999999 2023-08-28T02:40:30.212 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:40:30.212 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:40:30.212 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:40:30.213 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.213 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.213 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.45647.0:129214 8.9 8:9d086036:::benchmark_data_smithi143_419289_object32304:head [delete] snapc 0=[] ondisk+write+known_if_redirected e343)", 2023-08-28T02:40:30.213 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:37:39.447216+0000", 2023-08-28T02:40:30.213 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 170.64281682399999, 2023-08-28T02:40:30.213 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.31526806400000001, 2023-08-28T02:40:30.213 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:40:30.213 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:40:30.213 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:40:30.213 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.45647", 2023-08-28T02:40:30.213 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3230872490", 2023-08-28T02:40:30.213 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 129214 2023-08-28T02:40:30.213 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.213 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:40:30.214 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.214 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:40:30.214 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:39.447216+0000", 2023-08-28T02:40:30.214 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:40:30.214 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.214 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.214 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:40:30.214 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:39.447216+0000", 2023-08-28T02:40:30.214 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:40:30.214 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.214 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.214 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:40:30.214 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:39.447214+0000", 2023-08-28T02:40:30.215 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.5499999999999997e-06 2023-08-28T02:40:30.215 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.215 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.215 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:40:30.215 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:39.447222+0000", 2023-08-28T02:40:30.215 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.6286999999999998e-05 2023-08-28T02:40:30.215 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.215 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.215 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:40:30.215 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:39.447238+0000", 2023-08-28T02:40:30.215 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.8569999999999997e-06 2023-08-28T02:40:30.215 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.215 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.216 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:40:30.216 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:39.447245+0000", 2023-08-28T02:40:30.216 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0021699470000000002 2023-08-28T02:40:30.216 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.216 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.216 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:40:30.216 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:39.449415+0000", 2023-08-28T02:40:30.216 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00013120100000000001 2023-08-28T02:40:30.216 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.216 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.216 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:40:30.216 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:39.449546+0000", 2023-08-28T02:40:30.217 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00013646 2023-08-28T02:40:30.217 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.217 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.217 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:40:30.217 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:39.449683+0000", 2023-08-28T02:40:30.217 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0048172019999999996 2023-08-28T02:40:30.217 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.217 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.217 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:40:30.217 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:39.454500+0000", 2023-08-28T02:40:30.217 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.002921655 2023-08-28T02:40:30.218 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.218 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.218 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:40:30.218 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:39.457422+0000", 2023-08-28T02:40:30.218 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.30497702300000001 2023-08-28T02:40:30.218 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.218 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.218 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:40:30.219 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:39.762399+0000", 2023-08-28T02:40:30.219 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.7709e-05 2023-08-28T02:40:30.219 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.219 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.219 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:40:30.219 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:39.762446+0000", 2023-08-28T02:40:30.220 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 3.7771999999999997e-05 2023-08-28T02:40:30.220 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.220 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.220 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:40:30.220 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:37:39.762484+0000", 2023-08-28T02:40:30.220 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.31526806400000001 2023-08-28T02:40:30.220 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:40:30.220 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:40:30.220 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:40:30.220 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.220 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.221 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.45647.0:161690 8.9 8:906e4066:::benchmark_data_smithi143_419289_object64780:head [delete] snapc 0=[] ondisk+write+known_if_redirected e356)", 2023-08-28T02:40:30.221 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:38:39.874151+0000", 2023-08-28T02:40:30.221 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 110.215882077, 2023-08-28T02:40:30.221 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.23764834300000001, 2023-08-28T02:40:30.221 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:40:30.221 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:40:30.221 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:40:30.222 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.45647", 2023-08-28T02:40:30.222 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3230872490", 2023-08-28T02:40:30.222 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 161690 2023-08-28T02:40:30.222 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.222 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:40:30.222 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.222 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:40:30.222 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:39.874151+0000", 2023-08-28T02:40:30.222 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:40:30.222 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.222 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.222 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:40:30.222 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:39.874151+0000", 2023-08-28T02:40:30.223 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999981 2023-08-28T02:40:30.223 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.223 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.223 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:40:30.223 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:39.874149+0000", 2023-08-28T02:40:30.223 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.4195999999999999e-05 2023-08-28T02:40:30.223 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.223 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.223 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:40:30.223 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:39.874163+0000", 2023-08-28T02:40:30.223 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.7898999999999999e-05 2023-08-28T02:40:30.223 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.223 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.223 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:40:30.224 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:39.874181+0000", 2023-08-28T02:40:30.224 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 7.1930000000000004e-06 2023-08-28T02:40:30.224 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.224 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.224 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:40:30.224 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:39.874188+0000", 2023-08-28T02:40:30.224 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0027148680000000001 2023-08-28T02:40:30.224 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.224 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.224 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:40:30.224 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:39.876903+0000", 2023-08-28T02:40:30.224 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000142538 2023-08-28T02:40:30.224 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.225 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.225 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:40:30.225 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:39.877046+0000", 2023-08-28T02:40:30.225 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.000166174 2023-08-28T02:40:30.225 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.225 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.225 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 0,5", 2023-08-28T02:40:30.225 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:39.877212+0000", 2023-08-28T02:40:30.225 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0070466510000000001 2023-08-28T02:40:30.225 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.225 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.225 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:40:30.225 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:39.884259+0000", 2023-08-28T02:40:30.226 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0014741820000000001 2023-08-28T02:40:30.226 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.226 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.226 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:40:30.226 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:39.885733+0000", 2023-08-28T02:40:30.226 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.225967315 2023-08-28T02:40:30.226 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.226 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.226 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:40:30.226 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:40.111700+0000", 2023-08-28T02:40:30.226 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.5291000000000003e-05 2023-08-28T02:40:30.226 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.227 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.227 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:40:30.227 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:40.111755+0000", 2023-08-28T02:40:30.227 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.3797000000000001e-05 2023-08-28T02:40:30.227 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.227 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.227 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:40:30.227 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:38:40.111799+0000", 2023-08-28T02:40:30.227 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.23764834300000001 2023-08-28T02:40:30.227 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:40:30.227 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:40:30.227 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:40:30.227 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.228 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.228 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_op(client.45647.0:175316 8.5 8:af7a9aae:::benchmark_data_smithi143_419289_object78406:head [delete] snapc 0=[] ondisk+write+known_if_redirected e358)", 2023-08-28T02:40:30.228 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:39:03.543078+0000", 2023-08-28T02:40:30.228 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 86.546955111000003, 2023-08-28T02:40:30.228 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.238712759, 2023-08-28T02:40:30.228 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:40:30.228 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:40:30.228 INFO:teuthology.orchestra.run.smithi169.stdout: "client_info": { 2023-08-28T02:40:30.228 INFO:teuthology.orchestra.run.smithi169.stdout: "client": "client.45647", 2023-08-28T02:40:30.228 INFO:teuthology.orchestra.run.smithi169.stdout: "client_addr": "172.21.15.143:0/3230872490", 2023-08-28T02:40:30.228 INFO:teuthology.orchestra.run.smithi169.stdout: "tid": 175316 2023-08-28T02:40:30.228 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.228 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:40:30.229 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.229 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:40:30.229 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:39:03.543078+0000", 2023-08-28T02:40:30.229 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:40:30.229 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.229 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.229 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:40:30.229 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:39:03.543078+0000", 2023-08-28T02:40:30.229 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.9999986 2023-08-28T02:40:30.229 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.229 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.229 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:40:30.229 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:39:03.543076+0000", 2023-08-28T02:40:30.229 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.8700000000000003e-06 2023-08-28T02:40:30.230 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.230 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.230 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:40:30.230 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:39:03.543083+0000", 2023-08-28T02:40:30.230 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.1548e-05 2023-08-28T02:40:30.230 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.230 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.230 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:40:30.230 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:39:03.543095+0000", 2023-08-28T02:40:30.230 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.4609999999999999e-06 2023-08-28T02:40:30.230 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.230 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.230 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:40:30.231 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:39:03.543100+0000", 2023-08-28T02:40:30.231 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.004444936 2023-08-28T02:40:30.231 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.231 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.231 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:40:30.231 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:39:03.547545+0000", 2023-08-28T02:40:30.231 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00015247200000000001 2023-08-28T02:40:30.231 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.231 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.231 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:40:30.231 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:39:03.547698+0000", 2023-08-28T02:40:30.231 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.00017465199999999999 2023-08-28T02:40:30.232 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.232 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.232 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "waiting for subops from 4,7", 2023-08-28T02:40:30.232 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:39:03.547872+0000", 2023-08-28T02:40:30.232 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.0063887659999999997 2023-08-28T02:40:30.232 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.232 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.232 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "op_commit", 2023-08-28T02:40:30.232 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:39:03.554261+0000", 2023-08-28T02:40:30.232 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.01226673 2023-08-28T02:40:30.232 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.232 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.232 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:40:30.232 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:39:03.566528+0000", 2023-08-28T02:40:30.233 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.4626999999999998e-05 2023-08-28T02:40:30.233 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.233 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.233 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "sub_op_commit_rec", 2023-08-28T02:40:30.233 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:39:03.566572+0000", 2023-08-28T02:40:30.233 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.0386999999999997e-05 2023-08-28T02:40:30.233 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.233 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.233 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:40:30.233 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:39:03.566623+0000", 2023-08-28T02:40:30.233 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.21516765700000001 2023-08-28T02:40:30.233 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.233 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.234 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:40:30.234 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:39:03.781791+0000", 2023-08-28T02:40:30.234 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.238712759 2023-08-28T02:40:30.234 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:40:30.234 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:40:30.234 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:40:30.234 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.234 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.234 INFO:teuthology.orchestra.run.smithi169.stdout: "description": "osd_repop(client.35967.0:6630 9.b e373/368 9:d632ab21:::benchmark_data_smithi143_479678_object6629:head v 373'390, mlcod=373'388)", 2023-08-28T02:40:30.234 INFO:teuthology.orchestra.run.smithi169.stdout: "initiated_at": "2023-08-28T02:40:06.191817+0000", 2023-08-28T02:40:30.234 INFO:teuthology.orchestra.run.smithi169.stdout: "age": 23.898216331, 2023-08-28T02:40:30.234 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.229364285, 2023-08-28T02:40:30.235 INFO:teuthology.orchestra.run.smithi169.stdout: "type_data": { 2023-08-28T02:40:30.235 INFO:teuthology.orchestra.run.smithi169.stdout: "flag_point": "commit sent; apply or cleanup", 2023-08-28T02:40:30.235 INFO:teuthology.orchestra.run.smithi169.stdout: "events": [ 2023-08-28T02:40:30.235 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.235 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "initiated", 2023-08-28T02:40:30.235 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:40:06.191817+0000", 2023-08-28T02:40:30.235 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0 2023-08-28T02:40:30.235 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.235 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.235 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "throttled", 2023-08-28T02:40:30.235 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:40:06.191817+0000", 2023-08-28T02:40:30.235 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4294967295.999999 2023-08-28T02:40:30.236 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.236 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.236 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "header_read", 2023-08-28T02:40:30.236 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:40:06.191816+0000", 2023-08-28T02:40:30.236 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 8.1318999999999998e-05 2023-08-28T02:40:30.236 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.236 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.236 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "all_read", 2023-08-28T02:40:30.236 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:40:06.191897+0000", 2023-08-28T02:40:30.236 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 1.2917e-05 2023-08-28T02:40:30.236 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.236 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.236 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "dispatched", 2023-08-28T02:40:30.237 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:40:06.191910+0000", 2023-08-28T02:40:30.237 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 6.5320000000000003e-06 2023-08-28T02:40:30.237 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.237 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.237 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "queued_for_pg", 2023-08-28T02:40:30.237 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:40:06.191916+0000", 2023-08-28T02:40:30.237 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 5.7482999999999998e-05 2023-08-28T02:40:30.237 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.237 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.237 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "reached_pg", 2023-08-28T02:40:30.237 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:40:06.191974+0000", 2023-08-28T02:40:30.237 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 2.0213000000000001e-05 2023-08-28T02:40:30.238 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.238 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.238 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "started", 2023-08-28T02:40:30.238 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:40:06.191994+0000", 2023-08-28T02:40:30.238 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.22913689100000001 2023-08-28T02:40:30.238 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.238 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.238 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "commit_sent", 2023-08-28T02:40:30.238 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:40:06.421131+0000", 2023-08-28T02:40:30.238 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 4.9969000000000002e-05 2023-08-28T02:40:30.238 INFO:teuthology.orchestra.run.smithi169.stdout: }, 2023-08-28T02:40:30.238 INFO:teuthology.orchestra.run.smithi169.stdout: { 2023-08-28T02:40:30.238 INFO:teuthology.orchestra.run.smithi169.stdout: "event": "done", 2023-08-28T02:40:30.239 INFO:teuthology.orchestra.run.smithi169.stdout: "time": "2023-08-28T02:40:06.421181+0000", 2023-08-28T02:40:30.239 INFO:teuthology.orchestra.run.smithi169.stdout: "duration": 0.229364285 2023-08-28T02:40:30.239 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:40:30.239 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:40:30.239 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:40:30.239 INFO:teuthology.orchestra.run.smithi169.stdout: } 2023-08-28T02:40:30.239 INFO:teuthology.orchestra.run.smithi169.stdout: ] 2023-08-28T02:40:30.239 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:40:30.240 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:40:30.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:30.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:40:30.342 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:40:30.352 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:40:30.431 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:30.737 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 27 16 40246 40230 93.0928 78.375 0.00960916 0.0107325 2023-08-28T02:40:30.752 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:40:30.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:30.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:40:30.862 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:40:30.882 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:30 smithi169 ceph-mon[159415]: pgmap v1041: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s; 3417/112005 objects degraded (3.051%); 1088/112005 objects misplaced (0.971%); 4.9 MiB/s, 1 keys/s, 81 objects/s recovering 2023-08-28T02:40:30.908 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.7 dump_ops_in_flight 2023-08-28T02:40:30.911 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:30.979 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:40:30.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:30.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:40:30.994 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:30 smithi143 ceph-mon[189871]: pgmap v1041: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s; 3417/112005 objects degraded (3.051%); 1088/112005 objects misplaced (0.971%); 4.9 MiB/s, 1 keys/s, 81 objects/s recovering 2023-08-28T02:40:30.994 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:30 smithi143 ceph-mon[183059]: pgmap v1041: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s; 3417/112005 objects degraded (3.051%); 1088/112005 objects misplaced (0.971%); 4.9 MiB/s, 1 keys/s, 81 objects/s recovering 2023-08-28T02:40:31.086 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:40:31.122 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:31.195 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:40:31.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:31.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:40:31.300 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:40:31.347 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:31.414 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:40:31.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:31.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:40:31.431 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:40:31.431 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:31 smithi143 conmon[183035]: 2023-08-28T02:40:31.426+0000 7fadc3887700 -1 mon.a@0(leader).osd e380 definitely_dead 0 2023-08-28T02:40:31.518 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:40:31.595 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:31.700 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:40:31.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:31.706 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:40:31.738 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 28 16 41465 41449 92.489 76.1875 0.00245073 0.0107964 2023-08-28T02:40:31.748 INFO:teuthology.orchestra.run.smithi143.stderr:marked out osd.2. 2023-08-28T02:40:31.807 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:40:31.836 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:31.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:31 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3798528754' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-08-28T02:40:31.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:31 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-08-28T02:40:31.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:31 smithi169 ceph-mon[159415]: Client client.admin marked osd.2 out, while it was still marked up 2023-08-28T02:40:31.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:31 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 3417/112005 objects degraded (3.051%), 2 pgs degraded (PG_DEGRADED) 2023-08-28T02:40:32.016 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:31 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3798528754' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-08-28T02:40:32.016 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:31 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-08-28T02:40:32.017 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:31 smithi143 ceph-mon[183059]: Client client.admin marked osd.2 out, while it was still marked up 2023-08-28T02:40:32.017 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:31 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 3417/112005 objects degraded (3.051%), 2 pgs degraded (PG_DEGRADED) 2023-08-28T02:40:32.017 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:31 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3798528754' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-08-28T02:40:32.017 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:31 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-08-28T02:40:32.017 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:31 smithi143 ceph-mon[189871]: Client client.admin marked osd.2 out, while it was still marked up 2023-08-28T02:40:32.017 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:31 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 3417/112005 objects degraded (3.051%), 2 pgs degraded (PG_DEGRADED) 2023-08-28T02:40:32.018 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:40:31 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:40:31] "GET /metrics HTTP/1.1" 200 36567 "" "Prometheus/2.33.4" 2023-08-28T02:40:32.242 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:40:32.383 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:40:32.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:32.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:40:32.487 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:40:32.514 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:32.738 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 29 16 41547 41531 89.4771 5.125 0.00282965 0.0109611 2023-08-28T02:40:32.765 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:40:32.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:32.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:40:32.873 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:40:32.902 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:32.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:32 smithi143 ceph-mon[183059]: pgmap v1042: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 70 MiB/s wr, 1.11k op/s; 3417/115731 objects degraded (2.953%); 1088/115731 objects misplaced (0.940%); 3.9 MiB/s, 1 keys/s, 65 objects/s recovering 2023-08-28T02:40:32.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:32 smithi143 ceph-mon[183059]: failed to encode map e381 with expected crc 2023-08-28T02:40:32.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:32 smithi143 ceph-mon[183059]: failed to encode map e381 with expected crc 2023-08-28T02:40:32.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:32 smithi143 ceph-mon[183059]: failed to encode map e381 with expected crc 2023-08-28T02:40:32.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:32 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-08-28T02:40:32.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:32 smithi143 ceph-mon[183059]: osdmap e381: 8 total, 8 up, 6 in 2023-08-28T02:40:32.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:32 smithi143 ceph-mon[183059]: failed to encode map e381 with expected crc 2023-08-28T02:40:32.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:32 smithi143 ceph-mon[183059]: failed to encode map e381 with expected crc 2023-08-28T02:40:32.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:32 smithi143 ceph-mon[189871]: pgmap v1042: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 70 MiB/s wr, 1.11k op/s; 3417/115731 objects degraded (2.953%); 1088/115731 objects misplaced (0.940%); 3.9 MiB/s, 1 keys/s, 65 objects/s recovering 2023-08-28T02:40:32.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:32 smithi143 ceph-mon[189871]: failed to encode map e381 with expected crc 2023-08-28T02:40:32.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:32 smithi143 ceph-mon[189871]: failed to encode map e381 with expected crc 2023-08-28T02:40:32.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:32 smithi143 ceph-mon[189871]: failed to encode map e381 with expected crc 2023-08-28T02:40:32.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:32 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-08-28T02:40:32.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:32 smithi143 ceph-mon[189871]: osdmap e381: 8 total, 8 up, 6 in 2023-08-28T02:40:32.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:32 smithi143 ceph-mon[189871]: failed to encode map e381 with expected crc 2023-08-28T02:40:32.952 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:32 smithi143 ceph-mon[189871]: failed to encode map e381 with expected crc 2023-08-28T02:40:32.952 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:40:32.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:32.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:40:33.057 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:40:33.088 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:33.161 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:32 smithi169 ceph-mon[159415]: pgmap v1042: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 70 MiB/s wr, 1.11k op/s; 3417/115731 objects degraded (2.953%); 1088/115731 objects misplaced (0.940%); 3.9 MiB/s, 1 keys/s, 65 objects/s recovering 2023-08-28T02:40:33.161 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:32 smithi169 ceph-mon[159415]: failed to encode map e381 with expected crc 2023-08-28T02:40:33.161 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:32 smithi169 ceph-mon[159415]: failed to encode map e381 with expected crc 2023-08-28T02:40:33.161 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:32 smithi169 ceph-mon[159415]: failed to encode map e381 with expected crc 2023-08-28T02:40:33.161 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:32 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-08-28T02:40:33.161 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:32 smithi169 ceph-mon[159415]: osdmap e381: 8 total, 8 up, 6 in 2023-08-28T02:40:33.161 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:32 smithi169 ceph-mon[159415]: failed to encode map e381 with expected crc 2023-08-28T02:40:33.161 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:32 smithi169 ceph-mon[159415]: failed to encode map e381 with expected crc 2023-08-28T02:40:33.363 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:40:33.363 INFO:teuthology.orchestra.run.smithi169.stdout: "ops": [], 2023-08-28T02:40:33.363 INFO:teuthology.orchestra.run.smithi169.stdout: "num_ops": 0 2023-08-28T02:40:33.363 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:40:33.379 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:40:33.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:33.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:40:33.481 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:40:33.484 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:40:33.519 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:33.738 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 30 16 41547 41531 86.4953 0 - 0.0109611 2023-08-28T02:40:33.789 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:40:33.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:33.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:40:33.900 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:40:33.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:33 smithi169 ceph-mon[159415]: failed to encode map e382 with expected crc 2023-08-28T02:40:33.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:33 smithi169 ceph-mon[159415]: failed to encode map e382 with expected crc 2023-08-28T02:40:33.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:33 smithi169 ceph-mon[159415]: osdmap e382: 8 total, 8 up, 6 in 2023-08-28T02:40:33.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:33 smithi169 ceph-mon[159415]: failed to encode map e382 with expected crc 2023-08-28T02:40:33.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:33 smithi169 ceph-mon[159415]: failed to encode map e382 with expected crc 2023-08-28T02:40:33.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:33 smithi169 ceph-mon[159415]: failed to encode map e382 with expected crc 2023-08-28T02:40:33.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:33 smithi169 ceph-mon[159415]: failed to encode map e382 with expected crc 2023-08-28T02:40:33.989 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:33 smithi169 ceph-mon[159415]: failed to encode map e382 with expected crc 2023-08-28T02:40:34.061 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:33 smithi143 ceph-mon[189871]: failed to encode map e382 with expected crc 2023-08-28T02:40:34.062 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:33 smithi143 ceph-mon[189871]: failed to encode map e382 with expected crc 2023-08-28T02:40:34.062 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:33 smithi143 ceph-mon[189871]: osdmap e382: 8 total, 8 up, 6 in 2023-08-28T02:40:34.062 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:33 smithi143 ceph-mon[189871]: failed to encode map e382 with expected crc 2023-08-28T02:40:34.062 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:33 smithi143 ceph-mon[189871]: failed to encode map e382 with expected crc 2023-08-28T02:40:34.062 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:33 smithi143 ceph-mon[189871]: failed to encode map e382 with expected crc 2023-08-28T02:40:34.062 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:33 smithi143 ceph-mon[189871]: failed to encode map e382 with expected crc 2023-08-28T02:40:34.062 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:33 smithi143 ceph-mon[189871]: failed to encode map e382 with expected crc 2023-08-28T02:40:34.062 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:33 smithi143 ceph-mon[183059]: failed to encode map e382 with expected crc 2023-08-28T02:40:34.062 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:33 smithi143 ceph-mon[183059]: failed to encode map e382 with expected crc 2023-08-28T02:40:34.062 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:33 smithi143 ceph-mon[183059]: osdmap e382: 8 total, 8 up, 6 in 2023-08-28T02:40:34.063 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:33 smithi143 ceph-mon[183059]: failed to encode map e382 with expected crc 2023-08-28T02:40:34.063 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:33 smithi143 ceph-mon[183059]: failed to encode map e382 with expected crc 2023-08-28T02:40:34.063 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:33 smithi143 ceph-mon[183059]: failed to encode map e382 with expected crc 2023-08-28T02:40:34.063 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:33 smithi143 ceph-mon[183059]: failed to encode map e382 with expected crc 2023-08-28T02:40:34.063 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:33 smithi143 ceph-mon[183059]: failed to encode map e382 with expected crc 2023-08-28T02:40:34.101 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:34.237 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:40:34.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:34.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:40:34.341 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:40:34.342 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.7 dump_blocked_ops 2023-08-28T02:40:34.408 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:34.518 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:40:34.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:34.526 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:40:34.627 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:40:34.693 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:34.738 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 31 16 42463 42447 85.5509 28.625 0.0080299 0.0116815 2023-08-28T02:40:34.779 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:40:34.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:34.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:40:34.801 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is unset 2023-08-28T02:40:34.890 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:40:34.938 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:40:34.956 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:34.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:34 smithi169 ceph-mon[159415]: pgmap v1045: 89 pgs: 2 remapped+peering, 3 peering, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 50 MiB/s wr, 800 op/s; 4143/118767 objects degraded (3.488%); 5.7 MiB/s, 91 objects/s recovering 2023-08-28T02:40:34.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:34 smithi169 ceph-mon[159415]: failed to encode map e382 with expected crc 2023-08-28T02:40:34.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:34 smithi169 ceph-mon[159415]: failed to encode map e383 with expected crc 2023-08-28T02:40:34.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:34 smithi169 ceph-mon[159415]: failed to encode map e383 with expected crc 2023-08-28T02:40:34.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:34 smithi169 ceph-mon[159415]: osdmap e383: 8 total, 8 up, 6 in 2023-08-28T02:40:34.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:34 smithi169 ceph-mon[159415]: failed to encode map e383 with expected crc 2023-08-28T02:40:34.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:34 smithi169 ceph-mon[159415]: failed to encode map e383 with expected crc 2023-08-28T02:40:34.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:34 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/3536452314' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:40:34.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:34 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:40:35.061 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:40:35.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:35.065 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:40:35.066 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:34 smithi143 ceph-mon[189871]: pgmap v1045: 89 pgs: 2 remapped+peering, 3 peering, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 50 MiB/s wr, 800 op/s; 4143/118767 objects degraded (3.488%); 5.7 MiB/s, 91 objects/s recovering 2023-08-28T02:40:35.066 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:34 smithi143 ceph-mon[189871]: failed to encode map e382 with expected crc 2023-08-28T02:40:35.066 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:34 smithi143 ceph-mon[189871]: failed to encode map e383 with expected crc 2023-08-28T02:40:35.066 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:34 smithi143 ceph-mon[189871]: failed to encode map e383 with expected crc 2023-08-28T02:40:35.066 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:34 smithi143 ceph-mon[189871]: osdmap e383: 8 total, 8 up, 6 in 2023-08-28T02:40:35.066 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:34 smithi143 ceph-mon[189871]: failed to encode map e383 with expected crc 2023-08-28T02:40:35.066 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:34 smithi143 ceph-mon[189871]: failed to encode map e383 with expected crc 2023-08-28T02:40:35.066 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:34 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/3536452314' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:40:35.066 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:34 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:40:35.067 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:34 smithi143 ceph-mon[183059]: pgmap v1045: 89 pgs: 2 remapped+peering, 3 peering, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 50 MiB/s wr, 800 op/s; 4143/118767 objects degraded (3.488%); 5.7 MiB/s, 91 objects/s recovering 2023-08-28T02:40:35.067 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:34 smithi143 ceph-mon[183059]: failed to encode map e382 with expected crc 2023-08-28T02:40:35.068 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:34 smithi143 ceph-mon[183059]: failed to encode map e383 with expected crc 2023-08-28T02:40:35.068 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:34 smithi143 ceph-mon[183059]: failed to encode map e383 with expected crc 2023-08-28T02:40:35.068 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:34 smithi143 ceph-mon[183059]: osdmap e383: 8 total, 8 up, 6 in 2023-08-28T02:40:35.068 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:34 smithi143 ceph-mon[183059]: failed to encode map e383 with expected crc 2023-08-28T02:40:35.068 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:34 smithi143 ceph-mon[183059]: failed to encode map e383 with expected crc 2023-08-28T02:40:35.068 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:34 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/3536452314' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:40:35.068 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:34 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:40:35.166 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:40:35.200 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:35.267 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:40:35.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:35.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:40:35.373 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:40:35.399 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:35.664 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:40:35.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:35.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:40:35.738 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 32 16 43378 43362 84.6646 57.1875 0.0101685 0.0118062 2023-08-28T02:40:35.770 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:40:35.817 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:35.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:35 smithi169 ceph-mon[159415]: failed to encode map e383 with expected crc 2023-08-28T02:40:35.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:35 smithi169 ceph-mon[159415]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:40:35.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:35 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:40:35.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:35 smithi169 ceph-mon[159415]: osdmap e384: 8 total, 8 up, 6 in 2023-08-28T02:40:35.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:35 smithi169 ceph-mon[159415]: failed to encode map e384 with expected crc 2023-08-28T02:40:35.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:35 smithi169 ceph-mon[159415]: failed to encode map e384 with expected crc 2023-08-28T02:40:35.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:35 smithi169 ceph-mon[159415]: failed to encode map e384 with expected crc 2023-08-28T02:40:35.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:35 smithi169 ceph-mon[159415]: failed to encode map e384 with expected crc 2023-08-28T02:40:35.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:35 smithi169 ceph-mon[159415]: failed to encode map e384 with expected crc 2023-08-28T02:40:36.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:35 smithi143 ceph-mon[183059]: failed to encode map e383 with expected crc 2023-08-28T02:40:36.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:35 smithi143 ceph-mon[183059]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:40:36.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:35 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:40:36.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:35 smithi143 ceph-mon[183059]: osdmap e384: 8 total, 8 up, 6 in 2023-08-28T02:40:36.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:35 smithi143 ceph-mon[183059]: failed to encode map e384 with expected crc 2023-08-28T02:40:36.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:35 smithi143 ceph-mon[183059]: failed to encode map e384 with expected crc 2023-08-28T02:40:36.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:35 smithi143 ceph-mon[183059]: failed to encode map e384 with expected crc 2023-08-28T02:40:36.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:35 smithi143 ceph-mon[183059]: failed to encode map e384 with expected crc 2023-08-28T02:40:36.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:35 smithi143 ceph-mon[183059]: failed to encode map e384 with expected crc 2023-08-28T02:40:36.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:35 smithi143 ceph-mon[189871]: failed to encode map e383 with expected crc 2023-08-28T02:40:36.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:35 smithi143 ceph-mon[189871]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-08-28T02:40:36.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:35 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-08-28T02:40:36.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:35 smithi143 ceph-mon[189871]: osdmap e384: 8 total, 8 up, 6 in 2023-08-28T02:40:36.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:35 smithi143 ceph-mon[189871]: failed to encode map e384 with expected crc 2023-08-28T02:40:36.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:35 smithi143 ceph-mon[189871]: failed to encode map e384 with expected crc 2023-08-28T02:40:36.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:35 smithi143 ceph-mon[189871]: failed to encode map e384 with expected crc 2023-08-28T02:40:36.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:35 smithi143 ceph-mon[189871]: failed to encode map e384 with expected crc 2023-08-28T02:40:36.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:35 smithi143 ceph-mon[189871]: failed to encode map e384 with expected crc 2023-08-28T02:40:36.449 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:40:36.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:36.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:40:36.560 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:40:36.641 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:36.739 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 33 16 44203 44187 83.6616 51.5625 0.0126411 0.0119445 2023-08-28T02:40:36.749 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:40:36.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:36.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:40:36.856 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:40:36.923 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:37.016 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:40:37.020 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:40:37.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:37.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:40:37.054 INFO:teuthology.orchestra.run.smithi169.stdout:{ 2023-08-28T02:40:37.054 INFO:teuthology.orchestra.run.smithi169.stdout: "ops": [], 2023-08-28T02:40:37.054 INFO:teuthology.orchestra.run.smithi169.stdout: "complaint_time": 30, 2023-08-28T02:40:37.054 INFO:teuthology.orchestra.run.smithi169.stdout: "num_blocked_ops": 0 2023-08-28T02:40:37.054 INFO:teuthology.orchestra.run.smithi169.stdout:} 2023-08-28T02:40:37.060 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:40:37.076 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:36 smithi169 ceph-mon[159415]: pgmap v1048: 89 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 2 remapped+peering, 3 peering, 3 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.6 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 52 MiB/s wr, 827 op/s; 3415/125673 objects degraded (2.717%); 3217/125673 objects misplaced (2.560%); 8.8 MiB/s, 2 keys/s, 144 objects/s recovering 2023-08-28T02:40:37.076 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:36 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 3415/125673 objects degraded (2.717%), 4 pgs degraded (PG_DEGRADED) 2023-08-28T02:40:37.093 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:40:37.128 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:40:37.129 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:40:37.156 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:40:37.186 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:37.192 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:40:37.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:36 smithi143 ceph-mon[189871]: pgmap v1048: 89 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 2 remapped+peering, 3 peering, 3 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.6 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 52 MiB/s wr, 827 op/s; 3415/125673 objects degraded (2.717%); 3217/125673 objects misplaced (2.560%); 8.8 MiB/s, 2 keys/s, 144 objects/s recovering 2023-08-28T02:40:37.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:36 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 3415/125673 objects degraded (2.717%), 4 pgs degraded (PG_DEGRADED) 2023-08-28T02:40:37.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:36 smithi143 ceph-mon[183059]: pgmap v1048: 89 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 2 remapped+peering, 3 peering, 3 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.6 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 52 MiB/s wr, 827 op/s; 3415/125673 objects degraded (2.717%); 3217/125673 objects misplaced (2.560%); 8.8 MiB/s, 2 keys/s, 144 objects/s recovering 2023-08-28T02:40:37.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:36 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 3415/125673 objects degraded (2.717%), 4 pgs degraded (PG_DEGRADED) 2023-08-28T02:40:37.236 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:40:37.270 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:40:37.285 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:40:37.285 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:40:37.286 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:40:37.286 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:40:37.286 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:40:37.286 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:40:37.286 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:40:37.286 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:40:37.315 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:40:37.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:37.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:40:37.422 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:40:37.466 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:37.519 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:40:37.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:37.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:40:37.622 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:40:37.650 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:37.711 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:40:37.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:37.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:40:37.738 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 34 16 45122 45106 82.8902 57.4375 0.00743055 0.0120581 2023-08-28T02:40:37.817 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:40:37.884 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:37.950 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:37 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/4258475747' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:40:37.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:37 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/4258475747' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:40:38.123 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:37 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/4258475747' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:40:38.142 DEBUG:teuthology.orchestra.run.smithi169:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.7 dump_historic_ops 2023-08-28T02:40:38.143 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:40:38.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:38.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:40:38.249 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:40:38.297 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:38.395 INFO:tasks.thrashosds.thrasher:in_osds: [0, 5, 3, 6, 1, 7] out_osds: [4, 2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-08-28T02:40:38.395 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-08-28T02:40:38.395 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_4 2023-08-28T02:40:38.395 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph pg dump --format=json 2023-08-28T02:40:38.404 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:40:38.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:38.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:40:38.510 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:40:38.554 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:38.558 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set noscrub 2023-08-28T02:40:38.639 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:40:38.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:38.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:40:38.679 INFO:teuthology.orchestra.run.smithi169.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.b/config 2023-08-28T02:40:38.741 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 35 16 46046 46030 82.1673 57.75 0.0130656 0.0121632 2023-08-28T02:40:38.746 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:40:38.815 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:38 smithi143 ceph-mon[183059]: pgmap v1049: 89 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 2 remapped+peering, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.6 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 35 MiB/s wr, 557 op/s; 3424/125673 objects degraded (2.725%); 3217/125673 objects misplaced (2.560%); 5.9 MiB/s, 1 keys/s, 97 objects/s recovering 2023-08-28T02:40:38.867 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:38.981 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:40:38.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:38.992 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:40:39.092 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:40:39.103 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:38 smithi169 ceph-mon[159415]: pgmap v1049: 89 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 2 remapped+peering, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.6 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 35 MiB/s wr, 557 op/s; 3424/125673 objects degraded (2.725%); 3217/125673 objects misplaced (2.560%); 5.9 MiB/s, 1 keys/s, 97 objects/s recovering 2023-08-28T02:40:39.137 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:39.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:38 smithi143 ceph-mon[189871]: pgmap v1049: 89 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 2 remapped+peering, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.6 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 35 MiB/s wr, 557 op/s; 3424/125673 objects degraded (2.725%); 3217/125673 objects misplaced (2.560%); 5.9 MiB/s, 1 keys/s, 97 objects/s recovering 2023-08-28T02:40:39.206 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:40:39.213 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:40:39.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:39.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:40:39.293 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:40:39.319 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:40:39.388 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:39.577 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:40:39.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:39.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:40:39.683 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:40:39.712 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:39.741 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 36 16 46833 46817 81.2513 49.1875 0.0042276 0.0122454 2023-08-28T02:40:39.965 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:40:39.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:39.985 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:40:40.086 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:40:40.153 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:40.466 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:40:40.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:40.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:40:40.576 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:40:40.636 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:40.711 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:40:40.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:40.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:40:40.741 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 37 16 47830 47814 80.7394 62.3125 0.0115677 0.0123637 2023-08-28T02:40:40.764 INFO:teuthology.orchestra.run.smithi169.stdout:ERROR: (22) Invalid argument 2023-08-28T02:40:40.764 INFO:teuthology.orchestra.run.smithi169.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:40:40.814 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:40:40.878 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:40.986 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:40:41.062 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:40:41.064 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:40 smithi169 ceph-mon[159415]: pgmap v1050: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.8 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s; 8548/136695 objects degraded (6.253%); 2716/136695 objects misplaced (1.987%); 9.6 MiB/s, 1 keys/s, 155 objects/s recovering 2023-08-28T02:40:41.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:41.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:40:41.086 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:40 smithi143 ceph-mon[183059]: pgmap v1050: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.8 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s; 8548/136695 objects degraded (6.253%); 2716/136695 objects misplaced (1.987%); 9.6 MiB/s, 1 keys/s, 155 objects/s recovering 2023-08-28T02:40:41.087 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:40 smithi143 ceph-mon[189871]: pgmap v1050: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.8 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s; 8548/136695 objects degraded (6.253%); 2716/136695 objects misplaced (1.987%); 9.6 MiB/s, 1 keys/s, 155 objects/s recovering 2023-08-28T02:40:41.166 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:40:41.227 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:41.696 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.0 dump_ops_in_flight 2023-08-28T02:40:41.741 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 38 16 48825 48809 80.2511 62.1875 0.0161087 0.0124566 2023-08-28T02:40:42.056 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:40:41 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:40:41] "GET /metrics HTTP/1.1" 200 36593 "" "Prometheus/2.33.4" 2023-08-28T02:40:42.234 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:40:42.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:42.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:40:42.338 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:42 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:40:42.338 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:42 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:40:42.338 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:42 smithi143 ceph-mon[189871]: pgmap v1051: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.8 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 53 MiB/s wr, 850 op/s; 8548/139194 objects degraded (6.141%); 2716/139194 objects misplaced (1.951%); 5.5 MiB/s, 1 keys/s, 89 objects/s recovering 2023-08-28T02:40:42.338 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:42 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 8548/136695 objects degraded (6.253%), 7 pgs degraded (PG_DEGRADED) 2023-08-28T02:40:42.339 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:42 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:40:42.339 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:42 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:40:42.339 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:42 smithi143 ceph-mon[183059]: pgmap v1051: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.8 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 53 MiB/s wr, 850 op/s; 8548/139194 objects degraded (6.141%); 2716/139194 objects misplaced (1.951%); 5.5 MiB/s, 1 keys/s, 89 objects/s recovering 2023-08-28T02:40:42.339 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:42 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 8548/136695 objects degraded (6.253%), 7 pgs degraded (PG_DEGRADED) 2023-08-28T02:40:42.345 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:40:42.419 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:42.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:42 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:40:42.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:42 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:40:42.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:42 smithi169 ceph-mon[159415]: pgmap v1051: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.8 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 53 MiB/s wr, 850 op/s; 8548/139194 objects degraded (6.141%); 2716/139194 objects misplaced (1.951%); 5.5 MiB/s, 1 keys/s, 89 objects/s recovering 2023-08-28T02:40:42.487 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:42 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 8548/136695 objects degraded (6.253%), 7 pgs degraded (PG_DEGRADED) 2023-08-28T02:40:42.513 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:40:42.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:42.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:40:42.630 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:40:42.706 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:42.742 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 39 16 49844 49828 79.8234 63.6875 0.0183632 0.012521 2023-08-28T02:40:42.860 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:40:42.865 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:40:42.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:42.869 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:40:42.970 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:40:43.014 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:43.078 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:40:43.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:43.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:40:43.111 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:40:43.188 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:40:43.307 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:43.375 INFO:teuthology.orchestra.run.smithi143.stdout: 2023-08-28T02:40:43.403 INFO:teuthology.orchestra.run.smithi143.stderr:dumped all 2023-08-28T02:40:43.613 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:40:43.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:43.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:40:43.652 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is set 2023-08-28T02:40:43.726 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:40:43.745 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:2023-08-28T02:40:43.746544+0000 min lat: 0.00102595 max lat: 2.1786 avg lat: 0.0126253 2023-08-28T02:40:43.745 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-08-28T02:40:43.745 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 40 16 50700 50684 79.1606 53.5 0.0228674 0.0126253 2023-08-28T02:40:43.801 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:43 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/963885581' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:40:43.802 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:43 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:40:43.802 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:43 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/963885581' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:40:43.802 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:43 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:40:43.813 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:43.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:43 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/963885581' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:40:43.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:43 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-08-28T02:40:44.747 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 41 16 51746 51730 78.8203 65.375 0.00762124 0.0126816 2023-08-28T02:40:44.950 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:44 smithi143 ceph-mon[189871]: from='client.56011 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:40:44.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:44 smithi143 ceph-mon[189871]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:40:44.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:44 smithi143 ceph-mon[189871]: failed to encode map e385 with expected crc 2023-08-28T02:40:44.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:44 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:40:44.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:44 smithi143 ceph-mon[189871]: osdmap e385: 8 total, 8 up, 6 in 2023-08-28T02:40:44.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:44 smithi143 ceph-mon[189871]: failed to encode map e385 with expected crc 2023-08-28T02:40:44.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:44 smithi143 ceph-mon[189871]: failed to encode map e385 with expected crc 2023-08-28T02:40:44.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:44 smithi143 ceph-mon[189871]: failed to encode map e385 with expected crc 2023-08-28T02:40:44.951 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:44 smithi143 ceph-mon[189871]: pgmap v1053: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 46 MiB/s wr, 741 op/s; 8548/143667 objects degraded (5.950%); 2716/143667 objects misplaced (1.890%); 3.5 MiB/s, 56 objects/s recovering 2023-08-28T02:40:44.951 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:44 smithi143 ceph-mon[183059]: from='client.56011 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:40:44.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:44 smithi143 ceph-mon[183059]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:40:44.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:44 smithi143 ceph-mon[183059]: failed to encode map e385 with expected crc 2023-08-28T02:40:44.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:44 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:40:44.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:44 smithi143 ceph-mon[183059]: osdmap e385: 8 total, 8 up, 6 in 2023-08-28T02:40:44.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:44 smithi143 ceph-mon[183059]: failed to encode map e385 with expected crc 2023-08-28T02:40:44.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:44 smithi143 ceph-mon[183059]: failed to encode map e385 with expected crc 2023-08-28T02:40:44.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:44 smithi143 ceph-mon[183059]: failed to encode map e385 with expected crc 2023-08-28T02:40:44.952 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:44 smithi143 ceph-mon[183059]: pgmap v1053: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 46 MiB/s wr, 741 op/s; 8548/143667 objects degraded (5.950%); 2716/143667 objects misplaced (1.890%); 3.5 MiB/s, 56 objects/s recovering 2023-08-28T02:40:44.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:44 smithi169 ceph-mon[159415]: from='client.56011 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-28T02:40:44.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:44 smithi169 ceph-mon[159415]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:40:44.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:44 smithi169 ceph-mon[159415]: failed to encode map e385 with expected crc 2023-08-28T02:40:44.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:44 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-08-28T02:40:44.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:44 smithi169 ceph-mon[159415]: osdmap e385: 8 total, 8 up, 6 in 2023-08-28T02:40:44.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:44 smithi169 ceph-mon[159415]: failed to encode map e385 with expected crc 2023-08-28T02:40:44.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:44 smithi169 ceph-mon[159415]: failed to encode map e385 with expected crc 2023-08-28T02:40:44.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:44 smithi169 ceph-mon[159415]: failed to encode map e385 with expected crc 2023-08-28T02:40:44.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:44 smithi169 ceph-mon[159415]: pgmap v1053: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 46 MiB/s wr, 741 op/s; 8548/143667 objects degraded (5.950%); 2716/143667 objects misplaced (1.890%); 3.5 MiB/s, 56 objects/s recovering 2023-08-28T02:40:45.747 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 42 16 52357 52341 77.8531 38.1875 0.117641 0.0128292 2023-08-28T02:40:46.747 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 43 16 53029 53013 77.0195 42 0.0176403 0.0129786 2023-08-28T02:40:47.098 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:46 smithi143 ceph-mon[183059]: pgmap v1054: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 58 MiB/s wr, 927 op/s; 8548/153513 objects degraded (5.568%); 2302/153513 objects misplaced (1.500%); 5.7 MiB/s, 91 objects/s recovering 2023-08-28T02:40:47.099 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:46 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 8548/153513 objects degraded (5.568%), 7 pgs degraded (PG_DEGRADED) 2023-08-28T02:40:47.099 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:46 smithi143 ceph-mon[189871]: pgmap v1054: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 58 MiB/s wr, 927 op/s; 8548/153513 objects degraded (5.568%); 2302/153513 objects misplaced (1.500%); 5.7 MiB/s, 91 objects/s recovering 2023-08-28T02:40:47.099 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:46 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 8548/153513 objects degraded (5.568%), 7 pgs degraded (PG_DEGRADED) 2023-08-28T02:40:47.185 INFO:teuthology.orchestra.run.smithi143.stdout:{"pg_ready":true,"pg_map":{"version":1051,"stamp":"2023-08-28T02:40:41.682705+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3043694515,"num_objects":46398,"num_object_clones":0,"num_object_copies":139194,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8548,"num_objects_misplaced":2716,"num_objects_unfound":0,"num_objects_dirty":46398,"num_whiteouts":0,"num_read":6274,"num_read_kb":13707,"num_write":47476,"num_write_kb":2982821,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2006,"num_bytes_recovered":121345904,"num_keys_recovered":50,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":47540,"ondisk_log_size":47540,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":12039152,"kb_used_data":8234688,"kb_used_omap":27,"kb_used_meta":3804388,"kb_avail":550431760,"statfs":{"total":575970213888,"available":563642122240,"internally_reserved":0,"allocated":8432320512,"data_stored":8424007358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28409,"internal_metadata":3895693575},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,0,0,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":446234624,"num_objects":6809,"num_object_clones":0,"num_object_copies":20427,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4405,"num_objects_misplaced":2716,"num_objects_unfound":0,"num_objects_dirty":6809,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6809,"num_write_kb":435776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":718,"num_bytes_recovered":45946058,"num_keys_recovered":10,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.003654"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":331,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.743953+0000","last_change":"2023-08-28T02:40:32.756907+0000","last_active":"2023-08-28T02:40:33.743953+0000","last_peered":"2023-08-28T02:40:33.743953+0000","last_clean":"2023-08-28T02:40:33.743953+0000","last_became_active":"2023-08-28T02:40:32.756096+0000","last_became_peered":"2023-08-28T02:40:32.756096+0000","last_unstale":"2023-08-28T02:40:33.743953+0000","last_undegraded":"2023-08-28T02:40:33.743953+0000","last_fullsized":"2023-08-28T02:40:33.743953+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:52:03.101633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":301,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.747306+0000","last_change":"2023-08-28T02:40:18.795155+0000","last_active":"2023-08-28T02:40:33.747306+0000","last_peered":"2023-08-28T02:40:33.747306+0000","last_clean":"2023-08-28T02:40:33.747306+0000","last_became_active":"2023-08-28T02:40:18.794789+0000","last_became_peered":"2023-08-28T02:40:18.794789+0000","last_unstale":"2023-08-28T02:40:33.747306+0000","last_undegraded":"2023-08-28T02:40:33.747306+0000","last_fullsized":"2023-08-28T02:40:33.747306+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:00:46.533597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":115,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.743693+0000","last_change":"2023-08-28T02:40:18.791599+0000","last_active":"2023-08-28T02:40:33.743693+0000","last_peered":"2023-08-28T02:40:33.743693+0000","last_clean":"2023-08-28T02:40:33.743693+0000","last_became_active":"2023-08-28T02:40:18.791292+0000","last_became_peered":"2023-08-28T02:40:18.791292+0000","last_unstale":"2023-08-28T02:40:33.743693+0000","last_undegraded":"2023-08-28T02:40:33.743693+0000","last_fullsized":"2023-08-28T02:40:33.743693+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:02:16.854523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":413,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-08-28T02:40:34.799594+0000","last_change":"2023-08-28T02:37:45.580751+0000","last_active":"2023-08-28T02:40:34.799594+0000","last_peered":"2023-08-28T02:40:34.799594+0000","last_clean":"2023-08-28T02:40:34.799594+0000","last_became_active":"2023-08-28T02:37:45.580446+0000","last_became_peered":"2023-08-28T02:37:45.580446+0000","last_unstale":"2023-08-28T02:40:34.799594+0000","last_undegraded":"2023-08-28T02:40:34.799594+0000","last_fullsized":"2023-08-28T02:40:34.799594+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:59:21.094880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":283,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.746575+0000","last_change":"2023-08-28T02:32:58.779602+0000","last_active":"2023-08-28T02:40:33.746575+0000","last_peered":"2023-08-28T02:40:33.746575+0000","last_clean":"2023-08-28T02:40:33.746575+0000","last_became_active":"2023-08-28T02:32:58.778945+0000","last_became_peered":"2023-08-28T02:32:58.778945+0000","last_unstale":"2023-08-28T02:40:33.746575+0000","last_undegraded":"2023-08-28T02:40:33.746575+0000","last_fullsized":"2023-08-28T02:40:33.746575+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:35:55.832928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":190,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.750544+0000","last_change":"2023-08-28T02:32:58.626766+0000","last_active":"2023-08-28T02:40:33.750544+0000","last_peered":"2023-08-28T02:40:33.750544+0000","last_clean":"2023-08-28T02:40:33.750544+0000","last_became_active":"2023-08-28T02:32:58.626415+0000","last_became_peered":"2023-08-28T02:32:58.626415+0000","last_unstale":"2023-08-28T02:40:33.750544+0000","last_undegraded":"2023-08-28T02:40:33.750544+0000","last_fullsized":"2023-08-28T02:40:33.750544+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":356,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.770611+0000","last_change":"2023-08-28T02:23:59.227592+0000","last_active":"2023-08-28T02:40:33.770611+0000","last_peered":"2023-08-28T02:40:33.770611+0000","last_clean":"2023-08-28T02:40:33.770611+0000","last_became_active":"2023-08-28T02:23:59.227342+0000","last_became_peered":"2023-08-28T02:23:59.227342+0000","last_unstale":"2023-08-28T02:40:33.770611+0000","last_undegraded":"2023-08-28T02:40:33.770611+0000","last_fullsized":"2023-08-28T02:40:33.770611+0000","mapping_epoch":153,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:08:19.874396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":302,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.746651+0000","last_change":"2023-08-28T02:32:58.779435+0000","last_active":"2023-08-28T02:40:33.746651+0000","last_peered":"2023-08-28T02:40:33.746651+0000","last_clean":"2023-08-28T02:40:33.746651+0000","last_became_active":"2023-08-28T02:32:58.778629+0000","last_became_peered":"2023-08-28T02:32:58.778629+0000","last_unstale":"2023-08-28T02:40:33.746651+0000","last_undegraded":"2023-08-28T02:40:33.746651+0000","last_fullsized":"2023-08-28T02:40:33.746651+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:25:32.193898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":49,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.748543+0000","last_change":"2023-08-28T02:37:45.579563+0000","last_active":"2023-08-28T02:40:33.748543+0000","last_peered":"2023-08-28T02:40:33.748543+0000","last_clean":"2023-08-28T02:40:33.748543+0000","last_became_active":"2023-08-28T02:37:45.579133+0000","last_became_peered":"2023-08-28T02:37:45.579133+0000","last_unstale":"2023-08-28T02:40:33.748543+0000","last_undegraded":"2023-08-28T02:40:33.748543+0000","last_fullsized":"2023-08-28T02:40:33.748543+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":284,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.747455+0000","last_change":"2023-08-28T02:37:45.580277+0000","last_active":"2023-08-28T02:40:33.747455+0000","last_peered":"2023-08-28T02:40:33.747455+0000","last_clean":"2023-08-28T02:40:33.747455+0000","last_became_active":"2023-08-28T02:37:45.579888+0000","last_became_peered":"2023-08-28T02:37:45.579888+0000","last_unstale":"2023-08-28T02:40:33.747455+0000","last_undegraded":"2023-08-28T02:40:33.747455+0000","last_fullsized":"2023-08-28T02:40:33.747455+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:18:54.085955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":70,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.748507+0000","last_change":"2023-08-28T02:40:19.210077+0000","last_active":"2023-08-28T02:40:33.748507+0000","last_peered":"2023-08-28T02:40:33.748507+0000","last_clean":"2023-08-28T02:40:33.748507+0000","last_became_active":"2023-08-28T02:40:18.793311+0000","last_became_peered":"2023-08-28T02:40:18.793311+0000","last_unstale":"2023-08-28T02:40:33.748507+0000","last_undegraded":"2023-08-28T02:40:33.748507+0000","last_fullsized":"2023-08-28T02:40:33.748507+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":142,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.750478+0000","last_change":"2023-08-28T02:40:32.755751+0000","last_active":"2023-08-28T02:40:33.750478+0000","last_peered":"2023-08-28T02:40:33.750478+0000","last_clean":"2023-08-28T02:40:33.750478+0000","last_became_active":"2023-08-28T02:40:32.755410+0000","last_became_peered":"2023-08-28T02:40:32.755410+0000","last_unstale":"2023-08-28T02:40:33.750478+0000","last_undegraded":"2023-08-28T02:40:33.750478+0000","last_fullsized":"2023-08-28T02:40:33.750478+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"86'22","reported_seq":463,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.748345+0000","last_change":"2023-08-28T02:40:32.937707+0000","last_active":"2023-08-28T02:40:33.748345+0000","last_peered":"2023-08-28T02:40:33.748345+0000","last_clean":"2023-08-28T02:40:33.748345+0000","last_became_active":"2023-08-28T02:40:32.752388+0000","last_became_peered":"2023-08-28T02:40:32.752388+0000","last_unstale":"2023-08-28T02:40:33.748345+0000","last_undegraded":"2023-08-28T02:40:33.748345+0000","last_fullsized":"2023-08-28T02:40:33.748345+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":33138,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":346,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.750805+0000","last_change":"2023-08-28T02:40:18.793531+0000","last_active":"2023-08-28T02:40:33.750805+0000","last_peered":"2023-08-28T02:40:33.750805+0000","last_clean":"2023-08-28T02:40:33.750805+0000","last_became_active":"2023-08-28T02:40:18.793062+0000","last_became_peered":"2023-08-28T02:40:18.793062+0000","last_unstale":"2023-08-28T02:40:33.750805+0000","last_undegraded":"2023-08-28T02:40:33.750805+0000","last_fullsized":"2023-08-28T02:40:33.750805+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":115,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.748453+0000","last_change":"2023-08-28T02:40:33.031300+0000","last_active":"2023-08-28T02:40:33.748453+0000","last_peered":"2023-08-28T02:40:33.748453+0000","last_clean":"2023-08-28T02:40:33.748453+0000","last_became_active":"2023-08-28T02:40:32.754783+0000","last_became_peered":"2023-08-28T02:40:32.754783+0000","last_unstale":"2023-08-28T02:40:33.748453+0000","last_undegraded":"2023-08-28T02:40:33.748453+0000","last_fullsized":"2023-08-28T02:40:33.748453+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":302,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.747888+0000","last_change":"2023-08-28T02:37:45.579069+0000","last_active":"2023-08-28T02:40:33.747888+0000","last_peered":"2023-08-28T02:40:33.747888+0000","last_clean":"2023-08-28T02:40:33.747888+0000","last_became_active":"2023-08-28T02:37:45.578728+0000","last_became_peered":"2023-08-28T02:37:45.578728+0000","last_unstale":"2023-08-28T02:40:33.747888+0000","last_undegraded":"2023-08-28T02:40:33.747888+0000","last_fullsized":"2023-08-28T02:40:33.747888+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:58:05.172104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":410,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.771406+0000","last_change":"2023-08-28T02:37:45.579678+0000","last_active":"2023-08-28T02:40:33.771406+0000","last_peered":"2023-08-28T02:40:33.771406+0000","last_clean":"2023-08-28T02:40:33.771406+0000","last_became_active":"2023-08-28T02:37:45.579390+0000","last_became_peered":"2023-08-28T02:37:45.579390+0000","last_unstale":"2023-08-28T02:40:33.771406+0000","last_undegraded":"2023-08-28T02:40:33.771406+0000","last_fullsized":"2023-08-28T02:40:33.771406+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:31:04.974167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":49,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.748488+0000","last_change":"2023-08-28T02:37:45.584893+0000","last_active":"2023-08-28T02:40:33.748488+0000","last_peered":"2023-08-28T02:40:33.748488+0000","last_clean":"2023-08-28T02:40:33.748488+0000","last_became_active":"2023-08-28T02:37:45.584568+0000","last_became_peered":"2023-08-28T02:37:45.584568+0000","last_unstale":"2023-08-28T02:40:33.748488+0000","last_undegraded":"2023-08-28T02:40:33.748488+0000","last_fullsized":"2023-08-28T02:40:33.748488+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":58,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.748186+0000","last_change":"2023-08-28T02:40:18.787361+0000","last_active":"2023-08-28T02:40:33.748186+0000","last_peered":"2023-08-28T02:40:33.748186+0000","last_clean":"2023-08-28T02:40:33.748186+0000","last_became_active":"2023-08-28T02:40:18.787023+0000","last_became_peered":"2023-08-28T02:40:18.787023+0000","last_unstale":"2023-08-28T02:40:33.748186+0000","last_undegraded":"2023-08-28T02:40:33.748186+0000","last_fullsized":"2023-08-28T02:40:33.748186+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":393,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.771173+0000","last_change":"2023-08-28T02:37:45.580634+0000","last_active":"2023-08-28T02:40:33.771173+0000","last_peered":"2023-08-28T02:40:33.771173+0000","last_clean":"2023-08-28T02:40:33.771173+0000","last_became_active":"2023-08-28T02:37:45.580322+0000","last_became_peered":"2023-08-28T02:37:45.580322+0000","last_unstale":"2023-08-28T02:40:33.771173+0000","last_undegraded":"2023-08-28T02:40:33.771173+0000","last_fullsized":"2023-08-28T02:40:33.771173+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:24:37.014061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":182,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.749794+0000","last_change":"2023-08-28T02:28:25.703333+0000","last_active":"2023-08-28T02:40:33.749794+0000","last_peered":"2023-08-28T02:40:33.749794+0000","last_clean":"2023-08-28T02:40:33.749794+0000","last_became_active":"2023-08-28T02:28:25.343515+0000","last_became_peered":"2023-08-28T02:28:25.343515+0000","last_unstale":"2023-08-28T02:40:33.749794+0000","last_undegraded":"2023-08-28T02:40:33.749794+0000","last_fullsized":"2023-08-28T02:40:33.749794+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":310,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.748055+0000","last_change":"2023-08-28T02:37:45.579694+0000","last_active":"2023-08-28T02:40:33.748055+0000","last_peered":"2023-08-28T02:40:33.748055+0000","last_clean":"2023-08-28T02:40:33.748055+0000","last_became_active":"2023-08-28T02:37:45.579219+0000","last_became_peered":"2023-08-28T02:37:45.579219+0000","last_unstale":"2023-08-28T02:40:33.748055+0000","last_undegraded":"2023-08-28T02:40:33.748055+0000","last_fullsized":"2023-08-28T02:40:33.748055+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:22:51.406552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"384'2961","reported_seq":2986,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-08-28T02:40:40.179498+0000","last_change":"2023-08-28T02:39:56.814577+0000","last_active":"2023-08-28T02:40:40.179498+0000","last_peered":"2023-08-28T02:40:40.179498+0000","last_clean":"2023-08-28T02:40:40.179498+0000","last_became_active":"2023-08-28T02:39:56.814271+0000","last_became_peered":"2023-08-28T02:39:56.814271+0000","last_unstale":"2023-08-28T02:40:40.179498+0000","last_undegraded":"2023-08-28T02:40:40.179498+0000","last_fullsized":"2023-08-28T02:40:40.179498+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":368,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:39:55.797153+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:39:55.797153+0000","last_clean_scrub_stamp":"2023-08-28T02:39:55.797153+0000","objects_scrubbed":0,"log_size":2961,"log_dups_size":0,"ondisk_log_size":2961,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:14:22.829693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194052096,"num_objects":2961,"num_object_clones":0,"num_object_copies":8883,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2961,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2961,"num_write_kb":189504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":79,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.748731+0000","last_change":"2023-08-28T02:40:18.858032+0000","last_active":"2023-08-28T02:40:33.748731+0000","last_peered":"2023-08-28T02:40:33.748731+0000","last_clean":"2023-08-28T02:40:33.748731+0000","last_became_active":"2023-08-28T02:40:18.788333+0000","last_became_peered":"2023-08-28T02:40:18.788333+0000","last_unstale":"2023-08-28T02:40:33.748731+0000","last_undegraded":"2023-08-28T02:40:33.748731+0000","last_fullsized":"2023-08-28T02:40:33.748731+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":49,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.748099+0000","last_change":"2023-08-28T02:37:45.585246+0000","last_active":"2023-08-28T02:40:33.748099+0000","last_peered":"2023-08-28T02:40:33.748099+0000","last_clean":"2023-08-28T02:40:33.748099+0000","last_became_active":"2023-08-28T02:37:45.584391+0000","last_became_peered":"2023-08-28T02:37:45.584391+0000","last_unstale":"2023-08-28T02:40:33.748099+0000","last_undegraded":"2023-08-28T02:40:33.748099+0000","last_fullsized":"2023-08-28T02:40:33.748099+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.4","version":"384'2926","reported_seq":4930,"reported_epoch":384,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:40:38.651270+0000","last_change":"2023-08-28T02:40:32.758776+0000","last_active":"2023-08-28T02:40:38.651270+0000","last_peered":"2023-08-28T02:40:38.651270+0000","last_clean":"2023-08-28T02:40:17.747482+0000","last_became_active":"2023-08-28T02:40:19.830259+0000","last_became_peered":"2023-08-28T02:40:19.830259+0000","last_unstale":"2023-08-28T02:40:38.651270+0000","last_undegraded":"2023-08-28T02:40:32.757384+0000","last_fullsized":"2023-08-28T02:40:19.765642+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":368,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:39:55.797153+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:39:55.797153+0000","last_clean_scrub_stamp":"2023-08-28T02:39:55.797153+0000","objects_scrubbed":0,"log_size":2926,"log_dups_size":0,"ondisk_log_size":2926,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:11:14.824946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191758336,"num_objects":2926,"num_object_clones":0,"num_object_copies":8778,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":726,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2926,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2926,"num_write_kb":187264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":985,"num_bytes_recovered":64552960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3],"avail_no_missing":["1","3","4"],"object_location_counts":[{"shards":"1,3","objects":2200},{"shards":"1,3,4","objects":726}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":66,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.748455+0000","last_change":"2023-08-28T02:40:19.256270+0000","last_active":"2023-08-28T02:40:33.748455+0000","last_peered":"2023-08-28T02:40:33.748455+0000","last_clean":"2023-08-28T02:40:33.748455+0000","last_became_active":"2023-08-28T02:40:18.794981+0000","last_became_peered":"2023-08-28T02:40:18.794981+0000","last_unstale":"2023-08-28T02:40:33.748455+0000","last_undegraded":"2023-08-28T02:40:33.748455+0000","last_fullsized":"2023-08-28T02:40:33.748455+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":356,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.743919+0000","last_change":"2023-08-28T02:40:18.791117+0000","last_active":"2023-08-28T02:40:33.743919+0000","last_peered":"2023-08-28T02:40:33.743919+0000","last_clean":"2023-08-28T02:40:33.743919+0000","last_became_active":"2023-08-28T02:40:18.790782+0000","last_became_peered":"2023-08-28T02:40:18.790782+0000","last_unstale":"2023-08-28T02:40:33.743919+0000","last_undegraded":"2023-08-28T02:40:33.743919+0000","last_fullsized":"2023-08-28T02:40:33.743919+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:44:49.264567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.7","version":"384'2949","reported_seq":2973,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-08-28T02:40:38.921801+0000","last_change":"2023-08-28T02:39:56.827126+0000","last_active":"2023-08-28T02:40:38.921801+0000","last_peered":"2023-08-28T02:40:38.921801+0000","last_clean":"2023-08-28T02:40:38.921801+0000","last_became_active":"2023-08-28T02:39:56.826836+0000","last_became_peered":"2023-08-28T02:39:56.826836+0000","last_unstale":"2023-08-28T02:40:38.921801+0000","last_undegraded":"2023-08-28T02:40:38.921801+0000","last_fullsized":"2023-08-28T02:40:38.921801+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":368,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:39:55.797153+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:39:55.797153+0000","last_clean_scrub_stamp":"2023-08-28T02:39:55.797153+0000","objects_scrubbed":0,"log_size":2949,"log_dups_size":0,"ondisk_log_size":2949,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193265664,"num_objects":2949,"num_object_clones":0,"num_object_copies":8847,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2949,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2949,"num_write_kb":188736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":53,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.748382+0000","last_change":"2023-08-28T02:37:46.190365+0000","last_active":"2023-08-28T02:40:33.748382+0000","last_peered":"2023-08-28T02:40:33.748382+0000","last_clean":"2023-08-28T02:40:33.748382+0000","last_became_active":"2023-08-28T02:37:45.584408+0000","last_became_peered":"2023-08-28T02:37:45.584408+0000","last_unstale":"2023-08-28T02:40:33.748382+0000","last_undegraded":"2023-08-28T02:40:33.748382+0000","last_fullsized":"2023-08-28T02:40:33.748382+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":226,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.750403+0000","last_change":"2023-08-28T02:38:35.888252+0000","last_active":"2023-08-28T02:40:33.750403+0000","last_peered":"2023-08-28T02:40:33.750403+0000","last_clean":"2023-08-28T02:40:33.750403+0000","last_became_active":"2023-08-28T02:38:35.887901+0000","last_became_peered":"2023-08-28T02:38:35.887901+0000","last_unstale":"2023-08-28T02:40:33.750403+0000","last_undegraded":"2023-08-28T02:40:33.750403+0000","last_fullsized":"2023-08-28T02:40:33.750403+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"384'2835","reported_seq":2867,"reported_epoch":384,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:40:38.622233+0000","last_change":"2023-08-28T02:40:33.924182+0000","last_active":"2023-08-28T02:40:38.622233+0000","last_peered":"2023-08-28T02:40:38.622233+0000","last_clean":"2023-08-28T02:40:31.707617+0000","last_became_active":"2023-08-28T02:40:33.919046+0000","last_became_peered":"2023-08-28T02:40:33.919046+0000","last_unstale":"2023-08-28T02:40:38.622233+0000","last_undegraded":"2023-08-28T02:40:33.787918+0000","last_fullsized":"2023-08-28T02:40:33.744045+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":368,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:39:55.797153+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:39:55.797153+0000","last_clean_scrub_stamp":"2023-08-28T02:39:55.797153+0000","objects_scrubbed":0,"log_size":2835,"log_dups_size":0,"ondisk_log_size":2835,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:09:02.818532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185794560,"num_objects":2835,"num_object_clones":0,"num_object_copies":8505,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2556,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2835,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2835,"num_write_kb":181440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5],"avail_no_missing":["1","2","5"],"object_location_counts":[{"shards":"1,2,5","objects":2556},{"shards":"1,5","objects":279}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":390,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.747987+0000","last_change":"2023-08-28T02:28:25.445982+0000","last_active":"2023-08-28T02:40:33.747987+0000","last_peered":"2023-08-28T02:40:33.747987+0000","last_clean":"2023-08-28T02:40:33.747987+0000","last_became_active":"2023-08-28T02:28:25.443577+0000","last_became_peered":"2023-08-28T02:28:25.443577+0000","last_unstale":"2023-08-28T02:40:33.747987+0000","last_undegraded":"2023-08-28T02:40:33.747987+0000","last_fullsized":"2023-08-28T02:40:33.747987+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:24:03.381751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":395,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.770285+0000","last_change":"2023-08-28T02:40:18.789038+0000","last_active":"2023-08-28T02:40:33.770285+0000","last_peered":"2023-08-28T02:40:33.770285+0000","last_clean":"2023-08-28T02:40:33.770285+0000","last_became_active":"2023-08-28T02:40:18.788722+0000","last_became_peered":"2023-08-28T02:40:18.788722+0000","last_unstale":"2023-08-28T02:40:33.770285+0000","last_undegraded":"2023-08-28T02:40:33.770285+0000","last_fullsized":"2023-08-28T02:40:33.770285+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:00:29.658962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"384'2847","reported_seq":2872,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-08-28T02:40:39.193169+0000","last_change":"2023-08-28T02:39:56.817546+0000","last_active":"2023-08-28T02:40:39.193169+0000","last_peered":"2023-08-28T02:40:39.193169+0000","last_clean":"2023-08-28T02:40:39.193169+0000","last_became_active":"2023-08-28T02:39:56.817272+0000","last_became_peered":"2023-08-28T02:39:56.817272+0000","last_unstale":"2023-08-28T02:40:39.193169+0000","last_undegraded":"2023-08-28T02:40:39.193169+0000","last_fullsized":"2023-08-28T02:40:39.193169+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":368,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:39:55.797153+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:39:55.797153+0000","last_clean_scrub_stamp":"2023-08-28T02:39:55.797153+0000","objects_scrubbed":0,"log_size":2847,"log_dups_size":0,"ondisk_log_size":2847,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:47:51.610739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186580992,"num_objects":2847,"num_object_clones":0,"num_object_copies":8541,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2847,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2847,"num_write_kb":182208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":265,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.747220+0000","last_change":"2023-08-28T02:40:18.859764+0000","last_active":"2023-08-28T02:40:33.747220+0000","last_peered":"2023-08-28T02:40:33.747220+0000","last_clean":"2023-08-28T02:40:33.747220+0000","last_became_active":"2023-08-28T02:40:18.796061+0000","last_became_peered":"2023-08-28T02:40:18.796061+0000","last_unstale":"2023-08-28T02:40:33.747220+0000","last_undegraded":"2023-08-28T02:40:33.747220+0000","last_fullsized":"2023-08-28T02:40:33.747220+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:09:33.630861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":252,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.747938+0000","last_change":"2023-08-28T02:40:18.796332+0000","last_active":"2023-08-28T02:40:33.747938+0000","last_peered":"2023-08-28T02:40:33.747938+0000","last_clean":"2023-08-28T02:40:33.747938+0000","last_became_active":"2023-08-28T02:40:18.796020+0000","last_became_peered":"2023-08-28T02:40:18.796020+0000","last_unstale":"2023-08-28T02:40:33.747938+0000","last_undegraded":"2023-08-28T02:40:33.747938+0000","last_fullsized":"2023-08-28T02:40:33.747938+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:26:00.261091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.0","version":"384'2842","reported_seq":2875,"reported_epoch":384,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:40:38.642717+0000","last_change":"2023-08-28T02:40:33.855056+0000","last_active":"2023-08-28T02:40:38.642717+0000","last_peered":"2023-08-28T02:40:38.642717+0000","last_clean":"2023-08-28T02:40:31.709088+0000","last_became_active":"2023-08-28T02:40:33.849744+0000","last_became_peered":"2023-08-28T02:40:33.849744+0000","last_unstale":"2023-08-28T02:40:38.642717+0000","last_undegraded":"2023-08-28T02:40:33.765265+0000","last_fullsized":"2023-08-28T02:40:33.743258+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":368,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:39:55.797153+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:39:55.797153+0000","last_clean_scrub_stamp":"2023-08-28T02:39:55.797153+0000","objects_scrubbed":0,"log_size":2842,"log_dups_size":0,"ondisk_log_size":2842,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:33:50.979595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186253312,"num_objects":2842,"num_object_clones":0,"num_object_copies":8526,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2568,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2842,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2842,"num_write_kb":181888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"1,2,6","objects":2568},{"shards":"1,6","objects":274}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":199,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.750293+0000","last_change":"2023-08-28T02:40:18.790958+0000","last_active":"2023-08-28T02:40:33.750293+0000","last_peered":"2023-08-28T02:40:33.750293+0000","last_clean":"2023-08-28T02:40:33.750293+0000","last_became_active":"2023-08-28T02:40:18.790643+0000","last_became_peered":"2023-08-28T02:40:18.790643+0000","last_unstale":"2023-08-28T02:40:33.750293+0000","last_undegraded":"2023-08-28T02:40:33.750293+0000","last_fullsized":"2023-08-28T02:40:33.750293+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":466,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.749955+0000","last_change":"2023-08-28T02:40:32.755489+0000","last_active":"2023-08-28T02:40:33.749955+0000","last_peered":"2023-08-28T02:40:33.749955+0000","last_clean":"2023-08-28T02:40:33.749955+0000","last_became_active":"2023-08-28T02:40:32.755083+0000","last_became_peered":"2023-08-28T02:40:32.755083+0000","last_unstale":"2023-08-28T02:40:33.749955+0000","last_undegraded":"2023-08-28T02:40:33.749955+0000","last_fullsized":"2023-08-28T02:40:33.749955+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.3","version":"384'2917","reported_seq":2941,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-08-28T02:40:38.905758+0000","last_change":"2023-08-28T02:39:56.831175+0000","last_active":"2023-08-28T02:40:38.905758+0000","last_peered":"2023-08-28T02:40:38.905758+0000","last_clean":"2023-08-28T02:40:38.905758+0000","last_became_active":"2023-08-28T02:39:56.830898+0000","last_became_peered":"2023-08-28T02:39:56.830898+0000","last_unstale":"2023-08-28T02:40:38.905758+0000","last_undegraded":"2023-08-28T02:40:38.905758+0000","last_fullsized":"2023-08-28T02:40:38.905758+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":368,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:39:55.797153+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:39:55.797153+0000","last_clean_scrub_stamp":"2023-08-28T02:39:55.797153+0000","objects_scrubbed":0,"log_size":2917,"log_dups_size":0,"ondisk_log_size":2917,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191168512,"num_objects":2917,"num_object_clones":0,"num_object_copies":8751,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2917,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2917,"num_write_kb":186688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":338,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.748030+0000","last_change":"2023-08-28T02:40:32.759153+0000","last_active":"2023-08-28T02:40:33.748030+0000","last_peered":"2023-08-28T02:40:33.748030+0000","last_clean":"2023-08-28T02:40:33.748030+0000","last_became_active":"2023-08-28T02:40:32.758781+0000","last_became_peered":"2023-08-28T02:40:32.758781+0000","last_unstale":"2023-08-28T02:40:33.748030+0000","last_undegraded":"2023-08-28T02:40:33.748030+0000","last_fullsized":"2023-08-28T02:40:33.748030+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":288,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.749982+0000","last_change":"2023-08-28T02:40:18.792323+0000","last_active":"2023-08-28T02:40:33.749982+0000","last_peered":"2023-08-28T02:40:33.749982+0000","last_clean":"2023-08-28T02:40:33.749982+0000","last_became_active":"2023-08-28T02:40:18.791985+0000","last_became_peered":"2023-08-28T02:40:18.791985+0000","last_unstale":"2023-08-28T02:40:33.749982+0000","last_undegraded":"2023-08-28T02:40:33.749982+0000","last_fullsized":"2023-08-28T02:40:33.749982+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.2","version":"384'2978","reported_seq":3010,"reported_epoch":384,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:40:39.208408+0000","last_change":"2023-08-28T02:40:33.840389+0000","last_active":"2023-08-28T02:40:39.208408+0000","last_peered":"2023-08-28T02:40:39.208408+0000","last_clean":"2023-08-28T02:40:31.724493+0000","last_became_active":"2023-08-28T02:40:33.832905+0000","last_became_peered":"2023-08-28T02:40:33.832905+0000","last_unstale":"2023-08-28T02:40:39.208408+0000","last_undegraded":"2023-08-28T02:40:33.762276+0000","last_fullsized":"2023-08-28T02:40:33.746790+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":368,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:39:55.797153+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:39:55.797153+0000","last_clean_scrub_stamp":"2023-08-28T02:39:55.797153+0000","objects_scrubbed":0,"log_size":2978,"log_dups_size":0,"ondisk_log_size":2978,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:12:32.581550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195166208,"num_objects":2978,"num_object_clones":0,"num_object_copies":8934,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2673,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2978,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2978,"num_write_kb":190592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1],"avail_no_missing":["3","1","2"],"object_location_counts":[{"shards":"1,2,3","objects":2673},{"shards":"1,3","objects":305}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":302,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.746933+0000","last_change":"2023-08-28T02:37:45.579112+0000","last_active":"2023-08-28T02:40:33.746933+0000","last_peered":"2023-08-28T02:40:33.746933+0000","last_clean":"2023-08-28T02:40:33.746933+0000","last_became_active":"2023-08-28T02:37:45.578787+0000","last_became_peered":"2023-08-28T02:37:45.578787+0000","last_unstale":"2023-08-28T02:40:33.746933+0000","last_undegraded":"2023-08-28T02:40:33.746933+0000","last_fullsized":"2023-08-28T02:40:33.746933+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:20:59.994447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":115,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.744262+0000","last_change":"2023-08-28T02:40:18.786444+0000","last_active":"2023-08-28T02:40:33.744262+0000","last_peered":"2023-08-28T02:40:33.744262+0000","last_clean":"2023-08-28T02:40:33.744262+0000","last_became_active":"2023-08-28T02:40:18.786107+0000","last_became_peered":"2023-08-28T02:40:18.786107+0000","last_unstale":"2023-08-28T02:40:33.744262+0000","last_undegraded":"2023-08-28T02:40:33.744262+0000","last_fullsized":"2023-08-28T02:40:33.744262+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:32:41.355183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"384'2839","reported_seq":2864,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-08-28T02:40:38.636195+0000","last_change":"2023-08-28T02:39:56.829894+0000","last_active":"2023-08-28T02:40:38.636195+0000","last_peered":"2023-08-28T02:40:38.636195+0000","last_clean":"2023-08-28T02:40:38.636195+0000","last_became_active":"2023-08-28T02:39:56.829595+0000","last_became_peered":"2023-08-28T02:39:56.829595+0000","last_unstale":"2023-08-28T02:40:38.636195+0000","last_undegraded":"2023-08-28T02:40:38.636195+0000","last_fullsized":"2023-08-28T02:40:38.636195+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":368,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:39:55.797153+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:39:55.797153+0000","last_clean_scrub_stamp":"2023-08-28T02:39:55.797153+0000","objects_scrubbed":0,"log_size":2839,"log_dups_size":0,"ondisk_log_size":2839,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:01:54.942248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186056704,"num_objects":2839,"num_object_clones":0,"num_object_copies":8517,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2839,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2839,"num_write_kb":181696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"78'126","reported_seq":831,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.746901+0000","last_change":"2023-08-28T02:37:48.647489+0000","last_active":"2023-08-28T02:40:33.746901+0000","last_peered":"2023-08-28T02:40:33.746901+0000","last_clean":"2023-08-28T02:40:33.746901+0000","last_became_active":"2023-08-28T02:37:48.647002+0000","last_became_peered":"2023-08-28T02:37:48.647002+0000","last_unstale":"2023-08-28T02:40:33.746901+0000","last_undegraded":"2023-08-28T02:40:33.746901+0000","last_fullsized":"2023-08-28T02:40:33.746901+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:04:10.819878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":743,"num_read_kb":1331,"num_write":125,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":254,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.747382+0000","last_change":"2023-08-28T02:40:32.755572+0000","last_active":"2023-08-28T02:40:33.747382+0000","last_peered":"2023-08-28T02:40:33.747382+0000","last_clean":"2023-08-28T02:40:33.747382+0000","last_became_active":"2023-08-28T02:40:32.755256+0000","last_became_peered":"2023-08-28T02:40:32.755256+0000","last_unstale":"2023-08-28T02:40:33.747382+0000","last_undegraded":"2023-08-28T02:40:33.747382+0000","last_fullsized":"2023-08-28T02:40:33.747382+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:59:39.105195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":50,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.743862+0000","last_change":"2023-08-28T02:40:18.790060+0000","last_active":"2023-08-28T02:40:33.743862+0000","last_peered":"2023-08-28T02:40:33.743862+0000","last_clean":"2023-08-28T02:40:33.743862+0000","last_became_active":"2023-08-28T02:40:18.789732+0000","last_became_peered":"2023-08-28T02:40:18.789732+0000","last_unstale":"2023-08-28T02:40:33.743862+0000","last_undegraded":"2023-08-28T02:40:33.743862+0000","last_fullsized":"2023-08-28T02:40:33.743862+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:23:11.465730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"384'2937","reported_seq":2962,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-08-28T02:40:40.184184+0000","last_change":"2023-08-28T02:39:56.813983+0000","last_active":"2023-08-28T02:40:40.184184+0000","last_peered":"2023-08-28T02:40:40.184184+0000","last_clean":"2023-08-28T02:40:40.184184+0000","last_became_active":"2023-08-28T02:39:56.813635+0000","last_became_peered":"2023-08-28T02:39:56.813635+0000","last_unstale":"2023-08-28T02:40:40.184184+0000","last_undegraded":"2023-08-28T02:40:40.184184+0000","last_fullsized":"2023-08-28T02:40:40.184184+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":368,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:39:55.797153+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:39:55.797153+0000","last_clean_scrub_stamp":"2023-08-28T02:39:55.797153+0000","objects_scrubbed":0,"log_size":2937,"log_dups_size":0,"ondisk_log_size":2937,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:46:19.333100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192479232,"num_objects":2937,"num_object_clones":0,"num_object_copies":8811,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2937,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2937,"num_write_kb":187968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"78'204","reported_seq":1470,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.744107+0000","last_change":"2023-08-28T02:32:59.644574+0000","last_active":"2023-08-28T02:40:33.744107+0000","last_peered":"2023-08-28T02:40:33.744107+0000","last_clean":"2023-08-28T02:40:33.744107+0000","last_became_active":"2023-08-28T02:32:59.644053+0000","last_became_peered":"2023-08-28T02:32:59.644053+0000","last_unstale":"2023-08-28T02:40:33.744107+0000","last_undegraded":"2023-08-28T02:40:33.744107+0000","last_fullsized":"2023-08-28T02:40:33.744107+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":204,"log_dups_size":0,"ondisk_log_size":204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:55:51.460483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1521,"num_read_kb":2511,"num_write":178,"num_write_kb":1175,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":226,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.770593+0000","last_change":"2023-08-28T02:40:18.791989+0000","last_active":"2023-08-28T02:40:33.770593+0000","last_peered":"2023-08-28T02:40:33.770593+0000","last_clean":"2023-08-28T02:40:33.770593+0000","last_became_active":"2023-08-28T02:40:18.789727+0000","last_became_peered":"2023-08-28T02:40:18.789727+0000","last_unstale":"2023-08-28T02:40:33.770593+0000","last_undegraded":"2023-08-28T02:40:33.770593+0000","last_fullsized":"2023-08-28T02:40:33.770593+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:19:48.191236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":133,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.746822+0000","last_change":"2023-08-28T02:40:32.758669+0000","last_active":"2023-08-28T02:40:33.746822+0000","last_peered":"2023-08-28T02:40:33.746822+0000","last_clean":"2023-08-28T02:40:33.746822+0000","last_became_active":"2023-08-28T02:40:32.757733+0000","last_became_peered":"2023-08-28T02:40:32.757733+0000","last_unstale":"2023-08-28T02:40:33.746822+0000","last_undegraded":"2023-08-28T02:40:33.746822+0000","last_fullsized":"2023-08-28T02:40:33.746822+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:58:54.852642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"384'2938","reported_seq":2963,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-08-28T02:40:38.621065+0000","last_change":"2023-08-28T02:39:56.813784+0000","last_active":"2023-08-28T02:40:38.621065+0000","last_peered":"2023-08-28T02:40:38.621065+0000","last_clean":"2023-08-28T02:40:38.621065+0000","last_became_active":"2023-08-28T02:39:56.813207+0000","last_became_peered":"2023-08-28T02:39:56.813207+0000","last_unstale":"2023-08-28T02:40:38.621065+0000","last_undegraded":"2023-08-28T02:40:38.621065+0000","last_fullsized":"2023-08-28T02:40:38.621065+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":368,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:39:55.797153+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:39:55.797153+0000","last_clean_scrub_stamp":"2023-08-28T02:39:55.797153+0000","objects_scrubbed":0,"log_size":2938,"log_dups_size":0,"ondisk_log_size":2938,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:43:05.337535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192544768,"num_objects":2938,"num_object_clones":0,"num_object_copies":8814,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2938,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2938,"num_write_kb":188032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"78'47","reported_seq":392,"reported_epoch":383,"state":"active+recovery_wait+degraded","last_fresh":"2023-08-28T02:40:33.747328+0000","last_change":"2023-08-28T02:40:32.765040+0000","last_active":"2023-08-28T02:40:33.747328+0000","last_peered":"2023-08-28T02:40:33.747328+0000","last_clean":"2023-08-28T02:39:50.740580+0000","last_became_active":"2023-08-28T02:40:32.764603+0000","last_became_peered":"2023-08-28T02:40:32.764603+0000","last_unstale":"2023-08-28T02:40:33.747328+0000","last_undegraded":"2023-08-28T02:40:32.747207+0000","last_fullsized":"2023-08-28T02:40:33.747328+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":47,"log_dups_size":0,"ondisk_log_size":47,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:01:50.275795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":162,"num_read_kb":1281,"num_write":48,"num_write_kb":1153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":["3","0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":133,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.743522+0000","last_change":"2023-08-28T02:40:32.757962+0000","last_active":"2023-08-28T02:40:33.743522+0000","last_peered":"2023-08-28T02:40:33.743522+0000","last_clean":"2023-08-28T02:40:33.743522+0000","last_became_active":"2023-08-28T02:40:32.757660+0000","last_became_peered":"2023-08-28T02:40:32.757660+0000","last_unstale":"2023-08-28T02:40:33.743522+0000","last_undegraded":"2023-08-28T02:40:33.743522+0000","last_fullsized":"2023-08-28T02:40:33.743522+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:01:21.071322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":106,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.744340+0000","last_change":"2023-08-28T02:32:58.724424+0000","last_active":"2023-08-28T02:40:33.744340+0000","last_peered":"2023-08-28T02:40:33.744340+0000","last_clean":"2023-08-28T02:40:33.744340+0000","last_became_active":"2023-08-28T02:32:58.724127+0000","last_became_peered":"2023-08-28T02:32:58.724127+0000","last_unstale":"2023-08-28T02:40:33.744340+0000","last_undegraded":"2023-08-28T02:40:33.744340+0000","last_fullsized":"2023-08-28T02:40:33.744340+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:02:25.178712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"384'2939","reported_seq":2964,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-08-28T02:40:39.208105+0000","last_change":"2023-08-28T02:39:56.817603+0000","last_active":"2023-08-28T02:40:39.208105+0000","last_peered":"2023-08-28T02:40:39.208105+0000","last_clean":"2023-08-28T02:40:39.208105+0000","last_became_active":"2023-08-28T02:39:56.817236+0000","last_became_peered":"2023-08-28T02:39:56.817236+0000","last_unstale":"2023-08-28T02:40:39.208105+0000","last_undegraded":"2023-08-28T02:40:39.208105+0000","last_fullsized":"2023-08-28T02:40:39.208105+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":368,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:39:55.797153+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:39:55.797153+0000","last_clean_scrub_stamp":"2023-08-28T02:39:55.797153+0000","objects_scrubbed":0,"log_size":2939,"log_dups_size":0,"ondisk_log_size":2939,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T12:50:58.948699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192610304,"num_objects":2939,"num_object_clones":0,"num_object_copies":8817,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2939,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2939,"num_write_kb":188096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"78'114","reported_seq":461,"reported_epoch":383,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:40:33.756998+0000","last_change":"2023-08-28T02:40:33.756998+0000","last_active":"2023-08-28T02:40:33.756998+0000","last_peered":"2023-08-28T02:40:33.756998+0000","last_clean":"2023-08-28T02:39:50.747613+0000","last_became_active":"2023-08-28T02:40:33.756663+0000","last_became_peered":"2023-08-28T02:40:33.756663+0000","last_unstale":"2023-08-28T02:40:33.756998+0000","last_undegraded":"2023-08-28T02:40:33.748872+0000","last_fullsized":"2023-08-28T02:40:33.748376+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":114,"log_dups_size":0,"ondisk_log_size":114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":358,"num_read_kb":1352,"num_write":102,"num_write_kb":1104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5],"avail_no_missing":["7","2","5"],"object_location_counts":[{"shards":"2,5,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"86'139","reported_seq":478,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.747969+0000","last_change":"2023-08-28T02:37:48.651113+0000","last_active":"2023-08-28T02:40:33.747969+0000","last_peered":"2023-08-28T02:40:33.747969+0000","last_clean":"2023-08-28T02:40:33.747969+0000","last_became_active":"2023-08-28T02:37:48.650691+0000","last_became_peered":"2023-08-28T02:37:48.650691+0000","last_unstale":"2023-08-28T02:40:33.747969+0000","last_undegraded":"2023-08-28T02:40:33.747969+0000","last_fullsized":"2023-08-28T02:40:33.747969+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:08:02.043718+0000","last_clean_scrub_stamp":"2023-08-28T02:08:02.043718+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":267,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-08-28T02:40:34.790532+0000","last_change":"2023-08-28T02:28:25.347600+0000","last_active":"2023-08-28T02:40:34.790532+0000","last_peered":"2023-08-28T02:40:34.790532+0000","last_clean":"2023-08-28T02:40:34.790532+0000","last_became_active":"2023-08-28T02:28:25.347237+0000","last_became_peered":"2023-08-28T02:28:25.347237+0000","last_unstale":"2023-08-28T02:40:34.790532+0000","last_undegraded":"2023-08-28T02:40:34.790532+0000","last_fullsized":"2023-08-28T02:40:34.790532+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:29:26.767559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":199,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.750225+0000","last_change":"2023-08-28T02:40:18.794826+0000","last_active":"2023-08-28T02:40:33.750225+0000","last_peered":"2023-08-28T02:40:33.750225+0000","last_clean":"2023-08-28T02:40:33.750225+0000","last_became_active":"2023-08-28T02:40:18.794221+0000","last_became_peered":"2023-08-28T02:40:18.794221+0000","last_unstale":"2023-08-28T02:40:33.750225+0000","last_undegraded":"2023-08-28T02:40:33.750225+0000","last_fullsized":"2023-08-28T02:40:33.750225+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"384'2940","reported_seq":2965,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-08-28T02:40:40.154174+0000","last_change":"2023-08-28T02:39:56.818655+0000","last_active":"2023-08-28T02:40:40.154174+0000","last_peered":"2023-08-28T02:40:40.154174+0000","last_clean":"2023-08-28T02:40:40.154174+0000","last_became_active":"2023-08-28T02:39:56.818421+0000","last_became_peered":"2023-08-28T02:39:56.818421+0000","last_unstale":"2023-08-28T02:40:40.154174+0000","last_undegraded":"2023-08-28T02:40:40.154174+0000","last_fullsized":"2023-08-28T02:40:40.154174+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":368,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:39:55.797153+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:39:55.797153+0000","last_clean_scrub_stamp":"2023-08-28T02:39:55.797153+0000","objects_scrubbed":0,"log_size":2940,"log_dups_size":0,"ondisk_log_size":2940,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:47:45.657810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192675840,"num_objects":2940,"num_object_clones":0,"num_object_copies":8820,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2940,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2940,"num_write_kb":188160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"78'148","reported_seq":702,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.747912+0000","last_change":"2023-08-28T02:37:48.650996+0000","last_active":"2023-08-28T02:40:33.747912+0000","last_peered":"2023-08-28T02:40:33.747912+0000","last_clean":"2023-08-28T02:40:33.747912+0000","last_became_active":"2023-08-28T02:37:48.650016+0000","last_became_peered":"2023-08-28T02:37:48.650016+0000","last_unstale":"2023-08-28T02:40:33.747912+0000","last_undegraded":"2023-08-28T02:40:33.747912+0000","last_fullsized":"2023-08-28T02:40:33.747912+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":148,"log_dups_size":0,"ondisk_log_size":148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1092,"num_read_kb":1955,"num_write":127,"num_write_kb":5177,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":366,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.772700+0000","last_change":"2023-08-28T02:40:18.794134+0000","last_active":"2023-08-28T02:40:33.772700+0000","last_peered":"2023-08-28T02:40:33.772700+0000","last_clean":"2023-08-28T02:40:33.772700+0000","last_became_active":"2023-08-28T02:40:18.793807+0000","last_became_peered":"2023-08-28T02:40:18.793807+0000","last_unstale":"2023-08-28T02:40:33.772700+0000","last_undegraded":"2023-08-28T02:40:33.772700+0000","last_fullsized":"2023-08-28T02:40:33.772700+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:18:09.575169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":447,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.770882+0000","last_change":"2023-08-28T02:40:32.757659+0000","last_active":"2023-08-28T02:40:33.770882+0000","last_peered":"2023-08-28T02:40:33.770882+0000","last_clean":"2023-08-28T02:40:33.770882+0000","last_became_active":"2023-08-28T02:40:32.757411+0000","last_became_peered":"2023-08-28T02:40:32.757411+0000","last_unstale":"2023-08-28T02:40:33.770882+0000","last_undegraded":"2023-08-28T02:40:33.770882+0000","last_fullsized":"2023-08-28T02:40:33.770882+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:21:37.321734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"384'2855","reported_seq":3097,"reported_epoch":384,"state":"active+recovering+undersized+remapped","last_fresh":"2023-08-28T02:40:38.908267+0000","last_change":"2023-08-28T02:40:33.037636+0000","last_active":"2023-08-28T02:40:38.908267+0000","last_peered":"2023-08-28T02:40:38.908267+0000","last_clean":"2023-08-28T02:40:17.741060+0000","last_became_active":"2023-08-28T02:40:19.832010+0000","last_became_peered":"2023-08-28T02:40:19.832010+0000","last_unstale":"2023-08-28T02:40:38.908267+0000","last_undegraded":"2023-08-28T02:40:38.908267+0000","last_fullsized":"2023-08-28T02:40:19.767074+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":368,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:39:55.797153+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:39:55.797153+0000","last_clean_scrub_stamp":"2023-08-28T02:39:55.797153+0000","objects_scrubbed":0,"log_size":2855,"log_dups_size":0,"ondisk_log_size":2855,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187105280,"num_objects":2855,"num_object_clones":0,"num_object_copies":8565,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1629,"num_objects_unfound":0,"num_objects_dirty":2855,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2855,"num_write_kb":182720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":105,"num_bytes_recovered":6881280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1],"avail_no_missing":["7","1","4"],"object_location_counts":[{"shards":"1,4,7","objects":1629},{"shards":"1,7","objects":1226}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"78'103","reported_seq":759,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.748411+0000","last_change":"2023-08-28T02:40:21.993389+0000","last_active":"2023-08-28T02:40:33.748411+0000","last_peered":"2023-08-28T02:40:33.748411+0000","last_clean":"2023-08-28T02:40:33.748411+0000","last_became_active":"2023-08-28T02:40:21.992789+0000","last_became_peered":"2023-08-28T02:40:21.992789+0000","last_unstale":"2023-08-28T02:40:33.748411+0000","last_undegraded":"2023-08-28T02:40:33.748411+0000","last_fullsized":"2023-08-28T02:40:33.748411+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":103,"log_dups_size":0,"ondisk_log_size":103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":803,"num_read_kb":2080,"num_write":73,"num_write_kb":1244,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":133,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.745734+0000","last_change":"2023-08-28T02:40:32.760154+0000","last_active":"2023-08-28T02:40:33.745734+0000","last_peered":"2023-08-28T02:40:33.745734+0000","last_clean":"2023-08-28T02:40:33.745734+0000","last_became_active":"2023-08-28T02:40:32.758259+0000","last_became_peered":"2023-08-28T02:40:32.758259+0000","last_unstale":"2023-08-28T02:40:33.745734+0000","last_undegraded":"2023-08-28T02:40:33.745734+0000","last_fullsized":"2023-08-28T02:40:33.745734+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:16:32.639064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":379,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.770515+0000","last_change":"2023-08-28T02:37:45.579740+0000","last_active":"2023-08-28T02:40:33.770515+0000","last_peered":"2023-08-28T02:40:33.770515+0000","last_clean":"2023-08-28T02:40:33.770515+0000","last_became_active":"2023-08-28T02:37:45.579412+0000","last_became_peered":"2023-08-28T02:37:45.579412+0000","last_unstale":"2023-08-28T02:40:33.770515+0000","last_undegraded":"2023-08-28T02:40:33.770515+0000","last_fullsized":"2023-08-28T02:40:33.770515+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:31:18.483532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"384'2826","reported_seq":2850,"reported_epoch":384,"state":"active+clean","last_fresh":"2023-08-28T02:40:38.872387+0000","last_change":"2023-08-28T02:39:56.816682+0000","last_active":"2023-08-28T02:40:38.872387+0000","last_peered":"2023-08-28T02:40:38.872387+0000","last_clean":"2023-08-28T02:40:38.872387+0000","last_became_active":"2023-08-28T02:39:56.816283+0000","last_became_peered":"2023-08-28T02:39:56.816283+0000","last_unstale":"2023-08-28T02:40:38.872387+0000","last_undegraded":"2023-08-28T02:40:38.872387+0000","last_fullsized":"2023-08-28T02:40:38.872387+0000","mapping_epoch":368,"log_start":"0'0","ondisk_log_start":"0'0","created":368,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:39:55.797153+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:39:55.797153+0000","last_clean_scrub_stamp":"2023-08-28T02:39:55.797153+0000","objects_scrubbed":0,"log_size":2826,"log_dups_size":0,"ondisk_log_size":2826,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185204736,"num_objects":2826,"num_object_clones":0,"num_object_copies":8478,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2826,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2826,"num_write_kb":180864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":278,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.748134+0000","last_change":"2023-08-28T02:37:45.590339+0000","last_active":"2023-08-28T02:40:33.748134+0000","last_peered":"2023-08-28T02:40:33.748134+0000","last_clean":"2023-08-28T02:40:33.748134+0000","last_became_active":"2023-08-28T02:37:45.578751+0000","last_became_peered":"2023-08-28T02:37:45.578751+0000","last_unstale":"2023-08-28T02:40:33.748134+0000","last_undegraded":"2023-08-28T02:40:33.748134+0000","last_fullsized":"2023-08-28T02:40:33.748134+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:27:24.226372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"78'112","reported_seq":576,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.744396+0000","last_change":"2023-08-28T02:40:21.801641+0000","last_active":"2023-08-28T02:40:33.744396+0000","last_peered":"2023-08-28T02:40:33.744396+0000","last_clean":"2023-08-28T02:40:33.744396+0000","last_became_active":"2023-08-28T02:40:21.800886+0000","last_became_peered":"2023-08-28T02:40:21.800886+0000","last_unstale":"2023-08-28T02:40:33.744396+0000","last_undegraded":"2023-08-28T02:40:33.744396+0000","last_fullsized":"2023-08-28T02:40:33.744396+0000","mapping_epoch":378,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":379,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T13:44:49.480872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":444,"num_read_kb":473,"num_write":94,"num_write_kb":127,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":285,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.746727+0000","last_change":"2023-08-28T02:37:45.578790+0000","last_active":"2023-08-28T02:40:33.746727+0000","last_peered":"2023-08-28T02:40:33.746727+0000","last_clean":"2023-08-28T02:40:33.746727+0000","last_became_active":"2023-08-28T02:37:45.578473+0000","last_became_peered":"2023-08-28T02:37:45.578473+0000","last_unstale":"2023-08-28T02:40:33.746727+0000","last_undegraded":"2023-08-28T02:40:33.746727+0000","last_fullsized":"2023-08-28T02:40:33.746727+0000","mapping_epoch":344,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:23:51.722772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"384'2841","reported_seq":4070,"reported_epoch":384,"state":"active+recovering+undersized+remapped","last_fresh":"2023-08-28T02:40:39.210311+0000","last_change":"2023-08-28T02:40:32.944544+0000","last_active":"2023-08-28T02:40:39.210311+0000","last_peered":"2023-08-28T02:40:39.210311+0000","last_clean":"2023-08-28T02:40:17.753189+0000","last_became_active":"2023-08-28T02:40:19.903139+0000","last_became_peered":"2023-08-28T02:40:19.903139+0000","last_unstale":"2023-08-28T02:40:39.210311+0000","last_undegraded":"2023-08-28T02:40:39.210311+0000","last_fullsized":"2023-08-28T02:40:19.762984+0000","mapping_epoch":376,"log_start":"0'0","ondisk_log_start":"0'0","created":368,"last_epoch_clean":369,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:39:55.797153+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:39:55.797153+0000","last_clean_scrub_stamp":"2023-08-28T02:39:55.797153+0000","objects_scrubbed":0,"log_size":2841,"log_dups_size":0,"ondisk_log_size":2841,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:15:28.527856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186187776,"num_objects":2841,"num_object_clones":0,"num_object_copies":8523,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1087,"num_objects_unfound":0,"num_objects_dirty":2841,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2841,"num_write_kb":181824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":597,"num_bytes_recovered":39124992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7],"avail_no_missing":["3","4","7"],"object_location_counts":[{"shards":"3,4,7","objects":1087},{"shards":"3,7","objects":1754}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"78'122","reported_seq":846,"reported_epoch":383,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-08-28T02:40:33.756963+0000","last_change":"2023-08-28T02:40:33.756963+0000","last_active":"2023-08-28T02:40:33.756963+0000","last_peered":"2023-08-28T02:40:33.756963+0000","last_clean":"2023-08-28T02:40:25.198400+0000","last_became_active":"2023-08-28T02:40:33.756632+0000","last_became_peered":"2023-08-28T02:40:33.756632+0000","last_unstale":"2023-08-28T02:40:33.756963+0000","last_undegraded":"2023-08-28T02:40:33.750176+0000","last_fullsized":"2023-08-28T02:40:33.749573+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:10:31.171243+0000","last_clean_scrub_stamp":"2023-08-28T02:10:31.171243+0000","objects_scrubbed":0,"log_size":122,"log_dups_size":0,"ondisk_log_size":122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":857,"num_read_kb":2224,"num_write":99,"num_write_kb":2312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":45,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7],"avail_no_missing":["6","2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":408,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.770963+0000","last_change":"2023-08-28T02:40:18.796862+0000","last_active":"2023-08-28T02:40:33.770963+0000","last_peered":"2023-08-28T02:40:33.770963+0000","last_clean":"2023-08-28T02:40:33.770963+0000","last_became_active":"2023-08-28T02:40:18.789217+0000","last_became_peered":"2023-08-28T02:40:18.789217+0000","last_unstale":"2023-08-28T02:40:33.770963+0000","last_undegraded":"2023-08-28T02:40:33.770963+0000","last_fullsized":"2023-08-28T02:40:33.770963+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:06:42.193561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":201,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.751545+0000","last_change":"2023-08-28T02:40:18.787650+0000","last_active":"2023-08-28T02:40:33.751545+0000","last_peered":"2023-08-28T02:40:33.751545+0000","last_clean":"2023-08-28T02:40:33.751545+0000","last_became_active":"2023-08-28T02:40:18.787308+0000","last_became_peered":"2023-08-28T02:40:18.787308+0000","last_unstale":"2023-08-28T02:40:33.751545+0000","last_undegraded":"2023-08-28T02:40:33.751545+0000","last_fullsized":"2023-08-28T02:40:33.751545+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":389,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.770439+0000","last_change":"2023-08-28T02:40:18.792269+0000","last_active":"2023-08-28T02:40:33.770439+0000","last_peered":"2023-08-28T02:40:33.770439+0000","last_clean":"2023-08-28T02:40:33.770439+0000","last_became_active":"2023-08-28T02:40:18.789331+0000","last_became_peered":"2023-08-28T02:40:18.789331+0000","last_unstale":"2023-08-28T02:40:33.770439+0000","last_undegraded":"2023-08-28T02:40:33.770439+0000","last_fullsized":"2023-08-28T02:40:33.770439+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T10:29:55.050977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":420,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.745628+0000","last_change":"2023-08-28T02:40:32.757415+0000","last_active":"2023-08-28T02:40:33.745628+0000","last_peered":"2023-08-28T02:40:33.745628+0000","last_clean":"2023-08-28T02:40:33.745628+0000","last_became_active":"2023-08-28T02:40:32.756567+0000","last_became_peered":"2023-08-28T02:40:32.756567+0000","last_unstale":"2023-08-28T02:40:33.745628+0000","last_undegraded":"2023-08-28T02:40:33.745628+0000","last_fullsized":"2023-08-28T02:40:33.745628+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T11:44:48.741261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":135,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.745492+0000","last_change":"2023-08-28T02:40:18.812248+0000","last_active":"2023-08-28T02:40:33.745492+0000","last_peered":"2023-08-28T02:40:33.745492+0000","last_clean":"2023-08-28T02:40:33.745492+0000","last_became_active":"2023-08-28T02:40:18.789784+0000","last_became_peered":"2023-08-28T02:40:18.789784+0000","last_unstale":"2023-08-28T02:40:33.745492+0000","last_undegraded":"2023-08-28T02:40:33.745492+0000","last_fullsized":"2023-08-28T02:40:33.745492+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:42:27.332223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":347,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.747145+0000","last_change":"2023-08-28T02:40:32.759664+0000","last_active":"2023-08-28T02:40:33.747145+0000","last_peered":"2023-08-28T02:40:33.747145+0000","last_clean":"2023-08-28T02:40:33.747145+0000","last_became_active":"2023-08-28T02:40:32.759333+0000","last_became_peered":"2023-08-28T02:40:32.759333+0000","last_unstale":"2023-08-28T02:40:33.747145+0000","last_undegraded":"2023-08-28T02:40:33.747145+0000","last_fullsized":"2023-08-28T02:40:33.747145+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:28:51.735503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":51,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.750178+0000","last_change":"2023-08-28T02:40:18.814493+0000","last_active":"2023-08-28T02:40:33.750178+0000","last_peered":"2023-08-28T02:40:33.750178+0000","last_clean":"2023-08-28T02:40:33.750178+0000","last_became_active":"2023-08-28T02:40:18.792124+0000","last_became_peered":"2023-08-28T02:40:18.792124+0000","last_unstale":"2023-08-28T02:40:33.750178+0000","last_undegraded":"2023-08-28T02:40:33.750178+0000","last_fullsized":"2023-08-28T02:40:33.750178+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":173,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.763891+0000","last_change":"2023-08-28T02:40:32.754128+0000","last_active":"2023-08-28T02:40:33.763891+0000","last_peered":"2023-08-28T02:40:33.763891+0000","last_clean":"2023-08-28T02:40:33.763891+0000","last_became_active":"2023-08-28T02:40:32.753759+0000","last_became_peered":"2023-08-28T02:40:32.753759+0000","last_unstale":"2023-08-28T02:40:33.763891+0000","last_undegraded":"2023-08-28T02:40:33.763891+0000","last_fullsized":"2023-08-28T02:40:33.763891+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T09:20:19.070942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":303,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.747755+0000","last_change":"2023-08-28T02:40:32.759864+0000","last_active":"2023-08-28T02:40:33.747755+0000","last_peered":"2023-08-28T02:40:33.747755+0000","last_clean":"2023-08-28T02:40:33.747755+0000","last_became_active":"2023-08-28T02:40:32.759565+0000","last_became_peered":"2023-08-28T02:40:32.759565+0000","last_unstale":"2023-08-28T02:40:33.747755+0000","last_undegraded":"2023-08-28T02:40:33.747755+0000","last_fullsized":"2023-08-28T02:40:33.747755+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:57:08.355365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":274,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.747528+0000","last_change":"2023-08-28T02:40:32.756491+0000","last_active":"2023-08-28T02:40:33.747528+0000","last_peered":"2023-08-28T02:40:33.747528+0000","last_clean":"2023-08-28T02:40:33.747528+0000","last_became_active":"2023-08-28T02:40:32.756184+0000","last_became_peered":"2023-08-28T02:40:32.756184+0000","last_unstale":"2023-08-28T02:40:33.747528+0000","last_undegraded":"2023-08-28T02:40:33.747528+0000","last_fullsized":"2023-08-28T02:40:33.747528+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:09:42.362236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":418,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.770690+0000","last_change":"2023-08-28T02:40:32.940379+0000","last_active":"2023-08-28T02:40:33.770690+0000","last_peered":"2023-08-28T02:40:33.770690+0000","last_clean":"2023-08-28T02:40:33.770690+0000","last_became_active":"2023-08-28T02:40:32.755284+0000","last_became_peered":"2023-08-28T02:40:32.755284+0000","last_unstale":"2023-08-28T02:40:33.770690+0000","last_undegraded":"2023-08-28T02:40:33.770690+0000","last_fullsized":"2023-08-28T02:40:33.770690+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":382,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:21.154436+0000","last_clean_scrub_stamp":"2023-08-28T02:11:21.154436+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:42:10.444475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2770,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":361,"reported_epoch":383,"state":"active+clean","last_fresh":"2023-08-28T02:40:33.750107+0000","last_change":"2023-08-28T02:40:18.790819+0000","last_active":"2023-08-28T02:40:33.750107+0000","last_peered":"2023-08-28T02:40:33.750107+0000","last_clean":"2023-08-28T02:40:33.750107+0000","last_became_active":"2023-08-28T02:40:18.790577+0000","last_became_peered":"2023-08-28T02:40:18.790577+0000","last_unstale":"2023-08-28T02:40:33.750107+0000","last_undegraded":"2023-08-28T02:40:33.750107+0000","last_fullsized":"2023-08-28T02:40:33.750107+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-28T02:11:22.162328+0000","last_clean_scrub_stamp":"2023-08-28T02:11:22.162328+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":85,"num_bytes_recovered":36554,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":143360,"data_stored":26714,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35337,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":3038904320,"num_objects":46370,"num_object_clones":0,"num_object_copies":139110,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8523,"num_objects_misplaced":2716,"num_objects_unfound":0,"num_objects_dirty":46370,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":46370,"num_write_kb":2967680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1687,"num_bytes_recovered":110559232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9160556544,"data_stored":9160556544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":46370,"ondisk_log_size":46370,"up":48,"acting":42,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":25,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5980,"num_read_kb":13207,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":226,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":311,"seq":1335734829126,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1989448,"kb_used_data":1658304,"kb_used_omap":8,"kb_used_meta":331127,"kb_avail":91755704,"statfs":{"total":95995035648,"available":93957840896,"internally_reserved":0,"allocated":1698103296,"data_stored":1697155676,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8527,"internal_metadata":339074737},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:40:05 2023","interfaces":[{"interface":"back","average":{"1min":2.109,"5min":6.133,"15min":3.0550000000000002},"min":{"1min":0.23599999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":14.913,"5min":140.35499999999999,"15min":140.35499999999999},"last":0.246},{"interface":"front","average":{"1min":1.3600000000000001,"5min":5.7910000000000004,"15min":2.8050000000000002},"min":{"1min":0.38500000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":7.6790000000000003,"5min":140.40199999999999,"15min":140.40199999999999},"last":0.32400000000000001}]},{"osd":1,"last update":"Mon Aug 28 02:40:05 2023","interfaces":[{"interface":"back","average":{"1min":1.518,"5min":7.4320000000000004,"15min":6.6239999999999997},"min":{"1min":0.27200000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":9.3179999999999996,"5min":151.048,"15min":151.048},"last":54.813000000000002},{"interface":"front","average":{"1min":1.718,"5min":7.6879999999999997,"15min":6.6040000000000001},"min":{"1min":0.34000000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":6.5039999999999996,"5min":150.56299999999999,"15min":150.56299999999999},"last":54.363}]},{"osd":2,"last update":"Mon Aug 28 02:40:05 2023","interfaces":[{"interface":"back","average":{"1min":1.7410000000000001,"5min":2.9020000000000001,"15min":3.431},"min":{"1min":0.32700000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.8799999999999999,"5min":46.042999999999999,"15min":46.042999999999999},"last":0.377},{"interface":"front","average":{"1min":1.7509999999999999,"5min":2.3260000000000001,"15min":3.1779999999999999},"min":{"1min":0.35499999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":9.3810000000000002,"5min":46.020000000000003,"15min":46.020000000000003},"last":0.30599999999999999}]},{"osd":3,"last update":"Mon Aug 28 02:40:05 2023","interfaces":[{"interface":"back","average":{"1min":1.3320000000000001,"5min":5.3789999999999996,"15min":4.633},"min":{"1min":0.36199999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":7.8499999999999996,"5min":129.20099999999999,"15min":129.20099999999999},"last":0.41699999999999998},{"interface":"front","average":{"1min":2.0750000000000002,"5min":5.5819999999999999,"15min":5.1660000000000004},"min":{"1min":0.34499999999999997,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":9.3460000000000001,"5min":129.512,"15min":129.512},"last":0.35699999999999998}]},{"osd":4,"last update":"Mon Aug 28 02:40:05 2023","interfaces":[{"interface":"back","average":{"1min":1.3460000000000001,"5min":1.163,"15min":1.0980000000000001},"min":{"1min":0.307,"5min":0.25,"15min":0.25},"max":{"1min":5.5780000000000003,"5min":14.984999999999999,"15min":14.984999999999999},"last":0.28999999999999998},{"interface":"front","average":{"1min":1.6950000000000001,"5min":1.5569999999999999,"15min":1.353},"min":{"1min":0.33600000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":14.077,"5min":21.18,"15min":21.18},"last":0.33600000000000002}]},{"osd":5,"last update":"Mon Aug 28 02:40:05 2023","interfaces":[{"interface":"back","average":{"1min":1.5489999999999999,"5min":1.3839999999999999,"15min":1.996},"min":{"1min":0.32900000000000001,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":9.4109999999999996,"5min":36.476999999999997,"15min":36.476999999999997},"last":0.27500000000000002},{"interface":"front","average":{"1min":1.3069999999999999,"5min":1.518,"15min":2.0979999999999999},"min":{"1min":0.32500000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":6.4790000000000001,"5min":36.872,"15min":36.872},"last":0.34899999999999998}]},{"osd":6,"last update":"Mon Aug 28 02:40:05 2023","interfaces":[{"interface":"back","average":{"1min":1.498,"5min":3.4249999999999998,"15min":3.419},"min":{"1min":0.35899999999999999,"5min":0.307,"15min":0.307},"max":{"1min":8.3970000000000002,"5min":204.48400000000001,"15min":204.48400000000001},"last":0.37},{"interface":"front","average":{"1min":1.397,"5min":3.9780000000000002,"15min":3.7250000000000001},"min":{"1min":0.36499999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":7.8230000000000004,"5min":204.52099999999999,"15min":204.52099999999999},"last":0.40000000000000002}]}]},{"osd":6,"up_from":42,"seq":180388626807,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1151812,"kb_used_data":562108,"kb_used_omap":0,"kb_used_meta":589696,"kb_avail":92593340,"statfs":{"total":95995035648,"available":94815580160,"internally_reserved":0,"allocated":575598592,"data_stored":574655214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603848704},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:40:30 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":3.7730000000000001,"15min":3.6070000000000002},"min":{"1min":0.249,"5min":0.23000000000000001,"15min":0.20799999999999999},"max":{"1min":1.149,"5min":185.09800000000001,"15min":185.09800000000001},"last":1.8999999999999999},{"interface":"front","average":{"1min":1.3620000000000001,"5min":4.29,"15min":3.8530000000000002},"min":{"1min":0.33300000000000002,"5min":0.26800000000000002,"15min":0.23200000000000001},"max":{"1min":9.4860000000000007,"5min":185.017,"15min":185.017},"last":0.315}]},{"osd":1,"last update":"Mon Aug 28 02:40:30 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":2.802,"15min":2.9180000000000001},"min":{"1min":0.29699999999999999,"5min":0.26400000000000001,"15min":0.23799999999999999},"max":{"1min":5.1959999999999997,"5min":62.981000000000002,"15min":141.87299999999999},"last":0.29899999999999999},{"interface":"front","average":{"1min":1.5369999999999999,"5min":3.23,"15min":3.1699999999999999},"min":{"1min":0.311,"5min":0.25,"15min":0.24299999999999999},"max":{"1min":9.5169999999999995,"5min":63.725999999999999,"15min":141.904},"last":0.52200000000000002}]},{"osd":2,"last update":"Mon Aug 28 02:40:17 2023","interfaces":[{"interface":"back","average":{"1min":1.5800000000000001,"5min":4.3380000000000001,"15min":4.6909999999999998},"min":{"1min":0.32400000000000001,"5min":0.255,"15min":0.255},"max":{"1min":9.407,"5min":149.321,"15min":234.876},"last":0.34999999999999998},{"interface":"front","average":{"1min":2.0539999999999998,"5min":4.4029999999999996,"15min":4.7119999999999997},"min":{"1min":0.28399999999999997,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":11.08,"5min":149.34,"15min":234.482},"last":1.7569999999999999}]},{"osd":3,"last update":"Mon Aug 28 02:40:30 2023","interfaces":[{"interface":"back","average":{"1min":1.357,"5min":5.6059999999999999,"15min":4.4770000000000003},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":5.1680000000000001,"5min":167.233,"15min":167.233},"last":2.0960000000000001},{"interface":"front","average":{"1min":1.728,"5min":5.734,"15min":4.4649999999999999},"min":{"1min":0.40799999999999997,"5min":0.38,"15min":0.27400000000000002},"max":{"1min":9.5470000000000006,"5min":167.25,"15min":167.25},"last":2.089}]},{"osd":4,"last update":"Mon Aug 28 02:40:30 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.97399999999999998,"15min":0.94299999999999995},"min":{"1min":0.28999999999999998,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":2.1110000000000002,"5min":9.6519999999999992,"15min":18.308},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.55000000000000004,"5min":1.6359999999999999,"15min":1.2470000000000001},"min":{"1min":0.34599999999999997,"5min":0.26400000000000001,"15min":0.25900000000000001},"max":{"1min":2.6960000000000002,"5min":26.465,"15min":26.465},"last":0.38400000000000001}]},{"osd":5,"last update":"Mon Aug 28 02:40:30 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":1.446,"15min":1.1120000000000001},"min":{"1min":0.27300000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":1.6319999999999999,"5min":17.419,"15min":17.419},"last":1.9370000000000001},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.92400000000000004,"15min":0.89600000000000002},"min":{"1min":0.375,"5min":0.32100000000000001,"15min":0.28399999999999997},"max":{"1min":0.82899999999999996,"5min":9.6940000000000008,"15min":9.6940000000000008},"last":0.54700000000000004}]},{"osd":7,"last update":"Mon Aug 28 02:40:05 2023","interfaces":[{"interface":"back","average":{"1min":2.7450000000000001,"5min":1.3340000000000001,"15min":0.81499999999999995},"min":{"1min":0.34699999999999998,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":10.877000000000001,"5min":10.877000000000001,"15min":10.877000000000001},"last":0.48899999999999999},{"interface":"front","average":{"1min":1.6799999999999999,"5min":1.0089999999999999,"15min":0.67400000000000004},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":9.7249999999999996,"5min":9.7249999999999996,"15min":9.7249999999999996},"last":0.746}]}]},{"osd":1,"up_from":98,"seq":420906795260,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2238004,"kb_used_data":1645484,"kb_used_omap":7,"kb_used_meta":592504,"kb_avail":91507148,"statfs":{"total":95995035648,"available":93703319552,"internally_reserved":0,"allocated":1684975616,"data_stored":1683016935,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":606724427},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:40:26 2023","interfaces":[{"interface":"back","average":{"1min":8.8350000000000009,"5min":4.5910000000000002,"15min":5.226},"min":{"1min":0.183,"5min":0.183,"15min":0.183},"max":{"1min":152.33600000000001,"5min":152.33600000000001,"15min":152.33600000000001},"last":0.245},{"interface":"front","average":{"1min":9.2089999999999996,"5min":4.5659999999999998,"15min":5.2119999999999997},"min":{"1min":0.254,"5min":0.19800000000000001,"15min":0.188},"max":{"1min":151.82400000000001,"5min":151.82400000000001,"15min":151.82400000000001},"last":0.88300000000000001}]},{"osd":2,"last update":"Mon Aug 28 02:39:58 2023","interfaces":[{"interface":"back","average":{"1min":2.48,"5min":2.9049999999999998,"15min":4.5819999999999999},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.189},"max":{"1min":12.385999999999999,"5min":25.283000000000001,"15min":135.25999999999999},"last":0.23200000000000001},{"interface":"front","average":{"1min":2.0979999999999999,"5min":2.3820000000000001,"15min":4.1699999999999999},"min":{"1min":0.224,"5min":0.221,"15min":0.218},"max":{"1min":7.9409999999999998,"5min":24.850999999999999,"15min":135.10300000000001},"last":0.29099999999999998}]},{"osd":3,"last update":"Mon Aug 28 02:40:26 2023","interfaces":[{"interface":"back","average":{"1min":1.1599999999999999,"5min":2.5760000000000001,"15min":4.7530000000000001},"min":{"1min":0.22900000000000001,"5min":0.21299999999999999,"15min":0.19700000000000001},"max":{"1min":5.3399999999999999,"5min":16.239999999999998,"15min":126.95699999999999},"last":29.129999999999999},{"interface":"front","average":{"1min":0.93100000000000005,"5min":2.8450000000000002,"15min":4.9269999999999996},"min":{"1min":0.25900000000000001,"5min":0.217,"15min":0.217},"max":{"1min":2.5099999999999998,"5min":17.332999999999998,"15min":128.00200000000001},"last":33.582999999999998}]},{"osd":4,"last update":"Mon Aug 28 02:40:26 2023","interfaces":[{"interface":"back","average":{"1min":1.0580000000000001,"5min":2.4169999999999998,"15min":1.8939999999999999},"min":{"1min":0.34300000000000003,"5min":0.29699999999999999,"15min":0.27500000000000002},"max":{"1min":4.8330000000000002,"5min":24.809999999999999,"15min":24.809999999999999},"last":0.35199999999999998},{"interface":"front","average":{"1min":1.5,"5min":2.1899999999999999,"15min":1.704},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":8.109,"5min":24.835999999999999,"15min":24.835999999999999},"last":0.55700000000000005}]},{"osd":5,"last update":"Mon Aug 28 02:40:26 2023","interfaces":[{"interface":"back","average":{"1min":1.4159999999999999,"5min":3.2999999999999998,"15min":2.2109999999999999},"min":{"1min":0.40999999999999998,"5min":0.30499999999999999,"15min":0.252},"max":{"1min":5.3929999999999998,"5min":82.722999999999999,"15min":82.722999999999999},"last":0.47599999999999998},{"interface":"front","average":{"1min":1.488,"5min":3.8740000000000001,"15min":2.423},"min":{"1min":0.36599999999999999,"5min":0.32900000000000001,"15min":0.307},"max":{"1min":10.331,"5min":82.733999999999995,"15min":82.733999999999995},"last":0.38500000000000001}]},{"osd":6,"last update":"Mon Aug 28 02:40:26 2023","interfaces":[{"interface":"back","average":{"1min":8.0259999999999998,"5min":3.6589999999999998,"15min":2.3660000000000001},"min":{"1min":0.45100000000000001,"5min":0.35499999999999998,"15min":0.311},"max":{"1min":135.75,"5min":135.75,"15min":135.75},"last":0.51900000000000002},{"interface":"front","average":{"1min":7.9279999999999999,"5min":4.2210000000000001,"15min":2.5910000000000002},"min":{"1min":0.38100000000000001,"5min":0.35699999999999998,"15min":0.32200000000000001},"max":{"1min":135.77699999999999,"5min":135.77699999999999,"15min":135.77699999999999},"last":0.40899999999999997}]},{"osd":7,"last update":"Mon Aug 28 02:40:07 2023","interfaces":[{"interface":"back","average":{"1min":1.508,"5min":1.788,"15min":1.2030000000000001},"min":{"1min":0.40400000000000003,"5min":0.38,"15min":0.38},"max":{"1min":7.5110000000000001,"5min":10.872,"15min":10.872},"last":0.54700000000000004},{"interface":"front","average":{"1min":2.5630000000000002,"5min":1.8779999999999999,"15min":1.1659999999999999},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":8.3460000000000001,"5min":9.4900000000000002,"15min":9.4900000000000002},"last":0.59699999999999998}]}]},{"osd":0,"up_from":90,"seq":386547056906,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1969900,"kb_used_data":1273700,"kb_used_omap":8,"kb_used_meta":696183,"kb_avail":91775252,"statfs":{"total":95995035648,"available":93977858048,"internally_reserved":0,"allocated":1304268800,"data_stored":1302494194,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":712891753},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Aug 28 02:40:33 2023","interfaces":[{"interface":"back","average":{"1min":1.0760000000000001,"5min":6.4740000000000002,"15min":3.9990000000000001},"min":{"1min":0.22600000000000001,"5min":0.16500000000000001,"15min":0.16500000000000001},"max":{"1min":4.5300000000000002,"5min":150.00700000000001,"15min":150.00700000000001},"last":1.2390000000000001},{"interface":"front","average":{"1min":1.4990000000000001,"5min":6.2350000000000003,"15min":4.5019999999999998},"min":{"1min":0.23699999999999999,"5min":0.214,"15min":0.19500000000000001},"max":{"1min":10.138999999999999,"5min":150.27699999999999,"15min":150.27699999999999},"last":7.681}]},{"osd":2,"last update":"Mon Aug 28 02:39:55 2023","interfaces":[{"interface":"back","average":{"1min":1.238,"5min":2.3940000000000001,"15min":3.875},"min":{"1min":0.223,"5min":0.20999999999999999,"15min":0.19500000000000001},"max":{"1min":4.375,"5min":24.030999999999999,"15min":100.718},"last":11.33},{"interface":"front","average":{"1min":2.6560000000000001,"5min":3.113,"15min":4.5259999999999998},"min":{"1min":0.23699999999999999,"5min":0.23000000000000001,"15min":0.20000000000000001},"max":{"1min":9.9109999999999996,"5min":28.050999999999998,"15min":100.962},"last":0.39300000000000002}]},{"osd":3,"last update":"Mon Aug 28 02:40:40 2023","interfaces":[{"interface":"back","average":{"1min":1.3280000000000001,"5min":5.5190000000000001,"15min":6.1189999999999998},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":7.4969999999999999,"5min":196.702,"15min":237.43899999999999},"last":1.369},{"interface":"front","average":{"1min":1.337,"5min":5.806,"15min":6.1429999999999998},"min":{"1min":0.29899999999999999,"5min":0.26800000000000002,"15min":0.24399999999999999},"max":{"1min":7.6239999999999997,"5min":197.75399999999999,"15min":237.48099999999999},"last":2.7530000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:40:33 2023","interfaces":[{"interface":"back","average":{"1min":1.1899999999999999,"5min":4.0490000000000004,"15min":2.4239999999999999},"min":{"1min":0.34200000000000003,"5min":0.29299999999999998,"15min":0.23999999999999999},"max":{"1min":4.3120000000000003,"5min":223.64599999999999,"15min":223.64599999999999},"last":1.2030000000000001},{"interface":"front","average":{"1min":1.018,"5min":2.411,"15min":2.1299999999999999},"min":{"1min":0.32300000000000001,"5min":0.32000000000000001,"15min":0.309},"max":{"1min":4.2149999999999999,"5min":14.241,"15min":27.271999999999998},"last":1.3049999999999999}]},{"osd":5,"last update":"Mon Aug 28 02:40:33 2023","interfaces":[{"interface":"back","average":{"1min":1.0109999999999999,"5min":2.4889999999999999,"15min":2.133},"min":{"1min":0.35699999999999998,"5min":0.309,"15min":0.29599999999999999},"max":{"1min":4.2320000000000002,"5min":13.547000000000001,"15min":27.367000000000001},"last":1.3400000000000001},{"interface":"front","average":{"1min":1.206,"5min":1.885,"15min":1.784},"min":{"1min":0.33300000000000002,"5min":0.316,"15min":0.307},"max":{"1min":4.2709999999999999,"5min":13.103,"15min":13.103},"last":1.139}]},{"osd":6,"last update":"Mon Aug 28 02:40:33 2023","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":2.5409999999999999,"15min":2.234},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.29999999999999999},"max":{"1min":4.1859999999999999,"5min":16.777999999999999,"15min":27.331},"last":1.2749999999999999},{"interface":"front","average":{"1min":1.254,"5min":2.0640000000000001,"15min":1.919},"min":{"1min":0.42599999999999999,"5min":0.35099999999999998,"15min":0.32700000000000001},"max":{"1min":4.2919999999999998,"5min":9.9640000000000004,"15min":27.247},"last":1.1759999999999999}]},{"osd":7,"last update":"Mon Aug 28 02:40:05 2023","interfaces":[{"interface":"back","average":{"1min":2.7130000000000001,"5min":2.2530000000000001,"15min":1.5149999999999999},"min":{"1min":0.45500000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":9.8879999999999999,"5min":13.071,"15min":13.071},"last":1.1000000000000001},{"interface":"front","average":{"1min":2.3889999999999998,"5min":1.9079999999999999,"15min":1.9199999999999999},"min":{"1min":0.371,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":9.4570000000000007,"5min":16.529,"15min":16.529},"last":0.42499999999999999}]}]},{"osd":2,"up_from":0,"seq":446676599030,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":112,"seq":481036337392,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2197976,"kb_used_data":1496848,"kb_used_omap":3,"kb_used_meta":701116,"kb_avail":91547176,"statfs":{"total":95995035648,"available":93744308224,"internally_reserved":0,"allocated":1532772352,"data_stored":1531007912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":717943714},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:40:30 2023","interfaces":[{"interface":"back","average":{"1min":0.746,"5min":2.9929999999999999,"15min":3.6869999999999998},"min":{"1min":0.25800000000000001,"5min":0.14999999999999999,"15min":0.14999999999999999},"max":{"1min":2.355,"5min":59.256999999999998,"15min":138.39699999999999},"last":0.217},{"interface":"front","average":{"1min":1.2509999999999999,"5min":3.0049999999999999,"15min":2.899},"min":{"1min":0.26000000000000001,"5min":0.20100000000000001,"15min":0.192},"max":{"1min":9.6509999999999998,"5min":59.341999999999999,"15min":78.555000000000007},"last":0.38700000000000001}]},{"osd":1,"last update":"Mon Aug 28 02:40:30 2023","interfaces":[{"interface":"back","average":{"1min":1.3899999999999999,"5min":3.7930000000000001,"15min":2.8260000000000001},"min":{"1min":0.218,"5min":0.191,"15min":0.191},"max":{"1min":7.181,"5min":105.04600000000001,"15min":105.04600000000001},"last":3.4750000000000001},{"interface":"front","average":{"1min":1.0409999999999999,"5min":4.3689999999999998,"15min":3.681},"min":{"1min":0.24299999999999999,"5min":0.193,"15min":0.193},"max":{"1min":4.3019999999999996,"5min":105.55800000000001,"15min":143.54400000000001},"last":1.6180000000000001}]},{"osd":2,"last update":"Mon Aug 28 02:40:30 2023","interfaces":[{"interface":"back","average":{"1min":1.405,"5min":2.8929999999999998,"15min":3.7759999999999998},"min":{"1min":0.30399999999999999,"5min":0.20799999999999999,"15min":0.20100000000000001},"max":{"1min":7.5890000000000004,"5min":42.417999999999999,"15min":143.50299999999999},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.94899999999999995,"5min":2.4159999999999999,"15min":2.7130000000000001},"min":{"1min":0.34200000000000003,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":3.319,"5min":42.229999999999997,"15min":65.120999999999995},"last":0.47699999999999998}]},{"osd":4,"last update":"Mon Aug 28 02:40:30 2023","interfaces":[{"interface":"back","average":{"1min":0.73399999999999999,"5min":3.399,"15min":2.5219999999999998},"min":{"1min":0.33500000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":2.371,"5min":158.739,"15min":158.739},"last":0.45300000000000001},{"interface":"front","average":{"1min":1.272,"5min":3.4609999999999999,"15min":2.665},"min":{"1min":0.436,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":7.1600000000000001,"5min":158.72,"15min":158.72},"last":3.0470000000000002}]},{"osd":5,"last update":"Mon Aug 28 02:40:30 2023","interfaces":[{"interface":"back","average":{"1min":1.238,"5min":3.3639999999999999,"15min":2.827},"min":{"1min":0.35199999999999998,"5min":0.315,"15min":0.29099999999999998},"max":{"1min":6.2649999999999997,"5min":163.55199999999999,"15min":163.55199999999999},"last":4.6639999999999997},{"interface":"front","average":{"1min":1.0680000000000001,"5min":3.7639999999999998,"15min":3.5950000000000002},"min":{"1min":0.31900000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.6459999999999999,"5min":163.45699999999999,"15min":163.45699999999999},"last":0.42499999999999999}]},{"osd":6,"last update":"Mon Aug 28 02:40:30 2023","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":2.093,"15min":2.581},"min":{"1min":0.34399999999999997,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":3.6789999999999998,"5min":11.324999999999999,"15min":138.37},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.72599999999999998,"5min":1.6879999999999999,"15min":1.6539999999999999},"min":{"1min":0.39200000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":1.9350000000000001,"5min":12.193,"15min":12.193},"last":2.496}]},{"osd":7,"last update":"Mon Aug 28 02:40:09 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":2.1230000000000002,"15min":1.4770000000000001},"min":{"1min":0.39800000000000002,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.6859999999999999,"5min":9.2530000000000001,"15min":9.2530000000000001},"last":0.57199999999999995},{"interface":"front","average":{"1min":1,"5min":1.8999999999999999,"15min":1.4910000000000001},"min":{"1min":0.40999999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":4.5490000000000004,"5min":12.226000000000001,"15min":12.226000000000001},"last":0.50600000000000001}]}]},{"osd":4,"up_from":0,"seq":128849019265,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":36,"seq":154618823037,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2492012,"kb_used_data":1598244,"kb_used_omap":0,"kb_used_meta":893760,"kb_avail":91253140,"statfs":{"total":95995035648,"available":93443215360,"internally_reserved":0,"allocated":1636601856,"data_stored":1635677427,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":915210240},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Aug 28 02:40:23 2023","interfaces":[{"interface":"back","average":{"1min":6.9809999999999999,"5min":4.1680000000000001,"15min":3.7280000000000002},"min":{"1min":0.33200000000000002,"5min":0.216,"15min":0.214},"max":{"1min":111.084,"5min":111.084,"15min":128.72800000000001},"last":82.563000000000002},{"interface":"front","average":{"1min":6.5339999999999998,"5min":4.5350000000000001,"15min":3.9729999999999999},"min":{"1min":0.314,"5min":0.245,"15min":0.24199999999999999},"max":{"1min":111.098,"5min":111.098,"15min":128.75200000000001},"last":82.542000000000002}]},{"osd":1,"last update":"Mon Aug 28 02:39:36 2023","interfaces":[{"interface":"back","average":{"1min":5.085,"5min":8.3390000000000004,"15min":4.2290000000000001},"min":{"1min":0.33100000000000002,"5min":0.22800000000000001,"15min":0.216},"max":{"1min":61.218000000000004,"5min":148.01400000000001,"15min":148.01400000000001},"last":0.48999999999999999},{"interface":"front","average":{"1min":5.7110000000000003,"5min":8.5060000000000002,"15min":4.4870000000000001},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.23200000000000001},"max":{"1min":62.253999999999998,"5min":148.08000000000001,"15min":148.08000000000001},"last":0.47899999999999998}]},{"osd":2,"last update":"Mon Aug 28 02:40:03 2023","interfaces":[{"interface":"back","average":{"1min":4.0549999999999997,"5min":2.0590000000000002,"15min":3.8919999999999999},"min":{"1min":0.32700000000000001,"5min":0.247,"15min":0.247},"max":{"1min":44.088000000000001,"5min":44.088000000000001,"15min":176.69499999999999},"last":0.52600000000000002},{"interface":"front","average":{"1min":4.2210000000000001,"5min":3.0249999999999999,"15min":4.3860000000000001},"min":{"1min":0.36199999999999999,"5min":0.33200000000000002,"15min":0.27600000000000002},"max":{"1min":44.100999999999999,"5min":50.279000000000003,"15min":176.71199999999999},"last":0.54800000000000004}]},{"osd":3,"last update":"Mon Aug 28 02:39:36 2023","interfaces":[{"interface":"back","average":{"1min":6.1299999999999999,"5min":3.7770000000000001,"15min":4.415},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.26100000000000001},"max":{"1min":57.956000000000003,"5min":87.233999999999995,"15min":128.571},"last":0.501},{"interface":"front","average":{"1min":6.4619999999999997,"5min":3.802,"15min":4.6959999999999997},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.26200000000000001},"max":{"1min":58.139000000000003,"5min":87.337000000000003,"15min":128.684},"last":0.51100000000000001}]},{"osd":4,"last update":"Mon Aug 28 02:40:13 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":0.94699999999999995,"15min":1.258},"min":{"1min":0.32600000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":3.6459999999999999,"5min":11.206,"15min":85.769999999999996},"last":0.33200000000000002},{"interface":"front","average":{"1min":1.6180000000000001,"5min":1.0649999999999999,"15min":1.2869999999999999},"min":{"1min":0.32400000000000001,"5min":0.29899999999999999,"15min":0.248},"max":{"1min":12.215,"5min":12.215,"15min":85.825000000000003},"last":0.42699999999999999}]},{"osd":6,"last update":"Mon Aug 28 02:39:36 2023","interfaces":[{"interface":"back","average":{"1min":2.4980000000000002,"5min":1.1000000000000001,"15min":1.0089999999999999},"min":{"1min":0.32900000000000001,"5min":0.27800000000000002,"15min":0.25700000000000001},"max":{"1min":18.512,"5min":18.512,"15min":18.512},"last":0.45100000000000001},{"interface":"front","average":{"1min":2.02,"5min":1.032,"15min":0.997},"min":{"1min":0.31,"5min":0.307,"15min":0.28799999999999998},"max":{"1min":12.193,"5min":12.193,"15min":12.193},"last":0.39200000000000002}]},{"osd":7,"last update":"Mon Aug 28 02:40:03 2023","interfaces":[{"interface":"back","average":{"1min":1.6859999999999999,"5min":1.2929999999999999,"15min":0.83299999999999996},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":12.169,"5min":12.169,"15min":12.169},"last":0.46700000000000003},{"interface":"front","average":{"1min":1.1100000000000001,"5min":1.1060000000000001,"15min":0.83699999999999997},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.9089999999999998,"5min":8.9199999999999999,"15min":8.9199999999999999},"last":0.441}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3864,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8855,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7861,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8527,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1158348800,"data_stored":1158348800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1696858112,"data_stored":1696858112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":510984192,"data_stored":510984192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1530134528,"data_stored":1530134528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":336134144,"data_stored":336134144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1648492544,"data_stored":1648492544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":561774592,"data_stored":561774592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1717829632,"data_stored":1717829632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-08-28T02:40:47.191 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd pool set unique_pool_4 pgp_num 16 2023-08-28T02:40:47.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:46 smithi169 ceph-mon[159415]: pgmap v1054: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 58 MiB/s wr, 927 op/s; 8548/153513 objects degraded (5.568%); 2302/153513 objects misplaced (1.500%); 5.7 MiB/s, 91 objects/s recovering 2023-08-28T02:40:47.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:46 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 8548/153513 objects degraded (5.568%), 7 pgs degraded (PG_DEGRADED) 2023-08-28T02:40:47.281 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:40:47.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:47.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:40:47.397 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:40:47.500 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:47.746 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:40:47.747 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 44 16 54034 54018 76.6966 62.8125 0.0115711 0.0130318 2023-08-28T02:40:47.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:47.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:40:47.852 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:40:47.937 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:48.069 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:40:48.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:48.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:40:48.092 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:40:48.093 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [], 2023-08-28T02:40:48.093 INFO:teuthology.orchestra.run.smithi143.stdout: "num_ops": 0 2023-08-28T02:40:48.093 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:40:48.105 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:40:48.148 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:40:48.166 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:40:48.183 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:40:48.214 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:40:48.241 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:48.250 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:40:48.292 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:40:48.311 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:40:48.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:48.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:40:48.336 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:40:48.370 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:40:48.401 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:40:48.419 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:40:48.422 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:40:48.422 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:40:48.422 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:40:48.422 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:40:48.422 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:40:48.423 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:40:48.423 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:40:48.423 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:40:48.462 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:48.526 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:40:48.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:48.532 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:40:48.632 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:40:48.686 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:48.747 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 45 16 55021 55005 76.363 61.6875 0.0257944 0.0130892 2023-08-28T02:40:48.763 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:40:48.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:48.773 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:40:48.874 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:40:48.902 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:48.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:48 smithi169 ceph-mon[159415]: pgmap v1055: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.1 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 58 MiB/s wr, 927 op/s; 8548/153513 objects degraded (5.568%); 2302/153513 objects misplaced (1.500%); 5.7 MiB/s, 91 objects/s recovering 2023-08-28T02:40:48.988 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:48 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/300531076' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:40:49.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:48 smithi143 ceph-mon[183059]: pgmap v1055: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.1 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 58 MiB/s wr, 927 op/s; 8548/153513 objects degraded (5.568%); 2302/153513 objects misplaced (1.500%); 5.7 MiB/s, 91 objects/s recovering 2023-08-28T02:40:49.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:48 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/300531076' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:40:49.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:48 smithi143 ceph-mon[189871]: pgmap v1055: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.1 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 58 MiB/s wr, 927 op/s; 8548/153513 objects degraded (5.568%); 2302/153513 objects misplaced (1.500%); 5.7 MiB/s, 91 objects/s recovering 2023-08-28T02:40:49.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:48 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/300531076' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:40:49.368 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd set nodeep-scrub 2023-08-28T02:40:49.538 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:40:49.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:49.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:40:49.652 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:40:49.715 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:49.751 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 46 16 55724 55708 75.6519 43.9375 0.0234567 0.0132116 2023-08-28T02:40:50.357 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.0 dump_blocked_ops 2023-08-28T02:40:50.485 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:40:50.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:50.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:40:50.594 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:40:50.708 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:50.756 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 47 16 56732 56716 75.3808 63 0.017833 0.0132596 2023-08-28T02:40:50.776 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:50 smithi143 ceph-mon[189871]: pgmap v1056: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.3 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 56 MiB/s wr, 898 op/s; 8548/163665 objects degraded (5.223%); 1845/163665 objects misplaced (1.127%); 5.4 MiB/s, 87 objects/s recovering 2023-08-28T02:40:50.802 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:40:50.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:50.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:40:50.913 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:40:50.991 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:51.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:50 smithi143 ceph-mon[183059]: pgmap v1056: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.3 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 56 MiB/s wr, 898 op/s; 8548/163665 objects degraded (5.223%); 1845/163665 objects misplaced (1.127%); 5.4 MiB/s, 87 objects/s recovering 2023-08-28T02:40:51.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:50 smithi169 ceph-mon[159415]: pgmap v1056: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.3 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 56 MiB/s wr, 898 op/s; 8548/163665 objects degraded (5.223%); 1845/163665 objects misplaced (1.127%); 5.4 MiB/s, 87 objects/s recovering 2023-08-28T02:40:51.589 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:40:51.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:51.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:40:51.675 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:40:51.693 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:40:51.753 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 48 16 57624 57608 74.9719 55.75 0.0221846 0.0133319 2023-08-28T02:40:51.800 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:51.903 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:40:51.927 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:40:51.928 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:51 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 8548/163665 objects degraded (5.223%), 7 pgs degraded (PG_DEGRADED) 2023-08-28T02:40:51.929 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:51 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 8548/163665 objects degraded (5.223%), 7 pgs degraded (PG_DEGRADED) 2023-08-28T02:40:51.930 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:40:51 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:40:51] "GET /metrics HTTP/1.1" 200 36593 "" "Prometheus/2.33.4" 2023-08-28T02:40:51.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:51.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:40:52.035 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:40:52.134 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:52.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:51 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 8548/163665 objects degraded (5.223%), 7 pgs degraded (PG_DEGRADED) 2023-08-28T02:40:52.388 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-08-28T02:40:52.570 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:40:52.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:52.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:40:52.678 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:40:52.753 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 49 16 58517 58501 74.581 55.8125 0.00808719 0.0133993 2023-08-28T02:40:52.793 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:53.041 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:52 smithi143 ceph-mon[183059]: pgmap v1057: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 56 MiB/s wr, 894 op/s; 8548/166029 objects degraded (5.148%); 1845/166029 objects misplaced (1.111%); 5.4 MiB/s, 87 objects/s recovering 2023-08-28T02:40:53.042 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:52 smithi143 ceph-mon[189871]: pgmap v1057: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 56 MiB/s wr, 894 op/s; 8548/166029 objects degraded (5.148%); 1845/166029 objects misplaced (1.111%); 5.4 MiB/s, 87 objects/s recovering 2023-08-28T02:40:53.172 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:40:53.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:53.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:40:53.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:52 smithi169 ceph-mon[159415]: pgmap v1057: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 56 MiB/s wr, 894 op/s; 8548/166029 objects degraded (5.148%); 1845/166029 objects misplaced (1.111%); 5.4 MiB/s, 87 objects/s recovering 2023-08-28T02:40:53.286 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:40:53.369 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:53.753 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 50 16 59378 59362 74.1657 53.8125 0.0120579 0.0134767 2023-08-28T02:40:53.853 INFO:teuthology.orchestra.run.smithi143.stderr:set pool 9 pgp_num to 16 2023-08-28T02:40:54.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:53 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2664992503' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-08-28T02:40:54.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:53 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-08-28T02:40:54.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:53 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2664992503' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-08-28T02:40:54.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:53 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-08-28T02:40:54.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:53 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2664992503' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-08-28T02:40:54.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:53 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-08-28T02:40:54.753 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 51 16 60373 60357 73.9308 62.1875 0.0138815 0.0135211 2023-08-28T02:40:55.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:54 smithi143 ceph-mon[183059]: pgmap v1058: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.5 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 64 MiB/s wr, 1.03k op/s; 8548/170190 objects degraded (5.023%); 1845/170190 objects misplaced (1.084%); 5.4 MiB/s, 86 objects/s recovering 2023-08-28T02:40:55.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:54 smithi143 ceph-mon[183059]: failed to encode map e386 with expected crc 2023-08-28T02:40:55.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:54 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-08-28T02:40:55.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:54 smithi143 ceph-mon[183059]: osdmap e386: 8 total, 8 up, 6 in 2023-08-28T02:40:55.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:54 smithi143 ceph-mon[183059]: failed to encode map e386 with expected crc 2023-08-28T02:40:55.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:54 smithi143 ceph-mon[183059]: failed to encode map e386 with expected crc 2023-08-28T02:40:55.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:54 smithi143 ceph-mon[183059]: failed to encode map e386 with expected crc 2023-08-28T02:40:55.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:54 smithi143 ceph-mon[183059]: failed to encode map e386 with expected crc 2023-08-28T02:40:55.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:54 smithi143 ceph-mon[183059]: failed to encode map e386 with expected crc 2023-08-28T02:40:55.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:54 smithi143 ceph-mon[183059]: failed to encode map e386 with expected crc 2023-08-28T02:40:55.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:54 smithi143 ceph-mon[189871]: pgmap v1058: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.5 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 64 MiB/s wr, 1.03k op/s; 8548/170190 objects degraded (5.023%); 1845/170190 objects misplaced (1.084%); 5.4 MiB/s, 86 objects/s recovering 2023-08-28T02:40:55.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:54 smithi143 ceph-mon[189871]: failed to encode map e386 with expected crc 2023-08-28T02:40:55.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:54 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-08-28T02:40:55.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:54 smithi143 ceph-mon[189871]: osdmap e386: 8 total, 8 up, 6 in 2023-08-28T02:40:55.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:54 smithi143 ceph-mon[189871]: failed to encode map e386 with expected crc 2023-08-28T02:40:55.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:54 smithi143 ceph-mon[189871]: failed to encode map e386 with expected crc 2023-08-28T02:40:55.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:54 smithi143 ceph-mon[189871]: failed to encode map e386 with expected crc 2023-08-28T02:40:55.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:54 smithi143 ceph-mon[189871]: failed to encode map e386 with expected crc 2023-08-28T02:40:55.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:54 smithi143 ceph-mon[189871]: failed to encode map e386 with expected crc 2023-08-28T02:40:55.202 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:54 smithi143 ceph-mon[189871]: failed to encode map e386 with expected crc 2023-08-28T02:40:55.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:54 smithi169 ceph-mon[159415]: pgmap v1058: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.5 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 64 MiB/s wr, 1.03k op/s; 8548/170190 objects degraded (5.023%); 1845/170190 objects misplaced (1.084%); 5.4 MiB/s, 86 objects/s recovering 2023-08-28T02:40:55.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:54 smithi169 ceph-mon[159415]: failed to encode map e386 with expected crc 2023-08-28T02:40:55.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:54 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-08-28T02:40:55.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:54 smithi169 ceph-mon[159415]: osdmap e386: 8 total, 8 up, 6 in 2023-08-28T02:40:55.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:54 smithi169 ceph-mon[159415]: failed to encode map e386 with expected crc 2023-08-28T02:40:55.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:54 smithi169 ceph-mon[159415]: failed to encode map e386 with expected crc 2023-08-28T02:40:55.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:54 smithi169 ceph-mon[159415]: failed to encode map e386 with expected crc 2023-08-28T02:40:55.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:54 smithi169 ceph-mon[159415]: failed to encode map e386 with expected crc 2023-08-28T02:40:55.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:54 smithi169 ceph-mon[159415]: failed to encode map e386 with expected crc 2023-08-28T02:40:55.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:54 smithi169 ceph-mon[159415]: failed to encode map e386 with expected crc 2023-08-28T02:40:55.498 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:40:55.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:55.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:40:55.603 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:40:55.648 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:55.691 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:40:55.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:55.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:40:55.753 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 52 16 60654 60638 72.8472 17.5625 0.00360055 0.0135294 2023-08-28T02:40:55.797 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:40:55.822 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:55.888 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:40:55.899 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:55 smithi169 ceph-mon[159415]: osdmap e387: 8 total, 8 up, 6 in 2023-08-28T02:40:55.900 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:55 smithi169 ceph-mon[159415]: failed to encode map e387 with expected crc 2023-08-28T02:40:55.900 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:55 smithi169 ceph-mon[159415]: failed to encode map e387 with expected crc 2023-08-28T02:40:55.900 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:55 smithi169 ceph-mon[159415]: failed to encode map e387 with expected crc 2023-08-28T02:40:55.900 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:55 smithi169 ceph-mon[159415]: failed to encode map e387 with expected crc 2023-08-28T02:40:55.900 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:55 smithi169 ceph-mon[159415]: failed to encode map e387 with expected crc 2023-08-28T02:40:55.900 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:55 smithi169 ceph-mon[159415]: failed to encode map e387 with expected crc 2023-08-28T02:40:55.900 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:55 smithi169 ceph-mon[159415]: failed to encode map e387 with expected crc 2023-08-28T02:40:55.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:55.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:40:56.002 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:40:56.035 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:56.066 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:40:56.087 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:40:56.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:56.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:40:56.192 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:40:56.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:55 smithi143 ceph-mon[183059]: osdmap e387: 8 total, 8 up, 6 in 2023-08-28T02:40:56.201 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:55 smithi143 ceph-mon[183059]: failed to encode map e387 with expected crc 2023-08-28T02:40:56.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:55 smithi143 ceph-mon[183059]: failed to encode map e387 with expected crc 2023-08-28T02:40:56.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:55 smithi143 ceph-mon[183059]: failed to encode map e387 with expected crc 2023-08-28T02:40:56.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:55 smithi143 ceph-mon[183059]: failed to encode map e387 with expected crc 2023-08-28T02:40:56.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:55 smithi143 ceph-mon[183059]: failed to encode map e387 with expected crc 2023-08-28T02:40:56.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:55 smithi143 ceph-mon[183059]: failed to encode map e387 with expected crc 2023-08-28T02:40:56.202 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:55 smithi143 ceph-mon[183059]: failed to encode map e387 with expected crc 2023-08-28T02:40:56.214 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:55 smithi143 ceph-mon[189871]: osdmap e387: 8 total, 8 up, 6 in 2023-08-28T02:40:56.214 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:55 smithi143 ceph-mon[189871]: failed to encode map e387 with expected crc 2023-08-28T02:40:56.214 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:55 smithi143 ceph-mon[189871]: failed to encode map e387 with expected crc 2023-08-28T02:40:56.214 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:55 smithi143 ceph-mon[189871]: failed to encode map e387 with expected crc 2023-08-28T02:40:56.214 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:55 smithi143 ceph-mon[189871]: failed to encode map e387 with expected crc 2023-08-28T02:40:56.214 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:55 smithi143 ceph-mon[189871]: failed to encode map e387 with expected crc 2023-08-28T02:40:56.215 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:55 smithi143 ceph-mon[189871]: failed to encode map e387 with expected crc 2023-08-28T02:40:56.215 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:55 smithi143 ceph-mon[189871]: failed to encode map e387 with expected crc 2023-08-28T02:40:56.275 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:40:56.276 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [], 2023-08-28T02:40:56.276 INFO:teuthology.orchestra.run.smithi143.stdout: "complaint_time": 30, 2023-08-28T02:40:56.276 INFO:teuthology.orchestra.run.smithi143.stdout: "num_blocked_ops": 0 2023-08-28T02:40:56.276 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:40:56.294 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:56.539 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:40:56.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:56.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:40:56.648 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:40:56.674 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:56.723 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:40:56.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:56.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:40:56.753 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 53 16 61366 61350 72.3124 44.5 0.0110289 0.0138055 2023-08-28T02:40:56.828 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:40:56.855 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:56.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:56 smithi169 ceph-mon[159415]: pgmap v1061: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 67 MiB/s wr, 1.07k op/s; 8548/179349 objects degraded (4.766%); 1368/179349 objects misplaced (0.763%); 7.3 MiB/s, 116 objects/s recovering 2023-08-28T02:40:56.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:56 smithi169 ceph-mon[159415]: failed to encode map e388 with expected crc 2023-08-28T02:40:56.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:56 smithi169 ceph-mon[159415]: failed to encode map e388 with expected crc 2023-08-28T02:40:56.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:56 smithi169 ceph-mon[159415]: osdmap e388: 8 total, 8 up, 6 in 2023-08-28T02:40:56.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:56 smithi169 ceph-mon[159415]: failed to encode map e388 with expected crc 2023-08-28T02:40:56.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:56 smithi169 ceph-mon[159415]: failed to encode map e388 with expected crc 2023-08-28T02:40:56.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:56 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:40:56.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:56 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:40:56.990 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:56 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/622469105' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:40:56.991 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:56 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:40:56.991 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:56 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 8548/179349 objects degraded (4.766%), 7 pgs degraded (PG_DEGRADED) 2023-08-28T02:40:57.124 INFO:teuthology.orchestra.run.smithi143.stderr:nodeep-scrub is set 2023-08-28T02:40:57.139 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:56 smithi143 ceph-mon[189871]: pgmap v1061: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 67 MiB/s wr, 1.07k op/s; 8548/179349 objects degraded (4.766%); 1368/179349 objects misplaced (0.763%); 7.3 MiB/s, 116 objects/s recovering 2023-08-28T02:40:57.140 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:56 smithi143 ceph-mon[189871]: failed to encode map e388 with expected crc 2023-08-28T02:40:57.140 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:56 smithi143 ceph-mon[189871]: failed to encode map e388 with expected crc 2023-08-28T02:40:57.140 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:56 smithi143 ceph-mon[189871]: osdmap e388: 8 total, 8 up, 6 in 2023-08-28T02:40:57.140 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:56 smithi143 ceph-mon[189871]: failed to encode map e388 with expected crc 2023-08-28T02:40:57.140 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:56 smithi143 ceph-mon[189871]: failed to encode map e388 with expected crc 2023-08-28T02:40:57.140 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:56 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:40:57.140 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:56 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:40:57.140 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:56 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/622469105' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:40:57.140 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:56 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:40:57.140 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:56 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 8548/179349 objects degraded (4.766%), 7 pgs degraded (PG_DEGRADED) 2023-08-28T02:40:57.142 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:56 smithi143 ceph-mon[183059]: pgmap v1061: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 67 MiB/s wr, 1.07k op/s; 8548/179349 objects degraded (4.766%); 1368/179349 objects misplaced (0.763%); 7.3 MiB/s, 116 objects/s recovering 2023-08-28T02:40:57.142 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:56 smithi143 ceph-mon[183059]: failed to encode map e388 with expected crc 2023-08-28T02:40:57.142 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:56 smithi143 ceph-mon[183059]: failed to encode map e388 with expected crc 2023-08-28T02:40:57.142 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:56 smithi143 ceph-mon[183059]: osdmap e388: 8 total, 8 up, 6 in 2023-08-28T02:40:57.142 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:56 smithi143 ceph-mon[183059]: failed to encode map e388 with expected crc 2023-08-28T02:40:57.143 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:56 smithi143 ceph-mon[183059]: failed to encode map e388 with expected crc 2023-08-28T02:40:57.143 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:56 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' 2023-08-28T02:40:57.143 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:56 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:40:57.143 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:56 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/622469105' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:40:57.143 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:56 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-08-28T02:40:57.143 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:56 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 8548/179349 objects degraded (4.766%), 7 pgs degraded (PG_DEGRADED) 2023-08-28T02:40:57.753 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 54 16 62094 62078 71.816 45.5 0.0125295 0.013919 2023-08-28T02:40:58.034 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:40:58.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:58.044 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:40:58.145 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:40:58.176 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:57 smithi143 ceph-mon[183059]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:40:58.176 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:57 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:40:58.176 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:57 smithi143 ceph-mon[183059]: osdmap e389: 8 total, 8 up, 6 in 2023-08-28T02:40:58.176 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:57 smithi143 ceph-mon[183059]: failed to encode map e389 with expected crc 2023-08-28T02:40:58.176 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:57 smithi143 ceph-mon[183059]: failed to encode map e389 with expected crc 2023-08-28T02:40:58.176 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:57 smithi143 ceph-mon[183059]: failed to encode map e389 with expected crc 2023-08-28T02:40:58.176 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:57 smithi143 ceph-mon[189871]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:40:58.176 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:57 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:40:58.177 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:57 smithi143 ceph-mon[189871]: osdmap e389: 8 total, 8 up, 6 in 2023-08-28T02:40:58.177 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:57 smithi143 ceph-mon[189871]: failed to encode map e389 with expected crc 2023-08-28T02:40:58.177 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:57 smithi143 ceph-mon[189871]: failed to encode map e389 with expected crc 2023-08-28T02:40:58.177 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:57 smithi143 ceph-mon[189871]: failed to encode map e389 with expected crc 2023-08-28T02:40:58.196 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:57 smithi169 ceph-mon[159415]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:40:58.196 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:57 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-08-28T02:40:58.196 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:57 smithi169 ceph-mon[159415]: osdmap e389: 8 total, 8 up, 6 in 2023-08-28T02:40:58.196 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:57 smithi169 ceph-mon[159415]: failed to encode map e389 with expected crc 2023-08-28T02:40:58.196 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:57 smithi169 ceph-mon[159415]: failed to encode map e389 with expected crc 2023-08-28T02:40:58.196 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:57 smithi169 ceph-mon[159415]: failed to encode map e389 with expected crc 2023-08-28T02:40:58.200 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:58.210 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.0 dump_historic_ops 2023-08-28T02:40:58.287 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:40:58.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:58.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:40:58.396 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:40:58.422 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:58.754 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 55 16 63005 62989 71.5455 56.9375 0.0130824 0.0139714 2023-08-28T02:40:59.031 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:40:59.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:59.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:40:59.116 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:58 smithi143 ceph-mon[183059]: failed to encode map e389 with expected crc 2023-08-28T02:40:59.117 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:58 smithi143 ceph-mon[183059]: pgmap v1064: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 48 MiB/s wr, 763 op/s; 8539/179349 objects degraded (4.761%); 1368/179349 objects misplaced (0.763%); 7.5 MiB/s, 121 objects/s recovering 2023-08-28T02:40:59.117 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:58 smithi143 ceph-mon[189871]: failed to encode map e389 with expected crc 2023-08-28T02:40:59.117 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:58 smithi143 ceph-mon[189871]: pgmap v1064: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 48 MiB/s wr, 763 op/s; 8539/179349 objects degraded (4.761%); 1368/179349 objects misplaced (0.763%); 7.5 MiB/s, 121 objects/s recovering 2023-08-28T02:40:59.141 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:40:59.185 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:58 smithi169 ceph-mon[159415]: failed to encode map e389 with expected crc 2023-08-28T02:40:59.185 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:58 smithi169 ceph-mon[159415]: pgmap v1064: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 48 MiB/s wr, 763 op/s; 8539/179349 objects degraded (4.761%); 1368/179349 objects misplaced (0.763%); 7.5 MiB/s, 121 objects/s recovering 2023-08-28T02:40:59.188 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:59.246 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:40:59.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:59.318 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:40:59.419 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:40:59.480 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:59.510 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:40:59.592 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:40:59.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:59.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:40:59.705 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:40:59.751 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:40:59.760 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 56 16 63999 63983 71.3763 62.125 0.0254392 0.0140039 2023-08-28T02:40:59.761 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:40:59.796 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:40:59.822 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:40:59.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:40:59.827 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:40:59.839 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:40:59.872 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:40:59.918 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:40:59.928 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:40:59.964 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:40:59.999 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:41:00.003 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:41:00.040 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:41:00.083 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-08-28T02:41:00.083 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-08-28T02:41:00.083 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-08-28T02:41:00.083 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-08-28T02:41:00.083 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-08-28T02:41:00.083 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-08-28T02:41:00.084 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-08-28T02:41:00.084 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-08-28T02:41:00.103 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:40:59 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/4291348890' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:41:00.103 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:40:59 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/4291348890' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:41:00.242 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:40:59 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/4291348890' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:41:00.287 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:41:00.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:41:00.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:41:00.395 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:41:00.476 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:41:00.515 INFO:tasks.thrashosds.thrasher:in_osds: [0, 5, 3, 6, 1, 7] out_osds: [4, 2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-08-28T02:41:00.515 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-08-28T02:41:00.516 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-08-28T02:41:00.516 INFO:tasks.thrashosds.thrasher:Setting osd 0 primary_affinity to 0.634090 2023-08-28T02:41:00.516 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd primary-affinity 0 0.6340899891385338 2023-08-28T02:41:00.754 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 57 16 65006 64990 71.2282 62.9375 0.0168204 0.0140343 2023-08-28T02:41:01.158 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset noscrub 2023-08-28T02:41:01.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:41:00 smithi143 ceph-mon[183059]: pgmap v1065: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 64 MiB/s wr, 1.02k op/s; 8532/188259 objects degraded (4.532%); 1274/188259 objects misplaced (0.677%); 6.1 MiB/s, 99 objects/s recovering 2023-08-28T02:41:01.203 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:41:00 smithi143 ceph-mon[189871]: pgmap v1065: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 64 MiB/s wr, 1.02k op/s; 8532/188259 objects degraded (4.532%); 1274/188259 objects misplaced (0.677%); 6.1 MiB/s, 99 objects/s recovering 2023-08-28T02:41:01.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:41:00 smithi169 ceph-mon[159415]: pgmap v1065: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 64 MiB/s wr, 1.02k op/s; 8532/188259 objects degraded (4.532%); 1274/188259 objects misplaced (0.677%); 6.1 MiB/s, 99 objects/s recovering 2023-08-28T02:41:01.759 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 58 16 65731 65715 70.7786 45.3125 0.00916928 0.014123 2023-08-28T02:41:01.988 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:41:01.995 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:41:01 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:41:01] "GET /metrics HTTP/1.1" 200 36588 "" "Prometheus/2.33.4" 2023-08-28T02:41:01.995 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:41:01 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 8532/188259 objects degraded (4.532%), 5 pgs degraded (PG_DEGRADED) 2023-08-28T02:41:01.998 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:41:01 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 8532/188259 objects degraded (4.532%), 5 pgs degraded (PG_DEGRADED) 2023-08-28T02:41:01.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:41:01.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:41:02.100 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:41:02.174 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:41:02.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:41:01 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 8532/188259 objects degraded (4.532%), 5 pgs degraded (PG_DEGRADED) 2023-08-28T02:41:02.296 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:41:02.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:41:02.307 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:41:02.408 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:41:02.455 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:41:02.519 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:41:02.551 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:41:02.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:41:02.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:41:02.576 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:41:02.661 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:41:02.712 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:41:02.757 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 59 16 66702 66686 70.6075 60.6875 0.0235977 0.0141566 2023-08-28T02:41:02.774 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:41:02.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:41:02.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:41:02.791 INFO:teuthology.orchestra.run.smithi143.stdout:ERROR: (22) Invalid argument 2023-08-28T02:41:02.792 INFO:teuthology.orchestra.run.smithi143.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-08-28T02:41:02.880 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:41:02.929 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:41:02.987 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:41:02 smithi169 ceph-mon[159415]: pgmap v1066: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 41 MiB/s wr, 658 op/s; 8532/190299 objects degraded (4.483%); 1274/190299 objects misplaced (0.669%); 877 KiB/s, 16 objects/s recovering 2023-08-28T02:41:03.011 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:41:03.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:41:03.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:41:03.115 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:41:02 smithi143 ceph-mon[189871]: pgmap v1066: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 41 MiB/s wr, 658 op/s; 8532/190299 objects degraded (4.483%); 1274/190299 objects misplaced (0.669%); 877 KiB/s, 16 objects/s recovering 2023-08-28T02:41:03.115 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:41:02 smithi143 ceph-mon[183059]: pgmap v1066: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 41 MiB/s wr, 658 op/s; 8532/190299 objects degraded (4.483%); 1274/190299 objects misplaced (0.669%); 877 KiB/s, 16 objects/s recovering 2023-08-28T02:41:03.119 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:41:03.172 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:41:03.757 INFO:tasks.radosbench.radosbench.0.smithi143.stdout:2023-08-28T02:41:03.758638+0000 min lat: 0.00102595 max lat: 2.1786 avg lat: 0.0142178 2023-08-28T02:41:03.757 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-08-28T02:41:03.757 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 60 16 67545 67529 70.3089 52.6875 0.0104669 0.0142178 2023-08-28T02:41:03.822 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:41:03.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:41:03.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:41:03.932 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:41:03.974 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:41:04.068 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:41:04.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:41:04.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:41:04.090 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-08-28T02:41:04.176 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:41:04.227 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.1 dump_ops_in_flight 2023-08-28T02:41:04.240 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:41:04.312 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:41:04.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:41:04.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:41:04.423 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:41:04.462 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:41:04.504 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:41:04.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:41:04.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:41:04.611 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:41:04.637 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:41:04.683 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:41:04.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:41:04.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:41:04.757 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 61 16 68234 68218 69.8623 43.0625 0.0109144 0.014309 2023-08-28T02:41:04.789 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:41:04.869 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:41:05.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:41:04 smithi143 ceph-mon[183059]: pgmap v1067: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 29 MiB/s wr, 456 op/s; 8532/190299 objects degraded (4.483%); 1274/190299 objects misplaced (0.669%); 752 KiB/s, 13 objects/s recovering 2023-08-28T02:41:05.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:41:04 smithi143 ceph-mon[189871]: pgmap v1067: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 29 MiB/s wr, 456 op/s; 8532/190299 objects degraded (4.483%); 1274/190299 objects misplaced (0.669%); 752 KiB/s, 13 objects/s recovering 2023-08-28T02:41:05.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:41:04 smithi169 ceph-mon[159415]: pgmap v1067: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 29 MiB/s wr, 456 op/s; 8532/190299 objects degraded (4.483%); 1274/190299 objects misplaced (0.669%); 752 KiB/s, 13 objects/s recovering 2023-08-28T02:41:05.631 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:41:05.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:41:05.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:41:05.736 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:41:05.757 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 62 16 69161 69145 69.67 57.9375 0.00744429 0.0143471 2023-08-28T02:41:05.836 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:41:06.063 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:41:06.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:41:06.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:41:06.172 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:41:06.219 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:41:06.286 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:41:06.307 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:41:06.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:41:06.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:41:06.344 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:41:06.414 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:41:06.504 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:41:06.758 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 63 16 69971 69955 69.3669 50.625 0.0140925 0.0144108 2023-08-28T02:41:06.764 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:41:06.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:41:06.780 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:41:06.880 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:41:06.952 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:41:07.098 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:41:06 smithi143 ceph-mon[183059]: pgmap v1068: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 52 MiB/s wr, 825 op/s; 8532/203769 objects degraded (4.187%); 1188/203769 objects misplaced (0.583%); 1.1 MiB/s, 19 objects/s recovering 2023-08-28T02:41:07.099 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:41:06 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 8532/203769 objects degraded (4.187%), 5 pgs degraded (PG_DEGRADED) 2023-08-28T02:41:07.099 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:41:06 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2772285158' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:41:07.099 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:41:06 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:41:07.099 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:41:06 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/1245918971' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.6340899891385338}]: dispatch 2023-08-28T02:41:07.099 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:41:06 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.6340899891385338}]: dispatch 2023-08-28T02:41:07.099 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:41:06 smithi143 ceph-mon[189871]: pgmap v1068: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 52 MiB/s wr, 825 op/s; 8532/203769 objects degraded (4.187%); 1188/203769 objects misplaced (0.583%); 1.1 MiB/s, 19 objects/s recovering 2023-08-28T02:41:07.099 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:41:06 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 8532/203769 objects degraded (4.187%), 5 pgs degraded (PG_DEGRADED) 2023-08-28T02:41:07.099 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:41:06 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2772285158' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:41:07.099 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:41:06 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:41:07.099 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:41:06 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/1245918971' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.6340899891385338}]: dispatch 2023-08-28T02:41:07.100 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:41:06 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.6340899891385338}]: dispatch 2023-08-28T02:41:07.102 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:41:07.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:41:07.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:41:07.220 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:41:07.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:41:06 smithi169 ceph-mon[159415]: pgmap v1068: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 52 MiB/s wr, 825 op/s; 8532/203769 objects degraded (4.187%); 1188/203769 objects misplaced (0.583%); 1.1 MiB/s, 19 objects/s recovering 2023-08-28T02:41:07.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:41:06 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 8532/203769 objects degraded (4.187%), 5 pgs degraded (PG_DEGRADED) 2023-08-28T02:41:07.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:41:06 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2772285158' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:41:07.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:41:06 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-08-28T02:41:07.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:41:06 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/1245918971' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.6340899891385338}]: dispatch 2023-08-28T02:41:07.238 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:41:06 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.6340899891385338}]: dispatch 2023-08-28T02:41:07.308 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:41:07.747 INFO:teuthology.orchestra.run.smithi143.stderr:set osd.0 primary-affinity to 0.63409 (8415552) 2023-08-28T02:41:07.747 INFO:teuthology.orchestra.run.smithi143.stderr:noscrub is unset 2023-08-28T02:41:07.758 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 64 16 70817 70801 69.1092 52.875 0.00735032 0.0144422 2023-08-28T02:41:08.189 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:41:07 smithi143 ceph-mon[183059]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:41:08.189 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:41:07 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:41:08.190 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:41:07 smithi143 ceph-mon[183059]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.6340899891385338}]': finished 2023-08-28T02:41:08.190 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:41:07 smithi143 ceph-mon[183059]: osdmap e390: 8 total, 8 up, 6 in 2023-08-28T02:41:08.190 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:41:07 smithi143 ceph-mon[189871]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:41:08.190 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:41:07 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:41:08.190 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:41:07 smithi143 ceph-mon[189871]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.6340899891385338}]': finished 2023-08-28T02:41:08.190 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:41:07 smithi143 ceph-mon[189871]: osdmap e390: 8 total, 8 up, 6 in 2023-08-28T02:41:08.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:41:07 smithi169 ceph-mon[159415]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-08-28T02:41:08.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:41:07 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-08-28T02:41:08.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:41:07 smithi169 ceph-mon[159415]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.6340899891385338}]': finished 2023-08-28T02:41:08.237 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:41:07 smithi169 ceph-mon[159415]: osdmap e390: 8 total, 8 up, 6 in 2023-08-28T02:41:08.758 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 65 16 70898 70882 68.1242 5.0625 0.00366634 0.0144758 2023-08-28T02:41:09.115 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:41:08 smithi143 ceph-mon[189871]: pgmap v1069: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 48 MiB/s wr, 767 op/s; 8532/203769 objects degraded (4.187%); 1188/203769 objects misplaced (0.583%); 1.1 MiB/s, 18 objects/s recovering 2023-08-28T02:41:09.116 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:41:08 smithi143 ceph-mon[189871]: failed to encode map e390 with expected crc 2023-08-28T02:41:09.116 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:41:08 smithi143 ceph-mon[189871]: failed to encode map e390 with expected crc 2023-08-28T02:41:09.116 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:41:08 smithi143 ceph-mon[189871]: failed to encode map e390 with expected crc 2023-08-28T02:41:09.116 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:41:08 smithi143 ceph-mon[189871]: failed to encode map e390 with expected crc 2023-08-28T02:41:09.116 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:41:08 smithi143 ceph-mon[189871]: osdmap e391: 8 total, 8 up, 6 in 2023-08-28T02:41:09.116 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:41:08 smithi143 ceph-mon[183059]: pgmap v1069: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 48 MiB/s wr, 767 op/s; 8532/203769 objects degraded (4.187%); 1188/203769 objects misplaced (0.583%); 1.1 MiB/s, 18 objects/s recovering 2023-08-28T02:41:09.116 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:41:08 smithi143 ceph-mon[183059]: failed to encode map e390 with expected crc 2023-08-28T02:41:09.116 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:41:08 smithi143 ceph-mon[183059]: failed to encode map e390 with expected crc 2023-08-28T02:41:09.116 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:41:08 smithi143 ceph-mon[183059]: failed to encode map e390 with expected crc 2023-08-28T02:41:09.116 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:41:08 smithi143 ceph-mon[183059]: failed to encode map e390 with expected crc 2023-08-28T02:41:09.117 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:41:08 smithi143 ceph-mon[183059]: osdmap e391: 8 total, 8 up, 6 in 2023-08-28T02:41:09.131 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:41:09.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:41:09.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:41:09.238 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:41:09.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:41:08 smithi169 ceph-mon[159415]: pgmap v1069: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 48 MiB/s wr, 767 op/s; 8532/203769 objects degraded (4.187%); 1188/203769 objects misplaced (0.583%); 1.1 MiB/s, 18 objects/s recovering 2023-08-28T02:41:09.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:41:08 smithi169 ceph-mon[159415]: failed to encode map e390 with expected crc 2023-08-28T02:41:09.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:41:08 smithi169 ceph-mon[159415]: failed to encode map e390 with expected crc 2023-08-28T02:41:09.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:41:08 smithi169 ceph-mon[159415]: failed to encode map e390 with expected crc 2023-08-28T02:41:09.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:41:08 smithi169 ceph-mon[159415]: failed to encode map e390 with expected crc 2023-08-28T02:41:09.240 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:41:08 smithi169 ceph-mon[159415]: osdmap e391: 8 total, 8 up, 6 in 2023-08-28T02:41:09.307 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:41:09.758 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 66 16 71869 71853 68.0114 60.6875 0.0135564 0.0146964 2023-08-28T02:41:10.046 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:41:10.051 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:41:09 smithi143 ceph-mon[183059]: failed to encode map e391 with expected crc 2023-08-28T02:41:10.051 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:41:09 smithi143 ceph-mon[183059]: failed to encode map e391 with expected crc 2023-08-28T02:41:10.051 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:41:09 smithi143 ceph-mon[183059]: failed to encode map e391 with expected crc 2023-08-28T02:41:10.051 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:41:09 smithi143 ceph-mon[183059]: failed to encode map e391 with expected crc 2023-08-28T02:41:10.051 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:41:09 smithi143 ceph-mon[183059]: failed to encode map e391 with expected crc 2023-08-28T02:41:10.051 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:41:09 smithi143 ceph-mon[183059]: failed to encode map e391 with expected crc 2023-08-28T02:41:10.051 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:41:09 smithi143 ceph-mon[183059]: failed to encode map e391 with expected crc 2023-08-28T02:41:10.052 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:41:09 smithi143 ceph-mon[183059]: failed to encode map e391 with expected crc 2023-08-28T02:41:10.052 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:41:09 smithi143 ceph-mon[183059]: failed to encode map e391 with expected crc 2023-08-28T02:41:10.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:41:10.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:41:10.053 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:41:09 smithi143 ceph-mon[189871]: failed to encode map e391 with expected crc 2023-08-28T02:41:10.053 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:41:09 smithi143 ceph-mon[189871]: failed to encode map e391 with expected crc 2023-08-28T02:41:10.053 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:41:09 smithi143 ceph-mon[189871]: failed to encode map e391 with expected crc 2023-08-28T02:41:10.053 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:41:09 smithi143 ceph-mon[189871]: failed to encode map e391 with expected crc 2023-08-28T02:41:10.053 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:41:09 smithi143 ceph-mon[189871]: failed to encode map e391 with expected crc 2023-08-28T02:41:10.054 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:41:09 smithi143 ceph-mon[189871]: failed to encode map e391 with expected crc 2023-08-28T02:41:10.054 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:41:09 smithi143 ceph-mon[189871]: failed to encode map e391 with expected crc 2023-08-28T02:41:10.054 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:41:09 smithi143 ceph-mon[189871]: failed to encode map e391 with expected crc 2023-08-28T02:41:10.054 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:41:09 smithi143 ceph-mon[189871]: failed to encode map e391 with expected crc 2023-08-28T02:41:10.154 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 2023-08-28T02:41:10.222 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:41:09 smithi169 ceph-mon[159415]: failed to encode map e391 with expected crc 2023-08-28T02:41:10.222 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:41:09 smithi169 ceph-mon[159415]: failed to encode map e391 with expected crc 2023-08-28T02:41:10.222 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:41:09 smithi169 ceph-mon[159415]: failed to encode map e391 with expected crc 2023-08-28T02:41:10.222 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:41:09 smithi169 ceph-mon[159415]: failed to encode map e391 with expected crc 2023-08-28T02:41:10.222 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:41:09 smithi169 ceph-mon[159415]: failed to encode map e391 with expected crc 2023-08-28T02:41:10.222 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:41:09 smithi169 ceph-mon[159415]: failed to encode map e391 with expected crc 2023-08-28T02:41:10.222 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:41:09 smithi169 ceph-mon[159415]: failed to encode map e391 with expected crc 2023-08-28T02:41:10.222 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:41:09 smithi169 ceph-mon[159415]: failed to encode map e391 with expected crc 2023-08-28T02:41:10.222 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:41:09 smithi169 ceph-mon[159415]: failed to encode map e391 with expected crc 2023-08-28T02:41:10.225 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:41:10.305 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5 found: no such container 2023-08-28T02:41:10.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:41:10.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.5' 2023-08-28T02:41:10.417 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:41:10.466 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:41:10.510 INFO:teuthology.orchestra.run.smithi143.stdout:osd.0: {} 2023-08-28T02:41:10.541 INFO:teuthology.orchestra.run.smithi143.stdout:osd.1: {} 2023-08-28T02:41:10.545 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:41:10.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:41:10.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:41:10.572 INFO:teuthology.orchestra.run.smithi143.stdout:osd.2: {} 2023-08-28T02:41:10.627 INFO:teuthology.orchestra.run.smithi143.stdout:osd.3: {} 2023-08-28T02:41:10.655 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 2023-08-28T02:41:10.661 INFO:teuthology.orchestra.run.smithi143.stdout:osd.4: {} 2023-08-28T02:41:10.698 INFO:teuthology.orchestra.run.smithi143.stdout:{ 2023-08-28T02:41:10.698 INFO:teuthology.orchestra.run.smithi143.stdout: "ops": [ 2023-08-28T02:41:10.698 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.698 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_repop(client.35967.0:72770 9.3 e391/368 9:c9c14b5f:::benchmark_data_smithi143_479678_object72769:head v 391'4534, mlcod=391'4532)", 2023-08-28T02:41:10.698 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:41:10.694586+0000", 2023-08-28T02:41:10.699 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.0028744180000000001, 2023-08-28T02:41:10.699 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.002903536, 2023-08-28T02:41:10.699 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:41:10.699 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "started", 2023-08-28T02:41:10.699 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:41:10.699 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.699 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:41:10.699 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:41:10.694586+0000", 2023-08-28T02:41:10.699 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:41:10.699 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:41:10.699 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.699 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:41:10.700 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:41:10.694586+0000", 2023-08-28T02:41:10.700 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:41:10.700 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:41:10.700 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.700 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:41:10.700 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:41:10.694586+0000", 2023-08-28T02:41:10.700 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:41:10.700 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:41:10.700 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.700 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:41:10.700 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:41:10.694624+0000", 2023-08-28T02:41:10.700 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 3.8881000000000003e-05 2023-08-28T02:41:10.701 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:41:10.701 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.701 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:41:10.701 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:41:10.694631+0000", 2023-08-28T02:41:10.701 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.7390000000000002e-06 2023-08-28T02:41:10.701 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:41:10.701 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.701 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:41:10.701 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:41:10.694636+0000", 2023-08-28T02:41:10.701 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 5.1429999999999997e-06 2023-08-28T02:41:10.701 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:41:10.701 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.702 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:41:10.702 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:41:10.694700+0000", 2023-08-28T02:41:10.702 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.3225999999999994e-05 2023-08-28T02:41:10.702 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:41:10.702 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.702 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:41:10.702 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:41:10.694721+0000", 2023-08-28T02:41:10.702 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.1766000000000001e-05 2023-08-28T02:41:10.702 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:41:10.702 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:41:10.702 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:41:10.703 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:41:10.703 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.703 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35967.0:72772 9.0 9:017a8285:::benchmark_data_smithi143_479678_object72771:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e391)", 2023-08-28T02:41:10.703 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:41:10.696073+0000", 2023-08-28T02:41:10.703 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.0013875879999999999, 2023-08-28T02:41:10.703 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.0014958930000000001, 2023-08-28T02:41:10.703 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:41:10.703 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "waiting for sub ops", 2023-08-28T02:41:10.703 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:41:10.703 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35967", 2023-08-28T02:41:10.704 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3130613759", 2023-08-28T02:41:10.704 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 72772 2023-08-28T02:41:10.704 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:41:10.704 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:41:10.704 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.704 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:41:10.704 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:41:10.696073+0000", 2023-08-28T02:41:10.704 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:41:10.704 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:41:10.704 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.704 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:41:10.704 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:41:10.696073+0000", 2023-08-28T02:41:10.705 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:41:10.705 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:41:10.705 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.705 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:41:10.705 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:41:10.696071+0000", 2023-08-28T02:41:10.705 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999986 2023-08-28T02:41:10.705 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:41:10.705 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.705 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:41:10.705 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:41:10.696101+0000", 2023-08-28T02:41:10.705 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.9850999999999999e-05 2023-08-28T02:41:10.706 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:41:10.706 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.706 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:41:10.706 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:41:10.696114+0000", 2023-08-28T02:41:10.706 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 1.3257e-05 2023-08-28T02:41:10.706 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:41:10.706 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.706 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:41:10.706 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:41:10.696122+0000", 2023-08-28T02:41:10.706 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 7.8849999999999999e-06 2023-08-28T02:41:10.706 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:41:10.706 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.707 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:41:10.707 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:41:10.696212+0000", 2023-08-28T02:41:10.707 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.9865999999999996e-05 2023-08-28T02:41:10.707 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:41:10.707 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.707 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:41:10.707 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:41:10.696364+0000", 2023-08-28T02:41:10.707 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00015178699999999999 2023-08-28T02:41:10.707 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:41:10.707 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.707 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 0,6", 2023-08-28T02:41:10.707 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:41:10.696474+0000", 2023-08-28T02:41:10.707 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000110612 2023-08-28T02:41:10.708 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:41:10.708 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:41:10.708 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:41:10.708 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:41:10.708 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.708 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35967.0:72773 9.6 9:673dd6d4:::benchmark_data_smithi143_479678_object72772:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e391)", 2023-08-28T02:41:10.708 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:41:10.696139+0000", 2023-08-28T02:41:10.708 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.001321083, 2023-08-28T02:41:10.708 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.001526999, 2023-08-28T02:41:10.708 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:41:10.708 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "waiting for sub ops", 2023-08-28T02:41:10.708 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:41:10.709 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35967", 2023-08-28T02:41:10.709 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3130613759", 2023-08-28T02:41:10.709 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 72773 2023-08-28T02:41:10.709 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:41:10.709 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:41:10.709 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.709 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:41:10.709 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:41:10.696139+0000", 2023-08-28T02:41:10.709 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:41:10.709 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:41:10.709 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.709 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:41:10.710 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:41:10.696139+0000", 2023-08-28T02:41:10.710 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:41:10.710 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:41:10.710 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.710 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:41:10.710 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:41:10.696139+0000", 2023-08-28T02:41:10.710 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:41:10.710 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:41:10.710 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.710 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:41:10.710 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:41:10.696162+0000", 2023-08-28T02:41:10.710 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.3291000000000001e-05 2023-08-28T02:41:10.711 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:41:10.711 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.711 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:41:10.711 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:41:10.696169+0000", 2023-08-28T02:41:10.711 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.5259999999999999e-06 2023-08-28T02:41:10.711 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:41:10.711 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.711 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:41:10.711 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:41:10.696173+0000", 2023-08-28T02:41:10.711 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.8679999999999998e-06 2023-08-28T02:41:10.711 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:41:10.711 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.711 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:41:10.712 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:41:10.697103+0000", 2023-08-28T02:41:10.712 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00092970699999999999 2023-08-28T02:41:10.712 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:41:10.712 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.712 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:41:10.712 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:41:10.697256+0000", 2023-08-28T02:41:10.712 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00015256800000000001 2023-08-28T02:41:10.712 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:41:10.712 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.712 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 5,6", 2023-08-28T02:41:10.712 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:41:10.697425+0000", 2023-08-28T02:41:10.712 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00016902400000000001 2023-08-28T02:41:10.712 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:41:10.712 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:41:10.713 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:41:10.713 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:41:10.713 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.713 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35967.0:72780 9.a 9:50939d26:::benchmark_data_smithi143_479678_object72779:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e391)", 2023-08-28T02:41:10.713 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:41:10.696785+0000", 2023-08-28T02:41:10.713 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.00067558699999999995, 2023-08-28T02:41:10.713 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00095516600000000002, 2023-08-28T02:41:10.713 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:41:10.713 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "waiting for sub ops", 2023-08-28T02:41:10.713 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:41:10.713 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35967", 2023-08-28T02:41:10.713 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3130613759", 2023-08-28T02:41:10.714 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 72780 2023-08-28T02:41:10.714 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:41:10.714 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:41:10.714 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.714 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:41:10.714 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:41:10.696785+0000", 2023-08-28T02:41:10.714 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:41:10.714 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:41:10.714 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.714 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:41:10.714 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:41:10.696785+0000", 2023-08-28T02:41:10.714 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:41:10.714 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:41:10.715 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.715 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:41:10.715 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:41:10.696784+0000", 2023-08-28T02:41:10.715 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.999999 2023-08-28T02:41:10.715 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:41:10.715 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.715 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:41:10.715 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:41:10.696805+0000", 2023-08-28T02:41:10.715 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 2.0834000000000001e-05 2023-08-28T02:41:10.715 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:41:10.715 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.715 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:41:10.715 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:41:10.696814+0000", 2023-08-28T02:41:10.715 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 9.6970000000000005e-06 2023-08-28T02:41:10.716 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:41:10.716 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.716 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:41:10.716 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:41:10.696821+0000", 2023-08-28T02:41:10.716 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.3749999999999999e-06 2023-08-28T02:41:10.716 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:41:10.716 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.716 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:41:10.716 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:41:10.697023+0000", 2023-08-28T02:41:10.716 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000201645 2023-08-28T02:41:10.716 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:41:10.716 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.716 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:41:10.717 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:41:10.697141+0000", 2023-08-28T02:41:10.717 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000118712 2023-08-28T02:41:10.717 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:41:10.717 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.717 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 5,7", 2023-08-28T02:41:10.717 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:41:10.697267+0000", 2023-08-28T02:41:10.717 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00012606999999999999 2023-08-28T02:41:10.717 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:41:10.717 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:41:10.717 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:41:10.717 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:41:10.717 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.717 INFO:teuthology.orchestra.run.smithi143.stdout: "description": "osd_op(client.35967.0:72781 9.f 9:f35d661d:::benchmark_data_smithi143_479678_object72780:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e391)", 2023-08-28T02:41:10.717 INFO:teuthology.orchestra.run.smithi143.stdout: "initiated_at": "2023-08-28T02:41:10.697129+0000", 2023-08-28T02:41:10.718 INFO:teuthology.orchestra.run.smithi143.stdout: "age": 0.00033148100000000001, 2023-08-28T02:41:10.718 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00068292400000000001, 2023-08-28T02:41:10.718 INFO:teuthology.orchestra.run.smithi143.stdout: "type_data": { 2023-08-28T02:41:10.718 INFO:teuthology.orchestra.run.smithi143.stdout: "flag_point": "waiting for sub ops", 2023-08-28T02:41:10.718 INFO:teuthology.orchestra.run.smithi143.stdout: "client_info": { 2023-08-28T02:41:10.718 INFO:teuthology.orchestra.run.smithi143.stdout: "client": "client.35967", 2023-08-28T02:41:10.718 INFO:teuthology.orchestra.run.smithi143.stdout: "client_addr": "172.21.15.143:0/3130613759", 2023-08-28T02:41:10.718 INFO:teuthology.orchestra.run.smithi143.stdout: "tid": 72781 2023-08-28T02:41:10.718 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:41:10.718 INFO:teuthology.orchestra.run.smithi143.stdout: "events": [ 2023-08-28T02:41:10.718 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.719 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "initiated", 2023-08-28T02:41:10.719 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:41:10.697129+0000", 2023-08-28T02:41:10.719 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:41:10.719 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:41:10.719 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.719 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "throttled", 2023-08-28T02:41:10.719 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:41:10.697129+0000", 2023-08-28T02:41:10.719 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0 2023-08-28T02:41:10.719 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:41:10.719 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.719 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "header_read", 2023-08-28T02:41:10.720 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:41:10.697128+0000", 2023-08-28T02:41:10.720 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4294967295.9999995 2023-08-28T02:41:10.720 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:41:10.720 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.720 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "all_read", 2023-08-28T02:41:10.720 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:41:10.697177+0000", 2023-08-28T02:41:10.720 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.8919999999999999e-05 2023-08-28T02:41:10.720 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:41:10.720 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.720 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "dispatched", 2023-08-28T02:41:10.720 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:41:10.697186+0000", 2023-08-28T02:41:10.720 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 8.9260000000000006e-06 2023-08-28T02:41:10.720 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:41:10.721 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.721 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "queued_for_pg", 2023-08-28T02:41:10.721 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:41:10.697192+0000", 2023-08-28T02:41:10.721 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 6.3040000000000001e-06 2023-08-28T02:41:10.721 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:41:10.721 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.721 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "reached_pg", 2023-08-28T02:41:10.721 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:41:10.697237+0000", 2023-08-28T02:41:10.721 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 4.5015000000000002e-05 2023-08-28T02:41:10.721 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:41:10.721 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.721 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "started", 2023-08-28T02:41:10.721 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:41:10.697361+0000", 2023-08-28T02:41:10.722 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.000123905 2023-08-28T02:41:10.722 INFO:teuthology.orchestra.run.smithi143.stdout: }, 2023-08-28T02:41:10.722 INFO:teuthology.orchestra.run.smithi143.stdout: { 2023-08-28T02:41:10.722 INFO:teuthology.orchestra.run.smithi143.stdout: "event": "waiting for subops from 3,5", 2023-08-28T02:41:10.722 INFO:teuthology.orchestra.run.smithi143.stdout: "time": "2023-08-28T02:41:10.697507+0000", 2023-08-28T02:41:10.722 INFO:teuthology.orchestra.run.smithi143.stdout: "duration": 0.00014559100000000001 2023-08-28T02:41:10.722 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:41:10.722 INFO:teuthology.orchestra.run.smithi143.stdout: ] 2023-08-28T02:41:10.722 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:41:10.722 INFO:teuthology.orchestra.run.smithi143.stdout: } 2023-08-28T02:41:10.722 INFO:teuthology.orchestra.run.smithi143.stdout: ], 2023-08-28T02:41:10.722 INFO:teuthology.orchestra.run.smithi143.stdout: "num_ops": 5 2023-08-28T02:41:10.722 INFO:teuthology.orchestra.run.smithi143.stdout:} 2023-08-28T02:41:10.723 INFO:teuthology.orchestra.run.smithi143.stdout:osd.5: {} 2023-08-28T02:41:10.731 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:41:10.748 INFO:teuthology.orchestra.run.smithi143.stdout:osd.6: {} 2023-08-28T02:41:10.760 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 67 16 72833 72817 67.8936 60.25 0.0180424 0.0147236 2023-08-28T02:41:10.783 INFO:teuthology.orchestra.run.smithi143.stdout:osd.7: {} 2023-08-28T02:41:10.803 INFO:teuthology.orchestra.run.smithi143.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-08-28T02:41:10.803 INFO:teuthology.orchestra.run.smithi143.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-08-28T02:41:10.804 INFO:teuthology.orchestra.run.smithi143.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-08-28T02:41:10.804 INFO:teuthology.orchestra.run.smithi143.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-08-28T02:41:10.804 INFO:teuthology.orchestra.run.smithi143.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-08-28T02:41:10.804 INFO:teuthology.orchestra.run.smithi143.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-08-28T02:41:10.804 INFO:teuthology.orchestra.run.smithi143.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-08-28T02:41:10.804 INFO:teuthology.orchestra.run.smithi143.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-08-28T02:41:10.932 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:41:10 smithi169 ceph-mon[159415]: pgmap v1072: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 59 MiB/s wr, 947 op/s; 8532/213042 objects degraded (4.005%); 1100/213042 objects misplaced (0.516%); 1.4 MiB/s, 21 objects/s recovering 2023-08-28T02:41:10.932 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:41:10 smithi169 ceph-mon[159415]: from='client.? 172.21.15.143:0/2651911697' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:41:11.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:41:10 smithi143 ceph-mon[183059]: pgmap v1072: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 59 MiB/s wr, 947 op/s; 8532/213042 objects degraded (4.005%); 1100/213042 objects misplaced (0.516%); 1.4 MiB/s, 21 objects/s recovering 2023-08-28T02:41:11.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:41:10 smithi143 ceph-mon[183059]: from='client.? 172.21.15.143:0/2651911697' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:41:11.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:41:10 smithi143 ceph-mon[189871]: pgmap v1072: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 59 MiB/s wr, 947 op/s; 8532/213042 objects degraded (4.005%); 1100/213042 objects misplaced (0.516%); 1.4 MiB/s, 21 objects/s recovering 2023-08-28T02:41:11.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:41:10 smithi143 ceph-mon[189871]: from='client.? 172.21.15.143:0/2651911697' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-08-28T02:41:11.761 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 68 16 73737 73721 67.7261 56.5 0.0171187 0.0147601 2023-08-28T02:41:11.802 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph osd unset nodeep-scrub 2023-08-28T02:41:11.907 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1 found: no such container 2023-08-28T02:41:11.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:41:11.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.1' 2023-08-28T02:41:12.018 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 2023-08-28T02:41:12.044 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:41:12.086 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4 found: no such container 2023-08-28T02:41:12.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:41:12.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.4' 2023-08-28T02:41:12.193 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 2023-08-28T02:41:12.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:41:11 smithi143 ceph-mon[183059]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:41:12.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:41:11 smithi143 ceph-mon[183059]: pgmap v1073: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 61 MiB/s wr, 978 op/s; 8532/213801 objects degraded (3.991%); 1100/213801 objects misplaced (0.514%); 1.4 MiB/s, 21 objects/s recovering 2023-08-28T02:41:12.200 INFO:journalctl@ceph.mon.a.smithi143.stdout:Aug 28 02:41:11 smithi143 ceph-mon[183059]: Health check update: Degraded data redundancy: 8532/213042 objects degraded (4.005%), 5 pgs degraded (PG_DEGRADED) 2023-08-28T02:41:12.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:41:11 smithi143 ceph-mon[189871]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:41:12.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:41:11 smithi143 ceph-mon[189871]: pgmap v1073: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 61 MiB/s wr, 978 op/s; 8532/213801 objects degraded (3.991%); 1100/213801 objects misplaced (0.514%); 1.4 MiB/s, 21 objects/s recovering 2023-08-28T02:41:12.201 INFO:journalctl@ceph.mon.c.smithi143.stdout:Aug 28 02:41:11 smithi143 ceph-mon[189871]: Health check update: Degraded data redundancy: 8532/213042 objects degraded (4.005%), 5 pgs degraded (PG_DEGRADED) 2023-08-28T02:41:12.201 INFO:journalctl@ceph.mgr.y.smithi143.stdout:Aug 28 02:41:11 smithi143 conmon[176053]: ::ffff:172.21.15.169 - - [28/Aug/2023:02:41:11] "GET /metrics HTTP/1.1" 200 36570 "" "Prometheus/2.33.4" 2023-08-28T02:41:12.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:41:11 smithi169 ceph-mon[159415]: from='mgr.34107 172.21.15.143:0/1426912649' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:41:12.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:41:11 smithi169 ceph-mon[159415]: pgmap v1073: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 61 MiB/s wr, 978 op/s; 8532/213801 objects degraded (3.991%); 1100/213801 objects misplaced (0.514%); 1.4 MiB/s, 21 objects/s recovering 2023-08-28T02:41:12.239 INFO:journalctl@ceph.mon.b.smithi169.stdout:Aug 28 02:41:11 smithi169 ceph-mon[159415]: Health check update: Degraded data redundancy: 8532/213042 objects degraded (4.005%), 5 pgs degraded (PG_DEGRADED) 2023-08-28T02:41:12.253 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:41:12.291 DEBUG:teuthology.orchestra.run.smithi143:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid 2fcb0bfa-4547-11ee-817c-1b7a5f1ba315 -- ceph daemon osd.1 dump_blocked_ops 2023-08-28T02:41:12.560 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2 found: no such container 2023-08-28T02:41:12.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:41:12.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.2' 2023-08-28T02:41:12.674 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:41:12.761 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 69 16 74533 74517 67.4655 49.75 0.0126529 0.0148168 2023-08-28T02:41:12.772 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:41:12.948 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 found: no such container 2023-08-28T02:41:12.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:41:12.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3' 2023-08-28T02:41:12.998 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:41:13.064 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 2023-08-28T02:41:13.069 INFO:teuthology.orchestra.run.smithi143.stderr:Inferring config /var/lib/ceph/2fcb0bfa-4547-11ee-817c-1b7a5f1ba315/mon.a/config 2023-08-28T02:41:13.127 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:41:13.301 INFO:teuthology.orchestra.run.smithi143.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0 found: no such container 2023-08-28T02:41:13.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:41:13.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi143 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.0' 2023-08-28T02:41:13.407 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 2023-08-28T02:41:13.471 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:41:13.544 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7 found: no such container 2023-08-28T02:41:13.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:41:13.553 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.7' 2023-08-28T02:41:13.654 DEBUG:teuthology.orchestra.run.smithi169:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 2023-08-28T02:41:13.700 INFO:teuthology.orchestra.run.smithi169.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-08-28T02:41:13.761 INFO:tasks.radosbench.radosbench.0.smithi143.stdout: 70 16 75436 75420 67.3079 56.4375 0.0198964 0.0148526 2023-08-28T02:41:13.789 INFO:teuthology.orchestra.run.smithi169.stderr:Error: no container with name or ID ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6 found: no such container 2023-08-28T02:41:13.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-08-28T02:41:13.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi169 with status 125: 'sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.6' 2023-08-28T02:41:13.900 DEBUG:teuthology.orchestra.run.smithi143:> sudo docker kill -s 1 ceph-2fcb0bfa-4547-11ee-817c-1b7a5f1ba315-osd.3 2023-08-28T02:41:13.981 INFO:teuthology.orchestra.run.smithi143.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg.